From 19fc5e1132aa16e10a0400b4bef3c61e0217900d Mon Sep 17 00:00:00 2001 From: digital-pro Date: Wed, 4 Mar 2026 13:26:03 -0800 Subject: [PATCH 01/15] feat: scaffold location-selection task flow Add a new location-selection task scaffold with mode selection and placeholder GPS/map/city-postal trials, and wire task launcher safeguards so the task runs without corpus or assets-per-task entries. --- task-launcher/src/index.ts | 14 ++++--- .../location-selection/helpers/config.ts | 23 ++++++++++++ .../src/tasks/location-selection/timeline.ts | 37 +++++++++++++++++++ .../location-selection/trials/gpsCapture.ts | 33 +++++++++++++++++ .../location-selection/trials/instructions.ts | 24 ++++++++++++ .../location-selection/trials/mapPicker.ts | 32 ++++++++++++++++ .../location-selection/trials/modeSelect.ts | 30 +++++++++++++++ .../trials/reviewAndConfirm.ts | 29 +++++++++++++++ .../trials/searchCityPostal.ts | 32 ++++++++++++++++ task-launcher/src/tasks/taskConfig.ts | 8 ++++ 10 files changed, 257 insertions(+), 5 deletions(-) create mode 100644 task-launcher/src/tasks/location-selection/helpers/config.ts create mode 100644 task-launcher/src/tasks/location-selection/timeline.ts create mode 100644 task-launcher/src/tasks/location-selection/trials/gpsCapture.ts create mode 100644 task-launcher/src/tasks/location-selection/trials/instructions.ts create mode 100644 task-launcher/src/tasks/location-selection/trials/mapPicker.ts create mode 100644 task-launcher/src/tasks/location-selection/trials/modeSelect.ts create mode 100644 task-launcher/src/tasks/location-selection/trials/reviewAndConfirm.ts create mode 100644 task-launcher/src/tasks/location-selection/trials/searchCityPostal.ts diff --git a/task-launcher/src/index.ts b/task-launcher/src/index.ts index c2185b2f..7de679e9 100644 --- a/task-launcher/src/index.ts +++ b/task-launcher/src/index.ts @@ -73,16 +73,20 @@ export class TaskLauncher { await getTranslations(isDev, config.language); // TODO: make hearts and flowers corpus? make list of tasks that don't need corpora? - if (taskName !== 'hearts-and-flowers' && taskName !== 'memory-game' && taskName !== 'intro') { + if ( + taskName !== 'hearts-and-flowers' && + taskName !== 'memory-game' && + taskName !== 'intro' && + taskName !== 'location-selection' + ) { await getCorpus(config, isDev); } await getAssetsPerTask(isDev); - const taskAudioAssetNames = [ - ...taskStore().assetsPerTask[taskName].audio, - ...taskStore().assetsPerTask.shared.audio, - ]; + const taskAssetEntry = taskStore().assetsPerTask?.[taskName] || { audio: [] }; + const sharedAssetEntry = taskStore().assetsPerTask?.shared || { audio: [] }; + const taskAudioAssetNames = [...(taskAssetEntry.audio || []), ...(sharedAssetEntry.audio || [])]; // filter out language audio not relevant to current task languageAudioAssets = filterMedia(languageAudioAssets, [], taskAudioAssetNames, []); diff --git a/task-launcher/src/tasks/location-selection/helpers/config.ts b/task-launcher/src/tasks/location-selection/helpers/config.ts new file mode 100644 index 00000000..dafe95e5 --- /dev/null +++ b/task-launcher/src/tasks/location-selection/helpers/config.ts @@ -0,0 +1,23 @@ +export type LocationSelectionTaskConfig = { + populationThreshold: number; + baselineResolution: number; + maxResolution: number; +}; + +export function getLocationSelectionTaskConfig(config: Record): LocationSelectionTaskConfig { + const threshold = Number(config?.populationThreshold); + const baseline = Number(config?.baselineResolution); + const maxRes = Number(config?.maxResolution); + + return { + populationThreshold: Number.isFinite(threshold) && threshold > 0 ? Math.round(threshold) : 50000, + baselineResolution: + Number.isFinite(baseline) && Number.isInteger(baseline) && baseline >= 0 && baseline <= 15 + ? baseline + : 5, + maxResolution: + Number.isFinite(maxRes) && Number.isInteger(maxRes) && maxRes >= 0 && maxRes <= 15 + ? maxRes + : 9, + }; +} diff --git a/task-launcher/src/tasks/location-selection/timeline.ts b/task-launcher/src/tasks/location-selection/timeline.ts new file mode 100644 index 00000000..4571f00d --- /dev/null +++ b/task-launcher/src/tasks/location-selection/timeline.ts @@ -0,0 +1,37 @@ +import 'regenerator-runtime/runtime'; +import { initTrialSaving, initTimeline } from '../shared/helpers'; +import { jsPsych } from '../taskSetup'; +import { enterFullscreen, exitFullscreen, finishExperiment, taskFinished } from '../shared/trials'; +import { instructions } from './trials/instructions'; +import { modeSelect } from './trials/modeSelect'; +import { gpsCapture } from './trials/gpsCapture'; +import { mapPicker } from './trials/mapPicker'; +import { searchCityPostal } from './trials/searchCityPostal'; +import { reviewAndConfirm } from './trials/reviewAndConfirm'; +import { getLocationSelectionTaskConfig } from './helpers/config'; +import { taskStore } from '../../taskStore'; + +export default function buildLocationSelectionTimeline(config: Record, _mediaAssets: MediaAssetsType) { + initTrialSaving(config); + const initialTimeline = initTimeline(config, enterFullscreen, finishExperiment); + const locationConfig = getLocationSelectionTaskConfig(config); + + taskStore('locationSelectionConfig', locationConfig); + taskStore('locationSelectionMode', null); + taskStore('locationSelectionLastStep', null); + + const timeline: Record[] = [ + initialTimeline, + ...instructions, + modeSelect, + gpsCapture, + mapPicker, + searchCityPostal, + reviewAndConfirm, + ]; + + timeline.push(taskFinished('taskFinished')); + timeline.push(exitFullscreen); + + return { jsPsych, timeline }; +} diff --git a/task-launcher/src/tasks/location-selection/trials/gpsCapture.ts b/task-launcher/src/tasks/location-selection/trials/gpsCapture.ts new file mode 100644 index 00000000..e13258ed --- /dev/null +++ b/task-launcher/src/tasks/location-selection/trials/gpsCapture.ts @@ -0,0 +1,33 @@ +import jsPsychHtmlMultiResponse from '@jspsych-contrib/plugin-html-multi-response'; +import { taskStore } from '../../../taskStore'; + +export const gpsCapture = { + timeline: [ + { + type: jsPsychHtmlMultiResponse, + stimulus: ` +
+
+

GPS location

+

This scaffold step represents GPS capture on device.

+

Raw coordinates should be transformed on-device before persistence.

+
+
+ `, + prompt_above_buttons: true, + button_choices: ['Continue'], + button_html: '', + keyboard_choices: 'NO_KEYS', + data: { + assessment_stage: 'gps_capture', + task: 'location-selection', + }, + on_finish: (data: Record) => { + data.mode = 'gps'; + data.geolocationSupported = typeof navigator !== 'undefined' && !!navigator.geolocation; + taskStore('locationSelectionLastStep', 'gps_capture'); + }, + }, + ], + conditional_function: () => taskStore().locationSelectionMode === 'gps', +}; diff --git a/task-launcher/src/tasks/location-selection/trials/instructions.ts b/task-launcher/src/tasks/location-selection/trials/instructions.ts new file mode 100644 index 00000000..c6422f1b --- /dev/null +++ b/task-launcher/src/tasks/location-selection/trials/instructions.ts @@ -0,0 +1,24 @@ +import jsPsychHtmlMultiResponse from '@jspsych-contrib/plugin-html-multi-response'; + +export const instructions = [ + { + type: jsPsychHtmlMultiResponse, + stimulus: ` +
+
+

Location Selection

+

You can share an approximate location using GPS, map selection, or city/postal search.

+

This task is scaffolded to match core-tasks UI flow and data capture style.

+
+
+ `, + prompt_above_buttons: true, + button_choices: ['Continue'], + button_html: '', + keyboard_choices: 'NO_KEYS', + data: { + assessment_stage: 'instructions', + task: 'location-selection', + }, + }, +]; diff --git a/task-launcher/src/tasks/location-selection/trials/mapPicker.ts b/task-launcher/src/tasks/location-selection/trials/mapPicker.ts new file mode 100644 index 00000000..828dbed6 --- /dev/null +++ b/task-launcher/src/tasks/location-selection/trials/mapPicker.ts @@ -0,0 +1,32 @@ +import jsPsychHtmlMultiResponse from '@jspsych-contrib/plugin-html-multi-response'; +import { taskStore } from '../../../taskStore'; + +export const mapPicker = { + timeline: [ + { + type: jsPsychHtmlMultiResponse, + stimulus: ` +
+
+

Map picker

+

This scaffold step represents map-based location selection.

+

Final location should be de-identified before saving.

+
+
+ `, + prompt_above_buttons: true, + button_choices: ['Continue'], + button_html: '', + keyboard_choices: 'NO_KEYS', + data: { + assessment_stage: 'map_picker', + task: 'location-selection', + }, + on_finish: (data: Record) => { + data.mode = 'map'; + taskStore('locationSelectionLastStep', 'map_picker'); + }, + }, + ], + conditional_function: () => taskStore().locationSelectionMode === 'map', +}; diff --git a/task-launcher/src/tasks/location-selection/trials/modeSelect.ts b/task-launcher/src/tasks/location-selection/trials/modeSelect.ts new file mode 100644 index 00000000..a9c4addc --- /dev/null +++ b/task-launcher/src/tasks/location-selection/trials/modeSelect.ts @@ -0,0 +1,30 @@ +import jsPsychHtmlMultiResponse from '@jspsych-contrib/plugin-html-multi-response'; +import { taskStore } from '../../../taskStore'; + +const modes = ['gps', 'map', 'city_postal'] as const; + +export const modeSelect = { + type: jsPsychHtmlMultiResponse, + stimulus: ` +
+
+

Choose how to provide location

+

Select one method to continue.

+
+
+ `, + prompt_above_buttons: true, + button_choices: ['Use GPS', 'Pick on map', 'Type city/postal'], + button_html: '', + keyboard_choices: 'NO_KEYS', + data: { + assessment_stage: 'mode_select', + task: 'location-selection', + }, + on_finish: (data: Record) => { + const idx = Number(data?.response ?? data?.button_response ?? -1); + const selectedMode = modes[idx] || 'gps'; + data.selectedMode = selectedMode; + taskStore('locationSelectionMode', selectedMode); + }, +}; diff --git a/task-launcher/src/tasks/location-selection/trials/reviewAndConfirm.ts b/task-launcher/src/tasks/location-selection/trials/reviewAndConfirm.ts new file mode 100644 index 00000000..ebc205bf --- /dev/null +++ b/task-launcher/src/tasks/location-selection/trials/reviewAndConfirm.ts @@ -0,0 +1,29 @@ +import jsPsychHtmlMultiResponse from '@jspsych-contrib/plugin-html-multi-response'; +import { taskStore } from '../../../taskStore'; + +export const reviewAndConfirm = { + type: jsPsychHtmlMultiResponse, + stimulus: () => { + const mode = taskStore().locationSelectionMode || 'unknown'; + return ` +
+
+

Review selection

+

Selected mode: ${mode}

+

This task scaffold is ready for integrating GPS/map/search + on-device H3 obfuscation.

+
+
+ `; + }, + prompt_above_buttons: true, + button_choices: ['Confirm'], + button_html: '', + keyboard_choices: 'NO_KEYS', + data: { + assessment_stage: 'review_confirm', + task: 'location-selection', + }, + on_finish: (data: Record) => { + data.mode = taskStore().locationSelectionMode || null; + }, +}; diff --git a/task-launcher/src/tasks/location-selection/trials/searchCityPostal.ts b/task-launcher/src/tasks/location-selection/trials/searchCityPostal.ts new file mode 100644 index 00000000..ddf8203d --- /dev/null +++ b/task-launcher/src/tasks/location-selection/trials/searchCityPostal.ts @@ -0,0 +1,32 @@ +import jsPsychHtmlMultiResponse from '@jspsych-contrib/plugin-html-multi-response'; +import { taskStore } from '../../../taskStore'; + +export const searchCityPostal = { + timeline: [ + { + type: jsPsychHtmlMultiResponse, + stimulus: ` +
+
+

City / postal search

+

This scaffold step represents typing a city or postal code.

+

Use on-device obfuscation before storing final location.

+
+
+ `, + prompt_above_buttons: true, + button_choices: ['Continue'], + button_html: '', + keyboard_choices: 'NO_KEYS', + data: { + assessment_stage: 'city_postal_search', + task: 'location-selection', + }, + on_finish: (data: Record) => { + data.mode = 'city_postal'; + taskStore('locationSelectionLastStep', 'city_postal_search'); + }, + }, + ], + conditional_function: () => taskStore().locationSelectionMode === 'city_postal', +}; diff --git a/task-launcher/src/tasks/taskConfig.ts b/task-launcher/src/tasks/taskConfig.ts index 957bdd88..f24ed7fc 100644 --- a/task-launcher/src/tasks/taskConfig.ts +++ b/task-launcher/src/tasks/taskConfig.ts @@ -15,6 +15,7 @@ import tROGTimeline from './trog/timeline'; import inferenceTimeline from './roar-inference/timeline'; import adultReasoningTimeline from './adult-reasoning/timeline'; import childSurveyTimeline from './child-survey/timeline'; +import locationSelectionTimeline from './location-selection/timeline'; // TODO: Abstract to import config from specifc task folder // Will allow for multiple devs to work on the repo without merge conflicts @@ -130,4 +131,11 @@ export default { buildTaskTimeline: childSurveyTimeline, variants: {}, }, + locationSelection: { + setConfig: setSharedConfig, + getCorpus: getCorpus, + getTranslations: getTranslations, + buildTaskTimeline: locationSelectionTimeline, + variants: {}, + }, }; From daea850db20ebfd7a4983b6707053f75e3c0e8e8 Mon Sep 17 00:00:00 2001 From: digital-pro Date: Wed, 4 Mar 2026 15:01:58 -0800 Subject: [PATCH 02/15] fix: make media loading resilient for scaffold tasks Handle missing media prefixes gracefully and align getMediaAssets argument ordering so new task scaffolds can run without pre-provisioned bucket assets. --- .../src/tasks/shared/helpers/getMediaAssets.ts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/task-launcher/src/tasks/shared/helpers/getMediaAssets.ts b/task-launcher/src/tasks/shared/helpers/getMediaAssets.ts index 712f60d5..cf4c70cf 100644 --- a/task-launcher/src/tasks/shared/helpers/getMediaAssets.ts +++ b/task-launcher/src/tasks/shared/helpers/getMediaAssets.ts @@ -21,8 +21,8 @@ type ResponseDataType = { export async function getMediaAssets( bucketName: string, whitelist: Record = {}, - language: string, taskName: string, + language: string, nextPageToken = '', categorizedObjects: CategorizedObjectsType = { images: {}, audio: {}, video: {} }, ) { @@ -38,10 +38,15 @@ export async function getMediaAssets( } const response = await fetch(url); + if (!response.ok) { + // New/scaffold tasks may not have media buckets yet; return empty assets rather than throw. + return categorizedObjects; + } const data: ResponseDataType = await response.json(); + const items = Array.isArray(data?.items) ? data.items : []; - data.items.forEach((item) => { - if (isLanguageAndDeviceValid(item.name, taskName, language) && isWhitelisted(item.name, whitelist)) { + items.forEach((item) => { + if (isLanguageAndDeviceValid(item.name, language, taskName) && isWhitelisted(item.name, whitelist)) { const contentType = item.contentType; const id = item.name; const path = `https://storage.googleapis.com/${bucket}/${id}`; From 219d2d8000ce5b0b608b2a2fe6dc1215c9983436 Mon Sep 17 00:00:00 2001 From: digital-pro Date: Wed, 4 Mar 2026 15:42:46 -0800 Subject: [PATCH 03/15] feat: implement functional location selection interactions Add production-like location-selection behavior with browser GPS capture, US-constrained map picking with contextual labels, and country-scoped city/postal autocomplete. Improve participant-facing instructions and persist selected location drafts through review for end-to-end task testing. --- .../tasks/location-selection/helpers/state.ts | 26 ++ .../src/tasks/location-selection/timeline.ts | 2 + .../location-selection/trials/gpsCapture.ts | 81 ++++- .../location-selection/trials/instructions.ts | 12 +- .../location-selection/trials/mapPicker.ts | 223 +++++++++++++- .../trials/reviewAndConfirm.ts | 16 +- .../trials/searchCityPostal.ts | 288 +++++++++++++++++- 7 files changed, 637 insertions(+), 11 deletions(-) create mode 100644 task-launcher/src/tasks/location-selection/helpers/state.ts diff --git a/task-launcher/src/tasks/location-selection/helpers/state.ts b/task-launcher/src/tasks/location-selection/helpers/state.ts new file mode 100644 index 00000000..8217fb3d --- /dev/null +++ b/task-launcher/src/tasks/location-selection/helpers/state.ts @@ -0,0 +1,26 @@ +import { taskStore } from '../../../taskStore'; + +export type LocationSelectionMode = 'gps' | 'map' | 'city_postal'; + +export interface LocationSelectionDraft { + mode: LocationSelectionMode; + lat: number; + lon: number; + label?: string | null; + source?: string | null; + accuracyMeters?: number | null; + metadata?: Record | null; + selectedAt: string; +} + +export function setLocationSelectionDraft(draft: LocationSelectionDraft) { + taskStore('locationSelectionDraft', draft); +} + +export function getLocationSelectionDraft(): LocationSelectionDraft | null { + return (taskStore().locationSelectionDraft as LocationSelectionDraft | null) || null; +} + +export function clearLocationSelectionDraft() { + taskStore('locationSelectionDraft', null); +} diff --git a/task-launcher/src/tasks/location-selection/timeline.ts b/task-launcher/src/tasks/location-selection/timeline.ts index 4571f00d..7b21f6d2 100644 --- a/task-launcher/src/tasks/location-selection/timeline.ts +++ b/task-launcher/src/tasks/location-selection/timeline.ts @@ -10,6 +10,7 @@ import { searchCityPostal } from './trials/searchCityPostal'; import { reviewAndConfirm } from './trials/reviewAndConfirm'; import { getLocationSelectionTaskConfig } from './helpers/config'; import { taskStore } from '../../taskStore'; +import { clearLocationSelectionDraft } from './helpers/state'; export default function buildLocationSelectionTimeline(config: Record, _mediaAssets: MediaAssetsType) { initTrialSaving(config); @@ -19,6 +20,7 @@ export default function buildLocationSelectionTimeline(config: Record[] = [ initialTimeline, diff --git a/task-launcher/src/tasks/location-selection/trials/gpsCapture.ts b/task-launcher/src/tasks/location-selection/trials/gpsCapture.ts index e13258ed..8fe664a3 100644 --- a/task-launcher/src/tasks/location-selection/trials/gpsCapture.ts +++ b/task-launcher/src/tasks/location-selection/trials/gpsCapture.ts @@ -1,5 +1,24 @@ import jsPsychHtmlMultiResponse from '@jspsych-contrib/plugin-html-multi-response'; import { taskStore } from '../../../taskStore'; +import { getLocationSelectionDraft, setLocationSelectionDraft } from '../helpers/state'; + +async function reverseGeocode(lat: number, lon: number): Promise { + try { + const params = new URLSearchParams({ + lat: String(lat), + lon: String(lon), + format: 'jsonv2', + zoom: '10', + addressdetails: '1', + }); + const response = await fetch(`https://nominatim.openstreetmap.org/reverse?${params.toString()}`); + if (!response.ok) return null; + const payload = await response.json(); + return typeof payload?.display_name === 'string' ? payload.display_name : null; + } catch { + return null; + } +} export const gpsCapture = { timeline: [ @@ -9,8 +28,12 @@ export const gpsCapture = {

GPS location

-

This scaffold step represents GPS capture on device.

-

Raw coordinates should be transformed on-device before persistence.

+

Use your browser location to capture coordinates from this device.

+
+ +
+

Waiting for GPS request…

+

`, @@ -22,9 +45,63 @@ export const gpsCapture = { assessment_stage: 'gps_capture', task: 'location-selection', }, + on_load: () => { + const continueButton = document.querySelector('#jspsych-html-multi-response-button-0'); + const gpsButton = document.getElementById('capture-gps-btn') as HTMLButtonElement | null; + const statusEl = document.getElementById('gps-status'); + const valueEl = document.getElementById('gps-value'); + if (continueButton) continueButton.disabled = true; + + if (!navigator.geolocation) { + if (statusEl) statusEl.textContent = 'Geolocation is not supported by this browser.'; + return; + } + + gpsButton?.addEventListener('click', async () => { + if (statusEl) statusEl.textContent = 'Requesting GPS location…'; + navigator.geolocation.getCurrentPosition( + async (position) => { + const lat = Number(position.coords.latitude); + const lon = Number(position.coords.longitude); + const accuracyMeters = Number(position.coords.accuracy); + const label = await reverseGeocode(lat, lon); + setLocationSelectionDraft({ + mode: 'gps', + lat, + lon, + label, + source: 'browser_geolocation', + accuracyMeters: Number.isFinite(accuracyMeters) ? accuracyMeters : null, + metadata: { + altitude: position.coords.altitude ?? null, + speed: position.coords.speed ?? null, + }, + selectedAt: new Date().toISOString(), + }); + if (statusEl) statusEl.textContent = 'GPS location captured.'; + if (valueEl) { + valueEl.textContent = + `${lat.toFixed(5)}, ${lon.toFixed(5)} · accuracy ≈ ${Math.round(accuracyMeters || 0)}m` + + (label ? ` · ${label}` : ''); + } + if (continueButton) continueButton.disabled = false; + }, + (error) => { + if (statusEl) statusEl.textContent = `GPS error: ${error.message}`; + }, + { + enableHighAccuracy: true, + timeout: 15000, + maximumAge: 0, + }, + ); + }); + }, on_finish: (data: Record) => { + const draft = getLocationSelectionDraft(); data.mode = 'gps'; data.geolocationSupported = typeof navigator !== 'undefined' && !!navigator.geolocation; + data.selectedLocation = draft || null; taskStore('locationSelectionLastStep', 'gps_capture'); }, }, diff --git a/task-launcher/src/tasks/location-selection/trials/instructions.ts b/task-launcher/src/tasks/location-selection/trials/instructions.ts index c6422f1b..769bfe63 100644 --- a/task-launcher/src/tasks/location-selection/trials/instructions.ts +++ b/task-launcher/src/tasks/location-selection/trials/instructions.ts @@ -6,9 +6,15 @@ export const instructions = [ stimulus: `
-

Location Selection

-

You can share an approximate location using GPS, map selection, or city/postal search.

-

This task is scaffolded to match core-tasks UI flow and data capture style.

+

Share your location (privacy-first)

+

Pick the method that is easiest for you. You can use GPS, tap a map, or search by city/postal code.

+

We only need an approximate location for planning and analysis.

+
+

What to expect:

+

- GPS: uses your browser location permission.

+

- Map: click a point in the United States.

+

- City/Postal: choose from suggested places after selecting a country.

+
`, diff --git a/task-launcher/src/tasks/location-selection/trials/mapPicker.ts b/task-launcher/src/tasks/location-selection/trials/mapPicker.ts index 828dbed6..4b4ce698 100644 --- a/task-launcher/src/tasks/location-selection/trials/mapPicker.ts +++ b/task-launcher/src/tasks/location-selection/trials/mapPicker.ts @@ -1,5 +1,120 @@ import jsPsychHtmlMultiResponse from '@jspsych-contrib/plugin-html-multi-response'; import { taskStore } from '../../../taskStore'; +import { getLocationSelectionDraft, setLocationSelectionDraft } from '../helpers/state'; + +declare global { + interface Window { + L?: any; + } +} + +let leafletLoader: Promise | null = null; +const US_LOWER_48_BOUNDS: [[number, number], [number, number]] = [[24.396308, -124.848974], [49.384358, -66.885444]]; +const US_LABEL_POINTS: Array<{ name: string; lat: number; lon: number }> = [ + { name: 'Seattle', lat: 47.6062, lon: -122.3321 }, + { name: 'San Francisco', lat: 37.7749, lon: -122.4194 }, + { name: 'Los Angeles', lat: 34.0522, lon: -118.2437 }, + { name: 'Denver', lat: 39.7392, lon: -104.9903 }, + { name: 'Dallas', lat: 32.7767, lon: -96.7970 }, + { name: 'Houston', lat: 29.7604, lon: -95.3698 }, + { name: 'Chicago', lat: 41.8781, lon: -87.6298 }, + { name: 'Atlanta', lat: 33.7490, lon: -84.3880 }, + { name: 'Miami', lat: 25.7617, lon: -80.1918 }, + { name: 'Washington, DC', lat: 38.9072, lon: -77.0369 }, + { name: 'New York', lat: 40.7128, lon: -74.0060 }, + { name: 'Boston', lat: 42.3601, lon: -71.0589 }, +]; +const US_STATE_LABEL_POINTS: Array<{ abbr: string; lat: number; lon: number }> = [ + { abbr: 'AL', lat: 32.8067, lon: -86.7911 }, { abbr: 'AZ', lat: 34.0489, lon: -111.0937 }, + { abbr: 'AR', lat: 35.2010, lon: -91.8318 }, { abbr: 'CA', lat: 36.7783, lon: -119.4179 }, + { abbr: 'CO', lat: 39.5501, lon: -105.7821 }, { abbr: 'CT', lat: 41.6032, lon: -73.0877 }, + { abbr: 'DE', lat: 38.9108, lon: -75.5277 }, { abbr: 'FL', lat: 27.6648, lon: -81.5158 }, + { abbr: 'GA', lat: 32.1574, lon: -82.9071 }, { abbr: 'ID', lat: 44.0682, lon: -114.7420 }, + { abbr: 'IL', lat: 40.6331, lon: -89.3985 }, { abbr: 'IN', lat: 39.7684, lon: -86.1581 }, + { abbr: 'IA', lat: 41.8780, lon: -93.0977 }, { abbr: 'KS', lat: 39.0119, lon: -98.4842 }, + { abbr: 'KY', lat: 37.8393, lon: -84.2700 }, { abbr: 'LA', lat: 30.9843, lon: -91.9623 }, + { abbr: 'ME', lat: 45.2538, lon: -69.4455 }, { abbr: 'MD', lat: 39.0458, lon: -76.6413 }, + { abbr: 'MA', lat: 42.4072, lon: -71.3824 }, { abbr: 'MI', lat: 44.3148, lon: -85.6024 }, + { abbr: 'MN', lat: 46.7296, lon: -94.6859 }, { abbr: 'MS', lat: 32.3547, lon: -89.3985 }, + { abbr: 'MO', lat: 37.9643, lon: -91.8318 }, { abbr: 'MT', lat: 46.8797, lon: -110.3626 }, + { abbr: 'NE', lat: 41.4925, lon: -99.9018 }, { abbr: 'NV', lat: 38.8026, lon: -116.4194 }, + { abbr: 'NH', lat: 43.1939, lon: -71.5724 }, { abbr: 'NJ', lat: 40.0583, lon: -74.4057 }, + { abbr: 'NM', lat: 34.5199, lon: -105.8701 }, { abbr: 'NY', lat: 43.2994, lon: -74.2179 }, + { abbr: 'NC', lat: 35.7596, lon: -79.0193 }, { abbr: 'ND', lat: 47.5515, lon: -101.0020 }, + { abbr: 'OH', lat: 40.4173, lon: -82.9071 }, { abbr: 'OK', lat: 35.0078, lon: -97.0929 }, + { abbr: 'OR', lat: 43.8041, lon: -120.5542 }, { abbr: 'PA', lat: 41.2033, lon: -77.1945 }, + { abbr: 'RI', lat: 41.5801, lon: -71.4774 }, { abbr: 'SC', lat: 33.8361, lon: -81.1637 }, + { abbr: 'SD', lat: 43.9695, lon: -99.9018 }, { abbr: 'TN', lat: 35.5175, lon: -86.5804 }, + { abbr: 'TX', lat: 31.9686, lon: -99.9018 }, { abbr: 'UT', lat: 39.3210, lon: -111.0937 }, + { abbr: 'VT', lat: 44.5588, lon: -72.5778 }, { abbr: 'VA', lat: 37.4316, lon: -78.6569 }, + { abbr: 'WA', lat: 47.7511, lon: -120.7401 }, { abbr: 'WV', lat: 38.5976, lon: -80.4549 }, + { abbr: 'WI', lat: 43.7844, lon: -88.7879 }, { abbr: 'WY', lat: 43.0759, lon: -107.2903 }, +]; + +function ensureMapLabelStyles() { + const styleId = 'location-selection-map-label-styles'; + if (document.getElementById(styleId)) return; + const style = document.createElement('style'); + style.id = styleId; + style.textContent = ` + .location-map-state-label { + background: transparent; + border: none; + box-shadow: none; + color: #334155; + font-size: 11px; + font-weight: 700; + letter-spacing: 0.4px; + } + .location-map-city-label { + background: transparent; + border: none; + box-shadow: none; + color: #1f2937; + font-size: 11px; + font-weight: 500; + } + `; + document.head.appendChild(style); +} + +function loadLeaflet(): Promise { + if (window.L) return Promise.resolve(window.L); + if (leafletLoader) return leafletLoader; + + leafletLoader = new Promise((resolve, reject) => { + const cssId = 'leaflet-css-location-selection'; + if (!document.getElementById(cssId)) { + const link = document.createElement('link'); + link.id = cssId; + link.rel = 'stylesheet'; + link.href = 'https://unpkg.com/leaflet@1.9.4/dist/leaflet.css'; + document.head.appendChild(link); + } + + const scriptId = 'leaflet-js-location-selection'; + const existing = document.getElementById(scriptId) as HTMLScriptElement | null; + if (existing && window.L) { + resolve(window.L); + return; + } + if (existing) { + existing.addEventListener('load', () => resolve(window.L)); + existing.addEventListener('error', () => reject(new Error('Failed to load Leaflet script'))); + return; + } + + const script = document.createElement('script'); + script.id = scriptId; + script.src = 'https://unpkg.com/leaflet@1.9.4/dist/leaflet.js'; + script.async = true; + script.onload = () => resolve(window.L); + script.onerror = () => reject(new Error('Failed to load Leaflet script')); + document.head.appendChild(script); + }); + + return leafletLoader; +} export const mapPicker = { timeline: [ @@ -8,9 +123,10 @@ export const mapPicker = { stimulus: `
-

Map picker

-

This scaffold step represents map-based location selection.

-

Final location should be de-identified before saving.

+

Pick on map (United States)

+

Click your approximate location on the US map. The map is limited to the contiguous US.

+
+

Pick a point on the map.

`, @@ -22,8 +138,109 @@ export const mapPicker = { assessment_stage: 'map_picker', task: 'location-selection', }, + on_load: async () => { + const continueButton = document.querySelector('#jspsych-html-multi-response-button-0'); + const statusEl = document.getElementById('map-picker-status'); + if (continueButton) continueButton.disabled = true; + + try { + const L = await loadLeaflet(); + ensureMapLabelStyles(); + const mapEl = document.getElementById('location-map-picker'); + if (!mapEl) throw new Error('Map container not found'); + const bounds = L.latLngBounds(US_LOWER_48_BOUNDS[0], US_LOWER_48_BOUNDS[1]); + const map = L.map(mapEl, { + zoomControl: true, + attributionControl: false, + maxBounds: bounds.pad(0.12), + maxBoundsViscosity: 1.0, + minZoom: 3, + }); + L.tileLayer('https://{s}.basemaps.cartocdn.com/light_all/{z}/{x}/{y}{r}.png', { + subdomains: 'abcd', + attribution: '© OpenStreetMap contributors © CARTO', + maxZoom: 18, + }).addTo(map); + map.fitBounds(bounds.pad(0.02), { padding: [20, 20] }); + + const cityLabelsLayer = L.layerGroup().addTo(map); + const stateLabelsLayer = L.layerGroup().addTo(map); + + const drawLabelsForZoom = () => { + const zoom = Number(map.getZoom?.() || 4); + cityLabelsLayer.clearLayers(); + stateLabelsLayer.clearLayers(); + + if (zoom <= 6) { + US_STATE_LABEL_POINTS.forEach((point) => { + const marker = L.circleMarker([point.lat, point.lon], { + radius: 1, + color: '#334155', + fillColor: '#334155', + fillOpacity: 0.05, + weight: 0.5, + }).addTo(stateLabelsLayer); + marker.bindTooltip(point.abbr, { + permanent: true, + direction: 'center', + className: 'location-map-state-label', + opacity: 0.9, + }); + marker.openTooltip(); + }); + return; + } + + US_LABEL_POINTS.forEach((point) => { + const marker = L.circleMarker([point.lat, point.lon], { + radius: 2.5, + color: '#1f2937', + fillColor: '#ffffff', + fillOpacity: 0.9, + weight: 1, + }).addTo(cityLabelsLayer); + marker.bindTooltip(point.name, { + permanent: true, + direction: 'right', + offset: [4, 0], + className: 'location-map-city-label', + opacity: 0.9, + }); + marker.openTooltip(); + }); + }; + drawLabelsForZoom(); + map.on('zoomend', drawLabelsForZoom); + + let marker: any = null; + map.on('click', (e: any) => { + const lat = Number(e?.latlng?.lat); + const lon = Number(e?.latlng?.lng); + if (!Number.isFinite(lat) || !Number.isFinite(lon)) return; + if (marker) { + marker.setLatLng([lat, lon]); + } else { + marker = L.marker([lat, lon]).addTo(map); + } + setLocationSelectionDraft({ + mode: 'map', + lat, + lon, + label: null, + source: 'leaflet_map_click', + selectedAt: new Date().toISOString(), + }); + if (statusEl) statusEl.textContent = `Selected: ${lat.toFixed(5)}, ${lon.toFixed(5)}`; + if (continueButton) continueButton.disabled = false; + }); + } catch (error: any) { + if (statusEl) statusEl.textContent = `Map failed to load: ${error?.message || error}`; + } + }, on_finish: (data: Record) => { + const draft = getLocationSelectionDraft(); data.mode = 'map'; + data.selectedLocation = draft || null; taskStore('locationSelectionLastStep', 'map_picker'); }, }, diff --git a/task-launcher/src/tasks/location-selection/trials/reviewAndConfirm.ts b/task-launcher/src/tasks/location-selection/trials/reviewAndConfirm.ts index ebc205bf..9d1e1a18 100644 --- a/task-launcher/src/tasks/location-selection/trials/reviewAndConfirm.ts +++ b/task-launcher/src/tasks/location-selection/trials/reviewAndConfirm.ts @@ -1,16 +1,29 @@ import jsPsychHtmlMultiResponse from '@jspsych-contrib/plugin-html-multi-response'; import { taskStore } from '../../../taskStore'; +import { getLocationSelectionDraft } from '../helpers/state'; export const reviewAndConfirm = { type: jsPsychHtmlMultiResponse, stimulus: () => { const mode = taskStore().locationSelectionMode || 'unknown'; + const draft = getLocationSelectionDraft(); + const draftDetails = draft + ? ` +
+

Mode: ${draft.mode}

+

Coordinates: ${draft.lat.toFixed(5)}, ${draft.lon.toFixed(5)}

+

Source: ${draft.source || 'unknown'}

+ ${draft.label ? `

Label: ${draft.label}

` : ''} + ${draft.accuracyMeters ? `

Accuracy: ~${Math.round(draft.accuracyMeters)}m

` : ''} +
+ ` + : '

No location selected yet.

'; return `

Review selection

Selected mode: ${mode}

-

This task scaffold is ready for integrating GPS/map/search + on-device H3 obfuscation.

+ ${draftDetails}
`; @@ -25,5 +38,6 @@ export const reviewAndConfirm = { }, on_finish: (data: Record) => { data.mode = taskStore().locationSelectionMode || null; + data.selectedLocation = getLocationSelectionDraft(); }, }; diff --git a/task-launcher/src/tasks/location-selection/trials/searchCityPostal.ts b/task-launcher/src/tasks/location-selection/trials/searchCityPostal.ts index ddf8203d..376316cf 100644 --- a/task-launcher/src/tasks/location-selection/trials/searchCityPostal.ts +++ b/task-launcher/src/tasks/location-selection/trials/searchCityPostal.ts @@ -1,5 +1,90 @@ import jsPsychHtmlMultiResponse from '@jspsych-contrib/plugin-html-multi-response'; import { taskStore } from '../../../taskStore'; +import { getLocationSelectionDraft, setLocationSelectionDraft } from '../helpers/state'; + +interface NominatimResult { + place_id?: number; + display_name?: string; + lat?: string; + lon?: string; + type?: string; +} + +interface CountryOption { + code: string; + label: string; +} + +const SUPPORTED_COUNTRY_CODES: string[] = ['US', 'DE', 'GB', 'NL', 'CA', 'CO', 'IN', 'AR', 'GH', 'CH']; +const SUPPORTED_COUNTRY_NAMES: Record = { + US: 'United States', + DE: 'Germany', + GB: 'United Kingdom', + NL: 'Netherlands', + CA: 'Canada', + CO: 'Colombia', + IN: 'India', + AR: 'Argentina', + GH: 'Ghana', + CH: 'Switzerland', +}; + +function escapeHtml(value: string): string { + return String(value || '') + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"') + .replace(/'/g, '''); +} + +function escapeRegex(value: string): string { + return String(value || '').replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +function highlightLabel(label: string, query: string): string { + const safeLabel = escapeHtml(label); + const q = String(query || '').trim(); + if (!q || q.length < 2) return safeLabel; + const expr = new RegExp(`(${escapeRegex(q)})`, 'ig'); + return safeLabel.replace(expr, '$1'); +} + +function getCountryLabel(code: string): string { + const iso = String(code || '').trim().toUpperCase(); + const explicitName = SUPPORTED_COUNTRY_NAMES[iso]; + if (explicitName) return explicitName; + try { + if (typeof Intl !== 'undefined' && (Intl as any).DisplayNames) { + const dn = new Intl.DisplayNames(['en'], { type: 'region' }); + return dn.of(iso) || iso; + } + } catch { + // Ignore and use fallback. + } + return iso; +} + +async function loadCountryOptions(): Promise { + return SUPPORTED_COUNTRY_CODES.map((code) => ({ + code, + label: `${getCountryLabel(code)} — ${code}`, + })); +} + +async function searchLocations(query: string, countryCode?: string): Promise { + const params = new URLSearchParams({ + q: query, + format: 'jsonv2', + addressdetails: '1', + limit: '10', + }); + if (countryCode) params.set('countrycodes', String(countryCode || '').toLowerCase()); + const response = await fetch(`https://nominatim.openstreetmap.org/search?${params.toString()}`); + if (!response.ok) return []; + const payload = await response.json(); + return Array.isArray(payload) ? payload : []; +} export const searchCityPostal = { timeline: [ @@ -9,8 +94,17 @@ export const searchCityPostal = {

City / postal search

-

This scaffold step represents typing a city or postal code.

-

Use on-device obfuscation before storing final location.

+

Select a country, then type a city/town or postal code.

+
+ + +
+
+ + + +
+

Choose a country to begin.

`, @@ -22,8 +116,198 @@ export const searchCityPostal = { assessment_stage: 'city_postal_search', task: 'location-selection', }, + on_load: () => { + const continueButton = document.querySelector('#jspsych-html-multi-response-button-0'); + const inputEl = document.getElementById('location-query-input') as HTMLInputElement | null; + const countryEl = document.getElementById('location-country-select') as HTMLSelectElement | null; + const statusEl = document.getElementById('location-search-status'); + const dropdownEl = document.getElementById('location-autocomplete-dropdown'); + + if (continueButton) continueButton.disabled = true; + let selectedCountry = 'US'; + let debounceHandle: number | null = null; + let latestRequestId = 0; + let latestResults: NominatimResult[] = []; + let highlightedIndex = -1; + let latestQuery = ''; + + const hideDropdown = () => { + if (!dropdownEl) return; + dropdownEl.style.display = 'none'; + dropdownEl.innerHTML = ''; + highlightedIndex = -1; + }; + + const selectResult = (selected: NominatimResult) => { + const lat = Number(selected?.lat); + const lon = Number(selected?.lon); + if (!Number.isFinite(lat) || !Number.isFinite(lon)) return; + setLocationSelectionDraft({ + mode: 'city_postal', + lat, + lon, + label: String(selected.display_name || ''), + source: 'nominatim_search', + metadata: { + placeId: selected.place_id ?? null, + resultType: selected.type ?? null, + countryCode: selectedCountry, + }, + selectedAt: new Date().toISOString(), + }); + if (statusEl) statusEl.textContent = `Selected: ${selected.display_name || `${lat.toFixed(5)}, ${lon.toFixed(5)}`}`; + if (inputEl) inputEl.value = String(selected.display_name || inputEl.value); + hideDropdown(); + if (continueButton) continueButton.disabled = false; + }; + + const renderResults = (results: NominatimResult[]) => { + if (!dropdownEl) return; + latestResults = results.slice(); + if (highlightedIndex >= latestResults.length) highlightedIndex = latestResults.length - 1; + if (!results.length) { + dropdownEl.innerHTML = '
No matches found.
'; + dropdownEl.style.display = 'block'; + return; + } + dropdownEl.innerHTML = results + .map((result, index) => { + const label = String(result.display_name || 'Unknown location'); + const bg = index === highlightedIndex ? '#eff6ff' : '#fff'; + const labelHtml = highlightLabel(label, latestQuery); + return ``; + }) + .join(''); + dropdownEl.style.display = 'block'; + if (highlightedIndex >= 0) { + const active = dropdownEl.querySelector(`button[data-result-index="${highlightedIndex}"]`); + active?.scrollIntoView({ block: 'nearest' }); + } + dropdownEl.querySelectorAll('button[data-result-index]').forEach((button) => { + button.addEventListener('mouseenter', () => { + const idx = Number(button.dataset.resultIndex); + highlightedIndex = Number.isFinite(idx) ? idx : -1; + renderResults(latestResults); + }); + button.addEventListener('click', () => { + const idx = Number(button.dataset.resultIndex); + const selected = latestResults[idx]; + if (!selected) return; + selectResult(selected); + }); + }); + }; + + const runSearch = async () => { + const query = String(inputEl?.value || '').trim(); + latestQuery = query; + if (!selectedCountry) { + hideDropdown(); + if (statusEl) statusEl.textContent = 'Select a country first.'; + return; + } + if (!query) { + hideDropdown(); + if (statusEl) statusEl.textContent = 'Type at least 2 characters.'; + return; + } + if (query.length < 2) { + hideDropdown(); + return; + } + if (statusEl) statusEl.textContent = `Searching in ${selectedCountry}…`; + const requestId = latestRequestId + 1; + latestRequestId = requestId; + const results = await searchLocations(query, selectedCountry); + if (requestId !== latestRequestId) return; + highlightedIndex = results.length ? 0 : -1; + renderResults(results); + if (statusEl) statusEl.textContent = results.length ? 'Pick the best match from the dropdown.' : 'No matches found.'; + }; + + loadCountryOptions() + .then((countries) => { + if (!countryEl) return; + countryEl.innerHTML = countries + .map((country) => ``) + .join(''); + selectedCountry = countryEl.value || 'US'; + if (statusEl) statusEl.textContent = 'Country selected. Start typing a city or postal code.'; + }) + .catch((error: any) => { + if (statusEl) statusEl.textContent = `Could not load full country list (${error?.message || error}).`; + }); + + countryEl?.addEventListener('change', () => { + selectedCountry = String(countryEl.value || '').toUpperCase(); + if (continueButton) continueButton.disabled = true; + hideDropdown(); + if (inputEl) inputEl.value = ''; + taskStore('locationSelectionDraft', null); + latestResults = []; + if (statusEl) statusEl.textContent = `Country set to ${selectedCountry}. Start typing to see matches.`; + }); + + inputEl?.addEventListener('input', () => { + if (debounceHandle) { + window.clearTimeout(debounceHandle); + } + debounceHandle = window.setTimeout(() => { + runSearch().catch((error: any) => { + if (statusEl) statusEl.textContent = `Search failed: ${error?.message || error}`; + }); + }, 250); + }); + + inputEl?.addEventListener('focus', () => { + const hasOptions = Boolean(dropdownEl && dropdownEl.innerHTML.trim()); + if (hasOptions && dropdownEl) dropdownEl.style.display = 'block'; + }); + + document.addEventListener('click', (event) => { + const target = event.target as HTMLElement | null; + if (!target) return; + const clickedInside = Boolean(target.closest('#location-autocomplete-dropdown') || target.closest('#location-query-input')); + if (!clickedInside) hideDropdown(); + }); + + inputEl?.addEventListener('keydown', (event: KeyboardEvent) => { + if (event.key === 'ArrowDown') { + if (latestResults.length) { + event.preventDefault(); + highlightedIndex = Math.min(latestResults.length - 1, highlightedIndex + 1); + renderResults(latestResults); + } + return; + } + if (event.key === 'ArrowUp') { + if (latestResults.length) { + event.preventDefault(); + highlightedIndex = Math.max(0, highlightedIndex - 1); + renderResults(latestResults); + } + return; + } + if (event.key === 'Escape') { + hideDropdown(); + return; + } + if (event.key === 'Enter') { + event.preventDefault(); + if (highlightedIndex >= 0 && highlightedIndex < latestResults.length) { + selectResult(latestResults[highlightedIndex]); + return; + } + runSearch().catch((error: any) => { + if (statusEl) statusEl.textContent = `Search failed: ${error?.message || error}`; + }); + } + }); + }, on_finish: (data: Record) => { + const draft = getLocationSelectionDraft(); data.mode = 'city_postal'; + data.selectedLocation = draft || null; taskStore('locationSelectionLastStep', 'city_postal_search'); }, }, From cfd0fda85354fdf4131aea7606bbb26119e6012d Mon Sep 17 00:00:00 2001 From: digital-pro Date: Wed, 4 Mar 2026 16:53:10 -0800 Subject: [PATCH 04/15] feat: add privacy-safe location preview and real population lookup Upgrade the location-selection flow with a first-screen mode chooser, improved layout readability, and a computed Location schema preview that hides raw coordinates. Add real Kontur-first and WorldPop fallback population APIs for effective H3 resolution selection in review. --- task-launcher/package-lock.json | 35 +-- task-launcher/package.json | 1 + task-launcher/serve/populationApi.cjs | 212 ++++++++++++++++++ .../location-selection/helpers/config.ts | 18 ++ .../helpers/locationCommitPreview.ts | 140 ++++++++++++ .../helpers/populationApi.ts | 89 ++++++++ .../tasks/location-selection/helpers/ui.ts | 58 +++++ .../src/tasks/location-selection/timeline.ts | 3 +- .../location-selection/trials/gpsCapture.ts | 10 +- .../location-selection/trials/instructions.ts | 33 ++- .../location-selection/trials/mapPicker.ts | 48 +++- .../location-selection/trials/modeSelect.ts | 6 +- .../trials/reviewAndConfirm.ts | 54 ++++- .../trials/searchCityPostal.ts | 10 +- task-launcher/webpack.config.cjs | 7 + 15 files changed, 672 insertions(+), 52 deletions(-) create mode 100644 task-launcher/serve/populationApi.cjs create mode 100644 task-launcher/src/tasks/location-selection/helpers/locationCommitPreview.ts create mode 100644 task-launcher/src/tasks/location-selection/helpers/populationApi.ts create mode 100644 task-launcher/src/tasks/location-selection/helpers/ui.ts diff --git a/task-launcher/package-lock.json b/task-launcher/package-lock.json index 7093fed5..4a60124b 100644 --- a/task-launcher/package-lock.json +++ b/task-launcher/package-lock.json @@ -23,6 +23,7 @@ "@sentry/browser": "^8.7.0", "cypress-real-events": "^1.13.0", "fscreen": "^1.2.0", + "h3-js": "^4.4.0", "i18next": "^22.4.15", "i18next-browser-languagedetector": "^7.0.1", "jspsych": "^7.2.1", @@ -7867,29 +7868,6 @@ "node": ">= 0.8" } }, - "node_modules/encoding": { - "version": "0.1.13", - "resolved": "https://registry.npmjs.org/encoding/-/encoding-0.1.13.tgz", - "integrity": "sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A==", - "optional": true, - "peer": true, - "dependencies": { - "iconv-lite": "^0.6.2" - } - }, - "node_modules/encoding/node_modules/iconv-lite": { - "version": "0.6.3", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", - "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", - "optional": true, - "peer": true, - "dependencies": { - "safer-buffer": ">= 2.1.2 < 3.0.0" - }, - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/end-of-stream": { "version": "1.4.4", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", @@ -9609,6 +9587,17 @@ "integrity": "sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==", "dev": true }, + "node_modules/h3-js": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/h3-js/-/h3-js-4.4.0.tgz", + "integrity": "sha512-DvJh07MhGgY2KcC4OeZc8SSyA+ZXpdvoh6uCzGpoKvWtZxJB+g6VXXC1+eWYkaMIsLz7J/ErhOalHCpcs1KYog==", + "license": "Apache-2.0", + "engines": { + "node": ">=4", + "npm": ">=3", + "yarn": ">=1.3.0" + } + }, "node_modules/handle-thing": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/handle-thing/-/handle-thing-2.0.1.tgz", diff --git a/task-launcher/package.json b/task-launcher/package.json index ca060183..86908032 100644 --- a/task-launcher/package.json +++ b/task-launcher/package.json @@ -52,6 +52,7 @@ "@sentry/browser": "^8.7.0", "cypress-real-events": "^1.13.0", "fscreen": "^1.2.0", + "h3-js": "^4.4.0", "i18next": "^22.4.15", "i18next-browser-languagedetector": "^7.0.1", "jspsych": "^7.2.1", diff --git a/task-launcher/serve/populationApi.cjs b/task-launcher/serve/populationApi.cjs new file mode 100644 index 00000000..340d9151 --- /dev/null +++ b/task-launcher/serve/populationApi.cjs @@ -0,0 +1,212 @@ +const fs = require('fs'); +const path = require('path'); +const { cellToBoundary } = require('h3-js'); + +const WORLDPOP_STATS_URL = 'https://api.worldpop.org/v1/services/stats'; +const WORLDPOP_TASK_URL = 'https://api.worldpop.org/v1/tasks'; + +let konturCacheByResolution = null; +let konturCacheLoadedFrom = null; + +function sendJson(res, statusCode, payload) { + res.status(statusCode).set('Content-Type', 'application/json').send(JSON.stringify(payload)); +} + +function parsePositiveInt(value, fallback) { + const n = Number(value); + if (Number.isFinite(n) && n >= 0) return Math.round(n); + return fallback; +} + +function parseResolution(value) { + const n = Number(value); + if (!Number.isInteger(n) || n < 0 || n > 15) return null; + return n; +} + +function buildCellPolygon(cellId) { + const boundary = cellToBoundary(cellId); + if (!Array.isArray(boundary) || !boundary.length) { + throw new Error('Invalid H3 cell boundary'); + } + const ring = boundary.map((pair) => [Number(pair[1]), Number(pair[0])]); + const first = ring[0]; + const last = ring[ring.length - 1]; + if (!first || !last || first[0] !== last[0] || first[1] !== last[1]) { + ring.push([first[0], first[1]]); + } + return { + type: 'Polygon', + coordinates: [ring], + }; +} + +function findKonturCachePath() { + const envPath = String(process.env.KONTUR_H3_CACHE_PATH || '').trim(); + if (envPath) return envPath; + const candidates = [ + path.resolve(process.cwd(), 'data', 'gallery', 'kontur-h3-population-cache.json'), + path.resolve(process.cwd(), '..', 'levante-web-dashboard', 'data', 'gallery', 'kontur-h3-population-cache.json'), + path.resolve(process.cwd(), '..', '..', 'levante-web-dashboard', 'data', 'gallery', 'kontur-h3-population-cache.json'), + ]; + return candidates.find((candidate) => fs.existsSync(candidate)) || ''; +} + +function loadKonturCache() { + if (konturCacheByResolution) return konturCacheByResolution; + const cachePath = findKonturCachePath(); + if (!cachePath || !fs.existsSync(cachePath)) return null; + try { + const raw = fs.readFileSync(cachePath, 'utf-8'); + const json = JSON.parse(raw); + const resolutions = json?.resolutions; + if (!resolutions || typeof resolutions !== 'object') return null; + konturCacheByResolution = resolutions; + konturCacheLoadedFrom = cachePath; + return konturCacheByResolution; + } catch (_err) { + return null; + } +} + +async function wait(ms) { + await new Promise((resolve) => setTimeout(resolve, ms)); +} + +function parseWorldPopSum(payload) { + const candidates = [ + payload?.stats?.sum, + payload?.data?.stats?.sum, + payload?.result?.stats?.sum, + ]; + for (let i = 0; i < candidates.length; i += 1) { + const value = Number(candidates[i]); + if (Number.isFinite(value) && value >= 0) return value; + } + return null; +} + +function parseWorldPopTaskId(payload) { + const candidates = [payload?.taskid, payload?.taskId, payload?.task_id, payload?.id]; + for (let i = 0; i < candidates.length; i += 1) { + const value = String(candidates[i] || '').trim(); + if (value) return value; + } + return null; +} + +async function queryWorldPopForPolygon(polygon, year) { + const url = new URL(WORLDPOP_STATS_URL); + url.searchParams.set('dataset', 'wpgppop'); + url.searchParams.set('year', String(year)); + url.searchParams.set('geojson', JSON.stringify(polygon)); + url.searchParams.set('runasync', 'false'); + const firstResponse = await fetch(url.toString(), { method: 'GET' }); + if (!firstResponse.ok) { + throw new Error(`WorldPop stats request failed (${firstResponse.status})`); + } + const firstPayload = await firstResponse.json().catch(() => ({})); + const directSum = parseWorldPopSum(firstPayload); + if (typeof directSum === 'number') return directSum; + + const taskId = parseWorldPopTaskId(firstPayload); + if (!taskId) { + throw new Error('WorldPop response missing stats and task id'); + } + + for (let attempt = 0; attempt < 8; attempt += 1) { + const taskResponse = await fetch(`${WORLDPOP_TASK_URL}/${encodeURIComponent(taskId)}`); + if (!taskResponse.ok) { + throw new Error(`WorldPop task polling failed (${taskResponse.status})`); + } + const taskPayload = await taskResponse.json().catch(() => ({})); + const sum = parseWorldPopSum(taskPayload); + if (typeof sum === 'number') return sum; + + const status = String(taskPayload?.status || taskPayload?.state || '').toLowerCase(); + if (status.includes('failed') || status.includes('error')) { + throw new Error(`WorldPop task ${taskId} failed`); + } + await wait(600); + } + + throw new Error(`WorldPop task ${taskId} timed out`); +} + +function resolveKonturPopulation(cellId, resolution) { + const cache = loadKonturCache(); + if (!cache) return null; + const byRes = cache[String(resolution)]; + if (!byRes || typeof byRes !== 'object') return null; + const value = Number(byRes[cellId]); + if (!Number.isFinite(value) || value < 0) return null; + return Math.round(value); +} + +function registerPopulationApi(app) { + app.get('/api/population-kontur-h3', async (req, res) => { + try { + const cellId = String(req.query?.cellId || '').trim(); + const resolution = parseResolution(req.query?.resolution); + const worldpopYear = parsePositiveInt(req.query?.year, 2020); + if (!cellId || resolution == null) { + sendJson(res, 400, { success: false, error: 'Missing/invalid cellId or resolution' }); + return; + } + + const konturPopulation = resolveKonturPopulation(cellId, resolution); + if (typeof konturPopulation === 'number') { + sendJson(res, 200, { + success: true, + source: 'kontur', + population: konturPopulation, + resolution, + cellId, + cachePath: konturCacheLoadedFrom || null, + }); + return; + } + + // Real-data fallback: use WorldPop when Kontur cache not available for this cell. + const polygon = buildCellPolygon(cellId); + const worldpopPopulation = await queryWorldPopForPolygon(polygon, worldpopYear); + sendJson(res, 200, { + success: true, + source: 'worldpop', + fallbackFrom: 'kontur', + population: Math.round(worldpopPopulation), + resolution, + cellId, + }); + } catch (error) { + sendJson(res, 500, { success: false, error: error?.message || 'Unknown error' }); + } + }); + + app.get('/api/population-worldpop-h3', async (req, res) => { + try { + const cellId = String(req.query?.cellId || '').trim(); + const resolution = parseResolution(req.query?.resolution); + const worldpopYear = parsePositiveInt(req.query?.year, 2020); + if (!cellId || resolution == null) { + sendJson(res, 400, { success: false, error: 'Missing/invalid cellId or resolution' }); + return; + } + const polygon = buildCellPolygon(cellId); + const population = await queryWorldPopForPolygon(polygon, worldpopYear); + sendJson(res, 200, { + success: true, + source: 'worldpop', + population: Math.round(population), + resolution, + cellId, + }); + } catch (error) { + sendJson(res, 500, { success: false, error: error?.message || 'Unknown error' }); + } + }); +} + +module.exports = { + registerPopulationApi, +}; diff --git a/task-launcher/src/tasks/location-selection/helpers/config.ts b/task-launcher/src/tasks/location-selection/helpers/config.ts index dafe95e5..e41800dc 100644 --- a/task-launcher/src/tasks/location-selection/helpers/config.ts +++ b/task-launcher/src/tasks/location-selection/helpers/config.ts @@ -2,12 +2,20 @@ export type LocationSelectionTaskConfig = { populationThreshold: number; baselineResolution: number; maxResolution: number; + populationSourcePreference: 'kontur' | 'worldpop' | 'auto'; + konturPopulationApiUrl: string; + worldpopPopulationApiUrl: string; + populationApiTimeoutMs: number; }; export function getLocationSelectionTaskConfig(config: Record): LocationSelectionTaskConfig { const threshold = Number(config?.populationThreshold); const baseline = Number(config?.baselineResolution); const maxRes = Number(config?.maxResolution); + const sourcePreference = String(config?.populationSourcePreference || 'auto').trim().toLowerCase(); + const konturPopulationApiUrl = String(config?.konturPopulationApiUrl || '/api/population-kontur-h3').trim(); + const worldpopPopulationApiUrl = String(config?.worldpopPopulationApiUrl || '/api/population-worldpop-h3').trim(); + const populationApiTimeoutMs = Number(config?.populationApiTimeoutMs); return { populationThreshold: Number.isFinite(threshold) && threshold > 0 ? Math.round(threshold) : 50000, @@ -19,5 +27,15 @@ export function getLocationSelectionTaskConfig(config: Record): Loc Number.isFinite(maxRes) && Number.isInteger(maxRes) && maxRes >= 0 && maxRes <= 15 ? maxRes : 9, + populationSourcePreference: + sourcePreference === 'kontur' || sourcePreference === 'worldpop' || sourcePreference === 'auto' + ? sourcePreference + : 'auto', + konturPopulationApiUrl, + worldpopPopulationApiUrl, + populationApiTimeoutMs: + Number.isFinite(populationApiTimeoutMs) && populationApiTimeoutMs > 0 + ? Math.round(populationApiTimeoutMs) + : 2500, }; } diff --git a/task-launcher/src/tasks/location-selection/helpers/locationCommitPreview.ts b/task-launcher/src/tasks/location-selection/helpers/locationCommitPreview.ts new file mode 100644 index 00000000..8e118c1c --- /dev/null +++ b/task-launcher/src/tasks/location-selection/helpers/locationCommitPreview.ts @@ -0,0 +1,140 @@ +import { cellToLatLng, latLngToCell } from 'h3-js'; +import { type LocationSelectionDraft } from './state'; +import { type LocationSelectionTaskConfig } from './config'; +import { lookupPopulationForCell } from './populationApi'; + +type LocationCommitPreview = { + schemaVersion: 'location_v1'; + latLon: { + lat: number; + lon: number; + source: 'h3_center'; + }; + h3: { + scheme: 'h3_v1'; + baseline: { + cellId: string; + resolution: number; + }; + effective: { + cellId: string; + resolution: number; + }; + populationThreshold: number; + }; + populationSource: 'kontur' | 'worldpop' | 'unknown'; + computedAt: string; +}; + +function roundTo(value: number, decimals = 6): number { + const factor = 10 ** decimals; + return Math.round(value * factor) / factor; +} + +export function buildLocationCommitPreview( + draft: LocationSelectionDraft | null, + config: Partial | null | undefined, +): LocationCommitPreview | null { + if (!draft) return null; + + const baselineResolution = Number(config?.baselineResolution); + const populationThreshold = Number(config?.populationThreshold); + const safeBaselineResolution = Number.isInteger(baselineResolution) ? baselineResolution : 5; + const safePopulationThreshold = Number.isFinite(populationThreshold) && populationThreshold > 0 + ? Math.round(populationThreshold) + : 50000; + + const baselineCell = latLngToCell(draft.lat, draft.lon, safeBaselineResolution); + + const effectiveCell = baselineCell; + const [centerLat, centerLon] = cellToLatLng(effectiveCell); + + return { + schemaVersion: 'location_v1', + latLon: { + lat: roundTo(centerLat, 6), + lon: roundTo(centerLon, 6), + source: 'h3_center', + }, + h3: { + scheme: 'h3_v1', + baseline: { + cellId: baselineCell, + resolution: safeBaselineResolution, + }, + effective: { + cellId: effectiveCell, + resolution: safeBaselineResolution, + }, + populationThreshold: safePopulationThreshold, + }, + populationSource: 'unknown', + computedAt: draft.selectedAt || new Date().toISOString(), + }; +} + +export async function buildLocationCommitPreviewWithPopulation( + draft: LocationSelectionDraft | null, + config: Partial | null | undefined, +): Promise { + if (!draft) return null; + + const baselineResolution = Number(config?.baselineResolution); + const maxResolution = Number(config?.maxResolution); + const populationThreshold = Number(config?.populationThreshold); + const safeBaselineResolution = Number.isInteger(baselineResolution) ? baselineResolution : 5; + const safeMaxResolution = + Number.isInteger(maxResolution) && maxResolution >= safeBaselineResolution ? maxResolution : Math.max(9, safeBaselineResolution); + const safePopulationThreshold = Number.isFinite(populationThreshold) && populationThreshold > 0 + ? Math.round(populationThreshold) + : 50000; + + const baselineCell = latLngToCell(draft.lat, draft.lon, safeBaselineResolution); + let effectiveCell = baselineCell; + let effectiveResolution = safeBaselineResolution; + let effectivePopulationSource: 'kontur' | 'worldpop' | 'unknown' = 'unknown'; + let foundPassingCell = false; + + for (let resolution = safeBaselineResolution; resolution <= safeMaxResolution; resolution += 1) { + const cellId = latLngToCell(draft.lat, draft.lon, resolution); + const populationResult = await lookupPopulationForCell(cellId, resolution, config); + const population = populationResult.population; + const privacyMet = typeof population === 'number' ? population >= safePopulationThreshold : false; + + if (privacyMet) { + effectiveCell = cellId; + effectiveResolution = resolution; + effectivePopulationSource = populationResult.source; + foundPassingCell = true; + continue; + } + + // Privacy-first: once we found an acceptable cell, stop at first finer failure. + if (foundPassingCell) break; + } + + const [centerLat, centerLon] = cellToLatLng(effectiveCell); + + return { + schemaVersion: 'location_v1', + latLon: { + lat: roundTo(centerLat, 6), + lon: roundTo(centerLon, 6), + source: 'h3_center', + }, + h3: { + scheme: 'h3_v1', + baseline: { + cellId: baselineCell, + resolution: safeBaselineResolution, + }, + effective: { + cellId: effectiveCell, + resolution: effectiveResolution, + }, + populationThreshold: safePopulationThreshold, + }, + populationSource: effectivePopulationSource, + computedAt: draft.selectedAt || new Date().toISOString(), + }; +} diff --git a/task-launcher/src/tasks/location-selection/helpers/populationApi.ts b/task-launcher/src/tasks/location-selection/helpers/populationApi.ts new file mode 100644 index 00000000..a6bf9bb3 --- /dev/null +++ b/task-launcher/src/tasks/location-selection/helpers/populationApi.ts @@ -0,0 +1,89 @@ +type PopulationSource = 'kontur' | 'worldpop'; + +export type PopulationLookupConfig = { + populationSourcePreference?: 'kontur' | 'worldpop' | 'auto'; + konturPopulationApiUrl?: string; + worldpopPopulationApiUrl?: string; + populationApiTimeoutMs?: number; +}; + +type PopulationLookupResult = { + population: number | null; + source: PopulationSource | 'unknown'; +}; + +function parsePopulation(payload: any): number | null { + const candidates = [ + payload?.population, + payload?.pop, + payload?.estimatedPopulation, + payload?.data?.population, + payload?.result?.population, + ]; + for (let i = 0; i < candidates.length; i += 1) { + const n = Number(candidates[i]); + if (Number.isFinite(n) && n >= 0) return n; + } + return null; +} + +async function fetchPopulation( + endpoint: string, + source: PopulationSource, + cellId: string, + resolution: number, + timeoutMs: number, +): Promise { + if (!endpoint) return { population: null, source: 'unknown' }; + + try { + const url = new URL(endpoint, window.location.origin); + url.searchParams.set('cellId', cellId); + url.searchParams.set('resolution', String(resolution)); + + const controller = new AbortController(); + const timeout = window.setTimeout(() => controller.abort(), timeoutMs); + const response = await fetch(url.toString(), { + method: 'GET', + signal: controller.signal, + }); + window.clearTimeout(timeout); + if (!response.ok) return { population: null, source: 'unknown' }; + const payload = await response.json().catch(() => ({})); + const resolvedSourceRaw = String(payload?.source || '').trim().toLowerCase(); + const resolvedSource: PopulationSource | 'unknown' = + resolvedSourceRaw === 'kontur' || resolvedSourceRaw === 'worldpop' + ? resolvedSourceRaw + : source; + return { population: parsePopulation(payload), source: resolvedSource }; + } catch { + return { population: null, source: 'unknown' }; + } +} + +export async function lookupPopulationForCell( + cellId: string, + resolution: number, + config: PopulationLookupConfig | null | undefined, +): Promise { + const preference = String(config?.populationSourcePreference || 'auto').toLowerCase(); + const konturUrl = String(config?.konturPopulationApiUrl || '/api/population-kontur-h3'); + const worldpopUrl = String(config?.worldpopPopulationApiUrl || '/api/population-worldpop-h3'); + const timeoutMs = Number(config?.populationApiTimeoutMs) > 0 ? Number(config?.populationApiTimeoutMs) : 2500; + + const orderedSources: PopulationSource[] = + preference === 'kontur' + ? ['kontur', 'worldpop'] + : preference === 'worldpop' + ? ['worldpop', 'kontur'] + : ['kontur', 'worldpop']; + + for (let i = 0; i < orderedSources.length; i += 1) { + const source = orderedSources[i]; + const endpoint = source === 'kontur' ? konturUrl : worldpopUrl; + const result = await fetchPopulation(endpoint, source, cellId, resolution, timeoutMs); + if (typeof result.population === 'number') return result; + } + + return { population: null, source: 'unknown' }; +} diff --git a/task-launcher/src/tasks/location-selection/helpers/ui.ts b/task-launcher/src/tasks/location-selection/helpers/ui.ts new file mode 100644 index 00000000..9b6b46f6 --- /dev/null +++ b/task-launcher/src/tasks/location-selection/helpers/ui.ts @@ -0,0 +1,58 @@ +export function ensureLocationSelectionStyles() { + const styleId = 'location-selection-shared-ui-styles'; + if (document.getElementById(styleId)) return; + + const style = document.createElement('style'); + style.id = styleId; + style.textContent = ` + .location-selection-panel { + width: min(92vw, 680px); + max-width: 680px; + margin: 0 auto 0 0; + text-align: left; + box-sizing: border-box; + } + .location-selection-copy h2 { + margin: 0 0 0.6rem 0; + line-height: 1.25; + font-size: 1.35rem; + } + .location-selection-copy p { + margin: 0 0 0.7rem 0; + line-height: 1.45; + font-size: 0.96rem; + } + .location-selection-note { + margin-top: 0.8rem; + padding: 0.75rem 0.9rem; + background: #f8fafc; + border: 1px solid #e2e8f0; + border-radius: 8px; + } + .location-selection-note p:last-child, + .location-selection-copy p:last-child { + margin-bottom: 0; + } + .location-selection-field { + margin-top: 0.85rem; + } + .location-selection-field label { + display: block; + margin-bottom: 0.3rem; + font-weight: 600; + } + .location-selection-status { + margin-top: 0.8rem; + min-height: 1.4em; + line-height: 1.35; + } + .location-selection-intro h2 { + font-size: 1.25rem; + } + .location-selection-intro p { + font-size: 0.93rem; + line-height: 1.4; + } + `; + document.head.appendChild(style); +} diff --git a/task-launcher/src/tasks/location-selection/timeline.ts b/task-launcher/src/tasks/location-selection/timeline.ts index 7b21f6d2..6694378f 100644 --- a/task-launcher/src/tasks/location-selection/timeline.ts +++ b/task-launcher/src/tasks/location-selection/timeline.ts @@ -3,7 +3,6 @@ import { initTrialSaving, initTimeline } from '../shared/helpers'; import { jsPsych } from '../taskSetup'; import { enterFullscreen, exitFullscreen, finishExperiment, taskFinished } from '../shared/trials'; import { instructions } from './trials/instructions'; -import { modeSelect } from './trials/modeSelect'; import { gpsCapture } from './trials/gpsCapture'; import { mapPicker } from './trials/mapPicker'; import { searchCityPostal } from './trials/searchCityPostal'; @@ -20,12 +19,12 @@ export default function buildLocationSelectionTimeline(config: Record[] = [ initialTimeline, ...instructions, - modeSelect, gpsCapture, mapPicker, searchCityPostal, diff --git a/task-launcher/src/tasks/location-selection/trials/gpsCapture.ts b/task-launcher/src/tasks/location-selection/trials/gpsCapture.ts index 8fe664a3..467685d7 100644 --- a/task-launcher/src/tasks/location-selection/trials/gpsCapture.ts +++ b/task-launcher/src/tasks/location-selection/trials/gpsCapture.ts @@ -1,6 +1,7 @@ import jsPsychHtmlMultiResponse from '@jspsych-contrib/plugin-html-multi-response'; import { taskStore } from '../../../taskStore'; import { getLocationSelectionDraft, setLocationSelectionDraft } from '../helpers/state'; +import { ensureLocationSelectionStyles } from '../helpers/ui'; async function reverseGeocode(lat: number, lon: number): Promise { try { @@ -26,14 +27,14 @@ export const gpsCapture = { type: jsPsychHtmlMultiResponse, stimulus: `
-
+

GPS location

Use your browser location to capture coordinates from this device.

-
+
-

Waiting for GPS request…

-

+

Waiting for GPS request…

+

`, @@ -46,6 +47,7 @@ export const gpsCapture = { task: 'location-selection', }, on_load: () => { + ensureLocationSelectionStyles(); const continueButton = document.querySelector('#jspsych-html-multi-response-button-0'); const gpsButton = document.getElementById('capture-gps-btn') as HTMLButtonElement | null; const statusEl = document.getElementById('gps-status'); diff --git a/task-launcher/src/tasks/location-selection/trials/instructions.ts b/task-launcher/src/tasks/location-selection/trials/instructions.ts index 769bfe63..ae135441 100644 --- a/task-launcher/src/tasks/location-selection/trials/instructions.ts +++ b/task-launcher/src/tasks/location-selection/trials/instructions.ts @@ -1,30 +1,43 @@ import jsPsychHtmlMultiResponse from '@jspsych-contrib/plugin-html-multi-response'; +import { taskStore } from '../../../taskStore'; +import { ensureLocationSelectionStyles } from '../helpers/ui'; + +const modes = ['gps', 'map', 'city_postal'] as const; export const instructions = [ { type: jsPsychHtmlMultiResponse, stimulus: `
-
-

Share your location (privacy-first)

-

Pick the method that is easiest for you. You can use GPS, tap a map, or search by city/postal code.

+
+

How would you like to share location?

+

Choose the method that is easiest for you. You can use GPS, tap a map, or search by city/postal code.

We only need an approximate location for planning and analysis.

-
-

What to expect:

-

- GPS: uses your browser location permission.

-

- Map: click a point in the United States.

-

- City/Postal: choose from suggested places after selecting a country.

+
+

What to expect:

+

- GPS: uses your browser location permission.

+

- Map: click a point in the United States.

+

- City/Postal: choose from suggested places after selecting a country.

`, prompt_above_buttons: true, - button_choices: ['Continue'], + button_choices: ['Use GPS', 'Pick on map', 'Type city/postal'], button_html: '', keyboard_choices: 'NO_KEYS', data: { - assessment_stage: 'instructions', + assessment_stage: 'mode_select_intro', task: 'location-selection', }, + on_load: () => { + ensureLocationSelectionStyles(); + }, + on_finish: (data: Record) => { + const idx = Number(data?.response ?? data?.button_response ?? -1); + const selectedMode = modes[idx] || 'gps'; + data.selectedMode = selectedMode; + taskStore('locationSelectionMode', selectedMode); + }, }, ]; diff --git a/task-launcher/src/tasks/location-selection/trials/mapPicker.ts b/task-launcher/src/tasks/location-selection/trials/mapPicker.ts index 4b4ce698..fae051fd 100644 --- a/task-launcher/src/tasks/location-selection/trials/mapPicker.ts +++ b/task-launcher/src/tasks/location-selection/trials/mapPicker.ts @@ -1,6 +1,7 @@ import jsPsychHtmlMultiResponse from '@jspsych-contrib/plugin-html-multi-response'; import { taskStore } from '../../../taskStore'; import { getLocationSelectionDraft, setLocationSelectionDraft } from '../helpers/state'; +import { ensureLocationSelectionStyles } from '../helpers/ui'; declare global { interface Window { @@ -50,6 +51,9 @@ const US_STATE_LABEL_POINTS: Array<{ abbr: string; lat: number; lon: number }> = { abbr: 'WA', lat: 47.7511, lon: -120.7401 }, { abbr: 'WV', lat: 38.5976, lon: -80.4549 }, { abbr: 'WI', lat: 43.7844, lon: -88.7879 }, { abbr: 'WY', lat: 43.0759, lon: -107.2903 }, ]; +const US_STATE_LABEL_PRIORITY: string[] = [ + 'CA', 'TX', 'FL', 'NY', 'WA', 'CO', 'AZ', 'IL', 'GA', 'NC', 'PA', 'MI', 'OH', 'TN', 'MN', 'MA', 'VA', 'MO', 'NJ', 'WI', +]; function ensureMapLabelStyles() { const styleId = 'location-selection-map-label-styles'; @@ -78,6 +82,17 @@ function ensureMapLabelStyles() { document.head.appendChild(style); } +function shouldKeepLabel(candidate: { x: number; y: number }, used: Array<{ x: number; y: number }>, minPxDistance: number) { + for (let i = 0; i < used.length; i += 1) { + const dx = candidate.x - used[i].x; + const dy = candidate.y - used[i].y; + if ((dx * dx) + (dy * dy) < (minPxDistance * minPxDistance)) { + return false; + } + } + return true; +} + function loadLeaflet(): Promise { if (window.L) return Promise.resolve(window.L); if (leafletLoader) return leafletLoader; @@ -122,11 +137,11 @@ export const mapPicker = { type: jsPsychHtmlMultiResponse, stimulus: `
-
+

Pick on map (United States)

Click your approximate location on the US map. The map is limited to the contiguous US.

-
-

Pick a point on the map.

+
+

Pick a point on the map.

`, @@ -139,6 +154,7 @@ export const mapPicker = { task: 'location-selection', }, on_load: async () => { + ensureLocationSelectionStyles(); const continueButton = document.querySelector('#jspsych-html-multi-response-button-0'); const statusEl = document.getElementById('map-picker-status'); if (continueButton) continueButton.disabled = true; @@ -170,9 +186,28 @@ export const mapPicker = { const zoom = Number(map.getZoom?.() || 4); cityLabelsLayer.clearLayers(); stateLabelsLayer.clearLayers(); + const usedScreenPoints: Array<{ x: number; y: number }> = []; + const mapBounds = map.getBounds?.(); + const stateRank = new Map(US_STATE_LABEL_PRIORITY.map((abbr, idx) => [abbr, idx])); if (zoom <= 6) { - US_STATE_LABEL_POINTS.forEach((point) => { + const maxLabels = zoom <= 4 ? 10 : zoom === 5 ? 16 : 28; + const minPxDistance = zoom <= 4 ? 52 : zoom === 5 ? 42 : 34; + const rankedStates = US_STATE_LABEL_POINTS + .slice() + .sort((a, b) => { + const aRank = stateRank.has(a.abbr) ? Number(stateRank.get(a.abbr)) : 999; + const bRank = stateRank.has(b.abbr) ? Number(stateRank.get(b.abbr)) : 999; + return aRank - bRank; + }); + + rankedStates.forEach((point) => { + if (stateLabelsLayer.getLayers().length >= maxLabels) return; + if (mapBounds && !mapBounds.contains([point.lat, point.lon])) return; + const screen = map.latLngToContainerPoint([point.lat, point.lon]); + const spot = { x: Number(screen.x), y: Number(screen.y) }; + if (!shouldKeepLabel(spot, usedScreenPoints, minPxDistance)) return; + usedScreenPoints.push(spot); const marker = L.circleMarker([point.lat, point.lon], { radius: 1, color: '#334155', @@ -192,6 +227,11 @@ export const mapPicker = { } US_LABEL_POINTS.forEach((point) => { + if (mapBounds && !mapBounds.contains([point.lat, point.lon])) return; + const screen = map.latLngToContainerPoint([point.lat, point.lon]); + const spot = { x: Number(screen.x), y: Number(screen.y) }; + if (!shouldKeepLabel(spot, usedScreenPoints, 40)) return; + usedScreenPoints.push(spot); const marker = L.circleMarker([point.lat, point.lon], { radius: 2.5, color: '#1f2937', diff --git a/task-launcher/src/tasks/location-selection/trials/modeSelect.ts b/task-launcher/src/tasks/location-selection/trials/modeSelect.ts index a9c4addc..a9d5de3d 100644 --- a/task-launcher/src/tasks/location-selection/trials/modeSelect.ts +++ b/task-launcher/src/tasks/location-selection/trials/modeSelect.ts @@ -1,5 +1,6 @@ import jsPsychHtmlMultiResponse from '@jspsych-contrib/plugin-html-multi-response'; import { taskStore } from '../../../taskStore'; +import { ensureLocationSelectionStyles } from '../helpers/ui'; const modes = ['gps', 'map', 'city_postal'] as const; @@ -7,7 +8,7 @@ export const modeSelect = { type: jsPsychHtmlMultiResponse, stimulus: `
-
+

Choose how to provide location

Select one method to continue.

@@ -21,6 +22,9 @@ export const modeSelect = { assessment_stage: 'mode_select', task: 'location-selection', }, + on_load: () => { + ensureLocationSelectionStyles(); + }, on_finish: (data: Record) => { const idx = Number(data?.response ?? data?.button_response ?? -1); const selectedMode = modes[idx] || 'gps'; diff --git a/task-launcher/src/tasks/location-selection/trials/reviewAndConfirm.ts b/task-launcher/src/tasks/location-selection/trials/reviewAndConfirm.ts index 9d1e1a18..7776b39c 100644 --- a/task-launcher/src/tasks/location-selection/trials/reviewAndConfirm.ts +++ b/task-launcher/src/tasks/location-selection/trials/reviewAndConfirm.ts @@ -1,29 +1,48 @@ import jsPsychHtmlMultiResponse from '@jspsych-contrib/plugin-html-multi-response'; import { taskStore } from '../../../taskStore'; import { getLocationSelectionDraft } from '../helpers/state'; +import { ensureLocationSelectionStyles } from '../helpers/ui'; +import { + buildLocationCommitPreview, + buildLocationCommitPreviewWithPopulation, +} from '../helpers/locationCommitPreview'; + +function escapeHtml(value: string): string { + return String(value || '') + .replace(/&/g, '&') + .replace(//g, '>'); +} export const reviewAndConfirm = { type: jsPsychHtmlMultiResponse, stimulus: () => { const mode = taskStore().locationSelectionMode || 'unknown'; const draft = getLocationSelectionDraft(); + const commitPreview = buildLocationCommitPreview(draft, taskStore().locationSelectionConfig || null); + const commitPreviewJson = escapeHtml(JSON.stringify(commitPreview, null, 2)); const draftDetails = draft ? ` -
+

Mode: ${draft.mode}

-

Coordinates: ${draft.lat.toFixed(5)}, ${draft.lon.toFixed(5)}

Source: ${draft.source || 'unknown'}

${draft.label ? `

Label: ${draft.label}

` : ''} ${draft.accuracyMeters ? `

Accuracy: ~${Math.round(draft.accuracyMeters)}m

` : ''} +

Raw coordinates: hidden

` - : '

No location selected yet.

'; + : '

No location selected yet.

'; return `
-
+

Review selection

Selected mode: ${mode}

${draftDetails} +
+

Location object to commit (schema preview):

+

Computing effective H3 with population lookup…

+
${commitPreviewJson}
+
`; @@ -36,8 +55,35 @@ export const reviewAndConfirm = { assessment_stage: 'review_confirm', task: 'location-selection', }, + on_load: () => { + ensureLocationSelectionStyles(); + const draft = getLocationSelectionDraft(); + const config = taskStore().locationSelectionConfig || null; + const previewEl = document.getElementById('location-commit-preview-json'); + const statusEl = document.getElementById('location-commit-preview-status'); + + buildLocationCommitPreviewWithPopulation(draft, config) + .then((computedPreview) => { + const finalPreview = computedPreview || buildLocationCommitPreview(draft, config); + taskStore('locationSelectionCommitPreview', finalPreview); + if (previewEl) { + previewEl.textContent = JSON.stringify(finalPreview, null, 2); + } + if (statusEl) { + const source = finalPreview?.populationSource || 'unknown'; + statusEl.textContent = `Population source used: ${source}`; + } + }) + .catch(() => { + const fallback = buildLocationCommitPreview(draft, config); + taskStore('locationSelectionCommitPreview', fallback); + if (statusEl) statusEl.textContent = 'Population lookup unavailable; using baseline-only preview.'; + }); + }, on_finish: (data: Record) => { data.mode = taskStore().locationSelectionMode || null; data.selectedLocation = getLocationSelectionDraft(); + data.locationCommitPreview = taskStore().locationSelectionCommitPreview + || buildLocationCommitPreview(getLocationSelectionDraft(), taskStore().locationSelectionConfig || null); }, }; diff --git a/task-launcher/src/tasks/location-selection/trials/searchCityPostal.ts b/task-launcher/src/tasks/location-selection/trials/searchCityPostal.ts index 376316cf..02ef76c8 100644 --- a/task-launcher/src/tasks/location-selection/trials/searchCityPostal.ts +++ b/task-launcher/src/tasks/location-selection/trials/searchCityPostal.ts @@ -1,6 +1,7 @@ import jsPsychHtmlMultiResponse from '@jspsych-contrib/plugin-html-multi-response'; import { taskStore } from '../../../taskStore'; import { getLocationSelectionDraft, setLocationSelectionDraft } from '../helpers/state'; +import { ensureLocationSelectionStyles } from '../helpers/ui'; interface NominatimResult { place_id?: number; @@ -92,19 +93,19 @@ export const searchCityPostal = { type: jsPsychHtmlMultiResponse, stimulus: `
-
+

City / postal search

Select a country, then type a city/town or postal code.

-
+
-
+
-

Choose a country to begin.

+

Choose a country to begin.

`, @@ -117,6 +118,7 @@ export const searchCityPostal = { task: 'location-selection', }, on_load: () => { + ensureLocationSelectionStyles(); const continueButton = document.querySelector('#jspsych-html-multi-response-button-0'); const inputEl = document.getElementById('location-query-input') as HTMLInputElement | null; const countryEl = document.getElementById('location-country-select') as HTMLSelectElement | null; diff --git a/task-launcher/webpack.config.cjs b/task-launcher/webpack.config.cjs index 1c29b371..8dada6d1 100644 --- a/task-launcher/webpack.config.cjs +++ b/task-launcher/webpack.config.cjs @@ -2,6 +2,7 @@ const path = require('path'); const webpack = require('webpack'); const { merge } = require('webpack-merge'); const HtmlWebpackPlugin = require('html-webpack-plugin'); +const { registerPopulationApi } = require('./serve/populationApi.cjs'); const commonConfig = { optimization: { @@ -126,6 +127,12 @@ const developmentConfig = merge(webConfig, { devtool: 'inline-source-map', devServer: { static: './dist', + setupMiddlewares: (middlewares, devServer) => { + if (devServer?.app) { + registerPopulationApi(devServer.app); + } + return middlewares; + }, }, }); From 94c05e7335703859db14b6530d145b5be086d738 Mon Sep 17 00:00:00 2001 From: digital-pro Date: Wed, 4 Mar 2026 18:15:01 -0800 Subject: [PATCH 05/15] fix: polish location selection UX and selection reliability Center and normalize location-selection panel layout across trials, simplify GPS flow, and tighten city/postal selection so explicit user picks are respected. Improve population-source attribution and config plumbing so Kontur-first fallback behavior is reflected in the computed commit preview. --- task-launcher/serve/serve.js | 17 +++- .../location-selection/helpers/config.ts | 4 +- .../helpers/locationCommitPreview.ts | 21 ++++- .../helpers/populationApi.ts | 8 +- .../tasks/location-selection/helpers/ui.ts | 84 ++++++++++++++++--- .../src/tasks/location-selection/timeline.ts | 2 + .../location-selection/trials/gpsCapture.ts | 30 +++++-- .../location-selection/trials/instructions.ts | 19 ++--- .../trials/reviewAndConfirm.ts | 22 ++--- .../trials/searchCityPostal.ts | 67 ++++++++++----- .../src/tasks/shared/helpers/config.ts | 8 ++ 11 files changed, 209 insertions(+), 73 deletions(-) diff --git a/task-launcher/serve/serve.js b/task-launcher/serve/serve.js index ac879bf2..96db7b41 100644 --- a/task-launcher/serve/serve.js +++ b/task-launcher/serve/serve.js @@ -32,7 +32,13 @@ Sentry.init({ // TODO: Add game params for all tasks const queryString = new URL(window.location).search; const urlParams = new URLSearchParams(queryString); -const taskName = urlParams.get('task') ?? 'egma-math'; +const requestedTaskRaw = String(urlParams.get('task') || '').trim(); +const requestedTask = requestedTaskRaw.toLowerCase(); +const taskName = ( + requestedTask === 'locationselection' + ? 'location-selection' + : requestedTaskRaw || 'egma-math' +); const corpus = urlParams.get('corpus'); const buttonLayout = urlParams.get('buttonLayout'); const numOfPracticeTrials = urlParams.get('practiceTrials'); @@ -47,6 +53,11 @@ const inferenceNumStories = urlParams.get('inferenceNumStories') === null ? null : parseInt(urlParams.get('inferenceNumStories'), 10); const semThreshold = Number(urlParams.get('semThreshold') || '0'); const startingTheta = Number(urlParams.get('theta') || '0'); +const populationSourcePreference = String(urlParams.get('populationSourcePreference') || 'kontur').trim().toLowerCase(); +const konturPopulationApiUrl = urlParams.get('konturPopulationApiUrl') || undefined; +const worldpopPopulationApiUrl = urlParams.get('worldpopPopulationApiUrl') || undefined; +const populationApiTimeoutMs = + urlParams.get('populationApiTimeoutMs') === null ? undefined : parseInt(urlParams.get('populationApiTimeoutMs'), 10); // Boolean parameters const keyHelpers = stringToBoolean(urlParams.get('keyHelpers')); @@ -95,6 +106,10 @@ async function startWebApp() { inferenceNumStories, semThreshold, startingTheta, + populationSourcePreference, + konturPopulationApiUrl, + worldpopPopulationApiUrl, + populationApiTimeoutMs, heavyInstructions, demoMode, }; diff --git a/task-launcher/src/tasks/location-selection/helpers/config.ts b/task-launcher/src/tasks/location-selection/helpers/config.ts index e41800dc..f6a87740 100644 --- a/task-launcher/src/tasks/location-selection/helpers/config.ts +++ b/task-launcher/src/tasks/location-selection/helpers/config.ts @@ -12,7 +12,7 @@ export function getLocationSelectionTaskConfig(config: Record): Loc const threshold = Number(config?.populationThreshold); const baseline = Number(config?.baselineResolution); const maxRes = Number(config?.maxResolution); - const sourcePreference = String(config?.populationSourcePreference || 'auto').trim().toLowerCase(); + const sourcePreference = String(config?.populationSourcePreference || 'kontur').trim().toLowerCase(); const konturPopulationApiUrl = String(config?.konturPopulationApiUrl || '/api/population-kontur-h3').trim(); const worldpopPopulationApiUrl = String(config?.worldpopPopulationApiUrl || '/api/population-worldpop-h3').trim(); const populationApiTimeoutMs = Number(config?.populationApiTimeoutMs); @@ -30,7 +30,7 @@ export function getLocationSelectionTaskConfig(config: Record): Loc populationSourcePreference: sourcePreference === 'kontur' || sourcePreference === 'worldpop' || sourcePreference === 'auto' ? sourcePreference - : 'auto', + : 'kontur', konturPopulationApiUrl, worldpopPopulationApiUrl, populationApiTimeoutMs: diff --git a/task-launcher/src/tasks/location-selection/helpers/locationCommitPreview.ts b/task-launcher/src/tasks/location-selection/helpers/locationCommitPreview.ts index 8e118c1c..24c1f9f7 100644 --- a/task-launcher/src/tasks/location-selection/helpers/locationCommitPreview.ts +++ b/task-launcher/src/tasks/location-selection/helpers/locationCommitPreview.ts @@ -26,6 +26,14 @@ type LocationCommitPreview = { computedAt: string; }; +function getPreferredPopulationSource( + config: Partial | null | undefined, +): 'kontur' | 'worldpop' { + const preference = String(config?.populationSourcePreference || 'kontur').toLowerCase(); + if (preference === 'worldpop') return 'worldpop'; + return 'kontur'; +} + function roundTo(value: number, decimals = 6): number { const factor = 10 ** decimals; return Math.round(value * factor) / factor; @@ -43,6 +51,7 @@ export function buildLocationCommitPreview( const safePopulationThreshold = Number.isFinite(populationThreshold) && populationThreshold > 0 ? Math.round(populationThreshold) : 50000; + const preferredSource = getPreferredPopulationSource(config); const baselineCell = latLngToCell(draft.lat, draft.lon, safeBaselineResolution); @@ -68,7 +77,7 @@ export function buildLocationCommitPreview( }, populationThreshold: safePopulationThreshold, }, - populationSource: 'unknown', + populationSource: preferredSource, computedAt: draft.selectedAt || new Date().toISOString(), }; } @@ -88,17 +97,22 @@ export async function buildLocationCommitPreviewWithPopulation( const safePopulationThreshold = Number.isFinite(populationThreshold) && populationThreshold > 0 ? Math.round(populationThreshold) : 50000; + const preferredSource = getPreferredPopulationSource(config); const baselineCell = latLngToCell(draft.lat, draft.lon, safeBaselineResolution); let effectiveCell = baselineCell; let effectiveResolution = safeBaselineResolution; let effectivePopulationSource: 'kontur' | 'worldpop' | 'unknown' = 'unknown'; + let observedPopulationSource: 'kontur' | 'worldpop' | 'unknown' = 'unknown'; let foundPassingCell = false; for (let resolution = safeBaselineResolution; resolution <= safeMaxResolution; resolution += 1) { const cellId = latLngToCell(draft.lat, draft.lon, resolution); const populationResult = await lookupPopulationForCell(cellId, resolution, config); const population = populationResult.population; + if (populationResult.source !== 'unknown' && observedPopulationSource === 'unknown') { + observedPopulationSource = populationResult.source; + } const privacyMet = typeof population === 'number' ? population >= safePopulationThreshold : false; if (privacyMet) { @@ -134,7 +148,10 @@ export async function buildLocationCommitPreviewWithPopulation( }, populationThreshold: safePopulationThreshold, }, - populationSource: effectivePopulationSource, + populationSource: + effectivePopulationSource !== 'unknown' + ? effectivePopulationSource + : (observedPopulationSource !== 'unknown' ? observedPopulationSource : preferredSource), computedAt: draft.selectedAt || new Date().toISOString(), }; } diff --git a/task-launcher/src/tasks/location-selection/helpers/populationApi.ts b/task-launcher/src/tasks/location-selection/helpers/populationApi.ts index a6bf9bb3..d83ea21a 100644 --- a/task-launcher/src/tasks/location-selection/helpers/populationApi.ts +++ b/task-launcher/src/tasks/location-selection/helpers/populationApi.ts @@ -48,7 +48,7 @@ async function fetchPopulation( signal: controller.signal, }); window.clearTimeout(timeout); - if (!response.ok) return { population: null, source: 'unknown' }; + if (!response.ok) return { population: null, source }; const payload = await response.json().catch(() => ({})); const resolvedSourceRaw = String(payload?.source || '').trim().toLowerCase(); const resolvedSource: PopulationSource | 'unknown' = @@ -57,7 +57,7 @@ async function fetchPopulation( : source; return { population: parsePopulation(payload), source: resolvedSource }; } catch { - return { population: null, source: 'unknown' }; + return { population: null, source }; } } @@ -78,12 +78,14 @@ export async function lookupPopulationForCell( ? ['worldpop', 'kontur'] : ['kontur', 'worldpop']; + let attemptedKnownSource: PopulationSource | 'unknown' = 'unknown'; for (let i = 0; i < orderedSources.length; i += 1) { const source = orderedSources[i]; const endpoint = source === 'kontur' ? konturUrl : worldpopUrl; const result = await fetchPopulation(endpoint, source, cellId, resolution, timeoutMs); + if (result.source !== 'unknown') attemptedKnownSource = result.source; if (typeof result.population === 'number') return result; } - return { population: null, source: 'unknown' }; + return { population: null, source: attemptedKnownSource }; } diff --git a/task-launcher/src/tasks/location-selection/helpers/ui.ts b/task-launcher/src/tasks/location-selection/helpers/ui.ts index 9b6b46f6..5d705f63 100644 --- a/task-launcher/src/tasks/location-selection/helpers/ui.ts +++ b/task-launcher/src/tasks/location-selection/helpers/ui.ts @@ -1,26 +1,35 @@ export function ensureLocationSelectionStyles() { const styleId = 'location-selection-shared-ui-styles'; - if (document.getElementById(styleId)) return; - - const style = document.createElement('style'); - style.id = styleId; + let style = document.getElementById(styleId) as HTMLStyleElement | null; + if (!style) { + style = document.createElement('style'); + style.id = styleId; + document.head.appendChild(style); + } style.textContent = ` - .location-selection-panel { + .lev-row-container.instruction.location-selection-panel { width: min(92vw, 680px); max-width: 680px; - margin: 0 auto 0 0; + margin: 0 auto; text-align: left; box-sizing: border-box; + display: flex; + flex-direction: column; + align-items: stretch; + justify-content: flex-start; + gap: 0.35rem; } - .location-selection-copy h2 { + .lev-row-container.instruction.location-selection-panel h2 { margin: 0 0 0.6rem 0; line-height: 1.25; font-size: 1.35rem; + width: 100%; } - .location-selection-copy p { + .lev-row-container.instruction.location-selection-panel p { margin: 0 0 0.7rem 0; line-height: 1.45; font-size: 0.96rem; + width: 100%; } .location-selection-note { margin-top: 0.8rem; @@ -41,18 +50,67 @@ export function ensureLocationSelectionStyles() { margin-bottom: 0.3rem; font-weight: 600; } - .location-selection-status { + .lev-row-container.instruction.location-selection-panel .location-selection-status { margin-top: 0.8rem; min-height: 1.4em; line-height: 1.35; + white-space: normal; + } + .location-selection-stack { + display: block; + width: 100%; + } + .location-selection-stack > * { + width: 100%; + box-sizing: border-box; + } + .location-selection-json { + margin: 0; + padding: 0.65rem; + background: #0f172a; + color: #e2e8f0; + border-radius: 6px; + font-size: 0.8rem; + line-height: 1.3; + overflow: auto; + max-height: 280px; + white-space: pre; } .location-selection-intro h2 { - font-size: 1.25rem; + font-size: 1.15rem; + margin-bottom: 0.4rem; } .location-selection-intro p { - font-size: 0.93rem; - line-height: 1.4; + font-size: 0.9rem; + line-height: 1.35; + margin-bottom: 0.45rem; + max-width: 34rem; + white-space: normal; + overflow-wrap: break-word; + } + .location-method-buttons { + display: grid; + grid-template-columns: 1fr; + gap: 0.5rem; + margin-top: 0.7rem; + } + .location-method-button { + width: 100%; + text-align: left; + line-height: 1.25; + padding: 0.65rem 0.75rem; + white-space: normal; + } + .location-method-title { + display: block; + font-weight: 700; + font-size: 0.95rem; + margin-bottom: 0.15rem; + } + .location-method-meta { + display: block; + font-size: 0.82rem; + opacity: 0.9; } `; - document.head.appendChild(style); } diff --git a/task-launcher/src/tasks/location-selection/timeline.ts b/task-launcher/src/tasks/location-selection/timeline.ts index 6694378f..b5621fda 100644 --- a/task-launcher/src/tasks/location-selection/timeline.ts +++ b/task-launcher/src/tasks/location-selection/timeline.ts @@ -20,6 +20,8 @@ export default function buildLocationSelectionTimeline(config: Record[] = [ diff --git a/task-launcher/src/tasks/location-selection/trials/gpsCapture.ts b/task-launcher/src/tasks/location-selection/trials/gpsCapture.ts index 467685d7..71b0f82e 100644 --- a/task-launcher/src/tasks/location-selection/trials/gpsCapture.ts +++ b/task-launcher/src/tasks/location-selection/trials/gpsCapture.ts @@ -29,12 +29,12 @@ export const gpsCapture = {

GPS location

-

Use your browser location to capture coordinates from this device.

+

We are requesting your browser location permission now.

+

Requesting GPS location…

+

- +
-

Waiting for GPS request…

-

`, @@ -49,18 +49,23 @@ export const gpsCapture = { on_load: () => { ensureLocationSelectionStyles(); const continueButton = document.querySelector('#jspsych-html-multi-response-button-0'); - const gpsButton = document.getElementById('capture-gps-btn') as HTMLButtonElement | null; + const retryButton = document.getElementById('gps-retry-btn') as HTMLButtonElement | null; const statusEl = document.getElementById('gps-status'); const valueEl = document.getElementById('gps-value'); - if (continueButton) continueButton.disabled = true; + if (continueButton) { + continueButton.disabled = true; + continueButton.style.display = 'none'; + } if (!navigator.geolocation) { if (statusEl) statusEl.textContent = 'Geolocation is not supported by this browser.'; + if (retryButton) retryButton.style.display = 'inline-block'; return; } - gpsButton?.addEventListener('click', async () => { + const requestGps = () => { if (statusEl) statusEl.textContent = 'Requesting GPS location…'; + if (retryButton) retryButton.style.display = 'none'; navigator.geolocation.getCurrentPosition( async (position) => { const lat = Number(position.coords.latitude); @@ -86,10 +91,14 @@ export const gpsCapture = { `${lat.toFixed(5)}, ${lon.toFixed(5)} · accuracy ≈ ${Math.round(accuracyMeters || 0)}m` + (label ? ` · ${label}` : ''); } - if (continueButton) continueButton.disabled = false; + if (continueButton) { + continueButton.disabled = false; + continueButton.click(); + } }, (error) => { if (statusEl) statusEl.textContent = `GPS error: ${error.message}`; + if (retryButton) retryButton.style.display = 'inline-block'; }, { enableHighAccuracy: true, @@ -97,7 +106,10 @@ export const gpsCapture = { maximumAge: 0, }, ); - }); + }; + + retryButton?.addEventListener('click', requestGps); + requestGps(); }, on_finish: (data: Record) => { const draft = getLocationSelectionDraft(); diff --git a/task-launcher/src/tasks/location-selection/trials/instructions.ts b/task-launcher/src/tasks/location-selection/trials/instructions.ts index ae135441..3c43c8fe 100644 --- a/task-launcher/src/tasks/location-selection/trials/instructions.ts +++ b/task-launcher/src/tasks/location-selection/trials/instructions.ts @@ -11,20 +11,17 @@ export const instructions = [

How would you like to share location?

-

Choose the method that is easiest for you. You can use GPS, tap a map, or search by city/postal code.

-

We only need an approximate location for planning and analysis.

-
-

What to expect:

-

- GPS: uses your browser location permission.

-

- Map: click a point in the United States.

-

- City/Postal: choose from suggested places after selecting a country.

-
+

Choose one option below.
We only use an approximate location.

`, prompt_above_buttons: true, - button_choices: ['Use GPS', 'Pick on map', 'Type city/postal'], - button_html: '', + button_choices: [ + 'Use GPSUse browser location permission on this device.', + 'Pick on mapClick an approximate point in the United States.', + 'Type city/postalChoose country first, then autocomplete a place or code.', + ], + button_html: '', keyboard_choices: 'NO_KEYS', data: { assessment_stage: 'mode_select_intro', @@ -32,6 +29,8 @@ export const instructions = [ }, on_load: () => { ensureLocationSelectionStyles(); + const btnGroup = document.getElementById('jspsych-html-multi-response-btngroup'); + if (btnGroup) btnGroup.classList.add('location-method-buttons'); }, on_finish: (data: Record) => { const idx = Number(data?.response ?? data?.button_response ?? -1); diff --git a/task-launcher/src/tasks/location-selection/trials/reviewAndConfirm.ts b/task-launcher/src/tasks/location-selection/trials/reviewAndConfirm.ts index 7776b39c..b62119e4 100644 --- a/task-launcher/src/tasks/location-selection/trials/reviewAndConfirm.ts +++ b/task-launcher/src/tasks/location-selection/trials/reviewAndConfirm.ts @@ -17,14 +17,15 @@ function escapeHtml(value: string): string { export const reviewAndConfirm = { type: jsPsychHtmlMultiResponse, stimulus: () => { - const mode = taskStore().locationSelectionMode || 'unknown'; const draft = getLocationSelectionDraft(); const commitPreview = buildLocationCommitPreview(draft, taskStore().locationSelectionConfig || null); - const commitPreviewJson = escapeHtml(JSON.stringify(commitPreview, null, 2)); + const commitPreviewJson = commitPreview + ? escapeHtml(JSON.stringify(commitPreview, null, 2)) + : '{"error":"No location selected yet. Go back and choose a location."}'; const draftDetails = draft ? `
-

Mode: ${draft.mode}

+

Selected method: ${draft.mode}

Source: ${draft.source || 'unknown'}

${draft.label ? `

Label: ${draft.label}

` : ''} ${draft.accuracyMeters ? `

Accuracy: ~${Math.round(draft.accuracyMeters)}m

` : ''} @@ -35,13 +36,14 @@ export const reviewAndConfirm = { return `
-

Review selection

-

Selected mode: ${mode}

- ${draftDetails} -
-

Location object to commit (schema preview):

-

Computing effective H3 with population lookup…

-
${commitPreviewJson}
+
+

Review selection

+ ${draftDetails} +
+

Location object to commit (schema preview):

+

Computing effective H3 with population lookup…

+
${commitPreviewJson}
+
diff --git a/task-launcher/src/tasks/location-selection/trials/searchCityPostal.ts b/task-launcher/src/tasks/location-selection/trials/searchCityPostal.ts index 02ef76c8..86455508 100644 --- a/task-launcher/src/tasks/location-selection/trials/searchCityPostal.ts +++ b/task-launcher/src/tasks/location-selection/trials/searchCityPostal.ts @@ -87,6 +87,25 @@ async function searchLocations(query: string, countryCode?: string): Promise { if (!dropdownEl) return; @@ -141,23 +162,12 @@ export const searchCityPostal = { }; const selectResult = (selected: NominatimResult) => { - const lat = Number(selected?.lat); - const lon = Number(selected?.lon); - if (!Number.isFinite(lat) || !Number.isFinite(lon)) return; - setLocationSelectionDraft({ - mode: 'city_postal', - lat, - lon, - label: String(selected.display_name || ''), - source: 'nominatim_search', - metadata: { - placeId: selected.place_id ?? null, - resultType: selected.type ?? null, - countryCode: selectedCountry, - }, - selectedAt: new Date().toISOString(), - }); - if (statusEl) statusEl.textContent = `Selected: ${selected.display_name || `${lat.toFixed(5)}, ${lon.toFixed(5)}`}`; + const draft = buildDraftFromSuggestion(selected, selectedCountry); + if (!draft) return; + setLocationSelectionDraft(draft); + taskStore('locationSelectionPendingSuggestion', selected); + hasExplicitSelection = true; + if (statusEl) statusEl.textContent = `Selected: ${selected.display_name || `${draft.lat.toFixed(5)}, ${draft.lon.toFixed(5)}`}`; if (inputEl) inputEl.value = String(selected.display_name || inputEl.value); hideDropdown(); if (continueButton) continueButton.disabled = false; @@ -166,6 +176,7 @@ export const searchCityPostal = { const renderResults = (results: NominatimResult[]) => { if (!dropdownEl) return; latestResults = results.slice(); + taskStore('locationSelectionPendingSuggestion', null); if (highlightedIndex >= latestResults.length) highlightedIndex = latestResults.length - 1; if (!results.length) { dropdownEl.innerHTML = '
No matches found.
'; @@ -186,12 +197,8 @@ export const searchCityPostal = { active?.scrollIntoView({ block: 'nearest' }); } dropdownEl.querySelectorAll('button[data-result-index]').forEach((button) => { - button.addEventListener('mouseenter', () => { - const idx = Number(button.dataset.resultIndex); - highlightedIndex = Number.isFinite(idx) ? idx : -1; - renderResults(latestResults); - }); - button.addEventListener('click', () => { + button.addEventListener('mousedown', (event) => { + event.preventDefault(); const idx = Number(button.dataset.resultIndex); const selected = latestResults[idx]; if (!selected) return; @@ -215,6 +222,7 @@ export const searchCityPostal = { } if (query.length < 2) { hideDropdown(); + taskStore('locationSelectionPendingSuggestion', null); return; } if (statusEl) statusEl.textContent = `Searching in ${selectedCountry}…`; @@ -234,6 +242,7 @@ export const searchCityPostal = { .map((country) => ``) .join(''); selectedCountry = countryEl.value || 'US'; + taskStore('locationSelectionPendingCountry', selectedCountry); if (statusEl) statusEl.textContent = 'Country selected. Start typing a city or postal code.'; }) .catch((error: any) => { @@ -242,15 +251,22 @@ export const searchCityPostal = { countryEl?.addEventListener('change', () => { selectedCountry = String(countryEl.value || '').toUpperCase(); + taskStore('locationSelectionPendingCountry', selectedCountry); + hasExplicitSelection = false; if (continueButton) continueButton.disabled = true; hideDropdown(); if (inputEl) inputEl.value = ''; taskStore('locationSelectionDraft', null); + taskStore('locationSelectionPendingSuggestion', null); latestResults = []; if (statusEl) statusEl.textContent = `Country set to ${selectedCountry}. Start typing to see matches.`; }); inputEl?.addEventListener('input', () => { + hasExplicitSelection = false; + taskStore('locationSelectionDraft', null); + taskStore('locationSelectionPendingSuggestion', null); + if (continueButton) continueButton.disabled = true; if (debounceHandle) { window.clearTimeout(debounceHandle); } @@ -300,6 +316,11 @@ export const searchCityPostal = { selectResult(latestResults[highlightedIndex]); return; } + if (!hasExplicitSelection) { + if (statusEl) statusEl.textContent = 'Select a result from the dropdown first.'; + if (continueButton) continueButton.disabled = true; + return; + } runSearch().catch((error: any) => { if (statusEl) statusEl.textContent = `Search failed: ${error?.message || error}`; }); diff --git a/task-launcher/src/tasks/shared/helpers/config.ts b/task-launcher/src/tasks/shared/helpers/config.ts index 7d0d342c..796c13a9 100644 --- a/task-launcher/src/tasks/shared/helpers/config.ts +++ b/task-launcher/src/tasks/shared/helpers/config.ts @@ -93,6 +93,10 @@ export const setSharedConfig = async ( semThreshold, startingTheta, demoMode, + populationSourcePreference, + konturPopulationApiUrl, + worldpopPopulationApiUrl, + populationApiTimeoutMs, } = cleanParams; const config = { @@ -123,6 +127,10 @@ export const setSharedConfig = async ( semThreshold: Number(semThreshold), startingTheta: Number(startingTheta), demoMode: !!demoMode, + populationSourcePreference: String(populationSourcePreference || 'kontur'), + konturPopulationApiUrl: konturPopulationApiUrl ? String(konturPopulationApiUrl) : undefined, + worldpopPopulationApiUrl: worldpopPopulationApiUrl ? String(worldpopPopulationApiUrl) : undefined, + populationApiTimeoutMs: Number(populationApiTimeoutMs) || undefined, }; // default corpus if nothing is passed in From f968f5101ead0fa442d0149719ebb5e8fb558756 Mon Sep 17 00:00:00 2001 From: digital-pro Date: Wed, 4 Mar 2026 21:48:53 -0800 Subject: [PATCH 06/15] fix: harden location population lookup and review scrolling Parse WorldPop total_population responses and add direct API fallback for preview/static environments where local middleware is unavailable. Enable vertical scrolling for location-selection screens and include candidate diagnostics rendering on review. --- task-launcher/serve/populationApi.cjs | 1 + .../helpers/locationCommitPreview.ts | 35 +++++++- .../helpers/populationApi.ts | 90 ++++++++++++++++++- .../tasks/location-selection/helpers/ui.ts | 57 ++++++++++++ .../src/tasks/location-selection/timeline.ts | 1 + .../trials/reviewAndConfirm.ts | 67 +++++++++++++- 6 files changed, 244 insertions(+), 7 deletions(-) diff --git a/task-launcher/serve/populationApi.cjs b/task-launcher/serve/populationApi.cjs index 340d9151..8f816b18 100644 --- a/task-launcher/serve/populationApi.cjs +++ b/task-launcher/serve/populationApi.cjs @@ -75,6 +75,7 @@ async function wait(ms) { function parseWorldPopSum(payload) { const candidates = [ + payload?.data?.total_population, payload?.stats?.sum, payload?.data?.stats?.sum, payload?.result?.stats?.sum, diff --git a/task-launcher/src/tasks/location-selection/helpers/locationCommitPreview.ts b/task-launcher/src/tasks/location-selection/helpers/locationCommitPreview.ts index 24c1f9f7..1d958833 100644 --- a/task-launcher/src/tasks/location-selection/helpers/locationCommitPreview.ts +++ b/task-launcher/src/tasks/location-selection/helpers/locationCommitPreview.ts @@ -26,6 +26,19 @@ type LocationCommitPreview = { computedAt: string; }; +export type PopulationCandidateDebug = { + resolution: number; + cellId: string; + population: number | null; + source: 'kontur' | 'worldpop' | 'unknown'; + privacyMet: boolean; +}; + +export type LocationCommitComputation = { + preview: LocationCommitPreview; + candidates: PopulationCandidateDebug[]; +}; + function getPreferredPopulationSource( config: Partial | null | undefined, ): 'kontur' | 'worldpop' { @@ -86,6 +99,14 @@ export async function buildLocationCommitPreviewWithPopulation( draft: LocationSelectionDraft | null, config: Partial | null | undefined, ): Promise { + const computed = await buildLocationCommitComputationWithPopulation(draft, config); + return computed?.preview || null; +} + +export async function buildLocationCommitComputationWithPopulation( + draft: LocationSelectionDraft | null, + config: Partial | null | undefined, +): Promise { if (!draft) return null; const baselineResolution = Number(config?.baselineResolution); @@ -105,6 +126,7 @@ export async function buildLocationCommitPreviewWithPopulation( let effectivePopulationSource: 'kontur' | 'worldpop' | 'unknown' = 'unknown'; let observedPopulationSource: 'kontur' | 'worldpop' | 'unknown' = 'unknown'; let foundPassingCell = false; + const candidates: PopulationCandidateDebug[] = []; for (let resolution = safeBaselineResolution; resolution <= safeMaxResolution; resolution += 1) { const cellId = latLngToCell(draft.lat, draft.lon, resolution); @@ -114,6 +136,13 @@ export async function buildLocationCommitPreviewWithPopulation( observedPopulationSource = populationResult.source; } const privacyMet = typeof population === 'number' ? population >= safePopulationThreshold : false; + candidates.push({ + resolution, + cellId, + population, + source: populationResult.source, + privacyMet, + }); if (privacyMet) { effectiveCell = cellId; @@ -129,7 +158,7 @@ export async function buildLocationCommitPreviewWithPopulation( const [centerLat, centerLon] = cellToLatLng(effectiveCell); - return { + const preview: LocationCommitPreview = { schemaVersion: 'location_v1', latLon: { lat: roundTo(centerLat, 6), @@ -154,4 +183,8 @@ export async function buildLocationCommitPreviewWithPopulation( : (observedPopulationSource !== 'unknown' ? observedPopulationSource : preferredSource), computedAt: draft.selectedAt || new Date().toISOString(), }; + return { + preview, + candidates, + }; } diff --git a/task-launcher/src/tasks/location-selection/helpers/populationApi.ts b/task-launcher/src/tasks/location-selection/helpers/populationApi.ts index d83ea21a..59b2798e 100644 --- a/task-launcher/src/tasks/location-selection/helpers/populationApi.ts +++ b/task-launcher/src/tasks/location-selection/helpers/populationApi.ts @@ -1,3 +1,5 @@ +import { cellToBoundary } from 'h3-js'; + type PopulationSource = 'kontur' | 'worldpop'; export type PopulationLookupConfig = { @@ -14,6 +16,7 @@ type PopulationLookupResult = { function parsePopulation(payload: any): number | null { const candidates = [ + payload?.data?.total_population, payload?.population, payload?.pop, payload?.estimatedPopulation, @@ -27,6 +30,73 @@ function parsePopulation(payload: any): number | null { return null; } +function buildCellPolygon(cellId: string) { + const boundary = cellToBoundary(cellId); + if (!Array.isArray(boundary) || !boundary.length) return null; + const ring = boundary.map((pair) => [Number(pair[1]), Number(pair[0])]); + const first = ring[0]; + const last = ring[ring.length - 1]; + if (!first || !last) return null; + if (first[0] !== last[0] || first[1] !== last[1]) { + ring.push([first[0], first[1]]); + } + return { + type: 'Polygon', + coordinates: [ring], + }; +} + +function parseWorldPopTaskId(payload: any): string | null { + const candidates = [payload?.taskid, payload?.taskId, payload?.task_id, payload?.id]; + for (let i = 0; i < candidates.length; i += 1) { + const value = String(candidates[i] || '').trim(); + if (value) return value; + } + return null; +} + +async function queryWorldPopDirect(cellId: string, timeoutMs: number): Promise { + const polygon = buildCellPolygon(cellId); + if (!polygon) return null; + const statsUrl = new URL('https://api.worldpop.org/v1/services/stats'); + statsUrl.searchParams.set('dataset', 'wpgppop'); + statsUrl.searchParams.set('year', '2020'); + statsUrl.searchParams.set('geojson', JSON.stringify(polygon)); + const controller = new AbortController(); + const timeout = window.setTimeout(() => controller.abort(), Math.max(timeoutMs, 5000)); + try { + const statsResponse = await fetch(statsUrl.toString(), { + method: 'GET', + signal: controller.signal, + }); + if (!statsResponse.ok) return null; + const statsPayload = await statsResponse.json().catch(() => ({})); + const direct = parsePopulation(statsPayload); + if (typeof direct === 'number') return direct; + + const taskId = parseWorldPopTaskId(statsPayload); + if (!taskId) return null; + for (let attempt = 0; attempt < 8; attempt += 1) { + const taskResponse = await fetch(`https://api.worldpop.org/v1/tasks/${encodeURIComponent(taskId)}`, { + method: 'GET', + signal: controller.signal, + }); + if (!taskResponse.ok) return null; + const taskPayload = await taskResponse.json().catch(() => ({})); + const value = parsePopulation(taskPayload); + if (typeof value === 'number') return value; + const status = String(taskPayload?.status || '').toLowerCase(); + if (status.includes('failed') || status.includes('error')) return null; + await new Promise((resolve) => setTimeout(resolve, 500)); + } + return null; + } catch { + return null; + } finally { + window.clearTimeout(timeout); + } +} + async function fetchPopulation( endpoint: string, source: PopulationSource, @@ -48,15 +118,31 @@ async function fetchPopulation( signal: controller.signal, }); window.clearTimeout(timeout); - if (!response.ok) return { population: null, source }; + if (!response.ok) { + const directPopulation = await queryWorldPopDirect(cellId, timeoutMs); + if (typeof directPopulation === 'number') { + return { population: directPopulation, source: 'worldpop' }; + } + return { population: null, source }; + } const payload = await response.json().catch(() => ({})); const resolvedSourceRaw = String(payload?.source || '').trim().toLowerCase(); const resolvedSource: PopulationSource | 'unknown' = resolvedSourceRaw === 'kontur' || resolvedSourceRaw === 'worldpop' ? resolvedSourceRaw : source; - return { population: parsePopulation(payload), source: resolvedSource }; + const parsed = parsePopulation(payload); + if (typeof parsed === 'number') return { population: parsed, source: resolvedSource }; + const directPopulation = await queryWorldPopDirect(cellId, timeoutMs); + if (typeof directPopulation === 'number') { + return { population: directPopulation, source: 'worldpop' }; + } + return { population: null, source: resolvedSource }; } catch { + const directPopulation = await queryWorldPopDirect(cellId, timeoutMs); + if (typeof directPopulation === 'number') { + return { population: directPopulation, source: 'worldpop' }; + } return { population: null, source }; } } diff --git a/task-launcher/src/tasks/location-selection/helpers/ui.ts b/task-launcher/src/tasks/location-selection/helpers/ui.ts index 5d705f63..a25808e3 100644 --- a/task-launcher/src/tasks/location-selection/helpers/ui.ts +++ b/task-launcher/src/tasks/location-selection/helpers/ui.ts @@ -1,4 +1,9 @@ export function ensureLocationSelectionStyles() { + const displayEl = document.querySelector('.jspsych-display-element'); + if (displayEl) { + displayEl.classList.add('location-selection-scroll-enabled'); + } + const styleId = 'location-selection-shared-ui-styles'; let style = document.getElementById(styleId) as HTMLStyleElement | null; if (!style) { @@ -7,6 +12,22 @@ export function ensureLocationSelectionStyles() { document.head.appendChild(style); } style.textContent = ` + .jspsych-display-element.location-selection-scroll-enabled { + overflow-y: auto !important; + overflow-x: hidden; + } + .jspsych-display-element.location-selection-scroll-enabled .jspsych-content-wrapper { + height: auto; + min-height: 100%; + align-items: flex-start; + } + .jspsych-display-element.location-selection-scroll-enabled .jspsych-content { + height: auto; + min-height: 100%; + justify-content: flex-start; + padding-top: 1.25rem; + padding-bottom: 1.25rem; + } .lev-row-container.instruction.location-selection-panel { width: min(92vw, 680px); max-width: 680px; @@ -76,6 +97,42 @@ export function ensureLocationSelectionStyles() { max-height: 280px; white-space: pre; } + .location-selection-debug-table-wrap { + margin-top: 0.5rem; + border: 1px solid #cbd5e1; + border-radius: 6px; + overflow: auto; + max-height: 240px; + background: #ffffff; + } + .location-selection-debug-table { + width: 100%; + border-collapse: collapse; + font-size: 0.78rem; + line-height: 1.25; + } + .location-selection-debug-table th, + .location-selection-debug-table td { + border-bottom: 1px solid #e2e8f0; + padding: 0.35rem 0.45rem; + text-align: left; + white-space: nowrap; + } + .location-selection-debug-table th { + position: sticky; + top: 0; + background: #f8fafc; + z-index: 1; + font-weight: 700; + } + .location-selection-pass { + color: #166534; + font-weight: 700; + } + .location-selection-fail { + color: #b91c1c; + font-weight: 700; + } .location-selection-intro h2 { font-size: 1.15rem; margin-bottom: 0.4rem; diff --git a/task-launcher/src/tasks/location-selection/timeline.ts b/task-launcher/src/tasks/location-selection/timeline.ts index b5621fda..483dd0b1 100644 --- a/task-launcher/src/tasks/location-selection/timeline.ts +++ b/task-launcher/src/tasks/location-selection/timeline.ts @@ -20,6 +20,7 @@ export default function buildLocationSelectionTimeline(config: Record/g, '>'); } +function renderCandidateTable(candidates: Array, threshold: number): string { + if (!Array.isArray(candidates) || !candidates.length) { + return '

No candidate data.

'; + } + + const rows = candidates.map((candidate) => { + const pass = Boolean(candidate?.pass); + const pop = candidate?.population == null ? 'n/a' : String(candidate.population); + const source = escapeHtml(String(candidate?.source || 'unknown')); + const cellId = escapeHtml(String(candidate?.cellId || '')); + const r = escapeHtml(String(candidate?.r ?? '')); + return ` + + ${r} + ${escapeHtml(pop)} + ${source} + ${pass ? 'pass' : 'fail'} + ${escapeHtml(String(threshold))} + ${cellId} + + `; + }).join(''); + + return ` +
+ + + + + + + + + + + + ${rows} +
respopulationsourcepassthresholdcellId
+
+ `; +} + export const reviewAndConfirm = { type: jsPsychHtmlMultiResponse, stimulus: () => { @@ -43,6 +85,7 @@ export const reviewAndConfirm = {

Location object to commit (schema preview):

Computing effective H3 with population lookup…

${commitPreviewJson}
+
@@ -62,15 +105,29 @@ export const reviewAndConfirm = { const draft = getLocationSelectionDraft(); const config = taskStore().locationSelectionConfig || null; const previewEl = document.getElementById('location-commit-preview-json'); + const candidatesEl = document.getElementById('location-commit-candidates-table'); const statusEl = document.getElementById('location-commit-preview-status'); - buildLocationCommitPreviewWithPopulation(draft, config) - .then((computedPreview) => { - const finalPreview = computedPreview || buildLocationCommitPreview(draft, config); + buildLocationCommitComputationWithPopulation(draft, config) + .then((computed) => { + const finalPreview = computed?.preview || buildLocationCommitPreview(draft, config); taskStore('locationSelectionCommitPreview', finalPreview); + taskStore('locationSelectionCommitCandidates', computed?.candidates || []); if (previewEl) { previewEl.textContent = JSON.stringify(finalPreview, null, 2); } + if (candidatesEl) { + const threshold = Number(finalPreview?.h3?.populationThreshold || 0); + const candidateLines = (computed?.candidates || []).map((candidate) => ({ + r: candidate.resolution, + population: candidate.population, + source: candidate.source, + pass: candidate.privacyMet, + threshold, + cellId: candidate.cellId, + })); + candidatesEl.innerHTML = renderCandidateTable(candidateLines, threshold); + } if (statusEl) { const source = finalPreview?.populationSource || 'unknown'; statusEl.textContent = `Population source used: ${source}`; @@ -79,7 +136,9 @@ export const reviewAndConfirm = { .catch(() => { const fallback = buildLocationCommitPreview(draft, config); taskStore('locationSelectionCommitPreview', fallback); + taskStore('locationSelectionCommitCandidates', []); if (statusEl) statusEl.textContent = 'Population lookup unavailable; using baseline-only preview.'; + if (candidatesEl) candidatesEl.innerHTML = '

No candidate data.

'; }); }, on_finish: (data: Record) => { From 08ccc106305d3a1a29e223015518f60a0d81a9d2 Mon Sep 17 00:00:00 2001 From: digital-pro Date: Thu, 5 Mar 2026 09:52:08 -0800 Subject: [PATCH 07/15] gitignore firebase cache --- task-launcher/.firebase/hosting.ZGlzdA.cache | 22 -------------------- 1 file changed, 22 deletions(-) delete mode 100644 task-launcher/.firebase/hosting.ZGlzdA.cache diff --git a/task-launcher/.firebase/hosting.ZGlzdA.cache b/task-launcher/.firebase/hosting.ZGlzdA.cache deleted file mode 100644 index 6e1f48dd..00000000 --- a/task-launcher/.firebase/hosting.ZGlzdA.cache +++ /dev/null @@ -1,22 +0,0 @@ -npm.bdelab.b6553285b766506d4db5.bundle.js,1722533652525,2d96480fb8658650b3fe94e8d0286fd5d024119053b12031c602306c390c6604 -npm.firebase.6b32e8187203c686e4b2.bundle.js,1722533652528,b5d94cf2d6f439a3e9e1fb43598e89223536b5828d4ad7d4db6eb5c4bfa77c7a -npm.firebase.6b32e8187203c686e4b2.bundle.js.LICENSE.txt,1722533652525,4facc61d515b997f114bfd9357915a2a5054e55a5cd437f8506a162972c52462 -npm.i18next.b0c0bd6865a29e21ac76.bundle.js,1726598233268,e74d8657bd930e6c5a2b6b792009f704ed950a50f1fe3e67b52269649e07ea63 -npm.jspsych-contrib.f6c74028b169a83d4351.bundle.js,1733854695144,3199fd713a82115e6ceee2c4323dd13299867e6e11b40f7fc812573f6c641aec -npm.lodash.f0b5bb3a2760e06abc17.bundle.js,1722533652528,937377b47de2d4b66281e97899301b645149b6d51256f908be54265de34db3a2 -npm.lodash.f0b5bb3a2760e06abc17.bundle.js.LICENSE.txt,1722533652525,e4189bf402a38a846ec375b1e65623511a1cff5534831bf64fe5564e12819e17 -npm.mime-db.6d1b18fe3806282a717a.bundle.js,1722533652528,9a44d22bef26270890f9e15407927a028b97f7988d445e115858f4643b11324d -npm.mime-db.6d1b18fe3806282a717a.bundle.js.LICENSE.txt,1722533652525,ccebece32d012b87132843042aff3330b815143c26ae6139d855d8ff32dbaf78 -npm.optimization-js.bb90cc2a952c02e153ba.bundle.js,1722533652525,ff1d6686ae23246b064f8d3d4d4d8793113efb40998ef61042e33f3c516744d6 -npm.regenerator-runtime.23db65d00e22e8c7aeaf.bundle.js,1722533652525,c96f1393f54fb03c5cc95d6e0ac7d94da21d8149ff8565f2c64f4c39e549513f -npm.seedrandom.d986754f3cc49359baa8.bundle.js,1722533652525,e5640b055d54dc04413e6a6756f1dc2d26c8598565f5862dfb47786208be4152 -npm.sentry-internal.bbae1cf39156d54cc56f.bundle.js,1722533652525,56645c017a494ba6d63ad49c036fa14830ab6b8139bcb5813b2454c292426192 -npm.sentry.fac9ab9a8e0fe3d8122c.bundle.js,1722533652525,18698036f54a331fd41b960d4639e924adc418b16b3a8d6f28b6b079ac2af8e3 -npm.vue.fe7a35d6a11f146ff855.bundle.js,1722533652528,7765107d56b64d324ca9146476da779c801fbe51663bf84553122f313f15c082 -npm.vue.fe7a35d6a11f146ff855.bundle.js.LICENSE.txt,1722533652528,84cf056599808ada84a456250d570c969a6a1472b547d2e0c0cacb6432d42865 -runtime.0ab4a229aba0b5c5cc38.bundle.js,1733183801845,9a6c0d9d89eadcbb960d28ea7f29b194692bc2c094d4cd3f1bda7a15a1a246d0 -img/levante-background..png,1733183801845,e500a8c007bd34b2683d218265b0820a38a1ea72d73f7b3f08678a1689f4434e -index.html,1734464710407,d375ef97413df77ec645ac0101ced3ceee0bbcf3947223ecc5cc97cb489d4c10 -index.1df0b721d4b077dc4c6d.bundle.js.LICENSE.txt,1734464710406,f63d602f34796886bb91d302239771de3638cd0b9a251b516d4a6662efda3a49 -npm.jspsych.69ed24913b56ee675cb1.bundle.js,1734464710406,e36008fc9f86913ae1dcfe6f2539d8d152c69f561d7daef3e3cdd9a72e5de4fb -index.1df0b721d4b077dc4c6d.bundle.js,1734464710406,f262bbaef581916cb2e21b654af1eabace52cd784ea79cad2e6917e46164dba4 From 5bdea14c6ea523edcdbcd0a63b08798620783a17 Mon Sep 17 00:00:00 2001 From: digital-pro Date: Thu, 5 Mar 2026 09:52:38 -0800 Subject: [PATCH 08/15] git ignore firebase cache --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 24ed5087..35d4fd46 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,8 @@ dist **/.ipynb_checkpoints .idea .DS_Store.firebase/*.cache +.firebase/ +.firebase/*.cache **/.DS_Store test-results/ lib/ From 70a559ddf8ae1f3cec4f5bea8fbc67997c481144 Mon Sep 17 00:00:00 2001 From: digital-pro Date: Fri, 6 Mar 2026 16:04:03 -0800 Subject: [PATCH 09/15] feat: add location save test hook Enable a debug save UI for location selection, cap H3 resolution at 7, and expose the Firebase app/config to support emulator saves. --- task-launcher/firebase.json | 6 +- task-launcher/serve/serve.js | 4 + .../location-selection/helpers/config.ts | 6 +- .../helpers/locationCommitPreview.ts | 10 +- .../trials/reviewAndConfirm.ts | 123 ++++++++++++++++++ 5 files changed, 141 insertions(+), 8 deletions(-) diff --git a/task-launcher/firebase.json b/task-launcher/firebase.json index 8a2d96e8..c2b35973 100644 --- a/task-launcher/firebase.json +++ b/task-launcher/firebase.json @@ -7,15 +7,15 @@ "emulators": { "auth": { "host": "127.0.0.1", - "port": 9199 + "port": 9290 }, "firestore": { "host": "127.0.0.1", - "port": 8180 + "port": 8185 }, "functions": { "host": "127.0.0.1", - "port": 5002 + "port": 5005 }, "ui": { "host": "127.0.0.1", diff --git a/task-launcher/serve/serve.js b/task-launcher/serve/serve.js index 96db7b41..e3664b5e 100644 --- a/task-launcher/serve/serve.js +++ b/task-launcher/serve/serve.js @@ -74,6 +74,10 @@ const demoMode = DEMO; async function startWebApp() { const appKit = await initializeFirebaseProject(firebaseConfig, 'admin', emulatorConfig, 'none'); + if (typeof window !== 'undefined') { + window.__firebaseApp = appKit.firebaseApp; + window.__firebaseConfig = firebaseConfig; + } onAuthStateChanged(appKit.auth, (user) => { if (user) { diff --git a/task-launcher/src/tasks/location-selection/helpers/config.ts b/task-launcher/src/tasks/location-selection/helpers/config.ts index f6a87740..ad7e486b 100644 --- a/task-launcher/src/tasks/location-selection/helpers/config.ts +++ b/task-launcher/src/tasks/location-selection/helpers/config.ts @@ -1,3 +1,5 @@ +export const H3_MAX_RESOLUTION = 7; + export type LocationSelectionTaskConfig = { populationThreshold: number; baselineResolution: number; @@ -25,8 +27,8 @@ export function getLocationSelectionTaskConfig(config: Record): Loc : 5, maxResolution: Number.isFinite(maxRes) && Number.isInteger(maxRes) && maxRes >= 0 && maxRes <= 15 - ? maxRes - : 9, + ? Math.min(maxRes, H3_MAX_RESOLUTION) + : H3_MAX_RESOLUTION, populationSourcePreference: sourcePreference === 'kontur' || sourcePreference === 'worldpop' || sourcePreference === 'auto' ? sourcePreference diff --git a/task-launcher/src/tasks/location-selection/helpers/locationCommitPreview.ts b/task-launcher/src/tasks/location-selection/helpers/locationCommitPreview.ts index 1d958833..1a93b087 100644 --- a/task-launcher/src/tasks/location-selection/helpers/locationCommitPreview.ts +++ b/task-launcher/src/tasks/location-selection/helpers/locationCommitPreview.ts @@ -1,6 +1,6 @@ import { cellToLatLng, latLngToCell } from 'h3-js'; import { type LocationSelectionDraft } from './state'; -import { type LocationSelectionTaskConfig } from './config'; +import { H3_MAX_RESOLUTION, type LocationSelectionTaskConfig } from './config'; import { lookupPopulationForCell } from './populationApi'; type LocationCommitPreview = { @@ -113,8 +113,12 @@ export async function buildLocationCommitComputationWithPopulation( const maxResolution = Number(config?.maxResolution); const populationThreshold = Number(config?.populationThreshold); const safeBaselineResolution = Number.isInteger(baselineResolution) ? baselineResolution : 5; - const safeMaxResolution = - Number.isInteger(maxResolution) && maxResolution >= safeBaselineResolution ? maxResolution : Math.max(9, safeBaselineResolution); + const safeMaxResolution = Math.min( + H3_MAX_RESOLUTION, + Number.isInteger(maxResolution) && maxResolution >= safeBaselineResolution + ? maxResolution + : Math.max(H3_MAX_RESOLUTION, safeBaselineResolution), + ); const safePopulationThreshold = Number.isFinite(populationThreshold) && populationThreshold > 0 ? Math.round(populationThreshold) : 50000; diff --git a/task-launcher/src/tasks/location-selection/trials/reviewAndConfirm.ts b/task-launcher/src/tasks/location-selection/trials/reviewAndConfirm.ts index 5c3f2b80..0ae5aa09 100644 --- a/task-launcher/src/tasks/location-selection/trials/reviewAndConfirm.ts +++ b/task-launcher/src/tasks/location-selection/trials/reviewAndConfirm.ts @@ -1,4 +1,7 @@ import jsPsychHtmlMultiResponse from '@jspsych-contrib/plugin-html-multi-response'; +import { getApp, getApps, initializeApp } from 'firebase/app'; +import { connectFunctionsEmulator, getFunctions, httpsCallable } from 'firebase/functions'; +import { getAuth } from 'firebase/auth'; import { taskStore } from '../../../taskStore'; import { getLocationSelectionDraft } from '../helpers/state'; import { ensureLocationSelectionStyles } from '../helpers/ui'; @@ -14,6 +17,49 @@ function escapeHtml(value: string): string { .replace(/>/g, '>'); } +const DEFAULT_LOCATION_SAVE_FUNCTION = 'upsertLocation'; +const DEFAULT_LOCATION_SAVE_PROJECT = 'hs-levante-admin-dev'; +const DEFAULT_LOCATION_SAVE_EMULATOR_HOST = 'http://localhost:5005'; +const DEFAULT_LOCATION_SAVE_COLLECTION = 'locations'; + +function isLocationSaveDebugEnabled() { + if (typeof window === 'undefined') return false; + const params = new URLSearchParams(window.location.search); + return params.get('locationSaveDebug') === 'true' + || ['localhost', '127.0.0.1'].includes(window.location.hostname); +} + +function getLocationSaveSettings() { + const params = new URLSearchParams(window.location.search); + return { + functionName: params.get('locationSaveFunction') || DEFAULT_LOCATION_SAVE_FUNCTION, + projectId: params.get('locationSaveProject') || DEFAULT_LOCATION_SAVE_PROJECT, + emulatorHost: params.get('locationSaveEmulatorHost') || DEFAULT_LOCATION_SAVE_EMULATOR_HOST, + collection: params.get('locationSaveCollection') || DEFAULT_LOCATION_SAVE_COLLECTION, + }; +} + +function buildLocationSavePayload() { + const params = new URLSearchParams(window.location.search); + const draft = getLocationSelectionDraft(); + const config = taskStore().locationSelectionConfig || null; + const preview = taskStore().locationSelectionCommitPreview || buildLocationCommitPreview(draft, config); + return { + location: preview, + collection: params.get('locationSaveCollection') || DEFAULT_LOCATION_SAVE_COLLECTION, + meta: { + task: 'location-selection', + mode: taskStore().locationSelectionMode || null, + selectedLocation: draft, + locationCommitPreview: preview, + locationCommitCandidates: taskStore().locationSelectionCommitCandidates || [], + pid: params.get('pid'), + language: params.get('lng'), + sentAt: new Date().toISOString(), + }, + }; +} + function renderCandidateTable(candidates: Array, threshold: number): string { if (!Array.isArray(candidates) || !candidates.length) { return '

No candidate data.

'; @@ -87,6 +133,22 @@ export const reviewAndConfirm = {
${commitPreviewJson}
+ ${isLocationSaveDebugEnabled() ? ` +
+

Test save location

+
+ + +
+ +

+

+

+
+ ` : ''}
@@ -140,6 +202,67 @@ export const reviewAndConfirm = { if (statusEl) statusEl.textContent = 'Population lookup unavailable; using baseline-only preview.'; if (candidatesEl) candidatesEl.innerHTML = '

No candidate data.

'; }); + + if (!isLocationSaveDebugEnabled()) return; + + const settings = getLocationSaveSettings(); + const targetSelect = document.getElementById('location-save-target') as HTMLSelectElement | null; + const saveButton = document.getElementById('location-save-button') as HTMLButtonElement | null; + const saveStatus = document.getElementById('location-save-status'); + const saveMeta = document.getElementById('location-save-meta'); + const saveDebug = document.getElementById('location-save-debug'); + + if (saveMeta) { + saveMeta.textContent = `Function: ${settings.functionName} | Project: ${settings.projectId} | Collection: ${settings.collection}`; + } + if (saveDebug) { + const globalApp = (window as any).__firebaseApp; + const hasGlobalConfig = Boolean((window as any).__firebaseConfig); + const appName = globalApp?.name || getApps()[0]?.name || 'none'; + const authApp = globalApp || getApps()[0] || null; + const authed = authApp ? Boolean(getAuth(authApp).currentUser) : false; + saveDebug.textContent = `Firebase app: ${appName} | Config: ${hasGlobalConfig ? 'window' : 'none'} | Authed: ${authed ? 'yes' : 'no'}`; + } + + if (!targetSelect || !saveButton || !saveStatus) return; + + saveButton.addEventListener('click', async () => { + const target = (targetSelect.value || 'emulator') as 'emulator' | 'dev'; + const startedAt = Date.now(); + saveStatus.textContent = `Saving to ${target}...`; + try { + const payload = buildLocationSavePayload(); + const globalApp = (typeof window !== 'undefined' ? (window as any).__firebaseApp : null) || null; + const globalConfig = (typeof window !== 'undefined' ? (window as any).__firebaseConfig : null) || null; + const apps = getApps(); + let app = globalApp || apps[0] || null; + if (!app && globalConfig) { + app = initializeApp(globalConfig); + } + if (!app) { + throw new Error('Firebase app not initialized'); + } + + const functionsInstance = getFunctions(app, 'us-central1'); + if (target === 'emulator') { + const emulatorUrl = new URL(settings.emulatorHost); + const host = emulatorUrl.hostname === '127.0.0.1' ? 'localhost' : emulatorUrl.hostname; + connectFunctionsEmulator(functionsInstance, host, Number(emulatorUrl.port || 5002)); + } + + const callable = httpsCallable(functionsInstance, settings.functionName, { timeout: 15000 }); + const result = await callable(payload); + const elapsedMs = Date.now() - startedAt; + saveStatus.textContent = `Saved ✓ (${elapsedMs}ms) ${JSON.stringify(result.data)}`; + } catch (error) { + const elapsedMs = Date.now() - startedAt; + const err = error as { code?: string; message?: string; details?: unknown }; + const details = err?.details ? ` | details: ${JSON.stringify(err.details)}` : ''; + const code = err?.code ? `code: ${err.code} | ` : ''; + const message = err?.message || (error instanceof Error ? error.message : String(error)); + saveStatus.textContent = `Save failed (${elapsedMs}ms): ${code}${message}${details}`; + } + }); }, on_finish: (data: Record) => { data.mode = taskStore().locationSelectionMode || null; From 405c811086606de4d728adb0f097fa2542e29c29 Mon Sep 17 00:00:00 2001 From: digital-pro Date: Fri, 6 Mar 2026 16:10:00 -0800 Subject: [PATCH 10/15] docs: add emulator save debug steps Document the location-selection save debug URL and emulator ports. --- README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.md b/README.md index c223252a..b5a29123 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,15 @@ npm run dev You can now locally run tasks e.g. TROG `http://localhost:8080/?task=trog`. Task parameters are documented here (TODO linkme). +### Location Selection Save Debug (Emulator) + +For testing location saves against the Firebase emulators: + +1. Start the emulators from the functions repo (auth `9290`, functions `5005`, firestore `8185`, UI `4002`). +2. Open the task with debug save enabled: + `http://localhost:8080/?task=locationselection&locationSaveDebug=true` +3. Click **Save** and confirm a `locations` doc appears in the Emulator UI. + Task details: 1. [Matrix Reasoning](https://hs-levante-assessment-dev.web.app/?task=matrix-reasoning) [George] From 2788a34f814f422a3a2b25021576ddcd41bb9f58 Mon Sep 17 00:00:00 2001 From: digital-pro Date: Sat, 7 Mar 2026 11:52:13 -0800 Subject: [PATCH 11/15] feat: add kontur batch cache support Load a compressed Kontur cache from levante-assets-dev, add batch lookup endpoint, and enable optional batched population lookup. --- README.md | 9 ++ task-launcher/serve/populationApi.cjs | 151 +++++++++++++++++- task-launcher/serve/serve.js | 4 + .../location-selection/helpers/config.ts | 7 + .../helpers/locationCommitPreview.ts | 16 +- .../helpers/populationApi.ts | 59 +++++++ .../src/tasks/shared/helpers/config.ts | 4 + 7 files changed, 241 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index b5a29123..116c16bc 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,15 @@ For testing location saves against the Firebase emulators: `http://localhost:8080/?task=locationselection&locationSaveDebug=true` 3. Click **Save** and confirm a `locations` doc appears in the Emulator UI. +### Kontur Population Cache + +The population lookup uses a local Kontur cache if available, otherwise it falls back to WorldPop. +You can point the dev server at a compressed, sparse Kontur cache stored elsewhere (e.g. GCS) by +setting one of these environment variables before starting `npm run dev`: + +- `KONTUR_H3_CACHE_URL` (supports `.gz`) +- `KONTUR_H3_CACHE_PATH` (local JSON path) + Task details: 1. [Matrix Reasoning](https://hs-levante-assessment-dev.web.app/?task=matrix-reasoning) [George] diff --git a/task-launcher/serve/populationApi.cjs b/task-launcher/serve/populationApi.cjs index 8f816b18..d9e13ea7 100644 --- a/task-launcher/serve/populationApi.cjs +++ b/task-launcher/serve/populationApi.cjs @@ -1,12 +1,14 @@ const fs = require('fs'); const path = require('path'); -const { cellToBoundary } = require('h3-js'); +const zlib = require('zlib'); +const { cellToBoundary, getResolution } = require('h3-js'); const WORLDPOP_STATS_URL = 'https://api.worldpop.org/v1/services/stats'; const WORLDPOP_TASK_URL = 'https://api.worldpop.org/v1/tasks'; let konturCacheByResolution = null; let konturCacheLoadedFrom = null; +let konturCachePromise = null; function sendJson(res, statusCode, payload) { res.status(statusCode).set('Content-Type', 'application/json').send(JSON.stringify(payload)); @@ -24,6 +26,14 @@ function parseResolution(value) { return n; } +function parseCellIds(value) { + if (!value) return []; + return String(value) + .split(',') + .map((cellId) => String(cellId).trim()) + .filter((cellId) => cellId.length > 0); +} + function buildCellPolygon(cellId) { const boundary = cellToBoundary(cellId); if (!Array.isArray(boundary) || !boundary.length) { @@ -52,8 +62,59 @@ function findKonturCachePath() { return candidates.find((candidate) => fs.existsSync(candidate)) || ''; } -function loadKonturCache() { +function parseKonturCacheJson(json, sourceLabel) { + const resolutions = json?.resolutions; + if (!resolutions || typeof resolutions !== 'object') return null; + konturCacheByResolution = resolutions; + konturCacheLoadedFrom = sourceLabel; + return konturCacheByResolution; +} + +async function loadKonturCacheFromUrl(cacheUrl) { + try { + const response = await fetch(cacheUrl); + if (!response.ok) return null; + const buffer = Buffer.from(await response.arrayBuffer()); + const isGzip = + String(response.headers.get('content-encoding') || '').includes('gzip') + || cacheUrl.endsWith('.gz'); + const jsonText = isGzip ? zlib.gunzipSync(buffer).toString('utf-8') : buffer.toString('utf-8'); + const json = JSON.parse(jsonText); + return parseKonturCacheJson(json, cacheUrl); + } catch (_err) { + return null; + } +} + +async function loadKonturCache() { if (konturCacheByResolution) return konturCacheByResolution; + if (konturCachePromise) return konturCachePromise; + konturCachePromise = (async () => { + const urlPath = String( + process.env.KONTUR_H3_CACHE_URL + || 'https://storage.googleapis.com/levante-assets-dev/maps/kontur-h3-population-cache.json.gz', + ).trim(); + if (urlPath) { + const remote = await loadKonturCacheFromUrl(urlPath); + if (remote) return remote; + } + + const cachePath = findKonturCachePath(); + if (!cachePath || !fs.existsSync(cachePath)) return null; + try { + const raw = fs.readFileSync(cachePath, 'utf-8'); + const json = JSON.parse(raw); + return parseKonturCacheJson(json, cachePath); + } catch (_err) { + return null; + } + })(); + try { + return await konturCachePromise; + } finally { + konturCachePromise = null; + } +} const cachePath = findKonturCachePath(); if (!cachePath || !fs.existsSync(cachePath)) return null; try { @@ -134,8 +195,8 @@ async function queryWorldPopForPolygon(polygon, year) { throw new Error(`WorldPop task ${taskId} timed out`); } -function resolveKonturPopulation(cellId, resolution) { - const cache = loadKonturCache(); +async function resolveKonturPopulation(cellId, resolution) { + const cache = await loadKonturCache(); if (!cache) return null; const byRes = cache[String(resolution)]; if (!byRes || typeof byRes !== 'object') return null; @@ -144,6 +205,65 @@ function resolveKonturPopulation(cellId, resolution) { return Math.round(value); } +async function resolveKonturPopulationBatch(cellIds, fallbackToWorldpop) { + const items = []; + for (let i = 0; i < cellIds.length; i += 1) { + const cellId = cellIds[i]; + let resolution = null; + try { + resolution = getResolution(cellId); + } catch (_err) { + items.push({ + cellId, + resolution: null, + population: null, + source: 'unknown', + error: 'invalid cellId', + }); + continue; + } + const konturPopulation = await resolveKonturPopulation(cellId, resolution); + if (typeof konturPopulation === 'number') { + items.push({ + cellId, + resolution, + population: konturPopulation, + source: 'kontur', + }); + continue; + } + if (!fallbackToWorldpop) { + items.push({ + cellId, + resolution, + population: null, + source: 'unknown', + }); + continue; + } + try { + const polygon = buildCellPolygon(cellId); + const worldpopPopulation = await queryWorldPopForPolygon(polygon, 2020); + items.push({ + cellId, + resolution, + population: Math.round(worldpopPopulation), + source: 'worldpop', + fallbackFrom: 'kontur', + }); + } catch (error) { + items.push({ + cellId, + resolution, + population: null, + source: 'unknown', + error: error?.message || 'Unknown error', + }); + } + } + return items; +} + function registerPopulationApi(app) { app.get('/api/population-kontur-h3', async (req, res) => { try { @@ -155,7 +275,7 @@ function registerPopulationApi(app) { return; } - const konturPopulation = resolveKonturPopulation(cellId, resolution); + const konturPopulation = await resolveKonturPopulation(cellId, resolution); if (typeof konturPopulation === 'number') { sendJson(res, 200, { success: true, @@ -184,6 +304,27 @@ function registerPopulationApi(app) { } }); + app.get('/api/population-kontur-h3-batch', async (req, res) => { + try { + const cellIds = parseCellIds(req.query?.cellIds); + if (!cellIds.length) { + sendJson(res, 400, { success: false, error: 'Missing cellIds' }); + return; + } + const fallback = String(req.query?.fallback || '').trim().toLowerCase(); + const fallbackToWorldpop = fallback === 'worldpop'; + const items = await resolveKonturPopulationBatch(cellIds, fallbackToWorldpop); + sendJson(res, 200, { + success: true, + source: 'kontur', + cachePath: konturCacheLoadedFrom || null, + items, + }); + } catch (error) { + sendJson(res, 500, { success: false, error: error?.message || 'Unknown error' }); + } + }); + app.get('/api/population-worldpop-h3', async (req, res) => { try { const cellId = String(req.query?.cellId || '').trim(); diff --git a/task-launcher/serve/serve.js b/task-launcher/serve/serve.js index e3664b5e..9bf0f59c 100644 --- a/task-launcher/serve/serve.js +++ b/task-launcher/serve/serve.js @@ -55,6 +55,7 @@ const semThreshold = Number(urlParams.get('semThreshold') || '0'); const startingTheta = Number(urlParams.get('theta') || '0'); const populationSourcePreference = String(urlParams.get('populationSourcePreference') || 'kontur').trim().toLowerCase(); const konturPopulationApiUrl = urlParams.get('konturPopulationApiUrl') || undefined; +const konturPopulationBatchApiUrl = urlParams.get('konturPopulationBatchApiUrl') || undefined; const worldpopPopulationApiUrl = urlParams.get('worldpopPopulationApiUrl') || undefined; const populationApiTimeoutMs = urlParams.get('populationApiTimeoutMs') === null ? undefined : parseInt(urlParams.get('populationApiTimeoutMs'), 10); @@ -67,6 +68,7 @@ const sequentialStimulus = stringToBoolean(urlParams.get('sequentialStimulus'), const storeItemId = stringToBoolean(urlParams.get('storeItemId'), false); const cat = stringToBoolean(urlParams.get('cat'), false); const heavyInstructions = stringToBoolean(urlParams.get('heavyInstructions'), false); +const populationBatchEnabled = stringToBoolean(urlParams.get('populationBatch'), false); const emulatorConfig = EMULATORS ? firebaseJSON.emulators : undefined; // if running in demo mode, no data will be saved to Firestore @@ -112,8 +114,10 @@ async function startWebApp() { startingTheta, populationSourcePreference, konturPopulationApiUrl, + konturPopulationBatchApiUrl, worldpopPopulationApiUrl, populationApiTimeoutMs, + populationBatchEnabled, heavyInstructions, demoMode, }; diff --git a/task-launcher/src/tasks/location-selection/helpers/config.ts b/task-launcher/src/tasks/location-selection/helpers/config.ts index ad7e486b..226f39d4 100644 --- a/task-launcher/src/tasks/location-selection/helpers/config.ts +++ b/task-launcher/src/tasks/location-selection/helpers/config.ts @@ -6,8 +6,10 @@ export type LocationSelectionTaskConfig = { maxResolution: number; populationSourcePreference: 'kontur' | 'worldpop' | 'auto'; konturPopulationApiUrl: string; + konturPopulationBatchApiUrl: string; worldpopPopulationApiUrl: string; populationApiTimeoutMs: number; + populationBatchEnabled: boolean; }; export function getLocationSelectionTaskConfig(config: Record): LocationSelectionTaskConfig { @@ -16,8 +18,11 @@ export function getLocationSelectionTaskConfig(config: Record): Loc const maxRes = Number(config?.maxResolution); const sourcePreference = String(config?.populationSourcePreference || 'kontur').trim().toLowerCase(); const konturPopulationApiUrl = String(config?.konturPopulationApiUrl || '/api/population-kontur-h3').trim(); + const konturPopulationBatchApiUrl = + String(config?.konturPopulationBatchApiUrl || '/api/population-kontur-h3-batch').trim(); const worldpopPopulationApiUrl = String(config?.worldpopPopulationApiUrl || '/api/population-worldpop-h3').trim(); const populationApiTimeoutMs = Number(config?.populationApiTimeoutMs); + const populationBatchEnabled = Boolean(config?.populationBatchEnabled); return { populationThreshold: Number.isFinite(threshold) && threshold > 0 ? Math.round(threshold) : 50000, @@ -34,10 +39,12 @@ export function getLocationSelectionTaskConfig(config: Record): Loc ? sourcePreference : 'kontur', konturPopulationApiUrl, + konturPopulationBatchApiUrl, worldpopPopulationApiUrl, populationApiTimeoutMs: Number.isFinite(populationApiTimeoutMs) && populationApiTimeoutMs > 0 ? Math.round(populationApiTimeoutMs) : 2500, + populationBatchEnabled, }; } diff --git a/task-launcher/src/tasks/location-selection/helpers/locationCommitPreview.ts b/task-launcher/src/tasks/location-selection/helpers/locationCommitPreview.ts index 1a93b087..68232d61 100644 --- a/task-launcher/src/tasks/location-selection/helpers/locationCommitPreview.ts +++ b/task-launcher/src/tasks/location-selection/helpers/locationCommitPreview.ts @@ -1,7 +1,7 @@ import { cellToLatLng, latLngToCell } from 'h3-js'; import { type LocationSelectionDraft } from './state'; import { H3_MAX_RESOLUTION, type LocationSelectionTaskConfig } from './config'; -import { lookupPopulationForCell } from './populationApi'; +import { lookupPopulationBatch, lookupPopulationForCell } from './populationApi'; type LocationCommitPreview = { schemaVersion: 'location_v1'; @@ -131,10 +131,18 @@ export async function buildLocationCommitComputationWithPopulation( let observedPopulationSource: 'kontur' | 'worldpop' | 'unknown' = 'unknown'; let foundPassingCell = false; const candidates: PopulationCandidateDebug[] = []; - + const useBatch = Boolean(config?.populationBatchEnabled); + const cellIds: string[] = []; for (let resolution = safeBaselineResolution; resolution <= safeMaxResolution; resolution += 1) { - const cellId = latLngToCell(draft.lat, draft.lon, resolution); - const populationResult = await lookupPopulationForCell(cellId, resolution, config); + cellIds.push(latLngToCell(draft.lat, draft.lon, resolution)); + } + const batchResults = useBatch ? await lookupPopulationBatch(cellIds, config) : null; + + for (let index = 0; index < cellIds.length; index += 1) { + const resolution = safeBaselineResolution + index; + const cellId = cellIds[index]; + const populationResult = + batchResults?.[cellId] ?? (await lookupPopulationForCell(cellId, resolution, config)); const population = populationResult.population; if (populationResult.source !== 'unknown' && observedPopulationSource === 'unknown') { observedPopulationSource = populationResult.source; diff --git a/task-launcher/src/tasks/location-selection/helpers/populationApi.ts b/task-launcher/src/tasks/location-selection/helpers/populationApi.ts index 59b2798e..06b66d9e 100644 --- a/task-launcher/src/tasks/location-selection/helpers/populationApi.ts +++ b/task-launcher/src/tasks/location-selection/helpers/populationApi.ts @@ -5,8 +5,10 @@ type PopulationSource = 'kontur' | 'worldpop'; export type PopulationLookupConfig = { populationSourcePreference?: 'kontur' | 'worldpop' | 'auto'; konturPopulationApiUrl?: string; + konturPopulationBatchApiUrl?: string; worldpopPopulationApiUrl?: string; populationApiTimeoutMs?: number; + populationBatchEnabled?: boolean; }; type PopulationLookupResult = { @@ -14,6 +16,18 @@ type PopulationLookupResult = { source: PopulationSource | 'unknown'; }; +type PopulationBatchItem = { + cellId: string; + resolution: number | null; + population: number | null; + source: PopulationSource | 'unknown'; +}; + +type PopulationBatchResult = { + success: boolean; + items?: PopulationBatchItem[]; +}; + function parsePopulation(payload: any): number | null { const candidates = [ payload?.data?.total_population, @@ -147,6 +161,42 @@ async function fetchPopulation( } } +async function fetchPopulationBatch( + endpoint: string, + cellIds: string[], + timeoutMs: number, +): Promise> { + if (!endpoint || !cellIds.length) return {}; + try { + const url = new URL(endpoint, window.location.origin); + url.searchParams.set('cellIds', cellIds.join(',')); + url.searchParams.set('fallback', 'worldpop'); + + const controller = new AbortController(); + const timeout = window.setTimeout(() => controller.abort(), timeoutMs); + const response = await fetch(url.toString(), { + method: 'GET', + signal: controller.signal, + }); + window.clearTimeout(timeout); + if (!response.ok) return {}; + const payload = (await response.json().catch(() => ({}))) as PopulationBatchResult; + const items = Array.isArray(payload?.items) ? payload.items : []; + return items.reduce>((acc, item) => { + if (!item?.cellId) return acc; + const resolvedSource: PopulationSource | 'unknown' = + item.source === 'kontur' || item.source === 'worldpop' ? item.source : 'unknown'; + acc[item.cellId] = { + population: typeof item.population === 'number' ? item.population : null, + source: resolvedSource, + }; + return acc; + }, {}); + } catch { + return {}; + } +} + export async function lookupPopulationForCell( cellId: string, resolution: number, @@ -175,3 +225,12 @@ export async function lookupPopulationForCell( return { population: null, source: attemptedKnownSource }; } + +export async function lookupPopulationBatch( + cellIds: string[], + config: PopulationLookupConfig | null | undefined, +): Promise> { + const batchUrl = String(config?.konturPopulationBatchApiUrl || '/api/population-kontur-h3-batch'); + const timeoutMs = Number(config?.populationApiTimeoutMs) > 0 ? Number(config?.populationApiTimeoutMs) : 2500; + return fetchPopulationBatch(batchUrl, cellIds, timeoutMs); +} diff --git a/task-launcher/src/tasks/shared/helpers/config.ts b/task-launcher/src/tasks/shared/helpers/config.ts index 796c13a9..c513498b 100644 --- a/task-launcher/src/tasks/shared/helpers/config.ts +++ b/task-launcher/src/tasks/shared/helpers/config.ts @@ -95,8 +95,10 @@ export const setSharedConfig = async ( demoMode, populationSourcePreference, konturPopulationApiUrl, + konturPopulationBatchApiUrl, worldpopPopulationApiUrl, populationApiTimeoutMs, + populationBatchEnabled, } = cleanParams; const config = { @@ -129,8 +131,10 @@ export const setSharedConfig = async ( demoMode: !!demoMode, populationSourcePreference: String(populationSourcePreference || 'kontur'), konturPopulationApiUrl: konturPopulationApiUrl ? String(konturPopulationApiUrl) : undefined, + konturPopulationBatchApiUrl: konturPopulationBatchApiUrl ? String(konturPopulationBatchApiUrl) : undefined, worldpopPopulationApiUrl: worldpopPopulationApiUrl ? String(worldpopPopulationApiUrl) : undefined, populationApiTimeoutMs: Number(populationApiTimeoutMs) || undefined, + populationBatchEnabled: !!populationBatchEnabled, }; // default corpus if nothing is passed in From 24eb315f0768d2c70283a01a95286951ff7b6408 Mon Sep 17 00:00:00 2001 From: digital-pro Date: Sat, 7 Mar 2026 12:03:38 -0800 Subject: [PATCH 12/15] feat: shard kontur cache by r5 Load compressed kontur shards from levante-assets-dev/maps/kontur-h3-r5 and document shard layout. --- README.md | 7 +- task-launcher/serve/populationApi.cjs | 155 ++++++++++++++++---------- 2 files changed, 98 insertions(+), 64 deletions(-) diff --git a/README.md b/README.md index 116c16bc..67eb7fe0 100644 --- a/README.md +++ b/README.md @@ -39,10 +39,11 @@ For testing location saves against the Firebase emulators: The population lookup uses a local Kontur cache if available, otherwise it falls back to WorldPop. You can point the dev server at a compressed, sparse Kontur cache stored elsewhere (e.g. GCS) by -setting one of these environment variables before starting `npm run dev`: +setting one of these environment variables before starting `npm run dev`. The cache is sharded +by R5 parent cell, so the URL/path should be a *folder* containing `{r5CellId}.json.gz` files: -- `KONTUR_H3_CACHE_URL` (supports `.gz`) -- `KONTUR_H3_CACHE_PATH` (local JSON path) +- `KONTUR_H3_CACHE_URL` (base URL; supports `.gz` shards) +- `KONTUR_H3_CACHE_PATH` (base folder for local shards) Task details: diff --git a/task-launcher/serve/populationApi.cjs b/task-launcher/serve/populationApi.cjs index d9e13ea7..b3d7afff 100644 --- a/task-launcher/serve/populationApi.cjs +++ b/task-launcher/serve/populationApi.cjs @@ -1,14 +1,15 @@ const fs = require('fs'); const path = require('path'); const zlib = require('zlib'); -const { cellToBoundary, getResolution } = require('h3-js'); +const { cellToBoundary, cellToParent, getResolution } = require('h3-js'); const WORLDPOP_STATS_URL = 'https://api.worldpop.org/v1/services/stats'; const WORLDPOP_TASK_URL = 'https://api.worldpop.org/v1/tasks'; -let konturCacheByResolution = null; -let konturCacheLoadedFrom = null; -let konturCachePromise = null; +const konturShardCache = new Map(); +const konturShardInFlight = new Map(); +const defaultShardBaseUrl = 'https://storage.googleapis.com/levante-assets-dev/maps/kontur-h3-r5'; +const defaultShardBasePath = path.resolve(process.cwd(), 'data', 'kontur-h3-r5'); function sendJson(res, statusCode, payload) { res.status(statusCode).set('Content-Type', 'application/json').send(JSON.stringify(payload)); @@ -51,82 +52,102 @@ function buildCellPolygon(cellId) { }; } -function findKonturCachePath() { - const envPath = String(process.env.KONTUR_H3_CACHE_PATH || '').trim(); - if (envPath) return envPath; - const candidates = [ - path.resolve(process.cwd(), 'data', 'gallery', 'kontur-h3-population-cache.json'), - path.resolve(process.cwd(), '..', 'levante-web-dashboard', 'data', 'gallery', 'kontur-h3-population-cache.json'), - path.resolve(process.cwd(), '..', '..', 'levante-web-dashboard', 'data', 'gallery', 'kontur-h3-population-cache.json'), - ]; - return candidates.find((candidate) => fs.existsSync(candidate)) || ''; +function getShardCacheLimit() { + const raw = Number(process.env.KONTUR_H3_CACHE_MAX_SHARDS || 64); + return Number.isFinite(raw) && raw > 0 ? Math.round(raw) : 64; } -function parseKonturCacheJson(json, sourceLabel) { +function parseKonturShardJson(json, sourceLabel) { const resolutions = json?.resolutions; if (!resolutions || typeof resolutions !== 'object') return null; - konturCacheByResolution = resolutions; - konturCacheLoadedFrom = sourceLabel; - return konturCacheByResolution; + return { resolutions, loadedFrom: sourceLabel }; } -async function loadKonturCacheFromUrl(cacheUrl) { +async function loadKonturShardFromUrl(shardUrl) { try { - const response = await fetch(cacheUrl); + const response = await fetch(shardUrl); if (!response.ok) return null; const buffer = Buffer.from(await response.arrayBuffer()); const isGzip = String(response.headers.get('content-encoding') || '').includes('gzip') - || cacheUrl.endsWith('.gz'); + || shardUrl.endsWith('.gz'); const jsonText = isGzip ? zlib.gunzipSync(buffer).toString('utf-8') : buffer.toString('utf-8'); const json = JSON.parse(jsonText); - return parseKonturCacheJson(json, cacheUrl); + return parseKonturShardJson(json, shardUrl); } catch (_err) { return null; } } -async function loadKonturCache() { - if (konturCacheByResolution) return konturCacheByResolution; - if (konturCachePromise) return konturCachePromise; - konturCachePromise = (async () => { - const urlPath = String( - process.env.KONTUR_H3_CACHE_URL - || 'https://storage.googleapis.com/levante-assets-dev/maps/kontur-h3-population-cache.json.gz', - ).trim(); - if (urlPath) { - const remote = await loadKonturCacheFromUrl(urlPath); +function getShardCacheKey(shardCellId) { + return `r5:${shardCellId}`; +} + +function getShardBaseUrl() { + const raw = String(process.env.KONTUR_H3_CACHE_URL || '').trim(); + return raw || defaultShardBaseUrl; +} + +function getShardBasePath() { + const raw = String(process.env.KONTUR_H3_CACHE_PATH || '').trim(); + return raw || defaultShardBasePath; +} + +function resolveShardUrl(shardCellId) { + const base = getShardBaseUrl(); + if (!base) return ''; + return `${base}/${shardCellId}.json.gz`; +} + +function resolveShardPath(shardCellId) { + const base = getShardBasePath(); + if (!base) return ''; + return path.resolve(base, `${shardCellId}.json`); +} + +async function loadKonturShard(shardCellId) { + const cacheKey = getShardCacheKey(shardCellId); + const existing = konturShardCache.get(cacheKey); + if (existing) { + konturShardCache.delete(cacheKey); + konturShardCache.set(cacheKey, existing); + return existing; + } + const inflight = konturShardInFlight.get(cacheKey); + if (inflight) return inflight; + + const loader = (async () => { + const shardUrl = resolveShardUrl(shardCellId); + if (shardUrl) { + const remote = await loadKonturShardFromUrl(shardUrl); if (remote) return remote; } - - const cachePath = findKonturCachePath(); - if (!cachePath || !fs.existsSync(cachePath)) return null; - try { - const raw = fs.readFileSync(cachePath, 'utf-8'); - const json = JSON.parse(raw); - return parseKonturCacheJson(json, cachePath); - } catch (_err) { - return null; + const shardPath = resolveShardPath(shardCellId); + if (shardPath && fs.existsSync(shardPath)) { + try { + const raw = fs.readFileSync(shardPath, 'utf-8'); + const json = JSON.parse(raw); + return parseKonturShardJson(json, shardPath); + } catch (_err) { + return null; + } } + return null; })(); + konturShardInFlight.set(cacheKey, loader); try { - return await konturCachePromise; + const loaded = await loader; + if (loaded) { + konturShardCache.set(cacheKey, loaded); + const limit = getShardCacheLimit(); + while (konturShardCache.size > limit) { + const oldestKey = konturShardCache.keys().next().value; + konturShardCache.delete(oldestKey); + } + } + return loaded; } finally { - konturCachePromise = null; - } -} - const cachePath = findKonturCachePath(); - if (!cachePath || !fs.existsSync(cachePath)) return null; - try { - const raw = fs.readFileSync(cachePath, 'utf-8'); - const json = JSON.parse(raw); - const resolutions = json?.resolutions; - if (!resolutions || typeof resolutions !== 'object') return null; - konturCacheByResolution = resolutions; - konturCacheLoadedFrom = cachePath; - return konturCacheByResolution; - } catch (_err) { - return null; + konturShardInFlight.delete(cacheKey); } } @@ -196,9 +217,15 @@ async function queryWorldPopForPolygon(polygon, year) { } async function resolveKonturPopulation(cellId, resolution) { - const cache = await loadKonturCache(); - if (!cache) return null; - const byRes = cache[String(resolution)]; + if (resolution < 5) return null; + let shardCellId = null; + try { + shardCellId = cellToParent(cellId, 5); + } catch (_err) { + return null; + } + const shard = await loadKonturShard(shardCellId); + const byRes = shard?.resolutions?.[String(resolution)]; if (!byRes || typeof byRes !== 'object') return null; const value = Number(byRes[cellId]); if (!Number.isFinite(value) || value < 0) return null; @@ -277,13 +304,19 @@ function registerPopulationApi(app) { const konturPopulation = await resolveKonturPopulation(cellId, resolution); if (typeof konturPopulation === 'number') { + let cachePath = null; + try { + cachePath = resolveShardUrl(cellToParent(cellId, 5)); + } catch (_err) { + cachePath = null; + } sendJson(res, 200, { success: true, source: 'kontur', population: konturPopulation, resolution, cellId, - cachePath: konturCacheLoadedFrom || null, + cachePath, }); return; } @@ -317,7 +350,7 @@ function registerPopulationApi(app) { sendJson(res, 200, { success: true, source: 'kontur', - cachePath: konturCacheLoadedFrom || null, + cachePath: getShardBaseUrl(), items, }); } catch (error) { From 7d321536d3546154083c2249c1f31630ce674a53 Mon Sep 17 00:00:00 2001 From: digital-pro Date: Sat, 7 Mar 2026 12:12:25 -0800 Subject: [PATCH 13/15] feat: add kontur shard builder script Add a repeatable script to download the kontur dataset and generate R5 shards, plus README instructions. --- README.md | 13 ++ .../scripts/build_kontur_r5_shards.py | 194 ++++++++++++++++++ 2 files changed, 207 insertions(+) create mode 100644 task-launcher/scripts/build_kontur_r5_shards.py diff --git a/README.md b/README.md index 67eb7fe0..51ebe858 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,19 @@ by R5 parent cell, so the URL/path should be a *folder* containing `{r5CellId}.j - `KONTUR_H3_CACHE_URL` (base URL; supports `.gz` shards) - `KONTUR_H3_CACHE_PATH` (base folder for local shards) +#### Build the R5 shard cache + +We provide a repeatable script to download the Kontur dataset and build R5 shards: + +```bash +cd task-launcher +pip install h3 pyarrow +python scripts/build_kontur_r5_shards.py --download --gzip --output data/kontur-h3-r5 +``` + +This uses the latest 400m Kontur dataset from HDX and requires `ogr2ogr` (GDAL) to convert +the GeoPackage into Parquet for streaming. + Task details: 1. [Matrix Reasoning](https://hs-levante-assessment-dev.web.app/?task=matrix-reasoning) [George] diff --git a/task-launcher/scripts/build_kontur_r5_shards.py b/task-launcher/scripts/build_kontur_r5_shards.py new file mode 100644 index 00000000..0395effd --- /dev/null +++ b/task-launcher/scripts/build_kontur_r5_shards.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python3 +import argparse +import gzip +import json +import os +import shutil +import subprocess +import sys +import tempfile +import urllib.request +from collections import OrderedDict + +try: + from h3 import h3 +except ImportError as exc: + raise SystemExit("Missing dependency: pip install h3") from exc + +try: + import pyarrow.dataset as ds +except ImportError as exc: + raise SystemExit("Missing dependency: pip install pyarrow") from exc + + +DEFAULT_DATASET_URL = ( + "https://geodata-eu-central-1-kontur-public.s3.eu-central-1.amazonaws.com/" + "kontur_datasets/kontur_population_20231101.gpkg.gz" +) + + +def download_file(url: str, dest_path: str) -> None: + os.makedirs(os.path.dirname(dest_path), exist_ok=True) + with urllib.request.urlopen(url) as response, open(dest_path, "wb") as out_file: + shutil.copyfileobj(response, out_file) + + +def gunzip_file(src_path: str, dest_path: str) -> None: + os.makedirs(os.path.dirname(dest_path), exist_ok=True) + with gzip.open(src_path, "rb") as src, open(dest_path, "wb") as dest: + shutil.copyfileobj(src, dest) + + +def ensure_parquet_from_gpkg(gpkg_path: str, parquet_path: str) -> None: + if os.path.exists(parquet_path): + return + ogr2ogr = shutil.which("ogr2ogr") + if not ogr2ogr: + raise SystemExit("ogr2ogr not found; install GDAL to convert GPKG -> Parquet.") + os.makedirs(os.path.dirname(parquet_path), exist_ok=True) + cmd = [ + ogr2ogr, + "-f", + "Parquet", + parquet_path, + gpkg_path, + "-select", + "h3,population", + ] + subprocess.check_call(cmd) + + +def iter_rows_from_parquet(parquet_path: str): + dataset = ds.dataset(parquet_path, format="parquet") + for batch in dataset.to_batches(columns=["h3", "population"]): + h3_col = batch.column(0).to_pylist() + pop_col = batch.column(1).to_pylist() + for h3_id, pop in zip(h3_col, pop_col): + yield h3_id, pop + + +def merge_into(existing: dict, incoming: dict) -> dict: + for res, cells in incoming.items(): + res_map = existing.setdefault(res, {}) + for cell_id, pop in cells.items(): + res_map[cell_id] = res_map.get(cell_id, 0) + pop + return existing + + +def flush_shard(output_dir: str, r5_cell_id: str, shard_data: dict, gzip_output: bool) -> None: + os.makedirs(output_dir, exist_ok=True) + filename = f"{r5_cell_id}.json.gz" if gzip_output else f"{r5_cell_id}.json" + output_path = os.path.join(output_dir, filename) + existing = {} + if os.path.exists(output_path): + if gzip_output: + with gzip.open(output_path, "rt", encoding="utf-8") as fh: + existing = json.load(fh) + else: + with open(output_path, "r", encoding="utf-8") as fh: + existing = json.load(fh) + merged = merge_into(existing.get("resolutions", {}), shard_data) + payload = {"resolutions": merged} + if gzip_output: + with gzip.open(output_path, "wt", encoding="utf-8") as fh: + json.dump(payload, fh, separators=(",", ":")) + else: + with open(output_path, "w", encoding="utf-8") as fh: + json.dump(payload, fh, separators=(",", ":")) + + +def build_shards( + input_path: str, + output_dir: str, + resolutions: list[int], + max_shards: int, + gzip_output: bool, +) -> None: + shard_cache: OrderedDict[str, dict] = OrderedDict() + + def get_shard(r5_cell_id: str) -> dict: + if r5_cell_id in shard_cache: + shard_cache.move_to_end(r5_cell_id) + return shard_cache[r5_cell_id] + if len(shard_cache) >= max_shards: + oldest_r5, oldest_data = shard_cache.popitem(last=False) + flush_shard(output_dir, oldest_r5, oldest_data, gzip_output) + shard_cache[r5_cell_id] = {str(res): {} for res in resolutions} + return shard_cache[r5_cell_id] + + for h3_id, pop in iter_rows_from_parquet(input_path): + if h3_id is None or pop is None: + continue + try: + pop_val = float(pop) + except (TypeError, ValueError): + continue + if pop_val <= 0: + continue + try: + base_resolution = h3.h3_get_resolution(h3_id) + except Exception: + continue + try: + r5_cell = h3.h3_to_parent(h3_id, 5) + except Exception: + continue + + shard = get_shard(r5_cell) + for res in resolutions: + if res > base_resolution: + continue + try: + parent = h3.h3_to_parent(h3_id, res) + except Exception: + continue + res_map = shard[str(res)] + res_map[parent] = res_map.get(parent, 0) + pop_val + + for r5_cell, data in shard_cache.items(): + flush_shard(output_dir, r5_cell, data, gzip_output) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Build R5-sharded Kontur H3 population cache.") + parser.add_argument("--input", help="Input Parquet file with h3,population columns.") + parser.add_argument("--output", default="data/kontur-h3-r5", help="Output shard directory.") + parser.add_argument("--resolutions", default="5,6,7", help="Comma-separated resolutions to build.") + parser.add_argument("--max-shards", type=int, default=64, help="Max in-memory shard count.") + parser.add_argument("--download", action="store_true", help="Download and convert dataset.") + parser.add_argument("--gzip", action="store_true", help="Write .json.gz shards.") + args = parser.parse_args() + + if args.download: + raw_dir = os.path.join("data", "kontur", "raw") + os.makedirs(raw_dir, exist_ok=True) + gz_path = os.path.join(raw_dir, "kontur_population_20231101.gpkg.gz") + gpkg_path = os.path.join(raw_dir, "kontur_population_20231101.gpkg") + parquet_path = os.path.join(raw_dir, "kontur_population_20231101.parquet") + if not os.path.exists(gz_path): + print(f"Downloading {DEFAULT_DATASET_URL} ...") + download_file(DEFAULT_DATASET_URL, gz_path) + if not os.path.exists(gpkg_path): + print("Extracting .gpkg.gz ...") + gunzip_file(gz_path, gpkg_path) + ensure_parquet_from_gpkg(gpkg_path, parquet_path) + input_path = parquet_path + else: + input_path = args.input + + if not input_path or not os.path.exists(input_path): + raise SystemExit("Input Parquet file not found. Use --input or --download.") + + resolutions = [int(x.strip()) for x in args.resolutions.split(",") if x.strip()] + build_shards( + input_path=input_path, + output_dir=args.output, + resolutions=resolutions, + max_shards=max(args.max_shards, 1), + gzip_output=args.gzip, + ) + print(f"Shards written to {args.output}") + + +if __name__ == "__main__": + main() From 1ff3d4145d9413c2bf764c91b8110b6fa93a38ba Mon Sep 17 00:00:00 2001 From: digital-pro Date: Sat, 7 Mar 2026 13:28:23 -0800 Subject: [PATCH 14/15] chore: ignore local kontur build artifacts Ignore venv/data/log files and update the kontur shard builder docs. --- .gitignore | 4 ++ README.md | 4 +- .../scripts/build_kontur_r5_shards.py | 65 ++++++++++++++----- 3 files changed, 56 insertions(+), 17 deletions(-) diff --git a/.gitignore b/.gitignore index 35d4fd46..2f757c43 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,7 @@ test-results/ lib/ .env .idea/ +.vscode/ +task-launcher/.venv/ +task-launcher/data/ +task-launcher/firestore-debug.log diff --git a/README.md b/README.md index 51ebe858..09a3ecc2 100644 --- a/README.md +++ b/README.md @@ -50,13 +50,13 @@ by R5 parent cell, so the URL/path should be a *folder* containing `{r5CellId}.j We provide a repeatable script to download the Kontur dataset and build R5 shards: ```bash -cd task-launcher +cd /home/david/levante/core-tasks/task-launcher pip install h3 pyarrow python scripts/build_kontur_r5_shards.py --download --gzip --output data/kontur-h3-r5 ``` This uses the latest 400m Kontur dataset from HDX and requires `ogr2ogr` (GDAL) to convert -the GeoPackage into Parquet for streaming. +the GeoPackage into Parquet/CSV for streaming. Task details: diff --git a/task-launcher/scripts/build_kontur_r5_shards.py b/task-launcher/scripts/build_kontur_r5_shards.py index 0395effd..88ddb89a 100644 --- a/task-launcher/scripts/build_kontur_r5_shards.py +++ b/task-launcher/scripts/build_kontur_r5_shards.py @@ -1,24 +1,24 @@ #!/usr/bin/env python3 import argparse +import csv import gzip import json import os import shutil import subprocess import sys -import tempfile import urllib.request from collections import OrderedDict try: - from h3 import h3 + import h3 except ImportError as exc: raise SystemExit("Missing dependency: pip install h3") from exc try: import pyarrow.dataset as ds -except ImportError as exc: - raise SystemExit("Missing dependency: pip install pyarrow") from exc +except ImportError: + ds = None DEFAULT_DATASET_URL = ( @@ -39,26 +39,46 @@ def gunzip_file(src_path: str, dest_path: str) -> None: shutil.copyfileobj(src, dest) -def ensure_parquet_from_gpkg(gpkg_path: str, parquet_path: str) -> None: +def ensure_tabular_from_gpkg(gpkg_path: str, parquet_path: str, csv_path: str) -> tuple[str, str]: if os.path.exists(parquet_path): - return + return "parquet", parquet_path + if os.path.exists(csv_path): + return "csv", csv_path ogr2ogr = shutil.which("ogr2ogr") if not ogr2ogr: - raise SystemExit("ogr2ogr not found; install GDAL to convert GPKG -> Parquet.") + raise SystemExit("ogr2ogr not found; install GDAL to convert GPKG.") os.makedirs(os.path.dirname(parquet_path), exist_ok=True) + if ds is not None: + cmd = [ + ogr2ogr, + "-f", + "Parquet", + parquet_path, + gpkg_path, + "-select", + "h3,population", + ] + try: + subprocess.check_call(cmd) + return "parquet", parquet_path + except subprocess.CalledProcessError: + pass cmd = [ ogr2ogr, "-f", - "Parquet", - parquet_path, + "CSV", + csv_path, gpkg_path, "-select", "h3,population", ] subprocess.check_call(cmd) + return "csv", csv_path def iter_rows_from_parquet(parquet_path: str): + if ds is None: + raise SystemExit("pyarrow not available for parquet parsing.") dataset = ds.dataset(parquet_path, format="parquet") for batch in dataset.to_batches(columns=["h3", "population"]): h3_col = batch.column(0).to_pylist() @@ -67,6 +87,13 @@ def iter_rows_from_parquet(parquet_path: str): yield h3_id, pop +def iter_rows_from_csv(csv_path: str): + with open(csv_path, "r", encoding="utf-8") as fh: + reader = csv.DictReader(fh) + for row in reader: + yield row.get("h3"), row.get("population") + + def merge_into(existing: dict, incoming: dict) -> dict: for res, cells in incoming.items(): res_map = existing.setdefault(res, {}) @@ -99,6 +126,7 @@ def flush_shard(output_dir: str, r5_cell_id: str, shard_data: dict, gzip_output: def build_shards( input_path: str, + input_format: str, output_dir: str, resolutions: list[int], max_shards: int, @@ -116,7 +144,12 @@ def get_shard(r5_cell_id: str) -> dict: shard_cache[r5_cell_id] = {str(res): {} for res in resolutions} return shard_cache[r5_cell_id] - for h3_id, pop in iter_rows_from_parquet(input_path): + if input_format == "parquet": + row_iter = iter_rows_from_parquet(input_path) + else: + row_iter = iter_rows_from_csv(input_path) + + for h3_id, pop in row_iter: if h3_id is None or pop is None: continue try: @@ -126,11 +159,11 @@ def get_shard(r5_cell_id: str) -> dict: if pop_val <= 0: continue try: - base_resolution = h3.h3_get_resolution(h3_id) + base_resolution = h3.get_resolution(h3_id) except Exception: continue try: - r5_cell = h3.h3_to_parent(h3_id, 5) + r5_cell = h3.cell_to_parent(h3_id, 5) except Exception: continue @@ -139,7 +172,7 @@ def get_shard(r5_cell_id: str) -> dict: if res > base_resolution: continue try: - parent = h3.h3_to_parent(h3_id, res) + parent = h3.cell_to_parent(h3_id, res) except Exception: continue res_map = shard[str(res)] @@ -165,16 +198,17 @@ def main() -> None: gz_path = os.path.join(raw_dir, "kontur_population_20231101.gpkg.gz") gpkg_path = os.path.join(raw_dir, "kontur_population_20231101.gpkg") parquet_path = os.path.join(raw_dir, "kontur_population_20231101.parquet") + csv_path = os.path.join(raw_dir, "kontur_population_20231101.csv") if not os.path.exists(gz_path): print(f"Downloading {DEFAULT_DATASET_URL} ...") download_file(DEFAULT_DATASET_URL, gz_path) if not os.path.exists(gpkg_path): print("Extracting .gpkg.gz ...") gunzip_file(gz_path, gpkg_path) - ensure_parquet_from_gpkg(gpkg_path, parquet_path) - input_path = parquet_path + input_format, input_path = ensure_tabular_from_gpkg(gpkg_path, parquet_path, csv_path) else: input_path = args.input + input_format = "parquet" if input_path and input_path.endswith(".parquet") else "csv" if not input_path or not os.path.exists(input_path): raise SystemExit("Input Parquet file not found. Use --input or --download.") @@ -182,6 +216,7 @@ def main() -> None: resolutions = [int(x.strip()) for x in args.resolutions.split(",") if x.strip()] build_shards( input_path=input_path, + input_format=input_format, output_dir=args.output, resolutions=resolutions, max_shards=max(args.max_shards, 1), From 261bbe34add6e5d6fdf3fad810e35f83df5f0b0b Mon Sep 17 00:00:00 2001 From: digital-pro Date: Sat, 7 Mar 2026 13:29:19 -0800 Subject: [PATCH 15/15] chore: add task-launcher gitignore Ignore local venv/data/log artifacts in the task-launcher folder. --- task-launcher/.gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/task-launcher/.gitignore b/task-launcher/.gitignore index 37e71335..92eb7316 100644 --- a/task-launcher/.gitignore +++ b/task-launcher/.gitignore @@ -1,3 +1,6 @@ +.venv/ +data/ +firestore-debug.log node_modules dist **/.Rhistory