diff --git a/task-launcher/src/index.ts b/task-launcher/src/index.ts index c2185b2f..28ef159f 100644 --- a/task-launcher/src/index.ts +++ b/task-launcher/src/index.ts @@ -56,9 +56,16 @@ export class TaskLauncher { const languageAudioBucket = getBucketName('shared', isDev, 'audio', language); const sharedAudioBucket = getBucketName('shared', isDev, 'audio', 'shared'); + await getAssetsPerTask(isDev); + + const taskAudioAssetNames = [ + ...taskStore().assetsPerTask[taskName].audio, + ...taskStore().assetsPerTask.shared.audio, + ]; + try { // will avoid language folder if not provided - languageAudioAssets = await getMediaAssets(languageAudioBucket, {}, taskName, language); + languageAudioAssets = await getMediaAssets(languageAudioBucket, {}, taskName, language, taskAudioAssetNames); sharedAudioAssets = await getMediaAssets(sharedAudioBucket, {}, taskName, 'shared'); taskVisualAssets = await getMediaAssets(taskVisualBucket, {}, taskName, language); sharedVisualAssets = await getMediaAssets(sharedVisualBucket, {}, 'shared', language); @@ -77,16 +84,6 @@ export class TaskLauncher { await getCorpus(config, isDev); } - await getAssetsPerTask(isDev); - - const taskAudioAssetNames = [ - ...taskStore().assetsPerTask[taskName].audio, - ...taskStore().assetsPerTask.shared.audio, - ]; - - // filter out language audio not relevant to current task - languageAudioAssets = filterMedia(languageAudioAssets, [], taskAudioAssetNames, []); - mediaAssets = combineMediaAssets([languageAudioAssets, sharedAudioAssets, taskVisualAssets, sharedVisualAssets]); // Expose resolved media assets for e2e validation (dev/test only) diff --git a/task-launcher/src/tasks/shared/helpers/getBucketName.ts b/task-launcher/src/tasks/shared/helpers/getBucketName.ts index 4a7deabe..db453fc0 100644 --- a/task-launcher/src/tasks/shared/helpers/getBucketName.ts +++ b/task-launcher/src/tasks/shared/helpers/getBucketName.ts @@ -10,5 +10,9 @@ export function getBucketName( ? TASK_BUCKET_NAMES_DEV[assetType as keyof typeof TASK_BUCKET_NAMES_DEV] : TASK_BUCKET_NAMES_PROD[assetType as keyof typeof TASK_BUCKET_NAMES_PROD]; - return `${bucket}/${assetType === 'audio' ? language : taskName}`; + if (assetType === 'audio') { + return language ? `${bucket}/${language}` : bucket; + } + + return `${bucket}/${taskName}`; } diff --git a/task-launcher/src/tasks/shared/helpers/getMediaAssets.ts b/task-launcher/src/tasks/shared/helpers/getMediaAssets.ts index 712f60d5..d888075e 100644 --- a/task-launcher/src/tasks/shared/helpers/getMediaAssets.ts +++ b/task-launcher/src/tasks/shared/helpers/getMediaAssets.ts @@ -1,5 +1,3 @@ -//@ts-ignore -import { getDevice } from '@bdelab/roar-utils'; import { camelize } from './camelize'; type CategorizedObjectsType = { @@ -21,64 +19,82 @@ type ResponseDataType = { export async function getMediaAssets( bucketName: string, whitelist: Record = {}, - language: string, taskName: string, + language: string, + requiredAssetNames?: string[], nextPageToken = '', - categorizedObjects: CategorizedObjectsType = { images: {}, audio: {}, video: {} }, + categorizedObjects: CategorizedObjectsType = { images: {}, audio: {}, video: {} } ) { const parts = bucketName.split('/'); const bucket = parts[0]; const folder = parts.slice(1).join('/'); + const prefix = folder ? `${folder}/` : ''; - const baseUrl = `https://storage.googleapis.com/storage/v1/b/${bucket}/o?prefix=${folder}/`; + if (requiredAssetNames) { + requiredAssetNames.forEach((assetName) => { + const path = `https://storage.googleapis.com/${bucket}/${prefix}${assetName}.mp3`; - let url = baseUrl; - if (nextPageToken) { - url += `&pageToken=${nextPageToken}`; - } + categorizedObjects.audio[camelize(assetName)] = path; + }); - const response = await fetch(url); - const data: ResponseDataType = await response.json(); - - data.items.forEach((item) => { - if (isLanguageAndDeviceValid(item.name, taskName, language) && isWhitelisted(item.name, whitelist)) { - const contentType = item.contentType; - const id = item.name; - const path = `https://storage.googleapis.com/${bucket}/${id}`; - const fileName = id.split('/').pop()?.split('.')[0] || ''; - const camelCaseFileName = camelize(fileName); - - if (contentType.startsWith('image/')) { - categorizedObjects.images[camelCaseFileName] = path; - } else if (contentType.startsWith('audio/')) { - categorizedObjects.audio[camelCaseFileName] = path; - } else if (contentType.startsWith('video/')) { - categorizedObjects.video[camelCaseFileName] = path; - } + return categorizedObjects; + } else { + const baseUrl = `https://storage.googleapis.com/storage/v1/b/${bucket}/o`; + const params = new URLSearchParams({ prefix, fields: 'items(name,contentType),nextPageToken' }); + + let url = baseUrl; + if (nextPageToken) { + params.set('pageToken', nextPageToken); + } + if (params.toString()) { + url += `?${params.toString()}`; } - }); - if (data.nextPageToken) { - return getMediaAssets(bucketName, whitelist, taskName, language, data.nextPageToken, categorizedObjects); - } else { - return categorizedObjects; + const response = await fetch(url); + const data: ResponseDataType = await response.json(); + + data.items.forEach((item) => { + if (isLanguageAndTaskValid(item.name, taskName, language) && isWhitelisted(item.name, whitelist)) { + const contentType = item.contentType; + const id = item.name; + const path = `https://storage.googleapis.com/${bucket}/${id}`; + const fileName = id.split('/').pop()?.split('.')[0] || ''; + const camelCaseFileName = camelize(fileName); + + if (contentType.startsWith('image/')) { + categorizedObjects.images[camelCaseFileName] = path; + } else if (contentType.startsWith('audio/')) { + categorizedObjects.audio[camelCaseFileName] = path; + } else if (contentType.startsWith('video/')) { + categorizedObjects.video[camelCaseFileName] = path; + } + } + }); + + if (data.nextPageToken) { + return getMediaAssets(bucketName, whitelist, taskName, language, requiredAssetNames, data.nextPageToken, categorizedObjects); + } else { + return categorizedObjects; + } } } -function isLanguageAndDeviceValid(filePath: string, languageCode: string, taskName: string) { +function isLanguageAndTaskValid(filePath: string, taskName: string, languageCode: string) { const parts = filePath.split('/'); - - if (parts.length !== 3) { + if (parts.length < 3) { return false; - } else if (parts[0] === 'visual') { - // visual assets have task prefix + } + + const assetType = parts[0]; + if (assetType === 'audio') { + return parts[1] === languageCode && parts[2].length !== 0; + } + + if (assetType === 'visual') { return parts[1] === taskName && parts[2].length !== 0; - } else if (parts[0] === 'audio') { - // audio assets have language prefix - return parts[1] == languageCode && parts[2].length !== 0; } - return false; // Not a valid path + return false; } // TODO: allow nested whitelisting (whitelisting within an already whitelisted folder)