From 207a00311e4b8def25026140fc7b9a1bd92e69a1 Mon Sep 17 00:00:00 2001 From: Sebastien DUMETZ Date: Wed, 21 Jan 2026 15:53:05 +0100 Subject: [PATCH 01/14] VFS improvements: additional directories, copyFile method, and test refactoring --- source/server/vfs/Base.ts | 45 +- source/server/vfs/Files.ts | 45 +- source/server/vfs/index.ts | 14 +- source/server/vfs/vfs.test.ts | 3570 +++++++++++++++++---------------- 4 files changed, 1888 insertions(+), 1786 deletions(-) diff --git a/source/server/vfs/Base.ts b/source/server/vfs/Base.ts index 3b444fd51..7525396bd 100644 --- a/source/server/vfs/Base.ts +++ b/source/server/vfs/Base.ts @@ -1,8 +1,9 @@ -import open, {Database, DbController} from "./helpers/db.js"; +import {Database, DbController} from "./helpers/db.js"; import path from "path"; -import { InternalError, NotFoundError } from "../utils/errors.js"; +import { NotFoundError } from "../utils/errors.js"; import { FileProps } from "./types.js"; +import { mkdir } from "fs/promises"; export type Isolate = (this: that, vfs :that)=> Promise; @@ -13,8 +14,23 @@ export default abstract class BaseVfs extends DbController{ super(db); } + public get baseDir(){ return this.rootDir; } + /** + * Temporary directory to store in-transit files. + * should be in the same volume as `Vfs.objectsDir` and `Vfs.artifactsDir` + * to ensure files can be moved atomically between those folders. + */ public get uploadsDir(){ return path.join(this.rootDir, "uploads"); } + /** + * Main objects directory + */ public get objectsDir(){ return path.join(this.rootDir, "objects"); } + /** + * Secondary directory used to store task artifacts + * Those are not considered long-term persistent: They can be cleaned-up to save space. + * On the other hand, they _shouldn't_ be cleaned before their referencing task + */ + public get artifactsDir(){ return path.join(this.rootDir, "artifacts"); } public filepath(f :FileProps|string|{hash:string}){ if(typeof f ==="string"){ @@ -26,6 +42,31 @@ export default abstract class BaseVfs extends DbController{ } } + + /** + * Compute path to a file relative to {@link baseDir }. Accepts either absolute paths or paths that are already relative to baseDir. + * This is not a sanitize function and won't prevent bad paths from breaking out of baseDir. + * @param filepath + */ + public relative(filepath: string){ + return path.relative(this.baseDir, path.resolve(this.baseDir, filepath)); + } + + + + /** + * Create an artifact directory for this task + */ + public async createTaskWorkspace(id: number){ + const dir = this.getTaskWorkspace(id); + await mkdir( dir, {recursive: true}); + return dir; + } + + public getTaskWorkspace(id: number){ + return path.join(this.artifactsDir, id.toString(10)); + } + abstract close() :Promise; public abstract isOpen :boolean; } \ No newline at end of file diff --git a/source/server/vfs/Files.ts b/source/server/vfs/Files.ts index c2d94c28d..67eda93a2 100644 --- a/source/server/vfs/Files.ts +++ b/source/server/vfs/Files.ts @@ -8,7 +8,7 @@ import { CommonFileParams, DataStream, DocProps, FileProps, GetFileParams, GetFi import { Transaction } from "./helpers/db.js"; import { FileHandle } from "fs/promises"; -import { Duplex, Readable, Transform } from "stream"; +import { Duplex, Readable, Transform, Writable } from "stream"; import { pipeline } from "stream/promises"; import { transform } from "typescript"; @@ -85,6 +85,49 @@ export default abstract class FilesVfs extends BaseVfs{ } } + /** + * Faster alternative to {@link writeFile} that only computes the file's hash in-place then hard-links it to its destination + * + * The source file can then safely be unlinked + */ + async copyFile( + filepath :string, + params :WriteFileParams + ) :Promise{ + let handle = await fs.open(filepath, constants.O_RDONLY); + let hashsum = createHash("sha256"); + let size = 0; + try{ + let rs = handle.createReadStream(); + await pipeline( + rs, + new Writable({ + write(chunk, encoding, callback){ + hashsum.update(chunk); + size += chunk.length; + callback(null); + } + }), + ); + let hash = hashsum.digest("base64url"); + let destfile = path.join(this.objectsDir, hash); + + return await this.createFile(params, async ({id})=>{ + try{ + // It's always possible for the transaction to fail afterwards, creating a loose object + // However it's not possible to safely clean it up without race conditions over any other row that may be referencing it + await fs.link(filepath, destfile); + }catch(e){ + if((e as any).code != "EEXIST") throw e; + //If a file with the same hash exists, we presume it's the same file and don't overwrite it. + } + return {hash, size}; + }); + }finally{ + await handle.close(); + } + } + /** * Write a document for a scene diff --git a/source/server/vfs/index.ts b/source/server/vfs/index.ts index 49793b54b..148954e09 100644 --- a/source/server/vfs/index.ts +++ b/source/server/vfs/index.ts @@ -34,17 +34,21 @@ class Vfs extends BaseVfs{ static async Open(rootDir :string, opts:VfsOptions&Required> ):Promise static async Open(rootDir :string, {db, database_uri, createDirs=true, forceMigration = true} :VfsOptions = {} ){ if(!db && !database_uri) throw new Error(`No DB connection method provided. Can't open VFS`); - if(createDirs){ - await fs.mkdir(path.join(rootDir, "objects"), {recursive: true}); - await fs.rm(path.join(rootDir, "uploads"), {recursive: true, force: true}); - await fs.mkdir(path.join(rootDir, "uploads"), {recursive: true}); - } + db ??= await open({ uri: database_uri!, forceMigration, }); let vfs = new Vfs(rootDir, db); + + if(createDirs){ + await Promise.all([ + fs.mkdir(vfs.objectsDir, {recursive: true}), + fs.mkdir(vfs.uploadsDir, {recursive: true}), + fs.mkdir(vfs.artifactsDir, {recursive: true}), + ]); + } return vfs; } diff --git a/source/server/vfs/vfs.test.ts b/source/server/vfs/vfs.test.ts index c2dc68cbc..8a0cbb89d 100755 --- a/source/server/vfs/vfs.test.ts +++ b/source/server/vfs/vfs.test.ts @@ -42,2073 +42,2087 @@ function sceneProps(id:number): {[P in keyof Required]: Function|any}{ } describe("Vfs", function(){ - this.beforeEach(async function(){ - this.dir = await fs.mkdtemp(path.join(tmpdir(), `vfs_tests`)); - this.uploads = path.join(this.dir, "uploads"); //For quick reference - }); - this.afterEach(async function(){ - await fs.rm(this.dir, {recursive: true}); - }) - it("creates upload directory", async function(){ - let vfs = await Vfs.Open(this.dir, {db: {} as any}); - await expect(fs.access(path.join(this.dir, "uploads"))).to.be.fulfilled; - }); - - describe("isolate", function(){ - let vfs :Vfs; - this.beforeEach(async function(){ - this.db_uri = await getUniqueDb(this.test?.title); - vfs = await Vfs.Open(this.dir, {database_uri: this.db_uri}); - }) - this.afterEach(async function(){ - await vfs.close(); - await dropDb(this.db_uri); + describe("relative()", function(){ + let vfs: Vfs; + this.beforeEach(function(){ + vfs = new Vfs("/path/to/data", {} as any); }) - it("can rollback on error", async function(){ - await expect(vfs.isolate(async (vfs)=>{ - await vfs.createScene("foo"); - await vfs.createScene("foo"); - })).to.be.rejected; - expect(await vfs.getScenes()).to.have.property("length", 0); - }); - - it("reuses a connection when nested", async function(){ - await expect(vfs.isolate( async (v2)=>{ - await v2.isolate(async (v3)=>{ - expect(v3._db).to.equal(v2._db); - }); - })).to.be.fulfilled; + it("computes path relative to baseDir", function(){ + expect(vfs.relative("/path/to/data/test.bin")).to.equal("test.bin"); }); - - it("can be nested (success)", async function(){ - let scenes = await expect(vfs.isolate( async (v2)=>{ - await v2.getScenes(); - await v2.isolate(async (v3)=>{ - await v3.getScenes(); - await v3.createScene("foo"); - }); - await v2.getScenes(); - await v2.createScene("bar") - return await v2.getScenes(); - })).to.be.fulfilled; - expect(scenes).to.have.property("length", 2); - await expect(vfs.getScenes()).to.eventually.deep.equal(scenes); + it("works with paths that are already relative", function(){ + expect(vfs.relative("test.bin")).to.equal("test.bin"); }); - - it("can be nested (with caught error)", async function(){ - let scenes = await expect(vfs.isolate( async (v2)=>{ - await v2.createScene("foo"); - //This isolate rolls back but since we don't propagate the error - //the parent will succeed - await v2.isolate(async (v3)=>{ - await v3.createScene("bar"); - //Force this transaction to roll back - throw new Error("TEST"); - }).catch(e=>{ - if(e.message !== "TEST") throw e; - }); - return await v2.getScenes(); - })).to.be.fulfilled; - expect(scenes).to.have.property("length", 1); - expect(scenes[0]).to.have.property("name", "foo"); - expect(await vfs.getScenes()).to.deep.equal(scenes); + }) + describe("", function(){ + this.beforeEach(async function(){ + this.dir = await fs.mkdtemp(path.join(tmpdir(), `vfs_tests`)); + this.uploads = path.join(this.dir, "uploads"); //For quick reference }); - - it("is properly closed on success", async function(){ - let _transaction:Vfs|null =null; - await expect(vfs.isolate(async tr=>{ - _transaction = tr; - expect(_transaction).to.have.property("isOpen", true); - })).to.be.fulfilled; - expect(_transaction).to.have.property("isOpen", false); - }) - - it("is properly closed on error", async function(){ - let _transaction:Vfs|null =null; - await expect(vfs.isolate(async tr=>{ - _transaction = tr; - expect(_transaction).to.have.property("isOpen", true); - throw new Error("dummy"); - })).to.be.rejectedWith("dummy"); - expect(_transaction).to.have.property("isOpen", false); + this.afterEach(async function(){ + await fs.rm(this.dir, {recursive: true}); }) - }); - - describe("validate search params", function(){ - it("accepts no parameters", function(){ - expect(()=>ScenesVfs._validateSceneQuery({})).not.to.throw(); + it("creates upload directory", async function(){ + let vfs = await Vfs.Open(this.dir, {db: {} as any}); + await expect(fs.access(path.join(this.dir, "uploads"))).to.be.fulfilled; }); - it("requires limit to be a positive integer", function(){ - [null, "foo", 0.5, "0", 0, -1, 101].forEach((limit)=>{ - expect(()=>ScenesVfs._validateSceneQuery({limit} as any), `{limit: ${limit}}`).to.throw(); + describe("isolate", function(){ + let vfs :Vfs; + this.beforeEach(async function(){ + this.db_uri = await getUniqueDb(this.test?.title); + vfs = await Vfs.Open(this.dir, {database_uri: this.db_uri}); + }) + this.afterEach(async function(){ + await vfs.close(); + await dropDb(this.db_uri); + }) + it("can rollback on error", async function(){ + await expect(vfs.isolate(async (vfs)=>{ + await vfs.createScene("foo"); + await vfs.createScene("foo"); + })).to.be.rejected; + expect(await vfs.getScenes()).to.have.property("length", 0); }); - [1, 10, 100].forEach((limit)=>{ - expect(()=>ScenesVfs._validateSceneQuery({limit} as any)).not.to.throw(); + it("reuses a connection when nested", async function(){ + await expect(vfs.isolate( async (v2)=>{ + await v2.isolate(async (v3)=>{ + expect(v3._db).to.equal(v2._db); + }); + })).to.be.fulfilled; }); - }); - it("requires offset to be a positive integer", function(){ - [null, "foo", 0.5, "0", -1].forEach((offset)=>{ - expect(()=>ScenesVfs._validateSceneQuery({offset} as any), `{offset: ${offset}}`).to.throw(); + it("can be nested (success)", async function(){ + let scenes = await expect(vfs.isolate( async (v2)=>{ + await v2.getScenes(); + await v2.isolate(async (v3)=>{ + await v3.getScenes(); + await v3.createScene("foo"); + }); + await v2.getScenes(); + await v2.createScene("bar") + return await v2.getScenes(); + })).to.be.fulfilled; + expect(scenes).to.have.property("length", 2); + await expect(vfs.getScenes()).to.eventually.deep.equal(scenes); }); - [0, 1, 10, 100, 1000].forEach((offset)=>{ - expect(()=>ScenesVfs._validateSceneQuery({offset} as any)).not.to.throw(); + it("can be nested (with caught error)", async function(){ + let scenes = await expect(vfs.isolate( async (v2)=>{ + await v2.createScene("foo"); + //This isolate rolls back but since we don't propagate the error + //the parent will succeed + await v2.isolate(async (v3)=>{ + await v3.createScene("bar"); + //Force this transaction to roll back + throw new Error("TEST"); + }).catch(e=>{ + if(e.message !== "TEST") throw e; + }); + return await v2.getScenes(); + })).to.be.fulfilled; + expect(scenes).to.have.property("length", 1); + expect(scenes[0]).to.have.property("name", "foo"); + expect(await vfs.getScenes()).to.deep.equal(scenes); }); - }); - it("requires orderDirection to match", function(){ - ["AS", "DE", null, 0, -1, 1, "1"].forEach((orderDirection)=>{ - expect(()=>ScenesVfs._validateSceneQuery({orderDirection} as any), `{orderDirection: ${orderDirection}}`).to.throw("Invalid orderDirection"); - }); - ["ASC", "DESC", "asc", "desc"].forEach((orderDirection)=>{ - expect(()=>ScenesVfs._validateSceneQuery({orderDirection} as any)).not.to.throw(); + it("is properly closed on success", async function(){ + let _transaction:Vfs|null =null; + await expect(vfs.isolate(async tr=>{ + _transaction = tr; + expect(_transaction).to.have.property("isOpen", true); + })).to.be.fulfilled; + expect(_transaction).to.have.property("isOpen", false); }) - }); - - it("requires orderBy to match", function(){ - ["foo", 1, -1, null].forEach((orderBy)=>{ - expect(()=>ScenesVfs._validateSceneQuery({orderBy} as any), `{orderBy: ${orderBy}}`).to.throw(`Invalid orderBy`); - }); - ["ctime", "mtime", "name"].forEach((orderBy)=>{ - expect(()=>ScenesVfs._validateSceneQuery({orderBy} as any), `{orderBy: "${orderBy}"}`).not.to.throw(); - }); + it("is properly closed on error", async function(){ + let _transaction:Vfs|null =null; + await expect(vfs.isolate(async tr=>{ + _transaction = tr; + expect(_transaction).to.have.property("isOpen", true); + throw new Error("dummy"); + })).to.be.rejectedWith("dummy"); + expect(_transaction).to.have.property("isOpen", false); + }) }); - it("sanitizes access values", function(){ - ["read","write","admin","none"] - .forEach( a => { - expect(()=>ScenesVfs._validateSceneQuery({access: a as any}),`expected ${a ?? typeof a} to be an accepted access value`).not.to.throw(); + describe("validate search params", function(){ + it("accepts no parameters", function(){ + expect(()=>ScenesVfs._validateSceneQuery({})).not.to.throw(); }); - ["foo",true, 1] - .forEach ( a=> { - expect(()=>ScenesVfs._validateSceneQuery({access: a as any}),`expected ${a ?? typeof a} to not be an accepted access value`).to.throw(`Invalid access type requested : ${a.toString()}`); - }); - }); + it("requires limit to be a positive integer", function(){ + [null, "foo", 0.5, "0", 0, -1, 101].forEach((limit)=>{ + expect(()=>ScenesVfs._validateSceneQuery({limit} as any), `{limit: ${limit}}`).to.throw(); + }); - it("sanitizes authors username", function(){ - [ "Jane" ].forEach(a=>{ - expect(()=>ScenesVfs._validateSceneQuery({author: a})).not.to.throw(); - }); - [ null, 0].forEach(a=>{ - expect(()=>ScenesVfs._validateSceneQuery({author: a as any}),`expected ${a ?? typeof a} to not be an accepted author value`).to.throw(`[400] Invalid author filter request: ${a}`); + [1, 10, 100].forEach((limit)=>{ + expect(()=>ScenesVfs._validateSceneQuery({limit} as any)).not.to.throw(); + }); }); - }) - }); - - describe("", function(){ - let vfs :Vfs, database_uri:string; - //@ts-ignore - const run = async (sql: ISqlite.SqlType, ...params: any[])=> await vfs.db.run(sql, ...params); - //@ts-ignore - const get = async (sql: ISqlite.SqlType, ...params: any[])=> await vfs.db.get(sql, ...params); - //@ts-ignore - const all = async (sql: ISqlite.SqlType, ...params: any[])=> await vfs.db.all(sql, ...params); - this.beforeEach(async function(){ - database_uri = await getUniqueDb(); - vfs = await Vfs.Open(this.dir, {database_uri}); - }); - this.afterEach(async function(){ - await vfs.close(); - await dropDb(database_uri); - }) + it("requires offset to be a positive integer", function(){ + [null, "foo", 0.5, "0", -1].forEach((offset)=>{ + expect(()=>ScenesVfs._validateSceneQuery({offset} as any), `{offset: ${offset}}`).to.throw(); + }); - describe("createScene()", function(){ - it("insert a new scene", async function(){ - await expect(vfs.createScene("foo")).to.be.fulfilled; - }) - it("throws on duplicate name", async function(){ - await expect(vfs.createScene("foo")).to.be.fulfilled; - await expect(vfs.createScene("foo")).to.be.rejectedWith("exist"); + [0, 1, 10, 100, 1000].forEach((offset)=>{ + expect(()=>ScenesVfs._validateSceneQuery({offset} as any)).not.to.throw(); + }); }); - describe("uid handling", function(){ - let old :typeof Uid.make; - let returns :number[] = []; - this.beforeEach(function(){ - old = Uid.make; - returns = []; - Uid.make = ()=> { - let r = returns.pop(); - if (typeof r === "undefined") throw new Error("No mock result provided"); - return r; - }; - }); - this.afterEach(function(){ - Uid.make = old; + it("requires orderDirection to match", function(){ + ["AS", "DE", null, 0, -1, 1, "1"].forEach((orderDirection)=>{ + expect(()=>ScenesVfs._validateSceneQuery({orderDirection} as any), `{orderDirection: ${orderDirection}}`).to.throw("Invalid orderDirection"); }); + ["ASC", "DESC", "asc", "desc"].forEach((orderDirection)=>{ + expect(()=>ScenesVfs._validateSceneQuery({orderDirection} as any)).not.to.throw(); + }) + }); - it("fails if no free uid can be found", async function(){ - returns = [1, 1, 1, 1]; - await expect(vfs.createScene("bar")).to.be.fulfilled; - await expect(vfs.createScene("baz")).to.be.rejectedWith("Unable to find a free id"); + it("requires orderBy to match", function(){ + ["foo", 1, -1, null].forEach((orderBy)=>{ + expect(()=>ScenesVfs._validateSceneQuery({orderBy} as any), `{orderBy: ${orderBy}}`).to.throw(`Invalid orderBy`); }); - it("retry", async function(){ - returns = [1, 1, 2]; - await expect(vfs.createScene("bar")).to.be.fulfilled; - await expect(vfs.createScene("baz")).to.be.fulfilled; + ["ctime", "mtime", "name"].forEach((orderBy)=>{ + expect(()=>ScenesVfs._validateSceneQuery({orderBy} as any), `{orderBy: "${orderBy}"}`).not.to.throw(); }); + }); - it("prevents scene name containing uid", async function(){ - returns = [1, 2]; - let scene_id = await expect(vfs.createScene("bar#1")).to.be.fulfilled; - expect(scene_id).to.equal(2); + it("sanitizes access values", function(){ + ["read","write","admin","none"] + .forEach( a => { + expect(()=>ScenesVfs._validateSceneQuery({access: a as any}),`expected ${a ?? typeof a} to be an accepted access value`).not.to.throw(); }); - }) - - it("sets scene author", async function(){ - const userManager = new UserManager(vfs._db); - const user = await userManager.addUser("alice", "xxxxxxxx", "create"); - let id = await expect(vfs.createScene("foo", user.uid)).to.be.fulfilled; - try{ - let s = await vfs.getScene(id, user.uid); - expect(s).to.have.property("access").to.be.equal("admin"); - }catch(e){ - console.log("createScene :", e); - throw e; - } - }); - it("sets custom scene permissions", async function(){ - const userManager = new UserManager(vfs._db); - const user = await userManager.addUser("alice", "xxxxxxxx", "create"); - let id = await expect(vfs.createScene("foo", user.uid)).to.be.fulfilled; - await userManager.grant(id, user.uid, "write"); - await userManager.setPublicAccess(id, "none"); - let s = await vfs.getScene(id, user.uid); - expect(s.access).to.deep.equal("write"); + ["foo",true, 1] + .forEach ( a=> { + expect(()=>ScenesVfs._validateSceneQuery({access: a as any}),`expected ${a ?? typeof a} to not be an accepted access value`).to.throw(`Invalid access type requested : ${a.toString()}`); + }); }); - }); - describe("getScenes()", function(){ - it("get an empty list", async function(){ - let scenes = await vfs.getScenes(); - expect(scenes).to.have.property("length", 0); + it("sanitizes authors username", function(){ + [ "Jane" ].forEach(a=>{ + expect(()=>ScenesVfs._validateSceneQuery({author: a})).not.to.throw(); + }); + [ null, 0].forEach(a=>{ + expect(()=>ScenesVfs._validateSceneQuery({author: a as any}),`expected ${a ?? typeof a} to not be an accepted author value`).to.throw(`[400] Invalid author filter request: ${a}`); + }); }) + }); - it("get a list of scenes", async function(){ - let scene_id = await vfs.createScene("foo"); - let scenes = await vfs.getScenes(); - expect(scenes).to.have.property("length", 1); - let scene = scenes[0]; - - let props = sceneProps(scene_id); - let key:keyof Scene; - for(key in props){ - if(typeof props[key] ==="undefined"){ - expect(scene, `${(scene as any)[key]}`).not.to.have.property(key); - }else if(typeof props[key] === "function"){ - expect(scene, `scene.${key} should match expected class ${props[key].constructor.name}`).to.have.property(key).instanceof(props[key]); - }else{ - expect(scene, `scene.${key} should match expected value ${props[key]}`).to.have.property(key).to.deep.equal(props[key]); - } - } - }); - - it("get proper ctime and mtime from last document edit", async function(){ - let t2 = new Date(); - let t1 = new Date(Date.now()-100000); - let scene_id = await vfs.createScene("foo"); - await vfs.writeDoc("{}", {scene: scene_id, user_id: null, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); - let $doc_id = (await vfs.writeDoc("{}", {scene: scene_id, user_id: null, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"})).id; - //Force ctime - await run(`UPDATE scenes SET ctime = $1`, [t1]); - await run(`UPDATE files SET ctime = $1 WHERE file_id = $2`, [t2, $doc_id]); - let scenes = await vfs.getScenes(); - expect(scenes).to.have.property("length", 1); - expect(scenes[0].ctime.valueOf(), `ctime is ${scenes[0].ctime}, expected ${t1}`).to.equal(t1.valueOf()); - expect(scenes[0].mtime.valueOf(), `mtime is ${scenes[0].mtime}, expected ${t2}`).to.equal(t2.valueOf()); - }); + describe("", function(){ + let vfs :Vfs, database_uri:string; + //@ts-ignore + const run = async (sql: ISqlite.SqlType, ...params: any[])=> await vfs.db.run(sql, ...params); + //@ts-ignore + const get = async (sql: ISqlite.SqlType, ...params: any[])=> await vfs.db.get(sql, ...params); + //@ts-ignore + const all = async (sql: ISqlite.SqlType, ...params: any[])=> await vfs.db.all(sql, ...params); - it("orders by names, case-insensitive and ascending", async function(){ - await Promise.all([ - vfs.createScene("a1"), - vfs.createScene("aa"), - vfs.createScene("Ab"), - ]); - let scenes = await vfs.getScenes(null, {orderBy: "name"}); - let names = scenes.map(s=>s.name); - expect(names).to.deep.equal(["a1", "aa", "Ab"]); + this.beforeEach(async function(){ + database_uri = await getUniqueDb(); + vfs = await Vfs.Open(this.dir, {database_uri}); }); + this.afterEach(async function(){ + await vfs.close(); + await dropDb(database_uri); + }) - it("can return existing thumbnails", async function(){ - let s1 = await vfs.createScene("01"); - await vfs.writeDoc("{}", {scene: s1, user_id: null, name: "scene-image-thumb.jpg", mime: "image/jpeg"}); - let s2 = await vfs.createScene("02"); - await vfs.writeDoc("{}", {scene: s2, user_id: null, name: "scene-image-thumb.png", mime: "image/jpeg"}); - - let s = await vfs.getScenes(0); - expect(s).to.have.property("length", 2); - expect(s[0]).to.have.property("thumb", "scene-image-thumb.png"); - expect(s[1]).to.have.property("thumb", "scene-image-thumb.jpg"); - }); + describe("createScene()", function(){ + it("insert a new scene", async function(){ + await expect(vfs.createScene("foo")).to.be.fulfilled; + }) + it("throws on duplicate name", async function(){ + await expect(vfs.createScene("foo")).to.be.fulfilled; + await expect(vfs.createScene("foo")).to.be.rejectedWith("exist"); + }); + + describe("uid handling", function(){ + let old :typeof Uid.make; + let returns :number[] = []; + this.beforeEach(function(){ + old = Uid.make; + returns = []; + Uid.make = ()=> { + let r = returns.pop(); + if (typeof r === "undefined") throw new Error("No mock result provided"); + return r; + }; + }); + this.afterEach(function(){ + Uid.make = old; + }); - it("returns the last-saved thumbnail", async function(){ - let s1 = await vfs.createScene("01"); - let times = [ - new Date("2022-01-01"), - new Date("2023-01-01"), - new Date("2024-01-01") - ]; - const setDate = (i:number, d:Date)=>vfs._db.run(`UPDATE files SET ctime = $2 WHERE file_id = $1`, [i, d]); - let png = await vfs.writeDoc("{}", {scene: s1, user_id: null, name: "scene-image-thumb.png", mime: "image/png"}); - let jpg = await vfs.writeDoc("{}", {scene: s1, user_id: null, name: "scene-image-thumb.jpg", mime: "image/jpeg"}); - - let r = await setDate(jpg.id, times[1]); - await setDate(png.id, times[2]); - let s = await vfs.getScenes(0); - expect(s).to.have.length(1); - expect(s[0], `use PNG thumbnail if it's the most recent`).to.have.property("thumb", "scene-image-thumb.png"); - - await setDate(png.id, times[0]); - s = await vfs.getScenes(0); - expect(s[0], `use JPG thumbnail if it's the most recent`).to.have.property("thumb", "scene-image-thumb.jpg"); - - //If date is equal, prioritize jpg - await setDate(png.id, times[1]); - s = await vfs.getScenes(0); - expect(s[0], `With equal dates, alphanumeric order shopuld prioritize JPG over PNG file`).to.have.property("thumb", "scene-image-thumb.jpg"); - }); + it("fails if no free uid can be found", async function(){ + returns = [1, 1, 1, 1]; + await expect(vfs.createScene("bar")).to.be.fulfilled; + await expect(vfs.createScene("baz")).to.be.rejectedWith("Unable to find a free id"); + }); - it("can get archived scenes", async function(){ - let scene_id = await vfs.createScene("foo"); - await vfs.writeDoc(JSON.stringify({foo: "bar"}), {scene: scene_id, user_id: null, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); - await vfs.archiveScene(scene_id); - let scenes = await vfs.getScenes(); - expect(scenes.map(({name})=>({name}))).to.deep.equal([{name: `foo#${scene_id}`}]); - }); - - - it("Can't get archived scenes without being authenticated", async function(){ - await vfs.createScene("bar"); - let scene_id = await vfs.createScene("foo"); - await vfs.writeDoc(JSON.stringify({foo: "bar"}), {scene: scene_id, user_id: null, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); - await vfs.archiveScene(scene_id); - - //Two scenes total - expect(await vfs.getScenes()).to.have.length(2); - //Filter only scenes with access: none - await expect(vfs.getScenes(null, {archived: true})).to.be.rejectedWith(UnauthorizedError); - }); + it("retry", async function(){ + returns = [1, 1, 2]; + await expect(vfs.createScene("bar")).to.be.fulfilled; + await expect(vfs.createScene("baz")).to.be.fulfilled; + }); - it("can get an author's own archived scenes", async function(){ - let um :UserManager= new UserManager(vfs._db); - let user = await um.addUser("bob", "12345678", "create", "bob@example.com") - //Create a reference non-archived scene (shouldn't be shown) - await vfs.createScene("bar", user.uid); - //Create a scene owned by someone else - await vfs.createScene("baz"); - //Create our archived scene - let scene_id = await vfs.createScene("foo", user.uid); - await vfs.writeDoc(JSON.stringify({foo: "bar"}), {scene: scene_id, user_id: null, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); - await vfs.archiveScene(scene_id); - - //Three scenes total - expect(await vfs.getScenes()).to.have.length(3); - //Two "existing" scenes - expect(await vfs.getScenes(user.uid, {archived: false})).to.have.length(2); - - //Filter only scenes with access: none - let scenes = await vfs.getScenes(user.uid, {archived: true}); - expect(scenes.map(({name})=>({name}))).to.deep.equal([{name: `foo#${scene_id}`}]); - }) + it("prevents scene name containing uid", async function(){ + returns = [1, 2]; + let scene_id = await expect(vfs.createScene("bar#1")).to.be.fulfilled; + expect(scene_id).to.equal(2); + }); + }) - describe("with permissions", function(){ - let userManager :UserManager, user :User; - this.beforeEach(async function(){ - userManager = new UserManager(vfs._db); - user = await userManager.addUser("alice", "xxxxxxxx", "create"); + it("sets scene author", async function(){ + const userManager = new UserManager(vfs._db); + const user = await userManager.addUser("alice", "xxxxxxxx", "create"); + let id = await expect(vfs.createScene("foo", user.uid)).to.be.fulfilled; + try{ + let s = await vfs.getScene(id, user.uid); + expect(s).to.have.property("access").to.be.equal("admin"); + }catch(e){ + console.log("createScene :", e); + throw e; + } }); - it("can filter accessible scenes by user_id", async function(){ - let scene_id = await vfs.createScene("foo", user.uid); - await userManager.setPublicAccess("foo", "none"); - await userManager.setDefaultAccess("foo", "none"); - await userManager.grant("foo", user.uid, "none"); - await run(`UPDATE scenes SET public_access = 0`); - await run(`INSERT INTO users_acl (fk_scene_id, fk_user_id, access_level) VALUES ($1, $2, 3)`, [scene_id, user.uid]); - expect((await vfs.getScenes(0)), `private scene shouldn't be returned to default user`).to.have.property("length", 0); - expect(await vfs.getScenes(user.uid), `private scene should be returned to its author`).to.have.property("length", 1); + it("sets custom scene permissions", async function(){ + const userManager = new UserManager(vfs._db); + const user = await userManager.addUser("alice", "xxxxxxxx", "create"); + let id = await expect(vfs.createScene("foo", user.uid)).to.be.fulfilled; + await userManager.grant(id, user.uid, "write"); + await userManager.setPublicAccess(id, "none"); + let s = await vfs.getScene(id, user.uid); + expect(s.access).to.deep.equal("write"); }); + }); - it("get proper author id and name", async function(){ - let scene_id = await vfs.createScene("foo", user.uid); - await vfs.writeDoc("{}", {scene: scene_id, user_id: user.uid, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); + describe("getScenes()", function(){ + it("get an empty list", async function(){ let scenes = await vfs.getScenes(); - expect(scenes).to.have.property("length", 1); - expect(scenes[0]).to.have.property("author", user.username); - expect(scenes[0]).to.have.property("author_id", user.uid); - }); - - it("get proper user own access", async function(){ - let scene_id = await vfs.createScene("foo", user.uid); - await userManager.setDefaultAccess(scene_id, "write"); - await userManager.setPublicAccess(scene_id, "none"); - let scenes = await vfs.getScenes(user.uid); - expect(scenes).to.have.property("length", 1); - expect(scenes[0]).to.have.property("access").to.equal("admin"); - }); - it("get proper \"any\" access", async function(){ + expect(scenes).to.have.property("length", 0); + }) + + it("get a list of scenes", async function(){ let scene_id = await vfs.createScene("foo"); - await userManager.setDefaultAccess(scene_id, "write"); - await userManager.setPublicAccess(scene_id, "read"); - let scenes = await vfs.getScenes(user.uid); + let scenes = await vfs.getScenes(); expect(scenes).to.have.property("length", 1); - expect(scenes[0]).to.have.property("access").to.equal("write"); + let scene = scenes[0]; + + let props = sceneProps(scene_id); + let key:keyof Scene; + for(key in props){ + if(typeof props[key] ==="undefined"){ + expect(scene, `${(scene as any)[key]}`).not.to.have.property(key); + }else if(typeof props[key] === "function"){ + expect(scene, `scene.${key} should match expected class ${props[key].constructor.name}`).to.have.property(key).instanceof(props[key]); + }else{ + expect(scene, `scene.${key} should match expected value ${props[key]}`).to.have.property(key).to.deep.equal(props[key]); + } + } }); - - it("get proper group access", async function(){ + + it("get proper ctime and mtime from last document edit", async function(){ + let t2 = new Date(); + let t1 = new Date(Date.now()-100000); let scene_id = await vfs.createScene("foo"); - let group = await userManager.addGroup("My group"); - await userManager.addMemberToGroup(user.uid, group.groupUid); - await userManager.setDefaultAccess(scene_id, "read"); - await userManager.setPublicAccess(scene_id, "read"); - await userManager.grantGroup(scene_id, group.groupUid, "write") - let scenes = await vfs.getScenes(user.uid); - expect(scenes).to.have.property("length", 1); - expect(scenes[0]).to.have.property("access").to.equal("write"); - }); - - it("Do not show non-public scene when there is no requester", async function(){ - let scene_id = await vfs.createScene("foo", user.uid); - await userManager.setDefaultAccess(scene_id, "write"); - await userManager.setPublicAccess(scene_id, "none"); + await vfs.writeDoc("{}", {scene: scene_id, user_id: null, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); + let $doc_id = (await vfs.writeDoc("{}", {scene: scene_id, user_id: null, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"})).id; + //Force ctime + await run(`UPDATE scenes SET ctime = $1`, [t1]); + await run(`UPDATE files SET ctime = $1 WHERE file_id = $2`, [t2, $doc_id]); let scenes = await vfs.getScenes(); - expect(scenes).to.have.property("length", 0); - }); - }); - - describe("search", async function(){ - let userManager :UserManager, user :User, sceneAdmin :User, admin :User; - this.beforeEach(async function(){ - userManager = new UserManager(vfs._db); - user = await userManager.addUser("bob", "xxxxxxxx", "create"); - sceneAdmin = await userManager.addUser("alice", "xxxxxxxx", "create"); - admin = await userManager.addUser("adele", "xxxxxxxx", "admin"); + expect(scenes).to.have.property("length", 1); + expect(scenes[0].ctime.valueOf(), `ctime is ${scenes[0].ctime}, expected ${t1}`).to.equal(t1.valueOf()); + expect(scenes[0].mtime.valueOf(), `mtime is ${scenes[0].mtime}, expected ${t2}`).to.equal(t2.valueOf()); }); - it("filters by access-level", async function(){ - await vfs.createScene("foo", sceneAdmin.uid); - await userManager.grant("foo", user.uid, "read"); - expect(await vfs.getScenes(user.uid, {})).to.have.property("length", 1); - expect(await vfs.getScenes(user.uid, {access:"admin"})).to.have.property("length", 0); + it("orders by names, case-insensitive and ascending", async function(){ + await Promise.all([ + vfs.createScene("a1"), + vfs.createScene("aa"), + vfs.createScene("Ab"), + ]); + let scenes = await vfs.getScenes(null, {orderBy: "name"}); + let names = scenes.map(s=>s.name); + expect(names).to.deep.equal(["a1", "aa", "Ab"]); }); - it("won't return inaccessible content", async function(){ - await vfs.createScene("foo", sceneAdmin.uid); - await userManager.setPublicAccess("foo", "none"); - await userManager.setDefaultAccess("foo", "none"); - expect(await vfs.getScenes(user.uid, {access:"none"})).to.have.property("length", 0); - }); + it("can return existing thumbnails", async function(){ + let s1 = await vfs.createScene("01"); + await vfs.writeDoc("{}", {scene: s1, user_id: null, name: "scene-image-thumb.jpg", mime: "image/jpeg"}); + let s2 = await vfs.createScene("02"); + await vfs.writeDoc("{}", {scene: s2, user_id: null, name: "scene-image-thumb.png", mime: "image/jpeg"}); - it("will return everything to admin level user", async function(){ - await vfs.createScene("foo", sceneAdmin.uid); - await userManager.setPublicAccess("foo", "none"); - await userManager.setDefaultAccess("foo", "none"); - expect(await vfs.getScenes(admin.uid)).to.have.property("length", 1); - }); - - it("will return only scenes with specicfic rights to admin level user", async function(){ - await vfs.createScene("foo", sceneAdmin.uid); - await userManager.setPublicAccess("foo", "none"); - await userManager.setDefaultAccess("foo", "none"); - expect(await vfs.getScenes(admin.uid, {access:"read"})).to.have.property("length", 0); - expect(await vfs.getScenes(admin.uid, {access:"write"})).to.have.property("length", 0); - expect(await vfs.getScenes(admin.uid, {access:"admin"})).to.have.property("length", 0); - }); - - it("can select by specific user access level", async function(){ - await vfs.createScene("foo", sceneAdmin.uid); - await userManager.grant("foo", user.uid, "read"); - await userManager.setPublicAccess("foo", "read"); - await userManager.setDefaultAccess("foo", "read"); - expect(await vfs.getScenes(user.uid, {access:"read"})).to.have.property("length", 1); + let s = await vfs.getScenes(0); + expect(s).to.have.property("length", 2); + expect(s[0]).to.have.property("thumb", "scene-image-thumb.png"); + expect(s[1]).to.have.property("thumb", "scene-image-thumb.jpg"); }); - it("filters by author", async function(){ - await vfs.createScene("User Authored", user.uid); - await vfs.createScene("Scene Admin Authored", sceneAdmin.uid); - let s = await vfs.getScenes(user.uid, {author: user.username}); - expect(s, `Matched Scenes: [${s.map(s=>s.name).join(", ")}]`).to.have.property("length", 1); - }); + it("returns the last-saved thumbnail", async function(){ + let s1 = await vfs.createScene("01"); + let times = [ + new Date("2022-01-01"), + new Date("2023-01-01"), + new Date("2024-01-01") + ]; + const setDate = (i:number, d:Date)=>vfs._db.run(`UPDATE files SET ctime = $2 WHERE file_id = $1`, [i, d]); + let png = await vfs.writeDoc("{}", {scene: s1, user_id: null, name: "scene-image-thumb.png", mime: "image/png"}); + let jpg = await vfs.writeDoc("{}", {scene: s1, user_id: null, name: "scene-image-thumb.jpg", mime: "image/jpeg"}); - it("filters by name match", async function(){ - let hello_scene_id = await vfs.createScene("Hello World", user.uid); - await vfs.writeDoc(JSON.stringify( { - metas: [{collection:{ - titles:{EN: "", FR: ""} - }}]} - ), {scene: hello_scene_id, user_id: user.uid, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); - let goodbye_scene_id = await vfs.createScene("Goodbye World", user.uid); - await vfs.writeDoc(JSON.stringify({ - metas: [{collection:{ - titles:{EN: "", FR: ""} - }}]} - ), {scene: goodbye_scene_id, user_id: user.uid, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); - let s = await vfs.getScenes(user.uid, {match: "Hello"}) - expect(s, `Matched Scenes: [${s.map(s=>s.name).join(", ")}]`).to.have.property("length", 1); - }); + let r = await setDate(jpg.id, times[1]); + await setDate(png.id, times[2]); + let s = await vfs.getScenes(0); + expect(s).to.have.length(1); + expect(s[0], `use PNG thumbnail if it's the most recent`).to.have.property("thumb", "scene-image-thumb.png"); - it("can match a document's meta title", async function(){ - let scene_id = await vfs.createScene("foo", user.uid); - await vfs.writeDoc(JSON.stringify({ - metas: [{collection:{ - titles:{EN: "Hello World", FR: "Bonjour, monde"} - }}] - }), {scene: scene_id, user_id: user.uid, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); - let s = await vfs.getScenes(user.uid, {match: "Hello"}); - expect(s, `[${s.map(s=>s.name).join(", ")}]`).to.have.property("length", 1); - }); + await setDate(png.id, times[0]); + s = await vfs.getScenes(0); + expect(s[0], `use JPG thumbnail if it's the most recent`).to.have.property("thumb", "scene-image-thumb.jpg"); - it("can match a document's intros", async function(){ - let scene_id = await vfs.createScene("foo", user.uid); - await vfs.writeDoc(JSON.stringify({ - metas: [{collection:{ - intros:{EN: "Hello World", FR: "Bonjour, monde"} - }}] - }), {scene: scene_id, user_id: user.uid, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); - - let s = await vfs.getScenes(user.uid, {match: "Hello"}); - expect(s, `[${s.map(s=>s.name).join(", ")}]`).to.have.property("length", 1); + //If date is equal, prioritize jpg + await setDate(png.id, times[1]); + s = await vfs.getScenes(0); + expect(s[0], `With equal dates, alphanumeric order shopuld prioritize JPG over PNG file`).to.have.property("thumb", "scene-image-thumb.jpg"); }); - it("can match a document's copyright", async function(){ - let scene_id = await vfs.createScene("foo", user.uid); - await vfs.writeDoc(JSON.stringify({ - metas: [{collection:{ - titles:{EN: " ", FR: " "} - }}], - asset: {copyright: "Hello World"} - }), {scene: scene_id, user_id: user.uid, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); - - let s = await vfs.getScenes(user.uid, {match: "Hello"}); - expect(s, `[${s.map(s=>s.name).join(", ")}]`).to.have.property("length", 1); + it("can get archived scenes", async function(){ + let scene_id = await vfs.createScene("foo"); + await vfs.writeDoc(JSON.stringify({foo: "bar"}), {scene: scene_id, user_id: null, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); + await vfs.archiveScene(scene_id); + let scenes = await vfs.getScenes(); + expect(scenes.map(({name})=>({name}))).to.deep.equal([{name: `foo#${scene_id}`}]); }); + - it("can match a document's article title", async function(){ - let scene_id = await vfs.createScene("foo", user.uid); - await vfs.writeDoc(JSON.stringify({ - metas: [{ - articles:[ - {titles:{EN: "Hello"}} - ] - }] - }), {scene: scene_id, user_id: user.uid, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); - let s = await vfs.getScenes(user.uid, {match: "Hello"}); - expect(s, `[${s.map(s=>s.name).join(", ")}]`).to.have.property("length", 1); - }); + it("Can't get archived scenes without being authenticated", async function(){ + await vfs.createScene("bar"); + let scene_id = await vfs.createScene("foo"); + await vfs.writeDoc(JSON.stringify({foo: "bar"}), {scene: scene_id, user_id: null, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); + await vfs.archiveScene(scene_id); - it("can match a document's annotation title", async function(){ - let scene_id = await vfs.createScene("foo", user.uid); - await vfs.writeDoc(JSON.stringify({ - models: [{ - annotations:[ - {titles:{EN: "Hello"}} - ] - }] - }), {scene: scene_id, user_id: user.uid, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); - let s = await vfs.getScenes(user.uid, {match: "Hello"}); - expect(s, `[${s.map(s=>s.name).join(", ")}]`).to.have.property("length", 1); + //Two scenes total + expect(await vfs.getScenes()).to.have.length(2); + //Filter only scenes with access: none + await expect(vfs.getScenes(null, {archived: true})).to.be.rejectedWith(UnauthorizedError); }); - it("can match a document's tour title", async function(){ + it("can get an author's own archived scenes", async function(){ + let um :UserManager= new UserManager(vfs._db); + let user = await um.addUser("bob", "12345678", "create", "bob@example.com") + //Create a reference non-archived scene (shouldn't be shown) + await vfs.createScene("bar", user.uid); + //Create a scene owned by someone else + await vfs.createScene("baz"); + //Create our archived scene let scene_id = await vfs.createScene("foo", user.uid); - await vfs.writeDoc(JSON.stringify({ - setups: [{ - tours:[ - {titles:{EN: "Hello"}} - ] - }] - }), {scene: scene_id, user_id: user.uid, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); - let s = await vfs.getScenes(user.uid, {match: "Hello"}); - expect(s, `[${s.map(s=>s.name).join(", ")}]`).to.have.property("length", 1); - }); + await vfs.writeDoc(JSON.stringify({foo: "bar"}), {scene: scene_id, user_id: null, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); + await vfs.archiveScene(scene_id); - it("can match a document's article lead", async function(){ - let scene_id = await vfs.createScene("foo", user.uid); - await vfs.writeDoc(JSON.stringify({ - metas: [{ - articles:[ - {leads:{EN: "Hello"}} - ] - }] - }), {scene: scene_id, user_id: user.uid, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); - let s = await vfs.getScenes(user.uid, {match: "Hello"}); - expect(s, `[${s.map(s=>s.name).join(", ")}]`).to.have.property("length", 1); - }); + //Three scenes total + expect(await vfs.getScenes()).to.have.length(3); + //Two "existing" scenes + expect(await vfs.getScenes(user.uid, {archived: false})).to.have.length(2); - it("can match a document's annotation leads", async function(){ - let scene_id = await vfs.createScene("foo", user.uid); - await vfs.writeDoc(JSON.stringify({ - models: [{ - annotations:[ - {leads:{EN: "Hello"}} - ] - }] - }), {scene: scene_id, user_id: user.uid, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); - let s = await vfs.getScenes(user.uid, {match: "Hello"}); - expect(s, `[${s.map(s=>s.name).join(", ")}]`).to.have.property("length", 1); - }); + //Filter only scenes with access: none + let scenes = await vfs.getScenes(user.uid, {archived: true}); + expect(scenes.map(({name})=>({name}))).to.deep.equal([{name: `foo#${scene_id}`}]); + }) - it("can match a document's tour leads", async function(){ - let scene_id = await vfs.createScene("foo", user.uid); - await vfs.writeDoc(JSON.stringify({ - setups: [{ - tours:[ - {leads:{EN: "Hello"}} - ] - }] - }), {scene: scene_id, user_id: user.uid, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); - let s = await vfs.getScenes(user.uid, {match: "Hello"}); - expect(s, `[${s.map(s=>s.name).join(", ")}]`).to.have.property("length", 1); - }); + describe("with permissions", function(){ + let userManager :UserManager, user :User; + this.beforeEach(async function(){ + userManager = new UserManager(vfs._db); + user = await userManager.addUser("alice", "xxxxxxxx", "create"); + }); + it("can filter accessible scenes by user_id", async function(){ + let scene_id = await vfs.createScene("foo", user.uid); + await userManager.setPublicAccess("foo", "none"); + await userManager.setDefaultAccess("foo", "none"); + await userManager.grant("foo", user.uid, "none"); + await run(`UPDATE scenes SET public_access = 0`); + await run(`INSERT INTO users_acl (fk_scene_id, fk_user_id, access_level) VALUES ($1, $2, 3)`, [scene_id, user.uid]); + expect((await vfs.getScenes(0)), `private scene shouldn't be returned to default user`).to.have.property("length", 0); + expect(await vfs.getScenes(user.uid), `private scene should be returned to its author`).to.have.property("length", 1); + }); - it("can match a document's article text", async function(){ - let scene_id = await vfs.createScene("foo", user.uid); - await vfs.writeDoc("Hello\n", {scene: scene_id, mime: "text/html", name: "articles/foo.html", user_id: user.uid}); - await vfs.writeDoc(JSON.stringify({ - metas: [{ - articles:[{ - titles: {EN: " ", FR: ""}, - uris: {EN: "articles/foo.html", FR: "articles/foo.html"} - }] - }] - }), {scene: scene_id, user_id: user.uid, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); - let s = await vfs.getScenes(user.uid, {match: "Hello"}); - expect(s, `[${s.map(s=>s.name).join(", ")}]`).to.have.property("length", 1); + it("get proper author id and name", async function(){ + let scene_id = await vfs.createScene("foo", user.uid); + await vfs.writeDoc("{}", {scene: scene_id, user_id: user.uid, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); + let scenes = await vfs.getScenes(); + expect(scenes).to.have.property("length", 1); + expect(scenes[0]).to.have.property("author", user.username); + expect(scenes[0]).to.have.property("author_id", user.uid); + }); + + it("get proper user own access", async function(){ + let scene_id = await vfs.createScene("foo", user.uid); + await userManager.setDefaultAccess(scene_id, "write"); + await userManager.setPublicAccess(scene_id, "none"); + let scenes = await vfs.getScenes(user.uid); + expect(scenes).to.have.property("length", 1); + expect(scenes[0]).to.have.property("access").to.equal("admin"); + }); + it("get proper \"any\" access", async function(){ + let scene_id = await vfs.createScene("foo"); + await userManager.setDefaultAccess(scene_id, "write"); + await userManager.setPublicAccess(scene_id, "read"); + let scenes = await vfs.getScenes(user.uid); + expect(scenes).to.have.property("length", 1); + expect(scenes[0]).to.have.property("access").to.equal("write"); + }); + + it("get proper group access", async function(){ + let scene_id = await vfs.createScene("foo"); + let group = await userManager.addGroup("My group"); + await userManager.addMemberToGroup(user.uid, group.groupUid); + await userManager.setDefaultAccess(scene_id, "read"); + await userManager.setPublicAccess(scene_id, "read"); + await userManager.grantGroup(scene_id, group.groupUid, "write") + let scenes = await vfs.getScenes(user.uid); + expect(scenes).to.have.property("length", 1); + expect(scenes[0]).to.have.property("access").to.equal("write"); + }); + + it("Do not show non-public scene when there is no requester", async function(){ + let scene_id = await vfs.createScene("foo", user.uid); + await userManager.setDefaultAccess(scene_id, "write"); + await userManager.setPublicAccess(scene_id, "none"); + let scenes = await vfs.getScenes(); + expect(scenes).to.have.property("length", 0); + }); }); + describe("search", async function(){ + let userManager :UserManager, user :User, sceneAdmin :User, admin :User; + this.beforeEach(async function(){ + userManager = new UserManager(vfs._db); + user = await userManager.addUser("bob", "xxxxxxxx", "create"); + sceneAdmin = await userManager.addUser("alice", "xxxxxxxx", "create"); + admin = await userManager.addUser("adele", "xxxxxxxx", "admin"); + }); - it("is case-insensitive", async function(){ - const scene = await vfs.createScene("Hello World", user.uid); - await vfs.writeDoc(JSON.stringify( { - metas: [{collection:{ - titles:{EN: "", FR: ""} - }}]} - ), {scene: scene, user_id: sceneAdmin.uid, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); - let s = await vfs.getScenes(user.uid, {match: "hello"}) - expect(s, `[${s.map(s=>s.name).join(", ")}]`).to.have.property("length", 1); - }); + it("filters by access-level", async function(){ + await vfs.createScene("foo", sceneAdmin.uid); + await userManager.grant("foo", user.uid, "read"); + expect(await vfs.getScenes(user.uid, {})).to.have.property("length", 1); + expect(await vfs.getScenes(user.uid, {access:"admin"})).to.have.property("length", 0); + }); + it("won't return inaccessible content", async function(){ + await vfs.createScene("foo", sceneAdmin.uid); + await userManager.setPublicAccess("foo", "none"); + await userManager.setDefaultAccess("foo", "none"); + expect(await vfs.getScenes(user.uid, {access:"none"})).to.have.property("length", 0); + }); - it("can search against multiple search terms", async function(){ - let scene_id = await vfs.createScene("bar", user.uid); - await vfs.writeDoc(JSON.stringify({ - metas: [{ - articles:[ - {leads:{EN: "Hello World, this is User"}} - ] - }] - }), {scene: scene_id, user_id: user.uid, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); - - scene_id = await vfs.createScene("foo 1", sceneAdmin.uid); - await vfs.writeDoc(JSON.stringify( { - metas: [{collection:{ - titles:{EN: "", FR: ""} - }}]} - ), {scene: scene_id, user_id: sceneAdmin.uid, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); - - scene_id = await vfs.createScene("foo 2", user.uid); - await vfs.writeDoc(JSON.stringify( { - metas: [{collection:{ - titles:{EN: "fizz", FR: ""} - }}]} - ), {scene: scene_id, user_id: sceneAdmin.uid, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); + it("will return everything to admin level user", async function(){ + await vfs.createScene("foo", sceneAdmin.uid); + await userManager.setPublicAccess("foo", "none"); + await userManager.setDefaultAccess("foo", "none"); + expect(await vfs.getScenes(admin.uid)).to.have.property("length", 1); + }); - let s = await vfs.getScenes(user.uid, {match: `foo fizz`}); - expect(s, `[${s.map(s=>s.name).join(", ")}]`).to.have.property("length", 1); - expect(s[0]).to.have.property("name", "foo 2"); + it("will return only scenes with specicfic rights to admin level user", async function(){ + await vfs.createScene("foo", sceneAdmin.uid); + await userManager.setPublicAccess("foo", "none"); + await userManager.setDefaultAccess("foo", "none"); + expect(await vfs.getScenes(admin.uid, {access:"read"})).to.have.property("length", 0); + expect(await vfs.getScenes(admin.uid, {access:"write"})).to.have.property("length", 0); + expect(await vfs.getScenes(admin.uid, {access:"admin"})).to.have.property("length", 0); + }); - s = await vfs.getScenes(user.uid, {match: `foo OR fizz`}); - expect(s, `[${s.map(s=>s.name).join(", ")}]`).to.have.property("length", 2); - expect(s[0]).to.have.property("name", "foo 2"); - expect(s[1]).to.have.property("name", "foo 1"); - + it("can select by specific user access level", async function(){ + await vfs.createScene("foo", sceneAdmin.uid); + await userManager.grant("foo", user.uid, "read"); + await userManager.setPublicAccess("foo", "read"); + await userManager.setDefaultAccess("foo", "read"); + expect(await vfs.getScenes(user.uid, {access:"read"})).to.have.property("length", 1); + }); - s = await vfs.getScenes(user.uid, {match: `Hello User`}); - expect(s, `[${s.map(s=>s.name).join(", ")}]`).to.have.property("length", 1); - expect(s[0]).to.have.property("name", "bar"); - }); + it("filters by author", async function(){ + await vfs.createScene("User Authored", user.uid); + await vfs.createScene("Scene Admin Authored", sceneAdmin.uid); + let s = await vfs.getScenes(user.uid, {author: user.username}); + expect(s, `Matched Scenes: [${s.map(s=>s.name).join(", ")}]`).to.have.property("length", 1); + }); + it("filters by name match", async function(){ + let hello_scene_id = await vfs.createScene("Hello World", user.uid); + await vfs.writeDoc(JSON.stringify( { + metas: [{collection:{ + titles:{EN: "", FR: ""} + }}]} + ), {scene: hello_scene_id, user_id: user.uid, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); + let goodbye_scene_id = await vfs.createScene("Goodbye World", user.uid); + await vfs.writeDoc(JSON.stringify({ + metas: [{collection:{ + titles:{EN: "", FR: ""} + }}]} + ), {scene: goodbye_scene_id, user_id: user.uid, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); + let s = await vfs.getScenes(user.uid, {match: "Hello"}) + expect(s, `Matched Scenes: [${s.map(s=>s.name).join(", ")}]`).to.have.property("length", 1); + }); - it("can match an empty string", async function(){ - await vfs.createScene("Hello World", user.uid); - await vfs.createScene("Goodbye World", user.uid); - let s = await vfs.getScenes(user.uid, {match: ""}) - expect(s, `Matched Scenes: [${s.map(s=>s.name).join(", ")}]`).to.have.property("length", 2); - }); + it("can match a document's meta title", async function(){ + let scene_id = await vfs.createScene("foo", user.uid); + await vfs.writeDoc(JSON.stringify({ + metas: [{collection:{ + titles:{EN: "Hello World", FR: "Bonjour, monde"} + }}] + }), {scene: scene_id, user_id: user.uid, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); + let s = await vfs.getScenes(user.uid, {match: "Hello"}); + expect(s, `[${s.map(s=>s.name).join(", ")}]`).to.have.property("length", 1); + }); + it("can match a document's intros", async function(){ + let scene_id = await vfs.createScene("foo", user.uid); + await vfs.writeDoc(JSON.stringify({ + metas: [{collection:{ + intros:{EN: "Hello World", FR: "Bonjour, monde"} + }}] + }), {scene: scene_id, user_id: user.uid, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); - it("can match partial words in titles of scenes, including the ones without json", async function(){ - await vfs.createScene("EAD.A.Nom1.Nom2", user.uid); - await vfs.createScene("GlobeAppli", user.uid); - let s = await vfs.getScenes(user.uid, {match: "lobe"}); - expect(s, `Globe Matched Scenes: [${s.map(s=>s.name).join(", ")}]`).to.have.property("length", 1); - s = await vfs.getScenes(user.uid, {match: "EAD"}); - expect(s, `EAD Matched Scenes: [${s.map(s=>s.name).join(", ")}]`).to.have.property("length", 1); - }); - }); - - describe("ordering", function(){ - it("rejects bad orderBy key", async function(){ - await expect(vfs.getScenes(0, {orderBy: "bad" as any})).to.be.rejectedWith("Invalid orderBy: bad"); - }) - it("rejects bad orderDirection key", async function(){ - await expect(vfs.getScenes(0, {orderDirection: "bad" as any})).to.be.rejectedWith("Invalid orderDirection: bad"); - }); - it("can order by name descending", async function(){ - for(let i = 0; i < 10; i++){ - await vfs.createScene(`${i}_scene`); - } - const scenes = await vfs.getScenes(0, {orderBy: "name", orderDirection: "desc"}); - expect(scenes.map(s=>s.name)).to.deep.equal([9,8,7,6,5,4,3,2,1,0].map(n=>n+"_scene")); - }); - }); + let s = await vfs.getScenes(user.uid, {match: "Hello"}); + expect(s, `[${s.map(s=>s.name).join(", ")}]`).to.have.property("length", 1); + }); - describe("pagination", function(){ - it("rejects bad LIMIT", async function(){ - let fixtures = [-1, "10", null]; - for(let f of fixtures){ - await expect(vfs.getScenes(0, {limit: f as any})).to.be.rejectedWith(BadRequestError); - } - }); + it("can match a document's copyright", async function(){ + let scene_id = await vfs.createScene("foo", user.uid); + await vfs.writeDoc(JSON.stringify({ + metas: [{collection:{ + titles:{EN: " ", FR: " "} + }}], + asset: {copyright: "Hello World"} + }), {scene: scene_id, user_id: user.uid, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); + + let s = await vfs.getScenes(user.uid, {match: "Hello"}); + expect(s, `[${s.map(s=>s.name).join(", ")}]`).to.have.property("length", 1); + }); - it("rejects bad OFFSET", async function(){ - let fixtures = [-1, "10", null]; - for(let f of fixtures){ - await expect(vfs.getScenes(0, {limit: f as any})).to.be.rejectedWith(BadRequestError); - } - }); + it("can match a document's article title", async function(){ + let scene_id = await vfs.createScene("foo", user.uid); + await vfs.writeDoc(JSON.stringify({ + metas: [{ + articles:[ + {titles:{EN: "Hello"}} + ] + }] + }), {scene: scene_id, user_id: user.uid, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); + let s = await vfs.getScenes(user.uid, {match: "Hello"}); + expect(s, `[${s.map(s=>s.name).join(", ")}]`).to.have.property("length", 1); + }); - it("respects pagination options", async function(){ - for(let i = 0; i < 10; i++){ - await vfs.createScene(`scene_${i}`); - } - let res = await vfs.getScenes(0, {limit: 1, offset: 0}) - expect(res).to.have.property("length", 1); - expect(res[0]).to.have.property("name", "scene_9"); - - res = await vfs.getScenes(0, {limit: 2, offset: 2}) - expect(res).to.have.property("length", 2); - expect(res[0]).to.have.property("name", "scene_7"); - expect(res[1]).to.have.property("name", "scene_6"); - }); + it("can match a document's annotation title", async function(){ + let scene_id = await vfs.createScene("foo", user.uid); + await vfs.writeDoc(JSON.stringify({ + models: [{ + annotations:[ + {titles:{EN: "Hello"}} + ] + }] + }), {scene: scene_id, user_id: user.uid, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); + let s = await vfs.getScenes(user.uid, {match: "Hello"}); + expect(s, `[${s.map(s=>s.name).join(", ")}]`).to.have.property("length", 1); + }); - it("limits LIMIT to 100", async function(){ - await expect(vfs.getScenes(0, {limit: 110, offset: 0})).to.be.rejectedWith("[400]"); - }); - }); - }); + it("can match a document's tour title", async function(){ + let scene_id = await vfs.createScene("foo", user.uid); + await vfs.writeDoc(JSON.stringify({ + setups: [{ + tours:[ + {titles:{EN: "Hello"}} + ] + }] + }), {scene: scene_id, user_id: user.uid, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); + let s = await vfs.getScenes(user.uid, {match: "Hello"}); + expect(s, `[${s.map(s=>s.name).join(", ")}]`).to.have.property("length", 1); + }); - describe("createFolder(), removeFolder(), listFolders()", function(){ - let scene_id :number; - this.beforeEach(async function(){ - scene_id = await vfs.createScene("foo"); - }) + it("can match a document's article lead", async function(){ + let scene_id = await vfs.createScene("foo", user.uid); + await vfs.writeDoc(JSON.stringify({ + metas: [{ + articles:[ + {leads:{EN: "Hello"}} + ] + }] + }), {scene: scene_id, user_id: user.uid, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); + let s = await vfs.getScenes(user.uid, {match: "Hello"}); + expect(s, `[${s.map(s=>s.name).join(", ")}]`).to.have.property("length", 1); + }); - it("create a folder in a scene", async function(){ - await vfs.createFolder({scene:scene_id, name: "videos", user_id: null}); - await vfs.writeDoc("foo", {scene: scene_id, name: "videos/foo.txt", user_id: null}); - let folders = await collapseAsync(vfs.listFolders(scene_id)); - //order is by mtime descending, name ascending so we can't rely on it - expect(folders.map(f=>f.name)).to.have.members(["articles", "models", "videos"]); - expect(folders).to.have.length(3); - }); + it("can match a document's annotation leads", async function(){ + let scene_id = await vfs.createScene("foo", user.uid); + await vfs.writeDoc(JSON.stringify({ + models: [{ + annotations:[ + {leads:{EN: "Hello"}} + ] + }] + }), {scene: scene_id, user_id: user.uid, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); + let s = await vfs.getScenes(user.uid, {match: "Hello"}); + expect(s, `[${s.map(s=>s.name).join(", ")}]`).to.have.property("length", 1); + }); - it("create a tree of folders", async function(){ - await vfs.createFolder({scene:scene_id, name: "articles/videos", user_id: null}); - let folders = await collapseAsync(vfs.listFolders(scene_id)); - expect(folders.map(f=>f.name)).to.deep.equal(["articles/videos", "articles" , "models"]); - }); + it("can match a document's tour leads", async function(){ + let scene_id = await vfs.createScene("foo", user.uid); + await vfs.writeDoc(JSON.stringify({ + setups: [{ + tours:[ + {leads:{EN: "Hello"}} + ] + }] + }), {scene: scene_id, user_id: user.uid, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); + let s = await vfs.getScenes(user.uid, {match: "Hello"}); + expect(s, `[${s.map(s=>s.name).join(", ")}]`).to.have.property("length", 1); + }); - it("don't accept a trailing slash", async function(){ - await expect(vfs.createFolder({scene:scene_id, name: "videos/", user_id: null})).to.be.rejectedWith(BadRequestError); - }); - it("don't accept absolute paths", async function(){ - await expect(vfs.createFolder({scene:scene_id, name: "/videos", user_id: null})).to.be.rejectedWith(BadRequestError); - }); + it("can match a document's article text", async function(){ + let scene_id = await vfs.createScene("foo", user.uid); + await vfs.writeDoc("Hello\n", {scene: scene_id, mime: "text/html", name: "articles/foo.html", user_id: user.uid}); + await vfs.writeDoc(JSON.stringify({ + metas: [{ + articles:[{ + titles: {EN: " ", FR: ""}, + uris: {EN: "articles/foo.html", FR: "articles/foo.html"} + }] + }] + }), {scene: scene_id, user_id: user.uid, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); + let s = await vfs.getScenes(user.uid, {match: "Hello"}); + expect(s, `[${s.map(s=>s.name).join(", ")}]`).to.have.property("length", 1); + }); - it("throws an error if folder exists", async function(){ - await vfs.createFolder({scene: scene_id, name: "videos", user_id: null}); - await expect( vfs.createFolder({scene: scene_id, name: "videos", user_id: null}) ).to.be.rejectedWith(ConflictError); - }); - it("throws an error if folder doesn't exist", async function(){ - await expect(vfs.removeFolder({scene: scene_id, name: "videos", user_id: null})).to.be.rejectedWith(NotFoundError); - }); + it("is case-insensitive", async function(){ + const scene = await vfs.createScene("Hello World", user.uid); + await vfs.writeDoc(JSON.stringify( { + metas: [{collection:{ + titles:{EN: "", FR: ""} + }}]} + ), {scene: scene, user_id: sceneAdmin.uid, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); + let s = await vfs.getScenes(user.uid, {match: "hello"}) + expect(s, `[${s.map(s=>s.name).join(", ")}]`).to.have.property("length", 1); + }); - it("remove a scene's folder", async function(){ - await vfs.createFolder({scene:scene_id, name: "videos", user_id: null}); - await vfs.removeFolder({scene: scene_id, name: "videos", user_id: null}); - let folders = await collapseAsync(vfs.listFolders(scene_id)); - expect(folders.map(f=>f.name)).to.deep.equal(["articles", "models"]); - await vfs.createFolder({scene:scene_id, name: "videos", user_id: null}); - folders = await collapseAsync(vfs.listFolders(scene_id)); - expect(folders.map(f=>f.name).sort()).to.deep.equal(["videos", "models", "articles"].sort()); - }); - it("removeFolder() removes all files in the folder", async function(){ - let userManager = new UserManager(vfs._db); - let user = await userManager.addUser("alice", "xxxxxxxx", "create"); - await vfs.createFolder({scene:scene_id, name: "videos", user_id: null}); - await vfs.writeFile(dataStream(), {scene: scene_id, name: "videos/foo.mp4", mime:"video/mp4", user_id: null}); + it("can search against multiple search terms", async function(){ + let scene_id = await vfs.createScene("bar", user.uid); + await vfs.writeDoc(JSON.stringify({ + metas: [{ + articles:[ + {leads:{EN: "Hello World, this is User"}} + ] + }] + }), {scene: scene_id, user_id: user.uid, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); + + scene_id = await vfs.createScene("foo 1", sceneAdmin.uid); + await vfs.writeDoc(JSON.stringify( { + metas: [{collection:{ + titles:{EN: "", FR: ""} + }}]} + ), {scene: scene_id, user_id: sceneAdmin.uid, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); + + scene_id = await vfs.createScene("foo 2", user.uid); + await vfs.writeDoc(JSON.stringify( { + metas: [{collection:{ + titles:{EN: "fizz", FR: ""} + }}]} + ), {scene: scene_id, user_id: sceneAdmin.uid, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); + + let s = await vfs.getScenes(user.uid, {match: `foo fizz`}); + expect(s, `[${s.map(s=>s.name).join(", ")}]`).to.have.property("length", 1); + expect(s[0]).to.have.property("name", "foo 2"); + + s = await vfs.getScenes(user.uid, {match: `foo OR fizz`}); + expect(s, `[${s.map(s=>s.name).join(", ")}]`).to.have.property("length", 2); + expect(s[0]).to.have.property("name", "foo 2"); + expect(s[1]).to.have.property("name", "foo 1"); + + + s = await vfs.getScenes(user.uid, {match: `Hello User`}); + expect(s, `[${s.map(s=>s.name).join(", ")}]`).to.have.property("length", 1); + expect(s[0]).to.have.property("name", "bar"); + }); - await vfs.removeFolder({scene: scene_id, name: "videos", user_id: user.uid }); - let files = await collapseAsync(vfs.listFiles(scene_id)); - expect(files).to.deep.equal([]); - }); - }); + it("can match an empty string", async function(){ + await vfs.createScene("Hello World", user.uid); + await vfs.createScene("Goodbye World", user.uid); + let s = await vfs.getScenes(user.uid, {match: ""}) + expect(s, `Matched Scenes: [${s.map(s=>s.name).join(", ")}]`).to.have.property("length", 2); + }); - describe("tags", function(){ - let scene_id :number; - //Create a dummy scene for future tests - this.beforeEach(async function(){ - scene_id = await vfs.createScene("foo"); - }); - describe("addSceneTag() / removeSceneTag()", function(){ - it("adds a tag to a scene", async function(){ - await vfs.addTag(scene_id, "foo"); - let s = await vfs.getScene(scene_id); - expect(s).to.have.property("tags").to.deep.equal(["foo"]); - await vfs.addTag(scene_id, "bar"); - s = await vfs.getScene(scene_id); - //Ordering is loosely expected to hold: we do not enforce AUTOINCREMENT on rowids but it's generally true - expect(s).to.have.property("tags").to.deep.equal(["foo", "bar"]); + it("can match partial words in titles of scenes, including the ones without json", async function(){ + await vfs.createScene("EAD.A.Nom1.Nom2", user.uid); + await vfs.createScene("GlobeAppli", user.uid); + let s = await vfs.getScenes(user.uid, {match: "lobe"}); + expect(s, `Globe Matched Scenes: [${s.map(s=>s.name).join(", ")}]`).to.have.property("length", 1); + s = await vfs.getScenes(user.uid, {match: "EAD"}); + expect(s, `EAD Matched Scenes: [${s.map(s=>s.name).join(", ")}]`).to.have.property("length", 1); + }); }); - - it("can remove tag", async function(){ - await expect(vfs.addTag(scene_id, "foo")).to.eventually.equal(true); - await expect(vfs.addTag(scene_id, "bar")).to.eventually.equal(true); - await expect(vfs.removeTag(scene_id, "foo")).to.eventually.equal(true); - - let s = await vfs.getScene(scene_id); - expect(s).to.have.property("tags").to.deep.equal(["bar"]); + + describe("ordering", function(){ + it("rejects bad orderBy key", async function(){ + await expect(vfs.getScenes(0, {orderBy: "bad" as any})).to.be.rejectedWith("Invalid orderBy: bad"); + }) + it("rejects bad orderDirection key", async function(){ + await expect(vfs.getScenes(0, {orderDirection: "bad" as any})).to.be.rejectedWith("Invalid orderDirection: bad"); + }); + it("can order by name descending", async function(){ + for(let i = 0; i < 10; i++){ + await vfs.createScene(`${i}_scene`); + } + const scenes = await vfs.getScenes(0, {orderBy: "name", orderDirection: "desc"}); + expect(scenes.map(s=>s.name)).to.deep.equal([9,8,7,6,5,4,3,2,1,0].map(n=>n+"_scene")); + }); }); - it("can be called with scene name", async function(){ - await expect(vfs.addTag("foo", "foo")).to.eventually.equal(true); - let s = await vfs.getScene(scene_id); - expect(s).to.have.property("tags").to.deep.equal(["foo"]); + describe("pagination", function(){ + it("rejects bad LIMIT", async function(){ + let fixtures = [-1, "10", null]; + for(let f of fixtures){ + await expect(vfs.getScenes(0, {limit: f as any})).to.be.rejectedWith(BadRequestError); + } + }); - await expect(vfs.removeTag("foo", "foo")).to.eventually.equal(true); - }); + it("rejects bad OFFSET", async function(){ + let fixtures = [-1, "10", null]; + for(let f of fixtures){ + await expect(vfs.getScenes(0, {limit: f as any})).to.be.rejectedWith(BadRequestError); + } + }); - it("throws a 404 errors if scene doesn't exist", async function(){ - // by id - await expect(vfs.addTag(scene_id+1, "foo")).to.be.rejectedWith(NotFoundError); - // by name - await expect(vfs.addTag("baz", "foo")).to.be.rejectedWith(NotFoundError); + it("respects pagination options", async function(){ + for(let i = 0; i < 10; i++){ + await vfs.createScene(`scene_${i}`); + } + let res = await vfs.getScenes(0, {limit: 1, offset: 0}) + expect(res).to.have.property("length", 1); + expect(res[0]).to.have.property("name", "scene_9"); + + res = await vfs.getScenes(0, {limit: 2, offset: 2}) + expect(res).to.have.property("length", 2); + expect(res[0]).to.have.property("name", "scene_7"); + expect(res[1]).to.have.property("name", "scene_6"); + }); + it("limits LIMIT to 100", async function(){ + await expect(vfs.getScenes(0, {limit: 110, offset: 0})).to.be.rejectedWith("[400]"); + }); }); + }); + describe("createFolder(), removeFolder(), listFolders()", function(){ + let scene_id :number; + this.beforeEach(async function(){ + scene_id = await vfs.createScene("foo"); + }) - it("returns false if nothing was changed", async function(){ - await vfs.addTag(scene_id, "foo"); - //When tag is added twice, by scene_id - await expect(vfs.addTag(scene_id, "foo")).to.eventually.equal(false); - //When tag is added twice, by name - await expect(vfs.addTag("foo", "foo")).to.eventually.equal(false); - - //When tag doesn't exist - await expect(vfs.removeTag(scene_id, "bar")).to.be.eventually.equal(false); - //When scene doesn't exist - await expect(vfs.removeTag(scene_id+1, "foo")).to.eventually.equal(false); - }); - it("store case and accents", async function(){ - await expect(vfs.addTag(scene_id, "Électricité")).to.eventually.equal(true); - expect(await vfs.getTags()).to.deep.equal([{name: "Électricité", size: 1}]); + it("create a folder in a scene", async function(){ + await vfs.createFolder({scene:scene_id, name: "videos", user_id: null}); + await vfs.writeDoc("foo", {scene: scene_id, name: "videos/foo.txt", user_id: null}); + let folders = await collapseAsync(vfs.listFolders(scene_id)); + //order is by mtime descending, name ascending so we can't rely on it + expect(folders.map(f=>f.name)).to.have.members(["articles", "models", "videos"]); + expect(folders).to.have.length(3); }); - it("collate case and accents (same scene)", async function(){ - await expect(vfs.addTag(scene_id, "Électricité")).to.eventually.equal(true); - await expect(vfs.addTag(scene_id, "électricité")).to.eventually.equal(false); - await expect(vfs.addTag(scene_id, "electricite")).to.eventually.equal(false); - - expect(await vfs.getTags()).to.deep.equal([{name: "Électricité", size: 1}]); + it("create a tree of folders", async function(){ + await vfs.createFolder({scene:scene_id, name: "articles/videos", user_id: null}); + let folders = await collapseAsync(vfs.listFolders(scene_id)); + expect(folders.map(f=>f.name)).to.deep.equal(["articles/videos", "articles" , "models"]); }); - it("collate case and accents (multiple scenes)", async function(){ - let s2 = await vfs.createScene("tags-collate-s2"); - let s3 = await vfs.createScene("tags-collate-s3"); - await expect(vfs.addTag(scene_id, "Électricité")).to.eventually.equal(true); - await expect(vfs.addTag(s2, "électricité")).to.eventually.equal(true); - await expect(vfs.addTag(s3, "electricite")).to.eventually.equal(true); - - expect(await vfs.getTags()).to.deep.equal([{name: "Électricité", size: 3}]); - }) - }); - - - describe("getTags()", function(){ - it("get all tags", async function(){ - //Create a bunch of additional test scenes - for(let i=0; i < 3; i++){ - let id = await vfs.createScene(`test_${i}`); - for(let j=0; j <= i; j++ ){ - await vfs.addTag(id, `tag_${j}`); - } - } - expect(await vfs.getTags()).to.deep.equal([ - {name: "tag_0", size: 3}, - {name: "tag_1", size: 2}, - {name: "tag_2", size: 1}, - ]); + it("don't accept a trailing slash", async function(){ + await expect(vfs.createFolder({scene:scene_id, name: "videos/", user_id: null})).to.be.rejectedWith(BadRequestError); }); - it("get tags matching a string", async function(){ - await vfs.addTag(scene_id, `tag_foo`); - await vfs.addTag(scene_id, `foo_tag`); - await vfs.addTag(scene_id, `tag_bar`); - - expect(await vfs.getTags({like: "foo"})).to.deep.equal([ - {name:"foo_tag", size: 1}, - {name: "tag_foo", size: 1}, - ]); - - //Match should be case-insensitive - expect(await vfs.getTags({like: "Foo"})).to.deep.equal([ - {name:"foo_tag", size: 1}, - {name: "tag_foo", size: 1}, - ]); + it("don't accept absolute paths", async function(){ + await expect(vfs.createFolder({scene:scene_id, name: "/videos", user_id: null})).to.be.rejectedWith(BadRequestError); }); - it("supports pagination", async function(){ - //Create a bunch of additional test scenes - for(let i=0; i < 3; i++){ - let id = await vfs.createScene(`test_${i}`); - for(let j=0; j <= i; j++ ){ - await vfs.addTag(id, `tag_${j}`); - } - } - expect(await vfs.getTags({limit: 1})).to.deep.equal([ - {name: "tag_0", size: 3}, - ]); - expect(await vfs.getTags({limit: 2, offset: 1})).to.deep.equal([ - {name: "tag_1", size: 2}, - {name: "tag_2", size: 1}, - ]); - expect(await vfs.getTags({offset: 3})).to.deep.equal([ ]); + it("throws an error if folder exists", async function(){ + await vfs.createFolder({scene: scene_id, name: "videos", user_id: null}); + await expect( vfs.createFolder({scene: scene_id, name: "videos", user_id: null}) ).to.be.rejectedWith(ConflictError); }); - it("don't count archived scenes", async function(){ - await expect(vfs.addTag(scene_id, "foo")).to.eventually.equal(true); - await vfs.archiveScene(scene_id); - expect(await vfs.getTags()).to.deep.equal([]); + it("throws an error if folder doesn't exist", async function(){ + await expect(vfs.removeFolder({scene: scene_id, name: "videos", user_id: null})).to.be.rejectedWith(NotFoundError); }); - }); - describe("getTag()", function(){ - it("Get all scenes attached to a tag", async function(){ - let ids = []; - for(let i=0; i < 3; i++){ - let id = await vfs.createScene(`test_${i}`); - ids.push(id); - await vfs.addTag(id, `tag_foo`); - } - for(let i=3; i < 6; i++){ - let id = await vfs.createScene(`test_${i}`); - await vfs.addTag(id, `tag_bar`); - } - let scenes = await vfs.getTag("tag_foo"); - expect(scenes).to.deep.equal(ids); + it("remove a scene's folder", async function(){ + await vfs.createFolder({scene:scene_id, name: "videos", user_id: null}); + await vfs.removeFolder({scene: scene_id, name: "videos", user_id: null}); + let folders = await collapseAsync(vfs.listFolders(scene_id)); + expect(folders.map(f=>f.name)).to.deep.equal(["articles", "models"]); + await vfs.createFolder({scene:scene_id, name: "videos", user_id: null}); + folders = await collapseAsync(vfs.listFolders(scene_id)); + expect(folders.map(f=>f.name).sort()).to.deep.equal(["videos", "models", "articles"].sort()); }); - it("Ignore archived scenes", async function(){ - await expect(vfs.addTag(scene_id, "foo")).to.eventually.equal(true); + it("removeFolder() removes all files in the folder", async function(){ + let userManager = new UserManager(vfs._db); + let user = await userManager.addUser("alice", "xxxxxxxx", "create"); + await vfs.createFolder({scene:scene_id, name: "videos", user_id: null}); + await vfs.writeFile(dataStream(), {scene: scene_id, name: "videos/foo.mp4", mime:"video/mp4", user_id: null}); - let s2 = await vfs.createScene(`test_scene_2`); - await expect(vfs.addTag(s2, "foo")).to.eventually.equal(true); - expect(await vfs.getTag("foo")).to.deep.equal([scene_id, s2]); + await vfs.removeFolder({scene: scene_id, name: "videos", user_id: user.uid }); - await vfs.archiveScene(s2); - expect(await vfs.getTag("foo")).to.deep.equal([scene_id]); + let files = await collapseAsync(vfs.listFiles(scene_id)); + expect(files).to.deep.equal([]); }); + }); + describe("tags", function(){ + let scene_id :number; + //Create a dummy scene for future tests + this.beforeEach(async function(){ + scene_id = await vfs.createScene("foo"); + }); + + describe("addSceneTag() / removeSceneTag()", function(){ + it("adds a tag to a scene", async function(){ + await vfs.addTag(scene_id, "foo"); + let s = await vfs.getScene(scene_id); + expect(s).to.have.property("tags").to.deep.equal(["foo"]); + await vfs.addTag(scene_id, "bar"); + s = await vfs.getScene(scene_id); + //Ordering is loosely expected to hold: we do not enforce AUTOINCREMENT on rowids but it's generally true + expect(s).to.have.property("tags").to.deep.equal(["foo", "bar"]); + }); - describe("respects permissions", function(){ - let userManager :UserManager, alice :User, bob :User; - this.beforeEach(async function(){ - userManager = new UserManager(vfs._db); - alice = await userManager.addUser("alice", "12345678", "admin"); - bob = await userManager.addUser("bob", "12345678", "create"); + it("can remove tag", async function(){ + await expect(vfs.addTag(scene_id, "foo")).to.eventually.equal(true); + await expect(vfs.addTag(scene_id, "bar")).to.eventually.equal(true); + await expect(vfs.removeTag(scene_id, "foo")).to.eventually.equal(true); + + let s = await vfs.getScene(scene_id); + expect(s).to.have.property("tags").to.deep.equal(["bar"]); }); - it("return scenes with public read access", async function(){ - await vfs.addTag("foo", "foo"); - expect(await vfs.getTag("foo", alice.uid), "with admin user_id").to.deep.equal([scene_id]); + it("can be called with scene name", async function(){ + await expect(vfs.addTag("foo", "foo")).to.eventually.equal(true); + let s = await vfs.getScene(scene_id); + expect(s).to.have.property("tags").to.deep.equal(["foo"]); - expect(await vfs.getTag("foo", bob.uid), "with normal user id").to.deep.equal([scene_id]); + await expect(vfs.removeTag("foo", "foo")).to.eventually.equal(true); }); - it("return all scenes for admins", async function(){ - //Scene if from bob, alice has no special rights over it, but should see it anyways - const id = await vfs.createScene("bob-private", bob.uid); - await userManager.setPublicAccess("bob-private", "none"); - await userManager.setDefaultAccess("bob-private", "none"); - await vfs.addTag("bob-private", "foo"); - expect(await vfs.getTag("foo"), "without user id").to.deep.equal([]); - expect(await vfs.getTag("foo", alice.uid), "with admin id").to.deep.equal([id]); - }) + it("throws a 404 errors if scene doesn't exist", async function(){ + // by id + await expect(vfs.addTag(scene_id+1, "foo")).to.be.rejectedWith(NotFoundError); + // by name + await expect(vfs.addTag("baz", "foo")).to.be.rejectedWith(NotFoundError); + + }); - it("won't return non-readable scene", async function(){ - const id = await vfs.createScene("admin-only", alice.uid); - await userManager.setPublicAccess("admin-only", "none"); - await userManager.setDefaultAccess("admin-only", "none"); - await vfs.addTag("admin-only", "foo"); - expect(await vfs.getTag("foo"), "without user_id").to.deep.equal([]); - expect(await vfs.getTag("foo", alice.uid), "with admin user_id").to.deep.equal([id]); + it("returns false if nothing was changed", async function(){ + await vfs.addTag(scene_id, "foo"); + //When tag is added twice, by scene_id + await expect(vfs.addTag(scene_id, "foo")).to.eventually.equal(false); + //When tag is added twice, by name + await expect(vfs.addTag("foo", "foo")).to.eventually.equal(false); - expect(await vfs.getTag("foo", bob.uid)).to.deep.equal([]); + //When tag doesn't exist + await expect(vfs.removeTag(scene_id, "bar")).to.be.eventually.equal(false); + //When scene doesn't exist + await expect(vfs.removeTag(scene_id+1, "foo")).to.eventually.equal(false); + }); + it("store case and accents", async function(){ + await expect(vfs.addTag(scene_id, "Électricité")).to.eventually.equal(true); + expect(await vfs.getTags()).to.deep.equal([{name: "Électricité", size: 1}]); }); - }) - }); - }); - describe("", function(){ - let scene_id :number; - //Create a dummy scene for future tests - this.beforeEach(async function(){ - scene_id = await vfs.createScene("foo"); - }); - - describe("renameScene()", function(){ - it("can change a scene name", async function(){ - await expect(vfs.renameScene(scene_id, "bar")).to.be.fulfilled; - }); - it("throw a 404 error", async function(){ - await expect(vfs.renameScene(404, "bar")).to.be.rejectedWith("404"); - }); - }) + it("collate case and accents (same scene)", async function(){ + await expect(vfs.addTag(scene_id, "Électricité")).to.eventually.equal(true); + await expect(vfs.addTag(scene_id, "électricité")).to.eventually.equal(false); + await expect(vfs.addTag(scene_id, "electricite")).to.eventually.equal(false); - describe("archiveScene()", function(){ - it("makes scene hidden", async function(){ - await vfs.archiveScene("foo"); - expect(await vfs.getScenes(0, {archived: false})).to.have.property("length", 0); - }); + expect(await vfs.getTags()).to.deep.equal([{name: "Électricité", size: 1}]); + }); + it("collate case and accents (multiple scenes)", async function(){ + let s2 = await vfs.createScene("tags-collate-s2"); + let s3 = await vfs.createScene("tags-collate-s3"); + await expect(vfs.addTag(scene_id, "Électricité")).to.eventually.equal(true); + await expect(vfs.addTag(s2, "électricité")).to.eventually.equal(true); + await expect(vfs.addTag(s3, "electricite")).to.eventually.equal(true); - it("can't archive twice", async function(){ - await vfs.archiveScene(scene_id); - await expect(vfs.archiveScene(scene_id), (await vfs._db.all("SELECT * FROM scenes"))[0].scene_name).to.be.rejectedWith(NotFoundError); + expect(await vfs.getTags()).to.deep.equal([{name: "Électricité", size: 3}]); + }) }); - it("can remove archived scene (by id)", async function(){ - await vfs.archiveScene(scene_id); - await expect(vfs.removeScene(scene_id)).to.be.fulfilled; - }); - - it("can remove archived scene (by name)", async function(){ - await vfs.archiveScene(scene_id); - await expect(vfs.removeScene(`foo#${scene_id}`)).to.be.fulfilled; - }); - it("store archive time", async function(){ - //To be used later - await vfs.archiveScene(scene_id); - let {archived} = await vfs._db.get(`SELECT archived FROM scenes WHERE scene_id= $1`, [scene_id]); - expect(archived).to.be.instanceof(Date); - expect(archived.toString()).not.to.equal('Invalid Date'); - expect(archived.valueOf(), archived.toUTCString()).to.be.above(new Date().valueOf() - 2000); - expect(archived.valueOf(), archived.toUTCString()).to.be.below(new Date().valueOf()+1); - }) - }); + describe("getTags()", function(){ + it("get all tags", async function(){ + //Create a bunch of additional test scenes + for(let i=0; i < 3; i++){ + let id = await vfs.createScene(`test_${i}`); + for(let j=0; j <= i; j++ ){ + await vfs.addTag(id, `tag_${j}`); + } + } + expect(await vfs.getTags()).to.deep.equal([ + {name: "tag_0", size: 3}, + {name: "tag_1", size: 2}, + {name: "tag_2", size: 1}, + ]); + }); - describe("unarchiveScene()", function(){ - it("restores an archived scene", async function(){ - await vfs.archiveScene(scene_id); - await vfs.unarchiveScene(`foo#${scene_id}`); - expect(await vfs.getScene("foo")).to.have.property("archived", null); - }); + it("get tags matching a string", async function(){ + await vfs.addTag(scene_id, `tag_foo`); + await vfs.addTag(scene_id, `foo_tag`); + await vfs.addTag(scene_id, `tag_bar`); - it("throws if archive doesn't exist", async function(){ - await expect(vfs.unarchiveScene("xxx")).to.be.rejectedWith(NotFoundError); - }) - }) + expect(await vfs.getTags({like: "foo"})).to.deep.equal([ + {name:"foo_tag", size: 1}, + {name: "tag_foo", size: 1}, + ]); - describe("createFile()", function(){ - it("can create an empty file", async function(){ - let r = await vfs.createFile( {scene: "foo", mime: "text/html", name: "articles/foo.txt", user_id: null}, {hash: null, size: 0}); - expect(r).to.have.property("id"); - expect(r).to.have.property("generation", 1); - expect(r).to.have.property("hash", null); - }); + //Match should be case-insensitive + expect(await vfs.getTags({like: "Foo"})).to.deep.equal([ + {name:"foo_tag", size: 1}, + {name: "tag_foo", size: 1}, + ]); + }); - it("can create a dummy file", async function(){ - let r = await vfs.createFile( {scene: "foo", mime: "text/html", name: "articles/foo.txt", user_id: null}, {hash: "xxxxxx", size: 150}); - }) + it("supports pagination", async function(){ + //Create a bunch of additional test scenes + for(let i=0; i < 3; i++){ + let id = await vfs.createScene(`test_${i}`); + for(let j=0; j <= i; j++ ){ + await vfs.addTag(id, `tag_${j}`); + } + } + expect(await vfs.getTags({limit: 1})).to.deep.equal([ + {name: "tag_0", size: 3}, + ]); + expect(await vfs.getTags({limit: 2, offset: 1})).to.deep.equal([ + {name: "tag_1", size: 2}, + {name: "tag_2", size: 1}, + ]); + expect(await vfs.getTags({offset: 3})).to.deep.equal([ ]); + }); - it("autoincrements generation", async function(){ - await vfs.createFile( {scene: "foo", mime: "text/html", name: "articles/foo.txt", user_id: null}, {hash: "xxxxxx", size: 150}); - let r = await vfs.createFile( {scene: "foo", mime: "text/html", name: "articles/foo.txt", user_id: null}, {hash: "yyyyy", size: 150}); - expect(r).to.have.property("generation", 2); - }) - it("can copy a file", async function(){ - let foo = await vfs.createFile( {scene: "foo", mime: "text/html", name: "articles/foo.txt", user_id: null}, {hash: "xxxxxx", size: 150}); - let bar = await vfs.createFile( {scene: "foo", mime: "text/html", name: "articles/bar.txt", user_id: null}, {hash: "xxxxxx", size: 150}); - expect(bar).to.have.property("id").not.equal(foo.id); - expect(bar).to.have.property("generation", 1); - expect(bar).to.have.property("hash", foo.hash); - expect(bar).to.have.property("size", foo.size); + it("don't count archived scenes", async function(){ + await expect(vfs.addTag(scene_id, "foo")).to.eventually.equal(true); + await vfs.archiveScene(scene_id); + expect(await vfs.getTags()).to.deep.equal([]); + }); }); - it("can use custom callbacks", async function(){ - let foo = await vfs.createFile( {scene: "foo", mime: "text/html", name: "articles/foo.txt", user_id: null}, ()=>Promise.resolve({hash: null, size: 150})); - expect(foo).to.have.property("hash", null); - expect(foo).to.have.property("size", 150); - - let bar = await vfs.createFile( {scene: "foo", mime: "text/html", name: "articles/bar.txt", user_id: null}, ()=>Promise.resolve({hash: "xxxxxx", size: 0})); - expect(bar).to.have.property("hash", "xxxxxx"); - expect(bar).to.have.property("size", 0); - }); + describe("getTag()", function(){ + it("Get all scenes attached to a tag", async function(){ + let ids = []; + for(let i=0; i < 3; i++){ + let id = await vfs.createScene(`test_${i}`); + ids.push(id); + await vfs.addTag(id, `tag_foo`); + } + for(let i=3; i < 6; i++){ + let id = await vfs.createScene(`test_${i}`); + await vfs.addTag(id, `tag_bar`); + } + let scenes = await vfs.getTag("tag_foo"); + expect(scenes).to.deep.equal(ids); + }); - it("Set scene type to html when a file named index.html is created", async function(){ - await vfs.createFile( {scene: "foo", mime: "text/html", name: "index.html", user_id: null}, {hash: "xxxxxx", size: 150}); - let scene = await vfs.getScene("foo"); - expect(scene.type).to.equal("html"); - }); + it("Ignore archived scenes", async function(){ + await expect(vfs.addTag(scene_id, "foo")).to.eventually.equal(true); - it("Set scene type to html when a file named scene.svx.json is created", async function(){ - await vfs.createFile( {scene: "foo", mime: "application/si-dpo-3d.document+json", name: "scene.svx.json", user_id: null}, {hash: "xxxxxx", size: 150}); - let scene = await vfs.getScene("foo"); - expect(scene.type).to.equal("voyager"); - }); + let s2 = await vfs.createScene(`test_scene_2`); + await expect(vfs.addTag(s2, "foo")).to.eventually.equal(true); + expect(await vfs.getTag("foo")).to.deep.equal([scene_id, s2]); - it("Voyager scene type overrides html scene types", async function(){ - await vfs.createFile( {scene: "foo", mime: "text/html", name: "index.html", user_id: null}, {hash: "xxxxxx", size: 150}); - let scene = await vfs.getScene("foo"); - expect(scene.type).to.equal("html"); - await vfs.createFile( {scene: "foo", mime: "application/si-dpo-3d.document+json", name: "scene.svx.json", user_id: null}, {hash: "xxxxxx", size: 150}); - scene = await vfs.getScene("foo"); - expect(scene.type).to.equal("voyager"); - }); - - it("html scene type does not override voyager scene type", async function(){ - await vfs.createFile( {scene: "foo", mime: "application/si-dpo-3d.document+json", name: "scene.svx.json", user_id: null}, {hash: "xxxxxx", size: 150}); - let scene = await vfs.getScene("foo"); - expect(scene.type).to.equal("voyager"); - await vfs.createFile( {scene: "foo", mime: "text/html", name: "index.html", user_id: null}, {hash: "xxxxxx", size: 150}); - scene = await vfs.getScene("foo"); - expect(scene.type).to.equal("voyager"); - }); - }); + await vfs.archiveScene(s2); + expect(await vfs.getTag("foo")).to.deep.equal([scene_id]); + }); - describe("writeFile()", function(){ - it("can upload a file (relative)", async function(){ - let r = await vfs.writeFile(dataStream(["foo","\n"]), {scene: "foo", mime: "text/html", name: "articles/foo.txt", user_id: null}); - expect(r).to.have.property("id").a("number"); - expect(r).to.have.property("generation", 1); - await expect(fs.access(path.join(this.dir, "objects", r.hash as any)), "can't access object file").to.be.fulfilled; - await expect(empty(this.uploads)); - }); - it("can upload a file (absolute)", async function(){ - let r = await expect( - vfs.writeFile(dataStream(["foo","\n"]), {scene: "foo", mime: "text/html", name: "articles/foo.txt", user_id: null}) - ).to.be.fulfilled; - expect(r).to.have.property("generation", 1); - expect(r).to.have.property("id").a("number"); + describe("respects permissions", function(){ + let userManager :UserManager, alice :User, bob :User; + this.beforeEach(async function(){ + userManager = new UserManager(vfs._db); + alice = await userManager.addUser("alice", "12345678", "admin"); + bob = await userManager.addUser("bob", "12345678", "create"); + }); - await expect(fs.access(path.join(this.dir, "objects", r.hash)), "can't access object file").to.be.fulfilled; - await expect(empty(this.uploads)); - }); - it("gets proper generation", async function(){ - await vfs.writeFile(dataStream(["foo","\n"]), {scene: "foo", mime: "text/html", name: "articles/foo.txt", user_id: null}); - for(let i=2; i < 5; i++){ - let foo = await vfs.writeFile(dataStream(["bar","\n"]), {scene: "foo", mime: "text/html", name: "articles/foo.txt", user_id: null}); - expect(foo).to.have.property("generation", i); - } - let bar = await vfs.writeFile(dataStream(["bar","\n"]), {scene: "foo", mime: "text/html", name: "articles/bar.txt", user_id: null}); - expect(bar).to.have.property("generation", 1); - }); - it("can upload over an existing file", async function(){ - await expect( - vfs.writeFile(dataStream(["foo","\n"]), {scene: "foo", mime: "text/html", name: "articles/foo.txt", user_id: null}) - ).to.eventually.have.property("generation", 1); - let r = await expect( - vfs.writeFile(dataStream(["bar","\n"]), {scene: "foo", mime: "text/html", name: "articles/foo.txt", user_id: null}) - ).to.be.fulfilled; - - expect(r).to.have.property("generation", 2); - await expect(fs.access(path.join(this.dir, "objects", r.hash)), "can't access object file").to.be.fulfilled; - await expect(empty(this.uploads)); - }); - - it("cleans up on errors", async function(){ - async function* badStream(){ - yield Promise.resolve(Buffer.from("foo")); - yield Promise.reject(new Error("CONNRESET")); - } - await expect(vfs.writeFile(badStream(), {scene: "foo", mime: "text/html", name: "articles/foo.txt", user_id: null})).to.be.rejectedWith("CONNRESET"); - await expect(fs.access(path.join(this.dir, "foo.txt")), "can't access foo.txt").to.be.rejectedWith("ENOENT"); - await expect(empty(this.uploads)); - }); - }); + it("return scenes with public read access", async function(){ + await vfs.addTag("foo", "foo"); + expect(await vfs.getTag("foo", alice.uid), "with admin user_id").to.deep.equal([scene_id]); + expect(await vfs.getTag("foo", bob.uid), "with normal user id").to.deep.equal([scene_id]); + }); + it("return all scenes for admins", async function(){ + //Scene if from bob, alice has no special rights over it, but should see it anyways + const id = await vfs.createScene("bob-private", bob.uid); + await userManager.setPublicAccess("bob-private", "none"); + await userManager.setDefaultAccess("bob-private", "none"); + await vfs.addTag("bob-private", "foo"); + expect(await vfs.getTag("foo"), "without user id").to.deep.equal([]); + expect(await vfs.getTag("foo", alice.uid), "with admin id").to.deep.equal([id]); + }) + + it("won't return non-readable scene", async function(){ + const id = await vfs.createScene("admin-only", alice.uid); + await userManager.setPublicAccess("admin-only", "none"); + await userManager.setDefaultAccess("admin-only", "none"); + await vfs.addTag("admin-only", "foo"); + expect(await vfs.getTag("foo"), "without user_id").to.deep.equal([]); + + expect(await vfs.getTag("foo", alice.uid), "with admin user_id").to.deep.equal([id]); + + expect(await vfs.getTag("foo", bob.uid)).to.deep.equal([]); + }); + }) + }); + }); describe("", function(){ - let r:FileProps, ctime :Date; - let props :GetFileParams = {scene: "foo", name: "articles/foo.txt"}; + let scene_id :number; + //Create a dummy scene for future tests this.beforeEach(async function(){ - r = await vfs.writeFile(dataStream(["foo","\n"]), {...props, mime: "text/html", user_id: null} ); - ctime = r.ctime; + scene_id = await vfs.createScene("foo"); }); - describe("getFileProps", function(){ - it("get a file properties", async function(){ - let r = await expect(vfs.getFileProps(props)).to.be.fulfilled; - expect(r).to.have.property("generation", 1); - expect(r).to.have.property("ctime").instanceof(Date); - expect(r).to.have.property("mtime").instanceof(Date); - expect(r.ctime.valueOf()).to.equal(ctime.valueOf()); - expect(r.mtime.valueOf()).to.equal(ctime.valueOf()); + + describe("renameScene()", function(){ + it("can change a scene name", async function(){ + await expect(vfs.renameScene(scene_id, "bar")).to.be.fulfilled; }); - it("uses the same format as writeFile", async function(){ - await expect(vfs.getFileProps(props)).to.eventually.deep.equal(r); - }) - it("get proper mtime and ctime", async function(){ - let mtime = new Date(Math.floor(Date.now())+100*1000); - let r = await vfs.writeFile(dataStream(["foo","\n"]), {...props, user_id: null}); - r = await expect(run(`UPDATE files SET ctime = $2 WHERE file_id = $1`, [ r.id, mtime.toISOString()])).to.be.fulfilled; - expect(r).to.have.property("changes", 1); - r = await expect(vfs.getFileProps(props)).to.be.fulfilled; - expect(r.ctime.valueOf()).to.equal(ctime.valueOf()); - expect(r.mtime.valueOf()).to.equal(mtime.valueOf()); - }); - - it("can use a scene ID", async function(){ - let r = await expect(vfs.getFileProps({...props, scene: scene_id})).to.be.fulfilled; - expect(r).to.have.property("name", props.name); - }) - - it("throw 404 error if file doesn't exist", async function(){ - await expect(vfs.getFileProps({...props, name: "bar.html"})).to.be.rejectedWith("404"); + it("throw a 404 error", async function(){ + await expect(vfs.renameScene(404, "bar")).to.be.rejectedWith("404"); }); + }) - it("get archived file", async function(){ - let id = await vfs.removeFile({...props, user_id: null}); - await expect(vfs.getFileProps(props), `File with id ${id} shouldn't be returned`).to.be.rejectedWith("[404]"); - await expect(vfs.getFileProps({...props, archive: true})).to.eventually.have.property("id", id); + describe("archiveScene()", function(){ + it("makes scene hidden", async function(){ + await vfs.archiveScene("foo"); + expect(await vfs.getScenes(0, {archived: false})).to.have.property("length", 0); }); - it("get by generation", async function(){ - let r = await vfs.writeFile(dataStream(["foo","\n"]), {...props, user_id: null}); - expect(r).to.have.property("generation", 2); - await expect(vfs.getFileProps({...props, generation: 2})).to.eventually.have.property("generation", 2); - await expect(vfs.getFileProps({...props, generation: 1})).to.eventually.have.property("generation", 1); + + it("can't archive twice", async function(){ + await vfs.archiveScene(scene_id); + await expect(vfs.archiveScene(scene_id), (await vfs._db.all("SELECT * FROM scenes"))[0].scene_name).to.be.rejectedWith(NotFoundError); }); - it("get archived by generation", async function(){ - await vfs.writeFile(dataStream(["foo","\n"]), {...props, user_id: null}); - let id = await vfs.removeFile({...props, user_id: null}); - await expect(vfs.getFileProps({...props, archive: true, generation: 3})).to.eventually.have.property("id", id); - await expect(vfs.getFileProps({...props, archive: true, generation: 3}, true)).to.eventually.have.property("id", id); + it("can remove archived scene (by id)", async function(){ + await vfs.archiveScene(scene_id); + await expect(vfs.removeScene(scene_id)).to.be.fulfilled; }); - it("get document", async function(){ - let {ctime:docCtime, ...doc} = await vfs.writeDoc("{}", {...props, user_id: null}); - await expect(vfs.getFileProps({...props, archive: true, generation: doc.generation}, true)).to.eventually.deep.equal({...doc, ctime, data: "{}"}); + it("can remove archived scene (by name)", async function(){ + await vfs.archiveScene(scene_id); + await expect(vfs.removeScene(`foo#${scene_id}`)).to.be.fulfilled; }); - + it("store archive time", async function(){ + //To be used later + await vfs.archiveScene(scene_id); + let {archived} = await vfs._db.get(`SELECT archived FROM scenes WHERE scene_id= $1`, [scene_id]); + expect(archived).to.be.instanceof(Date); + expect(archived.toString()).not.to.equal('Invalid Date'); + expect(archived.valueOf(), archived.toUTCString()).to.be.above(new Date().valueOf() - 2000); + expect(archived.valueOf(), archived.toUTCString()).to.be.below(new Date().valueOf()+1); + }) }); - describe("getFileBefore()", function(){ - - it("a file is \"before\" another", async function (){ - //Source file - let {id:expectedId} = r; - let {id: refId} = await vfs.writeDoc("", {scene: scene_id, name: "reference.txt", mime: "text/html", user_id: null}); - expect(refId).to.be.a("number"); + describe("unarchiveScene()", function(){ + it("restores an archived scene", async function(){ + await vfs.archiveScene(scene_id); + await vfs.unarchiveScene(`foo#${scene_id}`); + expect(await vfs.getScene("foo")).to.have.property("archived", null); + }); - //Ensure a matching file exists AFTER our reference - await vfs.writeDoc("", {...props, user_id: null}); + it("throws if archive doesn't exist", async function(){ + await expect(vfs.unarchiveScene("xxx")).to.be.rejectedWith(NotFoundError); + }) + }) - let f = await vfs.getFileBefore({...props, before: refId, scene: scene_id}); - expect(f).to.have.property("id", expectedId); + describe("createFile()", function(){ + it("can create an empty file", async function(){ + let r = await vfs.createFile( {scene: "foo", mime: "text/html", name: "articles/foo.txt", user_id: null}, {hash: null, size: 0}); + expect(r).to.have.property("id"); + expect(r).to.have.property("generation", 1); + expect(r).to.have.property("hash", null); }); - it("a file is \"before\" itself", async function (){ - //Ensure a matching file exists AFTER our reference - await vfs.writeDoc("", {...props, user_id: null}); + it("can create a dummy file", async function(){ + let r = await vfs.createFile( {scene: "foo", mime: "text/html", name: "articles/foo.txt", user_id: null}, {hash: "xxxxxx", size: 150}); + }) - let f = await vfs.getFileBefore({...props, before: r.id, scene: scene_id}); - expect(f).to.have.property("id", r.id); + it("autoincrements generation", async function(){ + await vfs.createFile( {scene: "foo", mime: "text/html", name: "articles/foo.txt", user_id: null}, {hash: "xxxxxx", size: 150}); + let r = await vfs.createFile( {scene: "foo", mime: "text/html", name: "articles/foo.txt", user_id: null}, {hash: "yyyyy", size: 150}); + expect(r).to.have.property("generation", 2); + }) + it("can copy a file", async function(){ + let foo = await vfs.createFile( {scene: "foo", mime: "text/html", name: "articles/foo.txt", user_id: null}, {hash: "xxxxxx", size: 150}); + let bar = await vfs.createFile( {scene: "foo", mime: "text/html", name: "articles/bar.txt", user_id: null}, {hash: "xxxxxx", size: 150}); + expect(bar).to.have.property("id").not.equal(foo.id); + expect(bar).to.have.property("generation", 1); + expect(bar).to.have.property("hash", foo.hash); + expect(bar).to.have.property("size", foo.size); }); - it("throws an error if reference file doesn't exist", async function(){ - await expect(vfs.getFileBefore({...props, before: -1, scene: scene_id})).to.be.rejectedWith(NotFoundError); + it("can use custom callbacks", async function(){ + let foo = await vfs.createFile( {scene: "foo", mime: "text/html", name: "articles/foo.txt", user_id: null}, ()=>Promise.resolve({hash: null, size: 150})); + expect(foo).to.have.property("hash", null); + expect(foo).to.have.property("size", 150); + + let bar = await vfs.createFile( {scene: "foo", mime: "text/html", name: "articles/bar.txt", user_id: null}, ()=>Promise.resolve({hash: "xxxxxx", size: 0})); + expect(bar).to.have.property("hash", "xxxxxx"); + expect(bar).to.have.property("size", 0); }); - it("throws an error if referenced file is from another scene", async function(){ - let name = randomBytes(6).toString("base64url"); - await vfs.createScene(name); - let {id} = await vfs.writeDoc("", {name: "foo.txt", scene: name, mime: "text/plain", user_id: null}); - await expect(vfs.getFileBefore({...props, before: id, scene: scene_id})).to.be.rejectedWith(NotFoundError); + it("Set scene type to html when a file named index.html is created", async function(){ + await vfs.createFile( {scene: "foo", mime: "text/html", name: "index.html", user_id: null}, {hash: "xxxxxx", size: 150}); + let scene = await vfs.getScene("foo"); + expect(scene.type).to.equal("html"); }); - it("throws if a a file was removed at the reference point", async function(){ - await vfs.removeFile({...props, user_id: null}); - - //The reference point - let {id: refId} = await vfs.writeDoc("", {scene: scene_id, name: "reference.txt", mime: "text/html", user_id: null}); - expect(refId).to.be.a("number"); - - await vfs.writeDoc("foo", {...props, user_id: null}); - - await expect(vfs.getFileBefore({...props, before: refId, scene: scene_id})).to.be.rejectedWith(NotFoundError); + it("Set scene type to html when a file named scene.svx.json is created", async function(){ + await vfs.createFile( {scene: "foo", mime: "application/si-dpo-3d.document+json", name: "scene.svx.json", user_id: null}, {hash: "xxxxxx", size: 150}); + let scene = await vfs.getScene("foo"); + expect(scene.type).to.equal("voyager"); }); - }); - describe("getFile()", function(){ - it("get a file", async function(){ - let {stream} = await vfs.getFile(props); - let str = ""; - for await (let d of stream!){ - str += d.toString("utf8"); - } - expect(str).to.equal("foo\n"); + it("Voyager scene type overrides html scene types", async function(){ + await vfs.createFile( {scene: "foo", mime: "text/html", name: "index.html", user_id: null}, {hash: "xxxxxx", size: 150}); + let scene = await vfs.getScene("foo"); + expect(scene.type).to.equal("html"); + await vfs.createFile( {scene: "foo", mime: "application/si-dpo-3d.document+json", name: "scene.svx.json", user_id: null}, {hash: "xxxxxx", size: 150}); + scene = await vfs.getScene("foo"); + expect(scene.type).to.equal("voyager"); }); + + it("html scene type does not override voyager scene type", async function(){ + await vfs.createFile( {scene: "foo", mime: "application/si-dpo-3d.document+json", name: "scene.svx.json", user_id: null}, {hash: "xxxxxx", size: 150}); + let scene = await vfs.getScene("foo"); + expect(scene.type).to.equal("voyager"); + await vfs.createFile( {scene: "foo", mime: "text/html", name: "index.html", user_id: null}, {hash: "xxxxxx", size: 150}); + scene = await vfs.getScene("foo"); + expect(scene.type).to.equal("voyager"); + }); + }); - it("get a document", async function(){ - //getFile can sometimes be used to get a stream to an existing document. Its shouldn't care and do it. - await vfs.writeDoc("Hello World\n", {...props, user_id: null}); - let {stream} = await vfs.getFile(props); - let str = ""; - for await (let d of stream!){ - expect(Buffer.isBuffer(d), `chunk is a ${typeof d}. Expected a buffer`).to.be.true; - str += d.toString("utf8"); - } - expect(str).to.equal("Hello World\n"); - }); - - it("get a range of a document", async function(){ - await vfs.writeDoc("Hello World\n", {...props, user_id: null}); - let start = 3; - let end = 7; - let {stream} = await vfs.getFile({...props,start,end}); - let str = ""; - for await (let d of stream!){ - expect(Buffer.isBuffer(d), `chunk is a ${typeof d}. Expected a buffer`).to.be.true; - str += d.toString("utf8"); - } - expect(str).to.equal("lo W"); + describe("writeFile()", function(){ + it("can upload a file (relative)", async function(){ + let r = await vfs.writeFile(dataStream(["foo","\n"]), {scene: "foo", mime: "text/html", name: "articles/foo.txt", user_id: null}); + expect(r).to.have.property("id").a("number"); + expect(r).to.have.property("generation", 1); + await expect(fs.access(path.join(this.dir, "objects", r.hash as any)), "can't access object file").to.be.fulfilled; + await expect(empty(this.uploads)); }); + it("can upload a file (absolute)", async function(){ + let r = await expect( + vfs.writeFile(dataStream(["foo","\n"]), {scene: "foo", mime: "text/html", name: "articles/foo.txt", user_id: null}) + ).to.be.fulfilled; + expect(r).to.have.property("generation", 1); + expect(r).to.have.property("id").a("number"); - it("get a document range of a document with NO end", async function(){ - await vfs.writeDoc("Hello World\n", {...props, user_id: null}); - let start = 3; - let {stream} = await vfs.getFile({...props,start}); - let str = ""; - for await (let d of stream!){ - expect(Buffer.isBuffer(d), `chunk is a ${typeof d}. Expected a buffer`).to.be.true; - str += d.toString("utf8"); + await expect(fs.access(path.join(this.dir, "objects", r.hash)), "can't access object file").to.be.fulfilled; + await expect(empty(this.uploads)); + }); + it("gets proper generation", async function(){ + await vfs.writeFile(dataStream(["foo","\n"]), {scene: "foo", mime: "text/html", name: "articles/foo.txt", user_id: null}); + for(let i=2; i < 5; i++){ + let foo = await vfs.writeFile(dataStream(["bar","\n"]), {scene: "foo", mime: "text/html", name: "articles/foo.txt", user_id: null}); + expect(foo).to.have.property("generation", i); } - expect(str).to.equal("lo World\n"); + let bar = await vfs.writeFile(dataStream(["bar","\n"]), {scene: "foo", mime: "text/html", name: "articles/bar.txt", user_id: null}); + expect(bar).to.have.property("generation", 1); }); + it("can upload over an existing file", async function(){ + await expect( + vfs.writeFile(dataStream(["foo","\n"]), {scene: "foo", mime: "text/html", name: "articles/foo.txt", user_id: null}) + ).to.eventually.have.property("generation", 1); + let r = await expect( + vfs.writeFile(dataStream(["bar","\n"]), {scene: "foo", mime: "text/html", name: "articles/foo.txt", user_id: null}) + ).to.be.fulfilled; - - it("get a document of a document with NO start", async function(){ - await vfs.writeDoc("Hello World\n", {...props, user_id: null}); - let end = 3; - let {stream} = await vfs.getFile({...props,end}); - let str = ""; - for await (let d of stream!){ - expect(Buffer.isBuffer(d), `chunk is a ${typeof d}. Expected a buffer`).to.be.true; - str += d.toString("utf8"); + expect(r).to.have.property("generation", 2); + await expect(fs.access(path.join(this.dir, "objects", r.hash)), "can't access object file").to.be.fulfilled; + await expect(empty(this.uploads)); + }); + + it("cleans up on errors", async function(){ + async function* badStream(){ + yield Promise.resolve(Buffer.from("foo")); + yield Promise.reject(new Error("CONNRESET")); } - expect(str).to.equal("Hel"); + await expect(vfs.writeFile(badStream(), {scene: "foo", mime: "text/html", name: "articles/foo.txt", user_id: null})).to.be.rejectedWith("CONNRESET"); + await expect(fs.access(path.join(this.dir, "foo.txt")), "can't access foo.txt").to.be.rejectedWith("ENOENT"); + await expect(empty(this.uploads)); }); + }); - it("get a document with end after end of file", async function(){ - await vfs.writeDoc("Hello World\n", {...props, user_id: null}); - let start = 3; - let end = 100; - let {stream} = await vfs.getFile({...props,start,end}); - let str = ""; - for await (let d of stream!){ - expect(Buffer.isBuffer(d), `chunk is a ${typeof d}. Expected a buffer`).to.be.true; - str += d.toString("utf8"); - } - expect(str).to.equal("lo World\n"); + + describe("", function(){ + let r:FileProps, ctime :Date; + let props :GetFileParams = {scene: "foo", name: "articles/foo.txt"}; + this.beforeEach(async function(){ + r = await vfs.writeFile(dataStream(["foo","\n"]), {...props, mime: "text/html", user_id: null} ); + ctime = r.ctime; }); + describe("getFileProps", function(){ + it("get a file properties", async function(){ + let r = await expect(vfs.getFileProps(props)).to.be.fulfilled; + expect(r).to.have.property("generation", 1); + expect(r).to.have.property("ctime").instanceof(Date); + expect(r).to.have.property("mtime").instanceof(Date); + expect(r.ctime.valueOf()).to.equal(ctime.valueOf()); + expect(r.mtime.valueOf()).to.equal(ctime.valueOf()); + }); + it("uses the same format as writeFile", async function(){ + await expect(vfs.getFileProps(props)).to.eventually.deep.equal(r); + }) + it("get proper mtime and ctime", async function(){ + let mtime = new Date(Math.floor(Date.now())+100*1000); + let r = await vfs.writeFile(dataStream(["foo","\n"]), {...props, user_id: null}); + r = await expect(run(`UPDATE files SET ctime = $2 WHERE file_id = $1`, [ r.id, mtime.toISOString()])).to.be.fulfilled; + expect(r).to.have.property("changes", 1); + r = await expect(vfs.getFileProps(props)).to.be.fulfilled; + expect(r.ctime.valueOf()).to.equal(ctime.valueOf()); + expect(r.mtime.valueOf()).to.equal(mtime.valueOf()); + }); - it("get a document with start after end of file", async function(){ - await vfs.writeDoc("Hello World\n", {...props, user_id: null}); - let start = 50; //getFile can sometimes be used to get a stream to an existing document. Its shouldn't care and do it. + it("can use a scene ID", async function(){ + let r = await expect(vfs.getFileProps({...props, scene: scene_id})).to.be.fulfilled; + expect(r).to.have.property("name", props.name); + }) - let end = 100; - let {stream} = await vfs.getFile({...props,start,end}); - let str = ""; - for await (let d of stream!){ - expect(Buffer.isBuffer(d), `chunk is a ${typeof d}. Expected a buffer`).to.be.true; - str += d.toString("utf8"); - } - expect(str).to.equal(""); - }); - - it("get a range of bytes of a document with start and end", async function(){ - // getFile can get start and end properties to read parts of a file - let start = 1; - let end = 3; - let {stream} = await vfs.getFile({...props, start, end}); - let str = ""; - for await (let d of stream!){ - expect(Buffer.isBuffer(d), `chunk is a ${typeof d}. Expected a buffer`).to.be.true; - str += d.toString("utf8"); - } - expect(str.length).to.equal(end-start); - expect(str).to.equal("oo"); - }); - - it("get a range of bytes of a document with start and NO end", async function(){ - // When getting only a start, getFile goes from start property to end of the file - let start = 1; - let {stream} = await vfs.getFile({...props, start}); - let str = ""; - for await (let d of stream!){ - expect(Buffer.isBuffer(d), `chunk is a ${typeof d}. Expected a buffer`).to.be.true; - str += d.toString("utf8"); - } - expect(str.length).to.equal("foo\n".length-start); - expect(str).to.equal("oo\n"); - }); - - it("get a range of bytes of a document with NO start and end", async function(){ - // When getting only an end, getFile goes from the start of the file to end property - let end = 2; - let {stream} = await vfs.getFile({...props, end}); - let str = ""; - for await (let d of stream!){ - expect(Buffer.isBuffer(d), `chunk is a ${typeof d}. Expected a buffer`).to.be.true; - str += d.toString("utf8"); - } - expect(str.length).to.equal(end); - expect(str).to.equal("fo"); - }); + it("throw 404 error if file doesn't exist", async function(){ + await expect(vfs.getFileProps({...props, name: "bar.html"})).to.be.rejectedWith("404"); + }); + it("get archived file", async function(){ + let id = await vfs.removeFile({...props, user_id: null}); + await expect(vfs.getFileProps(props), `File with id ${id} shouldn't be returned`).to.be.rejectedWith("[404]"); + await expect(vfs.getFileProps({...props, archive: true})).to.eventually.have.property("id", id); + }); - it("get a range of bytes of a document with end after end of file", async function(){ - let start = 1; - let end = 50; - let {stream} = await vfs.getFile({...props, start, end}); - let str = ""; - for await (let d of stream!){ - expect(Buffer.isBuffer(d), `chunk is a ${typeof d}. Expected a buffer`).to.be.true; - str += d.toString("utf8"); - } - expect(str).to.equal("oo\n"); - }); + it("get by generation", async function(){ + let r = await vfs.writeFile(dataStream(["foo","\n"]), {...props, user_id: null}); + expect(r).to.have.property("generation", 2); + await expect(vfs.getFileProps({...props, generation: 2})).to.eventually.have.property("generation", 2); + await expect(vfs.getFileProps({...props, generation: 1})).to.eventually.have.property("generation", 1); + }); - it("get a range of bytes of a document with start after end of file", async function(){ - let start = 20; - let end = 50; - let {stream} = await vfs.getFile({...props, start, end}); - let str = ""; - for await (let d of stream!){ - expect(Buffer.isBuffer(d), `chunk is a ${typeof d}. Expected a buffer`).to.be.true; - str += d.toString("utf8"); - } - expect(str).to.equal(""); - }); + it("get archived by generation", async function(){ + await vfs.writeFile(dataStream(["foo","\n"]), {...props, user_id: null}); + let id = await vfs.removeFile({...props, user_id: null}); + await expect(vfs.getFileProps({...props, archive: true, generation: 3})).to.eventually.have.property("id", id); + await expect(vfs.getFileProps({...props, archive: true, generation: 3}, true)).to.eventually.have.property("id", id); + }); + + it("get document", async function(){ + let {ctime:docCtime, ...doc} = await vfs.writeDoc("{}", {...props, user_id: null}); + await expect(vfs.getFileProps({...props, archive: true, generation: doc.generation}, true)).to.eventually.deep.equal({...doc, ctime, data: "{}"}); + }); - it("throw 404 error if file doesn't exist", async function(){ - await expect(vfs.getFile({...props, name: "bar.html"})).to.be.rejectedWith("404"); }); - it("throw 404 error if file was deleted", async function(){ - await vfs.removeFile({...props, user_id: null}); - await expect(vfs.getFile(props)).to.be.rejectedWith("404"); - }); + describe("getFileBefore()", function(){ - it("won't try to open a folder, just returns props", async function(){ - let file = await expect(vfs.getFile({scene: props.scene, name: "articles"})).to.be.fulfilled; - expect(file).to.have.property("mime", "text/directory"); - expect(file).to.not.have.property("stream"); - }); - }); + it("a file is \"before\" another", async function (){ + //Source file + let {id:expectedId} = r; + let {id: refId} = await vfs.writeDoc("", {scene: scene_id, name: "reference.txt", mime: "text/html", user_id: null}); + expect(refId).to.be.a("number"); - describe("getFileById()", function(){ - it("gets file props using its id", async function(){ - const {scene_id:stored_scene_id, data, ...file} = await vfs.getFileById(r.id); - expect(file).to.deep.equal(r); - expect(data).to.be.null; - expect(stored_scene_id).to.equal(scene_id); - }); + //Ensure a matching file exists AFTER our reference + await vfs.writeDoc("", {...props, user_id: null}); - it("gets a document's data using its id", async function(){ - let doc = await vfs.writeDoc("Hello!", {...props, user_id: null}); - const {scene_id:stored_scene_id, data, ...file} = await vfs.getFileById(doc.id); - expect(file.id).to.equal(doc.id); - expect(data).to.equal("Hello!"); - expect(stored_scene_id).to.equal(scene_id); - }); + let f = await vfs.getFileBefore({...props, before: refId, scene: scene_id}); + expect(f).to.have.property("id", expectedId); + }); - it("throws 404 if id doesn't map to a file", async function(){ - await expect(vfs.getFileById(-1)).to.be.rejectedWith(NotFoundError); - }); - }); - - describe("getFileHistory()", function(){ - it("get previous versions of a file", async function(){ - let r2 = await vfs.writeFile(dataStream(["foo2","\n"]), {...props, user_id: null} ); - let r3 = await vfs.writeFile(dataStream(["foo3","\n"]), {...props, user_id: null} ); - await vfs.writeFile(dataStream(["bar","\n"]), {...props, name:"bar", user_id: null} ); //another file - let versions = await vfs.getFileHistory(props); - let fileProps = await vfs.getFileProps(props); - //Expect reverse order - expect(versions.map(v=>v.generation)).to.deep.equal([3, 2, 1]); - versions.forEach((version, i)=>{ - expect(Object.keys(version).sort(),`Bad file properties at index ${i}`).to.deep.equal(Object.keys(fileProps).sort()) + it("a file is \"before\" itself", async function (){ + //Ensure a matching file exists AFTER our reference + await vfs.writeDoc("", {...props, user_id: null}); + + let f = await vfs.getFileBefore({...props, before: r.id, scene: scene_id}); + expect(f).to.have.property("id", r.id); }); - }); - it("works using a scene's name", async function(){ - await expect(vfs.getFileHistory({...props, scene: "foo"})).to.be.fulfilled; - }); - it("throw a 404 if file doesn't exist", async function(){ - await expect(vfs.getFileHistory({...props, name: "missing"})).to.be.rejectedWith("404"); - }); - it("throw a 404 if scene doesn't exist (by name)", async function(){ - await expect(vfs.getFileHistory({...props, scene: "missing"})).to.be.rejectedWith("404"); - }); - it("throw a 404 if scene doesn't exist (by id)", async function(){ - await expect(vfs.getFileHistory({...props, scene: scene_id+1})).to.be.rejectedWith("404"); - }); - }); - describe("removeFile()", function(){ - it("add an entry with state = REMOVED", async function(){ - await vfs.removeFile({...props, user_id: null}); - let files = await all(`SELECT * FROM files WHERE name = '${props.name}'`); - expect(files).to.have.property("length", 2); - expect(files[0]).to.include({ - hash: "tbudgBSg-bHWHiHnlteNzN8TUvI80ygS9IULh4rklEw", - generation: 1 + it("throws an error if reference file doesn't exist", async function(){ + await expect(vfs.getFileBefore({...props, before: -1, scene: scene_id})).to.be.rejectedWith(NotFoundError); }); - expect(files[1]).to.include({ - hash: null, - generation: 2 + + it("throws an error if referenced file is from another scene", async function(){ + let name = randomBytes(6).toString("base64url"); + await vfs.createScene(name); + let {id} = await vfs.writeDoc("", {name: "foo.txt", scene: name, mime: "text/plain", user_id: null}); + await expect(vfs.getFileBefore({...props, before: id, scene: scene_id})).to.be.rejectedWith(NotFoundError); }); - }); - it("requires the file to actually exist", async function(){ - await expect(vfs.removeFile({...props, name: "bar.txt", user_id: null})).to.be.rejectedWith("404"); - }); - it("require file to be in active state", async function(){ - await expect(vfs.removeFile({...props, user_id: null})).to.be.fulfilled, - await expect(vfs.removeFile({...props, user_id: null})).to.be.rejectedWith("already deleted"); - }); - }); - - describe("renameFile()", function(){ - it("rename a file", async function(){ - await vfs.renameFile({...props, user_id: null}, "bar.txt"); - await expect(vfs.getFileProps(props), "old file should not be reported anymore").to.be.rejectedWith("404"); - let file = await expect(vfs.getFileProps({...props, name: "bar.txt"})).to.be.fulfilled; - expect(file).to.have.property("mime", "text/html"); - }); + it("throws if a a file was removed at the reference point", async function(){ + await vfs.removeFile({...props, user_id: null}); - it("throw 404 error if scene doesn't exist", async function(){ - await expect(vfs.renameFile({...props, user_id: null, scene: "bar"}, "bar.txt")).to.be.rejectedWith("404"); + //The reference point + let {id: refId} = await vfs.writeDoc("", {scene: scene_id, name: "reference.txt", mime: "text/html", user_id: null}); + expect(refId).to.be.a("number"); + + await vfs.writeDoc("foo", {...props, user_id: null}); + + await expect(vfs.getFileBefore({...props, before: refId, scene: scene_id})).to.be.rejectedWith(NotFoundError); + }); }); + + describe("getFile()", function(){ + it("get a file", async function(){ + let {stream} = await vfs.getFile(props); + let str = ""; + for await (let d of stream!){ + str += d.toString("utf8"); + } + expect(str).to.equal("foo\n"); + }); + + it("get a document", async function(){ + //getFile can sometimes be used to get a stream to an existing document. Its shouldn't care and do it. + await vfs.writeDoc("Hello World\n", {...props, user_id: null}); + let {stream} = await vfs.getFile(props); + let str = ""; + for await (let d of stream!){ + expect(Buffer.isBuffer(d), `chunk is a ${typeof d}. Expected a buffer`).to.be.true; + str += d.toString("utf8"); + } + expect(str).to.equal("Hello World\n"); + }); + + it("get a range of a document", async function(){ + await vfs.writeDoc("Hello World\n", {...props, user_id: null}); + let start = 3; + let end = 7; + let {stream} = await vfs.getFile({...props,start,end}); + let str = ""; + for await (let d of stream!){ + expect(Buffer.isBuffer(d), `chunk is a ${typeof d}. Expected a buffer`).to.be.true; + str += d.toString("utf8"); + } + expect(str).to.equal("lo W"); + }); + + + it("get a document range of a document with NO end", async function(){ + await vfs.writeDoc("Hello World\n", {...props, user_id: null}); + let start = 3; + let {stream} = await vfs.getFile({...props,start}); + let str = ""; + for await (let d of stream!){ + expect(Buffer.isBuffer(d), `chunk is a ${typeof d}. Expected a buffer`).to.be.true; + str += d.toString("utf8"); + } + expect(str).to.equal("lo World\n"); + }); + - it("throw 404 error if file doesn't exist", async function(){ - await expect(vfs.renameFile({...props, user_id: null, name: "bar.html"}, "baz.html")).to.be.rejectedWith("404"); - }); + it("get a document of a document with NO start", async function(){ + await vfs.writeDoc("Hello World\n", {...props, user_id: null}); + let end = 3; + let {stream} = await vfs.getFile({...props,end}); + let str = ""; + for await (let d of stream!){ + expect(Buffer.isBuffer(d), `chunk is a ${typeof d}. Expected a buffer`).to.be.true; + str += d.toString("utf8"); + } + expect(str).to.equal("Hel"); + }); - it("throw 409 error if destination file already exist", async function(){ - await vfs.writeDoc("Hello World\n", {...props, user_id: null, name: "baz.txt"}); - await expect(vfs.renameFile({...props, user_id: null}, "baz.txt")).to.be.rejectedWith("409"); - }); - it("file can be created back after rename", async function(){ - await vfs.renameFile({...props, user_id: null}, "bar.txt"); - await vfs.writeFile(dataStream(["foo","\n"]), {...props, user_id: null} ); - await expect(vfs.getFileProps({...props, name: "bar.txt"})).to.be.fulfilled; - //Check if it doesn't mess with the history - let hist = await vfs.getFileHistory(props); - expect(hist.map(f=>f.hash)).to.deep.equal([ - "tbudgBSg-bHWHiHnlteNzN8TUvI80ygS9IULh4rklEw", - null, - "tbudgBSg-bHWHiHnlteNzN8TUvI80ygS9IULh4rklEw" - ]); + it("get a document with end after end of file", async function(){ + await vfs.writeDoc("Hello World\n", {...props, user_id: null}); + let start = 3; + let end = 100; + let {stream} = await vfs.getFile({...props,start,end}); + let str = ""; + for await (let d of stream!){ + expect(Buffer.isBuffer(d), `chunk is a ${typeof d}. Expected a buffer`).to.be.true; + str += d.toString("utf8"); + } + expect(str).to.equal("lo World\n"); + }); + + it("get a document with start after end of file", async function(){ + await vfs.writeDoc("Hello World\n", {...props, user_id: null}); + let start = 50; //getFile can sometimes be used to get a stream to an existing document. Its shouldn't care and do it. + + let end = 100; + let {stream} = await vfs.getFile({...props,start,end}); + let str = ""; + for await (let d of stream!){ + expect(Buffer.isBuffer(d), `chunk is a ${typeof d}. Expected a buffer`).to.be.true; + str += d.toString("utf8"); + } + expect(str).to.equal(""); + }); + + it("get a range of bytes of a document with start and end", async function(){ + // getFile can get start and end properties to read parts of a file + let start = 1; + let end = 3; + let {stream} = await vfs.getFile({...props, start, end}); + let str = ""; + for await (let d of stream!){ + expect(Buffer.isBuffer(d), `chunk is a ${typeof d}. Expected a buffer`).to.be.true; + str += d.toString("utf8"); + } + expect(str.length).to.equal(end-start); + expect(str).to.equal("oo"); + }); + + it("get a range of bytes of a document with start and NO end", async function(){ + // When getting only a start, getFile goes from start property to end of the file + let start = 1; + let {stream} = await vfs.getFile({...props, start}); + let str = ""; + for await (let d of stream!){ + expect(Buffer.isBuffer(d), `chunk is a ${typeof d}. Expected a buffer`).to.be.true; + str += d.toString("utf8"); + } + expect(str.length).to.equal("foo\n".length-start); + expect(str).to.equal("oo\n"); + }); + + it("get a range of bytes of a document with NO start and end", async function(){ + // When getting only an end, getFile goes from the start of the file to end property + let end = 2; + let {stream} = await vfs.getFile({...props, end}); + let str = ""; + for await (let d of stream!){ + expect(Buffer.isBuffer(d), `chunk is a ${typeof d}. Expected a buffer`).to.be.true; + str += d.toString("utf8"); + } + expect(str.length).to.equal(end); + expect(str).to.equal("fo"); + }); + + + it("get a range of bytes of a document with end after end of file", async function(){ + let start = 1; + let end = 50; + let {stream} = await vfs.getFile({...props, start, end}); + let str = ""; + for await (let d of stream!){ + expect(Buffer.isBuffer(d), `chunk is a ${typeof d}. Expected a buffer`).to.be.true; + str += d.toString("utf8"); + } + expect(str).to.equal("oo\n"); + }); + + it("get a range of bytes of a document with start after end of file", async function(){ + let start = 20; + let end = 50; + let {stream} = await vfs.getFile({...props, start, end}); + let str = ""; + for await (let d of stream!){ + expect(Buffer.isBuffer(d), `chunk is a ${typeof d}. Expected a buffer`).to.be.true; + str += d.toString("utf8"); + } + expect(str).to.equal(""); + }); + + + it("throw 404 error if file doesn't exist", async function(){ + await expect(vfs.getFile({...props, name: "bar.html"})).to.be.rejectedWith("404"); + }); + + it("throw 404 error if file was deleted", async function(){ + await vfs.removeFile({...props, user_id: null}); + await expect(vfs.getFile(props)).to.be.rejectedWith("404"); + }); + + it("won't try to open a folder, just returns props", async function(){ + let file = await expect(vfs.getFile({scene: props.scene, name: "articles"})).to.be.fulfilled; + expect(file).to.have.property("mime", "text/directory"); + expect(file).to.not.have.property("stream"); + }); }); - it("can move to a deleted file", async function(){ - await vfs.renameFile({...props, user_id: null}, "bar.txt"); - //move it back in place after it was deleted - await vfs.renameFile({...props, name: "bar.txt", user_id: null}, props.name); - let hist = await vfs.getFileHistory(props); - expect(hist.map(f=>`${f.name}#${f.generation}: ${f.hash}`)).to.deep.equal([ - `articles/foo.txt#3: tbudgBSg-bHWHiHnlteNzN8TUvI80ygS9IULh4rklEw`, - `articles/foo.txt#2: null`, - `articles/foo.txt#1: tbudgBSg-bHWHiHnlteNzN8TUvI80ygS9IULh4rklEw` - ]); - await expect(vfs.getFile({...props, name: "bar.txt"})).to.be.rejectedWith(NotFoundError); + + describe("getFileById()", function(){ + it("gets file props using its id", async function(){ + const {scene_id:stored_scene_id, data, ...file} = await vfs.getFileById(r.id); + expect(file).to.deep.equal(r); + expect(data).to.be.null; + expect(stored_scene_id).to.equal(scene_id); + }); + + it("gets a document's data using its id", async function(){ + let doc = await vfs.writeDoc("Hello!", {...props, user_id: null}); + const {scene_id:stored_scene_id, data, ...file} = await vfs.getFileById(doc.id); + expect(file.id).to.equal(doc.id); + expect(data).to.equal("Hello!"); + expect(stored_scene_id).to.equal(scene_id); + }); + + it("throws 404 if id doesn't map to a file", async function(){ + await expect(vfs.getFileById(-1)).to.be.rejectedWith(NotFoundError); + }); }); - it("can move in a folder", async function(){ - await vfs.renameFile({...props, user_id: null}, "articles/bar.txt"); - await expect(vfs.getFileProps(props)).to.be.rejectedWith(NotFoundError); - expect(await vfs.getFileProps({...props, name: "articles/bar.txt"})).to.have.property("hash", "tbudgBSg-bHWHiHnlteNzN8TUvI80ygS9IULh4rklEw"); + + describe("getFileHistory()", function(){ + it("get previous versions of a file", async function(){ + let r2 = await vfs.writeFile(dataStream(["foo2","\n"]), {...props, user_id: null} ); + let r3 = await vfs.writeFile(dataStream(["foo3","\n"]), {...props, user_id: null} ); + await vfs.writeFile(dataStream(["bar","\n"]), {...props, name:"bar", user_id: null} ); //another file + let versions = await vfs.getFileHistory(props); + let fileProps = await vfs.getFileProps(props); + //Expect reverse order + expect(versions.map(v=>v.generation)).to.deep.equal([3, 2, 1]); + versions.forEach((version, i)=>{ + expect(Object.keys(version).sort(),`Bad file properties at index ${i}`).to.deep.equal(Object.keys(fileProps).sort()) + }); + }); + it("works using a scene's name", async function(){ + await expect(vfs.getFileHistory({...props, scene: "foo"})).to.be.fulfilled; + }); + it("throw a 404 if file doesn't exist", async function(){ + await expect(vfs.getFileHistory({...props, name: "missing"})).to.be.rejectedWith("404"); + }); + it("throw a 404 if scene doesn't exist (by name)", async function(){ + await expect(vfs.getFileHistory({...props, scene: "missing"})).to.be.rejectedWith("404"); + }); + it("throw a 404 if scene doesn't exist (by id)", async function(){ + await expect(vfs.getFileHistory({...props, scene: scene_id+1})).to.be.rejectedWith("404"); + }); }); - it("can move a document", async function(){ - const props = {scene: scene_id, user_id: null, name:"foo.json", mime: "application/json"}; - let doc = await vfs.writeDoc("{}",props); - expect(doc).to.have.property("hash").ok; - await expect(vfs.renameFile(props, "bar.json")).to.be.fulfilled; - expect(await vfs.getFileProps({...props, name: "bar.json"})).to.have.property("hash", doc.hash); + describe("removeFile()", function(){ + it("add an entry with state = REMOVED", async function(){ + await vfs.removeFile({...props, user_id: null}); + let files = await all(`SELECT * FROM files WHERE name = '${props.name}'`); + expect(files).to.have.property("length", 2); + expect(files[0]).to.include({ + hash: "tbudgBSg-bHWHiHnlteNzN8TUvI80ygS9IULh4rklEw", + generation: 1 + }); + expect(files[1]).to.include({ + hash: null, + generation: 2 + }); + }); + it("requires the file to actually exist", async function(){ + await expect(vfs.removeFile({...props, name: "bar.txt", user_id: null})).to.be.rejectedWith("404"); + }); + it("require file to be in active state", async function(){ + await expect(vfs.removeFile({...props, user_id: null})).to.be.fulfilled, + await expect(vfs.removeFile({...props, user_id: null})).to.be.rejectedWith("already deleted"); + }); }); - }); - }) - + + describe("renameFile()", function(){ + + it("rename a file", async function(){ + await vfs.renameFile({...props, user_id: null}, "bar.txt"); + await expect(vfs.getFileProps(props), "old file should not be reported anymore").to.be.rejectedWith("404"); + let file = await expect(vfs.getFileProps({...props, name: "bar.txt"})).to.be.fulfilled; + expect(file).to.have.property("mime", "text/html"); + }); - describe("writeDoc()", function(){ - it("insert a new document using scene_id", async function(){ - await vfs.writeDoc("{}", {scene: scene_id, user_id: null, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); - await expect(all(`SELECT * FROM files WHERE name = 'scene.svx.json'`)).to.eventually.have.property("length", 1); - }) - it("insert a new document using scene_name", async function(){ - await vfs.writeDoc("{}", {scene: "foo", user_id: null, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); - await expect(all(`SELECT * FROM files WHERE name = 'scene.svx.json'`)).to.eventually.have.property("length", 1); - }) - it("requires a scene to exist", async function(){ - await expect(vfs.writeDoc("{}", {scene: 125 /*arbitrary non-existent scene id */, user_id: null, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"})).to.be.rejectedWith("404"); - await expect(all(`SELECT * FROM files WHERE fk_scene_id = 125`)).to.eventually.have.property("length", 0); - }); - it("can provide an author", async function(){ - let user_id = Uid.make(); - await get(`INSERT INTO users ( user_id, username ) VALUES ($1, 'alice')`, [user_id]); - await expect(vfs.writeDoc("{}", {scene: scene_id, user_id: user_id, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"})).to.be.fulfilled; - let files = await all(`SELECT data, fk_author_id AS fk_author_id FROM files WHERE name = 'scene.svx.json'`); - expect(files).to.have.length(1); - expect(files[0]).to.deep.equal({ - data: "{}", - fk_author_id: user_id, - }); - }); - it("updates scene's current doc", async function(){ - for(let i = 1; i<=3; i++){ - let id = (await vfs.writeDoc(`{"i":${i}}`, {scene: scene_id, user_id: null, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"})).id; - await expect(vfs.getDoc(scene_id)).to.eventually.deep.include({id}); - } - }); + it("throw 404 error if scene doesn't exist", async function(){ + await expect(vfs.renameFile({...props, user_id: null, scene: "bar"}, "bar.txt")).to.be.rejectedWith("404"); + }); + + it("throw 404 error if file doesn't exist", async function(){ + await expect(vfs.renameFile({...props, user_id: null, name: "bar.html"}, "baz.html")).to.be.rejectedWith("404"); + }); - it("reports byte size, not character size", async function(){ - let str = `{"id":"你好"}`; - expect(str.length).not.to.equal(Buffer.byteLength(str)); - const doc = await vfs.writeDoc(str, {scene: scene_id, user_id: null, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); - expect(doc).to.have.property("size", Buffer.byteLength(str)); - }) + it("throw 409 error if destination file already exist", async function(){ + await vfs.writeDoc("Hello World\n", {...props, user_id: null, name: "baz.txt"}); + await expect(vfs.renameFile({...props, user_id: null}, "baz.txt")).to.be.rejectedWith("409"); + }); - }); + it("file can be created back after rename", async function(){ + await vfs.renameFile({...props, user_id: null}, "bar.txt"); + await vfs.writeFile(dataStream(["foo","\n"]), {...props, user_id: null} ); + await expect(vfs.getFileProps({...props, name: "bar.txt"})).to.be.fulfilled; + //Check if it doesn't mess with the history + let hist = await vfs.getFileHistory(props); + expect(hist.map(f=>f.hash)).to.deep.equal([ + "tbudgBSg-bHWHiHnlteNzN8TUvI80ygS9IULh4rklEw", + null, + "tbudgBSg-bHWHiHnlteNzN8TUvI80ygS9IULh4rklEw" + ]); + }); + it("can move to a deleted file", async function(){ + await vfs.renameFile({...props, user_id: null}, "bar.txt"); + //move it back in place after it was deleted + await vfs.renameFile({...props, name: "bar.txt", user_id: null}, props.name); + let hist = await vfs.getFileHistory(props); + expect(hist.map(f=>`${f.name}#${f.generation}: ${f.hash}`)).to.deep.equal([ + `articles/foo.txt#3: tbudgBSg-bHWHiHnlteNzN8TUvI80ygS9IULh4rklEw`, + `articles/foo.txt#2: null`, + `articles/foo.txt#1: tbudgBSg-bHWHiHnlteNzN8TUvI80ygS9IULh4rklEw` + ]); + await expect(vfs.getFile({...props, name: "bar.txt"})).to.be.rejectedWith(NotFoundError); + }); + it("can move in a folder", async function(){ + await vfs.renameFile({...props, user_id: null}, "articles/bar.txt"); + await expect(vfs.getFileProps(props)).to.be.rejectedWith(NotFoundError); + expect(await vfs.getFileProps({...props, name: "articles/bar.txt"})).to.have.property("hash", "tbudgBSg-bHWHiHnlteNzN8TUvI80ygS9IULh4rklEw"); + }); - - describe("getScene()", function(){ - this.beforeEach(async function(){ - await vfs.writeDoc("{}", {scene: scene_id, user_id: null, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); - }); + it("can move a document", async function(){ + const props = {scene: scene_id, user_id: null, name:"foo.json", mime: "application/json"}; + let doc = await vfs.writeDoc("{}",props); + expect(doc).to.have.property("hash").ok; + await expect(vfs.renameFile(props, "bar.json")).to.be.fulfilled; + expect(await vfs.getFileProps({...props, name: "bar.json"})).to.have.property("hash", doc.hash); + }); + }); + }) + - it("throw an error if not found", async function(){ - await expect(vfs.getScene("bar")).to.be.rejectedWith("scene_name"); - }); + describe("writeDoc()", function(){ + it("insert a new document using scene_id", async function(){ + await vfs.writeDoc("{}", {scene: scene_id, user_id: null, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); + await expect(all(`SELECT * FROM files WHERE name = 'scene.svx.json'`)).to.eventually.have.property("length", 1); + }) + it("insert a new document using scene_name", async function(){ + await vfs.writeDoc("{}", {scene: "foo", user_id: null, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); + await expect(all(`SELECT * FROM files WHERE name = 'scene.svx.json'`)).to.eventually.have.property("length", 1); + }) + it("requires a scene to exist", async function(){ + await expect(vfs.writeDoc("{}", {scene: 125 /*arbitrary non-existent scene id */, user_id: null, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"})).to.be.rejectedWith("404"); + await expect(all(`SELECT * FROM files WHERE fk_scene_id = 125`)).to.eventually.have.property("length", 0); + }); + it("can provide an author", async function(){ + let user_id = Uid.make(); + await get(`INSERT INTO users ( user_id, username ) VALUES ($1, 'alice')`, [user_id]); + await expect(vfs.writeDoc("{}", {scene: scene_id, user_id: user_id, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"})).to.be.fulfilled; + let files = await all(`SELECT data, fk_author_id AS fk_author_id FROM files WHERE name = 'scene.svx.json'`); + expect(files).to.have.length(1); + expect(files[0]).to.deep.equal({ + data: "{}", + fk_author_id: user_id, + }); + }); + it("updates scene's current doc", async function(){ + for(let i = 1; i<=3; i++){ + let id = (await vfs.writeDoc(`{"i":${i}}`, {scene: scene_id, user_id: null, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"})).id; + await expect(vfs.getDoc(scene_id)).to.eventually.deep.include({id}); + } + }); - it("get a valid scene", async function(){ - let scene = await vfs.getScene("foo"); + it("reports byte size, not character size", async function(){ + let str = `{"id":"你好"}`; + expect(str.length).not.to.equal(Buffer.byteLength(str)); + const doc = await vfs.writeDoc(str, {scene: scene_id, user_id: null, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); + expect(doc).to.have.property("size", Buffer.byteLength(str)); + }) - let props = sceneProps(scene_id); - props.type = "voyager"; - let key:keyof Scene; - for(key in props){ - if(typeof props[key] ==="undefined"){ - expect(scene, `${(scene as any)[key]}`).not.to.have.property(key); - }else if(typeof props[key] === "function"){ - expect(scene).to.have.property(key).instanceof(props[key]); - }else{ - expect(scene).to.have.property(key).to.deep.equal(props[key]); - } - } }); - it("get an empty scene", async function(){ - let id = await vfs.createScene("empty"); - let scene = await vfs.getScene("empty"); - expect(scene).to.have.property("ctime").instanceof(Date); - expect(scene).to.have.property("mtime").instanceof(Date); - expect(scene).to.have.property("id", id).a("number"); - expect(scene).to.have.property("name", "empty"); - expect(scene).to.have.property("author", "default"); - }); + + describe("getScene()", function(){ + this.beforeEach(async function(){ + await vfs.writeDoc("{}", {scene: scene_id, user_id: null, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); + }); - it("get a scene's thumbnail if it exist (jpg)", async function(){ - await vfs.writeDoc("{}", {scene: scene_id, user_id: null, name: "scene-image-thumb.jpg", mime: "image/jpeg"}); - let s = await vfs.getScene(scene_id); - expect(s).to.have.property("thumb", "scene-image-thumb.jpg"); - }); + it("throw an error if not found", async function(){ + await expect(vfs.getScene("bar")).to.be.rejectedWith("scene_name"); + }); - it("get a scene's thumbnail if it exist (png)", async function(){ - await vfs.writeDoc("{}", {scene: scene_id, user_id: null, name: "scene-image-thumb.png", mime: "image/png"}); - let s = await vfs.getScene(scene_id); - expect(s).to.have.property("thumb", "scene-image-thumb.png"); - }); + it("get a valid scene", async function(){ + let scene = await vfs.getScene("foo"); + + let props = sceneProps(scene_id); + props.type = "voyager"; + let key:keyof Scene; + for(key in props){ + if(typeof props[key] ==="undefined"){ + expect(scene, `${(scene as any)[key]}`).not.to.have.property(key); + }else if(typeof props[key] === "function"){ + expect(scene).to.have.property(key).instanceof(props[key]); + }else{ + expect(scene).to.have.property(key).to.deep.equal(props[key]); + } + } + }); - it("get a scene's thumbnail if it exist (prioritized)", async function(){ - let times = [ - new Date("2022-01-01"), - new Date("2023-01-01"), - new Date("2024-01-01") - ]; - const setDate = (i:number, d:Date)=>vfs._db.run(`UPDATE files SET ctime = $2 WHERE file_id = $1`, [ i, d ]); - let png = await vfs.writeDoc("{}", {scene: scene_id, user_id: null, name: "scene-image-thumb.png", mime: "image/png"}); - let jpg = await vfs.writeDoc("{}", {scene: scene_id, user_id: null, name: "scene-image-thumb.jpg", mime: "image/jpeg"}); + it("get an empty scene", async function(){ + let id = await vfs.createScene("empty"); + let scene = await vfs.getScene("empty"); + expect(scene).to.have.property("ctime").instanceof(Date); + expect(scene).to.have.property("mtime").instanceof(Date); + expect(scene).to.have.property("id", id).a("number"); + expect(scene).to.have.property("name", "empty"); + expect(scene).to.have.property("author", "default"); + }); - let r = await setDate(jpg.id, times[1]); - await setDate(png.id, times[2]); - let s = await vfs.getScene(scene_id); - expect(s, `use PNG thumbnail if it's the most recent`).to.have.property("thumb", "scene-image-thumb.png"); + it("get a scene's thumbnail if it exist (jpg)", async function(){ + await vfs.writeDoc("{}", {scene: scene_id, user_id: null, name: "scene-image-thumb.jpg", mime: "image/jpeg"}); + let s = await vfs.getScene(scene_id); + expect(s).to.have.property("thumb", "scene-image-thumb.jpg"); + }); - await setDate(png.id, times[0]); - s = await vfs.getScene(scene_id); - expect(s, `use JPG thumbnail if it's the most recent`).to.have.property("thumb", "scene-image-thumb.jpg"); + it("get a scene's thumbnail if it exist (png)", async function(){ + await vfs.writeDoc("{}", {scene: scene_id, user_id: null, name: "scene-image-thumb.png", mime: "image/png"}); + let s = await vfs.getScene(scene_id); + expect(s).to.have.property("thumb", "scene-image-thumb.png"); + }); - //If date is equal, prioritize jpg - await setDate(png.id, times[1]); - s = await vfs.getScene(scene_id); - expect(s, `With equal dates, alphanumeric order shopuld prioritize JPG over PNG file`).to.have.property("thumb", "scene-image-thumb.jpg"); - }); + it("get a scene's thumbnail if it exist (prioritized)", async function(){ + let times = [ + new Date("2022-01-01"), + new Date("2023-01-01"), + new Date("2024-01-01") + ]; + const setDate = (i:number, d:Date)=>vfs._db.run(`UPDATE files SET ctime = $2 WHERE file_id = $1`, [ i, d ]); + let png = await vfs.writeDoc("{}", {scene: scene_id, user_id: null, name: "scene-image-thumb.png", mime: "image/png"}); + let jpg = await vfs.writeDoc("{}", {scene: scene_id, user_id: null, name: "scene-image-thumb.jpg", mime: "image/jpeg"}); + + let r = await setDate(jpg.id, times[1]); + await setDate(png.id, times[2]); + let s = await vfs.getScene(scene_id); + expect(s, `use PNG thumbnail if it's the most recent`).to.have.property("thumb", "scene-image-thumb.png"); + + await setDate(png.id, times[0]); + s = await vfs.getScene(scene_id); + expect(s, `use JPG thumbnail if it's the most recent`).to.have.property("thumb", "scene-image-thumb.jpg"); + + //If date is equal, prioritize jpg + await setDate(png.id, times[1]); + s = await vfs.getScene(scene_id); + expect(s, `With equal dates, alphanumeric order shopuld prioritize JPG over PNG file`).to.have.property("thumb", "scene-image-thumb.jpg"); + }); - it("get requester's access right", async function(){ - let userManager = new UserManager(vfs._db); - let alice = await userManager.addUser("alice", "xxxxxxxx", "create"); + it("get requester's access right", async function(){ + let userManager = new UserManager(vfs._db); + let alice = await userManager.addUser("alice", "xxxxxxxx", "create"); - let id = await vfs.createScene("alice's", alice.uid); - await vfs.writeDoc("{}", {scene: id, user_id: alice.uid, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); - let scene = await vfs.getScene("alice's", alice.uid); - expect(scene).to.have.property("access").to.equal("admin"); - }); + let id = await vfs.createScene("alice's", alice.uid); + await vfs.writeDoc("{}", {scene: id, user_id: alice.uid, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); + let scene = await vfs.getScene("alice's", alice.uid); + expect(scene).to.have.property("access").to.equal("admin"); + }); - it("get requester's group access right", async function(){ - let userManager = new UserManager(vfs._db); - let alice = await userManager.addUser("alice", "xxxxxxxx", "create"); - let group = await userManager.addGroup("My Group"); - await userManager.addMemberToGroup(alice.uid, group.groupUid); + it("get requester's group access right", async function(){ + let userManager = new UserManager(vfs._db); + let alice = await userManager.addUser("alice", "xxxxxxxx", "create"); + let group = await userManager.addGroup("My Group"); + await userManager.addMemberToGroup(alice.uid, group.groupUid); - let id = await vfs.createScene("foo2"); - await userManager.grantGroup(id, group.groupUid, "write"); + let id = await vfs.createScene("foo2"); + await userManager.grantGroup(id, group.groupUid, "write"); - await vfs.writeDoc("{}", {scene: id, user_id: alice.uid, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); - let scene = await vfs.getScene("foo2", alice.uid); - expect(scene).to.have.property("access").to.equal("write"); - }); + await vfs.writeDoc("{}", {scene: id, user_id: alice.uid, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); + let scene = await vfs.getScene("foo2", alice.uid); + expect(scene).to.have.property("access").to.equal("write"); + }); - it("performs requests for default user", async function(){ - let scene = await vfs.getScene("foo", 0); - expect(scene).to.be.ok; - expect(scene).to.have.property("access").to.equal("read"); + it("performs requests for default user", async function(){ + let scene = await vfs.getScene("foo", 0); + expect(scene).to.be.ok; + expect(scene).to.have.property("access").to.equal("read"); + }); }); - }); - describe("getSceneHistory()", function(){ - let default_folders = 2 - describe("get an ordered history", function(){ - this.beforeEach(async function(){ - let fileProps :WriteFileParams = {user_id: null, scene:scene_id, mime: "model/gltf-binary", name:"models/foo.glb"} - await vfs.writeFile(dataStream(), fileProps); - await vfs.writeDoc("{}", {scene: scene_id, user_id: null, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); - await vfs.writeFile(dataStream(), fileProps); - await vfs.writeDoc("{}", {scene: scene_id, user_id: null, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); - //Ensure all ctime are equal to prevent ordering issues - await vfs._db.run(`UPDATE files SET ctime = $1 WHERE fk_scene_id = $2`, [new Date(), scene_id]); - }); - - it("all events", async function(){ - let history = await vfs.getSceneHistory(scene_id); - expect(history).to.have.property("length", 4 + default_folders); - //Couln't easily test ctime sort - expect(history.map(e=>e.name)).to.deep.equal([ - "scene.svx.json", - "scene.svx.json", - "models/foo.glb", - "models/foo.glb", - "models", - "articles", - ]); - expect(history.map(e=>e.generation)).to.deep.equal([2,1,2,1,1,1]); + describe("getSceneHistory()", function(){ + let default_folders = 2 + describe("get an ordered history", function(){ + this.beforeEach(async function(){ + let fileProps :WriteFileParams = {user_id: null, scene:scene_id, mime: "model/gltf-binary", name:"models/foo.glb"} + await vfs.writeFile(dataStream(), fileProps); + await vfs.writeDoc("{}", {scene: scene_id, user_id: null, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); + await vfs.writeFile(dataStream(), fileProps); + await vfs.writeDoc("{}", {scene: scene_id, user_id: null, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); + //Ensure all ctime are equal to prevent ordering issues + await vfs._db.run(`UPDATE files SET ctime = $1 WHERE fk_scene_id = $2`, [new Date(), scene_id]); + }); + + it("all events", async function(){ + let history = await vfs.getSceneHistory(scene_id); + expect(history).to.have.property("length", 4 + default_folders); + //Couln't easily test ctime sort + expect(history.map(e=>e.name)).to.deep.equal([ + "scene.svx.json", + "scene.svx.json", + "models/foo.glb", + "models/foo.glb", + "models", + "articles", + ]); + expect(history.map(e=>e.generation)).to.deep.equal([2,1,2,1,1,1]); + }); + + it("with limit", async function(){ + let history = await vfs.getSceneHistory(scene_id, {limit: 1}); + expect(history).to.have.property("length", 1); + //Couln't easily test ctime sort + expect(history.map(e=>e.name)).to.deep.equal([ + "scene.svx.json", + ]); + expect(history.map(e=>e.generation)).to.deep.equal([2]); + }); + it("with offset", async function(){ + let history = await vfs.getSceneHistory(scene_id, {limit: 2, offset: 1}); + expect(history).to.have.property("length", 2); + //Couln't easily test ctime sort + expect(history.map(e=>e.name)).to.deep.equal([ + "scene.svx.json", + "models/foo.glb", + ]); + expect(history.map(e=>e.generation)).to.deep.equal([1,2]); + }); }); - - it("with limit", async function(){ - let history = await vfs.getSceneHistory(scene_id, {limit: 1}); - expect(history).to.have.property("length", 1); - //Couln't easily test ctime sort - expect(history.map(e=>e.name)).to.deep.equal([ - "scene.svx.json", + + it("supports pagination", async function(){ + for(let i=0; i < 20; i++){ + await vfs.writeDoc("{}", {scene: scene_id, user_id: null, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); + } + + let history = await vfs.getSceneHistory(scene_id, {limit: 2, offset: 0}); + expect(history.map(e=>e.generation)).to.deep.equal([ + 20, + 19, ]); - expect(history.map(e=>e.generation)).to.deep.equal([2]); - }); - it("with offset", async function(){ - let history = await vfs.getSceneHistory(scene_id, {limit: 2, offset: 1}); + history = await vfs.getSceneHistory(scene_id, {limit: 2, offset: 2}); expect(history).to.have.property("length", 2); - //Couln't easily test ctime sort - expect(history.map(e=>e.name)).to.deep.equal([ - "scene.svx.json", - "models/foo.glb", + expect(history.map(e=>e.generation)).to.deep.equal([ + 18, + 17, ]); - expect(history.map(e=>e.generation)).to.deep.equal([1,2]); }); }); - - it("supports pagination", async function(){ - for(let i=0; i < 20; i++){ - await vfs.writeDoc("{}", {scene: scene_id, user_id: null, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); - } - - let history = await vfs.getSceneHistory(scene_id, {limit: 2, offset: 0}); - expect(history.map(e=>e.generation)).to.deep.equal([ - 20, - 19, - ]); - history = await vfs.getSceneHistory(scene_id, {limit: 2, offset: 2}); - expect(history).to.have.property("length", 2); - expect(history.map(e=>e.generation)).to.deep.equal([ - 18, - 17, - ]); - }); - }); - - describe("getSceneMeta()", function() { - it("can get default meta (0) data", async function(){ - await vfs.writeDoc( JSON.stringify({ - metas: [ - {collection: - {titles: { - "EN": "English title", - "FR": "French title" - }} - } - ] - }) - , {scene: scene_id, user_id: null, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); - const meta = await vfs.getSceneMeta("foo"); - expect(meta.titles).to.be.deep.equal({ - "EN": "English title", - "FR": "French title" - }) - }); - it("can get non-default meta data", async function(){ - await vfs.writeDoc( JSON.stringify({ - scenes: [{meta: 1}], - metas: [ - {}, - {collection: - {titles: { - "EN": "English title", - "FR": "French title" - }} - } - ] - }) - , {scene: scene_id, user_id: null, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); - const meta = await vfs.getSceneMeta("foo"); - expect(meta.titles).to.deep.equal({ - "EN": "English title", - "FR": "French title" - }) - }); - - it("can get primary title and intros", async function(){ - await vfs.writeDoc( JSON.stringify( - { - scenes: [{meta: 1}], - setups: [ {language: {language: "FR"}}], - metas: [ - {}, - {collection: - {titles: { - "EN": "English title", - "FR": "French title" - }, - intros: { - "EN": "English intro", - "FR": "French intro" - }} - }] - }) - , {scene: scene_id, user_id: null, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); - const meta = await vfs.getSceneMeta("foo"); - expect(meta.primary_title).to.equal("French title"); - expect(meta.primary_intro).to.equal("French intro"); - }); - - }); + describe("getSceneMeta()", function() { + it("can get default meta (0) data", async function(){ + await vfs.writeDoc( JSON.stringify({ + metas: [ + {collection: + {titles: { + "EN": "English title", + "FR": "French title" + }} + } + ] + }) + , {scene: scene_id, user_id: null, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); + const meta = await vfs.getSceneMeta("foo"); + expect(meta.titles).to.be.deep.equal({ + "EN": "English title", + "FR": "French title" + }) + }); + + it("can get non-default meta data", async function(){ + await vfs.writeDoc( JSON.stringify({ + scenes: [{meta: 1}], + metas: [ + {}, + {collection: + {titles: { + "EN": "English title", + "FR": "French title" + }} + } + ] + }) + , {scene: scene_id, user_id: null, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); + const meta = await vfs.getSceneMeta("foo"); + expect(meta.titles).to.deep.equal({ + "EN": "English title", + "FR": "French title" + }) + }); + + it("can get primary title and intros", async function(){ + await vfs.writeDoc( JSON.stringify( + { + scenes: [{meta: 1}], + setups: [ {language: {language: "FR"}}], + metas: [ + {}, + {collection: + {titles: { + "EN": "English title", + "FR": "French title" + }, + intros: { + "EN": "English intro", + "FR": "French intro" + }} + }] + }) + , {scene: scene_id, user_id: null, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); + const meta = await vfs.getSceneMeta("foo"); + expect(meta.primary_title).to.equal("French title"); + expect(meta.primary_intro).to.equal("French intro"); + }); + + }); - describe("listFiles()", function(){ - let tref = new Date("2022-12-08T10:49:46.196Z"); + describe("listFiles()", function(){ + let tref = new Date("2022-12-08T10:49:46.196Z"); + + it("Get files created for a scene", async function(){ + let f1 = await vfs.writeFile(dataStream(), {user_id: null, scene:"foo", mime: "model/gltf-binary", name:"models/foo.glb"}); + let f2 = await vfs.writeFile(dataStream(), {user_id: null, scene:"foo", mime: "image/jpeg", name:"foo.jpg"}); + let d1 = await vfs.writeDoc('{}', {user_id: null, scene: "foo", mime: "application/si-dpo-3d.document+json", name: "scene.svx.json"}); + await run(`UPDATE files SET ctime = $1`, [tref.toISOString()]); + let files = await collapseAsync(vfs.listFiles(scene_id)); + expect(files).to.deep.equal([ + { + size: 4, + hash: 'tbudgBSg-bHWHiHnlteNzN8TUvI80ygS9IULh4rklEw', + generation: 1, + id: f2.id, + name: 'foo.jpg', + mime: "image/jpeg", + ctime: tref, + mtime: tref, + author_id: null, + author: "default", + },{ + size: 4, + hash: 'tbudgBSg-bHWHiHnlteNzN8TUvI80ygS9IULh4rklEw', + generation: 1, + id: f1.id, + name: 'models/foo.glb', + mime: "model/gltf-binary", + ctime: tref, + mtime: tref, + author_id: null, + author: "default", + }, + { + size: 2, + hash: "RBNvo1WzZ4oRRq0W9-hknpT7T8If536DEMBg9hyq_4o", + generation: 1, + id: d1.id, + mime: "application/si-dpo-3d.document+json", + name: "scene.svx.json", + ctime: tref, + mtime: tref, + author_id: null, + author: "default", + } + ]); + }); - it("Get files created for a scene", async function(){ - let f1 = await vfs.writeFile(dataStream(), {user_id: null, scene:"foo", mime: "model/gltf-binary", name:"models/foo.glb"}); - let f2 = await vfs.writeFile(dataStream(), {user_id: null, scene:"foo", mime: "image/jpeg", name:"foo.jpg"}); - let d1 = await vfs.writeDoc('{}', {user_id: null, scene: "foo", mime: "application/si-dpo-3d.document+json", name: "scene.svx.json"}); - await run(`UPDATE files SET ctime = $1`, [tref.toISOString()]); - let files = await collapseAsync(vfs.listFiles(scene_id)); - expect(files).to.deep.equal([ - { - size: 4, - hash: 'tbudgBSg-bHWHiHnlteNzN8TUvI80ygS9IULh4rklEw', - generation: 1, + it("Groups files versions", async function(){ + let tnext = new Date(tref.getTime()+8000); + let originalFiles = (await all("SELECT * FROM files")).length + let f1 = await vfs.writeFile(dataStream(["foo", "\n"]), {user_id: null, scene:"foo", mime: "model/gltf-binary", name:"models/foo.glb"}); + let del = await vfs.createFile({user_id: null, scene:"foo", mime: "model/gltf-binary", name:"models/foo.glb"}, {hash: null, size: 0}); + let f2 = await vfs.writeFile(dataStream(["hello world", "\n"]), {user_id: null, scene:"foo", mime: "model/gltf-binary", name:"models/foo.glb"}); + await expect(all("SELECT * FROM files")).to.eventually.have.property("length", 3+originalFiles); + await run(`UPDATE files SET ctime = $1 WHERE file_id = $2`, [tref.toISOString(), f1.id]); + await run(`UPDATE files SET ctime = $1 WHERE file_id = $2`, [tref.toISOString(), del.id]); + await run(`UPDATE files SET ctime = $1 WHERE file_id = $2`, [tnext.toISOString(), f2.id]); + + let files = await collapseAsync(vfs.listFiles(scene_id)); + expect(files).to.have.property("length", 1); + expect(files).to.deep.equal([{ + size: 12, + hash: 'qUiQTy8PR5uPgZdpSzAYSw0u0cHNKh7A-4XSmaGSpEc', + generation: 3, id: f2.id, - name: 'foo.jpg', - mime: "image/jpeg", - ctime: tref, - mtime: tref, - author_id: null, - author: "default", - },{ - size: 4, - hash: 'tbudgBSg-bHWHiHnlteNzN8TUvI80ygS9IULh4rklEw', - generation: 1, - id: f1.id, name: 'models/foo.glb', mime: "model/gltf-binary", ctime: tref, - mtime: tref, + mtime: tnext, author_id: null, author: "default", - }, - { - size: 2, - hash: "RBNvo1WzZ4oRRq0W9-hknpT7T8If536DEMBg9hyq_4o", - generation: 1, - id: d1.id, - mime: "application/si-dpo-3d.document+json", - name: "scene.svx.json", - ctime: tref, - mtime: tref, - author_id: null, - author: "default", - } - ]); - }); + }]); + }); - it("Groups files versions", async function(){ - let tnext = new Date(tref.getTime()+8000); - let originalFiles = (await all("SELECT * FROM files")).length - let f1 = await vfs.writeFile(dataStream(["foo", "\n"]), {user_id: null, scene:"foo", mime: "model/gltf-binary", name:"models/foo.glb"}); - let del = await vfs.createFile({user_id: null, scene:"foo", mime: "model/gltf-binary", name:"models/foo.glb"}, {hash: null, size: 0}); - let f2 = await vfs.writeFile(dataStream(["hello world", "\n"]), {user_id: null, scene:"foo", mime: "model/gltf-binary", name:"models/foo.glb"}); - await expect(all("SELECT * FROM files")).to.eventually.have.property("length", 3+originalFiles); - await run(`UPDATE files SET ctime = $1 WHERE file_id = $2`, [tref.toISOString(), f1.id]); - await run(`UPDATE files SET ctime = $1 WHERE file_id = $2`, [tref.toISOString(), del.id]); - await run(`UPDATE files SET ctime = $1 WHERE file_id = $2`, [tnext.toISOString(), f2.id]); + it("returns only files that are not removed", async function(){ + let props :WriteFileParams = {user_id: null, scene:"foo", mime: "model/gltf-binary", name:"models/foo.glb"} + let f1 = await vfs.writeFile(dataStream(), props); + await vfs.removeFile(props); + let files = await collapseAsync(vfs.listFiles(scene_id)); + expect(files).to.have.property("length", 0); + }); - let files = await collapseAsync(vfs.listFiles(scene_id)); - expect(files).to.have.property("length", 1); - expect(files).to.deep.equal([{ - size: 12, - hash: 'qUiQTy8PR5uPgZdpSzAYSw0u0cHNKh7A-4XSmaGSpEc', - generation: 3, - id: f2.id, - name: 'models/foo.glb', - mime: "model/gltf-binary", - ctime: tref, - mtime: tnext, - author_id: null, - author: "default", - }]); - }); + it("can get a list of archived files", async function(){ + await vfs.writeFile(dataStream(["foo", "\n"]), {user_id: null, scene: scene_id, mime: "text/html", name:"articles/hello.txt"}); + let del = await vfs.createFile({user_id: null, scene: scene_id, name:"articles/hello.txt"}, {hash: null, size: 0}); - it("returns only files that are not removed", async function(){ - let props :WriteFileParams = {user_id: null, scene:"foo", mime: "model/gltf-binary", name:"models/foo.glb"} - let f1 = await vfs.writeFile(dataStream(), props); - await vfs.removeFile(props); - let files = await collapseAsync(vfs.listFiles(scene_id)); - expect(files).to.have.property("length", 0); + let files = await collapseAsync(vfs.listFiles(scene_id, {withArchives: true})); + expect(files).to.have.property("length", 1); + expect(files[0]).to.have.property("hash", null); + expect(files[0]).to.have.property("id", del.id); + }); + + it("can get file data", async function(){ + await vfs.writeDoc(`{"foo":"bar"}`, {user_id: null, scene: scene_id, mime: "text/html", name: "foo.txt"}); + let files = await collapseAsync(vfs.listFiles(scene_id, {withData: true})); + expect(files).to.have.property("length", 1); + expect(files[0]).to.have.property("data", `{"foo":"bar"}`); + }); }); + + describe("getDoc()", function(){ + it("throw if not found", async function(){ + await expect(vfs.getDoc(scene_id)).to.be.rejectedWith("[404]"); + }); - it("can get a list of archived files", async function(){ - await vfs.writeFile(dataStream(["foo", "\n"]), {user_id: null, scene: scene_id, mime: "text/html", name:"articles/hello.txt"}); - let del = await vfs.createFile({user_id: null, scene: scene_id, name:"articles/hello.txt"}, {hash: null, size: 0}); + it("get document data as a string", async function(){ + await vfs.writeDoc(Buffer.from("{}"), {scene: scene_id, user_id: null, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); + let doc = await vfs.getDoc(scene_id); + expect(doc).to.have.property("data").a("string"); + }); - let files = await collapseAsync(vfs.listFiles(scene_id, {withArchives: true})); - expect(files).to.have.property("length", 1); - expect(files[0]).to.have.property("hash", null); - expect(files[0]).to.have.property("id", del.id); - }); + it("throws if file is not a document", async function(){ + await vfs.writeFile(dataStream(["{}"]), {scene: scene_id, user_id: null, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); + await expect(vfs.getDoc(scene_id)).to.be.rejectedWith(BadRequestError); + }) - it("can get file data", async function(){ - await vfs.writeDoc(`{"foo":"bar"}`, {user_id: null, scene: scene_id, mime: "text/html", name: "foo.txt"}); - let files = await collapseAsync(vfs.listFiles(scene_id, {withData: true})); - expect(files).to.have.property("length", 1); - expect(files[0]).to.have.property("data", `{"foo":"bar"}`); - }); - }); - - describe("getDoc()", function(){ - it("throw if not found", async function(){ - await expect(vfs.getDoc(scene_id)).to.be.rejectedWith("[404]"); + it("fetch currently active document", async function(){ + let id = (await vfs.writeDoc("{}", {scene: scene_id, user_id: null, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"})).id; + let doc = await expect(vfs.getDoc(scene_id)).to.be.fulfilled; + expect(doc).to.have.property("id", id); + expect(doc).to.have.property("ctime").instanceof(Date); + expect(doc).to.have.property("mtime").instanceof(Date); + expect(doc).to.have.property("author_id", null); + expect(doc).to.have.property("author", "default"); + expect(doc).to.have.property("data", "{}"); + expect(doc).to.have.property("generation", 1); + }); }); - it("get document data as a string", async function(){ - await vfs.writeDoc(Buffer.from("{}"), {scene: scene_id, user_id: null, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); - let doc = await vfs.getDoc(scene_id); - expect(doc).to.have.property("data").a("string"); + describe("cleanLooseObjects()", function(){ + it("remove old dangling blobs", async function(){ + let file = await vfs.writeFile(dataStream(["Hello World\n"]), {scene: scene_id, name: "foo.txt", mime: "text/plain", user_id: null}); + //lie about the file's mtime: it is old enough + await fs.utimes(vfs.filepath(file), new Date(Date.now() - 3600*1000*2), new Date(Date.now()- 3600*1000*3)); + await vfs.removeScene(scene_id); + //Blob should still be here + await expect(fs.access(vfs.filepath(file as any), constants.R_OK)).to.be.fulfilled; + let report = await vfs.cleanLooseObjects(); + expect(report).to.equal(`Cleaned 1 loose object`); + await expect(fs.access(vfs.filepath(file as any), constants.R_OK)).to.be.rejectedWith("ENOENT"); + }); }); - it("throws if file is not a document", async function(){ - await vfs.writeFile(dataStream(["{}"]), {scene: scene_id, user_id: null, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); - await expect(vfs.getDoc(scene_id)).to.be.rejectedWith(BadRequestError); + describe("checkForMissingObjects()", function(){ + it("reports missing blobs", async function(){ + let file = await vfs.writeFile(dataStream(["Hello World\n"]), {scene: scene_id, name: "foo.txt", mime: "text/plain", user_id: null}); + //force: false, so it throws if file is not here + await expect(fs.rm(vfs.getPath(file as any), {force: false})).to.be.fulfilled; + let report = await vfs.checkForMissingObjects(); + expect(report).to.equal("File 0qhPS4tlCTfsj3PNi-LHSt1akRumTfJ0WO2CKdqASiY can't be read on disk (can't fix). Some data have been lost!") + }); }) - - it("fetch currently active document", async function(){ - let id = (await vfs.writeDoc("{}", {scene: scene_id, user_id: null, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"})).id; - let doc = await expect(vfs.getDoc(scene_id)).to.be.fulfilled; - expect(doc).to.have.property("id", id); - expect(doc).to.have.property("ctime").instanceof(Date); - expect(doc).to.have.property("mtime").instanceof(Date); - expect(doc).to.have.property("author_id", null); - expect(doc).to.have.property("author", "default"); - expect(doc).to.have.property("data", "{}"); - expect(doc).to.have.property("generation", 1); - }); }); - - describe("cleanLooseObjects()", function(){ - it("remove old dangling blobs", async function(){ - let file = await vfs.writeFile(dataStream(["Hello World\n"]), {scene: scene_id, name: "foo.txt", mime: "text/plain", user_id: null}); - //lie about the file's mtime: it is old enough - await fs.utimes(vfs.filepath(file), new Date(Date.now() - 3600*1000*2), new Date(Date.now()- 3600*1000*3)); - await vfs.removeScene(scene_id); - //Blob should still be here - await expect(fs.access(vfs.filepath(file as any), constants.R_OK)).to.be.fulfilled; - let report = await vfs.cleanLooseObjects(); - expect(report).to.equal(`Cleaned 1 loose object`); - await expect(fs.access(vfs.filepath(file as any), constants.R_OK)).to.be.rejectedWith("ENOENT"); - }); - }); - - describe("checkForMissingObjects()", function(){ - it("reports missing blobs", async function(){ - let file = await vfs.writeFile(dataStream(["Hello World\n"]), {scene: scene_id, name: "foo.txt", mime: "text/plain", user_id: null}); - //force: false, so it throws if file is not here - await expect(fs.rm(vfs.getPath(file as any), {force: false})).to.be.fulfilled; - let report = await vfs.checkForMissingObjects(); - expect(report).to.equal("File 0qhPS4tlCTfsj3PNi-LHSt1akRumTfJ0WO2CKdqASiY can't be read on disk (can't fix). Some data have been lost!") - }); - }) }); }); -}); +}) From d76cb177bfd41aa772771ab0f0cc4435385e1912 Mon Sep 17 00:00:00 2001 From: Sebastien DUMETZ Date: Thu, 22 Jan 2026 12:02:57 +0100 Subject: [PATCH 02/14] refactor system initialization to be more readable with a separate create script --- source/server/create.ts | 63 +++++++++++++++++++++++++++++++++++ source/server/index.ts | 23 ++----------- source/server/routes/index.ts | 62 +++++++++------------------------- source/server/tests-common.ts | 7 ++-- source/server/utils/locals.ts | 2 ++ 5 files changed, 88 insertions(+), 69 deletions(-) create mode 100644 source/server/create.ts diff --git a/source/server/create.ts b/source/server/create.ts new file mode 100644 index 000000000..dcb457fe2 --- /dev/null +++ b/source/server/create.ts @@ -0,0 +1,63 @@ +import { debuglog } from "node:util"; + +import type express from "express"; + + +import UserManager from "./auth/UserManager.js"; +import { mkdir } from "fs/promises"; + +import openDatabase from "./vfs/helpers/db.js"; +import Vfs from "./vfs/index.js"; +import defaultConfig from "./utils/config.js"; +import createServer from "./routes/index.js"; + + +const debug = debuglog("pg:connect"); + +export interface Services{ + app: express.Application; + vfs: Vfs; + userManager: UserManager; + close: ()=>Promise; +} + +export default async function createService(config = defaultConfig) :Promise{ + + await Promise.all([config.files_dir].map(d=>mkdir(d, {recursive: true}))); + let db = await openDatabase({uri: config.database_uri, forceMigration: config.force_migration}); + let uri = new URL(config.database_uri); + debug(`Connected to database ${uri.hostname}:${uri.port}${uri.pathname}`) + const vfs = await Vfs.Open(config.files_dir, {db}); + const userManager = new UserManager(db); + + + + + if(config.clean_database){ + setTimeout(()=>{ + //Clean file system after a while to prevent delaying startup + vfs.clean().then(()=>console.log("Cleanup done."), e=> console.error("Cleanup failed :", e)); + }, 6000).unref(); + + + setInterval(()=>{ + vfs.optimize(); + }, 2*3600*1000).unref(); + } + + const app = await createServer({ + userManager, + fileDir: config.files_dir, + vfs, + config, + }); + + return { + app, + vfs, + userManager, + async close(){ + await vfs.close(); + } + }; +} diff --git a/source/server/index.ts b/source/server/index.ts index 41b817d87..f710bc7f1 100644 --- a/source/server/index.ts +++ b/source/server/index.ts @@ -1,22 +1,5 @@ -/** - * 3D Foundation Project - * Copyright 2019 Smithsonian Institution - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - import path from "path"; -import createServer from "./routes/index.js"; +import createService from "./create.js"; import config from "./utils/config.js"; //@ts-ignore @@ -28,8 +11,8 @@ import("source-map-support").then((s)=>{ (async ()=>{ let root = path.resolve(config.root_dir); console.info("Serve directory : "+root+" on "+config.port); - const app = await createServer(config); - app.listen(config.port, () => { + const services = await createService(config); + services.app.listen(config.port, () => { console.info(`Server ready and listening on port ${config.port}\n`); }); })(); diff --git a/source/server/routes/index.ts b/source/server/routes/index.ts index dea84de75..f11e37276 100644 --- a/source/server/routes/index.ts +++ b/source/server/routes/index.ts @@ -1,68 +1,38 @@ -import path from "path"; -import util, { debuglog } from "util"; +import { debuglog } from "util"; import cookieSession from "cookie-session"; -import express, { Request, Response } from "express"; +import express from "express"; -import UserManager from "../auth/UserManager.js"; -import { BadRequestError, HTTPError, UnauthorizedError } from "../utils/errors.js"; +import { HTTPError, UnauthorizedError } from "../utils/errors.js"; import { errorHandlerMdw, LogLevel, notFoundHandlerMdw } from "../utils/errorHandler.js"; -import { mkdir } from "fs/promises"; -import {AppLocals, getHost, getLocals, getUser, getUserManager, isUser} from "../utils/locals.js"; +import {AppLocals, AppParameters, getLocals, getUserManager} from "../utils/locals.js"; -import openDatabase from "../vfs/helpers/db.js"; -import Vfs from "../vfs/index.js"; -import defaultConfig from "../utils/config.js"; import User from "../auth/User.js"; -import Templates, { locales } from "../utils/templates.js"; +import Templates from "../utils/templates.js"; const debug = debuglog("pg:connect"); -export default async function createServer(config = defaultConfig) :Promise{ +export default async function createServer(locals:AppParameters) :Promise{ - await Promise.all([config.files_dir].map(d=>mkdir(d, {recursive: true}))); - let db = await openDatabase({uri: config.database_uri, forceMigration: config.force_migration}); - let uri = new URL(config.database_uri); - debug(`Connected to database ${uri.hostname}:${uri.port}${uri.pathname}`) - const vfs = await Vfs.Open(config.files_dir, {db}); - const userManager = new UserManager(db); - - const templates = new Templates({dir: config.templates_dir, cache: config.node_env == "production"}); + const templates = new Templates({dir: locals.config.templates_dir, cache: locals.config.node_env == "production"}); const app = express(); app.disable('x-powered-by'); - app.set("trust proxy", config.trust_proxy); - - if(config.clean_database){ - setTimeout(()=>{ - //Clean file system after a while to prevent delaying startup - vfs.clean().then(()=>console.log("Cleanup done."), e=> console.error("Cleanup failed :", e)); - }, 6000).unref(); - - - setInterval(()=>{ - vfs.optimize(); - }, 2*3600*1000).unref(); - } - + app.set("trust proxy", locals.config.trust_proxy); app.locals = Object.assign(app.locals, { - userManager, - fileDir: config.files_dir, - vfs, - templates, - config, sessionMaxAge: 31 * 24 * 60 * 60*1000, // 1 month, in milliseconds - }) as AppLocals; + templates, + }, locals) as AppLocals; app.use(cookieSession({ name: 'session', - keys: await userManager.getKeys(), + keys: await locals.userManager.getKeys(), // Cookie Options maxAge: (app.locals as AppLocals).sessionMaxAge, sameSite: "lax" @@ -112,7 +82,7 @@ export default async function createServer(config = defaultConfig) :Promise={}){ - let {default:createServer} = await import("./routes/index.js"); + let {default:createService} = await import("./create.js"); let titleSlug = "t_"+ (c.currentTest?.title.replace(/[^\w]/g, "_") ?? `eCorpus_integration`)+"_"+randomBytes(4).toString("hex"); c.db_uri = await getUniqueDb(titleSlug); c.dir = await fs.mkdtemp(path.join(tmpdir(), titleSlug)); @@ -83,12 +83,13 @@ global.createIntegrationContext = async function(c :Mocha.Context, config_overri //Options we might want to customize config_override ); - c.server = await createServer( c.config ); + c.services = await createService( c.config ); + c.server = c.services.app; return c.server.locals; } global.cleanIntegrationContext = async function(c :Mocha.Context){ - await c.server.locals.vfs.close(); + await c.services.close(); await dropDb(c.db_uri); if(c.dir) await fs.rm(c.dir, {recursive: true}); } \ No newline at end of file diff --git a/source/server/utils/locals.ts b/source/server/utils/locals.ts index 87e6200d0..c6185f0a0 100644 --- a/source/server/utils/locals.ts +++ b/source/server/utils/locals.ts @@ -20,6 +20,8 @@ export interface AppLocals extends Record{ sessionMaxAge: number; } +export type AppParameters = Omit; + export function getLocals(req :Request){ return req.app.locals as AppLocals; } From ed3ab9c938eebe1a0cf8b247e241f9982e2f1ad5 Mon Sep 17 00:00:00 2001 From: Sebastien DUMETZ Date: Thu, 22 Jan 2026 12:21:02 +0100 Subject: [PATCH 03/14] server utilities: filetypes probing, HTTP error codes, and UserManager.getUserById --- source/server/auth/UserManager.ts | 13 +++- .../routes/scenes/scene/files/get/file.ts | 6 +- source/server/utils/errors.ts | 23 +++++- source/server/utils/filetypes.test.ts | 77 +++++++++++++++++- source/server/utils/filetypes.ts | 78 ++++++++++++++----- 5 files changed, 168 insertions(+), 29 deletions(-) diff --git a/source/server/auth/UserManager.ts b/source/server/auth/UserManager.ts index 13f50b9aa..945527967 100644 --- a/source/server/auth/UserManager.ts +++ b/source/server/auth/UserManager.ts @@ -180,13 +180,20 @@ export default class UserManager extends DbController { ]); } + /** + * Get a user using its unique identifier + * @throws {NotFoundError} if user_id is not found + */ + async getUserById(user_id: number) :Promise{ + let u = await this.db.get(`SELECT * FROM users WHERE user_id = $1`, [ user_id ]); + if(!u) throw new NotFoundError(`no user with user_id ${user_id}`); + return UserManager.deserialize(u); + } /** - * Reads users file and checks users validity. - * Also allow requests by email + * Also allow requests by username or email * @throws {BadRequestError} is username is invalid * @throws {NotFoundError} is username is not found - * @throws {Error} if fs.readFile fails (generally with error.code == ENOENT) */ async getUserByName(username : string) :Promise{ if(!UserManager.isValidUserName(username) && username.indexOf("@")== -1) throw new BadRequestError(`Invalid user name`); diff --git a/source/server/routes/scenes/scene/files/get/file.ts b/source/server/routes/scenes/scene/files/get/file.ts index 097ac499a..027383478 100644 --- a/source/server/routes/scenes/scene/files/get/file.ts +++ b/source/server/routes/scenes/scene/files/get/file.ts @@ -2,7 +2,7 @@ import {pipeline} from "node:stream/promises"; import { Request, Response } from "express"; import { getVfs, getFileParams } from "../../../../../utils/locals.js"; -import { BadRequestError, RangeNotSatisfiable} from "../../../../../utils/errors.js"; +import { BadRequestError, RangeNotSatisfiableError} from "../../../../../utils/errors.js"; async function handleGetFileRange(req :Request, res :Response){ const vfs = getVfs(req); @@ -24,9 +24,9 @@ async function handleGetFileRange(req :Request, res :Response){ if (end && end > file.size){ res.set("Content-Range", "bytes */" + file.size); if(startRange.length > 0){ - throw new RangeNotSatisfiable("Range Not Satisfiable: end after end of file") + throw new RangeNotSatisfiableError("Range Not Satisfiable: end after end of file") }else{ - throw new RangeNotSatisfiable("Range Not Satisfiable: Suffix-length is bigger than lenght of file") + throw new RangeNotSatisfiableError("Range Not Satisfiable: Suffix-length is bigger than lenght of file") } } diff --git a/source/server/utils/errors.ts b/source/server/utils/errors.ts index 9edcf76fd..753b15066 100644 --- a/source/server/utils/errors.ts +++ b/source/server/utils/errors.ts @@ -1,6 +1,3 @@ -import util from "node:util"; -import { NextFunction, Request, Response } from "express"; -import { useTemplateProperties } from "./locals.js"; export class HTTPError extends Error{ @@ -32,13 +29,25 @@ export class NotFoundError extends HTTPError { } } +export class MethodNotAllowedError extends HTTPError{ + constructor(reason :string = "Method Not Allowed"){ + super(405, reason); + } +} + export class ConflictError extends HTTPError { constructor(reason :string="Conflict"){ super(409, reason); } } -export class RangeNotSatisfiable extends HTTPError { +export class LengthRequiredError extends HTTPError{ + constructor(reason: string = "Length Required"){ + super(411, reason); + } +} + +export class RangeNotSatisfiableError extends HTTPError { constructor(reason :string="Range Not Satisfiable"){ super(416, reason); } @@ -49,3 +58,9 @@ export class InternalError extends HTTPError { super(500, reason); } } + +export class NotImplementedError extends HTTPError{ + constructor(reason :string="Not Implemented"){ + super(501, reason); + } +} \ No newline at end of file diff --git a/source/server/utils/filetypes.test.ts b/source/server/utils/filetypes.test.ts index c00a7c468..d399a5fa3 100644 --- a/source/server/utils/filetypes.test.ts +++ b/source/server/utils/filetypes.test.ts @@ -1,8 +1,11 @@ +import path from "node:path"; import express from "express"; -import { compressedMime, getContentType, getMimeType } from "./filetypes.js"; +import { compressedMime, extFromType, getContentType, getMimeType, parseMagicBytes, readMagicBytes } from "./filetypes.js"; import request from "supertest"; +import { fixturesDir } from "../__test_fixtures/fixtures.js"; + describe("getMimeType",function(){ //Check proper operation of mimetype guessing function //This is mostly out of our hands, but if types we tend to rely on do change, we want to know! @@ -93,4 +96,76 @@ describe("compressedMime", function(){ expect(compressedMime(t)).to.be.true; }); }); +}); + + +describe("extFromType()", function(){ + + Object.entries({ + "model/gltf-binary": ".glb", + "image/jpeg": ".jpeg", + "application/zip": ".zip", + "application/si-dpo-3d.document+json": ".svx.json", + }).forEach(function([type, extname]){ + it(`${type} => ${extname}`, function(){ + expect(extFromType(type)).to.equal(extname); + }); + }); + + it("returns an empty string for extensionless files", function(){ + expect(extFromType("application/octet-stream")).to.equal(""); + }); +}); + +describe("parseMagicBytes()", function(){ + //Header of a 1156x420 png + //Header of a 1067x800 jpeg + it("image/png", function(){ + expect(parseMagicBytes(Buffer.from('89504e470d0a1a0a', "hex"))).to.equal("image/png"); + }); + + it("image/jpeg", function(){ + [ + "FFD8FFDB", //Raw jpeg + "FFD8FFE000104A4649460001", //JFIF + "FFD8FFEE", //Also jpeg + "FFD8FFE0", // still jpeg + ].forEach((str)=>{ + expect(parseMagicBytes(Buffer.from(str, "hex")), `0x${str} should be a valid jpeg header`).to.equal("image/jpeg") + }) + }); + it("image/webp", function(){ + //Header with a size of 0 + expect(parseMagicBytes(Buffer.from('524946460000000057454250', 'hex'))).to.equal("image/webp"); + //header with a real size + expect(parseMagicBytes(Buffer.from('52494646b254000057454250', 'hex'))).to.equal("image/webp"); + //header from another RIFF container + expect(parseMagicBytes(Buffer.from('524946460000000057415645', 'hex'))).to.equal("application/octet-stream"); + }); + + it("model/gltf-binary", function(){ + // See https://registry.khronos.org/glTF/specs/2.0/glTF-2.0.html#binary-header + // maps to glTF ascii string, but backwards, because it's a little-endian format + const b= Buffer.alloc(4); + b.writeUint32LE(0x46546C67); + expect(parseMagicBytes(b)).to.equal("model/gltf-binary"); + }); + + it("application/zip", function(){ + // See https://en.wikipedia.org/wiki/ZIP_(file_format)#File_headers + const b = Buffer.alloc(4); + b.writeUint32LE(0x04034b50); + expect(parseMagicBytes(b)).to.equal("application/zip"); + }); +}); + +describe("readMagicBytes()", function(){ + it("reads first bytes of a glb file", async function(){ + let type = await readMagicBytes(path.join(fixturesDir, "cube.glb")); + expect(type).to.equal("model/gltf-binary"); + }); + + it("throws if file doesn't exist", async function(){ + await expect(readMagicBytes(path.join(fixturesDir, "not_cube.glb"))).to.be.rejectedWith({code: "ENOENT"} as any); + }); }); \ No newline at end of file diff --git a/source/server/utils/filetypes.ts b/source/server/utils/filetypes.ts index c9808a4b6..1fcf0afe0 100644 --- a/source/server/utils/filetypes.ts +++ b/source/server/utils/filetypes.ts @@ -1,21 +1,10 @@ -import path from 'path'; +import fs from 'fs/promises'; + import {Request} from "express"; -import {lookup, types} from "mime-types"; - -const mimeMap :Record = { - ".jpg": "image.jpeg", - ".jpeg": "image.jpeg", - ".png": "image/png", - ".webp": "image/webp", - ".mp4": "video/mp4", - ".glb": "model/gltf-binary", - ".htm": "text/html", - ".html": "text/html", - ".txt": "text/plain", - ".json": "application/json", - ".svx.json":"application/si-dpo-3d.document+json", -} -const mimeList = Object.values(mimeMap); +import {lookup, types, extension, extensions} from "mime-types"; + +//Add non-standard extension +extensions["application/si-dpo-3d.document+json"] = ["svx.json"]; export function getMimeType(name:string){ //Special case because extname doesn't recognize double-extensions @@ -42,4 +31,57 @@ export function compressedMime(mime :string) :boolean{ if(mime.endsWith("zip")) return false; //A whole lot of things are application/**+zip } return true; -} \ No newline at end of file +} + +/** + * Returns standard extension with a leading dot for a given mime type + * Returns an empty string for extensionless types (eg: application/octet-stream) + */ +export function extFromType(type: string){ + let ext = extension(type); + if(ext === "bin") return ""; + return `.${ext}`; +} + +export async function readMagicBytes(filepath: string): Promise{ + let handle = await fs.open(filepath, fs.constants.O_RDONLY); + try{ + const b = Buffer.allocUnsafe(12); + const {bytesRead} = await handle.read({buffer: b}); + return parseMagicBytes(b.subarray(0, bytesRead)); + }finally{ + await handle.close(); + } +} + +/** + * Seek the first bytes of a file to find type signatures. + * Most files have 4 bytes signatures. + * RIFF containers (eg: webp) needs at least 12 bytes of content + * @see {@link https://en.wikipedia.org/wiki/List_of_file_signatures File Signatures } + */ +export function parseMagicBytes(src: Buffer|Uint8Array) :string{ + if(src[0] == 0x89 && src.subarray(0, 8).toString("hex") == "89504e470d0a1a0a"){ + return "image/png"; + } + + if(src[0] == 0xFF && src.subarray(0, 3).toString("hex") == "ffd8ff"){ + return "image/jpeg"; + } + + if(src[0] == 0x52 && src.subarray(0, 4).toString("ascii") == "RIFF"){ + //RIFF files. Next 4 bytes are for the file size. + let sig = src.subarray(8, 12).toString(); + if(sig == "WEBP") return "image/webp"; + } + + if(src[0] == 0x67 && src.subarray(0, 4).toString("ascii") == "glTF"){ + return "model/gltf-binary"; + } + + if(src[0] == 0x50 && src.subarray(0, 4).toString("hex") == "504b0304"){ + return "application/zip"; + } + + return "application/octet-stream"; +} From 5b1f8aac1d866d7e0be66d3b30e4d508c91eb379 Mon Sep 17 00:00:00 2001 From: Sebastien DUMETZ Date: Thu, 22 Jan 2026 11:44:01 +0100 Subject: [PATCH 04/14] task processing system with scheduler integration --- source/server/create.ts | 7 +- source/server/migrations/005-tasks.sql | 56 ++++++++++ source/server/tasks/errors.test.ts | 85 ++++++++++++++ source/server/tasks/errors.ts | 35 ++++++ source/server/tasks/logger.ts | 38 +++++++ source/server/tasks/manager.test.ts | 79 +++++++++++++ source/server/tasks/manager.ts | 90 +++++++++++++++ source/server/tasks/queue.test.ts | 33 ++++++ source/server/tasks/queue.ts | 78 +++++++++++++ source/server/tasks/scheduler.test.ts | 112 +++++++++++++++++++ source/server/tasks/scheduler.ts | 102 +++++++++++++++++ source/server/tasks/types.ts | 146 +++++++++++++++++++++++++ source/server/tsconfig.json | 2 +- source/server/utils/locals.ts | 9 ++ 14 files changed, 870 insertions(+), 2 deletions(-) create mode 100644 source/server/migrations/005-tasks.sql create mode 100644 source/server/tasks/errors.test.ts create mode 100644 source/server/tasks/errors.ts create mode 100644 source/server/tasks/logger.ts create mode 100644 source/server/tasks/manager.test.ts create mode 100644 source/server/tasks/manager.ts create mode 100644 source/server/tasks/queue.test.ts create mode 100644 source/server/tasks/queue.ts create mode 100644 source/server/tasks/scheduler.test.ts create mode 100644 source/server/tasks/scheduler.ts create mode 100644 source/server/tasks/types.ts diff --git a/source/server/create.ts b/source/server/create.ts index dcb457fe2..ed1133f72 100644 --- a/source/server/create.ts +++ b/source/server/create.ts @@ -10,6 +10,7 @@ import openDatabase from "./vfs/helpers/db.js"; import Vfs from "./vfs/index.js"; import defaultConfig from "./utils/config.js"; import createServer from "./routes/index.js"; +import { TaskScheduler } from "./tasks/scheduler.js"; const debug = debuglog("pg:connect"); @@ -17,6 +18,7 @@ const debug = debuglog("pg:connect"); export interface Services{ app: express.Application; vfs: Vfs; + taskScheduler: TaskScheduler; userManager: UserManager; close: ()=>Promise; } @@ -30,7 +32,7 @@ export default async function createService(config = defaultConfig) :Promise{ + it(`parses ${v}`, function(){ + const out = parseTaskError(v); + expect(out).to.be.instanceof(Error); + }); + }); + it(`parses empty strings`, function(){ + const out = parseTaskError(""); + expect(out).to.be.instanceof(Error); + expect(out.message).to.have.length.above(0); + }); + }); + + describe("parses serialized error", function(){ + it("Error", function(){ + const outputString = serializeTaskError(new Error("The message")); + const out = parseTaskError(JSON.parse(outputString)); + expect(out).to.be.instanceOf(Error); + expect(out).to.have.property("message", "The message"); + }); + it.skip("HTTPError", function(){ + const _e = new HTTPError(401, "The message") + const outputString = serializeTaskError(_e); + const out = parseTaskError(JSON.parse(outputString)); + expect(out).to.be.instanceOf(HTTPError); + expect(out).to.have.property("code", 401); + expect(out).to.have.property("message", "[401] The message"); + }); + it.skip("BadRequestError", function(){ + const _e = new BadRequestError("The message"); + const outputString = serializeTaskError(_e); + const out = parseTaskError(JSON.parse(outputString)); + expect(out).to.be.instanceOf(HTTPError); + expect(out).to.have.property("code", 400); + expect(out).to.have.property("message", "[400] The message"); + }); + + }); +}) +}) diff --git a/source/server/tasks/errors.ts b/source/server/tasks/errors.ts new file mode 100644 index 000000000..f184dc77d --- /dev/null +++ b/source/server/tasks/errors.ts @@ -0,0 +1,35 @@ + +import { HTTPError } from "../utils/errors.js"; + + +/** + * Constructs an appropriate error from a generic object + * returned from the database + */ +export function parseTaskError(output: any) :Error|HTTPError{ + if(typeof output === "string" && output.length) return new Error(output); + let err :any = (output?.name == "HTTPError" && typeof output.code === "number")?new HTTPError(output.code, "Unknown task error"):new Error("Unknown task error"); + if(! output || typeof output !== "object") return err; + for(let key in output){ + err[key] = output[key]; + } + return err; +} +/** + * Serializes an `Error` or {@link HTTPError}. + * + * The string is guaranteed to be the serialization of an object with at least a "message" property + * Iterates over the `message` and `stack` properties of standard errors, which `JSON.stringify` wouldn't do + * @fixme we should possibly try to handle errors issued from postgresql too + */ +export function serializeTaskError(e: HTTPError|Error|string):string{ + if(typeof e === "string") return JSON.stringify({message: e}); + else if(!e || typeof e !== "object") return JSON.stringify({message: `Error: ${e}`}); + else if(!e.message) return JSON.stringify({message: `Error: ${JSON.stringify(e)}`}); + let obj :any = {} + for(let key of Object.getOwnPropertyNames(e)){ + obj[key] = (e as any)[key]; + } + obj["name"] = e.name; + return JSON.stringify(obj); +} diff --git a/source/server/tasks/logger.ts b/source/server/tasks/logger.ts new file mode 100644 index 000000000..e5110f444 --- /dev/null +++ b/source/server/tasks/logger.ts @@ -0,0 +1,38 @@ +import { Writable } from "node:stream"; +import { ITaskLogger, LogSeverity } from "./types.js"; +import { DatabaseHandle } from "../vfs/helpers/db.js"; +import { format } from "node:util"; + +/** + * Disposable logger that queues messages and waits for all logs to be flushed when closed + */ +export function createLogger(db: DatabaseHandle, task_id: number){ + + const stream = new Writable({ + objectMode: true, + write: async ({severity, message}, encoding, callback) => { + try { + await db.run(`INSERT INTO tasks_logs(fk_task_id, severity, message) VALUES ($1, $2, $3)`, [task_id, severity, message]); + callback(); + } catch (err) { + callback(err as Error); + } + } + }); + + function log(severity: LogSeverity, message: string){ + stream.write({severity, message}); + } + + return { + debug: (...args:any[])=>log('debug', format(...args)), + log: (...args:any[])=>log('log', format(...args)), + warn: (...args:any[])=>log('warn', format(...args)), + error: (...args:any[])=>log('error', format(...args)), + [Symbol.asyncDispose]: function (): PromiseLike { + return new Promise((resolve) => { + stream.end(() => resolve()); + }); + } + } satisfies ITaskLogger & AsyncDisposable; +} \ No newline at end of file diff --git a/source/server/tasks/manager.test.ts b/source/server/tasks/manager.test.ts new file mode 100644 index 000000000..6b01591e6 --- /dev/null +++ b/source/server/tasks/manager.test.ts @@ -0,0 +1,79 @@ + + +// Tests for tasks management +// The system is a bit intricate and hard to test in isolation + +import { Client } from "pg"; +import openDatabase, { Database } from "../vfs/helpers/db.js"; + +import { Uid } from "../utils/uid.js"; +import { randomBytes } from "node:crypto"; +import { TaskManager } from "./manager.js"; + +describe("TaskManager", function(){ + let db_uri: string, scene_id: number, handle: Database, client: Client; + + let listener :TaskManager; + this.beforeAll(async function(){ + db_uri = await getUniqueDb(this.currentTest?.title.replace(/[^\w]/g, "_")); + handle = await openDatabase({uri: db_uri}); + let s = await handle.get( + "INSERT INTO scenes(scene_id, scene_name) VALUES ( $1, $2 ) RETURNING scene_id", + [Uid.make(), randomBytes(8).toString("base64url")]); + scene_id = s.scene_id; + }); + + this.afterAll(async function(){ + await handle?.end(); + await dropDb(db_uri); + }); + + this.beforeEach(async function(){ + await Promise.all([ + handle.run(`DELETE FROM tasks`), + handle.run(`DELETE FROM tasks_logs`), + ]); + + listener = new TaskManager(handle); + }) + + + //Non-connected functions that should work well in isolation + it("create tasks", async function(){ + let task = await listener.create({ + scene_id, + user_id: null, + type: "delayTask", + data: {time: 0} + }); + expect(task.task_id).to.be.a("number"); + expect(await handle.all("SELECT * FROM tasks")).to.have.length(1); + }); + + + it("update task status", async function(){ + let t = await listener.create({ + scene_id, + user_id: null, + type: "delayTask", + data: {time: 0} + }); + expect(await handle.get("SELECT * FROM tasks")).to.have.property("status", "pending"); + await listener.setTaskStatus(t.task_id, "success"); + expect(await handle.get("SELECT * FROM tasks")).to.have.property("status", "success"); + }); + + it("get a task", async function(){ + //Create a task + let t = await listener.create({ + scene_id, + user_id: null, + type: "delayTask", + data: {time: 0} + }); + + let resolved = await listener.getTask(t.task_id); + expect(resolved).to.deep.equal(t); + }); + +}); \ No newline at end of file diff --git a/source/server/tasks/manager.ts b/source/server/tasks/manager.ts new file mode 100644 index 000000000..29f10295b --- /dev/null +++ b/source/server/tasks/manager.ts @@ -0,0 +1,90 @@ +import { debuglog } from "node:util"; +import { HTTPError, NotFoundError } from "../utils/errors.js"; +import { DatabaseHandle } from "../vfs/helpers/db.js"; +import { serializeTaskError } from "./errors.js"; +import { TaskStatus, TaskData, TaskDefinition, CreateTaskParams } from "./types.js"; + +const debug_outputs = debuglog("tasks:outputs"); + +/** + * Contains base interface to manage tasks: creation, status changes, etc... + * To be used through a {@link TaskScheduler} or directly for externally-managed tasks + */ +export class TaskManager{ + public get db(){ + return this._db; + } + + constructor(private _db: DatabaseHandle){ + + } + /** + * Internal method to adjust task status + * @param id + * @param status + */ + public async setTaskStatus(id: number, status:Omit): Promise{ + await this.db.run(`UPDATE tasks SET status = $2 WHERE task_id = $1`, [id, status]); + } + + /** + * Marks a task as completed + * Output is serialized using `JSON.stringify()` + */ + async releaseTask(id: number, output: any = null){ + debug_outputs(`Release task #${id}`, output); + await this.db.run(`UPDATE tasks SET status = 'success', output = $2 WHERE task_id = $1`, [id, JSON.stringify(output)]); + } + + /** + * @fixme use task.output to store the error message? + */ + async errorTask(id: number, reason: HTTPError|Error|string){ + try{ + debug_outputs(`Task #${id} Error : `, reason); + await this.db.run(`UPDATE tasks SET status = 'error', output = $2 WHERE task_id = $1`, [id, serializeTaskError(reason)]); + }catch(e:any){ + console.error("While trying to set task status:", e); + } + } + + public async create({scene_id, user_id, type, data, status='pending'}: CreateTaskParams): Promise>{ + let args = [scene_id, type, data, status, user_id]; + let task = await this.db.get>(` + INSERT INTO tasks(fk_scene_id, type, data, status, fk_user_id) + VALUES ($1, $2, $3, $4, $5) + RETURNING * + `, args); + return task; + } + + public async getTask(id: number):Promise{ + let task = await this.db.get(` + SELECT + fk_scene_id, + fk_user_id, + task_id, + ctime, + type, + parent, + data, + output, + status + FROM tasks + WHERE task_id = $1 + `, [id]); + if(!task) throw new NotFoundError(`No task found with id ${id}`); + return task; + } + + /** + * Deletes a task from the database + * Deletion will cascade to any dependents. + */ + async deleteTask(id: number): Promise{ + let r = await this.db.run(`DELETE FROM tasks WHERE task_id = $1`, [id]); + if(r.changes !== 1) return false; + return true; + } + +} \ No newline at end of file diff --git a/source/server/tasks/queue.test.ts b/source/server/tasks/queue.test.ts new file mode 100644 index 000000000..3495a92a2 --- /dev/null +++ b/source/server/tasks/queue.test.ts @@ -0,0 +1,33 @@ +import { Queue } from "./queue.js"; + + + +describe("Queue", function(){ + let q: Queue; + this.beforeEach(function(){ + q = new Queue(); + }); + describe("close()", function(){ + it("close enmpty queue", async function(){ + await q.close(); + }); + it("can't close twice", async function(){ + await q.close(); + await expect(q.close()).to.be.rejectedWith("already closed"); + }); + it("can't add jobs once closed", async function(){ + await q.close(); + //add() throws synchronously + expect(()=>q.add(()=>Promise.resolve())).to.throw("Can't add new tasks"); + }); + + it.skip("cancels running jobs") + }); + + describe("add()", function(){ + it("can process a task", async function(){ + const result = await q.add(()=>Promise.resolve("Hello")); + expect(result).to.equal("Hello"); + }); + }); +}) \ No newline at end of file diff --git a/source/server/tasks/queue.ts b/source/server/tasks/queue.ts new file mode 100644 index 000000000..78e65009b --- /dev/null +++ b/source/server/tasks/queue.ts @@ -0,0 +1,78 @@ +import EventEmitter from "node:events"; +import { TaskData, TaskHandler, TaskPackage } from "./types.js"; + + + +interface WorkPackage{ + work: TaskPackage; + resolve: (value: ReturnType) => void; + reject: (err: Error)=>void; +} + + +export class Queue{ + #queue: WorkPackage[] = []; + #active = 0; + #limit: number; + #c = new AbortController(); + + constructor(limit = Infinity) { + this.#limit = limit; + } + + + /** + * Adds a task to the queue. + */ + add(work: TaskPackage):Promise { + if(this.#c.signal.aborted){ + throw new Error(`Queue is closed. Can't add new tasks`); + } + return new Promise((resolve, reject) => { + this.#queue.push({ work, resolve, reject }); + this.#processNext(); + }); + } + + #onSettled = ()=>{ + this.#active--; + this.#processNext(); + } + + #processNext(){ + // Stop if we are busy or if the queue is empty + if (this.#active >= this.#limit || this.#queue.length === 0 ) { + return; + } + + // 3. Dequeue the next task + this.#active++; + const { work, resolve, reject } = this.#queue.shift()!; + + Promise.resolve(work({signal: this.#c.signal})) + .then(result => resolve(result)) + .catch(error => reject(error)) + .finally(this.#onSettled); + } + + /** + * Close the queue + * @todo Add a more graceful close that waits for jobs to complete instead of cancelling + */ + async close(){ + if(this.#c.signal.aborted){ + throw new Error(`Queue is already closed.`); + } + this.#c.abort(); + for (let item of this.#queue){ + item.reject(this.#c.signal.reason); + } + this.#queue = []; + //@FIXME We do not actually wait for jobs to be cancelled! + //Jobs that incompletely implement the cancellation interface could still complete after close(). + } + + // Optional getters for observability + get pendingCount() { return this.#queue.length; } + get activeCount() { return this.#active; } +} \ No newline at end of file diff --git a/source/server/tasks/scheduler.test.ts b/source/server/tasks/scheduler.test.ts new file mode 100644 index 000000000..0c29aff0c --- /dev/null +++ b/source/server/tasks/scheduler.test.ts @@ -0,0 +1,112 @@ + + +// Tests for tasks management +// The system is a bit intricate and hard to test in isolation + +import { Client } from "pg"; +import openDatabase, { Database, DatabaseHandle } from "../vfs/helpers/db.js"; + +import { Uid } from "../utils/uid.js"; +import { randomBytes } from "node:crypto"; +import { TaskScheduler } from "./scheduler.js"; + +describe("TaskScheduler", function(){ + let db_uri: string, scene_id: number, handle: Database, client: Client; + + //Create a taskScheduler with minimal context + let scheduler :TaskScheduler<{db:DatabaseHandle}>; + this.beforeAll(async function(){ + db_uri = await getUniqueDb(this.currentTest?.title.replace(/[^\w]/g, "_")); + handle = await openDatabase({uri: db_uri}); + let s = await handle.get( + "INSERT INTO scenes(scene_id, scene_name) VALUES ( $1, $2 ) RETURNING scene_id", + [Uid.make(), randomBytes(8).toString("base64url")]); + scene_id = s.scene_id; + }); + + this.afterAll(async function(){ + await handle?.end(); + await dropDb(db_uri); + }); + + this.beforeEach(async function(){ + await Promise.all([ + handle.run(`DELETE FROM tasks`), + handle.run(`DELETE FROM tasks_logs`), + ]); + + scheduler = new TaskScheduler({db: handle}); + }) + this.afterEach(async function(){ + await scheduler.close(); + }); + + it("creates an immediately-executed task", async function(){ + const result = await scheduler.run({ + scene_id: null, + user_id: null, + type: "testTask", + data: {}, + handler: async ({task})=>{ + return task.task_id; + } + }); + expect(result).to.be.a("number"); + + const task = await scheduler.getTask(result); + expect(task).to.have.property("status", "success"); + expect(task).to.have.property("output", result); + }); + + it("initialized a task for later execution", async function(){ + let task = await scheduler.create({ + scene_id: null, + user_id: null, + type: "testTask", + data: {}, + }); + const output = await scheduler.run({task, handler: async ({task})=> task.task_id}); + expect(output).to.equal(task.task_id); + + task = await scheduler.getTask(output); + expect(task).to.have.property("status", "success"); + expect(task).to.have.property("output", output); + }); + + it("handles tasks errors", async function(){ + let id_ref :number; + await expect(scheduler.run({ + scene_id: null, + user_id: null, + type: "testTask", + data: {}, + handler: async ({task})=>{ + id_ref = task.task_id; + await Promise.reject(new Error("Some message")); + } + })).to.be.rejectedWith("Some message"); + + expect(id_ref!).to.be.a("number"); + + const task = await scheduler.getTask(id_ref!); + expect(task).to.have.property("status", "error"); + expect(task).to.have.property("output").ok; + }); + + it("use a named function for task type", async function(){ + const result = await scheduler.run({ + scene_id: null, + user_id: null, + data: {}, + handler: async function testTask({task}){ + return task.task_id; + } + }); + expect(result).to.be.a("number"); + + const task = await scheduler.getTask(result); + expect(task).to.have.property("status", "success"); + expect(task).to.have.property("type", "testTask"); + }); + +}); \ No newline at end of file diff --git a/source/server/tasks/scheduler.ts b/source/server/tasks/scheduler.ts new file mode 100644 index 000000000..2abf6382d --- /dev/null +++ b/source/server/tasks/scheduler.ts @@ -0,0 +1,102 @@ +import { debuglog, format } from "node:util"; +import { Writable } from 'node:stream'; +import { HTTPError } from "../utils/errors.js"; +import { Database, DatabaseHandle } from "../vfs/helpers/db.js"; +import { Queue } from "./queue.js"; +import { CreateRunTaskParams, RunTaskParams, TaskContextHandlers, TaskData, TaskDefinition, TaskHandler, TaskHandlerContext, TaskPackage, TaskSchedulerContext, TaskSettledCallback, } from "./types.js"; +import { createLogger } from "./logger.js"; +import { TaskManager } from "./manager.js"; + + + + +export class TaskScheduler extends TaskManager{ + readonly #queue = new Queue(4); + + + public get context(){ + return this._context; + } + + + constructor(protected _context :TContext){ + super(_context.db); + } + + async close(){ + await this.#queue.close(); + } + + private async _run(handler:TaskHandler,task: TaskDefinition, {signal:taskSignal}:{signal?:AbortSignal}= {}){ + const work :TaskPackage = async ({signal: queueSignal})=>{ + await using logger = createLogger(this.db, task.task_id); + const context: TaskHandlerContext = { + ...this.context, + tasks: this.childContext(task.task_id), + logger, + signal: taskSignal? (AbortSignal as any).any([taskSignal, queueSignal]): queueSignal, + }; + + await this.setTaskStatus(task.task_id, "running"); + return await handler.call(context, {context, inputs: new Map(), task}); + } + try{ + const output = await this.#queue.add(work); + await this.releaseTask(task.task_id, output); + return output; + }catch(e: any){ + await this.errorTask(task.task_id, e).catch(e=> console.error("Failed to set task error : ", e)); + throw e; + } + } + + /** + * Registers a task to run immediately and wait for its completion + */ + async run({task, handler}: RunTaskParams, callback?:TaskSettledCallback ): Promise; + async run({scene_id, user_id, type, data, handler, signal}: CreateRunTaskParams, callback?:TaskSettledCallback): Promise; + async run(params: CreateRunTaskParams|RunTaskParams, callback?:TaskSettledCallback): Promise{ + let task: TaskDefinition; + if("task" in params){ + task = params.task; + }else{ + task = await this.create({...params, type: params.type ?? params.handler.name}); + } + const _p = this._run(params.handler, task); + + if(typeof callback=== "function"){ + _p.then((value)=>callback(null, value), (err)=>callback(err)); + } + return _p; + } + + /** + * Join a task that has been created through {@link queue} + */ + async join(task_id: number){ + //It's yet unclear if this is really needed + throw new Error("Unimplemented"); + } + + /** + * in-context task scheduling methods + */ + private childContext(id: number) :TaskContextHandlers&AsyncDisposable{ + + return { + create: function (p: CreateRunTaskParams): Promise { + throw new Error("Function not implemented."); + }, + getTask: function (id: number): Promise> { + throw new Error("Function not implemented."); + }, + group: function (cb: (context: TaskContextHandlers) => Promise | AsyncGenerator, remap?: any): Promise { + throw new Error("Function not implemented."); + }, + async [Symbol.asyncDispose](){ + + } + } + } + +} \ No newline at end of file diff --git a/source/server/tasks/types.ts b/source/server/tasks/types.ts new file mode 100644 index 000000000..c2964b986 --- /dev/null +++ b/source/server/tasks/types.ts @@ -0,0 +1,146 @@ +import type UserManager from "../auth/UserManager.js"; +import { Config } from "../utils/config.js"; +import { TDerivativeQuality } from "../utils/schema/model.js"; +import { DatabaseHandle } from "../vfs/helpers/db.js"; +import type Vfs from "../vfs/index.js"; + + +export type TaskData = Record; + +export type TaskStatus = 'initializing'|'pending'|'aborting'|'running'|'success'|'error'; +export enum ETaskStatus{ + 'aborting' = -2, + 'error' = -1, + 'initializing', + 'running', + 'pending', + 'success', +} + +export interface TaskDefinition{ + fk_scene_id: number; + fk_user_id: number; + task_id: number; + ctime: Date; + type :string; + parent: number|null; + /** **Unordered** list of task requirements */ + after: number[]; + data: T; + output: any; + status: TaskStatus; +} + + +export interface TaskSchedulerContext{ + vfs: Vfs, + db: DatabaseHandle, + userManager: UserManager, + config: Config, +} + +export type TaskHandlerContext = T & { + tasks: TaskContextHandlers, + logger: ITaskLogger, + signal: AbortSignal, +}; + + +export interface TaskHandlerParams{ + task: TaskDefinition; + inputs: Map; + context: TaskHandlerContext; +} + +/** + * In the future we might want to support tasks that yield sub-tasks using return value `AsyncGenerator` + */ +export type TaskHandler = (this: TaskHandlerContext, params:TaskHandlerParams)=> Promise; + +/** + * Bound TaskHandler work package + */ +export type TaskPackage = (params: {signal: AbortSignal})=>Promise; + +export interface CreateTaskParams{ + data: T; + type: string; + scene_id?: number|null; + user_id?: number|null; + status?: TaskStatus; +} + +export interface CreateRunTaskParams extends Omit, "type">{ + handler: TaskHandler; + type?:string; + signal?: AbortSignal; + /** Can't create an immediately-running task with a status other than pending */ + status?:"pending"; +} + +export interface RunTaskParams{ + task: TaskDefinition; + handler: TaskHandler; + signal?: AbortSignal; +} + +export interface TaskSettledCallback { + (err:null, value:T):unknown + (err:any):unknown +} + +export interface ITaskLogger{ + debug: (message?: any, ...optionalParams: any[]) => void, + log: (message?: any, ...optionalParams: any[]) => void, + warn: (message?: any, ...optionalParams: any[]) => void, + error: (message?: any, ...optionalParams: any[]) => void, +} + +export type LogSeverity = keyof ITaskLogger; + +export interface TaskContextHandlers{ + create(p: CreateRunTaskParams): Promise; + getTask(id: number):Promise>; + /** + * Creates a group and encapsulates any task N° returned from the inner function as a dependency of this group. + * The group's output is an array of all its dependencies outputs. + * @param cb + */ + group(cb: (context: TaskContextHandlers)=>Promise|AsyncGenerator, remap?: any) :Promise; +} + + +export type GroupCallback = (context: TaskContextHandlers)=>Promise|AsyncGenerator; + + +export interface TaskHandlerDefinition{ + readonly type: string; + handle: TaskHandler; +}; + + +export interface ProcessFileParams{ + file?:string; + preset: TDerivativeQuality; +} + +export function requireFileInput(inputs:Map){ + let file:string|undefined = undefined; + for(let input of inputs.values()){ + if(file){ + throw new Error("More than one input could be used as input file"); + } + if(typeof input === "string") file = input; + } + if(!file){ + throw new Error(`No input file provided and none was found in task inputs\n${JSON.stringify([...inputs.entries()], null, 2)}`); + } + return file; +} + + +export interface ImportSceneResult{ + name: string; + action: "create"|"update"|"error"; + error?: string; +} \ No newline at end of file diff --git a/source/server/tsconfig.json b/source/server/tsconfig.json index 1b5a0baaa..def3511e0 100644 --- a/source/server/tsconfig.json +++ b/source/server/tsconfig.json @@ -1,6 +1,6 @@ { "compilerOptions": { - "lib": ["ES2021"], + "lib": ["ESNext"], "module": "NodeNext", "target": "ES2021", "strict": true, diff --git a/source/server/utils/locals.ts b/source/server/utils/locals.ts index c6185f0a0..c076a5b8e 100644 --- a/source/server/utils/locals.ts +++ b/source/server/utils/locals.ts @@ -8,12 +8,14 @@ import { BadRequestError, ForbiddenError, HTTPError, InternalError, NotFoundErro import Templates, { AcceptedLocales, locales } from "./templates.js"; import { Config } from "./config.js"; import { isEmbeddable } from "../routes/services/oembed.js"; +import { TaskScheduler } from "../tasks/scheduler.js"; export interface AppLocals extends Record{ fileDir :string; userManager :UserManager; vfs :Vfs; + taskScheduler :TaskScheduler; templates :Templates; config: Config; /** Length of a session, in milliseconds since epoch */ @@ -216,6 +218,13 @@ export function getVfs(req :Request){ return vfs; } + +export function getTaskScheduler(req: Request){ + const scheduler = getLocals(req).taskScheduler; + if(!scheduler) throw new InternalError("Badly configured app: task scheduler is not defined in app.locals"); + return scheduler; +} + export function getHost(req :Request) :URL{ let host = (req.app.get("trust proxy")? req.get("X-Forwarded-Host") : null) ?? req.get("Host"); return new URL(`${req.protocol}://${host}`); From cb00eb7d7b64beb9b04231d74f1807ce910590f5 Mon Sep 17 00:00:00 2001 From: Sebastien DUMETZ Date: Thu, 22 Jan 2026 16:19:35 +0100 Subject: [PATCH 05/14] task upload routes and TaskScheduler fixes --- source/server/migrations/005-tasks.sql | 17 - source/server/package-lock.json | 60 ++- source/server/package.json | 2 + source/server/routes/index.ts | 1 + source/server/routes/scenes/post.test.ts | 50 ++- source/server/routes/scenes/post.ts | 208 +++------- .../routes/scenes/scene/files/get/document.ts | 2 +- source/server/routes/tasks/artifacts/get.ts | 28 ++ .../server/routes/tasks/artifacts/put.test.ts | 124 ++++++ source/server/routes/tasks/artifacts/put.ts | 136 +++++++ source/server/routes/tasks/index.ts | 26 ++ source/server/routes/tasks/post.ts | 30 ++ source/server/routes/tasks/task/delete.ts | 29 ++ source/server/routes/tasks/task/get.ts | 25 ++ source/server/tasks/handlers/extractZip.ts | 168 ++++++++ source/server/tasks/handlers/index.ts | 5 + source/server/tasks/handlers/uploads.ts | 157 ++++++++ source/server/tasks/logger.ts | 5 +- source/server/tasks/manager.test.ts | 205 +++++++++- source/server/tasks/manager.ts | 109 ++++-- source/server/tasks/queue.ts | 48 ++- source/server/tasks/scheduler.test.ts | 359 +++++++++++++++++- source/server/tasks/scheduler.ts | 223 +++++++++-- source/server/tasks/types.ts | 107 +++--- source/server/utils/archives.test.ts | 22 ++ source/server/utils/archives.ts | 36 ++ source/server/vfs/helpers/db.ts | 6 +- 27 files changed, 1825 insertions(+), 363 deletions(-) create mode 100644 source/server/routes/tasks/artifacts/get.ts create mode 100644 source/server/routes/tasks/artifacts/put.test.ts create mode 100644 source/server/routes/tasks/artifacts/put.ts create mode 100644 source/server/routes/tasks/index.ts create mode 100644 source/server/routes/tasks/post.ts create mode 100644 source/server/routes/tasks/task/delete.ts create mode 100644 source/server/routes/tasks/task/get.ts create mode 100644 source/server/tasks/handlers/extractZip.ts create mode 100644 source/server/tasks/handlers/index.ts create mode 100644 source/server/tasks/handlers/uploads.ts create mode 100644 source/server/utils/archives.test.ts create mode 100644 source/server/utils/archives.ts diff --git a/source/server/migrations/005-tasks.sql b/source/server/migrations/005-tasks.sql index 9591ab1c4..ad83b9168 100644 --- a/source/server/migrations/005-tasks.sql +++ b/source/server/migrations/005-tasks.sql @@ -18,23 +18,6 @@ CREATE TABLE tasks ( status task_status NOT NULL DEFAULT 'pending' ); - --- create a trigger that will notify clients on every task update -CREATE OR REPLACE FUNCTION tasks_status_notify() -RETURNS trigger AS $$ -BEGIN - PERFORM pg_notify( 'tasks_' || NEW.status, NEW.task_id::text); - RETURN NEW; -END; -$$ LANGUAGE plpgsql; - -CREATE TRIGGER tasks_status - AFTER INSERT OR UPDATE OF status - ON tasks - FOR EACH ROW -EXECUTE PROCEDURE tasks_status_notify(); - - CREATE TYPE log_severity AS ENUM('debug', 'log', 'warn', 'error'); CREATE TABLE tasks_logs ( diff --git a/source/server/package-lock.json b/source/server/package-lock.json index 113f0df63..2ee67252c 100644 --- a/source/server/package-lock.json +++ b/source/server/package-lock.json @@ -10,6 +10,7 @@ "license": "Apache-2.0", "dependencies": { "body-parser": "^1.20.3", + "content-disposition": "^1.0.1", "cookie-parser": "^1.4.7", "cookie-session": "^2.1.0", "express": "^4.21.2", @@ -30,6 +31,7 @@ "devDependencies": { "@types/chai": "^5.0.1", "@types/chai-as-promised": "^8.0.1", + "@types/content-disposition": "^0.5.9", "@types/cookie-parser": "^1.4.8", "@types/cookie-session": "^2.0.49", "@types/express": "^5.0.0", @@ -163,6 +165,13 @@ "@types/node": "*" } }, + "node_modules/@types/content-disposition": { + "version": "0.5.9", + "resolved": "https://registry.npmjs.org/@types/content-disposition/-/content-disposition-0.5.9.tgz", + "integrity": "sha512-8uYXI3Gw35MhiVYhG3s295oihrxRyytcRHjSjqnqZVDDy/xcGBRny7+Xj1Wgfhv5QzRtN2hB2dVRBUX9XW3UcQ==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/cookie-parser": { "version": "1.4.8", "resolved": "https://registry.npmjs.org/@types/cookie-parser/-/cookie-parser-1.4.8.tgz", @@ -775,14 +784,16 @@ } }, "node_modules/content-disposition": { - "version": "0.5.4", - "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", - "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", - "dependencies": { - "safe-buffer": "5.2.1" - }, + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.0.1.tgz", + "integrity": "sha512-oIXISMynqSqm241k6kcQ5UwttDILMK4BiurCfGEREw6+X9jkkpEe5T9FZaApyLGGOnFuyMWZpdolTXMtvEJ08Q==", + "license": "MIT", "engines": { - "node": ">= 0.6" + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/content-type": { @@ -1110,6 +1121,18 @@ "express": "^4.11 || 5 || ^5.0.0-beta.1" } }, + "node_modules/express/node_modules/content-disposition": { + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", + "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", + "license": "MIT", + "dependencies": { + "safe-buffer": "5.2.1" + }, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/express/node_modules/cookie": { "version": "0.7.1", "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.1.tgz", @@ -2896,6 +2919,12 @@ "@types/node": "*" } }, + "@types/content-disposition": { + "version": "0.5.9", + "resolved": "https://registry.npmjs.org/@types/content-disposition/-/content-disposition-0.5.9.tgz", + "integrity": "sha512-8uYXI3Gw35MhiVYhG3s295oihrxRyytcRHjSjqnqZVDDy/xcGBRny7+Xj1Wgfhv5QzRtN2hB2dVRBUX9XW3UcQ==", + "dev": true + }, "@types/cookie-parser": { "version": "1.4.8", "resolved": "https://registry.npmjs.org/@types/cookie-parser/-/cookie-parser-1.4.8.tgz", @@ -3408,12 +3437,9 @@ "dev": true }, "content-disposition": { - "version": "0.5.4", - "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", - "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", - "requires": { - "safe-buffer": "5.2.1" - } + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.0.1.tgz", + "integrity": "sha512-oIXISMynqSqm241k6kcQ5UwttDILMK4BiurCfGEREw6+X9jkkpEe5T9FZaApyLGGOnFuyMWZpdolTXMtvEJ08Q==" }, "content-type": { "version": "1.0.5", @@ -3651,6 +3677,14 @@ "vary": "~1.1.2" }, "dependencies": { + "content-disposition": { + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", + "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", + "requires": { + "safe-buffer": "5.2.1" + } + }, "cookie": { "version": "0.7.1", "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.1.tgz", diff --git a/source/server/package.json b/source/server/package.json index 67fee6e7b..34d4e70d7 100755 --- a/source/server/package.json +++ b/source/server/package.json @@ -41,6 +41,7 @@ }, "dependencies": { "body-parser": "^1.20.3", + "content-disposition": "^1.0.1", "cookie-parser": "^1.4.7", "cookie-session": "^2.1.0", "express": "^4.21.2", @@ -61,6 +62,7 @@ "devDependencies": { "@types/chai": "^5.0.1", "@types/chai-as-promised": "^8.0.1", + "@types/content-disposition": "^0.5.9", "@types/cookie-parser": "^1.4.8", "@types/cookie-session": "^2.0.49", "@types/express": "^5.0.0", diff --git a/source/server/routes/index.ts b/source/server/routes/index.ts index f11e37276..f1c9d2c84 100644 --- a/source/server/routes/index.ts +++ b/source/server/routes/index.ts @@ -124,6 +124,7 @@ export default async function createServer(locals:AppParameters) :Promise { }); - throw e; - } - finally{ - await handle.close(); +export function getFilename(headers: Request["headers"]) :string|undefined{ + const disposition = headers["content-disposition"]?contentDisposition.parse(headers["content-disposition"]): null; + if(disposition?.parameters?.filename) return disposition?.parameters?.filename; + const mimeType = headers["content-type"] ?? "application/octet-stream"; + const ext = extFromType(mimeType); + if(ext){ + return uid(12) + ext; } - await vfs.isolate(async (vfs)=>{ - let zip = await new Promise((resolve,reject)=>yauzl.open(tmpfile, {lazyEntries: true, autoClose: true}, (err, zip)=>(err?reject(err): resolve(zip)))); - const openZipEntry = (record:Entry)=> new Promise((resolve, reject)=>zip.openReadStream(record, (err, rs)=>(err?reject(err): resolve(rs)))); - - //Directory entries are optional in a zip file so we should handle their absence - let scenes = new Map>(); + return undefined; +} - const onEntry = async (record :Entry) =>{ - const pathParts = record.fileName.split("/").filter(p=>!!p); - if(pathParts[0] == "scenes") pathParts.shift(); - if(pathParts.length === 0) return; //Skip "scenes/" - const scene = pathParts.shift(); - const name = pathParts.join("/"); - if(!record.fileName.endsWith("/")) pathParts.pop();//Drop the file name unless it's a directory - if(!scene){ - results.fail[`${record.fileName}`] = "not matching pattern"; - return - } - if(!scenes.has(scene)){ - //Create the scene - try{ - if (isUserAtLeast(requester, "create")) { - await vfs.createScene(scene, requester.uid); - results.ok.push(scene); - } - }catch(e){ - if((e as HTTPError).code != 409) throw e; - //409 == Scene already exist, it's OK. - } - scenes.set(scene, new Set()); - } - if ((Object.keys(results.fail) && !Object.keys(results.fail).includes(scene)) || (Object.keys(results.fail).length == 0)) { - if (!results.ok.includes(scene)) { - try { - let rights = await userManager.getAccessRights(scene, requester.uid); - if ((rights != "write" && rights != "admin") && requester.level != "admin") { - results.fail[scene] = "User does not have writting rights on the scene"; - throw new UnauthorizedError("User does not have writting rights on the scene"); - } else { - results.ok.push(scene); - } - } - catch (e) { - // If the scene is not found, the actual error is that the user cannot create it - if ((e as HTTPError).code == 404) { - results.fail[scene] = "User cannot create a scene"; - throw new UnauthorizedError("User cannot create a scene"); - } - else throw e; - } - } - if (!name) return; - let folders = scenes.get(scene)!; - let dirpath = ""; - while(pathParts.length){ - dirpath = path.join(dirpath, pathParts.shift()!); - if(folders.has(dirpath)) continue; - folders.add(dirpath); - try{ - await vfs.createFolder({scene, name: dirpath, user_id: requester.uid}); - results.ok.push(`${scene}/${dirpath}/`); - }catch(e){ - if((e as HTTPError).code != 409) throw e; - //409 == Folder already exist, it's OK. - } - } +export async function postRawZipfile(req: Request, res: Response){ + const requester = getUser(req); + if (requester === null){ throw new UnauthorizedError("No identified user")} + let userManager = getUserManager(req); + let taskScheduler = getTaskScheduler(req); - if(/\/$/.test(record.fileName)){ - // Is a directory. Do nothing, handled above. - }else if(name.endsWith(".svx.json")){ - let data = Buffer.alloc(record.uncompressedSize), size = 0; - let rs = await openZipEntry(record); - rs.on("data", (chunk)=>{ - chunk.copy(data, size); - size += chunk.length; - }); - await finished(rs); - await vfs.writeDoc(data, {scene, user_id: requester.uid, name, mime: "application/si-dpo-3d.document+json"}); - }else{ - //Add the file - let rs = await openZipEntry(record); - let mime = getMimeType(name); - if (mime.startsWith('text/')){ - await vfs.writeDoc(await text(rs), {user_id: requester.uid, scene, name, mime}); - } - else { - await vfs.writeFile(rs, {user_id: requester.uid, scene, name, mime}); - } - } - - results.ok.push(`${scene}/${name}`); - } - }; + let filename = getFilename(req.headers) as string; + if(!filename) throw new BadRequestError(`Can't detect file type from Content-Disposition or Content-Type headers`); + let size = parseInt(req.headers["content-length"]!); + if(!size || !Number.isInteger(size)) throw new BadRequestError(`Chunked encoding not supported for this request. Use upload tasks to transfer large files`); - zip.on("entry", (record)=>{ - onEntry(record).then(()=>{ - zip.readEntry() - }, (e)=>{ - if ((e as HTTPError).code == 401) { // If the error is unauthorised, we keep checking the rest of the scenes - zip.readEntry(); - } - else { - zip.close(); - zipError=e; - } + const output = await taskScheduler.run({ + scene_id: null, + user_id: requester.uid, + handler: async function handlePostScene({task, context:{vfs, logger}}) :Promise{ + const dir = await vfs.createTaskWorkspace(task.task_id); + const abs_filepath = path.join(dir, filename); + const relPath = vfs.relative(abs_filepath); + logger.log("Write upload artifact to :", relPath); + const ws = createWriteStream(abs_filepath); + await pipeline( + req, + ws, + ); + logger.log("artifact uploaded to :", relPath); + return await taskScheduler.run({ + scene_id: null, + user_id: requester.uid, + data: {filepath: relPath, size}, + handler: extractScenesArchive, }); - }); - zip.readEntry(); - await once(zip, "close"); - // If one or several files have raised unauthorised errors, we send the unauthorized http error after going through the whole zip to check for all errors - if (Object.keys(results.fail).length > 0 || zipError) { - res.status(zipError? ((zipError as HTTPError).code? (zipError as HTTPError).code : 500) : 401) - .send({failed_scenes: results.fail, message: zipError? zipError.message:""}); } - }).finally(() => fs.rm(tmpfile, { force: true })); + }); - if (Object.keys(results.fail).length == 0) { - res.status(200).send({ok : results.ok}); + res.status(200).send(output); +} + +export default async function postScenes(req :Request, res :Response){ + const requester = getUser(req); + if (requester === null){ throw new UnauthorizedError("No identified user")} + let userManager = getUserManager(req); + + if(req.is("multipart") || req.is("application/x-www-form-urlencoded")){ + throw new BadRequestError(`Form data is not supported on this route. Provide a raw Zip attachment`); } + + return await postRawZipfile(req, res); }; \ No newline at end of file diff --git a/source/server/routes/scenes/scene/files/get/document.ts b/source/server/routes/scenes/scene/files/get/document.ts index 65c865e56..eefddfa98 100644 --- a/source/server/routes/scenes/scene/files/get/document.ts +++ b/source/server/routes/scenes/scene/files/get/document.ts @@ -20,7 +20,7 @@ export default async function handleGetDocument(req :Request, res :Response){ } let data = Buffer.from(JSON.stringify(doc), "utf-8"); - let hash = createHash("sha256").update(data).digest("base64url"); + let hash = createHash("sha256").update(data as any).digest("base64url"); res.set("ETag", hash); diff --git a/source/server/routes/tasks/artifacts/get.ts b/source/server/routes/tasks/artifacts/get.ts new file mode 100644 index 000000000..ab1001fd0 --- /dev/null +++ b/source/server/routes/tasks/artifacts/get.ts @@ -0,0 +1,28 @@ +import { Request, Response } from "express"; +import { getVfs, getUser, getTaskScheduler } from "../../../utils/locals.js"; +import { MethodNotAllowedError, NotImplementedError, UnauthorizedError } from "../../../utils/errors.js"; + +import { isUploadTask } from "./put.js"; +import path from "node:path"; + + + +export async function getUploadTask(req: Request, res: Response){ + const vfs = getVfs(req); + const taskScheduler = getTaskScheduler(req); + const requester = getUser(req)!; + const {id:idString} = req.params; + const id = parseInt(idString); + const task = await taskScheduler.getTask(id); + if(task.fk_user_id !== requester.uid){ + throw new UnauthorizedError(`This task does not belong to this user`); + } + if(!isUploadTask(task)){ + throw new NotImplementedError(`Artifacts download not yet supported for this task`); + } + if(task.status == "initializing"){ + throw new MethodNotAllowedError(`GET not allowed on artifacts that are not yet complete`); + } + const {filename, size} = task.data; + res.sendFile(path.join(vfs.getTaskWorkspace(task.task_id), filename)); +} \ No newline at end of file diff --git a/source/server/routes/tasks/artifacts/put.test.ts b/source/server/routes/tasks/artifacts/put.test.ts new file mode 100644 index 000000000..bbe00a5c0 --- /dev/null +++ b/source/server/routes/tasks/artifacts/put.test.ts @@ -0,0 +1,124 @@ + +import request from "supertest"; + +import User from "../../../auth/User.js"; +import UserManager from "../../../auth/UserManager.js"; +import Vfs from "../../../vfs/index.js"; +import { randomBytes, randomInt } from "node:crypto"; + + + + + +describe("PUT /tasks/artifacts/:id", function(){ + let vfs :Vfs, userManager :UserManager, user :User, admin :User; + let filename: string, size: number, task: number; + + this.beforeAll(async function(){ + let locals = await createIntegrationContext(this); + vfs = locals.vfs; + userManager = locals.userManager; + user = await userManager.addUser("bob", "12345678"); + admin = await userManager.addUser("alice", "12345678", "admin"); + }); + + this.afterAll(async function(){ + await cleanIntegrationContext(this); + }); + + this.beforeEach(async function(){ + filename = randomBytes(4).toString("hex")+".bin"; + size = randomInt(16, 512); + const {body} = await request(this.server).post(`/tasks`) + .auth("alice", "12345678") + .set("Content-Type", "application/json") + .send({filename, size}) + .expect(200); + expect(body).to.have.property("task_id").a("number"); + task = body.task_id; + }); + + it("Can handle a single-chunk upload (no headers)", async function(){ + const data = randomBytes(size); + await request(this.server).put(`/tasks/artifacts/${task}`) + .auth("alice", "12345678") + .send(data) + .expect(201); + + const {body} = await request(this.server).get(`/tasks/artifacts/${task}`) + .auth("alice", "12345678") + .expect(200); + + expect(body.toString("hex")).to.equal(data.toString("hex")); + }); + + it("Can handle a single-chunk upload (chunk headers)", async function(){ + const data = randomBytes(size); + await request(this.server).put(`/tasks/artifacts/${task}`) + .auth("alice", "12345678") + .set("Content-Length", size.toString()) + .set("Content-Range", `bytes 0-${size-1 /*end is inclusive*/}/${size}`) + .send(data) + .expect(201); + + const {body} = await request(this.server).get(`/tasks/artifacts/${task}`) + .auth("alice", "12345678") + .expect(200); + + expect(body.toString("hex")).to.equal(data.toString("hex")); + }); + + it("Can handle a multi-chunk upload", async function(){ + const data = randomBytes(size); + + let offset = 0; + let chunkSize = Math.floor(size / 4); + while(offset < size){ + const len = Math.min(size - offset, chunkSize); + await request(this.server).put(`/tasks/artifacts/${task}`) + .auth("alice", "12345678") + .set("Content-Length", len.toString()) + .set("Content-Range", `bytes ${offset}-${offset+len-1 /*end is inclusive*/}/${size}`) + .send(data.subarray(offset, offset + len)) + .expect(offset + len < size? 206 : 201); + offset += len; + } + + const {body} = await request(this.server).get(`/tasks/artifacts/${task}`) + .auth("alice", "12345678") + .expect(200); + + expect(body.toString("hex")).to.equal(data.toString("hex")); + }); + + it("can parse uploaded contents (simple)", async function(){ + const data = randomBytes(size); + await request(this.server).put(`/tasks/artifacts/${task}`) + .auth("alice", "12345678") + .send(data) + .expect(201); + + const {body} = await request(this.server).get(`/tasks/task/${task}`) + .auth("alice", "12345678") + .expect(200); + + expect(body).to.have.property("status", "success"); + expect(body).to.have.property("output").to.deep.equal({ + filepath: `artifacts/${task}/${filename}`, + files: [filename], + scenes: [], + }); + }); + + it("rejects out-of-order bytes", async function(){ + const data = randomBytes(6); + console.log() + const res = await request(this.server).put(`/tasks/artifacts/${task}`) + .auth("alice", "12345678") + .set("Content-Length", '6') + .set("Content-Range", `bytes 10-15/${size}`) + .send(data) + .expect(416); + expect(res.body).to.have.property("message", `Error: [416] Missing bytes 0-10`); + }); +}) \ No newline at end of file diff --git a/source/server/routes/tasks/artifacts/put.ts b/source/server/routes/tasks/artifacts/put.ts new file mode 100644 index 000000000..fd8303100 --- /dev/null +++ b/source/server/routes/tasks/artifacts/put.ts @@ -0,0 +1,136 @@ +import { Request, Response } from "express"; +import parseRange from "range-parser"; +import path from "node:path"; +import { stat } from "node:fs/promises"; +import { getVfs, getUser, getTaskScheduler } from "../../../utils/locals.js"; +import { BadRequestError, LengthRequiredError, RangeNotSatisfiableError, UnauthorizedError } from "../../../utils/errors.js"; +import { TaskDefinition } from "../../../tasks/types.js"; +import { createWriteStream } from "node:fs"; +import { pipeline } from "node:stream/promises"; + +export function isUploadTask(t:TaskDefinition) : boolean{ + return t.type === "userUploads"; +} + +/** + * File upload handler + * Files can be sent in chunks. Uses `Content-Range` for the client to communicate chunk position + * and responds with a `Range` header to confirm current state. + * + * @see {@link https://docs.cloud.google.com/storage/docs/performing-resumable-uploads?hl=fr#chunked-upload Google Cloud Storage: Chunked upload} for a similar feature + */ +export async function putUploadTask(req: Request, res: Response){ + const vfs = getVfs(req); + const taskScheduler = getTaskScheduler(req); + const requester = getUser(req)!; + const {id:idString} = req.params; + const id = parseInt(idString); + const task = await taskScheduler.getTask(id); + if(task.fk_user_id !== requester.uid){ + throw new UnauthorizedError(`This task does not belong to this user`); + } + if(!isUploadTask(task)){ + throw new BadRequestError(`Task ${id} is not a user upload task`); + } + + + const {filename, size: filesize} = task.data; + const filepath = path.join(vfs.getTaskWorkspace(task.task_id), filename); + const contentRange = req.get("Content-Range"); + + const contentLength = parseInt(req.get("Content-Length")!); + if(!contentLength || !Number.isInteger(contentLength)) throw new LengthRequiredError(`A valid Content-Length header must be provided`); + + if(task.status !== "initializing"){ + throw new BadRequestError(`Task ${id} is in state ${task.status}, which does not allow further data to be sent`); + } + + + async function processUpload(){ + await taskScheduler.run({ + task, + handler: createUserUploadParser(), + }) + } + + if(!contentRange){ + const ws = createWriteStream(filepath) + await pipeline( + req, + ws + ); + await processUpload(); + res.status(201); + return res.format({ + "text/plain": ()=>{ + res.send("Created") + }, + "application/json": ()=>{ + res.send({code: 201, message: "Created"}) + } + }); + } + const ranges = parseRange(filesize, contentRange.replace("bytes ", "bytes=")); + if(ranges == -1 ) throw new RangeNotSatisfiableError(); + else if(ranges == -2 || !Array.isArray(ranges) || ranges.length != 1 || ranges.type !== "bytes") throw new BadRequestError(`Malformed range header`); + + const { start, end} = ranges[0]; + + + if(contentLength !== end - start + 1) throw new BadRequestError(`a Content-Length of ${contentLength} can't satisfy a range of ${end - start +1}`); + + // @fixme check current file size + if(start !== 0){ + const {size: currentSize} = await stat(filepath).catch(e=>{if(e.code !=="ENOENT") throw e; return {size: 0}}); + if(currentSize < start){ + //Slightly non-standard use of the Range header to communicate what is the current file size + res.set("Range", `bytes=0-${currentSize}/${filesize}`); + throw new RangeNotSatisfiableError(`Missing bytes ${currentSize}-${start}`); + } + } + const ws = createWriteStream(filepath, {flags: start == 0? 'w':'r+', start}); + await pipeline( + req, + async function* rangeLength(source){ + let len = 0; + for await (const chunk of source){ + len += chunk.length; + if(contentLength < len) throw new BadRequestError(`Expected a content length of ${contentLength}. Received ${len} bytes.`); + yield chunk; + } + if(len < contentLength){ + throw new BadRequestError(`Expected a content length of ${contentLength}. Received ${len} bytes.`); + } + }, + ws + ); + + res.set("Range", `bytes=0-${end}/${filesize}`); + + if(end == filesize - 1){ + await processUpload(); + res.status(201); + res.format({ + "text/plain": ()=>{ + res.send("Created") + }, + "application/json": ()=>{ + res.send({code: 201, message: "Created"}) + } + }); + }else{ + res.status(206); + res.format({ + "text/plain": ()=>{ + res.send("Partial Content") + }, + "application/json": ()=>{ + res.send({code: 206, message: "Partial Content"}) + } + }); + } +} + +function createUserUploadParser(): import("../../../tasks/types.js").TaskHandler { + throw new Error("Function not implemented."); +} diff --git a/source/server/routes/tasks/index.ts b/source/server/routes/tasks/index.ts new file mode 100644 index 000000000..a4202eac7 --- /dev/null +++ b/source/server/routes/tasks/index.ts @@ -0,0 +1,26 @@ +import { Router } from "express"; + +import wrap from "../../utils/wrapAsync.js"; + +import { canAdmin, canRead, isCreator, isUser } from "../../utils/locals.js"; + +import { createTask } from "./post.js"; +import { putUploadTask } from "./artifacts/put.js"; +import bodyParser from "body-parser"; +import { getUploadTask } from "./artifacts/get.js"; +import { getTask } from "./task/get.js"; +import { deleteTask } from "./task/delete.js"; + +const router = Router(); + +router.use("/", isUser); +router.post("/", isCreator, bodyParser.json(), wrap(createTask)); + + +router.put("/artifacts/:id", wrap(putUploadTask)); +router.get("/artifacts/:id", wrap(getUploadTask)); + +router.delete("/task/:id(\\d+)", wrap(deleteTask)); +router.get("/task/:id(\\d+)", wrap(getTask)); + +export default router; diff --git a/source/server/routes/tasks/post.ts b/source/server/routes/tasks/post.ts new file mode 100644 index 000000000..465ac1e11 --- /dev/null +++ b/source/server/routes/tasks/post.ts @@ -0,0 +1,30 @@ +import { Request, Response } from "express"; +import { getTaskScheduler, getUser, getVfs } from "../../utils/locals.js"; +import { BadRequestError } from "../../utils/errors.js"; + +export async function createTask(req: Request, res: Response){ + const vfs = getVfs(req); + const taskScheduler = getTaskScheduler(req); + const requester = getUser(req)!; + let {filename, size} = req.body; + if(!filename){ + throw new BadRequestError(`No filename provided`); + } + if(!size || !Number.isInteger(size)){ + throw new BadRequestError(`Bad file size provided`); + } + + const task = await taskScheduler.create({ + scene_id:null, + user_id:requester.uid, + type: "userUploads", + status: "initializing", + data: { + size, + filename, + } + }); + //Create the workspace immediately + await vfs.createTaskWorkspace(task.task_id); + res.status(200).send(task); +} \ No newline at end of file diff --git a/source/server/routes/tasks/task/delete.ts b/source/server/routes/tasks/task/delete.ts new file mode 100644 index 000000000..cce5ed104 --- /dev/null +++ b/source/server/routes/tasks/task/delete.ts @@ -0,0 +1,29 @@ +import { rm } from "node:fs/promises"; + +import { Request, Response } from "express"; +import { getUser, getLocals } from "../../../utils/locals.js"; +import { UnauthorizedError } from "../../../utils/errors.js"; + + +export async function deleteTask(req: Request, res: Response){ + const { + vfs, + taskScheduler, + userManager, + } = getLocals(req); + const requester = getUser(req)!; + const {id:idString} = req.params; + const id = parseInt(idString); + const task = await taskScheduler.getTask(id); + + if(requester.level !== "admin" + && task.fk_user_id !== requester.uid + && (await userManager.getAccessRights(task.fk_scene_id, requester.uid)) != "admin" + ){ + throw new UnauthorizedError(`Administrative rights are required to delete tasks`); + } + + await taskScheduler.deleteTask(id); + await rm(vfs.getTaskWorkspace(id), {force: true, recursive: true}); + res.status(204).send(); +} diff --git a/source/server/routes/tasks/task/get.ts b/source/server/routes/tasks/task/get.ts new file mode 100644 index 000000000..c912993c9 --- /dev/null +++ b/source/server/routes/tasks/task/get.ts @@ -0,0 +1,25 @@ +import { Request, Response } from "express"; +import { UnauthorizedError } from "../../../utils/errors.js"; +import { getLocals, getUser } from "../../../utils/locals.js"; +import { toAccessLevel } from "../../../auth/UserManager.js"; + + +export async function getTask(req: Request, res: Response){ + const { + vfs, + taskScheduler, + userManager, + } = getLocals(req); + const requester = getUser(req)!; + const {id:idString} = req.params; + const id = parseInt(idString); + let task = await taskScheduler.getTask(id); + + if(requester.level !== "admin" + && task.fk_user_id !== requester.uid + && toAccessLevel(await userManager.getAccessRights(task.fk_scene_id, requester.uid)) < toAccessLevel("read") + ){ + throw new UnauthorizedError(`Read rights are required to delete tasks`); + } + res.status(200).send(task); +} \ No newline at end of file diff --git a/source/server/tasks/handlers/extractZip.ts b/source/server/tasks/handlers/extractZip.ts new file mode 100644 index 000000000..f8b2970f6 --- /dev/null +++ b/source/server/tasks/handlers/extractZip.ts @@ -0,0 +1,168 @@ +import { Readable } from "node:stream"; +import { once } from "node:events"; +import { TaskHandlerParams } from "../types.js"; +import yauzl, { Entry, ZipFile } from "yauzl"; +import path from "node:path"; +import { isUserAtLeast } from "../../auth/User.js"; +import { BadRequestError, HTTPError, InternalError, UnauthorizedError } from "../../utils/errors.js"; +import { parseFilepath } from "../../utils/archives.js"; +import { toAccessLevel } from "../../auth/UserManager.js"; +import { getMimeType } from "../../utils/filetypes.js"; +import { text } from "node:stream/consumers"; +import { finished } from "node:stream/promises"; + + + +export interface ImportSuccessResult{ + name: string; + action: "create"|"update"; +} +interface ImportErrorResult{ + name: string; + action: "error"; + error: HTTPError|Error; + +} +type ImportSceneResult = ImportSuccessResult|ImportErrorResult; + +interface ExtractZipParams{ + /**filepath, relative to Vfs.baseDir */ + filepath: string, +} + +/** + * Analyze an uploaded file and create child tasks accordingly + */ +export async function extractScenesArchive({task: {scene_id: scene_id, user_id: user_id, data: {filepath}}, context:{vfs, logger, userManager, tasks}}:TaskHandlerParams):Promise{ + if(!filepath || typeof filepath !== "string") throw new Error(`invalid filepath provided`); + + if(!user_id) throw new Error(`This task requires an authenticated user`); + const requester = await userManager.getUserById(user_id); + + let zipError: Error; + logger.debug("Open Zip file"); + const zip = await new Promise((resolve,reject)=>yauzl.open(path.join(vfs.baseDir, filepath!), {lazyEntries: true, autoClose: true}, (err, zip)=>(err?reject(err): resolve(zip)))); + const openZipEntry = (record:Entry)=> new Promise((resolve, reject)=>zip.openReadStream(record, (err, rs)=>(err?reject(err): resolve(rs)))); + + logger.debug("Open database transaction"); + const results = await vfs.isolate(async (vfs)=>{ + //Directory entries are optional in a zip file so we should handle their absence + //We do this by maintaining a Map of scenes, and for each scene a Set of files + let scenes = new Map}>(); + // As soon as we have one or more errors, we skip trying to copy files + // but we continue processing entries to get a list of all unauthorized scenes + let has_errors = false; + /** + * Handles a zip entry. + */ + const onEntry = async (record :Entry) =>{ + const {scene, name, isDirectory} = parseFilepath(record.fileName); + if(!scene ) return; + if(!scenes.has(scene)){ + let result:ImportSceneResult; + //Create the scene + try{ + const rights = await userManager.getAccessRights(scene, requester.uid) + if(toAccessLevel("write") <= toAccessLevel(rights)){ + result = {name: scene, action: "update"}; + logger.log(`Scene ${scene} will be updated`); + }else{ + logger.warn(`Scene ${scene} can't be updated: User only has access level "${toAccessLevel(rights)}"`); + result = {name: scene, action: "error", error: new UnauthorizedError(`User doesn't have write permissions on scene "${scene}"`)}; + has_errors = true; + } + }catch(e){ + if((e as HTTPError).code != 404) throw e; + //404 == Not Found. Check if user can create the scene + if (isUserAtLeast(requester, "create")) { + await vfs.createScene(scene, requester.uid); + result = {name: scene, action: "create"}; + logger.log(`Scene ${scene} will be created`); + }else{ + result = {name: scene, action: "error", error: new UnauthorizedError(`User doesn't have write permissions on scene "${scene}"`)}; + has_errors = true; + } + } + scenes.set(scene, {folders: new Set(), ...result}); + } + if(has_errors) return; + const _s = scenes.get(scene); + if(!_s || _s.action === "error") throw new Error(`Scene ${scene} wasn't properly checked for permissions`); + const { folders } = _s; + if (!name) return; + let dirpath = ""; + let pathParts = name.split("/"); + if(!isDirectory) pathParts.pop(); //Remove last segment except for directories + while(pathParts.length){ + dirpath = path.join(dirpath, pathParts.shift()!); + if(folders.has(dirpath)) continue; + folders.add(dirpath); + try{ + await vfs.createFolder({scene, name: dirpath, user_id: requester.uid}); + }catch(e){ + if((e as HTTPError).code != 409) throw e; + //409 == Folder already exist, it's OK. + } + } + + if(isDirectory){ + // Is a directory. Do nothing, handled above. + }else if(name.endsWith(".svx.json")){ + let data = Buffer.alloc(record.uncompressedSize), size = 0; + let rs = await openZipEntry(record); + rs.on("data", (chunk)=>{ + chunk.copy(data, size); + size += chunk.length; + }); + await finished(rs); + await vfs.writeDoc(data, {scene, user_id: requester.uid, name, mime: "application/si-dpo-3d.document+json"}); + }else{ + //Add the file + let rs = await openZipEntry(record); + let mime = getMimeType(name); + if (mime.startsWith('text/')){ + await vfs.writeDoc(await text(rs), {user_id: requester.uid, scene, name, mime}); + } + else { + await vfs.writeFile(rs, {user_id: requester.uid, scene, name, mime}); + } + } + }; + + zip.on("entry", (record)=>{ + onEntry(record).then(()=>{ + zip.readEntry() + }, (e)=>{ + zip.close(); + zipError=e; + }); + }); + zip.readEntry(); + logger.debug("Start extracting zip"); + await once(zip, "close"); + if(zipError){ + logger.error("Zip extraction encountered an error. This is most probably due to an invalid zip"); + throw zipError; + } + const results = [...scenes.values()].map(({folders, ...r})=> r as any); + if(has_errors){ + let errors = results.filter(function(r):r is ImportErrorResult {return r.action == "error"}); + if(errors.length == 1){ + throw errors[0].error; + }else { + let unauthorized = errors.filter(r=>r.error instanceof UnauthorizedError); + if(unauthorized.length === errors.length){ + throw new UnauthorizedError( + `Multiple unauthorized scenes : ${errors.map(r=>r.name).join(",")}` + ); + } else{ + throw new InternalError(`Mixed errors : ${errors.map(r=>r.error.message).join(", ")}`) + } + } + }else{ + logger.debug("zip file closed"); + return results as ImportSuccessResult[]; + } + }); + return results; +}; diff --git a/source/server/tasks/handlers/index.ts b/source/server/tasks/handlers/index.ts new file mode 100644 index 000000000..bb952b4c3 --- /dev/null +++ b/source/server/tasks/handlers/index.ts @@ -0,0 +1,5 @@ +/** + * Gather all handlers that can be called from user-created tasks + */ + +export {processUploadedFiles, parseUserUpload} from "./uploads.js"; \ No newline at end of file diff --git a/source/server/tasks/handlers/uploads.ts b/source/server/tasks/handlers/uploads.ts new file mode 100644 index 000000000..93927a88c --- /dev/null +++ b/source/server/tasks/handlers/uploads.ts @@ -0,0 +1,157 @@ +import { once } from "node:events"; +import { stat } from "node:fs/promises"; +import path, { extname } from "node:path"; + +import yauzl, { ZipFile } from "yauzl"; +import { isUserAtLeast } from "../../auth/User.js"; +import { toAccessLevel } from "../../auth/UserManager.js"; +import { parseFilepath, isMainSceneFile } from "../../utils/archives.js"; +import { TaskHandlerParams } from "../types.js"; +import { BadRequestError, UnauthorizedError } from "../../utils/errors.js"; +import { extractScenesArchive, ImportSuccessResult } from "./extractZip.js"; + + + +export interface ImportSceneResult{ + name: string; + action: "create"|"update"|"error"; + error?: string; +} + +export interface UploadHandlerParams{ + filename: string; + size: number; +} + +export interface UserUploadResult{ + filepath: string; + files: string[]; + scenes: ImportSceneResult[]; +} + +/** + * Inspect a user-uploaded file to detect its contents + * @param param0 + * @returns + */ +export async function parseUserUpload({task:{task_id, user_id, data:{filename, size}}, context: {vfs, userManager, logger}}:TaskHandlerParams):Promise{ + + let files :string[] = []; + logger.debug("Requester :", user_id); + const requester = await userManager.getUserById(user_id); + const filepath = path.join(vfs.getTaskWorkspace(task_id), filename); + logger.debug(`Checking size of uploaded file ${filepath}`); + let diskSize: number; + try{ + const stats= await stat(filepath); + diskSize = stats.size; + }catch(e:any){ + if(e.code === "ENOENT"){ + logger.error(`File ${filepath} does not exist. Maybe it wasn't uploaded properly?`); + } + throw e; + } + if(diskSize != size){ + throw new Error(`Expected a file of size ${size}, found ${diskSize}`); + } + + const ext = extname(filename).toLowerCase(); + if(ext == ".zip"){ + logger.debug(`Open ${filename} to list entries`); + let zip = await new Promise((resolve,reject)=>yauzl.open(filepath, {lazyEntries: false, autoClose: true}, (err, zip)=>(err?reject(err): resolve(zip)))); + zip.on("entry", (record)=>{ + files.push(record.fileName); + }); + await once(zip, "close"); + logger.debug(`Found ${files.length} entries in zip`); + }else{ + files.push(filename); + } + + let scenes :ImportSceneResult[] = []; + for(let file of files){ + const {scene, name} = parseFilepath(file); + if(!scene || !name || !isMainSceneFile(file)) continue; + let action :"create"|"update"|"error"; + let error: string; + try{ + const level = toAccessLevel(await userManager.getAccessRights(scene, user_id)); + if(level < toAccessLevel("write")){ + action = "error"; + error = `User doesn't have write permissions on scene ${scene}`; + }else{ + action = "update"; + } + }catch(e:any){ + if(e.code !== 404) throw e; + if(isUserAtLeast(requester, "create")){ + action = "create" + }else{ + action = "error"; + error = "User doesn't have permission to create a new scene"; + } + } + scenes.push({name: scene, action, error: error!}); + } + // @FIXME maybe we should already delete the file if it has errors? + // It depends on the behaviour we expect of a "partial success" zip upload. + + return { + filepath: vfs.relative(filepath), + files: files, + scenes + } satisfies UserUploadResult; +} + + + +export interface ProcessUploadedFilesParams{ + files: UserUploadResult[]; + name?: string; + lang?: string; +} + +/** + * Process file(s) that have been uploaded through `userUploads` task(s). + * The file(s) are expected to come from previous tasks + */ +export async function processUploadedFiles({context:{tasks, logger}, task: {data:{files, name, lang}}}: TaskHandlerParams):Promise{ + + if(!files.length) throw new BadRequestError(`This task requires at least one source file`); + + const upload_scenes = files.findIndex(u=>u.scenes.length) !== -1 //A file containing at least one complete scene + const upload_files = files.findIndex(u=>!u.scenes.length) !== -1 // A file NOT containing any complete scene + // Don't allow mixed-content. ie. a scene archive with some asset files + // In reality we _could_ probably, though we'd have to think carefully about edge cases? + if( upload_scenes && upload_files ){ + throw new BadRequestError(`Can't do mixed-content processing. Provide EITHER scene archive(s) OR source file(s)`); + } + if(upload_files && (!name || !lang)){ + throw new BadRequestError(`scene name and default language are required when creating a scene from a set files`); + } + + //Check that everything _should_ be error-free + //We might still have fails because of race conditions or _things_, but it is preferable to have a preflight check + const errors = files.map(u=>u.scenes.filter(s=>s.action === "error")).flat(); + for(let {error, name} of errors){ + logger.error(error ?? `Unspecified error on scene ${name}`); + } + + // @FIXME should we clean up some things immediately when we abort due to planned errors? + if(errors.length){ + throw new UnauthorizedError(`Insufficient permissions on scene${1 < errors.length?"s":""} [${errors.slice(0, 3).map(({name})=>name)}${3 < errors.length?", ...":""}] for this user. Aborting.`); + } + + const results = await tasks.group(function* createChildren(){ + for(let upload of files){ + if(upload_scenes){ + logger.debug("Extract uploaded scenes archive "+upload.filepath); + yield tasks.run({ + data: {filepath: upload.filepath}, + handler: extractScenesArchive, + }); + } + } + }); + return results.flat(); +} \ No newline at end of file diff --git a/source/server/tasks/logger.ts b/source/server/tasks/logger.ts index e5110f444..71b50ec76 100644 --- a/source/server/tasks/logger.ts +++ b/source/server/tasks/logger.ts @@ -1,7 +1,9 @@ import { Writable } from "node:stream"; import { ITaskLogger, LogSeverity } from "./types.js"; import { DatabaseHandle } from "../vfs/helpers/db.js"; -import { format } from "node:util"; +import { debuglog, format } from "node:util"; + +const debug = debuglog("tasks:logs"); /** * Disposable logger that queues messages and waits for all logs to be flushed when closed @@ -21,6 +23,7 @@ export function createLogger(db: DatabaseHandle, task_id: number){ }); function log(severity: LogSeverity, message: string){ + debug(`[${severity.toUpperCase()}] ${message}`); stream.write({severity, message}); } diff --git a/source/server/tasks/manager.test.ts b/source/server/tasks/manager.test.ts index 6b01591e6..0d50ce67f 100644 --- a/source/server/tasks/manager.test.ts +++ b/source/server/tasks/manager.test.ts @@ -9,18 +9,21 @@ import openDatabase, { Database } from "../vfs/helpers/db.js"; import { Uid } from "../utils/uid.js"; import { randomBytes } from "node:crypto"; import { TaskManager } from "./manager.js"; +import Vfs from "../vfs/index.js"; +import UserManager from "../auth/UserManager.js"; +import { BadRequestError, NotFoundError } from "../utils/errors.js"; describe("TaskManager", function(){ - let db_uri: string, scene_id: number, handle: Database, client: Client; + let db_uri: string, scene_id: number, user_id: number, handle: Database; let listener :TaskManager; this.beforeAll(async function(){ db_uri = await getUniqueDb(this.currentTest?.title.replace(/[^\w]/g, "_")); handle = await openDatabase({uri: db_uri}); - let s = await handle.get( - "INSERT INTO scenes(scene_id, scene_name) VALUES ( $1, $2 ) RETURNING scene_id", - [Uid.make(), randomBytes(8).toString("base64url")]); - scene_id = s.scene_id; + const vfs = new Vfs("/dev/null", handle); + scene_id = await vfs.createScene(randomBytes(8).toString("base64url")); + const userManager = new UserManager(handle); + user_id = (await userManager.addUser(randomBytes(8).toString("base64url"), randomBytes(8).toString("base64url"))).uid; }); this.afterAll(async function(){ @@ -41,7 +44,7 @@ describe("TaskManager", function(){ //Non-connected functions that should work well in isolation it("create tasks", async function(){ let task = await listener.create({ - scene_id, + scene_id: null, user_id: null, type: "delayTask", data: {time: 0} @@ -49,7 +52,28 @@ describe("TaskManager", function(){ expect(task.task_id).to.be.a("number"); expect(await handle.all("SELECT * FROM tasks")).to.have.length(1); }); - + + it("create a task attached to a scene", async function(){ + let task = await listener.create({ + scene_id, + user_id: null, + type: "delayTask", + data: {time: 0} + }); + expect(task.task_id).to.be.a("number"); + expect(task.scene_id).to.equal(scene_id); + }) + + it("create a task attached to a user", async function(){ + let task = await listener.create({ + scene_id: null, + user_id, + type: "delayTask", + data: {time: 0} + }); + expect(task.task_id).to.be.a("number"); + expect(task.user_id).to.equal(user_id); + }); it("update task status", async function(){ let t = await listener.create({ @@ -74,6 +98,171 @@ describe("TaskManager", function(){ let resolved = await listener.getTask(t.task_id); expect(resolved).to.deep.equal(t); + expect(resolved.data).to.deep.equal({time: 0}); + }); + + + it("creates a task with a parent", async function(){ + let parent = await listener.create({ + scene_id, + user_id: null, + type: "delayTask", + data: {time: 0} + }); + + let child = await listener.create({ + scene_id, + user_id: null, + type: "delayTask", + data: {time: 0}, + parent: parent.task_id, + }); + + expect(child).to.have.property("parent", parent.task_id); + }); + + it("raises NotFoundError when getting non-existent task", async function(){ + await expect(listener.getTask(99999)).to.be.rejectedWith("No task found"); + }); + + it("raises NotFoundError calling setTaskStatus on non-existent task", async function(){ + // the scheduler shouldn't assume the update succeeded. + await expect(listener.setTaskStatus(99999, "running")).to.be.rejectedWith(NotFoundError); + }); + + it("raises NotFoundError calling takeTask on non-existent task", async function(){ + // the scheduler shouldn't assume the update succeeded. + await expect(listener.takeTask(99999)).to.be.rejectedWith(NotFoundError); + }); + + it("raises NotFoundError calling releaseTask on non-existent task", async function(){ + // the scheduler shouldn't assume the update succeeded. + await expect(listener.releaseTask(99999)).to.be.rejectedWith(NotFoundError); + }); + + it("raises NotFoundError calling errorTask on non-existent task", async function(){ + // the scheduler shouldn't assume the update succeeded. + await expect(listener.errorTask(99999, new Error("some error"))).to.be.rejectedWith(NotFoundError); + }); + + it("handles database errors during task creation", async function(){ + await expect(listener.create({ + scene_id: -1, // Invalid ID + user_id: null, + type: "delayTask", + data: {}, + })).to.be.rejected; + }); + + + it("close() prevents further database operations", async function(){ + // PURPOSE: Verify that calling close() properly invalidates the manager + // and prevents accidental use after close, which could cause connection leaks. + const manager = new TaskManager(handle); + await manager.close(); + + await expect(manager.create({ + scene_id: null, + user_id: null, + type: "test", + data: {}, + })).to.be.rejectedWith("TaskManager has been closed"); + }); + + + it("deleteTask properly cleans up cascade deletions", async function(){ + // PURPOSE: Verify that deleteTask with cascading deletes doesn't leave + // orphaned records or cause constraint violations on subsequent operations. + const parent = await listener.create({ + scene_id: null, + user_id: null, + type: "parent", + data: {}, + }); + + const child = await listener.create({ + scene_id: null, + user_id: null, + type: "child", + data: {}, + parent: parent.task_id, + }); + + const deleted = await listener.deleteTask(parent.task_id); + expect(deleted).to.be.true; + + // Child should be cascaded deleted + const rows = await handle.all("SELECT * FROM tasks"); + expect(rows).to.have.length(0); + }); + + it("takeTask only transitions from pending or initializing status", async function(){ + let task = await listener.create({ + scene_id: null, + user_id: null, + type: "test", + data: {}, + status: "pending", + }); + + await listener.takeTask(task.task_id); + let updated = await listener.getTask(task.task_id); + expect(updated.status).to.equal("running"); + + // Try to take again - should throw. + await expect(listener.takeTask(task.task_id)).to.be.rejectedWith(BadRequestError); + }); + + it("setTaskStatus validates status values", async function(){ + // PURPOSE: Verify that invalid status transitions are handled correctly. + // The type system prevents "success" and "error" here, but "running"->pending + // should be allowed even if not semantically correct. Tests the type boundaries. + const task = await listener.create({ + scene_id: null, + user_id: null, + type: "test", + data: {}, + }); + + await listener.setTaskStatus(task.task_id, "running"); + let updated = await listener.getTask(task.task_id); + expect(updated.status).to.equal("running"); + + // Setting to initializing should also work + await listener.setTaskStatus(task.task_id, "initializing"); + updated = await listener.getTask(task.task_id); + expect(updated.status).to.equal("initializing"); + }); + + it("task output is properly serialized and deserialized", async function(){ + // PURPOSE: Verify round-trip serialization of various data types. + // Ensures JSON serialization doesn't lose precision or corrupt data. + const task = await listener.create({ + scene_id: null, + user_id: null, + type: "test", + data: {}, + }); + + const testOutput = { + string: "test", + number: 42, + float: 3.14159, + boolean: true, + null: null, + array: [1, 2, 3], + nested: { + deep: { + value: "found", + } + } + }; + + await listener.releaseTask(task.task_id, testOutput); + const retrieved = await listener.getTask(task.task_id); + + // Output is stored as JSON string, so we need to parse it if comparing + expect(retrieved.output).to.be.an("object"); + expect(retrieved.output).to.deep.equal(testOutput); }); - }); \ No newline at end of file diff --git a/source/server/tasks/manager.ts b/source/server/tasks/manager.ts index 29f10295b..3655190c8 100644 --- a/source/server/tasks/manager.ts +++ b/source/server/tasks/manager.ts @@ -1,10 +1,11 @@ import { debuglog } from "node:util"; -import { HTTPError, NotFoundError } from "../utils/errors.js"; +import { BadRequestError, HTTPError, NotFoundError } from "../utils/errors.js"; import { DatabaseHandle } from "../vfs/helpers/db.js"; import { serializeTaskError } from "./errors.js"; -import { TaskStatus, TaskData, TaskDefinition, CreateTaskParams } from "./types.js"; +import { TaskStatus, TaskDataPayload, TaskDefinition, CreateTaskParams } from "./types.js"; -const debug_outputs = debuglog("tasks:outputs"); +const debug_status = debuglog("tasks:status"); +const debug_logs = debuglog("tasks:logs"); /** * Contains base interface to manage tasks: creation, status changes, etc... @@ -12,69 +13,115 @@ const debug_outputs = debuglog("tasks:outputs"); */ export class TaskManager{ public get db(){ + if(!this._db) throw new Error(`TaskManager has been closed`); return this._db; } constructor(private _db: DatabaseHandle){ - + if(!this._db) throw new Error("A valid database handle is required to instanciate a TaskManager"); + } + + + close(){ + this._db = null as any; } /** * Internal method to adjust task status * @param id * @param status + * @warning it's almost always better to use the narrowed-down versions : + * {@link TaskManager.takeTask}, {@link TaskManager.releaseTask} and {@link TaskManager.errorTask} + * Because they contain more robust assertions about the task's current sate that might prevent a number of race conditions + * eg. {@link TaskManager.takeTask} will only work on tasks that have not already started */ public async setTaskStatus(id: number, status:Omit): Promise{ - await this.db.run(`UPDATE tasks SET status = $2 WHERE task_id = $1`, [id, status]); + debug_status("Set task %d to status %s", id, status); + let r = await this.db.run(`UPDATE tasks SET status = $2 WHERE task_id = $1`, [id, status]); + if(!r.changes) throw new NotFoundError(`No task found with id ${id}`); + } + + /** + * "take" a task that has status "pending" or "initializing" and switch it to status "running" + * @param id + * @throws {NotFoundError} if task doesn't exist + * @throws {BadRequestError} if task status doesn't match 'pending' or 'initializing' + */ + public async takeTask(id: number): Promise{ + debug_status("Take task %d", id); + const r = await this.db.run(`UPDATE tasks SET status = 'running' WHERE task_id = $1 AND status IN ('initializing', 'pending')`, [id]); + if(!r.changes){ + const t = await this.getTask(id); //will throw NotFoundError if task doesn't exist + throw new BadRequestError(`Can't take task #${id} with status ${t.status}`); + } } /** * Marks a task as completed * Output is serialized using `JSON.stringify()` + * @throws {NotFoundError} if task doesn't exist */ async releaseTask(id: number, output: any = null){ - debug_outputs(`Release task #${id}`, output); - await this.db.run(`UPDATE tasks SET status = 'success', output = $2 WHERE task_id = $1`, [id, JSON.stringify(output)]); + if(debug_logs.enabled) debug_logs(`Release task #${id}`, output); + else debug_status(`Release task #${id}`); + + const result = await this.db.run(`UPDATE tasks SET status = 'success', output = $2 WHERE task_id = $1`, [id, JSON.stringify(output)]); + if(!result.changes){ + throw new NotFoundError(`No task found with id ${id}`); + } } /** - * @fixme use task.output to store the error message? + * Marks a task as failed with an error message + * @throws {NotFoundError} if task doesn't exist + * @throws {Error} if error serialization or database update fails */ async errorTask(id: number, reason: HTTPError|Error|string){ try{ - debug_outputs(`Task #${id} Error : `, reason); - await this.db.run(`UPDATE tasks SET status = 'error', output = $2 WHERE task_id = $1`, [id, serializeTaskError(reason)]); + debug_status(`Task #${id} Error : `, reason); + const serialized = serializeTaskError(reason) + const result = await this.db.run(`UPDATE tasks SET status = 'error', output = $2 WHERE task_id = $1`, [id, serialized]); + + if(!result.changes){ + throw new NotFoundError(`No task found with id ${id}`); + } }catch(e:any){ - console.error("While trying to set task status:", e); + console.error("While trying to set task status:", e); + throw e; } } - public async create({scene_id, user_id, type, data, status='pending'}: CreateTaskParams): Promise>{ - let args = [scene_id, type, data, status, user_id]; - let task = await this.db.get>(` - INSERT INTO tasks(fk_scene_id, type, data, status, fk_user_id) - VALUES ($1, $2, $3, $4, $5) - RETURNING * + static #taskColumns = ` + fk_scene_id AS scene_id, + fk_user_id AS user_id, + task_id, + ctime, + type, + parent, + data, + output, + status + `; + + public async create({scene_id, user_id, type, data, status='pending', parent=null}: CreateTaskParams): Promise>{ + let args = [scene_id, type, data ?? {}, status, user_id, parent]; + let task = await this.db.all>(` + INSERT INTO tasks(fk_scene_id, type, data, status, fk_user_id, parent) + VALUES ($1, $2, $3, $4, $5, $6) + RETURNING ${TaskManager.#taskColumns} `, args); - return task; + return task[0]; } - public async getTask(id: number):Promise{ - let task = await this.db.get(` + public async getTask(id: number):Promise>{ + let task = await this.db.all>(` SELECT - fk_scene_id, - fk_user_id, - task_id, - ctime, - type, - parent, - data, - output, - status + ${TaskManager.#taskColumns} FROM tasks WHERE task_id = $1 + LIMIT 1 `, [id]); - if(!task) throw new NotFoundError(`No task found with id ${id}`); - return task; + if(!task.length) throw new NotFoundError(`No task found with id ${id}`); + return task[0]; } /** diff --git a/source/server/tasks/queue.ts b/source/server/tasks/queue.ts index 78e65009b..34f60a3b8 100644 --- a/source/server/tasks/queue.ts +++ b/source/server/tasks/queue.ts @@ -1,5 +1,6 @@ import EventEmitter from "node:events"; -import { TaskData, TaskHandler, TaskPackage } from "./types.js"; +import timers from "node:timers/promises"; +import { TaskDataPayload, TaskHandler, TaskPackage } from "./types.js"; @@ -13,11 +14,14 @@ interface WorkPackage{ export class Queue{ #queue: WorkPackage[] = []; #active = 0; - #limit: number; #c = new AbortController(); + #settleResolve: (() => void) | null = null; - constructor(limit = Infinity) { - this.#limit = limit; + constructor(public limit = Infinity, public name?:string) { + } + + toString(){ + return `Queue(${this.limit}, ${this.name || "anonymous"})` } @@ -36,12 +40,17 @@ export class Queue{ #onSettled = ()=>{ this.#active--; + // Notify waiters if all active tasks have completed + if(this.#active === 0 && this.#settleResolve){ + this.#settleResolve(); + this.#settleResolve = null; + } this.#processNext(); } #processNext(){ // Stop if we are busy or if the queue is empty - if (this.#active >= this.#limit || this.#queue.length === 0 ) { + if (this.#active >= this.limit || this.#queue.length === 0 ) { return; } @@ -56,23 +65,40 @@ export class Queue{ } /** - * Close the queue - * @todo Add a more graceful close that waits for jobs to complete instead of cancelling + * Close the queue gracefully + * Waits for active tasks to complete before aborting pending ones + * @param timeoutMs Maximum time to wait for active tasks. Defaults to 30 seconds. */ - async close(){ + async close(timeoutMs: number = 1000){ if(this.#c.signal.aborted){ throw new Error(`Queue is already closed.`); } + // Now abort and reject pending tasks this.#c.abort(); + //Empty the queue (work not yet started) for (let item of this.#queue){ - item.reject(this.#c.signal.reason); + item.reject(this.#c.signal.reason ?? new Error("Queue closed")); } this.#queue = []; - //@FIXME We do not actually wait for jobs to be cancelled! - //Jobs that incompletely implement the cancellation interface could still complete after close(). + //Return immediately if no jobs are currently processed + if(this.#active == 0) return; + //Otherwise wait for timeout to let jobs resolve properly + await new Promise((resolve, reject)=>{ + const _t = setTimeout(()=>{ + this.#settleResolve = null; + reject(new Error(`Queue close timeout: active tasks did not complete within ${timeoutMs}ms`)); + }, timeoutMs); + this.#settleResolve = ()=>{ + clearTimeout(_t); + resolve(); + }; + + }); + } // Optional getters for observability get pendingCount() { return this.#queue.length; } get activeCount() { return this.#active; } + get closed(){ return this.#c.signal.aborted; } } \ No newline at end of file diff --git a/source/server/tasks/scheduler.test.ts b/source/server/tasks/scheduler.test.ts index 0c29aff0c..6d8706ef7 100644 --- a/source/server/tasks/scheduler.test.ts +++ b/source/server/tasks/scheduler.test.ts @@ -2,26 +2,38 @@ // Tests for tasks management // The system is a bit intricate and hard to test in isolation - +import timers from "timers/promises"; import { Client } from "pg"; import openDatabase, { Database, DatabaseHandle } from "../vfs/helpers/db.js"; import { Uid } from "../utils/uid.js"; import { randomBytes } from "node:crypto"; import { TaskScheduler } from "./scheduler.js"; +import { CreateRunTaskParams, TaskDefinition } from "./types.js"; +import Vfs from "../vfs/index.js"; +import UserManager from "../auth/UserManager.js"; + + +const makeTask = (props:Partial> = {})=>({ + scene_id: null, user_id: null, data: {}, + handler: ()=>Promise.resolve(), + ...props, + }) describe("TaskScheduler", function(){ - let db_uri: string, scene_id: number, handle: Database, client: Client; + let db_uri: string, scene_id: number, user_id: number, handle: Database, client: Client; //Create a taskScheduler with minimal context let scheduler :TaskScheduler<{db:DatabaseHandle}>; this.beforeAll(async function(){ db_uri = await getUniqueDb(this.currentTest?.title.replace(/[^\w]/g, "_")); handle = await openDatabase({uri: db_uri}); - let s = await handle.get( - "INSERT INTO scenes(scene_id, scene_name) VALUES ( $1, $2 ) RETURNING scene_id", - [Uid.make(), randomBytes(8).toString("base64url")]); - scene_id = s.scene_id; + + const userManager = new UserManager(handle); + const user = await userManager.addUser("alice", "12345678", "admin"); + user_id = user.uid; + const vfs = new Vfs("/dev/null", handle); + scene_id = await vfs.createScene(randomBytes(8).toString("base64url"), user_id); }); this.afterAll(async function(){ @@ -38,7 +50,7 @@ describe("TaskScheduler", function(){ scheduler = new TaskScheduler({db: handle}); }) this.afterEach(async function(){ - await scheduler.close(); + if(!scheduler.closed) await scheduler.close(); }); it("creates an immediately-executed task", async function(){ @@ -58,7 +70,7 @@ describe("TaskScheduler", function(){ expect(task).to.have.property("output", result); }); - it("initialized a task for later execution", async function(){ + it("initialize a task for later execution", async function(){ let task = await scheduler.create({ scene_id: null, user_id: null, @@ -73,7 +85,7 @@ describe("TaskScheduler", function(){ expect(task).to.have.property("output", output); }); - it("handles tasks errors", async function(){ + it("handles async tasks errors", async function(){ let id_ref :number; await expect(scheduler.run({ scene_id: null, @@ -93,6 +105,26 @@ describe("TaskScheduler", function(){ expect(task).to.have.property("output").ok; }); + + it("handles synchronous errors in handlers", async function(){ + // PURPOSE: Verify that synchronous errors (not returned from async) are caught. + // This catches programming errors like accessing undefined properties without returning a rejected promise. + let id_ref: number; + + await expect(scheduler.run({ + scene_id: null, + user_id: null, + data: {}, + handler: ({task})=>{ + id_ref = task.task_id; + throw new Error("Sync error"); + } + })).to.be.rejectedWith("Sync error"); + + const task = await scheduler.getTask(id_ref!); + expect(task.status).to.equal("error"); + }); + it("use a named function for task type", async function(){ const result = await scheduler.run({ scene_id: null, @@ -108,5 +140,314 @@ describe("TaskScheduler", function(){ expect(task).to.have.property("status", "success"); expect(task).to.have.property("type", "testTask"); }); + + it("data is never null inside a task", async function(){ + let ok = false; + await scheduler.run(makeTask({handler: ({task})=>{ + expect(task).to.have.property("data").an("object"); + ok = true; + }})); + expect(ok, `Task seems to not have been run`).to.be.true; + }); + + it("won't deadlock itself", async function(){ + scheduler.concurrency = 2; + + let tasks = []; + for(let i = 0; i < 4; i++){ + tasks.push(scheduler.run(makeTask({handler: async ()=>{ + await timers.setTimeout(1); + await scheduler.run(makeTask()); + }}))); + } + await Promise.all(tasks); + }); + + it("nested tasks with limited concurrency don't deadlock (deep nesting)", async function(){ + scheduler.concurrency = 2; + + const executed: number[] = []; + + const result = await scheduler.run(makeTask({ + handler: async ()=>{ + executed.push(1); + await scheduler.run(makeTask({ + handler: async ()=>{ + executed.push(2); + await timers.setTimeout(0); + await scheduler.run(makeTask({ + handler: async ()=>{ + executed.push(3); + } + })); + } + })); + } + })); + + expect(executed).to.deep.equal([1, 2, 3]); + }); + + it("nested tasks get their parent's attributes", async function(){ + let children:Array = []; + + const parent_id = await scheduler.run(makeTask({ + scene_id, + user_id, + handler: async function({task, context:{tasks}}){ + await tasks.run(makeTask({handler: async ({task})=>{ + children.push(task); + }})); + await scheduler.run(makeTask({handler: async ({task})=>{ + children.push(task); + }})); + return task.task_id; + } + })); + + expect(parent_id).to.be.a("number"); + expect(children).to.have.length(2); + for(let i = 0; i < children.length; i++){ + const child = children[i]; + expect(child).to.have.property("parent", parent_id); + expect(child).to.have.property("scene_id", scene_id); + expect(child).to.have.property("user_id", user_id); + } + }); + + it("group runs function-items inside nest and preserves order (Promise.all-like)", async function(){ + //Also, checks for deadlocks + function* gen() { + for (let i = 0; i < 100; i++){ + yield timers.setTimeout(1, i); + } + } + + const res = await scheduler.group(() => gen()); + expect(res).to.have.length(100); + for(let i = 0; i < 100; i++){ + expect(res[i]).to.equal(i); + } + }); + it("Can group promises", async function(){ + const calls: string[] = []; + + const res = await scheduler.group(() => ([ + timers.setTimeout(5, 1), + timers.setTimeout(5, 2), + ])); + expect(res).to.deep.equal([1,2]); + }); + + it("accepts a generator function (callable) and preserves results", async function(){ + function *gen(){ + yield timers.setTimeout(1, 1); + yield timers.setTimeout(1, 2); + } + const res = await scheduler.group(gen); + expect(res).to.deep.equal([1,2]); + }); + + it("generator function runs inside nest and shares context name", async function(){ + const ctxNames: string[] = []; + function *gen(){ + yield (async ()=>{ ctxNames.push(String(scheduler.context().queue.name)); return 1; })(); + yield (async ()=>{ ctxNames.push(String(scheduler.context().queue.name)); return 2; })(); + } + const res = await scheduler.group(gen); + expect(res).to.deep.equal([1,2]); + expect(ctxNames).to.have.length(2); + for (const n of ctxNames) expect(n).to.match(/\[GROUP\]/); + expect(ctxNames[0], `Expected contexts to all have the same name`).to.equal(ctxNames[1]); + }); + + + it("handles errors in nested tasks without deadlocking", async function(){ + // PURPOSE: Verify that if a nested task throws an error, the parent task + // receives that error and can handle it appropriately without deadlocking. + // This is critical because async context nesting could easily cause deadlocks + // if error propagation isn't handled correctly. + let parentTaskId: number; + let childTaskId: number; + + await expect(scheduler.run({ + scene_id: null, + user_id: null, + data: {}, + handler: async ({task})=>{ + parentTaskId = task.task_id; + await expect(scheduler.run({ + scene_id: null, + user_id: null, + data: {}, + handler: async ({task})=>{ + childTaskId = task.task_id; + throw new Error("Child failed"); + } + })).to.be.rejectedWith("Child failed"); + throw new Error("Parent failed"); + } + })).to.be.rejectedWith("Parent failed"); + + const parent = await scheduler.getTask(parentTaskId!); + const child = await scheduler.getTask(childTaskId!); + expect(parent.status).to.equal("error"); + expect(child.status).to.equal("error"); + }); + + + + it("rejects new tasks after close()", async function(){ + // PURPOSE: Verify that Queue.close() properly transitions to a closed state + // and prevents new tasks from being added. This prevents memory leaks from + // accumulated tasks that will never execute. + await scheduler.close(); + + // Attempting to add a task to closed scheduler should fail + await expect(scheduler.run({ + scene_id: null, + user_id: null, + data: {}, + handler: async ()=> "test" + })).to.be.rejected; + }); + + it("many sequential tasks don't leak memory or resources", async function(){ + // PURPOSE: Run many tasks in sequence to verify that AsyncLocalStorage + // contexts and queue workers are properly cleaned up after each task, + // not accumulating in memory. This tests for classic event listener leaks, + // promise chain leaks, etc. + this.timeout(10000); + + const count = 50; + const results = []; + + for(let i = 0; i < count; i++){ + const result = await scheduler.run({ + scene_id: null, + user_id: null, + data: {}, + handler: async ({task})=> task.task_id + }); + results.push(result); + } + + expect(results).to.have.length(count); + // Verify all tasks completed successfully + for(let id of results){ + const task = await scheduler.getTask(id); + expect(task.status).to.equal("success"); + } + }); + + it("many nested tasks don't cause stack overflow or memory issues", async function(){ + // PURPOSE: Test that deeply nested async contexts don't cause stack overflow + // or accumulate memory. AsyncLocalStorage should clean up properly as + // contexts exit. This is a regression test for context chain leaks. + this.timeout(10000); + + const depth = 10; + let maxDepth = 0; + + const runNested = async (d: number): Promise => { + if(d === depth) return d; + maxDepth = Math.max(maxDepth, d); + + return await scheduler.run({ + scene_id: null, + user_id: null, + data: {}, + handler: async ()=> runNested(d + 1) + }); + }; + + const result = await runNested(0); + expect(result).to.equal(depth); + expect(maxDepth).to.be.greaterThan(0); + }); + + it("group can run tasks concurrently", async function(){ + this.timeout(500) + const results = await scheduler.group(function *(){ + for(let i = 0; i < 60; i++){ + yield timers.setTimeout(10, i); + } + }); + expect(results).to.have.length(60); + }); + + it("callback-based error handling doesn't cause unhandled rejections", async function(){ + // PURPOSE: When using the optional callback parameter on run(), + // ensure errors are passed to the callback, not left as unhandled rejections. + // Unhandled rejections can crash the process in strict environments. + let callbackError: any = null; + + const promise = scheduler.run({ + scene_id: null, + user_id: null, + data: {}, + handler: async ()=>{ + throw new Error("Error in handler"); + } + }, (err)=>{ + callbackError = err; + }); + + // The promise should also reject + await expect(promise).to.be.rejected; + // The callback should have received the error + await timers.setTimeout(10); // Give callback time to execute + expect(callbackError).to.be.instanceof(Error); + expect(callbackError.message).to.include("Error in handler"); + }); + + it("closing scheduler with pending tasks cleans up properly", async function(){ + // PURPOSE: Verify that if we close the scheduler while tasks are still + // pending, we don't leak resources or leave zombie tasks hanging. + this.timeout(5000); + + scheduler.concurrency = 1; + + let task1Executed = false; + let task2Executed = false; + + // Task 1: Will be running when we close + const runningTask = scheduler.run({ + scene_id: null, + user_id: null, + data: {}, + handler: async ({context:{signal}})=>{ + task1Executed = true; + await timers.setTimeout(100, {signal}); // Slow task, with abort + return "task1"; + } + }); + + // Task 2: Will be pending when we close (due to concurrency=1) + const pendingTask = scheduler.run({ + scene_id: null, + user_id: null, + data: {}, + handler: async ()=>{ + task2Executed = true; + return "task2"; + } + }); + + // Give task1 time to start executing + await timers.setTimeout(10); + + // Close scheduler - task1 should be allowed to complete, task2 should be rejected + await scheduler.close(1000); + + // The running task should have completed successfully + const result1 = await runningTask; + expect(result1).to.equal("task1"); + expect(task1Executed).to.be.true; + + // The pending task should be rejected + await expect(pendingTask).to.be.rejected; + expect(task2Executed).to.be.false; + }); }); \ No newline at end of file diff --git a/source/server/tasks/scheduler.ts b/source/server/tasks/scheduler.ts index 2abf6382d..be2cf62bb 100644 --- a/source/server/tasks/scheduler.ts +++ b/source/server/tasks/scheduler.ts @@ -1,68 +1,179 @@ -import { debuglog, format } from "node:util"; -import { Writable } from 'node:stream'; -import { HTTPError } from "../utils/errors.js"; -import { Database, DatabaseHandle } from "../vfs/helpers/db.js"; +import {debuglog} from "node:util"; +import { AsyncLocalStorage } from 'node:async_hooks'; +import { DatabaseHandle } from "../vfs/helpers/db.js"; import { Queue } from "./queue.js"; -import { CreateRunTaskParams, RunTaskParams, TaskContextHandlers, TaskData, TaskDefinition, TaskHandler, TaskHandlerContext, TaskPackage, TaskSchedulerContext, TaskSettledCallback, } from "./types.js"; +import { CreateRunTaskParams, CreateTaskParams, ITaskLogger, RunTaskParams, TaskDataPayload, TaskDefinition, TaskHandler, TaskHandlerContext, TaskPackage, TaskSchedulerContext, TaskSettledCallback, TaskStatus, } from "./types.js"; import { createLogger } from "./logger.js"; import { TaskManager } from "./manager.js"; +import { BadRequestError, InternalError } from "../utils/errors.js"; +import * as userHandlers from "./handlers/index.js"; +// Note: previously used stream/once for generator bridge; no longer required for Promise.all-style group + + +const debug = debuglog("tasks:scheduler"); + + +interface AsyncContext{ + queue: Queue; + parent: { + task_id: number; + user_id: number|null; + scene_id: number|null; + logger:ITaskLogger; + } | null; +} + +type NestContextProps = { + name: string; + parent: AsyncContext["parent"]; + /** + * Tasks concurrency for this context. Default is 1 (except for the root context). + * Infinity can be provided if we don't care. + */ + concurrency: number; +} + +// Work must be a callable that, when invoked inside `nest`, returns +// an iterable of promises. This ensures the promises are created +// in the nested async context (covers both factory functions and +// generator functions — calling them produces an iterator). +type GroupWorkload = () => Iterable>; +export function isUserHandlerType(t: string):t is keyof typeof userHandlers{ + return t in userHandlers; +} export class TaskScheduler extends TaskManager{ - readonly #queue = new Queue(4); + //Do not use "real" private members here because they would be missed by Object.create + /** Work queue. One should never use this directly */ + private readonly rootQueue = new Queue(4, "root"); - public get context(){ + public get taskContext(){ return this._context; } + public get concurrency(){ + return this.rootQueue.limit; + } + public set concurrency(value){ + this.rootQueue.limit = value; + } + + public get closed(){ + return this.rootQueue.closed; + } constructor(protected _context :TContext){ super(_context.db); + + //AsyncLocalStorage is here instead of as a class member because we want to only use it through the interfaces provided below + const asyncStore = new AsyncLocalStorage(); + this.context = () =>{ + return (asyncStore.getStore() as any ?? {queue: this.rootQueue, parent: null}) satisfies AsyncContext; + } + this.nest = async ({parent, name, concurrency=1}, work, ...args)=>{ + const q = new Queue(concurrency, name); + try{ + return await asyncStore.run({queue:q, parent} satisfies AsyncContext, work, ...args); + }finally{ + await q.close(); + } + } } - async close(){ - await this.#queue.close(); + /** + * Close the scheduler gracefully + * @param timeoutMs Maximum time to wait for active tasks to complete. Defaults to 30 seconds. + */ + async close(timeoutMs?: number){ + await this.rootQueue.close(timeoutMs); + super.close(); } + /** + * Retrieve the current async context + * + * Async contexts allow nesting calls to {@link TaskScheduler.run()} without risking a deadlock: + * Each nesting level gets its own concurrency context with a default concurrency of one. + */ + public readonly context: () => AsyncContext; + /** + * Run `work` inside a new async context with the given name and concurrency settings + */ + public readonly nest: (props:NestContextProps, work:(...args: T)=>U, ...args: T)=>Promise; + - private async _run(handler:TaskHandler,task: TaskDefinition, {signal:taskSignal}:{signal?:AbortSignal}= {}){ + private async _run(handler:TaskHandler,task: TaskDefinition, {signal:taskSignal}:{signal?:AbortSignal}= {}){ const work :TaskPackage = async ({signal: queueSignal})=>{ await using logger = createLogger(this.db, task.task_id); const context: TaskHandlerContext = { - ...this.context, - tasks: this.childContext(task.task_id), + ...this.taskContext, + tasks: Object.create(this), logger, signal: taskSignal? (AbortSignal as any).any([taskSignal, queueSignal]): queueSignal, }; - await this.setTaskStatus(task.task_id, "running"); - return await handler.call(context, {context, inputs: new Map(), task}); + const thisContext:AsyncContext["parent"] = { + task_id: task.task_id, + scene_id: task.scene_id, + user_id: task.user_id, + logger, + }; + + await this.takeTask(task.task_id); + return await this.nest({concurrency: 1, name: `${task.type}#${task.task_id.toString()}`, parent: thisContext}, handler.bind(context), {context, task}) } + try{ - const output = await this.#queue.add(work); + const async_ctx = this.context(); + //Custom name for work to be shown in stack traces + Object.defineProperty(work, 'name', { value: `TaskScheduler.payload<${task.type}>(${task.task_id})@${async_ctx.queue.name}` }); + if(async_ctx.parent?.logger){ + async_ctx.parent.logger.debug(`Schedule child task ${task.type}#${task.task_id}`); + } + debug("Schedule work for task #%d on Queue(%s)", task.task_id, async_ctx.queue.name); + const output = await async_ctx.queue.add(work); await this.releaseTask(task.task_id, output); return output; }catch(e: any){ + //Here we might make an exception if e.name === "AbortError" and the database is closed await this.errorTask(task.task_id, e).catch(e=> console.error("Failed to set task error : ", e)); throw e; } } /** - * Registers a task to run immediately and wait for its completion + * Registers a task to run as soon as possible and wait for its completion + * It's OK to ignore the returned promise if a callback is provided to at least properly log the error + * + * `TaskScheduler.run()` uses async context tracking to inherit **scene_id**, **user_id** and **parent** from it's context + * However those can still be forced to another value if deemed necessary. + * Whether or not this override is desirable is yet unclear. */ - async run({task, handler}: RunTaskParams, callback?:TaskSettledCallback ): Promise; - async run({scene_id, user_id, type, data, handler, signal}: CreateRunTaskParams, callback?:TaskSettledCallback): Promise; - async run(params: CreateRunTaskParams|RunTaskParams, callback?:TaskSettledCallback): Promise{ + async run({task, handler}: RunTaskParams, callback?:TaskSettledCallback ): Promise; + async run({scene_id, user_id, type, data, handler, signal}: CreateRunTaskParams, callback?:TaskSettledCallback): Promise; + async run(params: CreateRunTaskParams|RunTaskParams, callback?:TaskSettledCallback): Promise{ let task: TaskDefinition; if("task" in params){ - task = params.task; + //We allow externally-created tasks here. But we want to limit error sources so we re-fetch the task anyway + //If this ends up being used in practice, we may need perform validation that the given task actually matches the stored one. + task = await this.getTask(params.task.task_id); }else{ - task = await this.create({...params, type: params.type ?? params.handler.name}); + //We use context to inherit parent, user_id and scene_id + //But if different values are explicitly specified it's possible to "break out" + //Whether or not this is + const {parent} = this.context(); + task = await this.create({ + ...params, + data: params.data as any, + type: (params.type ?? params.handler.name) as string, + status: "pending" as TaskStatus, + parent: parent?.task_id ?? null, + }); } - const _p = this._run(params.handler, task); + const _p = this._run( params.handler, task); if(typeof callback=== "function"){ _p.then((value)=>callback(null, value), (err)=>callback(err)); @@ -70,6 +181,38 @@ export class TaskScheduler(params: CreateTaskParams): Promise> { + const {parent} = this.context(); + if(parent) debug(`Inherit values from Parent task #${parent.task_id}: ${parent.scene_id?"Scene: "+parent.scene_id:""} ${parent.user_id?"User: "+parent.user_id:""}`); + if(!params.scene_id && parent?.scene_id) params.scene_id = parent.scene_id; + if(!params.user_id && parent?.user_id) params.user_id = parent.user_id; + if(!params.parent && parent?.task_id) params.parent = parent.task_id; + return await super.create(params); + } + /** * Join a task that has been created through {@link queue} */ @@ -79,24 +222,28 @@ export class TaskScheduler(p: CreateRunTaskParams): Promise { - throw new Error("Function not implemented."); - }, - getTask: function (id: number): Promise> { - throw new Error("Function not implemented."); - }, - group: function (cb: (context: TaskContextHandlers) => Promise | AsyncGenerator, remap?: any): Promise { - throw new Error("Function not implemented."); - }, - async [Symbol.asyncDispose](){ - - } + // Accept either a generator function or a factory that returns an iterable of promises. + group(work: () => Generator, any, any>): Promise; + group(work: () => Iterable>): Promise; + group(work: () => Generator, any, any>|Iterable>): Promise { + const async_ctx = this.context(); + if (async_ctx.parent?.logger) { + async_ctx.parent.logger.debug(`Create tasks group`); } + + if (typeof work !== 'function') throw new TypeError('group expects a function (factory or generator function)'); + + // Run once inside nest so the iterable is created and consumed in the nested async context + return this.nest({ name: `${async_ctx.queue.name}[GROUP]`, parent: async_ctx.parent, concurrency: Infinity }, async () => { + const iterable = (work as () => Iterable>)(); + // materialize inside nested context so iterator.next() runs under nest + const arr = [...iterable]; + return await Promise.all(arr); + }) as unknown as Promise; } } \ No newline at end of file diff --git a/source/server/tasks/types.ts b/source/server/tasks/types.ts index c2964b986..5a8f5f550 100644 --- a/source/server/tasks/types.ts +++ b/source/server/tasks/types.ts @@ -3,9 +3,11 @@ import { Config } from "../utils/config.js"; import { TDerivativeQuality } from "../utils/schema/model.js"; import { DatabaseHandle } from "../vfs/helpers/db.js"; import type Vfs from "../vfs/index.js"; +import { TaskScheduler } from "./scheduler.js"; -export type TaskData = Record; +export type TaskData = Record +export type TaskDataPayload = undefined|TaskData; export type TaskStatus = 'initializing'|'pending'|'aborting'|'running'|'success'|'error'; export enum ETaskStatus{ @@ -17,19 +19,24 @@ export enum ETaskStatus{ 'success', } -export interface TaskDefinition{ - fk_scene_id: number; - fk_user_id: number; + +/** + * Task Creation parameters + */ +export interface TaskDefinition{ + scene_id: number; + user_id: number; task_id: number; ctime: Date; type :string; parent: number|null; /** **Unordered** list of task requirements */ after: number[]; - data: T; - output: any; + data: TData extends undefined? {}: TData; + output: TReturn; status: TaskStatus; -} +}; + export interface TaskSchedulerContext{ @@ -40,45 +47,64 @@ export interface TaskSchedulerContext{ } export type TaskHandlerContext = T & { - tasks: TaskContextHandlers, + tasks: TaskScheduler, logger: ITaskLogger, signal: AbortSignal, }; - -export interface TaskHandlerParams{ +/** + * Parameters passed to a task handler when it is invoked + */ +export interface TaskHandlerParams{ task: TaskDefinition; - inputs: Map; context: TaskHandlerContext; } /** * In the future we might want to support tasks that yield sub-tasks using return value `AsyncGenerator` */ -export type TaskHandler = (this: TaskHandlerContext, params:TaskHandlerParams)=> Promise; +export type TaskHandler = (this: TaskHandlerContext, params:TaskHandlerParams)=> TReturn|Promise; /** * Bound TaskHandler work package */ export type TaskPackage = (params: {signal: AbortSignal})=>Promise; -export interface CreateTaskParams{ - data: T; - type: string; + +type TaskDataRequirement = T extends undefined + ? { data?: never } + : { data: T }; + + +type TaskCreateCommonParameters = { scene_id?: number|null; user_id?: number|null; - status?: TaskStatus; + parent?: number|null; } -export interface CreateRunTaskParams extends Omit, "type">{ - handler: TaskHandler; - type?:string; +/** + * Parameters to create a task + */ +export type CreateTaskParams = TaskCreateCommonParameters &{ + type: string; + status?: TaskStatus; + data: TData; +}; + + +export type CreateRunTaskParams = + TaskCreateCommonParameters & TaskDataRequirement & { + handler: TaskHandler; + type?: string; + status?:"pending"; signal?: AbortSignal; /** Can't create an immediately-running task with a status other than pending */ - status?:"pending"; -} +}; -export interface RunTaskParams{ +/** + * Run a task that was previously created + */ +export interface RunTaskParams{ task: TaskDefinition; handler: TaskHandler; signal?: AbortSignal; @@ -98,22 +124,7 @@ export interface ITaskLogger{ export type LogSeverity = keyof ITaskLogger; -export interface TaskContextHandlers{ - create(p: CreateRunTaskParams): Promise; - getTask(id: number):Promise>; - /** - * Creates a group and encapsulates any task N° returned from the inner function as a dependency of this group. - * The group's output is an array of all its dependencies outputs. - * @param cb - */ - group(cb: (context: TaskContextHandlers)=>Promise|AsyncGenerator, remap?: any) :Promise; -} - - -export type GroupCallback = (context: TaskContextHandlers)=>Promise|AsyncGenerator; - - -export interface TaskHandlerDefinition{ +export interface TaskHandlerDefinition{ readonly type: string; handle: TaskHandler; }; @@ -124,23 +135,3 @@ export interface ProcessFileParams{ preset: TDerivativeQuality; } -export function requireFileInput(inputs:Map){ - let file:string|undefined = undefined; - for(let input of inputs.values()){ - if(file){ - throw new Error("More than one input could be used as input file"); - } - if(typeof input === "string") file = input; - } - if(!file){ - throw new Error(`No input file provided and none was found in task inputs\n${JSON.stringify([...inputs.entries()], null, 2)}`); - } - return file; -} - - -export interface ImportSceneResult{ - name: string; - action: "create"|"update"|"error"; - error?: string; -} \ No newline at end of file diff --git a/source/server/utils/archives.test.ts b/source/server/utils/archives.test.ts new file mode 100644 index 000000000..e2336281d --- /dev/null +++ b/source/server/utils/archives.test.ts @@ -0,0 +1,22 @@ +import { parseFilepath } from "./archives.js" + + + +describe("parseFilepath()", function(){ + it("ignores base `scenes` directory", function(){ + expect(parseFilepath(`/scenes/`)).to.deep.equal({isDirectory: true}); + }); + it("finds scene name and relative file path", function(){ + expect(parseFilepath(`/scenes/foo/scene.svx.json`)).to.deep.equal({scene: "foo", name:"scene.svx.json", isDirectory: false}); + }); + it("finds nested paths", function(){ + expect(parseFilepath(`/scenes/foo/articles/hello.html`)).to.deep.equal({scene: "foo", name:"articles/hello.html", isDirectory: false}); + }); + it("find nested folders", function(){ + //Also, strips trailing slash. + expect(parseFilepath(`/scenes/foo/articles/`)).to.deep.equal({scene: "foo", name:"articles", isDirectory: true}); + }) + it("find scene folder", function(){ + expect(parseFilepath(`/scenes/foo/`)).to.deep.equal({scene: "foo", name: undefined, isDirectory: true}); + }) +}) \ No newline at end of file diff --git a/source/server/utils/archives.ts b/source/server/utils/archives.ts new file mode 100644 index 000000000..660d9960d --- /dev/null +++ b/source/server/utils/archives.ts @@ -0,0 +1,36 @@ + +export interface ParsedFileEntry{ + scene?: string; + /** + * full path to the file from inside the scene's scope + * eg: `articles/foo-xyz.html` + */ + name?: string; + + /**True if file is a directory. */ + isDirectory: boolean; +} + +/** + * Parse an archive entry's name to extract its scene name + * @param filepath + * @returns Undefined if a file is definitely not scoped to a scene + */ +export function parseFilepath(filepath: string): ParsedFileEntry{ + const isDirectory = filepath.endsWith("/"); + const pathParts = filepath.split("/").filter(p=>!!p); + if(pathParts[0] == "scenes") pathParts.shift(); + if(pathParts.length === 0) return {isDirectory}; + const scene = pathParts.shift()!; + const name = pathParts.join("/"); + return { + scene, + name: name.length? name: undefined, + isDirectory, + } +} + +export function isMainSceneFile(filename: string){ + const name = filename.toLowerCase(); + return name.endsWith(".svx.json") || name.endsWith("index.html"); +} \ No newline at end of file diff --git a/source/server/vfs/helpers/db.ts b/source/server/vfs/helpers/db.ts index 8e62a7156..37d598943 100644 --- a/source/server/vfs/helpers/db.ts +++ b/source/server/vfs/helpers/db.ts @@ -24,6 +24,7 @@ pgtypes.setTypeParser(20 /* BIGINT */, function parseBigInt(val){ const debug = debuglog("pg:trace"); function safeDebugError(e:Error|unknown, sql: string){ + if(!debug.enabled) return; try{ debug(expandSQLError(e, sql).toString()); }catch(e){ @@ -33,6 +34,7 @@ function safeDebugError(e:Error|unknown, sql: string){ export interface DatabaseHandle{ /** + * @deprecated it is way more efficient to run `db.all("... LIMIT 1")[0]` because it will not deadlock as easily * Creates a cursor that will only fetch the first row that would be returned by the query */ get(sql: string, params?: any[]):Promise; @@ -120,7 +122,9 @@ export function toHandle(db:Pool|PoolClient|Client) :Omit { debug("connect to database at : "+ uri) - let pool = new Pool({connectionString: uri}); + let pool = new Pool({ + connectionString: uri, + }); pool.on("error", (err, client)=>{ console.error("psql client pool error :", err); From a65be904e7838afac1147e5c7fd596f9e383c334 Mon Sep 17 00:00:00 2001 From: Sebastien DUMETZ Date: Fri, 13 Feb 2026 15:02:26 +0100 Subject: [PATCH 06/14] UploadManager UI prototype --- source/server/templates/upload.hbs | 99 +++--- source/ui/MainView.ts | 6 +- source/ui/composants/Size.ts | 28 +- source/ui/composants/UploadManager.ts | 450 ++++++++++++++++++++++++++ source/ui/state/uploader.ts | 290 +++++++++++++++++ 5 files changed, 817 insertions(+), 56 deletions(-) create mode 100644 source/ui/composants/UploadManager.ts create mode 100644 source/ui/state/uploader.ts diff --git a/source/server/templates/upload.hbs b/source/server/templates/upload.hbs index 9e9109a5f..5d589814c 100644 --- a/source/server/templates/upload.hbs +++ b/source/server/templates/upload.hbs @@ -1,42 +1,63 @@ -

{{i18n "titles.upload"}}

- -
+
+ +

{{i18n "titles.upload"}}

+
-

{{i18n "titles.createOrUpdateScene"}} - {{#> popover id="upload-tooltip" type="button" }}{{i18n "tooltips.upload"}}{{/popover}} -

-
-
- - -
-
-
-
- - -
- -
- - -
-
- -
-
+ + +

{{i18n "titles.createOrUpdateScene"}}

+ {{i18n "labels.selectFile_s"}} +

{{i18n "leads.uploadFiles"}}

+

{{i18n "leads.uploadMixedContent"}}

+ +
+ +
+
+ + +
+
+ +
+
+ + +
+
+ +
+ +
+

{{i18n "titles.advancedConfiguration"}}

+
+ + +
+
+ + + + + + + {{i18n "leads.uploadErrors"}} + +
- - +
+
diff --git a/source/ui/MainView.ts b/source/ui/MainView.ts index 162af8116..391c2c808 100644 --- a/source/ui/MainView.ts +++ b/source/ui/MainView.ts @@ -5,7 +5,9 @@ import "./styles/globals.scss"; import "./screens/SceneHistory"; +import "./handlers/toggle"; + import "./composants/SubmitFragment"; -import "./composants/UploadForm"; -import "./handlers/toggle"; \ No newline at end of file +import "./composants/UploadManager"; +import "./composants/SceneSelectionForm"; diff --git a/source/ui/composants/Size.ts b/source/ui/composants/Size.ts index 2ab8a55b3..5effcac59 100644 --- a/source/ui/composants/Size.ts +++ b/source/ui/composants/Size.ts @@ -1,23 +1,23 @@ import { LitElement, html } from 'lit'; import { customElement, property } from 'lit/decorators.js'; +type BinaryUnit = 'B'|'kB'|'MB'|'GB'|'TB'|'PB'|'EB'|'ZB'|'YB'; +export function formatBytes(bytes: number, unit?: BinaryUnit){ - - -export function formatBytes(bytes, si=true){ - const thresh = si ? 1000 : 1024; - if(Math.abs(bytes) < thresh) { - return bytes + ' B'; + if(unit === "B" || Math.abs(bytes) < 1000) { + return bytes + (unit?'':' B'); } - let units = si - ? ['kB','MB','GB','TB','PB','EB','ZB','YB'] - : ['KiB','MiB','GiB','TiB','PiB','EiB','ZiB','YiB']; + let units = ['kB','MB','GB','TB','PB','EB','ZB','YB']; let u = -1; do { - bytes /= thresh; + bytes /= 1000; ++u; - } while(Math.abs(bytes) >= thresh && u < units.length - 1); - return Math.round(bytes*100)/100 + ' '+units[u]; + } while(unit? units[u] !== unit : Math.abs(bytes) >= 1000 && u < units.length - 1); + return Math.round(bytes*100)/100 + (unit? '' : ' '+units[u]); +} + +export function binaryUnit(bytes: number) :BinaryUnit{ + return formatBytes(bytes).split(" ").pop() as any; } @@ -26,10 +26,8 @@ export default class Size extends LitElement{ @property({type: Number}) b :number; - @property({type: Boolean}) - i :boolean = false; render(){ - return html`${formatBytes(this.b, !this.i)}`; + return html`${formatBytes(this.b)}`; } } \ No newline at end of file diff --git a/source/ui/composants/UploadManager.ts b/source/ui/composants/UploadManager.ts new file mode 100644 index 000000000..b03045ae8 --- /dev/null +++ b/source/ui/composants/UploadManager.ts @@ -0,0 +1,450 @@ +import { css, html, LitElement, PropertyValues, TemplateResult } from "lit"; +import { customElement, state } from "lit/decorators.js"; + +import Notification from "./Notification"; +import { formatBytes, binaryUnit } from "./Size"; +import HttpError from "../state/HttpError"; +import { SceneUploadResult, Uploader, UploadOperation } from "../state/uploader"; + + + + +@customElement("upload-manager") +export default class UploadManager extends LitElement{ + //static shadowRootOptions = {...LitElement.shadowRootOptions, delegatesFocus: true}; + private uploader = new Uploader(this); + @state() + busy: boolean = false; + + @state() + error:string|null = null; + + @state() + scenes + + connectedCallback(): void { + super.connectedCallback(); + // We register a number of global events that might influence app behaviour + window.addEventListener("drop", this.handleGlobalDrop); + window.addEventListener("dragover", this.handleGlobalDragover); + } + + disconnectedCallback(): void { + super.disconnectedCallback(); + window.removeEventListener("drop", this.handleGlobalDrop); + window.removeEventListener("dragover", this.handleGlobalDragover); + } + + /** + * Prevent default action on drop at the window level when files are dropped in the page + */ + private handleGlobalDrop = (e: DragEvent)=>{ + if(e.defaultPrevented) return; + if([...e.dataTransfer.items].every((item) => item.kind !== "file")) return; + e.preventDefault(); + this.classList.remove("drag-active"); + } + + /** + * Prevent default action on drop at the window level when files are dragged through the page + */ + private handleGlobalDragover = (e: DragEvent)=>{ + if([...e.dataTransfer.items].every((item) => item.kind !== "file")) return; + e.preventDefault(); + if (!this.shadowRoot.contains(e.target as Node)) { + e.dataTransfer.dropEffect = "none"; + } + } + + + + handleSubmit = (ev:MouseEvent)=>{ + ev.preventDefault(); + ev.stopPropagation(); + const form = this.uploadForm; + const data= new FormData(form); + if(!form.checkValidity()){ + Notification.show("Upload form is invalid", "warning", 1500); + return; + } + const tasks = this.uploader.uploads.map(u=>u.task_id) + console.log("Submit, form :", data, tasks); + this.busy = true; + this.error = null; + fetch("/tasks", { + method: "POST", + headers: { + "Content-Type": "application/json; charset=utf-8" + }, + body: JSON.stringify({ + type: "createSceneFromFiles", + data: { + tasks, + name: data.get("name"), + language: data.get("language")?.toString().toUpperCase(), + options: { + optimize: data.get("optimize") ?? false + } + } + }) + }).then(async (res)=>{ + await HttpError.okOrThrow(res); + let task = await res.json(); + console.log("Body : ", task); + if(!task.task_id) throw new Error(`Unexpected body shape: ${JSON.stringify(task)}`); + const u = new URL(window.location.href); + u.searchParams.append("task", task.task_id); + window.location.href = u.toString(); + }).catch((e)=>{ + console.error(e); + this.error = e.message; + }).finally(()=> this.busy = false); + //Reset everything + return false; + } + + + + + + /** + * Handles files being dropped onto the upload zone + * @fixme files is empty? + */ + public ondrop = (ev: DragEvent)=>{ + ev.preventDefault(); + this.classList.remove("drag-active"); + console.log("Drop :", ev.dataTransfer.files); + // @todo add the files + this.uploader.handleFiles(ev.dataTransfer.files); + } + + public ondragover = (ev: DragEvent)=>{ + ev.preventDefault(); + ev.dataTransfer.dropEffect = "copy"; + this.classList.add("drag-active"); + } + + public ondragleave = (ev: DragEvent)=>{ + this.classList.remove("drag-active"); + } + + /** + * Handles "change" events in the file input + * @param ev + */ + protected handleChange = (ev: Event)=>{ + ev.preventDefault(); + this.uploader.handleFiles((ev.target as HTMLInputElement).files); + } + + /** + * Proxy scene default title to prevent having to query slotted input repeatedly + */ + private _defaultTitle: string = ""; + /** + * slotted element selector to edit proposed scene title + */ + get nameInput() :HTMLInputElement|undefined{ + const slot = this.shadowRoot.querySelector('slot[name="upload-form"]'); + return slot?.assignedElements().map(e=> e.querySelector(`input[name="name"]`)).filter(n=>!!n)[0]; + } + + get uploadForm() :HTMLFormElement|undefined{ + const slot = this.shadowRoot.querySelector('slot[name="upload-form"]'); + return slot?.assignedElements().map(e=> e instanceof HTMLFormElement? e: e.querySelector(`FORM`)).filter(n=>!!n)[0]; + } + + + protected update(changedProperties: PropertyValues): void { + const models = this.uploader.uploads.filter(u=>u.filetype === "model" || u.filetype === "source"); + const defaultTitle = models[0]?.filename.split(".").slice(0, -1).join(".") ?? ""; + + if(this._defaultTitle != defaultTitle && !this.uploader.has_pending_uploads){ + console.log("Assign default title %s to ", defaultTitle, ["", defaultTitle].indexOf(this.nameInput.value) != -1, this.nameInput) + if(this.nameInput){ + if(["", this._defaultTitle].indexOf(this.nameInput.value) != -1) this.nameInput.value = defaultTitle; + this._defaultTitle = defaultTitle + } + } + super.update(changedProperties); + } + + /** + * Renders an individual upload operation + */ + private renderUploadItem = (u:UploadOperation)=>{ + const onActionClick = (ev: MouseEvent)=>{ + ev.preventDefault(); + ev.stopPropagation(); + if(u.error || u.done){ + u.task_id && fetch(`/tasks/${u.task_id}`, {method: "DELETE"}) + .then(res=> HttpError.okOrThrow(res)) + .catch(err=>{ + Notification.show(`Failed to delete upload task for ${u.filename}. Data may remain on the server`, "warning", 10000); + console.warn(err) + }); + this.uploader.remove(u.id); + }else{ + console.log("Abort upload: ", u.id); + u.abort(); + } + } + let state = "pending"; + let stateText: TemplateResult|string = html``; + let unit = binaryUnit(u.total); + let progress = `${formatBytes(u.progress, unit)}/${formatBytes(u.total)}`; + if(u.error){ + state = "error"; + stateText = "⚠"; + progress= `${u.error.message}`; + }else if(u.done){ + state = "done"; + stateText = "✓"; + progress = formatBytes(u.total); + } + + return html` +
  • + ${stateText} + ${u.filetype?.[0].toUpperCase() ?? ""} + + ${u.filename} + + ${progress} + 🗙 +
    • + `; + } + + /** + * When it looks like the user is uploading model(s) for a scene creation, show this form. + * The submit button is shown only when all uploads have settled. + */ + private renderSceneCreationForm(){ + const has_model = this.uploader.uploads.findIndex(u=>u.filetype === "model" || u.filetype === "source") !== -1 + return html` +
    +
    ${this.error}
    + ${(()=>{ + if(this.uploader.has_pending_uploads|| this.busy) return html`` + else if(this.uploader.has_errors) return html`Some uploads have failed` + else if(this.uploader.size && has_model) return html`` + else if(this.uploader.size) return html`Provide at least one model` + else return null; + })()} +
    + `; + } + + /** + * Renders the details of zipfiles contents + */ + private renderScenesContentSummary(){ + return html` + Scenes: +
      + ${this.uploader.uploads.map(u=>(u.scenes?.map(s=>html`
    • [${s.action.toUpperCase()}] ${s.name}
      • `)) ?? [])} +
    + + `; + } + + /** + * Prints a warning when mixed content prevents any action. + */ + private renderMixedContentWarning(){ + return html`Mixed content: can't proceed. Remove some of the uploaded files.`; + } + + protected render(): unknown { + const uploads = this.uploader.uploads; + const is_active = uploads.some(u=>!u.done && !u.error ); + const scene_archives = uploads.filter(u=>u.filetype === "archive"); + let form_content :TemplateResult|null = null; + if(scene_archives.length && scene_archives.length == uploads.length){ + form_content = this.renderScenesContentSummary(); + }else if(scene_archives.length){ + form_content = this.renderMixedContentWarning(); + }else if(uploads.length){ + form_content = this.renderSceneCreationForm(); + }else{ + form_content = html`Start uploading files in the box above.` + } + return html` + Create or Update a scene +
    +
      + ${uploads.map(this.renderUploadItem)} +
    + +
    + ${form_content} + `; + } + + static styles = [css` + .dropzone{ + display: block; + max-width: 100%; + border: 1px solid #99999988; + border-radius: 2px; + transition: background-color .1s ease; + &:hover{ + background-color: #99999910; + } + + label { + display: block; + cursor: pointer; + padding: 1rem; + } + input[type=file] { + display: none; + } + } + + + :host(.drag-active) .dropzone{ + border: 1px dotted #99999988; + background-color: #99999905; + } + + + @keyframes l1 { + 0% {background-size: 20% 100%,20% 100%,20% 100%} + 33% {background-size: 20% 10% ,20% 100%,20% 100%} + 50% {background-size: 20% 100%,20% 10% ,20% 100%} + 66% {background-size: 20% 100%,20% 100%,20% 10% } + 100%{background-size: 20% 100%,20% 100%,20% 100%} + } + + .loader{ + display: block; + width: 24px; + height: 24px; + aspect-ratio: 1; + --c: no-repeat linear-gradient(var(--color-loader, var(--color-info)) 0 0); + background: + var(--c) 0% 50%, + var(--c) 50% 50%, + var(--c) 100% 50%; + background-size: 20% 100%; + animation: l1 1s infinite linear; + } + + .upload-list{ + max-width: 100%; + margin: .25rem; + padding: 0 .5rem; + display: flex; + flex-direction: column; + gap: 3px; + + .upload-line{ + display: flex; + justify-content: stretch; + gap: .5rem; + + &:not(:last-child){ + border-bottom: 2px solid #00000010; + } + + .upload-filetype{ + font-family: monospace; + font-weight: bold; + align-self: center; + &:not(:empty)::before{ + content: "["; + } + &:not(:empty)::after{ + content: "]"; + } + &.filetype-u{ + color: var(--color-info); + } + &.filetype-s{ + color: var(--color-warning); + } + &.filetype-m{ + color: var(--color-success); + } + } + + .upload-filename{ + flex-grow: 1; + } + + .upload-state{ + width: 24px; + } + + + .upload-action{ + cursor: pointer; + &.action-cancel{ + color: var(--color-error); + &:hover{ + filter: saturate(0.6); + } + } + } + + &.upload-done{ + .upload-state{ + color: var(--color-success); + } + } + + &.upload-error{ + color: var(--color-error); + } + } + } + .drop-label{ + opacity: 0; + transition: opacity 0.2s ease; + display: flex; + justify-content: center; + } + + .upload-list:empty ~ .drop-label, + .dropzone.empty .drop-label { + opacity: 1; + } + + .dropzone:hover, + :host(.drag-active) { + .drop-label{ + opacity: 0.7; + } + } + + .scenes-list-actions{ + list-style: none; + .scene-action{ + display: inline-block; + min-width: 4.6rem; + } + .scene-action-create{ + color: var(--color-success); + } + .scene-action-update{ + color: var(--color-info); + } + .scene-action-error{ + color: var(--color-error); + } + } + `]; +} \ No newline at end of file diff --git a/source/ui/state/uploader.ts b/source/ui/state/uploader.ts new file mode 100644 index 000000000..69c0bfb82 --- /dev/null +++ b/source/ui/state/uploader.ts @@ -0,0 +1,290 @@ +import { ReactiveController, ReactiveControllerHost } from "lit"; +import HttpError from "./HttpError"; +import Notification from "../composants/Notification"; + +export type SceneUploadResult = {name: string, action: "create"|"update"|"error"} + + +// @FIXME use more like 100MB in production +const CHUNK_SIZE = 10000000; + +export interface UploadOperation{ + //Unique ID of the upload. Might be different from "name" when we upload a scene zip + id: string; + filename: string; + filetype?: "archive"|"model"|"source"|"unknown"; + /** The file to upload. Should be required? */ + file?: File; + /** When the file is an archive we will store the list of files it contains here */ + files?:string[]; + scenes?: SceneUploadResult[]; + //An array to be able to show a list of imported scenes in case of zip uploads + error ?:{code?:number, name?: string, message:string}; + done :boolean; + active ?:boolean; + task_id?:number; + total ?:number; + progress :number; + signal: AbortSignal; + abort: ()=>void; +} + +export interface ParsedUploadTaskOutput{ + filetype: "archive"|"model", + scenes?: SceneUploadResult[], + files?: string[], +} + + +export class Uploader implements ReactiveController{ + host: ReactiveControllerHost; + /** + * List of upload operations + * Shouldn't be mutated as it will trigger an update when reassigned + */ + uploads: readonly UploadOperation[]; + has_pending_uploads: boolean = false; + has_errors: boolean = true; + + + get size(){ + return this.uploads.length; + } + + hostConnected() { + //Initialize + this.uploads = []; + window.addEventListener("online", this.handleGlobalOnline); + } + + hostDisconnected() { + // Clear uploads when host disconnects + for(let op of this.uploads){ + op.abort(); + } + this.uploads = []; + window.removeEventListener("online", this.handleGlobalOnline); + } + + constructor(host: ReactiveControllerHost){ + (this.host = host).addController(this); + //Make a closure around uploads list to prevent mutation + let _uploads : readonly UploadOperation[] = []; + const self = this; + Object.defineProperties(this,{ + "uploads": { + get() { + return _uploads; + }, + set(value){ + if(_uploads === value) return + _uploads = value; + this.has_pending_uploads = this.uploads.some(u=>!u.done && !u.error ); + this.has_errors = this.uploads.some(u=>!!u.error && u.error.name !== "AbortError" ); + this.processUploads(); + self.host?.requestUpdate(); + } + } + }) + } + + /** + * Amend a running download operation + * @param name scene name that uniquely identifies the operation + * @param changes partial object to merge into operation + */ + protected splice(id: string, changes?: Partial) { + this.uploads = this.uploads.map(current => { + if (current.id !== id) return current; + else if (changes && current) return { ...current, ...changes }; + else return undefined; + }).filter(u=>!!u); + } + + public remove(id: string){ + let size = this.uploads.length; + this.uploads = this.uploads.map(current => { + if (current.id !== id) return current; + else return undefined; + }).filter(u=>!!u); + return size != this.uploads.length; + } + + public reset(){ + this.uploads = []; + } + + /** + * Listen for the global "online" event to resume downloads that had a NETWORK_ERR + */ + private handleGlobalOnline = ()=>{ + this.uploads = this.uploads.map(u=>{ + if(u.error && u.error.code == DOMException.NETWORK_ERR) return {...u, error: undefined}; + else return u; + }); + this.processUploads(); + } + + /** + * Handles a list of files that was submitted through dragdrop or the file input + * @param files + */ + public handleFiles(files: Iterable): void{ + const uploads = new Map(this.uploads.map(u=>([u.id, u]))); + for(let file of files){ + console.log("Handle file : ", file); + const prev = uploads.get(file.name); + if(prev){ + if(!prev.done && ! prev.error) prev.abort(); + // @fixme remove from server? + uploads.delete(file.name); + } + + uploads.set(file.name, this.createUploadOperation(file)); + } + this.uploads = Array.from(uploads.values()); + } + + protected createUploadOperation(file: File) :UploadOperation{ + const c = new AbortController(); + const task :UploadOperation = { + id: file.name, + filename: file.name, + file, + done: false, + active: false, + progress: 0, + total: file.size, + signal: c.signal, + abort: ()=> { + c.abort(); + this.splice(file.name, {error: {message: "Upload was aborted"}}); + } + }; + return task; + } + + + async initUpload(task: UploadOperation) :Promise{ + console.debug("Initializing upload task for %s", task.filename); + + const res = await fetch(`/tasks`, { + method: "POST", + signal: task.signal, + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + type: "parseUserUpload", + status: "initializing", + data:{ + filename: task.filename, + size: task.total, + } + }), + }); + await HttpError.okOrThrow(res); + const body = await res.json() + if(typeof body.task_id !== "number"){ + console.warn("Can't use body:", body); + throw new HttpError(500, "Server answered with an unreadable task identifier"); + } + return body.task_id; + } + + + async finalizeUpload(task: UploadOperation) :Promise{ + const res = await fetch(`/tasks/${task.task_id}`, { + method: "GET", + headers:{"Content-Type": "application/json"}, + }); + await HttpError.okOrThrow(res); + const body = await res.json(); + if(!body.output || typeof body.output !== "object" || !body.output.filetype){ + console.warn("Unexpected format for scene output:", body); + throw new Error("Invalid response body"); + } + + return body.output; + } + + + /** + * Process queued uploads up to a max of 5 concurrent requests + */ + private processUploads(){ + const inFlight = this.uploads.filter(u=>u.active && (!u.done && !u.error)).length; + if(2 <= inFlight) return; + const task = this.uploads.find(u=>!u.active && !u.done && !u.error); + if(!task) return; + + if(!task.file){ + Notification.show(`Can't upload : No file provided`, "error", 4000); + return; + } + + const starting_offset = task.progress; + const end_offset = Math.min(task.progress+CHUNK_SIZE, task.total) + const chunk = task.file.slice(task.progress, end_offset); + + const update :((changes :Partial)=>void) = this.splice.bind(this, task.id); + const setError = (err: Error|{code: number, message: string})=>{ + console.error("Upload request failed :", err); + update({active: false, progress: starting_offset, error: err}); + } + update({active: true}); + //Initialize upload + if(typeof task.task_id === "undefined"){ + this.initUpload(task).then( + (id)=> update({active: false, task_id: id}), + setError, + ); + return; + } + + let xhr = new XMLHttpRequest(); + task.signal.addEventListener("abort", xhr.abort) + xhr.onload = ()=>{ + if (299 < xhr.status) { + const fail_response = JSON.parse(xhr.responseText) as { message?: string }; + console.error("Upload Request failed :", fail_response.message ? fail_response.message : xhr.statusText) + setError({code: xhr.status, message: fail_response.message ? fail_response.message : xhr.statusText}); + }else if(xhr.status === 201){ + this.finalizeUpload(task).then((output)=>{ + console.debug("Finalized upload task. Parsed content :", output); + update({active: false, progress: task.total, done: true, ...output}); + }, setError); + }else{ + console.debug("Chunk uploaded. Set progress to :", end_offset); + update({active: false, progress: end_offset}); + } + } + + xhr.upload.onprogress = (evt)=>{ + if(evt.lengthComputable){ + console.debug("Progress event : %d (%d)", starting_offset + evt.loaded, evt.loaded); + update({progress: starting_offset + evt.loaded}); + } + } + xhr.ontimeout = function(ev){ + console.log("XHR Timeout", ev); + } + xhr.onerror = function onUploadError(ev){ + console.log("XHR Error", ev); + setError({ code: xhr.status ||DOMException.NETWORK_ERR, message: xhr.response.message || xhr.statusText || navigator.onLine? "Server is unreachable": "Disconnected" }); + } + + xhr.onabort = function onUploadAbort(){ + setError({ code: 20, name: "AbortError", message: "Upload was aborted"}); + } + + let url = new URL(`/tasks/${task.task_id}/artifact`, window.location.href); + + + xhr.open('PUT', url); + xhr.setRequestHeader("Content-Range", `bytes ${starting_offset}-${end_offset-1}/${task.total}`); + xhr.send(chunk); + } + + +} \ No newline at end of file From 1a92b5cd4f1f55c6f42af12edf158f0a98f4c581 Mon Sep 17 00:00:00 2001 From: Sebastien DUMETZ Date: Mon, 16 Feb 2026 10:36:18 +0100 Subject: [PATCH 07/14] task system: scheduler, handlers, routes, and upload implementation --- .github/workflows/build.yml | 2 +- source/e2e/__test_fixtures/Diffuse.jpg | Bin 0 -> 1318 bytes source/e2e/__test_fixtures/cube.blend | Bin 0 -> 116332 bytes source/e2e/__test_fixtures/cube.mtl | 12 + source/e2e/__test_fixtures/cube.obj | 39 + source/server/migrations/005-tasks.sql | 8 +- source/server/package-lock.json | 1084 ++++++++++++++++- source/server/package.json | 9 +- source/server/routes/scenes/index.ts | 2 +- source/server/routes/scenes/post.ts | 10 +- source/server/routes/services/opensearch.ts | 2 +- source/server/routes/tasks/artifacts/get.ts | 28 - source/server/routes/tasks/index.ts | 55 +- source/server/routes/tasks/post.ts | 57 +- .../server/routes/tasks/task/artifacts/get.ts | 29 + .../tasks/{ => task}/artifacts/put.test.ts | 61 +- .../routes/tasks/{ => task}/artifacts/put.ts | 33 +- source/server/routes/tasks/task/delete.ts | 4 +- source/server/routes/tasks/task/get.ts | 4 +- source/server/routes/views/index.ts | 59 +- source/server/scripts/obj2gltf.py | 93 ++ source/server/tasks/handlers/extractZip.ts | 73 +- source/server/tasks/handlers/index.ts | 7 +- source/server/tasks/handlers/inspectGlb.ts | 17 + source/server/tasks/handlers/optimizeGlb.ts | 139 +++ .../server/tasks/handlers/runBlenderScript.ts | 28 + source/server/tasks/handlers/toGlb.ts | 21 + source/server/tasks/handlers/uploads.ts | 394 ++++-- source/server/tasks/logger.ts | 101 +- source/server/tasks/queue.test.ts | 28 +- source/server/tasks/queue.ts | 82 +- source/server/tasks/scheduler.test.ts | 42 +- source/server/tasks/scheduler.ts | 141 +-- source/server/tasks/types.ts | 25 +- source/server/templates/locales/en.yml | 3 +- source/server/templates/locales/fr.yml | 3 +- source/server/templates/upload.hbs | 66 +- source/server/tests-common.ts | 2 +- source/server/utils/config.ts | 1 + source/server/utils/exec.test.ts | 28 + source/server/utils/exec.ts | 102 ++ source/server/utils/filetypes.test.ts | 18 + source/server/utils/filetypes.ts | 4 + source/server/utils/format.test.ts | 37 + source/server/utils/format.ts | 29 + source/server/utils/glTF.ts | 8 +- source/server/utils/gltf/inspect.ts | 75 ++ source/server/utils/gltf/io.ts | 19 + source/server/utils/gltf/toktx.ts | 297 +++++ source/server/utils/schema/default.ts | 6 +- source/server/utils/wrapAsync.ts | 2 +- source/server/vfs/Base.ts | 11 +- source/server/vfs/vfs.test.ts | 14 + source/ui/composants/UploadManager.ts | 98 +- source/ui/state/uploader.ts | 7 +- 55 files changed, 3137 insertions(+), 382 deletions(-) create mode 100644 source/e2e/__test_fixtures/Diffuse.jpg create mode 100644 source/e2e/__test_fixtures/cube.blend create mode 100644 source/e2e/__test_fixtures/cube.mtl create mode 100644 source/e2e/__test_fixtures/cube.obj delete mode 100644 source/server/routes/tasks/artifacts/get.ts create mode 100644 source/server/routes/tasks/task/artifacts/get.ts rename source/server/routes/tasks/{ => task}/artifacts/put.test.ts (62%) rename source/server/routes/tasks/{ => task}/artifacts/put.ts (80%) create mode 100644 source/server/scripts/obj2gltf.py create mode 100644 source/server/tasks/handlers/inspectGlb.ts create mode 100644 source/server/tasks/handlers/optimizeGlb.ts create mode 100644 source/server/tasks/handlers/runBlenderScript.ts create mode 100644 source/server/tasks/handlers/toGlb.ts create mode 100644 source/server/utils/exec.test.ts create mode 100644 source/server/utils/exec.ts create mode 100644 source/server/utils/format.test.ts create mode 100644 source/server/utils/format.ts create mode 100644 source/server/utils/gltf/inspect.ts create mode 100644 source/server/utils/gltf/io.ts create mode 100644 source/server/utils/gltf/toktx.ts diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a4237adcb..7be08d58a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -38,7 +38,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - node-version: [16, 18, 20, 22] + node-version: [18, 20, 22] services: postgres: image: postgres diff --git a/source/e2e/__test_fixtures/Diffuse.jpg b/source/e2e/__test_fixtures/Diffuse.jpg new file mode 100644 index 0000000000000000000000000000000000000000..0bf7d2a47322c425d067c9242104f8eef2d58983 GIT binary patch literal 1318 zcmex=|EI`$@KzRlhK~^C}Lq|5@z(jVXLJ_0J zi3>TDoi-j64Z8S2#W<;`iIYoATtZSxRZU$(Q_IBE%-q7#%Gt%$&E3P(D>x)HEIcAI zDmf)JEj=SMtGJ}Jth}PKs=1}Lt-YhOYtrN?Q>RUzF>}_U#Y>hhTfSoDs!f}>Y~8kf z$Ie}c4j(ys?D&b3r!HN-a`oEv8#iw~eDwIq(`V0LynOZX)8{W=zkUDl^B2fpj10^W zZvjcH{t^WGi;0DWnS~wXFGi+vAZ8Y1VO2C_6LJh>Pb?HxGHT=yahkYr<3UbkKb$@|Xn~>=`aC>IztMoQu=$(Bw?* z4MJVTi`|Y06wc$6&D^qB)nuvog2}$Ia;2`KDq^4S?DVcknc8qN;>yP3>qTyR#`YIk zX@~igrQJo@IzEgcCLOd8>+54Ycb*X2{r73aEnak6vbVY(yrddoc*`&o(IL%@e z%j`1e7b|ktCGYBtmGso&*2sB#J~=qZ*R;g)P{8D#sk68=7R^-JsB^SGsMyn0WRhnj zYQ6Fd&TR;DbIjN=k&a4 zsacaUS1&v#)G{Y!%|)OCgBZ5a}_NH>>MQh`4_%FY+C+N6B zz?&6sT3s6C7JvEkYOTgeUrpB+X0yr?T}47UCOszF&I3m&jdJ-tEJ$S=!EW6311 z6Xv#JU7C};WxRJqiZWW9mI;>f-L3c~#O%7^U6*w|OQy0;F)Mq%apg+2S-WH&YIq*= zI*T=K%_i+uCoXP-TGK27JimcbLx-2}$4K^s#xf?9 IH1Pi>0A^eSTmS$7 literal 0 HcmV?d00001 diff --git a/source/e2e/__test_fixtures/cube.blend b/source/e2e/__test_fixtures/cube.blend new file mode 100644 index 0000000000000000000000000000000000000000..6d6a3683637f04101edce409fcfe8577009bff7b GIT binary patch literal 116332 zcmV)nK%KuRwJ-gk^aKF_9TWj-S40v=@QiK_Nd8)6F5D)vD)YHoq;7#HvI$>@36&a; zn$sG7@Gkk$8&G85_N)KIdszWTh8%8(NXoW$fjI{KcVM9J6^|&v3PJ1y0TTfp0jq_D z#e+bgcK3FD}F%Mx#AURWTUMk47Vh!C=3xYe`ATlaP>HE|>4<==jxY)w9{` zU$56=j5nLjn@pzfZtszikylmK<#JtITs%pVpQh=$t}m0x-sN&(j13qt@G6z6RaN~2 z2@-tH`868NpXcfAtyb$pq0pyk+TQKm?d{&}?d|RD-tOMr-R|w)-Rn zcvIjjHXM(0ew;6>)rw_t@4D`exBFR&`!%8NQwF(*5#9HFd%a%Hxk7{E2W)&^KOWh^CeBYy=gRB z`?`;EfO`%Kw>KlOUlMeAJRZilBD3)n9@0~Eklv!{boXpFd!^uhwZeXE*#1kU-R*Wc zXA4I2*aG7RYHRpXR}!* zlX;WLN0?-vq@ zzPnvj)nG6fv{)>o(TIeEFq!u@OeC5 zMp1OP+qJ{@c|f1f_xt^F&Xx?$V~i0XK=9q|ZQFJO_tyoxo-f3cDqT?_qEJwKM zfu}^oI0_g8jF|xE2n{&i2hccL@NkkSLF0%4fEr`06$rFJfUD-%l6LQIcW>`@ckgy@ zZ}0Z*c5iog@9y^YcK7a1)6`F=(`vQqQmN#wQmNi#GI^`jGEK8AOBRcjN~Qih&(Ub4 zQmNb>MbWYhg+e4rrc$X^nm!7JI??~&ir>)|Ka|EE1aqI`h42TwcHqE)QmK@tX|Gl* z6h+Na$b+P@FEFg|8Y>uHqln=>x(83DMxI0}pF|>sLLrhQQ7BZm%jFUZg({WmbUK}; z7r}BrO4T<@r7ue%Pqx$-rLrf1fPj!BDH4e|I5_@txw!kt$VjnRBoc|Q*Xwi&n6s#K z-2npt3IGCt0096q7XU%ofR;MRY5){~FcAm_r5+50({U6jG6p0BNB~d(Fu(u+K?C_Q z08U&l-dAoU+^+#3uxrpdg8-cKqwwhp(;bg2b#I5?Kk=QmvQKOx+`bL&t5HXP_rqV` z`$HqkcDkG65mP-He+Y_q4NL2>gjpK2VZmSMOuMqrWc8S%8J~XO%|N*K{XGXqe_~!} zdJDWkPMAt~a0ROIzQc-6aC3TT-ryUTah`zCUmBELMD^3#$2B~h_WGv-dXw96Kq1a7 zhy6G3J?OyKgSGpXGX4_q#;zT2|6_5Nu1LW54@3-m1Q zy$$Ls+Io-5v9^=2#li74PQYDz`cFst=<3O4Q=A^PgShbH^d?Osa_}EsM_@#G>CL)u ziJcvP@?p3~624@%M{xIB#hRL2z07IxxnoMRaLcIl7iv3$e8Q6YZEwp@kN7Um<#3Hn z+8WrN|6cvMrMePVa(RJ#Q7*oGettQJ&NOq0$yNtZ=apJuTIyvYoX-d0kJ%b-@o&$5 z?c#g8_l5i{Jq&;5z5<9bvo2tDYX0|QWpe(K{>`s&BA*X-A+Jnwc5UBnv+>lPL#=Er z%j8W+FGVAR8WdHm|nRr3Bry1oa3J`LxSkz(l_HvK)PSZo4 z5aGlfed6ZS=>9YP+zYyJRk>x~#dGk>IYdABo$=Q4Tn=Y<+($y@H=1(8ZS#ZCReADm z^Z(5o<=I0c@yg zT7M1i$&ro8!5hC|CW867cA)G1%{#4z9lA1d?<#)^&Hi79&zk>u`R7J0I_zGnGFQnQm%nUAlE_V~@bENzBL}e7q<-^<*3R#)I%v&rRdj{5?jl`-R^H z9rtRV?(tY!U)6q~K%OIGW{3Z7-<9xr^vP=br)rU;9sc@P>0RtrQyY6B594;|F-rM}JZ}-hlgfRMVbp`O*Pc@mo_M|at-sj( z%*O2=x+hRK>QR>jktwXfG3n{Pbb}d#{jB zfxw?KE8DrSya%tocG{Hizi+k@`GIk5+L zO<+g)OLg}57!KTriF5=E}hmO3rdLKUS z@^X3Gjj)k+Lgq?Pui|JTIp;mJkBs;B99cBZ8eBe>8Aiw6RJhO2>43)h@v&1D9o-#O zGPH<%Xi z&>T~r{Ql+G7kr<0k(gZ>%E?9A`OimF%Zqw0v0$FBUtZYzGly~QcqB(KYZDf4;en<5 z$nfd=e}Yt<_Q;A&8{$NZr5>)>e(1B#m#BQAYaHjLdxyb>6@D^<$+_5>s=K}}J>a$0 zN9`;5^?HaGbNUb1d4r31o^vQN&_>4N?qHQ>S3HEd zPrCTzV~JkBk9(AF+-Ss0Fs=i6dzORidLI6VG#(xA%P-73`+Or^ypS_L4KX?P&d$FB zLv`oda&tny8>-IW8rSHLCQQ^VzJ|)<{?6t0ANlP8-dUe>$e5(3&N>!GvOhQHxv4d| z?nK-o$&Ck>;`GJ5)Wx|fZy7%@veXx&SZ`{skhd+i; z_xr^$fWJ$!$^07pGM#e*c9CN{uV>{?_ZxXb*LLjKBV|2fZ|~U@Ugi4#!(0PC_@*%W zy^hV&-fiGgR%wxarj7SU#$$zi_@ugQGwC=_n3T-c_^x;!HS^f4J09b~IZnMF0Z#Q0 z)6YMNPTyqv9A={jrx9OoxEK9LmIpF<6TpLGX8;H z^dep-O&W*mb`OzqSIQS_`=D{c83a*bzRL?QxaKaKi~yW^!Gwgv6?t{cJe|q-%MA4Y z>}kgz+ZFUi|Lnc}Y33@j;cxsU_p|Tsf1=*)unp!dm1&*gYdP$2_1o_g&JFU@O@dvE z_M5SdkH1v(Z!W}Lo(BDckvpJVY*{>5E$hu?&CCZe)=4jX9SL82zvn*Q)ZP1j!ohaE z)}&@|06kMaJThfoxrloae$PKU9ji5-LqBbe%VVu))vh;lf&-E4 zK7PG6=T^q@kX;Y>FYx&t>}B8IxDyP$D?ksiJpRVDYt11fJG-NK^#@(Jd^Dm5NM9}x3nnj2u0jyqGbhe2wIWu9~N;m-e*qnOJ}f8&=CJACJS zk6ZNb^A8_0^0$CLob^oOFFqhXf#U=S9-RIt!VTgGYl3`~GhfXdC7J#A zd!6Owz=u$efF0wN{E6ntxEiwsEV4Pup3XNu0KZk5@FDp&J5OvpK;@f2#1GK(`H)L- zv$37qbPgk)=6Vc0EHVS1o#A4xUiju`uF5`?tpa(Uhcd(S*rhIuiL^^JQxJ*Po7vM)SOF8cKpP$;!jO)_<*v#oI&tn*}tpDL34ep2GstYiJrl2eQ^ucqQ zFxj*Rruy#>rq^0o%cq~h?P3&c?)RS7OXktPFHbDLo%|CF~~neD?R*tvv%=9@d4C;$KLEdbB(1w7~~ zFWAp{-1pk?IXtt6DelVy@`B>`+!*%NLe&7G2SE1(h42N-E?BTK$T}-(V`u7!1F}nV zQrzDBzR&wsUT+0%1pWa7EV>QQeYkaaPUs!Q^O*k`UYm5ouD9=7 z-{PlEIt%_Msm!-`TG=|lYc?5AWWEVF7RYLvyC=Yr8!kIloqZw7o1 zdl!N?Q+M)_bm+CT=LYVWK_2aUxx&jY%@P0IyEzQN3G@kz+)aKxZaku~$1_q9gns50JtnXPc|6q3Fme?-U z{|EZ^1NK3dYrBie|HF>I>bYMTtU9yIso)bKR~=m)!ee*j%I*)YrUtB`hQLO z`;ZIfK{F?wH(mZ;e)xb&*)+#qD?NPdVBdHZnM=wA?S27qKOq0M*)19EGwTq^I$OQiA0#Yj z|NJ%Fo5)^Hmzht$yPH3^O&!C82<;auA`G4-hF%JTdhcQao)JTCIa`+>+jAkGN5_j))xNZ ziDmMEpv}jM3>fRA%m343e2=Tq;jr@kbkMA)bM4Vw?Xyol0ZYu`i8~e340!Ca3qMq` zcxi!o$?|_M=szI)KweS68M!ZzisrA@+i`g)RQ!iFuIr-`y!_NSEc`zN1880POS6Za zUd$J_C+hK#zMt#)_LjaNpQ=Pkb6@cSAEfS0AdvGP>+NwJ1x6-0 z-coDx?Y3wN@yC@|^2#yJ8YcHQ=oa1Od{*_H%ghaYm%qM-@TH&R+rd;%73n|~Z#V;& zjacvk*<_EnSM+W&pY8+f%TN^Tk~P&o%=(5#iQZ_XoSv`+&ydDEBL`j3eg5f+E}usS zj}r4EF$D~F0*~1BrDi(s0Q#+hi#SP~r9RyJGDexh9${WTKR(T3{qRPnu4SBeAok?I z8V`1d6V{%d*u@D_aM)gu5DEb6&>=Avi3ixNzkkZl+wmVellaGoKK|R>60w=6{sTLw zsQ9BTid@nd(gyyRJ%3-e#NXr@(xBsW%l422X-U(PS9n){%Kvn|EM@+Ws;vQ3mSPGnRvDCeBv}7Bkza&9;4!Kq3&{k zH;!!4eYXwPb?>yRb6a^t)*~)+Ig0q+9s7OW`B`7~7GL=j?P@uIqU;2#x$}4beS^pW zYjC`8DSfumFH#vj{SAlxh%nYn^Ej9Dd!Uq_CvS5-#u^thH64yEGtZ5j9X`8}Z|A}&M~SarWGXB!oR9yQ zLN0iz6N-F5rh#we?pkX5uge}OT-NnA)UEJoR3`J6Gvece%iP!bIoo`)#aQi6;gQ3t zi3%Smr)Ds*y=88-4Y-3J@KCPjN;%NdbqBPG{vK*wBBX{L-Y0rJX3$Am5`_1(=is*$ ztlF${6a&++?{QvsX_pP-S54-$bS7k<s*yet|yQ@9DD#TK7lcmivGA zZ|akV!}#sDEC2IO2G5S{k3g@QFIsU5N+%75|M|A{A&;LAGA;!2^AHU%o!?I94Ik&E z^6z-Sn)Iypg<8O{d1gRkc4`$l7C@- z=lbg5NJ6jI2$g@jOtv1deS0qT_;;S%1G`6h;NvMZj_BO!zdbwO)V4!m5sY^e(S)ofN51(HjJgEo@8)-^4@gX^-Hmr#Y0!=5w`yN@ z8#x@`v%bP)rM|pX@AC0i8>iHVclqwWwl%OrVA7pUUS{W$8|YqZr#^V82f`kg^_?67 z|9I5Bo$I)p+ZR6b;9~Q#PU6(*pM&SAX~c)-1^9l^ z_TwA1JUY*L43|HqSLc7vWtf~sza_MJA^M4-Z2!&qvP8M}_1}Y^?EHb{gzx{d**IJv zKELeEIBzEOd+hIq>v|WQ=<~LociHJ@2HzVUnl8KOYq_%byF6rAeir(%XZpvP`r}Qt zU*gh!8=m^0xcs@^Kid0me;IYNZsO86|H@}2$0L|wTNL*dR>R=IvBOtA{-T3Q@v}DG zdvsP0MxR?A^a~E!+cN|C)jSaA9yliQ2Jvob{jh=FBcHn3cxEQNFRd@88C&&2tc9b~ zo_p@L4tnK#{lI6#h=;VhVEVj}bnqQKB;~O_>@Aa7`$b@ioa1J<*4pS&i%|u*r~lwH zt3FYlJ@*HN8%WxR)SPb*`ivS4?)ftZuYI%eWgI&H8yrM< z+|l34@&DA8{qhPY{$=(ytW^1(n3}#}UGs7O<6Apq--j{Xg~r7$z!&l#Jn;YIM<^ee zjfZ5nN)VqGvp6{0ammitZ-PH{_8W_PJk7bQv%@nS{S=Rs?K^+OXvW&+HPCON0b6*0 z%YOqmO}_m;!9CoBGhFK0uXKdsU%cO!ZHPF+zsY}-6M#kg&bYg~Jcj!YzBW*sW+f*F z&||n-+b18eGX9R0$+awat`A=YSnTEJdoAjBudtR+4D}rKJ%G%MpmHeuvEY31dzN?! z?h_!BuwdIBcydPcHiKRu^T+!K_^$7hw~Xr}Db&6Q`G^lZ9i2=c`%->GCfNgUH#owd zH=&(&;P-^}`7wR+{l_@m#=3`3$|BbDJwk(XUi`*p`q&Ngjb7_Q>kz>yyY$DmSs<9ewQda#>0O9MhL?rEjz51m0C(?#bsU6R-I_4%j!; z$awtTa@cnuUuV<9egK}N>Pt`mde%rzxb0CU=s2LC$Y^r+DF6PF&IP_cKPP&Jx5Iaa z|J46*rXOj3HfYkTzJ5T5ME`u^mQIStZztaGm~r4XymFCV`0_dnYqrD1_WQqX=H6wW z>GeOWxsd-Y?tm|WqhWoA|3qC~f9+$FF-Uj@-tGY&+4lL+Dtlj@8t`rIZ8UxF)2G|k z_AtEO@)8H|m*9fSzQ`Z%7kuf@|Ft9F_9*ZOXP@YIOP0+(u^)X-yyED6bjf?txP4{Py~GT=S36eOde^r0LVuBFc%E<6iFnp4LE`LyHW-^e_WiH@m{t!RD|X z)UXAdr9T2UnIqE}C084Vt)5Aa87j-Zji0Um&26}SZBKgz>aVQ#U%QcvUa_bqi545h zxaQn_0dWNO42~U89wJf9-%{DkGEZDc^l1%?u{LjDOTU^c2!yEIn|U^7-m3rpd()ye zt2-XZ(c(Zrr@eYfyPDQTGzC)WVWK>Q<2Ysq1ur|IX9V#4LGszBcNf!Y*Yc}(@bYcP zr0=DuO>^{s@BwVEMZ}J0#`)HZ7FtK$t*) zfBv%>nmB)5%Y)pf5pzU6Y%Kr-ciS59vOKi;1g4wPSoh%WRi^MYMJKr(ZG*Q8t?o3D6DJ>o*hhbkH94oO_%qaDtL$CsMsllPtMUBCp{oG*x`oqYYu z^qo!NAC`4a(r3iJH2}8 zBAqEkrIEG=^m4tj*LV_I_kMogOeR5x{gdxI`r(04XPyI3sO1v|0)Ef8 z%XiGL_bDsMp&gyOT{KVFztTbdCcAl7+mk!u1{hrW2dr)M-1~?oV#Q@=ZnCLx(BqLB zOvArzFL6pPLp!oxU%{H+j{0Z9`q-mAi8SMjs^7jfyTx$-pOd`XljkGYc@O&I>h$~HnA=XQ&_=dy|>B@ZKq^R z!rg+yo<4SS`j9>9>gX7@jDK&(*Zh7{UhH>iMyq$0@pe4A4t;E9^PQlh^GDNOmleJR zx9I(oy)6{c{zd8^DSxWR>;&@#zltoPkqJxuy{7)`+1p%qXAO7B4vyq32XYwh?~mqB zb~HY`i>J+-=T~k5_3eIVn2nd-w$=1@?vJblyXi>$*O>LXFT9Bt9=~htvCH+yY<+)> z>x`2b{*&YSbe+>d&JQTHFa2N`6ult;q#ajH006KB001DW<1N8|004gj00031S4~%G zZcSls00RIZ00aPFYU%^X8ov)nt-vU!6)*n(|Nls1p&Ce9>&RQ{2pIbOFNjC|FUZSv zm>uCQ0|Nj65HkTk&CE=IKY%?w0000A*CRXt0D443K~zC_0f2jdAR_r7#8XXHX=Qgz zVR>b8poR3{pocKo;sDooeJ>z(DfKOY2loR30RsR500032005Z*003nI0Eo)+Sb3lY z_yN^e6jLsUtXJp|2w%4|IxPu|pbhMxx#KMepaS5>0w9L;+yh|t1E7Bgpx|KOActUw z6kwo#2cUlkC;-Ud*y50oKtS-*;yCI$c*uuw$+0LVcC00RRA000CE04D|-000PN z|1=mh^)E<#@`!u#XrW(d^53YcfT%-&tV37<0LDUD0U9R;Bme{;Au-5$Xt7A~>n|9p zfI$ENK=4HHRz-jZ0^9=u2xG`7cqoEk3L+H%AQJ!p02US$Ky8pFl%WAqXL z(ny}Hm_bM>MkE*!p+G_ri6cqVX4K@kV-GH9KZvCM6?!*k>ocD8J3U#9XX%{O=-2wp zw{NZ<>_Bm^vz8?AdUf>gCsnwQPW8S^hKqwX8K3&|%^={p3cu#s^3*?lTmj=t7Gm~+ zrT3^m!PT2nnAXv)ESq|_n$ak{Hcg2X^jr36gpFL=ys4v6#>UGiw51m&#?%^qNs0Md zJ<14G6@tcKIK+O#?oAJSA{>Na;B4PFpWu8Q^fq{7mUkZuTkp`6X4*vg6a{ad^VN~E z)8P@FJgZr+34m>s)91g|(KJ*F|ZG>-e}72fF|}#JyM zc?&l9{Z?*XtxtILGW|Y*U)BB7*KX9D2LKUJ-`|){6~K%~jrQ1H-fEcwSpZo8TLAgG z@MU0m^K0RmGVeYw_s zp09kL4c?jaisglX@jvPKV|?~x*5nm&$mL6%5~;z{15gA?jT)d*IA)4Z7AQ)9i6eN= z6fG}^-Z2N=-PeU0u_}_iEt>&m1$Z)V<#p>!1r0D!^Qd{19;qlK760EFapb6x&Eq;Q zmdzR>R&$PAj$z5~ndXvv(zpe+?1#G6I#dqc=5Gl6yRSP?uqCkFZZz747=du8!M!vh zl7f;X8P*X729cpN;sz2xK_tpZ5QxAaL=i%WfEYuBOlS-t1R*kk32Osk!h=xE@2IXX z(dbc?7B4_&YmQ4(E$QgFM@kmVvp2_yN&D;GIJi@{#G?mQTD$-qH>ExJTtBNIukN@> zITLf%FuTc_DoWc@&6#2_eDI3tKE%2>njO-&_c%~y&>_H?ihHHXFV;m&9oiKZz&(%% zIP4a=D>7_w|D0L%8zq(Nf+_iM;q0E@D4lRQiUAy7R+T5Slzm-pI<7Ib9?9*@*;}_+ zFvD0ZSQK3==;hhclvjIHoGUQ1=Gje8p01x&kXP%Jsd}d5f?+l&$NTpmq(V2_HqDv# zeDYt`VGLKWS1&v=z%}iV7V=rnE&K#&$E;bIctrF!U6za^cQJjy_Frddy$>r}R9&5+ zM)3Vn^QI<9%xQ}_pL9pSlDBQ6QBytx1KahR^wueMdv4NY-h^f&WNnX+JDxs4@(E&u zTI2}`V`Cp4QC=})ZL^Rs3iC@d0SZfemve-ly77rX7fkIvfN29&i;5SDC1}ng9o#-! zb8a7x<2A3<;-kXMI_BuAW1Kg@v9Jbljw`PHsG+zNTbtO}(M#5#xv=@;ugIA$JEMB9 z)}z8>bZ+PiWb`iaTh%6beHZ^c0h)|So#@7&hzo$y&+ehz<^& zWK&B?@E0PO(-8*FoU?+>b3LZmFRu+Pw^-b~v47 z009gs5}^*ReNkOXb>mR6!`tl)&z`_N8LA0A=BCB`T*N{qAnvW3dY->|EE+$Yo1&gX z4F=3uSil`P1>c;~7SGK1Cnr8L<7Sxn!ZVfMdQFo zN7pD&rrLNU#!wmld7t^ud-(U3Da{1=1T+P8aVTm9uB<$(W>cq`5}xFmcu?cto-X{1 z3-j~$Cx^jAf( z!}yJYAEtYjbp4M`^8nTK=R_XjpXfuBlG#eZ%+eCuy`IMhnAbJfI>hBa68XmNAqm zs9aoIDUOs|Cy+k!4=SuEkAWdTJ@3)YqN%pgXK^-Pd+k7>k{8ssLYz*Sqe3keX;rzo{& z)$EDI5-!=gxD!q@B}lc!tO76v`FLmfg75PS@5yWvb)j5AEL`IGJAC1R)FBjU?Jo3fzxcr==2g~0D9De1T}){aB9c_fgBgg zo2=fTlI)(fY_5RaIx+v{J%N;*o4_FfWN> zwiUtFFt5pB;ge{Al%&oibOB$EdXSjP%J`7uL4QuYK_yQr(`{M*Yk`!UoJnX8Po-(N z9-`}!wSWYkrwKM4*(pB#*Va%ZbI*|@^Zb_wCfna0bK~XkhjItsMDF0R@MBvW8Z7_c zq?=ZQm|H@B^r#KTIY9jO%qRXQE&2Ef^Z$+JMo{^0qTBfO9iQVDKZZUnUXJX09QD?f zX`fHLln$Y#NU4eczRm}D^J(dL@$#Q@-t-?EVc%%Iue||u!nd2v1)VU?ZqwI!EE866 zKmVaAtGOQgSWEs_h@b@&0XaDTpYt@?6|8K_s#C{+8088~3SC^6zuQr9OV5~NrTnHVfl(r{SSx#a@YTF_%EXU z6HlzMV0-4d-1m2lwgcyHulm|{6<`(M!`W~)c1!;yw^yv)^@M)AvAge@UUQ?o@3pl& z6K3xd#mjg(MqaW#$V)fPiAxFk{YC$`&528!L;r93i)i+be#>6;ZQGoYvIg##jO}ADsv1yPNWTaEBG-(XVJ_nygxg6A)yE|&3_^gA4ocDN9aMim`C={xxN-2K0ixCN#Y`%v&GIP8O?KU8fUD_DSA zvw#(13i1nlieLD_kv{79-P~fq_Y>+NVnH%v;c&Rhk`X@4n*0i-$MimYg5XHM&`b3>nWPoAIBkm72C?`Fc`hr+1SRt`5hTE#{hE zpBCk!-lm;y+ZhyT(wZ=LO5D=*-Uy1ex%D}w2eT|Hf|Q*05C_wb93ZkePrOcu2~v{# zX#o5r`r$f_h`0e&+q(f(lY?&Ifjm&G5&6mPIjLr$3}NJo_%)qfN)Yh8Kxloo1?6a2 zERoZ)U{Xd)M8(iEgP1tztH9gZi7*5GvcQ24*)Ajsc#^G)D`8dVG*d!TZSRH_6!oL1 zQhs<+gP_PMa7E&S^C`;4vudV7E?XD3bebuFDQQIv26x&1`*t@Ui{Wq2R`*3W2PtmD5z+`vr9mHdn^pIioOP)JQmmwg0G*dg+Zr0E)d~ zmcLX2Dtpl~u&5(o$9XZ}i7>yt+Fj$VxZBy)^j&kMecM{!HCylJZ}@B9a8-2p;o#^k zU6|Z%uh$ehars>TH(f;A>Tw`(8RNENAvX;eqsP$y+a@mUTZ`7Rwfb!r(d?(`HDrEG zx7L%Bc3U*vb;i04F)}I*D;27MpJ7_Ka0CfFybl)=kr)v!5q|H73(-ap@gkx=6i6a) z=;x;giVmbw#jdDaRw^;;F$u)IVhn~qYw+OsH2D8CSj1T&6^aV$hKblfDu5erb05l_ zQSW~f-+vVEAKL$gm@x*=Y@)le(1#d<$BQv2#$LSt|7EUd`_UKK|LBb2Ll05+jzK68 zL+|rJqe&)J^c;~!^}TK^#T2eE_DK{}8dh|RgSl$)>?tkrBFbcU|5Dp_^RdZl5F^3?b5JdYgb zQX|<^&?8HvGs&hov-d=TA$`_OP=#!wZgzX!2wdn%dr%Z{p;F-MKB>SHjerM^xgIaN zENVSfPCY7tOc`MCk(H_@v7**fCHHe9`y|>KavBMNf%So_)5TNGT9OhRoMEs*OvY%b z2ARxg?8vrM&Q3t3SuNgFw`a6xVIDR;5X4c5tEIP<6dwTcT}A?m6a$K{Hok zrYvSNvP(=$uBb_pL+%H=!INl*b{a7N3sh|n2UJb2Y4F9+9h4)_wjd0&7R)n)m@W6# zP6P(4Z-+Uw{EG z!AdY-<{r#9sR`%2Rgh91+bUDQNP<)T$PAA}+iMM*#48D(`Xm%k@beZW%3Bn{H^8pN zOUspV3aV#e)JQn&k0l2gb`s`;+X&4~;rZEk{XeFbeLBhB3Jdd-6JzG?cqcH77K zYgc`38?LIt!x`baifD2daXBo2xb*5>cbDBocXb&zaZO$qVXJ4MroCu;0~nRdI#t4} z6efGfy1CNK7Lo2`3!p{dc@&p?)=-30(fR)B>O_EzUV#d7q25fpP!>O;IwIKUUXkMC z>=Isp2}lVX*#mOaM^#YXAYr*t@*I&Q?3BO^3q6XFp+~biXLipyW4=JtKo+bT8z&X& zo|A*F1y3hyQSFrnA=<0{M7u-CfhUm%6tlCk?Gj(kB!opgvs9?@^U4Deq76fn}@7P_Mt+4-ar>12_*MNy%?lf)GsPcX2RD-S}l8$1m;zhKb-DDDOj zAz{B##Cnak6)f8hhraK?N7=27DUG zZ*tT#<)~qXC9;YZ^_-+5>123_%OMhQ9UroFjbn^R8H$sU4WVjw7*Kwjx^I z5$@8w*@8v>ybIK_4&t&CTksK831o?s82^#d2i2tP3 zUqHZN}K~6dWQv0^fPQ5TLmH&5d9FNeP^t4 zbV1P@H3peiR~0Oa?2HZJsW zSh&aJvsSsvv>Y%>rx&zPz=l&8ZX+5aqmE^Dg;i}(zr?UB-lBKI_nDG%dn=<+9kUvp zh>!R7c#7nYCA8`@ot5%F!zi%jd9Wu*}S zW1W{z2M-1*udOgGvaztL3nO0+tP9^&foO{g24>xAfu%FvU_&Bz@PSdqRv5<+o!!M3 ze)#{UMFvLY0sz*GM~s&R1N`$I4i_phdi_5QF{_NRFfB2%sZoPz8P<`@9oZIjMht7# zRn#e}X@v_Z){7lu`+uQuIE)v@dUi%73$t$Ju`q^t^CPlDbPP1~4UBh)kt<^}oG9Dl zU~_(8+5Q5WaJcXQ6Jmy5F|8O`UDX;|VOdvISyoOeu39)%t!Pn&+^aK=naxZmFq-`^ zd!X2?Mh&1X*2hsHk^gVTB{t$*_!hjVkBj*1am>i5hAS5+T=hV%eMU#;KNdo>ai9gk zgp#bBnI6xm2q-j(m=Hj{F^jr6GopJ0DNb5c#m0piY@mBW1#t8T*NM3z9v!k^X{_^( zkOLD2blq%I868)QG*<5Cf#Q>hhXAJ$qONUy0TWUakDdt!cw%OO5ekfOpo7{5+qoB9 zXjWGRH?${Y@P%YyTLf|UvtdK$4a7z*+H@j3foJjra9l{KQK)Lu#!my|6MCn9MoOR9 z$gdBD7T`V(ICu$SBO+)yY{LpXF_zrH@c%gEWnofKFGyFZ7Azf65}HlE*zg+(Z@j^i_ z)L+QMPlp%UfZ@Um2~~I@zoGq{Ki7-WlXgzOU?cm{uF#7%QSN@+HqoC0lIYI?MfB%@ zLiTgcOZ4ZQa}oa`HgGF%OD6bH5B=!K#=^h8+eZ#DFBn%*@eD@J%G;6&e$+0cnwUjG zK0F!dmv={I&I{br(vqlv4acH?EmMkyYi(?4JZ@ZFNHsBwg!=QaPk$ax_U9+K_2(xf z>CaCX(x1Qia6kX&|Nl|!f4?dA6HcKm%+{opwi-iple&EByIk%U!vAiJy6aYd)ahmc z0R(rIyGn=XFeb;?0sl35*?ysJ+78}|09Y$;a!}@J>Ylw~PBHB&IWW=RSqu)g|`UY!3 zT!Mjpfq{L2eSLj>ea-m#`kG-x3-)X|Tyy5I``_qV72C1`1kvSTt>X*=0)&4q^JmR0 z0cT4IjA^BGg~%2qHA8UCqLMPr9^0=uf`J&#BQ;k|rW+2;qBatmC7|>*hct}PBoflp z^nk^t`5^%@&7D6g%^oZsG&vxtGr=askDb|PQ&DAF3};I zvknr?CNXo(=xsOWAv;Z|+z3r47C_P`WtwxExu%L8qNWoRQBz85)r9g=q^V;1*UT|t zgA!MYp3+q{Dg~CyU>2!}_8pTX{rv$+1W0j≠M}0-zTrKZ7#CVv_`gD#rt7&?+jT zHvt!ttTHwiTEa&5!y0YYhp;Hf(>@$m6tsl^=R>j$1lLEfAY z_(1Tyk!I>Vas&zV`?c1IINXUlB5LAtnLsn!<;9vO6b$6DZf#1CM(~xQV#-b0ytHBN z6e=m+L~inUaud1D>kFS$R8;psk4Y552M;_Ee&C%B&C@5UCoo)>2|)ZGYW1q6kH9$PS1uvPFDWT?<`S6M`#{iqsta=Xer>x7 zt_s7$(ObG-FC+t!vZ%xn&o-q~Mg&9v5C8xGa~O~)3guZGB`KtE4-|kF5-uf&kOks_ zxNIJVp-6}UaRx#d1fl?77=(;7N+Ce50YrGvu^@zsiO1$y>;!3>I;JQa8M)YBwM@Lj zFqyGkMN9`7q;BY531+Sgm{zT~u?Lzv*1*(L%M|5k3$@kw0=b9{mMb6ng5p za(729Z8vBr4x336;<9OsD#oY+g(bjmMIcN$t#gj_w@&R>vzn&6%wdCEz*ivRcqGJz)oO1PZ z+CV>k)n#cc_qbwM2X?UvNH*pa&+TxZ@mqEEMHP*`_f3blDZ~JePfS_^z*QwRN(} zwpM#HP|PC0tFR;TZ}SAXe7zY~eO3b;y_x-qUfIw}%?v3HLw@U+t;Te7*m;$|^dPRk zu(l7$xu^E#M7fHunIHovu9dgR?MYeC1(PfMq|gf{W`aqcU6?waVLlxmb{;r-wpS{~h95qh+tR6cV6Fo{%v`fiat4@ol|`2@^h-=h z8Pm^w_6Z$+NDVlNDHMNF7vL!KYKs z$^LUl4jswCJ`CS>tNZKu_TD)jHAl1Ls=g~*a@50%ne08wD}Ej9wIvKb3kwK`e_(4f zaVrQS>K0fxzoTa+;L8zt>x(!)N8nc_=)s8N3yU}xuK1&}Qn2Bd`zod(f;=Dh@e97? zHKY&mYX-X#nrluOEjH6x2ERX0q6cwx>@0P~o8cV1Dm! z8*l0~Z4+I-5ST5e0_eRGw@%*@m~LTpUZ-NfxNnN`E|ec}=+LqJDaMd!l*%RdPN$+Ht+4;pFl+V|SrxgXK!o2D)}?XlhP&@;w1@ zZAvac2l!ysd)M)govyr8v}$j`GpC61FaFf;6qZn?rABQie@Rp?;O4h4W}L3Ozn(L) z#~-mOv?ithpn2!O6v3G0VOud<*}1@@0EEJk)m!hf9gyNQ=%()cb2CZMoDae^;TKhG z{Larcbv8%2P)@FBZFLVGx=`RWoc;R}aN7O1ps6?y#xym2@PT>FZb$0bbLSVa6ozIv zvlth~S{!)v1Exs-WL&ZDOAr)xonv3w&(#vK0rbJ{b8C|DeaSxJ?ZPi^WE#?XknBr2 z4_G_?22u*|hi5<8aD$ogDIXZAax~svTLyor!8Yt|nKRie2xl`m1Kw`gl2BfQVa9=K z+ZNS~9$Iym%oSAiL(THaelEtl!smdw zV{X!9wnOL;=2mj*88;Z~*o{4VBY*1jiab2Fnx=AiZ?(Nnq|AW&pS@x*87SXO!_UoG zgF_WVriAk%Xqs0rpawm*uz4HMM5&3b{x!Xy9I)-o={jCIy}xJKw7bL8q~Za-p@HppmtVy{@$Qb)ppQ!JSQGstD>9|k>W z1{%&V$Q~~1xoxa{f$-=EHxWYe6aKIV6+-8Ks$g1e7EJeuedjmh9^t=#1JoxOs)pw0 zM!5W4QBS#mdp_j@X{P7R%>O*SwRs4xCRgp*)aVtMPVgRN{5n2VzZM^_EGQDKBSB^OCT7o6wX;tX z%YHh79ZLDXI@L5<(MS071Fcfg7g1 zvOQ3b{2t9L;tg!BbJ&qY;XAw>kWSWV>pVAH(W_jYA11Lq`~&u7-f#mMa2#F44$qER zy{fCd*Im}t%cdgEEUZ`o&e8CW*=LSfCzMd5t1SXH;n zY9N&-mu=2JGFoAuqM&&BOvf@`g*BM|-*hA{gvn&agVn09xKQ}yO>T6$<*Zk`SRc3m zQT+7KL1eV`tKn6uY1?er&oG@OQoMo=@(r}n!M*K5XKl=2vM{~06DtFd+E#J4oEYpx z2R+HvjLM7lns#|ovhv3yit?*T>H==4n7{b*lpA8f#Sz2 zmxI9&`%RJj7T8|gfVKmmdVD&v+7&q7>i%Y?k%}H-MK|b+!J#8s>pZ?=M}s^P5wHun zOr3Fnd3t3}+?a!Z8Fl{3t_E|?hAS)XeebbAlw&?(rH3U$0x(a4JKj(&AYcvNJLIl2 zV~F|JVA+Xp1_i|$>U-d3zc2?1d5x*uTQ_2=bsr|JfS8EjGGV~pJZ6~)1E_Y(+-Xc( z$*LJS1Ctu3IN~2BH@dB?SWCHfFPMqebpH9gTSlu~18pyGc(c(z?+?jnlCvWd22x=$ssrGLM^3b$zzk>b40T zoizD+=IM;ASl06P%T4Fg{bo>P-~w|5WXn@6o!Dr&1Pg5>t(M9)jRf>4C(Z`Kf>aLu z!Ik5c13^esB5k?rkDXq5+%G$+Z8~v+xncL5F@QqKJp{bURwM(v`xtpqU%(SacPiAM( zf5+`Cp=R%rDD#SC*!_2^4ChDAi5-e$2s*0LzFgA^u+{XgRJW!)H7ipln3KFQA3N33 zHFY+lSe;ybY4x?0hA4!ghsAaohU3lX$1_zZMu1{9BW9ZJkb@G<0&zntZ~&`4ufw&Nvad^#!An~JnP^JP*1*N25;1jmuF%#1*?bqyRX_{+yPceG5rak^P=$EXRSnB{_trH*u>H|BEdjA^6^ zdEm=otE5HA#7mroicg8RUNp_q#s7YzAd<$lYCS{}$b6%U+iL8kQXGDtww&Z@luNgq zICTDFZ=0YwE^rG=*_sgwrq7;WqkoaHEXum>h|dDru6EE}Ozu>eb|RVjo{{x^Cb<0x zfzJLPB>a4%(yRZtW*RAa_zh*J<-U~lbAWLHs#Vxe_Vs5>cd-M|=ZxG0%Qevh&S9;% z(xG{z_xae%4!B21uVj&(VE-XR}8p zOD?{A#6lg{!~6)Ji9K3Sl@g+ua_HQs<5Il1bmmRipXGiS4T2%j0lA#Py|xCzWDO$` z8Kh-_V}p-a4NWYtN*sQvRC9xTB|#1xusUdho$MMsh5ML{pP%I+zfd<(mcwxIsuaFg zydwPv%h~<6{T84THIWtw+^8ojauhf5tlG|Q=Tissi=<{P8U4a#yZSZ|NC|=}`O)W) z0iO)zmzJVF46rPAE8{y>H=4R0qoQGBJ%XE$zrn{a?WHk|su*r)V7 zP7N6Qi@Y>6h6o@Z2l8nWfpdh0b@OyOU2kxt_3HixjMNv^_#DiCAkK7-B@wr?wctc> zQ&`nU+knU9j=g-D($+tQZkAyb|$lwJ-!w|yQWNbhQ5v($!olsoy<9%;!Z}bXt4mKHGpD5C_xfC?zzULbZURdD zS_ik5olTApCVABYTxBRw_K7F1AvNAv+5pSJeRBAc{l-Dr9^K{IdQS)%tep}hDix@%Sr-0gu zSp1(KVw98DbK2nV>w~{YNcR)xP5?0Y-M?c#j0ERx*BiMRQc-YExtZvpWcn!UV$+)# zuSfcH-J<6$Cyt8X-lG1kF8_@(h#+j&1*6>mGj184Zt~DBr&VB*!5ihi9*nDu>w4X; zZ;st3R-$LU_qC@))>L{-sBKG3zk+&!grc>YSrk0Y6{M}$oY7#kthqT5#qCy*?Iq^w zFx$*-o95@!z^F^OosEnorjbPyyfbsRb8{4&m+=+j=Yi4oMe#KEo$%d0=b5k;@Uqx_ z!tzEcYG<8qo92hFjdYpdDVB?VdGY?tlpk{YpJ)0M!OQs31xup-5lPHP+gi*)VXBrT z(<0~9H(;mwFc4m>!E%Q%#VvfUkOp$ghDIdJU13tnT!hZ0S$$H;vq1afd72tfiU`dX zI9K+%igo$vCFJ$*vWWIsx3;+{wzdQ%n8m3WBnL#^3g*gkvaBMAqFz#Oc{U*S9nKcb z*Uw(UddekRV%rm;G$6kMhPIYXX5ny^5xi*>Uju8aG~awl<;~CeV9PDakn|wxEyq0j zyd}i(OlLb*1UW1sgt@w$%J!LOC15$=HsCX~#&%jZPT|T0Hq>o@ndZ(3jiPWXc84pv zxGfYA+KNckFsI!9SR{Ex%xrsg5Nl%Is-HDwB~1)QT=Jd*q%G`E<^Y?yh0G0hX4z6b z9K(w3Ea%j(c(r={8xYyR0-7segZZ+d!PwMQo1uDkijb9SM!bznBC7&Lsq!eqRsICi zp7PxB8tM%-m&d;A#NDND)tlUb%z_O_pP@hg=soiTH2i;Ndbef#OVRjzO z0g;cNv&~9&XP8Z}PET)brC!_lYt?0-hp6|9;NLlwY}OnE7mRNJkSdAIZ23E*=+sjB z0f*+_ej$+OR8!VB*gl_uz_zqmsoqb-0N6~l-DgpIAf9p9FuwsnzhQXyZ2WP*lX?Te zfN$H(7Io94H&QZqZx;9aWq=|g>WTl~A>iSRe}iL}xye!14xPdBR&Mk25M+o*ugrLO z@1~8B!sX(G&3fW3;IFIE=SKED!NGvFfc4VEp>sMm&@GZHr30R9xV>%FFcN*5p)KmN zbCMTJHdUkg|I*{jIj5&2sH&Rh2bb@^`nm&l%J7R|~$xXi{zJ5=Ex@F)k!gd_a0idZmHGuV(kl=C88)R zM|+(r+%qn?l!M~PG|B%sK)fbg>4(rP21U*#8{@>2oE(dtaM@}xu}^Sj&5+5@ISn>M z+A+~WGMwvaav|W0F~6ESxnkesWq=d-b|=T%rs=I5*vk*I+ZX0tx#F@#l>e1doZ6F_OZXMC{)0@?V)8OeExY+9UbWHW$x*phcjz^1YH`ux)O!c=7PVjkeu z1-N#Lu9wksRSevmdhtxoNG{E9yKbDbFkngB-{sr#hsq-{iO|N$H~%2e`yt~GJwLl~ zwt=BmH_l1*`jHUp|4;AGOGFLc#!0{9a265=W&{7+xeNoxC5vfOy}lYFbx0e%0;hq~ zuxh8oTk~#nT+_zV9644$)71!keM==bW-`#|n~tkUX{;j6LRQgwb5b&wao5lK4)J|h z`*~xnzP5UpbS?&a&L zzNpYMrGf{FtjAN)A0Z|NNT0y7EdKMdQ2~0*HeIm2x{raKhJx~syL;*jtz5N!Vp4Xn zpEkASdu-Z}?zom4qfV)0MU&?q(Bb}mQao;3vO%nDLitJdErEgpkIi6XSlR2(_BF>Y zho6x(HaD4f!9v0NRr4`O{HWPdzcGMRA$;6zM!;f!+`a3T2EcDj)TRSX9WvEyS?3il zi)3Q6o+YcJzM+BdR7$O%$h)!`X!POER-VDx29=;Qj5ki3+CDl!xvwpYBA*YJd4k7HR!OWL)661z@hkdrc8BD**Vb(^M zZR#Aq^d`7sKftdX^U$N)M2fR3H19P;Za@H77e6=7Vty67R4jx6uv@o}=Aqi6EGh-Lh_Ry7#Z(dL+@6=+1IP6TeJ51Kkvsb12flrh~g&uypY zJP@-**_jUDI{2Vz^ai^cLLFvjVkjoUAE61CTnFm;?wzS=%i-V18VuE9x(frt(!}Tx zJ#R41b|$kD;z{N;AKgOM!OPh2M-vMxgkHCtvvRdLldGOSK@b?2_P6e{NhZ?=sFEU{ zvHP;AM!2(xZyTCAsBePQ?4d+GJLXNb3tk6HYu&lfrZ091stz9ykF;b zU-8>aR7%+%<$L;?rfyqvn?Z)I9aMrqJ-Ktp!v&!jT~506O4rMpYGI)@?QY!+GZm z>NClXI5EDs4jn?P5C-6G-41dg6f;RZ+=gt)U%N26b8UysSAE z`qp1>J=68y0`z98MI?OlG?pZWz}cAueKS?kVfXMTvYYzXzpQCy>o{GAjC$-$5CAQy&B|cNn>eGh?t;FbNG9|$Ut2a$ZH`CD8!4By3Se~%x z{D#srI+3vIQOfZ>0+A+vzyp z)#XETS4er&CC@*0_{@uKdugzF^!&*o_(&v!$N6+zd#Sms-R#rZ0C7mlW)*BbLq{08 z*fCT_mfXE8IVcrK^O$*eSm``7v=XSSUNuMzfmjem>GEWNY=CM*^!6fdUJ=|RkCSaU zs>EPOL(X?H_nA9QYl(QkZI1wVY24P-Ds}`^q!FlgLW*HJP2M(al~2_f9j)!bCFs>`rTQI|4q3{jjYk zj%x@*nCjD^*d|Q|M)H`?`s}($meSiJCyk=_jd7R9#M%*xgUOxln_u0X;S}@74c=41 zE%VwgZsL*&9kvM$v}x?CriV<3%BIeFv2n6>rPrZ`)*Oj0BIb6`Yuh6kE+aqfcem8z z`>xkf&u=d_ohU93;JYjB=s|yA3DVORo86sT;A0cfxNg*m(EsMxpSj?ld3-uLxDP}S z+7149gTBjNM_Z%e+=aAT&l~e#cj4~Qr{;h^gI+`gO*9Mt7(&Us2dklN%&^?t_xkXH zC&~k>eA0S%)yu;M{cRrezx;#jhI?QTUFWAG^s>-hr2}}DcOAXsdWRVQn`rtTOpZR2 z{*4ml=;Page3q)gu1xHIM!%=Q!``=KP>O%|it4A3b!8BSLD25ChXYWrgMmOEG-*&_k=_Ll%fqZNDZm#3)1S)aowwGT^G{cfunL0E&X?7M^Ot$ti zttno+{_$_q?=boH}E7uezc%gC0%#zsyu0@*eo;aGZy?JwcW`_4i((_lr+vm+m$2uY%kMdUp;*OAB)oHF5Gu5OL{Ymih8x0$wb73?FY zaeV)zS6_e98Lk_zr=wu_NzZvTc9=1p^hcXmYDhhTpvmrxI}t;syVV_AFDp_0L3whF z@-1pQ;_%=CPS!CrWv2nLD&Ff6I$&Vk_aS-=9~e-(`JO}^u64(L!*)5i*4C(3d}L~I zAz@z;o>s)VvND5LuEug?CzsBEI1~8>%2_VkQ!sl&wf2?#F#Kq@T%JO-Nshp9)@rEs z(9LHU@{|o-gQT1A{COGhhds*4$m#1W%&3&-M0mG=_KMw&H|*~o8zYUxfjJDjsrA#W z2i!0pbI14FAg+ON17xQ%ejmP5k|AY(w(;4gF>|bANOQB=_MWXqMuM_Q*FfBVHbG%d zHsK-o)<};tG}2|k#`03=LCXIhHzl1sbO>apGj-Ngtq>E8G$`lw^^+ZcLs^W%EssV^f|t6JCpK zu#i8Ge*ZD! zj0<{yNBLwq8%g=J3zoDa+bA_6;l_-_P<0 zYm`K$1*J0Ze~Z8DLBhmFW5zR9-x3|)8^=GxInyNGl_#Q zRqlcmQbCDeDDCc&7nI;{+Z_YgySw~nhm`#~py+BzzTG0uWd7X7o z-wvVUJ;%g#J;&HsF*5Q7$8@UcS{-L(v^sRaJ!^Kv(7Bl*v88~_*iu5@DZXN77vX)- z4&F-b^@BYdF0gBx@w#!Tgh2MR!b`LIZKZ&~TH@IlI;;2GF>zf+0E7u-+AN3YI3r_| zr65B$Wdm-dfXq0fXoAjix$GQF(WfPzeQgHlyvL)NjicA;9V&I+Lz`tb%*JZ7vLY67 zSv@8m6Cn?xlR@ZKxz&0qwjrK+@$z&F5%M8M3~?A?jGBaA#%VU4l-@(JRaG!x)Kpc3 zRCOQFZsYZpXPMlu;H%!@39g^Oy8d*0+Bxok?d;&YzT@G#2HU}O|9mo^Jk}jNETRU( zcbG#bIMQm3u-fT#Y6vlx$<1`nj!s!rXuq7HcnUi|{C%h~$ocN4?=Jq%dU{{3=D`jE-O>H4D--@Q-ZCgxyxDR8b>{DXd0Q@*v52VAtgA38jVTy)>kG^)mrWwC^hz0A zUTNl)Us0|q^$IIWRYtwQq{?U)SBknTsiq>;(XlHiT0;vz31)>l<-fzM_?W$)*Tn!b zp!&zz%1PN8y-k8Ba@W8hepsttt+qo$^hhNxF&`8at=S^tIL98lJt*8_iskF$R`Qb zE1B7$N8dFA448$amhp_5jFpZxBfT#cyRNvV(dt!`_a2%hzDY)tg<{sJpJmKyllZ2VbnE(sB=LH4k{QVz7JrAcrIp2!Ax!Q_^0w81S8m$jlc>>vU&cQPNX~jJR^nVGt$m_uf@H$)4ZQ@ zga^J?bXXcN&$^W=sGkD)rM<(!VNK2~SLykLi z^u=Yd*&SjXbzq6PiK&7+$CQ)zGNRP$?A6@-DDrK7SQu_1E^^)(2oM$7@3jI60T)Xz zkeZe`YgtyZQQEvGc(OkDdv`gr#_+%2AG(KPiCir;NX9O4=*dH$Kog7#pGfxg2pz;y zAG7ZJ6-;%ddg4>y?o?8LntIx&{4R%c@h~YJE;lSHjP~)Vi)ff5_7g_vBk&P*a!=|> zyhc69g)6X(s!8C<+Iy{~S}ICON0H+E;&~^inpp4dDhC)^zWwZ&%J(Gut&YPB^JNq#!YM!9!xoQ7F=5)`1ilMAGQ+ z>?_bS94=PdgjmLNu0#~>m5mG1PZB6hUGVheA`oBcA#$P9d)g^3B$LHQ;_x_J3u?jQ zRdDC5fLH{Fc<3Pk5-79^)_{d$z`-c=5KY6i5_P&r)5T}){kG0~+k~gzGI{>pp?RP5 z2c~HvUXLbHLl4PPMaDC146e)U<1%|=_T=9Fd@8-+T^A$7xj>RmOLP0@6QPDl>i(Fg}#qu?qPrkjqoo~F?Nj#dDw_9H)di zL-mX6>Zy)S#j0h|7ejn^l^UIbZAFPX#w`VZTv`%F9N?tUz<>gN5Hgv%bhkupd|UrR zt2N?PcA8++W?Q46WE}#4uE7y05ovs2$8?6p9a)x@yEPGlR3RP-G9kBxbh=G>GDU>l z1yh&|Svpx;VozW#kiib}U9`FhLL{dpGm=vjl3PUvZg+s9v~PY)?VBIrlZh?xgq5~0 zS}hQapylE=Wa}SrY^zEfu&p9uU~7dx+47)Wks^D}1p?yIWSV>2iOglFf(4cETMzFi*D!U_6oFoO2 zV()s`^gt2vQN=DIqaHaG;h}w6QBCXt6w;NTl(KWW44phU!aTWDyiaaVp<0VN+X511 zo(#G6IB#HJ#CgLB3rBcNIJOoCoVH)aeEW67)ozeCaq>0W>^v3YEmud|lkSL#^~i_W z7A>0F1lFeaTrL%_T8p7Q=D^Yu6hr$B42&H4FymsA9f|oe=2|p2IHwLg5<_L1xNRpp z1a@44U6PmSdbOIG!JNbj$6rU2q+yV#lXT7&a?%^*{UCU#b;});lfy&j&^S=8ZwA#! zwA%qHvUG@y9PHo;wEu@q%iqSIg_JOMfn#-aMQ)1nw1nT@I4b!69O6J9J+3_n{U!?L{GOxpIEF$ zcU{*wv=1K{M+^a9qnMv|`=kmy-);2q@jSGTlU0P$ZN;%0>kCVHF}YXfbAi01uh2fU zo(P>Jjz=NPpRp*ur%3@@NB(3Q6exC_3|I1hhX3am&m8?KSQm9g6gCMcwepi7X%u+- z0m%c6YN$4x@yk-fwmt?Z!y^E`Cd9XTrG=FnIco4pMyFQa<|b!262NMKoF;ojhiWKU z6~p41R;yQ5R3C{>lI0Xqvyix_U|nEYFY<7f_~E=bzK{(Gb}D8yCic|gBN4I3n;JC= zn{dc|Q$E-goMMeQeuM=?ZdDeoPNQQgzTgi+g~20{bx5S%OR3SP6nS6v1@^*?ShWlE zxkz1jZN}H5Y|lS)xA(7GvMx3HMI(z#T7~<>I|a5lqgdpCZqzH9<(A)lmApi>Balz!h8}%& zuAb2?_-wgNj^mN2hQ_jTNpu>BpRn-lEa|% z_7ngW7%wD<4aWgu$Uq_lLSYnzp#a1%2t!c_z(5ScG7d6?Jc9s@7dVgLTLVet#K4*8 zov#~5)Lh$R77rdcr$?m^C*lcJaLp69+aIvu$~1gH9!AC$77tn43eD7dRw; zcli1w21*a|jU}K&_MEHU?US9j)CW*VtdN^} z4@OzT5?6}De)IQmdE@y^%LDfN1rBp^XS_%HRh*4-g0u?9_}E>5zQE}kx~Ui2AoUjL zrD2JB)$bNrFsSk4Y`~$8tn+}G&N8+%Qjtep5Cc0*I^`PV z7%gk?iL5%dvy{{eoZwhLm*VM6JLi)x>t~jsg?aRYTFzM8FL0XQ+IE7t!(P_GF@zWR zbzIieZwK`(4l(!wCpxn#o3%C)Qj7((rI2=33yZDkX3s$HQ@AFUyfDr zS;)Q#P;%WVk~yAQ)z1d#;BLG8(Pr> z;J;>egD&it%XK}w0oZESy|BzWJH1(EBaooee}108i1q-s?F#^G`!0dFqjVyaj5>Iz zkPivMU6tZG6xV?qrw6P~|8-2d(bMtkC~~?W&%Lc~55vH$fJO)*X!~P+rvn6YKUIw> zD-5<1Dr5Xni#A)6mk3(M|u^{-o}WUQfFy zl{XO}b%Tb^@Qq$wc{pCEc;);c`QTk7co=bT5lPbyCjl2Vv?9pZ0KKwjmQ{2;{a4vk zFNxMSji3wXZ}60@I(CeG#h^on*0AhFS9|8v^q-NiTe#7K+(5^CuyoRjSFPY~`ftmx z`wI26GWJLH(wj?H4c_$sd`#26v(xJ8jf9>2c2C1o%4ParpMBeC@4mnKX!SELf3<{1 ze{wqHPsIVm9=}vJP!T0)uU1&cXj>+Vi%9t?)4F)*OmP1@hN%{S4CamOcNK-ph6@ z>Y#0=Y+c1rFKSY@_<7@|+IS$czokjtg{2I!g;@5rIYr!usj~Ya@%Y`Exrt@zXHjtb z1BIXMgO1^6r#rG`%)7O)wXx;;y+QPN)7_3p%}?bBU{|xn6D7U{oA;=ZY?;LG>jUiO_2J8$e&&*TF*(v7(n+aXg{O1W&(4JMh7|jc@Ei+HtGg|iY9y>Q%c_u=L zXEZN}x2$=WAC@_32@(1-Hd+(DL$82fZ@aBkBl{j z+o4mI(bKzxa%-@QRl#E~>PeuUTiI?E&!e*FGR@17Q-%b`x8F^|g{sBuqRm!^y${rH z7+$Ny2=TEU@E?xmT*Y^!f&nM|vGr_^pX1Tz?_zq(n7`mvYt zJ;c#=Zsqf(i!Fz>y&Y<{+Bw5yt7Y>yw0~=!PjcnsF=WU{E$rU9z$%(@L@pp69 z2igpZ=R59Z+~V=i=Gxs-q~uQG_`+LcyHcrPch!R4+dz&Inl@tI0aoeR=K@X!KTIt3 z>4kSSMrpcdX$Y?pN2jEfbX?mJJg zk@SP^Bf8#_=$k0yGgrCrmLdxDyJS?3Ex-cY$>8_*-4NFC*`5s?=)Ghs zm~9|E(5PqRWiQzh4bnPHwWVfOZi_Ah*~NB4Cpn@)-i@5hmUp@|Ang`pbj&$&GRORX#5altne;#q;wTW zoD1^xPC@OZ?z?1E&Qoy;$=4xP;_~)zZo7IFT=S5jRMdJ5vi^ zty6CDCI1ykOf9-!Bl#0q{;P!xvJmZ+xWB*7ol+ZV2ZKNj7>?u=%UIdXM0mIy(;8ig zD-a>@D`|E{P_OfLWW+96Y0%bVcw#z>G>w&b5Bl)`;Q#!8Mt2%4_v?Bqaw$``N9I|f z9f;ouEfji|*aw3e?O|nl#6CV?qeBmZFov9leaDqJ(G3nWyWY4O=78zh*iVDG7#sJu zp|uCEBVeiWwGszx>)P@TT@~mjN#F2zbwd&i`?uyfT{;I17ty85$Ftu@qKQn`i(o;- zSEr^+cN>y1kGu!1i{_<_i^XsFdaBa0LHi7Ih#gkeJHWG<75gi30+%FxcPv)O0x!Rz zg0frCuN5>iS7hw|F4)+)BEYU-wBq>!)(7eKGbV>0jdle}f4@d6ZlK*2g#H$_o_GWh zXOvG7ubL)cZjp^0b+9Xltg9Eb^E65|-M#%{)TTYI&*L_Y^DfAM`b~c~Eb?vZvdP6- zM({)&*QD-7VEVLAp4LHG>(QE^bwas@ZnhQ&?-QqB;nJSs*NR!K7EhwV=^W3I72g>}yIka-D?!WhWM{IAW~6`;p+ zzCI=ta^JB6R_$rzDsP$~0~GE--5pMo9iPsGSp#n!s7z}cfamGfNFC1w`l+PZs1HbD z%?pt@h6z+0upoH2$PaNZu;V$5yBf*n>1k^7rRdJn%GrcpZR*L3?!c zB=h=$$nl)i+8&+D77z;r2mWEi6wQW7hP#mryq(78Oj&+I)c(EsWi|DKgzZPx`{kIy zntcbIZP~37rA3v&ZL5m)1TXHJZ3Ir@pgRAph~aLfm=?)?vB1mn!82rUcyve7K_703 z!)Xa+bOPBx`i{cRZ{hAfheYrU$0mFW$a#i8cpI&>ylqf&Z>S8#>4Z8xi>qb$jaV*dQp#MmKH!o%V7Uu<-HDZ`u9-@(t8|No0orC>e0 zJ2Cq7KRmX7V5Fy^O!aUnxuN%#8-9cD<#J%1|Ng)X6&&{A_k4Cu6Sb1i+P(!qYsPbuax+3~4=Q(67??bn=x^=o{4?CWSrv z{$3f4eeRwch4=r$8wL?QWYeeH^YDAU)fX>(u3Eqjo{acHR$mzfbmO|4g~-dJO3iVU zzDJeF4>D|L8q7V~>)5r1+*sKZIgZ&8&b?4zuZ;XX%dyTh{kx&d5(-uq0?r#(9aZ$` zkMaE7RXD{(aRkHxyQDC3UJyG2Iwt zA9JU#_uk?4%J^ds7SsgXIan}8aw2{ESHN(VAI4^Vcd#tNFntwSvTh?+x40_YuKm-I zPjneWHAoGRL+a>&A|Ljl_2VZ+4;f)k{m`5DJTW2Wd$q(nU{aSEa81dw)_jDg9@?-5 zz>)HdN!1#iE026|sKb|fXn}gB>G&gfv>oD<1m4}zr(CR;ndxx5%Hw}YFqUvP5U^iJ zPXF(5MvDsmuURmhkFgFa_)E@nJ327g^u1plPek%U-Gr%>v_e7@WBTa`*VH{Q1wNp# zN3HyTd{nxBUKR&Ws6ZCv&}=jnI6@<8ALkuLUQu=rjIcZ9_+gt2TI+gPKx+d?TSZ#@ ziLU#siaPr^S8#0m?7jyY^rCiZ-o11h>96rN5?a@Xos47KXp%NsCKDn*0NrdxxQQK0 zi@EF6AW~{8C&^vixrmqdlRI0Xy1&u)yZB>>Hb=Otze_e0>sU#1IZ>rf&65?Wj2CaqGIiMAgAy^A%1={L zV>cmO8jVL6)w5@hWM@G8%bh(xXCU^o{JTDKLiyxJ7>FAl_&@Xdp3;H;YB@q0`{FNm z)mI=1OSP<{Sm!p@oxu97DPZa|vmxzHDYEq{5qc%soG+sh5ZG$4`Ngm^7|2psq{2LKnf<5CEbNTegy$E=ThS1ogBSr3KN}J>UqiVli*QoQ*9HYJ90vD^l?gckq zt3x&7hBsyD=nI@!QiF=83Fb1*@zB9X!+n%nj|z?T#pMjm9SbX_JCF|8z*? zJE`e#HHzx#nNu81I83(F;sF-&Q9c8W=+h7V48$xrT`eJlGTu*&zP|l5)q)WmIhVw% z?+fOW8DC~hH(X2CG4H=?hVbt&6wR>WV1}74T0;8K4qK^Xg@TV+x{D#UCh5OmFBjfV zj=xI~n+`3n7}5k&X>!`3c3DE%YIR+!y1iDds_+j2Jxsdoj2aw>`K zRNEhRo1u{B)-bD6gwp|@BZ*rf>mCjG=F&}(W-e(zy3ED#^&GL$YbW3y)nj5P9=WxQ zt)5uqFe9~P;(dA{pYArlZHM#`M~)-U+#j~#-GNHD%oRl3r}}lJO(vq#iGDHpHT}3O z8{70%o?phsmg|l6E~dlk3_ng;`HpL#5AJBfnzx~JXt*SGQ&)!Gx$#K729=l=lLpkQrWa&BpmHa zV6B-T@?m&0vn}C)n1rL2`P)JGqO+loZRX)=&4b&*nXiTLp6hSXJnrNOGap*V>Oek^ z^eYd{2m3NY^23=Kt7(%;YuwY&%pcaDSMI_DhhMHK?w#eTPZ@JiM)mMBe%LvU3m(f< zw=-ouo2l!m%+A}$t=tb6xWC3-O0IcgEoSR>dlQt&>0IG`eqTClT}*1aar>T7W^_VM#8dSO(=N2cpZ7;7|z1cV2*(}8?7l9mwt45)l><#hb$yRs1A(;F`TX6vNpo;{a-$Cnm zdp+-mm$rOomZp$;9I~6a_tIEFb$~0S#dTHv*>~r*EBR7N8|`7qwuidRABgL91{8x^ znfZ7?@+U$Rc zzx>?zSfW_Z<7v-KTRx7r!*b`~qe8dvwzoL*U2A85p67;5{I`vg&*!UZ(_z`iGf#EC zn>G5p^!32LubGf_xtigAXqC&xL1@-JuwD1wHB&j)=l12mI|u%n$(oJZe%fi12 zBMwMjov&FcF;8DJ&7;HXWuV%qH}>?~<`W-QcqW(kI_K4bt7kHoFH3QxKX%f^BK~;GW*qgL&ILCpRq1AZ2ISh~iaI z5P##(c2N9c{asxU8KJ}f(AXehI$gx;uLE}H6`o!VeH3xER&7L^LVVTiJ2L@wW3_77p#}&r{<+%LwxxK zJrwl8eRFgq&-eCBY}>ZYjW*gi*4N z`qZ4Ro}TJk_o+Trck15zv^XxYA5rvUZCTM_q7kM>G-hLpBC1G?SyXD_c?0j_)`aS72{ig$Uz#_1=NzCJl7~^f>&bHpd zP})H#Xpe7?SlnPPpzpd(E?RDCLAW?t{44$IHFRm~2`RGsaM2KdC`J8j+x2{w+^nCge$=$&u}q;B@Z&fxR1Jw2hD|k%XDL*!5U9@K3_L{38c! z>!%3*plj~a?dk7N=t^nsgBYh)H-(#nCBoNo5I^#z++ZzbC&MmobDU<=Ep6B>Bhn6R zzYje*bcfb9e0sOJG#}QF1Qib+S18mb$vXNx~ zQ2YjdYW-s@u^q&SF=Q$$`!lpf&`l!hYk8qZQT-x$c{%iZ>+@nbykx4KPFFvN&PuDu z2Ivm71@h`&O-XM|nwGa*HhYZnZ46U3z7CDrE?4fhC4hX6Peg_~E=^)l;CY>LtVd!ZIdVR78-Nm z`@BSob4e7-$qc@m-e7BS^CF;kva22Rp8X7b;HBGYmw37Oam47v9 zLnF5+8&~Hmy|(eoQ6Efj*n}raou2U9J&a1w7~0-SbNN;>f1Ym+UL=A^bmnyCDpa}pIEb&oy0}{CSh!r89yrVFd^aDw zv4vD@t8M_3R@`*FD@oJTz|#~kjruz}nSaZ^8Ck!6L`y#$JmIEpj{xGm%F{D5vR!a2pS~qh>SGpF6kDMLZ?XUTr}+ z(!c$)=sN@Tby0z$1Fd{<@e$%=dh<@prE%SkG57qG6)m=x$FK0y`n+F*Z2dVerBr`2 zBU^ZAgdWNR#FR5K7msaeQef$^b!zGiTM2iZf162PXY`Wi?uRrO9_>Oa@7* z&cqa^WxiVfn3;3(7P~8IzxzVl^{G5~JnCskji}_;%6?HIV_qA-O08(l+XiXq>IUj7 z?NOW4<+{VsW(&SBz8SwydwS`{!0n_5BtT&h>`#K{oM^O$wT3pnMCRk=m(OkLi7bN z&Q%lMQSJ#<)AYCv_!b%fmS8~%0diU3pY==t(E8_ee}*Cr_OdGa95E&fx*Ah26IP+p z;ZHTy7jKnjJ55haabY?63%?2IR%K9`z!7_=u3Pa5*w>w1CO9TsBfP%OywflFUbqlp z1Ey2N8(JYsQ@>=eCV~=udfW{vCT(wG&umLEZ`RTSPM*R1g*hnUNNZ(V{f)oKN+o*w z8~lG%UZw=0Tx3;%wB!3)G{C1_zo4kL^@f_F5*f_;$7H6FvYeQHjFRLcaBJ8oacZ2) zas^3pI`1f&Jw>;TZDe zesnIYPw3zrBI((`>0vX4U|ZUZsi`pITj5}lii9HNJKd9oQ+hEJnU4`3H+x1Wo@^RN zCbNS!t+6rDUlE@KK5~y1@aJdJP3xy62xc1C3=aQX6590E*Zo@b*1O*A*SfrKcA9=$ zobA*#Hs08;tS-B@sBm!+UjDU7%hySlcWAJhYs9ZyUDa$*)l#9inwy^=S>M$I-k&@AEaS`C|nH$+i6K z`2LptKriorHUvTyr)3Iw?k149LFw~F5uy16K410z-X&c1MG`U$UMtzdnu`7>6;Ba7 zWQ1pr5ris8O)BGw4hR4f4>9*=fyrGK2t*NuAr1QP14lHB^ioPW3V24}w-0^x-A||z z942flT!5d{cF3~99A_PaW<!;aqux>x$ zkM!-e7tF8!4XV}k4j3n%UiYisOO28@SvdqRpvbop_@^do{(er=y(^`@g~Y=#V-J(F zM!v|Oo1c}slNBD1#6dftxHgdJr#I&uM=il>+I(|jwfx9(z=q@Og<~R7975>Zrn>yYAzy2U9 zWG#bG&{x+>b{|g;&CjFlF0QYLJnU@FbHo8zF8;*tz}k2s^|eF90Xn51Nis66^$sKT zIraG?jER!Ui!r?1Ns=x3J17wT2Fj_5V91S${1iv>=fsETo$C*NV|IK*v*l{kH}NpS zFIto65MJ%~tJHgal$7Xct$neWKR{oi1tiT(FH;fCMfZA5v(6Wm45dvw`4E#Fw-ZZk zeu~A_!apmWCm&Ef=dK?Pv^grBg-JVEl=Y(wCmBfZLnUKb4v5h`eJSsQ(jC!Y@Lk;@ zWKV1!6dC&GgK={D4F^Sz3?17Qg9dUF4G*D?3@O{z5$rZSjR%FZ8g2S35#U!N4F|!q z8hj{@Sf$Q^D_|>EYcx zN|eUXsi!H=(2;{1>as~7l^EExR67STYIn)_GS&xwM{MwqIV`+FZV5{qHwR<%6q~07 zqdv1k#%LwpRq0SZstMrTt}Wq=$<@M8q}iAr!7}@5T#%*ghVc5J;sp!*;9`tF(bO-3 z#R2O#p@)>5)E6vn=Kwc~Qalyku-X=n7~>WGMph&C8*J*bpmP!xLUbV(Ggcn{wZ7JS zBjFGy#;FG2)}yY?gnQBZpub=~p>)a6JgYGiQ#Dxad6%c`^F+rWE1e|&iQ;e&;zT3P zIC9sFc_-_vLF@mt($uMf&(4MJ>yYNVsOEXdY+2#l}F!GRuT3)M%!bED=B5pqE8efhdZLtIyXzB zA=6!E`Jb>WB1L>uKKfIJ_O4CAPA9{D2uGS37T{eMovSG~vM&+fUK0HAiczITBih=! zkcKP2A5yMUJtB;LM!U}DM#JK#Sg|G%d*}Z1G%DzK#D?CCv+*kCGPS14wr^p0TWKxX zsey~Ma3nFBBvTm%%CTslZ5Z{P1Ve70SR;TMhW(Vzuo~5)DD%!S+~}Xh*L^B3Sc68$DpfRlq?@x84-4Kb%xFV$Sv=c}?ujg_D^= z%pCV;K&%nI`)G3MTf?uYznx6oTDn{{TYJ&?DA1CdEZitjIts)V4Lmh?jr>8s-ZExY z=^rI=ZzHI}y?mLd&m|+iaT`Sp%)dYLr;`OHD6iAxPmo~y(Hqynx$|NPtwS9zhLHmO(Ce|Ribj(Pmxv44ziNPhv3qqV`8W`!UwblF^ZAYh$yI3CSv0{tD z`#V~_kd_bZv}!i5dEvDi#K?#aj&54%dB%h^gI+yL#8MBloWHN*It!GcKru zyVG>{u=bFE?9!h>O&&V?&ZRRRU9)DZf>bjxZz`i`SBbO3KkE=Ivtfy9WEnDM!X3RqHS>34Q zET+pF04-|PD;?I|s`iX`nQ-zMhd_pa8)b>4RIrzYgI1kx%Vbxynm!*FsOmvj-?Yu7^%_c2@F=NK(TqkuR=<`HM{QAM(gCmW$tyPYP(T?Z>B?O_UcF=H~ObySRtFB}n)ZDdO zhw8XHrT&7Rs`WaoX|EWf5__KPM9z-yFmu=PZV<-EkuBCKia9j|@`y|7jkB1@OZC!# zhO@!Ctona!FclO~00A4osfrZ3lzy5r^nELv87p|VNM$VV#yjdU&Q3bXO*-F}%kT#? zOor`Av&J8C!bHl|z0@YK-rV0d!jQd1*nd>84nf#`o~N9v41E$_#h5^1AQjNr$drIfXvmeLiQX+szg^2=PQu{(hRH=rS7RkSjNAk%m%S~lqfn$k|cdJ9mvPBz`I zIy%_Iq_t;SU8ZC=M*9j-wx9SA=pq5&F!D_75kmFhi@CCRxY%Ur6+%VX#Hyl|uGOy1 z1S0MS!K)^H_~tx?}`s)f|wFOML)YrjPe_1Zck z%mY~5)%Aexde?)1r2+J4<%6Xc>u)Z|z5^B2kz6{%>m^*G)px1>$i?58n{w<^*4&Rk z#3OK+Gz#{t0b+U={Lq^eUBg6ma4%U^lup zAvTGQn@rSH3)8P3syGNk!5cwiWjs#XFp$Dz)C3d7^qx$^Ee|?63I_6k>Q{Z{gj5O3 zd|+cEhj#n5pl1(RKYCK6Bfs`U+ZEp*`ISkCHH(YZa0_5_XdA z{}k$d6?Br&sKHO%!9MN7h(0!NWfG#%(Y95Eh4Hw7c-QVyO#QBgamq#e_r#b{s%^Q( z9EfI8!?PTqLxmZeK|n7eRIi_r*mTmfHGnCS&#dYr-Gb=|haIvTS6^OHS?GezG^AMC z_vuE5_$?TIZN_sftg`u=xWsRAPs%-hJ@gv*hs1=!?=Kf6X-!NA5-`SVIKms)rb>Qd zH))^lwIZz7-L6f!$MrAs&8`XO%0bv;$5CyQg?FN9YNz8;@8VKB{2hQ3NsGF?+mm$%c7`LPlh!sIQ9GMLu{f5tXwGC%c8-Z zp(1TL_B?l@g*5ZQ`BO&;ad%M?8;1~tk$)ocfMv2;-WjA0DKlzbusIgfGX=wY4k3tO zWjz!}?GIgw5|X-X<1cN^i>AMslO5!Z?bs?qojVzVbQZYj$wj#+;B%FPq%)Nk$fx{+ zf^A47(fqb8f2Kz5ciFQcUwnwI%qhp^Nm=g&H4E2R@Qa^N_t>GiPrc!Sp*#15teO`W3)2LJ=fok>s-AcJ~Ia9|*SaDlyk zEGP(2;G%vkFavB9qyK&$+Y<`sutb=RX(~w_Aa=?CQg771(?_hOI|v>uXrjvCGqte5 zbq`*l5G6N7;%WPFru0H$BR%5y(r-ymscKVBE}Tw**Af}o94L=oY}QIjG;PDpx_ zVaY0*bK@P0(B_AGPuX+{7Z3*j$Kx9K@)9E1vY?s!lTpiaR843oLMezWtd_aJl=HBi zqkCF7C#jMA%#D>RmfgD}Iiv=s%xy|MNtGM$ZgMWiII2Z)*@}P~op;g`SN!%Gods7# zb|Lucf{XQQWuX7|untS$(A=$V$*Zq$t7~9>L~Y9PGg?QlnIHCdHP$ooELIA%ozYn1 zcqU^xq{u~S|Ma3}OM~z1sEe7yWhKJlhw)Wch4z=1o0TM7z(L)$GQS_Wg5<~?ST-Ic zb5C$)HNh(3*X(K%%hP-#bZ?H(vx&AIw4y67RoRc@AG~z^Yo9};LqfFD^+~i)NhK|RG5YPx@e*U@QyYic?P_qF!eZyoo$w?3WbY!?gEB_vUAfOZ{jV%*-{ z(KgsqVsfZOsj9&?%W9>(*O`fwS+}ffJ>-Oj%i*XB#!Q}lWp`w_`SI`v*>lshlvqhf z)cli`xbePj%;5y_Z_Uo9^(G8;dGN~i|Vr&?KcWOe$8=Vo`d3+9a>La}eQS_3M_S54U?ZJPG z^k&L>T%L3gz-C6>rmrz;a`In;`FZbkQ7CH6IPV+$QxWqTg1o@?10H*r)XiBlTT?hG zqCGYZJMXN4Rf{*1T0QW={hU6cVtRUxJCyT0P}T;H9kxjyZ)EkcGYHc@ZQyo$W z#?&T(JroQNnH!?P0;-bvXH~=%BX$t2s;dE)XE{dGT9&D)X5M+sJK~=TGaGIjZPp+4 z0p9tAH?FNg7wz-O$;Hq_kjmF;le9G$+`K6~k+GK#?q z6K`oX>L!-r!BRo=IR_8hv@cz2>&WtZ`zJGn#F4f7eU zl(n}et;S{wot6-#aJ#^Dk5T&7*QR=r;{hloWed3++=cO+MvYqTAi9K8M)xPClY0Y$ z9AT4Y*=tgkSdsTTHNeO-k!4Nr454O)X3s2+J^TgvfeQZ|HD6GMNSzJyQ$|q7TTTJ# z3LGQ7yzZpoShkN;R!?hu=V9iNB6cqI?R_lqzdL^^u$f*&3`-VA5igD;c-xLXn_4eX zWOc?s2_}2QA$!OOaLdT8zAKC)IEy3tSEE18?!*Uunw`-EJ52T2vYY80?1X<(u6aX3 z(&o~(>iJkjI}PcVm}DI8AsKB2hSteA&CM1>y&Gf}2Vel|hPG_|{WIq^T8YX6?QC54iEblDLQRjn+AI zuml@s1oNsFJsMXTh5*H8oimvTFf}LyJ#`4jhn!wkx8o_dg_ zNl+A3>2lwBKI+I@wtsIppM#PEG#}h7IqO5`zoq=E#OkenV!z8 z*=@17uY6ML8hsfU196GXXm^vE@xLXwi(yS0lxDWLS|lB60v@Mu=G8!mzk}E2(v*(g z)H~i;f^#!Grz7lqCt<&5G3|@kxhW$Hs@2-~2QQ>XtbxC3o3L`U0IYJ@xi-PDi4Xta z<2%7dz&Z?ewsBzsdaiMDbUhKXXD5+KGc|EF7sE6f5KqMvfwj zk(t&xdrmInUJv#@3JglLeWY&y+(4m7Gb32?!!(ZknIu2*xDbp5in#}NK&lW1G(RS2`%?k%F0j(7h( zyId`Ekv$VFh|Z8jD{ug9h`0f?@mAbh^PQ-{StNM#wEfp(r*A`31*gyEKfiqzce`cD zjeFtYnio}jX52w1!t^cvEWz-;(`0m~xZ)!EAOirMcn-=Bd(A~41QzscpfW=eIa{*` z1I~2CyaM0V#;Z^Nsj>$MCY}EN^TTQ_@o$)EeuE{YD=3OyD?`dzP>F7ok9!D4(PRA> zwqjbXj`k7o{B@`*V$s-NUCbu)SII%`F%zy*{I_9Yfi5s7>F<2ffPF6_j>t zRl1g4$Wr6QzKF~I)=*(%NJzhgF!dBipR#VCU7(C&AuT*++GMRP9&iz<3u!;dEktt! z=#d)}OX{c4Acc}KqPrsbzVyfg%!uk{0NK&(s zLNptuy_epPZ-kB&P>Cy3J_VW5|out%B}CgE7u zfFu46B6N3kw>Q%_NNRb4jp7zJv%vGrgSV`21n-N$~=_-PX7A`luJ&% z^F_zQs|y#m^*g}3+A2maIJ()0Lp87D1U$CvjWIfvj%>69o4`p^zYm=KkV>!m{uQux zT2dg3f;Ci~ERBowQsaZ{0)s3ftV@Rim+m57cZNbx_lX6!qF3p!mjLS4P z0KVcSXV@>Wo2u1I<8|^9q29^{RjqO$l^(yqf>)$6DrF1L~DO*r}t2JsyS*e1O!Sid8 zi;5^=`+(VvjLxrIG|ED=2BxxNN+iq(MP97w4D{2Z^B!k!Fucbp3<+2hDrgcX{idqA z7L|S023WXZAQp_DM+>`a;-xbD_#_1~a=PS=TwfhqYS5nP48;tPZ^Bj+8v2x^Gn4I| zV%PUHHIeq)7?-LrE_BGUT=62M+VZ!}y%8lRV;(qxQJ&bA%yGhVP7V3J7>`<_Cf;R` z-ll@rQ<;d7ECa&1)hQ;$;ZS>?QdL!1bho{K><_x+;TFWxo9-vw)#iqze<>y$hPYAc z68Wmi%JW~?*40#1xZj^s`acQrEWdqxtAd{QraRM-q@kKPI6h9vyqnXGrgR{n!Jh9M z0$O76NSKDyvv?=4s=TQfyxMihRg?Bod&6cGi3#s^c6`2M=;K2xN0>TzwUH5t8 zO7xwwk%OvB@?Fh>3rejC&^GIGh&`-a?CqEhe+itb!#e}_UKQM zd`Y#FC-a?s$$J}+Q@d<2p!R)`Xdt+i+QH$nM|;96EJ6c$XYAbBlvT`#XkxTs#@sBW zJ$9f~o5F@1MXF!*e#*q6$tbB(;+^V2JE@c%CG23Mxu%0@IjTN>5LXYes;2gy)}`;4 z4%X^Q*SItr$8A@A!r!{>!xft8pxR2&I z=5N9241UTOXL~+8?>uUPvyi$a0+p%Bm*d&CtA-0zdMN3qto`8Ou^8 zyv?{h^nFNA1^OMBa`L?n6y4^XD&<#XOyCQM;nznRwnsjp|3_i1H+)Ic9awb;yxoU@ zglnC&iI!Md?v?W;vp^IJD{;F3-@Z`+A&HnY{$CNhXS$^gB3irAGL5mswVWU;CJ|`t z26BJa%gSbcg%^CWjZ>FCewu$ZCdsC54D}BUnMG`|^{J#v{;|$^1rRgrwT#c5lHUj^XYs3N9^|It;_@#++LO(`!$C%`xPL% zaL@gq)MIm%-gN*;-5Gkr5c1G+TqCkM+hjabOgj?ojQdi+KpV9(LZ4%xw{E03pGCen zr;MMIkWOtmZy(1vBiHQH#oUDkAgNzql0A4zUg+aj*jgn+7p+P@{;X7d#*I@Cd;hLP z=M!OT(J@4(ieWa196R(pS>4<>-JR+~N*+R`HAwvu>jXb!z5 z?4qT`OWrK0VB+5|8QhXH-kir$W!Qoz$jM|d$rF+25ROe~$I!}JszGQ&!&_XFEMo9S z#FH8G*%N-RShP!U>VUf>6U+ybm)ZdMXqXWW2>FfpKty+Ka(5O2!*?FPNHz1NHEUJd zvh;@>s%?ie7H zX60}D1jjBJHbBisK=>rQN^PjPDKiIU;S14HrJPLb|& z+o#7}N@vMAH6Ae0ZHl*Q^TbIGMOybD-fYpSYwuiJR<7#gE6_Hvinc$n%^)JTIiSMO zHVDSDpA-{(Fc}6gFZH{fz2Hs?bL&@?D~k9WY%VaR7jr-55n$Qw{y^Pc!1<#oP3Gl4 zokCZuFv~^E;qoAdtR)nbn9p&X><)~9HB4@@CTEod$azKH3ozQh3O zCTVf$+CPQLmh9Ol!>1@4`{~bV#?OGUwEjJ3YL2!>=oB~93UU1<1@#FEjw|i1;A47$ zrueJj!nl;9Ev4c$D5bY1!oLhvzP9SQITC|iE{fON5FD#G&AV(hIUQ{6O9S8Y^8ESqQ#)r5TGDHTem7+-sgrJXxQs5&*| z1r-ceb(>@yGg~bc&No`#}ATI<1@0O4r<~ z&O~fhlgPH99_ zT$0dJm7l(zXdqhX34NL+4MAn5I!=ieNA3)wgeFH+xGDp&9?Z5~zU(Gr*{%3<@YbjL zs&C4f1S+qKA_p3|HK|9$#opY+Wuk&ZD^YR{iT!PPISGgR7p3cZ2!!K@jX5En-G9$- zYfd~?j+p|u@%tqsteH3mVbDxkZdY}ZOpPX%4WqY;FYBxuPh-bcKv25FhLB$d^c6~n zkUGbZ>f3_u>|(`Gfik=A&w|AnNwS_fXkJ|ppelh1;L&|ny2rW{JTg{2=l%yu_?MwH zbd%q|=TH$4WP|bxqx465AI5^@=`S=JT7@_{JhnrTerf}O?~@&>!6 zmqnUjNcnpKonHsj94i*WUCb-|^^p}WzL5cgrJe)~&zd-+Rz<{=MH3~oc?y<#O~S7* z!k-M|6I|)PFf$C=z{0gKF=?^d(0|ZyLGHCUHqda3eJ2|`iJiXwD3GIT+0QOGZ0stl zO?*);QdvQ_GL&HmCv2@N%X4f4PGpfC78EH1j_W3*Z`8^o2 z!@t8wf)vFPAuuMVq?R!@vZre1PQv$&e;VDR7I9>J1_br4$(f^JidfOHkJT}ZnYSz&=!3-I$5cV~V}G|ZQ))(sD!B4{6Q-X^PkU^_3u;XXuysh@wfQGK_wD`RT$kG6DkKU!@4zvcxsGgisVjN7u69{z5*u zz_^%wNgHRQJ>R%$WhJQsax3GgrjaUz!l6NNr3Ymj)}HCHj4`qAf006c0s`9LiOOt) z`&C+uWUbC$m+IL=AnVUp018B-HYaAi8@YF<|;x6$bfOSs+sZXSAg7xXRt3Pl&g#9 z&nPWS3^uxPE`!^h6K$1w(MJIeAt@uG;ldg)A~*ZS+tRVx@I9O`7=LfS-M5Lx#0bXB z?@0>U%=b%VG4I|a7Dr%4OG!!$k{Mxr5)=q9e-b|f7951H6XE}@tpmV=g^>z?G5=hk z;ab96=It*a*@i|K*Y*<287U@Q8twoEfN(2=HS#*s`v$QM_0i%Hm8Uu|qAZc>s~eb! zv~^MK^!Md<$o8hvtlkZSlGI5VXeE{bR?jlGW0@fP&V^RT0~eUX-9gK!Os$qvfu%|Z zJ?%UFz3_N3SrBFaYFoK1F?#b%8+g2xv%X-g1xYXoEKomi7ZuEvs9y!*9M2lOihMYl zu@34h)n0k;oYS0$o@l!Se&sGF@C&KDJpw-}s+wswDRa?*IyPnR29-tB91%3KGs{-WjH21Xh_D{3Tgqr4Nvz z*_s;0`kzZ#w1d_&*#xYBwxSadVElt0Y;lU!6`omhf^*S+uF8MT4BOf$Lf?#%#IFad zDVnO;oYlpq7fWTc4j-&_!8Jf1UPsbN%e$AaS)U$vcmS*2FTQ*eN{9*zK-w7P$Jxrb z3S(DS_yi|Z>3^1hbiqkXuy0|7CFMDO3DLv2nnj^?67boy&6w9>0xw=?;^w!EwS_dI z@g=EA+jVHZDI7rf@Ex`8ORLQzyM|Q;i0SiS>*2q3NwwvCwdze{2N?ALtXf?vkDS3yqjfBezL9Fkgd#7DCIJzszVIs|)510vUT=763?ni1urAirnanz;cS>TyL=2 zzF&bvcV*M?2#{Igzo(RF@i~)S?>nKLA31(K1zr%_ zS|(Nz3@z|4cQ^0w=S>;0A0PWMGKWvl$GV2=IJU6H_TvnzHT;#V;jA(7JR$awH_XJm zu(owB)Xz=&9$dqvM* zJ&d`TS4nVArQhnXh^{$e0z%^z{^i>mKI{J?z9+{6&ZQhURr1v=XrS!rU?xAS}@eu`mJ3vIIrCVJYcOGyeP4^ z!H#a>SnLraURngA&Q=pQfpCB>BLMEfZo&DFWLaibw2Nq+xR8e#PzNC6VoFw z9^&YE3W$yz^JqD};5lofJRP$-!e2zXWM|B?ql3R~qkvIHEML$%jLoxU7U$}ms(sQ{ z{w$^YNvKDsZ)HL~Fo-~t`8uh=;Afu{(w9UY3fuoC3nw|P15Q(HfRR@(=hgsHyBpLH2akGeO}{z$L` zrfSA_T~v8~%9?5iwlyEQp62-(n)~@?eBXx2g^SS<77ToVM#G$9}DYm1ID?%duGd%U(#;Sx_4O;g8`T5vD(_ z-f-DP-qH+_s-j%fq*X;Q6P4}gS{)w*BeuTVZzyX!{d475>BG8r0BS)_9>7!X)e%xf z?`SQzQ7-lJU)_k>2S@K=#Xnbhy83(8@}bbM)$?fVky{@{nP~X{2Oz6LqkFYl#wbk1 zb3-$M_Ka^Vzd2_(pM~D0jbDZrG>jg-tB;vD$QjJQ zs#<6-;Ny9FiQ$RBOhLV&_dNwG5qZn+H!l<%AwPxox6yx&g5NquqQ|z?I&kwxSF{Hp4t?Ll86;E;Ukrgy1Gwvi?-YzFtj=!wl z5yKRtY{}^#qszbO-PD|4N;}!&>p>-i8cHs2zdRGZaw-0VWL|b2iW`!d;BTzux0yVs zv4A!z#Yn95N6dRR#^1mpYenyYOykTM>kpL*l@NWYB3W*AadsW}k=62C1t`A0EqfPY z(u7*lo+>l(I^(cpbV3b8_kMP=Q+!$ztoid}@-FxDn+fML*2~ z4U!FfB8L<%FED!cjK1Ae+5%Rsl>s}b7XL~Hq-0aX5r^by8u(*kuBV3Etw%Xa)2tXxFAIYmBJ__F%{jzl5Z!jq zp&*YbKg5R9z?qVd>P>~J>H=>E3;}uogC!A0F)_jWE8u}2_)$YQ{uEhRwaId7IwY+M}sGfFA3_nPTaMV~n35lXMo^-B~R7S#}3l7>SBFP}n=F|4nm)*I8V@-#3|J2T1jhBL@l8SWc#eEs}$1JJnR-~X1=9fT5(JI=Ek>RRXT6kt@rI) z*;5#cAr5#Vop@AUBrdZd@d%-SHMo3+j~SU4*h?s(&{dKP6B!v<)0S6BgxL90d$%I! z|PO&YMnXt<=FB4R$fxwt-da;2KDFU z)hgZ^XF%BvJeN%uuC_trIOUvRjqk?mM$noT3(i?v6*kK?(m(Lyj|<7iw~w{bWoEpO zHTRfda*0Q3ckxtDTffAEt%_X)JR}lM&cSaOWka$qt{D{6zlpPQUjSv~PdO?_?LjH5 zeESn|hHt5-=a=Y>o{Svb9p&(2&62gK8P1ItHMBcclIrXi>G{qMd@ z-;~PNlfS3q*LOEQ?86C8Ze}ag!$V_ts;g@(b#sUDzVu-yT5%}9nCL-h;lKF>Z?y8Z z`4F?M_2E|dd2mApgqGxqC5WKXmNbDoBH#d78%yi((e#yQ@EjuJrb;L%gI}lFpj6RG z0_r6Upu9(>?=eDUK!b|ur0&WG_~gUNp#b%W?QCH8iZyC5XU&l(S4THDc$vgt4mU0 z@Pox$JR5!VBgpqn#aekl%?T16Ui$3`G2fhWyMKY3S)==OYk{d-oqK5J*C*&2}p@oQjiR8H-QCx>Krk5kAysQZgIWWJKfHG zaF35fJAXvqoLtLlt&{Q-B}aFbTEx-%SMun05vL`yY{fWPgsMpwtciqj|!I0vWVY8&-E?RcG|59 zgQ0rVt-Dgib+RB>BI@;x-35+pXfds+_jG?b-IogPFT`?YoUq9gNE8c#}WXNUDOub39GF((Hk9~Ci^2ofnYhjLR+;WWl=D(YT;BA{$<($%_*wj^J zkz$In+#2rS5Q)BYQjMy|3U5}&M2MHkUwvX_mW$1Q(^lQ95Z}E}g$&%EEqVMsA%ku_ zn}+)<)hd3xV9gz0L0e_Cr?9UizT4%TBl_Q307q;i67qxEfAbp;kP!m#9YRtW0B{F> z&EdK4jJ>`9fc4jW4g^Rli`iLQn;1D;*x3SD0bnEm2&-*P8)khi(07raA|&P{B;D}( zB-Iy;`Xq___yDbGS$_Zt;C}5#P^Uv07%&z&R^v~P8bk`O>Ej0#cYPu&(F~M5SPOI# z44ek3E`9_65u#P51_o%331;}8I0Puc$bfq!1yKMX>}yf+)#`uZ1cCie90gG^0~-@Z z0{{~X%a`RsVzmeL`XwhE0M7eN7$aw+$Gt2|*iXn$i0(5=h%8%C2ng`|Ut$01SUrFp z@YP8DS1JCga()%sudhk~UsEyw00{tJ1OdQ+`jl_~>)5eJ&*a2};N*ntH@KyI0k9n5 zZg^%FLlc0#xrw8R4e<0bXX~Bi5C{Z-EkWwrC@{kSAOKQdYLK-sGk3Oy+b(%evkwq% zuJaSJKkqp3q7qW4HxxWLxCF3$b;FMh2%!FYKF8N7H6#qgHnmp?u&ymN085h}B`^-x z#tDt2o*+aCC6GKw>bvk?5?DA?4AHP2lzdqwK^q=?bo5>$J>f@MB*i z3yf=d<|4dO)VzP8=+`2@Jled6kNP>!a%qz)wrIU1(8-}Ij;xQ){|@!`hoIa>ySQ<@ zY@pO87qNx*=|mUlx6lb`8Jwk6FB}e8U$qKRrk&RX1OSRBX3ow(%_mehHXQ?=44p;Y zl=4SLxSH4PCRcu5Y;ZygVrkc=z?I`muEVIqt~``8^k4TtF?H_Qys8lH=Mw*!X1Gk) z&;6nKlwoUN&TWCmiBKSCD@a2%7%$(ez}?Y=o^Ds94*U=K95_rAKm)Rv2mmnh1=1I| z{~>Su-#`=-`Tvl&`?@3rT!4fS`X5MNw~Kz&MRSo(g#ll90{|#r^&XM0Hr9}(zb?^; ztRatfe-)vxdJppM_hzf^{0`K2{U*u}1+IbptvWFh5q5TVVL%H}uwZKHi%>jVh!9s) zvC#j9LKrL*36!`=ievc%NF$=t-lM9;cC`E0z+LD#EZWZ2>^M0`%HGb}!`9Bm!oXV3isd*g zht)hyijAGa&%cDViVW2PE*>umxDAGysx%2i(|5#y2!}y2#vlU|4O7JRM-~2BUhD+T zMgWC8k7uMdqc%Uop)5HfI5Nd-%o?drDT?+<9Da~zQ&VfkTD6#47F1t7y9imR3RR<7 zBI?!V@0uSQ?@Gk;C|$)F@V~TW%kqV3qn^o7Qit8t0K%dgN-b;-@Jd<^wDXx}s zc`fBIf+6jRai{(>hoJQ_4u~3tP*#%^a;|bTdM_Q&H(r8zGa}7_cEIC@2U+Kt>}F!#U`2=3noO; zn*9?~r}lbaZJkC;RT|HuUmw-$!!p$?%vC5emc!oN`kWe3ME4&S4k_-^Qk>9P09bel z1PTyN|6zd!cuq-Sq5EHyQ3L|d1pz2L*2Wn81H!)^%gW@v3B2jQp69q~aq}?%5eqF| znnPU+Z5oT4NnPJGM}`IU0xmyM2Oy@4I>Q+J>ts20fB|TEC>~SDB~8PQZol!YX-cJ_ zqq7U-!cxDGyzD1fr5wi4S1}Jv(SNP?hd7g4hXsF#6G*O}!j|LzG1nsNH*N|8c1uGF ziYhwb%eVLcJ7$yrj@i;s|NqkmMG!^c;xB+QC@7eL4R|jF53s%aCUhtw{}+HQ1q;Lb zmlvslCl@Fv7z(HY)2S{!)&^crq+Fo;5Ay)=%*Jisd^KCB@I0-HJqM~iK@e$y7a?`n>VyeKitqRQ22*-o`Pjr9(hWnk2@mJ)yv)U z?2UaF3d8|?`@d6vyz@@JCM!iG&B8OED&@_a3F^!FP@XuE3%8@0uSH)|hLx(xrj2QvCB7|I76%zI9{m zY@X>qg%io6oI4R2^CdU$A2vKen;e7ziwy5fM%RBEnG2c?`0vi&@1 z$d-kIlLq|0?Ds=)-aTNnG4F9)lyqX`-LCn*U}=rebo8fy-uMqs_6u|VPAP>`M4{+G zqW=4ZSVTLpo|*!~0O=g=^fcr-{!HM?4w}2+gp=Sl^m^4f(p&S(^Lral*KaHu1%!7r z8s8Vl?P{641RAEeZ3*Dx`Cga3J0N;#-RBV4A@ z^YXzCvn-9Xo=eH=N1sZBU-HNyoK^+R2&w{yB94JH#(@7tMW}n$E;aqRL~9w1JI?9Y zgM8fy`dJLU;!C+zAJqE0Zk>0j;q^(@PE_-6pJGaUa|Hc!MNyvP&5)dh%5kl#^RDa2 zUx+}?|6+HUeVP-913T;g#cuwG|HW=1fL}@qJ8QcD1RP3%pg?%q0Mz|2SqtXt035C> zsCx;3!i890F3tbK(*|`W_+j@Swgst(lva0aecYiigrHRb6c&tt!Dx`rHMW%QPzY8m zSLSm*lBKd4`>n*oxX*LsLpdv0nO`kvjS3D#1!Jc+HT5Yd-7u{T5jcHj{eS6Vt%zTn zr*7|3<%}%~0icnZ!r&jgDH!zjZ38_0V8I!Z=l25Os0T1=3Y}Rg!8*a_{8l}8Pd{%& z0LtGdzYD{@48b!*H`g@)`UprzRL`~YFR$>(tl0vG=ICoa1Gaia34WbS6w?}+Ypq}2FJGanr9^O-eu(+i!*qxE&+)#HF(DT=&_ zSdYQR*0`{hkdG%{S!dG927FxFLt51hU^I0T{X9Z(65g-YAox z0`*)Yrl(dc9B`*owh1o^w)p64k28KT!5=vn-|Zgp>|R^;yOrVxzqa_-vp)%IWL$VQ{w0#-P@e4nphkB% z+dol{lFnGiqD-FY?FRT_^{y2-^GYM!QPq@gnEy+Z(ax%XxPm*%n}VMIa+IgkJH@Pl zrhPBhS&AL{1%GyRU!mQ?BzXon?Cazl1eLIK0f`g+Tv46fW8;pj-D1!eJ96CDM zRN~bAdO9GZ3p%aK|1j%Aal_XAK_J}M-<$S7?iSSj*Tq~Eg7kIVuP6mI4eqG0cMUp8 zK3x{bFMAI&s#s>EE*7E)e(J{h;L5?wp9cxe17=ScRdrG?`TTFrnRg-ABPj02{Svg1 zvzk|re|LeJF;lvvrxDs%xdran+({ejYCBRf4?%qrc$+qRAtIR){l*}^4aVZiMz#7z zd51!o1@sg5R^ah93a{@j)&3(wyrwz&uQe5lzEb|C{7X4--SKE*U_3;lQOH-ru5gC9 zZ?jlr>v(^w@Sa||9j7jiWPigm>+_pax}e~KU!b{R3Cukm>ekk{y8?RPf^U+@K2K)VJ=F#i%C~m_WXZJPk zN2*Wsw*Jn^MSiYfOq=X)z42z)r3pD-_PVM1Vrt#}F{QWLU#0HKRtM4?k7^hr(%+F1 zHkBmOPwBkRA^+vtuA)f4dFJEy4{KB76@LA3X)%8Ew7*4?1YGQ%(;KZb&!tJe^>~aliO1 zq#X6(mahR~$@b;y8sgatya@@PQJ1@Bdvgl7@J|6ZuRQtglg0+|Iq>z?T#_6cxi63f zx7$u>RN%8!h92I=)EweAnn7lwG=wK z-;srd#qdGI7keWk(H|?_&p>*5dI}f}CNC^3%p!PmJc@#kFRSr%Df~dQKhaG1GA*Ct zx`EZ#+*37RNm8CPqp#;iE2#*y#)d4*@Dpu5j!W}m{iX}ed7Mq-tls_RWZ9ID4`3!d zm=6Y-7F~_+UvD{HV&7RH8qM@l@#u0Nk~qZ&sP1~+>+PqMV>8u_MPA9BSN4KwT1)4e zD49JCyx`c({G5@egN|bKM-r0tYfg9I(zoafo9#+t$%`@lmpD)8wqc(NePa*$`62xy z&G+AT#LmtYzE~9-xr4C`6rKaC5HCmP_*!diyvxmbTxrywPBFtlqxvd#FgY}y&y#3) zgW~RemNMQOmgHfJd(wfz?QayiT6K>;Q35NIJ@ty^P^u)2CnYpaI1P5v(;kw}=H&fS}>F7Wd5 zTHkB)j*{!3km^~TnwoNTcW+BiCp?LYYHn`s?CfmI&CTVKN!REe7Alt zR#zO!wc4Yn(Pv_Hr{&MjpFf2RZf?6*N+fqsUrg!oIHK>CdwE&Z!r6D?ZI5bm;E|H$-iyVB{4mqq%W z%4c@I{WNg6xX@EluCth(iNHK2CEliSJKnD8AC!s-th^YicgE2Cc;=(((DYd3ksoGqMSN0``p6 zv9uhatC30)Of##$Q+ik0-Q7KjhA~Cjv7BpYVIiACL7_%UH1!7*3Ozah9Vss_Z+-EK z=AwXr!0+klyFiPoWE1s_ji2eMWY?z;6dxradB;#nY>@VVOS_> zs7@ozUWMi7=fAtdCt~t)=i7z!p_tBS+80aVjo`rL$FvL#n@^rTg_xSgnN6KsjCJSb z=1!N2%gw!b;R0f%%P@EP{#4Tjde-%ImFte76iwgt-XIdgCl9JsX)q#P5z*_;Pjv(wP)6>(JoaX61dH?SC37?VCa|39!h-u17 zG?RF&t_iD=knokuZXIebx8)d~n9#B5F$2>Q2?li{gVEE=}*mdtw&mx z3^Ok+pHzcg$aIZz;mbTNj(?y2>I`v>%qZ82k`6kTba}iHes8yE@3))A%V02H>1}s~ zGx^#o<(chMy1>R>SO1CMyr{97|A$+6>{`q3B=tq@j-&0fP%uwkl;_bCtUAR>^3ugi z7zGVEN-S;n#U^@odUg7{%o$fWFe19lEux!RBOa79;MMo8t_tUitF%f(SqP1Z*?h-VKR7DFP~ zltcqx_w}kRvoa}EGx3hRMnvkUUXl75#p~L~$aq05|JT*s@6<`l7dXFmhRy29Ulfm} zNon_bhrROrvqGJek_Pj!tF=P@T-m|854(JU*tPsUgt5Bj@gOAad5P9s2TQP5?%PWn zZi<$)&11O@ZUW8jMELPmYf9{uR4Hfzvfz(x4m=tyFmdd9zM#Euzka<=jWSP@;;+#~ z-^Oo>)EQ4>`P^ymC_B&3s zjC;Hz=wKX2QRAf!?PvenKqh(^u2Q9J9-5WyC9h5J`hG}`xAse$`w5Ze_k}If-g|o! z)0TQH(8VabjApw!(A0RSa`a)5`vpRM3-Wg2)2<$S-+R%1c%;zO)d_i1|I-vMY}fG1 zEP^Yhgk&M=U3#qzx0(GKKHZoXvqb2MA+8jFEm=Rbr40PB4GU45cUh(*qUVS)C5ba< zmjd3^XiF{;Z!tt`WPI0oAKrDdJULX`q-_0LJBO{#P_`WvgKfpK2DG9gDWfCn=&n-H zwPDKQLx+hHxgh0w>N_MA*1FQtyt_YJ$FEWMpFN6~$ffm;Z`h`+Drq5p6pboA+e+47 zW-3!syBMS9gL&`Neo?hc(ZJOiX+a|ilM>WDs-pYhcJSUg`=x}dNT^BfKIim~#LQf_ zv>kA?YE?YAF0h=hJAtHpqf}tXI5D(RvdL^1@bWVa}gdF>T_zH&>Z@Y zy|E27x^ok?6s#sJ+BG4iZ*~TI>UL7jv0MeVVZJP*4&0tL$!E=RRw5r*6y8fyexHNd z>jKUxwCnPE*nPhnTba)|T0vK6>$PAv_uZiWxH}F(^U(9H3aqR4h)AkC~7_u8P-AyPKH{4@wcKgVv&vP_+;dG9@Zts}wnb8F3 zr<$!r;r-DYXmOj)i{{B9JxRH@!xp3F?!qfz%UQg_*);*?yd2Jw9=+DVpK0~^mNqWw z)eN56epUPw8XQI&@|VM?@*J~+>Tdv1>^%L`LRUAd_34VFu>Jc7mJ*`0MV-79GIQ;h z8^#?`FBhTK)C+;_NT1TJ>$m0ApUiJc)Eu$E2;B+$U!xW%hJTq9kLDQka? zi7etC`ttIJzjIv92x*r@b^mYObX((oW6%os;i z-xA^zi>2B=EJ)O_>)08T#MG`I$GD z=(dZ#wJm%w#Bk4_-`Rud2}AfjStkZ@^$a)hyndAp=yUmo(%X89n8Gxf@o2C%yxEWW z5?=?aHLo#W4V$xyrjH?(@7?>d{It{jpo-OJu|7{b>5S?;u71EggY;Fw4bwOhYbV^& zBA@AY4kT99Z_!!v@!xo&EpMpVm8oc~m3Qr-qnDRt&U>MButCV#c1oD@4u$XtYx2il z$J)2+Td6IB#4oq9xqcvh^Q_)h$SxBV)a;*foQPgcb%%c?yDGw1P8lwQPb89x{A6dcZgu{b`mG_+#7&Nawg8JiGsnb!lfv1iV~yO$8s!yt8Rm^fvf zX2?^&DEa$o&S!q5@s$tw-?Re%D`y1Cf4fBSnaR|WfD89M4jcscBCJ3uSjPI=F`d8AN{L-*}?;Y1?BS<@&)@8^YfJi`<3(aRa6Jm z9*?}M77{YMKIAV*ovh=P@|E&(2ZF=!IXHrmlzc(n~zH*%QdsVs@6& zGa-S0Udp>z{fgyL5mw30>AS!FPAK=E_m`OX4PCh#EV~dndVdAsEOj}w< z9zK24(eJy0l_E{ehA0auZ~pPDjB2cTl-9<*tD60tfHzDOa(p*)DZYw9Lou?*VP>5bH_3cx{8IlwuPLlw68v);%7j_nWDqlod&X+rgQcw1ofj>(Y?Gz%>!wNWkP0UwZ@QN{POQ! z*`9Oqw@lB$4g^<&n+Q*PS5VF(esJeL zO1`$B%|7(1+Cp%Ye2t$Alj+NoL4o<%pB?+04K1l5+5LgY$$f{VvDg!Q6`i5XhW(6A zGSgmc-jLUA6-VBWb?paJpuQWjl9jq_C&w)!O=KSSU;dc|JGd)nE)I5(+zPz7x>p!vJ?XPpr&Q?VWNT!zI;Mhbu~*vel@Esn`=8cw}^(%XTdtJ#d z!#{?ETVgjvfBfWJ%0Q#K_!UEYYtH>0mbfjLo$|tDty7f;`yb2Ai1LIi!vjNjMj{^Z^JE}`9y;I93xX8UYA(7>xQ!@v^JXpJd@z>S=Gd;JOb-v-a>8LjAk z$*}C+Jf!ZK@7r#1ET!(@PjO@1u|i6@tkZ{-Fve!D`;|=(6Vyf?CTc)_#DuEUguLx; ziDUI6OgngQ#Ft4@k5x=(b>fc|Wy`{eHT)m@tZ72=PME3#tE0h6uAM)h!f{K!S1@11 zckol^g&D62N(0<}waSuy5R;ny;q&A6p`XEMWm$XyF)2jL+G_@%vOWL>))axR@nFk& znDHLF>j{-vcF|S)N^Jy2zbKLzuoOpL{UOTya5}BnH>=no>))dTTQV`Nz38v z>%c~#uvSoP%5;=^NocgJP0or^bMl)ZiN4&K;oV;3_v8L*kv2)bAj=Hyc4+?Q)^JNH zb*#W{>$-HOV$MLz{0QY3LZmMRFYNv$@$Dw~N8V;I*v-8)*P@3nM2{G1L(AL=bQ(FZl9DBn?xnT(IuxNH>)&yj;50p4wPI3 z(Ic5#QPMf$s9dvE<_otbD$pxi$GyIYHB9qxP~PHj;F$cYX&?D? z2aRJQ?kh_#Mfwh`L9KIh6J9lwce`XH!rS`UR9If+x|Q-D z&~FC|pN-&MPWfG26?2jmY-sc|Hq+L?gQEJyMzI!&bnIFR(EG}cV$trQ(T(uELggC& zNxqR=Qhnk2wd|ea?45_~nj=mjB4>%^fwmzq(X#=aDaoS6-JgGMrt}~Z-8WI|sbb!Y zQ@k6dsr)%&1R7tvoZPh^Xw0>pnXpkKw%wZ!n;vam(ld;a+&NLoVUZ9pHkF?@=OMJz zaJ1bTFFjK>axmj>#Sm{tqn{-fUBfEclA|Qwc3eorzCxF9V^((9+iLvcuJH)NmKqgt zRn{kiLw=a*(lt~ux3J-uhItaGB+m3t9AvHyB4+S)(k4!*l9BuAR##9%Y2ep!~(%1Zt8I{ten;$vzKR z`NIXNvKLuiEmBm~8?-vmp?!%CS2l8S*SRo}SxGyPtG$1&@8`or(VZkHDZvvy#ncm& zwWxWmhw$4fy>UaKawt5xwlXB8h96rI7>puq#;k1qY(F?SICoy!8nCak8y0cTE))xI z+gDBH;(;JV{H)EA7EQO3RVB0#Gi9yEA*5-_${m~tkK)imii<-iq*lXIPC~V!)DyE} zKWJpv#L(+pa*Fm0Ge;atI=5siVhb1sdyg^FWVsVZ_sx1%{Yp-3g*C`U<9R;AO#jQw zHauHMyAeOP&x{AuYAfnXTKQU<+|&s^*fH|VVk(swz~Nq8*3;6X-bZGoH%sNWJ8hB0 zu@k1}js_3xj9WL_U|5Fr-Gr}F!4d7fxH*NP2{^jMVQ|nKcPjiBKPjof_G$vZKiwt@ zTOzbU(wB`zjcJhWU(Q;-oSr;9>tzecf^0q+**YF1pJ3!n39p|U(#6V~60{$#(AK_M zdNFAJ)q{MwdG@jV(9ZFoTsn<@ww#zaf=zZ9h3b`cka5s>;8+d>UY(;Gz@`14eO7+! z(zp`nv&33Ytk_j9zJA`b&R*TGGf|ry^>!moPD=Y>LHayrARSYilU1>iVn8!eHhefe#a9tPysmwE7+ z-$n#2zjBANoJ57nMexWG?rUQ1QiRKF=m9sT9LsDULLeoV2WeX8h%2Z?#krwNpJvvc z|4gAw%3K;|KnnYVQ*yTy1`)9LL?$Uj`y|ti# z&cQhaIZbBWNe=H9kYUmQG>C1_2>L<_P(nPdV9!#c^FRCL6EeKx!BHer?D9r# zp6S03u2jprD!zc<7@WfC<+5z!?c1WPImv4S>V3*tHPsIjOW;zjWh0W=aRi@@^5)*aEI;OF$Dd5u0~>t( z<2RnWspV;SZX8>Yt!Tc`Sju`~qMB5W+hAB@_mc^vHhbb}?wZ6X z#9YOgAQoLNhnK1TEe`63RUf+#)lK%)Dyf1ud^q=GU%#*et^{!2_4;Oyp6xvhRE*k^>tfm15&pzpzxj*u z{8mfJPD-&VYpSTK7u!|wY-NWo$Z2?+X6{hXWenYK1MmL0GfZiuPx4EF~yJ84Wb$BnN3zA4w6 z6?0uGqmVJO?0Tx`S?@{gVaA>L8UkYEtjOE)%)`=;46Q=IP*AcOI-vt|U&RnTc8d0AIrcSHY#{mnn|OKiz97WE^bCvRjHyt1!7HltQ?lV&3X`_q8qi5qcBoMFtnN5g3NX$3I-2lp6h2e$wGkGgEIS;8 zcEN59SV&m<`<%Y2X06;z)<&;BJ#kY7yIk4Js|&|f&5xjgxEnsXIr&66E^`UU-0aMz zY$whHpW2EBxRU8%*;ZJIA6|WFl!V3*f!(ncq80v;{k%i;moL(78DL3#CBlm}656cP8d)M#D6 z5GQF+{WHV$%&G4uT5Rz(PAvvDQkA)G3=KN$pWS0lX|&}|+=ModA%zC1 z_F!q81|_3)d_B5^F;YpExouL|dsWN&;5<=i2N)g=mMZ3zG!BJFDG&e+Tr#B5Gl9}R z_=c95TEM%SE=BJ>lqVxLUc@OP6_gKDz{ZmzBcTdMk_kK^%mKe?kC z3o?z9%Lu8{f3_2Uc$eJe?gp;8X)U|>W9CO*!~uQ=trc#$zd0Nd+#C9d>AYjEOqRDw z({hgG0FbdQ*Gr{fs`!ntKPJ(qIRh0$P}XVG3S)yP>ZVIk=`SO@A825F43Dt3&sL?e zCeQ4fWGyL8o>*jtj(Cf}jk>)4_ppm(9+6j)HJ>SF=4j((><-eylG4|X9Z6c2mqPRp zdYxAupjTq1eR9Lxp*pFwd8#yH0ei6(Sv$9(acNR6a|2Evj1K8i^wbPA+@y8^u9{6;LA#L`zw ztv|Wz0vB!qjb^AGguPEbN7X_{B#X#d#$^c+u+IpC-)#fxlp`s9^q^hmP1yV;Yw&QaGK zLcD*(k;HJdUc@Y3vK!$ks*1^T+G?D`GD!*LRr{x7e&h#e>_|kPdcbm~s87g+2+kvl zK|m6!>sP~O@Glod++EPFl@3GDw8r)3-qaAo*(zI_u~sJxbjxMxq<@N3t#|nzcD+8k z7A~44d%2gY=jh_A6`GGj+c#&_Y=XVk)Fv`5`JH4ANXf!m%~m!v8UQ!qOuOu#+|E(D z2J>QQR|koysw?JZjwXDMdM)CokiD@tm{WMOED_LZ;vdKm8y* zPELT!zAGfkox8n%T%)hHI|!jjqV@xHtJg4I0lA+1I?Pi>;hG-YWcZn?A|B6|s(w=Uf_V@2Sc;Oj#p$lDONvZ-bmdkQG06VQ`HZpb02-pDQ# z205GO)t2?j)2IutDJCLCckCb4aQo$mTl#wxb8F~k!e@SRV#?*8K8Rf|mGnMcnuOsL zxy@-{LgMF2C`Uyk`5Rv)Gl%Y+hNlB&Re9cPZ@o$>2W3C4e9H|+QTh_@nJNZ5) ze>glMeyvm!h-Gf%Uhs;ts`0ThU>?>5Hh@!Bb^&t_Vn@^Mm&-t{Ux`;Q7z!87ld(3& z!&H%o$$d+!c`cC3Mpy;X>4mD2=@@5h-ms=+;0AufKE)*r<^SRvFib1+Q+-yimRIKl znbdILp5R0I?*PHxa(DDoC}0H$<5a!p>2Z)@!xZTmbwx6$2L$}dodZP0u&sz}&;AJ% z@ErlmphaES9&!vyC-8|z7l(el%5y&8Csv*P>56IK0Vf6++>YE&blE)_lq5Ts$%jB3 zHjYiQh58*c$X)$X(KYiz;suuE-WvX~qg|-y5B0tpWTHDEal>TN#QJ;K_VZQy%Tt0)%(n}$nD2ZBCJI^+15ams-wz!n8;?xZla?U%Lk%zcD6B}mzWQIq|` zfl&9Avb*Pz1?l`48GePj9{%`c_R6Pm+){l3YAHpbOMxN|0wbxpYxXkPaYdn#fIqpq z0vhsl15%**0gx8ZU>s@Bzy@D7snL@g1QNA&&L8~^*cs|aP=$*sDRq*PeRfRuLdssN z`F8zO(q((qjJMkZ4FGghFpB0ACmrB=MpzKz&tAvgjeZFxM4j?!HS(1LwZc z*JdX)=YVhn8fn0Yuu&A+eV(XL3$i9ew8`v@kZ1fyS6O`Xc5zVLaWrH1&UTkMKPvF; z;_x9FS~3-}EcGx=5owcSNk$k{C1kElJAgVX@K-mfhU_*Is|dgUbXBKDt&2r#5O+|n zmF8rq9bKLro0MtoldauFYkWC&eIT}8rj1|Gnoy=WG8T{{rg|N2(>Ex3<^Vo1%3v7u z(poysTlb>(`_1>!b&W7$GFx69o_N(y+&V|eJ!LB9ia39dqgsm6Mc}4-rGpc_+@opY zd}8LZ`FdKxX7cOh9py+DoM6SoC^wH{Wm*AyBOw%26~_~3h`v^Yp+zLJ+{KXl@$+$GyN8o8DnK; zNXp*JYZClJcJz{!WvweFkyv#>U?DNAT8m6u;5wh`&^fwsSAB*tzP2JHxwD<5^8Uzl2K-&I?R-1yilyxn?XtdV|(*c;txR!*HV}mDt z0aa=kjte`~Mvgct%n(`JtB#;3aIsi+i1nNuNZ-zGiM%=-h)k&o&vV&Z$N`L^5okWc z;QpX_Nb&Ga``9^io6Jeaab8|R{ZM#rZ4lQ~CK2&H4&tDpD{x3-d6bpQ3*j>fIX%%k zTp+c1m3WVOTsMtXOgC-yMx!E8_1tQJZAj*jJwspLwL9I$A4ghN;$3U*HOyo_SbF&i z?G-v|!p4m1%3=qsW{XMWag66M9=6M2d>EO}dRFh{^^QHH=7_y*Qt|*DZTR%s**vKY z&%=vZQ)`YwE4-{nTib!#TR_}lS$2J-Xd7W;V9LdG=fskQR~WX-jZ(% z-qU-NILqSt=rm=0Wx9EV^xGg{V|0I*0T>zgY9I9CWxPS%+Eq+iY;;*gp{|WaGr;H9 z)X=LjjZoA^7%SN-BQ(fS zuq0M`95gVBS2{R|B;=qr)NoQW^dW0=J;M%X+rtNtqn+NFPzO~&9C_x}Nn#bd6H@_E zB(gQYs(bJYpTfL1Mf1_>9;Rk8H4P#r*~TsvU_k@uo+sq5>nG4>EB5|Nav+!28!TGK zWU@VR>$qi#gen=zu5Zpm5yzl50|x+YJs=>dDlqo+ZYt?45vuHbo=x=2@0_ z^QB2rZiOYlb?K+u!TzQ=PX_kRSuj?i_WV=$r2pP}&es>;OxGLivq{pGtoD%=rDPUU zUZ0ejiRm4oABKIfvAQzsG~A^z%kr&ujHe)Pu3Y z-EX;TCx1tkjncMv1iXqsXVnh#x7@FA!d5P*p9J5u`R)M8MQVl~w+xix-AtkZZG|G7 zcziw0&UG^x#1EaPdD<}<9|Ls;KZpY+pXp)S{-#F+;A;Q+#NY6JkU*4lU4OlqT(=OV zI0IaNk}{n zUxeksl~3Tcm7JqXG?#?ZOKnG;HGD6}dP+pQv!3xji3;5S4S^0bcJL(M^~dX+uCj+p zzIhCTRyH9cM`V8sV;cQ&4O8}Cy-^6^oqg58(du8Zaz_J+jFazjJi)B5vZu0n{h5XIKEyQelBz3-g;qh&*EzRH8l`3&3a=fK|G9yE#s)7@KaiN*!O5UGRzS^DdP(kN^vD^%va}Vkrqk>mO?`cE zBn`bPC;PA*X`TWU-G%(b0L{9&WdT&7L(tZ2Ak!JOTP+tkjK^tQ8b$ihU5+a38nz+U zRX`M&FHPmF$K{u4xE6p%QUS}ghd+dXuQ_Q^%t2S4SGN}v96>J2p!V0PKVT3BjBj0QLK?ihHsVOD40$5i2_qdnyxR>pCQBZ9b5e>H{2C0UP()D%H9)ko`m(?%m zRgr$4jRgjN?ED?XF`85m7@^r`%#w$3yV209TOa0s(JE=wM8=G`NwjX!R(rZUgY&KK+GgAgV`*=D#eTJNl>=HYqOBKS>_eiPLs$) zY5HjCl3v_-2hQAc=uI_gbQa+S2VaE0R*TufocyVrtf-u1s9uph22q_ouby^vToKOtGv z22Dph0Qoz?yUbHX;;Q1rKA=AE8UDir06Duj=s9ott@MJMQ(~vQpqC4EN+Uhpi{RI93s$%3*e5 z9sHiNk!JxGv#l7oS1~4^iIBHYm8}3VJMG6vZ*<}O+=r?gKuk{l12`vl>3eNG;+pi2 zrAxYS=I%pps))UspFW;_Jsz)f7E~zrX4<--kP{=JE!=2%n{_qche$&4}Hu=2S|m&R0?!Od5_E zbu))m70J)u)@%S`HKr3cUtNTPO9(jqjFv`GKIpXN(sb#>jaC=kn7s{dP@;9>>%+Zu zGhdmVRBia2D6Q2wLv!OH?E_D9^Qt0|*-19|dj?QPZ(6LIIbrtpCHQ+jr=L+Jz97>0 zJsYLbS%3F;ymq=;a-4MuCHc^YpGU5-zSMl;i{Ij=N;Pe8tBONw2 z%$h7~Vw<#ij>{4}){=omuFev)c~;BsY5rW+bK;}i?wOLwLmY8XS=WZO^eCNB3N2E6 z=uu{(b5^AOK1>ncLsfyMM9Myf1QoH`bWpzIbaLz=2Gcsfz8zF#Ymxd`M0h!iEl_~+ zPP}xYhxwaX|HeCXVw>#gZ&+q;(>ZTZsZiYQFd3(TOHeg@j+55vz^POhS-=~08-n8` z@2;A$R27-ReKREuSIv@|CaIhyC@X>~oXpnE{G7}dCUYgH=O*WHc~1BJeVdDnoF&jj zYiaRLH(DnT&5#v!&n0Oql^TkwBqLn5h?56qE>83Fw*$=MW+(T}XakGX?9@NumYm;T zQhF{;>tWsp-@52@!^^xI{(c1avx=BpqNPboe)X)xH4lTHCvCA>qrwJBTeuR(2Bjfe zDeFqj`s2Dml!B8P$qdITvjnkRWGIAWzER$3$wMKV=^#Fw*^T*a@Hi=^ONDbg-05)?Nn?yiXbS-Y+j zlbJ7GMx(WI{T-uh^&1)eH0Q@uWi-+&3Q7Z0S*m7TY1pbFA-FGtQ~c9OX9)(n{Hr-M zx!}0uu$KAT(=eZH1r8K1riEGAB^D>6Z<_w5V8~_y=Cf(0>vX^Iq_!wey4-fqQlf2g z&+MbW)~vtgtbY=`Q2;J5=dV8Np9ohqs#p*rNYgJ^CQB#p#P*{!zP9ipq@j~W{Q*zl zsvrwh-Qoq7;%t^;C&2}lmwVs)Ldc%2->Qjcv;NAn{^9U+h07Rtqh1NNEJyewr(vcR3C|wYFQba}J`R6}p_RQ|= zWKU-1&YjtNzi;m&K%N*740Lh>of-rX6Rv;)BuF|+YXp&wXjVk=Nct$%0W~CO0g&SF zS7$>nP={w7m|_l~m?LQ_6%e@=HEB)uk{5GC7!9Ky2$P5<)COJkhgrJyR!~NIX~!Zz zYvGFoR;H;*Rtbk>R9JI{vI)_C&7}GUW&T483{nQhM|u2%G-$gIXY28Fg~3 zd{P0ggTd>ASkK=6hrHp`+nr38KWMYls4{Z&4EsET0)mSAH*V)UwE+PR*+YKUK3|gx zTo(^K-||0nEwrX+63|f#XB&em6F@Y066cD4%HfabPwia+4yc=obT=Or9wi^_MU%rx z@q9<}ytNIQfsUvEcVTTNC*HBH*9kH>!(a|1HJ#a}znj5V;RIHU$Ne-9A}@HI8@$dL zxXykY7X7L4SK9Lf!f>b(p@@?-O+iw6e)PCEZ#BRi=#(HJDG4A-0*QhED)2wb3SSE& zk;j?jPtbkkFr_jcIIW^Mqyp@zJscvOf`oVcRbAhIxC=eC{3nfMRUFXj)Ad5htFST2 z2O20&AbzAP@_dpubk!JQ@~o5et?;exKN1(;db*?hUjGDf;Qhj^);dBLtSA<&m}f$g zK4IJO-RJwuX^+X@6MJoyWQJtt3YvLnpwkm~0Nv{p!4+YBut%P4>raURy1-2^DJ}8n zhsU8uVdJC*pv@jMZLBzG3}^v2l}Y*a`G=}+JuX-$^Xl_xo8jH08d%%zF}pKW9V01l0^iv|tfWr4x3?XVth>7RiVQyisz9e~5eq2yQ|aZN zdyo1>QPQ!u7@G8e#E&7Ou>m(=H~9!iW>Q+m`hAM69;Jw>5BgKX<^&#IRoyAN=mXqf zh^0$U1-h-Ic^%djQ2T_~##WSlU{5Y?ecfKNXvDbE=-HdqylzSi5C{IUcxD~)4u%>9 zuX_`o?72|8^ge4|_au;fdtbw?EQxnBpo`9TPG9)_*LrY+W`%3b*z z8g%D0-;eNN2obikGVp2q#;GO|KMud+E@|EU&*Dv;5)qL@?|CPP(GTG;2m(_@55R zENbC2-$7ArtR#PI`q2&IznCK>Vk`PL6JfT+iFRQZ``WqfMI;XX%>LU3Ee4o4n+@WLNG(=rn)dIrlMig=D+?QYJ{qm ztJEV;;Aj626+%wSYiiOMaVF--75LO2#tO{#hp_-`ZhvSANVGOTNul{F<%pZ87jwi( z6plG^2Fz}KcGadKJZ!n5By|u=N5T{d4_ekKNsUD3kudp$Q*JYdWTZy2Pl9V<8UKW9 zLNIlH4CP;{DB4TLUmW1|=LhuOu7w5s?`|gqPuq;D8hpqtoV(`Z*TtL2lsz4C-w%; z@C6Bd5GGyg{jVpg>7O>`b&Q{O$$73rSLk}to#F5Q_E6yeaX!rv`7A;e|3oAHp|gwF zf1{l2GOQ2!opV`96wYT9}B$GMy+gWz39n5 zbilU-j+{QYh-(A};k=uY8Tg3rQ{u2)j%zXny>Sn_0*217$r(}ky3ge1dm-3)mA_diS?w?r0@ZVEDb2!5nbkERD zY>p*`I}Yc*i*<*!hX*|ekq25m%00rrzwhT>^?7ms{P3#N>a9=Yfu?-JFZR*B41Wu{ zw!Db+7X}dazX@8~X?q;8)H~g`bZk-`-(}~8N8f*E9CBh^NL9#iZOYtP#hi&DnQ3@= z1i>`a=MGu3Fen{jaq zKZ3~;4 z=wpqfHkuuvWjmmBC2div`n*@ok=~SgP@qXyS8khQdJ3)6^G;zctn?h*Q9=?_U>b7- zkDS)fmYCIG;!6NY*9KYrK>SedZww%za?~x5)t3Ma3NGR*%4MCBCfB?zkb2gY()(+c zUo9t(r7+EEq9!l8oIlg*9iTH7wx|Pzc)CmVws5@=NKQ*@%sgUY`u#IAwq*q8KNg0< zt+jAzw+Ml928iRzd%tvW{_UTMcw+i$c)I^P)0uG(x}oQv!_%9HnK05z*l^3?%g)Zj zAB1Nm-Pz`dK6hE?U}2E&doQ3SD^;FLn^)qUPkV{1k%TLE(mN0PV7|vKF%=e2S*YxP zf(D;L=@trd_ z%`1Q=XZbuh$!47@PlHChh#~f;7XHXS{5EX(<kZ7?2oXCbjrgDEI}GKY`*dhSBb73MUO%6Z_l2QlH7e38u+=_dT{kUGDGW+6)_ zWLJHp-7#hRr>)O=VG92v77gtY2(!kYiGvd8r}K*BJhHug&3#qiS=XS+|9LIRRtIpg zF>n(SBiAH1Iw4ECP#aM0P)Ug}0;h0Dv2DJ{+ToOVeV1ADG|)+yaa6|(juQ5%+D?-j z5CRG88-@FbVoM;4w=gdGmUN0J^hpe*9@cJR{aM5_# z4=P~Vlyge?zUAMucwu$Wx*D?8qty!xjt`E9@tU!RJu9c;8sNHY2I3Qy)Mpd@@j4a3 z4}|$`VG;h&0g*f3+MgG!r0^>X@hf#wSMAc@oqWkF1YwK1kLOEw!wNP{Fhr@!hLb{j~>HK^xVp9W9a%M)niGmN;YC1nc9JkN>s7e9Zr* zG*qmbj&#&Xs>EG)fgvEV6E(ElLV#Izrun`UpUyqCAoyK`>X z&T$rtEn)s|BTqQOzxl($qZl#G-e9RIveh&#Pf>lAz6g@aqB!&cOw)!E(R;WYTmO=3 zG<4{!5RGSbU{>Y&mgnEE+#%0rpDe+@jFR&H_|U#oO%XLCAQ#Md1M9IDY46JQ&{ElIrj-I1>7p zK+94a{mOhq;Wf5Xl_d6k;z^{~{ap)9=6X~P%}^#U8EecFbUAusPxU0S;;u;wo8F~p zFyo`QCn*R15l$li-i*3<-xKy&+4 zT(_pF@B5brB!Rb@>T`aRyQ2MC;qeszS>%ObDjAeJvBaS}w;TU*MJknHkNArsfiCi* z+4~|!@ijzQI4s~GGDd_yy`y(%TEGLUGN<L(z8K#Hf3KSxm7feq8qscg zVZHFkNu=*GmOogq_bIy8BS&pOL5iPaswFEp88sNxARc_u`T`sAI$Tq*G5n&ZPtm3C zsf!5j4QJ`GmXsHMn+&6qP-i_$c{lT37lMEU9W4DIqISf*DK5rk-=TYpM)93Cb3GP8 z)9ZaQ%wf-L2G+|&W99D@CZX>;gQ1A20*19*KFLS*#Nw^^=g#~Jh|&;ycejFF3w{t* zBjKKwf_`A_X-sv#DxF`N6!ys^_$TOXyR?V%i?_{!V*uIlxxK5be{4X@oG;S+HGG#x zvFz1KwZ+lD$L7HTx0M~<$mm5g4uJKF-MqJlW@!KE0$bB1-vfp@xFfncS=@{wou?GR z9Arn60@<})yPNNF3u_)v9>hBwEZZZao`Rj>C-{(qyJr*GOl%0cy{`WI{9s2%*M9E< zDpSit!G*K;PLx%<$uFiA(CzJgWI;?u|G&3VUGl-?N(C0!8~6H>%4I-JPd5~&T-&Vt zl$!YR>$lU$K|_xfb-{0)@4sO$hOZ`s4FP2AlGn&6?SxKCn1|}v_rBq&!|EI-hp}qQ zJfrLRCsM&JnudJ**(%D2E}M$rk!0S5%S}7Mw&IOMTG-LQ2B7-lb`b8nt9?LT+YP;8 zR4MC0p0=j={ZZf!S6^_tUe~B{)inm`sgy)J8Z#l^Pxp7JUC>jZA5Yt#mk9IU-E(?Z z5#ek+mm(MGffd0|wTV%$k$-ZN{_EBdzbCyinL6`+l(w|=Yx2gmMx?zXB24+$&|{a< z{fHKay}V{Jb4uwO>X=@U>>q9j(R2ehTk@ydkLF6`f4dp=j$=R>+VY5HBziuWR~Vm( z_TI&4RlOh+CFk#Y^zG{UH#~JNs;3@BN#%hs4gpiU4j1%AN}nT)*rqPzFv3{<`TA2o zdy`R*HP-Fj92PHS&_W8IlD+F*LxbyCj9VUAA-Y$*mrCK0uej;ky*ZUgE*QSUmYS!4 z8cy=jePgWrqG6y9?0n~VNF~o6dscNCL|zE;X8qcRS{A4mR=={3GQWi-Ja{9D2bUKZM-)yVQ2sYKjjc^>Y#QicObJ*?q63U#uL+-?Uz1RpnRM;dd6|u>3rd ztT9w=vfA8u0Zk-PCt(e`Qo2lgqI zgNOtC)SqB4)96Zmk>Eb6lW`k1; z+M1-$lw8KxA-{wb8teBd5fXknuY-zVKBvz+`!*GWJZGlnL9YjhJ{U5m&5$eC>ixfs z!Ktid)Q*SdD9mQ`CAC%Cp_YJVtFYw@8It?69v#*1gIO#?S?)>~)Y>-rXY|Pu3_0sL zp`oS%4!xQy0qL4i2_Ib|dh$by+Q8pY+Rk!eDw!s|S~!g;aMNXoX~BV0CJ-St^?k5V zGkakw>qBq#3)OUpEzRwq(aI9MKt!P%%V^I_{~5=1pP4tHM+NJ<;rTbR&MgITys>~LEFuHSl#J~5dR+ohfdXm zt-P3BDW97%-@^(+cE0xQu@0lM`o2?^bs;c595?P2+uIv?y%6LLRpgEL{0PadkZt|{R`CMNHF?bQ_L z4|Canz9Czv+~WB$n0w5+c-}bhm4wGnwb|2zpHqyt&zg%ve+qhZDh^opX>P9;koAJj zvc}!=VwK?H=baK}!8Bf~;{jib#dqlvpXY!a4oa$5v~IlS)hzvdaY6oX-M7=XyJe_n z)4KM$`I0}ASp>6YoI0~_Wwbu;Y5JAKws9?=-uc$6wX5N`K1&iH14F``W%JSIGkRDC z^4WxeEzKjCwcm@%n19!0hxh#Wlao(hMkoEc`)>hYvjz9C>u9bL^M)f<@!a;5@h43SW9lz3Rkp z$w|&SnPZtnGkv3P@1RGk^J%DtO+fhcaiLZl9zJ6Xz8qd9N0PKQ9qdt{c9Ohb`bphP zMbIi(a0r>O{;Oo`{+obG`Jg1(ZNCuWanmPGy z{4D{0?dke^b%lrBInveMdtExqO5fCag;s?%{rvN{4zkq9Ll+UbbEh!Vrc}cxd*S(= z!kv(tUef=*yq$jS*OjD~jONXE0%Py2_Or?@X$*sYFgcaWVRq#(4X1lu0aI`?YS;b7 z!!=z$^7(r|*T_$5{!NHi%5f%W?IrBZjP_{mi5B z%%hsjqom9u@600xdQz4sX`YoJp-qqg6>kaCs?hHI9t(4GIUK9W@RxGymD-H`Qcj45 zs?xVO{$yT7FO-{CnLX!sPMcM}#7$jW9Kp?J4bMYuabPbEq3R_u{C9Vv^2P{ogG z6kp*GUQt>f*iS!Xpj*ijN?#0O$`P0k%GJF`kIX*WrzK@E6Y!>{N<`NJvZ>!GwRxkb zrFk&FQj@=N!}mZOlw-m_Etc#O$*B1>S(>@q0j%5x#T1Zd|%<+|FBELX%Kz%j7Lq zMfSF2a%+wM)8`(C5g9mvMsAx0T<+88s|?tst`XBM3^)X`qJ>qi52#6S08dhh#0gm% z?#%h??HL>lIDuNo^gi{%hheoUau~AFj2SGZG<^z-2_y*f=d}oBwFrf7_=s9tfrZje zB(9|nW7DBvAs@8XM1M$JOsD3hY^vPN(cFdpeX4~lT3kOv>JbVs;~`#)IoeO*pf%+p zrb~j8`^QtXzPJvLFiH3U)=ZchDSgKOyg$_4*-Yx(&3*V0|1Y5nj*F@wZO%@K)e!C| zvI`gbeoWR&Ti#Whs09OPa{ z&a!p=XHG%ln6XVDek2>j0W-tYXrd#{*D>#@uFlkM>UgS?L%QNhr)(EwAWgNd{)z20kivrDg{zj-cij&n6ZcUgo`$l{-BF?kBfGz zi*_CPsz1bdC`8ReD_ccq8oa)NO|OKRslZC+^h|9CRWd7CIY%|`!iL?olU=kiF4~bU z+QBZp1%W~x1{eG-1PNY3l~ffEUV;KJlLyS?P-e1#nat&QoQRx2UbB$YYPxTf@nD$R zV3_1!824akoV)QOp_o9SXMsY~fLZ;Kd52InPGT`6WGx-GVC$nL?r&CgV6G+R@oqvKbHhvm(W&DF^?ry>mbOyg{#ZQ6 z_@61U$el*qvGm#iQrD0clo$J+JC40OS+F}f#K2b6V)(rViy@`pL$xhao~%1ICO3Y& zWyAYlDySRsWErQzBD0_t1{jW{=rr~QN(`UvHua5DX-CO4K0~A2SXJ+AtCtMt{etR7 zaSTdsZ0FHp9!b~4jNHdbDIpssqwG9`l(u57NldF049PcbsMRuK=J@#8IXDv-==s=L zPkDb2Qi%&jpNR9YGoHr3qUL5_M^lM|%ni85&(ll|Xp;CU^*=UH^$=zGK)IjG`<`Eh zw=sqH-epMo9nK+w{6M`_Nr#vh7J1*F@*O++o<8Z6nC#tLjfAP1b^MC^?9IDUHt<7| zJH=0y_lYd8xBy@R>DuBFY);}4W4$Lf7|X;VJZl1pXUOGkHch%7;bEfb+qgCNcgm;|Cs9s(u-fQbz8 zP9+O_0b0S=jk2sfniFOsiJh6JNu{6^=jcmHtCyu&WSqZlM^;aj;9E+lZbO2&I$i{u zzfbwTD|xQV`4(5DYE-&P%j@R2@znZpYJ{!8iZ7*7rmm@L}NleBWj*_usyx z^!?=eamqHsC(|Med9gHbQCA6$!BP_W|<95b@}**|{TxF~jwS{#KG`#IbN)VLFX^ek^oVB_c9R!}w1{Iv+Om-pj-te= z+IUWyucnM25X5hg4DV4?DkJ8vA(Wr<9UO}*+s)@Uny1K4-itE{@sMv->44!XR<{;v7A*tSUTBh9b=@&*Z zU2n`28W|??EnO8NTX{5B|Eq9|I<1tx;d)xhk0*U$BY*UOy4n>=IjL}cv@URcEy-Eq z`OPn}`|(%dRk}&e#0e0sMC7K#M_IN?zVuVmw@IjQDQ9D)5RMkY!w`B7W-Z`vTJUP+ zPGq%X>8&=hbJf^VmUo`1J(+<5k;SA&Z_D&OnSoY5c0XYd;P`ow0{M@ws9;QZ|b5ouvCYj-RQi26Xj6rphUO6<$)4MC-J-Fu0ykP0A zAofOYR`I;6xs%F8`?$3LIrb8Q_(B+DF@fyFyesSv<2AgumTZok_m=0hGO1u^+h;BM znUG|#kn~keeK8@&22={!(K7Q3XJU=sH}YbSkY%lmQBm z*kkWAy73qKzy<-e^42uO`zkiTw+Wam5D@$qGZ^qYL(HdXnR6A+fcrdfW`5r%8I0JFgbs znR*n?iHMwT8$APzo`wh{iWFGZ&@IdfFG#PWLfUxGFZzs_#%rW3ZIPbMk`7%Wd)D7lv>p_cYm_!(h%ZKAK z8Sr9Id<8qo$PcA&_?#Li6=%SHciTwhI2y7-Wz}kBrfv%8YiAzU-mR2??aqI7i(^1? zXWUhbtJSHzk2ep%m&vZ!Twh_o&P{XO(y%KEtq-0;l_gY#@B5}~*{Tm2!z)w?V&ud| z6hzt8)atalelS+NM<+DT^=i0kmPK0SP2632LIs4Nw7?-1WTC?pw4ovf(QBvFdc-Nl z8n-V?{cYaBM2rP6a*0z8WOe9&KPNq0^{DiHGiyu>D}3X+xGU)QmW82p1=N`G3!{~& zB&DMfN0*)vQ^ZMyvMg&*OXzo_S{WM3iFiU+f^gt^#^E!Rkz-tIIMVpABLK!L7B`ZJ z|D~HtlBJb0)DLN^Qt7R=j~z#YgcwUHd{ zKmOdGydAUQT4Co}nd4l^BqvIIg=Bi=n&A8`dTH3C8t1Jo+x?{L^8M<@(Oz+$Cl!=H zH@Wrh1Q9ZM!iII{f?cWotV}5d6{@GEZLxou8IgOh=0ig2FZK&fhBTyjImEdy!5wL0 zdRlR8qzPS6g4HDXF?hR~uFm6(xk^NB%w?UC#JNpm=H@w+nig-GIP9O zbh((zV{_27sF|>(>5fX82y+w59a4)MT1@{hfo=7;;<_GpG~4`fh1^PJuAg^;zw885 z(uiRK`rREA+{&uHnt%m5`3M#Kx4?kW4U~~G%18-i)Nn*6#tQ`%H`7v?q>w*~mz3t` z9q-3Lv69IY9e0Dwxs6Y~qX%?>kh$q=>>>yoR(!b*K}*NFkHI@b z$(&8gAOg5iep}c$mVO$|oISwH9jm~Qp|GE>z(9x06hhhvA=!nHbIb%$KER2>cVL-{ zotaS1gED*s7|IhdJH-1ok8zaM8+ryoeY6a;UX&O&B&AQ08|hxN$w>_f^RWx_p(`q> zYl_`^obtK^Z8?|DY58$zT3Y7K0ozO<2vCe(BPKw(0jPgCRKq~33uE3~;%_Jsp2;9Q zJj%_DIRc5&pjL<&gJ#;=7TRZMW{d6QQCo8K$MXrqoB(o;pG{@Ey`-Ebliri&-l!%; z)`^(Jpxg<=VvG|wfi?)u%;{#?5k5yJ{4z#PmfDV;4p$Dwl{3Q(d9j#&EJhwo`wT*B zi>Au=RE?n_d{{Q6NvAq>WyVm{q}*iZL#AMk9H`i72xTAxB+8Jj3)6nQPmw!fYj9tX zHiVpA1N%!7E5m78UH!W0;Sh}PD39@o7ndQe6~x1hJn=_=&joV###2a+NZ4>^l#|~# zv6$=N^*$_yA55!+X1d!S(%wjE(a#eXeP|wC96m?Jl>55s>&XDVENceGxn`o$c+07n zTw-0&G;XxoiZzoGs%E-rcpEVEAc#s5kUaWr0(apuClZe+g&?do6bnx&jH8vjDM)aN ztW}IJlHH$qZ50zj1{a;DLc9spLD{Tneo!@MfA3;S@d~PPG1Rk4mEqn-ul~Kv2GP=<2#Jb9!3J&wrsyc*8;a+vvKzH-ZRKDR zjRCZX#y}6?pg3=iprkqz75>j|DE3WsUv!U-cOv~kAZ<`Lh5#5k63P_?8JPDcQlcAB zhH$*7JdzKdq#&F4jbigys2uc$+I{G8v~L%NDv)+Qkk+a@KpIv5MbbT-BQrORd5#4{ zV_=J;eAn#1X}D=%UMU1D*zj@ zT{l{~z;wem6m8w=LzbnHu!TWrtBVWIs^D>_i;F<_n#sq#jU2!M)#ef8Jvk`fs@X5y z>g2+40^Cm}{K!1|5EifQ?#I09c&Opl6tY8(Exw0z+`u~Wq5PQ%`LtoQVd?eom^?Q>JM>Lm=lk?Q zz$=3o?m*xOgghMmp-Nig0vZ_!xzsDN<<0&me@1*h4PtV1Iep{#bUa{&=Cm5oU?4@dm^of&l^V4qbaQ#4|Il3H-{0f;`ILx-A4(gIR3D``da6H) zqfPz;ln;lOh0%URL`P{t*z-oXs~>A~$8EsBpwYzDFN3&7bq(-UEun&$wejZ3rfd9-gZ%fC<(kPxctfQ&^ zHS&*tqn{sgndB%=SVvO!ja*n6xJV496T*^?rkAhAEOli2cwGHriH@R#u&L$$$X!W* zPA#tijciEL&@yJ#Q5gSYayfE!O*t&FammHE8+Mf6xYXL=K;WHRc3yQ;h& za&nqs0E->-(Bhls4rQ{mtu#`S_?e@96#bap9Eb_cmNXOmpVF*G1vGaI$YlDwiHJ|j{7a#7 zbqTrp1-beTxjKeiEpGf?-Hv38gy~1X^wqF?syEfHK^Nj`_y~; z)OO%pVoaSDsTTw|wvu&QcYuR8?_a}#^RrR)cL}?NtdnJ|lZ*&__i2n> zyKzE#ZWgqV0a}=4#EWpj^!+bi(Y$Tcd5Y4P?20mN7JE^;B?rpOU?2$UgfyR&vBUKJ ziihcGkQ8xUQH#IQXXy#o-*-jL5lb5g0yD&@DPoik_$n77we8RK(b+xQdva3_bWJeN zr>imH6>#&Z*0C`6f2yO2)Nip>#b+WAd>JcjTVU93{AC6mK`xO%QM?8uy>=RugdkaE zR;dD~d83bbqq73bSmpw!O@~U;64>aVavijGgI&%y<;s`=)j6dJJE(v!`_RVR5BGpr z6{A8@=~KnPtLUJsdmCpXKa66Iyr__MbiCv>iEN2id=-0`QnRsh=HYjGvi1AGb#mk% z(c>~DLK*#~S;}8`$Q8Qf^_DD(h0NoITxg(EVoqC18h%;x{H<{PU9hEp_fjTViF;+1 zi)2kr5TOI~nThuE|8O6d(Ol9~{tepFr~hS0D3d0LNE6DW2qNS}4dnc?F8C`N#91am zIF;+S)uYjv@S-rg$SjERU0^%X*ZwWNfNKKNYx1|6WkyPPESkv3TS&d8A& z;VKT9!AvMC=G8O#V9bb+9}=%gJE!a{BmST!1t>ptRz%f}oS!|rbH8aPIi$*qp+XbfXn(II0|ppg3{YwHz5 z&=pP46-7%4#TtyfAVftGqlN)K#jL^{f`~9dgfI8$8$hZ#`)MDb??fmQ z`mbu}$B@ULip|;l1GC8=IUauoRIX43QJo?RB<@T=9gY_^z%QjLT*9?{cJzTS8= zq~4Kr+_EXDr;gI6Bu3ftDXJ^L4`o84 zCqK$=x+@CRmHxXOeAP@Sr5P~~n37sZ!x;>#o#tgWsx0~-KF`_{3>(4dRJsI;de1%VXVDCuR8BnX4H;qPn7YB z_5N>4fBj{*u^(sMQ95N>SAh#`B~gT#LFctIEOWJLu~sgi>RIuraRG2q$brD7}RjzV%Ucd^t9@(JZs}Go4)q>qoBfFo@X+ z#p&3e`7z$pW5XU#iIPFn(?zs|kATyswnX7dKj?g!RR|ksJZCJI4l&DeYXzt{;yoQl zGe@`vo^J$|kPJ$-h1ZAnPWsXH!rFu)_6(g+VA*KwiF4q27y8g=V0j4B=tYa16+~t* z=>*@iX8OoAR$d0P1)ui^o%aQucePwSX1fzS{bMsXd`ENON5y@-Fc(VKAEhhR6=@!L zo{B!?7!X<05r*PR*?|wC^@!Q)2=)(3Yh7$lv(ud0=Ji46H7!^DW$q;WD&dF_@xJQ^ zm~D<2IYW$`B1Y;oTit5hACT>Pm0F*e`UHHF4a$ozy$0yI0U{@X=<6B$$XPmqFe{+C zG0=a_^t2R=yB)SuD74xpQOevvP3A)?2A)R;o!{H|axrKWQ|m=lHaD(*v-Q9fdh4SE z`8t^%)h0S^PJ1T}7>OrF3L5wj(}$ll z5F_h|ku}6f95M1`S7aqIvYeRFGZGVC8lJ^D6Z4Aln@j%_-oSI&jW4(R)r4t2B#u4* zDY)^)XG<5>=k>hN?0Nkd1=5=Ho7|bO{m|vKl%MR3{CGCFxIMVEFt|Ll7I~i- znICT76-g0zUK4o!viZFHxRi@5EO=olq%|YbI=k_e=b@O(p@7RFhsz2k+b_f@vD)>NbCJCluo)ozFeSm zOY2_+=zdr>>dYgHF5$edQMb!p7&w$5^~Cyzx+6oI8`A&*O$#S?6W?q0FXdas)w6hN zlN!u*-A(Gm6z(!RWLb)^@uLwVMGgOY)SdjcqP793B8&uceFm*XPom)R|iVT zx0)M6fI|kt8#L0XS!kz_Uf^65mD9M$n;JWT@pl?n(R@>fcHG4-bHi>L;ndT8Oa|Hq zA&n1V^`M1FjF0+2I|0%tMYQJx%$gg;iS|%}It{63lz>4amx#-3fZuge@+hISrO^{r zPmk|N^^>X!LnFtq_Sazb{C0+Ths+1}GyOZ9;*VJI7j&_21{#_`*Z>SGewFSYg5gp> z!4r*yG&i~|GQ;X=aXhI$jr~@OHv%f}qLDJqjhw_~2H>VHVJXG`$Q9X5v}XW>uxnEF&({0SeTFP)br= zw7*q9K^%=NAudw`eq5x9;SS}XMvJHF{ZWgOz6E|JTaFnuv{-v8*dYZj57r1@>-ZOD z7y553^(Ot|sy$)x>EhY-;vHRlM21^$!tN<&e93QeeB;@jp6r2adfIW?F@caL6&j7W@GOGt4Ldd+ z{koPZbj+$qCUW=EfU9Ju_)- zkkCrVi|=u_ECLRh?CM4k6i?xseoP{W?o--H^QeTx+T?`7ox;K$Yb%vAk3u$({2vLRx&IQ7 zB^Ah!1Z2z#d`@RCS#pkQ15_c3R3-u=E$gUHMq{)6%_R{o+S`LLW|F}2`4Am5sX&TU zz(Y`_AkNe@m)v$iHZ`NQV+W@gNdjC1RSSZuRO%oTXa<)3djMutT_9Ew1y;RP-Mu`Q z%An6R2>(7V8g$_> zV1(hXxCbK}x9;54@4`g|Hj3y}&nO1NZx_rAhDeV@bSInzC!M(#V%k(qqU5HU0j@BeDQz#ntQL6m)U* zIyjsduHz36Ym|EIfdySCPb{;;=D=oiPgR0Fn3$O-^tR5;X!u@$$yn7AvvA4F;FWs; z{PI6@$x%Y14+rXKm+<#(^huOa9he8x1M|&y2h%8(XUmQkkbrrZ_Z5X%0CD1~{ z2drl(=5T^$1rF-KTY$M0IALvMOp5{s3<1CpV%U6(w~pKXv=|?I-+KY)F%$3}i>d%? zhl+s{bSRzK&_wFt1oaABU;*ehD;+#4502P@^yfQn+-Cj7Bk-^;D6K)O{~#LEa((-H z4_>1=?IMr=Qyj9J%M(QCl{oe z*z=7WG9bwAUym4T$zdUtU3Q0<6Dh%9*2m+*|7JefU^0B(rE=#{U>x?2U=Ey?P=AA< zz|>dR-y{iU9cihx=OE(yu=pMoS1%i-PVF_$lswi@sxbjtSjs!7qg>&k#+N~(az`K9U;0wM zy=%Z)ELKkg7&WjTd+!cc>7vBysR4u)3IPKUmH@&6K+qBK*+l#VfG`ITW&nbgh-YYa zVnz3gw>d2l6xPBz+MEsmMBCw)KK#^C-3NTC!5|0OuIw#s+A98!x-()cOp1gbJd*e7 zm3Qr>>Z~V*(US1aq@{KIo1=*irw)R`!(Q<%#Cj7=^Weo2``ez2JodLtTQR?uun(@v zJpCRDM&o5q@~;=Q3W}{Yxb&vn1)Kg zr@@_wF9#6T1O*uqJm)C+T^P?$80}El`C!=Y;150xLvlb_wTAm2SnU-X@-Ka9d#L(D zst4~;*K^|(xN-GdI0bT&T7WZ>;MqrYp1tBOz1L&5U8JbShU96|>8~Y?l?R7UNi=xG z@9>ac?v+MX`&ES$3XhoLj=sbmAvFYsc!GAnb_H+b)yF0-P5nq?ewj{)4j=tBq&m99 zhM&wMRj{R#^IVdP)-G_r@uPiRC;W?ld?B)A&tCdeSVN$}0wt`#X%@SbR5V~QJc8!= z&9jgAfhcWKjVBy_L7Z>Eb!>D`>x5A7bQoL#ofGc*@ z*&o+wI#**5?@sEt`{dJWnP}~J1nrhUc6lJXB}SaEjI#%(u`aE?jt0TQh^ym(!Nsw4 zL~PwHFuV~0kJ*6x_I;tQQzOW$Y3SVZ>YGTdouIaZ3)t}p*um)ubr8J#H5lBr&zJh4 zJZoRA%CS5P4DJGh^M&Wp;%s9eX?0W!HE*z{CNVzuxUi;DrnJEl>|_PB=IV#mJ`?`BV-*rO2le`4l3bI^C z9DGWUPu<{CHuzK^p917le|*Z1PxbLBK0Z~0Pwnw3JwBDkr)co08GK3xpNhe!@YtsA z_!JC2Wyh!L_!J$Vdcmh$@TnGjiUprq!KdcAod#81gS6@Dt`Q$>6V{8a2yuur`{ zbwBle%Jr$%r@T)Q@u}`p0{9fpr-t~{&8LL;R1lv6;!|Tj^~0ycd@9VRY(C|~r+WAl zm{0NWsV`g#QeHR|q`G`+hfnG7DK4Md@+mE!%JL~JpStoXE1#V{9* z@TnS70Z7sC8|YJ0J|*QrI5}qcDVKtCJ{4t~l3|;I@~J1Ea`LGrpJMW<7(NBVr(XD! z3!iG?Q>;(1@Tn!ATH#Xw_>>BtO5sx|Y*QzsO!$$8-{R0u&vb2`md72~aP%qFiuAwO|?)3%E75 z7>xn70*(z-8=MGGb8u>ey5K@^iMpTylpNEfU}`xNTptxj8`p4c{NdU_q2P*w0&Wcy31|$c5ln*;!8BW#?Iw)i8g2xW zxDlY(;68u~!4MRhU#111i*koz!v4i7IlCvs);R%i7jf0ElPWD3>09zCRwx|K1$V^eSEsC~9?Vxm~sM(`wI5q45wpNYphXfzE)>9T+)QK{0 z;t;KEWgG=nZc#>TOSoR5G@4|*2};IC&@!GP1X~ojrC6pLmMLnSqQogmc149(RL&Mv z#1;i^QL$|m0|lF+UQ^VaqP{80HAS_iC~t})VvFjwC;@CyI9t>ZThz@KCBzmL#1;j_ z7Byyz`eBO_g9@`n*=$ihY*9VbupcNemqFoCJZw>4wkR*CE>qMFQ~FGWra(licnP^Mddjlrl=gI+neBkupkoB zK;f`O-LOU3utn9dMFn7sqG5}gvPDVRqGs5lqHIYlpk&yhplnf3wkRiCRFf@=2`Yvy z3I@~*Ta*h^R11${y{M5#vGAxRk6PhFAwNDe90dSVlnPT+3e#732^0#0P$yhbCR|ZU zI5bctaA?#OrWplPl1G1jw3tyq)%+<3Xk5dg@rOfWM^GeeQ6p?oBA`OpqL6G+G+Pu1 z+Y^bq+!-VBs1LTNnJsFCQf*PGN1?8$({c&-wz-&?57WHWg6SxhWdhjP6JV}rbaT+*1ay{+fdUB+mbnvG} z{=~>Mr$weQaT&OTt7(YqDF-J-FmMfrfh~uDr*L7M=HM)l$kv?@`O_hPGGuxxr32A9Q& z3=WG^2d-NfQ?3yaxGhd&OmpJEG-5p98twsqI4$axKW$*TeKDzl=_vynal!zH#g@zB z6vm$}F#V8mrC|KY0&l8KPhgxXFg<;7JyGC#n!xoWf$J#(i%wqrsf*u)cy!`o5g;%< zZSf`>{OJLI(&86^!{U_1c9h1Su=vvzpUsLt+!asZ9IzeWKW+TU0bCU)D>y1nRd7>; z;@DItDz;PnNs2#3@t-$d1ZE4x^z_6uG=dRX3kjSRRZuYs+!Q0SmXiXmawF3bC&fK* zP&D9rV!-yafJW#^0arnBQJfMm%^U~CQ@AHi2$-In_|pOEsfi!SpP2a564R3s(~|+F zn{Z0vJWge%0fB44Q#b~kka%=DVi8USxNcWmw=o`_2vF&10Miqh>8Un7(WWD^mJhfE z&?JA_c@U?7&sk17SCR>?r`hRzMEQ7hG6!9M3uF7h_Ec_9Mx4m;6=)cbK;UiyL~alS z*f_3R3)_?I#5qyJ?I~xQ$5q4~popy&9j*y z16Ss5PalX=;vP79#?~@9xwfYkhs2gMz{%s1I1#}maq74vuJ7T>AaMXI z-QY)@LYo(^nJS+*xE*VC2hI9d5uoGH3uD{?@2s#Q?@55+tUo&LELzXay`j#Jwe%?o@`G}wx=fIiOKX7 z!}K%To?zJaz{O5496Gtcg`Od(Da6JLwdQ#zfO5sl^ z{ON>0nZSKYra6sHB}{WlGR-vH45u2dh!Y7MIOSqD9PAnnwughAM!23t;J#A`)0~h@ za}w@#4fp!Pz3zc?l?M3}&DC?qqZ0^R>-53)G_&2Pd=lBdkQ(rxLVjCaJwsdvIjv5r z>nU|Tp&p%1+mmU#so+$4bRu#0I;J^|rt#QZNjHy9BJMkdxbFnop6vD`gK18m%?j#) zsW^F#^2l6HmGe&A^+f4uo1QlH;A-N*)%3{KWXRT3$kw!jt;vz?#zVF8$fScuC*i(p zxNj=l_YOCC<6P&wsgbRTkwJ@WO*t4eh--at@C8hYaGfa;P5>rEIGj-x91)=syy*~b z2$LaKQz2IqA=k5b*Kpn?&fB28X^=PJU}_R%YP!J(j?Ie61|KQ`5h5ORA2P0{K(_P@ z6ChjDAKNV_0FxhAQy*6oA6HWia7=q_O?q5Sd0a(j48h%8!`;krHzp}KA>OzkOf=wr zFwJ0^DF=iMrlBr4F(xUvF`M_pH9w5Rn_@6E;W0Jc@g^9&$&P7Eb!@~*I5F37Vh(X) z?t$yUM8_l33*O`cE{v%K91SKGa5knDnC6(qq=IQoa!g}N!8E2grZGvu`Cvl9)C9-W z#KhEOfH$TlH>Rc&OigfYgX6(e0*;3y+>~p$DVxAeF_D0yVj96Tm_%?jv2itpfaAfm z#{c1ZxCgF_2?W!a)R^W2)0ooWb}&i7f%(as(3qa_CNric4@^yEOig5b7~yy@b>Kow zV{kQ?IKb8L>Ij)OI15;B5rX~zbO=0X{y1;c7Ocv1L8cv5l zoDN`YKY`0N9G5*D7gGheEq57->5Hq00$0-n-XwvmDFRoM7gtjkTwF~Mc#{;ImXmN< zuHmwL;<9WWmTNdHa~zhZa92!QOv6!`#`J(kCM~WiWHB{iaW!3W6;-*4s9a4BxSFiE znyPpc6>pk?vtp8ht8x;K$~7F7Kj5g)B%3LU9IoMV*u&+(aX4H}QxgfHyVqCMMpr z#G90OlL3CD$KQiDB{4OXA!Hg864RKDn8s9qX$mnB;L%e!DW(B%P)uaFCsqc|iK)iL zV4_`3J6Ds=NKCV>n9gbb<0U4O8<|Rz5$jgOdZ#PNdVjo z_GarwX>5WC$20^NWsBmGWo=;&O=G&*5R(ua4(G>FK6qTBVM9zoa57LVR}&D|DFe!Z~ zTuoZ8rYu(x9aj^Ut?A0vWMylrvNciJnsV5haM+q|*qUtEnrhgZ0$`%yO;ffeDO=MF zTT_&)NrtNl%GLB_YjUzRHQAb&Y)vt2O)y+dFI-J7Y)vg}O{}Ylg{x`F*0jRb1c0qc zg{>)ttqFy#>4dGxgsn-*)l|aOl;mov*_ue$nnt*qMA(`_*qV@RO*B^%2v^ewTR11C z8Pn=&Qf*DCtqHaL!!Xlt_DMO+M~&(`GGn(EfX z!7vJMqLq8$`ljtt55B~Mf8v>rL!1jG9c)XDY)g!6ON(qvIrx$yUrOXlhD0Lg@zFl9CNL0F(mRmH^q7{n3nqZ5+7fx!I$>< zk{(~m<4ZL7(hR;NgD=J4OL%|j z<(QUIP>SPADEJZ_UwY$9ZhYwkUoyd$+W1llrX><=OC#8pM8JtqVuM3c3IRt`T7y%e z1OhIF5BZWBU;4n8(%6>Jn3m4?k{Ms}z?aJS5*gD{2ezd#zQlnqiSeZkd`Sae3gSx{ z_!0)b6vno6fo;hG90?^bI1@@0;FOfU;7TY_fJ;)E09P}Lf*YYE0q%rS1h^CKxE4xY zTuWVCOI%z_5SW&>sDC(C=>gl47TZ!5+Y%OA3dWbN_>u#*B`db2Dz+smwxub48}paI zLP?5kDT-|gif!qMZK(m<5(BoS1$@RYDd0;9K)_$(mXw^}Rwx}{TWVrkVq#lbVp~#T zTQb15l*G1F=2}8xTRP%P1^5yHzBGU@k=d4N*Ai`8+S!(LuBF+wbY4s5wp8A*>7j%J zK*@;9ym2a&NDhV4c$t!TNwzJ8V;&9SQYhsd#luLch;vHdmttRn{UQ#9k^rvHT)7iW zP;&heI4;S}q>WWeugmP=YBr99(mhJ)y9|BeGPUH|mRj4Aw=EHIEp^+H0IpO3*AmVQ zH$rKMql{ofN;g;1b}b2UEd@~mVp|&XsPqFDLP^ZD6lPkonVvWiHWn@UFdg;bLMZj{ z6eTbpL6dxmhre=2t`G8DOJBAnFWY*4M5)V1ly>+C9i}BN)6$k{NlPgU2f`+f<{r42 zbH0RSTe@;BSt(Wd5|u}#94=GB;RKAtmu|S0Y~VgHaUg^Quo7?F2Q5rXHC*SonVA>~ z=f_btVRa3++H$K>0Inq(uB9oLDM`7OX4saZOiMCMKZQ(7P^KS5rm;&*OHZaHC)ZLF zE>&VO-3a+o3}1p_TYBMIa$#F)VOwHtODs%FOQxk2rX>J;Nrf+^@Ff(!bi%e|!nUO3 zKWu!dglQ?swN&#Z61Jrgwj~k16vDKGWLlz`mOz+>vY3`W*p_ClrPa2i+Llt=5^B4R z`O=A|WO_`g^d-`lMztimmO@_w(UP5#XfP$^F)e+rCC{}~uO*I_=(Mz5OK(S@X%DU> z9$ZO}TuD2)k{sD?XPG~ z#+LMfEh&vH35_l3j4jEGEy)90QW;wk8Cy~Zwxlt(Bo1szVr)qp*pf7`C1qes!oZdk z#+Gz}Ey)5~5*S-j1-7Iwwj>H{NfRJR;7Ji6d2tj{7e`qIipWR_j>N^51c5DSi!JE^ zTap%AQWjeh7F*I4TaLA*HXu1*O0wcQgNhzP6WLZVB~dXYO>yN!Bq_F0Y=s2HQMe|X zxDn}zM;s@rTpm4z3y~UNm$+6Tt{@z~gZo0_KqLlCGZIf)z?Gzc7q#B=$CDE9pTLPw zxi2IHTuDw`Ne8%+nz)jfxRRDgav#^+$06JYNr`Dl26$uu8UcF^X9sb1OFbeb!F{gb zK6|(ismy^?E{{k^Y)MCKNd@3Mpggu0kO;t4(g3a`GFMWKIzpmNL)v+g&XhEpl1`FI zDyJkPrX+IumMdwzlEf=Xb|r<^S_UbHRK%79ZnreHl?7ZCQAn|^1QP5hq}Pj(?u(GV zZ4Q!a8d7T-lE-l*A~=rJJt7H!qd!4+K*FItAPvE7CpnO74&)gQgmm+WB*bM%L2OAt zY)NCbqZzg%6g=sNttS9a5;G-*d6LbP9e;Ywm{Ni1whOSYsHw%Z7{BmitT7Dy^=NhxeeC~Qe5Y)K|;NlLb)61Joy zTT%^)gehr+D@lZF9fK6YmV{(WqS=x_*pfcjl4eM&ElIT{rM4v0mUP;ZOj}ZEOCn84 zqbW%=C51?!DalSnVN5F=q|cP(nUd<1#7#+bO4_ER>8J-&9!y7%Oh-G|jvTp;bTAz? zGG%16$dqGaw8U2W;2z~*0!+beBMGansKn5QgUU!ugCNBaGDx8r8rlW`qYwbVz!7mY zAQ1{?8?gfv00O1TXmBPC#Hdq9nVA6q0RR99KoS7FS&bc&bx8@4e=EKza{^1+C1uoB z#B**lcURqKHqY8c&yLykq0J!O0NZrXCip!m@@Gevy21Fimm^zn0=I>kFZu={1m zj%F)qBO`4-Y-NEe>y8B9SM~TSbJ6Q8=Z+-h6dZo^iT{U7h#?Xd9A-Z$mO4jC=-5m# z=>x?Tjh&Nuu`8he5?n z+HLIIqzco?@;2MCz-RmB?uM?xwscbZdlT}FPZazl=H}^0s{nfTSF7D0%J&N2NsCOb^ z5B+}u7kei5zUs86a(}<}0=VW`bSiAOacv4h&VCH~uxojGC00l1-+N2OzcuPIa1E-o zhwtu(_&&JKWJdL&&^LLiH#U6E(;Y2~X_KzA@qC+qKw~+%lVe!XkbIp$nH~J3AvQmU zUkttm-bFb5m3xBG)q23G0< z&r+$)B^W!4U@bMVH>&L5nAIn~!1LGSz)HuJyEo!;aD74aLkfx!5Q>#+f)l!j;@+B^ z!Q+0Jl94@<2E22))z{a{*$e5eGiQf7_xz3Lksh+iXf4?P{-f?gR>C)b^Qmg;H;4NT z+gfb>UevxTWj3FfADEi3h;+SUJ#TmT7q@`Ev_*WmJm^!lGZ2qO+7(?fRW0DwU(Viv z1&;g(I#$~bB`UUi%Pqg)H~Bm}o*(7D+b)fn&h8Vf!pCm8yRlb(m(rIW*lQeP)5zH@ z@ftm^>g z*^3w7Kw_Mku*mqy=`RkQc|qmx!gG)R&wxU|0`EptadK}n%^iK;UTAdXbemFlLG$%B^>L3cFo$374M|D+DUtzVgC=1*w&EaoR0tZW3cr1 z%-mepzv29H8zpo<<`dEOWtsnp35w_0Y0}EmfiPPZuTIoN@P5v{20Ke4V)Jp`Cnk?2 zhvRe2p*g*lYeMX%!f4&b{Pz(?PUK|N7y#mC>bjJo6r zjHfbr{ignp3ukR>=lGs*&tcS$3F2kF*HRBQZuC9O`c?U#-axzIJVwYCj$zJR@DlYYWgR_PvqtVfxNAE8ym*-6SH-cSYXA+GdtdI`5|+0@%NMnzN+C ze@@ENdl&7)1qS}mCt|ogAmd*metFEL{t#@3*7+lcOn*&1a(y zk0dOa4TUMM{l_(g2laoRQ5I=dwj|)`h{m)~&dLnvwKK*2f7yXp&(Wq;XztGSZpq0T zMm!I{C+E5`%&m5_#!%gw6=rxI3&V2y$C*0RQ>1f;h7Q9kD8Gg$-+gDBi5;V&M-gxT zCOb%CmxRoSL28!G%pLB9u;1sWb~1i)j|M(wIxv6j*inKDB> z!t@I_m=_uMm3IZJZc%jXOSf_woig$sfZ$e%S`u=@+^2KKaq*s8MKG_dUwg>y_La*2 z6Tzrp=NulT&kp3ZzkbQo8&cL87&t-O=j9QbP&L`{EJ1k*N5Et>)7>=TigHc?0{W8ni@=A0f*El=ph{eR{9+*lp^@s{uxM!XDr$pQv7WdDVymhTwlO{7ek|hH|TC$R3MM z?PaOoYl!eZ>NnDtHYXgMMspFt`FE5tH#$pf3_tg6 zEPTh%UkSDg!uD`Q`ZjvJN#*voF`C(k7w~4Z*6$EMI_YOXI)g>y&>ZNkk^iis`hv|# ztEnwaE|W%QB{giux1FXB1NxQQ#fe^^zeyfAdJJw;Zou4&APe2UFT@A?GZLdzI1~pS zXJH^}Vhevb1maWlPx4>(Q?l0IOAA!R6$gNAvgOXgw zY%6-!Z!Uq@EcOE~OFiX&eBtJ+yRG~75bJZ?OG|{174^%mjQJBU@RLrqu9@d)Wjf9o6RvS` zg;UK|5<9#_TbxH7M}u2oPOovm4S*$r#~)eEqg2BeZuykJ>mEEM4`MQzt@*ZIb-cPzcxBYhc5Uhn0*viq1{2+ zl{P$yd^i#SCfv(_`>mU@>oa9* z)qRh8V}#EW3~R#l5Spd&5(ia&G3Dc6$EE9Q3KWW z3((l}nD>5P;4*d8Ut~Rg%-FEWn+dW3n4mFx%(vfz_!Am1LqM4R_YY!5`3u4Nwj@&< z1lb#8xuxWq@6N=JuRn6%?=Rqh((zNHnV@K6{ZO(>z`*y$E-Of(JpuLi5ML{s}ej?-FS=JL<4LCTELO{(0lwO&T2QP zcc5;@$FIA~J8Q8&NS$sLc=?+0n(_Ayla&$;XqEa}*g*_rbzXwWKs)`diD^EaVI-6i!X@FaBq3<1tbi;L7%>|l3zJ|pMmW?;(R`8+R;&p2qSui$5mREup78| zzi^*stPikdmL`?*&Wrxr{BrdO!2Iue_-xp>cF@!Gc6F-uL({+98*d(+KCO0mx37BI zztd-NUwgld#8h^cMT<)x&u=I+FTG=SfI*IcLvdiyt2<-5`*6Esj5e1~TuAXcy*Hex z+rN<6@&_~Z`#W!{C#|N8w){I4W8D{CQIKomP7h?L#$7|6y00^i?* zLRrk;6*lJQ5UK8);PYs{v(W5~4Dz3q#GK0J$PF_AE*J>((~vaP)6->&d+{oLZvI}v z&}+iQS>=!uJphEgB5ohr#6Jv~>Cm8g8wb27vh~>IDI|ZPzDFo~j}}A3#o~c%8+H(( zevRNiUjKKjPgJX){B0SKKk?gzf8Fx1v6&^kY|8vK&n3!ziVxwv=_VReH6w%m#TpR+ zupuYLP0>gBiCt#w#mmrtSh3KZZmGXlxN9Wx#~1k!_UtTsh<+kQ&r`gg(lnc9Kgy@Y z{ERuz#Pf{Q4(PcA0YBXYrxAU^d;xPVXh|)Ui|GNVK3YFDn=V>gR+kw%+|X*Kg1#3^ z`xDs=`+JA^Gmq2oPu^0G1?Ks9fp>#K&gJ!qT_4u@DsSmXyMUg?vP}(H|o~G!!85X!|;pIc_4sKv2nZJW;~wL2mKdB{U0!|gFlV(JX7st z_AVOn%nS^_3JXIdiNF9`>Bq3vrwHfNJHPIACfKLXS(d<*2$_-2JOf{u9?+ zs;4n24m;!%d=?DSs*hVX74oTzzu)ulrw?!=GuSdT&Z7HaH-J8-zIzadAL{ViDiD& z?pAd%=pB?FqY$kB7UOI6ih&NaDpO5;p z3Gy@VDHQ*WVWxIUi{**G#qWC6uip`4Bt5mtBTtc=cY1EGoaNAMY5DwX_|tF;E*^0i zvF{&2I%>r^gv@G=PFKm6$|GH-a#g%KwQ#WU+w2?pb+%XDUBnrmLwR!B90%OkW@6?8Zl zXs1E@k>v9%G%Y+k{mi#Q>ajk;fSzhwc)$A(MkI)IxX#mI>ePw06|IRsm(W>@XFl)O z0GI2Iwe<7w=C#%5!HvIxW$uJ(r^F68#qnW(?e|&9yBV!pN_)s~*ef0Hcz)G&SG%V$ zx*0ybaW{PNs6YHq70=JEAigdYR`1k^!FL2TdNh8&euK%|Vn58`y_Uk^?B?Q9WIw#v z6G`KF}hIv><0KU_QeDMw7<%`->WL+1R`bum3 z@<#u+eQclW%w~2&OeJ-#CRIi>Duz} zPMrcgb4jlLsj()hmmFQ+I zOTsBOdgK+4kLo9mZa6@mwf*E<_?hnX{A6v)$ZCe&+L%9XGCd}fXUB_P*n`#{^|{tX zEv4(N=Pd4_)k^S#ikIw#b;alTya_vn+n~O1W_r~U%LQ3Kf?w>rGe5+9{9)CtF9ncl zp6K^lKz!`#gV zGxAf4`uowX@%u+hSdBzWaBr9RoO$Dk%Wc^^aj!b$X50?r_Ajl+_zM>Z38VOAQMmMy#!{Wk{qEd_o0GS>~<`xdTLbh;XW#RR+>BEWFR8vtD z3Xl(+!h8iIPD0}-G#co{fBAZ|16@xuRnm)x0B|ouEn*BafKHb~xz!{UPpuMl5EL*2 z`9KIAw+4mfp%YE64q{{?I@O@*%w=&yFhu(CdJazQJiI!MHi_I@x}v<#sG}%QfP@!m z0**^zD+a_c118-Q03kC+vB8wzW<%X!d^<{WfEeu6w@F4P z|NH|jED$Hk>NEEojYAkrj5iVR%`AIliF5$!SKx6zKk9n0^|Lv?@hS!p_w5;2c+`J! zS|{hyui~x37wCqn@cSc0)0$EcHG@%l*+w^#?~ETKplPB{oKB;~@iLmf?KS-Jigz+B z7z65Y_M!CTDV$4~CVzkKCXLS91}RJUynr!bU2Oum4h5yUVT)lml8hGp_DRt2?3Fs^ zZjCpiQRNwxGWP^QIt7NuPtNFDI)`krmIGhuP?Lp2 zR;<703SEY6`3D&$;$5UM=_iH}oJn4t!IzxUE+ThZBL5`qxSV!l-#Xp}xyjce3E=J2 zi{}zB7e0c32*on0^1`?x>!8M8&fhl9I3%g)&WIU$$$ix!tWf>3X}(^FBA3|{vK@I@ zZGX^b+lTJt4a~<5tiW=Eh))yt9EVUwqK@n@CW}@HFH!K}!xs~~Vzy}aEETF@798Y zhs`9khrR(o$u?1(e<&0G4l2?{a+XO4b`pWJs}3Rj!PRIM6t*nX7)!IO1}&QL`DJ2d zKAwhDDS^ohZldIEst$h(6a*KX*dNORd7?^V0uTe0CeORPLr-z{*CzcdL-eYX6X(2t_X097vQIp#h_(Ock37$k2JXaWo*t@vj z*~G*L{HC$Fp^9@`@78vo)&Nxd@|5fcLhLuuYy6m(S=iYyrUw~j?!J8;?COKh?2a04 zSjetiG!_dIa&fJHvE|$2m8na#$tFGuWygJxGkmZ?TLBRs^0{uBh)QIh!DFFZ@Ni?u zybi3W9tZvaCKl{%EG&RAn9R>mABgVN?(!J&^fP9q)d)1HJs zS|*Z`Bj;y8cLO|_<|WMOo^T+vB_FqS326c z3BZuhgD~&D=Mtx$cnFdUCFK#!FN7%sPW5z zGQEpY`s#PvihU4e#frmBg*ci!nH!2(`bI;{FKm8iRv!|=L%4@Pmh2A};P)gwH$y|0 z6b@Hy>9*0*8gVKblr?N8XHJKe`zgFJ{u0A>SQL|yrZ-Htq#=11Af(5-Z4&5S(N>j~^U8ds(y%;*Ss zwD)uiO3XFv#OJ48kcDn=x$%HTK)-ieP5vXd?ze9kk>CrtUmHzgpFR>G8%1<%O`f^a)zscT zR^bZ$dqCuwV%fy)tfsj228eIDA{CcyqnZ4SRWDTql#J~o^zA;Nf@+=<<+NfMB_S*R zoO2$mTca;^|6*3dmW(5Hu%9R-d}|f;4(yG;`ot z!VmSr6|3feYu`tkYbQ~(HHf}y~{hWZ|J-O?RrUkWVpkyw)c z`Xj2l)XMw8zN&(}gQX;~Vt%>C0 z%iTVrpnGNo?BkbRC%o{4^<#ulMA5JXlKl+<75Z-jh*(HuSNlEjCcOlSty>gfc6G#J zaf=b1E=AP2grga$9yx*ZW2#+{8K?ff)2#|VX9z`Ef1`SH%VLi&DFe%V>&p18F7?|+ z9o2W6WjHUj2bj!wQ_X;<`h8cgJ9yTVfm4X5HoX`@TPNmWRL@mjNn;Bd)`vK>>m1o| zxPB}L@lKVX?k_fejYKb%1SGJQnn@uX7ju2ARG6bX66^vNZ6|>&x*6oc3S^E2Ha|R6 ze5qwp)`J$ietnV=Fa9%Y>qz^cYP4DSH-Ok;4~0A#FS6knG27Pnz>6RsQSGbm=B(9t zA|fWZHtEX(N<}=TDevl5z?WQChPce_WkD119PqY??N*F_=K?;17C|>MVwbYfa^EKK zY@$`*codS;`uxQ}%>y2_6e`7@z}6m&bmgI-6n2f-8y1abY)?4K;1a%HKD}Z+^M7Uw z;k*y+k?Ogjn;$8QgpU#S@zX;ZgGcTd+J6y^)X^O5nsGK{#EX04eaWF=?=og&pzY57 zNtSAD+P`CKF*?->iFbmT!!!eC3Pd0V(-9YR^a`0^0Zv98gd4pS>68r!Yqqgmqy~`< zvizt5gO->Ob;@t%Q)VGbPo5<&5GMjQhLHRBmh+i@YjuOlZz3E;V$lvcwr-%}%;Rkf zhdU+{xxdv2<}$S^)AW2Ab-5Oh^Pg5hxKez-!{6^Uti&gf=GpdU6 z$EoRv6H$atBo^WIYB-~#K6fp}j1}hyG?lynMsYS!HSZ!DePV5$AjK^IoyC?xQ5_t( zPc2{xL_^D*&x{v4L!`Pqp1n53dpjwbl}95ra1@fv#)2Wb4rJgYvsA0k8XzT3%SdH& z2d|_I?j5Ao+O``dLcaYAm7%Ntw<0(O>?DrA{GgDRviA}++Pd>5PKXRjq#L1^dPRQX30`U? z%N}D4^t~}mz`eRs^e@Ah`QiFbFlCoLVs}v{k<1$!C(nG{Cf92WrJSl?NIoTtm&Vg^3SQao|WA2ht4KunAaNf^XQ77>i+ zntOUd%$N7uHIL;1pr#uY`D~o^v$o_AEnuk7K81Gb-#kx@{2Hn#>PcQ> z1YDYX)(@qM5mA+|Spl2x^ukzEwsj=DvX!`WaxK?GZr@8QEqwW50IkU)NU7CgKjq~E zS|EEs&>=CevEK{buYNEZFUhbECT`@$MCi5PMG1hgrp9ucAspmTZ%v@*!D(0N7Ptg! z&i0g`0xD#Nr^qvue0t-zm|Q(jkwTN-Zk|V&`Ha>&bE{_ zhBZV%CYmI1x=*s!c>3mgB%qzT%h=XI9#VeR%RYM&5pro?Jz|w@U)$1&daxuB-Wv{K z!{3fxrV84nMvm_Pl$*+)@G%zvlL|g&3E|dzs-^#r^~?k^`n7AY#p`4IxrX0IZ|LrqO8&c4VEgE8MkwaR3t6lTbx5N?aWa`C+(b?4*%f`wm;k@8GQ zfV;ml9>~9sO5}ZHlz>U3#?juyKa#3gUs;MQa*{NZx(k@zL9xX0nHZzp_uqYcWP<0rFZES z#l@~{JlzwnuO3jTN3V_DFJq=FlW?p^zHt=&N?Ck6k?7!QhBMNV@x;yp9Y%j&+f;K+E0&0>Vk((0IN+r-f#Y+R zL^QfTiZfEV&nRCW55Bw7+W}|QZp%Rqe&k@LGi!-ywTNV}%x7Wk7B%;n%){YmqxN5e zVLkK)#pO{{;p_qM_otnjASg-)Nd03eEph^;M-2D!GTsqNxlnJ06%!-Yw^1UPJenL( z)z-#HfhDd_YZ8{ahdi7bBjeUDWo%oMBe@jm#>US~v>tPp-uahcHjEw95?A-Aikad% z`LEVdKH-?1_KqU7N)jWfD&W1Y3`pCkG9^Ni+Vrrgu}h@s2Q5^#pY$nGP^Ucv$2YBU z`s7cq3uT~DTy!*f%?&=`D~PDJU>I1|aBBbEDJC>ok(?3XJeEw-{W+sl(&{VodLE7> z(8JRgXGXZo<@s~zop zLSHW4$FQv`ID!r3^+D_}=@02|;IyuXm~#ejzbn_7R7wVBvF<2GJy3QPJpwo<+gK#0 z$KMxBcd<1=x_YhyN`ssiY_H=w<d0FPF(Hg)jnY^bZ8r!RwrO2a&le1+$a zni9ncl7w8`(;_EAW~)Yzj};Nf=R|v)SB8<%K#Ka-!+}j zqE4U58GGztfiKsQ2~Q8u7%1+YUU3!&{zYJ< zY>BBGFba5X86hx$gd~Y=#7@=YI%&lJt3x4xvwZkld{c)gJBGVqr9)Cz93e&Tg$ zV7SlXkb_!nAHb4>nn*I;SfbsWpHc9&vm2agLKNJc_^jfdMR*B(7^pE4n$s#f_W(KF zIagSR-+mRL4a#r}9LTn*;o|GLl-Z%a>+5SuVq3^k(f@^V*|gg&$c#YUcXetuWvCL( z4_7H+ECBIZQw~=Sxa#QF51<*MBj|apZ^-CsSI9^7F@gayjijlGsiZbj$T@?3#%?B} z>H8aIaCb0Q#XXcdtDaTr)7E1`%R0i;-+)M>Ty3ihC!135R%rI>}(292M}T(Vl{xuvE3!_^kK`KE37*hCc$r1X|{ zOR6J+3LK2D7;TI#Rc5yBxr_f9A%A(}^vg9)c>*nVXu?so43QGXwwel<)V3WPG-kJR zFAg!NZ;{XU(iEDUTZIB-DpnK%X>Wt*J`j$wxt^aly9*Ot|u?Mk9n;1xI#_5 zahNoF&73AcJ@bz?WG_U|7dvtTsa~g^OF(0HG6%*aH@0yBv4-71{FX~cD&dUsI-Z1D zq0}XUOWE+NdVN}bY{Ed=f3kYyJrs_<)Zlpw|JP7~>K1wJcrZG}K@<67N>Cn1MZ{Zc zL?#+Y%YAso^w5#%e`%#!qkcD>I)Z7dL|uWa z9vqgi(%h`0iecaW@G3;8_f6ue(u4-R$BB#m+*RuDxG&KV1>05`IziKx28oQ!q(D`c8BgW`H2M1M-_H z>%OiC;xY1ya5=pzhGjLHs(h&hjO5ZsU$Qbo6)3euYDNj7`AA*|@jbBh zr-wEV;TgZbqrut}I-2l7U%dHe{(3(93)*-k+NQL;~O<8$myy3CFa1g}Y&oU!WR8;&#QZa0`>3j0E z@n5Iv*c~Gjpa^R=pyf?eemWJ;-m>S8W6LgTr1yMnp=Lygn1pTSnW*{^$)FC9%?iMt zq+N#vU-&Xv+Cn177G6_Q0Tv`gvTe=Mt&YmRLMxk?H(PJVIGI^@9puMOg>l{)RmohT zCw+IHR8xvLY$i}%w6jmhe1dq~fQ=&K1c)TuaM;rmR^+(?oHH2;qbklwlH~VAb;JgE zinoq9c^)2r6mw_02E`}=p}{jEv4^z6u%JSFY+8~)vu@TvV<^O}EaPynmF$xt7&p{Y z09!8;E{JnpqXQWbCBWbp@OAv2+S44XD})Pv4I1pLl9>&87P2!Tna0!@2>-cBW^|Gg z=cgqsLX&PFtkkChBnh8OWWq>y9P#KzNc$2@7?Xp}6+3a?74}hmpOQ<;6JtP=#k3QEc3u90#DgjwJ`> z%S8|kH2qy}{|Zw6D+Y=;L1RbG`d_B49|&)pzu%9VBZ(qytqjGNT`^wB6sUb=ldjg- ziwb!^VBP9Rq^qgz3|2BluKBv)B*oiu7rAHguyI|8^R-A*L0dc9v435kaK_#$6uIEm zR??nk9Zg^(i^e>PF*__wM~n;`tYo7oFJe!frKI3PlOH_ zLs$;SsXz_nI6Q=Ay%tD9pG95mrMBGa^#dhk4%pvN?b~P1h4l^G>7aDzO~K*JHo zvv(dkDYuD{%#o9l8kAIPoxPihZcQYU6n8q&W7Tw<>|~L6zkym;ZhTtECSPfT{ftPG zUK>}l`=o^UH;AU#VbQMUipFe%DcQihEpRv+T{j};nan9{q+|s@4k>9KAC{h?Nyh&LoV8jifshRM?WL<$SF>xoty?JMAjVI5B zD53&b^gQAXZU@dnn@fBnR-@w}0zf5}1JB=cUNS5VP+z+M5wTv7pqRn9kd=21YlB`u zatt8{qY*n9^u+rz_qJRQW%|p-*i{GIv|xhA+64w}Y)i_?TN@CDrP7%`#7mSB!l(!z zl}jjie-@5coy2;;=(fIouAyB{JT-;=uvw$LHt7pFM^bEHfl0^DkJKH3jM zXxw_Z1kq*|HbYISltkC$&Mx-FXY7v3&q#c}F9Vg33f~%-FQGN#rJp8(^d}}^0h+Ke9ZR^cy? zg`mguI`*9K1-#{BKL+Dsfbg%wj|z+93Sc-MpZS`2y(4k|VX;+d)}YJANNpy}DlA&} zQ-tmV=6AhTN0V}(afuO*9&`EV(PNzFsa{6$lx8ZJS9P52mfmC;Sv&{{%*BSzkk=Fg zuEO@)@gLgcWw*PBzhG;tUdXI`Hxl3Qt&r{%rvW*w?Y!LvW!;mafG)gS(=01fyLyd= zxAeq|X7FUFs1_y(Dvz1>gPcvBVfOG?X&(o-n#PPThdL+C@rttSWK$XfFEbKRvu&T} z=wf3PRybwWp#m>=s7c5?VR?e_P64ur+AJD}2MuvDyhoB2u@E#s634JAA{Z(C`cXW^ z>`8HEi80c3Lo$GfERmj%sVij+J{8DxR8EVIzg)Cf{V0iz4tdEO7J@NJG&>0MeMaMd zt@xygACtVquo+$$p3EqSjF?dbQdEs8E|_p7mZ`*$z-U7%XV!-+Y=zr>iGkNP zSmcVyHH-fy!*AT-n>MB!3cEa`z+~RGA_a{ur~aHz(b`m#C6=p7MC+=1Ai&Q|SuMj& zcV%8oYVYKxTKgEm%cs&j2sOVvkB38WBhxWflpZuvx<^&x6WTh9q~A;LogOAGb&5Ma zre243$N|1i6$0Dl_J9zLh5qXXZ8Q>YCCeMCm9FHQw7(Ay8L z_|USVk8xE8C5@gNDm;>L@r(Kq{ATEk@l&h6=`YtHwW!pAUZAO#44u_ZA7{VbFIl>gmt8jO6!Thud|l&I0MaqG zzY37YgVv_^OW#*B{CIK_K6JZC*jEJ5mdadi62sU?ghGngRt5pNc4D-Fa8*^0LY z<^u>WH4^8%da`Q=W9}CRpjB=t1$+nV*H6iJNvd`lMnT3_)GUwzf@v^Y#=8{BU8?z! zcxnsT{Np?V%mM4P4hu1RA2{j6f0xKYVgvOiBmvZU;)Exs5z?@#0z1kL>&?wX7rUwk2+#|7Z!Im>Bc;-s6zdqV{3L|PAqKVnz zt_Tz!trq}A-323)0oEg(W1NXbfjA0Om{*VYYX6TjLGW|k9Xw~^f0u~}3HWVgmX)?{ zYl=ZZD_y}O6#x(WtP;u1HUPDN|?^^|)e8m0ti{ccd1!!#Ky9PjN}G|Q;E?|1?@ zIBU{i`%Q)g+>*Ffc;E-Lmh)hE72~V)rQh$m%#tR{)NMO3oPaa_uM1cEzH6t9xZ5V@ zR1bcJaLlAZE0gfi+l2#D$erAG_dkq^%Z=ey@C49O^5I5RjwO9p#SaWefW6rsOH z0I~Oj6;oR(8aisvc)VY{pQ8B<2-9o;X*%zvp7}2Mu;2$6wX63yF?XQIpmC6L^^&>n z4;4b%nZ%*d0S2q-9({)T5_Tj@4YUg<++wTbeINQQgVS!wQQkvKJYQu?DW$T0l<_KsnV*E|C8s%{mnr=!O6UumI=r@^TFc_XyN`r)(h z+w?7BbC__xKpnmHfem{G-<4nS_P&7%Hjp)mT$9#N4>T;UdL_8@pkE#n%9=grZm$CK z^KjlQoinj=%aq8MJ@y^)Wd$mt=rRyLXc>Wk%`~{c6d%T?Mzh!_bGUPc5CzQ7UIE0{ znK}-92{qvvThB~`s7b7(ink!60f>&pi^A#?M!=ymcwU3uQHNiWt(7~7#j850deMo1 z3p<4K?k5~4fL2WC#oeeGpJW3kJ*CS|RYfaLCYL_2quS!8^P5%#@H$gR7a*}H)Fj-> zz52=ry&9~na{!K|PzXOIq?JR&f7v8>}`gDR9G&K5TjW>h60iTHXm_0=W?SyhMFo2Ra5^NO+aLnkVn zu;Gb;;OQwj$AnQ%nW!^Q80bbFH26e&dQYB*9VW6%2sdgJcdv<|#~Py7urLN4A^Dcx zIq`%15B=@oghkt-I}sTPhU71C8Qnei@hW}MjE90GiVeKLk&?xm&c5Iv(<1j_SP529 zgiGz?VN6?ttQmUYzF{}^Oc-MpI~#5J z<`Re){Z#H5=xym|2x|i!kX+<$(zqUyb1k$qL4fP&o`jc$e{GT}TrDHct#Wr46`5w1 za%9JQ;`RMxpwQ&Iuy&Akz5JJSC#$Dn z*JZEKs{ltps}&j|qPgCS;C&R_D!cq#b$XS~*^ZrZBn{`y}6Hl_nxYDD*ty*wch8 zlyi2}piwuSK)fcpS^bZK0w5engG<;#nr}$laz!ubhV_5gcSTm_z}mD-!@jNTR}wO5wj9suHV*EObH$=1}!MJ zzym^QLUgoUH-r9bNpC!cd$%f`BYh^*SRZqx>?pbT^=f2L4Y#&|(}BRbm=qWt$)W_s z9yF2?G$hR@Xn1}Vjn*E;!(C$km32yDlt%4CFo;dH%g@W!HrwUCqt&-0HD-SDB^qch z>hjF9d$Ao+HgG$JOpBUp0Jy!*@<0sc%yyx`bCj$Y%>+fv4D3U(a1bk>bVy}!hq<{0 zo87qu9KK?yPat!?a`O+ad@(OV*_1aG-?(cydMgq@N5#FsaHe-Sxounq>h=B0<77$m zPLMRO-4lA8V*YA${b1ux#17&?A5l?&0spi1tN#x%b(9u87C@zQzr$;JtAoIat}1@# zYUm|dIve+(N(}i8nNrQ@=#N;TaLi|`<78sxIGgVJ9-Gb#dsPt{fE>&>dxK>^1rjc(KvzV`j|1Nd^)pD=r1jN=e(Z~@!9Kw6KIf+T$P zYNUqTZj|dlg%HtdubvU9*jkeV=0A#h`{RG1bvkU;^tsNyLhRznZ1wR1&2V_fAdgG8Yd|=t52>h|^QPd2l?+p20*oP&d%Db~H?rkJ zuFtlL0Q5Y^RDR*xpwi%;%vEa0T3#o~zzxE}1-~ zV7Vk*6bYa~iSXXRB^x49)?y1=ly#RD*U4v0cOsN+AlTa7{rSe!sk0i#y~yw7_CwLI zxBj96Cr!wt7w2KACrXt+MW-D;?BKKDxuM}qI&&fIPG|U%#;M*b#)$mX8>PVzD)j0q zrXF#n9<*{kTHDz4PtMqxb`g$x%EJQ8=wiM0wY-Vi46AsMKF27M4^6Y{eHOqk3rE)% z0LE}J-r8yUeFl-2MQ+fwVp|CX9u&Q%KvF<6pE8leOd*VWq8+cGtj3|wx9WDNDBa7< z7NI1mvCyK2N)EGnaMpN&2SDnuaXkctEH)r`{_Wc>1#bw1tXT2~2yiS&%uv{+z~)+R zP}^$= zPcla7j3Jf6NXw|EKY-ZVHh&u6kdyqXMrrXqdWgyzYOf*u>5oS`Imw!Nb$Mqu+NYx{ zp3iR%Wu)Q-*47jK?>VWV4XJX3^bb9h_@aj4S=v=&ccwMJ^)mB1$7~2yd0{BFfwW8o zakdg2<((leSr5L5{l((5+junP&P)eh8kJXg8aA4S;tD)SKeX-w)wmFtonsOaQ0ylb z?ewu@IA{TRm&j;|z|0L1ZYn>47$DcEj58&cAu8GH=+hp&{v1)12mKK#d(I5XB5Jcx zD7Es@zz98-=8%c|9~f*C%>YY6t|2J9m(iYSGy_PNn!S{)S`y!cWQfk`fj`8030ctK z_NzvmVQfXF=oyyp$rmfpQeQY9a4VwcJv3`%-9ej!aTLLJKkMwY=agTmHx2|0n#Tk$ zxYV!W%W~}s;FfICdv6T>ppZ;7ItVftLpt}kX3j!qQn%~@1rCjsv3X8}lfdl+(7W!B)-?A$2Am1R?j#pw~=on>j!?Q$&DN+0_e`AunhDjhA%q$wm2N z=yIWm(6S&)gt82|I_$6w8Py_zh@pE$h0I-Qo!nTbxp-DL)^^rZORTmHqH(d_Y8<$lM` z{nvo{079C?P&!if)&uvhaf-^*%0aT@rzGMJ`2vz`LR{s z_ST4`-~%_!h!rwRt)E5UCk`0~*}`bb@~h5DI42>E7~G6i#R8NjXfoL)c&31WA-oPI z0mW6VAchr}BWO7XxaW2^Gq;HR24Dt{{SgiKkb_+Dwa7e;p)9UhcG4@h)TtApk_|hg z8iMGF@Gel#%ukmQ{zU9zA>;=tc%aemNxFr9sC>eoJZj(e8v_7NPc+z5n3y;pW>b4U z>G!Zh+p8Y4d<0u$-mE{o9t{VbY4W)f0hYL;+ivf~kJqbTgU@mFeZX&_yxp@1>cow7 zf_+A^Q98faKIX^zyWap2;9JhOJn9 z7i(>PXPII6Pg!=(0g;CZFPQ%s2=I0Xdw^!LQ6q#&@WG9MWsmQ8k9J(r8*6jlJ5NEu zumW-Y6Yr=L?S*+%B__)@1j>>j22)V>!ILW9s1q^)lsqMR&*?+sNU?(F(CcWRFkieJ z{YV=kYvuQ%)-Lh;&Lb}E+(C@%2o|$|PA(262;nQ9T&r{&{r-)Hb z+Q5jV_aqnHoe;xH_aNjV%gBdfBW)OdyU&^Dva3&xhe3W*LXMk!(oIM6yPY{-H$n_V zLUUu@B0{)dz>%k%&a@f|>Sq*vw){UAn5BrR?&V7C+;VB~jp5W7XwLurp}pWqYI6>< z+9ux-@Bv#uD`F?I6(Uman5&_X0S3_mkvu52IiL1%@nhdBTvM%9JR?6YyfG&F3}%}G z4$`^`puZIWxQqz~2cZCoPcOHY=j#fzARAMNMLcDSiw|kimV5=Uc39G(=99pFS?;mSvf+YfF^9DSa&51jdR0|LM{f?*M?v2`rTOU<3>j5y|6A70M5hm=MGBb47lb zN&vsnkXCq#FTgitp!U$_FAQv2?=QWlu}>V9!F8LIa=uRO0fn>jv(J$@=)rcOFhOUt zNPWBdSdPdCTxKw@P|kMwGtCo&kQlge1SQ7dHZqS6+xRe|iY`V&NM|$AU=XBbdgn6; zFj-=nH^Foxo8$}yQEQMCin0T+^#X@J)0ljw5%O82biohu3eI#QNdXhXNo*%DNjQ=h zOPSgU44HO^yZlwcECmv1Oa$FscMHyO{RvY*9FHU2SZ9_%Vk1JbBS!u0tx$S`BATM& zJbvZWP~x$8xPdAx!dvAfco$~a5ipy|8W!WaUCtpiKwf6IMPc9@qX9GV3U#H8CgeHn z;yjTPy-yyNx#R2rFj=0EmmG|rOYD?%5s?M*eFldw%3IE}Ku^5qi-WOdL&|z!zEEO> zYw%HqC{1+e;q0o`2g=c<$Q}d!40v@a8|UbQkaM7b_ajs&&4xcrgOuIm#DS9vAb?k& z!nA>_Ri~jKz2{Ok&4dh?H-+#^yVuqc%mqy-Os{cyv=k-!Td40fqqU46SHaX zd|V4U+npff-v-g{Zz6Z4(lC3V^EwB6ybDiD<*8#-Qh&G? zrw7R?@FRdc^zcyxB~E`z<}x-(EISCnbj>D?D8K|R!Gs>;3!zmtG4cy~wrYBwTG0m^ zRu39`xeW_(v%DG_JR$oU&D50dcxdM^Nh)9le#F6iA?>t_p9pxXkvpR6<3krK2`AWNFvlipT7aB0h#o@EPzTud#pfiKaJ^5MbZUc z%x-~kR56x2J;3sKa>m?rJ<*=YgLe^BQX96|#Q7LGN7h0>>B&biEzTEFZ6Z-pO#5LH z!FiONfe6k4IsdnO$DN>HvZ3;<$QW}D`RyJ7B^&m9V5{Cj#YKj4?)Ex#{l>U2Z)1!}g* zix}Uj#T*l?cn&f25&Os-QYA+!e|aTN3N0P959>$Py3I2T^P*%{I#32EiY}$PWuRTj z)8`TD<-pJ--eDwcp|4xO$(HJDXJ%Tn2Nd!|@JMeX=MQYU4Ar-h?`;^e-3gs&^Y@Pw zQ;Ors8~AZ>JF$)FsVK}qsk+=ragB!rt_y+0>9w=v$;pIU;xY0%k}{-Gt?(Zri;xCn zkI5xYXSPdSSu|$e*0|FqHjLqLaKphi8}?N{P`vsN9_dPZqVulA5%T>3O;QYD{6nh0 zvUA5}Pg%4N{I8Ub4(q=%?^cDF^Oc*I0JS%P3s7_u^9_In%nRK3vM*NKjv^B@XOgXn zWm}4x$=_+S7FuE5$JV{|)b#9tKeTyoY>!&=Kf(;&@aZKj%Q#c1zW;{H02DN!f=>kSvku4Yt7)ElzTcXS z@;vVm=RWa1R-*5?Vhx&GdFfwNTzb$}&aL3{Vv}+f$Q?tRX-^T!Cs0TpaoG)KdKa0J zo{{0rJ5gofo*|$`+w2yfDc6zU3~$$5I$Jc}h&JG^_T8mEloZ`M*V{7%%_i;~SKx3gh#JTH*PK53uq z?7EmQ?GC<@b+|Le0!#m`3@9mF3a;_uv;6#$w#DW#XH&tflxb#l5G9#dC3;oN=OTGG z(D_KgTh5S#74&HDL}H!GMrubB{W{Av|CspNxbW>@3$=^craFZqz*XF!js?c~MVcTV zk%N8MmC1$HbR2u&$yEQsCtesv70^H6pzffQNYGo3T0WDtfLyJ_%Mc2zasa!$qu?xz zA_Z}k#D~_%(`6qZB{Fp>z&xm!%^A7DG{nhm)L^d%FA8lY+GY$e^}75qY5SU+uH*$7 zn@1&Z?lp|ekrc2Xa;1Wd&Wft-BSIeV#)@F|v@I>Emk2CZ9-gPp_{s243JFBD*&Y%g z4|vCXo@VKRCuR@^vQpPZN5DOl)CM^~t}2!m=8uNg4`6xNGv+NjIWSjg+}a?2H7ikr zKs;^_T|(plH<=s!*-f}(4KzcG&?mG;golVJfD+qyAEp8mII;OK%xe_&AYV@ZJ@Db- z`NWlQ@!-b^z7Ei{;*zGkqXq{6!+@${x^~{i&pK2}Hbhc*Z3lbjAsYdl_~GJEtP-af zA(s1diFoFD>J_9waPdQ}SFW@NBrgc`(!Z92AEm_9pOsuw8bqXk1YAYw$aIZXo zTvm$MPCO!OZocN-i-QNy?JxU{y%5o}s&!SOb^C5Gb}mvqh@@1L=65j9aHHp@==;P5 z5(k4SU@q0gb&_`s+L|zAP0`62-GEH_UT?(FqlkSW4$T z29lVy_s|2JRBI=7vGzEf35$W|Ki*Fn?Gl%TT5Fg&GZ+&GIn}hh(fx@fQ2zlSX01qI zlkcGL%f$Fv$cM}>y(}VDFiGk3>T&AQmVD|MEmffY6?OCJ&;Ztxdx*qAv{0fyZz;F^ zZbTi^;5VDbDiDe1F}x&+T<5Uv;X|h&03Q|f$0;;GO%>D(U;+BZ2Qu@`816WJOZKoj z2f0kIu7rj0m2deLHpN){T+c4D7syYUE{CI*idJ|SG_P(5T>-A&Qpnv z1}?32#SIsHYfb>%oEUY}wIv1g9WJdi~h;P)(sUdnwo?kDyXa}|z-*25eIyCttP#$Gw1bNgGCwtGWLgXkI zwOmT_DvfF$#2cBJ*S1E7?$FIe()05Oiy4I#(lif<)K)RUK_5I$b8(WTaVME0O7mSI z*2XTKbhj!%>S52S>A{9l1_j@^M4wx@)JPzJVu+PLX3z@~#y2IyPk*x%rodFP1x9T| zEghH6{X^d^O;S8zJ}o|yZJi2Kvt)OPJW1?5idHO!xuf{vE^tMvy!?LE#N(n6iM@Xf zOgt2qZPkzYyPAz|2};6J zd~#@IrCaYmr~U%d-o@vY^UqI4tW9)lMjRnVNJ7=Su|5wp%fVF3KjiGLn|R#7>?jB# z0!;&@4u$^t#FQKaDTY9m0k*hFizzydA5u|C?FMX_J5x8`hBFyhmRIG&=_f3kTl+^n z3)1ICByr>{jYHwZkXF5hgyej>RcX$8*dw}gLEwWc%2(M0HM*W!iH=f>`L68M<^z zYN~U3@S58PSbu4%L9dI@#x1dSN=Ok$FfA`=W#Bk5%XfLE=oVZvcl#NVY?aV zTiTZ6#nDX;OHzoa4n|7K-1LPKq2fhZj-#Xzm;ChG{!g7fF(*G(_#2EOU6#J z&I?4t==>CDsSb+2d@3<%K97VfHB=-40l7#Cp)CanGag`uu3yX!Omr*BlVn~tdu?w$ zB%ciNKQo+9X{Cpw?DlF>$;hx#`q$fd3`VP=IbxD`+q4tk7MTp!GIn=>=Ct zEp*g9>2hBbY?EK_)0GA=v>lRn z3tQjj&K8Iks1*HE&FaofadZ=^y;elfjf3Ijs;BJjs-9z#8sEY5(~ZBFuPy;H8eJm zTVM+X)u?TT+m0K`3VF7j~E zK(YW`;H*zzj5!<{92H)dKc3fTB`M7r5T1IZUyuVR1JD(+!!w#Q2RzC<=!6-&C>e6D z>kvpRnh`h2+H>P*D;cjDMPggd9iBKy6vd* zhTP~lfU@Jcr z5+LpKBpK8<>;nt~sNGeJx|JUW1(<%`7XZL6@(P*Bsn;zH3Y4vMHJQ|>dwsL%#*fnJ zjqAl~FHuT(8+b<7H=|=+^v2mfP{9DGkh3$&OS@Ad@|X3%u;ykbs~z*iOncbfi}r3j zt>>JFO2b%L>$j{?pV-5bZ=J0))j5~X{jnVoI$PS4r1*=(V+J%y(EyFM&$@KSAd||R z{u}qf696kgUvWI6GE0Q-U>1XmPv;_QCXK)YIhY$6@Z^AGP{m$V7zwDGt8v}b42_Nh zg+X+)_Ps8|MqH(4uYAiPSliKNOxk!r6r{WY}`aB`#~4N?La zB*Qt;PJ}TfEi5pEkg*Q~5YI`@aUPdc)1 zy~S+5VL{AmXgv4_8eDXKJ3C{Ql-O3Tr;0crf1S-r-Fu z^1=Bx@W`BK_V>@4$P6!RB zuT#Dh+1@pjLn(s~sAB9Y)0o&yhC~_8KJam^cSGFWlI7mg{cEDGR&{FdT<5vF1P3v$ z0c*Npf~?_!6RsR9oWu#qX-W%h1^pQO3xqxMxVOqldI%>hC^n;3=yjeWr?&e{yMxY# zw=~e^RgwX{MOnt6FdjS5Tz?+h9h6vwxplLVtwz9H{|ZgV#B2sMBJt)63lcOp?5^~n zZ$w9EmBJ;fq@W5bZoyW4AVNZ}j7&Ho|LH=;+LXa3TG$!}gih4cyRi&m#!lwWkwg3dOt$#a)2CGKh04Zd`Fi*8Q0{$P9*X`5ZD)c6Y21)$;7j~9en zg+lC@7}gT}c_4sruA+x$i=vyL&}UgNPr<5Ld7kG+LQu4}!Gn!ITgy zv`S47H6&kT77>M*=yqW3ptI3x zz2r%1)AnVhK?H$?P~D3ba#2Qf3k(0VbIkKgI6!pl0^Z}#iin{wd5_9KROk$X^j4qp zBL@D}4d48dTV7tONA#qW{hw;hPBCx;k`It_Gbf6@{+@leNZ)8rlzAJ?y=b@Ifsz=J zt6GBu@tsq}D}Mys+;067KSC0B&N|1|jVwMZ#K9mWt}>f0X%2oD2++V4+?a1i3W;5` zAve;sAZgEwlmT&KwuCt$TsOMH1T^>A*Btmbu<(|CfQcgvU;@_PyNk}H?rGr;j=qs0 z?BR!oltcRtPjI&!b*f$$xf?Y&$>I|>M!6^p8{hnS0mJ1EU}#t(4SZw5e*f0Q2d9v3 zJl_e%Y`q!64pYE7Zv<_?LH53F`)vE_aC?v|HgVx>D#58?JyTJN7zB%E zY#!Abjc|?`Nn${0_SVoM(teMJJy&+c4DN|p?v#%hO}k_8T&-wBQ9+o%PI9Fnvx zN75!?(;ioAv#ey<6cfB-;1~A~2;okj@QZI}HLhK2;s0_s4c%jF;Z9%!qvSb53Vbn* z1UqxRt@3s7l1Wr;IA#Nw13`PGI%!b#GTT?5SSc<5A&;EfiV~$wVhcXo<&v856Fcgd zmy2(souJ4Y%KN;gK_SBpToO;1-yftI4f6trr-=!@BH&Lr(8AxGbWf1RQpVIdg(h^` z+zf9nZ!D=FgIr)1&IF{aD6%+AJ@omci0s87VYWP8*%|q$to(1F4tvS$J~Dv06s7#n zv)DuZgO^@%GQP%GBNPRPga5LV)F(OjCHAt~ks9Drio3wQPcW6KYw4lH#`IE?e5^5q zEdN3|?kAIc3$6&n2y2{KO%Nw#l-P|9A@2(4%7J8?!=LvHYsuIV`4GXMaM>|iVqqA8 zCtSg$4DoxbI)ExF(dM+lT{d7XKRis1hDz& zTccFnqi6wu+6eCTpPABV9BPeWD!M`1S!y~U{5o)>SUG9jc~ah7tz_YzK#)911ug08 zNIZoZqC539hR6FjrkgeWbG!Bxlspin5tc`E;rmAD_&osuSKJ;iw7#)fxJzUmKCb{$ z3L`H#b6mNyvYIX7->X0(SqH64LacAoZDKOCGz>TJ4TtpwW`hc8EWDyh3u#OJ+KN4e z=LO(G9o^tsX!=cC;eUBtYQ_XX<$ASW%!^X}nBv*h|9cm@!P=KQYSE z_f#9BWFO+9Be z25==uVb8lG?J0DsNj(0^1JE|6EecfNp(P$WpYzap|7}0QccX~!c}dQZjm269pSctA83Gp++dMD8IvyPn|?*PD5gcSNzmL z#NZZ3kUQ@4GZj|%r0WFJ>TyeELx3yp#~X6bGyXm2CApBEMNJ<}>OymskShb%$u$8+Hxm(J86Hr_Ukllr{t1#rg&Ssv*HV*_NC;6I zxVyZPbztG1N&IZ5W!TRW+SiepS;XJnY54Dl$jYXfd zk&Vr$6GMMzve{guSqD+g!K>Yjq*SgrOS^!rJ(t=l29X~I!sFU+da-QtNi0mqaUM$k zR^l|ggWd3LaYSrDLjvcxd*xpnvO`}BZ7bvfJC&W$a zI<{?K`?_1;V}%Vcl2v(ZxBaHXUtj7nTa+|WGFgQgE+;WL!-+rXm(9ZGKT3ykK)ci% ziNHmXd7T^y>Yxg|4$Q=8Ie_u0b}Eldj>~3;0<$#&Zb3ZM`c>`T@NbyE89->gI|vqQ z4`?{fu1KJX&?y?-^E^})$5aBAwTXBA7vzk`$mcVC$-3n1sF;zxv%fId*YANuV6AJ% z(1Ci2CJ_W2VQd?=@3Q<4n%Fyj17=?^eyVY0gH9cJ&4b2Rs1Qi6Lt%em=(xWIW}7$a z`8mX{f0L6G0RsOmGy0uMVDEAW{FMIWEGFPYvA?<&+sblW#J?&*f*Pgmx}14eCTE;7 zh)%6-9Wj)Eh{*Rl$?sY?7|dr3ItU6^7#RhljgWSb`{gNgcqB>rp7!^WS$m=lEe<2H z75RpLF&oJP9I^CaB8={lQ#`x0P7?QSUOo=q)0?yTjL^{0<8+o9f7t2#2znV8M;t^I zD6=cH$xJ^8%HuF!dFO3CRHZjJ&CZdEEXh|20Y!umWhNIzi%H)u%S=CX?SoUnEinoW@Nyj)SZLROr(& zeX&Kj{ZOC=1&Ht`D|7b5SH#nt{QrL^^zG-bjx>S1OSZ`EamtHDO7?A(~ z0FNR70Q3X_06_!*02mbj09OP60FWdA07+K>0KB0C0L2af02q(}03{^=0IVbi0N@4y z06-f60FM9w05}2w0Qvy{02~nj01N^E0H6y10JH!A0Pq0-00;vB0Nf8_U^oI4e;~rZ oFcm0%0LWek6kAcwz@PzS7Z@=xNWKQ@|H{Y!G=hO)=16.14" + "node": ">=18.17.0" } }, "node_modules/@babel/runtime": { @@ -89,6 +96,523 @@ "@jridgewell/sourcemap-codec": "^1.4.10" } }, + "node_modules/@emnapi/runtime": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.8.1.tgz", + "integrity": "sha512-mehfKSMWjjNol8659Z8KxEMrdSJDDot5SXMq00dM8BN4o+CLNXQ0xH2V7EchNHV4RmbZLmmPdEaXZc5H2FXmDg==", + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@gltf-transform/core": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/@gltf-transform/core/-/core-4.3.0.tgz", + "integrity": "sha512-ZeaQfszGJ9LYwELszu45CuDQCsE26lJNNe36FVmN8xclaT6WDdCj7fwGpQXo0/l/YgAVAHX+uO7YNBW75/SRYw==", + "license": "MIT", + "dependencies": { + "property-graph": "^4.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/donmccurdy" + } + }, + "node_modules/@gltf-transform/extensions": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/@gltf-transform/extensions/-/extensions-4.3.0.tgz", + "integrity": "sha512-XDAjQPYVMHa/VDpSbfCBwI+/1muwRJCaXhUpLgnUzAjn0D//PgvIAcbNm1EwBl3LIWBSwjDUCn2LiMAjp+aXVw==", + "license": "MIT", + "dependencies": { + "@gltf-transform/core": "^4.3.0", + "ktx-parse": "^1.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/donmccurdy" + } + }, + "node_modules/@gltf-transform/functions": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/@gltf-transform/functions/-/functions-4.3.0.tgz", + "integrity": "sha512-FZggHVgt3DHOezgESBrf2vDzuD2FYQYaNT2sT/aP316SIwhuiIwby3z7rhV9joDvWqqUaPkf1UmkjlOaY9riSQ==", + "license": "MIT", + "dependencies": { + "@gltf-transform/core": "^4.3.0", + "@gltf-transform/extensions": "^4.3.0", + "ktx-parse": "^1.0.1", + "ndarray": "^1.0.19", + "ndarray-lanczos": "^0.3.0", + "ndarray-pixels": "^5.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/donmccurdy" + } + }, + "node_modules/@img/colour": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.0.0.tgz", + "integrity": "sha512-A5P/LfWGFSl6nsckYtjw9da+19jB8hkJ6ACTGcDfEJ0aE+l2n2El7dsVM7UVHZQ9s2lmYMWlrS21YLy2IR1LUw==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/@img/sharp-darwin-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.34.5.tgz", + "integrity": "sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-arm64": "1.2.4" + } + }, + "node_modules/@img/sharp-darwin-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.34.5.tgz", + "integrity": "sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-x64": "1.2.4" + } + }, + "node_modules/@img/sharp-libvips-darwin-arm64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.2.4.tgz", + "integrity": "sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-darwin-x64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.2.4.tgz", + "integrity": "sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.2.4.tgz", + "integrity": "sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A==", + "cpu": [ + "arm" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.2.4.tgz", + "integrity": "sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-ppc64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.2.4.tgz", + "integrity": "sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA==", + "cpu": [ + "ppc64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-riscv64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.2.4.tgz", + "integrity": "sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA==", + "cpu": [ + "riscv64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-s390x": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.2.4.tgz", + "integrity": "sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ==", + "cpu": [ + "s390x" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-x64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.2.4.tgz", + "integrity": "sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-arm64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.2.4.tgz", + "integrity": "sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-x64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.2.4.tgz", + "integrity": "sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-linux-arm": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.34.5.tgz", + "integrity": "sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw==", + "cpu": [ + "arm" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.34.5.tgz", + "integrity": "sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm64": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-ppc64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.34.5.tgz", + "integrity": "sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA==", + "cpu": [ + "ppc64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-ppc64": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-riscv64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.34.5.tgz", + "integrity": "sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw==", + "cpu": [ + "riscv64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-riscv64": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-s390x": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.34.5.tgz", + "integrity": "sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg==", + "cpu": [ + "s390x" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-s390x": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.34.5.tgz", + "integrity": "sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-x64": "1.2.4" + } + }, + "node_modules/@img/sharp-linuxmusl-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.34.5.tgz", + "integrity": "sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-arm64": "1.2.4" + } + }, + "node_modules/@img/sharp-linuxmusl-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.34.5.tgz", + "integrity": "sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-x64": "1.2.4" + } + }, + "node_modules/@img/sharp-wasm32": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.34.5.tgz", + "integrity": "sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw==", + "cpu": [ + "wasm32" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", + "optional": true, + "dependencies": { + "@emnapi/runtime": "^1.7.0" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.34.5.tgz", + "integrity": "sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-ia32": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.34.5.tgz", + "integrity": "sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg==", + "cpu": [ + "ia32" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.34.5.tgz", + "integrity": "sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, "node_modules/@jridgewell/resolve-uri": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.0.tgz", @@ -203,6 +727,23 @@ "integrity": "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==", "dev": true }, + "node_modules/@types/draco3d": { + "version": "1.4.10", + "resolved": "https://registry.npmjs.org/@types/draco3d/-/draco3d-1.4.10.tgz", + "integrity": "sha512-AX22jp8Y7wwaBgAixaSvkoG4M/+PlAcm3Qs4OW8yT9DM4xUpWKeFhLueTAyZF39pviAdcDdeJoACapiAceqNcw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/draco3dgltf": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/@types/draco3dgltf/-/draco3dgltf-1.4.3.tgz", + "integrity": "sha512-JTY574f8xRI9+bOsDajeVSQ/gnIo0q3dt/MAJhNRKWJKdH2TAP3hld+lQ+eQnG9Eb6Ae493EiKi2oDZZpciQgw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/draco3d": "*" + } + }, "node_modules/@types/express": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/@types/express/-/express-5.0.0.tgz", @@ -267,6 +808,12 @@ "@types/node": "*" } }, + "node_modules/@types/ndarray": { + "version": "1.0.14", + "resolved": "https://registry.npmjs.org/@types/ndarray/-/ndarray-1.0.14.tgz", + "integrity": "sha512-oANmFZMnFQvb219SSBIhI1Ih/r4CvHDOzkWyJS/XRqkMrGH5/kaPSA1hQhdIBzouaE+5KpE/f5ylI9cujmckQg==", + "license": "MIT" + }, "node_modules/@types/node": { "version": "16.18.23", "resolved": "https://registry.npmjs.org/@types/node/-/node-16.18.23.tgz", @@ -880,6 +1427,15 @@ "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", "dev": true }, + "node_modules/cwise-compiler": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/cwise-compiler/-/cwise-compiler-1.1.3.tgz", + "integrity": "sha512-WXlK/m+Di8DMMcCjcWr4i+XzcQra9eCdXIJrgh4TUgh0pIS/yJduLxS9JgefsHJ/YVLdgPtXm9r62W92MvanEQ==", + "license": "MIT", + "dependencies": { + "uniq": "^1.0.0" + } + }, "node_modules/debug": { "version": "4.3.7", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz", @@ -950,6 +1506,15 @@ "npm": "1.2.8000 || >= 1.4.16" } }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, "node_modules/dezalgo": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/dezalgo/-/dezalgo-1.0.4.tgz", @@ -969,6 +1534,12 @@ "node": ">=0.3.1" } }, + "node_modules/draco3dgltf": { + "version": "1.5.7", + "resolved": "https://registry.npmjs.org/draco3dgltf/-/draco3dgltf-1.5.7.tgz", + "integrity": "sha512-LeqcpmoHIyYUi0z70/H3tMkGj8QhqVxq6FJGPjlzR24BNkQ6jyMheMvFKJBI0dzGZrEOUyQEmZ8axM1xRrbRiw==", + "license": "Apache-2.0" + }, "node_modules/dunder-proto": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", @@ -1501,6 +2072,12 @@ "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, + "node_modules/iota-array": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/iota-array/-/iota-array-1.0.0.tgz", + "integrity": "sha512-pZ2xT+LOHckCatGQ3DcG/a+QuEqvoxqkiL7tvE8nn3uuu+f6i1TtpB5/FtWFbxUuVr5PZCx8KskuGatbJDXOWA==", + "license": "MIT" + }, "node_modules/ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -1521,6 +2098,12 @@ "node": ">=8" } }, + "node_modules/is-buffer": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", + "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==", + "license": "MIT" + }, "node_modules/is-extglob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", @@ -1604,6 +2187,12 @@ "node": ">= 0.6" } }, + "node_modules/ktx-parse": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/ktx-parse/-/ktx-parse-1.1.0.tgz", + "integrity": "sha512-mKp3y+FaYgR7mXWAbyyzpa/r1zDWeaunH+INJO4fou3hb45XuNSwar+7llrRyvpMWafxSIi99RNFJ05MHedaJQ==", + "license": "MIT" + }, "node_modules/locate-path": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", @@ -1671,6 +2260,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/meshoptimizer": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/meshoptimizer/-/meshoptimizer-1.0.1.tgz", + "integrity": "sha512-Vix+QlA1YYT3FwmBBZ+49cE5y/b+pRrcXKqGpS5ouh33d3lSp2PoTpCw19E0cKDFWalembrHnIaZetf27a+W2g==", + "license": "MIT" + }, "node_modules/methods": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", @@ -1839,6 +2434,47 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" }, + "node_modules/ndarray": { + "version": "1.0.19", + "resolved": "https://registry.npmjs.org/ndarray/-/ndarray-1.0.19.tgz", + "integrity": "sha512-B4JHA4vdyZU30ELBw3g7/p9bZupyew5a7tX1Y/gGeF2hafrPaQZhgrGQfsvgfYbgdFZjYwuEcnaobeM/WMW+HQ==", + "license": "MIT", + "dependencies": { + "iota-array": "^1.0.0", + "is-buffer": "^1.0.2" + } + }, + "node_modules/ndarray-lanczos": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/ndarray-lanczos/-/ndarray-lanczos-0.3.0.tgz", + "integrity": "sha512-5kBmmG3Zvyj77qxIAC4QFLKuYdDIBJwCG+DukT6jQHNa1Ft74/hPH1z5mbQXeHBt8yvGPBGVrr3wEOdJPYYZYg==", + "license": "MIT", + "dependencies": { + "@types/ndarray": "^1.0.11", + "ndarray": "^1.0.19" + } + }, + "node_modules/ndarray-ops": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/ndarray-ops/-/ndarray-ops-1.2.2.tgz", + "integrity": "sha512-BppWAFRjMYF7N/r6Ie51q6D4fs0iiGmeXIACKY66fLpnwIui3Wc3CXiD/30mgLbDjPpSLrsqcp3Z62+IcHZsDw==", + "license": "MIT", + "dependencies": { + "cwise-compiler": "^1.0.0" + } + }, + "node_modules/ndarray-pixels": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ndarray-pixels/-/ndarray-pixels-5.0.1.tgz", + "integrity": "sha512-IBtrpefpqlI8SPDCGjXk4v5NV5z7r3JSuCbfuEEXaM0vrOJtNGgYUa4C3Lt5H+qWdYF4BCPVFsnXhNC7QvZwkw==", + "license": "MIT", + "dependencies": { + "@types/ndarray": "^1.0.14", + "ndarray": "^1.0.19", + "ndarray-ops": "^1.2.2", + "sharp": "^0.34.0" + } + }, "node_modules/negotiator": { "version": "0.6.3", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", @@ -2123,6 +2759,12 @@ "node": ">=0.10.0" } }, + "node_modules/property-graph": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/property-graph/-/property-graph-4.0.0.tgz", + "integrity": "sha512-I0hojAJfTbSCZy3y6xyK29eayxo14v1bj1VPiDkHjTdz33SV6RdfMz2AHnf4ai62Vng2mN5GkaKahkooBIo9gA==", + "license": "MIT" + }, "node_modules/proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", @@ -2235,6 +2877,18 @@ "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==" }, + "node_modules/semver": { + "version": "7.7.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", + "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/send": { "version": "0.19.0", "resolved": "https://registry.npmjs.org/send/-/send-0.19.0.tgz", @@ -2312,6 +2966,50 @@ "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==" }, + "node_modules/sharp": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.34.5.tgz", + "integrity": "sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg==", + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "@img/colour": "^1.0.0", + "detect-libc": "^2.1.2", + "semver": "^7.7.3" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-darwin-arm64": "0.34.5", + "@img/sharp-darwin-x64": "0.34.5", + "@img/sharp-libvips-darwin-arm64": "1.2.4", + "@img/sharp-libvips-darwin-x64": "1.2.4", + "@img/sharp-libvips-linux-arm": "1.2.4", + "@img/sharp-libvips-linux-arm64": "1.2.4", + "@img/sharp-libvips-linux-ppc64": "1.2.4", + "@img/sharp-libvips-linux-riscv64": "1.2.4", + "@img/sharp-libvips-linux-s390x": "1.2.4", + "@img/sharp-libvips-linux-x64": "1.2.4", + "@img/sharp-libvips-linuxmusl-arm64": "1.2.4", + "@img/sharp-libvips-linuxmusl-x64": "1.2.4", + "@img/sharp-linux-arm": "0.34.5", + "@img/sharp-linux-arm64": "0.34.5", + "@img/sharp-linux-ppc64": "0.34.5", + "@img/sharp-linux-riscv64": "0.34.5", + "@img/sharp-linux-s390x": "0.34.5", + "@img/sharp-linux-x64": "0.34.5", + "@img/sharp-linuxmusl-arm64": "0.34.5", + "@img/sharp-linuxmusl-x64": "0.34.5", + "@img/sharp-wasm32": "0.34.5", + "@img/sharp-win32-arm64": "0.34.5", + "@img/sharp-win32-ia32": "0.34.5", + "@img/sharp-win32-x64": "0.34.5" + } + }, "node_modules/side-channel": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz", @@ -2585,6 +3283,13 @@ "node": ">=0.3.1" } }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD", + "optional": true + }, "node_modules/tsscmp": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/tsscmp/-/tsscmp-1.0.6.tgz", @@ -2631,6 +3336,12 @@ "node": ">=0.8.0" } }, + "node_modules/uniq": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/uniq/-/uniq-1.0.1.tgz", + "integrity": "sha512-Gw+zz50YNKPDKXs+9d+aKAjVwpjNwqzvNpLigIruT4HA9lMZNdMqs9x07kKHB/L9WRzqp4+DlTU5s4wG2esdoA==", + "license": "MIT" + }, "node_modules/unpipe": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", @@ -2846,6 +3557,227 @@ } } }, + "@emnapi/runtime": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.8.1.tgz", + "integrity": "sha512-mehfKSMWjjNol8659Z8KxEMrdSJDDot5SXMq00dM8BN4o+CLNXQ0xH2V7EchNHV4RmbZLmmPdEaXZc5H2FXmDg==", + "optional": true, + "requires": { + "tslib": "^2.4.0" + } + }, + "@gltf-transform/core": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/@gltf-transform/core/-/core-4.3.0.tgz", + "integrity": "sha512-ZeaQfszGJ9LYwELszu45CuDQCsE26lJNNe36FVmN8xclaT6WDdCj7fwGpQXo0/l/YgAVAHX+uO7YNBW75/SRYw==", + "requires": { + "property-graph": "^4.0.0" + } + }, + "@gltf-transform/extensions": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/@gltf-transform/extensions/-/extensions-4.3.0.tgz", + "integrity": "sha512-XDAjQPYVMHa/VDpSbfCBwI+/1muwRJCaXhUpLgnUzAjn0D//PgvIAcbNm1EwBl3LIWBSwjDUCn2LiMAjp+aXVw==", + "requires": { + "@gltf-transform/core": "^4.3.0", + "ktx-parse": "^1.0.1" + } + }, + "@gltf-transform/functions": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/@gltf-transform/functions/-/functions-4.3.0.tgz", + "integrity": "sha512-FZggHVgt3DHOezgESBrf2vDzuD2FYQYaNT2sT/aP316SIwhuiIwby3z7rhV9joDvWqqUaPkf1UmkjlOaY9riSQ==", + "requires": { + "@gltf-transform/core": "^4.3.0", + "@gltf-transform/extensions": "^4.3.0", + "ktx-parse": "^1.0.1", + "ndarray": "^1.0.19", + "ndarray-lanczos": "^0.3.0", + "ndarray-pixels": "^5.0.1" + } + }, + "@img/colour": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.0.0.tgz", + "integrity": "sha512-A5P/LfWGFSl6nsckYtjw9da+19jB8hkJ6ACTGcDfEJ0aE+l2n2El7dsVM7UVHZQ9s2lmYMWlrS21YLy2IR1LUw==" + }, + "@img/sharp-darwin-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.34.5.tgz", + "integrity": "sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w==", + "optional": true, + "requires": { + "@img/sharp-libvips-darwin-arm64": "1.2.4" + } + }, + "@img/sharp-darwin-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.34.5.tgz", + "integrity": "sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw==", + "optional": true, + "requires": { + "@img/sharp-libvips-darwin-x64": "1.2.4" + } + }, + "@img/sharp-libvips-darwin-arm64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.2.4.tgz", + "integrity": "sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g==", + "optional": true + }, + "@img/sharp-libvips-darwin-x64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.2.4.tgz", + "integrity": "sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg==", + "optional": true + }, + "@img/sharp-libvips-linux-arm": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.2.4.tgz", + "integrity": "sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A==", + "optional": true + }, + "@img/sharp-libvips-linux-arm64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.2.4.tgz", + "integrity": "sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw==", + "optional": true + }, + "@img/sharp-libvips-linux-ppc64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.2.4.tgz", + "integrity": "sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA==", + "optional": true + }, + "@img/sharp-libvips-linux-riscv64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.2.4.tgz", + "integrity": "sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA==", + "optional": true + }, + "@img/sharp-libvips-linux-s390x": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.2.4.tgz", + "integrity": "sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ==", + "optional": true + }, + "@img/sharp-libvips-linux-x64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.2.4.tgz", + "integrity": "sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw==", + "optional": true + }, + "@img/sharp-libvips-linuxmusl-arm64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.2.4.tgz", + "integrity": "sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw==", + "optional": true + }, + "@img/sharp-libvips-linuxmusl-x64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.2.4.tgz", + "integrity": "sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg==", + "optional": true + }, + "@img/sharp-linux-arm": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.34.5.tgz", + "integrity": "sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw==", + "optional": true, + "requires": { + "@img/sharp-libvips-linux-arm": "1.2.4" + } + }, + "@img/sharp-linux-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.34.5.tgz", + "integrity": "sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg==", + "optional": true, + "requires": { + "@img/sharp-libvips-linux-arm64": "1.2.4" + } + }, + "@img/sharp-linux-ppc64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.34.5.tgz", + "integrity": "sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA==", + "optional": true, + "requires": { + "@img/sharp-libvips-linux-ppc64": "1.2.4" + } + }, + "@img/sharp-linux-riscv64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.34.5.tgz", + "integrity": "sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw==", + "optional": true, + "requires": { + "@img/sharp-libvips-linux-riscv64": "1.2.4" + } + }, + "@img/sharp-linux-s390x": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.34.5.tgz", + "integrity": "sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg==", + "optional": true, + "requires": { + "@img/sharp-libvips-linux-s390x": "1.2.4" + } + }, + "@img/sharp-linux-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.34.5.tgz", + "integrity": "sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ==", + "optional": true, + "requires": { + "@img/sharp-libvips-linux-x64": "1.2.4" + } + }, + "@img/sharp-linuxmusl-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.34.5.tgz", + "integrity": "sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg==", + "optional": true, + "requires": { + "@img/sharp-libvips-linuxmusl-arm64": "1.2.4" + } + }, + "@img/sharp-linuxmusl-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.34.5.tgz", + "integrity": "sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q==", + "optional": true, + "requires": { + "@img/sharp-libvips-linuxmusl-x64": "1.2.4" + } + }, + "@img/sharp-wasm32": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.34.5.tgz", + "integrity": "sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw==", + "optional": true, + "requires": { + "@emnapi/runtime": "^1.7.0" + } + }, + "@img/sharp-win32-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.34.5.tgz", + "integrity": "sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g==", + "optional": true + }, + "@img/sharp-win32-ia32": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.34.5.tgz", + "integrity": "sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg==", + "optional": true + }, + "@img/sharp-win32-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.34.5.tgz", + "integrity": "sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw==", + "optional": true + }, "@jridgewell/resolve-uri": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.0.tgz", @@ -2954,6 +3886,21 @@ "integrity": "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==", "dev": true }, + "@types/draco3d": { + "version": "1.4.10", + "resolved": "https://registry.npmjs.org/@types/draco3d/-/draco3d-1.4.10.tgz", + "integrity": "sha512-AX22jp8Y7wwaBgAixaSvkoG4M/+PlAcm3Qs4OW8yT9DM4xUpWKeFhLueTAyZF39pviAdcDdeJoACapiAceqNcw==", + "dev": true + }, + "@types/draco3dgltf": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/@types/draco3dgltf/-/draco3dgltf-1.4.3.tgz", + "integrity": "sha512-JTY574f8xRI9+bOsDajeVSQ/gnIo0q3dt/MAJhNRKWJKdH2TAP3hld+lQ+eQnG9Eb6Ae493EiKi2oDZZpciQgw==", + "dev": true, + "requires": { + "@types/draco3d": "*" + } + }, "@types/express": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/@types/express/-/express-5.0.0.tgz", @@ -3017,6 +3964,11 @@ "@types/node": "*" } }, + "@types/ndarray": { + "version": "1.0.14", + "resolved": "https://registry.npmjs.org/@types/ndarray/-/ndarray-1.0.14.tgz", + "integrity": "sha512-oANmFZMnFQvb219SSBIhI1Ih/r4CvHDOzkWyJS/XRqkMrGH5/kaPSA1hQhdIBzouaE+5KpE/f5ylI9cujmckQg==" + }, "@types/node": { "version": "16.18.23", "resolved": "https://registry.npmjs.org/@types/node/-/node-16.18.23.tgz", @@ -3512,6 +4464,14 @@ "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", "dev": true }, + "cwise-compiler": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/cwise-compiler/-/cwise-compiler-1.1.3.tgz", + "integrity": "sha512-WXlK/m+Di8DMMcCjcWr4i+XzcQra9eCdXIJrgh4TUgh0pIS/yJduLxS9JgefsHJ/YVLdgPtXm9r62W92MvanEQ==", + "requires": { + "uniq": "^1.0.0" + } + }, "debug": { "version": "4.3.7", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz", @@ -3557,6 +4517,11 @@ "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==" }, + "detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==" + }, "dezalgo": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/dezalgo/-/dezalgo-1.0.4.tgz", @@ -3573,6 +4538,11 @@ "integrity": "sha512-uIFDxqpRZGZ6ThOk84hEfqWoHx2devRFvpTZcTHur85vImfaxUbTW9Ryh4CpCuDnToOP1CEtXKIgytHBPVff5A==", "dev": true }, + "draco3dgltf": { + "version": "1.5.7", + "resolved": "https://registry.npmjs.org/draco3dgltf/-/draco3dgltf-1.5.7.tgz", + "integrity": "sha512-LeqcpmoHIyYUi0z70/H3tMkGj8QhqVxq6FJGPjlzR24BNkQ6jyMheMvFKJBI0dzGZrEOUyQEmZ8axM1xRrbRiw==" + }, "dunder-proto": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", @@ -3948,6 +4918,11 @@ "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, + "iota-array": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/iota-array/-/iota-array-1.0.0.tgz", + "integrity": "sha512-pZ2xT+LOHckCatGQ3DcG/a+QuEqvoxqkiL7tvE8nn3uuu+f6i1TtpB5/FtWFbxUuVr5PZCx8KskuGatbJDXOWA==" + }, "ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -3962,6 +4937,11 @@ "binary-extensions": "^2.0.0" } }, + "is-buffer": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", + "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==" + }, "is-extglob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", @@ -4018,6 +4998,11 @@ "tsscmp": "1.0.6" } }, + "ktx-parse": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/ktx-parse/-/ktx-parse-1.1.0.tgz", + "integrity": "sha512-mKp3y+FaYgR7mXWAbyyzpa/r1zDWeaunH+INJO4fou3hb45XuNSwar+7llrRyvpMWafxSIi99RNFJ05MHedaJQ==" + }, "locate-path": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", @@ -4064,6 +5049,11 @@ "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz", "integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==" }, + "meshoptimizer": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/meshoptimizer/-/meshoptimizer-1.0.1.tgz", + "integrity": "sha512-Vix+QlA1YYT3FwmBBZ+49cE5y/b+pRrcXKqGpS5ouh33d3lSp2PoTpCw19E0cKDFWalembrHnIaZetf27a+W2g==" + }, "methods": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", @@ -4194,6 +5184,43 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" }, + "ndarray": { + "version": "1.0.19", + "resolved": "https://registry.npmjs.org/ndarray/-/ndarray-1.0.19.tgz", + "integrity": "sha512-B4JHA4vdyZU30ELBw3g7/p9bZupyew5a7tX1Y/gGeF2hafrPaQZhgrGQfsvgfYbgdFZjYwuEcnaobeM/WMW+HQ==", + "requires": { + "iota-array": "^1.0.0", + "is-buffer": "^1.0.2" + } + }, + "ndarray-lanczos": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/ndarray-lanczos/-/ndarray-lanczos-0.3.0.tgz", + "integrity": "sha512-5kBmmG3Zvyj77qxIAC4QFLKuYdDIBJwCG+DukT6jQHNa1Ft74/hPH1z5mbQXeHBt8yvGPBGVrr3wEOdJPYYZYg==", + "requires": { + "@types/ndarray": "^1.0.11", + "ndarray": "^1.0.19" + } + }, + "ndarray-ops": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/ndarray-ops/-/ndarray-ops-1.2.2.tgz", + "integrity": "sha512-BppWAFRjMYF7N/r6Ie51q6D4fs0iiGmeXIACKY66fLpnwIui3Wc3CXiD/30mgLbDjPpSLrsqcp3Z62+IcHZsDw==", + "requires": { + "cwise-compiler": "^1.0.0" + } + }, + "ndarray-pixels": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ndarray-pixels/-/ndarray-pixels-5.0.1.tgz", + "integrity": "sha512-IBtrpefpqlI8SPDCGjXk4v5NV5z7r3JSuCbfuEEXaM0vrOJtNGgYUa4C3Lt5H+qWdYF4BCPVFsnXhNC7QvZwkw==", + "requires": { + "@types/ndarray": "^1.0.14", + "ndarray": "^1.0.19", + "ndarray-ops": "^1.2.2", + "sharp": "^0.34.0" + } + }, "negotiator": { "version": "0.6.3", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", @@ -4382,6 +5409,11 @@ "xtend": "^4.0.0" } }, + "property-graph": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/property-graph/-/property-graph-4.0.0.tgz", + "integrity": "sha512-I0hojAJfTbSCZy3y6xyK29eayxo14v1bj1VPiDkHjTdz33SV6RdfMz2AHnf4ai62Vng2mN5GkaKahkooBIo9gA==" + }, "proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", @@ -4459,6 +5491,11 @@ "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==" }, + "semver": { + "version": "7.7.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", + "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==" + }, "send": { "version": "0.19.0", "resolved": "https://registry.npmjs.org/send/-/send-0.19.0.tgz", @@ -4531,6 +5568,40 @@ "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==" }, + "sharp": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.34.5.tgz", + "integrity": "sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg==", + "requires": { + "@img/colour": "^1.0.0", + "@img/sharp-darwin-arm64": "0.34.5", + "@img/sharp-darwin-x64": "0.34.5", + "@img/sharp-libvips-darwin-arm64": "1.2.4", + "@img/sharp-libvips-darwin-x64": "1.2.4", + "@img/sharp-libvips-linux-arm": "1.2.4", + "@img/sharp-libvips-linux-arm64": "1.2.4", + "@img/sharp-libvips-linux-ppc64": "1.2.4", + "@img/sharp-libvips-linux-riscv64": "1.2.4", + "@img/sharp-libvips-linux-s390x": "1.2.4", + "@img/sharp-libvips-linux-x64": "1.2.4", + "@img/sharp-libvips-linuxmusl-arm64": "1.2.4", + "@img/sharp-libvips-linuxmusl-x64": "1.2.4", + "@img/sharp-linux-arm": "0.34.5", + "@img/sharp-linux-arm64": "0.34.5", + "@img/sharp-linux-ppc64": "0.34.5", + "@img/sharp-linux-riscv64": "0.34.5", + "@img/sharp-linux-s390x": "0.34.5", + "@img/sharp-linux-x64": "0.34.5", + "@img/sharp-linuxmusl-arm64": "0.34.5", + "@img/sharp-linuxmusl-x64": "0.34.5", + "@img/sharp-wasm32": "0.34.5", + "@img/sharp-win32-arm64": "0.34.5", + "@img/sharp-win32-ia32": "0.34.5", + "@img/sharp-win32-x64": "0.34.5", + "detect-libc": "^2.1.2", + "semver": "^7.7.3" + } + }, "side-channel": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz", @@ -4712,6 +5783,12 @@ } } }, + "tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "optional": true + }, "tsscmp": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/tsscmp/-/tsscmp-1.0.6.tgz", @@ -4738,6 +5815,11 @@ "integrity": "sha512-T9q82TJI9e/C1TAxYvfb16xO120tMVFZrGA3f9/P4424DNu6ypK103y0GPFVa17yotwSyZW5iYXgjYHkGrJW/g==", "optional": true }, + "uniq": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/uniq/-/uniq-1.0.1.tgz", + "integrity": "sha512-Gw+zz50YNKPDKXs+9d+aKAjVwpjNwqzvNpLigIruT4HA9lMZNdMqs9x07kKHB/L9WRzqp4+DlTU5s4wG2esdoA==" + }, "unpipe": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", diff --git a/source/server/package.json b/source/server/package.json index 34d4e70d7..431e1b400 100755 --- a/source/server/package.json +++ b/source/server/package.json @@ -22,7 +22,7 @@ "license": "Apache-2.0", "homepage": "https://github.com/Holusion/ecorpus", "engines": { - "node": ">=16.14" + "node": ">=18.17.0" }, "mocha": { "ui": "bdd", @@ -40,21 +40,27 @@ "spec": "./**/*.test.ts" }, "dependencies": { + "@gltf-transform/core": "^4.3.0", + "@gltf-transform/extensions": "^4.3.0", + "@gltf-transform/functions": "^4.3.0", "body-parser": "^1.20.3", "content-disposition": "^1.0.1", "cookie-parser": "^1.4.7", "cookie-session": "^2.1.0", + "draco3dgltf": "^1.5.7", "express": "^4.21.2", "express-rate-limit": "^7.5.0", "handlebars": "^4.7.8", "i18next": "^24.2.1", "i18next-fs-backend": "^2.6.0", + "meshoptimizer": "^1.0.1", "mime-types": "^2.1.35", "morgan": "^1.10.0", "nodemailer": "^6.9.16", "pg": "^8.16.0", "pg-cursor": "^2.15.0", "source-map-support": "^0.5.21", + "sharp": "^0.34.5", "xml-js": "^1.6.11", "yauzl": "^3.2.0", "yazl": "^3.3.1" @@ -65,6 +71,7 @@ "@types/content-disposition": "^0.5.9", "@types/cookie-parser": "^1.4.8", "@types/cookie-session": "^2.0.49", + "@types/draco3dgltf": "^1.4.3", "@types/express": "^5.0.0", "@types/mime-types": "^2.1.4", "@types/mocha": "^10.0.10", diff --git a/source/server/routes/scenes/index.ts b/source/server/routes/scenes/index.ts index 0fcf39c3e..7901a290d 100644 --- a/source/server/routes/scenes/index.ts +++ b/source/server/routes/scenes/index.ts @@ -39,7 +39,7 @@ router.use((req, res, next)=>{ router.get("/", wrap(getScenes)); router.propfind("/", wrap(handlePropfind)); // additional checks are used in postScenes to allow people to overrite scenes they have write access on - router.post("/", isUser, wrap(handlePostScenes)); +router.post("/", isUser, bodyParser.json(), wrap(handlePostScenes)); //allow POST outside of canRead : overwrite permissions are otherwise checked router.post("/:scene", isCreator, wrap(handlePostScene)); diff --git a/source/server/routes/scenes/post.ts b/source/server/routes/scenes/post.ts index b8b01eca1..6863e89fe 100644 --- a/source/server/routes/scenes/post.ts +++ b/source/server/routes/scenes/post.ts @@ -10,7 +10,7 @@ import uid, { Uid } from "../../utils/uid.js"; import { pipeline } from "stream/promises"; import { Dictionary } from "../../utils/schema/types.js"; -import { ImportSceneResult, parseUserUpload, UserUploadResult } from "../../tasks/handlers/uploads.js"; +import { ImportSceneResult, parseUserUpload, ParsedUserUpload } from "../../tasks/handlers/uploads.js"; import { extractScenesArchive } from "../../tasks/handlers/extractZip.js"; @@ -46,21 +46,23 @@ export async function postRawZipfile(req: Request, res: Response){ const output = await taskScheduler.run({ scene_id: null, user_id: requester.uid, + type: "handlePostScene", + immediate: true, handler: async function handlePostScene({task, context:{vfs, logger}}) :Promise{ const dir = await vfs.createTaskWorkspace(task.task_id); const abs_filepath = path.join(dir, filename); const relPath = vfs.relative(abs_filepath); - logger.log("Write upload artifact to :", relPath); + logger.log("Write upload file to :", relPath); const ws = createWriteStream(abs_filepath); await pipeline( req, ws, ); - logger.log("artifact uploaded to :", relPath); + logger.log("file uploaded to :", relPath); return await taskScheduler.run({ scene_id: null, user_id: requester.uid, - data: {filepath: relPath, size}, + data: {fileLocation: relPath, size}, handler: extractScenesArchive, }); } diff --git a/source/server/routes/services/opensearch.ts b/source/server/routes/services/opensearch.ts index 459afd73d..f16a9d9b2 100644 --- a/source/server/routes/services/opensearch.ts +++ b/source/server/routes/services/opensearch.ts @@ -31,7 +31,7 @@ export function renderOpenSearch(req:Request, res: Response):void{ })); let eTag = createHash("sha256"); - eTag.update(body); + eTag.update(body as any); res.set("Cache-Control", `max-age=${3600*24}, public`); res.set("ETag", "W/"+eTag.digest("base64url")); if(req.fresh){ diff --git a/source/server/routes/tasks/artifacts/get.ts b/source/server/routes/tasks/artifacts/get.ts deleted file mode 100644 index ab1001fd0..000000000 --- a/source/server/routes/tasks/artifacts/get.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { Request, Response } from "express"; -import { getVfs, getUser, getTaskScheduler } from "../../../utils/locals.js"; -import { MethodNotAllowedError, NotImplementedError, UnauthorizedError } from "../../../utils/errors.js"; - -import { isUploadTask } from "./put.js"; -import path from "node:path"; - - - -export async function getUploadTask(req: Request, res: Response){ - const vfs = getVfs(req); - const taskScheduler = getTaskScheduler(req); - const requester = getUser(req)!; - const {id:idString} = req.params; - const id = parseInt(idString); - const task = await taskScheduler.getTask(id); - if(task.fk_user_id !== requester.uid){ - throw new UnauthorizedError(`This task does not belong to this user`); - } - if(!isUploadTask(task)){ - throw new NotImplementedError(`Artifacts download not yet supported for this task`); - } - if(task.status == "initializing"){ - throw new MethodNotAllowedError(`GET not allowed on artifacts that are not yet complete`); - } - const {filename, size} = task.data; - res.sendFile(path.join(vfs.getTaskWorkspace(task.task_id), filename)); -} \ No newline at end of file diff --git a/source/server/routes/tasks/index.ts b/source/server/routes/tasks/index.ts index a4202eac7..ff05eaa15 100644 --- a/source/server/routes/tasks/index.ts +++ b/source/server/routes/tasks/index.ts @@ -1,26 +1,55 @@ -import { Router } from "express"; +import { NextFunction, Request, Response, Router } from "express"; import wrap from "../../utils/wrapAsync.js"; -import { canAdmin, canRead, isCreator, isUser } from "../../utils/locals.js"; +import { canAdmin, canRead, getLocals, getUser, isCreator, isUser } from "../../utils/locals.js"; -import { createTask } from "./post.js"; -import { putUploadTask } from "./artifacts/put.js"; +import { createUserTask } from "./post.js"; +import { putTaskArtifact } from "./task/artifacts/put.js"; import bodyParser from "body-parser"; -import { getUploadTask } from "./artifacts/get.js"; +import { getTaskArtifact } from "./task/artifacts/get.js"; import { getTask } from "./task/get.js"; import { deleteTask } from "./task/delete.js"; +import { UnauthorizedError } from "../../utils/errors.js"; +import { AccessType, toAccessLevel } from "../../auth/UserManager.js"; + +const jsonParser = bodyParser.json(); const router = Router(); router.use("/", isUser); -router.post("/", isCreator, bodyParser.json(), wrap(createTask)); - - -router.put("/artifacts/:id", wrap(putUploadTask)); -router.get("/artifacts/:id", wrap(getUploadTask)); - -router.delete("/task/:id(\\d+)", wrap(deleteTask)); -router.get("/task/:id(\\d+)", wrap(getTask)); +router.post("/", isCreator, jsonParser, wrap(createUserTask)); + + +function taskAccess(name:AccessType){ + const minLevel = toAccessLevel(name); + return function taskAccessMiddleware(req: Request, res: Response, next:NextFunction){ + const { + vfs, + taskScheduler, + userManager, + } = getLocals(req); + const requester = getUser(req)!; + if(!requester) return next(new UnauthorizedError(`Route requires a valid used`)); + + const {id:idString} = req.params; + const id = parseInt(idString); + taskScheduler.getTask(id).then(async (task)=>{ + if(requester.level == "admin") return next(); //Even if requester is admin, check for task existence + else if(task.user_id && task.user_id == requester.uid) return next(); + else if (!task.scene_id || toAccessLevel(await userManager.getAccessRights(task.scene_id, requester.uid)) < minLevel){ + return next(new UnauthorizedError(`Administrative rights are required to delete tasks`)) + }else{ + return next(); + } + }).catch(next); +} +} + + +router.get("/:id(\\d+)", isUser, wrap(getTask)); +router.delete("/:id(\\d+)", taskAccess("admin"), wrap(deleteTask)); +router.put("/:id(\\d+)/artifact", taskAccess("admin"), wrap(putTaskArtifact)); +router.get("/:id(\\d+)/artifact", taskAccess("read"), wrap(getTaskArtifact)); export default router; diff --git a/source/server/routes/tasks/post.ts b/source/server/routes/tasks/post.ts index 465ac1e11..f10b42e3f 100644 --- a/source/server/routes/tasks/post.ts +++ b/source/server/routes/tasks/post.ts @@ -2,29 +2,48 @@ import { Request, Response } from "express"; import { getTaskScheduler, getUser, getVfs } from "../../utils/locals.js"; import { BadRequestError } from "../../utils/errors.js"; -export async function createTask(req: Request, res: Response){ +import * as handlers from "../../tasks/handlers/index.js"; + +function isUserTaskType(t: any): t is keyof typeof handlers{ + return typeof t === "string" && typeof (handlers as any)[t] === "function"; +} + + +/** + * + */ +export async function createUserTask(req: Request, res: Response){ const vfs = getVfs(req); const taskScheduler = getTaskScheduler(req); const requester = getUser(req)!; - let {filename, size} = req.body; - if(!filename){ - throw new BadRequestError(`No filename provided`); + let {type, data, status = "pending"} = req.body; + if(!type || typeof type !=="string"){ + throw new BadRequestError(`No task type provided`); + }else if(!isUserTaskType(type) ){ + throw new BadRequestError(`Unsupported task type: ${type}`); + }else if(["initializing", "pending"].indexOf(status)=== -1){ + throw new BadRequestError(`Invalid task status: ${status}`); } - if(!size || !Number.isInteger(size)){ - throw new BadRequestError(`Bad file size provided`); - } - - const task = await taskScheduler.create({ + //We perform **NO** data validation here, which might be a security hole + //especially if task handlers can't be relied-upon to check their own data + let task = await taskScheduler.create({ scene_id:null, user_id:requester.uid, - type: "userUploads", - status: "initializing", - data: { - size, - filename, - } + type, + data, + status, }); - //Create the workspace immediately - await vfs.createTaskWorkspace(task.task_id); - res.status(200).send(task); -} \ No newline at end of file + + + if(status == "pending"){ + await taskScheduler.runTask({task, handler: handlers[type] as any}); + //We could just refresh task.output from the result of runTask and task.status + //But it's safer to just fetch the whole task again + task = await taskScheduler.getTask(task.task_id); + }else{ + //Create the workspace immediately + await vfs.createTaskWorkspace(task.task_id); + } + + res.status(201).send(task); +} diff --git a/source/server/routes/tasks/task/artifacts/get.ts b/source/server/routes/tasks/task/artifacts/get.ts new file mode 100644 index 000000000..035a7f1e4 --- /dev/null +++ b/source/server/routes/tasks/task/artifacts/get.ts @@ -0,0 +1,29 @@ +import { Request, Response } from "express"; +import { getVfs, getUser, getTaskScheduler } from "../../../../utils/locals.js"; +import { MethodNotAllowedError, NotImplementedError, UnauthorizedError } from "../../../../utils/errors.js"; + +import { isUploadTask } from "./put.js"; +import path from "node:path"; +import { UploadHandlerParams } from "../../../../tasks/handlers/uploads.js"; +import { isArtifactTask } from "../../../../tasks/types.js"; + + + +export async function getTaskArtifact(req: Request, res: Response){ + const vfs = getVfs(req); + const taskScheduler = getTaskScheduler(req); + const requester = getUser(req)!; + const {id:idString} = req.params; + const id = parseInt(idString); + const task = await taskScheduler.getTask(id); + if(task.user_id !== requester.uid){ + throw new UnauthorizedError(`This task does not belong to this user`); + } + if(task.status != 'success'){ + throw new MethodNotAllowedError(`Task status is ${task.status}. GET is only allowed on tasks that have status = "success"`); + } + if(!isArtifactTask(task.output)){ + throw new NotImplementedError(`Artifacts download not supported for task type ${task.type}`); + } + res.sendFile(task.output.fileLocation, {root: vfs.baseDir}); +} \ No newline at end of file diff --git a/source/server/routes/tasks/artifacts/put.test.ts b/source/server/routes/tasks/task/artifacts/put.test.ts similarity index 62% rename from source/server/routes/tasks/artifacts/put.test.ts rename to source/server/routes/tasks/task/artifacts/put.test.ts index bbe00a5c0..b44444685 100644 --- a/source/server/routes/tasks/artifacts/put.test.ts +++ b/source/server/routes/tasks/task/artifacts/put.test.ts @@ -1,16 +1,18 @@ - +import fs from "fs/promises"; import request from "supertest"; -import User from "../../../auth/User.js"; -import UserManager from "../../../auth/UserManager.js"; -import Vfs from "../../../vfs/index.js"; +import User from "../../../../auth/User.js"; +import UserManager from "../../../../auth/UserManager.js"; +import Vfs from "../../../../vfs/index.js"; import { randomBytes, randomInt } from "node:crypto"; +import path from "node:path"; +import { fixturesDir } from "../../../../__test_fixtures/fixtures.js"; -describe("PUT /tasks/artifacts/:id", function(){ +describe("PUT /tasks/:id/artifact", function(){ let vfs :Vfs, userManager :UserManager, user :User, admin :User; let filename: string, size: number, task: number; @@ -27,25 +29,31 @@ describe("PUT /tasks/artifacts/:id", function(){ }); this.beforeEach(async function(){ - filename = randomBytes(4).toString("hex")+".bin"; - size = randomInt(16, 512); - const {body} = await request(this.server).post(`/tasks`) - .auth("alice", "12345678") - .set("Content-Type", "application/json") - .send({filename, size}) - .expect(200); - expect(body).to.have.property("task_id").a("number"); - task = body.task_id; + //Create a task for each test + filename = randomBytes(4).toString("hex")+".bin"; + size = randomInt(16, 512); + const {body} = await request(this.server).post(`/tasks`) + .auth("alice", "12345678") + .set("Content-Type", "application/json") + .send({ + type: "parseUserUpload", + data: {filename, size}, + status: "initializing", + }) + .expect(201); + expect(body).to.have.property("task_id").a("number"); + task = body.task_id; }); it("Can handle a single-chunk upload (no headers)", async function(){ const data = randomBytes(size); - await request(this.server).put(`/tasks/artifacts/${task}`) + await request(this.server).put(`/tasks/${task}/artifact`) .auth("alice", "12345678") .send(data) .expect(201); - const {body} = await request(this.server).get(`/tasks/artifacts/${task}`) + console.log("GET to :", `/tasks/${task}/artifact`) + const {body} = await request(this.server).get(`/tasks/${task}/artifact`) .auth("alice", "12345678") .expect(200); @@ -54,14 +62,14 @@ describe("PUT /tasks/artifacts/:id", function(){ it("Can handle a single-chunk upload (chunk headers)", async function(){ const data = randomBytes(size); - await request(this.server).put(`/tasks/artifacts/${task}`) + await request(this.server).put(`/tasks/${task}/artifact`) .auth("alice", "12345678") .set("Content-Length", size.toString()) .set("Content-Range", `bytes 0-${size-1 /*end is inclusive*/}/${size}`) .send(data) .expect(201); - const {body} = await request(this.server).get(`/tasks/artifacts/${task}`) + const {body} = await request(this.server).get(`/tasks/${task}/artifact`) .auth("alice", "12345678") .expect(200); @@ -75,7 +83,7 @@ describe("PUT /tasks/artifacts/:id", function(){ let chunkSize = Math.floor(size / 4); while(offset < size){ const len = Math.min(size - offset, chunkSize); - await request(this.server).put(`/tasks/artifacts/${task}`) + await request(this.server).put(`/tasks/${task}/artifact`) .auth("alice", "12345678") .set("Content-Length", len.toString()) .set("Content-Range", `bytes ${offset}-${offset+len-1 /*end is inclusive*/}/${size}`) @@ -84,7 +92,7 @@ describe("PUT /tasks/artifacts/:id", function(){ offset += len; } - const {body} = await request(this.server).get(`/tasks/artifacts/${task}`) + const {body} = await request(this.server).get(`/tasks/${task}/artifact`) .auth("alice", "12345678") .expect(200); @@ -93,27 +101,26 @@ describe("PUT /tasks/artifacts/:id", function(){ it("can parse uploaded contents (simple)", async function(){ const data = randomBytes(size); - await request(this.server).put(`/tasks/artifacts/${task}`) + await request(this.server).put(`/tasks/${task}/artifact`) .auth("alice", "12345678") .send(data) .expect(201); - const {body} = await request(this.server).get(`/tasks/task/${task}`) + const {body} = await request(this.server).get(`/tasks/${task}`) .auth("alice", "12345678") .expect(200); expect(body).to.have.property("status", "success"); expect(body).to.have.property("output").to.deep.equal({ - filepath: `artifacts/${task}/${filename}`, - files: [filename], - scenes: [], + fileLocation: `artifacts/${task}/${filename}`, + isModel: false, + mime: "application/octet-stream", }); }); it("rejects out-of-order bytes", async function(){ const data = randomBytes(6); - console.log() - const res = await request(this.server).put(`/tasks/artifacts/${task}`) + const res = await request(this.server).put(`/tasks/${task}/artifact`) .auth("alice", "12345678") .set("Content-Length", '6') .set("Content-Range", `bytes 10-15/${size}`) diff --git a/source/server/routes/tasks/artifacts/put.ts b/source/server/routes/tasks/task/artifacts/put.ts similarity index 80% rename from source/server/routes/tasks/artifacts/put.ts rename to source/server/routes/tasks/task/artifacts/put.ts index fd8303100..266fac41c 100644 --- a/source/server/routes/tasks/artifacts/put.ts +++ b/source/server/routes/tasks/task/artifacts/put.ts @@ -2,14 +2,15 @@ import { Request, Response } from "express"; import parseRange from "range-parser"; import path from "node:path"; import { stat } from "node:fs/promises"; -import { getVfs, getUser, getTaskScheduler } from "../../../utils/locals.js"; -import { BadRequestError, LengthRequiredError, RangeNotSatisfiableError, UnauthorizedError } from "../../../utils/errors.js"; -import { TaskDefinition } from "../../../tasks/types.js"; +import { getVfs, getUser, getTaskScheduler } from "../../../../utils/locals.js"; +import { BadRequestError, LengthRequiredError, RangeNotSatisfiableError, UnauthorizedError } from "../../../../utils/errors.js"; +import { TaskDefinition } from "../../../../tasks/types.js"; import { createWriteStream } from "node:fs"; import { pipeline } from "node:stream/promises"; +import { parseUserUpload, UploadHandlerParams, ParsedUserUpload } from "../../../../tasks/handlers/uploads.js"; -export function isUploadTask(t:TaskDefinition) : boolean{ - return t.type === "userUploads"; +export function isUploadTask(t:TaskDefinition) : t is TaskDefinition{ + return t.type === parseUserUpload.name; } /** @@ -19,23 +20,23 @@ export function isUploadTask(t:TaskDefinition) : boolean{ * * @see {@link https://docs.cloud.google.com/storage/docs/performing-resumable-uploads?hl=fr#chunked-upload Google Cloud Storage: Chunked upload} for a similar feature */ -export async function putUploadTask(req: Request, res: Response){ +export async function putTaskArtifact(req: Request, res: Response){ const vfs = getVfs(req); const taskScheduler = getTaskScheduler(req); const requester = getUser(req)!; const {id:idString} = req.params; const id = parseInt(idString); const task = await taskScheduler.getTask(id); - if(task.fk_user_id !== requester.uid){ + if(task.user_id !== requester.uid){ throw new UnauthorizedError(`This task does not belong to this user`); } if(!isUploadTask(task)){ throw new BadRequestError(`Task ${id} is not a user upload task`); + }else if(!(typeof task.data?.filename === "string" && typeof task.data?.size === "number")){ + throw new BadRequestError(`Invalid task data: ${typeof task.data} ${JSON.stringify(task.data)}`); } - const {filename, size: filesize} = task.data; - const filepath = path.join(vfs.getTaskWorkspace(task.task_id), filename); const contentRange = req.get("Content-Range"); const contentLength = parseInt(req.get("Content-Length")!); @@ -46,13 +47,13 @@ export async function putUploadTask(req: Request, res: Response){ } + //Call this once the upload has completed async function processUpload(){ - await taskScheduler.run({ - task, - handler: createUserUploadParser(), - }) + await taskScheduler.runTask({ task, immediate: true, handler: parseUserUpload as any }); } + const filepath = path.join(vfs.getTaskWorkspace(task.task_id), filename); + if(!contentRange){ const ws = createWriteStream(filepath) await pipeline( @@ -108,6 +109,7 @@ export async function putUploadTask(req: Request, res: Response){ res.set("Range", `bytes=0-${end}/${filesize}`); if(end == filesize - 1){ + //Upload is complete await processUpload(); res.status(201); res.format({ @@ -119,6 +121,7 @@ export async function putUploadTask(req: Request, res: Response){ } }); }else{ + //Partial Content res.status(206); res.format({ "text/plain": ()=>{ @@ -130,7 +133,3 @@ export async function putUploadTask(req: Request, res: Response){ }); } } - -function createUserUploadParser(): import("../../../tasks/types.js").TaskHandler { - throw new Error("Function not implemented."); -} diff --git a/source/server/routes/tasks/task/delete.ts b/source/server/routes/tasks/task/delete.ts index cce5ed104..2936e1925 100644 --- a/source/server/routes/tasks/task/delete.ts +++ b/source/server/routes/tasks/task/delete.ts @@ -17,8 +17,8 @@ export async function deleteTask(req: Request, res: Response){ const task = await taskScheduler.getTask(id); if(requester.level !== "admin" - && task.fk_user_id !== requester.uid - && (await userManager.getAccessRights(task.fk_scene_id, requester.uid)) != "admin" + && task.user_id !== requester.uid + && (await userManager.getAccessRights(task.scene_id, requester.uid)) != "admin" ){ throw new UnauthorizedError(`Administrative rights are required to delete tasks`); } diff --git a/source/server/routes/tasks/task/get.ts b/source/server/routes/tasks/task/get.ts index c912993c9..1849ec050 100644 --- a/source/server/routes/tasks/task/get.ts +++ b/source/server/routes/tasks/task/get.ts @@ -16,8 +16,8 @@ export async function getTask(req: Request, res: Response){ let task = await taskScheduler.getTask(id); if(requester.level !== "admin" - && task.fk_user_id !== requester.uid - && toAccessLevel(await userManager.getAccessRights(task.fk_scene_id, requester.uid)) < toAccessLevel("read") + && task.user_id !== requester.uid + && toAccessLevel(await userManager.getAccessRights(task.scene_id, requester.uid)) < toAccessLevel("read") ){ throw new UnauthorizedError(`Read rights are required to delete tasks`); } diff --git a/source/server/routes/views/index.ts b/source/server/routes/views/index.ts index 8b5cee869..0b0d277f9 100644 --- a/source/server/routes/views/index.ts +++ b/source/server/routes/views/index.ts @@ -1,14 +1,17 @@ import { Router, Request, Response, NextFunction } from "express"; -import { canRead, getHost, canWrite, getSession, getVfs, getUser, isAdministrator, getUserManager, isMemberOrManage, isManage, isEmbed, useTemplateProperties } from "../../utils/locals.js"; +import { canRead, getHost, canWrite, getSession, getVfs, getUser, isAdministrator, getUserManager, isMemberOrManage, isManage, isEmbed, useTemplateProperties, getTaskScheduler } from "../../utils/locals.js"; import wrap from "../../utils/wrapAsync.js"; import path from "path"; import { Scene } from "../../vfs/types.js"; import ScenesVfs from "../../vfs/Scenes.js"; import { qsToBool, qsToInt } from "../../utils/query.js"; import { isUserAtLeast, UserRoles } from "../../auth/User.js"; -import { BadRequestError } from "../../utils/errors.js"; +import { BadRequestError, ForbiddenError } from "../../utils/errors.js"; +import { debuglog } from "util"; +const debug = debuglog("http:views"); + function mapScene(req :Request, {thumb, name, ...s}:Scene):Scene{ if(thumb){ @@ -67,11 +70,59 @@ routes.get("/", wrap(async (req, res)=>{ }); })); -routes.get("/upload", (req, res)=>{ + + + + +routes.get("/upload", wrap(async (req, res)=>{ + const requester = getUser(req); + const taskScheduler = getTaskScheduler(req); + const vfs = getVfs(req); + const {task} = req.query; + //Maybe we shouldn't fail on bad parameters and redirect to a blank page or just ignore them + const ids = [task].flat().filter(t=>typeof t === "string").map(t=>parseInt(t as string)); + if(ids.findIndex(t=>!Number.isInteger(t)) != -1){ + throw new BadRequestError(`Invalid list of tasks :${ids.join(", ")}`); + } + debug("Render previous upload tasks : ", ids); + let tasks = await Promise.all(ids.map(id=> taskScheduler.getTask(id))); + let scenes: Array<{name: string, action: "create"|"update"}|{error: string, action: "error"}> = []; + for(let task of tasks){ + if(!requester || task.user_id !== requester.uid && requester.level != "admin"){ + scenes.push({error: `Can't access results of task ${task.type}#${task.task_id}`, action: "error"}); + } + if(task.status !== "success"){ + console.warn(`Can't report on task ${task.type}#${task.task_id}: status is ${task.status}`); + scenes.push({error: `Task ${task.type}#${task.task_id} [${task.status}]${task.output?.message? " "+task.output.message: ""}`, action: "error"}); + }else if(task.type === "createSceneFromFiles"){ + if(typeof task.output !== "number"){ + console.warn("Unexpected output for %s :", task.type, task.output); + scenes.push({error: `Unexpected output for ${task.type}`, action: "error"}); + continue; + } + const scene = await vfs.getScene(task.output); + scenes.push({name: scene.name, action: "create"}); + }else if(task.type === "extractScenesArchives"){ + if(!Array.isArray(task.output)){ + console.warn("Unexpected output for %s :", task.type, task.output); + scenes.push({error: `Unexpected output for ${task.type}`, action: "error"}); + continue; + } + for(let {action, name } of task.output){ + console.log("Push :", action, name); + scenes.push({action, name}); + } + }else{ + console.warn("Unsupported task type: %s. not an upload task?", task.type); + scenes.push({error: `Unexpected task type: ${task.type} for task #${task.task_id}`, action: "error"}); + } + } + res.render("upload", { title: "eCorpus: Create new scene", + scenes, }); -}) +})) routes.get("/tags", wrap(async (req, res)=>{ const vfs = getVfs(req); diff --git a/source/server/scripts/obj2gltf.py b/source/server/scripts/obj2gltf.py new file mode 100644 index 000000000..c51e01c12 --- /dev/null +++ b/source/server/scripts/obj2gltf.py @@ -0,0 +1,93 @@ +from os import path +from contextlib import redirect_stdout +from sys import argv,stderr +import argparse +import io +import bpy + + +def fail(msg): + print(msg, file=stderr) + exit(1) + +def clean(): + bpy.ops.object.select_all(action='SELECT') + bpy.ops.object.delete() + if len(bpy.data.objects) != 0: + print('Error deleting Blender scene objects', file=stderr) + exit(1) + +def file_name(filepath): + return path.split(filepath)[1] + +def dir_path(filepath): + return path.split(filepath)[0] + +def file_suffix(filepath): + return path.splitext(file_name(filepath))[1] + +def import_func_wrapper(func, filepath): + func(filepath=filepath) + +def import_mesh(filepath): + import_func = { + '.obj': bpy.ops.wm.obj_import, + '.ply': bpy.ops.wm.ply_import, + '.stl': bpy.ops.wm.stl_import, + '.fbx': bpy.ops.import_scene.fbx, + '.blend': bpy.ops.wm.open_mainfile, + } + + stdout = io.StringIO() + with redirect_stdout(stdout): + import_func_wrapper(import_func[file_suffix(filepath)], filepath=filepath) + stdout.seek(0) + return stdout.read() + +if "--" not in argv: + argv = [] # as if no args are passed +else: + argv = argv[argv.index("--") + 1:] +parser = argparse.ArgumentParser(description='Blender mesh file to GLB conversion tool') +parser.add_argument('-i', '--input', help='mesh file to be converted') +parser.add_argument('-o', '--output', help='output GLB file') +parser.add_argument('--backface', action='store_true') +args = parser.parse_args(argv) + +if not (args.input and args.output): + fail('Command line arguments not supplied or inappropriate') + + +clean() + +try: + + stdout = import_mesh(args.input) + print("Imported source file. Starting conversion") + if len(bpy.data.objects) == 0: + # likely invalid file error, not an easy way to capture this from Blender + fail(stdout.replace("\n", "; ")) + + + for obj in bpy.data.objects: + if type(obj.data) != bpy.types.Mesh: + continue + bpy.context.view_layer.objects.active = obj + mesh = obj.data + for f in mesh.polygons: + f.use_smooth = True + mesh.shade_smooth() + + # Disable backface culling + for eachMat in bpy.data.materials: + eachMat.use_backface_culling = args.backface + # Compress later with gltf-transform + bpy.ops.export_scene.gltf( + filepath=args.output, + export_draco_mesh_compression_enable=False + ) + +except Exception as e: + fail(str(e).replace("\n", "; ")) + +print('Successfully converted') \ No newline at end of file diff --git a/source/server/tasks/handlers/extractZip.ts b/source/server/tasks/handlers/extractZip.ts index f8b2970f6..18672a401 100644 --- a/source/server/tasks/handlers/extractZip.ts +++ b/source/server/tasks/handlers/extractZip.ts @@ -1,6 +1,6 @@ import { Readable } from "node:stream"; import { once } from "node:events"; -import { TaskHandlerParams } from "../types.js"; +import { FileArtifact, TaskHandlerParams } from "../types.js"; import yauzl, { Entry, ZipFile } from "yauzl"; import path from "node:path"; import { isUserAtLeast } from "../../auth/User.js"; @@ -10,6 +10,7 @@ import { toAccessLevel } from "../../auth/UserManager.js"; import { getMimeType } from "../../utils/filetypes.js"; import { text } from "node:stream/consumers"; import { finished } from "node:stream/promises"; +import { UploadHandlerParams, ParsedUserUpload, UploadedArchive, UploadedFile } from "./uploads.js"; @@ -21,30 +22,26 @@ interface ImportErrorResult{ name: string; action: "error"; error: HTTPError|Error; - } type ImportSceneResult = ImportSuccessResult|ImportErrorResult; -interface ExtractZipParams{ - /**filepath, relative to Vfs.baseDir */ - filepath: string, -} /** * Analyze an uploaded file and create child tasks accordingly */ -export async function extractScenesArchive({task: {scene_id: scene_id, user_id: user_id, data: {filepath}}, context:{vfs, logger, userManager, tasks}}:TaskHandlerParams):Promise{ - if(!filepath || typeof filepath !== "string") throw new Error(`invalid filepath provided`); - +export async function extractScenesArchive({task: {scene_id: scene_id, user_id: user_id, data: {fileLocation}}, context:{vfs, logger, userManager, tasks}}:TaskHandlerParams):Promise{ + if(!fileLocation || typeof fileLocation !== "string") throw new Error(`invalid fileLocation provided`); + const filepath = vfs.absolute(fileLocation); if(!user_id) throw new Error(`This task requires an authenticated user`); const requester = await userManager.getUserById(user_id); let zipError: Error; logger.debug("Open Zip file"); - const zip = await new Promise((resolve,reject)=>yauzl.open(path.join(vfs.baseDir, filepath!), {lazyEntries: true, autoClose: true}, (err, zip)=>(err?reject(err): resolve(zip)))); + const zip = await new Promise((resolve,reject)=>yauzl.open(filepath, {lazyEntries: true, autoClose: true}, (err, zip)=>(err?reject(err): resolve(zip)))); const openZipEntry = (record:Entry)=> new Promise((resolve, reject)=>zip.openReadStream(record, (err, rs)=>(err?reject(err): resolve(rs)))); logger.debug("Open database transaction"); + const ts = Date.now(); const results = await vfs.isolate(async (vfs)=>{ //Directory entries are optional in a zip file so we should handle their absence //We do this by maintaining a Map of scenes, and for each scene a Set of files @@ -85,9 +82,16 @@ export async function extractScenesArchive({task: {scene_id: scene_id, user_id: } scenes.set(scene, {folders: new Set(), ...result}); } - if(has_errors) return; - const _s = scenes.get(scene); + //Don't create the files if any scene is rejected: The transaction will be reverted anyways + if(has_errors) return; + + const _s = scenes.get(scene); //having a scene not registered in this map is not supposed to happen. Something would be seriously wrong if(!_s || _s.action === "error") throw new Error(`Scene ${scene} wasn't properly checked for permissions`); + + + + // Proceed with content creation. Start with folders + // We have to manually re-create the structure because folder entries are optional in zip archives const { folders } = _s; if (!name) return; let dirpath = ""; @@ -122,8 +126,7 @@ export async function extractScenesArchive({task: {scene_id: scene_id, user_id: let mime = getMimeType(name); if (mime.startsWith('text/')){ await vfs.writeDoc(await text(rs), {user_id: requester.uid, scene, name, mime}); - } - else { + } else { await vfs.writeFile(rs, {user_id: requester.uid, scene, name, mime}); } } @@ -138,7 +141,7 @@ export async function extractScenesArchive({task: {scene_id: scene_id, user_id: }); }); zip.readEntry(); - logger.debug("Start extracting zip"); + logger.debug("Start extracting zip entries"); await once(zip, "close"); if(zipError){ logger.error("Zip extraction encountered an error. This is most probably due to an invalid zip"); @@ -155,14 +158,50 @@ export async function extractScenesArchive({task: {scene_id: scene_id, user_id: throw new UnauthorizedError( `Multiple unauthorized scenes : ${errors.map(r=>r.name).join(",")}` ); - } else{ + } else { throw new InternalError(`Mixed errors : ${errors.map(r=>r.error.message).join(", ")}`) } } }else{ - logger.debug("zip file closed"); + logger.debug("zip file closed successfully. Running database triggers."); return results as ImportSuccessResult[]; } }); + + logger.log("Database transaction took %dms", Date.now()- ts); return results; }; + +export function isUploadedArchive(t: UploadedFile): t is UploadedArchive { + return t.mime == "application/zip" && (t as any).scenes?.length; +} + +export async function extractScenesArchives({ context: { tasks, logger }, task: { data: { tasks: source_ids } } }: TaskHandlerParams<{ tasks: number[]; }>): Promise { + if (!source_ids.length) throw new BadRequestError(`This task requires at least one source file`); + + for (const task_id of source_ids) { + if (!Number.isInteger(task_id)) throw new BadRequestError(`Invalid source task id: ${task_id}`); + } + const source_tasks = await Promise.all(source_ids.map(id => tasks.getTask(id))); + const failed_tasks = source_tasks.filter(t => t.status !== "success"); + const invalid_outputs = source_tasks.filter(t => !isUploadedArchive(t.output)); + if (failed_tasks.length) throw new BadRequestError(`Source task${1 < failed_tasks.length ? "s" : ""} ${failed_tasks.map(t => t.task_id).join(", ")} has not completed successfully`); + if (invalid_outputs.length) throw new BadRequestError(`Source task${1 < invalid_outputs.length ? "s" : ""} ${invalid_outputs.map(t => t.task_id).join(", ")} did not output a zip file`); + + const archives = source_tasks.map(t => t.output as UploadedArchive); + // Unfortunately here it's possible to have partial failures if we have more than one file. + // it could be prevented with async-context support for database transactions + if(1 < archives.length) logger.warn(`Importing more than once archive file. Partial failures are possible`); + + let results = []; + for (let archive of archives) { + results.push(...await tasks.run({ + handler: extractScenesArchive, + data: { + fileLocation: archive.fileLocation, + } + })); + } + return results; +} + diff --git a/source/server/tasks/handlers/index.ts b/source/server/tasks/handlers/index.ts index bb952b4c3..32c1f6887 100644 --- a/source/server/tasks/handlers/index.ts +++ b/source/server/tasks/handlers/index.ts @@ -1,5 +1,4 @@ -/** - * Gather all handlers that can be called from user-created tasks - */ -export {processUploadedFiles, parseUserUpload} from "./uploads.js"; \ No newline at end of file +export {createSceneFromFiles, parseUserUpload} from "./uploads.js"; + +export {extractScenesArchives} from "./extractZip.js"; \ No newline at end of file diff --git a/source/server/tasks/handlers/inspectGlb.ts b/source/server/tasks/handlers/inspectGlb.ts new file mode 100644 index 000000000..1c742b9cc --- /dev/null +++ b/source/server/tasks/handlers/inspectGlb.ts @@ -0,0 +1,17 @@ + +import type { FileArtifact, TaskHandlerParams } from '../types.js'; + + + +/** + * Parse a glb file to gather its main attributes + * @param param0 + * @returns + */ +export async function inspectGlb({context: {vfs}, task: {data: {fileLocation}}}:TaskHandlerParams){ + const {io} = await import("../../utils/gltf/io.js"); + const {inspectDocument} = await import("../../utils/gltf/inspect.js"); + const document = await io.read(vfs.absolute(fileLocation)); // → Document + return inspectDocument(document); +} + diff --git a/source/server/tasks/handlers/optimizeGlb.ts b/source/server/tasks/handlers/optimizeGlb.ts new file mode 100644 index 000000000..7ade07a4e --- /dev/null +++ b/source/server/tasks/handlers/optimizeGlb.ts @@ -0,0 +1,139 @@ +import path from "node:path"; +import { dedup, flatten, getSceneVertexCount, join, meshopt, prune, resample, simplify, sparse, VertexCountMethod, weld } from '@gltf-transform/functions'; + +import { toktx } from '../../utils/gltf/toktx.js'; + +import {type ProcessFileParams, type TaskHandlerParams, type ITaskLogger, FileArtifact} from "../types.js"; + +import {io} from '../../utils/gltf/io.js'; +import { MeshoptEncoder, MeshoptSimplifier } from 'meshoptimizer'; +import { TDerivativeQuality } from "../../utils/schema/model.js"; + +export interface TransformGlbParams{ + logger: ITaskLogger; + preset: TDerivativeQuality; + tmpdir: string; + /** Whether to resize textures to match maxSize */ + resize: boolean; +} + + +interface PresetSettings{ + ratio: number, + error: number, + etc1s_quality: number, + maxSize: number, +} + +function getPreset(quality: TDerivativeQuality): PresetSettings { + return { + "Thumb": {ratio: 0, error: 0.01, etc1s_quality: 128, maxSize: 512}, + "Low": {ratio: 1/4, error: 0.005, etc1s_quality: 180, maxSize: 1024}, + "Medium": {ratio: 1/2, error: 0.001, etc1s_quality: 220, maxSize: 2048}, + "High": {ratio: 1, error: 0.0001, etc1s_quality: 240, maxSize: 4096}, + "Highest": {ratio: 1, error: 0, etc1s_quality: 250, maxSize: 8192}, + "AR": {ratio: 0, error: 0.001, etc1s_quality: 180, maxSize: 2048}, + }[quality]; +} + + +export async function optimizeGlb({task: {task_id, data:{fileLocation, preset: presetName}}, context:{ vfs, logger, config }}:TaskHandlerParams):Promise{ + if(!fileLocation) throw new Error(`A file is required for this task to run`); + + const resize = true; //Might make it configurable + + //Takes a glb file as input, outputs an optimized file + //It's not yet clear if the output file's path is determined beforehand or generated as an output + let tmpdir = await vfs.createTaskWorkspace(task_id); + const inputFile = vfs.absolute(fileLocation); + let outputFile = path.join(tmpdir, path.basename(inputFile, ".glb")+".glb"); + + logger.log("Optimize with preset %s using gltf-transform", presetName); + logger.debug("Input file:", inputFile); + + + const document = await io.read(inputFile); // → Document + + document.setLogger({...logger, info: logger.log}); + + const root = document.getRoot(); + const scene = root.getDefaultScene(); + if(!scene) throw new Error("Empty glb (no root scene)"); + + async function time(name: string, p: Promise):Promise{ + const t = Date.now(); + let res = await p; + logger.debug(`${name.padEnd(27, ' ')} ${Date.now() - t}ms`); + return res; + } + + /** + * Preset and heuristics + */ + let preset = getPreset(presetName); + + //FIXME : Don't touch geometry if it's already compressed using draco or meshopt? + logger.log("Optimize geometry"); + + + await time("Flatten", document.transform(flatten())); + + await time("Join", document.transform(join())); + + await time("Weld", document.transform(weld())); + + let vertexCount = getSceneVertexCount(root.getDefaultScene()!, VertexCountMethod.UPLOAD); + + await time("Simplify", document.transform(simplify({ + error:preset.error, + ratio: Math.min(1, preset.ratio, (preset.ratio*1000000/vertexCount)), + lockBorder: true, + simplifier: MeshoptSimplifier, + }))); + + await time("Resample", document.transform(resample())); + + await time("Sparse", document.transform(sparse())); + + await time("Compress meshs", document.transform(meshopt({ + ...preset, + encoder: MeshoptEncoder, + level: "medium", + }))); + + + //Remove draco extension as it is now unused + let ext_draco = root.listExtensionsUsed().find(e=> e.extensionName === 'KHR_draco_mesh_compression'); + ext_draco?.dispose(); + + logger.log(("Optimize textures")) + /// Textures + /** @fixme we should handle textures resize fallback when toktx is not available */ + try{ + await time("Compress ORM textures",document.transform(toktx({ + mode: "uastc", + slots: /^(normal|occlusion|metallicRoughness)/, + tmpdir, + maxSize: resize? preset.maxSize: undefined, + }))); + + await time("Compress Color textures",document.transform(toktx({ + mode: "etc1s", + quality: preset.etc1s_quality, + slots: /^baseColor/, + tmpdir, + maxSize: resize? preset.maxSize: undefined, + }))); + //Remove webp extension as it is now unused + let ext_webp = root.listExtensionsUsed().find(e=> e.extensionName === 'EXT_texture_webp'); + ext_webp?.dispose(); + }catch(e){ + logger.warn("Couldn't compress textures to KTX: ", e); + } + + logger.log("Output file uses extensions:", root.listExtensionsUsed().map(e=>e.extensionName)); + await io.write(outputFile, document); + + //Resize textures + return {fileLocation: vfs.relative(outputFile)}; +}; diff --git a/source/server/tasks/handlers/runBlenderScript.ts b/source/server/tasks/handlers/runBlenderScript.ts new file mode 100644 index 000000000..160fe33f5 --- /dev/null +++ b/source/server/tasks/handlers/runBlenderScript.ts @@ -0,0 +1,28 @@ +import path from "node:path"; +import { TaskHandlerParams } from "../types.js"; +import { taskRun } from "../../utils/exec.js"; + + +/** + * + * @param script absolute path to python script to load + * @param scriptArgs script arguments + * @param params Spawn parameters + * @returns + */ +export async function runBlenderScript({context: {config, logger, signal}, task: {data: {script, args}}}:TaskHandlerParams<{script: string, args: string[]}>){ + + return await taskRun("blender", [ + "--background", + "--offline-mode", + "--factory-startup", + "--threads", "1", + "--addons", "io_scene_gltf2", + "--python", path.isAbsolute(script)? script: path.join(config.scripts_dir, script), + "--", + ...args + ], { + logger, + signal, + }); +} \ No newline at end of file diff --git a/source/server/tasks/handlers/toGlb.ts b/source/server/tasks/handlers/toGlb.ts new file mode 100644 index 000000000..16759e3dc --- /dev/null +++ b/source/server/tasks/handlers/toGlb.ts @@ -0,0 +1,21 @@ +import path from "node:path"; +import { FileArtifact, TaskHandlerParams } from "../types.js"; +import { runBlenderScript } from "./runBlenderScript.js"; + + +export async function toGlb({context: {tasks, vfs, logger}, task:{task_id, data:{fileLocation}}}:TaskHandlerParams){ + const dir = await vfs.createTaskWorkspace(task_id); + const dest = path.join(dir, path.basename(fileLocation, path.extname(fileLocation))+".glb"); + logger.log("Transform %s to %s", fileLocation, dest); + await tasks.run({ + data: { + script: "obj2gltf.py", + args: [ + "-i", vfs.absolute(fileLocation), + "-o", dest, + ] + }, + handler: runBlenderScript, + }); + return vfs.relative(dest); +} \ No newline at end of file diff --git a/source/server/tasks/handlers/uploads.ts b/source/server/tasks/handlers/uploads.ts index 93927a88c..685bf1859 100644 --- a/source/server/tasks/handlers/uploads.ts +++ b/source/server/tasks/handlers/uploads.ts @@ -1,14 +1,21 @@ import { once } from "node:events"; -import { stat } from "node:fs/promises"; -import path, { extname } from "node:path"; +import fs from "node:fs/promises"; +import path from "node:path"; import yauzl, { ZipFile } from "yauzl"; import { isUserAtLeast } from "../../auth/User.js"; import { toAccessLevel } from "../../auth/UserManager.js"; import { parseFilepath, isMainSceneFile } from "../../utils/archives.js"; -import { TaskHandlerParams } from "../types.js"; -import { BadRequestError, UnauthorizedError } from "../../utils/errors.js"; -import { extractScenesArchive, ImportSuccessResult } from "./extractZip.js"; +import { FileArtifact, TaskDefinition, TaskHandlerParams } from "../types.js"; +import { BadRequestError, InternalError } from "../../utils/errors.js"; +import getDefaultDocument from "../../utils/schema/default.js"; +import { parse_glb } from "../../utils/glTF.js"; +import uid from "../../utils/uid.js"; +import { toGlb } from "./toGlb.js"; +import { getMimeType, isModelType, readMagicBytes } from "../../utils/filetypes.js"; +import { TDerivativeQuality, TDerivativeUsage } from "../../utils/schema/model.js"; +import { optimizeGlb } from "./optimizeGlb.js"; +import { IDocument } from "../../utils/schema/document.js"; @@ -23,50 +30,55 @@ export interface UploadHandlerParams{ size: number; } -export interface UserUploadResult{ - filepath: string; +export interface UploadedFile extends FileArtifact{ + mime: string; +} + +export interface UploadedArchive extends UploadedFile { + mime: "application/zip"; files: string[]; scenes: ImportSceneResult[]; } -/** - * Inspect a user-uploaded file to detect its contents - * @param param0 - * @returns - */ -export async function parseUserUpload({task:{task_id, user_id, data:{filename, size}}, context: {vfs, userManager, logger}}:TaskHandlerParams):Promise{ +export interface UploadedBinaryModel extends UploadedFile { + mime: "model/gltf-binary"; + isModel: boolean; + name?: string; + byteSize: number; + numFaces: number; + imageSize: number; + bounds: Awaited>["bounds"]; +} - let files :string[] = []; - logger.debug("Requester :", user_id); - const requester = await userManager.getUserById(user_id); - const filepath = path.join(vfs.getTaskWorkspace(task_id), filename); - logger.debug(`Checking size of uploaded file ${filepath}`); - let diskSize: number; - try{ - const stats= await stat(filepath); - diskSize = stats.size; - }catch(e:any){ - if(e.code === "ENOENT"){ - logger.error(`File ${filepath} does not exist. Maybe it wasn't uploaded properly?`); - } - throw e; - } - if(diskSize != size){ - throw new Error(`Expected a file of size ${size}, found ${diskSize}`); - } +export interface UploadedUsdModel extends UploadedFile{ + mime: "model/vnd.usdz+zip"; +} - const ext = extname(filename).toLowerCase(); - if(ext == ".zip"){ - logger.debug(`Open ${filename} to list entries`); - let zip = await new Promise((resolve,reject)=>yauzl.open(filepath, {lazyEntries: false, autoClose: true}, (err, zip)=>(err?reject(err): resolve(zip)))); - zip.on("entry", (record)=>{ - files.push(record.fileName); - }); - await once(zip, "close"); - logger.debug(`Found ${files.length} entries in zip`); - }else{ - files.push(filename); - } +export interface UploadedSource extends UploadedFile{ + isModel: boolean; +} + +export type ParsedUserUpload = UploadedArchive|UploadedBinaryModel|UploadedSource; + + +function isUploadedFile(output: any): output is UploadedFile{ + return typeof output === "object" && typeof output.fileLocation === "string" && typeof output.mime === "string"; +} + +function isUploadedBinaryModel(output: UploadedFile): output is UploadedBinaryModel{ + return isUploadedFile(output) && output.mime == "model/gltf-binary" +} + +async function parseUploadedArchive({task:{task_id, user_id, data:{fileLocation}}, context: {vfs, userManager, logger}}:TaskHandlerParams):Promise{ + const requester = await userManager.getUserById(user_id); + const filename = path.basename(fileLocation); + let files :string[] = []; + logger.debug(`Open ${fileLocation} to list entries`); + let zip = await new Promise((resolve,reject)=>yauzl.open(vfs.absolute(fileLocation), {lazyEntries: false, autoClose: true}, (err, zip)=>(err?reject(err): resolve(zip)))); + zip.on("entry", (record)=>{ + files.push(record.fileName); + }); + await once(zip, "close"); let scenes :ImportSceneResult[] = []; for(let file of files){ @@ -93,65 +105,283 @@ export async function parseUserUpload({task:{task_id, user_id, data:{filename, s } scenes.push({name: scene, action, error: error!}); } - // @FIXME maybe we should already delete the file if it has errors? - // It depends on the behaviour we expect of a "partial success" zip upload. - return { - filepath: vfs.relative(filepath), + mime: "application/zip", + fileLocation, files: files, scenes - } satisfies UserUploadResult; + }; +} + + +async function parseUploadedModel({task: {data: {fileLocation}}, context: {logger, vfs}}:TaskHandlerParams):Promise{ + const filepath = vfs.absolute(fileLocation); + logger.debug("Check mime type of "+fileLocation); + const mime = await readMagicBytes(filepath); + if(mime !== "model/gltf-binary"){ + throw new InternalError("This does not look like a GLB file"); + } + const meta = await parse_glb(filepath); + logger.log(`Parsed glb with ${meta.meshes.length} models`); + logger.warn("Using placeholder imageSize of 8192: Not compatible with LOD mode"); + return { + fileLocation, + mime, + isModel: true, + name: meta.meshes.find(m=>m.name)?.name, + bounds: meta.bounds, + byteSize: meta.byteSize, + numFaces: meta.meshes.reduce((acc, m)=> acc+m.numFaces, 0), + imageSize: 8192 + } +} + +/** + * Inspect a user-uploaded file to detect its contents + * @param param0 + * @returns + */ +export async function parseUserUpload({task:{task_id, user_id, data:{filename, size}}, context: {vfs, tasks, userManager, logger}}:TaskHandlerParams):Promise{ + const fileLocation = vfs.relative(path.join(vfs.getTaskWorkspace(task_id), filename)); + + logger.debug(`Checking size of uploaded file ${fileLocation}`); + let diskSize: number; + try{ + const stats= await fs.stat(vfs.absolute(fileLocation)); + diskSize = stats.size; + }catch(e:any){ + if(e.code === "ENOENT"){ + logger.error(`File ${fileLocation} does not exist. Maybe it wasn't uploaded properly?`); + } + throw e; + } + if(diskSize != size){ + throw new Error(`Expected a file of size ${size}, found ${diskSize}`); + } + + + const mime = getMimeType(filename); + + if(mime === "application/zip"){ + return await tasks.run({ + data: { fileLocation }, + handler: parseUploadedArchive, + }); + }else if (mime == "model/gltf-binary"){ + return await tasks.run({ + data: {fileLocation}, + handler: parseUploadedModel, + }); + }else{ + return { + mime, + fileLocation, + isModel: isModelType(filename), + } + } + + // @FIXME maybe we should already delete the file if it has errors? + // It depends on the behaviour we expect of a "partial success" zip upload. } export interface ProcessUploadedFilesParams{ - files: UserUploadResult[]; - name?: string; - lang?: string; + tasks: number[]; + name: string; + language: string; + options: { + optimize?:boolean; + } +} + + + +const sceneLanguages = ["EN", "ES", "DE", "NL", "JA", "FR", "HAW"] as const; +type SceneLanguage = typeof sceneLanguages[number]; +function isSceneLanguage(l:any) :l is SceneLanguage{ + return sceneLanguages.indexOf(l) !== -1; } + /** * Process file(s) that have been uploaded through `userUploads` task(s). * The file(s) are expected to come from previous tasks */ -export async function processUploadedFiles({context:{tasks, logger}, task: {data:{files, name, lang}}}: TaskHandlerParams):Promise{ - - if(!files.length) throw new BadRequestError(`This task requires at least one source file`); +export async function createSceneFromFiles({context:{tasks, vfs, logger}, task: {user_id, task_id, data:{tasks:source_ids, name, language, options}}}: TaskHandlerParams):Promise{ + if(!user_id) throw new InternalError(`Can't create an anonymous scene. Provide a user`); + if(!name) throw new BadRequestError(`Can't create a scene without a name`); + if(!language) throw new BadRequestError(`Default language is required for scene creation`); + if(!isSceneLanguage(language)) throw new BadRequestError(`Unsupported scene language ${language}`); + if(!source_ids.length) throw new BadRequestError(`This task requires at least one source file`); - const upload_scenes = files.findIndex(u=>u.scenes.length) !== -1 //A file containing at least one complete scene - const upload_files = files.findIndex(u=>!u.scenes.length) !== -1 // A file NOT containing any complete scene - // Don't allow mixed-content. ie. a scene archive with some asset files - // In reality we _could_ probably, though we'd have to think carefully about edge cases? - if( upload_scenes && upload_files ){ - throw new BadRequestError(`Can't do mixed-content processing. Provide EITHER scene archive(s) OR source file(s)`); + for(const task_id of source_ids){ + if(!Number.isInteger(task_id)) throw new BadRequestError(`Invalid source task id: ${task_id}`); } - if(upload_files && (!name || !lang)){ - throw new BadRequestError(`scene name and default language are required when creating a scene from a set files`); + const source_tasks = await Promise.all(source_ids.map(id=> tasks.getTask(id))); + const failed_tasks = source_tasks.filter(t=>t.status !== "success"); + const invalid_outputs = source_tasks.filter(t=>!isUploadedFile(t.output)); + if(failed_tasks.length) throw new BadRequestError(`Source task${1 < failed_tasks.length?"s":""} #${failed_tasks.map(t=>t.task_id).join(", #")} has not completed successfully`); + if(invalid_outputs.length){ + for(let t of invalid_outputs){ + console.log(`Task ${t.type}#${t.task_id} can't be used as a scene source: Output is ${JSON.stringify(t.output)}`); + } + throw new BadRequestError(`Source task${1 < invalid_outputs.length?"s":""} ${invalid_outputs.map(t=>`${t.type}#${t.task_id}`).join(", ")} did not output a file`); } - - //Check that everything _should_ be error-free - //We might still have fails because of race conditions or _things_, but it is preferable to have a preflight check - const errors = files.map(u=>u.scenes.filter(s=>s.action === "error")).flat(); - for(let {error, name} of errors){ - logger.error(error ?? `Unspecified error on scene ${name}`); + //If some source models are present, copy all files to the task's workspace + let sources :Array = source_tasks.map(t=>t.output); + if(source_tasks.some(t=>(t.output as UploadedSource).isModel)){ + const dir = await vfs.createTaskWorkspace(task_id); + sources = await Promise.all(source_tasks.map(async ({output})=>{ + const filepath = vfs.absolute(output.fileLocation); + const dest = path.join(dir, path.basename(filepath)); + await fs.link(filepath, dest); + return { + ...output, + fileLocation: vfs.relative(dest) + } satisfies ParsedUserUpload; + })); } - // @FIXME should we clean up some things immediately when we abort due to planned errors? - if(errors.length){ - throw new UnauthorizedError(`Insufficient permissions on scene${1 < errors.length?"s":""} [${errors.slice(0, 3).map(({name})=>name)}${3 < errors.length?", ...":""}] for this user. Aborting.`); - } + const scene_id = await vfs.createScene(name, user_id); - const results = await tasks.group(function* createChildren(){ - for(let upload of files){ - if(upload_scenes){ - logger.debug("Extract uploaded scenes archive "+upload.filepath); - yield tasks.run({ - data: {filepath: upload.filepath}, - handler: extractScenesArchive, - }); + async function moveModel(source: UploadedBinaryModel){ + let filepath = vfs.absolute(source.fileLocation); + let filename = path.basename(filepath); + let mime = source.mime; + if(options.optimize){ + const output = await tasks.run({ + data: { + fileLocation: source.fileLocation, + preset: "High", + }, + handler: optimizeGlb, + }); + if(typeof output.fileLocation !== "string") logger.warn("Model optimization output is unreadable : ", output); + else{ + filepath = vfs.absolute(output.fileLocation); + filename = path.basename(output.fileLocation); + mime = "model/gltf-binary"; + logger.debug("Optimized model %s to %s", source.fileLocation, output.fileLocation); } } + logger.debug("Copy %s to %s", filepath, filename); + await vfs.copyFile(filepath, {scene: scene_id, name: filename, user_id, mime }); + models.push({ + ...(source as UploadedBinaryModel), + filepath: filename, + quality: "High", + usage: "Web3D" + }); + } + + + // @TODO: reparent everything to this task and this task to the created scene for better discoverability + const models :Array = []; + // We could probably return the scene ID from here and let this all be out-of-band + for(let source of sources){ + let filepath = vfs.absolute(source.fileLocation); + const filename = path.basename(filepath); + if(source.mime === "application/zip"){ + logger.warn("in-scene Zip extraction is not yet implemented. Skipped."); + continue; + }else if(isUploadedBinaryModel(source)){ + await moveModel(source); + }else if((source as UploadedSource).isModel){ + logger.log("Convert source model %s to GLB", source.fileLocation); + const dest = await tasks.run({ + data: {fileLocation: vfs.relative(filepath)}, + handler: toGlb, + }); + logger.debug("Copy Converted source file to %s", dest); + + const meta = await tasks.run({ + handler: parseUploadedModel, + data: { + fileLocation: dest, + } + }); + logger.debug("Parsed converted file :", meta); + await moveModel(meta); + }else{ + logger.debug("Copy source file %s (%s)", filepath, source.mime); + const file = await vfs.copyFile(filepath, {scene: scene_id, name: filename, user_id, mime: source.mime }); + if(!file.hash) throw new BadRequestError(`File ${source.filepath} is empty`); + } + } + + logger.debug(`Create a new document for ${models.length} model${1 ; + language?:SceneLanguage; +} + + + +export async function createDocumentFromFiles( + { + task: { + data:{scene, models, language} + }, + }: TaskHandlerParams): Promise{ + + let document = getDefaultDocument(); + //dumb inefficient Deep copy because we want to mutate the doc in-place + document.models ??= []; + for(let model of models){ + const index = document.models.push({ + "units": "m", //glTF specification says it's always meters. It's what blender do. + "boundingBox": model.bounds, + "derivatives":[{ + "usage": model.usage, + "quality": model.quality, + "assets": [ + { + "uri": encodeURIComponent(model.filepath), + "type": "Model", + "byteSize": model.byteSize, + "numFaces": model.numFaces, + "imageSize": model.imageSize, + } + ] + }], + "annotations":[], + }) -1; + const nodeIndex = document.nodes.push({ + "id": uid(), + "name": model.name ?? scene, + "model": index, + } as any) -1; + document.scenes[0].nodes!.push(nodeIndex); + } + + + if(language){ + document.setups[0].language = {language: language}; + document.metas ??= []; + const meta_index = document.metas.push({ + "collection": { + "titles": { + [language]: scene, + } + }, + }) -1; + document.scenes[document.scene].meta = meta_index; + } + + + return document +} diff --git a/source/server/tasks/logger.ts b/source/server/tasks/logger.ts index 71b50ec76..de2c6b128 100644 --- a/source/server/tasks/logger.ts +++ b/source/server/tasks/logger.ts @@ -1,4 +1,4 @@ -import { Writable } from "node:stream"; +import { Writable, Transform } from "node:stream"; import { ITaskLogger, LogSeverity } from "./types.js"; import { DatabaseHandle } from "../vfs/helpers/db.js"; import { debuglog, format } from "node:util"; @@ -6,25 +6,95 @@ import { debuglog, format } from "node:util"; const debug = debuglog("tasks:logs"); /** - * Disposable logger that queues messages and waits for all logs to be flushed when closed + * Creates a Transform stream that batches logs by count or time + * @param batchSize max number of writes to batch + * @param debounceMs max time to wait before flushing writes */ -export function createLogger(db: DatabaseHandle, task_id: number){ +export function createBatcher(batchSize: number, debounceMs: number) { + let buffer: Array<{severity: LogSeverity; message: string}> = []; + let timer: NodeJS.Timeout | null = null; + + return new Transform({ + objectMode: true, + highWaterMark: batchSize, + transform(chunk, _, cb) { + buffer.push(chunk); + + // Flush immediately if batch is full + if (buffer.length >= batchSize) { + if (timer) clearTimeout(timer); + timer = null; + this.push(buffer); + buffer = []; + } else if (!timer) { + // Start timer only if one isn't already running + timer = setTimeout(() => { + if (buffer.length) { + this.push(buffer); + buffer = []; + } + timer = null; + }, debounceMs); + } + cb(); + }, + flush(cb) { + // Ensure remaining data is sent when the stream ends + if (timer) clearTimeout(timer); + if (buffer.length) this.push(buffer); + cb(); + } + }); +} - const stream = new Writable({ +/** + * Create a stream that batch-inserts logs into the database + * @param db + * @param task_id + * @returns + */ +export function createInserter(db: DatabaseHandle, task_id: number){ + return new Writable({ objectMode: true, - write: async ({severity, message}, encoding, callback) => { + write: async (batch: Array<{severity: LogSeverity; message: string, timestamp: Date}>, _, cb) => { try { - await db.run(`INSERT INTO tasks_logs(fk_task_id, severity, message) VALUES ($1, $2, $3)`, [task_id, severity, message]); - callback(); + const severities: LogSeverity[] =[]; + const messages: string[] = []; + const timestamps: Date[] = []; + for(let log of batch){ + severities.push(log.severity); + messages.push(log.message); + timestamps.push(log.timestamp); + } + + await db.run(` + INSERT INTO tasks_logs(fk_task_id, severity, message) + SELECT $1, * + FROM UNNEST($2::log_severity[], $3::text[]) AS t(severity, message)`, + [task_id, severities, messages] + ); + cb(); } catch (err) { - callback(err as Error); + cb(err as Error); } } - }); + }) +} + +/** + * Disposable logger that batches log inserts using Transform streams + * Reduces database lock contention by grouping multiple inserts + */ +export function createLogger(db: DatabaseHandle, task_id: number){ + const batcher = createBatcher(10, 100); + + const inserter = createInserter(db, task_id); + + batcher.pipe(inserter); function log(severity: LogSeverity, message: string){ debug(`[${severity.toUpperCase()}] ${message}`); - stream.write({severity, message}); + batcher.write({severity, message, timestamp: new Date()}); } return { @@ -32,10 +102,13 @@ export function createLogger(db: DatabaseHandle, task_id: number){ log: (...args:any[])=>log('log', format(...args)), warn: (...args:any[])=>log('warn', format(...args)), error: (...args:any[])=>log('error', format(...args)), - [Symbol.asyncDispose]: function (): PromiseLike { - return new Promise((resolve) => { - stream.end(() => resolve()); - }); + [Symbol.asyncDispose]: async function (): Promise { + // Close both streams and wait for them to finish + batcher.end(); + await new Promise((resolve, reject) => { + inserter.on('finish', resolve); + inserter.on('error', reject); + }); } } satisfies ITaskLogger & AsyncDisposable; } \ No newline at end of file diff --git a/source/server/tasks/queue.test.ts b/source/server/tasks/queue.test.ts index 3495a92a2..d3c552ffc 100644 --- a/source/server/tasks/queue.test.ts +++ b/source/server/tasks/queue.test.ts @@ -1,3 +1,4 @@ +import timers from "node:timers/promises"; import { Queue } from "./queue.js"; @@ -21,7 +22,32 @@ describe("Queue", function(){ expect(()=>q.add(()=>Promise.resolve())).to.throw("Can't add new tasks"); }); - it.skip("cancels running jobs") + it("cancels running jobs", async function(){ + let result: any; + let _op = q.add(async ({signal})=>{ + try{ + await timers.setTimeout(1000, null, {signal}) + result = "ok"; + }catch(e: any){ + result = e.name; + throw e; + } + }).catch(e=> e); + + //Shouldn't throw despite the task throwing an AbortError + await q.close(100); + expect(result).to.equal("AbortError"); + expect(await _op).to.have.property("name", "AbortError"); + }) + + it("force quit jobs after a timeout", async function(){ + let result: any; + let _op = q.add(async ({signal})=>timers.setTimeout(100, null, /*no signal support */)); + + //Shouldn't throw despite the task throwing an AbortError + await q.close(1); + await expect(_op).to.be.rejectedWith("Queue close timeout") + }) }); describe("add()", function(){ diff --git a/source/server/tasks/queue.ts b/source/server/tasks/queue.ts index 34f60a3b8..c60899bb6 100644 --- a/source/server/tasks/queue.ts +++ b/source/server/tasks/queue.ts @@ -13,8 +13,15 @@ interface WorkPackage{ export class Queue{ #queue: WorkPackage[] = []; - #active = 0; + /** + * Pointer to the jobs currently running. Allows force-stop in close() + */ + #current = new Set(); #c = new AbortController(); + + /** + * Callback defined when queue is closing + */ #settleResolve: (() => void) | null = null; constructor(public limit = Infinity, public name?:string) { @@ -27,6 +34,7 @@ export class Queue{ /** * Adds a task to the queue. + * The task will wait for an open slot before starting */ add(work: TaskPackage):Promise { if(this.#c.signal.aborted){ @@ -38,30 +46,47 @@ export class Queue{ }); } - #onSettled = ()=>{ - this.#active--; - // Notify waiters if all active tasks have completed - if(this.#active === 0 && this.#settleResolve){ - this.#settleResolve(); - this.#settleResolve = null; + /** + * Jumps the queue and start processing a job immediately + * + * It is still counted in {@link Queue.activeCount}, but will ignore the queue's concurrency limit. + */ + unshift(work: TaskPackage):Promise { + if(this.#c.signal.aborted){ + throw new Error(`Queue is closed. Can't add new tasks`); } - this.#processNext(); + return new Promise((resolve, reject) => { + this.#run({ work, resolve, reject }); + }); + } + + #run(job: WorkPackage){ + this.#current.add(job); + + //Execute the job. When it settles, check if it was interrupted before calling the resolvers + Promise.resolve(job.work({signal: this.#c.signal})) + .then(result => { + if(this.#current.delete(job)) job.resolve(result); + }, error =>{ + if(this.#current.delete(job)) job.reject(error); + }) + .finally(()=>{ + // Notify waiters if all active tasks have completed + if(this.#current.size === 0 && this.#settleResolve){ + this.#settleResolve(); + this.#settleResolve = null; + } + this.#processNext(); + }); } #processNext(){ // Stop if we are busy or if the queue is empty - if (this.#active >= this.limit || this.#queue.length === 0 ) { + if (this.#current.size >= this.limit || this.#queue.length === 0 ) { return; } - // 3. Dequeue the next task - this.#active++; - const { work, resolve, reject } = this.#queue.shift()!; - - Promise.resolve(work({signal: this.#c.signal})) - .then(result => resolve(result)) - .catch(error => reject(error)) - .finally(this.#onSettled); + this.#run(this.#queue.shift()!); } /** @@ -81,24 +106,35 @@ export class Queue{ } this.#queue = []; //Return immediately if no jobs are currently processed - if(this.#active == 0) return; + if(this.#current.size == 0) return; //Otherwise wait for timeout to let jobs resolve properly await new Promise((resolve, reject)=>{ const _t = setTimeout(()=>{ this.#settleResolve = null; - reject(new Error(`Queue close timeout: active tasks did not complete within ${timeoutMs}ms`)); + //Force-reject any running jobs + for(let job of this.#current){ + job.reject(new Error(`Queue close timeout: task did not stop within ${timeoutMs}ms`)); + } + this.#current.clear(); + resolve(); }, timeoutMs); + this.#settleResolve = ()=>{ clearTimeout(_t); resolve(); }; }); - } - - // Optional getters for observability + /** + * Number of jobs waiting for an execution slot + */ get pendingCount() { return this.#queue.length; } - get activeCount() { return this.#active; } + /** + * Number of jobs currently being executed + * It's possible to have `limit < activeCount` if {@link Queue.unshift} is used. + */ + get activeCount() { return this.#current.size; } + /** true if Queue has been closed */ get closed(){ return this.#c.signal.aborted; } } \ No newline at end of file diff --git a/source/server/tasks/scheduler.test.ts b/source/server/tasks/scheduler.test.ts index 6d8706ef7..b2b076e8a 100644 --- a/source/server/tasks/scheduler.test.ts +++ b/source/server/tasks/scheduler.test.ts @@ -12,6 +12,7 @@ import { TaskScheduler } from "./scheduler.js"; import { CreateRunTaskParams, TaskDefinition } from "./types.js"; import Vfs from "../vfs/index.js"; import UserManager from "../auth/UserManager.js"; +import EventEmitter, { once } from "node:events"; const makeTask = (props:Partial> = {})=>({ @@ -77,7 +78,8 @@ describe("TaskScheduler", function(){ type: "testTask", data: {}, }); - const output = await scheduler.run({task, handler: async ({task})=> task.task_id}); + + const output = await scheduler.runTask({task, handler: async ({task})=> task.task_id}); expect(output).to.equal(task.task_id); task = await scheduler.getTask(output); @@ -150,6 +152,25 @@ describe("TaskScheduler", function(){ expect(ok, `Task seems to not have been run`).to.be.true; }); + it("propagates abort signals", async function(){ + let progress = new EventEmitter(); + let c = new AbortController(); + const runningTask = scheduler.run({ + scene_id: null, + user_id: null, + data: {}, + signal: c.signal, + handler: async ({context:{signal}})=>{ + progress.emit("start"); + await timers.setTimeout(1000, null, {signal}); // Slow task, with abort + return "task1"; + } + }); + await once(progress, "start"); + c.abort(); + await expect(runningTask).to.be.rejected; + }); + it("won't deadlock itself", async function(){ scheduler.concurrency = 2; @@ -409,7 +430,7 @@ describe("TaskScheduler", function(){ scheduler.concurrency = 1; - let task1Executed = false; + let task1Executed = new EventEmitter(); let task2Executed = false; // Task 1: Will be running when we close @@ -418,8 +439,8 @@ describe("TaskScheduler", function(){ user_id: null, data: {}, handler: async ({context:{signal}})=>{ - task1Executed = true; - await timers.setTimeout(100, {signal}); // Slow task, with abort + task1Executed.emit("start"); + await timers.setTimeout(1000, null, {signal}); // Slow task, with abort return "task1"; } }); @@ -436,15 +457,14 @@ describe("TaskScheduler", function(){ }); // Give task1 time to start executing - await timers.setTimeout(10); + await once(task1Executed, "start"); + expect(task2Executed).to.be.false; - // Close scheduler - task1 should be allowed to complete, task2 should be rejected - await scheduler.close(1000); + // Close scheduler + await scheduler.close(100); - // The running task should have completed successfully - const result1 = await runningTask; - expect(result1).to.equal("task1"); - expect(task1Executed).to.be.true; + // The running task should abort + await expect(runningTask).to.be.rejectedWith("aborted"); // The pending task should be rejected await expect(pendingTask).to.be.rejected; diff --git a/source/server/tasks/scheduler.ts b/source/server/tasks/scheduler.ts index be2cf62bb..38882aefa 100644 --- a/source/server/tasks/scheduler.ts +++ b/source/server/tasks/scheduler.ts @@ -2,13 +2,11 @@ import {debuglog} from "node:util"; import { AsyncLocalStorage } from 'node:async_hooks'; import { DatabaseHandle } from "../vfs/helpers/db.js"; import { Queue } from "./queue.js"; -import { CreateRunTaskParams, CreateTaskParams, ITaskLogger, RunTaskParams, TaskDataPayload, TaskDefinition, TaskHandler, TaskHandlerContext, TaskPackage, TaskSchedulerContext, TaskSettledCallback, TaskStatus, } from "./types.js"; +import { CreateRunTaskParams, CreateTaskParams, ITaskLogger, RunOptions, RunTaskParams, TaskDataPayload, TaskDefinition, TaskHandler, TaskHandlerContext, TaskPackage, TaskSchedulerContext, TaskSettledCallback, TaskStatus, } from "./types.js"; import { createLogger } from "./logger.js"; import { TaskManager } from "./manager.js"; import { BadRequestError, InternalError } from "../utils/errors.js"; - -import * as userHandlers from "./handlers/index.js"; // Note: previously used stream/once for generator bridge; no longer required for Promise.all-style group @@ -35,19 +33,12 @@ type NestContextProps = { concurrency: number; } -// Work must be a callable that, when invoked inside `nest`, returns -// an iterable of promises. This ensures the promises are created -// in the nested async context (covers both factory functions and -// generator functions — calling them produces an iterator). -type GroupWorkload = () => Iterable>; -export function isUserHandlerType(t: string):t is keyof typeof userHandlers{ - return t in userHandlers; -} - export class TaskScheduler extends TaskManager{ //Do not use "real" private members here because they would be missed by Object.create - /** Work queue. One should never use this directly */ + /** + * Work queue. used internally by {@link TaskScheduler.run} to run jobs + * */ private readonly rootQueue = new Queue(4, "root"); @@ -69,7 +60,8 @@ export class TaskScheduler{ return (asyncStore.getStore() as any ?? {queue: this.rootQueue, parent: null}) satisfies AsyncContext; @@ -101,11 +93,30 @@ export class TaskScheduler AsyncContext; /** * Run `work` inside a new async context with the given name and concurrency settings + * Calls to {@link TaskScheduler._run} within this async context will resolve to the nested Queue */ public readonly nest: (props:NestContextProps, work:(...args: T)=>U, ...args: T)=>Promise; - private async _run(handler:TaskHandler,task: TaskDefinition, {signal:taskSignal}:{signal?:AbortSignal}= {}){ + /** + * Internal task running handler. + * + * Builds a context and actually schedule the task, handling the status change from `"pending"` to `"running"` and to `"complete"|"error"` + * + * Unless there is a problem with the database connection, the task is guaranteed to end up with `status = "complete"|"error"` + * @param handler Handler functionthat will perform the job + * @param task Task definition + * @param param2 Additional options for the task runner + * @param param2.immediate jumps the queue and run the task immediately regardless of concurrency settings + * @returns + */ + private async _run( + handler:TaskHandler, + task: TaskDefinition, + {signal:taskSignal, immediate }: RunOptions= {} + ){ + // Create a wrapper function around the handler to provide the task's execution context + // and set its status const work :TaskPackage = async ({signal: queueSignal})=>{ await using logger = createLogger(this.db, task.task_id); const context: TaskHandlerContext = { @@ -123,57 +134,52 @@ export class TaskScheduler(${task.task_id})@${async_ctx.queue.name}` }); - if(async_ctx.parent?.logger){ - async_ctx.parent.logger.debug(`Schedule child task ${task.type}#${task.task_id}`); + try{ + const output = await this.nest({concurrency: 1, name: `${task.type}#${task.task_id.toString()}`, parent: thisContext}, handler.bind(context), {context, task}) + await this.releaseTask(task.task_id, output); + return output; + }catch(e:any){ + //Here we might make an exception if e.name === "AbortError" and the database is closed + await this.errorTask(task.task_id, e).catch(e=> console.error("Failed to set task error : ", e)); + throw e; } - debug("Schedule work for task #%d on Queue(%s)", task.task_id, async_ctx.queue.name); - const output = await async_ctx.queue.add(work); - await this.releaseTask(task.task_id, output); - return output; - }catch(e: any){ - //Here we might make an exception if e.name === "AbortError" and the database is closed - await this.errorTask(task.task_id, e).catch(e=> console.error("Failed to set task error : ", e)); - throw e; } + + const async_ctx = this.context(); + //Custom name for work to be shown in stack traces + Object.defineProperty(work, 'name', { value: `TaskScheduler.payload<${task.type}>(${task.task_id})@${async_ctx.queue.name}` }); + if(async_ctx.parent?.logger){ + async_ctx.parent.logger.debug(`Schedule child task ${task.type}#${task.task_id}`); + } + debug("Schedule work for task #%d on Queue(%s)", task.task_id, async_ctx.queue.name); + return await (immediate? async_ctx.queue.unshift(work) : async_ctx.queue.add(work)); } /** - * Registers a task to run as soon as possible and wait for its completion + * Registers a task to run as soon as possible and wait for its completion. + * * It's OK to ignore the returned promise if a callback is provided to at least properly log the error * * `TaskScheduler.run()` uses async context tracking to inherit **scene_id**, **user_id** and **parent** from it's context * However those can still be forced to another value if deemed necessary. * Whether or not this override is desirable is yet unclear. */ - async run({task, handler}: RunTaskParams, callback?:TaskSettledCallback ): Promise; - async run({scene_id, user_id, type, data, handler, signal}: CreateRunTaskParams, callback?:TaskSettledCallback): Promise; - async run(params: CreateRunTaskParams|RunTaskParams, callback?:TaskSettledCallback): Promise{ - let task: TaskDefinition; - if("task" in params){ - //We allow externally-created tasks here. But we want to limit error sources so we re-fetch the task anyway - //If this ends up being used in practice, we may need perform validation that the given task actually matches the stored one. - task = await this.getTask(params.task.task_id); - }else{ - //We use context to inherit parent, user_id and scene_id - //But if different values are explicitly specified it's possible to "break out" - //Whether or not this is - const {parent} = this.context(); - task = await this.create({ - ...params, - data: params.data as any, - type: (params.type ?? params.handler.name) as string, - status: "pending" as TaskStatus, - parent: parent?.task_id ?? null, - }); - } - const _p = this._run( params.handler, task); + async run( + params: CreateRunTaskParams, + callback?:TaskSettledCallback + ): Promise{ + //We use context to inherit parent, user_id and scene_id + //But if different values are explicitly specified it's possible to "break out" + //Whether or not this is + const {parent} = this.context(); + const task : TaskDefinition= await this.create({ + ...params, + data: params.data as any, + type: (params.type ?? params.handler.name) as string, + status: "pending" as TaskStatus, + parent: parent?.task_id ?? null, + }); + const _p = this._run( params.handler, task, {signal: params.signal, immediate: params.immediate}); if(typeof callback=== "function"){ _p.then((value)=>callback(null, value), (err)=>callback(err)); @@ -181,24 +187,23 @@ export class TaskScheduler({task, signal, handler}: RunTaskParams, callback?:TaskSettledCallback ): Promise{ + const _p = this._run( handler, task, {signal: signal, immediate: false}); + if(typeof callback=== "function"){ + _p.then((value)=>callback(null, value), (err)=>callback(err)); } - return await this.run({task, handler: userHandlers[task.type] as any}); + return _p; } + /** * Create a task with async-context awareness * This is exposed in case it is ever needed but it's probably always better to call {@link TaskScheduler.run} diff --git a/source/server/tasks/types.ts b/source/server/tasks/types.ts index 5a8f5f550..93d941b78 100644 --- a/source/server/tasks/types.ts +++ b/source/server/tasks/types.ts @@ -1,6 +1,7 @@ import type UserManager from "../auth/UserManager.js"; import { Config } from "../utils/config.js"; import { TDerivativeQuality } from "../utils/schema/model.js"; +import { RootRelativePath } from "../vfs/Base.js"; import { DatabaseHandle } from "../vfs/helpers/db.js"; import type Vfs from "../vfs/index.js"; import { TaskScheduler } from "./scheduler.js"; @@ -20,6 +21,7 @@ export enum ETaskStatus{ } + /** * Task Creation parameters */ @@ -82,6 +84,12 @@ type TaskCreateCommonParameters = { parent?: number|null; } +export interface RunOptions{ + signal?:AbortSignal; + immediate?:boolean; +} + + /** * Parameters to create a task */ @@ -97,6 +105,7 @@ export type CreateRunTaskParams; type?: string; status?:"pending"; + immediate?: boolean; signal?: AbortSignal; /** Can't create an immediately-running task with a status other than pending */ }; @@ -107,6 +116,7 @@ export type CreateRunTaskParams{ task: TaskDefinition; handler: TaskHandler; + immediate?: boolean; signal?: AbortSignal; } @@ -130,8 +140,17 @@ export interface TaskHandlerDefinition{{i18n "titles.upload"}} @@ -54,7 +113,8 @@ - + + {{i18n "leads.uploadErrors"}} diff --git a/source/server/tests-common.ts b/source/server/tests-common.ts index 62093d499..72a94806b 100644 --- a/source/server/tests-common.ts +++ b/source/server/tests-common.ts @@ -89,7 +89,7 @@ global.createIntegrationContext = async function(c :Mocha.Context, config_overri } global.cleanIntegrationContext = async function(c :Mocha.Context){ - await c.services.close(); + await c.services?.close(); await dropDb(c.db_uri); if(c.dir) await fs.rm(c.dir, {recursive: true}); } \ No newline at end of file diff --git a/source/server/utils/config.ts b/source/server/utils/config.ts index 97f7cf34f..891a15436 100644 --- a/source/server/utils/config.ts +++ b/source/server/utils/config.ts @@ -12,6 +12,7 @@ const values = { root_dir: [ process.cwd(), toPath], migrations_dir: [path.join(process.cwd(),"migrations"), toPath], templates_dir: [path.join(process.cwd(),"templates"), toPath], + scripts_dir: [path.join(process.cwd(),"scripts"), toPath], files_dir: [({root_dir}:{root_dir:string})=> path.resolve(root_dir,"files"), toPath], dist_dir: [({root_dir}:{root_dir:string})=> path.resolve(root_dir,"dist"), toPath], assets_dir: [undefined, toPath], diff --git a/source/server/utils/exec.test.ts b/source/server/utils/exec.test.ts new file mode 100644 index 000000000..022ec8abc --- /dev/null +++ b/source/server/utils/exec.test.ts @@ -0,0 +1,28 @@ + +import {run} from "./exec.js"; + + +describe("run()", function(){ + it("runs a command", async function(){ + let {code, stdout, stderr} = await run("echo", ["Hello World"]); + expect(code).to.equal(0); + expect(stdout).to.equal("Hello World\n"); + expect(stderr).to.equal(""); + }); + + it("throw if interrupted by an AbortSignal", async function(){ + let c = new AbortController(); + setTimeout(()=> c.abort(), 10); + await expect(run("sleep", ["1"], {signal: c.signal})).to.be.rejectedWith("The operation was aborted"); + }); + + it("throw if interrupted by a signal", async function(){ + await expect(run("bash", ["-c", "kill -TERM $$"])).to.be.rejectedWith("Command bash was interrupted by signal SIGTERM"); + }); + + it("can return code != 0", async function(){ + let {code,} = await run("bash", ["-c", "exit 1"]); + expect(code).to.equal(1); + }); + +}) \ No newline at end of file diff --git a/source/server/utils/exec.ts b/source/server/utils/exec.ts new file mode 100644 index 000000000..e715d4a71 --- /dev/null +++ b/source/server/utils/exec.ts @@ -0,0 +1,102 @@ +import { spawn, SpawnOptionsWithoutStdio } from "child_process"; +import { once } from "events"; +import { ITaskLogger } from "../tasks/types.js"; + + +/** + * Run a command with a {@link ITaskLogger} instrumentation + * stdout will be piped to `logger.debug` and stderr to `logger.warn` + * The promise will then be rejected only if exit code != 0 + */ + +export async function taskRun(cmd: string, args: string[], { logger, ...opts }: RunCommandOpts): Promise { + let child = spawn(cmd, args, opts); + + child.stdout.setEncoding("utf-8"); + child.stdout.on("data", (chunk) => logger.debug(chunk)); + child.stderr.setEncoding("utf-8"); + child.stderr.on("data", (chunk) => logger.warn(chunk)); + + try { + let [code, signal] = await once(child, "close"); + if (typeof code !== "number") { + let e: any = new Error(`Command ${cmd} was interrupted by signal ${signal}`); + throw e; + } else if (code != 0) { + throw new Error(`Command ${cmd} exitted with non-zero error code: ${code}`); + } + } finally { + child.stdout.removeAllListeners(); + child.stderr.removeAllListeners(); + } +} + + +/** + * Wrapper around `spawn` that gathers stdout/stderr to a string + */ +export async function run(cmd: string, args: string[], opts?: SpawnOptionsWithoutStdio): Promise<{ code: number; stdout: string; stderr: string; }> { + let child = spawn(cmd, args, opts); + + let stdout: string = ""; + let stderr: string = ""; + child.stdout.setEncoding("utf-8"); + child.stdout.on("data", (chunk) => stdout += chunk); + child.stderr.setEncoding("utf-8"); + child.stderr.on("data", (chunk) => stderr += chunk); + + try { + let [code, signal] = await once(child, "close"); + if (typeof code !== "number") { + let e: any = new Error(`Command ${cmd} was interrupted by signal ${signal}`); + e.stdout = stdout; + e.stderr = stderr; + throw e; + } + return { code, stdout, stderr }; + } finally { + child.stdout.removeAllListeners(); + child.stderr.removeAllListeners(); + } +} +export interface RunCommandOpts extends SpawnOptionsWithoutStdio { + logger: ITaskLogger; +} + + +/** + * Spawn blender to get its version string and parse the result + */ +export async function getBlenderVersion(): Promise { + + const { stdout, stderr } = await run('blender', ['--version']); + + const versionMatch = stdout.match(/^Blender (\d+.\d+.\d+[^\s]*)/gm); + + if (!versionMatch) { + throw new Error( + `Unable to find "blender" version. Confirm Blender is installed.` + ); + } + return versionMatch[1]; +} + +/** + * spawn the ktx utility to get its version string and parse it + */ +export async function getKtxVersion(): Promise { + + const { stdout, stderr } = await run('ktx', ['--version']); + + const version = ((stdout || stderr) as string) + .replace(/ktx version:\s+/, '') + .replace(/~\d+/, '') + .trim(); + + if (!version) { + throw new Error( + `Unable to find "ktx" version. Confirm KTX-Software is installed.` + ); + } + return version; +} diff --git a/source/server/utils/filetypes.test.ts b/source/server/utils/filetypes.test.ts index d399a5fa3..52ef6c9c1 100644 --- a/source/server/utils/filetypes.test.ts +++ b/source/server/utils/filetypes.test.ts @@ -18,12 +18,30 @@ describe("getMimeType",function(){ expect(getMimeType("foo.glb")).to.equal("model/gltf-binary"); }); + it("model/obj", function(){ + // Not widely used, but recognized + // by IANA See: https://www.iana.org/assignments/media-types/media-types.xhtml#model + expect(getMimeType("foo.obj")).to.equal("model/obj"); + expect(getMimeType("foo.mtl")).to.equal("model/mtl") + }); + + it("model/stl", function(){ + expect(getMimeType("foo.stl")).to.equal("model/stl"); + }); + + it("model/vnd.usdz+zip", function(){ + expect(getMimeType("foo.usdz")).to.equal("model/vnd.usdz+zip"); + }) + it("text/html", function(){ expect(getMimeType("foo.html")).to.equal("text/html"); }); it("defaults to application/octet-stream", function(){ expect(getMimeType("foo.bar")).to.equal("application/octet-stream"); + //Test expectations for known file types + expect(getMimeType("foo.ply")).to.equal("application/octet-stream"); + expect(getMimeType("foo.blend")).to.equal("application/octet-stream"); }) }); diff --git a/source/server/utils/filetypes.ts b/source/server/utils/filetypes.ts index 1fcf0afe0..bf0976457 100644 --- a/source/server/utils/filetypes.ts +++ b/source/server/utils/filetypes.ts @@ -85,3 +85,7 @@ export function parseMagicBytes(src: Buffer|Uint8Array) :string{ return "application/octet-stream"; } + +export function isModelType(filename: string):boolean{ + return /\.(?:obj|stl|ply|gltf|blend)$/i.test(filename); +} \ No newline at end of file diff --git a/source/server/utils/format.test.ts b/source/server/utils/format.test.ts new file mode 100644 index 000000000..09a3558a7 --- /dev/null +++ b/source/server/utils/format.test.ts @@ -0,0 +1,37 @@ +import { formatBytes, isTimeInterval } from "./format.js"; + + + +describe("formatBytes()", function(){ + it("Format a number of bytes to a human readable string", function(){ + expect(formatBytes(1000)).to.equal("1 kB"); + }); + + it("format mibibytes", function(){ + expect(formatBytes(1024, false)).to.equal("1 KiB"); + expect(formatBytes(1024*1024, false)).to.equal("1 MiB"); + }); +}); + + +describe("isTimeInterval()", function(){ + it("Returns false for Date objects", function(){ + expect(isTimeInterval(new Date())).to.be.false; + }); + it("Returns false for timestamps", function(){ + expect(isTimeInterval(Date.now())).to.be.false; + }); + it("Returns false for ISO Date strings", function(){ + expect(isTimeInterval(new Date().toISOString())).to.be.false; + }); + it("Returns true for ISO8601 period strings", function(){ + [ + "P1Y", + "P1M1D", + "PT1M", // 1 minute + "PT1.5S", // Seconds are fractional + ].forEach(function(s){ + expect(isTimeInterval(s), `${s} should be a valid ISO8601 period string`).to.be.true; + }) + }) +}) \ No newline at end of file diff --git a/source/server/utils/format.ts b/source/server/utils/format.ts new file mode 100644 index 000000000..f80a93961 --- /dev/null +++ b/source/server/utils/format.ts @@ -0,0 +1,29 @@ + + + +/** + * Format a byte count into a human readable string + */ +export function formatBytes(bytes:number, si=true){ + const thresh = si ? 1000 : 1024; + if(Math.abs(bytes) < thresh) { + return bytes + ' B'; + } + let units = si + ? ['kB','MB','GB','TB','PB','EB','ZB','YB'] + : ['KiB','MiB','GiB','TiB','PiB','EiB','ZiB','YiB']; + let u = -1; + do { + bytes /= thresh; + ++u; + } while(Math.abs(bytes) >= thresh && u < units.length - 1); + return Math.round(bytes*100)/100 + ' '+units[u]; +} + +/** + * Checks if a Date-like parameter might be an interval + * It is not fool-proof: We don't want to match string that would be valid dates. But if it's neither a valid date nor a valid interval, we don't really care about trhe output here. + */ +export function isTimeInterval(t:any){ + return typeof t === "string" && /^[-+]?P?(\d+[YMD]|T[.\d]+[HMS])/.test(t); +} \ No newline at end of file diff --git a/source/server/utils/glTF.ts b/source/server/utils/glTF.ts index b53e883fe..35aa39d4b 100644 --- a/source/server/utils/glTF.ts +++ b/source/server/utils/glTF.ts @@ -98,7 +98,13 @@ export function parse_glTF({meshes = [], accessors = []}:JSONglTF) :SceneDescrip - +/** + * Opens a GLB file, reads its JSON portion and parse it through {@link parse_glTF} + * + * For large models, this is much more efficient than using `@gltf-transform` because it only reads what's needed + * @param filePath + * @returns + */ export async function parse_glb(filePath :string) :Promise{ let res :MeshDescription = {} as any; let handle = await fs.open(filePath, "r"); diff --git a/source/server/utils/gltf/inspect.ts b/source/server/utils/gltf/inspect.ts new file mode 100644 index 000000000..964128f4b --- /dev/null +++ b/source/server/utils/gltf/inspect.ts @@ -0,0 +1,75 @@ +import {getBounds, getPrimitiveVertexCount, VertexCountMethod} from '@gltf-transform/functions'; + +import { Document, ImageUtils, PropertyType } from '@gltf-transform/core'; + +export interface SceneDescription{ + name: string, + bounds :{ + min: [number, number, number], + max: [number, number, number] + }, + imageSize: number, + numFaces: number, + extensions: string[], +} + +export function inspectDocument(document: Document): SceneDescription{ + const root = document.getRoot(); + const scene = root.getDefaultScene(); + const name = scene?.listChildren().map((node=>node.getName())).find(n=>!!n) ?? scene?.getName() ?? ''; + if(!scene) throw new Error("Empty glb (no root scene)"); + const extensions = root.listExtensionsUsed().map(e=>e.extensionName); + const bounds = getBounds(scene); + let numFaces = 0; + for(const mesh of root.listMeshes()){ + for(let primitive of mesh.listPrimitives()){ + const mode = primitive.getMode(); + if(mode < 4) continue; // POINTS and LINES* have no faces + numFaces += Math.floor(getPrimitiveVertexCount(primitive, VertexCountMethod.RENDER) / 3); + if(4 < mode) numFaces -= 2; //TRIANGLES_STRIP and TRIANGLE_FAN have two shared vertices + } + } + + let imageSize = getMaxDiffuseSize(document); + + + return { + name, + bounds, + imageSize, + numFaces, + extensions, + } +} + + +/** + * Get the largest diffuse size in the document + * @param document + * @returns + */ +export function getMaxDiffuseSize(document: Document) :number{ + const root = document.getRoot(); + let imageSize = 0 + for(const texture of root.listTextures()){ + const slots = document + .getGraph() + .listParentEdges(texture) + .filter((edge) => edge.getParent().propertyType !== PropertyType.ROOT) + .map((edge) => edge.getName()); + + if(slots.indexOf("baseColorTexture") === -1) continue; //Ignore textures not used as baseColor + + const resolution = ImageUtils.getSize(texture.getImage()!, texture.getMimeType()); + if(resolution){ + imageSize = Math.max(imageSize, ...resolution); + } + } + return imageSize; +} + +export function getBaseTextureSizeMultiplier(originalMaxSize: number){ + //How much should we scale down to have High be a 8k texture? + const baseDivider = Math.max(1, originalMaxSize / 8192); + return 1/baseDivider; +} \ No newline at end of file diff --git a/source/server/utils/gltf/io.ts b/source/server/utils/gltf/io.ts new file mode 100644 index 000000000..f9ac22667 --- /dev/null +++ b/source/server/utils/gltf/io.ts @@ -0,0 +1,19 @@ +import { NodeIO } from '@gltf-transform/core'; +import { EXTMeshoptCompression, KHRDracoMeshCompression, KHRMeshQuantization, KHRTextureBasisu, EXTTextureWebP } from '@gltf-transform/extensions'; +import draco3d from 'draco3dgltf'; +import { MeshoptDecoder, MeshoptEncoder } from 'meshoptimizer'; + +await MeshoptEncoder.ready; +export const io = new NodeIO() + .registerExtensions([ + KHRDracoMeshCompression, + KHRMeshQuantization, + EXTMeshoptCompression, + KHRTextureBasisu, + EXTTextureWebP, + ]) + .registerDependencies({ + 'draco3d.decoder': await draco3d.createDecoderModule(), + 'meshopt.decoder': MeshoptDecoder, + 'meshopt.encoder': MeshoptEncoder, + }); diff --git a/source/server/utils/gltf/toktx.ts b/source/server/utils/gltf/toktx.ts new file mode 100644 index 000000000..1c431c976 --- /dev/null +++ b/source/server/utils/gltf/toktx.ts @@ -0,0 +1,297 @@ +import { + BufferUtils, + type Document, + FileUtils, + type ILogger, + ImageUtils, + type Texture, + TextureChannel, + type Transform, +} from '@gltf-transform/core'; + +import { KHRTextureBasisu } from '@gltf-transform/extensions'; +import { + createTransform, + getTextureChannelMask, + getTextureColorSpace, + listTextureSlots, +} from '@gltf-transform/functions'; +import fs from 'fs/promises'; +import os from 'os'; +import path, { join } from 'path'; +import sharp from 'sharp'; + +import { run } from "../exec.js"; +import { getKtxVersion } from "../exec.js"; +import { formatBytes } from '../format.js'; + +const NUM_CPUS = os.cpus().length || 1; + +const { R, G, A } = TextureChannel; + + + +/********************************************************************************************** + * Interfaces. + */ + +export const Mode = { + ETC1S: 'etc1s', + UASTC: 'uastc', +} as const; + + +interface GlobalOptions { + mode: typeof Mode[keyof typeof Mode]; + /** + * Pattern matching the material texture slot(s) to be compressed or converted. + */ + slots: RegExp; + tmpdir: string; + maxSize?: number; +} + +export interface ETC1SOptions extends GlobalOptions { + mode: typeof Mode.ETC1S; + /** [1..255] quality setting. Higher is better. */ + quality?: number; + compression?: number; + rdo?: boolean; +} + +export interface UASTCOptions extends GlobalOptions { + mode: typeof Mode.UASTC; + /** 0 (fastest) to 4 (slowest) */ + level?: number; + zstd?: number; +} + + +/********************************************************************************************** + * KTX conversion Implementation + * + * @todo There is some optimization to be had for the resize step where we may benefit from some experimentation over resize algorithms + */ +export const toktx = function (options: ETC1SOptions | UASTCOptions): Transform { + + return createTransform(options.mode, async (doc: Document): Promise => { + const logger = doc.getLogger(); + + // Confirm recent version of KTX-Software is installed. + logger.debug(`Found ktx version: ${await getKtxVersion()}`) ; + + const tmpdir = options.tmpdir; + const batchPrefix = path.basename(tmpdir); + + const basisuExtension = doc.createExtension(KHRTextureBasisu).setRequired(true); + + const textures = doc.getRoot().listTextures(); + + for (let textureIndex = 0; textureIndex< textures.length; textureIndex++){ + const texture = textures[textureIndex]; + const slots = listTextureSlots(texture); + const channels = getTextureChannelMask(texture); + const textureLabel = + texture.getURI() || + texture.getName() || + `${textureIndex + 1}/${doc.getRoot().listTextures().length}`; + const prefix = `ktx:texture(${textureLabel})`; + logger.debug(`${prefix}: Slots → [${slots.join(', ')}]`); + + // FILTER: Exclude textures that don't match (a) 'slots' or (b) expected formats. + + let srcMimeType = texture.getMimeType(); + + if (options.slots && !slots.find((slot) => slot.match(options.slots))) { + logger.debug(`${prefix}: Skipping, [${slots.join(', ')}] excluded by "slots" parameter.`); + return; + }else if (srcMimeType === 'image/ktx2') { + logger.debug(`${prefix}: Skipping, already KTX.`); + return; + } + + let srcImage = texture.getImage()!; + let srcExtension = texture.getURI() + ? FileUtils.extension(texture.getURI()) + : ImageUtils.mimeTypeToExtension(texture.getMimeType()); + const srcSize = texture.getSize(); + const srcBytes = srcImage ? srcImage.byteLength : null; + + if (!srcImage || !srcSize || !srcBytes) { + logger.warn(`${prefix}: Skipping, unreadable texture.`); + return; + } + + //Resize + if(typeof options.maxSize === "number" || !srcSize.every(n=>isMultipleOfFour(n)) || texture.getMimeType() === "image/webp"){ + // Constrain texture size to a multiple of four + // To be conservative with 3D API compatibility + // @see https://github.khronos.org/KTX-Specification/ktxspec.v2.html#dimensions + if(typeof options.maxSize ==="number"){ + logger.info(`Resizing images to fit within a ${options.maxSize}x${options.maxSize} pixels square`); + }else if(texture.getMimeType() === "image/webp"){ + // toktx doesn't support webp images + logger.info("Reencoding webp image to png before basisu conversion") + } + let dstSize:[number, number] = ((typeof options.maxSize === "number")?fitWithin(srcSize, options.maxSize):srcSize); + if(!dstSize.every(n=>isMultipleOfFour(n))){ + logger.warn(`Source image ${texture.getName()} at index ${textureIndex} ${typeof options.maxSize === "number"?"would be":"is"} ${dstSize.join("x")} px. Ceiling to a multiple of four`); + dstSize = dstSize.map(n=> ceilMultipleOfFour(n)) as [number, number]; + } + const encoder = sharp(srcImage, { limitInputPixels: 32768 * 32768 }).toFormat('png'); + + if(!dstSize.every(n=>isPowerofTwo(n))){ + logger.warn(`${prefix}: Resizing ${srcSize.join('x')} → ${dstSize.join('x')}px: This is not a power of two, which might not be optimal for performance`); + }else{ + logger.debug(`${prefix}: Resizing ${srcSize.join('x')} → ${dstSize.join('x')}px`); + } + + encoder.resize(dstSize[0], dstSize[1], { fit: 'fill', kernel: 'lanczos3' }); + + srcImage = BufferUtils.toView((await encoder.toBuffer()) as any); + srcExtension = 'png'; + srcMimeType = 'image/png'; + } + + // PREPARE: Create temporary in/out paths for the 'ktx' CLI tool, and determine + // necessary command-line flags. + + const srcPath = join(tmpdir, `${batchPrefix}_${textureIndex}.${srcExtension}`); + const dstPath = join(tmpdir, `${batchPrefix}_${textureIndex}.ktx2`); + + await fs.writeFile(srcPath, srcImage); + + const params = [ + 'create', + ...createParams(texture, slots, channels, options), + srcPath, + dstPath, + ]; + logger.debug(`${prefix}: Spawning → ktx ${params.join(' ')}`); + + // COMPRESS: Run `ktx create` CLI tool. + const { code, stdout, stderr} = await run('ktx', params as string[]); + + if (code !== 0) { + logger.error(`${prefix}: Failed with code [${code}]:\n\n${stderr.toString()}`); + } else { + // PACK: Replace image data in the glTF asset. + texture.setImage(await fs.readFile(dstPath) as any).setMimeType('image/ktx2'); + if (texture.getURI()) { + texture.setURI(FileUtils.basename(texture.getURI()) + '.ktx2'); + } + } + + const dstBytes = texture.getImage()!.byteLength; + logger.debug(`${prefix}: ${formatBytes(srcBytes)} → ${formatBytes(dstBytes)} bytes`); + }; + + const usesKTX2 = doc + .getRoot() + .listTextures() + .some((t) => t.getMimeType() === 'image/ktx2'); + + if (!usesKTX2) { + basisuExtension.dispose(); + } + }); +}; + +/********************************************************************************************** + * Utilities. + */ + +/** Create CLI parameters from the given options. Attempts to write only non-default options. */ +function createParams( + texture: Texture, + slots: string[], + channels: number, + options: ETC1SOptions | UASTCOptions, +): (string | number)[] { + const colorSpace = getTextureColorSpace(texture); + const params: (string | number)[] = ['--generate-mipmap']; + + + // See: https://github.com/KhronosGroup/KTX-Software/issues/600 + const isNormalMap = slots.find((slot) => /normal/.test(slot)); + + if (options.mode === Mode.UASTC) { + const _options = options as UASTCOptions; + params.push('--encode', 'uastc'); + params.push('--uastc-quality', _options.level ?? 2); + + if (_options.zstd !== 0) { + params.push('--zstd', _options.zstd ?? 18); + } + } else { + + const _options = options as ETC1SOptions; + params.push('--encode', 'basis-lz'); + + params.push('--qlevel', _options.quality ?? 128); + params.push('--clevel', _options.compression ?? 1); + + if ( _options.rdo === false || isNormalMap) { + params.push('--no-endpoint-rdo', '--no-selector-rdo'); + } + } + + // See: https://github.com/donmccurdy/glTF-Transform/issues/215 + if (colorSpace === 'srgb') { + params.push('--assign-oetf', 'srgb', '--assign-primaries', 'bt709'); + } else if (colorSpace === 'srgb-linear') { + params.push('--assign-oetf', 'linear', '--assign-primaries', 'bt709'); + } else if (slots.length && !colorSpace) { + params.push('--assign-oetf', 'linear', '--assign-primaries', 'none'); + } + + if (channels === R) { + params.push('--format', 'R8_UNORM'); + } else if (channels === G || channels === (R | G)) { + params.push('--format', 'R8G8_UNORM'); + } else if (!(channels & A)) { + params.push('--format', colorSpace === 'srgb' ? 'R8G8B8_SRGB' : 'R8G8B8_UNORM'); + } else { + params.push('--format', colorSpace === 'srgb' ? 'R8G8B8A8_SRGB' : 'R8G8B8A8_UNORM'); + } + + // See: https://github.com/donmccurdy/glTF-Transform/pull/389#issuecomment-1089842185 + const threads = Math.max(2, NUM_CPUS); + params.push('--threads', threads); + + return params; +} + + +function isMultipleOfFour(value: number): boolean { + return value % 4 === 0; +} + +function isPowerofTwo(n:number) { + // Check if n is positive and n & (n-1) is 0 + return (n > 0) && ((n & (n - 1)) === 0); +} + + +function ceilMultipleOfFour(value: number): number { + if (value <= 4) return 4; + return value % 4 ? value + 4 - (value % 4) : value; +} +export function fitWithin(size: [number, number], limit: number): [number, number] { + + let [width, height] = size; + + if (width <= limit && height <= limit) return size; + + if (width > limit) { + height = Math.floor(height * (limit / width)); + width = limit; + } + + if (height > limit) { + width = Math.floor(width * (limit / height)); + height = limit; + } + + return [width, height]; +} diff --git a/source/server/utils/schema/default.ts b/source/server/utils/schema/default.ts index 406bb6cee..1294c0cb0 100644 --- a/source/server/utils/schema/default.ts +++ b/source/server/utils/schema/default.ts @@ -1,3 +1,6 @@ +import { IDocument, IScene } from "./document.js"; +import { ISetup } from "./setup.js"; + /** * Copied from DPO-Voyager * @@ -136,12 +139,11 @@ const default_doc = { "type": "environment" }] } as const; - /** * This is a workaround for JSON imports syntax changing every other day and the fact we _might_ mutate the document in place one returned * @returns a pristine default document that we are free to mutate or otherwise modify */ -export default async function getDefaultDocument(){ +export default function getDefaultDocument(): IDocument&{nodes:number[], scene: 0,scenes:[IScene], setups:[ISetup]}{ /** @fixme structuredClone is only available starting with node-17. Remove this check once node-16 support is dropped */ //@ts-ignore return (typeof structuredClone ==="function")?structuredClone(default_doc) : JSON.parse(JSON.stringify(default_doc)); diff --git a/source/server/utils/wrapAsync.ts b/source/server/utils/wrapAsync.ts index 3e0ff357c..22783afd4 100644 --- a/source/server/utils/wrapAsync.ts +++ b/source/server/utils/wrapAsync.ts @@ -2,7 +2,7 @@ import { Request, RequestHandler, Response, NextFunction } from "express"; -interface AsyncRequestHandler{ +export interface AsyncRequestHandler{ ( req: Request, res: Response, diff --git a/source/server/vfs/Base.ts b/source/server/vfs/Base.ts index 7525396bd..2441bfffa 100644 --- a/source/server/vfs/Base.ts +++ b/source/server/vfs/Base.ts @@ -8,6 +8,11 @@ import { mkdir } from "fs/promises"; export type Isolate = (this: that, vfs :that)=> Promise; +/** + * Branded type to make a distinction between absolute path and ROOT_DIR relative paths + */ +export type RootRelativePath = string & {_brand: "RootRelativePath"}; + export default abstract class BaseVfs extends DbController{ constructor(protected rootDir :string, db :Database){ @@ -49,7 +54,11 @@ export default abstract class BaseVfs extends DbController{ * @param filepath */ public relative(filepath: string){ - return path.relative(this.baseDir, path.resolve(this.baseDir, filepath)); + return path.relative(this.baseDir, path.resolve(this.baseDir, filepath)) as RootRelativePath; + } + + public absolute(filepath: RootRelativePath|string){ + return path.isAbsolute(filepath)? filepath: path.resolve(this.baseDir, filepath); } diff --git a/source/server/vfs/vfs.test.ts b/source/server/vfs/vfs.test.ts index 8a0cbb89d..7f53f162f 100755 --- a/source/server/vfs/vfs.test.ts +++ b/source/server/vfs/vfs.test.ts @@ -53,6 +53,20 @@ describe("Vfs", function(){ it("works with paths that are already relative", function(){ expect(vfs.relative("test.bin")).to.equal("test.bin"); }); + }); + + describe("absolute()", function(){ + let vfs: Vfs; + this.beforeEach(function(){ + vfs = new Vfs("/path/to/data", {} as any); + }) + it("don't touch absoltue paths", function(){ + expect(vfs.absolute("/something/foo/bar")).to.equal("/something/foo/bar"); + }); + it("resolves paths from root dir", function(){ + expect(vfs.absolute("scenes/foo.txt")).to.equal("/path/to/data/scenes/foo.txt"); + }); + }) describe("", function(){ this.beforeEach(async function(){ diff --git a/source/ui/composants/UploadManager.ts b/source/ui/composants/UploadManager.ts index b03045ae8..d92493e44 100644 --- a/source/ui/composants/UploadManager.ts +++ b/source/ui/composants/UploadManager.ts @@ -58,7 +58,7 @@ export default class UploadManager extends LitElement{ - handleSubmit = (ev:MouseEvent)=>{ + createScene = (ev:MouseEvent)=>{ ev.preventDefault(); ev.stopPropagation(); const form = this.uploadForm; @@ -90,20 +90,50 @@ export default class UploadManager extends LitElement{ }).then(async (res)=>{ await HttpError.okOrThrow(res); let task = await res.json(); - console.log("Body : ", task); if(!task.task_id) throw new Error(`Unexpected body shape: ${JSON.stringify(task)}`); const u = new URL(window.location.href); u.searchParams.append("task", task.task_id); + //Reload the page window.location.href = u.toString(); }).catch((e)=>{ console.error(e); this.error = e.message; }).finally(()=> this.busy = false); - //Reset everything return false; } + extractArchives = (ev:MouseEvent)=>{ + ev.preventDefault(); + ev.stopPropagation(); + this.busy = true; + this.error = null; + const tasks = this.uploader.uploads.map(u=>u.task_id); + fetch("/tasks", { + method: "POST", + headers: { + "Content-Type": "application/json; charset=utf-8" + }, + body: JSON.stringify({ + type: "extractScenesArchives", + data: { + tasks, + } + }) + }).then(async (res)=>{ + await HttpError.okOrThrow(res); + let task = await res.json(); + if(!task.task_id) throw new Error(`Unexpected body shape: ${JSON.stringify(task)}`); + const u = new URL(window.location.href); + //Remove any existing "task" parameters to avoid confusion about what was imported + u.searchParams.set("task", task.task_id); + //Reload the page + window.location.href = u.toString(); + }).catch((e)=>{ + console.error(e); + this.error = e.message; + }).finally(()=> this.busy = false); + } @@ -157,7 +187,7 @@ export default class UploadManager extends LitElement{ protected update(changedProperties: PropertyValues): void { - const models = this.uploader.uploads.filter(u=>u.filetype === "model" || u.filetype === "source"); + const models = this.uploader.uploads.filter(u=>u.isModel); const defaultTitle = models[0]?.filename.split(".").slice(0, -1).join(".") ?? ""; if(this._defaultTitle != defaultTitle && !this.uploader.has_pending_uploads){ @@ -192,6 +222,7 @@ export default class UploadManager extends LitElement{ } let state = "pending"; let stateText: TemplateResult|string = html``; + let filetype: TemplateResult|null = null; let unit = binaryUnit(u.total); let progress = `${formatBytes(u.progress, unit)}/${formatBytes(u.total)}`; if(u.error){ @@ -203,12 +234,36 @@ export default class UploadManager extends LitElement{ stateText = "✓"; progress = formatBytes(u.total); } + if(u.isModel){ + filetype = html` + ` + }else if(u.scenes?.length){ + filetype = html` + + + + + + + ` + } return html`
  • ${stateText} - ${u.filetype?.[0].toUpperCase() ?? ""} + ${filetype} ${u.filename} ${progress} @@ -217,19 +272,19 @@ export default class UploadManager extends LitElement{ `; } + /** * When it looks like the user is uploading model(s) for a scene creation, show this form. * The submit button is shown only when all uploads have settled. */ private renderSceneCreationForm(){ - const has_model = this.uploader.uploads.findIndex(u=>u.filetype === "model" || u.filetype === "source") !== -1 + const can_submit = this.uploader.uploads.findIndex(u=>u.isModel) !== -1; return html` -
    -
    ${this.error}
    +
    ${this.error}
    ${(()=>{ if(this.uploader.has_pending_uploads|| this.busy) return html`` else if(this.uploader.has_errors) return html`Some uploads have failed` - else if(this.uploader.size && has_model) return html`` + else if(this.uploader.size && can_submit) return html`` else if(this.uploader.size) return html`Provide at least one model` else return null; })()} @@ -241,12 +296,20 @@ export default class UploadManager extends LitElement{ * Renders the details of zipfiles contents */ private renderScenesContentSummary(){ + const can_submit = this.uploader.uploads.findIndex(u=>u.scenes?.length) !== -1; return html` Scenes:
      ${this.uploader.uploads.map(u=>(u.scenes?.map(s=>html`
    • [${s.action.toUpperCase()}] ${s.name}
      • `)) ?? [])}
    - +
    ${this.error}
    + ${(()=>{ + if(this.uploader.has_pending_uploads|| this.busy) return html`` + else if(this.uploader.has_errors) return html`Some uploads have failed` + else if(this.uploader.size && can_submit) return html`` + else return null; + })()} +
    `; } @@ -260,7 +323,7 @@ export default class UploadManager extends LitElement{ protected render(): unknown { const uploads = this.uploader.uploads; const is_active = uploads.some(u=>!u.done && !u.error ); - const scene_archives = uploads.filter(u=>u.filetype === "archive"); + const scene_archives = uploads.filter(u=>u.mime === "application/zip"); let form_content :TemplateResult|null = null; if(scene_archives.length && scene_archives.length == uploads.length){ form_content = this.renderScenesContentSummary(); @@ -364,19 +427,10 @@ export default class UploadManager extends LitElement{ font-family: monospace; font-weight: bold; align-self: center; - &:not(:empty)::before{ - content: "["; - } - &:not(:empty)::after{ - content: "]"; - } - &.filetype-u{ - color: var(--color-info); - } - &.filetype-s{ + &.filetype-source{ color: var(--color-warning); } - &.filetype-m{ + &.filetype-glb{ color: var(--color-success); } } diff --git a/source/ui/state/uploader.ts b/source/ui/state/uploader.ts index 69c0bfb82..3b79da231 100644 --- a/source/ui/state/uploader.ts +++ b/source/ui/state/uploader.ts @@ -12,7 +12,8 @@ export interface UploadOperation{ //Unique ID of the upload. Might be different from "name" when we upload a scene zip id: string; filename: string; - filetype?: "archive"|"model"|"source"|"unknown"; + mime?: string; + isModel?: boolean; /** The file to upload. Should be required? */ file?: File; /** When the file is an archive we will store the list of files it contains here */ @@ -30,7 +31,7 @@ export interface UploadOperation{ } export interface ParsedUploadTaskOutput{ - filetype: "archive"|"model", + mime: string, scenes?: SceneUploadResult[], files?: string[], } @@ -200,7 +201,7 @@ export class Uploader implements ReactiveController{ }); await HttpError.okOrThrow(res); const body = await res.json(); - if(!body.output || typeof body.output !== "object" || !body.output.filetype){ + if(!body.output || typeof body.output !== "object" || !body.output.mime){ console.warn("Unexpected format for scene output:", body); throw new Error("Invalid response body"); } From 7207e9d5149121e90632dcfe83e67b126186b19d Mon Sep 17 00:00:00 2001 From: Sebastien DUMETZ Date: Fri, 20 Feb 2026 17:11:51 +0100 Subject: [PATCH 08/14] tasks UI: views, templates, and styles --- source/server/routes/tasks/index.ts | 4 +- .../routes/tasks/task/artifacts/put.test.ts | 4 +- source/server/routes/tasks/task/get.ts | 9 +- source/server/routes/tasks/task/tree/get.ts | 31 +++ source/server/routes/views/index.ts | 71 +++++- source/server/tasks/manager.test.ts | 166 ++++++++++++- source/server/tasks/manager.ts | 223 +++++++++++++++++- source/server/tasks/types.ts | 50 ++++ source/server/templates/task.hbs | 88 +++++++ source/server/templates/tasks.hbs | 93 ++++++++ source/ui/state/uploader.ts | 4 +- source/ui/styles/forms.scss | 3 + source/ui/styles/main.scss | 1 + source/ui/styles/tables.scss | 19 ++ source/ui/styles/tasks.scss | 105 +++++++++ source/ui/styles/titles.scss | 10 + 16 files changed, 872 insertions(+), 9 deletions(-) create mode 100644 source/server/routes/tasks/task/tree/get.ts create mode 100644 source/server/templates/task.hbs create mode 100644 source/server/templates/tasks.hbs create mode 100644 source/ui/styles/tasks.scss diff --git a/source/server/routes/tasks/index.ts b/source/server/routes/tasks/index.ts index ff05eaa15..f50dfe4e9 100644 --- a/source/server/routes/tasks/index.ts +++ b/source/server/routes/tasks/index.ts @@ -10,6 +10,7 @@ import bodyParser from "body-parser"; import { getTaskArtifact } from "./task/artifacts/get.js"; import { getTask } from "./task/get.js"; import { deleteTask } from "./task/delete.js"; +import { getTaskTree } from "./task/tree/get.js"; import { UnauthorizedError } from "../../utils/errors.js"; import { AccessType, toAccessLevel } from "../../auth/UserManager.js"; @@ -43,11 +44,12 @@ function taskAccess(name:AccessType){ return next(); } }).catch(next); -} + } } router.get("/:id(\\d+)", isUser, wrap(getTask)); +router.get("/:id(\\d+)/tree", taskAccess("read"), wrap(getTaskTree)); router.delete("/:id(\\d+)", taskAccess("admin"), wrap(deleteTask)); router.put("/:id(\\d+)/artifact", taskAccess("admin"), wrap(putTaskArtifact)); router.get("/:id(\\d+)/artifact", taskAccess("read"), wrap(getTaskArtifact)); diff --git a/source/server/routes/tasks/task/artifacts/put.test.ts b/source/server/routes/tasks/task/artifacts/put.test.ts index b44444685..8a39cd80c 100644 --- a/source/server/routes/tasks/task/artifacts/put.test.ts +++ b/source/server/routes/tasks/task/artifacts/put.test.ts @@ -110,8 +110,8 @@ describe("PUT /tasks/:id/artifact", function(){ .auth("alice", "12345678") .expect(200); - expect(body).to.have.property("status", "success"); - expect(body).to.have.property("output").to.deep.equal({ + expect(body).to.have.property("task").to.have.property("status", "success"); + expect(body).to.have.property("task").to.have.property("output").to.deep.equal({ fileLocation: `artifacts/${task}/${filename}`, isModel: false, mime: "application/octet-stream", diff --git a/source/server/routes/tasks/task/get.ts b/source/server/routes/tasks/task/get.ts index 1849ec050..3360fe8ce 100644 --- a/source/server/routes/tasks/task/get.ts +++ b/source/server/routes/tasks/task/get.ts @@ -2,6 +2,12 @@ import { Request, Response } from "express"; import { UnauthorizedError } from "../../../utils/errors.js"; import { getLocals, getUser } from "../../../utils/locals.js"; import { toAccessLevel } from "../../../auth/UserManager.js"; +import { TaskDefinition, TaskDataPayload, TaskLogEntry } from "../../../tasks/types.js"; + +export interface TaskResponse{ + task: TaskDefinition; + logs: TaskLogEntry[]; +} export async function getTask(req: Request, res: Response){ @@ -21,5 +27,6 @@ export async function getTask(req: Request, res: Response){ ){ throw new UnauthorizedError(`Read rights are required to delete tasks`); } - res.status(200).send(task); + const logs = await taskScheduler.getLogs(id); + res.status(200).send({task, logs}); } \ No newline at end of file diff --git a/source/server/routes/tasks/task/tree/get.ts b/source/server/routes/tasks/task/tree/get.ts new file mode 100644 index 000000000..3471a604b --- /dev/null +++ b/source/server/routes/tasks/task/tree/get.ts @@ -0,0 +1,31 @@ +import { Request, Response } from "express"; +import { UnauthorizedError } from "../../../../utils/errors.js"; +import { getLocals, getUser } from "../../../../utils/locals.js"; +import { toAccessLevel } from "../../../../auth/UserManager.js"; +import { TaskDataPayload, TaskLogEntry, TaskNode } from "../../../../tasks/types.js"; + +export interface TaskTreeResponse{ + task: TaskNode; + logs: TaskLogEntry[]; +} + + +export async function getTaskTree(req: Request, res: Response){ + const { + taskScheduler, + userManager, + } = getLocals(req); + const requester = getUser(req)!; + const {id: idString} = req.params; + const id = parseInt(idString); + + const {root, logs} = await taskScheduler.getTaskTree(id); + + if(requester.level !== "admin" + && root.user_id !== requester.uid + && toAccessLevel(await userManager.getAccessRights(root.scene_id, requester.uid)) < toAccessLevel("read") + ){ + throw new UnauthorizedError(`Read rights are required to access task trees`); + } + res.status(200).send({task: root, logs}); +} diff --git a/source/server/routes/views/index.ts b/source/server/routes/views/index.ts index 0b0d277f9..f00d7d865 100644 --- a/source/server/routes/views/index.ts +++ b/source/server/routes/views/index.ts @@ -1,5 +1,5 @@ import { Router, Request, Response, NextFunction } from "express"; -import { canRead, getHost, canWrite, getSession, getVfs, getUser, isAdministrator, getUserManager, isMemberOrManage, isManage, isEmbed, useTemplateProperties, getTaskScheduler } from "../../utils/locals.js"; +import { canRead, getHost, canWrite, getSession, getVfs, getUser, isAdministrator, getUserManager, isMemberOrManage, isManage, isEmbed, useTemplateProperties, getTaskScheduler, getLocals } from "../../utils/locals.js"; import wrap from "../../utils/wrapAsync.js"; import path from "path"; import { Scene } from "../../vfs/types.js"; @@ -519,6 +519,75 @@ routes.get("/standalone", (req, res)=>{ }); +routes.get("/tasks", wrap(async (req, res) => { + const user = getUser(req); + if (!user || user.level === "none") { + throw new ForbiddenError("You must be logged in to view your tasks"); + } + const taskScheduler = getTaskScheduler(req); + // Parse and validate query params + const rawOwner = typeof req.query.owner === 'string' ? req.query.owner : undefined; + const rawType = typeof req.query.type === 'string' ? req.query.type : undefined; + const rawStatus = typeof req.query.status === 'string' ? req.query.status : undefined; + + // owner: 'mine'|'all' (only admins may request 'all') + const owner = rawOwner === 'all' ? 'all' : 'mine'; + if (owner === 'all' && user.level !== 'admin') { + throw new ForbiddenError("Only administrators can list all users' tasks"); + } + + // status: 'all'|'success'|'error' + const allowedStatus = ['all', 'success', 'error']; + const status = allowedStatus.includes(rawStatus ?? '') ? (rawStatus as 'all'|'success'|'error') : 'all'; + if(rawStatus && status !== rawStatus){ + throw new BadRequestError(`Invalid status requested : ${rawStatus}`); + } + // type: optional string, limit length to avoid abuse + let type: string | undefined = undefined; + if (rawType) { + if (rawType.length > 200) throw new BadRequestError("type parameter too long"); + type = rawType; + } + + // rootOnly: optional boolean flag controlling whether only root tasks are returned. + const rootOnly = qsToBool(req.query.rootOnly) ?? true; + + const userId = owner === 'mine' ? user.uid : undefined; + + const tasks = await taskScheduler.getTasks({ user_id: userId, type, status, rootOnly }); + console.log("Root : ", typeof rootOnly, rootOnly) + res.render("tasks", { + title: "My tasks", + tasks, + params: { owner, type, status, rootOnly }, + }); +})); + +routes.get("/tasks/:id(\\d+)", wrap(async (req, res) => { + const { + taskScheduler, + userManager, + vfs, + requester + } = getLocals(req); + const id = parseInt(req.params.id); + const validLevels = ["debug", "log", "warn", "error"] as const; + type Level = typeof validLevels[number]; + const rawLevel = req.query.level as string | undefined; + const level: Level = (validLevels as readonly string[]).includes(rawLevel ?? "") ? rawLevel as Level : "log"; + const {root, logs} = await taskScheduler.getTaskTree(id, {level}); + + const owner = root.user_id? (root.user_id == requester?.uid ?requester.username :(await userManager.getUserById(root.user_id)).username):null; + const scene = root.scene_id? (await vfs.getScene(root.scene_id)).name : null; + res.render("task", { + title: `Task #${id} — ${root.type}`, + root, + logs, + level, + owner, + scene, + }); +})); export default routes; \ No newline at end of file diff --git a/source/server/tasks/manager.test.ts b/source/server/tasks/manager.test.ts index 0d50ce67f..5cecf6956 100644 --- a/source/server/tasks/manager.test.ts +++ b/source/server/tasks/manager.test.ts @@ -265,4 +265,168 @@ describe("TaskManager", function(){ expect(retrieved.output).to.be.an("object"); expect(retrieved.output).to.deep.equal(testOutput); }); -}); \ No newline at end of file + + describe("getTaskTree()", function(){ + it("returns the root task with empty logs when there are none", async function(){ + const root = await listener.create({scene_id: null, user_id: null, type: "root", data: {}}); + const {root: rootNode, logs} = await listener.getTaskTree(root.task_id); + + expect(logs).to.deep.equal([]); + expect(rootNode.task_id).to.equal(root.task_id); + expect(rootNode.children).to.deep.equal([]); + }); + + it("throws NotFoundError for a non-existent task id", async function(){ + await expect(listener.getTaskTree(99999)).to.be.rejectedWith(NotFoundError); + }); + + it("returns logs for a single task, ordered by log_id ASC", async function(){ + const root = await listener.create({scene_id: null, user_id: null, type: "root", data: {}}); + + await handle.run( + `INSERT INTO tasks_logs(fk_task_id, severity, message) VALUES ($1, 'log', 'first'), ($1, 'warn', 'second')`, + [root.task_id] + ); + + const {root: rootNode, logs} = await listener.getTaskTree(root.task_id); + + expect(rootNode.children).to.deep.equal([]); + expect(logs).to.have.length(2); + expect(logs[0].message).to.equal("first"); + expect(logs[1].message).to.equal("second"); + expect(logs[0].log_id).to.be.lessThan(logs[1].log_id); + expect(logs[0].task_id).to.equal(root.task_id); + expect(logs[1].task_id).to.equal(root.task_id); + }); + + it("returns parent and direct children with their respective logs", async function(){ + const root = await listener.create({scene_id: null, user_id: null, type: "root", data: {}}); + const child1 = await listener.create({scene_id: null, user_id: null, type: "child", data: {}, parent: root.task_id}); + const child2 = await listener.create({scene_id: null, user_id: null, type: "child", data: {}, parent: root.task_id}); + + await handle.run(`INSERT INTO tasks_logs(fk_task_id, severity, message) VALUES ($1, 'log', 'root-log')`, [root.task_id]); + await handle.run(`INSERT INTO tasks_logs(fk_task_id, severity, message) VALUES ($1, 'log', 'child1-log')`, [child1.task_id]); + await handle.run(`INSERT INTO tasks_logs(fk_task_id, severity, message) VALUES ($1, 'error', 'child2-log')`, [child2.task_id]); + + const {root: rootNode, logs} = await listener.getTaskTree(root.task_id); + + // All logs present, ordered by log_id + expect(logs).to.have.length(3); + const logIds = logs.map(l => l.log_id); + expect(logIds).to.deep.equal([...logIds].sort((a, b) => a - b)); + + // Root node carries both children + expect(rootNode.children.map(c => c.task_id).sort()).to.deep.equal([child1.task_id, child2.task_id].sort()); + + // Children carry the parent id and no children of their own + const childNode1 = rootNode.children.find(c => c.task_id === child1.task_id)!; + expect(childNode1.parent).to.equal(root.task_id); + expect(childNode1.children).to.deep.equal([]); + }); + + it("fetches deeply nested (grandchild) tasks recursively", async function(){ + const root = await listener.create({scene_id: null, user_id: null, type: "root", data: {}}); + const child = await listener.create({scene_id: null, user_id: null, type: "child", data: {}, parent: root.task_id}); + const grand = await listener.create({scene_id: null, user_id: null, type: "grandchild", data: {}, parent: child.task_id}); + + await handle.run(`INSERT INTO tasks_logs(fk_task_id, severity, message) VALUES ($1, 'log', 'grand-log')`, [grand.task_id]); + + const {root: rootNode, logs} = await listener.getTaskTree(root.task_id); + + // Log comes from the grandchild + expect(logs).to.have.length(1); + expect(logs[0].task_id).to.equal(grand.task_id); + + // Graph is properly linked: root -> child -> grandchild + expect(rootNode.children).to.have.length(1); + const childNode = rootNode.children[0]; + expect(childNode.task_id).to.equal(child.task_id); + expect(childNode.children).to.have.length(1); + expect(childNode.children[0].task_id).to.equal(grand.task_id); + }); + + it("does not include tasks outside the requested subtree", async function(){ + const root = await listener.create({scene_id: null, user_id: null, type: "root", data: {}}); + const child = await listener.create({scene_id: null, user_id: null, type: "child", data: {}, parent: root.task_id}); + const unrelated = await listener.create({scene_id: null, user_id: null, type: "other", data: {}}); + + // Fetching only the child subtree should not include root or unrelated + const {root: subtreeRoot} = await listener.getTaskTree(child.task_id); + expect(subtreeRoot.task_id).to.equal(child.task_id); + expect(subtreeRoot.parent).to.equal(root.task_id); // parent id is preserved + expect(subtreeRoot.children).to.deep.equal([]); // but parent node is not included + }); + + it("logs from all tasks in the tree are merged and sorted by log_id", async function(){ + const root = await listener.create({scene_id: null, user_id: null, type: "root", data: {}}); + const child = await listener.create({scene_id: null, user_id: null, type: "child", data: {}, parent: root.task_id}); + + // Interleave inserts from different tasks to test global ordering + await handle.run(`INSERT INTO tasks_logs(fk_task_id, severity, message) VALUES ($1, 'log', 'a')`, [root.task_id]); + await handle.run(`INSERT INTO tasks_logs(fk_task_id, severity, message) VALUES ($1, 'log', 'b')`, [child.task_id]); + await handle.run(`INSERT INTO tasks_logs(fk_task_id, severity, message) VALUES ($1, 'log', 'c')`, [root.task_id]); + + const {logs} = await listener.getTaskTree(root.task_id); + + expect(logs).to.have.length(3); + const logIds = logs.map(l => l.log_id); + expect(logIds).to.deep.equal([...logIds].sort((a, b) => a - b)); + expect(logs.map(l => l.message)).to.deep.equal(['a', 'b', 'c']); + }); + + it("filters logs by minimum severity level", async function(){ + const root = await listener.create({scene_id: null, user_id: null, type: "root", data: {}}); + + await handle.run(`INSERT INTO tasks_logs(fk_task_id, severity, message) VALUES ($1, 'debug', 'msg-debug')`, [root.task_id]); + await handle.run(`INSERT INTO tasks_logs(fk_task_id, severity, message) VALUES ($1, 'log', 'msg-log')`, [root.task_id]); + await handle.run(`INSERT INTO tasks_logs(fk_task_id, severity, message) VALUES ($1, 'warn', 'msg-warn')`, [root.task_id]); + await handle.run(`INSERT INTO tasks_logs(fk_task_id, severity, message) VALUES ($1, 'error', 'msg-error')`, [root.task_id]); + + // no filter → all four lines + const {logs: all} = await listener.getTaskTree(root.task_id); + expect(all.map(l => l.message)).to.deep.equal(['msg-debug', 'msg-log', 'msg-warn', 'msg-error']); + + // level: 'log' → excludes 'debug' + const {logs: fromLog} = await listener.getTaskTree(root.task_id, {level: 'log'}); + expect(fromLog.map(l => l.message)).to.deep.equal(['msg-log', 'msg-warn', 'msg-error']); + + // level: 'warn' → only warn and error + const {logs: fromWarn} = await listener.getTaskTree(root.task_id, {level: 'warn'}); + expect(fromWarn.map(l => l.message)).to.deep.equal(['msg-warn', 'msg-error']); + + // level: 'error' → only errors + const {logs: fromError} = await listener.getTaskTree(root.task_id, {level: 'error'}); + expect(fromError.map(l => l.message)).to.deep.equal(['msg-error']); + }); + }); + + describe("getTasks()", function(){ + it("returns only root tasks by default when filtering by user", async function(){ + const root = await listener.create({scene_id: null, user_id, type: "root", data: {}}); + const child = await listener.create({scene_id: null, user_id, type: "child", data: {}, parent: root.task_id}); + + const tasks = await listener.getTasks({ user_id }); + expect(tasks).to.have.length(1); + expect(tasks[0].task_id).to.equal(root.task_id); + }); + + it("returns child tasks as well when rootOnly is false", async function(){ + const root = await listener.create({scene_id: null, user_id, type: "root", data: {}}); + const child = await listener.create({scene_id: null, user_id, type: "child", data: {}, parent: root.task_id}); + + const tasks = await listener.getTasks({ user_id, rootOnly: false }); + // both root and child should be returned + const ids = tasks.map(t => t.task_id).sort((a,b)=>a-b); + expect(ids).to.deep.equal([root.task_id, child.task_id].sort((a,b)=>a-b)); + }); + + it("applies exact type matching", async function(){ + const a = await listener.create({scene_id: null, user_id, type: "delayTask", data: {}}); + const b = await listener.create({scene_id: null, user_id, type: "other", data: {}}); + + const tasks = await listener.getTasks({ user_id, type: 'delayTask' }); + expect(tasks).to.have.length(1); + expect(tasks[0].type).to.equal('delayTask'); + }); + }); +}); diff --git a/source/server/tasks/manager.ts b/source/server/tasks/manager.ts index 3655190c8..0ca628366 100644 --- a/source/server/tasks/manager.ts +++ b/source/server/tasks/manager.ts @@ -2,7 +2,7 @@ import { debuglog } from "node:util"; import { BadRequestError, HTTPError, NotFoundError } from "../utils/errors.js"; import { DatabaseHandle } from "../vfs/helpers/db.js"; import { serializeTaskError } from "./errors.js"; -import { TaskStatus, TaskDataPayload, TaskDefinition, CreateTaskParams } from "./types.js"; +import { TaskStatus, TaskDataPayload, TaskDefinition, CreateTaskParams, TaskNode, TaskLogEntry, TaskTreeResult, LogSeverity } from "./types.js"; const debug_status = debuglog("tasks:status"); const debug_logs = debuglog("tasks:logs"); @@ -102,6 +102,21 @@ export class TaskManager{ status `; + // Same as #taskColumns but with every column qualified by the tasks table alias, + // needed inside recursive CTEs where tasks is joined against a CTE that also + // exposes task_id (causing an "ambiguous column" error in PostgreSQL). + static #qualifiedTaskColumns = ` + tasks.fk_scene_id AS scene_id, + tasks.fk_user_id AS user_id, + tasks.task_id, + tasks.ctime, + tasks.type, + tasks.parent, + tasks.data, + tasks.output, + tasks.status + `; + public async create({scene_id, user_id, type, data, status='pending', parent=null}: CreateTaskParams): Promise>{ let args = [scene_id, type, data ?? {}, status, user_id, parent]; let task = await this.db.all>(` @@ -124,6 +139,212 @@ export class TaskManager{ return task[0]; } + /** + * Returns all log lines for a single task, ordered by `log_id ASC`. + * @throws {NotFoundError} if no task with `id` exists + */ + public async getLogs(id: number): Promise { + // Verify task existence first so callers get a proper NotFoundError + await this.getTask(id); + return this.db.all(` + SELECT + log_id, + fk_task_id AS task_id, + timestamp, + severity, + message + FROM tasks_logs + WHERE fk_task_id = $1 + ORDER BY log_id ASC + `, [id]); + } + + /** + * Fetches a task by id together with **all descendants** (tasks whose `parent` + * chain leads back to `id`), and every log line produced by any of those tasks. + * + * The query is issued as a **single atomic statement** using a recursive CTE so + * the result is a consistent snapshot even under concurrent writes. + * + * Returned structure: + * - `root` – the requested {@link TaskNode} with descendants nested under `children`. + * - `logs` – flat array of {@link TaskLogEntry} ordered by `log_id ASC`, + * optionally filtered to lines at or above `options.level`. + * + * @param options.level Minimum severity to include. Defaults to `'debug'` (all lines). + * Severity order: `debug` < `log` < `warn` < `error`. + * @throws {NotFoundError} when no task with `id` exists + */ + public async getTaskTree( + id: number, + options?: { level?: LogSeverity } + ): Promise> { + /* + * One atomic query: + * 1. Recursive CTE `tree` walks the task graph depth-first. + * 2. Left-join with tasks_logs to pull every log line in the same pass, + * filtered by minimum severity using PostgreSQL's ENUM ordering. + * 3. The outer SELECT returns every (task, log?) pair; rows with no + * matching logs still appear once thanks to LEFT JOIN. + * + * Post-processing in JS is O(n) and avoids a second round-trip. + */ + const level: LogSeverity = options?.level ?? 'debug'; + const rows = await this.db.all<{ + // task columns + scene_id: number; + user_id: number; + task_id: number; + ctime: Date; + type: string; + parent: number | null; + data: TData extends undefined ? {} : TData; + output: TReturn; + status: string; + // log columns (nullable – task may have no logs) + log_id: number | null; + log_task_id: number | null; + timestamp: Date | null; + severity: string | null; + message: string | null; + }>(` + WITH RECURSIVE tree AS ( + SELECT ${TaskManager.#taskColumns} + FROM tasks + WHERE task_id = $1 + + UNION ALL + + SELECT ${TaskManager.#qualifiedTaskColumns} + FROM tasks + INNER JOIN tree ON tasks.parent = tree.task_id + ) + SELECT + tree.*, + tl.log_id, + tl.fk_task_id AS log_task_id, + tl.timestamp, + tl.severity, + tl.message + FROM tree + LEFT JOIN tasks_logs tl ON tl.fk_task_id = tree.task_id + AND tl.severity >= $2::log_severity + ORDER BY tl.log_id ASC NULLS FIRST + `, [id, level]); + + if (!rows.length) { + throw new NotFoundError(`No task found with id ${id}`); + } + + // --- assemble tasks (deduplicate by task_id) and logs (deduplicate by log_id) --- + const taskMap = new Map>(); + const logs: TaskLogEntry[] = []; + const seenLogIds = new Set(); + + for (const row of rows) { + if (!taskMap.has(row.task_id)) { + taskMap.set(row.task_id, { + scene_id: row.scene_id, + user_id: row.user_id, + task_id: row.task_id, + ctime: row.ctime, + type: row.type, + parent: row.parent, + after: [], + data: row.data, + output: row.output, + status: row.status as any, + children: [], + }); + } + + if (row.log_id !== null && !seenLogIds.has(row.log_id)) { + seenLogIds.add(row.log_id); + logs.push({ + log_id: row.log_id, + task_id: row.log_task_id!, + timestamp: row.timestamp!, + severity: row.severity as any, + message: row.message!, + }); + } + } + + // Wire up children and find the root (the node whose parent is not in the tree) + let root: TaskNode | undefined; + for (const node of taskMap.values()) { + if (node.parent !== null && taskMap.has(node.parent)) { + taskMap.get(node.parent)!.children.push(node); + } else { + root = node; + } + } + + return { root: root!, logs }; + } + + /** + * Returns all **root** tasks (tasks without a parent) created by a given user, + * ordered by creation time descending (newest first). + * + * Only root tasks are returned because child tasks are accessible via + * {@link getTaskTree} from their parent. Returning the full flat list would + * duplicate every child in the summary and make the view noisy. + * + * @param userId The `uid` of the user whose tasks to fetch. + */ + /** + * Generic task listing with optional filters. + * + * Options: + * - `user_id` : filter by `fk_user_id` + * - `type` : regular expression match against `type` column + * - `scene_id`: filter by `fk_scene_id` + * + * Returns tasks ordered by `ctime DESC` (newest first). + */ + public async getTasks(options: { user_id?: number; type?: string; scene_id?: number; rootOnly?: boolean; status?: 'success'|'error'|'all' } = {}): Promise { + const where: string[] = []; + const params: any[] = []; + + if (options.user_id != null) { + where.push(`fk_user_id = $${params.push(options.user_id)}`); + } + + if (options.scene_id != null) { + where.push(`fk_scene_id = $${params.push(options.scene_id)}`); + } + + + if (options.type) { + // Exact match on `type` (previously supported regex). Use equality to + // avoid surprising regex semantics and to keep behavior predictable. + where.push(`type = $${params.push(options.type)}`); + } + + // By default, return only root tasks (no parent). Caller can set `rootOnly: false` + // to include child tasks as well. This preserves previous summary behaviour. + if (options.rootOnly !== false) { + where.push(`parent IS NULL`); + } + + // Status filter: 'success' or 'error' or 'all' (no filter) + if (options.status && options.status !== 'all') { + where.push(`status = $${params.push(options.status)}`); + } + + const whereClause = where.length ? `WHERE ${where.join(" AND ")}` : ""; + + return this.db.all(` + SELECT ${TaskManager.#qualifiedTaskColumns}, scenes.scene_name AS scene, users.username AS owner + FROM tasks + LEFT JOIN scenes ON scenes.scene_id = tasks.fk_scene_id + LEFT JOIN users ON users.user_id = tasks.fk_user_id + ${whereClause} + ORDER BY tasks.ctime DESC + `, params); + } + /** * Deletes a task from the database * Deletion will cascade to any dependents. diff --git a/source/server/tasks/types.ts b/source/server/tasks/types.ts index 93d941b78..8c67e5a9b 100644 --- a/source/server/tasks/types.ts +++ b/source/server/tasks/types.ts @@ -134,6 +134,56 @@ export interface ITaskLogger{ export type LogSeverity = keyof ITaskLogger; +/** + * A single log line produced by a task + */ +export interface TaskLogEntry{ + log_id: number; + task_id: number; + timestamp: Date; + severity: LogSeverity; + message: string; +} + +/** + * A task node in the tree, carrying its direct children. + * The `parent` field allows reconstructing the graph from a flat list if needed. + */ +export interface TaskNode + extends TaskDefinition { + children: TaskNode[]; +} + +/** + * Task list item returned by `getTasks()` in summary views. + * + * Includes nullable `scene` and `owner` fields which map to the referenced + * `scenes.scene_name` and `users.username` respectively. These fields may be + * `null` when the task has no linked scene or user. + */ +export interface TaskListItem + extends TaskDefinition { + scene: string | null; + owner: string | null; +} + +/** + * Result returned by {@link TaskManager.getTaskTree}. + * + * - `root` is the requested task as a {@link TaskNode}; its `children` array + * contains the direct children, each of which recursively carries their own + * `children`, forming a proper tree. The `parent` field on every node is + * preserved so the tree can also be flattened back to a list if needed. + * - `logs` is a **flat, ordered array** of every log line produced by any task in + * the tree, sorted by `log_id ASC` (i.e. insertion order). + */ +export interface TaskTreeResult{ + /** The requested task, with descendants nested under `children` recursively */ + root: TaskNode; + /** All log lines from every task in the tree, ordered by log_id ASC */ + logs: TaskLogEntry[]; +} + export interface TaskHandlerDefinition{ readonly type: string; handle: TaskHandler; diff --git a/source/server/templates/task.hbs b/source/server/templates/task.hbs new file mode 100644 index 000000000..12e856967 --- /dev/null +++ b/source/server/templates/task.hbs @@ -0,0 +1,88 @@ + +
    + +

    + Task #{{root.task_id}} — {{root.type}} + + ‹ + +

    + + {{#if root.scene_id}} +

    Scene #{{root.scene_id}}

    + {{/if}} + +
    +
    +

    {{i18n "titles.taskTree"}}

    + {{#if root.parent}}Child of #{{root.parent}}{{/if}} +
    +
    + {{> taskNode root}} +
    +
    + +
    +
    +

    {{i18n "titles.logs" default="Logs"}} ({{logs.length}})

    +
    +
    + +
    +
    +
    + {{#if logs.length}} +
    + + + + + + + + + + + {{#each logs}} + + + + + + + {{/each}} + +
    timetasklevelmessage
    {{dateString timestamp}}#{{task_id}}{{severity}}{{message}}
    +
    + {{else}} +

    {{i18n "labels.noLogs" default="No logs."}}

    + {{/if}} +
    + +
    + +{{!-- Recursive partial for task nodes --}} +{{#*inline "taskNode"}} +
    +
    + #{{task_id}} + {{type}} + {{status}} +
    + {{#if children.length}} +
    + {{#each children}} + {{> taskNode this}} + {{/each}} +
    + {{/if}} +
    +{{/inline}} diff --git a/source/server/templates/tasks.hbs b/source/server/templates/tasks.hbs new file mode 100644 index 000000000..19470250c --- /dev/null +++ b/source/server/templates/tasks.hbs @@ -0,0 +1,93 @@ + +
    + +

    {{i18n "titles.myTasks" default="My tasks"}}

    + +
    +
    + {{#if (test user.level "==" "admin")}} + + {{/if}} + + + + + + +
    + {{#if tasks.length}} + + + + + {{#if (test params.owner "==" "all") }}{{/if}} + {{#unless params.rootOnly }}{{/unless}} + + + + + + + {{#each tasks}} + + + {{#if (test @root.params.owner "==" "all") }}{{/if}} + {{#unless @root.params.rootOnly }} + + {{/unless}} + + + + + + {{/each}} + +
    {{i18n "fields.ctime"}}{{i18n "fields.owner"}}{{i18n "fields.parent"}}{{i18n "fields.scene"}}{{i18n "fields.type"}}{{i18n "fields.status"}}
    {{dateString ctime}}{{owner}} + {{#if parent}} + #{{ parent }} + {{else}} + — + {{/if}} + + {{#if scene}} + {{ scene }} + {{else}} + — + {{/if}} + + + {{type}}#{{task_id}} + + + + {{status}} + +
    + {{else}} +

    {{i18n "labels.noTasks" default="No tasks found."}}

    + {{/if}} +
    + +
    diff --git a/source/ui/state/uploader.ts b/source/ui/state/uploader.ts index 3b79da231..49658f9d8 100644 --- a/source/ui/state/uploader.ts +++ b/source/ui/state/uploader.ts @@ -201,12 +201,12 @@ export class Uploader implements ReactiveController{ }); await HttpError.okOrThrow(res); const body = await res.json(); - if(!body.output || typeof body.output !== "object" || !body.output.mime){ + if(!body?.task?.output || typeof body.task.output !== "object" || !body.task.output.mime){ console.warn("Unexpected format for scene output:", body); throw new Error("Invalid response body"); } - return body.output; + return body.task.output; } diff --git a/source/ui/styles/forms.scss b/source/ui/styles/forms.scss index 253cd5e99..d9c232ca3 100644 --- a/source/ui/styles/forms.scss +++ b/source/ui/styles/forms.scss @@ -19,6 +19,9 @@ display:flex; justify-content: stretch; flex-direction: row; + .form-item > select{ + width: auto; + } } &.column{ display:flex; diff --git a/source/ui/styles/main.scss b/source/ui/styles/main.scss index 00f67b89e..591ed8e17 100644 --- a/source/ui/styles/main.scss +++ b/source/ui/styles/main.scss @@ -3,6 +3,7 @@ @import "./landing.scss"; @import "./card.scss"; @import "./tags.scss"; +@import "./tasks.scss"; html { color: var(--color-text); diff --git a/source/ui/styles/tables.scss b/source/ui/styles/tables.scss index 81ef03bfa..b34c9f8ce 100644 --- a/source/ui/styles/tables.scss +++ b/source/ui/styles/tables.scss @@ -5,9 +5,22 @@ table.list-table { width: 100%; display: table; + @at-root section > &{ + --m: calc(var(--section-padding) * -1); + margin: 0 var(--m); + width: calc(-2 * var(--m) + 100%); + &:first-child{ + margin-top: var(--m); + } + &:last-child{ + margin-bottom: var(--m); + } + } + background: var(--color-element); color: var(--color-text); + tbody tr:nth-child(2n+1){ background: var(--color-section); } @@ -43,6 +56,12 @@ table.list-table { border-left: 0; border-right: 0; white-space: nowrap; + &.compact{ + width: 1%; + } + &.mono{ + font-family: var(--font-mono, monospace); + } } tbody tr:hover{ background: var(--color-dark); diff --git a/source/ui/styles/tasks.scss b/source/ui/styles/tasks.scss new file mode 100644 index 000000000..203cb4f9f --- /dev/null +++ b/source/ui/styles/tasks.scss @@ -0,0 +1,105 @@ +/* Shared styles for task views (compiled/copied to /dist/css/tasks.css by build) + Rules are chosen to prefer the definitions originally in `task.hbs` when conflicts arise. +*/ + +/* Task node/tree */ +.task-tree { + font-family: var(--font-mono, monospace); + container-type: inline-size; + > .task-node { + margin-left: 0; + border-left: none; + } +} +.task-node { + margin: 0; + padding: .25rem .5rem; + border-left: 2px solid var(--color-element, #444); + margin-left: .5rem; +} +.task-header { + display: flex; + flex-wrap: wrap; + align-items: baseline; + gap: .5rem; + padding: .25rem 0; +} +.task-id { color: var(--color-secondary, #aaa); } +.task-type { font-weight: bold; } +.task-children { margin-top: .25rem; } + +/* Status badges (preferred definitions from task.hbs) + Keep these synchronized with other UI components that use the same classes. */ +.task-status { + padding: .1em .45em; + border-radius: .25em; + font-weight: bold; + text-transform: uppercase; + font-size: .8em; + &.task-status-large{ + padding: .5rem ; + width: 75px; + text-align: center; + } +} + +.status-pending, .status-initializing { background: var(--color-element, #444); color: var(--color-light, #eee); } +.status-running { background: #2a6baa; color: #fff; } +.status-success { background: #2a7a4b; color: #fff; } +.status-error { background: #a03030; color: #fff; } +.status-aborting { background: #7a5a20; color: #fff; } + +/* Tasks list table */ +.tasks-table { + width: 100%; + border-collapse: collapse; + font-size: .9em; +} + +.tasks-table th { + text-align: left; + padding: .4rem .6rem; + border-bottom: 2px solid var(--color-element, #444); + white-space: nowrap; +} + +.tasks-table td { + padding: .3rem .6rem; + vertical-align: middle; + border-bottom: 1px solid color-mix(in srgb, var(--color-element, #444) 40%, transparent); +} + +/* Logs table */ +.logs-table { + font-family: var(--font-mono, monospace); +} + +.logs-table th { + text-align: left; + padding: .3rem .5rem; + border-bottom: 1px solid var(--color-element, #444); + color: var(--color-primary, #aaa); + font-weight: bold; + font-size: 110%; +} +.logs-table td { + padding: .2rem .5rem; + vertical-align: top; + border-bottom: 1px solid color-mix(in srgb, var(--color-element, #444) 40%, transparent); +} + +.log-task-id { color: #aaa; white-space: nowrap; width: 1%; } +.log-time { color: #aaa; white-space: nowrap; width: 1%; } +.log-level { white-space: nowrap; width: 1%; } +.log-debug { color: var(--color-info, #aaa); } +.log-log { color: var(--color-light, #eee); } +.log-warn { color: #d4a030; } +.log-error { color: #e05050; } +.log-message { white-space: pre-wrap; word-break: break-all; width: 100%; } + +/* Small responsive helpers */ +@media (max-width: 640px) { + .task-header { gap: .25rem; } + .tasks-table th, .tasks-table td { padding: .25rem .4rem; } + .logs-table th, .logs-table td { padding: .15rem .4rem; } +} diff --git a/source/ui/styles/titles.scss b/source/ui/styles/titles.scss index 57f871643..6a1ad4687 100644 --- a/source/ui/styles/titles.scss +++ b/source/ui/styles/titles.scss @@ -5,6 +5,7 @@ h1, h2, h3{ } h1{ + position: relative; display: inline-block; color: var(--color-text); padding: .5rem 0.2rem 0rem 1.5rem; @@ -13,6 +14,15 @@ h1{ @include text-colors { border-bottom-color: currentColor; } + > .title-caret{ + position: absolute; + left: 0; + height: 100%; + text-decoration: none; + &:hover{ + color: var(--color-primary); + } + } } h2{ From 70cfce1f96dc42ad9d8c981c7a30ac10646ae058 Mon Sep 17 00:00:00 2001 From: Sebastien DUMETZ Date: Tue, 24 Feb 2026 16:56:56 +0100 Subject: [PATCH 09/14] add test objects and scheduler tests --- source/server/__test_fixtures/cube_draco.glb | Bin 0 -> 1316 bytes source/server/__test_fixtures/cube_etc1s.glb | Bin 0 -> 2856 bytes .../server/__test_fixtures/cube_meshopt.glb | Bin 0 -> 2376 bytes .../server/__test_fixtures/cube_textured.glb | Bin 0 -> 2520 bytes source/server/__test_fixtures/cube_webp.glb | Bin 0 -> 2308 bytes source/server/tasks/scheduler.test.ts | 17 +------- source/server/utils/gltf/inspect.test.ts | 38 ++++++++++++++++++ 7 files changed, 39 insertions(+), 16 deletions(-) create mode 100644 source/server/__test_fixtures/cube_draco.glb create mode 100644 source/server/__test_fixtures/cube_etc1s.glb create mode 100644 source/server/__test_fixtures/cube_meshopt.glb create mode 100644 source/server/__test_fixtures/cube_textured.glb create mode 100644 source/server/__test_fixtures/cube_webp.glb create mode 100644 source/server/utils/gltf/inspect.test.ts diff --git a/source/server/__test_fixtures/cube_draco.glb b/source/server/__test_fixtures/cube_draco.glb new file mode 100644 index 0000000000000000000000000000000000000000..1c141961df5485025237fb24d94082fe7570a930 GIT binary patch literal 1316 zcmbVLT~E_c7=F8o8#3`DF(G=>9KG^g+UAgB>O*RE2+AuTJQ%(0s1)K(*7O4D@A zcDu?}X<#**j$QQHjV%}R;6obNnW6Y?RvFJRG$iYAg zvt$Eb|7@wru80|v5n29ApQkejmN_jaaCRer6$H$wwgbNHke6mDU(TBaQHp|DT+ObK zjINUM;;Wo#78fcS$r#uGcABcj%r_y^6ywx^&!{WAiA}r%e%pEo>>t zOmsy@EgLbu$NKH!L*=>D4ax`Gh?&%lDV5k>+r^nSCb-T%&VsJukC)iNf!%nbtKvpO zAXsF5F_*>}?jZAGxvS%i|L9|`QMhT0Jkl@avMbr5n9kGM_7m>!MOmr`0bQYPBfz;f zrcj6*%z9n?p*21xeQ-oPhvvmp6eS@6%;yWRP|W7cSWLn#j3r}9fT5G0-II72tIrJ{ zImAN%gh((f10Wh-k}S!?G2mQDipkRBHzPaA(b}^+ugAl?pDOva_?M5I7S8gu_xbdJ zxlxcM7?CCIfs~LE*CFxst=9uc{Ep)S-@`-bg~wqCA$kkZ?+tVTH)ZMe!?k-K?#K6z z=Hkk`BN*$!x5F?F;g4DPa}r9&9M3-hehn7%g5Pr6k67t_tAj*Z+wnC9`ll4#H*hq~-ew4IHjBKZ^h^l1myf_Q?uGw8D zDXyst;t~}EC2NV0t2{%c`-Y z={aE~*emt=MX$I*bwklqu<-XvV#zDY=`=MAO*h2ITv0SHkdGQK$^fKd%6bMnBoGV_Sd}_}tK!T)Ca3`6!H`#6)#ZEwp1%|fvsIFndy8=HWZ_PsG$|~H0)6-m5@t;WEhhw=dg`uRZ}subP=9j zPRUWbi*38U7tA$mlj^4?A&U*z31P?bTKJ%NsL#o;TSd;=7A(*-1A%Z+E2OeC0mWYw zBc@)UUNN82r>QAtvr0Oy6*9{z95~Eh=ZkSUZJk=Fw;uv9(+UVV$p#(KvYMW>%Z1L0 zx=7*lvAyT0vFzOYNFhZboxHB(6jNDYO4)?INEJ+67R6^{iOJ+-Y{shiOe{Vsu5F0zNU{Z#A z0dT@~I0J@)_9*rAT=yS6_;P>L;r*8<;Rd+S2iIKB=X&~Teutj%AU6uy35Y=qi@!?G zIqK=T4mlk99h(Pn6tu(m75*xHb^f*bYW|8p^W}aPKVTR?3IlQU<9g1q02ry!a~tFk zKo{luoqd1KkMZm2D;)g!FkdSVxaZ>V$2c&!;Jv`smS2^g+btl5HQcd%5wFAsBJ}4N zx2fjG_$Zc+aqzXJKyJ>@W5CP>{mVlx(ncO3E*KAyhshyw1g=NOqvUaNl(fUOog5~e zq=7WUwV6Cd8i|`U!L^CBkPgxcWF0U(Wb1VDoZy#kwKaD)V+0qBPBw6jf*2e-YrYM$ z7sl@}f-rhe2{OaxOZbrk0-re!xyW#j1101ohO?=nT{{3@IRM9;P~4=4eB^Qyk0TH~ z5+?tENN^9^Uey(NB@*~G5z;z^p8@>T3iSG49uWG3tdc?nLwr!t!{j;e8-o#ITzsnC zE8>pu%&>Nk2tNoDq79e9OkQ~FSfKcIv^?1AAx-Y#-NEMPTVJ^sE!*{fvtwO|k|*)G zHn?wmKe2MJ?Z(jdZ20ZV7r$}2uN~baf4^V8`RC{F{`|`u-<>i(`RvsVa&2>SZ)bZO z$M&ZnKYe`l>LyCZYwLTv-(I;LyaQuv{jboLJ0Cjc{`LBC_uvQJUGKctz4pi1zP+>k zcY+@B)u_F>-*%rWH%g%eIGdg2yPnpT30LmNL!|5Y6T;mq8yoFiuXmCu+f*o%o8)QN WC6DLi==ED)q!%UwA2v<1{`nUbUa1}c literal 0 HcmV?d00001 diff --git a/source/server/__test_fixtures/cube_meshopt.glb b/source/server/__test_fixtures/cube_meshopt.glb new file mode 100644 index 0000000000000000000000000000000000000000..d3483c0cd49ed8e0e911b97a15e80ede7186c91e GIT binary patch literal 2376 zcmcImPfrs;6n{udYtn?sO-?xHwb?&VY){lu1TC#;S7Jg4*_LU$vE9YpDFV&9Js6Ew zPu}F}x9A7pOL+HS!pQ^0H$zLeAQdI6()7?;<-4qNjO(J;^Fl|26@DdlQKus}I# z)mWXR7BL&laB94PwE1KLt4sG8wRn%9oi@q7)VG_|t=qe7n}c?+Sw#P|`yR2XV?C%z zOr^cUyGk#vrSng6*%BszcS(skCSMpD;9t5zIwfj}1~uJQ|HsNg{|bVWRqN z!{i=c~QRG{-TRd89* z!~6nzxN#wnEE{E`@`|K5ePBay#xbymOk`0(#g@_JL$lr&#${Q-mf9^l>y+VUvr@EM zwYr7#$mg_dxHMxoY-h!&h!bE_jz5ycrCzGZGnR|T&gW1UO;1nnM`rKieS)^5{3WcNXeS-tdyBW<@wvRi-@~z3?l=;O%tnq8Uo>)WR!Dr` S_k1qt_&sloWBBA1u$VRcN=iE>lFfZBNS$xA!*p-ZEx` zK^N!t!OTSBi)7R2zNncQA23UZBQZV_x0y^6jd3yCM+|D5y2SVYoO^F69V;_l_}~Bg z^*iS_EyVm42%)cQ5xP)?&_mI1D33WsAu9Ifahixk=2S_>Ui|R5EQyk0NkfXIS0G}N z$d=%quw~NOZf|!xuoX`dS>YuSg6-`cc-o4U1QfvD4qZ~PcQ}tlE(10kWj-qwr6i$_ zj*;n+RV|M*L>XtPdVF6lPB0D0a4L~`PSC0)rCeMfQ9enM*sIDp0yVS-LwXzxmW|6p zMCAm5Pei0#dR!z5Wan|5Q%IjANOFu!sW}>(TJ1zUUP zG@A;>WqTABWCKHQna}VlKdC!8Rh9X8PNixd?+ZtRv0ymF77c|XLp_7oYqR3mz`?$7 zIMP4bfxUK^1cu@hG{v#HKb?_brfS1$cqm*ZD-bv^3c8e&6BK_sAfMsV`V8@0Dn;Z2 zJUN8DPAJG{NbGQy9tsQElNHkdnhxV5qROiRpi7Hz37A5WWUW_m9cY)3GFeF^qN)|^ zwAs6;F(<-ZveP~s;5uxx!XK2%@NiLvAFx^3KNwVPgaZS84!ApTI7L$eBpX#Pmu~+r z>Gqqt1L!w*XGXT01J=~LNBepPdm;>~)$xnLV&@V+tdc<@rd7D$cDLJ#sUe(7v9At* zR5>|KKR}1m(*>t*K(TQyD|WltHV?p3+_}?57rD*eu99R|7wl1->?kw1(_WJ8cCtIF zUtSF`xUYU*fNvdpJubV4-I#*G<95;esDZ2lT!boIJtuq|mY?7^tv48gU$_~cPUkm} zso#jXmFz33aVy!VLH+B=hK=%avf(esefm5`9jceghEDl9GC;&0>Z1?EtC(w=ZZl#G zuE(y4DakkLts7IWPKnQ$Z%rK|ri8DF@oAbd$GUZln04w{s1K-=FFmN)OF5030bePn zanxSQX-4VFeQz&+yPW=NW%bgKXGyAV-vr<(cl`L^W)a)y&p8RoC}2Gvqyh@ zj@vnMQaN?igFbYwNJoD-x#aly)RECopSB%8 xvGCX5^RJ$oJ)1hV^vBA=J12f``QrUwwukQto`qHZKWr#C5b8hEJN67h{{k7_ivs`v literal 0 HcmV?d00001 diff --git a/source/server/__test_fixtures/cube_webp.glb b/source/server/__test_fixtures/cube_webp.glb new file mode 100644 index 0000000000000000000000000000000000000000..94d0755995f43b21f02337e8b7f87b439dd620d6 GIT binary patch literal 2308 zcmb7G&2Jk;6d$_{v;~^dQV=(5BreFf_S%W_{{a^`ao``|z!5Hm_hxsJ&Bk$xli4@(-tYb1$Go*Xb_T61 zA>{H^LOxz1BfV>o&a^o+1_JjP3XfUs>ePJZGJv7 z7(zl+FXpM-1;99^p6P;)_N6UPJ%7Z|P<(g<-0C!?E(;E%Ym9G(7NZDpO~JTrIuYB* z4=soF?Ga`nc*uYw3a}aD3dLdxik(<|lL^yt>|xIj#|It@fMH54Ghj{M@%bPs6LmH; z>j?yhmbkG`%N)wSLM>dmAL|M$M=~==S=Mq~BqpumbKA9reHa5wA-HXY0*_O=W%S#F zw$YUZca7fWT8HYTJRNL2Xc|WE?p~4V8e{{B*+XsWO&-jkb9#g#nL2Z`}o=)%rCFqVEisPPdd%_kD zTm`b2Lr697d6ZC+2%0$b-HGopPeg1hC2bXzp$CJkVIUJOV7HWqKcI4Ln9|)3N_qJY z3K8Y7v(YTW;KD5?(uBTU#J( zb~WW(U4=T5&gBJ(r)#cN8}$q3wPic-y+HMPRjbQ^IabtbRUF5t$8moXO|5G+ zcz<%auP+g@-tNM;>#Py4pDx&=U184KHxeA@?G*>w=k1ja<)Xc^FOrQoAH{>`^j@7& z&e=nY<}vH_2`C4ta}Qhv#+jCb>Z_lWXw2M&2e%BulQq^9p&5ER)xP za2eJm(rdR`Td=#ovA(slRm{ - task2Executed = true; - return "task2"; - } - }); + // Give task1 time to start executing await once(task1Executed, "start"); - expect(task2Executed).to.be.false; // Close scheduler await scheduler.close(100); // The running task should abort await expect(runningTask).to.be.rejectedWith("aborted"); - - // The pending task should be rejected - await expect(pendingTask).to.be.rejected; - expect(task2Executed).to.be.false; }); }); \ No newline at end of file diff --git a/source/server/utils/gltf/inspect.test.ts b/source/server/utils/gltf/inspect.test.ts new file mode 100644 index 000000000..8033c38e9 --- /dev/null +++ b/source/server/utils/gltf/inspect.test.ts @@ -0,0 +1,38 @@ +import { expect } from "chai"; +import path from "path"; + +import { fixturesDir } from "../../__test_fixtures/fixtures.js"; +import { io } from "./io.js"; +import { inspectDocument } from "./inspect.js"; + +describe("inspectDocument()", function(){ + + it("parse etc1s texture size", async function(){ + const document = await io.read(path.join(fixturesDir, "cube_etc1s.glb")); + const desc = inspectDocument(document); + expect(desc).to.be.an("object"); + expect(desc.name).to.equal("Cube"); + expect(desc.numFaces).to.equal(12); + expect(desc.imageSize).to.equal(16); + expect(desc.extensions).to.be.an("array"); + expect(desc.bounds).to.deep.equal({ + min: [-1, -1, -1], + max: [1, 1, 1], + }); + }); + + it("parse webp texture size", async function(){ + const document = await io.read(path.join(fixturesDir, "cube_webp.glb")); + const desc = inspectDocument(document); + expect(desc).to.be.an("object"); + expect(desc.name).to.equal("Cube"); + expect(desc.numFaces).to.equal(12); + expect(desc.imageSize).to.equal(16); + expect(desc.extensions).to.be.an("array"); + expect(desc.bounds).to.deep.equal({ + min: [-1, -1, -1], + max: [1, 1, 1], + }); + }); + +}); From 4919ad084bffa8d37e486d52ec3089c11083413e Mon Sep 17 00:00:00 2001 From: Sebastien DUMETZ Date: Tue, 24 Feb 2026 17:06:37 +0100 Subject: [PATCH 10/14] remove old gltf code to use gltf-transform everywhere --- Dockerfile | 39 ++++-- source/server/__test_fixtures/cube.glb | Bin 5500 -> 1936 bytes source/server/routes/scenes/scene/post.ts | 122 +++++++------------ source/server/tasks/handlers/uploads.ts | 48 +++++--- source/server/tasks/scheduler.test.ts | 5 - source/server/utils/glTF.test.ts | 68 ----------- source/server/utils/glTF.ts | 142 ---------------------- source/server/utils/gltf/inspect.ts | 12 +- 8 files changed, 119 insertions(+), 317 deletions(-) delete mode 100644 source/server/utils/glTF.test.ts delete mode 100644 source/server/utils/glTF.ts diff --git a/Dockerfile b/Dockerfile index 79938bd86..adeb1b798 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,5 @@ - - -################### -# Build source -################### FROM node:22-slim AS build +# Build artifacts from sources RUN mkdir -p /app/dist /app/source WORKDIR /app @@ -29,12 +25,30 @@ COPY source/ui /app/source/ui RUN npm run build-ui # outputs files in /app/dist +FROM node:22-slim AS utilities +# Fetch utilities that needs to be included + +#Install dependencies +RUN apt-get -qqy update && apt-get -qqy install --no-install-recommends \ + xz-utils lbzip2 \ + ca-certificates \ + curl \ + && rm -rf /var/lib/apt/lists/* /var/tmp/* +#Install blender +RUN curl -fsSL -o "/tmp/blender.tar.xz" "https://download.blender.org/release/Blender4.5/blender-4.5.4-linux-x64.tar.xz" \ + && mkdir -p /usr/local/lib/blender /usr/local/bin \ + && tar -xf "/tmp/blender.tar.xz" -C /usr/local/lib/blender --strip-components=1 \ + && ln -s /usr/local/lib/blender/blender /usr/local/bin/blender \ + && rm -rf /tmp/* + +#Install ktx +RUN curl -fsSL -o "/tmp/KTX-Software-Linux-x86_64.tar.bz2" "https://github.com/KhronosGroup/KTX-Software/releases/download/v4.4.2/KTX-Software-4.4.2-Linux-x86_64.tar.bz2"\ + && tar -xf "/tmp/KTX-Software-Linux-x86_64.tar.bz2" -C /usr/local/ --strip-components=1 \ + && rm -rf /tmp/* -################### -# The actual container to be published -################### FROM node:22-slim +# Build the distributed container LABEL org.opencontainers.image.source=https://github.com/Holusion/eCorpus LABEL org.opencontainers.image.description="eCorpus base image" LABEL org.opencontainers.image.documentation="https://ecorpus.eu" @@ -51,6 +65,15 @@ ENV BUILD_REF=${BUILD_REF} ENV NODE_ENV=production +#Install additional runtime dependencies +RUN apt-get -qqy update && apt-get -qqy install --no-install-recommends \ + ocl-icd-libopencl1 \ + && ln -s libOpenCL.so.1 /usr/lib/x86_64-linux-gnu/libOpenCL.so\ + && rm -rf /var/lib/apt/lists/* /var/tmp/* + +COPY --from=utilities /usr/local/lib /usr/local/lib +COPY --from=utilities /usr/local/bin /usr/local/bin + WORKDIR /app COPY source/server/package*.json /app/ diff --git a/source/server/__test_fixtures/cube.glb b/source/server/__test_fixtures/cube.glb index de8af0be6c659b6c9a4ea9d6a3e1146237c37bcf..190fe2a0aa77c73fb4ae2f6bfbc69a50d075524c 100644 GIT binary patch literal 1936 zcmb7EZBH6O5FV?o*0$Q(x36SBU+Hn6!hI-Jswu5<4G`1Bn0mm1mmGIyZ?QFmgrCrV z)c&;2?A`)-8?ru+&QxfEuMAq85v1=*Ej6G2qA% zCpb)Sc(C)9IONjxeXm^)2fed^M;JY1EjQwo&=1AF+g4Q`o7tkNMlM&f%#u|y?qkW( zv62Lu3X2vx*Oe{1IZ`EHZK~+2Zm8<;pu+2W(z{5Uu9U)S4J4^U_RVQjn^mW#x>9rM zhdT$%&Kaz^_r2mc^-n)cW?R^GFX(t}3dvtL3u8vA!EPH}Q7Gc>wFccT7e^ldsYu&l z|2zzMAe)!xct@K#Ys+Bma1h{YunNTX3vViC;D;*q@T!mBbFfoa5AFK+lg(0%YF$cxz=NK|Cu$Xo z1|x&h?W*4m-n?{0kA6b=LTL-HM7QGZSVeZ( z^z>fqBxmfgMQV_b2K8%F##&<27%da*6ebm~`DS9~@uYnEd{aCclaf=Ia;)id%;eFS z89W*C;Y`cFA7Xo25^jvXv?Lta(~@u>ah~D?oLVOw*#RZlwN8E-rzpqhBYm9sl>=>L zC;Yg)0MFnhEZ|&*$M6U?aNU5{@D^Ud8m?>b0-nPXtm3)~Z{Q(3fE8R<;0dh5QzTr+ Hxd{INQjgB- literal 5500 zcmd5Yh&n)oxTW;>K1MtsjvkjKoA6!vX%*mq(D-sgET0g z`mc)or2d3gHP zNfN~gtNl>BM=TsdBuO@!pPlje)aSnwqW-;tpp=asr+jdBmhf~zG>(E<6!I_~KAv%s zb{fr=sXLD5p-^kxG}(c7VFn^&Xmtvw_bs#^E~4uWaH=9T{aGp`momPa`A5_NIt zgO^Qs$YU>!Vmj2>>950++5|7IT63pf;}^3iPI=69G1(&?Cw?R@0EP(RLLTPEowedx zBS#BY+7S;Y={bpZFMB+tUObVvV5@!WW=oZK$Tqs!w|q9a+Yg^kMV~eH>Q)!dV_$56-h9M& zMtFPcRW%&=fj8k!`=*5lx4ne-qG=TG%)&`-OXs7vA7yQ+-QlS>o%-YBXg)a)@s(() z8=(XqZcN{sI-?1_Ks*8&C6w=AayT2s3vMWNBPg*NI)fpOZn|DQxRiE*Z>A)7T%) zQ;A+E2%=bjaC~R~h)QZUu+%5{Z-bM=;o+c92=kp=Acp(>pAY&&0+>$<18{J+Hy9lM z_@GXhc2O7!DV3j`;zIfOw}aqj-%C8x?c@?i55@Z1j*RuVuOcwlBSa-Jc^{$YKZ{ug{D!}c-KK|Q$1+KuW@EW`hZ@^X9f;V9s-h#K`9k>SX!h7&O zd;lN9M{pfJhEL#A_zXUWFW^h~3ciMKpa$OpgYVEz*2Uj5<4d%2{?EUE627}7^#qMP zjqe(C&QtcDm2^p7#&k87d>C?#)n6?u&xv0ZLm8*%^d%iVCz|xdx13`IpY+Oj!K<)K zd@a-8ZUs-{%3pp7&gm{wEsG@UTX2?sl&@b`IdX!U+bI^zoLtvNCtK_6A z{A(MOV~LjXzjy!YXzU`rtS)4ZY-}not6o^2OU;d(WImE1;-Z|hVlF8euZ7jV6oB%{ z^)(M|SvnPS0nL>-|p_L;}4VyK*F23{x|bCi)zZjaOyhgTn}9@{T2MIz!P5&T}xA+WTvhu)4HZy*vbkW YONM+sD)}haHgg>1rhYk3#Vl?72X9;%g#Z8m diff --git a/source/server/routes/scenes/scene/post.ts b/source/server/routes/scenes/scene/post.ts index ab872ac11..1ebe82d25 100644 --- a/source/server/routes/scenes/scene/post.ts +++ b/source/server/routes/scenes/scene/post.ts @@ -1,10 +1,9 @@ import { Request, Response } from "express"; -import { parse_glb } from "../../../utils/glTF.js"; -import { getVfs, getUserId } from "../../../utils/locals.js"; -import uid from "../../../utils/uid.js"; -import getDefaultDocument from "../../../utils/schema/default.js"; +import { getUserId, getLocals } from "../../../utils/locals.js"; import { BadRequestError, UnauthorizedError } from "../../../utils/errors.js"; +import { inspectGlb } from "../../../tasks/handlers/inspectGlb.js"; +import { createDocumentFromFiles } from "../../../tasks/handlers/uploads.js"; const sceneLanguages = ["EN", "ES", "DE", "NL", "JA", "FR", "HAW"] as const; type SceneLanguage = typeof sceneLanguages[number]; @@ -13,66 +12,6 @@ function isSceneLanguage(l:any) :l is SceneLanguage|undefined{ } -interface GetDocumentParams{ - scene :string; - filepath :string; - language?:SceneLanguage; -} - -/** - * Creates a new default document for a scene - * uses data embedded in the glb to fill the document where possible - * @param scene - * @param filepath - * @returns - */ -async function getDocument({scene, filepath, language}:GetDocumentParams){ - let orig = await getDefaultDocument(); - //dumb inefficient Deep copy because we want to mutate the doc in-place - let document = JSON.parse(JSON.stringify(orig)); - let meta = await parse_glb(filepath); - let mesh = meta.meshes[0]; //Take the first mesh for its name - document.nodes.push({ - "id": uid(), - "name": mesh?.name ?? scene, - "model": 0, - } as any); - document.scenes[0].nodes.push(document.nodes.length -1); - - if(language){ - document.setups[0].language = {language: language.toUpperCase()}; - } - - document.models = [{ - "units": "m", //glTF specification says it's always meters. It's what blender do. - "boundingBox": meta.bounds, - "derivatives":[{ - "usage": "Web3D", - "quality": "High", - "assets": [ - { - "uri": `models/${scene}.glb`, - "type": "Model", - "byteSize": meta.byteSize, - "numFaces": meta.meshes.reduce((acc, m)=> acc+m.numFaces, 0), - "imageSize": 8192 - } - ] - }], - "annotations":[], - }]; - document.metas = [{ - "collection": { - "titles": { - "EN": scene, - "FR": scene, - } - }, - }]; - document.scenes[document.scene].meta = 0; - - return document -} /** * Tries to create a scene. @@ -80,7 +19,7 @@ async function getDocument({scene, filepath, language}:GetDocumentParams){ * Whether or not it's desired behaviour remains to be defined */ export default async function postScene(req :Request, res :Response){ - let vfs = getVfs(req); + const {vfs, taskScheduler} = getLocals(req); let user_id = getUserId(req); let {scene} = req.params; let {language} = req.query; @@ -96,18 +35,51 @@ export default async function postScene(req :Request, res :Response){ } let scene_id = await vfs.createScene(scene, user_id); - try{ - let f = await vfs.writeFile(req, {user_id, scene: scene, mime:"model/gltf-binary", name: `models/${scene}.glb`}); - let document = await getDocument({ - scene, - filepath: vfs.filepath(f), - language, - }); - await vfs.writeDoc(JSON.stringify(document), {scene: scene_id, user_id: user_id, name: "scene.svx.json", mime: "application/si-dpo-3d.document+json"}); - }catch(e){ + await taskScheduler.run({ + scene_id, + user_id, + type: "postScene", + data: {}, + handler: async function postsSceneHandler({context: {logger, vfs}}){ + logger.debug("Draining the HTTP request into scene space"); + let f = await vfs.writeFile(req, {user_id, scene: scene, mime:"model/gltf-binary", name: `models/${scene}.glb`}); + if(f.size ==0 || !f.hash) throw new BadRequestError(`Body was empty. Can't create a scene.`); + logger.debug("Parse the created file"); + const meta = await taskScheduler.run({ + immediate: true, + data: {fileLocation: vfs.relative(vfs.getPath(f as {hash: string}))}, + handler: inspectGlb, + }); + + logger.debug(`Generate default document for model ${meta.name}`); + const document = await taskScheduler.run({ + immediate: true, + handler: createDocumentFromFiles, + data: { + scene: scene, + language: language, + models: [{ + uri: f.name, + quality: "High", + usage: "Web3D", + byteSize: f.size, + ...meta, + }], + } + }); + logger.debug(`Write scene document`); + await vfs.writeDoc(JSON.stringify(document), { + scene: scene_id, + user_id: user_id, + name: "scene.svx.json", + mime: "application/si-dpo-3d.document+json" + }); + } + }).catch(async e=>{ //If written, the file will stay as a loose object but will get cleaned-up later await vfs.removeScene(scene_id).catch(e=>console.warn(e)); throw e; - } + }); + res.status(201).send({code: 201, message: "created scene with id :"+scene_id}); }; diff --git a/source/server/tasks/handlers/uploads.ts b/source/server/tasks/handlers/uploads.ts index 685bf1859..3b3735ad8 100644 --- a/source/server/tasks/handlers/uploads.ts +++ b/source/server/tasks/handlers/uploads.ts @@ -9,13 +9,14 @@ import { parseFilepath, isMainSceneFile } from "../../utils/archives.js"; import { FileArtifact, TaskDefinition, TaskHandlerParams } from "../types.js"; import { BadRequestError, InternalError } from "../../utils/errors.js"; import getDefaultDocument from "../../utils/schema/default.js"; -import { parse_glb } from "../../utils/glTF.js"; import uid from "../../utils/uid.js"; import { toGlb } from "./toGlb.js"; import { getMimeType, isModelType, readMagicBytes } from "../../utils/filetypes.js"; import { TDerivativeQuality, TDerivativeUsage } from "../../utils/schema/model.js"; import { optimizeGlb } from "./optimizeGlb.js"; import { IDocument } from "../../utils/schema/document.js"; +import { inspectGlb } from "./inspectGlb.js"; +import { Bounds } from "../../utils/gltf/inspect.js"; @@ -47,7 +48,7 @@ export interface UploadedBinaryModel extends UploadedFile { byteSize: number; numFaces: number; imageSize: number; - bounds: Awaited>["bounds"]; + bounds: Bounds; } export interface UploadedUsdModel extends UploadedFile{ @@ -114,25 +115,30 @@ async function parseUploadedArchive({task:{task_id, user_id, data:{fileLocation} } -async function parseUploadedModel({task: {data: {fileLocation}}, context: {logger, vfs}}:TaskHandlerParams):Promise{ +async function parseUploadedModel({task: {data: {fileLocation}}, context: {logger, tasks, vfs}}:TaskHandlerParams):Promise{ const filepath = vfs.absolute(fileLocation); logger.debug("Check mime type of "+fileLocation); const mime = await readMagicBytes(filepath); if(mime !== "model/gltf-binary"){ throw new InternalError("This does not look like a GLB file"); } - const meta = await parse_glb(filepath); - logger.log(`Parsed glb with ${meta.meshes.length} models`); - logger.warn("Using placeholder imageSize of 8192: Not compatible with LOD mode"); + const meta = await tasks.run({ + handler: inspectGlb, + data: {fileLocation} + }); + + const stats = await fs.stat(filepath); + + logger.log(`Parsed glb file ${fileLocation}`); return { fileLocation, mime, isModel: true, - name: meta.meshes.find(m=>m.name)?.name, + name: meta.name, bounds: meta.bounds, - byteSize: meta.byteSize, - numFaces: meta.meshes.reduce((acc, m)=> acc+m.numFaces, 0), - imageSize: 8192 + imageSize: meta.imageSize, + numFaces: meta.numFaces, + byteSize: stats.size, } } @@ -269,7 +275,7 @@ export async function createSceneFromFiles({context:{tasks, vfs, logger}, task: await vfs.copyFile(filepath, {scene: scene_id, name: filename, user_id, mime }); models.push({ ...(source as UploadedBinaryModel), - filepath: filename, + uri: filename, quality: "High", usage: "Web3D" }); @@ -322,11 +328,25 @@ export async function createSceneFromFiles({context:{tasks, vfs, logger}, task: return scene_id; } -type DocumentModel = UploadedBinaryModel&{quality:TDerivativeQuality, usage: TDerivativeUsage}; +interface DocumentModel { + name?: string; + /** + * Uri is slightly misleading as it's "relative to scene root" + * `uri` **WILL** be urlencoded by {@link createDocumentFromFiles} so it should be given in clear text + */ + uri: string; + byteSize: number; + numFaces: number; + imageSize: number; + bounds: Bounds; + quality:TDerivativeQuality; + usage: TDerivativeUsage; +}; + interface GetDocumentParams{ scene :string; models :Array; - language?:SceneLanguage; + language:SceneLanguage|undefined; } @@ -350,7 +370,7 @@ export async function createDocumentFromFiles( "quality": model.quality, "assets": [ { - "uri": encodeURIComponent(model.filepath), + "uri": encodeURIComponent(model.uri), "type": "Model", "byteSize": model.byteSize, "numFaces": model.numFaces, diff --git a/source/server/tasks/scheduler.test.ts b/source/server/tasks/scheduler.test.ts index 17b4e6cc0..607dfce2f 100644 --- a/source/server/tasks/scheduler.test.ts +++ b/source/server/tasks/scheduler.test.ts @@ -424,8 +424,6 @@ describe("TaskScheduler", function(){ }); it("closing scheduler with pending tasks cleans up properly", async function(){ - // PURPOSE: Verify that if we close the scheduler while tasks are still - // pending, we don't leak resources or leave zombie tasks hanging. this.timeout(5000); scheduler.concurrency = 1; @@ -443,9 +441,6 @@ describe("TaskScheduler", function(){ return "task1"; } }); - - - // Give task1 time to start executing await once(task1Executed, "start"); diff --git a/source/server/utils/glTF.test.ts b/source/server/utils/glTF.test.ts deleted file mode 100644 index 4e6bef3bc..000000000 --- a/source/server/utils/glTF.test.ts +++ /dev/null @@ -1,68 +0,0 @@ -import { expect } from "chai"; -import fs from "fs/promises"; -import path from "path"; -import { fileURLToPath } from 'url'; -import { parse_glb, parse_glTF } from "./glTF.js"; - -const thisFile = fileURLToPath(import.meta.url); - -import { fixturesDir } from "../__test_fixtures/fixtures.js"; - -describe("parse_glb()", function(){ - - it("parse a glb file to extract data", async function(){ - let d = await parse_glb(path.resolve(fixturesDir, "cube.glb" )); - expect(d).to.deep.equal({ - meshes:[{ - numFaces: 6*2 /*it takes two triangles to make a square and 6 squares for a cube */, - bounds:{ - min: [-1,-1,-1], - max: [1,1,1.000001] - }, - position: [0,0,0], - name: "Cube", - }], - bounds:{ - min: [-1,-1,-1], - max: [1,1,1.000001] - }, - byteSize: 5500, - }) - }); - it("throw an error for invalid files", async function(){ - await expect(parse_glb( thisFile)).to.be.rejectedWith("bad magic number"); - }) -}); - - -describe("parse_gltf()", function(){ - it("handles morph targets", async function(){ - let data = await fs.readFile(path.resolve(fixturesDir, "morph.gltf" ), {encoding: "utf8"}); - let gltf = JSON.parse(data); - let res = parse_glTF(gltf); - expect(res).to.deep.equal({ - meshes:[{ - numFaces: 64, - position: [0,0,0], - bounds: {min: [-0.5,0,-0.5], max: [0.5,0.20000000298023224,0.5]}, - name: "mesh" - }], - bounds: {min: [-0.5,0,-0.5], max: [0.5,0.20000000298023224,0.5]}, - }); - }) - - it("handles empties", async function(){ - let gltf = { - "asset":{"generator":"Khronos glTF Blender I/O v4.2.69","version":"2.0"}, - "scene":0, - "scenes":[{"name":"Scene","nodes":[0]}], - "nodes":[{"name":"Empty"}] - }; - let res = parse_glTF(gltf); - expect(res).to.deep.equal({ - meshes:[], - bounds: {min: [0,0,0], max: [0,0,0]}, - }); - }) - // @todo add more edge-cases from https://github.com/KhronosGroup/glTF-Sample-Models -}) \ No newline at end of file diff --git a/source/server/utils/glTF.ts b/source/server/utils/glTF.ts deleted file mode 100644 index 35aa39d4b..000000000 --- a/source/server/utils/glTF.ts +++ /dev/null @@ -1,142 +0,0 @@ -import fs from "fs/promises"; -import assert from "assert/strict"; - - - -interface Accessor { - componentType :5120|5121|5122|5123|5125|5126; - type :"SCALAR"|"VEC1"|"VEC2"|"VEC3"|"VEC4"|"MAT2"|"MAT3"|"MAT4"; - count :number; - min?:number[]; - max?:number[]; -} - -interface Primitive { - attributes:{ - NORMAL :number; - POSITION :number; - TANGENT ?:number; - TEXCOORD_0 ?:number; - } - targets ?:[{POSITION :number}]; - indices :number; -} -interface Mesh { - name :string; - primitives: Primitive[]; -} -interface Bounds { - min: [number, number, number], - max: [number, number, number] -} - -interface MeshDescription { - position: [number, number, number]; - bounds :Bounds; - numFaces: number; - name ?:string; -} - - -export interface SceneDescription{ - meshes :MeshDescription[]; - bounds :Bounds; - byteSize ?:number; -} - -interface GlbDescription extends SceneDescription{ - byteSize :number; -} - -export interface JSONglTF extends Record{ - meshes?: Mesh[]; - accessors ?:Accessor[]; -} - -function asBounds(a :Accessor) :Bounds{ - if(!(a.min?.length == 3 && a.max?.length == 3 )) assert.fail("min and max MUST be defined in glTF mesh position"); - return a as any as Bounds; -} - -function mergeBounds(a:Bounds, b:Bounds) :Bounds{ - return { - min: a.min.map((value, index)=>(("min" in b)?Math.min(value, b.min[index]):value)) as any, - max:a.max.map((value, index)=>(("max" in b)?Math.max(value, b.max[index]):value)) as any, - }; -} - -/** - * - * Float values are rounded to single precision. - * @see https://github.com/KhronosGroup/glTF/blob/main/specification/2.0/Specification.adoc#3625-accessors-bounds - */ -export function parse_glTF({meshes = [], accessors = []}:JSONglTF) :SceneDescription { - let scene :SceneDescription = { - meshes:[], - bounds: {min:[0,0,0], max:[0,0,0]}, - }; - for(let mesh of meshes){ - let out :MeshDescription = { - position: [0, 0, 0], - bounds: {min:[0,0,0], max:[0,0,0]}, - numFaces: 0, - name: mesh.name - }; - for(let primitive of mesh.primitives){ - let positions = [primitive.attributes.POSITION, ...(primitive.targets ?? []).map(t=>t.POSITION)] - for (let positionIndex of positions){ - let position :Bounds|Accessor = accessors[positionIndex]; - out.bounds = mergeBounds(out.bounds, asBounds(position)); - out.numFaces+= accessors[primitive.indices].count /3; //every 3 indices form a triangle - } - } - scene.meshes.push(out); - scene.bounds = mergeBounds(scene.bounds, out.bounds); - } - return scene; -} - - - -/** - * Opens a GLB file, reads its JSON portion and parse it through {@link parse_glTF} - * - * For large models, this is much more efficient than using `@gltf-transform` because it only reads what's needed - * @param filePath - * @returns - */ -export async function parse_glb(filePath :string) :Promise{ - let res :MeshDescription = {} as any; - let handle = await fs.open(filePath, "r"); - //https://docs.fileformat.com/3d/glb/ - try{ - let header = Buffer.alloc(3*4); - let {bytesRead} = await handle.read({buffer:header}); - assert.equal(bytesRead, 3*4 as number, `Could not read glb header (file too short ${bytesRead})`); - let magic = header.readUint32LE(0); - assert(magic == 0x46546C67, "bad magic number : 0x"+magic.toString(16)) - let version = header.readUInt32LE(4); - assert(version == 0x2, `gltf files version ${version} not supported. Please provide a glTF2.0 file`); - let byteSize = header.readUInt32LE(8); - - let position = header.length; //position is not updated when we skip blocks - let chunkHeader = Buffer.allocUnsafe(4*2); - while( (await handle.read({buffer:chunkHeader, position})).bytesRead == 8){ - position += 8; - let chunkLength = chunkHeader.readUint32LE(0); - let chunkType = chunkHeader.readUInt32LE(4); - if(chunkType != 0x4E4F534A){ - position += chunkLength; - continue; - } - let data = Buffer.allocUnsafe(chunkLength) - let {bytesRead} = await handle.read({buffer:data, position}); - assert(bytesRead == chunkLength, "Reached end of file while trying to get JSON chunk"); - let gltfData :JSONglTF = JSON.parse(data.toString("utf-8")); - return {...parse_glTF(gltfData), byteSize}; - } - }finally{ - await handle.close(); - } - assert.fail("Can't find glTF data "); -} \ No newline at end of file diff --git a/source/server/utils/gltf/inspect.ts b/source/server/utils/gltf/inspect.ts index 964128f4b..79ca76b49 100644 --- a/source/server/utils/gltf/inspect.ts +++ b/source/server/utils/gltf/inspect.ts @@ -2,12 +2,14 @@ import {getBounds, getPrimitiveVertexCount, VertexCountMethod} from '@gltf-trans import { Document, ImageUtils, PropertyType } from '@gltf-transform/core'; +export interface Bounds { + min: [number, number, number], + max: [number, number, number], +} + export interface SceneDescription{ name: string, - bounds :{ - min: [number, number, number], - max: [number, number, number] - }, + bounds: Bounds, imageSize: number, numFaces: number, extensions: string[], @@ -31,7 +33,7 @@ export function inspectDocument(document: Document): SceneDescription{ } let imageSize = getMaxDiffuseSize(document); - + document return { name, From 8432eb4dbcd5ccaaa5b871b8adb3edfd2b4753f1 Mon Sep 17 00:00:00 2001 From: Sebastien DUMETZ Date: Fri, 27 Feb 2026 10:58:01 +0100 Subject: [PATCH 11/14] fix integration and e2e tests, remove blender from Docker --- Dockerfile | 7 - source/e2e/fixtures.ts | 25 +- source/e2e/package-lock.json | 18 +- source/e2e/package.json | 2 +- source/e2e/playwright.config.ts | 2 +- source/e2e/tests/accessRights.spec.ts | 55 +- source/e2e/tests/download.spec.ts | 2 +- source/e2e/tests/scene_settings.spec.ts | 6 +- source/e2e/tests/upload_object.spec.ts | 209 ++++- source/e2e/tests/upload_zip.spec.ts | 61 +- source/e2e/tests/userSettings.spec.ts | 19 +- source/e2e/tsconfig.json | 16 + source/server/__test_fixtures/cube.mtl | 12 + source/server/__test_fixtures/cube.obj | 40 + source/server/integration.test.ts | 4 +- source/server/package-lock.json | 858 +++++++++++++++++- source/server/package.json | 3 +- .../server/routes/scenes/scene/post.test.ts | 2 +- source/server/routes/scenes/scene/post.ts | 17 +- source/server/routes/views/index.ts | 10 +- .../server/tasks/handlers/runBlenderScript.ts | 28 - source/server/tasks/handlers/toGlb.ts | 42 +- source/server/tasks/handlers/uploads.test.ts | 67 ++ source/server/tasks/handlers/uploads.ts | 68 +- source/server/templates/locales/en.yml | 6 +- source/server/templates/upload.hbs | 10 +- source/server/utils/exec.ts | 18 - source/server/utils/gltf/inspect.test.ts | 49 + source/server/utils/gltf/inspect.ts | 8 +- source/server/utils/gltf/obj2gltf.d.ts | 85 ++ source/server/utils/templates.ts | 8 + source/ui/MainView.ts | 1 - source/ui/composants/UploadManager.ts | 13 +- 33 files changed, 1465 insertions(+), 306 deletions(-) create mode 100644 source/e2e/tsconfig.json create mode 100644 source/server/__test_fixtures/cube.mtl create mode 100644 source/server/__test_fixtures/cube.obj delete mode 100644 source/server/tasks/handlers/runBlenderScript.ts create mode 100644 source/server/tasks/handlers/uploads.test.ts create mode 100644 source/server/utils/gltf/obj2gltf.d.ts diff --git a/Dockerfile b/Dockerfile index adeb1b798..6ae35bba2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,13 +35,6 @@ RUN apt-get -qqy update && apt-get -qqy install --no-install-recommends \ curl \ && rm -rf /var/lib/apt/lists/* /var/tmp/* -#Install blender -RUN curl -fsSL -o "/tmp/blender.tar.xz" "https://download.blender.org/release/Blender4.5/blender-4.5.4-linux-x64.tar.xz" \ - && mkdir -p /usr/local/lib/blender /usr/local/bin \ - && tar -xf "/tmp/blender.tar.xz" -C /usr/local/lib/blender --strip-components=1 \ - && ln -s /usr/local/lib/blender/blender /usr/local/bin/blender \ - && rm -rf /tmp/* - #Install ktx RUN curl -fsSL -o "/tmp/KTX-Software-Linux-x86_64.tar.bz2" "https://github.com/KhronosGroup/KTX-Software/releases/download/v4.4.2/KTX-Software-4.4.2-Linux-x86_64.tar.bz2"\ && tar -xf "/tmp/KTX-Software-Linux-x86_64.tar.bz2" -C /usr/local/ --strip-components=1 \ diff --git a/source/e2e/fixtures.ts b/source/e2e/fixtures.ts index ed82c6468..3615dcb64 100644 --- a/source/e2e/fixtures.ts +++ b/source/e2e/fixtures.ts @@ -13,12 +13,15 @@ export type CreateSceneOptions = { autoDelete?:boolean, } +type AccessLevel = "use" | "create" | "admin"; + +type UniqueAccount = {username:string, password:string, uid:number}; + type TestFixture = { adminPage:Page, userPage:Page, createScene:(opts?:CreateSceneOptions)=>Promise, - uniqueAccount: {username:string, password:string, uid:number}, - + uniqueAccount: (level?:AccessLevel)=>Promise, } export {expect} from "@playwright/test"; @@ -70,16 +73,18 @@ export const test = base.extend({ await ctx.close(); }, uniqueAccount: async ({browser}, use)=>{ + let adminContext = await browser.newContext({storageState: "playwright/.auth/admin.json"}); + + await use(async (level :AccessLevel = "create")=>{ let username = `testUserLogin${randomBytes(2).readUInt16LE().toString(36)}`; let password = randomBytes(16).toString("base64"); - let adminContext = await browser.newContext({storageState: "playwright/.auth/admin.json"}); //Create a user for this specific test - let res = await adminContext.request.post("/users", { + let res = await adminContext.request.post("/users", { data: JSON.stringify({ username, email: `${username}@example.com`, password, - level: "create", + level, }), headers:{ "Content-Type": "application/json", @@ -87,8 +92,10 @@ export const test = base.extend({ }); let body = JSON.parse(await res.text()); expect(body).toHaveProperty("uid"); - let uid :number =body.uid; - await use({username, password, uid}); - await adminContext.close(); - }, + let uid :number = body.uid; + return {username, password, uid}; + }); + + await adminContext.close(); + } }); \ No newline at end of file diff --git a/source/e2e/package-lock.json b/source/e2e/package-lock.json index 2e450be13..290166d84 100644 --- a/source/e2e/package-lock.json +++ b/source/e2e/package-lock.json @@ -15,7 +15,7 @@ }, "devDependencies": { "@playwright/test": "^1.49.1", - "@types/node": "^22.10.7" + "@types/node": "^22.19.12" } }, "node_modules/@playwright/test": { @@ -34,11 +34,12 @@ } }, "node_modules/@types/node": { - "version": "22.10.7", - "resolved": "https://registry.npmjs.org/@types/node/-/node-22.10.7.tgz", - "integrity": "sha512-V09KvXxFiutGp6B7XkpaDXlNadZxrzajcY50EuoLIpQ6WWYCSvf19lVIazzfIzQvhUN2HjX12spLojTnhuKlGg==", + "version": "22.19.12", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.19.12.tgz", + "integrity": "sha512-0QEp0aPJYSyf6RrTjDB7HlKgNMTY+V2C7ESTaVt6G9gQ0rPLzTGz7OF2NXTLR5vcy7HJEtIUsyWLsfX0kTqJBA==", + "license": "MIT", "dependencies": { - "undici-types": "~6.20.0" + "undici-types": "~6.21.0" } }, "node_modules/@types/yauzl": { @@ -112,9 +113,10 @@ "integrity": "sha512-+aWOz7yVScEGoKNd4PA10LZ8sk0A/z5+nXQG5giUO5rprX9jgYsTdov9qCchZiPIZezbZH+jRut8nPodFAX4Jg==" }, "node_modules/undici-types": { - "version": "6.20.0", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.20.0.tgz", - "integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==" + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "license": "MIT" }, "node_modules/xml-js": { "version": "1.6.11", diff --git a/source/e2e/package.json b/source/e2e/package.json index 3afc70ceb..15beb2c72 100644 --- a/source/e2e/package.json +++ b/source/e2e/package.json @@ -11,7 +11,7 @@ "license": "ISC", "devDependencies": { "@playwright/test": "^1.49.1", - "@types/node": "^22.10.7" + "@types/node": "^22.19.12" }, "dependencies": { "@types/yauzl": "^2.10.3", diff --git a/source/e2e/playwright.config.ts b/source/e2e/playwright.config.ts index 156a0f742..e13329d11 100644 --- a/source/e2e/playwright.config.ts +++ b/source/e2e/playwright.config.ts @@ -1,5 +1,5 @@ import { defineConfig, devices } from '@playwright/test'; -import path from 'path'; +import path from 'node:path'; /** * Read environment variables from file. diff --git a/source/e2e/tests/accessRights.spec.ts b/source/e2e/tests/accessRights.spec.ts index d5e065e52..813107533 100644 --- a/source/e2e/tests/accessRights.spec.ts +++ b/source/e2e/tests/accessRights.spec.ts @@ -1,8 +1,6 @@ import path from "node:path"; -import fs, { readFile } from "node:fs/promises"; -import { randomBytes, randomUUID } from "node:crypto"; -import { test, expect } from '../fixtures'; +import { test, expect } from '../fixtures.js'; const fixtures = path.resolve(import.meta.dirname, "../__test_fixtures"); @@ -13,47 +11,22 @@ test.use({ storageState: { cookies: [], origins: [] }, locale: "cimode" }); //The _actual_ route access rights are tested in unit tests //We want to have the user properly informed and that's what we're testing here -test("can't create a new scene", async function({page, request}){ - const name = randomUUID(); - +test("can't see upload page anonymously", async function({page}){ await page.goto("/ui/upload/"); - const f = page.getByRole("form", {name: "titles.createOrUpdateScene"}); - await expect(f).toBeVisible(); - await f.getByRole("button", {name: "labels.selectFile"}).setInputFiles(path.join(fixtures, "cube.glb")); - await f.getByRole("textbox", {name: "labels.sceneTitle"}).fill(name); - await f.getByRole("button", {name: "buttons.upload"}).click(); - - await expect(page.getByRole("status").getByText("Unauthorized")).toBeVisible(); - - let res = await request.get(`/scenes/${name}`); - await expect(res).not.toBeOK(); + await expect(page.getByRole("heading", {name: "Error"})).toBeVisible(); + await expect(page.getByText('errors.requireUser')).toBeVisible(); }); +test("can't see upload page as non-creator user", async function({page, uniqueAccount}){ + const {username, password} = await uniqueAccount("use"); + await page.goto("/auth/login/"); + await page.getByRole("textbox", {name: "labels.username"}).fill(username); + await page.getByRole("textbox", {name: "labels.password"}).fill(password); + await page.getByRole("button", {name: "labels.signin"}).click(); + await page.goto(`/ui/upload`); -test("can't upload a zip", async function({page, userPage}){ - const name = randomUUID(); - let res = await userPage.request.post(`/scenes/${name}`,{ - data: await fs.readFile(path.join(fixtures, "cube.glb")), - }); - await expect(res).toBeOK(); - res = await userPage.request.get(`/scenes/${name}`, { - headers: { - "Accept": "application/zip", - } - }); + await expect(page.getByRole("heading", {name: "Error"})).toBeVisible(); + await expect(page.getByText('errors.requireCreate')).toBeVisible(); - let body = await res.body(); - await page.goto("/ui/upload/"); - const f = page.getByRole("form", {name: "titles.createOrUpdateScene"}); - await expect(f).toBeVisible(); - await f.getByRole("button", {name: "labels.selectFile"}).setInputFiles({ - name: "scene.zip", - mimeType: "application/zip", - buffer: body, - }); - - await f.getByRole("button", {name: "buttons.upload"}).click(); - - await expect(page.getByText("scene: Error: [401] Unauthorized", {exact: true})).toBeVisible(); -}); +}) diff --git a/source/e2e/tests/download.spec.ts b/source/e2e/tests/download.spec.ts index 2d5ece1d6..3f0b0cd3e 100644 --- a/source/e2e/tests/download.spec.ts +++ b/source/e2e/tests/download.spec.ts @@ -50,7 +50,7 @@ test("downloads a scene archive", async ({page, request})=>{ await once(zip, "end"); expect(entries).toHaveLength(2); expect(entries.map(e=>e.fileName).sort()).toEqual([ - `scenes/${name}/models/${name}.glb`, + `scenes/${name}/${name}.glb`, `scenes/${name}/scene.svx.json`, ]); }); diff --git a/source/e2e/tests/scene_settings.spec.ts b/source/e2e/tests/scene_settings.spec.ts index a4d2f39d2..980d7cad3 100644 --- a/source/e2e/tests/scene_settings.spec.ts +++ b/source/e2e/tests/scene_settings.spec.ts @@ -1,8 +1,4 @@ -import path from "node:path"; -import fs from "node:fs/promises"; - - -import { expect, test } from '../fixtures'; +import { expect, test } from '../fixtures.js'; import { randomUUID } from "node:crypto"; diff --git a/source/e2e/tests/upload_object.spec.ts b/source/e2e/tests/upload_object.spec.ts index bbdac7f70..8b550abd8 100644 --- a/source/e2e/tests/upload_object.spec.ts +++ b/source/e2e/tests/upload_object.spec.ts @@ -9,21 +9,24 @@ const fixtures = path.resolve(import.meta.dirname, "../__test_fixtures"); //Authenticated as admin test.use({ storageState: 'playwright/.auth/user.json' }); -test("uploads and rename a glb", async ({page, request})=>{ +test("uploads and rename a glb", async ({ page, request }) => { await page.goto("/ui/upload"); //We are forced to use the rename otherwise we'd have a name collision const name = randomUUID(); - const f = page.getByRole("form", {name: "create or update a scene"}); + await page.locator("input[type=\"file\"]").setInputFiles(path.join(fixtures, "cube.glb")); + + await expect(page.getByRole("listitem").getByText("✓")).toBeVisible(); + + const f = page.getByRole("form", { name: "Create or update a scene" }); await expect(f).toBeVisible(); - await expect(f.getByRole("combobox", {name: "language"})).toHaveValue("en"); - await f.getByRole("button", {name: "files"}).setInputFiles(path.join(fixtures, "cube.glb")); - await f.getByRole("textbox", {name: "scene title"}).fill(name) - await f.getByRole("button", {name: "create a scene"}).click(); + await expect(f.getByRole("combobox", { name: "Default language" })).toHaveValue("en"); + await f.getByRole("textbox", { name: "Scene name" }).fill(name); + await page.getByRole("button", { name: "create a scene" }).click(); - const uploads = page.getByRole("region", {name: "uploads"}); + const uploads = page.getByRole("region", { name: "Created Scenes" }); await expect(uploads).toBeVisible(); //Don't check for actual progress bar visibility because that could be too quick to register - const link = uploads.getByRole("link", {name: name}); + const link = uploads.getByRole("link", { name: name }); await link.click(); await expect(page).toHaveURL(`/ui/scenes/${name}`); await expect(page.locator("h1")).toHaveText(name); @@ -33,30 +36,36 @@ test("uploads and rename a glb", async ({page, request})=>{ let doc = JSON.parse((await res.body()).toString()); expect(doc).toHaveProperty("setups"); expect(doc.setups).toHaveLength(1); - expect(doc.setups[0]).toHaveProperty("language", {language: "EN"}); + expect(doc.setups[0]).toHaveProperty("language", { language: "EN" }); - res = await request.get(`/scenes/${name}/models/${name}.glb`); + res = await request.get(`/scenes/${name}/cube.glb`); await expect(res).toBeOK(); expect(res.headers()).toHaveProperty("etag", "W/4diz3Hx67bxWyU9b_iCJD864pVJ6OGYCPh9sU40QyLs"); }); -test("uploads and rename a glb (force FR)", async ({page, request})=>{ +test("uploads and rename a glb (force FR)", async ({ page, request }) => { await page.goto("/ui/upload"); //We are forced to use the rename otherwise we'd have a name collision const name = randomUUID(); - const uploads = page.getByRole("region", {name: "uploads"}); - const f = page.getByRole("form", {name: "create or update a scene"}); + await page.locator("input[type=\"file\"]").setInputFiles(path.join(fixtures, "cube.glb")); + + await expect(page.getByRole("listitem").getByText("✓")).toBeVisible(); + + const f = page.getByRole("form", { name: "Create or update a scene" }); + const uploads = page.getByRole("region", { name: "Created Scenes" }); + await expect(f).toBeVisible(); await expect(uploads).not.toBeVisible(); - await f.getByRole("combobox", {name: "language"}).selectOption("fr"); - await f.getByRole("button", {name: "files"}).setInputFiles(path.join(fixtures, "cube.glb")); - await f.getByRole("textbox", {name: "scene title"}).fill(name) - await f.getByRole("button", {name: "create a scene"}).click(); + + await f.getByRole("combobox", { name: "Default language" }).selectOption("fr"); + await f.getByRole("textbox", { name: "Scene name" }).fill(name); + + await page.getByRole("button", { name: "create a scene" }).click(); await expect(uploads).toBeVisible(); //Don't check for actual progress bar visibility because that could be too quick to register - const link = uploads.getByRole("link", {name: name}); + const link = uploads.getByRole("link", { name: name }); await link.click(); await expect(page).toHaveURL(`/ui/scenes/${name}`); await expect(page.locator("h1")).toHaveText(name); @@ -66,49 +75,177 @@ test("uploads and rename a glb (force FR)", async ({page, request})=>{ let doc = JSON.parse((await res.body()).toString()); expect(doc).toHaveProperty("setups"); expect(doc.setups).toHaveLength(1); - expect(doc.setups[0]).toHaveProperty("language", {language: "FR"}); + expect(doc.setups[0]).toHaveProperty("language", { language: "FR" }); - res = await request.get(`/scenes/${name}/models/${name}.glb`); + res = await request.get(`/scenes/${name}/cube.glb`); await expect(res).toBeOK(); expect(res.headers()).toHaveProperty("etag", "W/4diz3Hx67bxWyU9b_iCJD864pVJ6OGYCPh9sU40QyLs"); }); -test("upload many glb", async ({page, request})=>{ +test("upload many glb", async ({ page, request }) => { await page.goto("/ui/upload"); - //We are forced to use the rename otherwise we'd have a name collision - const f = page.getByRole("form", {name: "create or update a scene"}); - await expect(f).toBeVisible(); - await expect(f.getByRole("combobox", {name: "language"})).toHaveValue("en"); + const name = randomUUID(); const content = await readFile(path.join(fixtures, "cube.glb")); - let files :{ + let files: { name: string; mimeType: string; buffer: Buffer; }[] = []; - for(let i = 0; i < 10; i++){ + for (let i = 0; i < 10; i++) { const buffer = Buffer.from(content); files.push({ - name: randomUUID()+".glb", - mimeType: "model/gltf+binary", + name: randomUUID() + ".glb", + mimeType: "model/gltf-binary", buffer, }) } + const section = page.locator("section"); + //Check that we can actually open the filechooser by clicking on the button + const fileChooserPromise = page.waitForEvent('filechooser'); + await page.getByText("select one or several files").click(); + const fileChooser = await fileChooserPromise; + fileChooser.setFiles(files); - await f.getByRole("button", {name: "files"}).setInputFiles(files); - await f.getByRole("button", {name: "create a scene"}).click(); + //Don't check for actual progress bar visibility because that could be too quick to register + //Just wait for all files to be done + for (let file of files) { + await expect(page.locator(`#upload-${file.name.replace(/[^-_a-z0-9]/g, "_")}.upload-done`)).toBeVisible(); + } + + const f = section.getByRole("form", { name: "Create or update a scene" }); + const btn = section.getByRole("button", { name: "create a scene" }); + await expect(f).toBeVisible(); + await expect(btn).toBeVisible(); + await btn.scrollIntoViewIfNeeded(); + + await f.getByRole("textbox", { name: "Scene name" }).fill(name); + await f.getByRole("combobox", { name: "Default language" }).selectOption("fr"); - const uploads = page.getByRole("region", {name: "uploads"}); + //Submit + await btn.click(); + + const uploads = page.getByRole("region", { name: "Created Scenes" }); await expect(uploads).toBeVisible(); - //Don't check for actual progress bar visibility because that could be too quick to register - for(let file of files){ - const name = file.name.split(".").slice(0, -1).join("."); - await expect(uploads.getByRole("link", {name: name})).toHaveAttribute("href", `/ui/scenes/${name}`) + const link = uploads.getByRole("link", { name: name }); + await link.click(); + await expect(page).toHaveURL(`/ui/scenes/${name}`); + await expect(page.locator("h1")).toHaveText(name); + + let res = await request.get(`/scenes/${name}/scene.svx.json`); + await expect(res).toBeOK(); + let doc = JSON.parse((await res.body()).toString()); + expect(doc).toHaveProperty("setups"); + expect(doc.setups).toHaveLength(1); + expect(doc.setups[0]).toHaveProperty("language", { language: "FR" }); + expect(doc.models).toHaveProperty("length", files.length); + + for (let file of files) { + res = await request.get(`/scenes/${name}/${file.name}`); + await expect(res).toBeOK(); + expect(res.headers()).toHaveProperty("etag", "W/4diz3Hx67bxWyU9b_iCJD864pVJ6OGYCPh9sU40QyLs"); } +}); + +test("uploads an obj with mtl and texture", async ({ page, request }) => { + await page.goto("/ui/upload"); + //We are forced to use the rename otherwise we'd have a name collision + const name = randomUUID(); + await page.locator("input[type=\"file\"]").setInputFiles([ + path.join(fixtures, "cube.obj"), + path.join(fixtures, "cube.mtl"), + path.join(fixtures, "Diffuse.jpg"), + ]); + + await expect(page.getByRole("listitem").getByText("✓")).toHaveCount(3); + + const f = page.getByRole("form", { name: "Create or update a scene" }); + await expect(f).toBeVisible(); + await expect(f.getByRole("combobox", { name: "Default language" })).toHaveValue("en"); + await f.getByRole("textbox", { name: "Scene name" }).fill(name); + await page.getByRole("button", { name: "create a scene" }).click(); + + const uploads = page.getByRole("region", { name: "Created Scenes" }); + await expect(uploads).toBeVisible(); + //Don't check for actual progress bar visibility because that could be too quick to register + const link = uploads.getByRole("link", { name: name }); + await link.click(); + await expect(page).toHaveURL(`/ui/scenes/${name}`); + await expect(page.locator("h1")).toHaveText(name); + + let res = await request.get(`/scenes/${name}/scene.svx.json`); + await expect(res).toBeOK(); + let doc = JSON.parse((await res.body()).toString()); + + expect(doc).toHaveProperty("setups"); + expect(doc.setups).toHaveLength(1); + expect(doc.setups[0]).toHaveProperty("language", { language: "EN" }); + + expect(doc).toHaveProperty("models"); + expect(doc.models).toHaveLength(1); + expect(doc.models[0]).toHaveProperty("derivatives"); + expect(doc.models[0].derivatives).toEqual([{ + "assets": [{ + "byteSize": 3884, + "imageSize": 96, + "numFaces": 12, + "type": "Model", + "uri": "cube.glb" + }], + "quality": "High", + "usage": "Web3D" + }]); + + + res = await request.get(`/scenes/${name}/cube.glb`); + await expect(res).toBeOK(); + // It may change without it being a problem. Check the actual file if necessary. + expect(res.headers()).toHaveProperty("etag", "W/yhH03TGHdkBQgKlzJcPpDFD9XdQk9Wq_vBxCzThegYY"); +}); + + +test("uploads and optimize a glb", async ({ page, request }) => { + await page.goto("/ui/upload"); + //We are forced to use the rename otherwise we'd have a name collision + const name = randomUUID(); + await page.locator("input[type=\"file\"]").setInputFiles(path.join(fixtures, "cube.glb")); + + await expect(page.getByRole("listitem").getByText("✓")).toBeVisible(); + + const f = page.getByRole("form", { name: "Create or update a scene" }); + await expect(f).toBeVisible(); + await expect(f.getByRole("combobox", { name: "Default language" })).toHaveValue("en"); + await f.getByRole("textbox", { name: "Scene name" }).fill(name); + await page.getByRole("checkbox", { name: "Optimize models", exact: false }).click(); + + await page.getByRole("button", { name: "create a scene" }).click(); + + const uploads = page.getByRole("region", { name: "Created Scenes" }); + await expect(uploads).toBeVisible(); + //Don't check for actual progress bar visibility because that could be too quick to register + const link = uploads.getByRole("link", { name: name }); + await link.click(); + await expect(page).toHaveURL(`/ui/scenes/${name}`); + await expect(page.locator("h1")).toHaveText(name); + + let res = await request.get(`/scenes/${name}/scene.svx.json`); + await expect(res).toBeOK(); + let doc = JSON.parse((await res.body()).toString()); + expect(doc).toHaveProperty("setups"); + expect(doc.setups).toHaveLength(1); + expect(doc.setups[0]).toHaveProperty("language", { language: "EN" }); + + + res = await request.get(`/scenes/${name}/cube.glb`); + await expect(res).toBeOK(); + const headers = res.headers(); + //We check the etag is different from what we'd have if we didn't request optimization + expect(headers).toHaveProperty("etag"); + expect(headers).not.toEqual("W/4diz3Hx67bxWyU9b_iCJD864pVJ6OGYCPh9sU40QyLs"); }); \ No newline at end of file diff --git a/source/e2e/tests/upload_zip.spec.ts b/source/e2e/tests/upload_zip.spec.ts index e25e4e507..fb05acb03 100644 --- a/source/e2e/tests/upload_zip.spec.ts +++ b/source/e2e/tests/upload_zip.spec.ts @@ -21,18 +21,18 @@ function reducePropfind(text:string) :ReducedWebDAVProps[]{ const multistatus = root.elements[0]; expect(multistatus).toHaveProperty("name", "D:multistatus"); const responses = multistatus.elements; - return responses.map(({elements})=>{ - const href = elements.find(e=>e.name === "D:href"); - expect(href, `find D:href in ${elements.map(p=>p.name)}`).toBeTruthy(); + return responses.map(({elements}:any)=>{ + const href = elements.find((e: any)=>e.name === "D:href"); + expect(href, `find D:href in ${elements.map((p:any)=>p.name)}`).toBeTruthy(); let item: ReducedWebDAVProps = { - path: new URL(href.elements.find(e=>e.type === "text").text).pathname, + path: new URL(href.elements.find((e: any)=>e.type === "text").text).pathname, }; - const propstat = elements.find(e=>e.name === "D:propstat"); - expect(propstat, `find D:propstat in ${elements.map(p=>p.name)}`).toBeTruthy(); - const props = propstat.elements.find(e=>e.name === "D:prop"); + const propstat = elements.find((e: any)=>e.name === "D:propstat"); + expect(propstat, `find D:propstat in ${elements.map((p: any)=>p.name)}`).toBeTruthy(); + const props = propstat.elements.find((e: any)=>e.name === "D:prop"); for(const el of props.elements){ - const content = el.elements?.find(e=>e.type ==="text")?.text; + const content = el.elements?.find((e: any)=>e.type ==="text")?.text; switch(el.name){ case "D:getetag": item.etag = content; @@ -78,23 +78,27 @@ test("uploads a scene zip", async ({page, request})=>{ await page.goto("/ui/upload"); - const f = page.getByRole("form", {name: "titles.createOrUpdateScene"}); - await expect(f).toBeVisible(); - await f.getByRole("button", {name: "labels.selectFile"}).setInputFiles({ + + + await page.locator("input[type=\"file\"]").setInputFiles({ name: "scene.zip", mimeType: "application/zip", buffer: body, }); - await f.getByRole("button", {name: "buttons.upload"}).click(); + + //We expect to see a list of scenes that will be uploaded + //In our case we expect "create" status since the scene has been deleted + + await page.getByRole("button", {name: "buttons.upload"}).click(); - const uploads = page.getByRole("region", {name: "uploads"}); + const uploads = page.getByRole("region", {name: "titles.createdScenes"}); await expect(uploads).toBeVisible(); //Don't check for actual progress bar visibility because that could be too quick to register const link = uploads.getByRole("link", {name: name}); await link.click(); await expect(page).toHaveURL(`/ui/scenes/${name}`); - await expect(page.getByRole("heading", {name})).toBeVisible(); + await expect(page.getByRole("heading", {name, }).first()).toBeVisible(); }); @@ -127,27 +131,34 @@ test("uploads a multi-scene zip", async ({page, request})=>{ let body = await res.body(); - await Promise.all(names.map(async (name)=>{ - //Delete the scene - res = await request.delete(`/scenes/${name}?archive=false`); - await expect(res).toBeOK(); - })); - + //Delete the first scene + res = await request.delete(`/scenes/${names[0]}?archive=false`); + await expect(res).toBeOK(); await page.goto("/ui/upload"); - const f = page.getByRole("form", {name: "titles.createOrUpdateScene"}); - await expect(f).toBeVisible(); - await f.getByRole("button", {name: "labels.selectFile"}).setInputFiles({ + + + await page.locator("input[type=\"file\"]").setInputFiles({ name: "scene.zip", mimeType: "application/zip", buffer: body, }); - await f.getByRole("button", {name: "buttons.upload"}).click(); + + const btn = page.getByRole("button", {name: "buttons.upload"}); + + await expect(btn).toBeVisible(); + + //Expect first scene to be "create", because we just deleted it. Second scene should be "update" + await expect(page.getByText(`[CREATE] ${names[0]}`)).toBeVisible(); + await expect(page.getByText(`[UPDATE] ${names[1]}`)).toBeVisible(); + + + await btn.click(); - const uploads = page.getByRole("region", {name: "uploads"}); + const uploads = page.getByRole("region", {name: "titles.createdScenes"}); for (let name of names){ await expect(uploads).toBeVisible(); //Don't check for actual progress bar visibility because that could be too quick to register diff --git a/source/e2e/tests/userSettings.spec.ts b/source/e2e/tests/userSettings.spec.ts index ed3244013..a489b1dde 100644 --- a/source/e2e/tests/userSettings.spec.ts +++ b/source/e2e/tests/userSettings.spec.ts @@ -2,7 +2,7 @@ import path from "node:path"; import fs, { readFile } from "node:fs/promises"; import { randomBytes, randomUUID } from "node:crypto"; -import { test, expect } from '../fixtures'; +import { test, expect } from '../fixtures.js'; const fixtures = path.resolve(import.meta.dirname, "../__test_fixtures"); @@ -13,9 +13,12 @@ const fixtures = path.resolve(import.meta.dirname, "../__test_fixtures"); //Runs with a per-test storageState, in locale "cimode" test.use({ storageState: { cookies: [], origins: [] }, locale: "cimode" }); -test.beforeEach(async ({page, uniqueAccount:{username, password}})=>{ +let account: {username:string, password:string, uid:number}; + +test.beforeEach(async ({page, uniqueAccount})=>{ + account = await uniqueAccount(); let res = await page.request.post("/auth/login", { - data: JSON.stringify({username, password}), + data: JSON.stringify({username: account.username, password: account.password}), headers:{ "Content-Type": "application/json", } @@ -33,9 +36,9 @@ test("can read user settings page", async function({page}){ }); -test("can change email", async ({page, uniqueAccount})=>{ +test("can change email", async ({page})=>{ //Ensure this is unique, otherwise it is rejected - let new_email = `${uniqueAccount.username}-replacement@example2.com` + let new_email = `${account.username}-replacement@example2.com` await page.goto("/ui/user/"); const form = page.getByRole("form", {name: "titles.userProfile"}); await expect(form).toBeVisible(); @@ -53,7 +56,8 @@ test("can change email", async ({page, uniqueAccount})=>{ await expect(emailField).toHaveValue(new_email); }); -test("can change password", async ({baseURL, page, uniqueAccount:{username, password}})=>{ +test("can change password", async ({baseURL, page})=>{ + const {username, password} = account; const new_password = randomBytes(10).toString("base64"); let res = await fetch(new URL(`/auth/login`, baseURL), { @@ -109,7 +113,8 @@ test("can logout", async ({page})=>{ expect(await res.json()).toHaveProperty("level", "none"); }); -test("can show archived scenes", async ({page, uniqueAccount:{username}})=>{ +test("can show archived scenes", async ({page})=>{ + const {username} = account; const name = randomUUID(); const fs = await import("node:fs/promises"); const data = await fs.readFile(path.join(fixtures, "cube.glb")) diff --git a/source/e2e/tsconfig.json b/source/e2e/tsconfig.json new file mode 100644 index 000000000..34a478a74 --- /dev/null +++ b/source/e2e/tsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "lib": ["ESNext"], + "module": "NodeNext", + "target": "ES2021", + "strict": true, + "skipLibCheck": true, + "types": ["@playwright/test"], + "outDir": "./dist", + "rootDir": "." + }, + "include": ["./**/*.ts"], + "exclude": [ + "node_modules" + ] +} diff --git a/source/server/__test_fixtures/cube.mtl b/source/server/__test_fixtures/cube.mtl new file mode 100644 index 000000000..78a331544 --- /dev/null +++ b/source/server/__test_fixtures/cube.mtl @@ -0,0 +1,12 @@ +# Blender 4.2.18 LTS MTL File: 'None' +# www.blender.org + +newmtl Material +Ns 250.000000 +Ka 1.000000 1.000000 1.000000 +Kd 0.800000 0.800000 0.800000 +Ks 0.500000 0.500000 0.500000 +Ke 0.000000 0.000000 0.000000 +Ni 1.450000 +d 1.000000 +illum 2 diff --git a/source/server/__test_fixtures/cube.obj b/source/server/__test_fixtures/cube.obj new file mode 100644 index 000000000..17ac39311 --- /dev/null +++ b/source/server/__test_fixtures/cube.obj @@ -0,0 +1,40 @@ +# Blender 4.2.18 LTS +# www.blender.org +mtllib cube.mtl +o Cube +v 1.000000 1.000000 -1.000000 +v 1.000000 -1.000000 -1.000000 +v 1.000000 1.000000 1.000000 +v 1.000000 -1.000000 1.000000 +v -1.000000 1.000000 -1.000000 +v -1.000000 -1.000000 -1.000000 +v -1.000000 1.000000 1.000000 +v -1.000000 -1.000000 1.000000 +vn -0.0000 1.0000 -0.0000 +vn -0.0000 -0.0000 1.0000 +vn -1.0000 -0.0000 -0.0000 +vn -0.0000 -1.0000 -0.0000 +vn 1.0000 -0.0000 -0.0000 +vn -0.0000 -0.0000 -1.0000 +vt 0.625000 0.500000 +vt 0.875000 0.500000 +vt 0.875000 0.750000 +vt 0.625000 0.750000 +vt 0.375000 0.750000 +vt 0.625000 1.000000 +vt 0.375000 1.000000 +vt 0.375000 0.000000 +vt 0.625000 0.000000 +vt 0.625000 0.250000 +vt 0.375000 0.250000 +vt 0.125000 0.500000 +vt 0.375000 0.500000 +vt 0.125000 0.750000 +s 0 +usemtl Material +f 1/1/1 5/2/1 7/3/1 3/4/1 +f 4/5/2 3/4/2 7/6/2 8/7/2 +f 8/8/3 7/9/3 5/10/3 6/11/3 +f 6/12/4 2/13/4 4/5/4 8/14/4 +f 2/13/5 1/1/5 3/4/5 4/5/5 +f 6/11/6 5/10/6 1/1/6 2/13/6 diff --git a/source/server/integration.test.ts b/source/server/integration.test.ts index ad8300f85..768fe013a 100644 --- a/source/server/integration.test.ts +++ b/source/server/integration.test.ts @@ -67,11 +67,11 @@ describe("Web Server Integration", function(){ .set("Content-Type", "application/octet-stream") .send(content) expect(r.status, `Expected status code 201 but received [${r.status}]: ${r.text}`).to.equal(201); - let res = await this.agent.get("/scenes/bar/models/bar.glb").expect(200); + let res = await this.agent.get("/scenes/bar/bar.glb").expect(200); expect(res.text.slice(0,4).toString()).to.equal("glTF"); expect(res.text.length).to.equal(content.length); - let {body:doc} = await this.agent.get("/scenes/bar/bar.svx.json").expect(200); + let {body:doc} = await this.agent.get("/scenes/bar/scene.svx.json").expect(200); expect(doc).to.have.property("models").an("array").to.have.length(1); }); diff --git a/source/server/package-lock.json b/source/server/package-lock.json index 97ae35f88..94008d71d 100644 --- a/source/server/package-lock.json +++ b/source/server/package-lock.json @@ -26,6 +26,7 @@ "mime-types": "^2.1.35", "morgan": "^1.10.0", "nodemailer": "^6.9.16", + "obj2gltf": "^3.2.0", "pg": "^8.16.0", "pg-cursor": "^2.15.0", "sharp": "^0.34.5", @@ -74,6 +75,57 @@ "node": ">=6.9.0" } }, + "node_modules/@cesium/engine": { + "version": "22.3.0", + "resolved": "https://registry.npmjs.org/@cesium/engine/-/engine-22.3.0.tgz", + "integrity": "sha512-oDl+nWX/qfHYQ0lEdGxLqZoKEtTMghvJDzZKTycYfiIuDYDh8Kh0Oy45wr3mSJse3PuTj1e6hDmbw8vbycCOxw==", + "license": "Apache-2.0", + "dependencies": { + "@cesium/wasm-splats": "^0.1.0-alpha.2", + "@spz-loader/core": "0.3.0", + "@tweenjs/tween.js": "^25.0.0", + "@zip.js/zip.js": "^2.8.1", + "autolinker": "^4.0.0", + "bitmap-sdf": "^1.0.3", + "dompurify": "^3.3.0", + "draco3d": "^1.5.1", + "earcut": "^3.0.0", + "grapheme-splitter": "^1.0.4", + "jsep": "^1.3.8", + "kdbush": "^4.0.1", + "ktx-parse": "^1.0.0", + "lerc": "^2.0.0", + "mersenne-twister": "^1.1.0", + "meshoptimizer": "^1.0.1", + "pako": "^2.0.4", + "protobufjs": "^8.0.0", + "rbush": "^4.0.1", + "topojson-client": "^3.1.0", + "urijs": "^1.19.7" + }, + "engines": { + "node": ">=20.19.0" + } + }, + "node_modules/@cesium/wasm-splats": { + "version": "0.1.0-alpha.2", + "resolved": "https://registry.npmjs.org/@cesium/wasm-splats/-/wasm-splats-0.1.0-alpha.2.tgz", + "integrity": "sha512-t9pMkknv31hhIbLpMa8yPvmqfpvs5UkUjgqlQv9SeO8VerCXOYnyP8/486BDaFrztM0A7FMbRjsXtNeKvqQghA==", + "license": "Apache-2.0" + }, + "node_modules/@cesium/widgets": { + "version": "14.3.0", + "resolved": "https://registry.npmjs.org/@cesium/widgets/-/widgets-14.3.0.tgz", + "integrity": "sha512-1bS+Nv/uXwP0/NV0o4XeUA5nLCWttjTmKwl+pHnbZXp0ZwDmClb0xVDruDyVtLrUuRhsk84JZ4rXI/IT7HXOvA==", + "license": "Apache-2.0", + "dependencies": { + "@cesium/engine": "^22.3.0", + "nosleep.js": "^0.12.0" + }, + "engines": { + "node": ">=20.19.0" + } + }, "node_modules/@cspotcode/source-map-support": { "version": "0.8.1", "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", @@ -628,6 +680,80 @@ "integrity": "sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw==", "dev": true }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", + "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", + "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", + "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==", + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1", + "@protobufjs/inquire": "^1.1.0" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/inquire": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", + "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", + "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==", + "license": "BSD-3-Clause" + }, + "node_modules/@spz-loader/core": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@spz-loader/core/-/core-0.3.0.tgz", + "integrity": "sha512-sbStwMHb/MIE29st7rRuMYWqhX1UmLSFzdpyGtUZUXLkFNIuYKblzjQdtiet8bau8sUf21uL1DQ451zuySGmcA==", + "license": "Apache-2.0", + "engines": { + "node": ">=16", + "pnpm": ">=8" + } + }, "node_modules/@tsconfig/node10": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.9.tgz", @@ -652,6 +778,12 @@ "integrity": "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==", "dev": true }, + "node_modules/@tweenjs/tween.js": { + "version": "25.0.0", + "resolved": "https://registry.npmjs.org/@tweenjs/tween.js/-/tween.js-25.0.0.tgz", + "integrity": "sha512-XKLA6syeBUaPzx4j3qwMqzzq+V4uo72BnlbOjmuljLrRqdsd3qnzvZZoxvMHZ23ndsRS4aufU6JOZYpCbU6T1A==", + "license": "MIT" + }, "node_modules/@types/body-parser": { "version": "1.19.2", "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.2.tgz", @@ -817,8 +949,7 @@ "node_modules/@types/node": { "version": "16.18.23", "resolved": "https://registry.npmjs.org/@types/node/-/node-16.18.23.tgz", - "integrity": "sha512-XAMpaw1s1+6zM+jn2tmw8MyaRDIJfXxqmIQIS0HfoGYPuf7dUWeiUKopwq13KFX9lEp1+THGtlaaYx39Nxr58g==", - "dev": true + "integrity": "sha512-XAMpaw1s1+6zM+jn2tmw8MyaRDIJfXxqmIQIS0HfoGYPuf7dUWeiUKopwq13KFX9lEp1+THGtlaaYx39Nxr58g==" }, "node_modules/@types/nodemailer": { "version": "6.4.17", @@ -912,6 +1043,13 @@ "@types/superagent": "^8.1.0" } }, + "node_modules/@types/trusted-types": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.7.tgz", + "integrity": "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==", + "license": "MIT", + "optional": true + }, "node_modules/@types/yauzl": { "version": "2.10.3", "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.10.3.tgz", @@ -930,6 +1068,17 @@ "@types/node": "*" } }, + "node_modules/@zip.js/zip.js": { + "version": "2.8.21", + "resolved": "https://registry.npmjs.org/@zip.js/zip.js/-/zip.js-2.8.21.tgz", + "integrity": "sha512-fkyzXISE3IMrstDO1AgPkJCx14MYHP/suIGiAovEYEuBjq3mffsuL6aMV7ohOSjW4rXtuACuUfpA3GtITgdtYg==", + "license": "BSD-3-Clause", + "engines": { + "bun": ">=0.7.0", + "deno": ">=1.0.0", + "node": ">=18.0.0" + } + }, "node_modules/accepts": { "version": "1.3.8", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", @@ -976,7 +1125,6 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true, "engines": { "node": ">=8" } @@ -985,7 +1133,6 @@ "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dev": true, "dependencies": { "color-convert": "^2.0.1" }, @@ -1047,6 +1194,18 @@ "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", "dev": true }, + "node_modules/autolinker": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/autolinker/-/autolinker-4.1.5.tgz", + "integrity": "sha512-vEfYZPmvVOIuE567XBVCsx8SBgOYtjB2+S1iAaJ+HgH+DNjAcrHem2hmAeC9yaNGWayicv4yR+9UaJlkF3pvtw==", + "license": "MIT", + "dependencies": { + "tslib": "^2.8.1" + }, + "engines": { + "pnpm": ">=10.10.0" + } + }, "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", @@ -1078,6 +1237,18 @@ "node": ">=8" } }, + "node_modules/bitmap-sdf": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/bitmap-sdf/-/bitmap-sdf-1.0.4.tgz", + "integrity": "sha512-1G3U4n5JE6RAiALMxu0p1XmeZkTeCwGKykzsLTCqVzfSDaN6S7fKnkIkfejogz+iwqBWc0UYAIKnKHNN7pSfDg==", + "license": "MIT" + }, + "node_modules/bluebird": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", + "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==", + "license": "MIT" + }, "node_modules/body-parser": { "version": "1.20.3", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.3.tgz", @@ -1188,6 +1359,24 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/cesium": { + "version": "1.138.0", + "resolved": "https://registry.npmjs.org/cesium/-/cesium-1.138.0.tgz", + "integrity": "sha512-YX7Ttd4LzAxunuzcKPyOCQa+BPc2RmenqnkM5uZkk/GVwor724bd+F3kdVP4IyMbTgxFkchXuX2Aa8L1Y0/ZxA==", + "license": "Apache-2.0", + "workspaces": [ + "packages/engine", + "packages/widgets", + "packages/sandcastle" + ], + "dependencies": { + "@cesium/engine": "^22.3.0", + "@cesium/widgets": "^14.3.0" + }, + "engines": { + "node": ">=20.19.0" + } + }, "node_modules/chai": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/chai/-/chai-5.1.2.tgz", @@ -1295,7 +1484,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dev": true, "dependencies": { "color-name": "~1.1.4" }, @@ -1306,8 +1494,7 @@ "node_modules/color-name": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "dev": true + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" }, "node_modules/combined-stream": { "version": "1.0.8", @@ -1321,6 +1508,12 @@ "node": ">= 0.8" } }, + "node_modules/commander": { + "version": "2.20.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", + "license": "MIT" + }, "node_modules/component-emitter": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.3.1.tgz", @@ -1534,6 +1727,21 @@ "node": ">=0.3.1" } }, + "node_modules/dompurify": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.3.1.tgz", + "integrity": "sha512-qkdCKzLNtrgPFP1Vo+98FRzJnBRGe4ffyCea9IwHB1fyxPOeNTHpLKYGd4Uk9xvNoH0ZoOjwZxNptyMwqrId1Q==", + "license": "(MPL-2.0 OR Apache-2.0)", + "optionalDependencies": { + "@types/trusted-types": "^2.0.7" + } + }, + "node_modules/draco3d": { + "version": "1.5.7", + "resolved": "https://registry.npmjs.org/draco3d/-/draco3d-1.5.7.tgz", + "integrity": "sha512-m6WCKt/erDXcw+70IJXnG7M3awwQPAsZvJGX5zY7beBqpELw6RDGkYVU0W43AFxye4pDZ5i2Lbyc/NNGqwjUVQ==", + "license": "Apache-2.0" + }, "node_modules/draco3dgltf": { "version": "1.5.7", "resolved": "https://registry.npmjs.org/draco3dgltf/-/draco3dgltf-1.5.7.tgz", @@ -1553,6 +1761,12 @@ "node": ">= 0.4" } }, + "node_modules/earcut": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/earcut/-/earcut-3.0.2.tgz", + "integrity": "sha512-X7hshQbLyMJ/3RPhyObLARM2sNxxmRALLKx1+NVFFnQ9gKzmCrxm9+uLIAdBcvc8FNLpctqlQ2V6AE92Ol9UDQ==", + "license": "ISC" + }, "node_modules/ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", @@ -1561,8 +1775,7 @@ "node_modules/emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "dev": true + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" }, "node_modules/encodeurl": { "version": "2.0.0", @@ -1603,7 +1816,6 @@ "version": "3.2.0", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", - "dev": true, "engines": { "node": ">=6" } @@ -1832,6 +2044,20 @@ "node": ">= 0.6" } }, + "node_modules/fs-extra": { + "version": "11.3.3", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.3.3.tgz", + "integrity": "sha512-VWSRii4t0AFm6ixFFmLLx1t7wS1gh+ckoa84aOeapGum0h+EZd1EhEumSB+ZdDLnEPuucsVB9oB7cxJHap6Afg==", + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + }, + "engines": { + "node": ">=14.14" + } + }, "node_modules/fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", @@ -1864,7 +2090,6 @@ "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", - "dev": true, "engines": { "node": "6.* || 8.* || >= 10.*" } @@ -1927,6 +2152,18 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "license": "ISC" + }, + "node_modules/grapheme-splitter": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/grapheme-splitter/-/grapheme-splitter-1.0.4.tgz", + "integrity": "sha512-bzh50DW9kTPM00T8y4o8vQg89Di9oLJVLW/KaOGIXJWP/iqCN6WKYkbNOF04vFLJhwcpYUh9ydh/+5vpOqV4YQ==", + "license": "MIT" + }, "node_modules/handlebars": { "version": "4.7.8", "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.8.tgz", @@ -2117,7 +2354,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "dev": true, "engines": { "node": ">=8" } @@ -2164,6 +2400,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/jpeg-js": { + "version": "0.4.4", + "resolved": "https://registry.npmjs.org/jpeg-js/-/jpeg-js-0.4.4.tgz", + "integrity": "sha512-WZzeDOEtTOBK4Mdsar0IqEU5sMr3vSV2RqkAIzUEV2BHnUfKGyswWFPFwK5EeDo93K3FohSHbLAjj0s1Wzd+dg==", + "license": "BSD-3-Clause" + }, "node_modules/js-yaml": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", @@ -2176,6 +2418,33 @@ "js-yaml": "bin/js-yaml.js" } }, + "node_modules/jsep": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/jsep/-/jsep-1.4.0.tgz", + "integrity": "sha512-B7qPcEVE3NVkmSJbaYxvv4cHkVW7DQsZz13pUMrfS8z8Q/BuShN+gcTXrUlPiGqM2/t/EEaI030bpxMqY8gMlw==", + "license": "MIT", + "engines": { + "node": ">= 10.16.0" + } + }, + "node_modules/jsonfile": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.2.0.tgz", + "integrity": "sha512-FGuPw30AdOIUTRMC2OMRtQV+jkVj2cfPqSeWXv1NEAJ1qZ5zb1X6z1mFhbfOB/iy3ssJCD+3KuZ8r8C3uVFlAg==", + "license": "MIT", + "dependencies": { + "universalify": "^2.0.0" + }, + "optionalDependencies": { + "graceful-fs": "^4.1.6" + } + }, + "node_modules/kdbush": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/kdbush/-/kdbush-4.0.2.tgz", + "integrity": "sha512-WbCVYJ27Sz8zi9Q7Q0xHC+05iwkm3Znipc2XTlrnJbsHMYktW4hPhXUE8Ys1engBrvffoSCqbil1JQAa7clRpA==", + "license": "ISC" + }, "node_modules/keygrip": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/keygrip/-/keygrip-1.1.0.tgz", @@ -2193,6 +2462,12 @@ "integrity": "sha512-mKp3y+FaYgR7mXWAbyyzpa/r1zDWeaunH+INJO4fou3hb45XuNSwar+7llrRyvpMWafxSIi99RNFJ05MHedaJQ==", "license": "MIT" }, + "node_modules/lerc": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/lerc/-/lerc-2.0.0.tgz", + "integrity": "sha512-7qo1Mq8ZNmaR4USHHm615nEW2lPeeWJ3bTyoqFbd35DLx0LUH7C6ptt5FDCTAlbIzs3+WKrk5SkJvw8AFDE2hg==", + "license": "Apache-2.0" + }, "node_modules/locate-path": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", @@ -2224,6 +2499,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", + "license": "Apache-2.0" + }, "node_modules/loupe": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/loupe/-/loupe-3.1.2.tgz", @@ -2260,6 +2541,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/mersenne-twister": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/mersenne-twister/-/mersenne-twister-1.1.0.tgz", + "integrity": "sha512-mUYWsMKNrm4lfygPkL3OfGzOPTR2DBlTkBNHM//F6hGp8cLThY897crAlk3/Jo17LEOOjQUrNAx6DvgO77QJkA==", + "license": "MIT" + }, "node_modules/meshoptimizer": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/meshoptimizer/-/meshoptimizer-1.0.1.tgz", @@ -2505,6 +2792,86 @@ "node": ">=0.10.0" } }, + "node_modules/nosleep.js": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/nosleep.js/-/nosleep.js-0.12.0.tgz", + "integrity": "sha512-9d1HbpKLh3sdWlhXMhU6MMH+wQzKkrgfRkYV0EBdvt99YJfj0ilCJrWRDYG2130Tm4GXbEoTCx5b34JSaP+HhA==", + "license": "MIT" + }, + "node_modules/obj2gltf": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/obj2gltf/-/obj2gltf-3.2.0.tgz", + "integrity": "sha512-1pCbHSK55tiTkJG8Td0Nfqx97jcCtIKNeoukWhmuiyEtty3gmLBxHRN6WdYM6XKKAVgZVgeJ/PxXAizeRbQFxQ==", + "license": "Apache-2.0", + "dependencies": { + "bluebird": "^3.7.2", + "cesium": "^1.86.1", + "fs-extra": "^11.0.0", + "jpeg-js": "^0.4.3", + "mime": "^3.0.0", + "pngjs": "^7.0.0", + "yargs": "^17.2.1" + }, + "bin": { + "obj2gltf": "bin/obj2gltf.js" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/obj2gltf/node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "license": "ISC", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/obj2gltf/node_modules/mime": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-3.0.0.tgz", + "integrity": "sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A==", + "license": "MIT", + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/obj2gltf/node_modules/yargs": { + "version": "17.7.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", + "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", + "license": "MIT", + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/obj2gltf/node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, "node_modules/object-inspect": { "version": "1.13.3", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.3.tgz", @@ -2574,6 +2941,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/pako": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/pako/-/pako-2.1.0.tgz", + "integrity": "sha512-w+eufiZ1WuJYgPXbV/PO3NCMEc3xqylkKHzp8bxp1uW4qaSNQUkwmLLEc3kKsfz8lpV1F8Ht3U1Cm+9Srog2ug==", + "license": "(MIT AND Zlib)" + }, "node_modules/parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", @@ -2720,6 +3093,15 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/pngjs": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-7.0.0.tgz", + "integrity": "sha512-LKWqWJRhstyYo9pGvgor/ivk2w94eSjE3RGVuzLGlr3NmD8bf7RcYGze1mNdEHRP6TRP6rMuDHk5t44hnTRyow==", + "license": "MIT", + "engines": { + "node": ">=14.19.0" + } + }, "node_modules/postgres-array": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", @@ -2765,6 +3147,30 @@ "integrity": "sha512-I0hojAJfTbSCZy3y6xyK29eayxo14v1bj1VPiDkHjTdz33SV6RdfMz2AHnf4ai62Vng2mN5GkaKahkooBIo9gA==", "license": "MIT" }, + "node_modules/protobufjs": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-8.0.0.tgz", + "integrity": "sha512-jx6+sE9h/UryaCZhsJWbJtTEy47yXoGNYI4z8ZaRncM0zBKeRqjO2JEcOUYwrYGb1WLhXM1FfMzW3annvFv0rw==", + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.4", + "@protobufjs/eventemitter": "^1.1.0", + "@protobufjs/fetch": "^1.1.0", + "@protobufjs/float": "^1.0.2", + "@protobufjs/inquire": "^1.1.0", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.0", + "@types/node": ">=13.7.0", + "long": "^5.0.0" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", @@ -2791,6 +3197,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/quickselect": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/quickselect/-/quickselect-3.0.0.tgz", + "integrity": "sha512-XdjUArbK4Bm5fLLvlm5KpTFOiOThgfWWI4axAZDWg4E/0mKdZyI9tNEfds27qCi1ze/vwTR16kvmmGhRra3c2g==", + "license": "ISC" + }, "node_modules/randombytes": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", @@ -2822,6 +3234,15 @@ "node": ">= 0.8" } }, + "node_modules/rbush": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/rbush/-/rbush-4.0.1.tgz", + "integrity": "sha512-IP0UpfeWQujYC8Jg162rMNc01Rf0gWMMAb2Uxus/Q0qOFw4lCcq6ZnQEZwUoJqWyUGJ9th7JjwI4yIWo+uvoAQ==", + "license": "MIT", + "dependencies": { + "quickselect": "^3.0.0" + } + }, "node_modules/readdirp": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", @@ -2843,7 +3264,6 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", - "dev": true, "engines": { "node": ">=0.10.0" } @@ -3117,7 +3537,6 @@ "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dev": true, "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", @@ -3131,7 +3550,6 @@ "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, "dependencies": { "ansi-regex": "^5.0.1" }, @@ -3231,6 +3649,20 @@ "node": ">=0.6" } }, + "node_modules/topojson-client": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/topojson-client/-/topojson-client-3.1.0.tgz", + "integrity": "sha512-605uxS6bcYxGXw9qi62XyrV6Q3xwbndjachmNxu8HWTtVPxZfEJN9fd/SZS1Q54Sn2y0TMyMxFj/cJINqGHrKw==", + "license": "ISC", + "dependencies": { + "commander": "2" + }, + "bin": { + "topo2geo": "bin/topo2geo", + "topomerge": "bin/topomerge", + "topoquantize": "bin/topoquantize" + } + }, "node_modules/ts-node": { "version": "10.9.2", "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.2.tgz", @@ -3287,8 +3719,7 @@ "version": "2.8.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", - "license": "0BSD", - "optional": true + "license": "0BSD" }, "node_modules/tsscmp": { "version": "1.0.6", @@ -3342,6 +3773,15 @@ "integrity": "sha512-Gw+zz50YNKPDKXs+9d+aKAjVwpjNwqzvNpLigIruT4HA9lMZNdMqs9x07kKHB/L9WRzqp4+DlTU5s4wG2esdoA==", "license": "MIT" }, + "node_modules/universalify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz", + "integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==", + "license": "MIT", + "engines": { + "node": ">= 10.0.0" + } + }, "node_modules/unpipe": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", @@ -3350,6 +3790,12 @@ "node": ">= 0.8" } }, + "node_modules/urijs": { + "version": "1.19.11", + "resolved": "https://registry.npmjs.org/urijs/-/urijs-1.19.11.tgz", + "integrity": "sha512-HXgFDgDommxn5/bIv0cnQZsPhHDA90NPHD6+c/v21U5+Sx5hoP8+dP9IZXBU1gIfvdRfhG8cel9QNPeionfcCQ==", + "license": "MIT" + }, "node_modules/utils-merge": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", @@ -3387,7 +3833,6 @@ "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "dev": true, "dependencies": { "ansi-styles": "^4.0.0", "string-width": "^4.1.0", @@ -3430,7 +3875,6 @@ "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", - "dev": true, "engines": { "node": ">=10" } @@ -3536,6 +3980,48 @@ "regenerator-runtime": "^0.14.0" } }, + "@cesium/engine": { + "version": "22.3.0", + "resolved": "https://registry.npmjs.org/@cesium/engine/-/engine-22.3.0.tgz", + "integrity": "sha512-oDl+nWX/qfHYQ0lEdGxLqZoKEtTMghvJDzZKTycYfiIuDYDh8Kh0Oy45wr3mSJse3PuTj1e6hDmbw8vbycCOxw==", + "requires": { + "@cesium/wasm-splats": "^0.1.0-alpha.2", + "@spz-loader/core": "0.3.0", + "@tweenjs/tween.js": "^25.0.0", + "@zip.js/zip.js": "^2.8.1", + "autolinker": "^4.0.0", + "bitmap-sdf": "^1.0.3", + "dompurify": "^3.3.0", + "draco3d": "^1.5.1", + "earcut": "^3.0.0", + "grapheme-splitter": "^1.0.4", + "jsep": "^1.3.8", + "kdbush": "^4.0.1", + "ktx-parse": "^1.0.0", + "lerc": "^2.0.0", + "mersenne-twister": "^1.1.0", + "meshoptimizer": "^1.0.1", + "pako": "^2.0.4", + "protobufjs": "^8.0.0", + "rbush": "^4.0.1", + "topojson-client": "^3.1.0", + "urijs": "^1.19.7" + } + }, + "@cesium/wasm-splats": { + "version": "0.1.0-alpha.2", + "resolved": "https://registry.npmjs.org/@cesium/wasm-splats/-/wasm-splats-0.1.0-alpha.2.tgz", + "integrity": "sha512-t9pMkknv31hhIbLpMa8yPvmqfpvs5UkUjgqlQv9SeO8VerCXOYnyP8/486BDaFrztM0A7FMbRjsXtNeKvqQghA==" + }, + "@cesium/widgets": { + "version": "14.3.0", + "resolved": "https://registry.npmjs.org/@cesium/widgets/-/widgets-14.3.0.tgz", + "integrity": "sha512-1bS+Nv/uXwP0/NV0o4XeUA5nLCWttjTmKwl+pHnbZXp0ZwDmClb0xVDruDyVtLrUuRhsk84JZ4rXI/IT7HXOvA==", + "requires": { + "@cesium/engine": "^22.3.0", + "nosleep.js": "^0.12.0" + } + }, "@cspotcode/source-map-support": { "version": "0.8.1", "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", @@ -3790,6 +4276,65 @@ "integrity": "sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw==", "dev": true }, + "@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==" + }, + "@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==" + }, + "@protobufjs/codegen": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", + "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==" + }, + "@protobufjs/eventemitter": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", + "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==" + }, + "@protobufjs/fetch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", + "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==", + "requires": { + "@protobufjs/aspromise": "^1.1.1", + "@protobufjs/inquire": "^1.1.0" + } + }, + "@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==" + }, + "@protobufjs/inquire": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", + "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==" + }, + "@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==" + }, + "@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==" + }, + "@protobufjs/utf8": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", + "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==" + }, + "@spz-loader/core": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@spz-loader/core/-/core-0.3.0.tgz", + "integrity": "sha512-sbStwMHb/MIE29st7rRuMYWqhX1UmLSFzdpyGtUZUXLkFNIuYKblzjQdtiet8bau8sUf21uL1DQ451zuySGmcA==" + }, "@tsconfig/node10": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.9.tgz", @@ -3814,6 +4359,11 @@ "integrity": "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==", "dev": true }, + "@tweenjs/tween.js": { + "version": "25.0.0", + "resolved": "https://registry.npmjs.org/@tweenjs/tween.js/-/tween.js-25.0.0.tgz", + "integrity": "sha512-XKLA6syeBUaPzx4j3qwMqzzq+V4uo72BnlbOjmuljLrRqdsd3qnzvZZoxvMHZ23ndsRS4aufU6JOZYpCbU6T1A==" + }, "@types/body-parser": { "version": "1.19.2", "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.2.tgz", @@ -3972,8 +4522,7 @@ "@types/node": { "version": "16.18.23", "resolved": "https://registry.npmjs.org/@types/node/-/node-16.18.23.tgz", - "integrity": "sha512-XAMpaw1s1+6zM+jn2tmw8MyaRDIJfXxqmIQIS0HfoGYPuf7dUWeiUKopwq13KFX9lEp1+THGtlaaYx39Nxr58g==", - "dev": true + "integrity": "sha512-XAMpaw1s1+6zM+jn2tmw8MyaRDIJfXxqmIQIS0HfoGYPuf7dUWeiUKopwq13KFX9lEp1+THGtlaaYx39Nxr58g==" }, "@types/nodemailer": { "version": "6.4.17", @@ -4067,6 +4616,12 @@ "@types/superagent": "^8.1.0" } }, + "@types/trusted-types": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.7.tgz", + "integrity": "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==", + "optional": true + }, "@types/yauzl": { "version": "2.10.3", "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.10.3.tgz", @@ -4085,6 +4640,11 @@ "@types/node": "*" } }, + "@zip.js/zip.js": { + "version": "2.8.21", + "resolved": "https://registry.npmjs.org/@zip.js/zip.js/-/zip.js-2.8.21.tgz", + "integrity": "sha512-fkyzXISE3IMrstDO1AgPkJCx14MYHP/suIGiAovEYEuBjq3mffsuL6aMV7ohOSjW4rXtuACuUfpA3GtITgdtYg==" + }, "accepts": { "version": "1.3.8", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", @@ -4115,14 +4675,12 @@ "ansi-regex": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==" }, "ansi-styles": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dev": true, "requires": { "color-convert": "^2.0.1" } @@ -4172,6 +4730,14 @@ "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", "dev": true }, + "autolinker": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/autolinker/-/autolinker-4.1.5.tgz", + "integrity": "sha512-vEfYZPmvVOIuE567XBVCsx8SBgOYtjB2+S1iAaJ+HgH+DNjAcrHem2hmAeC9yaNGWayicv4yR+9UaJlkF3pvtw==", + "requires": { + "tslib": "^2.8.1" + } + }, "balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", @@ -4199,6 +4765,16 @@ "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==", "dev": true }, + "bitmap-sdf": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/bitmap-sdf/-/bitmap-sdf-1.0.4.tgz", + "integrity": "sha512-1G3U4n5JE6RAiALMxu0p1XmeZkTeCwGKykzsLTCqVzfSDaN6S7fKnkIkfejogz+iwqBWc0UYAIKnKHNN7pSfDg==" + }, + "bluebird": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", + "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==" + }, "body-parser": { "version": "1.20.3", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.3.tgz", @@ -4282,6 +4858,15 @@ "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", "dev": true }, + "cesium": { + "version": "1.138.0", + "resolved": "https://registry.npmjs.org/cesium/-/cesium-1.138.0.tgz", + "integrity": "sha512-YX7Ttd4LzAxunuzcKPyOCQa+BPc2RmenqnkM5uZkk/GVwor724bd+F3kdVP4IyMbTgxFkchXuX2Aa8L1Y0/ZxA==", + "requires": { + "@cesium/engine": "^22.3.0", + "@cesium/widgets": "^14.3.0" + } + }, "chai": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/chai/-/chai-5.1.2.tgz", @@ -4362,7 +4947,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dev": true, "requires": { "color-name": "~1.1.4" } @@ -4370,8 +4954,7 @@ "color-name": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "dev": true + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" }, "combined-stream": { "version": "1.0.8", @@ -4382,6 +4965,11 @@ "delayed-stream": "~1.0.0" } }, + "commander": { + "version": "2.20.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==" + }, "component-emitter": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.3.1.tgz", @@ -4538,6 +5126,19 @@ "integrity": "sha512-uIFDxqpRZGZ6ThOk84hEfqWoHx2devRFvpTZcTHur85vImfaxUbTW9Ryh4CpCuDnToOP1CEtXKIgytHBPVff5A==", "dev": true }, + "dompurify": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.3.1.tgz", + "integrity": "sha512-qkdCKzLNtrgPFP1Vo+98FRzJnBRGe4ffyCea9IwHB1fyxPOeNTHpLKYGd4Uk9xvNoH0ZoOjwZxNptyMwqrId1Q==", + "requires": { + "@types/trusted-types": "^2.0.7" + } + }, + "draco3d": { + "version": "1.5.7", + "resolved": "https://registry.npmjs.org/draco3d/-/draco3d-1.5.7.tgz", + "integrity": "sha512-m6WCKt/erDXcw+70IJXnG7M3awwQPAsZvJGX5zY7beBqpELw6RDGkYVU0W43AFxye4pDZ5i2Lbyc/NNGqwjUVQ==" + }, "draco3dgltf": { "version": "1.5.7", "resolved": "https://registry.npmjs.org/draco3dgltf/-/draco3dgltf-1.5.7.tgz", @@ -4553,6 +5154,11 @@ "gopd": "^1.2.0" } }, + "earcut": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/earcut/-/earcut-3.0.2.tgz", + "integrity": "sha512-X7hshQbLyMJ/3RPhyObLARM2sNxxmRALLKx1+NVFFnQ9gKzmCrxm9+uLIAdBcvc8FNLpctqlQ2V6AE92Ol9UDQ==" + }, "ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", @@ -4561,8 +5167,7 @@ "emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "dev": true + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" }, "encodeurl": { "version": "2.0.0", @@ -4590,8 +5195,7 @@ "escalade": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", - "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", - "dev": true + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==" }, "escape-html": { "version": "1.0.3", @@ -4763,6 +5367,16 @@ "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==" }, + "fs-extra": { + "version": "11.3.3", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.3.3.tgz", + "integrity": "sha512-VWSRii4t0AFm6ixFFmLLx1t7wS1gh+ckoa84aOeapGum0h+EZd1EhEumSB+ZdDLnEPuucsVB9oB7cxJHap6Afg==", + "requires": { + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + } + }, "fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", @@ -4784,8 +5398,7 @@ "get-caller-file": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", - "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", - "dev": true + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==" }, "get-intrinsic": { "version": "1.2.7", @@ -4827,6 +5440,16 @@ "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==" }, + "graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==" + }, + "grapheme-splitter": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/grapheme-splitter/-/grapheme-splitter-1.0.4.tgz", + "integrity": "sha512-bzh50DW9kTPM00T8y4o8vQg89Di9oLJVLW/KaOGIXJWP/iqCN6WKYkbNOF04vFLJhwcpYUh9ydh/+5vpOqV4YQ==" + }, "handlebars": { "version": "4.7.8", "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.8.tgz", @@ -4951,8 +5574,7 @@ "is-fullwidth-code-point": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "dev": true + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==" }, "is-glob": { "version": "4.0.3", @@ -4981,6 +5603,11 @@ "integrity": "sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw==", "dev": true }, + "jpeg-js": { + "version": "0.4.4", + "resolved": "https://registry.npmjs.org/jpeg-js/-/jpeg-js-0.4.4.tgz", + "integrity": "sha512-WZzeDOEtTOBK4Mdsar0IqEU5sMr3vSV2RqkAIzUEV2BHnUfKGyswWFPFwK5EeDo93K3FohSHbLAjj0s1Wzd+dg==" + }, "js-yaml": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", @@ -4990,6 +5617,25 @@ "argparse": "^2.0.1" } }, + "jsep": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/jsep/-/jsep-1.4.0.tgz", + "integrity": "sha512-B7qPcEVE3NVkmSJbaYxvv4cHkVW7DQsZz13pUMrfS8z8Q/BuShN+gcTXrUlPiGqM2/t/EEaI030bpxMqY8gMlw==" + }, + "jsonfile": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.2.0.tgz", + "integrity": "sha512-FGuPw30AdOIUTRMC2OMRtQV+jkVj2cfPqSeWXv1NEAJ1qZ5zb1X6z1mFhbfOB/iy3ssJCD+3KuZ8r8C3uVFlAg==", + "requires": { + "graceful-fs": "^4.1.6", + "universalify": "^2.0.0" + } + }, + "kdbush": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/kdbush/-/kdbush-4.0.2.tgz", + "integrity": "sha512-WbCVYJ27Sz8zi9Q7Q0xHC+05iwkm3Znipc2XTlrnJbsHMYktW4hPhXUE8Ys1engBrvffoSCqbil1JQAa7clRpA==" + }, "keygrip": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/keygrip/-/keygrip-1.1.0.tgz", @@ -5003,6 +5649,11 @@ "resolved": "https://registry.npmjs.org/ktx-parse/-/ktx-parse-1.1.0.tgz", "integrity": "sha512-mKp3y+FaYgR7mXWAbyyzpa/r1zDWeaunH+INJO4fou3hb45XuNSwar+7llrRyvpMWafxSIi99RNFJ05MHedaJQ==" }, + "lerc": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/lerc/-/lerc-2.0.0.tgz", + "integrity": "sha512-7qo1Mq8ZNmaR4USHHm615nEW2lPeeWJ3bTyoqFbd35DLx0LUH7C6ptt5FDCTAlbIzs3+WKrk5SkJvw8AFDE2hg==" + }, "locate-path": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", @@ -5022,6 +5673,11 @@ "is-unicode-supported": "^0.1.0" } }, + "long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==" + }, "loupe": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/loupe/-/loupe-3.1.2.tgz", @@ -5049,6 +5705,11 @@ "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz", "integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==" }, + "mersenne-twister": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/mersenne-twister/-/mersenne-twister-1.1.0.tgz", + "integrity": "sha512-mUYWsMKNrm4lfygPkL3OfGzOPTR2DBlTkBNHM//F6hGp8cLThY897crAlk3/Jo17LEOOjQUrNAx6DvgO77QJkA==" + }, "meshoptimizer": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/meshoptimizer/-/meshoptimizer-1.0.1.tgz", @@ -5242,6 +5903,61 @@ "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", "dev": true }, + "nosleep.js": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/nosleep.js/-/nosleep.js-0.12.0.tgz", + "integrity": "sha512-9d1HbpKLh3sdWlhXMhU6MMH+wQzKkrgfRkYV0EBdvt99YJfj0ilCJrWRDYG2130Tm4GXbEoTCx5b34JSaP+HhA==" + }, + "obj2gltf": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/obj2gltf/-/obj2gltf-3.2.0.tgz", + "integrity": "sha512-1pCbHSK55tiTkJG8Td0Nfqx97jcCtIKNeoukWhmuiyEtty3gmLBxHRN6WdYM6XKKAVgZVgeJ/PxXAizeRbQFxQ==", + "requires": { + "bluebird": "^3.7.2", + "cesium": "^1.86.1", + "fs-extra": "^11.0.0", + "jpeg-js": "^0.4.3", + "mime": "^3.0.0", + "pngjs": "^7.0.0", + "yargs": "^17.2.1" + }, + "dependencies": { + "cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "requires": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + } + }, + "mime": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-3.0.0.tgz", + "integrity": "sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A==" + }, + "yargs": { + "version": "17.7.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", + "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", + "requires": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + } + }, + "yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==" + } + } + }, "object-inspect": { "version": "1.13.3", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.3.tgz", @@ -5287,6 +6003,11 @@ "p-limit": "^3.0.2" } }, + "pako": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/pako/-/pako-2.1.0.tgz", + "integrity": "sha512-w+eufiZ1WuJYgPXbV/PO3NCMEc3xqylkKHzp8bxp1uW4qaSNQUkwmLLEc3kKsfz8lpV1F8Ht3U1Cm+9Srog2ug==" + }, "parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", @@ -5386,6 +6107,11 @@ "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", "dev": true }, + "pngjs": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-7.0.0.tgz", + "integrity": "sha512-LKWqWJRhstyYo9pGvgor/ivk2w94eSjE3RGVuzLGlr3NmD8bf7RcYGze1mNdEHRP6TRP6rMuDHk5t44hnTRyow==" + }, "postgres-array": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", @@ -5414,6 +6140,25 @@ "resolved": "https://registry.npmjs.org/property-graph/-/property-graph-4.0.0.tgz", "integrity": "sha512-I0hojAJfTbSCZy3y6xyK29eayxo14v1bj1VPiDkHjTdz33SV6RdfMz2AHnf4ai62Vng2mN5GkaKahkooBIo9gA==" }, + "protobufjs": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-8.0.0.tgz", + "integrity": "sha512-jx6+sE9h/UryaCZhsJWbJtTEy47yXoGNYI4z8ZaRncM0zBKeRqjO2JEcOUYwrYGb1WLhXM1FfMzW3annvFv0rw==", + "requires": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.4", + "@protobufjs/eventemitter": "^1.1.0", + "@protobufjs/fetch": "^1.1.0", + "@protobufjs/float": "^1.0.2", + "@protobufjs/inquire": "^1.1.0", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.0", + "@types/node": ">=13.7.0", + "long": "^5.0.0" + } + }, "proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", @@ -5431,6 +6176,11 @@ "side-channel": "^1.0.6" } }, + "quickselect": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/quickselect/-/quickselect-3.0.0.tgz", + "integrity": "sha512-XdjUArbK4Bm5fLLvlm5KpTFOiOThgfWWI4axAZDWg4E/0mKdZyI9tNEfds27qCi1ze/vwTR16kvmmGhRra3c2g==" + }, "randombytes": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", @@ -5456,6 +6206,14 @@ "unpipe": "1.0.0" } }, + "rbush": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/rbush/-/rbush-4.0.1.tgz", + "integrity": "sha512-IP0UpfeWQujYC8Jg162rMNc01Rf0gWMMAb2Uxus/Q0qOFw4lCcq6ZnQEZwUoJqWyUGJ9th7JjwI4yIWo+uvoAQ==", + "requires": { + "quickselect": "^3.0.0" + } + }, "readdirp": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", @@ -5473,8 +6231,7 @@ "require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", - "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", - "dev": true + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==" }, "safe-buffer": { "version": "5.2.1", @@ -5674,7 +6431,6 @@ "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dev": true, "requires": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", @@ -5685,7 +6441,6 @@ "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, "requires": { "ansi-regex": "^5.0.1" } @@ -5754,6 +6509,14 @@ "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==" }, + "topojson-client": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/topojson-client/-/topojson-client-3.1.0.tgz", + "integrity": "sha512-605uxS6bcYxGXw9qi62XyrV6Q3xwbndjachmNxu8HWTtVPxZfEJN9fd/SZS1Q54Sn2y0TMyMxFj/cJINqGHrKw==", + "requires": { + "commander": "2" + } + }, "ts-node": { "version": "10.9.2", "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.2.tgz", @@ -5786,8 +6549,7 @@ "tslib": { "version": "2.8.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", - "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", - "optional": true + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==" }, "tsscmp": { "version": "1.0.6", @@ -5820,11 +6582,21 @@ "resolved": "https://registry.npmjs.org/uniq/-/uniq-1.0.1.tgz", "integrity": "sha512-Gw+zz50YNKPDKXs+9d+aKAjVwpjNwqzvNpLigIruT4HA9lMZNdMqs9x07kKHB/L9WRzqp4+DlTU5s4wG2esdoA==" }, + "universalify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz", + "integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==" + }, "unpipe": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==" }, + "urijs": { + "version": "1.19.11", + "resolved": "https://registry.npmjs.org/urijs/-/urijs-1.19.11.tgz", + "integrity": "sha512-HXgFDgDommxn5/bIv0cnQZsPhHDA90NPHD6+c/v21U5+Sx5hoP8+dP9IZXBU1gIfvdRfhG8cel9QNPeionfcCQ==" + }, "utils-merge": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", @@ -5856,7 +6628,6 @@ "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "dev": true, "requires": { "ansi-styles": "^4.0.0", "string-width": "^4.1.0", @@ -5885,8 +6656,7 @@ "y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", - "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", - "dev": true + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==" }, "yargs": { "version": "16.2.0", diff --git a/source/server/package.json b/source/server/package.json index 431e1b400..7663d28e0 100755 --- a/source/server/package.json +++ b/source/server/package.json @@ -57,10 +57,11 @@ "mime-types": "^2.1.35", "morgan": "^1.10.0", "nodemailer": "^6.9.16", + "obj2gltf": "^3.2.0", "pg": "^8.16.0", "pg-cursor": "^2.15.0", - "source-map-support": "^0.5.21", "sharp": "^0.34.5", + "source-map-support": "^0.5.21", "xml-js": "^1.6.11", "yauzl": "^3.2.0", "yazl": "^3.3.1" diff --git a/source/server/routes/scenes/scene/post.test.ts b/source/server/routes/scenes/scene/post.test.ts index 753c0c625..9da78aeab 100644 --- a/source/server/routes/scenes/scene/post.test.ts +++ b/source/server/routes/scenes/scene/post.test.ts @@ -43,7 +43,7 @@ describe("POST /scenes/:scene", function(){ await expect(vfs.getScenes()).to.eventually.have.property("length", 1); - await request(app).get("/scenes/foo/models/foo.glb") + await request(app).get("/scenes/foo/foo.glb") .auth(user.username, "12345678") .expect(200) .expect("Content-Type", "model/gltf-binary"); diff --git a/source/server/routes/scenes/scene/post.ts b/source/server/routes/scenes/scene/post.ts index 1ebe82d25..5a05d154b 100644 --- a/source/server/routes/scenes/scene/post.ts +++ b/source/server/routes/scenes/scene/post.ts @@ -8,7 +8,7 @@ import { createDocumentFromFiles } from "../../../tasks/handlers/uploads.js"; const sceneLanguages = ["EN", "ES", "DE", "NL", "JA", "FR", "HAW"] as const; type SceneLanguage = typeof sceneLanguages[number]; function isSceneLanguage(l:any) :l is SceneLanguage|undefined{ - return typeof l === "undefined" || sceneLanguages.indexOf(l.toUpperCase()) !== -1; + return typeof l === "undefined" || sceneLanguages.indexOf(l) !== -1; } @@ -22,14 +22,14 @@ export default async function postScene(req :Request, res :Response){ const {vfs, taskScheduler} = getLocals(req); let user_id = getUserId(req); let {scene} = req.params; - let {language} = req.query; - + const {language: queryLanguage} = req.query; if(req.is("multipart")|| req.is("application/x-www-form-urlencoded")){ throw new BadRequestError(`${req.get("Content-Type")} content is not supported on this route. Provide a raw Zip attachment`); } - if(!isSceneLanguage(language)){ - throw new BadRequestError(`Invalid scene language requested: ${language}`) + if(queryLanguage && (typeof queryLanguage !== "string" || !isSceneLanguage(queryLanguage.toUpperCase()))){ + throw new BadRequestError(`Invalid scene language requested: ${queryLanguage}`) } + const language = queryLanguage?.toUpperCase() as SceneLanguage|undefined; if(!user_id){ throw new UnauthorizedError("Requires authenticated user"); } @@ -39,10 +39,13 @@ export default async function postScene(req :Request, res :Response){ scene_id, user_id, type: "postScene", - data: {}, + data: { + language, + scene, + }, handler: async function postsSceneHandler({context: {logger, vfs}}){ logger.debug("Draining the HTTP request into scene space"); - let f = await vfs.writeFile(req, {user_id, scene: scene, mime:"model/gltf-binary", name: `models/${scene}.glb`}); + let f = await vfs.writeFile(req, {user_id, scene: scene, mime:"model/gltf-binary", name: `${scene}.glb`}); if(f.size ==0 || !f.hash) throw new BadRequestError(`Body was empty. Can't create a scene.`); logger.debug("Parse the created file"); const meta = await taskScheduler.run({ diff --git a/source/server/routes/views/index.ts b/source/server/routes/views/index.ts index f00d7d865..1bb7f58d0 100644 --- a/source/server/routes/views/index.ts +++ b/source/server/routes/views/index.ts @@ -1,5 +1,5 @@ import { Router, Request, Response, NextFunction } from "express"; -import { canRead, getHost, canWrite, getSession, getVfs, getUser, isAdministrator, getUserManager, isMemberOrManage, isManage, isEmbed, useTemplateProperties, getTaskScheduler, getLocals } from "../../utils/locals.js"; +import { canRead, getHost, canWrite, getSession, getVfs, getUser, isAdministrator, getUserManager, isMemberOrManage, isManage, isEmbed, useTemplateProperties, getTaskScheduler, getLocals, isUser, isCreator } from "../../utils/locals.js"; import wrap from "../../utils/wrapAsync.js"; import path from "path"; import { Scene } from "../../vfs/types.js"; @@ -75,7 +75,15 @@ routes.get("/", wrap(async (req, res)=>{ routes.get("/upload", wrap(async (req, res)=>{ + const {templates}= getLocals(req); const requester = getUser(req); + if(!isUserAtLeast(requester!, "create")){ + return res.status(401).render("error", { + error: { + message: templates.t(requester? "errors.requireCreate": "errors.requireUser", {lng: res.locals.lang, what: "/ui/upload"}) + }, + }); + } const taskScheduler = getTaskScheduler(req); const vfs = getVfs(req); const {task} = req.query; diff --git a/source/server/tasks/handlers/runBlenderScript.ts b/source/server/tasks/handlers/runBlenderScript.ts deleted file mode 100644 index 160fe33f5..000000000 --- a/source/server/tasks/handlers/runBlenderScript.ts +++ /dev/null @@ -1,28 +0,0 @@ -import path from "node:path"; -import { TaskHandlerParams } from "../types.js"; -import { taskRun } from "../../utils/exec.js"; - - -/** - * - * @param script absolute path to python script to load - * @param scriptArgs script arguments - * @param params Spawn parameters - * @returns - */ -export async function runBlenderScript({context: {config, logger, signal}, task: {data: {script, args}}}:TaskHandlerParams<{script: string, args: string[]}>){ - - return await taskRun("blender", [ - "--background", - "--offline-mode", - "--factory-startup", - "--threads", "1", - "--addons", "io_scene_gltf2", - "--python", path.isAbsolute(script)? script: path.join(config.scripts_dir, script), - "--", - ...args - ], { - logger, - signal, - }); -} \ No newline at end of file diff --git a/source/server/tasks/handlers/toGlb.ts b/source/server/tasks/handlers/toGlb.ts index 16759e3dc..eb7fd055b 100644 --- a/source/server/tasks/handlers/toGlb.ts +++ b/source/server/tasks/handlers/toGlb.ts @@ -1,21 +1,35 @@ import path from "node:path"; +import fs from "node:fs/promises"; import { FileArtifact, TaskHandlerParams } from "../types.js"; -import { runBlenderScript } from "./runBlenderScript.js"; +import { BadRequestError } from "../../utils/errors.js"; +import { getMimeType } from "../../utils/filetypes.js"; export async function toGlb({context: {tasks, vfs, logger}, task:{task_id, data:{fileLocation}}}:TaskHandlerParams){ + const ext = path.extname(fileLocation).toLowerCase(); + const mime = getMimeType(fileLocation); + if(mime === "model/obj"){ + return await tasks.run({ + handler: objToGlb, + data: {fileLocation}, + }); + }else{ + throw new BadRequestError(`Unsupported file extension: ${ext}`); + } +} + +export async function objToGlb({context: {vfs}, task: {task_id, data:{fileLocation}}}:TaskHandlerParams):Promise{ + const {default: obj2gltf} = await import("obj2gltf"); + const inputFilename = path.basename(fileLocation); + const destFilename = (/\.obj/i.test(inputFilename)? inputFilename.slice(-4): inputFilename) + ".glb" const dir = await vfs.createTaskWorkspace(task_id); - const dest = path.join(dir, path.basename(fileLocation, path.extname(fileLocation))+".glb"); - logger.log("Transform %s to %s", fileLocation, dest); - await tasks.run({ - data: { - script: "obj2gltf.py", - args: [ - "-i", vfs.absolute(fileLocation), - "-o", dest, - ] - }, - handler: runBlenderScript, + const destPath = path.join(dir, destFilename); + const gltfBuffer = await obj2gltf(vfs.absolute(fileLocation), { + binary: true, + secure: true, //won't read outside of the source file's directory }); - return vfs.relative(dest); -} \ No newline at end of file + await fs.writeFile(vfs.absolute(destPath), gltfBuffer); + return { + fileLocation: vfs.relative(destPath), + } +} diff --git a/source/server/tasks/handlers/uploads.test.ts b/source/server/tasks/handlers/uploads.test.ts new file mode 100644 index 000000000..628377bd7 --- /dev/null +++ b/source/server/tasks/handlers/uploads.test.ts @@ -0,0 +1,67 @@ +import { TaskScheduler } from "../scheduler.js"; +import { createDocumentFromFiles, DocumentModel } from "./uploads.js"; + + +const makeModel = (opts: Partial = {}) => { + return { + uri: "foo.glb", + byteSize: 100, + numFaces: 0, + imageSize: 0, + bounds: null, + quality: "High", + usage: "Web3D", + ...opts, + } satisfies DocumentModel; +} + + +describe("createDocumentFromFiles() task", function () { + let taskScheduler: TaskScheduler; + this.beforeAll(async function () { + await createIntegrationContext(this); + taskScheduler = this.services.taskScheduler; + }); + this.afterAll(async function () { + await cleanIntegrationContext(this); + }) + it("initializes a task scheduler", function () { + expect(taskScheduler).to.be.instanceOf(TaskScheduler); + }); + + + it("removes optional properties when necessary", async function () { + //Handles a known exception where we report "no texture" as imageSize=0, while voyager requires imageSize >= 1 if present + // Same thing with numFaces and bounds + const outputDoc = await taskScheduler.run({ + handler: createDocumentFromFiles, + data: { + scene: "foo", + language: "FR", + models: [makeModel({ imageSize: 0 })] + } + }); + expect(outputDoc.models).to.have.length(1); + const model = outputDoc.models![0]; + expect(model).not.to.have.property("bounds"); + expect(model).to.have.property("derivatives").to.have.length(1); + expect(model.derivatives[0]).to.have.property("assets").to.have.length(1); + const asset = model.derivatives[0].assets[0] + expect(asset).to.have.property("type"); + expect(asset).not.to.have.property("imageSize"); + expect(asset).not.to.have.property("numFaces"); + expect(asset).not.to.have.property(""); + }); + + it("Initializes the scene's name", async function () { + const outputDoc = await taskScheduler.run({ + handler: createDocumentFromFiles, + data: { + scene: "foo", + language: "FR", + models: [] + } + }); + expect(outputDoc).to.have.property("metas").to.have.length(1); + }) +}); \ No newline at end of file diff --git a/source/server/tasks/handlers/uploads.ts b/source/server/tasks/handlers/uploads.ts index 3b3735ad8..7b60e0079 100644 --- a/source/server/tasks/handlers/uploads.ts +++ b/source/server/tasks/handlers/uploads.ts @@ -12,7 +12,7 @@ import getDefaultDocument from "../../utils/schema/default.js"; import uid from "../../utils/uid.js"; import { toGlb } from "./toGlb.js"; import { getMimeType, isModelType, readMagicBytes } from "../../utils/filetypes.js"; -import { TDerivativeQuality, TDerivativeUsage } from "../../utils/schema/model.js"; +import { IAsset, IModel, TDerivativeQuality, TDerivativeUsage } from "../../utils/schema/model.js"; import { optimizeGlb } from "./optimizeGlb.js"; import { IDocument } from "../../utils/schema/document.js"; import { inspectGlb } from "./inspectGlb.js"; @@ -48,7 +48,7 @@ export interface UploadedBinaryModel extends UploadedFile { byteSize: number; numFaces: number; imageSize: number; - bounds: Bounds; + bounds: Bounds | null; } export interface UploadedUsdModel extends UploadedFile{ @@ -251,6 +251,9 @@ export async function createSceneFromFiles({context:{tasks, vfs, logger}, task: const scene_id = await vfs.createScene(name, user_id); + /** + * Optimize the model if requested and perform the final move to its destination path + */ async function moveModel(source: UploadedBinaryModel){ let filepath = vfs.absolute(source.fileLocation); let filename = path.basename(filepath); @@ -299,12 +302,12 @@ export async function createSceneFromFiles({context:{tasks, vfs, logger}, task: data: {fileLocation: vfs.relative(filepath)}, handler: toGlb, }); - logger.debug("Copy Converted source file to %s", dest); + logger.debug("Copy Converted source file to %s", dest.fileLocation); const meta = await tasks.run({ handler: parseUploadedModel, data: { - fileLocation: dest, + fileLocation: dest.fileLocation, } }); logger.debug("Parsed converted file :", meta); @@ -328,7 +331,7 @@ export async function createSceneFromFiles({context:{tasks, vfs, logger}, task: return scene_id; } -interface DocumentModel { +export interface DocumentModel { name?: string; /** * Uri is slightly misleading as it's "relative to scene root" @@ -338,7 +341,7 @@ interface DocumentModel { byteSize: number; numFaces: number; imageSize: number; - bounds: Bounds; + bounds: Bounds|null; quality:TDerivativeQuality; usage: TDerivativeUsage; }; @@ -354,7 +357,7 @@ interface GetDocumentParams{ export async function createDocumentFromFiles( { task: { - data:{scene, models, language} + data:{scene, models, language = "EN"} }, }: TaskHandlerParams): Promise{ @@ -362,24 +365,28 @@ export async function createDocumentFromFiles( //dumb inefficient Deep copy because we want to mutate the doc in-place document.models ??= []; for(let model of models){ - const index = document.models.push({ + const asset :IAsset = { + "uri": encodeURIComponent(model.uri), + "type": "Model", + } + for(const k of ["byteSize", "numFaces", "imageSize"] as const){ + //Ignore values that does not match schema for those properties + if(!Number.isInteger(model[k]) || model[k] < 1) continue; + asset[k] = model[k]; + } + const _m:IModel = { "units": "m", //glTF specification says it's always meters. It's what blender do. - "boundingBox": model.bounds, "derivatives":[{ "usage": model.usage, "quality": model.quality, - "assets": [ - { - "uri": encodeURIComponent(model.uri), - "type": "Model", - "byteSize": model.byteSize, - "numFaces": model.numFaces, - "imageSize": model.imageSize, - } - ] + "assets": [asset] }], "annotations":[], - }) -1; + } + if(Array.isArray(model.bounds)){ + _m.boundingBox = model.bounds; + } + const index = document.models.push(_m) -1; const nodeIndex = document.nodes.push({ "id": uid(), "name": model.name ?? scene, @@ -389,19 +396,16 @@ export async function createDocumentFromFiles( } - if(language){ - document.setups[0].language = {language: language}; - document.metas ??= []; - const meta_index = document.metas.push({ - "collection": { - "titles": { - [language]: scene, - } - }, - }) -1; - document.scenes[document.scene].meta = meta_index; - } - + document.setups[0].language = {language: language}; + document.metas ??= []; + const meta_index = document.metas.push({ + "collection": { + "titles": { + [language]: scene, + } + }, + }) -1; + document.scenes[document.scene].meta = meta_index; return document } diff --git a/source/server/templates/locales/en.yml b/source/server/templates/locales/en.yml index 0013a9e8c..ec9cdbe5e 100644 --- a/source/server/templates/locales/en.yml +++ b/source/server/templates/locales/en.yml @@ -57,10 +57,11 @@ labels: batchTags: Change selection's tags batchAddTag: Add this tag to selected scenes batchRmTag: Remove this tag from selected scenes + sceneName: Scene name buttons: searchScene: search scenes - uploadScene: create the scene + uploadScene: create a scene uploadArchive: extract useStandalone: use standalone mode logout: Disconnect @@ -151,6 +152,7 @@ titles: modifiedToday: Scenes modified today buildRef: Current version previewEmail: Preview email templates + createdScenes: Created scenes tooltips: showPassword: Show the password's text @@ -221,3 +223,5 @@ errors: "Password not provided": Password not provided "Bad password": Invalid password "Username not found": Invalid Username + "requireCreate": "{{what}} requires \"create\" rights" + "requireUser": "Authentication is required for {{what}}" diff --git a/source/server/templates/upload.hbs b/source/server/templates/upload.hbs index ecc7cd6ba..c61c87585 100644 --- a/source/server/templates/upload.hbs +++ b/source/server/templates/upload.hbs @@ -4,8 +4,8 @@

    {{i18n "titles.upload"}}

    {{#if scenes.length }} -
    -

    {{i18n "titles.createdScenes" }}

    +
    +

    {{i18n "titles.createdScenes" }}

    🗙
    • + {{#if scenes.length }} +
    +

    {{i18n "titles.createdScenes" }}

    +     🗙 + + +
    + {{#each scenes}} + {{#if error }} + {{ error }} + {{else}} +
    + +  {{name}} + + + + {{i18n "labels.view"}} + + + + {{i18n "labels.edit"}} + +
    + + {{/if}} + {{/each}} +
    +
    + + {{/if}}
    - +

    {{i18n "titles.createOrUpdateScene"}}

    + {{i18n "labels.selectFile_s"}} +

    {{i18n "leads.uploadFiles"}}

    +

    {{i18n "leads.uploadMixedContent"}}

    +
    @@ -19,7 +78,7 @@ {{i18n "labels.sceneName"}} {{#> popover id="sceneName"}}{{i18n "leads.uploadName" }}{{/popover}} - +