feat: Further progress on the 530 branch.

package.json

@@ -12,6 +12,49 @@
     "db:reset": "prisma migrate reset --force --schema prisma/multiworld/schema.prisma",
     "db:schema": "prisma migrate dev --schema prisma/multiworld/schema.prisma",
     "dev": "bun run --watch src/app.ts",
+    "dump:enums": "bun run tools/cache/dump/dumpEnums.ts",
+    "dump:flos": "bun run tools/cache/dump/dumpFlos.ts",
+    "dump:flus": "bun run tools/cache/dump/dumpFlus.ts",
+    "dump:mesanim": "bun run tools/cache/dump/dumpMesanims.ts",
+    "dump:invs": "bun run tools/cache/dump/dumpInvs.ts",
+    "dump:objs": "bun run tools/cache/dump/dumpObjs.ts",
+    "dump:npcs": "bun run tools/cache/dump/dumpNpcs.ts",
+    "dump:param": "bun run tools/cache/dump/dumpParams.ts",
+    "dump:struct": "bun run tools/cache/dump/dumpStructs.ts",
+    "dump:quickchat": "bun run tools/cache/dump/dumpQuickChat.ts --archive 24 && bun run tools/cache/dump/dumpQuickChat.ts --archive 25",
+    "decode:group": "bun run tools/cache/debug/decodeGroup.ts --archive 2 --group 21",
+    "dump:hashes": "bun run tools/cache/debug/dumpJs5NameHashes.ts",
+    "match:hashes": "bun run tools/cache/debug/matchJs5Hashes.ts",
+    "pack:quickchat": "bun run tools/cache/pack/packQuickChat.ts --archive 24 --mode server && bun run tools/cache/pack/packQuickChat.ts --archive 24 --mode client --exact && bun run tools/cache/pack/packQuickChat.ts --archive 25 --mode server && bun run tools/cache/pack/packQuickChat.ts --archive 25 --mode client --exact",
+    "pack:flos": "bun run tools/cache/pack/packFlos.ts",
+    "pack:flus": "bun run tools/cache/pack/packFlus.ts",
+    "pack:mesanim": "bun run tools/cache/pack/packMesanims.ts",
+    "pack:invs": "bun run tools/cache/pack/packInvs.ts",
+    "pack:objs": "bun run tools/cache/pack/packObjs.ts",
+    "pack:npcs": "bun run tools/cache/pack/packNpcs.ts",
+    "pack:param": "bun run tools/cache/pack/packParams.ts",
+    "pack:struct": "bun run tools/cache/pack/packStructs.ts",
+    "pack:enums": "bun run tools/cache/pack/packEnums.ts --mode server --no-exact && bun run tools/cache/pack/packJs5Archive.ts --archive 17 --groups-dir data/cache --out data/pack/client/client.enum.config.js5",
+    "pack:midis": "bun run tools/cache/pack/packMidis.ts",
+    "pack:patches": "bun run tools/cache/pack/packPatches.ts",
+    "unpack:midis": "bun run tools/unpack/UnpackMidi.ts",
+    "unpack:patches": "bun run tools/unpack/UnpackPatch.ts",
+    "gen:midi:pack": "bun run tools/pack/generators/generateMidiPack.ts",
+    "gen:patch:pack": "bun run tools/pack/generators/generatePatchPack.ts",
+    "gen:enum:pack": "bun run tools/pack/generators/generateEnumPack.ts",
+    "gen:flo:pack": "bun run tools/pack/generators/generateFloPack.ts",
+    "gen:flu:pack": "bun run tools/pack/generators/generateFluPack.ts",
+    "gen:mesanim:pack": "bun run tools/pack/generators/generateMesanimPack.ts",
+    "gen:inv:pack": "bun run tools/pack/generators/generateInvPack.ts",
+    "gen:obj:pack": "bun run tools/pack/generators/generateObjPack.ts",
+    "gen:npc:pack": "bun run tools/pack/generators/generateNpcPack.ts",
+    "gen:param:pack": "bun run tools/pack/generators/generateParamPack.ts",
+    "gen:struct:pack": "bun run tools/pack/generators/generateStructPack.ts",
+    "gen:chatcat:pack": "bun run tools/pack/generators/generateChatCatPack.ts",
+    "gen:chatphrase:pack": "bun run tools/pack/generators/generateChatPhrasePack.ts",
+    "compare:groups": "bun run tools/cache/compare/compareGroups.ts",
+    "verify:js5:compressed": "bun run tools/cache/verify/verifyJs5Compressed.ts",
+    "group:filecount": "bun run tools/cache/debug/groupFileCount.ts",
     "friend": "bun run src/friend.ts",
     "lint": "eslint --no-warn-ignored src tools",
     "lint:staged": "eslint --no-warn-ignored",
@@ -37,6 +80,7 @@
     "@inquirer/prompts": "^7.2.3",
     "@jimp/js-png": "^1.6.0",
     "@jimp/plugin-quantize": "^1.6.0",
+    "@lostcityrs/runescript": "0.9.2",
     "axios": "^1.13.1",
     "bcrypt": "^5.1.1",
     "dotenv": "^16.4.7",
@@ -48,6 +92,7 @@
     "mysql2": "^3.12.0",
     "node-forge": "^1.3.1",
     "prom-client": "^15.1.3",
+    "pako": "^2.1.0",
     "ws": "^8.18.2"
   },
   "devDependencies": {
@@ -57,6 +102,7 @@
     "@types/ejs": "^3.1.5",
     "@types/node": "^22.10.2",
     "@types/node-forge": "^1.3.11",
+    "@types/pako": "^2.0.3",
     "@types/ws": "^8.18.1",
     "eslint": "^9.27.0",
     "globals": "^16.1.0",

src/cache/config/EnumType.ts

@@ -1,14 +1,84 @@
+import fs from 'fs';
+import path from 'path';
+
 import { ConfigType } from '#/cache/config/ConfigType.js';
 import ScriptVarType from '#/cache/config/ScriptVarType.js';
 import Packet from '#/io/Packet.js';
+import { unpackJs5Group } from '#/io/Js5Group.js';
+import Js5PackReader from '#/io/Js5PackReader.js';
+import { parseJs5ArchiveIndexFromPack, splitGroupFiles } from '#/io/Js5ArchiveIndex.js';
 
 export default class EnumType extends ConfigType {
     static configNames = new Map<string, number>();
     static configs: EnumType[] = [];
 
-    static load(_dir: string) {
+    static load(dir: string) {
+        const js5Path = path.join(dir, 'server.enum.config.js5');
+        if (fs.existsSync(js5Path)) {
+            this.loadFromJs5(js5Path);
+            return;
+        }
+
+        const candidates = [path.join(dir, 'server', 'enum.dat'), path.join(dir, 'enum.dat')];
+
+        let datPath: string | null = null;
+        for (const candidate of candidates) {
+            if (fs.existsSync(candidate)) {
+                datPath = candidate;
+                break;
+            }
+        }
+
+        if (!datPath) {
+            return;
+        }
+
+        const bytes = new Uint8Array(fs.readFileSync(datPath));
+        const unpacked = unpackJs5Group(bytes);
+        this.parse(new Packet(unpacked));
     }
 
+    private static loadFromJs5(js5Path: string): void {
+        const js5Data = new Uint8Array(fs.readFileSync(js5Path));
+        const indexInfo = parseJs5ArchiveIndexFromPack(js5Data);
+        const pack = new Js5PackReader(js5Data);
+
+        EnumType.configNames = new Map();
+        EnumType.configs = [];
+
+        for (const groupId of indexInfo.groupIds) {
+            const packedGroup = pack.getGroup(groupId);
+            if (!packedGroup || packedGroup.length === 0) {
+                continue;
+            }
+
+            const fileIds = indexInfo.fileIdsByGroup.get(groupId);
+            if (!fileIds || fileIds.length === 0) {
+                continue;
+            }
+
+            const groupData = unpackJs5Group(packedGroup);
+            const files = splitGroupFiles(groupData, fileIds);
+
+            for (const fileId of fileIds) {
+                const fileData = files.get(fileId);
+                if (!fileData) {
+                    continue;
+                }
+
+                const id = (groupId << 8) | fileId;
+                const config = new EnumType(id);
+                config.decodeType(new Packet(fileData));
+                EnumType.configs[id] = config;
+
+                if (config.debugname) {
+                    EnumType.configNames.set(config.debugname, id);
+                }
+            }
+        }
+    }
+
+
     static parse(dat: Packet) {
         EnumType.configNames = new Map();
         EnumType.configs = [];
@@ -49,12 +119,15 @@ export default class EnumType extends ConfigType {
     }
 
     // ----
-    // server-side
     inputtype = ScriptVarType.INT;
     outputtype = ScriptVarType.INT;
     defaultInt: number = 0;
     defaultString: string = 'null';
     values = new Map<number, number | string>();
+
+    // Flags to track if defaults were explicitly encoded (even if value is 0/null)
+    hasExplicitDefaultInt = false;
+    hasExplicitDefaultString = false;
 
     decode(code: number, dat: Packet): void {
         if (code === 1) {
@@ -63,8 +136,10 @@ export default class EnumType extends ConfigType {
             this.outputtype = dat.g1();
         } else if (code === 3) {
             this.defaultString = dat.gjstr();
+            this.hasExplicitDefaultString = true;
         } else if (code === 4) {
             this.defaultInt = dat.g4s();
+            this.hasExplicitDefaultInt = true;
         } else if (code === 5) {
             const count = dat.g2();
 

src/cache/config/FloType.ts

@@ -31,25 +31,6 @@ export default class FloType extends ConfigType {
         }
     }
 
-    static loadJag(config: Jagfile) {
-        FloType.configNames = new Map();
-        FloType.configs = [];
-
-        const client = config.read('flo.dat')!;
-        const count = client.g2();
-
-        for (let id = 0; id < count; id++) {
-            const config = new FloType(id);
-            config.decodeType(client);
-
-            FloType.configs[id] = config;
-
-            if (config.debugname) {
-                FloType.configNames.set(config.debugname, id);
-            }
-        }
-    }
-
     static get(id: number): FloType {
         return FloType.configs[id];
     }
@@ -69,22 +50,49 @@ export default class FloType extends ConfigType {
 
     // ----
 
-    rgb: number = 0;
-    texture: number = -1;
-    overlay: boolean = false;
+    colour: number = 0;
+    material: number = -1;
     occlude: boolean = true;
+    averagecolour: number = -1;
+    materialscale: number = 512;
+    hardshadow: boolean = true;
+    priority: number = 8;
+    blend: boolean = false;
+    waterfogcolour: number = 1190717;
+    waterfogscale: number = 512;
+    code8: boolean = false; // Client-only opcode marker
 
     decode(code: number, dat: Packet): void {
         if (code === 1) {
-            this.rgb = dat.g3();
+            this.colour = dat.g3();
         } else if (code === 2) {
-            this.texture = dat.g1();
+            this.material = dat.g1();
         } else if (code === 3) {
-            this.overlay = true;
+            this.material = dat.g2();
+            if (this.material === 65535) {
+                this.material = -1;
+            }
         } else if (code === 5) {
             this.occlude = false;
         } else if (code === 6) {
             this.debugname = dat.gjstr();
+        } else if (code === 7) {
+            this.averagecolour = dat.g3();
+        } else if (code === 8) {
+            // Client-only code, we store it only to write the opcode later during packing.
+            this.code8 = true;
+        } else if (code === 9) {
+            this.materialscale = dat.g2();
+        } else if (code === 10) {
+            this.hardshadow = false;
+        } else if (code === 11) {
+            this.priority = dat.g1();
+        } else if (code === 12) {
+            this.blend = true;
+        } else if (code === 13) {
+            this.waterfogcolour = dat.g3();
+        } else if (code === 14) {
+            this.waterfogscale = dat.g1();
         } else {
             throw new Error(`Unrecognized flo config code: ${code}`);
         }

src/cache/config/FluType.ts

@@ -0,0 +1,75 @@
+import { ConfigType } from '#/cache/config/ConfigType.js';
+import Jagfile from '#/io/Jagfile.js';
+import Packet from '#/io/Packet.js';
+
+export default class FluType extends ConfigType {
+    static configNames: Map<string, number> = new Map();
+    static configs: FluType[] = [];
+
+    static load(_dir: string) {
+    }
+
+    static parse(server: Packet, jag: Jagfile) {
+        FluType.configNames = new Map();
+        FluType.configs = [];
+
+        const count = server.g2();
+
+        const client = jag.read('flo.dat')!;
+        client.pos = 2;
+
+        for (let id = 0; id < count; id++) {
+            const config = new FluType(id);
+            config.decodeType(server);
+            config.decodeType(client);
+
+            FluType.configs[id] = config;
+
+            if (config.debugname) {
+                FluType.configNames.set(config.debugname, id);
+            }
+        }
+    }
+
+    static get(id: number): FluType {
+        return FluType.configs[id];
+    }
+
+    static getId(name: string): number {
+        return FluType.configNames.get(name) ?? -1;
+    }
+
+    static getByName(name: string): FluType | null {
+        const id = this.getId(name);
+        if (id === -1) {
+            return null;
+        }
+
+        return this.get(id);
+    }
+
+    // ----
+
+    colour: number = 0;
+    material: number = -1;
+    materialscale: number = 512;
+    hardshadow: boolean = true;
+
+
+    decode(code: number, dat: Packet): void {
+        if (code === 1) {
+            this.colour = dat.g3();
+        } else if (code === 2) {
+            this.material = dat.g2();
+            if (this.material === 65535) {
+                this.material = -1;
+            }
+        } else if (code === 3) {
+            this.materialscale = dat.g2();
+        } else if (code === 4) {
+            this.hardshadow = false;
+        } else {
+            throw new Error(`Unrecognized flo config code: ${code}`);
+        }
+    }
+}

src/cache/config/IdkType.ts

@@ -56,8 +56,10 @@ export default class IdkType extends ConfigType {
     type: number = -1;
     models: Uint16Array | null = null;
     heads: Uint16Array = new Uint16Array(5).fill(-1);
-    recol_s: Uint16Array = new Uint16Array(6).fill(0);
-    recol_d: Uint16Array = new Uint16Array(6).fill(0);
+    recol_s: Uint16Array | null = null;
+    recol_d: Uint16Array | null = null;
+    retex_s: Uint16Array | null = null;
+    retex_d: Uint16Array | null = null;
     disable: boolean = false;
 
     decode(code: number, dat: Packet): void {
@@ -72,10 +74,23 @@ export default class IdkType extends ConfigType {
             }
         } else if (code === 3) {
             this.disable = true;
-        } else if (code >= 40 && code < 50) {
-            this.recol_s[code - 40] = dat.g2();
-        } else if (code >= 50 && code < 60) {
-            this.recol_d[code - 50] = dat.g2();
+        } else if (code === 40) {
+            const count = dat.g1();
+            this.recol_s = new Uint16Array(count);
+            this.recol_d = new Uint16Array(count);
+            for (let i = 0; i < count; i++) {
+                this.recol_s[i] = dat.g2();
+                this.recol_d[i] = dat.g2();
+            }
+
+        } else if (code === 41) {
+            const count = dat.g1();
+            this.retex_s = new Uint16Array(count);
+            this.retex_d = new Uint16Array(count);
+            for (let i = 0; i < count; i++) {
+                this.retex_s[i] = dat.g2();
+                this.retex_d[i] = dat.g2();
+            }
         } else if (code >= 60 && code < 70) {
             this.heads[code - 60] = dat.g2();
         } else if (code === 250) {