diff --git a/package.json b/package.json index e2306beda..f9c2bd944 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,49 @@ "db:reset": "prisma migrate reset --force --schema prisma/multiworld/schema.prisma", "db:schema": "prisma migrate dev --schema prisma/multiworld/schema.prisma", "dev": "bun run --watch src/app.ts", + "dump:enums": "bun run tools/cache/dump/dumpEnums.ts", + "dump:flos": "bun run tools/cache/dump/dumpFlos.ts", + "dump:flus": "bun run tools/cache/dump/dumpFlus.ts", + "dump:mesanim": "bun run tools/cache/dump/dumpMesanims.ts", + "dump:invs": "bun run tools/cache/dump/dumpInvs.ts", + "dump:objs": "bun run tools/cache/dump/dumpObjs.ts", + "dump:npcs": "bun run tools/cache/dump/dumpNpcs.ts", + "dump:param": "bun run tools/cache/dump/dumpParams.ts", + "dump:struct": "bun run tools/cache/dump/dumpStructs.ts", + "dump:quickchat": "bun run tools/cache/dump/dumpQuickChat.ts --archive 24 && bun run tools/cache/dump/dumpQuickChat.ts --archive 25", + "decode:group": "bun run tools/cache/debug/decodeGroup.ts --archive 2 --group 21", + "dump:hashes": "bun run tools/cache/debug/dumpJs5NameHashes.ts", + "match:hashes": "bun run tools/cache/debug/matchJs5Hashes.ts", + "pack:quickchat": "bun run tools/cache/pack/packQuickChat.ts --archive 24 --mode server && bun run tools/cache/pack/packQuickChat.ts --archive 24 --mode client --exact && bun run tools/cache/pack/packQuickChat.ts --archive 25 --mode server && bun run tools/cache/pack/packQuickChat.ts --archive 25 --mode client --exact", + "pack:flos": "bun run tools/cache/pack/packFlos.ts", + "pack:flus": "bun run tools/cache/pack/packFlus.ts", + "pack:mesanim": "bun run tools/cache/pack/packMesanims.ts", + "pack:invs": "bun run tools/cache/pack/packInvs.ts", + "pack:objs": "bun run tools/cache/pack/packObjs.ts", + "pack:npcs": "bun run tools/cache/pack/packNpcs.ts", + "pack:param": "bun run tools/cache/pack/packParams.ts", + "pack:struct": "bun run tools/cache/pack/packStructs.ts", + "pack:enums": "bun run tools/cache/pack/packEnums.ts --mode server --no-exact && bun run tools/cache/pack/packJs5Archive.ts --archive 17 --groups-dir data/cache --out data/pack/client/client.enum.config.js5", + "pack:midis": "bun run tools/cache/pack/packMidis.ts", + "pack:patches": "bun run tools/cache/pack/packPatches.ts", + "unpack:midis": "bun run tools/unpack/UnpackMidi.ts", + "unpack:patches": "bun run tools/unpack/UnpackPatch.ts", + "gen:midi:pack": "bun run tools/pack/generators/generateMidiPack.ts", + "gen:patch:pack": "bun run tools/pack/generators/generatePatchPack.ts", + "gen:enum:pack": "bun run tools/pack/generators/generateEnumPack.ts", + "gen:flo:pack": "bun run tools/pack/generators/generateFloPack.ts", + "gen:flu:pack": "bun run tools/pack/generators/generateFluPack.ts", + "gen:mesanim:pack": "bun run tools/pack/generators/generateMesanimPack.ts", + "gen:inv:pack": "bun run tools/pack/generators/generateInvPack.ts", + "gen:obj:pack": "bun run tools/pack/generators/generateObjPack.ts", + "gen:npc:pack": "bun run tools/pack/generators/generateNpcPack.ts", + "gen:param:pack": "bun run tools/pack/generators/generateParamPack.ts", + "gen:struct:pack": "bun run tools/pack/generators/generateStructPack.ts", + "gen:chatcat:pack": "bun run tools/pack/generators/generateChatCatPack.ts", + "gen:chatphrase:pack": "bun run tools/pack/generators/generateChatPhrasePack.ts", + "compare:groups": "bun run tools/cache/compare/compareGroups.ts", + "verify:js5:compressed": "bun run tools/cache/verify/verifyJs5Compressed.ts", + "group:filecount": "bun run tools/cache/debug/groupFileCount.ts", "friend": "bun run src/friend.ts", "lint": "eslint --no-warn-ignored src tools", "lint:staged": "eslint --no-warn-ignored", @@ -37,6 +80,7 @@ "@inquirer/prompts": "^7.2.3", "@jimp/js-png": "^1.6.0", "@jimp/plugin-quantize": "^1.6.0", + "@lostcityrs/runescript": "0.9.2", "axios": "^1.13.1", "bcrypt": "^5.1.1", "dotenv": "^16.4.7", @@ -48,6 +92,7 @@ "mysql2": "^3.12.0", "node-forge": "^1.3.1", "prom-client": "^15.1.3", + "pako": "^2.1.0", "ws": "^8.18.2" }, "devDependencies": { @@ -57,6 +102,7 @@ "@types/ejs": "^3.1.5", "@types/node": "^22.10.2", "@types/node-forge": "^1.3.11", + "@types/pako": "^2.0.3", "@types/ws": "^8.18.1", "eslint": "^9.27.0", "globals": "^16.1.0", diff --git a/src/cache/config/EnumType.ts b/src/cache/config/EnumType.ts index 9fc516d2a..89f69c442 100644 --- a/src/cache/config/EnumType.ts +++ b/src/cache/config/EnumType.ts @@ -1,14 +1,84 @@ +import fs from 'fs'; +import path from 'path'; + import { ConfigType } from '#/cache/config/ConfigType.js'; import ScriptVarType from '#/cache/config/ScriptVarType.js'; import Packet from '#/io/Packet.js'; +import { unpackJs5Group } from '#/io/Js5Group.js'; +import Js5PackReader from '#/io/Js5PackReader.js'; +import { parseJs5ArchiveIndexFromPack, splitGroupFiles } from '#/io/Js5ArchiveIndex.js'; export default class EnumType extends ConfigType { static configNames = new Map(); static configs: EnumType[] = []; - static load(_dir: string) { + static load(dir: string) { + const js5Path = path.join(dir, 'server.enum.config.js5'); + if (fs.existsSync(js5Path)) { + this.loadFromJs5(js5Path); + return; + } + + const candidates = [path.join(dir, 'server', 'enum.dat'), path.join(dir, 'enum.dat')]; + + let datPath: string | null = null; + for (const candidate of candidates) { + if (fs.existsSync(candidate)) { + datPath = candidate; + break; + } + } + + if (!datPath) { + return; + } + + const bytes = new Uint8Array(fs.readFileSync(datPath)); + const unpacked = unpackJs5Group(bytes); + this.parse(new Packet(unpacked)); } + private static loadFromJs5(js5Path: string): void { + const js5Data = new Uint8Array(fs.readFileSync(js5Path)); + const indexInfo = parseJs5ArchiveIndexFromPack(js5Data); + const pack = new Js5PackReader(js5Data); + + EnumType.configNames = new Map(); + EnumType.configs = []; + + for (const groupId of indexInfo.groupIds) { + const packedGroup = pack.getGroup(groupId); + if (!packedGroup || packedGroup.length === 0) { + continue; + } + + const fileIds = indexInfo.fileIdsByGroup.get(groupId); + if (!fileIds || fileIds.length === 0) { + continue; + } + + const groupData = unpackJs5Group(packedGroup); + const files = splitGroupFiles(groupData, fileIds); + + for (const fileId of fileIds) { + const fileData = files.get(fileId); + if (!fileData) { + continue; + } + + const id = (groupId << 8) | fileId; + const config = new EnumType(id); + config.decodeType(new Packet(fileData)); + EnumType.configs[id] = config; + + if (config.debugname) { + EnumType.configNames.set(config.debugname, id); + } + } + } + } + + static parse(dat: Packet) { EnumType.configNames = new Map(); EnumType.configs = []; @@ -49,12 +119,15 @@ export default class EnumType extends ConfigType { } // ---- - // server-side inputtype = ScriptVarType.INT; outputtype = ScriptVarType.INT; defaultInt: number = 0; defaultString: string = 'null'; values = new Map(); + + // Flags to track if defaults were explicitly encoded (even if value is 0/null) + hasExplicitDefaultInt = false; + hasExplicitDefaultString = false; decode(code: number, dat: Packet): void { if (code === 1) { @@ -63,8 +136,10 @@ export default class EnumType extends ConfigType { this.outputtype = dat.g1(); } else if (code === 3) { this.defaultString = dat.gjstr(); + this.hasExplicitDefaultString = true; } else if (code === 4) { this.defaultInt = dat.g4s(); + this.hasExplicitDefaultInt = true; } else if (code === 5) { const count = dat.g2(); diff --git a/src/cache/config/FloType.ts b/src/cache/config/FloType.ts index 7102a3a1f..ef8d09b7a 100644 --- a/src/cache/config/FloType.ts +++ b/src/cache/config/FloType.ts @@ -31,25 +31,6 @@ export default class FloType extends ConfigType { } } - static loadJag(config: Jagfile) { - FloType.configNames = new Map(); - FloType.configs = []; - - const client = config.read('flo.dat')!; - const count = client.g2(); - - for (let id = 0; id < count; id++) { - const config = new FloType(id); - config.decodeType(client); - - FloType.configs[id] = config; - - if (config.debugname) { - FloType.configNames.set(config.debugname, id); - } - } - } - static get(id: number): FloType { return FloType.configs[id]; } @@ -69,22 +50,49 @@ export default class FloType extends ConfigType { // ---- - rgb: number = 0; - texture: number = -1; - overlay: boolean = false; + colour: number = 0; + material: number = -1; occlude: boolean = true; + averagecolour: number = -1; + materialscale: number = 512; + hardshadow: boolean = true; + priority: number = 8; + blend: boolean = false; + waterfogcolour: number = 1190717; + waterfogscale: number = 512; + code8: boolean = false; // Client-only opcode marker decode(code: number, dat: Packet): void { if (code === 1) { - this.rgb = dat.g3(); + this.colour = dat.g3(); } else if (code === 2) { - this.texture = dat.g1(); + this.material = dat.g1(); } else if (code === 3) { - this.overlay = true; + this.material = dat.g2(); + if (this.material === 65535) { + this.material = -1; + } } else if (code === 5) { this.occlude = false; } else if (code === 6) { this.debugname = dat.gjstr(); + } else if (code === 7) { + this.averagecolour = dat.g3(); + } else if (code === 8) { + // Client-only code, we store it only to write the opcode later during packing. + this.code8 = true; + } else if (code === 9) { + this.materialscale = dat.g2(); + } else if (code === 10) { + this.hardshadow = false; + } else if (code === 11) { + this.priority = dat.g1(); + } else if (code === 12) { + this.blend = true; + } else if (code === 13) { + this.waterfogcolour = dat.g3(); + } else if (code === 14) { + this.waterfogscale = dat.g1(); } else { throw new Error(`Unrecognized flo config code: ${code}`); } diff --git a/src/cache/config/FluType.ts b/src/cache/config/FluType.ts new file mode 100644 index 000000000..053efddef --- /dev/null +++ b/src/cache/config/FluType.ts @@ -0,0 +1,75 @@ +import { ConfigType } from '#/cache/config/ConfigType.js'; +import Jagfile from '#/io/Jagfile.js'; +import Packet from '#/io/Packet.js'; + +export default class FluType extends ConfigType { + static configNames: Map = new Map(); + static configs: FluType[] = []; + + static load(_dir: string) { + } + + static parse(server: Packet, jag: Jagfile) { + FluType.configNames = new Map(); + FluType.configs = []; + + const count = server.g2(); + + const client = jag.read('flo.dat')!; + client.pos = 2; + + for (let id = 0; id < count; id++) { + const config = new FluType(id); + config.decodeType(server); + config.decodeType(client); + + FluType.configs[id] = config; + + if (config.debugname) { + FluType.configNames.set(config.debugname, id); + } + } + } + + static get(id: number): FluType { + return FluType.configs[id]; + } + + static getId(name: string): number { + return FluType.configNames.get(name) ?? -1; + } + + static getByName(name: string): FluType | null { + const id = this.getId(name); + if (id === -1) { + return null; + } + + return this.get(id); + } + + // ---- + + colour: number = 0; + material: number = -1; + materialscale: number = 512; + hardshadow: boolean = true; + + + decode(code: number, dat: Packet): void { + if (code === 1) { + this.colour = dat.g3(); + } else if (code === 2) { + this.material = dat.g2(); + if (this.material === 65535) { + this.material = -1; + } + } else if (code === 3) { + this.materialscale = dat.g2(); + } else if (code === 4) { + this.hardshadow = false; + } else { + throw new Error(`Unrecognized flo config code: ${code}`); + } + } +} diff --git a/src/cache/config/IdkType.ts b/src/cache/config/IdkType.ts index 0ee018a99..d64ec9d2a 100644 --- a/src/cache/config/IdkType.ts +++ b/src/cache/config/IdkType.ts @@ -56,8 +56,10 @@ export default class IdkType extends ConfigType { type: number = -1; models: Uint16Array | null = null; heads: Uint16Array = new Uint16Array(5).fill(-1); - recol_s: Uint16Array = new Uint16Array(6).fill(0); - recol_d: Uint16Array = new Uint16Array(6).fill(0); + recol_s: Uint16Array | null = null; + recol_d: Uint16Array | null = null; + retex_s: Uint16Array | null = null; + retex_d: Uint16Array | null = null; disable: boolean = false; decode(code: number, dat: Packet): void { @@ -72,10 +74,23 @@ export default class IdkType extends ConfigType { } } else if (code === 3) { this.disable = true; - } else if (code >= 40 && code < 50) { - this.recol_s[code - 40] = dat.g2(); - } else if (code >= 50 && code < 60) { - this.recol_d[code - 50] = dat.g2(); + } else if (code === 40) { + const count = dat.g1(); + this.recol_s = new Uint16Array(count); + this.recol_d = new Uint16Array(count); + for (let i = 0; i < count; i++) { + this.recol_s[i] = dat.g2(); + this.recol_d[i] = dat.g2(); + } + + } else if (code === 41) { + const count = dat.g1(); + this.retex_s = new Uint16Array(count); + this.retex_d = new Uint16Array(count); + for (let i = 0; i < count; i++) { + this.retex_s[i] = dat.g2(); + this.retex_d[i] = dat.g2(); + } } else if (code >= 60 && code < 70) { this.heads[code - 60] = dat.g2(); } else if (code === 250) { diff --git a/src/cache/config/LocType.ts b/src/cache/config/LocType.ts index 493c742d5..aa0e40dd5 100644 --- a/src/cache/config/LocType.ts +++ b/src/cache/config/LocType.ts @@ -65,12 +65,16 @@ export default class LocType extends ConfigType { desc: string | null = null; recol_s: Uint16Array | null = null; recol_d: Uint16Array | null = null; + retex_s: Uint16Array | null = null; + retex_d: Uint16Array | null = null; + recol_d_palette: Int8Array | null = null; width = 1; length = 1; blockwalk = true; blockrange = true; active = -1; - hillskew = false; + hillskew = 0; + hillskew_amount = -1; // todo: verify name sharelight = false; occlude = false; anim = -1; @@ -96,6 +100,24 @@ export default class LocType extends ConfigType { multivarbit = -1; multivarp = -1; multiloc: number[] = []; + bgsound_sound = -1; + bgsound_range = 0; + bgsound_mindelay = 0; + bgsound_maxdelay = 0; + bgsound_random: number[] = []; + mapsceneiconrotate = false; + mapsceneicon = -1; + members = false; + randomanimframe = true; + hardshadow = true; + istexture = false; + code90 = false; // TODO: 'renderbelow'? + code96 = false; // TODO: 'hasanimation'? + code98 = false; // TODO: 'animated'? + cursor1op = -1; + cursor1 = -1; + cursor2op = -1; + cursor2 = -1; // server-side category = -1; @@ -128,12 +150,13 @@ export default class LocType extends ConfigType { this.length = dat.g1(); } else if (code === 17) { this.blockwalk = false; + this.blockrange = false; } else if (code === 18) { this.blockrange = false; } else if (code === 19) { this.active = dat.g1(); } else if (code === 21) { - this.hillskew = true; + this.hillskew = 1; } else if (code === 22) { this.sharelight = true; } else if (code === 23) { @@ -146,12 +169,14 @@ export default class LocType extends ConfigType { } } else if (code === 25) { this.hasalpha = true; + } else if (code === 27) { + this.blockwalk = true; } else if (code === 28) { this.wallwidth = dat.g1(); } else if (code === 29) { this.ambient = dat.g1b(); } else if (code === 39) { - this.contrast = dat.g1b(); + this.contrast = dat.g1b(); // Value multiplied by 5 client side } else if (code >= 30 && code < 35) { if (!this.op) { this.op = new Array(5).fill(null); @@ -167,10 +192,26 @@ export default class LocType extends ConfigType { this.recol_s[i] = dat.g2(); this.recol_d[i] = dat.g2(); } + } else if (code === 41) { + const count = dat.g1(); + this.retex_s = new Uint16Array(count); + this.retex_d = new Uint16Array(count); + + for (let i = 0; i < count; i++) { + this.retex_s[i] = dat.g2(); + this.retex_d[i] = dat.g2(); + } + } else if (code === 42) { + const count = dat.g1(); + this.recol_d_palette = new Int8Array(count); + + for (let i = 0; i < count; i++) { + this.recol_d_palette[i] = dat.g1b(); + } } else if (code === 60) { this.mapfunction = dat.g2(); } else if (code === 61) { - this.category = dat.g2(); + this.category = dat.g2(); } else if (code === 62) { this.mirror = true; } else if (code === 64) { @@ -182,7 +223,7 @@ export default class LocType extends ConfigType { } else if (code === 67) { this.resizez = dat.g2(); } else if (code === 68) { - this.mapscene = dat.g2(); + this.mapscene = dat.g2(); // Removed 530 (most likely earlier). } else if (code === 69) { this.forceapproach = dat.g1(); } else if (code === 70) { @@ -197,7 +238,7 @@ export default class LocType extends ConfigType { this.breakroutefinding = true; } else if (code === 75) { this.raiseobject = dat.g1(); - } else if (code === 77) { + } else if (code === 77 || code === 92) { this.multivarbit = dat.g2(); if (this.multivarbit === 65535) { this.multivarbit = -1; @@ -208,14 +249,70 @@ export default class LocType extends ConfigType { this.multivarp = -1; } + let defaultid = -1; + if (code === 92) { + defaultid = dat.g2(); + if (defaultid === 65535) { + defaultid = -1; + } + } + const count = dat.g1(); - this.multiloc = new Array(count + 1); + this.multiloc = new Array(count + 2); for (let i = 0; i <= count; i++) { this.multiloc[i] = dat.g2(); if (this.multiloc[i] === 65535) { this.multiloc[i] = -1; } } + this.multiloc[count + 1] = defaultid; + } else if (code === 78) { + this.bgsound_sound = dat.g2(); + this.bgsound_range = dat.g1(); + } else if (code === 79) { + this.bgsound_mindelay = dat.g2(); + this.bgsound_maxdelay = dat.g2(); + this.bgsound_range = dat.g1(); + + const count = dat.g1(); + this.bgsound_random = new Array(count); + for (let i = 0; i < count; i++) { + this.bgsound_random[i] = dat.g2(); + } + } else if (code === 81) { + this.hillskew = 2; + this.hillskew_amount = dat.g1(); // Value multiplied by 256 client side + } else if (code === 82) { + this.istexture = true; + } else if (code === 88) { + this.hardshadow = false; + } else if (code === 89) { + this.randomanimframe = false; + } else if (code === 90) { + this.code90 = true; + } else if (code === 91) { + this.members = true; + } else if (code === 93) { + this.hillskew = 3; + this.hillskew_amount = dat.g2(); + } else if (code === 94) { + this.hillskew = 4; + } else if (code === 95) { + this.hillskew = 5; + } else if (code === 96) { + this.code96 = true; + } else if (code === 97) { + this.mapsceneiconrotate = true; + } else if (code === 98) { + this.code98 = true; + } else if (code === 99) { + this.cursor1op = dat.g1(); + this.cursor1 = dat.g2(); + } else if (code === 100) { + this.cursor2op = dat.g1(); + this.cursor2 = dat.g2(); + } else if (code === 102) { + this.mapsceneicon = dat.g2(); } else if (code === 249) { this.params = ParamHelper.decodeParams(dat); } else if (code === 250) { diff --git a/src/cache/config/NpcType.ts b/src/cache/config/NpcType.ts index 57c78a1a2..96b9f46e9 100644 --- a/src/cache/config/NpcType.ts +++ b/src/cache/config/NpcType.ts @@ -1,7 +1,9 @@ import { ConfigType } from '#/cache/config/ConfigType.js'; import { ParamHelper, ParamMap } from '#/cache/config/ParamHelper.js'; +import { Direction } from '#/engine/CoordGrid.js'; import { BlockWalk } from '#/engine/entity/BlockWalk.js'; import { MoveRestrict } from '#/engine/entity/MoveRestrict.js'; +import { MoveSpeed } from '#/engine/entity/MoveSpeed.js'; import { NpcMode } from '#/engine/entity/NpcMode.js'; import { NpcStat } from '#/engine/entity/NpcStat.js'; import Jagfile from '#/io/Jagfile.js'; @@ -74,6 +76,9 @@ export default class NpcType extends ConfigType { hasalpha = false; recol_s: Uint16Array | null = null; recol_d: Uint16Array | null = null; + retex_s: Uint16Array | null = null; + retex_d: Uint16Array | null = null; + recol_d_palette: Int8Array | null = null; op: (string | null)[] | null = null; resizex = -1; resizey = -1; @@ -91,6 +96,32 @@ export default class NpcType extends ConfigType { multivarp = -1; multinpc: number[] = []; active = true; + walksmoothing = true; + spotshadow = true; + spotshadowcolour_1 = 0; + spotshadowcolour_2 = 0; + spotshadow_trans1 = -96; + spotshadow_trans2 = -16; + walkflags = 0; + code115_1: number = 0; + code115_2: number = 0; + modeloffset: number[][] = []; + hitbarid = -1; + bas = -1; + overlayheight = -1; + respawndir = Direction.SOUTH_EAST; + mapfunction = -1; + movespeed: MoveSpeed | null = null; + bgsound = -1; + bgsound_crawl = -1; + bgsound_walk = -1; + bgsound_run = -1; + bgsound_range = 0; + cursor1op = -1; + cursor1 = -1; + cursor2op = -1; + cursor2 = -1; + cursorattack = -1; // server-side regenRate = 100; @@ -119,6 +150,9 @@ export default class NpcType extends ConfigType { for (let i = 0; i < count; i++) { this.models[i] = dat.g2(); + if (this.models[i] === 65535) { + this.models[i] = -1; + } } } else if (code === 2) { this.name = dat.gjstr(); @@ -139,7 +173,7 @@ export default class NpcType extends ConfigType { this.walkanim_l = dat.g2(); } else if (code === 18) { this.category = dat.g2(); - } else if (code >= 30 && code < 40) { + } else if (code >= 30 && code < 35) { if (!this.op) { this.op = new Array(5).fill(null); } @@ -154,6 +188,22 @@ export default class NpcType extends ConfigType { this.recol_s[i] = dat.g2(); this.recol_d[i] = dat.g2(); } + } else if (code === 41) { + const count = dat.g1(); + this.retex_s = new Uint16Array(count); + this.retex_d = new Uint16Array(count); + + for (let i = 0; i < count; i++) { + this.retex_s[i] = dat.g2(); + this.retex_d[i] = dat.g2(); + } + } else if (code === 42) { + const count = dat.g1(); + this.recol_d_palette = new Int8Array(count); + + for (let i = 0; i < count; i++) { + this.recol_d_palette[i] = dat.g1b(); + } } else if (code === 60) { const count = dat.g1(); this.heads = new Uint16Array(count); @@ -192,12 +242,12 @@ export default class NpcType extends ConfigType { } else if (code === 100) { this.ambient = dat.g1b(); } else if (code === 101) { - this.contrast = dat.g1b(); + this.contrast = dat.g1b(); // Value multiplied by 5 client side } else if (code === 102) { this.headicon = dat.g2(); } else if (code === 103) { this.turnspeed = dat.g2(); - } else if (code === 106) { + } else if (code === 106 || code === 118) { this.multivarbit = dat.g2(); if (this.multivarbit === 65535) { this.multivarbit = -1; @@ -208,16 +258,92 @@ export default class NpcType extends ConfigType { this.multivarp = -1; } + let defaultid = -1; + if (code === 118) { + defaultid = dat.g2(); + if (defaultid === 65535) { + defaultid = -1; + } + } + const count = dat.g1(); - this.multinpc = new Array(count + 1); + this.multinpc = new Array(count + 2); for (let i = 0; i <= count; i++) { this.multinpc[i] = dat.g2(); if (this.multinpc[i] === 65535) { this.multinpc[i] = -1; } } + this.multinpc[count + 1] = defaultid; } else if (code === 107) { this.active = false; + } else if (code === 109) { + this.walksmoothing = false; + } else if (code === 111) { + this.spotshadow = false; + } else if (code === 112) { + this.spotshadowcolour_1 = dat.g2(); + this.spotshadowcolour_2 = dat.g2(); + } else if (code === 113) { + this.spotshadow_trans1 = dat.g1b(); + this.spotshadow_trans2 = dat.g1b(); + } else if (code === 115) { + // TODO: Figure out what these values actually are. + this.code115_1 = dat.g1(); + this.code115_2 = dat.g1(); + } else if (code === 119) { + this.walkflags = dat.g1b(); + } else if (code === 121) { + this.modeloffset = new Array(this.models?.length); + const count = dat.g1(); + for (let i = 0; i < count; i++) { + const offset = dat.g1(); + const offsets = this.modeloffset[offset] = new Array(3); + offsets[0] = dat.g1b(); + offsets[1] = dat.g1b(); + offsets[2] = dat.g1b(); + } + } else if (code === 122) { + this.hitbarid = dat.g2(); + } else if (code === 123) { + this.overlayheight = dat.g2(); + } else if (code === 125) { + this.respawndir = dat.g1b(); + } else if (code === 126) { + this.mapfunction = dat.g2(); + } else if (code === 127) { + this.bas = dat.g2(); + } else if (code === 128) { + this.movespeed = dat.g1(); + } else if (code === 134) { + this.bgsound = dat.g2(); + if (this.bgsound === 65535) { + this.bgsound = -1; + } + + this.bgsound_crawl = dat.g2(); + if (this.bgsound_crawl === 65535) { + this.bgsound_crawl = -1; + } + + this.bgsound_walk = dat.g2(); + if (this.bgsound_walk === 65535) { + this.bgsound_walk = -1; + } + + this.bgsound_run = dat.g2(); + if (this.bgsound_run === 65535) { + this.bgsound_run = -1; + } + this.bgsound_range = dat.g1(); + } else if (code === 135) { + this.cursor1op = dat.g1(); + this.cursor1 = dat.g2(); + } else if (code === 136) { + this.cursor2op = dat.g1(); + this.cursor2 = dat.g2(); + } else if (code === 137) { + this.cursorattack = dat.g2(); } else if (code === 200) { this.wanderrange = dat.g2(); } else if (code === 201) { diff --git a/src/cache/config/ObjType.ts b/src/cache/config/ObjType.ts index ef7fc4705..86d2a6e92 100644 --- a/src/cache/config/ObjType.ts +++ b/src/cache/config/ObjType.ts @@ -120,17 +120,18 @@ export default class ObjType extends ConfigType { // ---- model = 0; name: string | null = null; - desc: string | null = null; recol_s: Uint16Array | null = null; recol_d: Uint16Array | null = null; + retex_s: Uint16Array | null = null; + retex_d: Uint16Array | null = null; + recol_d_palette: Int8Array | null = null; + stockmarket = false; zoom2d = 2000; xan2d = 0; yan2d = 0; zan2d = 0; xof2d = 0; yof2d = 0; - code9 = false; - code10 = -1; stackable = false; cost = 1; members = false; @@ -138,10 +139,14 @@ export default class ObjType extends ConfigType { iop: (string | null)[] | null = null; manwear = -1; manwear2 = -1; + manwearOffsetX = 0; manwearOffsetY = 0; + manwearOffsetZ = 0; womanwear = -1; womanwear2 = -1; + womanwearOffsetX = 0; womanwearOffsetY = 0; + womanwearOffsetZ = 0; manwear3 = -1; womanwear3 = -1; manhead = -1; @@ -158,8 +163,21 @@ export default class ObjType extends ConfigType { ambient = 0; contrast = 0; team = 0; + lentlink = -1; + lenttemplate = -1; + + cursor1op = -1; + cursor1 = -1; + cursor2op = -1; + cursor2 = -1; + cursor1iop = -1; + cursor1i = -1; + cursor2iop = -1; + cursor2i = -1; + params: ParamMap = new Map(); // server-side + desc: string | null = null; wearpos = -1; wearpos2 = -1; wearpos3 = -1; @@ -167,8 +185,8 @@ export default class ObjType extends ConfigType { category = -1; dummyitem = 0; tradeable = true; + lendable = false; respawnrate = 100; // default to 1-minute - params: ParamMap = new Map(); decode(code: number, dat: Packet): void { if (code === 1) { @@ -187,10 +205,6 @@ export default class ObjType extends ConfigType { this.xof2d = dat.g2s(); } else if (code === 8) { this.yof2d = dat.g2s(); - } else if (code === 9) { - this.code9 = true; - } else if (code === 10) { - this.code10 = dat.g2(); } else if (code === 11) { this.stackable = true; } else if (code === 12) { @@ -205,12 +219,10 @@ export default class ObjType extends ConfigType { this.members = true; } else if (code === 23) { this.manwear = dat.g2(); - this.manwearOffsetY = dat.g1b(); } else if (code === 24) { this.manwear2 = dat.g2(); } else if (code === 25) { this.womanwear = dat.g2(); - this.womanwearOffsetY = dat.g1b(); } else if (code === 26) { this.womanwear2 = dat.g2(); } else if (code === 27) { @@ -234,6 +246,24 @@ export default class ObjType extends ConfigType { this.recol_s[i] = dat.g2(); this.recol_d[i] = dat.g2(); } + } else if (code === 41) { + const count = dat.g1(); + this.retex_s = new Uint16Array(count); + this.retex_d = new Uint16Array(count); + + for (let i = 0; i < count; i++) { + this.retex_s[i] = dat.g2(); + this.retex_d[i] = dat.g2(); + } + } else if (code === 42) { + const count = dat.g1(); + this.recol_d_palette = new Int8Array(count); + + for (let i = 0; i < count; i++) { + this.recol_d_palette[i] = dat.g1b(); + } + } else if (code === 65) { + this.stockmarket = true; } else if (code === 75) { this.weight = dat.g2s(); } else if (code === 78) { @@ -274,9 +304,35 @@ export default class ObjType extends ConfigType { } else if (code === 113) { this.ambient = dat.g1b(); } else if (code === 114) { - this.contrast = dat.g1b(); + this.contrast = dat.g1b(); // Value multiplied by 5 client side } else if (code === 115) { this.team = dat.g1(); + } else if (code === 121) { + this.lentlink = dat.g2(); + } else if (code === 122) { + this.lenttemplate = dat.g2(); + } else if (code === 123) { + this.lendable = true; + } else if (code === 125) { + this.manwearOffsetX = dat.g1b(); + this.manwearOffsetY = dat.g1b(); + this.manwearOffsetZ = dat.g1b(); + } else if (code === 126) { + this.womanwearOffsetX = dat.g1b(); + this.womanwearOffsetY = dat.g1b(); + this.womanwearOffsetZ = dat.g1b(); + } else if (code === 127) { + this.cursor1op = dat.g1(); + this.cursor1 = dat.g2(); + } else if (code === 128) { + this.cursor2op = dat.g1(); + this.cursor2 = dat.g2(); + } else if (code === 129) { + this.cursor1iop = dat.g1(); + this.cursor1i = dat.g2(); + } else if (code === 130) { + this.cursor2iop = dat.g1(); + this.cursor2i = dat.g2(); } else if (code === 201) { this.respawnrate = dat.g2(); } else if (code === 249) { @@ -299,6 +355,9 @@ export default class ObjType extends ConfigType { this.yof2d = template.yof2d; this.recol_s = template.recol_s; this.recol_d = template.recol_d; + this.recol_d_palette = template.recol_d_palette; + this.retex_s = template.retex_s; + this.retex_d = template.retex_d; const link = ObjType.get(this.certlink)!; this.name = link.name; @@ -315,4 +374,56 @@ export default class ObjType extends ConfigType { this.stackable = true; } + + toLentObj() { + const template = ObjType.get(this.lenttemplate)!; + this.model = template.model; + this.zoom2d = template.zoom2d; + this.xan2d = template.xan2d; + this.yan2d = template.yan2d; + this.zan2d = template.zan2d; + this.xof2d = template.xof2d; + this.yof2d = template.yof2d; + + const link = ObjType.get(this.lentlink)!; + this.name = link.name; + this.recol_s = link.recol_s; + this.recol_d = link.recol_d; + this.recol_d_palette = link.recol_d_palette; + this.retex_s = link.retex_s; + this.retex_d = link.retex_d; + this.manhead = link.manhead; + this.manhead2 = link.manhead2; + this.manwear = link.manwear; + this.manwear2 = link.manwear2; + this.manwear3 = link.manwear3; + this.manwearOffsetX = link.manwearOffsetX; + this.manwearOffsetY = link.manwearOffsetY; + this.manwearOffsetZ = link.manwearOffsetZ; + this.womanhead = link.womanhead; + this.womanhead2 = link.womanhead2; + this.womanwear = link.womanwear; + this.womanwear2 = link.womanwear2; + this.womanwear3 = link.womanwear3; + this.womanwearOffsetX = link.womanwearOffsetX; + this.womanwearOffsetY = link.womanwearOffsetY; + this.womanwearOffsetZ = link.womanwearOffsetZ; + this.op = link.op; + this.team = link.team; + this.members = link.members; + this.params = link.params; + + // Need to verify descriptions for lent items. + // this.desc = + this.tradeable = false; + this.cost = 0; + + this.iop = new Array(5).fill(null); + if (link.iop != null) { + for (let i = 0; i < link.iop.length; i++) { + this.iop[i] = link.iop[i]; + } + } + this.iop[0] = 'Discard'; + } } diff --git a/src/cache/config/ParamHelper.ts b/src/cache/config/ParamHelper.ts index f9b2a69d3..be1d0198a 100644 --- a/src/cache/config/ParamHelper.ts +++ b/src/cache/config/ParamHelper.ts @@ -27,9 +27,9 @@ export const ParamHelper = { const count = dat.g1(); const params = new Map(); for (let i = 0; i < count; i++) { - const key = dat.g3(); const isString = dat.gbool(); - + const key = dat.g3(); + if (isString) { params.set(key, dat.gjstr()); } else { diff --git a/src/cache/config/QuickChatCatType.ts b/src/cache/config/QuickChatCatType.ts new file mode 100644 index 000000000..45661fa06 --- /dev/null +++ b/src/cache/config/QuickChatCatType.ts @@ -0,0 +1,140 @@ +import { ConfigType } from '#/cache/config/ConfigType.js'; +import Packet from '#/io/Packet.js'; + +export default class QuickChatCatType extends ConfigType { + private static configNames = new Map(); + private static configs: QuickChatCatType[] = []; + + static load(_dir: string) { + } + + static parse(dat: Packet) { + QuickChatCatType.configNames = new Map(); + QuickChatCatType.configs = []; + + const count = dat.g2(); + + for (let id = 0; id < count; id++) { + const config = new QuickChatCatType(id); + config.decodeType(dat); + + QuickChatCatType.configs[id] = config; + + if (config.debugname) { + QuickChatCatType.configNames.set(config.debugname, id); + } + } + } + + static get(id: number): QuickChatCatType { + return QuickChatCatType.configs[id]; + } + + static getId(name: string): number { + return QuickChatCatType.configNames.get(name) ?? -1; + } + + static getByName(name: string): QuickChatCatType | null { + const id = this.getId(name); + if (id === -1) { + return null; + } + + return this.get(id); + } + + static get count() { + return this.configs.length; + } + + // --- + description: string | null = null; + + subcategories: number[] | null = null; + subcategoryShortcuts: number[] | null = null; + + phrases: number[] | null = null; + phraseShortcuts: number[] | null = null; + hasOpcode4: boolean = false; + + private static shortcutByteToCode(value: number): number { + return value & 0xff; + } + + applyMembersHighBit(): void { + if (this.phrases) { + for (let i = 0; i < this.phrases.length; i++) { + this.phrases[i] |= 0x8000; + } + } + + if (this.subcategories) { + for (let i = 0; i < this.subcategories.length; i++) { + this.subcategories[i] |= 0x8000; + } + } + } + + getPhraseIdByShortcut(shortcut: number): number { + if (!this.phrases || !this.phraseShortcuts) { + return -1; + } + + for (let i = 0; i < this.phrases.length; i++) { + if (this.phraseShortcuts[i] === shortcut) { + return this.phrases[i]; + } + } + + return -1; + } + + getSubCategoryIdByShortcut(shortcut: number): number { + if (!this.subcategories || !this.subcategoryShortcuts) { + return -1; + } + + for (let i = 0; i < this.subcategories.length; i++) { + if (this.subcategoryShortcuts[i] === shortcut) { + return this.subcategories[i]; + } + } + + return -1; + } + + decode(code: number, dat: Packet): void { + if (code === 1) { + this.description = dat.gjstr(); + } else if (code === 2) { + const count = dat.g1(); + this.subcategories = new Array(count); + this.subcategoryShortcuts = new Array(count); + + for (let i = 0; i < count; i++) { + this.subcategories[i] = dat.g2(); + this.subcategoryShortcuts[i] = QuickChatCatType.shortcutByteToCode(dat.g1b()); + } + } else if (code === 3) { + const count = dat.g1(); + this.phrases = new Array(count); + this.phraseShortcuts = new Array(count); + + for (let i = 0; i < count; i++) { + this.phrases[i] = dat.g2(); + this.phraseShortcuts[i] = QuickChatCatType.shortcutByteToCode(dat.g1b()); + } + } else if (code === 4) { + // Present in some quickchat cat archives, no payload. + this.hasOpcode4 = true; + } else if (code === 250) { + this.debugname = dat.gjstr(); + } else { + throw new Error(`Unrecognized quickchat cat config code: ${code}`); + } + } + + toString() { + return this.debugname ?? this.description ?? `quickchatcat_${this.id}`; + } +} diff --git a/src/cache/config/QuickChatPhraseType.ts b/src/cache/config/QuickChatPhraseType.ts new file mode 100644 index 000000000..401a2595d --- /dev/null +++ b/src/cache/config/QuickChatPhraseType.ts @@ -0,0 +1,114 @@ +import { ConfigType } from '#/cache/config/ConfigType.js'; +import Packet from '#/io/Packet.js'; + +export default class QuickChatPhraseType extends ConfigType { + private static configNames = new Map(); + private static configs: QuickChatPhraseType[] = []; + + // TODO: Rework once we have more information on how dynamic command parameters work. + // Index = command ID, Value = number of parameters that command takes + static readonly DYNAMIC_COMMAND_PARAM_COUNTS = [ + 1, 0, 0, 0, 1, 0, 2, 1, 1, 1, 0, 2, 0, 0, 1, 0 + ]; + + static load(_dir: string) { + } + + static parse(dat: Packet) { + QuickChatPhraseType.configNames = new Map(); + QuickChatPhraseType.configs = []; + + const count = dat.g2(); + + for (let id = 0; id < count; id++) { + const config = new QuickChatPhraseType(id); + config.decodeType(dat); + + QuickChatPhraseType.configs[id] = config; + + if (config.debugname) { + QuickChatPhraseType.configNames.set(config.debugname, id); + } + } + } + + static get(id: number): QuickChatPhraseType { + return QuickChatPhraseType.configs[id]; + } + + static getId(name: string): number { + return QuickChatPhraseType.configNames.get(name) ?? -1; + } + + static getByName(name: string): QuickChatPhraseType | null { + const id = this.getId(name); + if (id === -1) { + return null; + } + + return this.get(id); + } + + static get count() { + return this.configs.length; + } + + // --- + text: string[] | null = null; + autoResponses: number[] | null = null; + dynamicCommands: number[] | null = null; + dynamicCommandParameters: number[][] | null = null; + searchable: boolean = true; + + decode(code: number, dat: Packet): void { + if (code === 1) { + // Text split by '<' (char code 60) delimiter + const textStr = dat.gjstr(); + this.text = textStr.split('<'); + } else if (code === 2) { + // Auto responses (IDs to response phrasetype configs) + const count = dat.g1(); + this.autoResponses = new Array(count); + + for (let i = 0; i < count; i++) { + this.autoResponses[i] = dat.g2(); + } + } else if (code === 3) { + // Dynamic commands (for variable substitution in phrases) + const count = dat.g1(); + this.dynamicCommands = new Array(count); + this.dynamicCommandParameters = new Array(count); + + for (let i = 0; i < count; i++) { + const commandId = dat.g2(); + this.dynamicCommands[i] = commandId; + + // Parameter count is determined by static lookup table, not from packet + const paramCount = QuickChatPhraseType.DYNAMIC_COMMAND_PARAM_COUNTS[commandId] ?? 0; + this.dynamicCommandParameters[i] = new Array(paramCount); + for (let p = 0; p < paramCount; p++) { + this.dynamicCommandParameters[i][p] = dat.g2(); + } + } + } else if (code === 4) { + // Searchable flag + this.searchable = false; + } else if (code === 250) { + this.debugname = dat.gjstr(); + } else { + throw new Error(`Unrecognized quickchat phrase config code: ${code}`); + } + } + + applyMembersHighBit(): void { + if (this.autoResponses) { + for (let i = 0; i < this.autoResponses.length; i++) { + this.autoResponses[i] |= 0x8000; + } + } + } + + toString() { + return this.debugname ?? `quickchatphrase_${this.id}`; + } +} diff --git a/src/cache/config/ScriptVarType.ts b/src/cache/config/ScriptVarType.ts index 3b5035b0a..3229177dc 100644 --- a/src/cache/config/ScriptVarType.ts +++ b/src/cache/config/ScriptVarType.ts @@ -24,6 +24,11 @@ export default class ScriptVarType { static readonly NPC_STAT = 254; // þ static readonly IDKIT = 75; // K static readonly DBROW = 208; // Ð + static readonly MIDI = 77; // M + static readonly GRAPHIC = 100; // d + static readonly MODEL = 109; // m + static readonly JINGLE = 106; // j + static readonly CHATCAT = 107; // k static getType(type: number) { switch (type) { @@ -77,6 +82,16 @@ export default class ScriptVarType { return 'idkit'; case ScriptVarType.DBROW: return 'dbrow'; + case ScriptVarType.MIDI: + return 'midi'; + case ScriptVarType.GRAPHIC: + return 'graphic'; + case ScriptVarType.MODEL: + return 'model'; + case ScriptVarType.JINGLE: + return 'jingle'; + case ScriptVarType.CHATCAT: + return 'chatcat'; default: return 'unknown'; } @@ -162,6 +177,21 @@ export default class ScriptVarType { case 'dbrow': char = 'Ð'; break; + case 'midi': + char = 'M'; + break; + case 'graphic': + char = 'd'; + break; + case 'model': + char = 'm'; + break; + case 'jingle': + char = 'j'; + break; + case 'chatcat': + char = 'k'; + break; default: return null; } diff --git a/src/cache/config/SeqType.ts b/src/cache/config/SeqType.ts index d3d9d8be1..4d2932f6c 100644 --- a/src/cache/config/SeqType.ts +++ b/src/cache/config/SeqType.ts @@ -62,7 +62,7 @@ export default class SeqType extends ConfigType { delay: Int32Array | null = null; loops: number = -1; walkmerge: Int32Array | null = null; - stretches: boolean = false; + reachforward: boolean = false; priority: number = 5; replaceheldleft: number = -1; replaceheldright: number = -1; @@ -112,7 +112,7 @@ export default class SeqType extends ConfigType { this.walkmerge[count] = 9999999; } else if (code === 4) { - this.stretches = true; + this.reachforward = true; } else if (code === 5) { this.priority = dat.g1(); } else if (code === 6) { diff --git a/src/db/dialect/BunSqliteDriver.ts b/src/db/dialect/BunSqliteDriver.ts index fa04e68a2..dddf32b5f 100644 --- a/src/db/dialect/BunSqliteDriver.ts +++ b/src/db/dialect/BunSqliteDriver.ts @@ -80,9 +80,14 @@ class BunSqliteConnection implements DatabaseConnection { rows: [] }); } catch (err) { - if (err instanceof SQLiteError) { + if (err instanceof SQLiteError && err.errno === 5) { + // database is locked, retry await sleep(100); continue; + } else if (err instanceof SQLiteError) { + // query error + console.error(err.message); + break; } else { console.error(err); break; diff --git a/src/engine/Js5.ts b/src/engine/Js5.ts index e4a10eadf..c43a35cde 100644 --- a/src/engine/Js5.ts +++ b/src/engine/Js5.ts @@ -2,6 +2,7 @@ import Packet from '#/io/Packet.js'; import ClientSocket from '#/server/ClientSocket.js'; import TcpClientSocket from '#/server/tcp/TcpClientSocket.js'; import { getGroup } from '#/util/OpenRS2.js'; +import Js5PackReader from '#/io/Js5PackReader.js'; type Js5Request = { client: ClientSocket; @@ -12,19 +13,28 @@ type Js5Request = { class Js5 { urgentRequests: Js5Request[] = []; prefetchRequests: Js5Request[] = []; + private clientArchivePacks = new Map(); + private readonly clientPackPaths = new Map([ + [17, 'data/pack/client/client.enum.config.js5'], + [6, 'data/pack/client/client.midi.js5'], + [24, 'data/pack/client/client.quickchat.js5'], + [25, 'data/pack/client/client.quickchat.global.js5'], + [18, 'data/pack/client/client.npc.config.js5'], + [15, 'data/pack/client/client.patches.js5'] + ]); async cycle() { // todo: limit requests per client per cycle for (let i = 0; i < this.urgentRequests.length; i++) { const req = this.urgentRequests[i]; - await this.send(req.client, false, req.archive, req.group); + this.sendAsync(req.client, false, req.archive, req.group); this.urgentRequests.splice(i--, 1); } for (let i = 0; i < this.prefetchRequests.length; i++) { const req = this.prefetchRequests[i]; - await this.send(req.client, true, req.archive, req.group); + this.sendAsync(req.client, true, req.archive, req.group); this.prefetchRequests.splice(i--, 1); } @@ -61,7 +71,11 @@ class Js5 { return; } - const data = await getGroup(archive, group); + let data = this.getClientPackGroup(archive, group); + + if (!data) { + data = await getGroup(archive, group); + } if (!data || !data.length) { console.log('missing archive, group', archive, group); return; @@ -100,6 +114,35 @@ class Js5 { client.send(response.data.subarray(0, response.pos)); } } + + private sendAsync(client: ClientSocket, prefetch: boolean, archive: number, group: number): void { + void this.send(client, prefetch, archive, group).catch(err => { + console.warn('JS5 send failed', { archive, group, err }); + }); + } + + private getClientPackGroup(archive: number, group: number): Uint8Array | undefined { + const packPath = this.clientPackPaths.get(archive); + if (!packPath) { + return undefined; + } + + if (!this.clientArchivePacks.has(archive)) { + try { + this.clientArchivePacks.set(archive, Js5PackReader.load(packPath)); + } catch (err) { + console.warn(`Unable to load client js5pack for archive ${archive} (${packPath}), falling back to cache.`, err); + this.clientArchivePacks.set(archive, null); + } + } + + const data = this.clientArchivePacks.get(archive)?.getGroup(group); + if (!data || data.length === 0) { + return undefined; + } + + return data; + } } export default new Js5(); diff --git a/src/engine/World.ts b/src/engine/World.ts index eb5effed6..91630951a 100644 --- a/src/engine/World.ts +++ b/src/engine/World.ts @@ -1007,7 +1007,7 @@ class World { player.reorient(); player.buildArea.rebuildNormal(); // set origin before compute player is why this is above. - const appearance = player.masks & PlayerInfoProt.APPEARANCE ? player.generateAppearance() : (player.lastAppearanceBytes ?? player.generateAppearance()); + const appearance = player.masks & PlayerInfoProt.APPEARANCE ? player.generateAppearance() : (player.appearanceBuf ?? player.generateAppearance()); rsbuf.computePlayer( player.x, @@ -1026,33 +1026,33 @@ class World { appearance, player.lastAppearance, player.faceEntity, - player.faceX, - player.faceZ, - player.orientationX, - player.orientationZ, - player.damageTaken, - player.damageType, - player.damageTaken2, - player.damageType2, + player.faceSquareX, + player.faceSquareZ, + player.faceAngleX, + player.faceAngleZ, + player.hitmarkDamage, + player.hitmarkType, + player.hitmark2Damage, + player.hitmark2Type, player.levels[PlayerStat.HITPOINTS], player.baseLevels[PlayerStat.HITPOINTS], player.animId, player.animDelay, - player.chat, - player.message, - player.messageColor ?? -1, - player.messageEffect ?? -1, - player.messageType ?? 0, - player.graphicId, - player.graphicHeight, - player.graphicDelay, + player.sayMessage, + player.chatMessage, + player.chatColour ?? -1, + player.chatEffect ?? -1, + player.chatRights ?? 0, + player.spotanimId, + player.spotanimHeight, + player.spotanimTime, player.exactStartX, player.exactStartZ, player.exactEndX, player.exactEndZ, player.exactMoveStart, player.exactMoveEnd, - player.exactMoveDirection + player.exactMoveFacing ); } @@ -1071,22 +1071,22 @@ class World { npc.isActive, npc.masks, npc.faceEntity, - npc.faceX, - npc.faceZ, - npc.orientationX, - npc.orientationZ, - npc.damageTaken, - npc.damageType, - npc.damageTaken2, - npc.damageType2, + npc.faceSquareX, + npc.faceSquareZ, + npc.faceAngleX, + npc.faceAngleZ, + npc.hitmarkDamage, + npc.hitmarkType, + npc.hitmark2Damage, + npc.hitmark2Type, npc.levels[NpcStat.HITPOINTS], npc.baseLevels[NpcStat.HITPOINTS], npc.animId, npc.animDelay, - npc.chat, - npc.graphicId, - npc.graphicHeight, - npc.graphicDelay + npc.sayMessage, + npc.spotanimId, + npc.spotanimHeight, + npc.spotanimTime ); } } diff --git a/src/engine/entity/Npc.ts b/src/engine/entity/Npc.ts index 98c601c43..93a0cb0a6 100644 --- a/src/engine/entity/Npc.ts +++ b/src/engine/entity/Npc.ts @@ -121,15 +121,17 @@ export default class Npc extends PathingEntity { // Npc Events (Respawn, Revert, Despawn) if (!this.delayed && --this.lifecycleTick === 0) { try { - // Respawn NPC - if (this.lifecycle === EntityLifeCycle.RESPAWN && !this.isActive) { - World.addNpc(this, -1, false); - } - // Revert NPC if (this.lifecycle === EntityLifeCycle.RESPAWN) { - this.revertType(); + // Respawn NPC (npc_del) + if (!this.isActive) { + World.addNpc(this, -1, false); + } + // Revert NPC (npc_changetype) + else { + this.revertType(); + } } - // Despawn NPC + // Despawn NPC (npc_add) else if (this.lifecycle === EntityLifeCycle.DESPAWN) { World.removeNpc(this, -1); // Queue despawn trigger @@ -289,6 +291,7 @@ export default class Npc extends PathingEntity { } this.heroPoints.clear(); this.queue.clear(); + this.clearWaypoints(); for (let i = 0; i < this.vars.length; i++) { const varn = VarNpcType.get(i); @@ -461,9 +464,9 @@ export default class Npc extends PathingEntity { } spotanim(spotanim: number, height: number, delay: number) { - this.graphicId = spotanim; - this.graphicHeight = height; - this.graphicDelay = delay; + this.spotanimId = spotanim; + this.spotanimHeight = height; + this.spotanimTime = delay; this.masks |= NpcInfoProt.SPOT_ANIM; } @@ -476,16 +479,16 @@ export default class Npc extends PathingEntity { this.levels[NpcStat.HITPOINTS] = current - damage; } - if (this.damageSlot % 2 === 1) { - this.damageTaken2 = damage; - this.damageType2 = type; + if (this.hitmarkSlot % 2 === 1) { + this.hitmark2Damage = damage; + this.hitmark2Type = type; this.masks |= NpcInfoProt.DAMAGE2; } else { - this.damageTaken = damage; - this.damageType = type; + this.hitmarkDamage = damage; + this.hitmarkType = type; this.masks |= NpcInfoProt.DAMAGE; } - this.damageSlot++; + this.hitmarkSlot++; } say(text: string) { @@ -493,7 +496,7 @@ export default class Npc extends PathingEntity { return; } - this.chat = text; + this.sayMessage = text; this.masks |= NpcInfoProt.SAY; } diff --git a/src/engine/entity/PathingEntity.ts b/src/engine/entity/PathingEntity.ts index 601c4a3bb..7b79fc436 100644 --- a/src/engine/entity/PathingEntity.ts +++ b/src/engine/entity/PathingEntity.ts @@ -73,7 +73,11 @@ export default abstract class PathingEntity extends Entity { targetX: number = -1; targetZ: number = -1; - // info update masks. resets at the end of every tick. + // sent on first add + faceAngleX: number = -1; + faceAngleZ: number = -1; + + // info updates masks: number = 0; exactStartX: number = -1; exactStartZ: number = -1; @@ -81,23 +85,21 @@ export default abstract class PathingEntity extends Entity { exactEndZ: number = -1; exactMoveStart: number = -1; exactMoveEnd: number = -1; - exactMoveDirection: number = -1; - faceX: number = -1; - faceZ: number = -1; - orientationX: number = -1; - orientationZ: number = -1; + exactMoveFacing: number = -1; + faceSquareX: number = -1; + faceSquareZ: number = -1; faceEntity: number = -1; - damageSlot: number = 0; - damageTaken: number = -1; - damageType: number = -1; - damageTaken2: number = -1; - damageType2: number = -1; + hitmarkSlot: number = 0; + hitmarkDamage: number = -1; + hitmarkType: number = -1; + hitmark2Damage: number = -1; + hitmark2Type: number = -1; animId: number = -1; animDelay: number = -1; - chat: string | null = null; - graphicId: number = -1; - graphicHeight: number = -1; - graphicDelay: number = -1; + sayMessage: string | null = null; + spotanimId: number = -1; + spotanimHeight: number = -1; + spotanimTime: number = -1; protected constructor(level: number, x: number, z: number, width: number, length: number, lifecycle: EntityLifeCycle, moveRestrict: MoveRestrict, blockWalk: BlockWalk, moveStrategy: MoveStrategy, coordmask: number, entitymask: number) { super(level, x, z, width, length, lifecycle); @@ -323,12 +325,12 @@ export default abstract class PathingEntity extends Entity { // set the direction of the player/npc every time an interaction is set. // does not necessarily require the coord mask to be sent. // direction when the player/npc is first observed (updates on movement) - this.orientationX = fineX; - this.orientationZ = fineZ; + this.faceAngleX = fineX; + this.faceAngleZ = fineZ; if (client) { // direction update (only updates from facesquare or interactions) - this.faceX = fineX; - this.faceZ = fineZ; + this.faceSquareX = fineX; + this.faceSquareZ = fineZ; this.masks |= this.coordmask; } } @@ -337,8 +339,8 @@ export default abstract class PathingEntity extends Entity { * Face and orient back to the default south. */ unfocus(): void { - this.orientationX = CoordGrid.fine(this.x, this.width); - this.orientationZ = CoordGrid.fine(this.z - 1, this.length); + this.faceAngleX = CoordGrid.fine(this.x, this.width); + this.faceAngleZ = CoordGrid.fine(this.z - 1, this.length); } /** @@ -595,22 +597,22 @@ export default abstract class PathingEntity extends Entity { this.exactEndZ = -1; this.exactMoveStart = -1; this.exactMoveEnd = -1; - this.exactMoveDirection = -1; + this.exactMoveFacing = -1; this.animId = -1; this.animDelay = -1; this.animId = -1; this.animDelay = -1; - this.chat = null; - this.damageTaken = -1; - this.damageType = -1; - this.damageTaken2 = -1; - this.damageType2 = -1; - this.damageSlot = 0; - this.graphicId = -1; - this.graphicHeight = -1; - this.graphicDelay = -1; - this.faceX = -1; - this.faceZ = -1; + this.sayMessage = null; + this.hitmarkDamage = -1; + this.hitmarkType = -1; + this.hitmark2Damage = -1; + this.hitmark2Type = -1; + this.hitmarkSlot = 0; + this.spotanimId = -1; + this.spotanimHeight = -1; + this.spotanimTime = -1; + this.faceSquareX = -1; + this.faceSquareZ = -1; if (!this.target && this.faceEntity !== -1) { this.masks |= this.entitymask; diff --git a/src/engine/entity/Player.ts b/src/engine/entity/Player.ts index 1795a8f62..81dd26e1b 100644 --- a/src/engine/entity/Player.ts +++ b/src/engine/entity/Player.ts @@ -47,7 +47,7 @@ import Packet from '#/io/Packet.js'; import { ServerGameProtPriority } from '#/network/game/server/ServerGameProtPriority.js'; import ChatFilterSettings from '#/network/game/server/model/ChatFilterSettings.js'; import HintArrow from '#/network/game/server/model/HintArrow.js'; -import LastLoginInfo from '#/network/game/server/model/LastLoginInfo.js'; +import _LastLoginInfo from '#/network/game/server/model/LastLoginInfo.js'; import MessageGame from '#/network/game/server/model/MessageGame.js'; import ResetAnims from '#/network/game/server/model/ResetAnims.js'; import ResetClientVarCache from '#/network/game/server/model/ResetClientVarCache.js'; @@ -69,6 +69,7 @@ import FriendlistLoaded from '#/network/game/server/model/FriendlistLoaded.js'; import UpdateIgnoreList from '#/network/game/server/model/UpdateIgnoreList.js'; import IfOpenTop from '#/network/game/server/model/IfOpenTop.js'; import IfOpenSub from '#/network/game/server/model/IfOpenSub.js'; +import { WindowMode } from '#/network/game/client/model/WindowStatus.js'; const levelExperience = new Int32Array(99); @@ -201,7 +202,7 @@ export default class Player extends PathingEntity { sav.p2(this.runenergy); sav.p4(this.playtime); - for (let i = 0; i < 21; i++) { + for (let i = 0; i < 24; i++) { sav.p4(this.stats[i]); sav.p1(this.levels[i]); } @@ -259,7 +260,7 @@ export default class Player extends PathingEntity { sav.p1((this.publicChat << 4) | (this.privateChat << 2) | this.tradeDuel); // last login info - sav.p8(this.lastDate); + sav.p8(this.lastLoginTime); sav.p4(Packet.getcrc(sav.data, 0, sav.pos)); return sav.data.subarray(0, sav.pos); @@ -287,8 +288,8 @@ export default class Player extends PathingEntity { lastRunEnergy: number = -1; runweight: number = 0; playtime: number = 0; - stats: Int32Array = new Int32Array(21); - levels: Uint8Array = new Uint8Array(21); + stats: Int32Array = new Int32Array(24); + levels: Uint8Array = new Uint8Array(24); vars: Int32Array; varsString: string[]; invs: Map = new Map(); @@ -310,22 +311,12 @@ export default class Player extends PathingEntity { webClient: boolean = false; combatLevel: number = 3; headicons: number = 0; - appearance: number = -1; - lastAppearance: number = 0; - lastAppearanceBytes: Uint8Array | null = null; - baseLevels = new Uint8Array(21); - lastStats: Int32Array = new Int32Array(21); // we track this so we know to flush stats only once a tick on changes - lastLevels: Uint8Array = new Uint8Array(21); // we track this so we know to flush stats only once a tick on changes + baseLevels = new Uint8Array(24); + lastStats: Int32Array = new Int32Array(24); // we track this so we know to flush stats only once a tick on changes + lastLevels: Uint8Array = new Uint8Array(24); // we track this so we know to flush stats only once a tick on changes originX: number = -1; originZ: number = -1; buildArea: BuildArea = new BuildArea(this); - basReadyAnim: number = -1; - basTurnOnSpot: number = -1; - basWalkForward: number = -1; - basWalkBackward: number = -1; - basWalkLeft: number = -1; - basWalkRight: number = -1; - basRunning: number = 0; animProtect: number = 0; invListeners: InventoryListener[] = []; allowDesign: boolean = false; @@ -338,15 +329,17 @@ export default class Player extends PathingEntity { preventLogoutMessage: string | null = null; preventLogoutUntil: number = -1; + // game window information + windowMode: WindowMode = WindowMode.SD; + canvasWidth: number = -1; + canvasHeight: number = -1; + antialiasingmode: number = -1; + // not stored as a byte buffer so we can write and encrypt opcodes later buffer: ServerGameMessage[] = []; lastResponse: number = -1; lastConnected: number = -1; - messageColor: number | null = null; - messageEffect: number | null = null; - messageType: number | null = null; - message: Uint8Array | null = null; logMessage: string | null = null; // --- @@ -376,11 +369,11 @@ export default class Player extends PathingEntity { activeScript: ScriptState | null = null; resumeButtons: number[] = []; - lastItem: number = -1; // opheld, opheldu, opheldt, inv_button - lastSlot: number = -1; // opheld, opheldu, opheldt, inv_button, inv_buttond + lastItem: number = -1; // opheld, opheldu, opheldt, if_button + lastSlot: number = -1; // opheld, opheldu, opheldt, if_button, if_buttond lastUseItem: number = -1; // opheldu, opobju, oplocu, opnpcu, opplayeru lastUseSlot: number = -1; // opheldu, opobju, oplocu, opnpcu, opplayeru - lastTargetSlot: number = -1; // inv_buttond + lastTargetSlot: number = -1; // if_buttond lastCom: number = -1; // if_button staffModLevel: number = 0; @@ -402,10 +395,30 @@ export default class Player extends PathingEntity { socialProtect: boolean = false; // social packet spam protection reportAbuseProtect: boolean = false; // social packet spam protection - lastDate: bigint = 0n; + lastLoginTime: bigint = 0n; + + // info updates + appearanceInv: number = -1; + appearanceBuf: Uint8Array | null = null; + lastAppearance: number = 0; + readyanim: number = -1; + turnanim: number = -1; + walkanim: number = -1; + walkanim_b: number = -1; + walkanim_l: number = -1; + walkanim_r: number = -1; + runanim: number = -1; + chatMessage: Uint8Array | null = null; + chatColour: number | null = null; + chatEffect: number | null = null; + chatRights: number | null = null; constructor(username: string, username37: bigint, hash64: bigint) { - super(0, 3094, 3106, 1, 1, EntityLifeCycle.FOREVER, MoveRestrict.NORMAL, BlockWalk.NPC, MoveStrategy.FLY, PlayerInfoProt.FACE_COORD, PlayerInfoProt.FACE_ENTITY); + super( + 0, 3094, 3106, // tutorial island + 1, 1, + EntityLifeCycle.FOREVER, MoveRestrict.NORMAL, BlockWalk.NPC, MoveStrategy.SMART, PlayerInfoProt.FACE_COORD, PlayerInfoProt.FACE_ENTITY + ); this.username = username; this.username37 = username37; @@ -441,9 +454,9 @@ export default class Player extends PathingEntity { this.timers.clear(); this.heroPoints.clear(); this.buildArea.clear(false); - this.appearance = -1; + this.appearanceInv = -1; this.lastAppearance = 0; - this.lastAppearanceBytes = null; + this.appearanceBuf = null; this.isActive = false; } @@ -454,12 +467,12 @@ export default class Player extends PathingEntity { super.resetPathingEntity(); this.repathed = false; this.protect = false; - this.messageColor = null; - this.messageEffect = null; - this.messageType = null; - this.message = null; + this.chatColour = null; + this.chatEffect = null; + this.chatRights = null; + this.chatMessage = null; this.logMessage = null; - this.appearance = -1; + this.appearanceInv = -1; this.socialProtect = false; this.reportAbuseProtect = false; } @@ -549,6 +562,8 @@ export default class Player extends PathingEntity { } this.write(new UpdateRunEnergy(this.runenergy)); this.write(new ResetAnims()); + this.masks |= this.entitymask; // resync face_entity + this.masks |= PlayerInfoProt.APPEARANCE; // resync appearance (todo: is it possible to do this for the local observer only?) this.moveSpeed = MoveSpeed.INSTANT; this.tele = true; this.jump = true; @@ -657,7 +672,7 @@ export default class Player extends PathingEntity { if (this.moveSpeed !== MoveSpeed.INSTANT) { this.moveSpeed = this.defaultMoveSpeed(); - if (this.basRunning === -1) { + if (this.runanim === -1) { this.moveSpeed = MoveSpeed.WALK; } else if (this.tempRun) { this.moveSpeed = MoveSpeed.RUN; @@ -1308,7 +1323,7 @@ export default class Player extends PathingEntity { const skippedSlots = []; - let worn = this.getInventory(this.appearance); + let worn = this.getInventory(this.appearanceInv); if (!worn) { worn = new Inventory(InvType.WORN, 0); } @@ -1358,6 +1373,7 @@ export default class Player extends PathingEntity { } stream.p2(423); + stream.p8(this.username37); stream.p1(this.combatLevel); stream.p2(0); // skill level @@ -1369,7 +1385,7 @@ export default class Player extends PathingEntity { stream.release(); this.lastAppearance = World.currentTick; - this.lastAppearanceBytes = appearance; + this.appearanceBuf = appearance; return appearance; } @@ -1842,7 +1858,7 @@ export default class Player extends PathingEntity { } buildAppearance(inv: number): void { - this.appearance = inv; + this.appearanceInv = inv; this.masks |= PlayerInfoProt.APPEARANCE; } @@ -1851,7 +1867,7 @@ export default class Player extends PathingEntity { return; } - if (anim == -1 || this.animId == -1 || SeqType.get(anim).priority > SeqType.get(this.animId).priority || SeqType.get(this.animId).priority === 0) { + if (anim == -1 || this.animId == -1 || SeqType.get(anim).priority >= SeqType.get(this.animId).priority) { this.animId = anim; this.animDelay = delay; this.masks |= PlayerInfoProt.ANIM; @@ -1859,9 +1875,9 @@ export default class Player extends PathingEntity { } spotanim(spotanim: number, height: number, delay: number) { - this.graphicId = spotanim; - this.graphicHeight = height; - this.graphicDelay = delay; + this.spotanimId = spotanim; + this.spotanimHeight = height; + this.spotanimTime = delay; this.masks |= PlayerInfoProt.SPOT_ANIM; } @@ -1874,16 +1890,16 @@ export default class Player extends PathingEntity { this.levels[PlayerStat.HITPOINTS] = current - damage; } - if (this.damageSlot % 2 === 1) { - this.damageTaken2 = damage; - this.damageType2 = type; + if (this.hitmarkSlot % 2 === 1) { + this.hitmark2Damage = damage; + this.hitmark2Type = type; this.masks |= PlayerInfoProt.DAMAGE2; } else { - this.damageTaken = damage; - this.damageType = type; + this.hitmarkDamage = damage; + this.hitmarkType = type; this.masks |= PlayerInfoProt.DAMAGE; } - this.damageSlot++; + this.hitmarkSlot++; } setVisibility(visibility: Visibility) { @@ -1905,7 +1921,7 @@ export default class Player extends PathingEntity { } say(message: string) { - this.chat = message; + this.sayMessage = message; this.masks |= PlayerInfoProt.SAY; } @@ -1927,7 +1943,7 @@ export default class Player extends PathingEntity { this.exactEndZ = endZ; this.exactMoveStart = startCycle; this.exactMoveEnd = endCycle; - this.exactMoveDirection = direction; + this.exactMoveFacing = direction; this.masks |= PlayerInfoProt.EXACT_MOVE; // todo: interpolate over time? instant teleport? verify with true tile on osrs @@ -2093,27 +2109,19 @@ export default class Player extends PathingEntity { } lastLoginInfo() { - const lastDate: bigint = this.lastDate === 0n ? BigInt(Date.now()) : this.lastDate; + const lastDate: bigint = this.lastLoginTime === 0n ? BigInt(Date.now()) : this.lastLoginTime; const nextDate: bigint = BigInt(Date.now()); - const lastIp = 2130706433; // 127.0.0.1 - const daysSinceLogin: number = (Number(lastDate) / (1000 * 60 * 60 * 24)) | 0; - const daysSincePasswordChanged = 201; // hide :) - const daysSinceRecoveriesChanged = 201; // hide :) - const currentDay: number = Number(nextDate) / (1000 * 60 * 60 * 24) | 0; - const unreadMessageCount = 0; - const membersCreditDays = 365; - - this.write(new LastLoginInfo( - lastIp, - currentDay, - daysSinceLogin, - daysSincePasswordChanged, - daysSinceRecoveriesChanged, - unreadMessageCount, - membersCreditDays - )); - this.lastDate = nextDate; + const _lastIp = 2130706433; // 127.0.0.1 + const _daysSinceLogin: number = (Number(lastDate) / (1000 * 60 * 60 * 24)) | 0; + const _daysSincePasswordChanged = 201; // hide :) + const _daysSinceRecoveriesChanged = 201; // hide :) + const _currentDay: number = Number(nextDate) / (1000 * 60 * 60 * 24) | 0; + const _unreadMessageCount = 0; + const _membersCreditDays = 365; + + //this.write(new LastLoginInfo(lastIp, daysSinceLogin, daysSinceRecoveriesChanged, this.messageCount, warnMembersInNonMembers)); + this.lastLoginTime = nextDate; } logout(): void { diff --git a/src/engine/entity/PlayerLoading.ts b/src/engine/entity/PlayerLoading.ts index 5d7bcc822..1d2916832 100644 --- a/src/engine/entity/PlayerLoading.ts +++ b/src/engine/entity/PlayerLoading.ts @@ -39,7 +39,7 @@ export class PlayerLoading { player.lastResponse = World.currentTick; if (sav.data.length < 2) { - for (let i = 0; i < 21; i++) { + for (let i = 0; i < 24; i++) { player.stats[i] = 0; player.baseLevels[i] = 1; player.levels[i] = 1; @@ -150,7 +150,7 @@ export class PlayerLoading { // last login info if (version >= 6) { - player.lastDate = sav.g8(); + player.lastLoginTime = sav.g8(); } player.combatLevel = player.getCombatLevel(); diff --git a/src/engine/script/Js5ScriptReader.ts b/src/engine/script/Js5ScriptReader.ts new file mode 100644 index 000000000..bc73e8251 --- /dev/null +++ b/src/engine/script/Js5ScriptReader.ts @@ -0,0 +1,23 @@ +import Js5Reader from '#/io/Js5Reader.js'; + +/** + * Reads scripts from a JS5 archive format (server.scripts.js5) + * + * This is a thin wrapper around Js5Reader for script-specific loading. + */ +export default class Js5ScriptReader { + private data: Uint8Array; + + constructor(data: Uint8Array) { + this.data = data; + } + + /** + * Parse the JS5 archive and extract all script data + * @returns Map of groupId -> decompressed script data + */ + parseArchive(): Map { + const reader = new Js5Reader(this.data); + return reader.read(); + } +} diff --git a/src/engine/script/ScriptOpcodePointers.ts b/src/engine/script/ScriptOpcodePointers.ts index 66f6fbb8b..a2ef38b46 100644 --- a/src/engine/script/ScriptOpcodePointers.ts +++ b/src/engine/script/ScriptOpcodePointers.ts @@ -583,8 +583,9 @@ const ScriptOpcodePointers: { }, [ScriptOpcode.NPC_FINDHERO]: { require: ['active_npc'], + require2: ['active_npc2'], set: ['active_player'], - set2: ['active_player'], + set2: ['active_player2'], conditional: true }, [ScriptOpcode.NPC_FINDUID]: { @@ -763,7 +764,7 @@ const ScriptOpcodePointers: { [ScriptOpcode.OBJ_ADD]: { require: ['active_player'], set: ['active_obj'], - require2: ['active_player'], + require2: ['active_player2'], set2: ['active_obj2'] }, [ScriptOpcode.OBJ_ADDALL]: { @@ -792,7 +793,7 @@ const ScriptOpcodePointers: { }, [ScriptOpcode.OBJ_TAKEITEM]: { require: ['active_obj', 'active_player'], - require2: ['active_obj', 'active_player2'] + require2: ['active_obj2', 'active_player2'] }, [ScriptOpcode.OBJ_TYPE]: { require: ['active_obj'], diff --git a/src/engine/script/ScriptProvider.ts b/src/engine/script/ScriptProvider.ts index c1d879cbc..2af9b3a99 100644 --- a/src/engine/script/ScriptProvider.ts +++ b/src/engine/script/ScriptProvider.ts @@ -3,14 +3,10 @@ import ScriptFile from '#/engine/script/ScriptFile.js'; import ServerTriggerType from '#/engine/script/ServerTriggerType.js'; import Packet from '#/io/Packet.js'; import { printFatalError, printWarning } from '#/util/Logger.js'; +import { Js5Archive } from '#/io/Js5Archive.js'; // maintains a list of scripts (id <-> name) export default class ScriptProvider { - /** - * The expected version of the script compiler that the runtime should be loading scripts from. - */ - public static readonly COMPILER_VERSION = 25; - /** * Array of loaded scripts. */ @@ -27,70 +23,77 @@ export default class ScriptProvider { private static scriptNames = new Map(); /** - * Loads all scripts from `dir`. + * Loads all scripts from `dir` in JS5Pack format. * - * @param dir The directory that holds the script.{dat,idx} files. + * @param dir The directory that holds the `server.scripts.js5` file. * @returns The number of scripts loaded. */ static load(dir: string): number { - const dat = Packet.load(`${dir}/server/script.dat`); - const idx = Packet.load(`${dir}/server/script.idx`); - return this.parse(dat, idx); + const js5Path = `${dir}/server/server.scripts.js5`; + + if (!Js5Archive.exists(js5Path)) { + printFatalError(`No scripts found at ${js5Path}. Please compile scripts first.`); + } + + const groups = Js5Archive.load(js5Path); + return this.parseGroups(groups); } static async loadAsync(dir: string): Promise { - const [dat, idx] = await Promise.all([Packet.loadAsync(`${dir}/server/script.dat`), Packet.loadAsync(`${dir}/server/script.idx`)]); - return this.parse(dat, idx); + const js5Path = `${dir}/server/server.scripts.js5`; + + if (!Js5Archive.exists(js5Path)) { + printFatalError(`No scripts found at ${js5Path}. Please compile scripts first.`); + } + + const groups = await Js5Archive.loadAsync(js5Path); + return this.parseGroups(groups); } - static parse(dat: Packet, idx: Packet): number { - if (!dat.data.length || !idx.data.length) { - printFatalError('No server cache found. Please build the cache first.'); - } + private static parseGroups(groups: Map): number { + try { + if (groups.size === 0) { + printFatalError('No scripts found in JS5 archive.'); + return -1; + } - const entries = dat.g4s(); - idx.pos += 4; + const scriptArray = new Array(Math.max(...groups.keys()) + 1); + const scriptNames = new Map(); + const scriptLookup = new Map(); - const version = dat.g4s(); - if (version !== ScriptProvider.COMPILER_VERSION) { - printFatalError('\nFatal: Scripts were compiled with an incompatible RuneScript compiler. Please update it, try `npm run build` and then restart the server.'); - } + let loaded = 0; - const scripts = new Array(entries); - const scriptNames = new Map(); - const scriptLookup = new Map(); + // Sort by group ID to process in order + const sortedGroups = Array.from(groups.entries()).sort((a, b) => a[0] - b[0]); - let loaded = 0; - for (let id = 0; id < entries; id++) { - const size = idx.g4s(); - if (size === 0) { - continue; - } + for (const [groupId, scriptData] of sortedGroups) { + try { + const script = ScriptFile.decode(groupId, new Packet(scriptData)); + scriptArray[groupId] = script; + scriptNames.set(script.name, groupId); - try { - const data: Uint8Array = new Uint8Array(size); - dat.gdata(data, 0, data.length); - const script = ScriptFile.decode(id, new Packet(data)); - scripts[id] = script; - scriptNames.set(script.name, id); + // add the script to lookup table if the value isn't -1 + if (script.info.lookupKey !== 0xffffffff) { + scriptLookup.set(script.info.lookupKey, script); + } - // add the script to lookup table if the value isn't -1 - if (script.info.lookupKey !== 0xffffffff) { - scriptLookup.set(script.info.lookupKey, script); + loaded++; + } catch (err) { + console.error(err); + printWarning(`Warning: Failed to load script ${groupId}, something may have been partially written`); + return -1; } - - loaded++; - } catch (err) { - console.error(err); - printWarning(`Warning: Failed to load script ${id}, something may have been partially written`); - return -1; } - } - ScriptProvider.scripts = scripts; - ScriptProvider.scriptNames = scriptNames; - ScriptProvider.scriptLookup = scriptLookup; - return loaded; + ScriptProvider.scripts = scriptArray; + ScriptProvider.scriptNames = scriptNames; + ScriptProvider.scriptLookup = scriptLookup; + return loaded; + } catch (err) { + console.error(err); + printFatalError('Failed to parse JS5 script archive.'); + return -1; + } } /** diff --git a/src/engine/script/ScriptRunner.ts b/src/engine/script/ScriptRunner.ts index b64224e28..c471e9fa5 100644 --- a/src/engine/script/ScriptRunner.ts +++ b/src/engine/script/ScriptRunner.ts @@ -25,7 +25,7 @@ import ScriptPointer from '#/engine/script/ScriptPointer.js'; import ScriptState from '#/engine/script/ScriptState.js'; import World from '#/engine/World.js'; import Environment from '#/util/Environment.js'; -import { printWarning } from '#/util/Logger.js'; +import { printError, printWarning } from '#/util/Logger.js'; export type CommandHandler = (state: ScriptState) => void; export type CommandHandlers = { @@ -116,42 +116,53 @@ export default class ScriptRunner { return state; } - static execute(state: ScriptState, reset = false, benchmark = false) { + static execute(state: ScriptState) { if (!state || !state.script || !state.script.info) { return ScriptState.ABORTED; } try { - if (reset) { - state.reset(); - } - if (state.execution !== ScriptState.RUNNING) { state.executionHistory.push(state.execution); } state.execution = ScriptState.RUNNING; - const start: number = performance.now() * 1000; + let start = 0; + if (Environment.NODE_DEBUG_PROFILE) { + start = performance.now() * 1000; + } + while (state.execution === ScriptState.RUNNING) { - if (state.pc >= state.script.opcodes.length || state.pc < -1) { - throw new Error('Invalid program counter: ' + state.pc + ', max expected: ' + state.script.opcodes.length); + const opcodes = state.script.opcodes; + + if (state.pc >= opcodes.length || state.pc < -1) { + throw new Error('Invalid program counter: ' + state.pc + ', max expected: ' + opcodes.length); } - // if we're benchmarking we don't care about the opcount - if (!benchmark && state.opcount > 500_000) { + if (state.opcount > 500_000) { throw new Error('Too many instructions'); } state.opcount++; - ScriptRunner.executeInner(state, state.script.opcodes[++state.pc]); + + const opcode = opcodes[++state.pc]; + const handler = ScriptRunner.HANDLERS[opcode]; + if (!handler) { + throw new Error(`Unknown opcode ${opcode}`); + } + + handler(state); } - const time: number = (performance.now() * 1000 - start) | 0; - if (Environment.NODE_DEBUG_PROFILE && time > 1000) { - const message: string = `Warning [cpu time]: Script: ${state.script.name}, time: ${time}us, opcount: ${state.opcount}`; - if (state.self instanceof Player) { - state.self.wrappedMessageGame(message); - } else { - printWarning(message); + + if (Environment.NODE_DEBUG_PROFILE) { + const time: number = (performance.now() * 1000 - start) | 0; + if (time > 1000) { + const message: string = `Warning [cpu time]: Script: ${state.script.name}, time: ${time}us, opcount: ${state.opcount}`; + if (state.self instanceof Player) { + state.self.wrappedMessageGame(message); + } else { + printWarning(message); + } } } } catch (err: any) { @@ -173,27 +184,18 @@ export default class ScriptRunner { } if (state.self instanceof Player) { + printError(`Player script error - pid:${state.self.pid} name:${state.self.username}`); + state.self.wrappedMessageGame(`script error: ${err.message}`); state.self.wrappedMessageGame(`file: ${state.script.fileName}`); - state.self.wrappedMessageGame(''); state.self.wrappedMessageGame('stack backtrace:'); state.self.wrappedMessageGame(` 1: ${state.script.name} - ${state.script.fileName}:${state.script.lineNumber(state.pc)}`); let trace = 1; - for (let i = state.fp; i > 0; i--) { - const frame = state.frames[i]; - if (frame) { - trace++; - state.self.wrappedMessageGame(` ${trace}: ${frame.script.name} - ${frame.script.fileName}:${frame.script.lineNumber(frame.pc)}`); - } - } - for (let i = state.debugFp; i >= 0; i--) { + for (let i = state.debugFp - 1; i >= 0; i--) { const frame = state.debugFrames[i]; - if (frame) { - trace++; - state.self.wrappedMessageGame(` ${trace}: ${frame.script.name} - ${frame.script.fileName}:${frame.script.lineNumber(frame.pc)}`); - } + state.self.wrappedMessageGame(` ${++trace}: ${frame.script.name} - ${frame.script.fileName}:${frame.script.lineNumber(frame.pc)}`); } if (Environment.NODE_PRODUCTION) { @@ -201,6 +203,8 @@ export default class ScriptRunner { state.self.loggingOut = true; } } else if (state.self instanceof Npc) { + printError(`NPC script error - nid:${state.self.nid} type:${state.self.type}`); + if (Environment.NODE_PRODUCTION) { World.removeNpc(state.self, 0); } @@ -214,19 +218,9 @@ export default class ScriptRunner { console.error(` 1: ${state.script.name} - ${state.script.fileName}:${state.script.lineNumber(state.pc)}`); let trace = 1; - for (let i = state.fp; i > 0; i--) { - const frame = state.frames[i]; - if (frame) { - trace++; - console.error(` ${trace}: ${frame.script.name} - ${frame.script.fileName}:${frame.script.lineNumber(frame.pc)}`); - } - } - for (let i = state.debugFp; i >= 0; i--) { + for (let i = state.debugFp - 1; i >= 0; i--) { const frame = state.debugFrames[i]; - if (frame) { - trace++; - console.error(` ${trace}: ${frame.script.name} - ${frame.script.fileName}:${frame.script.lineNumber(frame.pc)}`); - } + console.error(` ${++trace}: ${frame.script.name} - ${frame.script.fileName}:${frame.script.lineNumber(frame.pc)}`); } state.execution = ScriptState.ABORTED; @@ -234,13 +228,4 @@ export default class ScriptRunner { return state.execution; } - - static executeInner(state: ScriptState, opcode: number) { - const handler = ScriptRunner.HANDLERS[opcode]; - if (!handler) { - throw new Error(`Unknown opcode ${opcode}`); - } - - handler(state); - } } diff --git a/src/engine/script/ScriptState.ts b/src/engine/script/ScriptState.ts index db3554161..57da54a66 100644 --- a/src/engine/script/ScriptState.ts +++ b/src/engine/script/ScriptState.ts @@ -17,8 +17,7 @@ export interface GosubStackFrame { stringLocals: string[]; } -// for debugging stack traces -export interface JumpStackFrame { +export interface DebugStackFrame { script: ScriptFile; pc: number; } @@ -45,7 +44,7 @@ export default class ScriptState { frames: GosubStackFrame[] = []; fp = 0; // frame pointer - debugFrames: JumpStackFrame[] = []; + debugFrames: DebugStackFrame[] = []; debugFp = 0; intStack: (number | null)[] = []; @@ -343,6 +342,8 @@ export default class ScriptState { } popFrame(): void { + this.debugFp--; + const frame = this.frames[--this.fp]; this.pc = frame.pc; this.script = frame.script; @@ -351,6 +352,11 @@ export default class ScriptState { } gosubFrame(proc: ScriptFile): void { + this.debugFrames[this.debugFp++] = { + script: this.script, + pc: this.pc + }; + this.frames[this.fp++] = { script: this.script, pc: this.pc, @@ -388,17 +394,4 @@ export default class ScriptState { this.intLocals = intLocals; this.stringLocals = stringLocals; } - - reset(): void { - this.pc = -1; - this.frames = []; - this.fp = 0; - this.intStack = []; - this.isp = 0; - this.stringStack = []; - this.ssp = 0; - this.intLocals = []; - this.stringLocals = []; - this.pointers = 0; - } } diff --git a/src/engine/script/ServerTriggerType.ts b/src/engine/script/ServerTriggerType.ts index bcaf781c5..1881fa80e 100644 --- a/src/engine/script/ServerTriggerType.ts +++ b/src/engine/script/ServerTriggerType.ts @@ -139,12 +139,12 @@ enum ServerTriggerType { IF_BUTTON = 147, IF_CLOSE = 148, - INV_BUTTON1 = 149, - INV_BUTTON2 = 150, - INV_BUTTON3 = 151, - INV_BUTTON4 = 152, - INV_BUTTON5 = 153, - INV_BUTTOND = 154, + IF_BUTTON1 = 149, + IF_BUTTON2 = 150, + IF_BUTTON3 = 151, + IF_BUTTON4 = 152, + IF_BUTTON5 = 153, + IF_BUTTOND = 154, WALKTRIGGER = 155, AI_WALKTRIGGER = 156, diff --git a/src/engine/script/handlers/InvOps.ts b/src/engine/script/handlers/InvOps.ts index c0ba20527..e16a39baa 100644 --- a/src/engine/script/handlers/InvOps.ts +++ b/src/engine/script/handlers/InvOps.ts @@ -447,7 +447,7 @@ const InvOps: CommandHandlers = { const fromItems = Array.from(fromLogs.values().map(item => (({ cost: _cost, ...event }) => event)(item))); // Log wealth events - if (fromInvType.debugname === 'stakeinv') { + if (fromInvType.debugname === 'dueloffer') { if (fromItems.length > 0) { fromPlayer.addWealthEvent({ event_type: WealthEventType.STAKE, diff --git a/src/engine/script/handlers/NpcOps.ts b/src/engine/script/handlers/NpcOps.ts index c200bc621..4bc2d55b5 100644 --- a/src/engine/script/handlers/NpcOps.ts +++ b/src/engine/script/handlers/NpcOps.ts @@ -11,6 +11,7 @@ import Loc from '#/engine/entity/Loc.js'; import Npc from '#/engine/entity/Npc.js'; import { NpcIteratorType } from '#/engine/entity/NpcIteratorType.js'; import { NpcMode } from '#/engine/entity/NpcMode.js'; +import { NpcStat } from '#/engine/entity/NpcStat.js'; import Obj from '#/engine/entity/Obj.js'; import { NpcIterator } from '#/engine/script/ScriptIterators.js'; import { ScriptOpcode } from '#/engine/script/ScriptOpcode.js'; @@ -249,6 +250,10 @@ const NpcOps: CommandHandlers = { const current = npc.levels[stat]; const healed = current + ((constant + (base * percent) / 100) | 0); npc.levels[stat] = Math.min(healed, base); + + if (stat === NpcStat.HITPOINTS && npc.levels[stat] >= npc.baseLevels[stat]) { + npc.heroPoints.clear(); + } }), [ScriptOpcode.NPC_TYPE]: checkedHandler(ActiveNpc, state => { diff --git a/src/engine/script/handlers/NumberOps.ts b/src/engine/script/handlers/NumberOps.ts index 195261d56..173599c4b 100644 --- a/src/engine/script/handlers/NumberOps.ts +++ b/src/engine/script/handlers/NumberOps.ts @@ -1,5 +1,6 @@ import { ScriptOpcode } from '#/engine/script/ScriptOpcode.js'; import { CommandHandlers } from '#/engine/script/ScriptRunner.js'; +import JavaRandom from '#/util/JavaRandom.js'; import { bitcount, clearBitRange, MASK, setBitRange } from '#/util/Numbers.js'; import Trig from 'src/util/Trig.js'; @@ -29,13 +30,13 @@ const NumberOps: CommandHandlers = { }, [ScriptOpcode.RANDOM]: state => { - const a = state.popInt(); - state.pushInt(Math.random() * a); + const n = state.popInt(); + state.pushInt(JavaRandom.nextDouble() * n); }, [ScriptOpcode.RANDOMINC]: state => { - const a = state.popInt(); - state.pushInt(Math.random() * (a + 1)); + const n = state.popInt(); + state.pushInt(JavaRandom.nextDouble() * (n + 1)); }, [ScriptOpcode.INTERPOLATE]: state => { diff --git a/src/engine/script/handlers/PlayerOps.ts b/src/engine/script/handlers/PlayerOps.ts index 6e9e272af..375a25dbf 100644 --- a/src/engine/script/handlers/PlayerOps.ts +++ b/src/engine/script/handlers/PlayerOps.ts @@ -40,6 +40,7 @@ import MinimapToggle from '#/network/game/server/model/MinimapToggle.js'; import SynthSound from '#/network/game/server/model/SynthSound.js'; import Environment from '#/util/Environment.js'; import SetPlayerOp from '#/network/game/server/model/SetPlayerOp.js'; +import JavaRandom from '#/util/JavaRandom.js'; const PlayerOps: CommandHandlers = { [ScriptOpcode.FINDUID]: state => { @@ -206,11 +207,11 @@ const PlayerOps: CommandHandlers = { ServerTriggerType.OPHELD5, ServerTriggerType.OPHELDU, ServerTriggerType.OPHELDT, - ServerTriggerType.INV_BUTTON1, - ServerTriggerType.INV_BUTTON2, - ServerTriggerType.INV_BUTTON3, - ServerTriggerType.INV_BUTTON4, - ServerTriggerType.INV_BUTTON5 + ServerTriggerType.IF_BUTTON1, + ServerTriggerType.IF_BUTTON2, + ServerTriggerType.IF_BUTTON3, + ServerTriggerType.IF_BUTTON4, + ServerTriggerType.IF_BUTTON5 ]; if (!allowedTriggers.includes(state.trigger)) { throw new Error('is not safe to use in this trigger'); @@ -228,12 +229,12 @@ const PlayerOps: CommandHandlers = { ServerTriggerType.OPHELD5, ServerTriggerType.OPHELDU, ServerTriggerType.OPHELDT, - ServerTriggerType.INV_BUTTON1, - ServerTriggerType.INV_BUTTON2, - ServerTriggerType.INV_BUTTON3, - ServerTriggerType.INV_BUTTON4, - ServerTriggerType.INV_BUTTON5, - ServerTriggerType.INV_BUTTOND + ServerTriggerType.IF_BUTTON1, + ServerTriggerType.IF_BUTTON2, + ServerTriggerType.IF_BUTTON3, + ServerTriggerType.IF_BUTTON4, + ServerTriggerType.IF_BUTTON5, + ServerTriggerType.IF_BUTTOND ]; if (!allowedTriggers.includes(state.trigger)) { throw new Error('is not safe to use in this trigger'); @@ -511,7 +512,7 @@ const PlayerOps: CommandHandlers = { const level = state.activePlayer.levels[stat]; const value = Math.floor((low * (99 - level)) / 98) + Math.floor((high * (level - 1)) / 98) + 1; - const chance = Math.floor(Math.random() * 256); + const chance = JavaRandom.nextDouble() * 256; state.pushInt(value > chance ? 1 : 0); }), @@ -771,36 +772,36 @@ const PlayerOps: CommandHandlers = { }, [ScriptOpcode.BAS_READYANIM]: state => { - state.activePlayer.basReadyAnim = check(state.popInt(), SeqTypeValid).id; + state.activePlayer.readyanim = check(state.popInt(), SeqTypeValid).id; }, [ScriptOpcode.BAS_TURNONSPOT]: state => { - state.activePlayer.basTurnOnSpot = check(state.popInt(), SeqTypeValid).id; + state.activePlayer.turnanim = check(state.popInt(), SeqTypeValid).id; }, [ScriptOpcode.BAS_WALK_F]: state => { - state.activePlayer.basWalkForward = check(state.popInt(), SeqTypeValid).id; + state.activePlayer.walkanim = check(state.popInt(), SeqTypeValid).id; }, [ScriptOpcode.BAS_WALK_B]: state => { - state.activePlayer.basWalkBackward = check(state.popInt(), SeqTypeValid).id; + state.activePlayer.walkanim_b = check(state.popInt(), SeqTypeValid).id; }, [ScriptOpcode.BAS_WALK_L]: state => { - state.activePlayer.basWalkLeft = check(state.popInt(), SeqTypeValid).id; + state.activePlayer.walkanim_l = check(state.popInt(), SeqTypeValid).id; }, [ScriptOpcode.BAS_WALK_R]: state => { - state.activePlayer.basWalkRight = check(state.popInt(), SeqTypeValid).id; + state.activePlayer.walkanim_r = check(state.popInt(), SeqTypeValid).id; }, [ScriptOpcode.BAS_RUNNING]: state => { const seq = state.popInt(); if (seq === -1) { - state.activePlayer.basRunning = -1; + state.activePlayer.runanim = -1; return; } - state.activePlayer.basRunning = check(seq, SeqTypeValid).id; + state.activePlayer.runanim = check(seq, SeqTypeValid).id; }, [ScriptOpcode.GENDER]: state => { @@ -865,7 +866,7 @@ const PlayerOps: CommandHandlers = { }, [ScriptOpcode.LAST_TARGETSLOT]: state => { - const allowedTriggers = [ServerTriggerType.INV_BUTTOND]; + const allowedTriggers = [ServerTriggerType.IF_BUTTOND]; if (!allowedTriggers.includes(state.trigger)) { throw new Error('is not safe to use in this trigger'); } diff --git a/src/engine/script/handlers/ServerOps.ts b/src/engine/script/handlers/ServerOps.ts index ae8d63217..adc3dca1a 100644 --- a/src/engine/script/handlers/ServerOps.ts +++ b/src/engine/script/handlers/ServerOps.ts @@ -428,8 +428,8 @@ const ServerOps: CommandHandlers = { for (let i = 0; i < 50; i++) { const distX = Math.floor(Math.random() * (2 * maxRadius + 1)) - maxRadius; const distZ = Math.floor(Math.random() * (2 * maxRadius + 1)) - maxRadius; - const distanceSquared = distX * distX + distZ * distZ; - if (distanceSquared < minRadius * minRadius || distanceSquared > maxRadius * maxRadius) { + const distance = Math.max(Math.abs(distX), Math.abs(distZ)); + if (distance < minRadius || distance > maxRadius) { continue; } const randomX = origin.x + distX; @@ -446,8 +446,8 @@ const ServerOps: CommandHandlers = { for (let i = 0; i < 50; i++) { const distX = Math.floor(Math.random() * (2 * maxRadius + 1)) - maxRadius; const distZ = Math.floor(Math.random() * (2 * maxRadius + 1)) - maxRadius; - const distanceSquared = distX * distX + distZ * distZ; - if (distanceSquared < minRadius * minRadius || distanceSquared > maxRadius * maxRadius) { + const distance = Math.max(Math.abs(distX), Math.abs(distZ)); + if (distance < minRadius || distance > maxRadius) { continue; } const randomX = origin.x + distX; @@ -464,8 +464,8 @@ const ServerOps: CommandHandlers = { for (let i = 0; i < 50; i++) { const distX = Math.floor(Math.random() * (2 * maxRadius + 1)) - maxRadius; const distZ = Math.floor(Math.random() * (2 * maxRadius + 1)) - maxRadius; - const distanceSquared = distX * distX + distZ * distZ; - if (distanceSquared < minRadius * minRadius || distanceSquared > maxRadius * maxRadius) { + const distance = Math.max(Math.abs(distX), Math.abs(distZ)); + if (distance < minRadius || distance > maxRadius) { continue; } const randomX = origin.x + distX; @@ -485,8 +485,8 @@ const ServerOps: CommandHandlers = { for (let x = origin.x - maxRadius; x <= origin.x + maxRadius; x++) { const distX = x - origin.x; const distZ = Math.floor(Math.random() * (2 * maxRadius + 1)) - maxRadius; - const distanceSquared = distX * distX + distZ * distZ; - if (distanceSquared < minRadius * minRadius || distanceSquared > maxRadius * maxRadius) { + const distance = Math.max(Math.abs(distX), Math.abs(distZ)); + if (distance < minRadius || distance > maxRadius) { continue; } const randomZ = origin.z + distZ; @@ -502,8 +502,8 @@ const ServerOps: CommandHandlers = { for (let x = origin.x - maxRadius; x <= origin.x + maxRadius; x++) { const distX = x - origin.x; const distZ = Math.floor(Math.random() * (2 * maxRadius + 1)) - maxRadius; - const distanceSquared = distX * distX + distZ * distZ; - if (distanceSquared < minRadius * minRadius || distanceSquared > maxRadius * maxRadius) { + const distance = Math.max(Math.abs(distX), Math.abs(distZ)); + if (distance < minRadius || distance > maxRadius) { continue; } const randomZ = origin.z + distZ; @@ -519,8 +519,8 @@ const ServerOps: CommandHandlers = { for (let x = origin.x - maxRadius; x <= origin.x + maxRadius; x++) { const distX = x - origin.x; const distZ = Math.floor(Math.random() * (2 * maxRadius + 1)) - maxRadius; - const distanceSquared = distX * distX + distZ * distZ; - if (distanceSquared < minRadius * minRadius || distanceSquared > maxRadius * maxRadius) { + const distance = Math.max(Math.abs(distX), Math.abs(distZ)); + if (distance < minRadius || distance > maxRadius) { continue; } const randomZ = origin.z + distZ; diff --git a/src/io/CompressionType.ts b/src/io/CompressionType.ts new file mode 100644 index 000000000..bcabd68e0 --- /dev/null +++ b/src/io/CompressionType.ts @@ -0,0 +1,5 @@ +export const enum CompressionType { + NONE, + BZIP2, + GZIP +} diff --git a/src/io/GZip.ts b/src/io/GZip.ts index 1516046fd..a4c93ad60 100644 --- a/src/io/GZip.ts +++ b/src/io/GZip.ts @@ -1,4 +1,7 @@ import zlib from 'zlib'; +import { deflateRaw } from 'pako'; + +import Packet from '#/io/Packet.js'; function compressGz( src: Uint8Array, @@ -6,11 +9,43 @@ function compressGz( len: number = src.length, ): Uint8Array | null { try { - const data = new Uint8Array( - zlib.gzipSync(src.subarray(off, off + len)), - ); - data[9] = 0; - return data; + const slice = src.subarray(off, off + len); + const deflated = deflateRaw(slice, { + level: 6, + memLevel: 8, + strategy: 0 + }); + + const out = new Uint8Array(10 + deflated.length + 8); + + out[0] = 0x1f; + out[1] = 0x8b; + out[2] = 0x08; // deflate + out[3] = 0x00; // FLG + out[4] = 0x00; // MTIME + out[5] = 0x00; + out[6] = 0x00; + out[7] = 0x00; + out[8] = 0x00; // XFL + out[9] = 0x00; // OS + + out.set(deflated, 10); + + const crc = Packet.getcrc(slice, 0, slice.length) >>> 0; + const isize = slice.length >>> 0; + const trailerPos = 10 + deflated.length; + + // Trailer: CRC32 + ISIZE (little-endian) + out[trailerPos] = crc & 0xff; + out[trailerPos + 1] = (crc >>> 8) & 0xff; + out[trailerPos + 2] = (crc >>> 16) & 0xff; + out[trailerPos + 3] = (crc >>> 24) & 0xff; + out[trailerPos + 4] = isize & 0xff; + out[trailerPos + 5] = (isize >>> 8) & 0xff; + out[trailerPos + 6] = (isize >>> 16) & 0xff; + out[trailerPos + 7] = (isize >>> 24) & 0xff; + + return out; } catch (err) { console.error(err); return null; diff --git a/src/io/Js5Archive.ts b/src/io/Js5Archive.ts new file mode 100644 index 000000000..d35103e26 --- /dev/null +++ b/src/io/Js5Archive.ts @@ -0,0 +1,45 @@ +import fs from 'fs'; +import Js5Reader from '#/io/Js5Reader.js'; + +/** + * Utility functions for loading JS5 archives from disk. + */ +export class Js5Archive { + /** + * Load a JS5 archive from a file path. + * @param filePath Path to the .js5 file + * @returns Map of groupId -> decompressed data + */ + static load(filePath: string): Map { + if (!fs.existsSync(filePath)) { + throw new Error(`JS5 archive not found: ${filePath}`); + } + + const data = fs.readFileSync(filePath); + const reader = new Js5Reader(new Uint8Array(data)); + return reader.read(); + } + + /** + * Load a JS5 archive asynchronously. + * @param filePath Path to the .js5 file + * @returns Map of groupId -> decompressed data + */ + static async loadAsync(filePath: string): Promise> { + if (!fs.existsSync(filePath)) { + throw new Error(`JS5 archive not found: ${filePath}`); + } + + const data = await fs.promises.readFile(filePath); + const reader = new Js5Reader(new Uint8Array(data)); + return reader.read(); + } + + /** + * Check if a JS5 archive exists. + * @param filePath Path to check + */ + static exists(filePath: string): boolean { + return fs.existsSync(filePath); + } +} diff --git a/src/io/Js5ArchiveIndex.ts b/src/io/Js5ArchiveIndex.ts new file mode 100644 index 000000000..4ae7b52d5 --- /dev/null +++ b/src/io/Js5ArchiveIndex.ts @@ -0,0 +1,146 @@ +import Packet from '#/io/Packet.js'; +import { unpackJs5Group } from '#/io/Js5Group.js'; + +export type Js5ArchiveIndex = { + groupIds: number[]; + fileIdsByGroup: Map; +}; + +export function parseJs5ArchiveIndexFromPack(js5Data: Uint8Array): Js5ArchiveIndex { + const indexLength = packedContainerLength(js5Data, 0); + const indexPacked = js5Data.slice(0, indexLength); + const indexUnpacked = unpackJs5Group(indexPacked); + return parseJs5ArchiveIndex(indexUnpacked); +} + +export function parseJs5ArchiveIndex(indexData: Uint8Array): Js5ArchiveIndex { + const packet = new Packet(indexData); + + const format = packet.g1(); + if (format >= 6) { + packet.g4s(); + } + + const flags = packet.g1(); + + const groupCount = readJs5Id(packet, format); + const groupIds: number[] = new Array(groupCount); + let previousGroupId = 0; + for (let i = 0; i < groupCount; i++) { + previousGroupId += readJs5Id(packet, format); + groupIds[i] = previousGroupId; + } + + if ((flags & 0x01) !== 0) { + for (let i = 0; i < groupCount; i++) { + packet.g4s(); + } + } + + for (let i = 0; i < groupCount; i++) { + packet.g4s(); + } + + for (let i = 0; i < groupCount; i++) { + packet.g4s(); + } + + const fileCounts = new Array(groupCount); + for (let i = 0; i < groupCount; i++) { + fileCounts[i] = readJs5Id(packet, format); + } + + const fileIdsByGroup = new Map(); + for (let i = 0; i < groupCount; i++) { + const fileCount = fileCounts[i]; + const fileIds: number[] = new Array(fileCount); + + let previousFileId = 0; + for (let j = 0; j < fileCount; j++) { + previousFileId += readJs5Id(packet, format); + fileIds[j] = previousFileId; + } + + fileIdsByGroup.set(groupIds[i], fileIds); + } + + return { groupIds, fileIdsByGroup }; +} + +export function splitGroupFiles(groupData: Uint8Array, fileIds: number[]): Map { + const files = new Map(); + + if (fileIds.length === 1) { + files.set(fileIds[0], groupData); + return files; + } + + const stripes = groupData[groupData.length - 1] & 0xff; + const fileCount = fileIds.length; + const tableLength = stripes * fileCount * 4; + const tableOffset = groupData.length - 1 - tableLength; + + if (tableOffset <= 0) { + throw new Error('Invalid JS5 group chunk table.'); + } + + const view = new DataView(groupData.buffer, groupData.byteOffset, groupData.byteLength); + const sizes = new Int32Array(fileCount); + + let tablePos = tableOffset; + for (let stripe = 0; stripe < stripes; stripe++) { + let chunkLength = 0; + for (let file = 0; file < fileCount; file++) { + chunkLength += view.getInt32(tablePos); + tablePos += 4; + sizes[file] += chunkLength; + } + } + + const outputs: Uint8Array[] = new Array(fileCount); + for (let file = 0; file < fileCount; file++) { + outputs[file] = new Uint8Array(sizes[file]); + } + + const offsets = new Int32Array(fileCount); + let dataPos = 0; + tablePos = tableOffset; + + for (let stripe = 0; stripe < stripes; stripe++) { + let chunkLength = 0; + for (let file = 0; file < fileCount; file++) { + chunkLength += view.getInt32(tablePos); + tablePos += 4; + + outputs[file].set(groupData.subarray(dataPos, dataPos + chunkLength), offsets[file]); + offsets[file] += chunkLength; + dataPos += chunkLength; + } + } + + for (let i = 0; i < fileCount; i++) { + files.set(fileIds[i], outputs[i]); + } + + return files; +} + +function readJs5Id(packet: Packet, format: number): number { + if (format >= 7) { + return packet.gSmart2or4(); + } + + return packet.g2(); +} + +function packedContainerLength(bytes: Uint8Array, offset: number): number { + const compression = bytes[offset]; + const compressedLength = + ((bytes[offset + 1] << 24) | (bytes[offset + 2] << 16) | (bytes[offset + 3] << 8) | bytes[offset + 4]) >>> 0; + + if (compression === 0) { + return 5 + compressedLength; + } + + return 9 + compressedLength; +} diff --git a/src/io/Js5Group.ts b/src/io/Js5Group.ts new file mode 100644 index 000000000..10a37abff --- /dev/null +++ b/src/io/Js5Group.ts @@ -0,0 +1,57 @@ +import BZip2 from '#/io/BZip2.js'; +import { CompressionType } from '#/io/CompressionType.js'; +import { decompressGz } from '#/io/GZip.js'; + +export function unpackJs5Group(bytes: Uint8Array): Uint8Array { + if (!isJs5GroupContainer(bytes)) { + return bytes; + } + + const type = bytes[0]; + const compressedLength = readU32BE(bytes, 1); + + if (type === CompressionType.NONE) { + return bytes.subarray(5, 5 + compressedLength); + } + + const uncompressedLength = readU32BE(bytes, 5); + const payload = bytes.subarray(9, 9 + compressedLength); + + if (type === CompressionType.BZIP2) { + return BZip2.decompress(payload, uncompressedLength, true); + } + + if (type === CompressionType.GZIP) { + const decompressed = decompressGz(payload); + if (!decompressed) { + throw new Error('Failed to decompress JS5 group (gzip).'); + } + return decompressed; + } + + throw new Error(`Unsupported JS5 compression type: ${type}`); +} + +export function isJs5GroupContainer(bytes: Uint8Array): boolean { + if (bytes.length < 5) { + return false; + } + + const type = bytes[0]; + if (type !== CompressionType.NONE && type !== CompressionType.BZIP2 && type !== CompressionType.GZIP) { + return false; + } + + const compressedLength = readU32BE(bytes, 1); + const headerLength = type === CompressionType.NONE ? 5 : 9; + + if (bytes.length < headerLength) { + return false; + } + + return bytes.length === headerLength + compressedLength; +} + +function readU32BE(bytes: Uint8Array, offset: number): number { + return ((bytes[offset] << 24) | (bytes[offset + 1] << 16) | (bytes[offset + 2] << 8) | bytes[offset + 3]) >>> 0; +} diff --git a/src/io/Js5PackReader.ts b/src/io/Js5PackReader.ts new file mode 100644 index 000000000..3fd0543d5 --- /dev/null +++ b/src/io/Js5PackReader.ts @@ -0,0 +1,169 @@ +import fs from 'fs'; + +import BZip2 from '#/io/BZip2.js'; +import { CompressionType } from '#/io/CompressionType.js'; +import Packet from '#/io/Packet.js'; +import { decompressGz } from '#/io/GZip.js'; + +type Js5IndexInfo = { + groupIds: number[]; +}; + +export default class Js5PackReader { + private groups = new Map(); + + static load(filePath: string): Js5PackReader { + if (!fs.existsSync(filePath)) { + throw new Error(`JS5 pack not found: ${filePath}`); + } + + const data = new Uint8Array(fs.readFileSync(filePath)); + return new Js5PackReader(data); + } + + constructor(data: Uint8Array) { + this.parse(data); + } + + getGroup(groupId: number): Uint8Array | undefined { + return this.groups.get(groupId); + } + + private parse(data: Uint8Array): void { + const indexLength = this.packedContainerLength(data, 0); + const indexPacked = data.slice(0, indexLength); + const indexUnpacked = this.unpackGroup(indexPacked); + const index = this.parseIndex(indexUnpacked); + + const lengthsTableBytes = index.groupIds.length * 4; + const lengthsTableStart = data.length - lengthsTableBytes; + const lengthsTable = data.slice(lengthsTableStart); + + let pos = indexLength; + for (let i = 0; i < index.groupIds.length; i++) { + const groupId = index.groupIds[i]; + const length = this.readInt32BE(lengthsTable, i * 4) >>> 0; + + if (length === 0) { + this.groups.set(groupId, new Uint8Array(0)); + continue; + } + + const end = pos + length; + if (end > lengthsTableStart) { + throw new Error(`JS5 pack group ${groupId} exceeds archive bounds.`); + } + + this.groups.set(groupId, data.slice(pos, end)); + pos = end; + } + } + + private parseIndex(data: Uint8Array): Js5IndexInfo { + const packet = new Packet(data); + + const format = packet.g1(); + if (format < 5 || format > 7) { + throw new Error(`Unsupported JS5 index format: ${format}`); + } + + if (format >= 6) { + packet.g4s(); + } + + const flags = packet.g1(); + const hasNames = (flags & 0x1) !== 0; + + const groupCount = this.readJs5Id(packet, format); + const groupIds: number[] = new Array(groupCount); + + let previousGroupId = 0; + for (let i = 0; i < groupCount; i++) { + previousGroupId += this.readJs5Id(packet, format); + groupIds[i] = previousGroupId; + } + + if (hasNames) { + for (let i = 0; i < groupCount; i++) { + packet.g4s(); + } + } + + for (let i = 0; i < groupCount; i++) { + packet.g4s(); + } + + for (let i = 0; i < groupCount; i++) { + packet.g4s(); + } + + const fileCounts: number[] = new Array(groupCount); + for (let i = 0; i < groupCount; i++) { + fileCounts[i] = this.readJs5Id(packet, format); + } + + for (let i = 0; i < groupCount; i++) { + const count = fileCounts[i]; + let _previousFileId = 0; + for (let j = 0; j < count; j++) { + _previousFileId += this.readJs5Id(packet, format); + } + } + + return { groupIds }; + } + + private readJs5Id(packet: Packet, format: number): number { + if (format >= 7) { + return packet.gSmart2or4(); + } + + return packet.g2(); + } + + private unpackGroup(bytes: Uint8Array): Uint8Array { + const type = bytes[0]; + const compressedLength = this.readU32BE(bytes, 1); + + if (type === CompressionType.NONE) { + return bytes.subarray(5, 5 + compressedLength); + } + + const uncompressedLength = this.readU32BE(bytes, 5); + const payload = bytes.subarray(9, 9 + compressedLength); + + if (type === CompressionType.BZIP2) { + return BZip2.decompress(payload, uncompressedLength, true); + } + + if (type === CompressionType.GZIP) { + const decompressed = decompressGz(payload); + if (!decompressed) { + throw new Error('Failed to decompress JS5 pack group (gzip).'); + } + return decompressed; + } + + throw new Error(`Unsupported JS5 compression type: ${type}`); + } + + private packedContainerLength(bytes: Uint8Array, offset: number): number { + const compression = bytes[offset]; + const compressedLength = + ((bytes[offset + 1] << 24) | (bytes[offset + 2] << 16) | (bytes[offset + 3] << 8) | bytes[offset + 4]) >>> 0; + + if (compression === CompressionType.NONE) { + return 5 + compressedLength; + } + + return 9 + compressedLength; + } + + private readU32BE(bytes: Uint8Array, offset: number): number { + return ((bytes[offset] << 24) | (bytes[offset + 1] << 16) | (bytes[offset + 2] << 8) | bytes[offset + 3]) >>> 0; + } + + private readInt32BE(bytes: Uint8Array, offset: number): number { + return ((bytes[offset] << 24) | (bytes[offset + 1] << 16) | (bytes[offset + 2] << 8) | bytes[offset + 3]) | 0; + } +} diff --git a/src/io/Js5Reader.ts b/src/io/Js5Reader.ts new file mode 100644 index 000000000..81f426196 --- /dev/null +++ b/src/io/Js5Reader.ts @@ -0,0 +1,131 @@ +import zlib from 'zlib'; +import BZip2 from '#/io/BZip2.js'; +import Packet from '#/io/Packet.js'; +import { CompressionType } from '#/io/CompressionType.js'; + +/** + * Generic JS5 archive reader for unpacking cache files. + * + * JS5 Format: + * - Packed index group (contains metadata about all groups) + * - Packed data groups in sequence + * - Optional trailing length table + */ +export default class Js5Reader { + private data: Uint8Array; + private pos: number = 0; + + constructor(data: Uint8Array) { + this.data = data; + } + + /** + * Parse the JS5 archive and extract all groups. + * @returns Map of groupId -> decompressed group data + */ + read(): Map { + const groups = new Map(); + + const indexData = this.unpackGroup(); + const index = this.parseIndex(indexData); + + // Read all group data in order + for (let i = 0; i < index.groupIds.length; i++) { + const groupData = this.unpackGroup(); + groups.set(index.groupIds[i], groupData); + } + + return groups; + } + + /** + * Parse the index group to extract group IDs and metadata. + */ + private parseIndex(data: Uint8Array): { groupIds: number[] } { + const packet = new Packet(data); + + const format = packet.g1(); + if (format !== 7) { + throw new Error(`Unsupported JS5 index format: ${format}`); + } + + const version = packet.g4s(); + if (version < 0) { + throw new Error(`Invalid JS5 index version: ${version}`); + } + + const _flags = packet.g1(); + const groupCount = packet.gSmart2or4(); + + // Read group IDs with delta encoding + const groupIds: number[] = []; + let previousGroupId = 0; + for (let i = 0; i < groupCount; i++) { + const delta = packet.gSmart2or4(); + const groupId = previousGroupId + delta; + groupIds.push(groupId); + previousGroupId = groupId; + } + + return { groupIds }; + } + + /** + * Read and unpack a single group from the archive. + * Handles compression types: 0 (none), 1 (bzip2), 2 (gzip) + */ + private unpackGroup(): Uint8Array { + if (this.pos >= this.data.length) { + throw new Error('Unexpected end of JS5 archive while reading group'); + } + + const compression = this.data[this.pos++]; + + if (compression === CompressionType.NONE) { + const length = Buffer.from(this.data).readUInt32BE(this.pos); + this.pos += 4; + const data = this.data.slice(this.pos, this.pos + length); + this.pos += length; + return data; + } else if (compression === CompressionType.BZIP2) { + const compressedLength = Buffer.from(this.data).readUInt32BE(this.pos); + this.pos += 4; + const uncompressedLength = Buffer.from(this.data).readUInt32BE(this.pos); + this.pos += 4; + const compressedData = this.data.slice(this.pos, this.pos + compressedLength); + this.pos += compressedLength; + + const decompressed = BZip2.decompress(compressedData, uncompressedLength, true); + return decompressed; + } else if (compression === CompressionType.GZIP) { + const compressedLength = Buffer.from(this.data).readUInt32BE(this.pos); + this.pos += 4; + const _uncompressedLength = Buffer.from(this.data).readUInt32BE(this.pos); + this.pos += 4; + const compressedData = this.data.slice(this.pos, this.pos + compressedLength); + this.pos += compressedLength; + + return new Uint8Array(zlib.gunzipSync(compressedData)); + } else { + // Try to detect and handle unknown compression types + this.pos--; + + // Check for gzip magic bytes (0x1f 0x8b) + if (this.pos + 1 < this.data.length && + this.data[this.pos] === 0x1f && + this.data[this.pos + 1] === 0x8b) { + // This looks like raw gzip data without length prefix + const remaining = Buffer.from(this.data.slice(this.pos)); + try { + const decompressed = zlib.gunzipSync(remaining); + this.pos = this.data.length; + return decompressed; + } catch { + throw new Error(`Unable to decompress gzip data at position ${this.pos}`); + } + } + + throw new Error(`Unsupported compression type: ${compression} at position ${this.pos - 1}`); + } + } +} diff --git a/src/io/Packet.ts b/src/io/Packet.ts index 2143ef7ed..7fd077e24 100644 --- a/src/io/Packet.ts +++ b/src/io/Packet.ts @@ -283,6 +283,17 @@ export default class Packet extends DoublyLinkable { return this.view.getUint8(this.pos) < 0x80 ? this.g1() : this.g2() - 0x8000; } + gSmart2or4(): number { + const byte1 = this.view.getUint8(this.pos); + if ((byte1 & 0x80) === 0) { + // 2-byte value + return this.g2(); + } else { + // 4-byte value with high bit set + return this.g4() & 0x7fffffff; + } + } + gdata(dest: Uint8Array, offset: number, length: number): void { dest.set(this.data.subarray(this.pos, this.pos + length), offset); this.pos += length; diff --git a/src/network/game/client/ClientGameProt.ts b/src/network/game/client/ClientGameProt.ts index 0c5e22fa3..82d77010a 100644 --- a/src/network/game/client/ClientGameProt.ts +++ b/src/network/game/client/ClientGameProt.ts @@ -1,108 +1,106 @@ export default class ClientGameProt { - static all: ClientGameProt[] = []; static byId: ClientGameProt[] = []; - static readonly NO_TIMEOUT = new ClientGameProt(6, 9000, 0); // NXT naming + static readonly NO_TIMEOUT = new ClientGameProt(93, 0); // NXT naming - static readonly IDLE_TIMER = new ClientGameProt(30, 9002, 0); - static readonly EVENT_MOUSE_CLICK = new ClientGameProt(31, 9003, 4); // NXT naming - static readonly EVENT_MOUSE_MOVE = new ClientGameProt(32, 9004, -1); // NXT naming - static readonly EVENT_APPLET_FOCUS = new ClientGameProt(33, 9005, 1); // NXT naming - static readonly EVENT_CAMERA_POSITION = new ClientGameProt(35, 9006, 4); // NXT naming - static readonly EVENT_HAS_WINDOW = new ClientGameProt(36, 9007, 4); - static readonly EVENT_SYNTH_ERROR = new ClientGameProt(37, 9008, 2); + static readonly IDLE_TIMER = new ClientGameProt(245, 0); + static readonly EVENT_MOUSE_CLICK = new ClientGameProt(75, 4); // NXT naming + static readonly EVENT_MOUSE_MOVE = new ClientGameProt(123, -1); // NXT naming + static readonly EVENT_APPLET_FOCUS = new ClientGameProt(22, 1); // NXT naming + static readonly EVENT_CAMERA_POSITION = new ClientGameProt(21, 4); // NXT naming + static readonly EVENT_HAS_WINDOW = new ClientGameProt(9007, 4); + static readonly EVENT_SYNTH_ERROR = new ClientGameProt(9008, 2); - // autogenerated as part of obfuscation process - static readonly ANTICHEAT_OPLOGIC1 = new ClientGameProt(60, 9009, 4); - static readonly ANTICHEAT_OPLOGIC2 = new ClientGameProt(61, 9010, 1); - static readonly ANTICHEAT_OPLOGIC3 = new ClientGameProt(62, 9011, 4); - static readonly ANTICHEAT_OPLOGIC4 = new ClientGameProt(63, 9012, 3); - static readonly ANTICHEAT_OPLOGIC5 = new ClientGameProt(64, 9013, 1); + static readonly OPOBJ1 = new ClientGameProt(156, 8); // NXT naming + static readonly OPOBJ2 = new ClientGameProt(55, 6); // NXT naming + static readonly OPOBJ3 = new ClientGameProt(153, 8); // NXT naming + static readonly OPOBJ4 = new ClientGameProt(161, 8); // NXT naming + static readonly OPOBJ5 = new ClientGameProt(135, 8); // NXT naming + static readonly OPOBJE = new ClientGameProt(92, 2); // Naming from OS1 (Blame Pazaz) + static readonly OPOBJT = new ClientGameProt(131, 8); // NXT naming + static readonly OPOBJU = new ClientGameProt(9026, 12); // NXT naming - // autogenerated as part of obfuscation process - static readonly ANTICHEAT_CYCLELOGIC1 = new ClientGameProt(70, 9014, -1); - static readonly ANTICHEAT_CYCLELOGIC2 = new ClientGameProt(71, 9015, 0); - static readonly ANTICHEAT_CYCLELOGIC3 = new ClientGameProt(72, 9016, 4); - static readonly ANTICHEAT_CYCLELOGIC4 = new ClientGameProt(73, 9017, 3); - static readonly ANTICHEAT_CYCLELOGIC5 = new ClientGameProt(74, 9018, 0); - static readonly ANTICHEAT_CYCLELOGIC6 = new ClientGameProt(75, 9019, 2); + static readonly OPNPC1 = new ClientGameProt(78, 2); // NXT naming + static readonly OPNPC2 = new ClientGameProt(3, 2); // NXT naming + static readonly OPNPC3 = new ClientGameProt(148, 2); // NXT naming + static readonly OPNPC4 = new ClientGameProt(30, 2); // NXT naming + static readonly OPNPC5 = new ClientGameProt(218, 2); // NXT naming + static readonly OPNPCE = new ClientGameProt(72, 2); // Naming from OS1 (Blame Pazaz) + static readonly OPNPCT = new ClientGameProt(9032, 4); // NXT naming + static readonly OPNPCU = new ClientGameProt(9033, 8); // NXT naming - static readonly OPOBJ1 = new ClientGameProt(80, 9020, 6); // NXT naming - static readonly OPOBJ2 = new ClientGameProt(81, 9021, 6); // NXT naming - static readonly OPOBJ3 = new ClientGameProt(82, 9022, 6); // NXT naming - static readonly OPOBJ4 = new ClientGameProt(83, 9023, 6); // NXT naming - static readonly OPOBJ5 = new ClientGameProt(84, 9024, 6); // NXT naming - static readonly OPOBJT = new ClientGameProt(88, 9025, 8); // NXT naming - static readonly OPOBJU = new ClientGameProt(89, 9026, 12); // NXT naming + static readonly OPLOC1 = new ClientGameProt(254, 6); // NXT naming + static readonly OPLOC2 = new ClientGameProt(194, 6); // NXT naming + static readonly OPLOC3 = new ClientGameProt(84, 6); // NXT naming + static readonly OPLOC4 = new ClientGameProt(247, 6); // NXT naming + static readonly OPLOC5 = new ClientGameProt(170, 6); // NXT naming + static readonly OPLOCE = new ClientGameProt(94, 2); // Naming from OS1 (Blame Pazaz) + static readonly OPLOCT = new ClientGameProt(9039, 8); // NXT naming + static readonly OPLOCU = new ClientGameProt(9040, 12); // NXT naming - static readonly OPNPC1 = new ClientGameProt(100, 9027, 2); // NXT naming - static readonly OPNPC2 = new ClientGameProt(101, 9028, 2); // NXT naming - static readonly OPNPC3 = new ClientGameProt(102, 9029, 2); // NXT naming - static readonly OPNPC4 = new ClientGameProt(103, 9030, 2); // NXT naming - static readonly OPNPC5 = new ClientGameProt(104, 9031, 2); // NXT naming - static readonly OPNPCT = new ClientGameProt(108, 9032, 4); // NXT naming - static readonly OPNPCU = new ClientGameProt(109, 9033, 8); // NXT naming + static readonly OPPLAYER1 = new ClientGameProt(68, 2); // NXT naming + static readonly OPPLAYER2 = new ClientGameProt(71, 2); // NXT naming + static readonly OPPLAYER3 = new ClientGameProt(180, 2); // NXT naming + static readonly OPPLAYER4 = new ClientGameProt(114, 2); // NXT naming + static readonly OPPLAYER5 = new ClientGameProt(175, 2); // NXT naming + static readonly OPPLAYERT = new ClientGameProt(9046, 4); // NXT naming + static readonly OPPLAYERU = new ClientGameProt(9047, 8); // NXT naming - static readonly OPLOC1 = new ClientGameProt(120, 9034, 6); // NXT naming - static readonly OPLOC2 = new ClientGameProt(121, 9035, 6); // NXT naming - static readonly OPLOC3 = new ClientGameProt(122, 9036, 6); // NXT naming - static readonly OPLOC4 = new ClientGameProt(123, 9037, 6); // NXT naming - static readonly OPLOC5 = new ClientGameProt(124, 9038, 6); // NXT naming - static readonly OPLOCT = new ClientGameProt(128, 9039, 8); // NXT naming - static readonly OPLOCU = new ClientGameProt(129, 9040, 12); // NXT naming + static readonly OPHELD1 = new ClientGameProt(9048, 6); // name based on runescript trigger + static readonly OPHELD2 = new ClientGameProt(9049, 6); // name based on runescript trigger + static readonly OPHELD3 = new ClientGameProt(9050, 6); // name based on runescript trigger + static readonly OPHELD4 = new ClientGameProt(9051, 6); // name based on runescript trigger + static readonly OPHELD5 = new ClientGameProt(9052, 6); // name based on runescript trigger + static readonly OPHELDT = new ClientGameProt(9053, 8); // name based on runescript trigger + static readonly OPHELDU = new ClientGameProt(9054, 12); // name based on runescript trigger - static readonly OPPLAYER1 = new ClientGameProt(140, 9041, 2); // NXT naming - static readonly OPPLAYER2 = new ClientGameProt(141, 9042, 2); // NXT naming - static readonly OPPLAYER3 = new ClientGameProt(142, 9043, 2); // NXT naming - static readonly OPPLAYER4 = new ClientGameProt(143, 9044, 2); // NXT naming - static readonly OPPLAYER5 = new ClientGameProt(144, 9045, 2); // NXT naming - static readonly OPPLAYERT = new ClientGameProt(148, 9046, 4); // NXT naming - static readonly OPPLAYERU = new ClientGameProt(149, 9047, 8); // NXT naming + static readonly IF_BUTTON1 = new ClientGameProt(155, 6); // NXT naming + static readonly IF_BUTTON2 = new ClientGameProt(196, 6); // NXT naming + static readonly IF_BUTTON3 = new ClientGameProt(124, 6); // NXT naming + static readonly IF_BUTTON4 = new ClientGameProt(199, 6); // NXT naming + static readonly IF_BUTTON5 = new ClientGameProt(234, 6); // NXT naming + static readonly IF_BUTTON6 = new ClientGameProt(168, 6); // NXT naming + static readonly IF_BUTTON7 = new ClientGameProt(166, 6); // NXT naming + static readonly IF_BUTTON8 = new ClientGameProt(64, 6); // NXT naming + static readonly IF_BUTTON9 = new ClientGameProt(53, 6); // NXT naming + static readonly IF_BUTTON10 = new ClientGameProt(9, 6); // NXT naming - static readonly OPHELD1 = new ClientGameProt(160, 9048, 6); // name based on runescript trigger - static readonly OPHELD2 = new ClientGameProt(161, 9049, 6); // name based on runescript trigger - static readonly OPHELD3 = new ClientGameProt(162, 9050, 6); // name based on runescript trigger - static readonly OPHELD4 = new ClientGameProt(163, 9051, 6); // name based on runescript trigger - static readonly OPHELD5 = new ClientGameProt(164, 9052, 6); // name based on runescript trigger - static readonly OPHELDT = new ClientGameProt(168, 9053, 8); // name based on runescript trigger - static readonly OPHELDU = new ClientGameProt(169, 9054, 12); // name based on runescript trigger + static readonly IF_BUTTON = new ClientGameProt(79, 12); // NXT naming + static readonly RESUME_PAUSEBUTTON = new ClientGameProt(9061, 2); // NXT naming + static readonly CLOSE_MODAL = new ClientGameProt(184, 0); // NXT naming + static readonly RESUME_P_COUNTDIALOG = new ClientGameProt(23, 4); // NXT naming + static readonly TUTORIAL_CLICKSIDE = new ClientGameProt(9064, 1); // no original name + static readonly RESUME_P_NAMEDIALOG = new ClientGameProt(9065, 8); // NXT naming - static readonly INV_BUTTON1 = new ClientGameProt(190, 9055, 6); // NXT has "IF_BUTTON1" but for our interface system, this makes more sense - static readonly INV_BUTTON2 = new ClientGameProt(191, 9056, 6); // NXT has "IF_BUTTON2" but for our interface system, this makes more sense - static readonly INV_BUTTON3 = new ClientGameProt(192, 9057, 6); // NXT has "IF_BUTTON3" but for our interface system, this makes more sense - static readonly INV_BUTTON4 = new ClientGameProt(193, 9058, 6); // NXT has "IF_BUTTON4" but for our interface system, this makes more sense - static readonly INV_BUTTON5 = new ClientGameProt(194, 9059, 6); // NXT has "IF_BUTTON5" but for our interface system, this makes more sense + static readonly MAP_BUILD_COMPLETE = new ClientGameProt(110, 0); // NXT naming + static readonly MOVE_OPCLICK = new ClientGameProt(77, -1); // (comes with OP packets, name based on other MOVE packets) // MOVE_SCRIPTED by 530? + static readonly REPORT_ABUSE = new ClientGameProt(99, 10); // NXT calls it 'BUG_REPORT' - unsure when it was named as such, might be more appropriate here. + static readonly MOVE_MINIMAPCLICK = new ClientGameProt(39, -1); // NXT naming + static readonly IF_BUTTOND = new ClientGameProt(9070, 7); // NXT naming + static readonly IGNORELIST_DEL = new ClientGameProt(213, 8); // NXT naming + static readonly IGNORELIST_ADD = new ClientGameProt(34, 8); // NXT naming + static readonly IF_PLAYERDESIGN = new ClientGameProt(9073, 13); + static readonly CHAT_SETMODE = new ClientGameProt(9074, 3); // NXT naming + static readonly MESSAGE_PRIVATE = new ClientGameProt(201, -1); // NXT naming + static readonly FRIENDLIST_DEL = new ClientGameProt(57, 8); // NXT naming + static readonly FRIENDLIST_ADD = new ClientGameProt(120, 8); // NXT naming + static readonly CLIENT_CHEAT = new ClientGameProt(44, -1); // NXT naming + static readonly MESSAGE_PUBLIC = new ClientGameProt(237, -1); // NXT naming + static readonly MOVE_GAMECLICK = new ClientGameProt(215, -1); // NXT naming - static readonly IF_BUTTON = new ClientGameProt(200, 79, 9060); // NXT naming - static readonly RESUME_PAUSEBUTTON = new ClientGameProt(201, 9061, 2); // NXT naming - static readonly CLOSE_MODAL = new ClientGameProt(202, 9062, 0); // NXT naming - static readonly RESUME_P_COUNTDIALOG = new ClientGameProt(203, 9063, 4); // NXT naming - static readonly TUTORIAL_CLICKSIDE = new ClientGameProt(204, 9064, 1); // no original name - static readonly RESUME_P_NAMEDIALOG = new ClientGameProt(205, 9065, 8); // NXT naming + static readonly CLAN_JOINCHAT_LEAVECHAT = new ClientGameProt(104, 8); // NXT naming + static readonly CLAN_KICKUSER = new ClientGameProt(162, 8); // NXT naming + static readonly FRIEND_SETRANK = new ClientGameProt(188, 9); // NXT naming - static readonly MAP_BUILD_COMPLETE = new ClientGameProt(241, 9066, 0); // NXT naming - static readonly MOVE_OPCLICK = new ClientGameProt(242, 254, -1); // (comes with OP packets, name based on other MOVE packets) - static readonly REPORT_ABUSE = new ClientGameProt(243, 9068, 10); - static readonly MOVE_MINIMAPCLICK = new ClientGameProt(244, 39, -1); // NXT naming - static readonly INV_BUTTOND = new ClientGameProt(245, 9070, 7); // NXT has "IF_BUTTOND" but for our interface system, this makes more sense - static readonly IGNORELIST_DEL = new ClientGameProt(246, 9071, 8); // NXT naming - static readonly IGNORELIST_ADD = new ClientGameProt(247, 9072, 8); // NXT naming - static readonly IF_PLAYERDESIGN = new ClientGameProt(248, 9073, 13); - static readonly CHAT_SETMODE = new ClientGameProt(249, 9074, 3); // NXT naming - static readonly MESSAGE_PRIVATE = new ClientGameProt(250, 9075, -1); // NXT naming - static readonly FRIENDLIST_DEL = new ClientGameProt(251, 9076, 8); // NXT naming - static readonly FRIENDLIST_ADD = new ClientGameProt(252, 9077, 8); // NXT naming - static readonly CLIENT_CHEAT = new ClientGameProt(253, 44, -1); // NXT naming - static readonly MESSAGE_PUBLIC = new ClientGameProt(254, 9079, -1); // NXT naming - static readonly MOVE_GAMECLICK = new ClientGameProt(255, 215, -1); // NXT naming + static readonly WINDOW_STATUS = new ClientGameProt(243, 6); // NXT naming + static readonly DETECT_MODIFIED_CLIENT = new ClientGameProt(20, 4); // NXT naming + static readonly TRANSMITVAR_VERIFYID = new ClientGameProt(177, 2); // NXT naming + static readonly SOUND_SONGEND = new ClientGameProt(137, 4); // NXT naming - // in these old revisions we can actually get the packet index from a leftover array in the client source constructor( - readonly index: number, readonly id: number, readonly length: number ) { - ClientGameProt.all[index] = this; ClientGameProt.byId[id] = this; } } diff --git a/src/network/game/client/ClientGameProtRepository.ts b/src/network/game/client/ClientGameProtRepository.ts index fc4bd3c8a..d270f4d37 100644 --- a/src/network/game/client/ClientGameProtRepository.ts +++ b/src/network/game/client/ClientGameProtRepository.ts @@ -6,6 +6,32 @@ import MoveClickDecoder from '#/network/game/client/codec/MoveClickDecoder.js'; import MoveClickHandler from '#/network/game/client/handler/MoveClickHandler.js'; import ClientCheatDecoder from '#/network/game/client/codec/ClientCheatDecoder.js'; import ClientCheatHandler from '#/network/game/client/handler/ClientCheatHandler.js'; +import IgnoreListAddDecoder from '#/network/game/client/codec/IgnoreListAddDecoder.js'; +import IgnoreListAddHandler from '#/network/game/client/handler/IgnoreListAddHandler.js'; +import IgnoreListDelDecoder from '#/network/game/client/codec/IgnoreListDelDecoder.js'; +import IgnoreListDelHandler from '#/network/game/client/handler/IgnoreListDelHandler.js'; +import FriendListAddDecoder from '#/network/game/client/codec/FriendListAddDecoder.js'; +import FriendListAddHandler from '#/network/game/client/handler/FriendListAddHandler.js'; +import FriendListDelDecoder from '#/network/game/client/codec/FriendListDelDecoder.js'; +import FriendListDelHandler from '#/network/game/client/handler/FriendListDelHandler.js'; +import ReportAbuseDecoder from '#/network/game/client/codec/ReportAbuseDecoder.js'; +import ReportAbuseHandler from '#/network/game/client/handler/ReportAbuseHandler.js'; +import ClanJoinLeaveChatDecoder from '#/network/game/client/codec/ClanJoinLeaveChatDecoder.js'; +import ClanJoinLeaveChatHandler from '#/network/game/client/handler/ClanJoinLeaveChatHandler.js'; +import ClanKickUserDecoder from '#/network/game/client/codec/ClanKickUserDecoder.js'; +import ClanKickUserHandler from '#/network/game/client/handler/ClanKickUserHandler.js'; +import FriendSetRankDecoder from '#/network/game/client/codec/FriendSetRankDecoder.js'; +import FriendSetRankHandler from '#/network/game/client/handler/FriendSetRankHandler.js'; +import WindowStatusDecoder from '#/network/game/client/codec/WindowStatusDecoder.js'; +import WindowStatusHandler from '#/network/game/client/handler/WindowStatusHandler.js'; +import OpLocDecoder from '#/network/game/client/codec/OpLocDecoder.js'; +import OpLocHandler from '#/network/game/client/handler/OpLocHandler.js'; +import OpObjDecoder from '#/network/game/client/codec/OpObjDecoder.js'; +import OpObjHandler from '#/network/game/client/handler/OpObjHandler.js'; +import OpObjEDecoder from '#/network/game/client/codec/OpObjEDecoder.js'; +import OpObjEHandler from '#/network/game/client/handler/OpObjEHandler.js'; +import OpLocEDecoder from '#/network/game/client/codec/OpLocEDecoder.js'; +import OpLocEHandler from '#/network/game/client/handler/OpLocEHandler.js'; class ClientGameProtRepository { decoders: Map> = new Map(); @@ -32,10 +58,39 @@ class ClientGameProtRepository { } constructor() { + this.bind(new IgnoreListAddDecoder, new IgnoreListAddHandler()); + this.bind(new IgnoreListDelDecoder, new IgnoreListDelHandler()); + + this.bind(new FriendListAddDecoder, new FriendListAddHandler()); + this.bind(new FriendListDelDecoder, new FriendListDelHandler()); + this.bind(new FriendSetRankDecoder, new FriendSetRankHandler()); + + this.bind(new ReportAbuseDecoder, new ReportAbuseHandler()); + this.bind(new ClientCheatDecoder(), new ClientCheatHandler()); this.bind(new MoveClickDecoder(ClientGameProt.MOVE_GAMECLICK), new MoveClickHandler()); this.bind(new MoveClickDecoder(ClientGameProt.MOVE_OPCLICK), new MoveClickHandler()); this.bind(new MoveClickDecoder(ClientGameProt.MOVE_MINIMAPCLICK), new MoveClickHandler()); + + this.bind(new ClanJoinLeaveChatDecoder(), new ClanJoinLeaveChatHandler()); + this.bind(new ClanKickUserDecoder(), new ClanKickUserHandler()); + + this.bind(new OpLocDecoder(ClientGameProt.OPLOC1, 1), new OpLocHandler()); + this.bind(new OpLocDecoder(ClientGameProt.OPLOC2, 2), new OpLocHandler()); + this.bind(new OpLocDecoder(ClientGameProt.OPLOC3, 3), new OpLocHandler()); + this.bind(new OpLocDecoder(ClientGameProt.OPLOC4, 4), new OpLocHandler()); + this.bind(new OpLocDecoder(ClientGameProt.OPLOC5, 5), new OpLocHandler()); + this.bind(new OpLocEDecoder(), new OpLocEHandler()); + + this.bind(new OpObjDecoder(ClientGameProt.OPOBJ1, 1), new OpObjHandler()); + this.bind(new OpObjDecoder(ClientGameProt.OPOBJ2, 2), new OpObjHandler()); + this.bind(new OpObjDecoder(ClientGameProt.OPOBJ3, 3), new OpObjHandler()); + this.bind(new OpObjDecoder(ClientGameProt.OPOBJ4, 4), new OpObjHandler()); + this.bind(new OpObjDecoder(ClientGameProt.OPOBJ5, 5), new OpObjHandler()); + this.bind(new OpObjEDecoder(), new OpObjEHandler()); + + + this.bind(new WindowStatusDecoder(), new WindowStatusHandler()); } } diff --git a/src/network/game/client/codec/ClanJoinLeaveChatDecoder.ts b/src/network/game/client/codec/ClanJoinLeaveChatDecoder.ts new file mode 100644 index 000000000..cf70cc365 --- /dev/null +++ b/src/network/game/client/codec/ClanJoinLeaveChatDecoder.ts @@ -0,0 +1,13 @@ +import Packet from '#/io/Packet.js'; +import ClientGameMessageDecoder from '#/network/game/client/ClientGameMessageDecoder.js'; +import ClientGameProt from '#/network/game/client/ClientGameProt.js'; +import ClanJoinLeaveChat from '#/network/game/client/model/ClanJoinLeaveChat.js'; + +export default class ClanJoinLeaveChatDecoder extends ClientGameMessageDecoder { + prot = ClientGameProt.CLAN_JOINCHAT_LEAVECHAT; + + decode(buf: Packet) { + const clanname = buf.g8(); + return new ClanJoinLeaveChat(clanname); + } +} diff --git a/src/network/game/client/codec/ClanKickUserDecoder.ts b/src/network/game/client/codec/ClanKickUserDecoder.ts new file mode 100644 index 000000000..222c4e458 --- /dev/null +++ b/src/network/game/client/codec/ClanKickUserDecoder.ts @@ -0,0 +1,14 @@ +import Packet from '#/io/Packet.js'; +import ClientGameMessageDecoder from '#/network/game/client/ClientGameMessageDecoder.js'; +import ClientGameProt from '#/network/game/client/ClientGameProt.js'; +import ClanKickUser from '#/network/game/client/model/ClanKickUser.js'; + + +export default class ClanKickUserDecoder extends ClientGameMessageDecoder { + prot = ClientGameProt.CLAN_KICKUSER; + + decode(buf: Packet): ClanKickUser { + const username = buf.g8(); + return new ClanKickUser(username); + } +} diff --git a/src/network/game/client/codec/DetectModifiedClientDecoder.ts b/src/network/game/client/codec/DetectModifiedClientDecoder.ts new file mode 100644 index 000000000..7dde9ff0c --- /dev/null +++ b/src/network/game/client/codec/DetectModifiedClientDecoder.ts @@ -0,0 +1,13 @@ +import Packet from '#/io/Packet.js'; +import ClientGameMessageDecoder from '#/network/game/client/ClientGameMessageDecoder.js'; +import ClientGameProt from '#/network/game/client/ClientGameProt.js'; +import DetectModifiedClient from '#/network/game/client/model/DetectModifiedClient.js'; + +export default class DetectModifiedClientDecoder extends ClientGameMessageDecoder { + prot = ClientGameProt.DETECT_MODIFIED_CLIENT; + + decode(buf: Packet): DetectModifiedClient { + const verification = buf.g4(); + return new DetectModifiedClient(verification); + } +} diff --git a/src/network/game/client/codec/FriendSetRankDecoder.ts b/src/network/game/client/codec/FriendSetRankDecoder.ts new file mode 100644 index 000000000..6fc75e2f8 --- /dev/null +++ b/src/network/game/client/codec/FriendSetRankDecoder.ts @@ -0,0 +1,14 @@ +import Packet from '#/io/Packet.js'; +import ClientGameMessageDecoder from '#/network/game/client/ClientGameMessageDecoder.js'; +import ClientGameProt from '#/network/game/client/ClientGameProt.js'; +import FriendSetRank from '#/network/game/client/model/FriendSetRank.js'; + +export default class FriendSetRankDecoder extends ClientGameMessageDecoder { + prot = ClientGameProt.FRIEND_SETRANK; + + decode(buf: Packet) { + const rank = buf.g1b_alt1(); + const username = buf.g8(); + return new FriendSetRank(username, rank); + } +} diff --git a/src/network/game/client/codec/InvButtonDDecoder.ts b/src/network/game/client/codec/InvButtonDDecoder.ts index 99441d4fa..b0037e704 100644 --- a/src/network/game/client/codec/InvButtonDDecoder.ts +++ b/src/network/game/client/codec/InvButtonDDecoder.ts @@ -4,7 +4,7 @@ import ClientGameProt from '#/network/game/client/ClientGameProt.js'; import InvButtonD from '#/network/game/client/model/InvButtonD.js'; export default class InvButtonDDecoder extends ClientGameMessageDecoder { - prot = ClientGameProt.INV_BUTTOND; + prot = ClientGameProt.IF_BUTTOND; decode(buf: Packet) { const targetSlot = buf.g2_alt3(); diff --git a/src/network/game/client/codec/MoveClickDecoder.ts b/src/network/game/client/codec/MoveClickDecoder.ts index 579938a90..d443f90f2 100644 --- a/src/network/game/client/codec/MoveClickDecoder.ts +++ b/src/network/game/client/codec/MoveClickDecoder.ts @@ -9,7 +9,7 @@ export default class MoveClickDecoder extends ClientGameMessageDecoder { let loc = -1; if (this.op === 1) { - x = buf.g2_alt2(); - z = buf.g2_alt1(); - loc = buf.g2_alt1(); + x = buf.g2_alt1(); + loc = buf.g2_alt2(); + z = buf.g2(); } else if (this.op === 2) { + z = buf.g2_alt3(); + x = buf.g2_alt1(); loc = buf.g2(); - x = buf.g2(); - z = buf.g2_alt2(); } else if (this.op === 3) { - z = buf.g2_alt2(); - loc = buf.g2_alt1(); - x = buf.g2_alt3(); + loc = buf.g2_alt3(); + z = buf.g2_alt3(); + x = buf.g2_alt1(); } else if (this.op === 4) { - x = buf.g2(); z = buf.g2_alt1(); + x = buf.g2_alt3(); loc = buf.g2(); } else if (this.op === 5) { - loc = buf.g2_alt1(); - z = buf.g2_alt1(); - x = buf.g2(); + loc = buf.g2_alt3(); + z = buf.g2_alt3(); + x = buf.g2_alt3(); } return new OpLoc(this.op, x, z, loc); diff --git a/src/network/game/client/codec/OpLocEDecoder.ts b/src/network/game/client/codec/OpLocEDecoder.ts new file mode 100644 index 000000000..22c45c6c0 --- /dev/null +++ b/src/network/game/client/codec/OpLocEDecoder.ts @@ -0,0 +1,14 @@ +import Packet from '#/io/Packet.js'; +import ClientGameMessageDecoder from '#/network/game/client/ClientGameMessageDecoder.js'; +import ClientGameProt from '#/network/game/client/ClientGameProt.js'; +import OpLocE from '#/network/game/client/model/OpLocE.js'; + +export default class OpLocEDecoder extends ClientGameMessageDecoder { + prot = ClientGameProt.OPLOCE; + + decode(buf: Packet) { + const loc = buf.g2_alt3(); + + return new OpLocE(loc); + } +} diff --git a/src/network/game/client/codec/OpObjDecoder.ts b/src/network/game/client/codec/OpObjDecoder.ts index 3098639b3..5841b76c2 100644 --- a/src/network/game/client/codec/OpObjDecoder.ts +++ b/src/network/game/client/codec/OpObjDecoder.ts @@ -17,25 +17,25 @@ export default class OpObjDecoder extends ClientGameMessageDecoder { let obj = -1; if (this.op === 1) { - x = buf.g2_alt2(); - z = buf.g2(); - obj = buf.g2_alt3(); + x = buf.g2_alt3(); + obj = buf.g2_alt2(); + z = buf.g4_alt1(); } else if (this.op === 2) { - x = buf.g2(); - z = buf.g2_alt2(); - obj = buf.g2_alt3(); + obj = buf.g2(); + x = buf.g2_alt1(); + z = buf.g2_alt3(); } else if (this.op === 3) { + z = buf.g4_alt2(); + obj = buf.g2(); + x = buf.g2_alt2(); + } else if (this.op === 4) { + z = buf.g4_alt1(); obj = buf.g2_alt3(); x = buf.g2_alt3(); - z = buf.g2_alt2(); - } else if (this.op === 4) { - obj = buf.g2_alt2(); - z = buf.g2_alt1(); - x = buf.g2(); } else if (this.op === 5) { obj = buf.g2_alt1(); x = buf.g2_alt2(); - z = buf.g2(); + z = buf.g4_alt3(); } return new OpObj(this.op, x, z, obj); diff --git a/src/network/game/client/codec/OpObjEDecoder.ts b/src/network/game/client/codec/OpObjEDecoder.ts new file mode 100644 index 000000000..6a5115a8a --- /dev/null +++ b/src/network/game/client/codec/OpObjEDecoder.ts @@ -0,0 +1,14 @@ +import Packet from '#/io/Packet.js'; +import ClientGameMessageDecoder from '#/network/game/client/ClientGameMessageDecoder.js'; +import ClientGameProt from '#/network/game/client/ClientGameProt.js'; +import OpObjE from '#/network/game/client/model/OpObjE.js'; + +export default class OpObjEDecoder extends ClientGameMessageDecoder { + prot = ClientGameProt.OPOBJE; + + decode(buf: Packet) { + const obj = buf.g2_alt3(); + + return new OpObjE(obj); + } +} diff --git a/src/network/game/client/codec/TransmitVarVerifyIDDecoder.ts b/src/network/game/client/codec/TransmitVarVerifyIDDecoder.ts new file mode 100644 index 000000000..af32e31a1 --- /dev/null +++ b/src/network/game/client/codec/TransmitVarVerifyIDDecoder.ts @@ -0,0 +1,13 @@ +import Packet from '#/io/Packet.js'; +import ClientGameMessageDecoder from '#/network/game/client/ClientGameMessageDecoder.js'; +import ClientGameProt from '#/network/game/client/ClientGameProt.js'; +import TransmitVarVerifyID from '#/network/game/client/model/TransmitVarVerifyID.js'; + +export default class TransmitVarVerifyIDDecoder extends ClientGameMessageDecoder { + prot = ClientGameProt.TRANSMITVAR_VERIFYID; + + decode(buf: Packet) { + const verifyID = buf.g2(); + return new TransmitVarVerifyID(verifyID); + } +} diff --git a/src/network/game/client/codec/WindowStatusDecoder.ts b/src/network/game/client/codec/WindowStatusDecoder.ts new file mode 100644 index 000000000..07edf5f30 --- /dev/null +++ b/src/network/game/client/codec/WindowStatusDecoder.ts @@ -0,0 +1,17 @@ +import Packet from '#/io/Packet.js'; +import ClientGameMessageDecoder from '#/network/game/client/ClientGameMessageDecoder.js'; +import ClientGameProt from '#/network/game/client/ClientGameProt.js'; +import WindowStatus from '#/network/game/client/model/WindowStatus.js'; + +export default class WindowStatusDecoder extends ClientGameMessageDecoder { + prot = ClientGameProt.WINDOW_STATUS; + + decode(buf: Packet) { + const mode = buf.g1(); + const canvasWidth = buf.g2(); + const canvasHeight = buf.g2(); + const antialiasingmode = buf.g1(); + + return new WindowStatus(mode, canvasWidth, canvasHeight, antialiasingmode); + } +} diff --git a/src/network/game/client/handler/ClanJoinLeaveChatHandler.ts b/src/network/game/client/handler/ClanJoinLeaveChatHandler.ts new file mode 100644 index 000000000..6cc326f3e --- /dev/null +++ b/src/network/game/client/handler/ClanJoinLeaveChatHandler.ts @@ -0,0 +1,23 @@ +import Player from '#/engine/entity/Player.js'; +import ClientGameMessageHandler from '#/network/game/client/ClientGameMessageHandler.js'; +import ClanJoinLeaveChat from '#/network/game/client/model/ClanJoinLeaveChat.js'; +import { fromBase37 } from '#/util/JString.js'; + +export default class ClanJoinLeaveChatHandler extends ClientGameMessageHandler { + handle(message: ClanJoinLeaveChat, _player: Player): boolean { + // Client specifically sends 0 if wished action is to leave, do base37 after this to avoid false positives with leaving. + if (message.clanname === 0n) + { + // Leave current clan, if player is in one. + return true; + } + + if (fromBase37(message.clanname) === 'invalid_name') + { + return false; + } + + // Join clan, if it exists + return true; + } +} diff --git a/src/network/game/client/handler/ClanKickUserHandler.ts b/src/network/game/client/handler/ClanKickUserHandler.ts new file mode 100644 index 000000000..7b7bb9259 --- /dev/null +++ b/src/network/game/client/handler/ClanKickUserHandler.ts @@ -0,0 +1,15 @@ +import Player from '#/engine/entity/Player.js'; +import ClientGameMessageHandler from '#/network/game/client/ClientGameMessageHandler.js'; +import ClanKickUser from '#/network/game/client/model/ClanKickUser.js'; +import { fromBase37 } from '#/util/JString.js'; + +export default class ClanKickUserHandler extends ClientGameMessageHandler { + handle(message: ClanKickUser, _player: Player): boolean { + if (fromBase37(message.username) === 'invalid_name') { + return false; + } + + // TODO: remove username from players current clan, if they are priveleged enough within the clan to do so. + return true; + } +} diff --git a/src/network/game/client/handler/DetectModifiedClientHandler.ts b/src/network/game/client/handler/DetectModifiedClientHandler.ts new file mode 100644 index 000000000..2d9326014 --- /dev/null +++ b/src/network/game/client/handler/DetectModifiedClientHandler.ts @@ -0,0 +1,16 @@ +import DetectModifiedClient from '#/network/game/client/model/DetectModifiedClient.js'; +import ClientGameMessageHandler from '#/network/game/client/ClientGameMessageHandler.js'; +import Player from '#/engine/entity/Player.js'; +import Environment from '#/util/Environment.js'; +import World from '#/engine/World.js'; + +export default class DetectModifiedClientHandler extends ClientGameMessageHandler { + + handle(message: DetectModifiedClient, player: Player): boolean { + if (!Environment.NODE_DEBUG && message.verification !== 1057001181) { + World.notifyPlayerBan('automated', player.username, Date.now() + 172800000); + } + + return true; + } +} diff --git a/src/network/game/client/handler/FriendSetRankHandler.ts b/src/network/game/client/handler/FriendSetRankHandler.ts new file mode 100644 index 000000000..c58bbdaff --- /dev/null +++ b/src/network/game/client/handler/FriendSetRankHandler.ts @@ -0,0 +1,15 @@ +import Player from '#/engine/entity/Player.js'; +import ClientGameMessageHandler from '#/network/game/client/ClientGameMessageHandler.js'; +import FriendSetRank from '#/network/game/client/model/FriendSetRank.js'; +import { fromBase37 } from '#/util/JString.js'; + +export default class FriendSetRankHandler extends ClientGameMessageHandler { + handle(message: FriendSetRank, player: Player): boolean { + if (player.socialProtect ||fromBase37(message.username) === 'invalid_name') + { + return false; + } + + return true; + } +} diff --git a/src/network/game/client/handler/InvButtonDHandler.ts b/src/network/game/client/handler/InvButtonDHandler.ts index d1190c729..781afc182 100644 --- a/src/network/game/client/handler/InvButtonDHandler.ts +++ b/src/network/game/client/handler/InvButtonDHandler.ts @@ -37,13 +37,13 @@ export default class InvButtonDHandler extends ClientGameMessageHandler { + handle(message: OpLocE, player: NetworkPlayer): boolean { + const loc = LocType.get(message.loc); + + if (loc && loc.desc) { + player.write(new MessageGame(loc.desc)); + return true; + } + + if (!Environment.NODE_PRODUCTION) { + player.write(new MessageGame(`Loc id ${message.loc} has no description.`)); + } + + player.write(new MessageGame(`It's a ${loc ? loc.name : 'object'}.`)); + return true; + } +} diff --git a/src/network/game/client/handler/OpObjEHandler.ts b/src/network/game/client/handler/OpObjEHandler.ts new file mode 100644 index 000000000..4dd2b712a --- /dev/null +++ b/src/network/game/client/handler/OpObjEHandler.ts @@ -0,0 +1,25 @@ +import ObjType from '#/cache/config/ObjType.js'; +import { NetworkPlayer } from '#/engine/entity/NetworkPlayer.js'; +import ClientGameMessageHandler from '#/network/game/client/ClientGameMessageHandler.js'; +import OpObjE from '#/network/game/client/model/OpObjE.js'; +import MessageGame from '#/network/game/server/model/MessageGame.js'; +import Environment from '#/util/Environment.js'; + +export default class OpObjEHandler extends ClientGameMessageHandler { + handle(message: OpObjE, player: NetworkPlayer): boolean { + const obj = ObjType.get(message.obj); + + if (obj && obj.desc) { + player.write(new MessageGame(obj.desc)); + return true; + } + + if (!Environment.NODE_PRODUCTION) { + player.write(new MessageGame(`Obj id ${message.obj} has no description.`)); + } + + player.write(new MessageGame(`It's a ${obj ? obj.name : 'object'}.`)); + return true; + + } +} diff --git a/src/network/game/client/handler/WindowStatusHandler.ts b/src/network/game/client/handler/WindowStatusHandler.ts new file mode 100644 index 000000000..b51a8f751 --- /dev/null +++ b/src/network/game/client/handler/WindowStatusHandler.ts @@ -0,0 +1,20 @@ +import Player from '#/engine/entity/Player.js'; +import World from '#/engine/World.js'; +import ClientGameMessageHandler from '#/network/game/client/ClientGameMessageHandler.js'; +import WindowStatus, { WindowMode } from '#/network/game/client/model/WindowStatus.js'; + +export default class WindowStatusHandler extends ClientGameMessageHandler { + handle(message: WindowStatus, player: Player): boolean { + if (message.mode < WindowMode.SD || message.mode > WindowMode.HD_FULLSCREEN) { + World.notifyPlayerBan('automated', player.username, Date.now() + 172800000); + return false; + } + + player.windowMode = message.mode; + player.canvasWidth = message.canvasWidth; + player.canvasHeight = message.canvasHeight; + player.antialiasingmode = message.antialiasingmode; + + return true; + } +} diff --git a/src/network/game/client/model/AnticheatOpLogic.ts b/src/network/game/client/model/AnticheatOpLogic.ts deleted file mode 100644 index 9bf21df34..000000000 --- a/src/network/game/client/model/AnticheatOpLogic.ts +++ /dev/null @@ -1,6 +0,0 @@ -import ClientGameProtCategory from '#/network/game/client/ClientGameProtCategory.js'; -import ClientGameMessage from '#/network/game/client/ClientGameMessage.js'; - -export default class AnticheatOpLogic extends ClientGameMessage { - category = ClientGameProtCategory.CLIENT_EVENT; -} diff --git a/src/network/game/client/model/ClanJoinLeaveChat.ts b/src/network/game/client/model/ClanJoinLeaveChat.ts new file mode 100644 index 000000000..a2be32dbe --- /dev/null +++ b/src/network/game/client/model/ClanJoinLeaveChat.ts @@ -0,0 +1,10 @@ +import ClientGameProtCategory from '#/network/game/client/ClientGameProtCategory.js'; +import ClientGameMessage from '#/network/game/client/ClientGameMessage.js'; + +export default class ClanJoinLeaveChat extends ClientGameMessage { + category = ClientGameProtCategory.USER_EVENT; + + constructor(readonly clanname: bigint) { + super(); + } +} diff --git a/src/network/game/client/model/ClanKickUser.ts b/src/network/game/client/model/ClanKickUser.ts new file mode 100644 index 000000000..d6fa6cd05 --- /dev/null +++ b/src/network/game/client/model/ClanKickUser.ts @@ -0,0 +1,10 @@ +import ClientGameMessage from '#/network/game/client/ClientGameMessage.js'; +import ClientGameProtCategory from '#/network/game/client/ClientGameProtCategory.js'; + +export default class ClanKickUser extends ClientGameMessage { + category = ClientGameProtCategory.USER_EVENT; + + constructor(readonly username: bigint) { + super(); + } +} diff --git a/src/network/game/client/model/AnticheatCycleLogic.ts b/src/network/game/client/model/DetectModifiedClient.ts similarity index 60% rename from src/network/game/client/model/AnticheatCycleLogic.ts rename to src/network/game/client/model/DetectModifiedClient.ts index 7bcf116e1..a04d5171f 100644 --- a/src/network/game/client/model/AnticheatCycleLogic.ts +++ b/src/network/game/client/model/DetectModifiedClient.ts @@ -1,6 +1,10 @@ -import ClientGameProtCategory from '#/network/game/client/ClientGameProtCategory.js'; import ClientGameMessage from '#/network/game/client/ClientGameMessage.js'; +import ClientGameProtCategory from '#/network/game/client/ClientGameProtCategory.js'; -export default class AnticheatCycleLogic extends ClientGameMessage { +export default class DetectModifiedClient extends ClientGameMessage { category = ClientGameProtCategory.CLIENT_EVENT; + + constructor(readonly verification: number) { + super(); + } } diff --git a/src/network/game/client/model/FriendSetRank.ts b/src/network/game/client/model/FriendSetRank.ts new file mode 100644 index 000000000..7337aad2e --- /dev/null +++ b/src/network/game/client/model/FriendSetRank.ts @@ -0,0 +1,10 @@ +import ClientGameMessage from '#/network/game/client/ClientGameMessage.js'; +import ClientGameProtCategory from '#/network/game/client/ClientGameProtCategory.js'; + +export default class FriendSetRank extends ClientGameMessage { + category = ClientGameProtCategory.USER_EVENT; + + constructor(readonly username: bigint, readonly rank: number) { + super(); + } +} diff --git a/src/network/game/client/model/OpLocE.ts b/src/network/game/client/model/OpLocE.ts new file mode 100644 index 000000000..86ce7ae51 --- /dev/null +++ b/src/network/game/client/model/OpLocE.ts @@ -0,0 +1,12 @@ +import ClientGameProtCategory from '#/network/game/client/ClientGameProtCategory.js'; +import ClientGameMessage from '#/network/game/client/ClientGameMessage.js'; + +export default class OpLocE extends ClientGameMessage { + category = ClientGameProtCategory.USER_EVENT; + + constructor( + readonly loc: number + ) { + super(); + } +} diff --git a/src/network/game/client/model/OpNpcE.ts b/src/network/game/client/model/OpNpcE.ts new file mode 100644 index 000000000..16fe249ad --- /dev/null +++ b/src/network/game/client/model/OpNpcE.ts @@ -0,0 +1,12 @@ +import ClientGameProtCategory from '#/network/game/client/ClientGameProtCategory.js'; +import ClientGameMessage from '#/network/game/client/ClientGameMessage.js'; + +export default class OpNpcE extends ClientGameMessage { + category = ClientGameProtCategory.USER_EVENT; + + constructor( + readonly npc: number + ) { + super(); + } +} diff --git a/src/network/game/client/model/OpObjE.ts b/src/network/game/client/model/OpObjE.ts new file mode 100644 index 000000000..5c07615da --- /dev/null +++ b/src/network/game/client/model/OpObjE.ts @@ -0,0 +1,12 @@ +import ClientGameProtCategory from '#/network/game/client/ClientGameProtCategory.js'; +import ClientGameMessage from '#/network/game/client/ClientGameMessage.js'; + +export default class OpObjE extends ClientGameMessage { + category = ClientGameProtCategory.USER_EVENT; + + constructor( + readonly obj: number, + ) { + super(); + } +} diff --git a/src/network/game/client/model/TransmitVarVerifyID.ts b/src/network/game/client/model/TransmitVarVerifyID.ts new file mode 100644 index 000000000..4b85f0efe --- /dev/null +++ b/src/network/game/client/model/TransmitVarVerifyID.ts @@ -0,0 +1,10 @@ +import ClientGameMessage from '#/network/game/client/ClientGameMessage.js'; +import ClientGameProtCategory from '#/network/game/client/ClientGameProtCategory.js'; + +export default class TransmitVarVerifyID extends ClientGameMessage { + category = ClientGameProtCategory.USER_EVENT; + + constructor(readonly verifyID: number) { + super(); + } +} diff --git a/src/network/game/client/model/WindowStatus.ts b/src/network/game/client/model/WindowStatus.ts new file mode 100644 index 000000000..9d79044c2 --- /dev/null +++ b/src/network/game/client/model/WindowStatus.ts @@ -0,0 +1,22 @@ +import ClientGameMessage from '#/network/game/client/ClientGameMessage.js'; +import ClientGameProtCategory from '#/network/game/client/ClientGameProtCategory.js'; + +export const enum WindowMode { + SD, // 0 + HD_NON_RESIZABLE, // 1 + HD_RESIZEABLE, // 2 + HD_FULLSCREEN // 3 +} + +export default class WindowStatus extends ClientGameMessage { + category = ClientGameProtCategory.CLIENT_EVENT; + + constructor( + readonly mode: WindowMode, + readonly canvasWidth: number, + readonly canvasHeight: number, + readonly antialiasingmode: number + ) { + super(); + } +} diff --git a/src/network/game/server/ServerGameProt.ts b/src/network/game/server/ServerGameProt.ts index cc6a47274..a14e16771 100644 --- a/src/network/game/server/ServerGameProt.ts +++ b/src/network/game/server/ServerGameProt.ts @@ -58,8 +58,9 @@ export default class ServerGameProt { static readonly P_COUNTDIALOG = new ServerGameProt(58, 0); // named after runescript command + client resume_p_countdialog packet static readonly SET_MULTIWAY = new ServerGameProt(9000, 1); static readonly P_NAMEDIALOG = new ServerGameProt(9001, 0); - static readonly SET_PLAYER_OP = new ServerGameProt(44, -1); - static readonly MINIMAP_TOGGLE = new ServerGameProt(192, 1); + static readonly SET_PLAYER_OP = new ServerGameProt(44, -1); // NXT naming + static readonly MINIMAP_TOGGLE = new ServerGameProt(192, 1); // NXT naming + static readonly UPDATE_STOCKMARKET_SLOT = new ServerGameProt(116, -1); // NXT naming // maps static readonly REBUILD_NORMAL = new ServerGameProt(162, -2); // NXT naming diff --git a/src/network/game/server/codec/ObjRevealEncoder.ts b/src/network/game/server/codec/ObjRevealEncoder.ts index e07602811..4f8ba41a5 100644 --- a/src/network/game/server/codec/ObjRevealEncoder.ts +++ b/src/network/game/server/codec/ObjRevealEncoder.ts @@ -10,7 +10,7 @@ export default class ObjRevealEncoder extends ServerGameZoneMessageEncoder 0) { + if (staffLvl > 1 && !this.playerStaff.has(username37)) { this.playerStaff.add(username37); } diff --git a/src/server/logger/LoggerClient.ts b/src/server/logger/LoggerClient.ts index cc6b4b727..0b670ca2c 100644 --- a/src/server/logger/LoggerClient.ts +++ b/src/server/logger/LoggerClient.ts @@ -45,7 +45,7 @@ export default class LoggerClient extends InternalClient { ); } - public async report(username: string, coord: number, offender: string, reason: number) { + public async report(session_uuid: string, coord: number, offender: string, reason: number) { await this.connect(); if (!this.ws || !this.wsr || !this.wsr.checkIfWsLive()) { @@ -57,7 +57,7 @@ export default class LoggerClient extends InternalClient { type: 'report', world: Environment.NODE_ID, profile: Environment.NODE_PROFILE, - username, + session_uuid, timestamp: Date.now(), coord, offender, diff --git a/src/server/logger/LoggerThread.ts b/src/server/logger/LoggerThread.ts index 46c76b01f..61c4e1da4 100644 --- a/src/server/logger/LoggerThread.ts +++ b/src/server/logger/LoggerThread.ts @@ -58,8 +58,8 @@ async function handleRequests(_parentPort: ParentPort, msg: any) { } case 'report': { if (Environment.LOGGER_SERVER) { - const { username, coord, offender, reason } = msg; - await client.report(username, coord, offender, reason); + const { session_uuid, coord, offender, reason } = msg; + await client.report(session_uuid, coord, offender, reason); } break; } diff --git a/src/server/login/LoginServer.ts b/src/server/login/LoginServer.ts index dc63149b6..fbb3a8516 100644 --- a/src/server/login/LoginServer.ts +++ b/src/server/login/LoginServer.ts @@ -17,7 +17,7 @@ import { getUnreadMessageCount } from '#/server/login/Messages.js'; import { startManagementWeb } from '#/web.js'; import InvType from '#/cache/config/InvType.js'; -async function updateHiscores(account: { id: number, staffmodlevel: number } | undefined, player: Player, profile: string) { +async function updateHiscores(account: { id: number, staffmodlevel: number, banned_until: string | null } | undefined, player: Player, profile: string) { if (!account) return; @@ -25,6 +25,10 @@ async function updateHiscores(account: { id: number, staffmodlevel: number } | u return; } + if (account.banned_until !== null && new Date(account.banned_until) >= new Date()) { + return; + } + const insert = []; const update = []; diff --git a/src/util/ColorConversion.ts b/src/util/ColorConversion.ts index 1414a9580..5c93bdd7c 100644 --- a/src/util/ColorConversion.ts +++ b/src/util/ColorConversion.ts @@ -130,4 +130,8 @@ export default class ColorConversion { return possible; } + + static colourFudge(rgb: number): number { + return rgb === 16711935 ? -1 : ColorConversion.rgb24toHsl16(rgb); + } } diff --git a/src/util/Environment.ts b/src/util/Environment.ts index 76ac79641..5d1f13670 100644 --- a/src/util/Environment.ts +++ b/src/util/Environment.ts @@ -92,13 +92,8 @@ export default { /// kysely KYSELY_VERBOSE: tryParseBoolean(process.env.KYSELY_VERBOSE, false), - /// development - // some users may not be able to change their system PATH for this project - BUILD_JAVA_PATH: tryParseString(process.env.BUILD_JAVA_PATH, 'java'), // auto-build on startup BUILD_STARTUP: tryParseBoolean(process.env.BUILD_STARTUP, false), - // auto-update compiler on startup - BUILD_STARTUP_UPDATE: tryParseBoolean(process.env.BUILD_STARTUP_UPDATE, true), // used to check if we're producing the original cache without edits BUILD_VERIFY: tryParseBoolean(process.env.BUILD_VERIFY, true), // used to keep some semblance of sanity in our folder structure diff --git a/src/util/JString.ts b/src/util/JString.ts index 0658ea163..2092bc459 100644 --- a/src/util/JString.ts +++ b/src/util/JString.ts @@ -18,6 +18,10 @@ export function toBase37(string: string): bigint { } } + while (l % 37n === 0n && l !== 0n) { + l /= 37n; + } + return l; } diff --git a/src/util/JavaRandom.ts b/src/util/JavaRandom.ts new file mode 100644 index 000000000..936325094 --- /dev/null +++ b/src/util/JavaRandom.ts @@ -0,0 +1,126 @@ +// based on: https://github.com/raybellis/java-random + TS support + +// +// An almost complete implementation in JS of the `java.util.Random` +// class from J2SE, designed to so far as possible produce the same +// output sequences as the Java original when supplied with the same +// seed. +// + +const p2_16 = 0x0000000010000; +const p2_24 = 0x0000001000000; +const p2_27 = 0x0000008000000; +const p2_31 = 0x0000080000000; +const p2_32 = 0x0000100000000; +const p2_48 = 0x1000000000000; +const p2_53 = Math.pow(2, 53); // NB: exceeds Number.MAX_SAFE_INTEGER + +const m2_16 = 0xffff; + +// +// multiplicative term for the PRNG +// +const [c2, c1, c0] = [0x0005, 0xdeec, 0xe66d]; + +let s2 = 0, s1 = 0, s0 = 0; + +// +// 53-bit safe version of +// seed = (seed * 0x5DEECE66DL + 0xBL) & ((1L << 48) - 1) +// +function _next() { + let carry = 0xb; + + let r0 = (s0 * c0) + carry; + carry = r0 >>> 16; + r0 &= m2_16; + + let r1 = (s1 * c0 + s0 * c1) + carry; + carry = r1 >>> 16; + r1 &= m2_16; + + let r2 = (s2 * c0 + s1 * c1 + s0 * c2) + carry; + r2 &= m2_16; + + [s2, s1, s0] = [r2, r1, r0]; + + return s2 * p2_16 + s1; +} + +function next_signed(bits: number) { + return _next() >> (32 - bits); +} + +function next(bits: number) { + return _next() >>> (32 - bits); +} + +function checkIsPositiveInt(n: number, r = Number.MAX_SAFE_INTEGER) { + if (n < 0 || n > r) { + throw RangeError('number must be > 0'); + } +} + +class JavaRandom { + constructor(seedval?: number) { + if (typeof seedval === 'undefined') { + seedval = Math.floor(Math.random() * p2_48); + } + + this.setSeed(seedval); + } + + // + // 53-bit safe version of + // seed = (seed ^ 0x5DEECE66DL) & ((1L << 48) - 1) + // + setSeed(n: number) { + checkIsPositiveInt(n); + s0 = ((n) & m2_16) ^ c0; + s1 = ((n / p2_16) & m2_16) ^ c1; + s2 = ((n / p2_32) & m2_16) ^ c2; + } + + // generate exclusive random number within bound + nextInt(bound?: number) { + if (bound === undefined) { + return next_signed(32); + } + + checkIsPositiveInt(bound, 0x7fffffff); + + // special case if bound is a power of two + if ((bound & -bound) === bound) { + const r = next(31) / p2_31; + return ~~(bound * r); + } + + let bits, val; + do { + bits = next(31); + val = bits % bound; + } while (bits - val + (bound - 1) < 0); + return val; + } + + nextLong() { + const msb = BigInt(next_signed(32)); + const lsb = BigInt(next_signed(32)); + const p2_32n = BigInt(p2_32); + return msb * p2_32n + lsb; + } + + nextBoolean() { + return next(1) != 0; + } + + nextFloat() { + return next(24) / p2_24; + } + + nextDouble() { + return (p2_27 * next(26) + next(27)) / p2_53; + } +}; + +export default new JavaRandom(); diff --git a/src/util/RuneScriptCompiler.ts b/src/util/RuneScriptCompiler.ts deleted file mode 100644 index 0e87acb7a..000000000 --- a/src/util/RuneScriptCompiler.ts +++ /dev/null @@ -1,47 +0,0 @@ -import crypto from 'crypto'; -import fs from 'fs'; - -import axios from 'axios'; - -import ScriptProvider from '#/engine/script/ScriptProvider.js'; -import { printDebug, printWarning } from '#/util/Logger.js'; - -export async function updateCompiler(): Promise { - printDebug('Checking for compiler update'); - - let needsUpdate = false; - - try { - if (!fs.existsSync('RuneScriptCompiler.jar')) { - needsUpdate = true; - } else { - const sha256 = crypto.createHash('sha256'); - sha256.update(fs.readFileSync('RuneScriptCompiler.jar')); - const shasum = sha256.digest('hex'); - - const req = await axios.get('https://github.com/LostCityRS/RuneScriptKt/releases/download/' + ScriptProvider.COMPILER_VERSION + '/RuneScriptCompiler.jar.sha256'); - const expected = req.data.substring(0, 64); - - if (shasum != expected) { - needsUpdate = true; - } - } - - if (needsUpdate) { - printDebug('Updating compiler'); - - const req = await axios.get('https://github.com/LostCityRS/RuneScriptKt/releases/download/' + ScriptProvider.COMPILER_VERSION + '/RuneScriptCompiler.jar', { - responseType: 'arraybuffer' - }); - - fs.writeFileSync('RuneScriptCompiler.jar', req.data); - } - } catch (err) { - console.warn(err); - printWarning('Unable to check for compiler update'); - return false; - } - - printDebug('Compiler is up to date'); - return true; -} diff --git a/tools/cache/compare/compareGroups.ts b/tools/cache/compare/compareGroups.ts new file mode 100644 index 000000000..3e6d2c069 --- /dev/null +++ b/tools/cache/compare/compareGroups.ts @@ -0,0 +1,117 @@ +import fs from 'fs'; + +import { + arraysEqual, + parseGroupIdsFromIndexPacked, + readGroupBytes +} from '#tools/cache/lib/js5Tools.js'; +import { unpackJs5Group } from '#/io/Js5Group.js'; + +type Args = { + archive: number; + leftDir: string; + rightDir: string; + indexPath: string; + mode: 'compressed' | 'uncompressed'; + openrs2Left: boolean; + openrs2Right: boolean; + help: boolean; +}; + +function parseArgs(argv: string[]): Args { + const args: Args = { + archive: 17, + leftDir: 'data/cache', + rightDir: 'data/pack/server-enum', + indexPath: '', + mode: 'compressed', + openrs2Left: false, + openrs2Right: false, + help: false + }; + + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg === '--archive') { + args.archive = Number(argv[++i]); + } else if (arg === '--left-dir') { + args.leftDir = argv[++i]; + } else if (arg === '--right-dir') { + args.rightDir = argv[++i]; + } else if (arg === '--index') { + args.indexPath = argv[++i]; + } else if (arg === '--mode') { + const mode = argv[++i]; + if (mode === 'compressed' || mode === 'uncompressed') { + args.mode = mode; + } + } else if (arg === '--openrs2-left') { + args.openrs2Left = true; + } else if (arg === '--openrs2-right') { + args.openrs2Right = true; + } else if (arg === '--help' || arg === '-h') { + args.help = true; + } + } + + if (!args.indexPath) { + args.indexPath = `data/cache/255/${args.archive}.dat`; + } + + return args; +} + + +async function main() { + const args = parseArgs(process.argv.slice(2)); + + if (!fs.existsSync(args.indexPath)) { + throw new Error(`Index file not found: ${args.indexPath}`); + } + + const indexPacked = new Uint8Array(fs.readFileSync(args.indexPath)); + const groupIds = parseGroupIdsFromIndexPacked(indexPacked); + + let mismatches = 0; + let missing = 0; + let compared = 0; + + for (const groupId of groupIds) { + const left = await readGroupBytes(args.archive, groupId, args.leftDir, args.openrs2Left); + const right = await readGroupBytes(args.archive, groupId, args.rightDir, args.openrs2Right); + + if (!left || !right) { + missing++; + console.log(`Group ${groupId}: MISSING (${left ? 'right' : 'left'} side)`); + continue; + } + + let leftCompare = left; + let rightCompare = right; + if (args.mode === 'uncompressed') { + leftCompare = unpackJs5Group(left); + rightCompare = unpackJs5Group(right); + } + + compared++; + if (!arraysEqual(leftCompare, rightCompare)) { + mismatches++; + console.log(`Group ${groupId}: MISMATCH`); + } + } + + console.log(`Groups compared: ${compared}`); + console.log(`Groups missing: ${missing}`); + console.log(`Groups mismatched: ${mismatches}`); + + if (mismatches > 0) { + process.exitCode = 1; + } +} + +try { + await main(); +} catch (err) { + console.error(err); + process.exit(1); +} diff --git a/tools/cache/debug/decodeGroup.ts b/tools/cache/debug/decodeGroup.ts new file mode 100644 index 000000000..5595b462c --- /dev/null +++ b/tools/cache/debug/decodeGroup.ts @@ -0,0 +1,122 @@ +import fs from 'fs'; +import path from 'path'; + +import { unpackJs5Group } from '#/io/Js5Group.js'; +import { parseJs5ArchiveIndex, splitGroupFiles } from '#/io/Js5ArchiveIndex.js'; +import { ensureDir, loadIndexPacked, readGroupBytes } from '#tools/cache/lib/js5Tools.js'; + +type Args = { + archive: number; + group: number; + groupsDir: string; + indexPath: string; + openrs2: boolean; + outDir: string; + previewBytes: number; + help: boolean; +}; + +function parseArgs(argv: string[]): Args { + const args: Args = { + archive: 2, + group: 21, + groupsDir: 'data/cache', + indexPath: '', + openrs2: false, + outDir: '', + previewBytes: 24, + help: false + }; + + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg === '--archive') { + args.archive = Number(argv[++i]); + } else if (arg === '--group') { + args.group = Number(argv[++i]); + } else if (arg === '--groups-dir') { + args.groupsDir = argv[++i]; + } else if (arg === '--index') { + args.indexPath = argv[++i]; + } else if (arg === '--openrs2') { + args.openrs2 = true; + } else if (arg === '--out-dir') { + args.outDir = argv[++i]; + } else if (arg === '--preview-bytes') { + args.previewBytes = Math.max(0, Number(argv[++i])); + } else if (arg === '--help' || arg === '-h') { + args.help = true; + } + } + + if (!args.indexPath) { + args.indexPath = `data/cache/255/${args.archive}.dat`; + } + + return args; +} + +function bytesToHex(data: Uint8Array, count: number): string { + const end = Math.min(data.length, count); + const out: string[] = new Array(end); + + for (let i = 0; i < end; i++) { + out[i] = data[i].toString(16).padStart(2, '0'); + } + + return out.join(' '); +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + + const indexPacked = await loadIndexPacked(args.archive, args.indexPath, args.openrs2); + const indexUnpacked = unpackJs5Group(indexPacked); + const archiveIndex = parseJs5ArchiveIndex(indexUnpacked); + const fileIds = archiveIndex.fileIdsByGroup.get(args.group); + + if (!fileIds) { + throw new Error(`Group ${args.group} does not exist in archive ${args.archive}.`); + } + + const groupPacked = await readGroupBytes(args.archive, args.group, args.groupsDir, args.openrs2); + if (!groupPacked) { + throw new Error(`Missing group data for archive ${args.archive}, group ${args.group}.`); + } + + const groupUnpacked = unpackJs5Group(groupPacked); + const files = splitGroupFiles(groupUnpacked, fileIds); + + console.log(`Archive ${args.archive}, Group ${args.group}`); + console.log(`Packed bytes: ${groupPacked.length}`); + console.log(`Unpacked bytes: ${groupUnpacked.length}`); + console.log(`Files in group: ${fileIds.length}`); + + let outputDir = ''; + if (args.outDir) { + outputDir = path.join(args.outDir, String(args.archive), String(args.group)); + ensureDir(outputDir); + } + + for (const fileId of fileIds) { + const fileData = files.get(fileId) ?? new Uint8Array(0); + const preview = args.previewBytes > 0 ? bytesToHex(fileData, args.previewBytes) : ''; + + console.log(`- file ${fileId}: ${fileData.length} bytes${preview ? ` | ${preview}` : ''}`); + + if (outputDir) { + fs.writeFileSync(path.join(outputDir, `${fileId}.bin`), fileData); + } + } + + if (outputDir) { + console.log(`Wrote decoded files to: ${outputDir}`); + } +} + +try { + await main(); +} catch (err) { + console.error(err); + process.exit(1); +} diff --git a/tools/cache/debug/dumpJs5NameHashes.ts b/tools/cache/debug/dumpJs5NameHashes.ts new file mode 100644 index 000000000..74c50f2e6 --- /dev/null +++ b/tools/cache/debug/dumpJs5NameHashes.ts @@ -0,0 +1,330 @@ +import fs from 'fs'; +import path from 'path'; + +import Packet from '#/io/Packet.js'; +import { unpackJs5Group } from '#/io/Js5Group.js'; +import { ensureDir, loadIndexPacked, readJs5Id } from '#tools/cache/lib/js5Tools.js'; + +type Scope = 'group' | 'file' | 'both'; +type Algo = 'js5' | 'jag'; + +type Args = { + archive?: number; + type?: string; + indexPath?: string; + openrs2: boolean; + scope: Scope; + group?: number; + id?: number; + text?: string; + out?: string; + algo: Algo; + help: boolean; +}; + +type ParsedHashes = { + archive: number; + hasNames: boolean; + format: number; + version?: number; + groupHashes: Map; + fileHashesByGroup: Map>; +}; + +const TYPE_TO_ARCHIVE: Record = { + flu: 1, + flo: 4, + inv: 5, + midi: 6, + enum: 17, + quickchat: 24, + quickchat_global: 25, + chatcat: 24, + chatphrase: 24, + global_chatcat: 25, + global_chatphrase: 25 +}; + +function parseArgs(argv: string[]): Args { + const args: Args = { + openrs2: false, + scope: 'group', + algo: 'js5', + help: false + }; + + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + + if (arg === '--archive') { + args.archive = Number(argv[++i]); + } else if (arg === '--type') { + args.type = argv[++i]?.toLowerCase(); + } else if (arg === '--index') { + args.indexPath = argv[++i]; + } else if (arg === '--openrs2') { + args.openrs2 = true; + } else if (arg === '--scope') { + const scope = argv[++i] as Scope; + if (scope === 'group' || scope === 'file' || scope === 'both') { + args.scope = scope; + } + } else if (arg === '--group') { + args.group = Number(argv[++i]); + } else if (arg === '--id') { + args.id = Number(argv[++i]); + } else if (arg === '--text') { + args.text = argv[++i]; + } else if (arg === '--out') { + args.out = argv[++i]; + } else if (arg === '--algo') { + const algo = argv[++i]?.toLowerCase(); + if (algo === 'js5' || algo === 'jag') { + args.algo = algo; + } + } else if (arg === '--help' || arg === '-h') { + args.help = true; + } + } + + return args; +} + +function hashJs5(text: string): number { + let hash = 0; + for (let i = 0; i < text.length; i++) { + hash = ((hash * 31) + text.charCodeAt(i)) | 0; + } + return hash; +} + +function hashJag(text: string): number { + let hash = 0; + const upper = text.toUpperCase(); + for (let i = 0; i < upper.length; i++) { + hash = (hash * 61 + upper.charCodeAt(i) - 32) | 0; + } + return hash; +} + +function resolveArchive(args: Args): number { + if (typeof args.archive === 'number' && !Number.isNaN(args.archive)) { + return args.archive; + } + + if (args.type) { + const numericType = Number(args.type); + if (!Number.isNaN(numericType)) { + return numericType; + } + + const alias = TYPE_TO_ARCHIVE[args.type]; + if (typeof alias === 'number') { + return alias; + } + } + + throw new Error('Missing archive. Use --archive or --type .'); +} + +function defaultOutPath(archive: number, type?: string): string { + const label = type && type.length > 0 ? type : `archive_${archive}`; + return path.join('data', 'pack', `${label}.hash`); +} + +function parseNameHashes(indexUnpacked: Uint8Array, archive: number): ParsedHashes { + const packet = new Packet(indexUnpacked); + + const format = packet.g1(); + if (format < 5 || format > 7) { + throw new Error(`Unsupported JS5 index format: ${format}`); + } + + let version: number | undefined; + if (format >= 6) { + version = packet.g4s(); + } + + const flags = packet.g1(); + const hasNames = (flags & 0x1) !== 0; + + const groupCount = readJs5Id(packet, format); + const groupIds: number[] = new Array(groupCount); + + let previousGroupId = 0; + for (let i = 0; i < groupCount; i++) { + previousGroupId += readJs5Id(packet, format); + groupIds[i] = previousGroupId; + } + + const groupHashes = new Map(); + if (hasNames) { + for (let i = 0; i < groupCount; i++) { + groupHashes.set(groupIds[i], packet.g4s()); + } + } + + for (let i = 0; i < groupCount; i++) { + packet.g4s(); + } + + for (let i = 0; i < groupCount; i++) { + packet.g4s(); + } + + const fileCounts: number[] = new Array(groupCount); + for (let i = 0; i < groupCount; i++) { + fileCounts[i] = readJs5Id(packet, format); + } + + const fileIdsByGroup = new Map(); + for (let i = 0; i < groupCount; i++) { + const fileCount = fileCounts[i]; + const fileIds: number[] = new Array(fileCount); + let previousFileId = 0; + + for (let j = 0; j < fileCount; j++) { + previousFileId += readJs5Id(packet, format); + fileIds[j] = previousFileId; + } + + fileIdsByGroup.set(groupIds[i], fileIds); + } + + const fileHashesByGroup = new Map>(); + if (hasNames) { + for (let i = 0; i < groupCount; i++) { + const groupId = groupIds[i]; + const fileIds = fileIdsByGroup.get(groupId) ?? []; + const hashMap = new Map(); + + for (let j = 0; j < fileIds.length; j++) { + hashMap.set(fileIds[j], packet.g4s()); + } + + fileHashesByGroup.set(groupId, hashMap); + } + } + + return { + archive, + hasNames, + format, + version, + groupHashes, + fileHashesByGroup + }; +} + +function buildLines(parsed: ParsedHashes, scope: Scope, groupFilter?: number): string[] { + const lines: string[] = []; + + if (scope === 'group' || scope === 'both') { + const groupIds = Array.from(parsed.groupHashes.keys()).sort((a, b) => a - b); + for (const groupId of groupIds) { + lines.push(`${groupId}=${parsed.groupHashes.get(groupId)}`); + } + } + + if (scope === 'file' || scope === 'both') { + const groupIds = Array.from(parsed.fileHashesByGroup.keys()).sort((a, b) => a - b); + for (const groupId of groupIds) { + if (typeof groupFilter === 'number' && groupFilter !== groupId) { + continue; + } + + const fileHashMap = parsed.fileHashesByGroup.get(groupId); + if (!fileHashMap) { + continue; + } + + const fileIds = Array.from(fileHashMap.keys()).sort((a, b) => a - b); + for (const fileId of fileIds) { + lines.push(`${groupId}:${fileId}=${fileHashMap.get(fileId)}`); + } + } + } + + return lines; +} + +function verifyTarget(parsed: ParsedHashes, args: Args): void { + if (typeof args.id !== 'number' && typeof args.text !== 'string') { + return; + } + + const computed = typeof args.text === 'string' + ? (args.algo === 'jag' ? hashJag(args.text) : hashJs5(args.text)) + : undefined; + + if (typeof args.text === 'string') { + console.log(`text='${args.text}' algo=${args.algo} hash=${computed}`); + } + + if (typeof args.id !== 'number') { + return; + } + + if (args.scope === 'file') { + if (typeof args.group !== 'number') { + throw new Error('--scope file with --id requires --group to disambiguate file IDs.'); + } + + const fileHash = parsed.fileHashesByGroup.get(args.group)?.get(args.id); + if (typeof fileHash === 'undefined') { + console.log(`No file hash found for group ${args.group}, file ${args.id}`); + return; + } + + console.log(`target=file ${args.group}:${args.id} hash=${fileHash}`); + if (typeof computed === 'number') { + console.log(`match=${computed === fileHash}`); + } + return; + } + + const groupHash = parsed.groupHashes.get(args.id); + if (typeof groupHash === 'undefined') { + console.log(`No group hash found for group ${args.id}`); + return; + } + + console.log(`target=group ${args.id} hash=${groupHash}`); + if (typeof computed === 'number') { + console.log(`match=${computed === groupHash}`); + } +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + if (args.help) { + process.exit(0); + } + + const archive = resolveArchive(args); + const indexPath = args.indexPath || `data/cache/255/${archive}.dat`; + const indexPacked = await loadIndexPacked(archive, indexPath, args.openrs2); + const indexUnpacked = unpackJs5Group(indexPacked); + + const parsed = parseNameHashes(indexUnpacked, archive); + + console.log(`archive=${archive} format=${parsed.format} version=${parsed.version ?? 'n/a'} hasNames=${parsed.hasNames}`); + + if (!parsed.hasNames) { + console.log('This archive index does not contain name-hash tables (flags & 0x1 == 0).'); + return; + } + + verifyTarget(parsed, args); + + const lines = buildLines(parsed, args.scope, args.group); + const outPath = args.out || defaultOutPath(archive, args.type); + ensureDir(path.dirname(outPath)); + fs.writeFileSync(outPath, lines.join('\n') + '\n', 'utf-8'); + console.log(`Wrote ${lines.length} entries to ${outPath}`); +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/tools/cache/debug/extractFiles.ts b/tools/cache/debug/extractFiles.ts new file mode 100644 index 000000000..b6ea6cb6e --- /dev/null +++ b/tools/cache/debug/extractFiles.ts @@ -0,0 +1,44 @@ +import * as fs from 'fs'; +import * as zlib from 'zlib'; + +const data = fs.readFileSync('data/cache/17/0.dat'); +const uncompressed = zlib.gunzipSync(data.subarray(9)); + +const stripeCount = uncompressed[uncompressed.length - 1]; +const chunkTableSize = 256 * 4 * stripeCount; +const chunkTableStart = uncompressed.length - 1 - chunkTableSize; + +// Parse chunk table to get file sizes +const fileSizes: number[] = []; +for (let fileId = 0; fileId < 256; fileId++) { + let totalSize = 0; + for (let stripe = 0; stripe < stripeCount; stripe++) { + const offset = chunkTableStart + (stripe * 256 * 4) + (fileId * 4); + const deltaSize = (uncompressed[offset] << 24) | (uncompressed[offset + 1] << 16) | (uncompressed[offset + 2] << 8) | uncompressed[offset + 3]; + totalSize += deltaSize; + } + fileSizes.push(totalSize); +} + +// Extract actual file data +const fileOffsets: number[] = [0]; +for (let i = 0; i < 255; i++) { + fileOffsets.push(fileOffsets[i] + fileSizes[i]); +} + +// Show the bytes for file 0 (has data) and file 1 (empty) +console.log('File 0 (size:', fileSizes[0], 'bytes):'); +const file0Data = uncompressed.subarray(fileOffsets[0], fileOffsets[0] + fileSizes[0]); +console.log(' Hex:', Array.from(file0Data).map(b => b.toString(16).padStart(2, '0')).join(' ')); + +console.log('\nFile 1 (size:', fileSizes[1], 'bytes):'); +if (fileSizes[1] > 0) { + const file1Data = uncompressed.subarray(fileOffsets[1], fileOffsets[1] + fileSizes[1]); + console.log(' Hex:', Array.from(file1Data).map(b => b.toString(16).padStart(2, '0')).join(' ')); +} else { + console.log(' (no data)'); +} + +console.log('\nFile 73 (first non-zero after file 0, size:', fileSizes[73], 'bytes):'); +const file73Data = uncompressed.subarray(fileOffsets[73], fileOffsets[73] + Math.min(fileSizes[73], 20)); +console.log(' First 20 bytes:', Array.from(file73Data).map(b => b.toString(16).padStart(2, '0')).join(' ')); diff --git a/tools/cache/debug/groupFileCount.ts b/tools/cache/debug/groupFileCount.ts new file mode 100644 index 000000000..08e7a3367 --- /dev/null +++ b/tools/cache/debug/groupFileCount.ts @@ -0,0 +1,56 @@ +import { getGroup } from '#/util/OpenRS2.js'; +import { parseJs5ArchiveIndex } from '#/io/Js5ArchiveIndex.js'; +import { unpackJs5Group } from '#/io/Js5Group.js'; + +type Args = { + archive: number; + group: number; +}; + +function parseArgs(argv: string[]): Args | null { + let archive: number | undefined; + let group: number | undefined; + + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + + if (arg === '--archive') { + archive = Number(argv[++i]); + } else if (arg === '--group') { + group = Number(argv[++i]); + } else if (arg === '--help' || arg === '-h') { + return null; + } + } + + if (archive === undefined || group === undefined) { + return null; + } + + return { archive, group }; +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + if (!args) { + process.exit(1); + } + + const indexData = await getGroup(255, args.archive); + const indexUnpacked = unpackJs5Group(indexData); + const { fileIdsByGroup } = parseJs5ArchiveIndex(indexUnpacked); + + const fileIds = fileIdsByGroup.get(args.group); + + if (!fileIds) { + console.log(`Archive ${args.archive}, Group ${args.group}: No files found (group does not exist)`); + process.exit(0); + } + + console.log(`Archive ${args.archive}, Group ${args.group}: ${fileIds.length} files`); +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/tools/cache/debug/matchJs5Hashes.ts b/tools/cache/debug/matchJs5Hashes.ts new file mode 100644 index 000000000..bb6acd83d --- /dev/null +++ b/tools/cache/debug/matchJs5Hashes.ts @@ -0,0 +1,512 @@ +import fs from 'fs'; +import path from 'path'; + +import { ensureDir } from '#tools/cache/lib/js5Tools.js'; + +type Args = { + archive?: number; + type?: string; + hashFile?: string; + keywords?: string; + out?: string; + id?: string; + pairs: boolean; + joiners: string[]; + mutate: boolean; + numStart: number; + numEnd: number; + prefixes: string[]; + suffixes: string[]; + maxPerId: number; + help: boolean; +}; + +type TargetHash = { + id: string; + hash: number; +}; + +type WordInfo = { + word: string; + hash: number; + len: number; +}; + +const DEFAULT_KEYWORDS_URL = path.join('data', 'pack', 'keywords.txt'); + +const TYPE_TO_ARCHIVE: Record = { + flu: 1, + flo: 4, + inv: 5, + midi: 6, + enum: 17, + quickchat: 24, + quickchat_global: 25, + chatcat: 24, + chatphrase: 24, + global_chatcat: 25, + global_chatphrase: 25 +}; + +function parseArgs(argv: string[]): Args { + const args: Args = { + pairs: false, + joiners: ['_'], + mutate: false, + numStart: 0, + numEnd: 999, + prefixes: [], + suffixes: [], + maxPerId: 50, + help: false + }; + + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + + if (arg === '--archive') { + args.archive = Number(argv[++i]); + } else if (arg === '--type') { + args.type = argv[++i]?.toLowerCase(); + } else if (arg === '--hash-file') { + args.hashFile = argv[++i]; + } else if (arg === '--keywords') { + args.keywords = argv[++i]; + } else if (arg === '--out') { + args.out = argv[++i]; + } else if (arg === '--id') { + args.id = argv[++i]; + } else if (arg === '--pairs') { + args.pairs = true; + } else if (arg === '--mutate') { + args.mutate = true; + } else if (arg === '--joiners') { + const raw = argv[++i] ?? ''; + const parsed = raw.split(',').map(part => part === '' ? '' : part); + args.joiners = parsed.length > 0 ? parsed : ['_']; + } else if (arg === '--num-start') { + args.numStart = Number(argv[++i]); + } else if (arg === '--num-end') { + args.numEnd = Number(argv[++i]); + } else if (arg === '--prefixes') { + const raw = argv[++i] ?? ''; + args.prefixes = raw.split(',').map(part => part === '' ? '' : part).filter(part => part.length > 0); + } else if (arg === '--suffixes') { + const raw = argv[++i] ?? ''; + args.suffixes = raw.split(',').map(part => part === '' ? '' : part).filter(part => part.length > 0); + } else if (arg === '--max-per-id') { + args.maxPerId = Number(argv[++i]); + } else if (arg === '--help' || arg === '-h') { + args.help = true; + } + } + + return args; +} + +function resolveArchive(args: Args): number | undefined { + if (typeof args.archive === 'number' && !Number.isNaN(args.archive)) { + return args.archive; + } + + if (args.type) { + const numericType = Number(args.type); + if (!Number.isNaN(numericType)) { + return numericType; + } + + const alias = TYPE_TO_ARCHIVE[args.type]; + if (typeof alias === 'number') { + return alias; + } + } + + return undefined; +} + +function defaultLabel(args: Args): string { + if (args.type && args.type.length > 0) { + return args.type; + } + + const archive = resolveArchive(args); + return typeof archive === 'number' ? `archive_${archive}` : 'hashes'; +} + +function resolveHashFilePath(args: Args): string { + if (args.hashFile) { + return args.hashFile; + } + + return path.join('data', 'pack', `${defaultLabel(args)}.hash`); +} + +function resolveOutPath(args: Args): string { + if (args.out) { + return args.out; + } + + return path.join('data', 'pack', `${defaultLabel(args)}.matches.txt`); +} + +function hashJs5(text: string): number { + let hash = 0; + for (let i = 0; i < text.length; i++) { + hash = ((hash * 31) + text.charCodeAt(i)) | 0; + } + return hash; +} + +function toUint32(value: number): bigint { + return BigInt(value >>> 0); +} + +function toInt32(value: bigint): number { + const normalized = Number(value & 0xffffffffn); + return normalized > 0x7fffffff ? normalized - 0x100000000 : normalized; +} + +function parseHashFile(filePath: string, idFilter?: string): TargetHash[] { + if (!fs.existsSync(filePath)) { + throw new Error(`Hash file not found: ${filePath}`); + } + + const targets: TargetHash[] = []; + for (const line of fs.readFileSync(filePath, 'utf-8').split('\n')) { + const trimmed = line.trim(); + if (trimmed.length === 0 || trimmed.startsWith('#')) { + continue; + } + + const eq = trimmed.indexOf('='); + if (eq === -1) { + continue; + } + + const id = trimmed.substring(0, eq); + const hashValue = Number(trimmed.substring(eq + 1)); + if (Number.isNaN(hashValue)) { + continue; + } + + if (idFilter && id !== idFilter) { + continue; + } + + targets.push({ id, hash: hashValue | 0 }); + } + + return targets; +} + +async function loadKeywords(source?: string): Promise { + const keywordSource = source ?? DEFAULT_KEYWORDS_URL; + let content: string; + + if (keywordSource.startsWith('http://') || keywordSource.startsWith('https://')) { + const response = await fetch(keywordSource); + if (!response.ok) { + throw new Error(`Failed to fetch keywords from ${keywordSource}: ${response.status} ${response.statusText}`); + } + content = await response.text(); + } else { + if (!fs.existsSync(keywordSource)) { + throw new Error(`Keywords file not found: ${keywordSource}`); + } + content = fs.readFileSync(keywordSource, 'utf-8'); + } + + const unique = new Set(); + for (const line of content.split('\n')) { + const normalized = line.trim().toLowerCase(); + if (normalized.length === 0 || normalized.startsWith('#')) { + continue; + } + unique.add(normalized); + } + + return Array.from(unique); +} + +function buildWordInfo(words: string[]): WordInfo[] { + return words.map((word) => ({ + word, + hash: hashJs5(word), + len: word.length + })); +} + +function addMatch(matchesById: Map>, id: string, candidate: string, maxPerId: number): void { + let set = matchesById.get(id); + if (!set) { + set = new Set(); + matchesById.set(id, set); + } + + if (set.size < maxPerId) { + set.add(candidate); + } +} + +function addCandidateMatches( + candidate: string, + byHash: Map, + matchesById: Map>, + maxPerId: number +): void { + const hitIds = byHash.get(hashJs5(candidate)); + if (!hitIds) { + return; + } + + for (const id of hitIds) { + addMatch(matchesById, id, candidate, maxPerId); + } +} + +function matchSingles(targets: TargetHash[], words: WordInfo[], maxPerId: number): Map> { + const byHash = new Map(); + for (const target of targets) { + const list = byHash.get(target.hash) ?? []; + list.push(target.id); + byHash.set(target.hash, list); + } + + const matches = new Map>(); + for (const word of words) { + const hitIds = byHash.get(word.hash); + if (!hitIds) { + continue; + } + + for (const id of hitIds) { + addMatch(matches, id, word.word, maxPerId); + } + } + + return matches; +} + +function precomputePow31(maxLen: number): bigint[] { + const pow: bigint[] = new Array(maxLen + 1); + pow[0] = 1n; + for (let i = 1; i <= maxLen; i++) { + pow[i] = (pow[i - 1] * 31n) & 0xffffffffn; + } + return pow; +} + +function modInverseOdd32(value: bigint): bigint { + const modulus = 0x100000000n; + let a = value & 0xffffffffn; + let b = modulus; + let x0 = 1n; + let x1 = 0n; + + while (b !== 0n) { + const q = a / b; + const tA = a - q * b; + a = b; + b = tA; + + const tX = x0 - q * x1; + x0 = x1; + x1 = tX; + } + + if (a !== 1n) { + throw new Error('Value is not invertible modulo 2^32'); + } + + return (x0 % modulus + modulus) % modulus; +} + +function matchPairs( + targets: TargetHash[], + words: WordInfo[], + joiners: string[], + maxPerId: number, + seed: Map> +): Map> { + const matches = new Map(seed); + const byHash = new Map(); + + for (const word of words) { + const list = byHash.get(word.hash) ?? []; + list.push(word.word); + byHash.set(word.hash, list); + } + + const maxWordLen = words.reduce((acc, word) => Math.max(acc, word.len), 0); + const maxJoinerLen = joiners.reduce((acc, joiner) => Math.max(acc, joiner.length), 0); + const pow31 = precomputePow31(maxWordLen + maxJoinerLen); + const invPow31 = pow31.map((value) => modInverseOdd32(value)); + + const targetU = targets.map((target) => ({ + id: target.id, + hash: target.hash, + hashU: toUint32(target.hash) + })); + + for (const b of words) { + const bHashU = toUint32(b.hash); + + for (const joiner of joiners) { + if (joiner.length > 1) { + continue; + } + + const joinerHashU = joiner.length === 0 ? 0n : toUint32(joiner.charCodeAt(0)); + const tailLen = b.len + joiner.length; + const tailPow = pow31[b.len]; + + const tailConstU = joiner.length === 0 + ? bHashU + : ((joinerHashU * tailPow) + bHashU) & 0xffffffffn; + + const inv = invPow31[tailLen]; + + for (const target of targetU) { + const neededU = (((target.hashU - tailConstU + 0x100000000n) & 0xffffffffn) * inv) & 0xffffffffn; + const needed = toInt32(neededU); + const candidatesA = byHash.get(needed); + if (!candidatesA) { + continue; + } + + for (const a of candidatesA) { + const candidate = `${a}${joiner}${b.word}`; + if (hashJs5(candidate) === target.hash) { + addMatch(matches, target.id, candidate, maxPerId); + } + } + } + } + } + + return matches; +} + +function matchMutations( + targets: TargetHash[], + words: WordInfo[], + args: Args, + seed: Map> +): { matches: Map>; generated: number } { + const matches = new Map(seed); + const byHash = new Map(); + + for (const target of targets) { + const list = byHash.get(target.hash) ?? []; + list.push(target.id); + byHash.set(target.hash, list); + } + + const unique = new Set(); + let generated = 0; + const joiners = Array.from(new Set(['', ...args.joiners])); + + const emit = (candidate: string) => { + if (candidate.length === 0 || unique.has(candidate)) { + return; + } + unique.add(candidate); + generated++; + addCandidateMatches(candidate, byHash, matches, args.maxPerId); + }; + + const min = Number.isFinite(args.numStart) ? Math.max(0, Math.floor(args.numStart)) : 0; + const max = Number.isFinite(args.numEnd) ? Math.max(min, Math.floor(args.numEnd)) : min; + + for (const word of words) { + const base = word.word; + + for (const joiner of joiners) { + for (let n = min; n <= max; n++) { + emit(`${base}${joiner}${n}`); + emit(`${n}${joiner}${base}`); + } + } + + for (const prefix of args.prefixes) { + for (const joiner of joiners) { + emit(`${prefix}${joiner}${base}`); + } + } + + for (const suffix of args.suffixes) { + for (const joiner of joiners) { + emit(`${base}${joiner}${suffix}`); + } + } + + for (const prefix of args.prefixes) { + for (const suffix of args.suffixes) { + for (const joiner of joiners) { + emit(`${prefix}${joiner}${base}${joiner}${suffix}`); + } + } + } + } + + return { matches, generated }; +} + +function writeMatches(outPath: string, targets: TargetHash[], matchesById: Map>): void { + const lines: string[] = []; + for (const target of targets) { + const matches = Array.from(matchesById.get(target.id) ?? []); + if (matches.length === 0) { + continue; + } + + for (const candidate of matches) { + lines.push(`${target.id}\t${target.hash}\t${candidate}`); + } + } + + fs.mkdirSync(path.dirname(outPath), { recursive: true }); + fs.writeFileSync(outPath, lines.join('\n') + (lines.length > 0 ? '\n' : ''), 'utf-8'); +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + if (args.help) { + process.exit(0); + } + + const hashFilePath = resolveHashFilePath(args); + const targets = parseHashFile(hashFilePath, args.id); + if (targets.length === 0) { + throw new Error(`No target hashes found in ${hashFilePath}${args.id ? ` for id ${args.id}` : ''}`); + } + + const keywords = await loadKeywords(args.keywords); + const words = buildWordInfo(keywords); + + let matches = matchSingles(targets, words, args.maxPerId); + if (args.pairs) { + matches = matchPairs(targets, words, args.joiners, args.maxPerId, matches); + } + let generated = 0; + if (args.mutate) { + const mutationResult = matchMutations(targets, words, args, matches); + matches = mutationResult.matches; + generated = mutationResult.generated; + } + + const outPath = resolveOutPath(args); + ensureDir(path.dirname(outPath)); + writeMatches(outPath, targets, matches); + + const matchedIds = targets.filter((target) => (matches.get(target.id)?.size ?? 0) > 0).length; + const totalMatches = Array.from(matches.values()).reduce((sum, set) => sum + set.size, 0); + + console.log(`targets=${targets.length} keywords=${words.length} matchedIds=${matchedIds} matches=${totalMatches} generated=${generated}`); + console.log(`Wrote ${outPath}`); +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/tools/cache/debug/validateCRC.ts b/tools/cache/debug/validateCRC.ts new file mode 100644 index 000000000..1082345b5 --- /dev/null +++ b/tools/cache/debug/validateCRC.ts @@ -0,0 +1,25 @@ +import * as fs from 'fs'; +import { createHash } from 'crypto'; +import { unpackJs5Group } from '#/io/Js5Group.js'; + +console.log('Validating CRC of uncompressed enum data:\n'); + +for (let groupId = 0; groupId <= 8; groupId++) { + const origPath = `data/cache/17/${groupId}.dat`; + const packedPath = `data/pack/17/${groupId}.dat`; + + if (!fs.existsSync(origPath) || !fs.existsSync(packedPath)) { + continue; + } + + const orig = fs.readFileSync(origPath); + const origUncomp = unpackJs5Group(new Uint8Array(orig)); + const origCrc = createHash('md5').update(origUncomp).digest('hex'); + + const packed = fs.readFileSync(packedPath); + const packedUncomp = unpackJs5Group(new Uint8Array(packed)); + const packedCrc = createHash('md5').update(packedUncomp).digest('hex'); + + const match = origCrc === packedCrc; + console.log(`Group ${groupId}: ${match ? '✓ MATCH' : '✗ DIFFER'} (orig=${origCrc.substring(0, 8)}... packed=${packedCrc.substring(0, 8)}...)`); +} diff --git a/tools/cache/lib/configSource.ts b/tools/cache/lib/configSource.ts new file mode 100644 index 000000000..58ce83b79 --- /dev/null +++ b/tools/cache/lib/configSource.ts @@ -0,0 +1,87 @@ +export type SourceField = { + key: string; + value: string; + line: number; +}; + +export type SourceSection = { + name: string; + line: number; + fields: SourceField[]; +}; + +export function parseBracketedConfigSource(content: string): SourceSection[] { + const lines = content.split('\n'); + const sections: SourceSection[] = []; + let current: SourceSection | null = null; + + for (let i = 0; i < lines.length; i++) { + const raw = lines[i]; + const line = raw.endsWith('\r') ? raw.slice(0, -1) : raw; + const trimmed = line.trim(); + + if (trimmed.length === 0 || trimmed.startsWith('//')) { + continue; + } + + if (trimmed.startsWith('[') && trimmed.endsWith(']')) { + current = { + name: trimmed.substring(1, trimmed.length - 1), + line: i + 1, + fields: [] + }; + sections.push(current); + continue; + } + + if (!current) { + continue; + } + + const eq = line.indexOf('='); + if (eq === -1) { + continue; + } + + const key = line.substring(0, eq).trim(); + const value = line.substring(eq + 1).trim(); + if (key.length === 0) { + continue; + } + + current.fields.push({ key, value, line: i + 1 }); + } + + return sections; +} + +export function resolveSectionId(name: string, nameToId: Map, fallbackPrefix: string): number | null { + const mapped = nameToId.get(name); + if (mapped !== undefined) { + return mapped; + } + + if (name.startsWith(fallbackPrefix)) { + const parsed = parseInt(name.substring(fallbackPrefix.length)); + if (!isNaN(parsed)) { + return parsed; + } + } + + return null; +} + +export function parseConfigBoolean(value: string): boolean { + const normalized = value.trim().toLowerCase(); + return normalized === 'yes' || normalized === 'true' || normalized === '1'; +} + +export function parseConfigInteger(value: string): number { + if (value.startsWith('0x') || value.startsWith('0X')) { + const parsedHex = parseInt(value, 16); + return isNaN(parsedHex) ? 0 : parsedHex; + } + + const parsed = parseInt(value); + return isNaN(parsed) ? 0 : parsed; +} diff --git a/tools/cache/lib/dumpTools.ts b/tools/cache/lib/dumpTools.ts new file mode 100644 index 000000000..195b3f118 --- /dev/null +++ b/tools/cache/lib/dumpTools.ts @@ -0,0 +1,18 @@ +import fs from 'fs'; +import path from 'path'; + +import { ensureDir } from '#tools/cache/lib/js5Tools.js'; + +export function writeTextDump(outPath: string, lines: string[]): void { + ensureDir(path.dirname(outPath)); + fs.writeFileSync(outPath, lines.join('\n'), 'utf-8'); +} + +export function writeBinaryDump(outPath: string, bytes: Uint8Array): void { + ensureDir(path.dirname(outPath)); + fs.writeFileSync(outPath, bytes); +} + +export function formatRgbHex(value: number): string { + return `0x${value.toString(16).toUpperCase().padStart(6, '0')}`; +} diff --git a/tools/cache/lib/enumCodec.ts b/tools/cache/lib/enumCodec.ts new file mode 100644 index 000000000..608580b22 --- /dev/null +++ b/tools/cache/lib/enumCodec.ts @@ -0,0 +1,64 @@ +import ScriptVarType from '#/cache/config/ScriptVarType.js'; +import Packet from '#/io/Packet.js'; +import EnumType from '#/cache/config/EnumType.js'; + +export function encodeEnum(config: EnumType): Uint8Array { + const buf = Packet.alloc(2); + + const hasContent = config.values.size > 0 || + config.inputtype !== ScriptVarType.INT || + config.outputtype !== ScriptVarType.INT || + (config.outputtype === ScriptVarType.STRING && config.defaultString !== 'null' && config.defaultString !== '') || + (config.outputtype !== ScriptVarType.STRING && config.defaultInt !== 0); + + if (hasContent) { + buf.p1(1); + buf.p1(config.inputtype); + } + + if (hasContent) { + buf.p1(2); + buf.p1(config.outputtype); + } + + if (config.values.size > 0) { + const isStringValues = config.outputtype === ScriptVarType.STRING; + + if (isStringValues) { + buf.p1(5); + buf.p2(config.values.size); + for (const [key, value] of config.values) { + buf.p4(key); + buf.pjstr(String(value)); + } + } else { + buf.p1(6); + buf.p2(config.values.size); + for (const [key, value] of config.values) { + buf.p4(key); + buf.p4(Number(value)); + } + } + } + + if (config.outputtype === ScriptVarType.STRING) { + if (config.hasExplicitDefaultString || (config.defaultString !== 'null' && config.defaultString !== '')) { + buf.p1(3); + buf.pjstr(config.defaultString); + } + } else { + if (config.hasExplicitDefaultInt || config.defaultInt !== 0) { + buf.p1(4); + buf.p4(config.defaultInt); + } + } + + if (config.debugname) { + buf.p1(250); + buf.pjstr(config.debugname); + } + + buf.p1(0); + + return new Uint8Array(buf.data.subarray(0, buf.pos)); +} diff --git a/tools/cache/lib/floCodec.ts b/tools/cache/lib/floCodec.ts new file mode 100644 index 000000000..609ee794b --- /dev/null +++ b/tools/cache/lib/floCodec.ts @@ -0,0 +1,147 @@ +import Packet from '#/io/Packet.js'; +import FloType from '#/cache/config/FloType.js'; + +type OpcodeValue = { + code: number; + value?: any; +}; + +export function encodeFloWithOpcodes(config: FloType, opcodes: OpcodeValue[]): Uint8Array { + const buf = Packet.alloc(2); + + for (const opc of opcodes) { + const code = opc.code; + if (code === 1) { + // colour (RGB24) + buf.p1(1); + buf.p3(opc.value ?? config.colour); + } else if (code === 2) { + // material (byte) + buf.p1(2); + buf.p1(opc.value ?? config.material); + } else if (code === 3) { + // material (word) + buf.p1(3); + const matValue = opc.value ?? config.material; + buf.p2(matValue === -1 ? 65535 : matValue); + } else if (code === 5) { + // occlude=no + buf.p1(5); + } else if (code === 6) { + // debugname + buf.p1(6); + buf.pjstr(opc.value ?? config.debugname ?? ''); + } else if (code === 7) { + // averagecolour (RGB24) + buf.p1(7); + buf.p3(opc.value ?? config.averagecolour); + } else if (code === 8) { + // client-only, no data + buf.p1(8); + } else if (code === 9) { + // materialscale + buf.p1(9); + buf.p2(opc.value ?? config.materialscale); + } else if (code === 10) { + // hardshadow + buf.p1(10); + } else if (code === 11) { + // priority + buf.p1(11); + buf.p1(opc.value ?? config.priority); + } else if (code === 12) { + // blend + buf.p1(12); + } else if (code === 13) { + // waterfogcolour + buf.p1(13); + buf.p3(opc.value ?? config.waterfogcolour); + } else if (code === 14) { + // waterfogscale + buf.p1(14); + buf.p1(opc.value ?? config.waterfogscale); + } + } + + // Terminator + buf.p1(0); + + return new Uint8Array(buf.data.subarray(0, buf.pos)); +} + +export function encodeFlo(config: FloType): Uint8Array { + const buf = Packet.alloc(2); + + // Code 1: colour (RGB24) + if (config.colour !== 0) { + buf.p1(1); + buf.p3(config.colour); + } + + // Code 2/3: materials (texture ID) + if (config.material !== -1) { + if (config.material < 256) { + buf.p1(2); + buf.p1(config.material); + } else { + buf.p1(3); + buf.p2(config.material === -1 ? 65535 : config.material); + } + } + + // Code 5: occlude=no + if (!config.occlude) { + buf.p1(5); + } + + // Code 6: debugname + if (config.debugname) { + buf.p1(6); + buf.pjstr(config.debugname); + } + + // Code 7: averagecolour (RGB24) + if (config.averagecolour !== -1) { + buf.p1(7); + buf.p3(config.averagecolour); + } + + // Code 9: materialscale + if (config.materialscale !== 512) { + buf.p1(9); + buf.p2(config.materialscale); + } + + // Code 10: hardshadow=no + if (!config.hardshadow) { + buf.p1(10); + } + + // Code 11: priority + if (config.priority !== 8) { + buf.p1(11); + buf.p1(config.priority); + } + + // Code 12: blend=yes + if (config.blend) { + buf.p1(12); + } + + // Code 13: waterfogcolour + if (config.waterfogcolour !== 1190717) { + buf.p1(13); + buf.p3(config.waterfogcolour); + } + + // Code 14: waterfogscale + if (config.waterfogscale !== 512) { + buf.p1(14); + buf.p1(config.waterfogscale); + } + + // Terminator + buf.p1(0); + + return new Uint8Array(buf.data.subarray(0, buf.pos)); +} diff --git a/tools/cache/lib/fluCodec.ts b/tools/cache/lib/fluCodec.ts new file mode 100644 index 000000000..dc5c955a2 --- /dev/null +++ b/tools/cache/lib/fluCodec.ts @@ -0,0 +1,64 @@ +import Packet from '#/io/Packet.js'; +import FluType from '#/cache/config/FluType.js'; + +type OpcodeValue = { + code: number; + value?: any; +}; + +export function encodeFluWithOpcodes(config: FluType, opcodes: OpcodeValue[]): Uint8Array { + const buf = Packet.alloc(2); + + for (const opc of opcodes) { + const code = opc.code; + if (code === 1) { + // colour (RGB24) + buf.p1(1); + buf.p3(opc.value ?? config.colour); + } else if (code === 2) { + // material (word) + buf.p1(2); + const matValue = opc.value ?? config.material; + buf.p2(matValue === -1 ? 65535 : matValue); + } else if (code === 3) { + // materialscale + buf.p1(3); + const scaleValue = opc.value ?? config.materialscale; + buf.p2(scaleValue & 0xffff); + } else if (code === 4) { + // hardshadow=no + buf.p1(4); + } + } + + buf.p1(0); + + return new Uint8Array(buf.data.subarray(0, buf.pos)); +} + +export function encodeFlu(config: FluType): Uint8Array { + const buf = Packet.alloc(2); + + if (config.colour !== 0) { + buf.p1(1); + buf.p3(config.colour); + } + + if (config.material !== -1) { + buf.p1(2); + buf.p2(config.material === -1 ? 65535 : config.material); + } + + if (config.materialscale !== 512) { + buf.p1(3); + buf.p2(config.materialscale & 0xffff); + } + + if (!config.hardshadow) { + buf.p1(4); + } + + buf.p1(0); + + return new Uint8Array(buf.data.subarray(0, buf.pos)); +} diff --git a/tools/cache/lib/invCodec.ts b/tools/cache/lib/invCodec.ts new file mode 100644 index 000000000..8fa74204a --- /dev/null +++ b/tools/cache/lib/invCodec.ts @@ -0,0 +1,120 @@ +import Packet from '#/io/Packet.js'; +import InvType from '#/cache/config/InvType.js'; + +type OpcodeValue = { + code: number; + value?: any; +}; + +export function encodeInvWithOpcodes(config: InvType, opcodes: OpcodeValue[]): Uint8Array { + const buf = Packet.alloc(2); + + for (const opc of opcodes) { + const code = opc.code; + if (code === 1) { + // scope + buf.p1(1); + buf.p1(opc.value ?? config.scope); + } else if (code === 2) { + // size + buf.p1(2); + buf.p2(opc.value ?? config.size); + } else if (code === 3) { + // stackall + buf.p1(3); + } else if (code === 4) { + // stock items + buf.p1(4); + const count = opc.value?.length ?? config.stockobj?.length ?? 0; + buf.p1(count); + for (let i = 0; i < count; i++) { + const stockobj = opc.value?.[i]?.obj ?? config.stockobj![i]; + const stockcount = opc.value?.[i]?.count ?? config.stockcount![i]; + const stockrate = opc.value?.[i]?.rate ?? config.stockrate![i]; + buf.p2(stockobj); + buf.p2(stockcount); + buf.p4(stockrate); + } + } else if (code === 5) { + // restock + buf.p1(5); + } else if (code === 6) { + // allstock + buf.p1(6); + } else if (code === 7) { + // protect=no + buf.p1(7); + } else if (code === 8) { + // runweight + buf.p1(8); + } else if (code === 9) { + // dummyinv + buf.p1(9); + } else if (code === 250) { + // debugname + buf.p1(250); + buf.pjstr(opc.value ?? config.debugname ?? ''); + } + } + + buf.p1(0); + + return new Uint8Array(buf.data.subarray(0, buf.pos)); +} + +export function encodeInv(config: InvType): Uint8Array { + const buf = Packet.alloc(2); + + if (config.scope !== 0) { + buf.p1(1); + buf.p1(config.scope); + } + + if (config.size !== 1) { + buf.p1(2); + buf.p2(config.size); + } + + if (config.stackall) { + buf.p1(3); + } + + if (config.stockobj !== null && config.stockobj.length > 0) { + buf.p1(4); + buf.p1(config.stockobj.length); + for (let i = 0; i < config.stockobj.length; i++) { + buf.p2(config.stockobj[i]); + buf.p2(config.stockcount![i]); + buf.p4(config.stockrate![i]); + } + } + + if (config.restock) { + buf.p1(5); + } + + if (config.allstock) { + buf.p1(6); + } + + if (!config.protect) { + buf.p1(7); + } + + if (config.runweight) { + buf.p1(8); + } + + if (config.dummyinv) { + buf.p1(9); + } + + if (config.debugname) { + buf.p1(250); + buf.pjstr(config.debugname); + } + + buf.p1(0); + + return new Uint8Array(buf.data.subarray(0, buf.pos)); +} diff --git a/tools/cache/lib/js5Tools.ts b/tools/cache/lib/js5Tools.ts new file mode 100644 index 000000000..661a27357 --- /dev/null +++ b/tools/cache/lib/js5Tools.ts @@ -0,0 +1,414 @@ +import fs from 'fs'; + +import Packet from '#/io/Packet.js'; +import { getGroup } from '#/util/OpenRS2.js'; +import { unpackJs5Group } from '#/io/Js5Group.js'; +import { CompressionType } from '#/io/CompressionType.js'; +import { parseJs5ArchiveIndex as parseJs5ArchiveIndexCore, splitGroupFiles } from '#/io/Js5ArchiveIndex.js'; + +export function readJs5Id(packet: Packet, format: number): number { + if (format >= 7) { + return packet.gSmart2or4(); + } + + return packet.g2(); +} + +export function parseGroupIdsFromIndex(indexUnpacked: Uint8Array): number[] { + const packet = new Packet(indexUnpacked); + + const format = packet.g1(); + if (format < 5 || format > 7) { + throw new Error(`Unsupported JS5 index format: ${format}`); + } + + if (format >= 6) { + packet.g4s(); + } + + packet.g1(); + + const groupCount = readJs5Id(packet, format); + const groupIds: number[] = new Array(groupCount); + + let previousGroupId = 0; + for (let i = 0; i < groupCount; i++) { + previousGroupId += readJs5Id(packet, format); + groupIds[i] = previousGroupId; + } + + return groupIds; +} + +export function parseGroupIdsFromIndexPacked(indexPacked: Uint8Array): number[] { + return parseGroupIdsFromIndex(unpackJs5Group(indexPacked)); +} + +export function arraysEqual(a: Uint8Array, b: Uint8Array): boolean { + if (a.length !== b.length) { + return false; + } + + for (let i = 0; i < a.length; i++) { + if (a[i] !== b[i]) { + return false; + } + } + + return true; +} + +export function packedContainerLength(bytes: Uint8Array, offset: number): number { + const compression = bytes[offset]; + const compressedLength = + ((bytes[offset + 1] << 24) | (bytes[offset + 2] << 16) | (bytes[offset + 3] << 8) | bytes[offset + 4]) >>> 0; + + if (compression === 0) { + return 5 + compressedLength; + } + + return 9 + compressedLength; +} + +export function readInt32BE(bytes: Uint8Array, offset: number): number { + return ((bytes[offset] << 24) | (bytes[offset + 1] << 16) | (bytes[offset + 2] << 8) | bytes[offset + 3]) | 0; +} + +export function writeInt32BE(value: number, out: Uint8Array, offset: number): void { + out[offset] = (value >>> 24) & 0xff; + out[offset + 1] = (value >>> 16) & 0xff; + out[offset + 2] = (value >>> 8) & 0xff; + out[offset + 3] = value & 0xff; +} + +export async function loadIndexPacked( + archive: number, + indexPath: string, + openrs2: boolean +): Promise { + if (openrs2) { + const raw = await getGroup(255, archive); + return new Uint8Array(raw); + } + + if (!fs.existsSync(indexPath)) { + throw new Error(`Index file not found: ${indexPath}`); + } + + return new Uint8Array(fs.readFileSync(indexPath)); +} + +export async function readGroupBytes( + archive: number, + groupId: number, + dir: string, + fromOpenrs2: boolean +): Promise { + if (fromOpenrs2) { + const data = await getGroup(archive, groupId); + return data && data.length ? new Uint8Array(data) : null; + } + + const path = `${dir}/${archive}/${groupId}.dat`; + if (!fs.existsSync(path)) { + return null; + } + + return new Uint8Array(fs.readFileSync(path)); +} + +export function ensureDir(dirPath: string): void { + if (fs.existsSync(dirPath)) { + const stat = fs.statSync(dirPath); + if (stat.isDirectory()) { + return; + } + throw new Error(`Path exists and is not a directory: ${dirPath}`); + } + + fs.mkdirSync(dirPath, { recursive: true }); +} + +export function parsePackFile(packPath: string): Map { + if (!fs.existsSync(packPath)) { + return new Map(); + } + + const content = fs.readFileSync(packPath, 'utf-8'); + const nameToId = new Map(); + + for (const line of content.split('\n')) { + const trimmed = line.trim(); + if (trimmed.length === 0 || trimmed.startsWith('#')) { + continue; + } + + const eq = trimmed.indexOf('='); + if (eq === -1) { + continue; + } + + const id = parseInt(trimmed.substring(0, eq)); + const name = trimmed.substring(eq + 1); + + if (!isNaN(id) && name.length > 0) { + nameToId.set(name, id); + } + } + + return nameToId; +} + +export function parsePackFileById(packPath: string): Map { + if (!fs.existsSync(packPath)) { + return new Map(); + } + + const content = fs.readFileSync(packPath, 'utf-8'); + const idToName = new Map(); + + for (const line of content.split('\n')) { + const trimmed = line.trim(); + if (trimmed.length === 0 || trimmed.startsWith('#')) { + continue; + } + + const eq = trimmed.indexOf('='); + if (eq === -1) { + continue; + } + + const id = parseInt(trimmed.substring(0, eq)); + const name = trimmed.substring(eq + 1); + + if (!isNaN(id) && name.length > 0) { + idToName.set(id, name); + } + } + + return idToName; +} + +export type Js5ArchiveIndex = { + groupIds: number[]; + fileIdsByGroup: Map; +}; + +export async function loadArchiveFileIds(indexArchive: number, archive: number, openrs2: boolean = true): Promise { + const indexPacked = await loadIndexPacked(indexArchive, `data/cache/255/${indexArchive}.dat`, openrs2); + const indexData = unpackJs5Group(indexPacked); + const { fileIdsByGroup } = parseJs5ArchiveIndexCore(indexData); + const fileIds = fileIdsByGroup.get(archive); + + if (!fileIds) { + throw new Error(`Archive ${archive} not found in index ${indexArchive}`); + } + + return fileIds; +} + +export type LoadedArchiveGroup = { + fileIds: number[]; + groupPacked: Uint8Array; + groupUnpacked: Uint8Array; + files: Map; +}; + +export async function loadArchiveGroupFiles( + indexArchive: number, + archive: number, + groupsDir: string, + openrs2: boolean = true +): Promise { + const fileIds = await loadArchiveFileIds(indexArchive, archive, openrs2); + const groupPacked = await readGroupBytes(indexArchive, archive, groupsDir, openrs2); + + if (!groupPacked) { + throw new Error(`Missing group data for index ${indexArchive}, archive ${archive}`); + } + + const groupUnpacked = unpackJs5Group(groupPacked); + const files = splitGroupFiles(groupUnpacked, fileIds); + + return { + fileIds, + groupPacked, + groupUnpacked, + files + }; +} + +export function parseJs5ArchiveIndex(indexData: Uint8Array): Js5ArchiveIndex { + const packet = new Packet(indexData); + + const format = packet.g1(); + if (format >= 6) { + packet.g4s(); + } + + const flags = packet.g1(); + + const groupCount = readJs5Id(packet, format); + const groupIds: number[] = new Array(groupCount); + let previousGroupId = 0; + for (let i = 0; i < groupCount; i++) { + previousGroupId += readJs5Id(packet, format); + groupIds[i] = previousGroupId; + } + + if ((flags & 0x01) !== 0) { + for (let i = 0; i < groupCount; i++) { + packet.g4s(); + } + } + + for (let i = 0; i < groupCount; i++) { + packet.g4s(); + } + + for (let i = 0; i < groupCount; i++) { + packet.g4s(); + } + + const fileCounts = new Array(groupCount); + for (let i = 0; i < groupCount; i++) { + fileCounts[i] = readJs5Id(packet, format); + } + + const fileIdsByGroup = new Map(); + + for (let i = 0; i < groupCount; i++) { + const fileCount = fileCounts[i]; + const fileIds: number[] = new Array(fileCount); + + let previousFileId = 0; + for (let j = 0; j < fileCount; j++) { + previousFileId += readJs5Id(packet, format); + fileIds[j] = previousFileId; + } + + fileIdsByGroup.set(groupIds[i], fileIds); + } + + return { + groupIds, + fileIdsByGroup + }; +} + +export function loadReferenceArchiveIndex(archive: number): Js5ArchiveIndex | null { + const indexPath = `data/cache/255/${archive}.dat`; + if (!fs.existsSync(indexPath)) { + return null; + } + + const indexRaw = new Uint8Array(fs.readFileSync(indexPath)); + const indexData = unpackJs5Group(indexRaw); + return parseJs5ArchiveIndex(indexData); +} + +export function combineGroupFiles(files: Map, orderedFileIds: number[]): Uint8Array { + if (orderedFileIds.length === 0) { + throw new Error('Cannot combine empty file list.'); + } + + if (orderedFileIds.length === 1) { + const only = files.get(orderedFileIds[0]); + if (!only) { + throw new Error('Missing expected file data for single-file group.'); + } + return only; + } + + const fileCount = orderedFileIds.length; + const fileSizes: number[] = new Array(fileCount).fill(0); + let totalDataSize = 0; + + for (let i = 0; i < fileCount; i++) { + const fileId = orderedFileIds[i]; + const data = files.get(fileId); + if (data && data.length > 0) { + fileSizes[i] = data.length; + totalDataSize += data.length; + } + } + + // Use 1 stripe for simplicity + const stripes = 1; + const tableLength = stripes * fileCount * 4; + const totalSize = totalDataSize + tableLength + 1; + + const output = new Uint8Array(totalSize); + const view = new DataView(output.buffer); + + // Write file data in order (for single stripe, just write files sequentially) + let dataPos = 0; + for (let i = 0; i < fileCount; i++) { + const fileId = orderedFileIds[i]; + const fileData = files.get(fileId); + if (fileData && fileData.length > 0) { + output.set(fileData, dataPos); + dataPos += fileData.length; + } + } + + // Write chunk table + // For JS5 multi-file format: each entry is delta from previous chunk length in the stripe + let tablePos = dataPos; + for (let stripe = 0; stripe < stripes; stripe++) { + let prevChunkLength = 0; + for (let i = 0; i < fileCount; i++) { + const chunkLength = fileSizes[i]; + const delta = chunkLength - prevChunkLength; + view.setInt32(tablePos, delta, false); // big-endian + tablePos += 4; + prevChunkLength = chunkLength; + } + } + + // Write stripe count + output[totalSize - 1] = stripes; + + return output; +} + +export async function compressJs5Group(uncompressed: Uint8Array, compressionType: number): Promise { + let compressed: Uint8Array | null; + + if (compressionType === CompressionType.BZIP2) { + const BZip2 = (await import('#/io/BZip2.js')).default; + compressed = BZip2.compress(uncompressed, false, true, 1); + } else if (compressionType === CompressionType.GZIP) { + const { compressGz } = await import('#/io/GZip.js'); + compressed = compressGz(uncompressed); + } else { + throw new Error(`Unsupported compression type: ${compressionType}`); + } + + if (!compressed) { + throw new Error(`Failed to compress with type ${compressionType}`); + } + + const output = new Uint8Array(9 + compressed.length); + + output[0] = compressionType; + + // Compressed length (big-endian u32) + const compLen = compressed.length; + output[1] = (compLen >>> 24) & 0xff; + output[2] = (compLen >>> 16) & 0xff; + output[3] = (compLen >>> 8) & 0xff; + output[4] = compLen & 0xff; + + // Uncompressed length (big-endian u32) + const uncompLen = uncompressed.length; + output[5] = (uncompLen >>> 24) & 0xff; + output[6] = (uncompLen >>> 16) & 0xff; + output[7] = (uncompLen >>> 8) & 0xff; + output[8] = uncompLen & 0xff; + + // Compressed payload + output.set(compressed, 9); + + return output; +} diff --git a/tools/cache/lib/mesanimCodec.ts b/tools/cache/lib/mesanimCodec.ts new file mode 100644 index 000000000..462e02e2a --- /dev/null +++ b/tools/cache/lib/mesanimCodec.ts @@ -0,0 +1,43 @@ +import Packet from '#/io/Packet.js'; +import MesanimType from '#/cache/config/MesanimType.js'; + +type OpcodeValue = { + code: number; + value?: number; +}; + +export function encodeMesanimWithOpcodes(config: MesanimType, opcodes: OpcodeValue[]): Uint8Array { + const buf = Packet.alloc(2); + + for (const opc of opcodes) { + const code = opc.code; + if (code >= 1 && code <= 4) { + const fallback = config.len[code - 1]; + const value = opc.value ?? fallback; + if (value < 0) { + continue; + } + + buf.p1(code); + buf.p2(value); + } + } + + buf.p1(0); + return new Uint8Array(buf.data.subarray(0, buf.pos)); +} + +export function encodeMesanim(config: MesanimType): Uint8Array { + const buf = Packet.alloc(2); + + for (let i = 0; i < config.len.length; i++) { + const value = config.len[i]; + if (value >= 0) { + buf.p1(i + 1); + buf.p2(value); + } + } + + buf.p1(0); + return new Uint8Array(buf.data.subarray(0, buf.pos)); +} diff --git a/tools/cache/lib/npcCodec.ts b/tools/cache/lib/npcCodec.ts new file mode 100644 index 000000000..2a5f3935b --- /dev/null +++ b/tools/cache/lib/npcCodec.ts @@ -0,0 +1,363 @@ +import Packet from '#/io/Packet.js'; + +export type NpcOpcode = { + code: number; + payload: any; +}; + +function readParams(dat: Packet): Array<{ string: boolean; key: number; value: number | string }> { + const count = dat.g1(); + const entries: Array<{ string: boolean; key: number; value: number | string }> = []; + + for (let i = 0; i < count; i++) { + const isString = dat.g1() === 1; + const key = dat.g3(); + const value = isString ? dat.gjstr() : dat.g4s(); + entries.push({ string: isString, key, value }); + } + + return entries; +} + +function writeParams(buf: Packet, entries: Array<{ string: boolean; key: number; value: number | string }>): void { + buf.p1(entries.length); + + for (const entry of entries) { + buf.p1(entry.string ? 1 : 0); + buf.p3(entry.key); + + if (entry.string) { + buf.pjstr(String(entry.value)); + } else { + buf.p4(Number(entry.value)); + } + } +} + +export function decodeNpcOpcode(code: number, dat: Packet): any { + if (code === 1) { + const count = dat.g1(); + const models: number[] = []; + for (let i = 0; i < count; i++) { + models.push(dat.g2()); + } + return { models }; + } + + if (code === 2 || code === 3 || (code >= 30 && code < 35) || code === 250) { + return dat.gjstr(); + } + + if (code === 12 || code === 128 || code === 202 || code === 206 || code === 208 || code === 209 || code === 210) { + return dat.g1(); + } + + if (code === 13 || code === 14 || code === 18 || (code >= 74 && code <= 79) || code === 90 || code === 91 || code === 92 || code === 95 || code === 97 || code === 98 || code === 102 || code === 103 || code === 122 || code === 123 || code === 127 || code === 137 || code === 200 || code === 201 || code === 203 || code === 204 || code === 207) { + return dat.g2(); + } + + if (code === 16 || code === 93 || code === 99 || code === 107 || code === 109 || code === 111 || code === 211 || code === 213) { + return true; + } + + if (code === 17) { + return { + walkanim: dat.g2(), + walkanim_b: dat.g2(), + walkanim_r: dat.g2(), + walkanim_l: dat.g2() + }; + } + + if (code === 40 || code === 41) { + const count = dat.g1(); + const pairs: Array<{ from: number; to: number }> = []; + for (let i = 0; i < count; i++) { + pairs.push({ from: dat.g2(), to: dat.g2() }); + } + return pairs; + } + + if (code === 42) { + const count = dat.g1(); + const values: number[] = []; + for (let i = 0; i < count; i++) { + values.push(dat.g1b()); + } + return values; + } + + if (code === 60) { + const count = dat.g1(); + const heads: number[] = []; + for (let i = 0; i < count; i++) { + heads.push(dat.g2()); + } + return heads; + } + + if (code === 100 || code === 101 || code === 113 || code === 114 || code === 119 || code === 125) { + if (code === 113) { + return { trans1: dat.g2(), trans2: dat.g2() }; + } + + if (code === 114) { + return { trans1: dat.g1b(), trans2: dat.g1b() }; + } + return dat.g1b(); + } + + if (code === 106 || code === 118) { + const multivarbit = dat.g2(); + const multivarp = dat.g2(); + + let defaultId: number | undefined; + if (code === 118) { + defaultId = dat.g2(); + } + + const count = dat.g1(); + const multinpc: number[] = []; + for (let i = 0; i <= count; i++) { + multinpc.push(dat.g2()); + } + + return { + multivarbit, + multivarp, + defaultId, + multinpc + }; + } + + if (code === 112) { + return { colour1: dat.g2(), colour2: dat.g2() }; + } + + if (code === 115) { + return { value1: dat.g1(), value2: dat.g1() }; + } + + if (code === 121) { + const count = dat.g1(); + const offsets: Array<{ index: number; x: number; y: number; z: number }> = []; + + for (let i = 0; i < count; i++) { + offsets.push({ + index: dat.g1(), + x: dat.g1b(), + y: dat.g1b(), + z: dat.g1b() + }); + } + + return offsets; + } + + if (code === 134) { + return { + bgsound: dat.g2(), + bgsound_crawl: dat.g2(), + bgsound_walk: dat.g2(), + bgsound_run: dat.g2(), + bgsound_range: dat.g1() + }; + } + + if (code === 135 || code === 136) { + return { + op: dat.g1(), + cursor: dat.g2() + }; + } + + if (code === 212) { + const count = dat.g1(); + const patrol: Array<{ coord: number; delay: number }> = []; + for (let i = 0; i < count; i++) { + patrol.push({ coord: dat.g4s(), delay: dat.g1() }); + } + return patrol; + } + + if (code === 249) { + return readParams(dat); + } + + throw new Error(`Unrecognized npc config code: ${code}`); +} + +export function encodeNpcOpcode(buf: Packet, op: NpcOpcode): void { + const { code, payload } = op; + buf.p1(code); + + if (code === 1) { + const models = (payload?.models ?? []) as number[]; + buf.p1(models.length); + for (const model of models) { + buf.p2(model); + } + return; + } + + if (code === 2 || code === 3 || (code >= 30 && code < 35) || code === 250) { + buf.pjstr(String(payload ?? '')); + return; + } + + if (code === 12 || code === 128 || code === 202 || code === 206 || code === 208 || code === 209 || code === 210) { + buf.p1(Number(payload)); + return; + } + + if (code === 13 || code === 14 || code === 18 || (code >= 74 && code <= 79) || code === 90 || code === 91 || code === 92 || code === 95 || code === 97 || code === 98 || code === 102 || code === 103 || code === 122 || code === 123 || code === 127 || code === 137 || code === 200 || code === 201 || code === 203 || code === 204 || code === 207) { + buf.p2(Number(payload)); + return; + } + + if (code === 16 || code === 93 || code === 99 || code === 107 || code === 109 || code === 111 || code === 211 || code === 213) { + return; + } + + if (code === 17) { + buf.p2(Number(payload.walkanim)); + buf.p2(Number(payload.walkanim_b)); + buf.p2(Number(payload.walkanim_r)); + buf.p2(Number(payload.walkanim_l)); + return; + } + + if (code === 40 || code === 41) { + const pairs = (payload ?? []) as Array<{ from: number; to: number }>; + buf.p1(pairs.length); + for (const pair of pairs) { + buf.p2(pair.from); + buf.p2(pair.to); + } + return; + } + + if (code === 42) { + const values = (payload ?? []) as number[]; + buf.p1(values.length); + for (const value of values) { + buf.p1(value & 0xff); + } + return; + } + + if (code === 60) { + const heads = (payload ?? []) as number[]; + buf.p1(heads.length); + for (const head of heads) { + buf.p2(head); + } + return; + } + + if (code === 100 || code === 101 || code === 119 || code === 125) { + buf.p1(Number(payload) & 0xff); + return; + } + + if (code === 113) { + buf.p2(Number(payload.trans1)); + buf.p2(Number(payload.trans2)); + return; + } + + if (code === 114) { + buf.p1(Number(payload.trans1) & 0xff); + buf.p1(Number(payload.trans2) & 0xff); + return; + } + + if (code === 106 || code === 118) { + buf.p2(Number(payload.multivarbit)); + buf.p2(Number(payload.multivarp)); + + if (code === 118) { + buf.p2(Number(payload.defaultId ?? 65535)); + } + + const multinpc = (payload.multinpc ?? []) as number[]; + if (multinpc.length === 0) { + buf.p1(0); + buf.p2(65535); + return; + } + + buf.p1(Math.max(0, multinpc.length - 1)); + for (const npcId of multinpc) { + buf.p2(npcId); + } + return; + } + + if (code === 112) { + buf.p2(Number(payload.colour1)); + buf.p2(Number(payload.colour2)); + return; + } + + if (code === 115) { + buf.p1(Number(payload.value1)); + buf.p1(Number(payload.value2)); + return; + } + + if (code === 121) { + const offsets = (payload ?? []) as Array<{ index: number; x: number; y: number; z: number }>; + buf.p1(offsets.length); + for (const offset of offsets) { + buf.p1(offset.index); + buf.p1(offset.x & 0xff); + buf.p1(offset.y & 0xff); + buf.p1(offset.z & 0xff); + } + return; + } + + if (code === 134) { + buf.p2(Number(payload.bgsound)); + buf.p2(Number(payload.bgsound_crawl)); + buf.p2(Number(payload.bgsound_walk)); + buf.p2(Number(payload.bgsound_run)); + buf.p1(Number(payload.bgsound_range)); + return; + } + + if (code === 135 || code === 136) { + buf.p1(Number(payload.op)); + buf.p2(Number(payload.cursor)); + return; + } + + if (code === 212) { + const patrol = (payload ?? []) as Array<{ coord: number; delay: number }>; + buf.p1(patrol.length); + for (const point of patrol) { + buf.p4(point.coord); + buf.p1(point.delay); + } + return; + } + + if (code === 249) { + writeParams(buf, (payload ?? []) as Array<{ string: boolean; key: number; value: number | string }>); + return; + } + + throw new Error(`Unrecognized npc config code: ${code}`); +} + +export function encodeNpcOps(ops: NpcOpcode[]): Uint8Array { + const buf = Packet.alloc(2); + + for (const op of ops) { + encodeNpcOpcode(buf, op); + } + + buf.p1(0); + return new Uint8Array(buf.data.subarray(0, buf.pos)); +} diff --git a/tools/cache/lib/objCodec.ts b/tools/cache/lib/objCodec.ts new file mode 100644 index 000000000..4ac7cedd1 --- /dev/null +++ b/tools/cache/lib/objCodec.ts @@ -0,0 +1,212 @@ +import Packet from '#/io/Packet.js'; + +export type ObjOpcode = { + code: number; + payload: any; +}; + +function readParams(dat: Packet): Array<{ string: boolean; key: number; value: number | string }> { + const entries: Array<{ string: boolean; key: number; value: number | string }> = []; + + const count = dat.g1(); + for (let i = 0; i < count; i++) { + const isString = dat.g1() === 1; + const key = dat.g3(); + const value = isString ? dat.gjstr() : dat.g4s(); + entries.push({ string: isString, key, value }); + } + + return entries; +} + +function writeParams(buf: Packet, entries: Array<{ string: boolean; key: number; value: number | string }>): void { + buf.p1(entries.length); + + for (const entry of entries) { + buf.p1(entry.string ? 1 : 0); + buf.p3(entry.key); + + if (entry.string) { + buf.pjstr(String(entry.value)); + } else { + buf.p4(Number(entry.value)); + } + } +} + +export function decodeObjOpcode(code: number, dat: Packet): any { + if (code === 1 || code === 23 || code === 24 || code === 25 || code === 26 || code === 78 || code === 79 || + code === 90 || code === 91 || code === 92 || code === 93 || code === 94 || code === 95 || code === 97 || + code === 98 || code === 110 || code === 111 || code === 112 || code === 121 || code === 122 || + code === 201 || code === 4 || code === 5 || code === 6) { + return dat.g2(); + } + + if (code === 2 || code === 3 || (code >= 30 && code < 40) || code === 250) { + return dat.gjstr(); + } + + if (code === 7 || code === 8 || code === 75) { + return dat.g2s(); + } + + if (code === 12) { + return dat.g4s(); + } + + if (code === 11 || code === 15 || code === 16 || code === 65 || code === 123) { + return true; + } + + if (code === 13 || code === 14 || code === 27 || code === 96 || code === 115) { + return dat.g1(); + } + + if (code === 113 || code === 114) { + return dat.g1b(); + } + + if (code === 40 || code === 41) { + const count = dat.g1(); + const pairs: Array<{ from: number; to: number }> = []; + for (let i = 0; i < count; i++) { + pairs.push({ from: dat.g2(), to: dat.g2() }); + } + return pairs; + } + + if (code === 42) { + const count = dat.g1(); + const values: number[] = []; + for (let i = 0; i < count; i++) { + values.push(dat.g1b()); + } + return values; + } + + if (code >= 100 && code < 110) { + return { + obj: dat.g2(), + count: dat.g2() + }; + } + + if (code === 125 || code === 126) { + return { + x: dat.g1b(), + y: dat.g1b(), + z: dat.g1b() + }; + } + + if (code === 127 || code === 128 || code === 129 || code === 130) { + return { + op: dat.g1(), + cursor: dat.g2() + }; + } + + if (code === 249) { + return readParams(dat); + } + + throw new Error(`Unrecognized obj config code: ${code}`); +} + +export function encodeObjOpcode(buf: Packet, op: ObjOpcode): void { + const { code, payload } = op; + buf.p1(code); + + if (code === 1 || code === 23 || code === 24 || code === 25 || code === 26 || code === 78 || code === 79 || + code === 90 || code === 91 || code === 92 || code === 93 || code === 94 || code === 95 || code === 97 || + code === 98 || code === 110 || code === 111 || code === 112 || code === 121 || code === 122 || + code === 4 || code === 5 || code === 6 || code === 201) { + buf.p2(Number(payload)); + return; + } + + if (code === 2 || code === 3 || (code >= 30 && code < 40) || code === 250) { + buf.pjstr(String(payload ?? '')); + return; + } + + if (code === 7 || code === 8 || code === 75) { + buf.p2(Number(payload) & 0xffff); + return; + } + + if (code === 12) { + buf.p4(Number(payload)); + return; + } + + if (code === 11 || code === 15 || code === 16 || code === 65 || code === 123) { + return; + } + + if (code === 13 || code === 14 || code === 27 || code === 96 || code === 115) { + buf.p1(Number(payload) & 0xff); + return; + } + + if (code === 113 || code === 114) { + buf.p1(Number(payload) & 0xff); + return; + } + + if (code === 40 || code === 41) { + const pairs = (payload ?? []) as Array<{ from: number; to: number }>; + buf.p1(pairs.length); + for (const pair of pairs) { + buf.p2(Number(pair.from)); + buf.p2(Number(pair.to)); + } + return; + } + + if (code === 42) { + const values = (payload ?? []) as number[]; + buf.p1(values.length); + for (const value of values) { + buf.p1(Number(value) & 0xff); + } + return; + } + + if (code >= 100 && code < 110) { + buf.p2(Number(payload.obj)); + buf.p2(Number(payload.count)); + return; + } + + if (code === 125 || code === 126) { + buf.p1(Number(payload.x) & 0xff); + buf.p1(Number(payload.y) & 0xff); + buf.p1(Number(payload.z) & 0xff); + return; + } + + if (code === 127 || code === 128 || code === 129 || code === 130) { + buf.p1(Number(payload.op)); + buf.p2(Number(payload.cursor)); + return; + } + + if (code === 249) { + writeParams(buf, (payload ?? []) as Array<{ string: boolean; key: number; value: number | string }>); + return; + } + + throw new Error(`Unrecognized obj config code: ${code}`); +} + +export function encodeObjOps(ops: ObjOpcode[]): Uint8Array { + const buf = Packet.alloc(2); + + for (const op of ops) { + encodeObjOpcode(buf, op); + } + + buf.p1(0); + return new Uint8Array(buf.data.subarray(0, buf.pos)); +} diff --git a/tools/cache/lib/quickchatCatCodec.ts b/tools/cache/lib/quickchatCatCodec.ts new file mode 100644 index 000000000..71af4a7b0 --- /dev/null +++ b/tools/cache/lib/quickchatCatCodec.ts @@ -0,0 +1,48 @@ +import Packet from '#/io/Packet.js'; +import QuickChatCatType from '#/cache/config/QuickChatCatType.js'; + +export function encodeQuickChatCat(config: QuickChatCatType): Uint8Array { + const buf = Packet.alloc(2); + + // Opcode 4: optional marker with no payload (appears first in archive 25) + if (config.hasOpcode4) { + buf.p1(4); + } + + // Opcode 1: description + if (config.description) { + buf.p1(1); + buf.pjstr(config.description); + } + + // Opcode 2: subcategories + if (config.subcategories && config.subcategories.length > 0) { + buf.p1(2); + buf.p1(config.subcategories.length); + for (let i = 0; i < config.subcategories.length; i++) { + buf.p2(config.subcategories[i]); + const shortcut = config.subcategoryShortcuts?.[i] ?? 0; + buf.p1(shortcut & 0xff); + } + } + + // Opcode 3: phrases + if (config.phrases && config.phrases.length > 0) { + buf.p1(3); + buf.p1(config.phrases.length); + for (let i = 0; i < config.phrases.length; i++) { + buf.p2(config.phrases[i]); + const shortcut = config.phraseShortcuts?.[i] ?? 0; + buf.p1(shortcut & 0xff); + } + } + + // Opcode 250: debugname + if (config.debugname) { + buf.p1(250); + buf.pjstr(config.debugname); + } + + buf.p1(0); + return new Uint8Array(buf.data.subarray(0, buf.pos)); +} diff --git a/tools/cache/lib/quickchatPhraseCodec.ts b/tools/cache/lib/quickchatPhraseCodec.ts new file mode 100644 index 000000000..1e11b7e4a --- /dev/null +++ b/tools/cache/lib/quickchatPhraseCodec.ts @@ -0,0 +1,51 @@ +import Packet from '#/io/Packet.js'; +import QuickChatPhraseType from '#/cache/config/QuickChatPhraseType.js'; + +export function encodeQuickChatPhrase(config: QuickChatPhraseType): Uint8Array { + const buf = Packet.alloc(2); + + // Opcode 1: text (split by '<' character) + if (config.text && config.text.length > 0) { + buf.p1(1); + const fullText = config.text.join('<'); + buf.pjstr(fullText); + } + + // Opcode 2: auto responses + if (config.autoResponses && config.autoResponses.length > 0) { + buf.p1(2); + buf.p1(config.autoResponses.length); + for (let i = 0; i < config.autoResponses.length; i++) { + buf.p2(config.autoResponses[i]); + } + } + + // Opcode 3: dynamic commands with parameters + if (config.dynamicCommands && config.dynamicCommands.length > 0) { + buf.p1(3); + buf.p1(config.dynamicCommands.length); + for (let i = 0; i < config.dynamicCommands.length; i++) { + const commandId = config.dynamicCommands[i]; + buf.p2(commandId); + + const params = config.dynamicCommandParameters?.[i] ?? []; + for (let p = 0; p < params.length; p++) { + buf.p2(params[p]); + } + } + } + + // Opcode 4: searchable flag (only if false) + if (!config.searchable) { + buf.p1(4); + } + + // Opcode 250: debugname + if (config.debugname) { + buf.p1(250); + buf.pjstr(config.debugname); + } + + buf.p1(0); + return new Uint8Array(buf.data.subarray(0, buf.pos)); +} diff --git a/tools/cache/pack/packEnums.ts b/tools/cache/pack/packEnums.ts new file mode 100644 index 000000000..441b38712 --- /dev/null +++ b/tools/cache/pack/packEnums.ts @@ -0,0 +1,539 @@ +import fs from 'fs'; +import path from 'path'; + +import EnumType from '#/cache/config/EnumType.js'; +import ScriptVarType from '#/cache/config/ScriptVarType.js'; +import { CompressionType } from '#/io/CompressionType.js'; +import Packet from '#/io/Packet.js'; +import { unpackJs5Group } from '#/io/Js5Group.js'; +import Environment from '#/util/Environment.js'; +import { parseBracketedConfigSource } from '#tools/cache/lib/configSource.js'; +import { encodeEnum } from '#tools/cache/lib/enumCodec.js'; +import { getMidiId, hasMidiId } from '#tools/pack/packs/MidiPack.js'; +import { + arraysEqual, + ensureDir, + parsePackFile, + combineGroupFiles, + compressJs5Group, + parseGroupIdsFromIndexPacked, + writeInt32BE, + type Js5ArchiveIndex as _Js5ArchiveIndex, + loadReferenceArchiveIndex +} from '#tools/cache/lib/js5Tools.js'; + +type Args = { + src: string; + out: string; + archive: number; + mode: 'server' | 'client'; + exact: boolean; + help: boolean; +}; + +type EnumPairInt = { + key: number; + value: number; +}; + +type EnumPairString = { + key: number; + value: string; +}; + +type EnumEncodeOp = + | { code: 1; value: number } + | { code: 2; value: number } + | { code: 3; value: string } + | { code: 4; value: number } + | { code: 5; values: EnumPairString[] } + | { code: 6; values: EnumPairInt[] } + | { code: 250; value: string }; + +type ParsedEnumSource = { + config: EnumType; + ops: EnumEncodeOp[]; + transmit: boolean; +}; + +function parseArgs(argv: string[]): Args { + const args: Args = { + src: path.join(Environment.BUILD_SRC_DIR, 'scripts', '_unpack', '530', 'all.enum'), + out: 'data/pack', + archive: 17, + mode: 'server', + exact: false, + help: false + }; + + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + + if (arg === '--src') { + args.src = argv[++i]; + } else if (arg === '--out') { + args.out = argv[++i]; + } else if (arg === '--archive') { + args.archive = Number(argv[++i]); + } else if (arg === '--mode') { + const mode = argv[++i]; + if (mode === 'server' || mode === 'client') { + args.mode = mode; + } + } else if (arg === '--exact') { + args.exact = true; + } else if (arg === '--no-exact') { + args.exact = false; + } else if (arg === '--help' || arg === '-h') { + args.help = true; + } + } + + return args; +} + +function parseSourceTypeName(typeName: string): number { + // Check for unknown_ format + if (typeName.startsWith('unknown_')) { + const code = parseInt(typeName.substring(8)); + if (!isNaN(code)) { + return code; + } + } + + // Use ScriptVarType.getTypeChar for standard type names + const typeCode = ScriptVarType.getTypeChar(typeName); + return typeCode ?? ScriptVarType.INT; +} + +function parseSourceScalar(value: string, type: number, sourceContext?: string): number | string { + // Handle null + if (value === 'null') { + if (type === ScriptVarType.STRING) { + return ''; + } + return -1; + } + + // Handle booleans + if (type === ScriptVarType.BOOLEAN) { + if (value === '^true') { + return 1; + } + if (value === '^false') { + return 0; + } + } + + // Handle strings + if (type === ScriptVarType.STRING) { + return value; + } + + if (type === ScriptVarType.MIDI) { + const trimmed = value.trim(); + + const namedId = getMidiId(trimmed); + if (namedId !== -1) { + return namedId; + } + + const numeric = parseInt(trimmed, 10); + if (!isNaN(numeric) && String(numeric) === trimmed && hasMidiId(numeric)) { + return numeric; + } + + throw new Error(`Unknown midi value '${trimmed}'${sourceContext ? ` at ${sourceContext}` : ''}. It must exist in midi.pack.`); + } + + // Handle numbers (including hex) + if (value.startsWith('0x')) { + const parsed = parseInt(value, 16); + return isNaN(parsed) ? 0 : parsed; + } + + const parsed = parseInt(value); + return isNaN(parsed) ? 0 : parsed; +} + +function parseSourceEnums(content: string, nameToId: Map): Map { + const enums = new Map(); + const sections = parseBracketedConfigSource(content); + + for (const section of sections) { + // Resolve enum ID from section name + let enumId = nameToId.get(section.name); + if (enumId === undefined && section.name.startsWith('enum_')) { + enumId = parseInt(section.name.substring(5)); + } + + if (enumId === undefined || isNaN(enumId)) { + continue; + } + + const currentEnum = new EnumType(enumId); + const currentOps: EnumEncodeOp[] = []; + let transmit = false; + let pendingValuesBlock: { code: 5 | 6; remaining: number } | null = null; + + // Store debugname if it's not the default enum_ format + if (!section.name.startsWith('enum_')) { + currentEnum.debugname = section.name; + } + + // Process fields + for (const field of section.fields) { + const { key, value } = field; + + if (key === 'inputtype') { + const parsed = parseSourceTypeName(value); + currentEnum.inputtype = parsed; + currentOps.push({ code: 1, value: parsed }); + } else if (key === 'outputtype') { + const parsed = parseSourceTypeName(value); + currentEnum.outputtype = parsed; + currentOps.push({ code: 2, value: parsed }); + } else if (key === 'debugname') { + currentEnum.debugname = value; + currentOps.push({ code: 250, value }); + } else if (key === 'transmit') { + const parsed = value.trim().toLowerCase(); + transmit = parsed === 'yes' || parsed === 'true' || parsed === '1'; + } else if (key === 'default' || key === 'default@3' || key === 'default@4') { + // Check for explicit flag (! suffix) + const isExplicit = value.endsWith('!'); + const cleanValue = isExplicit ? value.slice(0, -1) : value; + const opCode = key === 'default@3' ? 3 : key === 'default@4' ? 4 : (currentEnum.outputtype === ScriptVarType.STRING ? 3 : 4); + + if (opCode === 3) { + currentEnum.defaultString = cleanValue; + if (isExplicit) { + currentEnum.hasExplicitDefaultString = true; + } + currentOps.push({ code: 3, value: cleanValue }); + } else { + currentEnum.defaultInt = Number(parseSourceScalar(cleanValue, currentEnum.outputtype, `${section.name}:${field.line}`)); + if (isExplicit) { + currentEnum.hasExplicitDefaultInt = true; + } + currentOps.push({ code: 4, value: currentEnum.defaultInt }); + } + pendingValuesBlock = null; + } else if (key === 'values@5' || key === 'values@6') { + const count = Number(value); + const code: 5 | 6 = key === 'values@5' ? 5 : 6; + currentOps.push(code === 5 ? { code: 5, values: [] } : { code: 6, values: [] }); + pendingValuesBlock = { code, remaining: Number.isFinite(count) && count > 0 ? count : 0 }; + } else if (key === 'val' || key === 'val@5' || key === 'val@6') { + const commaIndex = value.indexOf(','); + if (commaIndex === -1) { + continue; + } + + const keyPart = value.substring(0, commaIndex).trim(); + const valuePart = value.substring(commaIndex + 1); + + const parsedKey = Number(parseSourceScalar(keyPart, currentEnum.inputtype, `${section.name}:${field.line}`)); + const opCode: 5 | 6 = key === 'val@5' ? 5 : key === 'val@6' ? 6 : (currentEnum.outputtype === ScriptVarType.STRING ? 5 : 6); + + if (opCode === 5) { + const parsedValue = valuePart; + currentEnum.values.set(parsedKey, parsedValue); + + const lastOp = currentOps[currentOps.length - 1]; + if (lastOp?.code === 5 && (!pendingValuesBlock || pendingValuesBlock.code === 5)) { + lastOp.values.push({ key: parsedKey, value: parsedValue }); + } else { + currentOps.push({ code: 5, values: [{ key: parsedKey, value: parsedValue }] }); + } + } else { + const parsedValue = Number(parseSourceScalar(valuePart, currentEnum.outputtype, `${section.name}:${field.line}`)); + currentEnum.values.set(parsedKey, parsedValue); + + const lastOp = currentOps[currentOps.length - 1]; + if (lastOp?.code === 6 && (!pendingValuesBlock || pendingValuesBlock.code === 6)) { + lastOp.values.push({ key: parsedKey, value: parsedValue }); + } else { + currentOps.push({ code: 6, values: [{ key: parsedKey, value: parsedValue }] }); + } + } + + if (pendingValuesBlock && pendingValuesBlock.code === opCode) { + pendingValuesBlock.remaining -= 1; + if (pendingValuesBlock.remaining <= 0) { + pendingValuesBlock = null; + } + } + } else { + pendingValuesBlock = null; + } + } + + enums.set(enumId, { config: currentEnum, ops: currentOps, transmit }); + } + + return enums; +} + +function encodeFromOps(ops: EnumEncodeOp[]): Uint8Array { + const buf = Packet.alloc(2); + + for (const op of ops) { + if (op.code === 1) { + buf.p1(1); + buf.p1(op.value); + } else if (op.code === 2) { + buf.p1(2); + buf.p1(op.value); + } else if (op.code === 3) { + buf.p1(3); + buf.pjstr(op.value); + } else if (op.code === 4) { + buf.p1(4); + buf.p4(op.value); + } else if (op.code === 5) { + buf.p1(5); + buf.p2(op.values.length); + for (const pair of op.values) { + buf.p4(pair.key); + buf.pjstr(pair.value); + } + } else if (op.code === 6) { + buf.p1(6); + buf.p2(op.values.length); + for (const pair of op.values) { + buf.p4(pair.key); + buf.p4(pair.value); + } + } else if (op.code === 250) { + buf.p1(250); + buf.pjstr(op.value); + } + } + + buf.p1(0); + return new Uint8Array(buf.data.subarray(0, buf.pos)); +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + + if (args.mode === 'client' && args.exact) { + throw new Error('Client mode intentionally empties non-transmit enums, so --exact is not valid. Use --no-exact.'); + } + + if (!fs.existsSync(args.src)) { + throw new Error(`Source file not found: ${args.src}`); + } + + // Read pack files to resolve names (prefer source-adjacent pack/, fallback to BUILD_SRC_DIR/pack) + const localPackDir = path.join(path.dirname(args.src), 'pack'); + const fallbackPackDir = path.join(Environment.BUILD_SRC_DIR, 'pack'); + + const enumLocalPackPath = path.join(localPackDir, 'enum.pack'); + const enumFallbackPackPath = path.join(fallbackPackDir, 'enum.pack'); + const nameToId = fs.existsSync(enumLocalPackPath) ? parsePackFile(enumLocalPackPath) : parsePackFile(enumFallbackPackPath); + + // Parse source enums + const content = fs.readFileSync(args.src, 'utf-8'); + const enums = parseSourceEnums(content, nameToId); + const transmitCount = Array.from(enums.values()).filter(parsed => parsed.transmit).length; + + if (args.mode === 'client' && transmitCount === 0) { + console.log('Client mode: no transmit=yes directives found; all enums will be filtered to empty'); + } + + // Encode enums to binary + const encodedFiles = new Map(); + let _emptyCount = 0; + let _filteredCount = 0; + let _nonEmptyCount = 0; + + for (const [id, parsed] of enums) { + const enumConfig = parsed.config; + // Check if this is an empty enum + const isEmpty = enumConfig.values.size === 0 && + enumConfig.inputtype === ScriptVarType.INT && + enumConfig.outputtype === ScriptVarType.INT && + enumConfig.defaultInt === 0 && + enumConfig.defaultString === 'null' && + !enumConfig.debugname && + !enumConfig.hasExplicitDefaultInt && + !enumConfig.hasExplicitDefaultString; + + const shouldPack = args.mode === 'server' || parsed.transmit; + + if (!shouldPack) { + encodedFiles.set(id, new Uint8Array([0x00])); + _filteredCount++; + continue; + } + + if (parsed.ops.length > 0) { + encodedFiles.set(id, encodeFromOps(parsed.ops)); + _nonEmptyCount++; + } else if (isEmpty) { + // Empty enum - encode as single terminator byte + encodedFiles.set(id, new Uint8Array([0x00])); + _emptyCount++; + } else { + encodedFiles.set(id, encodeEnum(enumConfig)); + _nonEmptyCount++; + } + } + + // Group by (id >> 8) + const groups = new Map>(); + for (const [id, data] of encodedFiles) { + const groupId = id >> 8; + const fileId = id & 0xff; + + if (!groups.has(groupId)) { + groups.set(groupId, new Map()); + } + + groups.get(groupId)!.set(fileId, data); + } + + const referenceIndex = loadReferenceArchiveIndex(args.archive); + const groupFileOrders = new Map(); + + for (const [groupId, files] of groups) { + const referenceFileIds = referenceIndex?.fileIdsByGroup.get(groupId); + if (referenceFileIds && referenceFileIds.length > 0) { + groupFileOrders.set(groupId, referenceFileIds); + for (const fileId of referenceFileIds) { + if (!files.has(fileId)) { + files.set(fileId, new Uint8Array([0x00])); + } + } + } else { + const fallbackIds: number[] = new Array(256); + for (let i = 0; i < 256; i++) { + fallbackIds[i] = i; + } + groupFileOrders.set(groupId, fallbackIds); + for (const fileId of fallbackIds) { + if (!files.has(fileId)) { + files.set(fileId, new Uint8Array([0x00])); + } + } + } + } + + // Pack and compress each group into memory + ensureDir(args.out); + + // Determine compression type by reading original groups from cache + const compressionTypes = new Map(); + for (const groupId of groups.keys()) { + const origPath = `data/cache/${args.archive}/${groupId}.dat`; + if (fs.existsSync(origPath)) { + const origData = fs.readFileSync(origPath); + compressionTypes.set(groupId, origData[0]); + } else { + // Default to GZIP if no original exists + compressionTypes.set(groupId, CompressionType.GZIP); + } + } + + // Build group buffers in memory + const compressedGroups = new Map(); + + for (const [groupId, files] of groups) { + const orderedFileIds = groupFileOrders.get(groupId); + if (!orderedFileIds || orderedFileIds.length === 0) { + throw new Error(`No file order available for group ${groupId}`); + } + + const combined = combineGroupFiles(files, orderedFileIds); + const origPath = `data/cache/${args.archive}/${groupId}.dat`; + + if (args.exact) { + if (!fs.existsSync(origPath)) { + throw new Error(`Exact mode requires reference cache group: ${origPath}`); + } + + const originalContainer = new Uint8Array(fs.readFileSync(origPath)); + const originalUncompressed = unpackJs5Group(originalContainer); + + if (!arraysEqual(originalUncompressed, combined)) { + throw new Error( + `Exact mode mismatch for group ${groupId}: generated uncompressed payload differs from reference cache.` + ); + } + + compressedGroups.set(groupId, originalContainer); + continue; + } + + const compressionType = compressionTypes.get(groupId) ?? CompressionType.GZIP; + const compressed = await compressJs5Group(combined, compressionType); + compressedGroups.set(groupId, compressed); + } + + // Load index to get group order + const indexPath = `data/cache/255/${args.archive}.dat`; + if (!fs.existsSync(indexPath)) { + throw new Error(`Index file not found: ${indexPath}`); + } + + const indexPacked = new Uint8Array(fs.readFileSync(indexPath)); + const groupIds = parseGroupIdsFromIndexPacked(indexPacked); + + // Build group buffers and length table in proper order + const groupBuffers: Uint8Array[] = new Array(groupIds.length); + const groupLengths: number[] = new Array(groupIds.length).fill(0); + + for (let i = 0; i < groupIds.length; i++) { + const groupId = groupIds[i]; + const compressed = compressedGroups.get(groupId); + + if (!compressed || compressed.length === 0) { + groupBuffers[i] = new Uint8Array(0); + continue; + } + + groupBuffers[i] = compressed; + groupLengths[i] = compressed.length; + } + + // Build length table + const lengthTable = new Uint8Array(groupIds.length * 4); + for (let i = 0; i < groupLengths.length; i++) { + writeInt32BE(groupLengths[i], lengthTable, i * 4); + } + + // Build final JS5 archive + const totalGroupBytes = groupLengths.reduce((sum, length) => sum + length, 0); + const totalSize = indexPacked.length + totalGroupBytes + lengthTable.length; + const output = new Uint8Array(totalSize); + + let pos = 0; + output.set(indexPacked, pos); + pos += indexPacked.length; + + for (const group of groupBuffers) { + if (group.length === 0) { + continue; + } + output.set(group, pos); + pos += group.length; + } + + output.set(lengthTable, pos); + + // Write to appropriate .js5 file + const modeDir = path.join(args.out, args.mode); + ensureDir(modeDir); + const outPath = path.join(modeDir, `${args.mode}.enum.config.js5`); + + fs.writeFileSync(outPath, output); + + console.log(`Wrote ${outPath}`); +} + +main().catch(err => { + console.error(err); + process.exit(1); +}); diff --git a/tools/cache/pack/packFlos.ts b/tools/cache/pack/packFlos.ts new file mode 100644 index 000000000..3c277e97f --- /dev/null +++ b/tools/cache/pack/packFlos.ts @@ -0,0 +1,238 @@ +import fs from 'fs'; +import path from 'path'; + +import FloType from '#/cache/config/FloType.js'; +import { CompressionType } from '#/io/CompressionType.js'; +import { unpackJs5Group } from '#/io/Js5Group.js'; +import { + parseBracketedConfigSource, + parseConfigBoolean, + parseConfigInteger, + resolveSectionId +} from '#tools/cache/lib/configSource.js'; +import { encodeFlo, encodeFloWithOpcodes } from '#tools/cache/lib/floCodec.js'; +import { + arraysEqual, + ensureDir, + parsePackFile, + loadArchiveFileIds, + combineGroupFiles, + compressJs5Group, +} from '#tools/cache/lib/js5Tools.js'; + +type Args = { + src: string; + out: string; + index: number; + archive: number; + exact: boolean; + help: boolean; +}; + +function parseArgs(argv: string[]): Args { + const args: Args = { + src: 'data/src/all.flo', + out: 'data/pack', + index: 2, + archive: 4, + exact: false, + help: false + }; + + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + + if (arg === '--src') { + args.src = argv[++i]; + } else if (arg === '--out') { + args.out = argv[++i]; + } else if (arg === '--index') { + args.index = Number(argv[++i]); + } else if (arg === '--archive') { + args.archive = Number(argv[++i]); + } else if (arg === '--exact') { + args.exact = true; + } else if (arg === '--no-exact') { + args.exact = false; + } else if (arg === '--help' || arg === '-h') { + args.help = true; + } + } + + return args; +} + +type OpcodeValue = { + code: number; + value?: any; +}; + +type ParsedFloSource = { + config: FloType; + opcodes: OpcodeValue[]; +}; + +function parseSourceFlos(content: string, nameToId: Map): Map { + const flos = new Map(); + const sections = parseBracketedConfigSource(content); + + for (const section of sections) { + const id = resolveSectionId(section.name, nameToId, 'flo_'); + if (id === null) { + throw new Error(`Unknown floor overlay name: ${section.name}`); + } + + const config = new FloType(id); + config.debugname = nameToId.get(section.name) !== undefined ? section.name : null; + const opcodes: OpcodeValue[] = []; + + for (const field of section.fields) { + const { key, value } = field; + + if (key === 'overlay') { + continue; + } + + if (key === 'code8') { + config.code8 = parseConfigBoolean(value); + opcodes.push({ code: 8 }); + } else if (key === 'colour') { + const colourValue = parseConfigInteger(value); + config.colour = colourValue; + opcodes.push({ code: 1, value: colourValue }); + } else if (key === 'texture') { + const matValue = parseConfigInteger(value); + config.material = matValue; + opcodes.push({ code: 2, value: matValue }); + } else if (key === 'material') { + const matValue = parseConfigInteger(value); + config.material = matValue; + opcodes.push({ code: 3, value: matValue }); + } else if (key === 'occlude') { + config.occlude = parseConfigBoolean(value); + opcodes.push({ code: 5 }); + } else if (key === 'averagecolour') { + const avgColourValue = parseConfigInteger(value); + config.averagecolour = avgColourValue; + opcodes.push({ code: 7, value: avgColourValue }); + } else if (key === 'materialscale') { + const scaleValue = parseConfigInteger(value); + config.materialscale = scaleValue; + opcodes.push({ code: 9, value: scaleValue }); + } else if (key === 'hardshadow') { + config.hardshadow = parseConfigBoolean(value); + if (!config.hardshadow) { + opcodes.push({ code: 10 }); + } + } else if (key === 'priority') { + const priorityValue = parseConfigInteger(value); + config.priority = priorityValue; + opcodes.push({ code: 11, value: priorityValue }); + } else if (key === 'blend') { + config.blend = parseConfigBoolean(value); + opcodes.push({ code: 12 }); + } else if (key === 'waterfogcolour') { + const fogColourValue = parseConfigInteger(value); + config.waterfogcolour = fogColourValue; + opcodes.push({ code: 13, value: fogColourValue }); + } else if (key === 'waterfogscale') { + const fogScaleValue = parseConfigInteger(value); + config.waterfogscale = fogScaleValue; + opcodes.push({ code: 14, value: fogScaleValue }); + } + } + + flos.set(id, { config, opcodes }); + } + + return flos; +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + + if (!fs.existsSync(args.src)) { + throw new Error(`Source file not found: ${args.src}`); + } + + const fileIds = await loadArchiveFileIds(args.index, args.archive, true); + + // Read pack file to resolve names + const packPath = path.join(path.dirname(args.src), 'pack', 'flo.pack'); + const nameToId = parsePackFile(packPath); + + // Parse source floor overlays + const content = fs.readFileSync(args.src, 'utf-8'); + const flos = parseSourceFlos(content, nameToId); + + // Encode floor overlays to binary + const encodedFiles = new Map(); + let emptyCount = 0; + let nonEmptyCount = 0; + + for (const [id, parsed] of flos) { + const floConfig = parsed.config; + + // Use opcode-ordered encoding from source file order + const encoded = parsed.opcodes.length > 0 + ? encodeFloWithOpcodes(floConfig, parsed.opcodes) + : encodeFlo(floConfig); + + encodedFiles.set(id, encoded); + + if (encoded.length === 1 && encoded[0] === 0) { + emptyCount++; + } else { + nonEmptyCount++; + } + } + + console.log(`Encoded: ${emptyCount} empty, ${nonEmptyCount} non-empty`); + + const files = encodedFiles; + + // Build ordered file IDs list - only include files that were actually parsed + const orderedFileIds = fileIds.filter(id => files.has(id)); + + const combined = combineGroupFiles(files, orderedFileIds); + + ensureDir(args.out); + const indexDir = path.join(args.out, String(args.index)); + ensureDir(indexDir); + + const groupPath = path.join(indexDir, `${args.archive}.dat`); + const origPath = `data/cache/${args.index}/${args.archive}.dat`; + + if (args.exact) { + if (!fs.existsSync(origPath)) { + throw new Error(`Exact mode requires reference cache group: ${origPath}`); + } + + const originalContainer = new Uint8Array(fs.readFileSync(origPath)); + const originalUncompressed = unpackJs5Group(originalContainer); + + if (!arraysEqual(originalUncompressed, combined)) { + throw new Error( + 'Exact mode mismatch: generated uncompressed payload differs from reference cache.' + ); + } + + fs.writeFileSync(groupPath, originalContainer); + } else { + // Determine compression type from original + let compressionType = CompressionType.GZIP; + if (fs.existsSync(origPath)) { + const origData = fs.readFileSync(origPath); + compressionType = origData[0]; + } + + const compressed = await compressJs5Group(combined, compressionType); + fs.writeFileSync(groupPath, compressed); + } + console.log(`Wrote ${groupPath}`); +} + +main().catch(err => { + console.error(err); + process.exit(1); +}); diff --git a/tools/cache/pack/packFlus.ts b/tools/cache/pack/packFlus.ts new file mode 100644 index 000000000..cece17c87 --- /dev/null +++ b/tools/cache/pack/packFlus.ts @@ -0,0 +1,295 @@ +import fs from 'fs'; +import path from 'path'; + +import FluType from '#/cache/config/FluType.js'; +import { CompressionType } from '#/io/CompressionType.js'; +import { unpackJs5Group } from '#/io/Js5Group.js'; +import Environment from '#/util/Environment.js'; +import { + parseBracketedConfigSource, + parseConfigBoolean, + parseConfigInteger, + resolveSectionId +} from '#tools/cache/lib/configSource.js'; +import { encodeFlu, encodeFluWithOpcodes } from '#tools/cache/lib/fluCodec.js'; +import { + arraysEqual, + ensureDir, + parsePackFile, + loadArchiveFileIds, + combineGroupFiles, + compressJs5Group, +} from '#tools/cache/lib/js5Tools.js'; + +type Args = { + src: string; + out: string; + index: number; + archive: number; + exact: boolean; + debug: boolean; + help: boolean; +}; + +function parseArgs(argv: string[]): Args { + const args: Args = { + src: path.join(Environment.BUILD_SRC_DIR, 'scripts', '_unpack', '530', 'all.flu'), + out: 'data/pack', + index: 2, + archive: 1, + exact: false, + debug: false, + help: false + }; + + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + + if (arg === '--src') { + args.src = argv[++i]; + } else if (arg === '--out') { + args.out = argv[++i]; + } else if (arg === '--index') { + args.index = Number(argv[++i]); + } else if (arg === '--archive') { + args.archive = Number(argv[++i]); + } else if (arg === '--exact') { + args.exact = true; + } else if (arg === '--no-exact') { + args.exact = false; + } else if (arg === '--debug') { + args.debug = true; + } else if (arg === '--help' || arg === '-h') { + args.help = true; + } + } + + return args; +} + +type OpcodeValue = { + code: number; + value?: any; +}; + +type ParsedFluSource = { + config: FluType; + opcodes: OpcodeValue[]; +}; + +function parseSourceFlus(content: string, nameToId: Map): Map { + const flus = new Map(); + const sections = parseBracketedConfigSource(content); + + for (const section of sections) { + const id = resolveSectionId(section.name, nameToId, 'flu_'); + if (id === null) { + throw new Error(`Unknown floor underlay name: ${section.name}`); + } + + const config = new FluType(id); + config.debugname = nameToId.get(section.name) !== undefined ? section.name : null; + const opcodes: OpcodeValue[] = []; + + for (const field of section.fields) { + const { key, value } = field; + + if (key === 'colour') { + const colourValue = parseConfigInteger(value); + config.colour = colourValue; + opcodes.push({ code: 1, value: colourValue }); + } else if (key === 'material') { + const matValue = parseConfigInteger(value); + config.material = matValue; + opcodes.push({ code: 2, value: matValue }); + } else if (key === 'materialscale') { + const scaleValue = parseConfigInteger(value); + config.materialscale = scaleValue; + opcodes.push({ code: 3, value: scaleValue }); + } else if (key === 'hardshadow') { + config.hardshadow = parseConfigBoolean(value); + if (!config.hardshadow) { + opcodes.push({ code: 4 }); + } + } + } + + flus.set(id, { config, opcodes }); + } + + return flus; +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + + + if (!fs.existsSync(args.src)) { + throw new Error(`Source file not found: ${args.src}`); + } + + const fileIds = await loadArchiveFileIds(args.index, args.archive, true); + + const primaryPackPath = path.join(Environment.BUILD_SRC_DIR, 'pack', 'flu.pack'); + const fallbackPackPath = path.join(path.dirname(args.src), 'pack', 'flu.pack'); + const packPath = fs.existsSync(primaryPackPath) ? primaryPackPath : fallbackPackPath; + const nameToId = parsePackFile(packPath); + + const content = fs.readFileSync(args.src, 'utf-8'); + const flus = parseSourceFlus(content, nameToId); + + const encodedFiles = new Map(); + let emptyCount = 0; + let nonEmptyCount = 0; + + for (const [id, parsed] of flus) { + const fluConfig = parsed.config; + const encoded = parsed.opcodes.length > 0 + ? encodeFluWithOpcodes(fluConfig, parsed.opcodes) + : encodeFlu(fluConfig); + + encodedFiles.set(id, encoded); + + if (encoded.length === 1 && encoded[0] === 0) { + emptyCount++; + } else { + nonEmptyCount++; + } + } + + console.log(`Encoded: ${emptyCount} empty, ${nonEmptyCount} non-empty`); + + const files = encodedFiles; + for (const fileId of fileIds) { + if (!files.has(fileId)) { + files.set(fileId, new Uint8Array([0x00])); + emptyCount++; + } + } + + const orderedFileIds = fileIds; + + const combined = combineGroupFiles(files, orderedFileIds); + + ensureDir(args.out); + const indexDir = path.join(args.out, String(args.index)); + ensureDir(indexDir); + + const groupPath = path.join(indexDir, `${args.archive}.dat`); + const origPath = `data/cache/${args.index}/${args.archive}.dat`; + + if (args.exact) { + if (!fs.existsSync(origPath)) { + throw new Error(`Exact mode requires reference cache group: ${origPath}`); + } + + const originalContainer = new Uint8Array(fs.readFileSync(origPath)); + const originalUncompressed = unpackJs5Group(originalContainer); + + if (!arraysEqual(originalUncompressed, combined)) { + if (args.debug) { + reportFirstMismatch(originalUncompressed, combined, fileIds); + } + throw new Error( + 'Exact mode mismatch: generated uncompressed payload differs from reference cache.' + ); + } + + fs.writeFileSync(groupPath, originalContainer); + } else { + let compressionType = CompressionType.GZIP; + if (fs.existsSync(origPath)) { + const origData = fs.readFileSync(origPath); + compressionType = origData[0]; + } + + const compressed = await compressJs5Group(combined, compressionType); + fs.writeFileSync(groupPath, compressed); + } + console.log(`Wrote ${groupPath}`); +} + +function splitGroupFiles(groupData: Uint8Array, fileIds: number[]): Map { + const files = new Map(); + + if (fileIds.length === 1) { + files.set(fileIds[0], groupData); + return files; + } + + const stripes = groupData[groupData.length - 1] & 0xff; + const fileCount = fileIds.length; + const tableLength = stripes * fileCount * 4; + const tableOffset = groupData.length - 1 - tableLength; + + if (tableOffset <= 0) { + throw new Error('Invalid JS5 group chunk table.'); + } + + const view = new DataView(groupData.buffer, groupData.byteOffset, groupData.byteLength); + const sizes = new Int32Array(fileCount); + + let tablePos = tableOffset; + for (let stripe = 0; stripe < stripes; stripe++) { + let chunkLength = 0; + for (let file = 0; file < fileCount; file++) { + chunkLength += view.getInt32(tablePos); + tablePos += 4; + sizes[file] += chunkLength; + } + } + + const outputs: Uint8Array[] = new Array(fileCount); + for (let file = 0; file < fileCount; file++) { + outputs[file] = new Uint8Array(sizes[file]); + } + + const offsets = new Int32Array(fileCount); + let dataPos = 0; + tablePos = tableOffset; + + for (let stripe = 0; stripe < stripes; stripe++) { + let chunkLength = 0; + for (let file = 0; file < fileCount; file++) { + chunkLength += view.getInt32(tablePos); + tablePos += 4; + + outputs[file].set(groupData.subarray(dataPos, dataPos + chunkLength), offsets[file]); + offsets[file] += chunkLength; + dataPos += chunkLength; + } + } + + for (let i = 0; i < fileCount; i++) { + files.set(fileIds[i], outputs[i]); + } + + return files; +} + +function reportFirstMismatch(reference: Uint8Array, generated: Uint8Array, fileIds: number[]): void { + const refFiles = splitGroupFiles(reference, fileIds); + const genFiles = splitGroupFiles(generated, fileIds); + + for (const fileId of fileIds) { + const ref = refFiles.get(fileId) ?? new Uint8Array(0); + const gen = genFiles.get(fileId) ?? new Uint8Array(0); + + if (ref.length !== gen.length) { + console.warn(`Mismatch file ${fileId}: size ${gen.length} vs ${ref.length}`); + return; + } + + for (let i = 0; i < ref.length; i++) { + if (ref[i] !== gen[i]) { + console.warn(`Mismatch file ${fileId}: first diff at offset ${i} (gen=${gen[i]}, ref=${ref[i]})`); + return; + } + } + } +} + +main().catch(err => { + console.error(err); + process.exit(1); +}); diff --git a/tools/cache/pack/packInvs.ts b/tools/cache/pack/packInvs.ts new file mode 100644 index 000000000..fa156ac72 --- /dev/null +++ b/tools/cache/pack/packInvs.ts @@ -0,0 +1,254 @@ +import fs from 'fs'; +import path from 'path'; + +import InvType from '#/cache/config/InvType.js'; +import { CompressionType } from '#/io/CompressionType.js'; +import Environment from '#/util/Environment.js'; +import { + parseBracketedConfigSource, + parseConfigBoolean, + parseConfigInteger, + resolveSectionId +} from '#tools/cache/lib/configSource.js'; +import { encodeInvWithOpcodes } from '#tools/cache/lib/invCodec.js'; +import { + arraysEqual, + ensureDir, + loadArchiveGroupFiles, + combineGroupFiles, + compressJs5Group, +} from '#tools/cache/lib/js5Tools.js'; + +type Args = { + src: string; + out: string; + index: number; + archive: number; + exact: boolean; + debug: boolean; + help: boolean; +}; + +function parseArgs(argv: string[]): Args { + const args: Args = { + src: path.join(Environment.BUILD_SRC_DIR, 'scripts', '_unpack', '530', 'all.inv'), + out: 'data/pack', + index: 2, + archive: 5, + exact: false, + debug: false, + help: false + }; + + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + + if (arg === '--src') { + args.src = argv[++i]; + } else if (arg === '--out') { + args.out = argv[++i]; + } else if (arg === '--index') { + args.index = Number(argv[++i]); + } else if (arg === '--archive') { + args.archive = Number(argv[++i]); + } else if (arg === '--exact') { + args.exact = true; + } else if (arg === '--no-exact') { + args.exact = false; + } else if (arg === '--debug') { + args.debug = true; + } else if (arg === '--help' || arg === '-h') { + args.help = true; + } + } + + return args; +} + +type OpcodeValue = { + code: number; + value?: any; +}; + +type ParsedInvSource = { + config: InvType; + opcodes: OpcodeValue[]; +}; + +function parseSourceInvs(content: string, nameToId: Map): Map { + const invs = new Map(); + const sections = parseBracketedConfigSource(content); + + for (const section of sections) { + const id = resolveSectionId(section.name, nameToId, 'inv_'); + if (id === null) { + throw new Error(`Unknown inventory name: ${section.name}`); + } + + const config = new InvType(id); + config.debugname = null; + const opcodes: OpcodeValue[] = []; + + for (const field of section.fields) { + const { key, value } = field; + + if (key === 'scope') { + const scopeValue = parseConfigInteger(value); + config.scope = scopeValue; + opcodes.push({ code: 1, value: scopeValue }); + } else if (key === 'size') { + const sizeValue = parseConfigInteger(value); + config.size = sizeValue; + opcodes.push({ code: 2, value: sizeValue }); + } else if (key === 'stackall') { + config.stackall = parseConfigBoolean(value); + if (config.stackall) { + opcodes.push({ code: 3 }); + } + } else if (key === 'stock_add') { + const parts = value.split(','); + const obj = parseConfigInteger(parts[0]); + const count = parseConfigInteger(parts[1]); + const rate = parseConfigInteger(parts[2]); + + if (!config.stockobj) { + config.stockobj = new Uint16Array([obj]); + config.stockcount = new Uint16Array([count]); + config.stockrate = new Int32Array([rate]); + } else { + const newStockobj = new Uint16Array(config.stockobj.length + 1); + newStockobj.set(config.stockobj); + newStockobj[config.stockobj.length] = obj; + + const newStockcount = new Uint16Array(config.stockcount.length + 1); + newStockcount.set(config.stockcount); + newStockcount[config.stockcount.length] = count; + + const newStockrate = new Int32Array(config.stockrate.length + 1); + newStockrate.set(config.stockrate); + newStockrate[config.stockrate.length] = rate; + + config.stockobj = newStockobj; + config.stockcount = newStockcount; + config.stockrate = newStockrate; + } + + const stockItems = Array.from({ length: config.stockobj!.length }, (_, i) => ({ + obj: config.stockobj![i], + count: config.stockcount![i], + rate: config.stockrate![i] + })); + const existingStockOpcode = opcodes.find((op) => op.code === 4); + if (existingStockOpcode) { + existingStockOpcode.value = stockItems; + } else { + opcodes.push({ code: 4, value: stockItems }); + } + } else if (key === 'restock') { + config.restock = parseConfigBoolean(value); + if (config.restock) { + opcodes.push({ code: 5 }); + } + } else if (key === 'allstock') { + config.allstock = parseConfigBoolean(value); + if (config.allstock) { + opcodes.push({ code: 6 }); + } + } else if (key === 'protect') { + config.protect = parseConfigBoolean(value); + if (!config.protect) { + opcodes.push({ code: 7 }); + } + } else if (key === 'runweight') { + config.runweight = parseConfigBoolean(value); + if (config.runweight) { + opcodes.push({ code: 8 }); + } + } else if (key === 'dummyinv') { + config.dummyinv = parseConfigBoolean(value); + if (config.dummyinv) { + opcodes.push({ code: 9 }); + } + } else if (key === 'debugname') { + config.debugname = value; + opcodes.push({ code: 250, value }); + } + } + + invs.set(id, { config, opcodes }); + } + + return invs; +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + + if (!fs.existsSync(args.src)) { + throw new Error(`Source file not found: ${args.src}`); + } + + const { + fileIds: currentGroupFileIds, + groupUnpacked: currentGroupUnpacked + } = await loadArchiveGroupFiles(args.index, args.archive, 'data/cache', true); + + const sourceContent = fs.readFileSync(args.src, 'utf-8'); + + const nameToId = new Map(); + for (let i = 0; i < 10000; i++) { + nameToId.set(`inv_${i}`, i); + } + + const sourceInvs = parseSourceInvs(sourceContent, nameToId); + + const fileData = new Map(); + + for (let id = 0; id < 10000; id++) { + const sourceInv = sourceInvs.get(id); + + if (!sourceInv) { + fileData.set(id, new Uint8Array(0)); + continue; + } + + const encoded = encodeInvWithOpcodes(sourceInv.config, sourceInv.opcodes); + fileData.set(id, encoded); + } + + const combined = combineGroupFiles(fileData, currentGroupFileIds); + + if (args.exact) { + const refGroupData = currentGroupUnpacked; + + if (!arraysEqual(combined, refGroupData)) { + if (args.debug) { + for (let i = 0; i < Math.min(combined.length, refGroupData.length); i++) { + if (combined[i] !== refGroupData[i]) { + console.error(`Mismatch at offset ${i}: generated=${combined[i]}, reference=${refGroupData[i]}`); + break; + } + } + } + throw new Error('Generated group does not match reference'); + } + } + + const compressed = await compressJs5Group(combined, CompressionType.GZIP); + const filename = `${args.archive}.dat`; + const filepath = path.join(args.out, filename); + + ensureDir(args.out); + if (fs.existsSync(filepath) && fs.statSync(filepath).isDirectory()) { + fs.rmSync(filepath, { recursive: true, force: true }); + } + fs.writeFileSync(filepath, compressed); + + console.log(`Packed ${sourceInvs.size} inventory configs`); + console.log(`Wrote ${filepath}`); +} + +main().catch(err => { + console.error(err); + process.exit(1); +}); diff --git a/tools/cache/pack/packJs5Archive.ts b/tools/cache/pack/packJs5Archive.ts new file mode 100644 index 000000000..e693229f9 --- /dev/null +++ b/tools/cache/pack/packJs5Archive.ts @@ -0,0 +1,116 @@ +import fs from 'fs'; +import path from 'path'; + +import { parseGroupIdsFromIndexPacked, writeInt32BE } from '#tools/cache/lib/js5Tools.js'; + +type Args = { + archive: number; + groupsDir: string; + indexPath: string; + out: string; + help: boolean; +}; + +function parseArgs(argv: string[]): Args { + const args: Args = { + archive: 17, + groupsDir: 'data/pack', + indexPath: '', + out: '', + help: false + }; + + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg === '--archive') { + args.archive = Number(argv[++i]); + } else if (arg === '--groups-dir') { + args.groupsDir = argv[++i]; + } else if (arg === '--index') { + args.indexPath = argv[++i]; + } else if (arg === '--out') { + args.out = argv[++i]; + } else if (arg === '--help' || arg === '-h') { + args.help = true; + } + } + + if (!args.indexPath) { + args.indexPath = `data/cache/255/${args.archive}.dat`; + } + + if (!args.out) { + args.out = `data/pack/archive.${args.archive}.js5`; + } + + return args; +} + + +function main() { + const args = parseArgs(process.argv.slice(2)); + + if (!fs.existsSync(args.indexPath)) { + throw new Error(`Index file not found: ${args.indexPath}`); + } + + const indexPacked = new Uint8Array(fs.readFileSync(args.indexPath)); + const groupIds = parseGroupIdsFromIndexPacked(indexPacked); + + const groupBuffers: Uint8Array[] = new Array(groupIds.length); + const groupLengths: number[] = new Array(groupIds.length).fill(0); + + for (let i = 0; i < groupIds.length; i++) { + const groupId = groupIds[i]; + const groupPath = path.join(args.groupsDir, String(args.archive), `${groupId}.dat`); + + if (!fs.existsSync(groupPath)) { + groupBuffers[i] = new Uint8Array(0); + continue; + } + + const bytes = new Uint8Array(fs.readFileSync(groupPath)); + groupBuffers[i] = bytes; + groupLengths[i] = bytes.length; + } + + const lengthTable = new Uint8Array(groupIds.length * 4); + for (let i = 0; i < groupLengths.length; i++) { + writeInt32BE(groupLengths[i], lengthTable, i * 4); + } + + const totalGroupBytes = groupLengths.reduce((sum, length) => sum + length, 0); + const totalSize = indexPacked.length + totalGroupBytes + lengthTable.length; + const output = new Uint8Array(totalSize); + + let pos = 0; + output.set(indexPacked, pos); + pos += indexPacked.length; + + for (const group of groupBuffers) { + if (group.length === 0) { + continue; + } + output.set(group, pos); + pos += group.length; + } + + output.set(lengthTable, pos); + + const outDir = path.dirname(args.out); + fs.mkdirSync(outDir, { recursive: true }); + fs.writeFileSync(args.out, output); + + console.log(`Archive ${args.archive}: ${groupIds.length} index groups`); + console.log(`Index bytes: ${indexPacked.length}`); + console.log(`Group bytes: ${totalGroupBytes}`); + console.log(`Length table bytes: ${lengthTable.length}`); + console.log(`Wrote ${args.out}`); +} + +try { + main(); +} catch (err) { + console.error(err); + process.exit(1); +} diff --git a/tools/cache/pack/packMesanims.ts b/tools/cache/pack/packMesanims.ts new file mode 100644 index 000000000..3e5e76246 --- /dev/null +++ b/tools/cache/pack/packMesanims.ts @@ -0,0 +1,211 @@ +import fs from 'fs'; +import path from 'path'; + +import MesanimType from '#/cache/config/MesanimType.js'; +import { splitGroupFiles } from '#/io/Js5ArchiveIndex.js'; +import { CompressionType } from '#/io/CompressionType.js'; +import { unpackJs5Group } from '#/io/Js5Group.js'; +import Environment from '#/util/Environment.js'; +import { + parseBracketedConfigSource, + parseConfigInteger, + resolveSectionId +} from '#tools/cache/lib/configSource.js'; +import { encodeMesanim, encodeMesanimWithOpcodes } from '#tools/cache/lib/mesanimCodec.js'; +import { + arraysEqual, + ensureDir, + parsePackFile, + loadArchiveFileIds, + combineGroupFiles, + compressJs5Group +} from '#tools/cache/lib/js5Tools.js'; + +type Args = { + src: string; + out: string; + index: number; + archive: number; + exact: boolean; + debug: boolean; + help: boolean; +}; + +type OpcodeValue = { + code: number; + value?: number; +}; + +type ParsedMesanimSource = { + config: MesanimType; + opcodes: OpcodeValue[]; +}; + +function parseArgs(argv: string[]): Args { + const args: Args = { + src: path.join(Environment.BUILD_SRC_DIR, 'scripts', '_unpack', '530', 'all.mesanim'), + out: 'data/pack', + index: 2, + archive: 7, + exact: false, + debug: false, + help: false + }; + + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + + if (arg === '--src') { + args.src = argv[++i]; + } else if (arg === '--out') { + args.out = argv[++i]; + } else if (arg === '--index') { + args.index = Number(argv[++i]); + } else if (arg === '--archive') { + args.archive = Number(argv[++i]); + } else if (arg === '--exact') { + args.exact = true; + } else if (arg === '--no-exact') { + args.exact = false; + } else if (arg === '--debug') { + args.debug = true; + } else if (arg === '--help' || arg === '-h') { + args.help = true; + } + } + + return args; +} + +function parseSourceMesanims(content: string, nameToId: Map): Map { + const mesanims = new Map(); + const sections = parseBracketedConfigSource(content); + + for (const section of sections) { + const id = resolveSectionId(section.name, nameToId, 'mesanim_'); + if (id === null) { + throw new Error(`Unknown mesanim name: ${section.name}`); + } + + const config = new MesanimType(id); + config.debugname = `mesanim_${id}`; + const opcodes: OpcodeValue[] = []; + + for (const field of section.fields) { + const { key, value } = field; + const lenMatch = /^len([1-4])$/.exec(key); + if (!lenMatch) { + continue; + } + + const code = Number(lenMatch[1]); + const parsed = parseConfigInteger(value); + config.len[code - 1] = parsed; + opcodes.push({ code, value: parsed }); + } + + mesanims.set(id, { config, opcodes }); + } + + return mesanims; +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + + if (!fs.existsSync(args.src)) { + throw new Error(`Source file not found: ${args.src}`); + } + + const fileIds = await loadArchiveFileIds(args.index, args.archive, true); + + const primaryPackPath = path.join(Environment.BUILD_SRC_DIR, 'pack', 'mesanim.pack'); + const fallbackPackPath = path.join(path.dirname(args.src), 'pack', 'mesanim.pack'); + const packPath = fs.existsSync(primaryPackPath) ? primaryPackPath : fallbackPackPath; + const nameToId = parsePackFile(packPath); + + const content = fs.readFileSync(args.src, 'utf-8'); + const mesanims = parseSourceMesanims(content, nameToId); + + const encodedFiles = new Map(); + + for (const [id, parsed] of mesanims) { + const encoded = parsed.opcodes.length > 0 + ? encodeMesanimWithOpcodes(parsed.config, parsed.opcodes) + : encodeMesanim(parsed.config); + + encodedFiles.set(id, encoded); + } + + for (const fileId of fileIds) { + if (!encodedFiles.has(fileId)) { + encodedFiles.set(fileId, new Uint8Array([0x00])); + } + } + + const combined = combineGroupFiles(encodedFiles, fileIds); + + ensureDir(args.out); + const indexDir = path.join(args.out, String(args.index)); + ensureDir(indexDir); + + const groupPath = path.join(indexDir, `${args.archive}.dat`); + const origPath = `data/cache/${args.index}/${args.archive}.dat`; + + if (args.exact) { + if (!fs.existsSync(origPath)) { + throw new Error(`Exact mode requires reference cache group: ${origPath}`); + } + + const originalContainer = new Uint8Array(fs.readFileSync(origPath)); + const originalUncompressed = unpackJs5Group(originalContainer); + + if (!arraysEqual(originalUncompressed, combined)) { + if (args.debug) { + reportFirstMismatch(originalUncompressed, combined, fileIds); + } + + throw new Error('Exact mode mismatch: generated uncompressed payload differs from reference cache.'); + } + + fs.writeFileSync(groupPath, originalContainer); + } else { + let compressionType = CompressionType.GZIP; + if (fs.existsSync(origPath)) { + const origData = fs.readFileSync(origPath); + compressionType = origData[0]; + } + + const compressed = await compressJs5Group(combined, compressionType); + fs.writeFileSync(groupPath, compressed); + } + + console.log(`Wrote ${groupPath}`); +} + +function reportFirstMismatch(reference: Uint8Array, generated: Uint8Array, fileIds: number[]): void { + const refFiles = splitGroupFiles(reference, fileIds); + const genFiles = splitGroupFiles(generated, fileIds); + + for (const fileId of fileIds) { + const ref = refFiles.get(fileId) ?? new Uint8Array(0); + const gen = genFiles.get(fileId) ?? new Uint8Array(0); + + if (ref.length !== gen.length) { + console.warn(`Mismatch file ${fileId}: size ${gen.length} vs ${ref.length}`); + return; + } + + for (let i = 0; i < ref.length; i++) { + if (ref[i] !== gen[i]) { + console.warn(`Mismatch file ${fileId}: first diff at offset ${i} (gen=${gen[i]}, ref=${ref[i]})`); + return; + } + } + } +} + +main().catch(err => { + console.error(err); + process.exit(1); +}); diff --git a/tools/cache/pack/packMidis.ts b/tools/cache/pack/packMidis.ts new file mode 100644 index 000000000..24056f3c7 --- /dev/null +++ b/tools/cache/pack/packMidis.ts @@ -0,0 +1,189 @@ +import fs from 'fs'; +import path from 'path'; + +import { CompressionType } from '#/io/CompressionType.js'; +import { unpackJs5Group } from '#/io/Js5Group.js'; +import { resolveMidiPathByName } from '#tools/pack/sources/MidiSource.js'; +import Environment from '#/util/Environment.js'; +import { arraysEqual, ensureDir, parsePackFile, compressJs5Group, parseGroupIdsFromIndexPacked, writeInt32BE } from '#tools/cache/lib/js5Tools.js'; + +type Args = { + src: string; + out: string; + archive: number; + exact: boolean; + help: boolean; +}; + +function parseArgs(argv: string[]): Args { + const args: Args = { + src: path.join(Environment.BUILD_SRC_DIR, 'pack', 'midi.pack'), + out: 'data/pack', + archive: 6, + exact: false, + help: false, + }; + + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg === '--src') { + args.src = argv[++i]; + } else if (arg === '--out') { + args.out = argv[++i]; + } else if (arg === '--archive') { + args.archive = Number(argv[++i]); + } else if (arg === '--exact') { + args.exact = true; + } else if (arg === '--no-exact') { + args.exact = false; + } else if (arg === '--help' || arg === '-h') { + args.help = true; + } + } + + return args; +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + + if (!fs.existsSync(args.src)) { + throw new Error(`Pack file not found: ${args.src}`); + } + + ensureDir(args.out); + + const midiPack = parsePackFile(args.src); + const indexPath = `data/cache/255/${args.archive}.dat`; + + if (!fs.existsSync(indexPath)) { + throw new Error(`Index file not found: ${indexPath}`); + } + + const indexPacked = new Uint8Array(fs.readFileSync(indexPath)); + const groupIds = parseGroupIdsFromIndexPacked(indexPacked); + const validGroupIds = new Set(groupIds); + + const midiData = new Map(); + let missingFiles = 0; + let skippedIds = 0; + + for (const [name, id] of midiPack) { + if (!validGroupIds.has(id)) { + skippedIds++; + continue; + } + + const filePath = resolveMidiPathByName(name); + if (!filePath) { + missingFiles++; + continue; + } + + midiData.set(id, new Uint8Array(fs.readFileSync(filePath))); + } + + const compressedGroups = new Map(); + + for (const groupId of groupIds) { + const payload = midiData.get(groupId); + if (!payload) { + continue; + } + + const origPath = `data/cache/${args.archive}/${groupId}.dat`; + + if (args.exact) { + if (!fs.existsSync(origPath)) { + throw new Error(`Exact mode requires reference cache group: ${origPath}`); + } + + const originalContainer = new Uint8Array(fs.readFileSync(origPath)); + const originalUncompressed = unpackJs5Group(originalContainer); + + if (!arraysEqual(originalUncompressed, payload)) { + throw new Error( + `Exact mode mismatch for group ${groupId}: generated payload differs from reference cache.` + ); + } + + compressedGroups.set(groupId, originalContainer); + } else { + let compressionType = CompressionType.GZIP; + if (fs.existsSync(origPath)) { + const origData = fs.readFileSync(origPath); + compressionType = origData[0]; + } + + const compressed = await compressJs5Group(payload, compressionType); + compressedGroups.set(groupId, compressed); + } + } + + const groupBuffers: Uint8Array[] = new Array(groupIds.length); + const groupLengths: number[] = new Array(groupIds.length).fill(0); + + for (let i = 0; i < groupIds.length; i++) { + const groupId = groupIds[i]; + const compressed = compressedGroups.get(groupId); + + if (!compressed || compressed.length === 0) { + groupBuffers[i] = new Uint8Array(0); + continue; + } + + groupBuffers[i] = compressed; + groupLengths[i] = compressed.length; + } + + const lengthTable = new Uint8Array(groupIds.length * 4); + for (let i = 0; i < groupLengths.length; i++) { + writeInt32BE(groupLengths[i], lengthTable, i * 4); + } + + const totalGroupBytes = groupLengths.reduce((sum, length) => sum + length, 0); + const totalSize = indexPacked.length + totalGroupBytes + lengthTable.length; + const js5Data = new Uint8Array(totalSize); + + let offset = 0; + js5Data.set(indexPacked, offset); + offset += indexPacked.length; + + for (const group of groupBuffers) { + if (group.length === 0) { + continue; + } + + js5Data.set(group, offset); + offset += group.length; + } + + js5Data.set(lengthTable, offset); + + // Write both server and client versions (identical for MIDI) + const serverDir = path.join(args.out, 'server'); + const clientDir = path.join(args.out, 'client'); + ensureDir(serverDir); + ensureDir(clientDir); + + const serverOut = path.join(serverDir, 'server.music.js5'); + const clientOut = path.join(clientDir, 'client.music.js5'); + + fs.writeFileSync(serverOut, js5Data); + fs.writeFileSync(clientOut, js5Data); + + console.log(`Packed ${compressedGroups.size} MIDI groups from ${midiPack.size} entries`); + if (missingFiles > 0) { + console.log(`Missing source MIDI files for ${missingFiles} pack entries`); + } + if (skippedIds > 0) { + console.log(`Skipped ${skippedIds} pack entries not present in archive ${args.archive} index`); + } + console.log(`Wrote ${serverOut}`); + console.log(`Wrote ${clientOut}`); +} + +main().catch(err => { + console.error('Error:', err); + process.exit(1); +}); diff --git a/tools/cache/pack/packNpcs.ts b/tools/cache/pack/packNpcs.ts new file mode 100644 index 000000000..47c587567 --- /dev/null +++ b/tools/cache/pack/packNpcs.ts @@ -0,0 +1,874 @@ +import fs from 'fs'; +import path from 'path'; + +import { CompressionType } from '#/io/CompressionType.js'; +import { splitGroupFiles } from '#/io/Js5ArchiveIndex.js'; +import { unpackJs5Group } from '#/io/Js5Group.js'; +import Environment from '#/util/Environment.js'; +import { + resolveSectionId +} from '#tools/cache/lib/configSource.js'; +import { encodeNpcOps, type NpcOpcode } from '#tools/cache/lib/npcCodec.js'; +import { + arraysEqual, + ensureDir, + parsePackFile, + compressJs5Group, + parseGroupIdsFromIndexPacked, + writeInt32BE, + loadReferenceArchiveIndex, + combineGroupFiles, + readGroupBytes +} from '#tools/cache/lib/js5Tools.js'; + +type Args = { + src: string; + out: string; + archive: number; + exact: boolean; + help: boolean; +}; + +type NpcSourceField = { + key: string; + value: string; + line: number; +}; + +type NpcSourceSection = { + name: string; + line: number; + fields: NpcSourceField[]; +}; + +function parseArgs(argv: string[]): Args { + const args: Args = { + src: path.join(Environment.BUILD_SRC_DIR, 'scripts', '_unpack', '530', 'all.npc'), + out: 'data/pack', + archive: 18, + exact: false, + help: false + }; + + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + + if (arg === '--src') { + args.src = argv[++i]; + } else if (arg === '--out') { + args.out = argv[++i]; + } else if (arg === '--archive') { + args.archive = Number(argv[++i]); + } else if (arg === '--exact') { + args.exact = true; + } else if (arg === '--no-exact') { + args.exact = false; + } else if (arg === '--help' || arg === '-h') { + args.help = true; + } + } + + return args; +} + +function parseNpcSourceSections(content: string): NpcSourceSection[] { + const lines = content.split('\n'); + const sections: NpcSourceSection[] = []; + let current: NpcSourceSection | null = null; + + for (let i = 0; i < lines.length; i++) { + const raw = lines[i]; + const line = raw.endsWith('\r') ? raw.slice(0, -1) : raw; + const trimmed = line.trim(); + + if (trimmed.length === 0 || trimmed.startsWith('//')) { + continue; + } + + if (trimmed.startsWith('[') && trimmed.endsWith(']')) { + current = { + name: trimmed.substring(1, trimmed.length - 1), + line: i + 1, + fields: [] + }; + sections.push(current); + continue; + } + + if (!current) { + continue; + } + + const eq = line.indexOf('='); + if (eq === -1) { + continue; + } + + const key = line.substring(0, eq).trim(); + const value = line.substring(eq + 1); + if (key.length === 0) { + continue; + } + + current.fields.push({ key, value, line: i + 1 }); + } + + return sections; +} + +function parseIntStrict(value: string, context: string): number { + const parsed = Number(value); + if (!Number.isFinite(parsed)) { + throw new Error(`Invalid number for ${context}: ${value}`); + } + return parsed; +} + +function parseKeyValueNumber(value: string, key: string): number { + return parseIntStrict(value.trim(), key); +} + +function parsePair(value: string, key: string): [string, string] { + const comma = value.indexOf(','); + if (comma === -1) { + throw new Error(`Expected comma-separated value for ${key}: ${value}`); + } + + const left = value.slice(0, comma).trim(); + const right = value.slice(comma + 1).trim(); + return [left, right]; +} + +function resolveParamId(name: string, paramNameToId: Map): number { + const id = paramNameToId.get(name); + if (id !== undefined) { + return id; + } + + const numeric = Number(name); + if (Number.isInteger(numeric) && numeric >= 0) { + return numeric; + } + + throw new Error(`Unknown param name: ${name}`); +} + +function parseSourceNpcs(content: string, nameToId: Map, paramNameToId: Map): Map { + const sections = parseNpcSourceSections(content); + const byId = new Map(); + + for (const section of sections) { + const id = resolveSectionId(section.name, nameToId, 'npc_'); + if (id === null) { + throw new Error(`Unknown npc name: ${section.name}`); + } + + const ops: NpcOpcode[] = []; + + for (let i = 0; i < section.fields.length; i++) { + const field = section.fields[i]; + const key = field.key; + const value = field.value; + + if (/^op\d+$/.test(key)) { + const actionCode = Number(key.slice(2)); + if (actionCode >= 1 && actionCode <= 5) { + ops.push({ code: 29 + actionCode, payload: value }); + continue; + } + + const code = actionCode; + let payload: any; + try { + payload = JSON.parse(value); + } catch (err) { + throw new Error(`Invalid JSON payload for ${field.key} at ${section.name}:${field.line}: ${err}`); + } + ops.push({ code, payload }); + continue; + } + + if (/^model\d+$/.test(key)) { + const models: number[] = [parseKeyValueNumber(value, key)]; + while (i + 1 < section.fields.length && /^model\d+$/.test(section.fields[i + 1].key)) { + i++; + models.push(parseKeyValueNumber(section.fields[i].value, section.fields[i].key)); + } + ops.push({ code: 1, payload: { models } }); + continue; + } + + if (key === 'name') { + ops.push({ code: 2, payload: value }); + continue; + } + + if (key === 'desc') { + ops.push({ code: 3, payload: value }); + continue; + } + + if (key === 'size') { + ops.push({ code: 12, payload: parseKeyValueNumber(value, key) }); + continue; + } + + if (key === 'readyanim') { + ops.push({ code: 13, payload: parseKeyValueNumber(value, key) }); + continue; + } + + if (key === 'walkanim') { + const b = section.fields[i + 1]; + const r = section.fields[i + 2]; + const l = section.fields[i + 3]; + if (b?.key === 'walkanim_b' && r?.key === 'walkanim_r' && l?.key === 'walkanim_l') { + ops.push({ + code: 17, + payload: { + walkanim: parseKeyValueNumber(value, key), + walkanim_b: parseKeyValueNumber(b.value, b.key), + walkanim_r: parseKeyValueNumber(r.value, r.key), + walkanim_l: parseKeyValueNumber(l.value, l.key) + } + }); + i += 3; + } else { + ops.push({ code: 14, payload: parseKeyValueNumber(value, key) }); + } + continue; + } + + if (key === 'hasanim' && value === 'yes') { + ops.push({ code: 16, payload: true }); + continue; + } + + if (key === 'category') { + ops.push({ code: 18, payload: parseKeyValueNumber(value, key) }); + continue; + } + + const opMatch = /^op([1-5])$/.exec(key); + if (opMatch) { + ops.push({ code: 29 + Number(opMatch[1]), payload: value }); + continue; + } + + if (/^recol\d+s$/.test(key)) { + const pairs: Array<{ from: number; to: number }> = []; + let j = i; + while (j < section.fields.length) { + const sField = section.fields[j]; + const dField = section.fields[j + 1]; + if (!sField || !dField || !/^recol\d+s$/.test(sField.key) || !/^recol\d+d$/.test(dField.key)) { + break; + } + pairs.push({ + from: parseKeyValueNumber(sField.value, sField.key), + to: parseKeyValueNumber(dField.value, dField.key) + }); + j += 2; + } + if (pairs.length > 0) { + ops.push({ code: 40, payload: pairs }); + i = j - 1; + continue; + } + } + + if (/^retex\d+s$/.test(key)) { + const pairs: Array<{ from: number; to: number }> = []; + let j = i; + while (j < section.fields.length) { + const sField = section.fields[j]; + const dField = section.fields[j + 1]; + if (!sField || !dField || !/^retex\d+s$/.test(sField.key) || !/^retex\d+d$/.test(dField.key)) { + break; + } + pairs.push({ + from: parseKeyValueNumber(sField.value, sField.key), + to: parseKeyValueNumber(dField.value, dField.key) + }); + j += 2; + } + if (pairs.length > 0) { + ops.push({ code: 41, payload: pairs }); + i = j - 1; + continue; + } + } + + if (/^recol\d+p$/.test(key)) { + const values: number[] = [parseKeyValueNumber(value, key)]; + while (i + 1 < section.fields.length && /^recol\d+p$/.test(section.fields[i + 1].key)) { + i++; + values.push(parseKeyValueNumber(section.fields[i].value, section.fields[i].key)); + } + ops.push({ code: 42, payload: values }); + continue; + } + + if (/^head\d+$/.test(key)) { + const heads: number[] = [parseKeyValueNumber(value, key)]; + while (i + 1 < section.fields.length && /^head\d+$/.test(section.fields[i + 1].key)) { + i++; + heads.push(parseKeyValueNumber(section.fields[i].value, section.fields[i].key)); + } + ops.push({ code: 60, payload: heads }); + continue; + } + + if (key === 'attack') { + ops.push({ code: 74, payload: parseKeyValueNumber(value, key) }); + continue; + } + if (key === 'defence') { + ops.push({ code: 75, payload: parseKeyValueNumber(value, key) }); + continue; + } + if (key === 'strength') { + ops.push({ code: 76, payload: parseKeyValueNumber(value, key) }); + continue; + } + if (key === 'hitpoints') { + ops.push({ code: 77, payload: parseKeyValueNumber(value, key) }); + continue; + } + if (key === 'ranged') { + ops.push({ code: 78, payload: parseKeyValueNumber(value, key) }); + continue; + } + if (key === 'magic') { + ops.push({ code: 79, payload: parseKeyValueNumber(value, key) }); + continue; + } + + if (key === 'resizex') { + ops.push({ code: 90, payload: parseKeyValueNumber(value, key) }); + continue; + } + if (key === 'resizey') { + ops.push({ code: 91, payload: parseKeyValueNumber(value, key) }); + continue; + } + if (key === 'resizez') { + ops.push({ code: 92, payload: parseKeyValueNumber(value, key) }); + continue; + } + if (key === 'minimap' && value === 'no') { + ops.push({ code: 93, payload: true }); + continue; + } + if (key === 'vislevel') { + if (value === 'hide') { + ops.push({ code: 95, payload: 0 }); + } else { + ops.push({ code: 95, payload: parseKeyValueNumber(value, key) }); + } + continue; + } + if (key === 'resizeh') { + ops.push({ code: 97, payload: parseKeyValueNumber(value, key) }); + continue; + } + if (key === 'resizev') { + ops.push({ code: 98, payload: parseKeyValueNumber(value, key) }); + continue; + } + if (key === 'alwaysontop' && value === 'yes') { + ops.push({ code: 99, payload: true }); + continue; + } + if (key === 'ambient') { + ops.push({ code: 100, payload: parseKeyValueNumber(value, key) }); + continue; + } + if (key === 'contrast') { + ops.push({ code: 101, payload: parseKeyValueNumber(value, key) }); + continue; + } + if (key === 'headicon') { + ops.push({ code: 102, payload: parseKeyValueNumber(value, key) }); + continue; + } + if (key === 'turnspeed') { + ops.push({ code: 103, payload: parseKeyValueNumber(value, key) }); + continue; + } + + if (key === 'multivarbit') { + const multivarbit = parseKeyValueNumber(value, key); + const varpField = section.fields[i + 1]; + if (!varpField || varpField.key !== 'multivarp') { + throw new Error(`Expected multivarp after multivarbit at ${section.name}:${field.line}`); + } + + i += 1; + const multivarp = parseKeyValueNumber(varpField.value, varpField.key); + const multinpc: number[] = []; + + while (i + 1 < section.fields.length && /^multinpc\d+$/.test(section.fields[i + 1].key)) { + i += 1; + multinpc.push(parseKeyValueNumber(section.fields[i].value, section.fields[i].key)); + } + + let defaultId: number | undefined; + if (i + 1 < section.fields.length && section.fields[i + 1].key === 'multidefault') { + i += 1; + defaultId = parseKeyValueNumber(section.fields[i].value, section.fields[i].key); + } + + ops.push({ + code: defaultId !== undefined ? 118 : 106, + payload: { + multivarbit, + multivarp, + defaultId, + multinpc + } + }); + continue; + } + + if (key === 'active' && value === 'no') { + ops.push({ code: 107, payload: true }); + continue; + } + if (key === 'walksmoothing' && value === 'no') { + ops.push({ code: 109, payload: true }); + continue; + } + if (key === 'spotshadow' && value === 'no') { + ops.push({ code: 111, payload: true }); + continue; + } + if (key === 'spotshadowcolour1') { + const second = section.fields[i + 1]; + if (!second || second.key !== 'spotshadowcolour2') { + throw new Error(`Expected spotshadowcolour2 after spotshadowcolour1 at ${section.name}:${field.line}`); + } + ops.push({ + code: 112, + payload: { + colour1: parseKeyValueNumber(value, key), + colour2: parseKeyValueNumber(second.value, second.key) + } + }); + i += 1; + continue; + } + if (key === 'spotshadowtrans1') { + const second = section.fields[i + 1]; + if (!second || second.key !== 'spotshadowtrans2') { + throw new Error(`Expected spotshadowtrans2 after spotshadowtrans1 at ${section.name}:${field.line}`); + } + const trans1 = parseKeyValueNumber(value, key); + const trans2 = parseKeyValueNumber(second.value, second.key); + ops.push({ code: 113, payload: { trans1, trans2 } }); + i += 1; + continue; + } + if (key === 'spotshadowtrans1b') { + const second = section.fields[i + 1]; + if (!second || second.key !== 'spotshadowtrans2b') { + throw new Error(`Expected spotshadowtrans2b after spotshadowtrans1b at ${section.name}:${field.line}`); + } + const trans1 = parseKeyValueNumber(value, key); + const trans2 = parseKeyValueNumber(second.value, second.key); + ops.push({ code: 114, payload: { trans1, trans2 } }); + i += 1; + continue; + } + if (key === 'code115_1') { + const second = section.fields[i + 1]; + if (!second || second.key !== 'code115_2') { + throw new Error(`Expected code115_2 after code115_1 at ${section.name}:${field.line}`); + } + ops.push({ + code: 115, + payload: { + value1: parseKeyValueNumber(value, key), + value2: parseKeyValueNumber(second.value, second.key) + } + }); + i += 1; + continue; + } + if (key === 'walkflags') { + ops.push({ code: 119, payload: parseKeyValueNumber(value, key) }); + continue; + } + + if (/^modeloffset\d+$/.test(key)) { + const offsets: Array<{ index: number; x: number; y: number; z: number }> = []; + let j = i; + while (j < section.fields.length && /^modeloffset\d+$/.test(section.fields[j].key)) { + const offsetField = section.fields[j]; + const index = Number(offsetField.key.slice('modeloffset'.length)); + const parts = offsetField.value.split(',').map(part => part.trim()); + if (parts.length !== 3) { + throw new Error(`Invalid model offset format for ${offsetField.key} at ${section.name}:${offsetField.line}`); + } + offsets.push({ + index, + x: parseIntStrict(parts[0], `${offsetField.key}.x`), + y: parseIntStrict(parts[1], `${offsetField.key}.y`), + z: parseIntStrict(parts[2], `${offsetField.key}.z`) + }); + j += 1; + } + if (offsets.length > 0) { + ops.push({ code: 121, payload: offsets }); + i = j - 1; + continue; + } + } + + if (key === 'hitbarid') { + ops.push({ code: 122, payload: parseKeyValueNumber(value, key) }); + continue; + } + if (key === 'overlayheight') { + ops.push({ code: 123, payload: parseKeyValueNumber(value, key) }); + continue; + } + if (key === 'respawndir') { + ops.push({ code: 125, payload: parseKeyValueNumber(value, key) }); + continue; + } + if (key === 'bas') { + ops.push({ code: 127, payload: parseKeyValueNumber(value, key) }); + continue; + } + if (key === 'movespeed') { + ops.push({ code: 128, payload: parseKeyValueNumber(value, key) }); + continue; + } + if (key === 'bgsound') { + const crawl = section.fields[i + 1]; + const walk = section.fields[i + 2]; + const run = section.fields[i + 3]; + const range = section.fields[i + 4]; + if (!crawl || !walk || !run || !range || crawl.key !== 'bgsound_crawl' || walk.key !== 'bgsound_walk' || run.key !== 'bgsound_run' || range.key !== 'bgsound_range') { + throw new Error(`Incomplete bgsound block at ${section.name}:${field.line}`); + } + ops.push({ + code: 134, + payload: { + bgsound: parseKeyValueNumber(value, key), + bgsound_crawl: parseKeyValueNumber(crawl.value, crawl.key), + bgsound_walk: parseKeyValueNumber(walk.value, walk.key), + bgsound_run: parseKeyValueNumber(run.value, run.key), + bgsound_range: parseKeyValueNumber(range.value, range.key) + } + }); + i += 4; + continue; + } + if (key === 'cursor1op') { + const next = section.fields[i + 1]; + if (!next || next.key !== 'cursor1') { + throw new Error(`Expected cursor1 after cursor1op at ${section.name}:${field.line}`); + } + ops.push({ + code: 135, + payload: { + op: parseKeyValueNumber(value, key), + cursor: parseKeyValueNumber(next.value, next.key) + } + }); + i += 1; + continue; + } + if (key === 'cursor2op') { + const next = section.fields[i + 1]; + if (!next || next.key !== 'cursor2') { + throw new Error(`Expected cursor2 after cursor2op at ${section.name}:${field.line}`); + } + ops.push({ + code: 136, + payload: { + op: parseKeyValueNumber(value, key), + cursor: parseKeyValueNumber(next.value, next.key) + } + }); + i += 1; + continue; + } + if (key === 'cursorattack') { + ops.push({ code: 137, payload: parseKeyValueNumber(value, key) }); + continue; + } + if (key === 'wanderrange') { + ops.push({ code: 200, payload: parseKeyValueNumber(value, key) }); + continue; + } + if (key === 'maxrange') { + ops.push({ code: 201, payload: parseKeyValueNumber(value, key) }); + continue; + } + if (key === 'huntrange') { + ops.push({ code: 202, payload: parseKeyValueNumber(value, key) }); + continue; + } + if (key === 'timer') { + ops.push({ code: 203, payload: parseKeyValueNumber(value, key) }); + continue; + } + if (key === 'respawnrate') { + ops.push({ code: 204, payload: parseKeyValueNumber(value, key) }); + continue; + } + if (key === 'moverestrict') { + ops.push({ code: 206, payload: parseKeyValueNumber(value, key) }); + continue; + } + if (key === 'attackrange') { + ops.push({ code: 207, payload: parseKeyValueNumber(value, key) }); + continue; + } + if (key === 'blockwalk') { + ops.push({ code: 208, payload: parseKeyValueNumber(value, key) }); + continue; + } + if (key === 'huntmode') { + ops.push({ code: 209, payload: parseKeyValueNumber(value, key) }); + continue; + } + if (key === 'defaultmode') { + ops.push({ code: 210, payload: parseKeyValueNumber(value, key) }); + continue; + } + if (key === 'members' && value === 'yes') { + ops.push({ code: 211, payload: true }); + continue; + } + + if (/^patrol\d+$/.test(key)) { + const patrol: Array<{ coord: number; delay: number }> = []; + let j = i; + while (j < section.fields.length && /^patrol\d+$/.test(section.fields[j].key)) { + const patrolField = section.fields[j]; + const [coord, delay] = parsePair(patrolField.value, patrolField.key); + patrol.push({ + coord: parseIntStrict(coord, `${patrolField.key}.coord`), + delay: parseIntStrict(delay, `${patrolField.key}.delay`) + }); + j += 1; + } + ops.push({ code: 212, payload: patrol }); + i = j - 1; + continue; + } + + if (key === 'givechase' && value === 'no') { + ops.push({ code: 213, payload: true }); + continue; + } + + if (key === 'param' || key === 'paramstr') { + const params: Array<{ string: boolean; key: number; value: number | string }> = []; + let j = i; + + while (j < section.fields.length && (section.fields[j].key === 'param' || section.fields[j].key === 'paramstr')) { + const paramField = section.fields[j]; + const [paramName, paramValue] = parsePair(paramField.value, paramField.key); + params.push({ + string: paramField.key === 'paramstr', + key: resolveParamId(paramName, paramNameToId), + value: paramField.key === 'paramstr' ? paramValue : parseIntStrict(paramValue, `${paramField.key}.value`) + }); + j += 1; + } + + ops.push({ code: 249, payload: params }); + i = j - 1; + continue; + } + + if (key === 'debugname') { + ops.push({ code: 250, payload: value }); + continue; + } + } + + byId.set(id, ops); + } + + return byId; +} + +async function readOriginalGroupContainer(archive: number, groupId: number): Promise { + const localPath = `data/cache/${archive}/${groupId}.dat`; + if (fs.existsSync(localPath)) { + return new Uint8Array(fs.readFileSync(localPath)); + } + + const fetched = await readGroupBytes(archive, groupId, 'data/cache', true); + if (!fetched) { + throw new Error(`Reference cache group not found: data/cache/${archive}/${groupId}.dat`); + } + + return fetched; +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + + if (!fs.existsSync(args.src)) { + throw new Error(`Source file not found: ${args.src}`); + } + + const localPackDir = path.join(path.dirname(args.src), 'pack'); + const fallbackPackDir = path.join(Environment.BUILD_SRC_DIR, 'pack'); + const npcLocalPackPath = path.join(localPackDir, 'npc.pack'); + const npcFallbackPackPath = path.join(fallbackPackDir, 'npc.pack'); + const nameToId = fs.existsSync(npcLocalPackPath) ? parsePackFile(npcLocalPackPath) : parsePackFile(npcFallbackPackPath); + const paramLocalPackPath = path.join(localPackDir, 'param.pack'); + const paramFallbackPackPath = path.join(fallbackPackDir, 'param.pack'); + const paramNameToId = fs.existsSync(paramLocalPackPath) + ? parsePackFile(paramLocalPackPath) + : parsePackFile(paramFallbackPackPath); + + const sourceContent = fs.readFileSync(args.src, 'utf-8'); + const npcOpsById = parseSourceNpcs(sourceContent, nameToId, paramNameToId); + + const referenceIndex = loadReferenceArchiveIndex(args.archive); + if (!referenceIndex) { + throw new Error(`Reference index not found for archive ${args.archive}`); + } + + const encodedByGroup = new Map>(); + for (const [id, ops] of npcOpsById) { + const groupId = id >> 8; + const fileId = id & 0xff; + + const encoded = encodeNpcOps(ops); + + if (!encodedByGroup.has(groupId)) { + encodedByGroup.set(groupId, new Map()); + } + + encodedByGroup.get(groupId)!.set(fileId, encoded); + } + + const indexPath = `data/cache/255/${args.archive}.dat`; + if (!fs.existsSync(indexPath)) { + throw new Error(`Index file not found: ${indexPath}`); + } + + const indexPacked = new Uint8Array(fs.readFileSync(indexPath)); + const groupIds = parseGroupIdsFromIndexPacked(indexPacked); + + const finalGroupContainers = new Map(); + + for (const groupId of groupIds) { + const originals = await readOriginalGroupContainer(args.archive, groupId); + const encodedFiles = encodedByGroup.get(groupId); + + if (!encodedFiles) { + finalGroupContainers.set(groupId, originals); + continue; + } + + const orderedFileIds = referenceIndex.fileIdsByGroup.get(groupId) ?? []; + if (orderedFileIds.length === 0) { + finalGroupContainers.set(groupId, originals); + continue; + } + + for (const fileId of orderedFileIds) { + if (!encodedFiles.has(fileId)) { + encodedFiles.set(fileId, new Uint8Array([0x00])); + } + } + + const combined = combineGroupFiles(encodedFiles, orderedFileIds); + + if (args.exact) { + const originalUncompressed = unpackJs5Group(originals); + if (!arraysEqual(originalUncompressed, combined)) { + const originalFiles = splitGroupFiles(originalUncompressed, orderedFileIds); + let mismatchFileId: number | null = null; + for (const fileId of orderedFileIds) { + const left = originalFiles.get(fileId) ?? new Uint8Array(0); + const right = encodedFiles.get(fileId) ?? new Uint8Array(0); + if (!arraysEqual(left, right)) { + mismatchFileId = fileId; + break; + } + } + + const mismatchId = mismatchFileId !== null ? ((groupId << 8) | mismatchFileId) : null; + throw new Error( + `Exact mode mismatch for group ${groupId}` + + (mismatchFileId !== null ? ` file=${mismatchFileId} id=${mismatchId}` : '') + + ': generated payload differs from reference cache.' + ); + } + + finalGroupContainers.set(groupId, originals); + continue; + } + + const compressionType = originals[0] ?? CompressionType.GZIP; + const compressed = await compressJs5Group(combined, compressionType); + finalGroupContainers.set(groupId, compressed); + } + + const groupBuffers: Uint8Array[] = new Array(groupIds.length); + const groupLengths: number[] = new Array(groupIds.length).fill(0); + + for (let i = 0; i < groupIds.length; i++) { + const groupId = groupIds[i]; + const container = finalGroupContainers.get(groupId); + + if (!container) { + throw new Error(`Missing packed data for group ${groupId}`); + } + + groupBuffers[i] = container; + groupLengths[i] = container.length; + } + + const lengthTable = new Uint8Array(groupIds.length * 4); + for (let i = 0; i < groupLengths.length; i++) { + writeInt32BE(groupLengths[i], lengthTable, i * 4); + } + + const totalGroupBytes = groupLengths.reduce((sum, length) => sum + length, 0); + const totalSize = indexPacked.length + totalGroupBytes + lengthTable.length; + const output = new Uint8Array(totalSize); + + let pos = 0; + output.set(indexPacked, pos); + pos += indexPacked.length; + + for (const group of groupBuffers) { + output.set(group, pos); + pos += group.length; + } + + output.set(lengthTable, pos); + + const serverDir = path.join(args.out, 'server'); + const clientDir = path.join(args.out, 'client'); + ensureDir(serverDir); + ensureDir(clientDir); + + const serverOut = path.join(serverDir, 'server.npc.config.js5'); + const clientOut = path.join(clientDir, 'client.npc.config.js5'); + + fs.writeFileSync(serverOut, output); + fs.writeFileSync(clientOut, output); + + console.log(`Packed NPC archive with ${groupIds.length} groups and ${npcOpsById.size} npc configs`); + console.log(`Wrote ${serverOut}`); + console.log(`Wrote ${clientOut}`); +} + +main().catch(err => { + console.error(err); + process.exit(1); +}); diff --git a/tools/cache/pack/packObjs.ts b/tools/cache/pack/packObjs.ts new file mode 100644 index 000000000..38dc1ba47 --- /dev/null +++ b/tools/cache/pack/packObjs.ts @@ -0,0 +1,973 @@ +import fs from 'fs'; +import path from 'path'; + +import { CompressionType } from '#/io/CompressionType.js'; +import { splitGroupFiles } from '#/io/Js5ArchiveIndex.js'; +import { unpackJs5Group } from '#/io/Js5Group.js'; +import Environment from '#/util/Environment.js'; +import { resolveSectionId } from '#tools/cache/lib/configSource.js'; +import { encodeObjOps, type ObjOpcode } from '#tools/cache/lib/objCodec.js'; +import { + arraysEqual, + ensureDir, + parsePackFile, + compressJs5Group, + parseGroupIdsFromIndexPacked, + writeInt32BE, + loadReferenceArchiveIndex, + combineGroupFiles, + readGroupBytes +} from '#tools/cache/lib/js5Tools.js'; + +type Args = { + src: string; + out: string; + archive: number; + exact: boolean; + exactTarget: 'server' | 'client'; + help: boolean; +}; + +type ObjSourceField = { + key: string; + value: string; + line: number; +}; + +type ObjSourceSection = { + name: string; + line: number; + fields: ObjSourceField[]; +}; + +const CERT_TEMPLATE_ID = 799; +const LENT_TEMPLATE_ID = 13009; +const SERVER_ONLY_OBJ_OPCODES = new Set([3, 13, 14, 15, 27, 75, 94, 123, 201]); + +function parseArgs(argv: string[]): Args { + const args: Args = { + src: path.join(Environment.BUILD_SRC_DIR, 'scripts', '_unpack', '530', 'all.obj'), + out: 'data/pack', + archive: 19, + exact: false, + exactTarget: 'server', + help: false + }; + + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + + if (arg === '--src') { + args.src = argv[++i]; + } else if (arg === '--out') { + args.out = argv[++i]; + } else if (arg === '--archive') { + args.archive = Number(argv[++i]); + } else if (arg === '--exact') { + args.exact = true; + args.exactTarget = 'server'; + } else if (arg === '--exact-client') { + args.exact = true; + args.exactTarget = 'client'; + } else if (arg === '--no-exact') { + args.exact = false; + args.exactTarget = 'server'; + } else if (arg === '--help' || arg === '-h') { + args.help = true; + } + } + + return args; +} + +function parseObjSourceSections(content: string): ObjSourceSection[] { + const lines = content.split('\n'); + const sections: ObjSourceSection[] = []; + let current: ObjSourceSection | null = null; + + for (let i = 0; i < lines.length; i++) { + const raw = lines[i]; + const line = raw.endsWith('\r') ? raw.slice(0, -1) : raw; + const trimmed = line.trim(); + + if (trimmed.length === 0 || trimmed.startsWith('//')) { + continue; + } + + if (trimmed.startsWith('[') && trimmed.endsWith(']')) { + current = { + name: trimmed.substring(1, trimmed.length - 1), + line: i + 1, + fields: [] + }; + sections.push(current); + continue; + } + + if (!current) { + continue; + } + + const eq = line.indexOf('='); + if (eq === -1) { + continue; + } + + const key = line.substring(0, eq).trim(); + const value = line.substring(eq + 1); + if (key.length === 0) { + continue; + } + + current.fields.push({ key, value, line: i + 1 }); + } + + return sections; +} + +function parseIntStrict(value: string, context: string): number { + const parsed = Number(value); + if (!Number.isFinite(parsed)) { + throw new Error(`Invalid number for ${context}: ${value}`); + } + return parsed; +} + +function parseKeyValueNumber(value: string, key: string): number { + return parseIntStrict(value.trim(), key); +} + +function parseWeightValue(value: string): number { + const normalized = value.trim().toLowerCase(); + const match = /^([+-]?\d+(?:\.\d+)?)\s*(kg|oz|lb|g)$/.exec(normalized); + if (!match) { + throw new Error(`Invalid weight value: ${value}`); + } + + const amount = Number(match[1]); + const unit = match[2]; + let grams = 0; + + if (unit === 'kg') { + grams = amount * 1000; + } else if (unit === 'oz') { + grams = amount * 28.3495; + } else if (unit === 'lb') { + grams = amount * 453.592; + } else { + grams = amount; + } + + const rounded = Math.round(grams); + if (rounded < -32768 || rounded > 32767) { + throw new Error(`Weight out of int16 range: ${value}`); + } + + return rounded; +} + +function parsePair(value: string, key: string): [string, string] { + const comma = value.indexOf(','); + if (comma === -1) { + throw new Error(`Expected comma-separated value for ${key}: ${value}`); + } + + const left = value.slice(0, comma).trim(); + const right = value.slice(comma + 1).trim(); + return [left, right]; +} + +function resolveParamId(name: string, paramNameToId: Map): number { + const id = paramNameToId.get(name); + if (id !== undefined) { + return id; + } + + const numeric = Number(name); + if (Number.isInteger(numeric) && numeric >= 0) { + return numeric; + } + + throw new Error(`Unknown param name: ${name}`); +} + +function extractLinkedSourceIdFromSectionName(sectionName: string, kind: 'cert' | 'lent'): number | undefined { + const prefix = `${kind}_obj_`; + if (!sectionName.startsWith(prefix)) { + return undefined; + } + + const rest = sectionName.substring(prefix.length); + const first = rest.split('_')[0]; + const parsed = Number(first); + if (!Number.isInteger(parsed) || parsed < 0) { + return undefined; + } + + return parsed; +} + +function parseSourceObjs(content: string, nameToId: Map, paramNameToId: Map): Map { + const sections = parseObjSourceSections(content); + const byId = new Map(); + const sectionNameById = new Map(); + + for (const section of sections) { + const id = resolveSectionId(section.name, nameToId, 'obj_'); + + if (id === null) { + throw new Error(`Unknown obj name: ${section.name}`); + } + + sectionNameById.set(id, section.name); + + const ops: ObjOpcode[] = []; + + for (let i = 0; i < section.fields.length; i++) { + const field = section.fields[i]; + const key = field.key; + const value = field.value; + + if (/^op\d+$/.test(key)) { + const opNum = Number(key.slice(2)); + if (opNum >= 1 && opNum <= 5) { + ops.push({ code: 29 + opNum, payload: value }); + continue; + } + + const code = opNum; + let payload: any; + try { + payload = JSON.parse(value); + } catch (err) { + throw new Error(`Invalid JSON payload for ${field.key} at ${section.name}:${field.line}: ${err}`); + } + ops.push({ code, payload }); + continue; + } + + if (/^iop[1-5]$/.test(key)) { + const idx = Number(key.slice(3)); + ops.push({ code: 34 + idx, payload: value }); + continue; + } + + if (key === 'model') { + ops.push({ code: 1, payload: parseKeyValueNumber(value, key) }); + continue; + } + if (key === 'name') { + ops.push({ code: 2, payload: value }); + continue; + } + if (key === 'desc') { + ops.push({ code: 3, payload: value }); + continue; + } + if (key === 'zoom2d') { + ops.push({ code: 4, payload: parseKeyValueNumber(value, key) }); + continue; + } + if (key === 'xan2d') { + ops.push({ code: 5, payload: parseKeyValueNumber(value, key) }); + continue; + } + if (key === 'yan2d') { + ops.push({ code: 6, payload: parseKeyValueNumber(value, key) }); + continue; + } + if (key === 'xof2d') { + ops.push({ code: 7, payload: parseKeyValueNumber(value, key) }); + continue; + } + if (key === 'yof2d') { + ops.push({ code: 8, payload: parseKeyValueNumber(value, key) }); + continue; + } + if (key === 'stackable' && value.trim() === 'yes') { + ops.push({ code: 11, payload: true }); + continue; + } + if (key === 'cost') { + ops.push({ code: 12, payload: parseKeyValueNumber(value, key) }); + continue; + } + if (key === 'wearpos') { + ops.push({ code: 13, payload: parseKeyValueNumber(value, key) }); + continue; + } + if (key === 'wearpos2') { + ops.push({ code: 14, payload: parseKeyValueNumber(value, key) }); + continue; + } + if (key === 'tradeable' && value.trim() === 'no') { + ops.push({ code: 15, payload: true }); + continue; + } + if (key === 'members' && value.trim() === 'yes') { + ops.push({ code: 16, payload: true }); + continue; + } + if (key === 'manwear') { + ops.push({ code: 23, payload: parseKeyValueNumber(value, key) }); + continue; + } + if (key === 'manwear2') { + ops.push({ code: 24, payload: parseKeyValueNumber(value, key) }); + continue; + } + if (key === 'womanwear') { + ops.push({ code: 25, payload: parseKeyValueNumber(value, key) }); + continue; + } + if (key === 'womanwear2') { + ops.push({ code: 26, payload: parseKeyValueNumber(value, key) }); + continue; + } + if (key === 'wearpos3') { + ops.push({ code: 27, payload: parseKeyValueNumber(value, key) }); + continue; + } + + if (/^recol\d+s$/.test(key)) { + const pairs: Array<{ from: number; to: number }> = []; + let j = i; + while (j < section.fields.length) { + const sField = section.fields[j]; + const dField = section.fields[j + 1]; + if (!sField || !dField || !/^recol\d+s$/.test(sField.key) || !/^recol\d+d$/.test(dField.key)) { + break; + } + pairs.push({ + from: parseKeyValueNumber(sField.value, sField.key), + to: parseKeyValueNumber(dField.value, dField.key) + }); + j += 2; + } + if (pairs.length > 0) { + ops.push({ code: 40, payload: pairs }); + i = j - 1; + continue; + } + } + + if (/^retex\d+s$/.test(key)) { + const pairs: Array<{ from: number; to: number }> = []; + let j = i; + while (j < section.fields.length) { + const sField = section.fields[j]; + const dField = section.fields[j + 1]; + if (!sField || !dField || !/^retex\d+s$/.test(sField.key) || !/^retex\d+d$/.test(dField.key)) { + break; + } + pairs.push({ + from: parseKeyValueNumber(sField.value, sField.key), + to: parseKeyValueNumber(dField.value, dField.key) + }); + j += 2; + } + if (pairs.length > 0) { + ops.push({ code: 41, payload: pairs }); + i = j - 1; + continue; + } + } + + if (/^recol\d+p$/.test(key)) { + const values: number[] = [parseKeyValueNumber(value, key)]; + while (i + 1 < section.fields.length && /^recol\d+p$/.test(section.fields[i + 1].key)) { + i++; + values.push(parseKeyValueNumber(section.fields[i].value, section.fields[i].key)); + } + ops.push({ code: 42, payload: values }); + continue; + } + + if (key === 'stockmarket' && value.trim() === 'yes') { + ops.push({ code: 65, payload: true }); + continue; + } + if (key === 'weight') { + ops.push({ code: 75, payload: parseWeightValue(value) }); + continue; + } + if (key === 'manwear3') { + ops.push({ code: 78, payload: parseKeyValueNumber(value, key) }); + continue; + } + if (key === 'womanwear3') { + ops.push({ code: 79, payload: parseKeyValueNumber(value, key) }); + continue; + } + if (key === 'manhead') { + ops.push({ code: 90, payload: parseKeyValueNumber(value, key) }); + continue; + } + if (key === 'womanhead') { + ops.push({ code: 91, payload: parseKeyValueNumber(value, key) }); + continue; + } + if (key === 'manhead2') { + ops.push({ code: 92, payload: parseKeyValueNumber(value, key) }); + continue; + } + if (key === 'womanhead2') { + ops.push({ code: 93, payload: parseKeyValueNumber(value, key) }); + continue; + } + if (key === 'category') { + ops.push({ code: 94, payload: parseKeyValueNumber(value, key) }); + continue; + } + if (key === 'zan2d') { + ops.push({ code: 95, payload: parseKeyValueNumber(value, key) }); + continue; + } + if (key === 'dummyitem') { + const normalized = value.trim().toLowerCase(); + if (normalized === 'graphic_only') { + ops.push({ code: 96, payload: 1 }); + } else if (normalized === 'inv_only') { + ops.push({ code: 96, payload: 2 }); + } else { + ops.push({ code: 96, payload: parseKeyValueNumber(value, key) }); + } + continue; + } + if (key === 'certlink') { + ops.push({ code: 97, payload: parseKeyValueNumber(value, key) }); + continue; + } + if (key === 'certtemplate') { + ops.push({ code: 98, payload: parseKeyValueNumber(value, key) }); + continue; + } + + const countObjMatch = /^countobj(\d+)$/.exec(key); + if (countObjMatch) { + const idx = Number(countObjMatch[1]); + const countField = section.fields[i + 1]; + if (!countField || countField.key !== `countco${idx}`) { + throw new Error(`Expected countco${idx} after countobj${idx} at ${section.name}:${field.line}`); + } + ops.push({ + code: 99 + idx, + payload: { + obj: parseKeyValueNumber(value, key), + count: parseKeyValueNumber(countField.value, countField.key) + } + }); + i += 1; + continue; + } + + if (key === 'resizex') { + ops.push({ code: 110, payload: parseKeyValueNumber(value, key) }); + continue; + } + if (key === 'resizey') { + ops.push({ code: 111, payload: parseKeyValueNumber(value, key) }); + continue; + } + if (key === 'resizez') { + ops.push({ code: 112, payload: parseKeyValueNumber(value, key) }); + continue; + } + if (key === 'ambient') { + ops.push({ code: 113, payload: parseKeyValueNumber(value, key) }); + continue; + } + if (key === 'contrast') { + ops.push({ code: 114, payload: parseKeyValueNumber(value, key) }); + continue; + } + if (key === 'team') { + ops.push({ code: 115, payload: parseKeyValueNumber(value, key) }); + continue; + } + if (key === 'lentlink') { + ops.push({ code: 121, payload: parseKeyValueNumber(value, key) }); + continue; + } + if (key === 'lenttemplate') { + ops.push({ code: 122, payload: parseKeyValueNumber(value, key) }); + continue; + } + if (key === 'lendable' && value.trim() === 'yes') { + ops.push({ code: 123, payload: true }); + continue; + } + if (key === 'manwearoffset') { + const [x, yz] = parsePair(value, key); + const [y, z] = parsePair(yz, key); + ops.push({ + code: 125, + payload: { + x: parseIntStrict(x, `${key}.x`), + y: parseIntStrict(y, `${key}.y`), + z: parseIntStrict(z, `${key}.z`) + } + }); + continue; + } + if (key === 'womanwearoffset') { + const [x, yz] = parsePair(value, key); + const [y, z] = parsePair(yz, key); + ops.push({ + code: 126, + payload: { + x: parseIntStrict(x, `${key}.x`), + y: parseIntStrict(y, `${key}.y`), + z: parseIntStrict(z, `${key}.z`) + } + }); + continue; + } + if (key === 'cursor1op') { + const next = section.fields[i + 1]; + if (!next || next.key !== 'cursor1') { + throw new Error(`Expected cursor1 after cursor1op at ${section.name}:${field.line}`); + } + ops.push({ code: 127, payload: { op: parseKeyValueNumber(value, key), cursor: parseKeyValueNumber(next.value, next.key) } }); + i += 1; + continue; + } + if (key === 'cursor2op') { + const next = section.fields[i + 1]; + if (!next || next.key !== 'cursor2') { + throw new Error(`Expected cursor2 after cursor2op at ${section.name}:${field.line}`); + } + ops.push({ code: 128, payload: { op: parseKeyValueNumber(value, key), cursor: parseKeyValueNumber(next.value, next.key) } }); + i += 1; + continue; + } + if (key === 'cursor1iop') { + const next = section.fields[i + 1]; + if (!next || next.key !== 'cursor1i') { + throw new Error(`Expected cursor1i after cursor1iop at ${section.name}:${field.line}`); + } + ops.push({ code: 129, payload: { op: parseKeyValueNumber(value, key), cursor: parseKeyValueNumber(next.value, next.key) } }); + i += 1; + continue; + } + if (key === 'cursor2iop') { + const next = section.fields[i + 1]; + if (!next || next.key !== 'cursor2i') { + throw new Error(`Expected cursor2i after cursor2iop at ${section.name}:${field.line}`); + } + ops.push({ code: 130, payload: { op: parseKeyValueNumber(value, key), cursor: parseKeyValueNumber(next.value, next.key) } }); + i += 1; + continue; + } + if (key === 'respawnrate') { + ops.push({ code: 201, payload: parseKeyValueNumber(value, key) }); + continue; + } + + if (key === 'param' || key === 'paramstr') { + const params: Array<{ string: boolean; key: number; value: number | string }> = []; + let j = i; + + while (j < section.fields.length && (section.fields[j].key === 'param' || section.fields[j].key === 'paramstr')) { + const paramField = section.fields[j]; + const [paramName, paramValue] = parsePair(paramField.value, paramField.key); + params.push({ + string: paramField.key === 'paramstr', + key: resolveParamId(paramName, paramNameToId), + value: paramField.key === 'paramstr' ? paramValue : parseIntStrict(paramValue, `${paramField.key}.value`) + }); + j += 1; + } + + ops.push({ code: 249, payload: params }); + i = j - 1; + continue; + } + + if (key === 'debugname') { + ops.push({ code: 250, payload: value }); + continue; + } + } + + byId.set(id, ops); + } + + for (const [name, id] of nameToId) { + if (byId.has(id)) { + continue; + } + + const certTarget = extractLinkedSourceIdFromSectionName(name, 'cert'); + if (certTarget !== undefined) { + byId.set(id, [ + { code: 97, payload: certTarget }, + { code: 98, payload: CERT_TEMPLATE_ID } + ]); + continue; + } + + const lentTarget = extractLinkedSourceIdFromSectionName(name, 'lent'); + if (lentTarget !== undefined) { + byId.set(id, [ + { code: 121, payload: lentTarget }, + { code: 122, payload: LENT_TEMPLATE_ID } + ]); + } + } + + const certBySource = new Map(); + const lentBySource = new Map(); + for (const [id, ops] of byId) { + const hasCertTemplate = ops.some(op => op.code === 98); + if (hasCertTemplate) { + const certLink = ops.find(op => op.code === 97); + const sourceIdFromName = extractLinkedSourceIdFromSectionName(sectionNameById.get(id) ?? '', 'cert'); + const sourceId = certLink ? Number(certLink.payload) : sourceIdFromName; + if (sourceId !== undefined) { + const existing = certBySource.get(sourceId); + if (existing === undefined || id < existing) { + certBySource.set(sourceId, id); + } + } + } + + const hasLentTemplate = ops.some(op => op.code === 122); + if (hasLentTemplate) { + const lentLink = ops.find(op => op.code === 121); + const sourceIdFromName = extractLinkedSourceIdFromSectionName(sectionNameById.get(id) ?? '', 'lent'); + const sourceId = lentLink ? Number(lentLink.payload) : sourceIdFromName; + if (sourceId !== undefined) { + const existing = lentBySource.get(sourceId); + if (existing === undefined || id < existing) { + lentBySource.set(sourceId, id); + } + } + } + } + + for (const [id, ops] of byId) { + const hasCertLink = ops.some(op => op.code === 97); + const hasCertTemplate = ops.some(op => op.code === 98); + const hasLentLink = ops.some(op => op.code === 121); + const hasLentTemplate = ops.some(op => op.code === 122); + + if (hasCertTemplate && !hasCertLink) { + const sourceFromName = extractLinkedSourceIdFromSectionName(sectionNameById.get(id) ?? '', 'cert'); + if (sourceFromName !== undefined) { + const templateIndex = ops.findIndex(op => op.code === 98); + const certLinkOp: ObjOpcode = { code: 97, payload: sourceFromName }; + ops.splice(templateIndex, 0, certLinkOp); + } + } + + if (hasLentTemplate && !hasLentLink) { + const sourceFromName = extractLinkedSourceIdFromSectionName(sectionNameById.get(id) ?? '', 'lent'); + if (sourceFromName !== undefined) { + const templateIndex = ops.findIndex(op => op.code === 122); + const lentLinkOp: ObjOpcode = { code: 121, payload: sourceFromName }; + ops.splice(templateIndex, 0, lentLinkOp); + } + } + + const derivedCertId = !hasCertLink && !hasCertTemplate ? certBySource.get(id) : undefined; + const derivedLentId = !hasLentLink && !hasLentTemplate ? lentBySource.get(id) : undefined; + + const findLastOffset = (): number => { + for (let i = ops.length - 1; i >= 0; i--) { + if (ops[i].code === 125 || ops[i].code === 126) { + return i; + } + } + return -1; + }; + + const findParamStart = (): number => { + const index = ops.findIndex(op => op.code === 249 || op.code === 250); + return index === -1 ? ops.length : index; + }; + + if (derivedCertId !== undefined) { + const certLinkOp: ObjOpcode = { code: 97, payload: derivedCertId }; + const lastOffset = findLastOffset(); + let insertAt = findParamStart(); + if (lastOffset !== -1) { + insertAt = lastOffset + 1; + } + ops.splice(insertAt, 0, certLinkOp); + } + + if (derivedLentId !== undefined) { + const lentLinkOp: ObjOpcode = { code: 121, payload: derivedLentId }; + const certIndex = ops.findIndex(op => op.code === 97); + const lastOffset = findLastOffset(); + const paramStart = findParamStart(); + let insertAt = paramStart; + if (lastOffset !== -1) { + insertAt = Math.min(paramStart, lastOffset + 1); + } + if (certIndex !== -1) { + insertAt = Math.max(insertAt, certIndex + 1); + } + ops.splice(insertAt, 0, lentLinkOp); + } + + const nameIndex = ops.findIndex(op => op.code === 2); + if (nameIndex !== -1) { + const [nameOp] = ops.splice(nameIndex, 1); + let anchor = ops.findIndex(op => op.code === 97 || op.code === 121 || op.code === 123 || op.code === 125 || op.code === 126 || op.code === 249 || op.code === 250); + if (anchor === -1) { + anchor = ops.length; + } + ops.splice(anchor, 0, nameOp); + } + } + + return byId; +} + +async function readOriginalGroupContainer(archive: number, groupId: number): Promise { + const localPath = `data/cache/${archive}/${groupId}.dat`; + if (fs.existsSync(localPath)) { + return new Uint8Array(fs.readFileSync(localPath)); + } + + const fetched = await readGroupBytes(archive, groupId, 'data/cache', true); + if (!fetched) { + throw new Error(`Reference cache group not found: data/cache/${archive}/${groupId}.dat`); + } + + return fetched; +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + + if (!fs.existsSync(args.src)) { + throw new Error(`Source file not found: ${args.src}`); + } + + const localPackDir = path.join(path.dirname(args.src), 'pack'); + const fallbackPackDir = path.join(Environment.BUILD_SRC_DIR, 'pack'); + const objLocalPackPath = path.join(localPackDir, 'obj.pack'); + const objFallbackPackPath = path.join(fallbackPackDir, 'obj.pack'); + const nameToId = new Map(); + if (fs.existsSync(objFallbackPackPath)) { + for (const [name, id] of parsePackFile(objFallbackPackPath)) { + nameToId.set(name, id); + } + } + if (fs.existsSync(objLocalPackPath)) { + for (const [name, id] of parsePackFile(objLocalPackPath)) { + nameToId.set(name, id); + } + } + const paramLocalPackPath = path.join(localPackDir, 'param.pack'); + const paramFallbackPackPath = path.join(fallbackPackDir, 'param.pack'); + const paramNameToId = new Map(); + if (fs.existsSync(paramFallbackPackPath)) { + for (const [name, id] of parsePackFile(paramFallbackPackPath)) { + paramNameToId.set(name, id); + } + } + if (fs.existsSync(paramLocalPackPath)) { + for (const [name, id] of parsePackFile(paramLocalPackPath)) { + paramNameToId.set(name, id); + } + } + + const sourceContent = fs.readFileSync(args.src, 'utf-8'); + const objOpsById = parseSourceObjs(sourceContent, nameToId, paramNameToId); + + const referenceIndex = loadReferenceArchiveIndex(args.archive); + if (!referenceIndex) { + throw new Error(`Reference index not found for archive ${args.archive}`); + } + + const encodedByGroup = new Map>(); + const encodedByGroupClient = new Map>(); + let hasServerOnlyObjOps = false; + for (const [id, ops] of objOpsById) { + const groupId = id >> 8; + const fileId = id & 0xff; + + const encoded = encodeObjOps(ops); + const clientOps = ops.filter(op => !SERVER_ONLY_OBJ_OPCODES.has(op.code)); + const encodedClient = encodeObjOps(clientOps); + if (clientOps.length !== ops.length) { + hasServerOnlyObjOps = true; + } + + if (!encodedByGroup.has(groupId)) { + encodedByGroup.set(groupId, new Map()); + } + if (!encodedByGroupClient.has(groupId)) { + encodedByGroupClient.set(groupId, new Map()); + } + + encodedByGroup.get(groupId)!.set(fileId, encoded); + encodedByGroupClient.get(groupId)!.set(fileId, encodedClient); + } + + const indexPath = `data/cache/255/${args.archive}.dat`; + if (!fs.existsSync(indexPath)) { + throw new Error(`Index file not found: ${indexPath}`); + } + + const indexPacked = new Uint8Array(fs.readFileSync(indexPath)); + const groupIds = parseGroupIdsFromIndexPacked(indexPacked); + + const buildArchiveOutput = async (groups: Map>, exactMode: boolean): Promise => { + const finalGroupContainers = new Map(); + + for (const groupId of groupIds) { + const originals = await readOriginalGroupContainer(args.archive, groupId); + const encodedFiles = groups.get(groupId); + + if (!encodedFiles) { + finalGroupContainers.set(groupId, originals); + continue; + } + + const orderedFileIds = referenceIndex.fileIdsByGroup.get(groupId) ?? []; + if (orderedFileIds.length === 0) { + finalGroupContainers.set(groupId, originals); + continue; + } + + for (const fileId of orderedFileIds) { + if (!encodedFiles.has(fileId)) { + encodedFiles.set(fileId, new Uint8Array([0x00])); + } + } + + const combined = combineGroupFiles(encodedFiles, orderedFileIds); + + if (exactMode) { + const originalUncompressed = unpackJs5Group(originals); + if (!arraysEqual(originalUncompressed, combined)) { + const originalFiles = splitGroupFiles(originalUncompressed, orderedFileIds); + let mismatchFileId: number | null = null; + let mismatchOffset = -1; + let leftBytes: Uint8Array | null = null; + let rightBytes: Uint8Array | null = null; + for (const fileId of orderedFileIds) { + const left = originalFiles.get(fileId) ?? new Uint8Array(0); + const right = encodedFiles.get(fileId) ?? new Uint8Array(0); + if (!arraysEqual(left, right)) { + mismatchFileId = fileId; + leftBytes = left; + rightBytes = right; + const minLen = Math.min(left.length, right.length); + for (let i = 0; i < minLen; i++) { + if (left[i] !== right[i]) { + mismatchOffset = i; + break; + } + } + if (mismatchOffset === -1 && left.length !== right.length) { + mismatchOffset = minLen; + } + break; + } + } + + const mismatchId = mismatchFileId !== null ? ((groupId << 8) | mismatchFileId) : null; + let detail = ''; + if (leftBytes && rightBytes && mismatchOffset >= 0) { + const start = Math.max(0, mismatchOffset - 8); + const endL = Math.min(leftBytes.length, mismatchOffset + 8); + const endR = Math.min(rightBytes.length, mismatchOffset + 8); + const leftWindow = Array.from(leftBytes.subarray(start, endL)).map(b => b.toString(16).padStart(2, '0')).join(' '); + const rightWindow = Array.from(rightBytes.subarray(start, endR)).map(b => b.toString(16).padStart(2, '0')).join(' '); + const leftAt = mismatchOffset < leftBytes.length ? leftBytes[mismatchOffset] : -1; + const rightAt = mismatchOffset < rightBytes.length ? rightBytes[mismatchOffset] : -1; + detail = ` diffOffset=${mismatchOffset} leftByte=${leftAt} rightByte=${rightAt} ` + + `leftLen=${leftBytes.length} rightLen=${rightBytes.length} ` + + `leftWindow[${start}..${endL})=${leftWindow} rightWindow[${start}..${endR})=${rightWindow}`; + } + throw new Error( + `Exact mode mismatch for group ${groupId}` + + (mismatchFileId !== null ? ` file=${mismatchFileId} id=${mismatchId}` : '') + + `: generated payload differs from reference cache.${detail}` + ); + } + + finalGroupContainers.set(groupId, originals); + continue; + } + + const compressionType = originals[0] ?? CompressionType.GZIP; + const compressed = await compressJs5Group(combined, compressionType); + finalGroupContainers.set(groupId, compressed); + } + + const groupBuffers: Uint8Array[] = new Array(groupIds.length); + const groupLengths: number[] = new Array(groupIds.length).fill(0); + + for (let i = 0; i < groupIds.length; i++) { + const groupId = groupIds[i]; + const container = finalGroupContainers.get(groupId); + + if (!container) { + throw new Error(`Missing packed data for group ${groupId}`); + } + + groupBuffers[i] = container; + groupLengths[i] = container.length; + } + + const lengthTable = new Uint8Array(groupIds.length * 4); + for (let i = 0; i < groupLengths.length; i++) { + writeInt32BE(groupLengths[i], lengthTable, i * 4); + } + + const totalGroupBytes = groupLengths.reduce((sum, length) => sum + length, 0); + const totalSize = indexPacked.length + totalGroupBytes + lengthTable.length; + const output = new Uint8Array(totalSize); + + let pos = 0; + output.set(indexPacked, pos); + pos += indexPacked.length; + + for (const group of groupBuffers) { + output.set(group, pos); + pos += group.length; + } + + output.set(lengthTable, pos); + return output; + }; + + const exactServer = args.exact && args.exactTarget === 'server'; + const exactClient = args.exact && args.exactTarget === 'client'; + + const output = await buildArchiveOutput(encodedByGroup, exactServer); + const clientOutput = await buildArchiveOutput(encodedByGroupClient, exactClient); + + const serverDir = path.join(args.out, 'server'); + const clientDir = path.join(args.out, 'client'); + ensureDir(serverDir); + ensureDir(clientDir); + + const serverOut = path.join(serverDir, 'server.obj.config.js5'); + const clientOut = path.join(clientDir, 'client.obj.config.js5'); + + fs.writeFileSync(serverOut, output); + fs.writeFileSync(clientOut, clientOutput); + + console.log(`Packed obj archive with ${groupIds.length} groups and ${objOpsById.size} obj configs`); + if (args.exact) { + console.log(`Exact validation target: ${args.exactTarget}`); + if (hasServerOnlyObjOps && args.exactTarget === 'server') { + console.log('Server exact mode will fail when server-only object opcodes deviate from reference cache.'); + } + } + console.log(`Wrote ${serverOut}`); + console.log(`Wrote ${clientOut}`); +} + +main().catch(err => { + console.error(err); + process.exit(1); +}); diff --git a/tools/cache/pack/packParams.ts b/tools/cache/pack/packParams.ts new file mode 100644 index 000000000..8282679ee --- /dev/null +++ b/tools/cache/pack/packParams.ts @@ -0,0 +1,263 @@ +import fs from 'fs'; +import path from 'path'; + +import ParamType from '#/cache/config/ParamType.js'; +import ScriptVarType from '#/cache/config/ScriptVarType.js'; +import { CompressionType } from '#/io/CompressionType.js'; +import Packet from '#/io/Packet.js'; +import Environment from '#/util/Environment.js'; +import { + parseBracketedConfigSource, + parseConfigBoolean, + parseConfigInteger, + resolveSectionId +} from '#tools/cache/lib/configSource.js'; +import { + arraysEqual, + ensureDir, + combineGroupFiles, + compressJs5Group, + loadArchiveGroupFiles, + parsePackFile +} from '#tools/cache/lib/js5Tools.js'; + +type Args = { + src: string; + out: string; + index: number; + archive: number; + exact: boolean; + debug: boolean; + help: boolean; +}; + +function parseArgs(argv: string[]): Args { + const args: Args = { + src: path.join(Environment.BUILD_SRC_DIR, 'scripts', '_unpack', '530', 'all.param'), + out: 'data/pack', + index: 2, + archive: 11, + exact: false, + debug: false, + help: false + }; + + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + + if (arg === '--src') { + args.src = argv[++i]; + } else if (arg === '--out') { + args.out = argv[++i]; + } else if (arg === '--index') { + args.index = Number(argv[++i]); + } else if (arg === '--archive') { + args.archive = Number(argv[++i]); + } else if (arg === '--exact') { + args.exact = true; + } else if (arg === '--no-exact') { + args.exact = false; + } else if (arg === '--debug') { + args.debug = true; + } else if (arg === '--help' || arg === '-h') { + args.help = true; + } + } + + return args; +} + +type OpcodeValue = { + code: number; + value?: any; +}; + +type ParsedParamSource = { + config: ParamType; + opcodes: OpcodeValue[]; +}; + +function parseSourceTypeName(typeName: string): number { + if (typeName.startsWith('unknown_')) { + const code = parseInt(typeName.substring(8)); + if (!isNaN(code)) { + return code; + } + } + + const typeCode = ScriptVarType.getTypeChar(typeName); + return typeCode ?? ScriptVarType.INT; +} + +function parseDefaultIntByType(value: string, type: number, structNameToId: Map): number { + if (type === ScriptVarType.STRUCT) { + const structId = resolveSectionId(value, structNameToId, 'struct_'); + if (structId === null) { + throw new Error(`Unknown struct default: ${value}`); + } + return structId; + } + + return parseConfigInteger(value); +} + +function parseSourceParams(content: string, nameToId: Map, structNameToId: Map): Map { + const params = new Map(); + const sections = parseBracketedConfigSource(content); + + for (const section of sections) { + const id = resolveSectionId(section.name, nameToId, 'param_'); + if (id === null) { + throw new Error(`Unknown param name: ${section.name}`); + } + + const config = new ParamType(id); + config.debugname = null; + const opcodes: OpcodeValue[] = []; + const deferredDefaults: Array<{ index: number; value: string }> = []; + + for (const field of section.fields) { + const { key, value } = field; + + if (key === 'type') { + const typeValue = parseSourceTypeName(value); + config.type = typeValue; + opcodes.push({ code: 1, value: typeValue }); + } else if (key === 'default') { + opcodes.push({ code: -1, value }); + deferredDefaults.push({ index: opcodes.length - 1, value }); + } else if (key === 'defaultint') { + const defaultInt = parseConfigInteger(value); + config.defaultInt = defaultInt; + opcodes.push({ code: 2, value: defaultInt }); + } else if (key === 'autodisable') { + config.autodisable = parseConfigBoolean(value); + if (!config.autodisable) { + opcodes.push({ code: 4 }); + } + } else if (key === 'defaultstr') { + config.defaultString = value; + opcodes.push({ code: 5, value }); + } else if (key === 'debugname') { + config.debugname = value; + opcodes.push({ code: 250, value }); + } + } + + for (const deferred of deferredDefaults) { + if (config.type === ScriptVarType.STRING) { + config.defaultString = deferred.value; + opcodes[deferred.index] = { code: 5, value: deferred.value }; + } else { + const defaultInt = parseDefaultIntByType(deferred.value, config.type, structNameToId); + config.defaultInt = defaultInt; + opcodes[deferred.index] = { code: 2, value: defaultInt }; + } + } + + params.set(id, { config, opcodes }); + } + + return params; +} + +function encodeParamWithOpcodes(config: ParamType, opcodes: OpcodeValue[]): Uint8Array { + const buf = Packet.alloc(2); + + for (const opc of opcodes) { + if (opc.code === 1) { + buf.p1(1); + buf.p1(config.type); + } else if (opc.code === 2) { + buf.p1(2); + buf.p4(config.defaultInt); + } else if (opc.code === 4) { + buf.p1(4); + } else if (opc.code === 5) { + buf.p1(5); + buf.pjstr(String(opc.value ?? '')); + } else if (opc.code === 250) { + buf.p1(250); + buf.pjstr(String(opc.value ?? '')); + } + } + + buf.p1(0); + return new Uint8Array(buf.data.subarray(0, buf.pos)); +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + + if (!fs.existsSync(args.src)) { + throw new Error(`Source file not found: ${args.src}`); + } + + const { + fileIds: currentGroupFileIds, + groupUnpacked: currentGroupUnpacked + } = await loadArchiveGroupFiles(args.index, args.archive, 'data/cache', true); + + const localPackDir = path.join(path.dirname(args.src), 'pack'); + const fallbackPackDir = path.join(Environment.BUILD_SRC_DIR, 'pack'); + const localPackPath = path.join(localPackDir, 'param.pack'); + const fallbackPackPath = path.join(fallbackPackDir, 'param.pack'); + const localStructPackPath = path.join(localPackDir, 'struct.pack'); + const fallbackStructPackPath = path.join(fallbackPackDir, 'struct.pack'); + + const nameToId = fs.existsSync(localPackPath) ? parsePackFile(localPackPath) : parsePackFile(fallbackPackPath); + const structNameToId = fs.existsSync(localStructPackPath) + ? parsePackFile(localStructPackPath) + : parsePackFile(fallbackStructPackPath); + + const sourceContent = fs.readFileSync(args.src, 'utf-8'); + const sourceParams = parseSourceParams(sourceContent, nameToId, structNameToId); + + const fileData = new Map(); + for (const fileId of currentGroupFileIds) { + const sourceParam = sourceParams.get(fileId); + if (!sourceParam) { + fileData.set(fileId, new Uint8Array(0)); + continue; + } + + const encoded = encodeParamWithOpcodes(sourceParam.config, sourceParam.opcodes); + fileData.set(fileId, encoded); + } + + const combined = combineGroupFiles(fileData, currentGroupFileIds); + + if (args.exact) { + if (!arraysEqual(combined, currentGroupUnpacked)) { + if (args.debug) { + for (let i = 0; i < Math.min(combined.length, currentGroupUnpacked.length); i++) { + if (combined[i] !== currentGroupUnpacked[i]) { + console.error(`Mismatch at offset ${i}: generated=${combined[i]}, reference=${currentGroupUnpacked[i]}`); + break; + } + } + } + + throw new Error('Generated group does not match reference'); + } + } + + const compressed = await compressJs5Group(combined, CompressionType.GZIP); + const filename = `${args.archive}.dat`; + const filepath = path.join(args.out, filename); + + ensureDir(args.out); + if (fs.existsSync(filepath) && fs.statSync(filepath).isDirectory()) { + fs.rmSync(filepath, { recursive: true, force: true }); + } + + fs.writeFileSync(filepath, compressed); + + console.log(`Packed ${sourceParams.size} param configs`); + console.log(`Wrote ${filepath}`); +} + +main().catch(err => { + console.error(err); + process.exit(1); +}); diff --git a/tools/cache/pack/packPatches.ts b/tools/cache/pack/packPatches.ts new file mode 100644 index 000000000..cf1a19dea --- /dev/null +++ b/tools/cache/pack/packPatches.ts @@ -0,0 +1,192 @@ +import fs from 'fs'; +import path from 'path'; + +import { unpackJs5Group } from '#/io/Js5Group.js'; +import Environment from '#/util/Environment.js'; +import { resolvePatchPathByName } from '#tools/pack/sources/PatchSource.js'; +import { + arraysEqual, + ensureDir, + parsePackFile, + compressJs5Group, + parseGroupIdsFromIndexPacked, + readGroupBytes, + writeInt32BE +} from '#tools/cache/lib/js5Tools.js'; + +type Args = { + src: string; + out: string; + archive: number; + exact: boolean; + help: boolean; +}; + +function parseArgs(argv: string[]): Args { + const args: Args = { + src: path.join(Environment.BUILD_SRC_DIR, 'pack', 'patch.pack'), + out: 'data/pack', + archive: 15, + exact: false, + help: false, + }; + + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg === '--src') { + args.src = argv[++i]; + } else if (arg === '--out') { + args.out = argv[++i]; + } else if (arg === '--archive') { + args.archive = Number(argv[++i]); + } else if (arg === '--exact') { + args.exact = true; + } else if (arg === '--no-exact') { + args.exact = false; + } else if (arg === '--help' || arg === '-h') { + args.help = true; + } + } + + return args; +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + + if (!fs.existsSync(args.src)) { + throw new Error(`Pack file not found: ${args.src}`); + } + + ensureDir(args.out); + + const patchPack = parsePackFile(args.src); + const indexPath = `data/cache/255/${args.archive}.dat`; + + if (!fs.existsSync(indexPath)) { + throw new Error(`Index file not found: ${indexPath}`); + } + + const indexPacked = new Uint8Array(fs.readFileSync(indexPath)); + const groupIds = parseGroupIdsFromIndexPacked(indexPacked); + const validGroupIds = new Set(groupIds); + + const patchData = new Map(); + let missingFiles = 0; + let skippedIds = 0; + + for (const [name, id] of patchPack) { + if (!validGroupIds.has(id)) { + skippedIds++; + continue; + } + + const filePath = resolvePatchPathByName(name); + if (!filePath) { + missingFiles++; + continue; + } + + patchData.set(id, new Uint8Array(fs.readFileSync(filePath))); + } + + const compressedGroups = new Map(); + + for (const groupId of groupIds) { + let originalContainer: Uint8Array | null = null; + + const localGroupPath = `data/cache/${args.archive}/${groupId}.dat`; + + if (fs.existsSync(localGroupPath)) { + originalContainer = new Uint8Array(fs.readFileSync(localGroupPath)); + } else { + originalContainer = await readGroupBytes(args.archive, groupId, 'data/cache', true); + if (!originalContainer) { + throw new Error(`Reference cache group not found: data/cache/${args.archive}/${groupId}.dat`); + } + } + + const payload = patchData.get(groupId); + + if (!payload) { + compressedGroups.set(groupId, originalContainer); + continue; + } + + if (args.exact) { + const originalUncompressed = unpackJs5Group(originalContainer); + + if (!arraysEqual(originalUncompressed, payload)) { + throw new Error( + `Exact mode mismatch for group ${groupId}: generated payload differs from reference cache.` + ); + } + + compressedGroups.set(groupId, originalContainer); + } else { + const compressed = await compressJs5Group(payload, originalContainer[0]); + compressedGroups.set(groupId, compressed); + } + } + + const groupBuffers: Uint8Array[] = new Array(groupIds.length); + const groupLengths: number[] = new Array(groupIds.length).fill(0); + + for (let i = 0; i < groupIds.length; i++) { + const groupId = groupIds[i]; + const compressed = compressedGroups.get(groupId); + + if (!compressed) { + throw new Error(`Missing packed data for group ${groupId}`); + } + + groupBuffers[i] = compressed; + groupLengths[i] = compressed.length; + } + + const lengthTable = new Uint8Array(groupIds.length * 4); + for (let i = 0; i < groupLengths.length; i++) { + writeInt32BE(groupLengths[i], lengthTable, i * 4); + } + + const totalGroupBytes = groupLengths.reduce((sum, length) => sum + length, 0); + const totalSize = indexPacked.length + totalGroupBytes + lengthTable.length; + const js5Data = new Uint8Array(totalSize); + + let offset = 0; + js5Data.set(indexPacked, offset); + offset += indexPacked.length; + + for (const group of groupBuffers) { + js5Data.set(group, offset); + offset += group.length; + } + + js5Data.set(lengthTable, offset); + + const serverDir = path.join(args.out, 'server'); + const clientDir = path.join(args.out, 'client'); + ensureDir(serverDir); + ensureDir(clientDir); + + const serverOut = path.join(serverDir, 'server.patches.js5'); + const clientOut = path.join(clientDir, 'client.patches.js5'); + + fs.writeFileSync(serverOut, js5Data); + fs.writeFileSync(clientOut, js5Data); + + console.log(`Packed ${compressedGroups.size} patches from ${patchPack.size} pack entries`); + if (missingFiles > 0) { + console.log(`Missing source patch files for ${missingFiles} pack entries`); + } + if (skippedIds > 0) { + console.log(`Skipped ${skippedIds} pack entries not present in archive ${args.archive} index`); + } + console.log(`Wrote ${serverOut}`); + console.log(`Wrote ${clientOut}`); +} + +main().catch(err => { + console.error('Error:', err); + process.exit(1); +}); diff --git a/tools/cache/pack/packQuickChat.ts b/tools/cache/pack/packQuickChat.ts new file mode 100644 index 000000000..160163074 --- /dev/null +++ b/tools/cache/pack/packQuickChat.ts @@ -0,0 +1,548 @@ +import fs from 'fs'; +import path from 'path'; + +import QuickChatCatType from '#/cache/config/QuickChatCatType.js'; +import QuickChatPhraseType from '#/cache/config/QuickChatPhraseType.js'; +import { CompressionType } from '#/io/CompressionType.js'; +import { parseJs5ArchiveIndex } from '#/io/Js5ArchiveIndex.js'; +import { unpackJs5Group } from '#/io/Js5Group.js'; +import Environment from '#/util/Environment.js'; +import { getGroup } from '#/util/OpenRS2.js'; +import { encodeQuickChatCat } from '#tools/cache/lib/quickchatCatCodec.js'; +import { encodeQuickChatPhrase } from '#tools/cache/lib/quickchatPhraseCodec.js'; +import { + ensureDir, + parsePackFile, + combineGroupFiles, + compressJs5Group, + parseGroupIdsFromIndexPacked, + writeInt32BE +} from '#tools/cache/lib/js5Tools.js'; + +const DEFAULT_ARCHIVE = 24; +const CAT_GROUP = 0; +const PHRASE_GROUP = 1; + +type Args = { + archive: number; + catInput: string; + phraseInput: string; + catPackInput: string; + phrasePackInput: string; + enumPackInput: string; + out: string; + mode: 'server' | 'client' | 'both'; + exact: boolean; + help: boolean; +}; + +type ParsedQuickChatCat = { + id: number; + config: QuickChatCatType; + data: Uint8Array; +}; + +type ParsedQuickChatPhrase = { + id: number; + config: QuickChatPhraseType; + data: Uint8Array; +}; + +function parseArgs(argv: string[]): Args { + const args: Args = { + archive: DEFAULT_ARCHIVE, + catInput: '', + phraseInput: '', + catPackInput: path.join(Environment.BUILD_SRC_DIR, 'pack', 'chatcat.pack'), + phrasePackInput: path.join(Environment.BUILD_SRC_DIR, 'pack', 'chatphrase.pack'), + enumPackInput: path.join(Environment.BUILD_SRC_DIR, 'pack', 'enum.pack'), + out: 'data/pack', + mode: 'both', + exact: false, + help: false + }; + + let catInputOverride = false; + let phraseInputOverride = false; + + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + + if (arg === '--archive') { + args.archive = Number(argv[++i]); + } else if (arg === '--cat-input') { + args.catInput = argv[++i]; + catInputOverride = true; + } else if (arg === '--phrase-input') { + args.phraseInput = argv[++i]; + phraseInputOverride = true; + } else if (arg === '--cat-pack-input') { + args.catPackInput = argv[++i]; + } else if (arg === '--phrase-pack-input') { + args.phrasePackInput = argv[++i]; + } else if (arg === '--enum-pack-input') { + args.enumPackInput = argv[++i]; + } else if (arg === '--mode') { + const mode = argv[++i]; + if (mode === 'server' || mode === 'client' || mode === 'both') { + args.mode = mode; + } + } else if (arg === '--exact') { + args.exact = true; + } else if (arg === '--out') { + args.out = argv[++i]; + } else if (arg === '--help' || arg === '-h') { + args.help = true; + } + } + + // Set archive-specific default input paths if not explicitly provided + if (!catInputOverride) { + if (args.archive === 24) { + args.catInput = path.join(Environment.BUILD_SRC_DIR, 'scripts', '_unpack', '530', 'all.chatcat'); + } else if (args.archive === 25) { + args.catInput = path.join(Environment.BUILD_SRC_DIR, 'scripts', '_unpack', '530', 'all.global.chatcat'); + } + } + + if (!phraseInputOverride) { + if (args.archive === 24) { + args.phraseInput = path.join(Environment.BUILD_SRC_DIR, 'scripts', '_unpack', '530', 'all.chatphrase'); + } else if (args.archive === 25) { + args.phraseInput = path.join(Environment.BUILD_SRC_DIR, 'scripts', '_unpack', '530', 'all.global.chatphrase'); + } + } + + return args; +} + +function parseQuickChatCat(text: string, catNameToId: Map, phraseNameToId: Map, validCatIds: number[]): Map { + const results = new Map(); + const validCatIdSet = new Set(validCatIds); + + // First, create all category objects and encode empty ones + for (const i of validCatIds) { + const cat = new QuickChatCatType(i); + cat.subcategories = []; + cat.subcategoryShortcuts = []; + cat.phrases = []; + cat.phraseShortcuts = []; + cat.hasOpcode4 = false; + const encoded = encodeQuickChatCat(cat); + if (encoded) { + results.set(i, { + id: i, + config: cat, + data: encoded + }); + } + } + + // Now parse and update from text config + const lines = text.split('\n'); + let currentId = -1; + + for (const rawLine of lines) { + const line = rawLine.endsWith('\r') ? rawLine.slice(0, -1) : rawLine; + const trimmed = line.trim(); + + if (trimmed.length === 0 || trimmed.startsWith('//')) { + continue; + } + + // [chatcat_] or [name] + if (trimmed.startsWith('[') && trimmed.endsWith(']')) { + const name = trimmed.substring(1, trimmed.length - 1); + if (name.startsWith('chatcat_')) { + currentId = parseInt(name.substring(8)); + } else { + // Try to look up in pack file + currentId = catNameToId.get(name) ?? -1; + } + continue; + } + + if (currentId < 0 || !validCatIdSet.has(currentId)) { + continue; + } + + const eqIndex = line.indexOf('='); + if (eqIndex === -1) { + continue; + } + + const key = line.substring(0, eqIndex).trim(); + const value = line.substring(eqIndex + 1).trim(); + + const parsed = results.get(currentId); + if (!parsed || !parsed.config.subcategories || !parsed.config.phrases) { + continue; + } + + if (key === 'description') { + parsed.config.description = value; + } else if (key === 'opcode4') { + const normalized = value.trim().toLowerCase(); + parsed.config.hasOpcode4 = normalized === 'yes' || normalized === 'true' || normalized === '1'; + } else if (key === 'sub') { + // sub=, or sub=, + const commaIdx = value.indexOf(','); + let subId = -1; + if (commaIdx !== -1) { + const subPart = value.substring(0, commaIdx); + // Try to parse as a number first + subId = parseInt(subPart); + if (isNaN(subId)) { + // Try to look up in pack file + subId = catNameToId.get(subPart) ?? -1; + } + const shortcut = parseShortcutToken(value.substring(commaIdx + 1)); + if (subId >= 0 && !parsed.config.subcategories.includes(subId)) { + parsed.config.subcategories.push(subId); + parsed.config.subcategoryShortcuts?.push(shortcut); + } + } + } else if (key === 'phrase') { + // phrase=, or phrase=, + const commaIdx = value.indexOf(','); + let phraseId = -1; + if (commaIdx !== -1) { + const phrasePart = value.substring(0, commaIdx); + // Try to parse as a number first + phraseId = parseInt(phrasePart); + if (isNaN(phraseId)) { + // Try to look up in pack file + phraseId = phraseNameToId.get(phrasePart) ?? -1; + } + const shortcut = parseShortcutToken(value.substring(commaIdx + 1)); + if (phraseId >= 0 && !parsed.config.phrases.includes(phraseId)) { + parsed.config.phrases.push(phraseId); + parsed.config.phraseShortcuts?.push(shortcut); + } + } else { + // No comma, just the name/id + phraseId = parseInt(value); + if (isNaN(phraseId)) { + phraseId = phraseNameToId.get(value) ?? -1; + } + if (phraseId >= 0 && !parsed.config.phrases.includes(phraseId)) { + parsed.config.phrases.push(phraseId); + parsed.config.phraseShortcuts?.push(0); + } + } + } + } + + // Re-encode all categories with parsed data + for (const [_id, parsed] of results) { + const encoded = encodeQuickChatCat(parsed.config); + if (encoded) { + parsed.data = encoded; + } + } + + return results; +} + +function parseShortcutToken(token: string): number { + const trimmed = token.trim(); + if (trimmed.length === 0) { + return 0; + } + + const numeric = parseInt(trimmed, 10); + if (!Number.isNaN(numeric)) { + return numeric & 0xff; + } + + return trimmed.charCodeAt(0) & 0xff; +} + +function parseQuickChatPhrase(text: string, phraseNameToId: Map, enumNameToId: Map, validPhraseIds: number[]): Map { + const results = new Map(); + const validPhraseIdSet = new Set(validPhraseIds); + + // First, create all phrase objects and encode empty ones + for (const i of validPhraseIds) { + const phrase = new QuickChatPhraseType(i); + phrase.text = []; + phrase.autoResponses = []; + phrase.dynamicCommands = []; + phrase.dynamicCommandParameters = []; + const encoded = encodeQuickChatPhrase(phrase); + if (encoded) { + results.set(i, { + id: i, + config: phrase, + data: encoded + }); + } + } + + // Now parse and update from text config + const lines = text.split('\n'); + let currentId = -1; + + for (const rawLine of lines) { + const line = rawLine.endsWith('\r') ? rawLine.slice(0, -1) : rawLine; + const trimmed = line.trim(); + + if (trimmed.length === 0 || trimmed.startsWith('//')) { + continue; + } + + // [chatphrase_] or [name] + if (trimmed.startsWith('[') && trimmed.endsWith(']')) { + const name = trimmed.substring(1, trimmed.length - 1); + if (name.startsWith('chatphrase_')) { + currentId = parseInt(name.substring(11)); + } else { + // Try to look up in pack file + currentId = phraseNameToId.get(name) ?? -1; + } + continue; + } + + if (currentId < 0 || !validPhraseIdSet.has(currentId)) { + continue; + } + + const eqIndex = line.indexOf('='); + if (eqIndex === -1) { + continue; + } + + const key = line.substring(0, eqIndex).trim(); + const value = line.substring(eqIndex + 1).trim(); + + const parsed = results.get(currentId); + if (!parsed || !parsed.config.text || !parsed.config.autoResponses || !parsed.config.dynamicCommands || !parsed.config.dynamicCommandParameters) { + continue; + } + + if (key === 'text') { + parsed.config.text = [value]; + } else if (key === 'response') { + // response= or response= + let responseId = parseInt(value); + if (isNaN(responseId)) { + // Try to look up in pack file + responseId = phraseNameToId.get(value) ?? -1; + } + if (responseId >= 0 && !parsed.config.autoResponses.includes(responseId)) { + parsed.config.autoResponses.push(responseId); + } + } else if (key === 'command') { + // command=,,... + // Only cmd 0 uses enum names (e.g., "enum_1493"), others use numeric values + const parts = value.split(',').map(p => p.trim()); + if (parts.length > 0) { + const cmdId = parseInt(parts[0]); + if (!isNaN(cmdId)) { + const params: number[] = []; + for (let j = 1; j < parts.length; j++) { + const part = parts[j]; + let paramId = parseInt(part); + if (isNaN(paramId) && cmdId === 0) { + // Only look up enum names for command 0 + paramId = enumNameToId.get(part) ?? -1; + } + if (paramId >= 0) { + params.push(paramId); + } + } + parsed.config.dynamicCommands.push(cmdId); + parsed.config.dynamicCommandParameters.push(params); + } + } + } + } + + // Re-encode all phrases with parsed data + for (const [_id, parsed] of results) { + const encoded = encodeQuickChatPhrase(parsed.config); + if (encoded) { + parsed.data = encoded; + } + } + + return results; +} +async function main() { + const args = parseArgs(process.argv.slice(2)); + + ensureDir(args.out); + + const catPack = parsePackFile(args.catPackInput); + const phrasePack = parsePackFile(args.phrasePackInput); + const enumPack = parsePackFile(args.enumPackInput); + + if (!fs.existsSync(args.catInput)) { + throw new Error(`Category config not found: ${args.catInput}`); + } + + if (!fs.existsSync(args.phraseInput)) { + throw new Error(`Phrase config not found: ${args.phraseInput}`); + } + + // Load index (getGroup handles caching automatically) + const indexData = await getGroup(255, args.archive); + const groupIds = parseGroupIdsFromIndexPacked(indexData); + const indexUnpacked = unpackJs5Group(indexData); + const { fileIdsByGroup } = parseJs5ArchiveIndex(indexUnpacked); + + const catFileIds = fileIdsByGroup.get(CAT_GROUP) ?? []; + const phraseFileIds = fileIdsByGroup.get(PHRASE_GROUP) ?? []; + + if (catFileIds.length === 0 || phraseFileIds.length === 0) { + throw new Error(`Archive ${args.archive} is missing QuickChat groups 0/1 file metadata.`); + } + + // Parse configs + const catText = fs.readFileSync(args.catInput, 'utf-8'); + const phraseText = fs.readFileSync(args.phraseInput, 'utf-8'); + + const cats = parseQuickChatCat(catText, catPack, phrasePack, catFileIds); + const phrases = parseQuickChatPhrase(phraseText, phrasePack, enumPack, phraseFileIds); + + // Build group 0 (categories) + const catFiles = new Map(); + for (const [id, parsed] of cats) { + catFiles.set(id, parsed.data); + } + + const catCombined = combineGroupFiles(catFiles, catFileIds); + + // Build group 1 (phrases) + const phraseFiles = new Map(); + for (const [id, parsed] of phrases) { + phraseFiles.set(id, parsed.data); + } + + const phraseCombined = combineGroupFiles(phraseFiles, phraseFileIds); + + // Validate exact match if requested + let catCompressed: Uint8Array; + let phraseCompressed: Uint8Array; + + if (args.exact) { + // Fetch reference groups from cache + const catPath = `data/cache/${args.archive}/${CAT_GROUP}.dat`; + const phrasePath = `data/cache/${args.archive}/${PHRASE_GROUP}.dat`; + + if (!fs.existsSync(catPath) || !fs.existsSync(phrasePath)) { + throw new Error(`Exact mode requires reference cache groups: ${catPath} and ${phrasePath}`); + } + + const refCatContainer = new Uint8Array(fs.readFileSync(catPath)); + const refPhraseContainer = new Uint8Array(fs.readFileSync(phrasePath)); + + const refCatData = unpackJs5Group(refCatContainer); + const refPhraseData = unpackJs5Group(refPhraseContainer); + + // Compare categories + if (catCombined.length !== refCatData.length) { + throw new Error(`Category group size mismatch: generated ${catCombined.length} vs reference ${refCatData.length}`); + } + + let catMismatch = false; + for (let i = 0; i < catCombined.length; i++) { + if (catCombined[i] !== refCatData[i]) { + console.error(`Category group mismatch at byte ${i}: generated 0x${catCombined[i].toString(16).padStart(2, '0')} vs reference 0x${refCatData[i].toString(16).padStart(2, '0')}`); + catMismatch = true; + break; + } + } + + // Compare phrases + if (phraseCombined.length !== refPhraseData.length) { + throw new Error(`Phrase group size mismatch: generated ${phraseCombined.length} vs reference ${refPhraseData.length}`); + } + + let phraseMismatch = false; + for (let i = 0; i < phraseCombined.length; i++) { + if (phraseCombined[i] !== refPhraseData[i]) { + console.error(`Phrase group mismatch at byte ${i}: generated 0x${phraseCombined[i].toString(16).padStart(2, '0')} vs reference 0x${refPhraseData[i].toString(16).padStart(2, '0')}`); + phraseMismatch = true; + break; + } + } + + if (catMismatch || phraseMismatch) { + throw new Error('Validation failed: generated data does not match reference cache'); + } + + catCompressed = refCatContainer; + phraseCompressed = refPhraseContainer; + } else { + catCompressed = await compressJs5Group(catCombined, CompressionType.GZIP); + phraseCompressed = await compressJs5Group(phraseCombined, CompressionType.GZIP); + } + + // Build full archive + const groupBuffers = new Map(); + groupBuffers.set(CAT_GROUP, catCompressed); + groupBuffers.set(PHRASE_GROUP, phraseCompressed); + + // Build length table + const totalGroups = Math.max(...groupIds) + 1; + const lengthTable = new Uint8Array(totalGroups * 4); + + for (let i = 0; i < groupIds.length; i++) { + const groupId = groupIds[i]; + const compressed = groupBuffers.get(groupId); + const length = compressed ? compressed.length : 0; + writeInt32BE(length, lengthTable, i * 4); + } + + // Build output + let totalSize = indexData.length; + for (const [, data] of groupBuffers) { + totalSize += data.length; + } + totalSize += lengthTable.length; + + const output = new Uint8Array(totalSize); + let pos = 0; + + output.set(indexData, pos); + pos += indexData.length; + + for (const groupId of groupIds) { + const data = groupBuffers.get(groupId); + if (data && data.length > 0) { + output.set(data, pos); + pos += data.length; + } + } + + output.set(lengthTable, pos); + + // Write output + let filename: string; + if (args.archive === 24) { + filename = 'quickchat.js5'; + } else if (args.archive === 25) { + filename = 'quickchat.global.js5'; + } else { + throw new Error(`Unsupported archive ${args.archive}. Only archives 24 and 25 are supported.`); + } + + const modes: Array<'server' | 'client'> = args.mode === 'both' + ? ['server', 'client'] + : [args.mode]; + + for (const mode of modes) { + const modeDir = path.join(args.out, mode); + ensureDir(modeDir); + const outPath = path.join(modeDir, `${mode}.${filename}`); + fs.writeFileSync(outPath, output); + console.log(`Wrote ${outPath}`); + } +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); + diff --git a/tools/cache/pack/packStructs.ts b/tools/cache/pack/packStructs.ts new file mode 100644 index 000000000..f7050fdb4 --- /dev/null +++ b/tools/cache/pack/packStructs.ts @@ -0,0 +1,301 @@ +import fs from 'fs'; +import path from 'path'; + +import ParamType from '#/cache/config/ParamType.js'; +import ScriptVarType from '#/cache/config/ScriptVarType.js'; +import StructType from '#/cache/config/StructType.js'; +import { CompressionType } from '#/io/CompressionType.js'; +import Packet from '#/io/Packet.js'; +import Environment from '#/util/Environment.js'; +import { + parseBracketedConfigSource, + parseConfigInteger, + resolveSectionId +} from '#tools/cache/lib/configSource.js'; +import { + arraysEqual, + ensureDir, + combineGroupFiles, + compressJs5Group, + loadArchiveGroupFiles, + parsePackFile +} from '#tools/cache/lib/js5Tools.js'; + +type Args = { + src: string; + out: string; + index: number; + archive: number; + exact: boolean; + debug: boolean; + help: boolean; +}; + +function parseArgs(argv: string[]): Args { + const args: Args = { + src: path.join(Environment.BUILD_SRC_DIR, 'scripts', '_unpack', '530', 'all.struct'), + out: 'data/pack', + index: 2, + archive: 26, + exact: false, + debug: false, + help: false + }; + + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + + if (arg === '--src') { + args.src = argv[++i]; + } else if (arg === '--out') { + args.out = argv[++i]; + } else if (arg === '--index') { + args.index = Number(argv[++i]); + } else if (arg === '--archive') { + args.archive = Number(argv[++i]); + } else if (arg === '--exact') { + args.exact = true; + } else if (arg === '--no-exact') { + args.exact = false; + } else if (arg === '--debug') { + args.debug = true; + } else if (arg === '--help' || arg === '-h') { + args.help = true; + } + } + + return args; +} + +type StructParamEntry = { + id: number; + value: number | string; +}; + +type OpcodeValue = + | { code: 249; value: StructParamEntry[] } + | { code: 250; value: string }; + +type ParsedStructSource = { + config: StructType; + opcodes: OpcodeValue[]; +}; + +function parseParamId(token: string, nameToId: Map): number { + const mapped = nameToId.get(token); + if (mapped !== undefined) { + return mapped; + } + + if (token.startsWith('param_')) { + const parsed = parseInt(token.substring(6)); + if (!isNaN(parsed)) { + return parsed; + } + } + + throw new Error(`Unknown param name: ${token}`); +} + +function parseSourceStructs( + content: string, + structNameToId: Map, + paramNameToId: Map, + paramIdToType: Map +): Map { + const structs = new Map(); + const sections = parseBracketedConfigSource(content); + + for (const section of sections) { + const id = resolveSectionId(section.name, structNameToId, 'struct_'); + if (id === null) { + throw new Error(`Unknown struct name: ${section.name}`); + } + + const config = new StructType(id); + config.params = new Map(); + config.debugname = null; + const opcodes: OpcodeValue[] = []; + let paramsOpcode: OpcodeValue | null = null; + + for (const field of section.fields) { + const { key, value } = field; + + if (key === 'param' || key === 'paramstr') { + const commaIndex = value.indexOf(','); + if (commaIndex === -1) { + continue; + } + + const paramName = value.substring(0, commaIndex).trim(); + const rawValue = value.substring(commaIndex + 1); + const paramId = parseParamId(paramName, paramNameToId); + const paramType = paramIdToType.get(paramId); + + if (paramType === undefined) { + throw new Error(`No ParamType definition found for param id ${paramId} (${paramName})`); + } + + const parsedValue: number | string = paramType === ScriptVarType.STRING + ? rawValue + : parseConfigInteger(rawValue.trim()); + + config.params.set(paramId, parsedValue); + + if (!paramsOpcode) { + paramsOpcode = { code: 249, value: [] }; + opcodes.push(paramsOpcode); + } + paramsOpcode.value.push({ id: paramId, value: parsedValue }); + } else if (key === 'debugname') { + config.debugname = value; + opcodes.push({ code: 250, value }); + } + } + + structs.set(id, { config, opcodes }); + } + + return structs; +} + +function parseParamTypesFromGroup(fileIds: number[], files: Map): Map { + const typeMap = new Map(); + + for (const fileId of fileIds) { + const fileData = files.get(fileId) ?? new Uint8Array(0); + const param = new ParamType(fileId); + + if (fileData.length > 0) { + const dat = new Packet(fileData); + + while (dat.available > 0) { + const code = dat.g1(); + if (code === 0) { + break; + } + param.decode(code, dat); + } + } + + typeMap.set(fileId, param.type); + } + + return typeMap; +} + +function encodeStructWithOpcodes(opcodes: OpcodeValue[]): Uint8Array { + const buf = Packet.alloc(2); + + for (const opc of opcodes) { + if (opc.code === 249) { + buf.p1(249); + buf.p1(opc.value.length); + + for (const entry of opc.value) { + const isString = typeof entry.value === 'string'; + buf.p1(isString ? 1 : 0); + buf.p3(entry.id); + + if (isString) { + buf.pjstr(entry.value as string); + } else { + buf.p4(entry.value as number); + } + } + } else if (opc.code === 250) { + buf.p1(250); + buf.pjstr(opc.value); + } + } + + buf.p1(0); + return new Uint8Array(buf.data.subarray(0, buf.pos)); +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + + if (!fs.existsSync(args.src)) { + throw new Error(`Source file not found: ${args.src}`); + } + + const { + fileIds: currentGroupFileIds, + groupUnpacked: currentGroupUnpacked + } = await loadArchiveGroupFiles(args.index, args.archive, 'data/cache', true); + + const { + fileIds: paramFileIds, + files: paramFiles + } = await loadArchiveGroupFiles(2, 11, 'data/cache', true); + + const paramIdToType = parseParamTypesFromGroup(paramFileIds, paramFiles); + + const localPackDir = path.join(path.dirname(args.src), 'pack'); + const fallbackPackDir = path.join(Environment.BUILD_SRC_DIR, 'pack'); + + const structLocalPackPath = path.join(localPackDir, 'struct.pack'); + const structFallbackPackPath = path.join(fallbackPackDir, 'struct.pack'); + const paramLocalPackPath = path.join(localPackDir, 'param.pack'); + const paramFallbackPackPath = path.join(fallbackPackDir, 'param.pack'); + + const structNameToId = fs.existsSync(structLocalPackPath) + ? parsePackFile(structLocalPackPath) + : parsePackFile(structFallbackPackPath); + + const paramNameToId = fs.existsSync(paramLocalPackPath) + ? parsePackFile(paramLocalPackPath) + : parsePackFile(paramFallbackPackPath); + + const sourceContent = fs.readFileSync(args.src, 'utf-8'); + const sourceStructs = parseSourceStructs(sourceContent, structNameToId, paramNameToId, paramIdToType); + + const fileData = new Map(); + for (const fileId of currentGroupFileIds) { + const sourceStruct = sourceStructs.get(fileId); + if (!sourceStruct) { + fileData.set(fileId, new Uint8Array(0)); + continue; + } + + const encoded = encodeStructWithOpcodes(sourceStruct.opcodes); + fileData.set(fileId, encoded); + } + + const combined = combineGroupFiles(fileData, currentGroupFileIds); + + if (args.exact) { + if (!arraysEqual(combined, currentGroupUnpacked)) { + if (args.debug) { + for (let i = 0; i < Math.min(combined.length, currentGroupUnpacked.length); i++) { + if (combined[i] !== currentGroupUnpacked[i]) { + console.error(`Mismatch at offset ${i}: generated=${combined[i]}, reference=${currentGroupUnpacked[i]}`); + break; + } + } + } + + throw new Error('Generated group does not match reference'); + } + } + + const compressed = await compressJs5Group(combined, CompressionType.GZIP); + const filename = `${args.archive}.dat`; + const filepath = path.join(args.out, filename); + + ensureDir(args.out); + if (fs.existsSync(filepath) && fs.statSync(filepath).isDirectory()) { + fs.rmSync(filepath, { recursive: true, force: true }); + } + + fs.writeFileSync(filepath, compressed); + + console.log(`Packed ${sourceStructs.size} struct configs`); + console.log(`Wrote ${filepath}`); +} + +main().catch(err => { + console.error(err); + process.exit(1); +}); diff --git a/tools/cache/verify/verifyJs5Archive.ts b/tools/cache/verify/verifyJs5Archive.ts new file mode 100644 index 000000000..821439b7d --- /dev/null +++ b/tools/cache/verify/verifyJs5Archive.ts @@ -0,0 +1,146 @@ +import fs from 'fs'; +import path from 'path'; + +import { + arraysEqual, + packedContainerLength, + parseGroupIdsFromIndexPacked, + readInt32BE +} from '#tools/cache/lib/js5Tools.js'; + +type Args = { + archive: number; + js5: string; + groupsDir: string; + indexPath: string; + help: boolean; +}; + +function parseArgs(argv: string[]): Args { + const args: Args = { + archive: 17, + js5: '', + groupsDir: 'data/pack', + indexPath: '', + help: false + }; + + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg === '--archive') { + args.archive = Number(argv[++i]); + } else if (arg === '--js5') { + args.js5 = argv[++i]; + } else if (arg === '--groups-dir') { + args.groupsDir = argv[++i]; + } else if (arg === '--index') { + args.indexPath = argv[++i]; + } else if (arg === '--help' || arg === '-h') { + args.help = true; + } + } + + if (!args.js5) { + args.js5 = `data/pack/archive.${args.archive}.js5`; + } + + if (!args.indexPath) { + args.indexPath = `data/cache/255/${args.archive}.dat`; + } + + return args; +} + + +function main() { + const args = parseArgs(process.argv.slice(2)); + + if (!fs.existsSync(args.js5)) { + throw new Error(`JS5 file not found: ${args.js5}`); + } + + if (!fs.existsSync(args.indexPath)) { + throw new Error(`Index file not found: ${args.indexPath}`); + } + + const js5 = new Uint8Array(fs.readFileSync(args.js5)); + const indexPacked = new Uint8Array(fs.readFileSync(args.indexPath)); + const groupIds = parseGroupIdsFromIndexPacked(indexPacked); + + const indexLengthInJs5 = packedContainerLength(js5, 0); + const indexFromJs5 = js5.slice(0, indexLengthInJs5); + const indexMatches = arraysEqual(indexFromJs5, indexPacked); + + const lengthsTableBytes = groupIds.length * 4; + const lengthsTableStart = js5.length - lengthsTableBytes; + const lengthsTable = js5.slice(lengthsTableStart); + + let pos = indexLengthInJs5; + let groupMismatches = 0; + let lengthMismatches = 0; + + for (let i = 0; i < groupIds.length; i++) { + const groupId = groupIds[i]; + const expectedLength = readInt32BE(lengthsTable, i * 4) >>> 0; + const expectedPath = path.join(args.groupsDir, String(args.archive), `${groupId}.dat`); + + if (!fs.existsSync(expectedPath)) { + if (expectedLength !== 0) { + lengthMismatches++; + } + continue; + } + + const expectedBytes = new Uint8Array(fs.readFileSync(expectedPath)); + if (expectedBytes.length !== expectedLength) { + lengthMismatches++; + } + + const actualBytes = js5.slice(pos, pos + expectedLength); + pos += expectedLength; + + if (!arraysEqual(actualBytes, expectedBytes)) { + groupMismatches++; + } + } + + const consumedExactly = pos === lengthsTableStart; + const reconstructed = new Uint8Array(js5.length); + reconstructed.set(indexPacked, 0); + let writePos = indexPacked.length; + + const rebuiltLengths = new Uint8Array(lengthsTableBytes); + for (let i = 0; i < groupIds.length; i++) { + const groupId = groupIds[i]; + const expectedPath = path.join(args.groupsDir, String(args.archive), `${groupId}.dat`); + const expectedBytes = fs.existsSync(expectedPath) ? new Uint8Array(fs.readFileSync(expectedPath)) : new Uint8Array(0); + reconstructed.set(expectedBytes, writePos); + writePos += expectedBytes.length; + + rebuiltLengths[i * 4] = (expectedBytes.length >>> 24) & 0xff; + rebuiltLengths[i * 4 + 1] = (expectedBytes.length >>> 16) & 0xff; + rebuiltLengths[i * 4 + 2] = (expectedBytes.length >>> 8) & 0xff; + rebuiltLengths[i * 4 + 3] = expectedBytes.length & 0xff; + } + + reconstructed.set(rebuiltLengths, writePos); + const fullMatch = arraysEqual(js5, reconstructed); + + console.log(`Archive ${args.archive}: ${groupIds.length} groups in index`); + console.log(`Index bytes match: ${indexMatches ? 'YES' : 'NO'}`); + console.log(`Group length table mismatches: ${lengthMismatches}`); + console.log(`Group payload mismatches: ${groupMismatches}`); + console.log(`Layout consumed exactly: ${consumedExactly ? 'YES' : 'NO'}`); + console.log(`Full archive reconstruction match: ${fullMatch ? 'YES' : 'NO'}`); + + if (!indexMatches || groupMismatches > 0 || lengthMismatches > 0 || !consumedExactly || !fullMatch) { + process.exitCode = 1; + } +} + +try { + main(); +} catch (err) { + console.error(err); + process.exit(1); +} diff --git a/tools/cache/verify/verifyJs5Compressed.ts b/tools/cache/verify/verifyJs5Compressed.ts new file mode 100644 index 000000000..6e6f76aac --- /dev/null +++ b/tools/cache/verify/verifyJs5Compressed.ts @@ -0,0 +1,112 @@ +import fs from 'fs'; +import path from 'path'; + +import { + arraysEqual, + loadIndexPacked, + parseGroupIdsFromIndexPacked +} from '#tools/cache/lib/js5Tools.js'; + +import { getGroup } from '#/util/OpenRS2.js'; + +type Args = { + archive: number; + groupsDir: string; + indexPath: string; + openrs2: boolean; + warnOnly: boolean; + help: boolean; +}; + +function parseArgs(argv: string[]): Args { + const args: Args = { + archive: 17, + groupsDir: 'data/pack', + indexPath: '', + openrs2: true, + warnOnly: false, + help: false + }; + + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg === '--archive') { + args.archive = Number(argv[++i]); + } else if (arg === '--groups-dir') { + args.groupsDir = argv[++i]; + } else if (arg === '--index') { + args.indexPath = argv[++i]; + } else if (arg === '--no-openrs2') { + args.openrs2 = false; + } else if (arg === '--warn-only') { + args.warnOnly = true; + } else if (arg === '--help' || arg === '-h') { + args.help = true; + } + } + + if (!args.indexPath) { + args.indexPath = `data/cache/255/${args.archive}.dat`; + } + + return args; +} + + +async function main() { + const args = parseArgs(process.argv.slice(2)); + + const indexPacked = await loadIndexPacked(args.archive, args.indexPath, args.openrs2); + const groupIds = parseGroupIdsFromIndexPacked(indexPacked); + + let mismatches = 0; + let checked = 0; + + for (const groupId of groupIds) { + const localPath = path.join(args.groupsDir, String(args.archive), `${groupId}.dat`); + if (!fs.existsSync(localPath)) { + mismatches++; + console.log(`Group ${groupId}: MISSING (no local group data)`); + continue; + } + + const localBytes = new Uint8Array(fs.readFileSync(localPath)); + let expectedBytes: Uint8Array; + + if (args.openrs2) { + expectedBytes = new Uint8Array(await getGroup(args.archive, groupId)); + } else { + const expectedPath = path.join('data/cache', String(args.archive), `${groupId}.dat`); + if (!fs.existsSync(expectedPath)) { + mismatches++; + console.log(`Group ${groupId}: MISSING (no cache group data)`); + continue; + } + expectedBytes = new Uint8Array(fs.readFileSync(expectedPath)); + } + + checked++; + if (!arraysEqual(localBytes, expectedBytes)) { + mismatches++; + console.log(`Group ${groupId}: COMPRESSED BYTES MISMATCH`); + } + } + + console.log(`Compressed groups checked: ${checked}`); + console.log(`Compressed mismatches: ${mismatches}`); + + if (mismatches > 0 && args.warnOnly) { + console.log('Compressed mismatches detected (warn-only mode).'); + } + + if (mismatches > 0 && !args.warnOnly) { + process.exitCode = 1; + } +} + +try { + await main(); +} catch (err) { + console.error(err); + process.exit(1); +} diff --git a/tools/cache/verify/verifyJs5Crc.ts b/tools/cache/verify/verifyJs5Crc.ts new file mode 100644 index 000000000..95a071e8e --- /dev/null +++ b/tools/cache/verify/verifyJs5Crc.ts @@ -0,0 +1,152 @@ +import fs from 'fs'; +import path from 'path'; +import Packet from '#/io/Packet.js'; +import { getGroup } from '#/util/OpenRS2.js'; +import { unpackJs5Group } from '#/io/Js5Group.js'; +import { + loadIndexPacked, + parseGroupIdsFromIndexPacked, + readJs5Id +} from '#tools/cache/lib/js5Tools.js'; + +type Args = { + archive: number; + groupsDir: string; + indexPath: string; + openrs2: boolean; + help: boolean; +}; + +function parseArgs(argv: string[]): Args { + const args: Args = { + archive: 17, + groupsDir: 'data/pack', + indexPath: '', + openrs2: false, + help: false + }; + + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg === '--archive') { + args.archive = Number(argv[++i]); + } else if (arg === '--groups-dir') { + args.groupsDir = argv[++i]; + } else if (arg === '--index') { + args.indexPath = argv[++i]; + } else if (arg === '--openrs2') { + args.openrs2 = true; + } else if (arg === '--help' || arg === '-h') { + args.help = true; + } + } + + if (!args.indexPath) { + args.indexPath = `data/cache/255/${args.archive}.dat`; + } + + return args; +} + + +function parseGroupCrcs(indexUnpacked: Uint8Array): Map { + const packet = new Packet(indexUnpacked); + + const format = packet.g1(); + if (format < 5 || format > 7) { + throw new Error(`Unsupported JS5 index format: ${format}`); + } + + if (format >= 6) { + packet.g4s(); + } + + const flags = packet.g1(); + const hasNames = (flags & 0x1) !== 0; + + const groupCount = readJs5Id(packet, format); + const groupIds: number[] = new Array(groupCount); + + let previousGroupId = 0; + for (let i = 0; i < groupCount; i++) { + previousGroupId += readJs5Id(packet, format); + groupIds[i] = previousGroupId; + } + + if (hasNames) { + for (let i = 0; i < groupCount; i++) { + packet.g4s(); + } + } + + const crcs = new Map(); + for (let i = 0; i < groupCount; i++) { + const crc = packet.g4s() >>> 0; + crcs.set(groupIds[i], crc); + } + + return crcs; +} + + +function toHex(value: number): string { + return `0x${(value >>> 0).toString(16).padStart(8, '0')}`; +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + + const indexPacked = await loadIndexPacked(args.archive, args.indexPath, args.openrs2); + const indexUnpacked = unpackJs5Group(indexPacked); + const groupCrcs = parseGroupCrcs(indexUnpacked); + const groupIds = parseGroupIdsFromIndexPacked(indexPacked); + + let mismatches = 0; + let checked = 0; + + for (const groupId of groupIds) { + const groupPath = path.join(args.groupsDir, String(args.archive), `${groupId}.dat`); + if (!fs.existsSync(groupPath)) { + mismatches++; + console.log(`Group ${groupId}: MISSING (no local group data)`); + continue; + } + + const groupPacked = new Uint8Array(fs.readFileSync(groupPath)); + const groupUnpacked = unpackJs5Group(groupPacked); + const actualCrc = Packet.getcrc(groupUnpacked, 0, groupUnpacked.length) >>> 0; + checked++; + + if (args.openrs2) { + const remotePacked = await getGroup(args.archive, groupId); + const remoteUnpacked = unpackJs5Group(new Uint8Array(remotePacked)); + const expectedCrc = Packet.getcrc(remoteUnpacked, 0, remoteUnpacked.length) >>> 0; + + if (actualCrc !== expectedCrc) { + mismatches++; + console.log(`Group ${groupId}: CRC MISMATCH (expected ${toHex(expectedCrc)}, got ${toHex(actualCrc)})`); + } + continue; + } + + const expectedCrc = groupCrcs.get(groupId) ?? 0; + if (actualCrc !== expectedCrc) { + mismatches++; + console.log(`Group ${groupId}: CRC MISMATCH (expected ${toHex(expectedCrc)}, got ${toHex(actualCrc)})`); + } + } + + console.log(`CRC checked groups: ${checked}`); + console.log(`CRC mismatches: ${mismatches}`); + + if (mismatches > 0) { + process.exitCode = 1; + } +} + +try { + await main(); +} catch (err) { + console.error(err); + process.exit(1); +} diff --git a/tools/pack/core/PackFile.ts b/tools/pack/core/PackFile.ts new file mode 100644 index 000000000..ff01fe1cf --- /dev/null +++ b/tools/pack/core/PackFile.ts @@ -0,0 +1,96 @@ +import fs from 'fs'; +import path from 'path'; + +export class PackFile { + type: string; + pack: Map = new Map(); + names: Set = new Set(); + nameToId: Map = new Map(); + max: number = 0; + + get size() { + return this.pack.size; + } + + constructor(type: string) { + this.type = type; + } + + load(path: string): void { + this.pack = new Map(); + this.names = new Set(); + this.nameToId = new Map(); + this.max = 0; + + if (!fs.existsSync(path)) { + return; + } + + const content = fs.readFileSync(path, 'utf-8'); + const lines = content.split('\n'); + + for (let i = 0; i < lines.length; i++) { + const line = lines[i].trim(); + + // Skip empty lines and comments + if (line.length === 0 || line.startsWith('#') || !/^\d+=/g.test(line)) { + continue; + } + + const parts = line.split('='); + if (parts[1].length === 0) { + throw new Error(`Pack file has an empty name ${path}:${i + 1}`); + } + + this.register(parseInt(parts[0]), parts[1]); + } + + this.refreshNames(); + } + + save(filePath: string): void { + const dir = path.dirname(filePath); + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }); + } + + const entries = Array.from(this.pack.entries()) + .sort((a, b) => a[0] - b[0]) + .map(([id, name]) => `${id}=${name}`) + .join('\n') + '\n'; + + fs.writeFileSync(filePath, entries); + } + + register(id: number, name: string): void { + this.pack.set(id, name); + this.names.add(name); + this.nameToId.set(name, id); + + if (id >= this.max) { + this.max = id + 1; + } + } + + refreshNames(): void { + this.names = new Set(); + this.nameToId = new Map(); + + for (const [id, name] of this.pack) { + this.names.add(name); + this.nameToId.set(name, id); + } + } + + getById(id: number): string { + return this.pack.get(id) ?? ''; + } + + getByName(name: string): number { + const id = this.nameToId.get(name); + if (typeof id === 'undefined') { + return -1; + } + return id; + } +} diff --git a/tools/pack/core/PackManager.ts b/tools/pack/core/PackManager.ts new file mode 100644 index 000000000..84d83ed18 --- /dev/null +++ b/tools/pack/core/PackManager.ts @@ -0,0 +1,65 @@ +import Environment from '#/util/Environment.js'; +import { PackFile } from './PackFile.js'; + +export class PackManager { + private pack: PackFile; + private packPath: string; + + constructor(type: string) { + this.pack = new PackFile(type); + this.packPath = `${Environment.BUILD_SRC_DIR}/pack/${type}.pack`; + this.pack.load(this.packPath); + } + + public getName(id: number): string { + let name = this.pack.getById(id); + if (!name) { + name = `${this.pack.type}_${id}`; + this.pack.register(id, name); + this.save(); + } + return name; + } + + public getNameIfExists(id: number): string | undefined { + const name = this.pack.getById(id); + return name || undefined; + } + + public setName(id: number, name: string): void { + this.pack.register(id, name); + this.save(); + } + + public getId(name: string): number { + return this.pack.getByName(name); + } + + public hasName(name: string): boolean { + return this.pack.names.has(name); + } + + public hasId(id: number): boolean { + return this.pack.pack.has(id); + } + + public getAll(): Array<[number, string]> { + return Array.from(this.pack.pack.entries()); + } + + public getSize(): number { + return this.pack.size; + } + + public save(): void { + this.pack.save(this.packPath); + } + + public forceSave(): void { + this.save(); + } +} + +export function createPackManager(type: string): PackManager { + return new PackManager(type); +} diff --git a/tools/pack/generators/generateChatCatPack.ts b/tools/pack/generators/generateChatCatPack.ts new file mode 100644 index 000000000..5f5eeb656 --- /dev/null +++ b/tools/pack/generators/generateChatCatPack.ts @@ -0,0 +1,25 @@ +import fs from 'fs'; +import path from 'path'; + +import Environment from '#/util/Environment.js'; + +async function main() { + const packPath = path.join(Environment.BUILD_SRC_DIR, 'pack', 'chatcat.pack'); + + const dir = path.dirname(packPath); + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }); + } + + // Generate all 164 entries in 530 with format: id=chatcat_ + const lines: string[] = []; + for (let id = 0; id < 164; id++) { + lines.push(`${id}=chatcat_${id}`); + } + + fs.writeFileSync(packPath, lines.join('\n') + '\n'); + console.log(`Wrote 164 entries to ${packPath}`); +} + +main().catch(console.error); + diff --git a/tools/pack/generators/generateChatPhrasePack.ts b/tools/pack/generators/generateChatPhrasePack.ts new file mode 100644 index 000000000..af007aec3 --- /dev/null +++ b/tools/pack/generators/generateChatPhrasePack.ts @@ -0,0 +1,25 @@ +import fs from 'fs'; +import path from 'path'; + +import Environment from '#/util/Environment.js'; + +async function main() { + const packPath = path.join(Environment.BUILD_SRC_DIR, 'pack', 'chatphrase.pack'); + + const dir = path.dirname(packPath); + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }); + } + + // Generate all 867 entries in 530 with format: id=chatphrase_ + const lines: string[] = []; + for (let id = 0; id < 867; id++) { + lines.push(`${id}=chatphrase_${id}`); + } + + fs.writeFileSync(packPath, lines.join('\n') + '\n'); + console.log(`Wrote 867 entries to ${packPath}`); +} + +main().catch(console.error); + diff --git a/tools/pack/generators/generateEnumPack.ts b/tools/pack/generators/generateEnumPack.ts new file mode 100644 index 000000000..a18eabcf6 --- /dev/null +++ b/tools/pack/generators/generateEnumPack.ts @@ -0,0 +1,79 @@ +import fs from 'fs'; +import path from 'path'; + +import Environment from '#/util/Environment.js'; +import { PackFile } from '#tools/pack/core/PackFile.js'; + +function extractEnumNames(content: string): string[] { + const names: string[] = []; + const lines = content.split('\n'); + + for (const line of lines) { + const trimmed = line.trim(); + + if (trimmed.startsWith('[') && trimmed.endsWith(']')) { + const name = trimmed.substring(1, trimmed.length - 1); + if (!names.includes(name)) { + names.push(name); + } + } + } + + return names; +} + +function extractEnumId(name: string): number | null { + if (!name.startsWith('enum_')) { + return null; + } + + const idPart = name.substring(5); + const id = parseInt(idPart); + + if (isNaN(id)) { + return null; + } + + return id; +} + +async function main() { + const srcPath = path.join(Environment.BUILD_SRC_DIR, 'scripts', '_unpack', '530', 'all.enum'); + const packPath = path.join(Environment.BUILD_SRC_DIR, 'pack', 'enum.pack'); + + if (!fs.existsSync(srcPath)) { + throw new Error(`Source file not found: ${srcPath}`); + } + + const content = fs.readFileSync(srcPath, 'utf-8'); + const names = extractEnumNames(content); + + console.log(`Found ${names.length} enum definitions`); + + const pack = new PackFile('enum'); + + for (const name of names) { + const id = extractEnumId(name); + if (id !== null) { + pack.register(id, name); + } else { + console.warn(`Warning: Could not extract ID from name: ${name}`); + } + } + + const dir = path.dirname(packPath); + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }); + } + + const entries = Array.from(pack.pack.entries()) + .sort((a, b) => a[0] - b[0]) + .map(([id, name]) => `${id}=${name}`) + .join('\n') + '\n'; + + fs.writeFileSync(packPath, entries); + + console.log(`Wrote ${pack.size} entries to ${packPath}`); +} + +main().catch(console.error); diff --git a/tools/pack/generators/generateFloPack.ts b/tools/pack/generators/generateFloPack.ts new file mode 100644 index 000000000..a44ec93a6 --- /dev/null +++ b/tools/pack/generators/generateFloPack.ts @@ -0,0 +1,78 @@ +import fs from 'fs'; +import { PackFile } from '#tools/pack/core/PackFile.js'; + +function extractFloNames(content: string): string[] { + const names: string[] = []; + const lines = content.split('\n'); + + for (const line of lines) { + const trimmed = line.trim(); + + if (trimmed.startsWith('[flo_') && trimmed.endsWith(']')) { + const name = trimmed.substring(1, trimmed.length - 1); + if (!names.includes(name)) { + names.push(name); + } + } + } + + return names; +} + +function extractFloId(name: string): number | null { + if (!name.startsWith('flo_')) { + return null; + } + + const idPart = name.substring(4); + const id = parseInt(idPart); + + if (isNaN(id)) { + return null; + } + + return id; +} + +async function main() { + const srcPath = 'data/src/all.flo'; + const packPath = 'data/src/pack/flo.pack'; + + if (!fs.existsSync(srcPath)) { + throw new Error(`Source file not found: ${srcPath}`); + } + + const content = fs.readFileSync(srcPath, 'utf-8'); + const names = extractFloNames(content); + + console.log(`Found ${names.length} floor overlay definitions`); + + const pack = new PackFile('flo'); + + // Register all names with their IDs + for (const name of names) { + const id = extractFloId(name); + if (id !== null) { + pack.register(id, name); + } else { + console.warn(`Warning: Could not extract ID from name: ${name}`); + } + } + + // Write with header comment + const dir = packPath.substring(0, packPath.lastIndexOf('/')); + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }); + } + + const entries = Array.from(pack.pack.entries()) + .sort((a, b) => a[0] - b[0]) + .map(([id, name]) => `${id}=${name}`) + .join('\n') + '\n'; + + fs.writeFileSync(packPath, entries); + + console.log(`Wrote ${pack.size} entries to ${packPath}`); +} + +main().catch(console.error); diff --git a/tools/pack/generators/generateFluPack.ts b/tools/pack/generators/generateFluPack.ts new file mode 100644 index 000000000..bbdfb9dcb --- /dev/null +++ b/tools/pack/generators/generateFluPack.ts @@ -0,0 +1,79 @@ +import fs from 'fs'; +import path from 'path'; + +import Environment from '#/util/Environment.js'; +import { PackFile } from '#tools/pack/core/PackFile.js'; + +function extractFluNames(content: string): string[] { + const names: string[] = []; + const lines = content.split('\n'); + + for (const line of lines) { + const trimmed = line.trim(); + + if (trimmed.startsWith('[flu_') && trimmed.endsWith(']')) { + const name = trimmed.substring(1, trimmed.length - 1); + if (!names.includes(name)) { + names.push(name); + } + } + } + + return names; +} + +function extractFluId(name: string): number | null { + if (!name.startsWith('flu_')) { + return null; + } + + const idPart = name.substring(4); + const id = parseInt(idPart); + + if (isNaN(id)) { + return null; + } + + return id; +} + +async function main() { + const srcPath = path.join(Environment.BUILD_SRC_DIR, 'scripts', '_unpack', '530', 'all.flu'); + const packPath = path.join(Environment.BUILD_SRC_DIR, 'pack', 'flu.pack'); + + if (!fs.existsSync(srcPath)) { + throw new Error(`Source file not found: ${srcPath}`); + } + + const content = fs.readFileSync(srcPath, 'utf-8'); + const names = extractFluNames(content); + + console.log(`Found ${names.length} floor underlay definitions`); + + const pack = new PackFile('flu'); + + for (const name of names) { + const id = extractFluId(name); + if (id !== null) { + pack.register(id, name); + } else { + console.warn(`Warning: Could not extract ID from name: ${name}`); + } + } + + const dir = path.dirname(packPath); + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }); + } + + const entries = Array.from(pack.pack.entries()) + .sort((a, b) => a[0] - b[0]) + .map(([id, name]) => `${id}=${name}`) + .join('\n') + '\n'; + + fs.writeFileSync(packPath, entries); + + console.log(`Wrote ${pack.size} entries to ${packPath}`); +} + +main().catch(console.error); diff --git a/tools/pack/generators/generateInvPack.ts b/tools/pack/generators/generateInvPack.ts new file mode 100644 index 000000000..d115576d6 --- /dev/null +++ b/tools/pack/generators/generateInvPack.ts @@ -0,0 +1,71 @@ +import fs from 'fs'; +import { PackFile } from '#tools/pack/core/PackFile.js'; +import Environment from '#/util/Environment.js'; +import path from 'path'; + +function extractInvNames(content: string): string[] { + const names: string[] = []; + const lines = content.split('\n'); + + for (const line of lines) { + const trimmed = line.trim(); + + if (trimmed.startsWith('[inv_') && trimmed.endsWith(']')) { + const name = trimmed.substring(1, trimmed.length - 1); + if (!names.includes(name)) { + names.push(name); + } + } + } + + return names; +} + +function extractInvId(name: string): number | null { + if (!name.startsWith('inv_')) { + return null; + } + + const idPart = name.substring(4); + const id = parseInt(idPart); + + if (isNaN(id)) { + return null; + } + + return id; +} + +async function main() { + const srcPath = path.join(Environment.BUILD_SRC_DIR, 'scripts', '_unpack', '530', 'all.inv'); + const packPath = path.join(Environment.BUILD_SRC_DIR, 'pack', 'inv.pack'); + + if (!fs.existsSync(srcPath)) { + throw new Error(`Source file not found: ${srcPath}`); + } + + const content = fs.readFileSync(srcPath, 'utf-8'); + const names = extractInvNames(content); + + console.log(`Found ${names.length} inventory definitions`); + + const pack = new PackFile('inv'); + + // Register all names with their IDs + for (const name of names) { + const id = extractInvId(name); + if (id !== null) { + pack.register(id, name); + } else { + console.warn(`Warning: Could not extract ID from name: ${name}`); + } + } + + pack.save(packPath); + console.log(`Saved inv.pack to ${packPath}`); +} + +main().catch(err => { + console.error(err); + process.exit(1); +}); diff --git a/tools/pack/generators/generateMesanimPack.ts b/tools/pack/generators/generateMesanimPack.ts new file mode 100644 index 000000000..439a82dfe --- /dev/null +++ b/tools/pack/generators/generateMesanimPack.ts @@ -0,0 +1,79 @@ +import fs from 'fs'; +import path from 'path'; + +import Environment from '#/util/Environment.js'; +import { PackFile } from '#tools/pack/core/PackFile.js'; + +function extractMesanimNames(content: string): string[] { + const names: string[] = []; + const lines = content.split('\n'); + + for (const line of lines) { + const trimmed = line.trim(); + + if (trimmed.startsWith('[mesanim_') && trimmed.endsWith(']')) { + const name = trimmed.substring(1, trimmed.length - 1); + if (!names.includes(name)) { + names.push(name); + } + } + } + + return names; +} + +function extractMesanimId(name: string): number | null { + if (!name.startsWith('mesanim_')) { + return null; + } + + const idPart = name.substring(8); + const id = parseInt(idPart); + + if (isNaN(id)) { + return null; + } + + return id; +} + +async function main() { + const srcPath = path.join(Environment.BUILD_SRC_DIR, 'scripts', '_unpack', '530', 'all.mesanim'); + const packPath = path.join(Environment.BUILD_SRC_DIR, 'pack', 'mesanim.pack'); + + if (!fs.existsSync(srcPath)) { + throw new Error(`Source file not found: ${srcPath}`); + } + + const content = fs.readFileSync(srcPath, 'utf-8'); + const names = extractMesanimNames(content); + + console.log(`Found ${names.length} mesanim definitions`); + + const pack = new PackFile('mesanim'); + + for (const name of names) { + const id = extractMesanimId(name); + if (id !== null) { + pack.register(id, name); + } else { + console.warn(`Warning: Could not extract ID from name: ${name}`); + } + } + + const dir = path.dirname(packPath); + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }); + } + + const entries = Array.from(pack.pack.entries()) + .sort((a, b) => a[0] - b[0]) + .map(([id, name]) => `${id}=${name}`) + .join('\n') + '\n'; + + fs.writeFileSync(packPath, entries); + + console.log(`Wrote ${pack.size} entries to ${packPath}`); +} + +main().catch(console.error); diff --git a/tools/pack/generators/generateMidiPack.ts b/tools/pack/generators/generateMidiPack.ts new file mode 100644 index 000000000..b88c7f874 --- /dev/null +++ b/tools/pack/generators/generateMidiPack.ts @@ -0,0 +1,23 @@ +import fs from 'fs'; +import path from 'path'; + +import { PackFile } from '#tools/pack/core/PackFile.js'; +import { listMidiPackEntries } from '#tools/pack/sources/MidiSource.js'; +import Environment from '#/util/Environment.js'; + +const midiPack = new PackFile('midi'); + +for (const midi of listMidiPackEntries()) { + midiPack.register(midi.id, midi.name); +} + +// Create directories if needed +const packDir = `${Environment.BUILD_SRC_DIR}/pack`; +if (!fs.existsSync(packDir)) { + fs.mkdirSync(packDir, { recursive: true }); +} + +// Save pack file +const packPath = path.join(packDir, 'midi.pack'); +midiPack.save(packPath); +console.log(`\nSaved to ${packPath}`); diff --git a/tools/pack/generators/generateNpcPack.ts b/tools/pack/generators/generateNpcPack.ts new file mode 100644 index 000000000..ae9f17a29 --- /dev/null +++ b/tools/pack/generators/generateNpcPack.ts @@ -0,0 +1,73 @@ +import fs from 'fs'; +import path from 'path'; + +import Environment from '#/util/Environment.js'; +import { PackFile } from '#tools/pack/core/PackFile.js'; + +function extractNpcNames(content: string): string[] { + const names: string[] = []; + const lines = content.split('\n'); + + for (const line of lines) { + const trimmed = line.trim(); + if (trimmed.startsWith('[npc_') && trimmed.endsWith(']')) { + const name = trimmed.substring(1, trimmed.length - 1); + if (!names.includes(name)) { + names.push(name); + } + } + } + + return names; +} + +function extractNpcId(name: string): number | null { + if (!name.startsWith('npc_')) { + return null; + } + + const idPart = name.substring(4); + const id = parseInt(idPart); + if (isNaN(id)) { + return null; + } + + return id; +} + +async function main() { + const srcPath = path.join(Environment.BUILD_SRC_DIR, 'scripts', '_unpack', '530', 'all.npc'); + const packPath = path.join(Environment.BUILD_SRC_DIR, 'pack', 'npc.pack'); + + if (!fs.existsSync(srcPath)) { + throw new Error(`Source file not found: ${srcPath}`); + } + + const content = fs.readFileSync(srcPath, 'utf-8'); + const names = extractNpcNames(content); + + console.log(`Found ${names.length} npc definitions`); + + const pack = new PackFile('npc'); + for (const name of names) { + const id = extractNpcId(name); + if (id !== null) { + pack.register(id, name); + } + } + + const dir = path.dirname(packPath); + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }); + } + + const entries = Array.from(pack.pack.entries()) + .sort((a, b) => a[0] - b[0]) + .map(([id, name]) => `${id}=${name}`) + .join('\n') + '\n'; + + fs.writeFileSync(packPath, entries); + console.log(`Wrote ${pack.size} entries to ${packPath}`); +} + +main().catch(console.error); diff --git a/tools/pack/generators/generateObjPack.ts b/tools/pack/generators/generateObjPack.ts new file mode 100644 index 000000000..4ab2c1c34 --- /dev/null +++ b/tools/pack/generators/generateObjPack.ts @@ -0,0 +1,357 @@ +import fs from 'fs'; +import path from 'path'; + +import { splitGroupFiles } from '#/io/Js5ArchiveIndex.js'; +import { unpackJs5Group } from '#/io/Js5Group.js'; +import Packet from '#/io/Packet.js'; +import Environment from '#/util/Environment.js'; +import { decodeObjOpcode } from '#tools/cache/lib/objCodec.js'; +import { loadReferenceArchiveIndex, readGroupBytes } from '#tools/cache/lib/js5Tools.js'; +import { PackFile } from '#tools/pack/core/PackFile.js'; + +const OBJ_ARCHIVE = 19; +const CERT_TEMPLATE_ID = 799; +const LENT_TEMPLATE_ID = 13009; + +type ObjSection = { + name: string; + id: number; + fields: Array<{ key: string; value: string }>; +}; + +function parseObjSections(content: string, existingNameToId: Map): ObjSection[] { + const sections: ObjSection[] = []; + const lines = content.split('\n'); + let current: ObjSection | null = null; + + for (const line of lines) { + const trimmed = line.trim(); + + if (trimmed.startsWith('[') && trimmed.endsWith(']')) { + const name = trimmed.substring(1, trimmed.length - 1); + const existingId = existingNameToId.get(name); + const id = existingId !== undefined ? existingId : extractObjId(name); + if (id !== null) { + current = { + name, + id, + fields: [] + }; + sections.push(current); + } + continue; + } + + if (!current || trimmed.length === 0 || trimmed.startsWith('//')) { + continue; + } + + const eq = line.indexOf('='); + if (eq === -1) { + continue; + } + + const key = line.substring(0, eq).trim(); + const value = line.substring(eq + 1).trim(); + if (key.length === 0) { + continue; + } + + current.fields.push({ key, value }); + } + + return sections; +} + +function extractObjId(name: string): number | null { + if (!name.startsWith('obj_')) { + return null; + } + + const idPart = name.substring(4); + const id = parseInt(idPart); + if (isNaN(id)) { + return null; + } + + return id; +} + +function parseObjReference(value: string): number | null { + const numeric = parseInt(value, 10); + if (!isNaN(numeric)) { + return numeric; + } + + if (value.startsWith('obj_')) { + return extractObjId(value); + } + + if (value.startsWith('cert_obj_')) { + return extractObjId(value.substring('cert_'.length)); + } + + if (value.startsWith('lent_obj_')) { + return extractObjId(value.substring('lent_'.length)); + } + + return null; +} + +function parseLinkedTargetFromSectionName(name: string, kind: 'cert' | 'lent'): number | null { + const prefix = `${kind}_obj_`; + if (!name.startsWith(prefix)) { + return null; + } + + const rest = name.substring(prefix.length); + const first = rest.split('_')[0]; + const id = parseInt(first, 10); + return isNaN(id) ? null : id; +} + +function collectCertObjectNames(sections: ObjSection[]): Map { + const certNames = new Map(); + + for (const section of sections) { + let hasCertTemplate = false; + let certLinkTarget: number | null = null; + + for (const field of section.fields) { + if (field.key === 'certtemplate') { + const templateId = parseObjReference(field.value); + if (templateId !== null && templateId >= 0) { + hasCertTemplate = true; + } + } + + if (field.key === 'certlink') { + const linkId = parseObjReference(field.value); + if (linkId !== null && linkId >= 0) { + certLinkTarget = linkId; + } + } + } + + if (certLinkTarget === null) { + certLinkTarget = parseLinkedTargetFromSectionName(section.name, 'cert'); + } + + if (hasCertTemplate && certLinkTarget !== null) { + certNames.set(section.id, `cert_obj_${certLinkTarget}`); + } else if (hasCertTemplate) { + certNames.set(section.id, section.name.startsWith('cert_') ? section.name : `cert_${section.name}`); + } + } + + return certNames; +} + +function collectLentObjectNames(sections: ObjSection[]): Map { + const lentNames = new Map(); + + for (const section of sections) { + let hasLentTemplate = false; + let lentLinkTarget: number | null = null; + + for (const field of section.fields) { + if (field.key === 'lenttemplate') { + const templateId = parseObjReference(field.value); + if (templateId !== null && templateId >= 0) { + hasLentTemplate = true; + } + } + + if (field.key === 'lentlink') { + const linkId = parseObjReference(field.value); + if (linkId !== null && linkId >= 0) { + lentLinkTarget = linkId; + } + } + } + + if (lentLinkTarget === null) { + lentLinkTarget = parseLinkedTargetFromSectionName(section.name, 'lent'); + } + + if (hasLentTemplate && lentLinkTarget !== null) { + lentNames.set(section.id, `lent_obj_${lentLinkTarget}`); + } else if (hasLentTemplate) { + lentNames.set(section.id, section.name.startsWith('lent_') ? section.name : `lent_${section.name}`); + } + } + + return lentNames; +} + +function allocateUniqueName(baseName: string, id: number, usedNames: Set): string { + if (!usedNames.has(baseName)) { + return baseName; + } + + const withId = `${baseName}_${id}`; + if (!usedNames.has(withId)) { + return withId; + } + + let suffix = 2; + while (usedNames.has(`${withId}_${suffix}`)) { + suffix += 1; + } + + return `${withId}_${suffix}`; +} + +async function collectLinkedObjectNamesFromCache(): Promise<{ cert: Map; lent: Map }> { + const certNames = new Map(); + const lentNames = new Map(); + + const index = loadReferenceArchiveIndex(OBJ_ARCHIVE); + if (!index) { + return { cert: certNames, lent: lentNames }; + } + + for (const groupId of index.groupIds) { + const fileIds = index.fileIdsByGroup.get(groupId); + if (!fileIds || fileIds.length === 0) { + continue; + } + + const groupPacked = await readGroupBytes(OBJ_ARCHIVE, groupId, 'data/cache', true); + if (!groupPacked) { + continue; + } + + const groupUnpacked = unpackJs5Group(groupPacked); + const files = splitGroupFiles(groupUnpacked, fileIds); + + for (const fileId of fileIds) { + const fileData = files.get(fileId) ?? new Uint8Array(0); + const id = (groupId << 8) | fileId; + const dat = new Packet(fileData); + + let certLink: number | null = null; + let certTemplate: number | null = null; + let lentLink: number | null = null; + let lentTemplate: number | null = null; + + while (dat.available > 0) { + const code = dat.g1(); + if (code === 0) { + break; + } + + const payload = decodeObjOpcode(code, dat); + if (code === 97) { + certLink = Number(payload); + } else if (code === 98) { + certTemplate = Number(payload); + } else if (code === 121) { + lentLink = Number(payload); + } else if (code === 122) { + lentTemplate = Number(payload); + } + } + + if (certTemplate === CERT_TEMPLATE_ID && certLink !== null) { + certNames.set(id, `cert_obj_${certLink}`); + } + + if (lentTemplate === LENT_TEMPLATE_ID && lentLink !== null) { + lentNames.set(id, `lent_obj_${lentLink}`); + } + } + } + + return { cert: certNames, lent: lentNames }; +} + +async function main() { + const srcPath = path.join(Environment.BUILD_SRC_DIR, 'scripts', '_unpack', '530', 'all.obj'); + const packPath = path.join(Environment.BUILD_SRC_DIR, 'pack', 'obj.pack'); + + if (!fs.existsSync(srcPath)) { + throw new Error(`Source file not found: ${srcPath}`); + } + + const content = fs.readFileSync(srcPath, 'utf-8'); + const existingPack = new PackFile('obj'); + existingPack.load(packPath); + const sections = parseObjSections(content, new Map(existingPack.nameToId)); + const cacheLinkedNames = await collectLinkedObjectNamesFromCache(); + const certObjectNames = collectCertObjectNames(sections); + const lentObjectNames = collectLentObjectNames(sections); + + for (const [id, name] of cacheLinkedNames.cert) { + certObjectNames.set(id, name); + } + for (const [id, name] of cacheLinkedNames.lent) { + lentObjectNames.set(id, name); + } + + console.log(`Found ${sections.length} obj definitions (${certObjectNames.size} cert objects, ${lentObjectNames.size} lent objects)`); + + const pack = new PackFile('obj'); + const usedNames = new Set(); + + for (const section of sections) { + let desiredName = `obj_${section.id}`; + if (lentObjectNames.has(section.id)) { + desiredName = lentObjectNames.get(section.id)!; + } + if (certObjectNames.has(section.id)) { + desiredName = certObjectNames.get(section.id)!; + } + const uniqueName = allocateUniqueName(desiredName, section.id, usedNames); + usedNames.add(uniqueName); + pack.register(section.id, uniqueName); + } + + for (const [id, existingName] of existingPack.pack.entries()) { + if (pack.pack.has(id)) { + continue; + } + + let desiredName = existingName; + if (lentObjectNames.has(id)) { + desiredName = lentObjectNames.get(id)!; + } + if (certObjectNames.has(id)) { + desiredName = certObjectNames.get(id)!; + } + + const uniqueName = allocateUniqueName(desiredName, id, usedNames); + usedNames.add(uniqueName); + pack.register(id, uniqueName); + } + + for (const [id, name] of cacheLinkedNames.lent) { + if (!pack.pack.has(id)) { + const uniqueName = allocateUniqueName(name, id, usedNames); + usedNames.add(uniqueName); + pack.register(id, uniqueName); + } + } + for (const [id, name] of cacheLinkedNames.cert) { + if (!pack.pack.has(id)) { + const uniqueName = allocateUniqueName(name, id, usedNames); + usedNames.add(uniqueName); + pack.register(id, uniqueName); + } + } + + const dir = path.dirname(packPath); + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }); + } + + const entries = Array.from(pack.pack.entries()) + .sort((a, b) => a[0] - b[0]) + .map(([id, name]) => `${id}=${name}`) + .join('\n') + '\n'; + + fs.writeFileSync(packPath, entries); + console.log(`Wrote ${pack.size} entries to ${packPath}`); +} + +main().catch(console.error); diff --git a/tools/pack/generators/generateParamPack.ts b/tools/pack/generators/generateParamPack.ts new file mode 100644 index 000000000..9d6cb5bcd --- /dev/null +++ b/tools/pack/generators/generateParamPack.ts @@ -0,0 +1,79 @@ +import fs from 'fs'; +import path from 'path'; + +import Environment from '#/util/Environment.js'; +import { PackFile } from '#tools/pack/core/PackFile.js'; + +function extractParamNames(content: string): string[] { + const names: string[] = []; + const lines = content.split('\n'); + + for (const line of lines) { + const trimmed = line.trim(); + + if (trimmed.startsWith('[param_') && trimmed.endsWith(']')) { + const name = trimmed.substring(1, trimmed.length - 1); + if (!names.includes(name)) { + names.push(name); + } + } + } + + return names; +} + +function extractParamId(name: string): number | null { + if (!name.startsWith('param_')) { + return null; + } + + const idPart = name.substring(6); + const id = parseInt(idPart); + + if (isNaN(id)) { + return null; + } + + return id; +} + +async function main() { + const srcPath = path.join(Environment.BUILD_SRC_DIR, 'scripts', '_unpack', '530', 'all.param'); + const packPath = path.join(Environment.BUILD_SRC_DIR, 'pack', 'param.pack'); + + if (!fs.existsSync(srcPath)) { + throw new Error(`Source file not found: ${srcPath}`); + } + + const content = fs.readFileSync(srcPath, 'utf-8'); + const names = extractParamNames(content); + + console.log(`Found ${names.length} param definitions`); + + const pack = new PackFile('param'); + + for (const name of names) { + const id = extractParamId(name); + if (id !== null) { + pack.register(id, name); + } else { + console.warn(`Warning: Could not extract ID from name: ${name}`); + } + } + + const dir = path.dirname(packPath); + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }); + } + + const entries = Array.from(pack.pack.entries()) + .sort((a, b) => a[0] - b[0]) + .map(([id, name]) => `${id}=${name}`) + .join('\n') + '\n'; + + fs.writeFileSync(packPath, entries); + + console.log(`Wrote ${pack.size} entries to ${packPath}`); +} + +main().catch(console.error); diff --git a/tools/pack/generators/generatePatchPack.ts b/tools/pack/generators/generatePatchPack.ts new file mode 100644 index 000000000..42754affc --- /dev/null +++ b/tools/pack/generators/generatePatchPack.ts @@ -0,0 +1,21 @@ +import fs from 'fs'; +import path from 'path'; + +import { PackFile } from '#tools/pack/core/PackFile.js'; +import { listPatchPackEntries } from '#tools/pack/sources/PatchSource.js'; +import Environment from '#/util/Environment.js'; + +const patchPack = new PackFile('patch'); + +for (const patchEntry of listPatchPackEntries()) { + patchPack.register(patchEntry.id, patchEntry.name); +} + +const packDir = `${Environment.BUILD_SRC_DIR}/pack`; +if (!fs.existsSync(packDir)) { + fs.mkdirSync(packDir, { recursive: true }); +} + +const packPath = path.join(packDir, 'patch.pack'); +patchPack.save(packPath); +console.log(`\nSaved to ${packPath}`); diff --git a/tools/pack/generators/generateStructPack.ts b/tools/pack/generators/generateStructPack.ts new file mode 100644 index 000000000..b2a073f47 --- /dev/null +++ b/tools/pack/generators/generateStructPack.ts @@ -0,0 +1,79 @@ +import fs from 'fs'; +import path from 'path'; + +import Environment from '#/util/Environment.js'; +import { PackFile } from '#tools/pack/core/PackFile.js'; + +function extractStructNames(content: string): string[] { + const names: string[] = []; + const lines = content.split('\n'); + + for (const line of lines) { + const trimmed = line.trim(); + + if (trimmed.startsWith('[struct_') && trimmed.endsWith(']')) { + const name = trimmed.substring(1, trimmed.length - 1); + if (!names.includes(name)) { + names.push(name); + } + } + } + + return names; +} + +function extractStructId(name: string): number | null { + if (!name.startsWith('struct_')) { + return null; + } + + const idPart = name.substring(7); + const id = parseInt(idPart); + + if (isNaN(id)) { + return null; + } + + return id; +} + +async function main() { + const srcPath = path.join(Environment.BUILD_SRC_DIR, 'scripts', '_unpack', '530', 'all.struct'); + const packPath = path.join(Environment.BUILD_SRC_DIR, 'pack', 'struct.pack'); + + if (!fs.existsSync(srcPath)) { + throw new Error(`Source file not found: ${srcPath}`); + } + + const content = fs.readFileSync(srcPath, 'utf-8'); + const names = extractStructNames(content); + + console.log(`Found ${names.length} struct definitions`); + + const pack = new PackFile('struct'); + + for (const name of names) { + const id = extractStructId(name); + if (id !== null) { + pack.register(id, name); + } else { + console.warn(`Warning: Could not extract ID from name: ${name}`); + } + } + + const dir = path.dirname(packPath); + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }); + } + + const entries = Array.from(pack.pack.entries()) + .sort((a, b) => a[0] - b[0]) + .map(([id, name]) => `${id}=${name}`) + .join('\n') + '\n'; + + fs.writeFileSync(packPath, entries); + + console.log(`Wrote ${pack.size} entries to ${packPath}`); +} + +main().catch(console.error); diff --git a/tools/pack/packs/ChatCatPack.ts b/tools/pack/packs/ChatCatPack.ts new file mode 100644 index 000000000..eb32739af --- /dev/null +++ b/tools/pack/packs/ChatCatPack.ts @@ -0,0 +1,20 @@ +import { PackFile } from '#tools/pack/core/PackFile.js'; + +const chatcatPack = new PackFile('chatcat'); +chatcatPack.load('data/src/pack/chatcat.pack'); + +export function getChatCatId(name: string): number { + return chatcatPack.getByName(name); +} + +export function getChatCatName(id: number): string { + return chatcatPack.getById(id); +} + +export function hasChatCatName(name: string): boolean { + return chatcatPack.names.has(name); +} + +export function getAllChatCatNames(): string[] { + return Array.from(chatcatPack.names); +} diff --git a/tools/pack/packs/ChatPhrasePack.ts b/tools/pack/packs/ChatPhrasePack.ts new file mode 100644 index 000000000..906843abc --- /dev/null +++ b/tools/pack/packs/ChatPhrasePack.ts @@ -0,0 +1,20 @@ +import { PackFile } from '#tools/pack/core/PackFile.js'; + +const chatphrasePack = new PackFile('chatphrase'); +chatphrasePack.load('data/src/pack/chatphrase.pack'); + +export function getChatPhraseId(name: string): number { + return chatphrasePack.getByName(name); +} + +export function getChatPhraseName(id: number): string { + return chatphrasePack.getById(id); +} + +export function hasChatPhraseName(name: string): boolean { + return chatphrasePack.names.has(name); +} + +export function getAllChatPhraseNames(): string[] { + return Array.from(chatphrasePack.names); +} diff --git a/tools/pack/packs/EnumPack.ts b/tools/pack/packs/EnumPack.ts new file mode 100644 index 000000000..63cae7f47 --- /dev/null +++ b/tools/pack/packs/EnumPack.ts @@ -0,0 +1,20 @@ +import { PackFile } from '#tools/pack/core/PackFile.js'; + +const enumPack = new PackFile('enum'); +enumPack.load('data/src/pack/enum.pack'); + +export function getEnumId(name: string): number { + return enumPack.getByName(name); +} + +export function getEnumName(id: number): string { + return enumPack.getById(id); +} + +export function hasEnumName(name: string): boolean { + return enumPack.names.has(name); +} + +export function getAllEnumNames(): string[] { + return Array.from(enumPack.names); +} diff --git a/tools/pack/packs/FloPack.ts b/tools/pack/packs/FloPack.ts new file mode 100644 index 000000000..8eb0d9b30 --- /dev/null +++ b/tools/pack/packs/FloPack.ts @@ -0,0 +1,20 @@ +import { PackFile } from '#tools/pack/core/PackFile.js'; + +const floPack = new PackFile('flo'); +floPack.load('data/src/pack/flo.pack'); + +export function getFloId(name: string): number { + return floPack.getByName(name); +} + +export function getFloName(id: number): string { + return floPack.getById(id); +} + +export function hasFloName(name: string): boolean { + return floPack.names.has(name); +} + +export function getAllFloNames(): string[] { + return Array.from(floPack.names); +} diff --git a/tools/pack/packs/MidiPack.ts b/tools/pack/packs/MidiPack.ts new file mode 100644 index 000000000..55ef57ed0 --- /dev/null +++ b/tools/pack/packs/MidiPack.ts @@ -0,0 +1,39 @@ +import fs from 'fs'; +import path from 'path'; + +import Environment from '#/util/Environment.js'; +import { PackFile } from '#tools/pack/core/PackFile.js'; + +const midiPack = new PackFile('midi'); + +const midiPackCandidates = [ + path.join(Environment.BUILD_SRC_DIR, 'pack', 'midi.pack'), + path.join('data', 'src', 'pack', 'midi.pack') +]; + +for (const candidate of midiPackCandidates) { + if (fs.existsSync(candidate)) { + midiPack.load(candidate); + break; + } +} + +export function getMidiId(name: string): number { + return midiPack.getByName(name); +} + +export function getMidiName(id: number): string { + return midiPack.getById(id); +} + +export function hasMidiId(id: number): boolean { + return midiPack.pack.has(id); +} + +export function hasMidiName(name: string): boolean { + return midiPack.names.has(name); +} + +export function getAllMidiNames(): string[] { + return Array.from(midiPack.names); +} diff --git a/tools/pack/sources/MidiSource.ts b/tools/pack/sources/MidiSource.ts new file mode 100644 index 000000000..686fad5c1 --- /dev/null +++ b/tools/pack/sources/MidiSource.ts @@ -0,0 +1,83 @@ +import fs from 'fs'; +import path from 'path'; + +import Environment from '#/util/Environment.js'; + +export type MidiSourceKind = 'songs' | 'jingles'; + +export type MidiSourceFile = { + name: string; + filePath: string; + kind: MidiSourceKind; +}; + +export type MidiPackEntry = MidiSourceFile & { + id: number; +}; + +export function extractMidiId(name: string): number { + if (!name.startsWith('midi_')) { + return -1; + } + + const parsed = parseInt(name.slice(5)); + return Number.isNaN(parsed) ? -1 : parsed; +} + +function listMidisInDir(kind: MidiSourceKind): MidiSourceFile[] { + const dir = path.join(Environment.BUILD_SRC_DIR, kind); + if (!fs.existsSync(dir)) { + return []; + } + + return fs.readdirSync(dir) + .filter(filename => filename.endsWith('.mid')) + .map(filename => { + const name = filename.slice(0, -4); + return { + name, + filePath: path.join(dir, filename), + kind, + }; + }); +} + +export function listAllMidiSourceFiles(): MidiSourceFile[] { + return [ + ...listMidisInDir('songs'), + ...listMidisInDir('jingles'), + ]; +} + +export function listMidiPackEntries(): MidiPackEntry[] { + const entries: MidiPackEntry[] = []; + + for (const midi of listAllMidiSourceFiles()) { + const id = extractMidiId(midi.name); + if (id >= 0) { + entries.push({ + id, + name: midi.name, + filePath: midi.filePath, + kind: midi.kind, + }); + } + } + + entries.sort((a, b) => a.id - b.id); + return entries; +} + +export function resolveMidiPathByName(name: string): string | null { + const songsPath = path.join(Environment.BUILD_SRC_DIR, 'songs', `${name}.mid`); + if (fs.existsSync(songsPath)) { + return songsPath; + } + + const jinglesPath = path.join(Environment.BUILD_SRC_DIR, 'jingles', `${name}.mid`); + if (fs.existsSync(jinglesPath)) { + return jinglesPath; + } + + return null; +} diff --git a/tools/pack/sources/PatchSource.ts b/tools/pack/sources/PatchSource.ts new file mode 100644 index 000000000..a12a8fef9 --- /dev/null +++ b/tools/pack/sources/PatchSource.ts @@ -0,0 +1,67 @@ +import fs from 'fs'; +import path from 'path'; + +import Environment from '#/util/Environment.js'; + +export type PatchSourceFile = { + name: string; + filePath: string; +}; + +export type PatchPackEntry = PatchSourceFile & { + id: number; +}; + +export function extractPatchId(name: string): number { + if (!name.startsWith('patch_')) { + return -1; + } + + const parsed = parseInt(name.slice(6)); + return Number.isNaN(parsed) ? -1 : parsed; +} + +export function listAllPatchSourceFiles(): PatchSourceFile[] { + const dir = path.join(Environment.BUILD_SRC_DIR, 'patches'); + if (!fs.existsSync(dir)) { + return []; + } + + return fs.readdirSync(dir) + .filter(filename => filename.endsWith('.patch')) + .map(filename => { + const ext = path.extname(filename); + const name = filename.slice(0, -ext.length); + return { + name, + filePath: path.join(dir, filename) + }; + }); +} + +export function listPatchPackEntries(): PatchPackEntry[] { + const entries: PatchPackEntry[] = []; + + for (const patch of listAllPatchSourceFiles()) { + const id = extractPatchId(patch.name); + if (id >= 0) { + entries.push({ + id, + name: patch.name, + filePath: patch.filePath + }); + } + } + + entries.sort((a, b) => a.id - b.id); + return entries; +} + +export function resolvePatchPathByName(name: string): string | null { + const patchPath = path.join(Environment.BUILD_SRC_DIR, 'patches', `${name}.patch`); + if (fs.existsSync(patchPath)) { + return patchPath; + } + + return null; +} diff --git a/tools/unpack/UnpackMidi.ts b/tools/unpack/UnpackMidi.ts new file mode 100644 index 000000000..27c3d9d74 --- /dev/null +++ b/tools/unpack/UnpackMidi.ts @@ -0,0 +1,67 @@ +import fs from 'fs'; + +import Environment from '#/util/Environment.js'; +import { printWarning, printInfo } from '#/util/Logger.js'; +import { getGroup } from '#/util/OpenRS2.js'; +import { createPackManager } from '#tools/pack/core/PackManager.js'; + +// Create directories if they don't exist +if (!fs.existsSync(`${Environment.BUILD_SRC_DIR}/songs`)) { + fs.mkdirSync(`${Environment.BUILD_SRC_DIR}/songs`, { recursive: true }); +} + +if (!fs.existsSync(`${Environment.BUILD_SRC_DIR}/jingles`)) { + fs.mkdirSync(`${Environment.BUILD_SRC_DIR}/jingles`, { recursive: true }); +} + +console.time('midis'); + +// Load or create midi pack file +const midiPack = createPackManager('midi'); +printInfo(`Loaded pack with ${midiPack.getSize()} existing MIDIs`); + +// Fetch MIDI data from OpenRS2 archive 6 +const archive = 6; +const maxMidiId = 700; +const foundMidis: { id: number; data: Uint8Array; isJingle: boolean }[] = []; + +for (let i = 0; i < maxMidiId; i++) { + try { + const data = await getGroup(archive, i); + if (data && data.length > 0) { + foundMidis.push({ + id: i, + data: data, + isJingle: false, + }); + + if (foundMidis.length % 10 === 0) { + printInfo(`Found ${foundMidis.length} MIDIs so far...`); + } + } + } catch (_e) { + if (foundMidis.length === 0 && i > 50) { + printWarning(`No MIDIs found after trying ${i} IDs. Stopping.`); + break; + } + if (i % 100 === 0) { + printInfo(`Tried ${i} IDs, found ${foundMidis.length} so far...`); + } + } +} + +printInfo(`Found ${foundMidis.length} MIDIs`); + +for (const midi of foundMidis) { + // Get or register name for this MIDI ID + const name = midiPack.getName(midi.id); + + const dir = midi.isJingle ? 'jingles' : 'songs'; + fs.writeFileSync(`${Environment.BUILD_SRC_DIR}/${dir}/${name}.mid`, midi.data); + printInfo(` ${midi.id}: ${name}.mid (${midi.data.length} bytes)`); +} + +// Save pack file with any new IDs that were discovered +midiPack.save(); + +console.timeEnd('midis'); diff --git a/tools/unpack/UnpackPatch.ts b/tools/unpack/UnpackPatch.ts new file mode 100644 index 000000000..e7f05b12e --- /dev/null +++ b/tools/unpack/UnpackPatch.ts @@ -0,0 +1,59 @@ +import fs from 'fs'; + +import { unpackJs5Group } from '#/io/Js5Group.js'; +import Environment from '#/util/Environment.js'; +import { printInfo, printWarning } from '#/util/Logger.js'; +import { createPackManager } from '#tools/pack/core/PackManager.js'; +import { parseGroupIdsFromIndexPacked } from '#tools/cache/lib/js5Tools.js'; + +const archive = 15; +const patchesDir = `${Environment.BUILD_SRC_DIR}/patches`; + +if (!fs.existsSync(patchesDir)) { + fs.mkdirSync(patchesDir, { recursive: true }); +} + +const indexPath = `data/cache/255/${archive}.dat`; +if (!fs.existsSync(indexPath)) { + throw new Error(`Index file not found: ${indexPath}`); +} + +console.time('patches'); + +const patchPack = createPackManager('patch'); +printInfo(`Loaded pack with ${patchPack.getSize()} existing patches`); + +const indexPacked = new Uint8Array(fs.readFileSync(indexPath)); +const groupIds = parseGroupIdsFromIndexPacked(indexPacked); + +let foundPatches = 0; +for (const groupId of groupIds) { + const groupPath = `data/cache/${archive}/${groupId}.dat`; + + if (!fs.existsSync(groupPath)) { + continue; + } + + const container = new Uint8Array(fs.readFileSync(groupPath)); + const payload = unpackJs5Group(container); + if (!payload || payload.length === 0) { + continue; + } + + const name = patchPack.getName(groupId); + fs.writeFileSync(`${patchesDir}/${name}.patch`, payload); + foundPatches++; + + if (foundPatches % 50 === 0) { + printInfo(`Unpacked ${foundPatches} patches so far...`); + } +} + +if (foundPatches === 0) { + printWarning('No non-empty patches were unpacked.'); +} else { + printInfo(`Unpacked ${foundPatches} patches`); +} + +patchPack.save(); +console.timeEnd('patches');