From befb3d8ada6cbd2389e02eaa187ce260f1ac9bb6 Mon Sep 17 00:00:00 2001 From: Miguel Duarte Date: Wed, 1 Dec 2021 10:43:42 -0300 Subject: [PATCH 01/82] just starting. nothing works. --- run-db.sqlite3 | Bin 0 -> 344064 bytes src/database.js | 112 +++++++++++++++++++++++++++--------------------- src/index.js | 2 +- src/indexer.js | 10 ++--- 4 files changed, 69 insertions(+), 55 deletions(-) create mode 100644 run-db.sqlite3 diff --git a/run-db.sqlite3 b/run-db.sqlite3 new file mode 100644 index 0000000000000000000000000000000000000000..692789d59c327b1f19799112ae6a8ba19b4e0b26 GIT binary patch literal 344064 zcmeI)OYG*?eb@0BI~gZ#+|+TVLkcK=Rfu{*fOEgh#WPMa#0F!xjzbc0anA2t>>;l0 zddAH}P=P567F~eYu*iZ12nj*eMK%Z)2z7xFAQlK^38)K%sDwm8Y(RW|&okG=sVr5- zk^CNc{(kQG7K6CLeT>R+^|N7#;y%h61`Cq-;K)i_bdio2`T|2q^_Q{hs zZ(ScJFJ1fkODA7_{@Gvn>b3Lsr_Y?b_Sw^?uD?F!tsB4en!eWea(nyc8;9RtzI*4( z_U_BGf9;t^A368<UKIS7PM&`5YfpdU zD<@BX<>blZY@hwg$(_ZYKlwAyzWCBtpMLKA^PfC(?vI{3{mS7XH{W{u#?6;^tC#Qn z{!_PQpMCyI*S>!8gO{AV@ci+;syX zmv`&22gZ2d!^6UE-Cot@pHQ$rar$eAnf%4;uN-Hg7oT`wfV-dUH|v*^C;Myd#*OnA zzjx}hd%XVYjR)fO%-J)Clzrw?GJE{CnI>@z$J<+Q>xl;*@dF>a-I{>T}<@R7q7{OU&!Gg3O~pN~E; zn?s7-z5nFo_dJ*Tr+P@;!&L8o_eOu~`kOaZ=wJPzkM#6!9p1b)-(UX8#s8W9R{b~ii}6>k*h?2K zek(mv|HkEyhhNYC^~x`&|M1j5C^s+u;Vb_%fAgvT_S7G|{QRZAa{2q|xu>@Be=q;- zOTStEcr4+6T=>+5uf>0H;ky_9)5YJ&fB%D86VF8e0R#|0009ILKmY**5I_I{1Q0*~ z0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~fe%99OCNdXv5)&z z#|+Ue(|xaLo7v|PTx+HVySW^!bJo_JQYkg%Ha2f_nA4Wcgr34-kZxM8v$VD4-nJQ+ zi`!_;*OH1KX{?z_bJe=d)irAjjOfGI)NyAy= zTDR6%6E&`zKBqNvGI@qs^5|t8263T}>60%lgwRVHzUJtYb-8$7a#>50v6}jbvBps9 zHe3xenvwH9^kI+@H2BEg)Mn<~%2wUTHclUVnlAe`lJCa2xK`@DRhPG6mf2F6qv#Qb zLA;-OT+2Q9Esflg=`P!$dAJ%^(@M8I>*~|U+pcA|=u%15r_ImQ=3$U5qb|fXeGR_O zu|!#Al2V;{3&T(Ck*jmg++A|ByP)MvKfN`x*25rX8_WCVeb6MTsY^qSw)qgFF@4j9 z`n9bQ&1Od})*0vWY|~V?72ExBkkD+^#GS}v%v#Rojl{HD#%|T;8jK0fYRgI;K5QZu zvXn~{u+7Jrc<;=zF6`oU+jPTBON-AgY|Gl1Y;mP36mu$lwJ@7@uF0`h>xVlGlKeE& z%4(Fh89mbWnD^K#Q&Fq?yVlf#*Ra~{s$Co=Y`M&BZfVAJ7-Utcx9*z4bHzEN()31H zKGY@awq=sCCZ?@wwa$2jfbCk7qRuPyqE&!p zmqtnxN~UUiXo|eq+tS{ZGBuIW z^u&3L*cg1=FGxn?#})P%Gqxw)=cWrQwH!jxugrdDO5%UPixl9FVUYTFpTD~h#%!*Y}?O6U zt7=lCO~&UE#>%~97j5mPCM{9B6i406@S`gox9PXseCn+@g+SA`9Zb5i-If^4a3#eq znHCkqDMd4z;;vh>gyDW zOy{i%VeCr*syb4%$;yzbA}XrGAS2m*_hL)V1qISrNl#Z97J+K?rZ@Jgs{*0Qkhirt zziDpr(!-;P$3e1c;FfYVEsUwiY}O`iOIyoKg`Y20RKJZ1Slic|mcW7<$+1w! zRz^?((}LA5q^3Y?-K&^J1?yC`m4@;r^)DhRT0Yp;6;f_tLQ1Y}L8YRV&kK$6y;-86K?zb< zHDzg!$&j4|5^YkyB|sHtx&P>e$VV4`H0BC`tpJvD|3lAbZPz2w`dNoa>- zNIguYd(R;&lG56?yhoBV$C)VB_C}{T%t~L=%@#GuytbBgUP#%`rjjY}pfV{Y#Y|OB zcJG?zYL7Efr25&XO8e6OWN(8?>uNgh#H^fDBkO9@d2Z?$acx#vsou6!F1*TZxZR73 zG*s~vX+`Vw*(k{N2-j=1T@CMa_ULYD%p7#E$gR)06e4kEWy--MTg}`T1@hiC_p{Gb zJRzGsdnd~vsNAIj@l;eLTSei7}blrv0i7Lt|W3@@or5MSPDAcUg>EW1wD)FkirRpGJy*k-8qDrFLWSQz;`!ai8aX;fJX>3;1YMWfzN>ak> zV0BoIE5fsCjk;RVW{cn3*``*yXI@o&s9mX^*`+LynU0Y7x z<-1W`NxE+@+1(ChgL-dQdW%Y}Sv7c68~5-3pSt=%oaK2Y0tg_000IagfB*srAb+Y=cW0R#|0 z009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{ z1Q0*~0R#|0009ILKmY**5I_I{1m1(dqx&^i-@}Dmf&c;tAba0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{ z1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_Kd_bkBo|KBqe7b1WF z0tg_000IagfB*srAbi-j#6anTvlpeZKzP%byP4FCV*dEB)`M z{!#gx7t57@mjCKg|NW^ybouL-e(myqOm9B*YW;nezkBK5mFMC&;~%*2y#L9Wz9?O!vK}ZDyZGaIKjd?B;T? z&RJV?N~P44+t|F#VNP2%6M72!7~Qm7XK8E8y=^lt7q`)xuO$^f(pWQ<=Bjm@t83O! z=CIxj8>Vm07qgGCtrl%DGZ{OpYs>jjOfGI)NuycgTDR6%6E&=xKBqNvGI@qs^5|vk zW4O@A^vRbNLg=LpUvu=yx?H?3xvZtBSj~IHSYs%48?J^K&B%Elc^_j04L!0qwV64$ zvQ;;-jnl`TrpvyKW6 zTIQB#U40r++qLW#T`H;iwE3CZ+{eh`>GE6C*Wl|MONvz z3tG$c(_1rZ-N!K7Sl&19gQieTT^ez;&4(C`>6>=auWgNJHalvu&N!E6o2I(0*zONw zgl4N|?Svg;)><}igr$8lcB?+uU`%jUn^o%YVUw^3zN!rcv5v^g!EV-b1cT#j0-aT2KpK zqiUP0_HUT55Z^_u*ufZ6xFH7ZlR`mpH*4!kplW+)3b@(Z(yo;z|7tmXE;OlMmwt1qK?Rd_|?Y8y_GmX(>NRwbXyS@9l{ zGGml#+Zetpb~XKdElLn&@CYGpN=YSe&$@9GNvyezA;YB@O(um=Q*Be5TGrTHoYq%= zFty9bw2T@;&YSI9%*oZE*()HDO)E`hk2bSynw0`_H06m+RW{1vL%#T=q^QNXlGlFS zjB|#~^yrK8&g$uL(;~b{#U!cP8LjEnS{t@Lk1=LdJ!-Vc_*}wRxtHvst=-h5^=W_N zsGAvnbYLXwWj&&!M>|9)iuI&-m2=wz7&?K8%4XTw5Y0>V%o}cX)jI4sUrCd!5V=5k- zwMiS&)-qF3=ZjVT&lsIbwNbrl`(o1)R#3+{)a2O82r5@vtonk$6kn}-70##-ovN&o zP_Cq2Z0ib}+Vn`3sAN<`?V-PTTZU`K)ah#9)H94aNwY~ds%?T$p)V>d>Q8D?8dbZj zsusEmV_NMHVDC?k_C?ds5RT_5hnr3Pb^H6;H*{6#5(#~XWgUaY? zI?lwbG*rLpYQ=eOY7KF1R*9$1Hnp#@=N0$kn-axl#i};QrL82TxQ(+>bK3ta_qNN^>IHM<)mFxK&YWRRdBWct_a)fMOW6Vtu3$C zp^ByhqYf=|D1+9qzbILBa!@jC>g`^~t)(ksRe~GUk)+%9>fG&63aGz!CA65-epPQr zwQ>Lc|Ea4Vs{b@#u?Qf500IagfB*srAbjRy@csWEyDect2q1s} z0tg_000IagfB*srAbE5vb literal 0 HcmV?d00001 diff --git a/src/database.js b/src/database.js index 22b43ec..7463cba 100644 --- a/src/database.js +++ b/src/database.js @@ -81,7 +81,7 @@ class Database { this.onRequestDownload = null } - open () { + async open () { this.logger.debug('Opening' + (this.readonly ? ' readonly' : '') + ' database') if (this.db) throw new Error('Database already open') @@ -100,13 +100,13 @@ class Database { if (!this.readonly) { // Initialise and perform upgrades - this.initializeV1() - this.initializeV2() - this.initializeV3() - this.initializeV4() - this.initializeV5() - this.initializeV6() - this.initializeV7() + await this.initializeV1() + await this.initializeV2() + await this.initializeV3() + await this.initializeV4() + await this.initializeV5() + await this.initializeV6() + await this.initializeV7() } this.addNewTransactionStmt = this.db.prepare('INSERT OR IGNORE INTO tx (txid, height, time, bytes, has_code, executable, executed, indexed) VALUES (?, null, ?, null, 0, 0, 0, 0)') @@ -194,12 +194,12 @@ class Database { this.unmarkExecutingStmt = this.db.prepare('DELETE FROM executing WHERE txid = ?') } - initializeV1 () { + async initializeV1 () { if (this.db.pragma('user_version')[0].user_version !== 0) return this.logger.info('Setting up database v1') - this.transaction(() => { + await this.transaction(() => { this.db.pragma('user_version = 1') this.db.prepare( @@ -283,12 +283,12 @@ class Database { }) } - initializeV2 () { + async initializeV2 () { if (this.db.pragma('user_version')[0].user_version !== 1) return this.logger.info('Setting up database v2') - this.transaction(() => { + await this.transaction(() => { this.db.pragma('user_version = 2') this.db.prepare( @@ -338,12 +338,12 @@ class Database { this.db.prepare('VACUUM').run() } - initializeV3 () { + async initializeV3 () { if (this.db.pragma('user_version')[0].user_version !== 2) return this.logger.info('Setting up database v3') - this.transaction(() => { + await this.transaction(() => { this.db.pragma('user_version = 3') this.db.prepare('CREATE INDEX IF NOT EXISTS deps_up_index ON deps (up)').run() @@ -354,12 +354,12 @@ class Database { }) } - initializeV4 () { + async initializeV4 () { if (this.db.pragma('user_version')[0].user_version !== 3) return this.logger.info('Setting up database v4') - this.transaction(() => { + await this.transaction(() => { this.db.pragma('user_version = 4') this.db.prepare('ALTER TABLE tx ADD COLUMN downloaded INTEGER GENERATED ALWAYS AS (bytes IS NOT NULL) VIRTUAL').run() @@ -370,12 +370,12 @@ class Database { }) } - initializeV5 () { + async initializeV5 () { if (this.db.pragma('user_version')[0].user_version !== 4) return this.logger.info('Setting up database v5') - this.transaction(() => { + await this.transaction(() => { this.db.pragma('user_version = 5') this.db.prepare('CREATE INDEX IF NOT EXISTS ban_txid_index ON ban (txid)').run() @@ -385,12 +385,12 @@ class Database { }) } - initializeV6 () { + async initializeV6 () { if (this.db.pragma('user_version')[0].user_version !== 5) return this.logger.info('Setting up database v6') - this.transaction(() => { + await this.transaction(() => { this.db.pragma('user_version = 6') const height = this.db.prepare('SELECT height FROM crawl WHERE role = \'tip\'').raw(true).all()[0] @@ -412,12 +412,12 @@ class Database { }) } - initializeV7 () { + async initializeV7 () { if (this.db.pragma('user_version')[0].user_version !== 6) return this.logger.info('Setting up database v7') - this.transaction(() => { + await this.transaction(() => { this.db.pragma('user_version = 7') this.logger.info('Getting possible transactions to execute') @@ -465,44 +465,58 @@ class Database { } } - transaction (f) { + async transaction (f) { if (!this.db) return - this.db.transaction(f)() + try { + this.db.raw('beggin') + await f() + this.db.raw('commit') + } catch (e) { + this.db.raw('rollback') + throw e + } + // this.db.transaction(f)() } // -------------------------------------------------------------------------- // tx // -------------------------------------------------------------------------- - addBlock (txids, txhexs, height, hash, time) { - this.transaction(() => { - txids.forEach((txid, i) => { - const txhex = txhexs && txhexs[i] - this.addTransaction(txid, txhex, height, time) - }) + async addBlock (txids, txhexs, height, hash, time) { + await this.transaction(async () => { + const indexes = new Array(txids.length).map((_, i) => i) + for(const index of indexes) { + const txid = txids[index] + const txHex = txhexs && txhexs[i] + await this.addTransaction(txid, txHex, height, time) + } + // txids.forEach(async (txid, i) => { + // const txhex = txhexs && txhexs[i] + // await this.addTransaction(txid, txhex, height, time) + // }) this.setHeight(height) this.setHash(hash) }) } - addTransaction (txid, txhex, height, time) { - this.transaction(() => { - this.addNewTransaction(txid) - if (height) this.setTransactionHeight(txid, height) - if (time) this.setTransactionTime(txid, time) + async addTransaction (txid, txhex, height, time) { + await this.transaction(async () => { + await this.addNewTransaction(txid) + if (height) { await this.setTransactionHeight(txid, height) } + if (time) { await this.setTransactionTime(txid, time) } }) - const downloaded = this.isTransactionDownloaded(txid) + const downloaded = await this.isTransactionDownloaded(txid) if (downloaded) return if (txhex) { - this.parseAndStoreTransaction(txid, txhex) + await this.parseAndStoreTransaction(txid, txhex) } else { - if (this.onRequestDownload) this.onRequestDownload(txid) + if (this.onRequestDownload) { await this.onRequestDownload(txid) } } } - parseAndStoreTransaction (txid, hex) { + async parseAndStoreTransaction (txid, hex) { if (this.isTransactionDownloaded(txid)) return let metadata = null @@ -581,7 +595,7 @@ class Database { } storeParsedNonExecutableTransaction (txid, hex, inputs, outputs) { - this.transaction(() => { + await this.transaction(() => { const bytes = Buffer.from(hex, 'hex') this.setTransactionBytesStmt.run(bytes, txid) this.setTransactionExecutableStmt.run(0, txid) @@ -599,7 +613,7 @@ class Database { } storeParsedExecutableTransaction (txid, hex, hasCode, deps, inputs, outputs) { - this.transaction(() => { + await this.transaction(() => { const bytes = Buffer.from(hex, 'hex') this.setTransactionBytesStmt.run(bytes, txid) this.setTransactionExecutableStmt.run(1, txid) @@ -625,7 +639,7 @@ class Database { storeExecutedTransaction (txid, result) { const { cache, classes, locks, scripthashes } = result - this.transaction(() => { + await this.transaction(() => { this.setTransactionExecutedStmt.run(1, txid) this.setTransactionIndexedStmt.run(1, txid) this.unmarkExecutingStmt.run(txid) @@ -665,7 +679,7 @@ class Database { } setTransactionExecutionFailed (txid) { - this.transaction(() => { + await this.transaction(() => { this.setTransactionExecutableStmt.run(0, txid) this.setTransactionExecutedStmt.run(1, txid) this.setTransactionIndexedStmt.run(0, txid) @@ -709,7 +723,7 @@ class Database { const txids = [txid] deleted.add(txid) - this.transaction(() => { + await this.transaction(() => { while (txids.length) { const txid = txids.shift() @@ -738,7 +752,7 @@ class Database { } unindexTransaction (txid) { - this.transaction(() => { + await this.transaction(() => { if (this.getTransactionIndexedStmt.raw(true).get(txid)[0]) { this.setTransactionExecutedStmt.run(0, txid) this.setTransactionIndexedStmt.run(0, txid) @@ -790,7 +804,7 @@ class Database { } addMissingDeps (txid, deptxids) { - this.transaction(() => deptxids.forEach(deptxid => this.addDep(txid, deptxid))) + await this.transaction(() => deptxids.forEach(deptxid => this.addDep(txid, deptxid))) this._checkExecutability(txid) } @@ -879,7 +893,7 @@ class Database { this.getUpstreamUnexecutedCodeStmt.raw(true).all(txid).forEach(x => queue.push(x[0])) } - this.transaction(() => trusted.forEach(txid => this.setTrustedStmt.run(txid, 1))) + await this.transaction(() => trusted.forEach(txid => this.setTrustedStmt.run(txid, 1))) trusted.forEach(txid => this._checkExecutability(txid)) @@ -888,7 +902,7 @@ class Database { untrust (txid) { if (!this.isTrusted(txid)) return - this.transaction(() => { + await this.transaction(() => { this.unindexTransaction(txid) this.setTrustedStmt.run(txid, 0) }) @@ -909,7 +923,7 @@ class Database { } ban (txid) { - this.transaction(() => { + await this.transaction(() => { this.unindexTransaction(txid) this.banStmt.run(txid) }) diff --git a/src/index.js b/src/index.js index 7a46349..7a32ce3 100644 --- a/src/index.js +++ b/src/index.js @@ -65,7 +65,7 @@ let started = false // ------------------------------------------------------------------------------------------------ async function main () { - database.open() + await database.open() if (!SERVE_ONLY) { await indexer.start() diff --git a/src/indexer.js b/src/indexer.js index b5005f0..64f5ee2 100644 --- a/src/indexer.js +++ b/src/indexer.js @@ -159,18 +159,18 @@ class Indexer { this.logger.error(`Crawl error: ${e.toString()}`) } - _onCrawlBlockTransactions (height, hash, time, txids, txhexs) { + async _onCrawlBlockTransactions (height, hash, time, txids, txhexs) { this.logger.info(`Crawled block ${height} for ${txids.length} transactions`) this.database.addBlock(txids, txhexs, height, hash, time) if (this.onBlock) this.onBlock(height) } - _onRewindBlocks (newHeight) { + async _onRewindBlocks (newHeight) { this.logger.info(`Rewinding to block ${newHeight}`) const txids = this.database.getTransactionsAboveHeight(newHeight) - this.database.transaction(() => { + await this.database.transaction(() => { // Put all transactions back into the mempool. This is better than deleting them, because // when we assume they will just go into a different block, we don't need to re-execute. // If they don't make it into a block, then they will be expired in time. @@ -187,12 +187,12 @@ class Indexer { this.database.addTransaction(txid, hex, Database.HEIGHT_MEMPOOL, null) } - _onExpireMempoolTransactions () { + async _onExpireMempoolTransactions () { const expirationTime = Math.round(Date.now() / 1000) - this.mempoolExpiration const expired = this.database.getMempoolTransactionsBeforeTime(expirationTime) const deleted = new Set() - this.database.transaction(() => expired.forEach(txid => this.database.deleteTransaction(txid, deleted))) + await this.database.transaction(() => expired.forEach(txid => this.database.deleteTransaction(txid, deleted))) } } From 83bb1f945ff0509532ff5a6d6319de2186456250 Mon Sep 17 00:00:00 2001 From: hojarasca Date: Wed, 1 Dec 2021 18:15:06 -0300 Subject: [PATCH 02/82] first version working. --- package-lock.json | 6082 ++++++++++++++++++++++++++++++++++++++++++++- src/database.js | 194 +- src/executor.js | 20 +- src/indexer.js | 64 +- src/server.js | 78 +- test/indexer.js | 2 +- 6 files changed, 6266 insertions(+), 174 deletions(-) diff --git a/package-lock.json b/package-lock.json index aabba9e..2429375 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,8 +1,6070 @@ { "name": "run-db", - "version": "1.0.31", - "lockfileVersion": 1, + "version": "1.0.28", + "lockfileVersion": 2, "requires": true, + "packages": { + "": { + "version": "1.0.28", + "license": "MIT", + "dependencies": { + "abort-controller": "^3.0.0", + "axios": "^0.21.1", + "better-sqlite3": "^7.4.1", + "body-parser": "^1.19.0", + "cors": "^2.8.5", + "dotenv": "^8.2.0", + "event-stream": "^4.0.1", + "eventsource": "^1.1.0", + "express": "^4.17.1", + "morgan": "^1.10.0", + "node-fetch": "^2.6.1", + "reconnecting-eventsource": "^1.1.0", + "run-sdk": "^0.6.18" + }, + "devDependencies": { + "chai": "^4.3.4", + "chai-as-promised": "^7.1.1", + "mocha": "^8.3.2", + "nyc": "^15.1.0", + "standard": "^16.0.3" + }, + "optionalDependencies": { + "zeromq": "^5.2.8" + } + }, + "node_modules/@babel/code-frame": { + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.14.5.tgz", + "integrity": "sha512-9pzDqyc6OLDaqe+zbACgFkb6fKMNG6CObKpnYXChRsvYGyEdc7CA2BaqeOM+vOtCS5ndmJicPJhKAwYRI6UfFw==", + "dev": true, + "dependencies": { + "@babel/highlight": "^7.14.5" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/compat-data": { + "version": "7.15.0", + "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.15.0.tgz", + "integrity": "sha512-0NqAC1IJE0S0+lL1SWFMxMkz1pKCNCjI4tr2Zx4LJSXxCLAdr6KyArnY+sno5m3yH9g737ygOyPABDsnXkpxiA==", + "dev": true, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/core": { + "version": "7.15.0", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.15.0.tgz", + "integrity": "sha512-tXtmTminrze5HEUPn/a0JtOzzfp0nk+UEXQ/tqIJo3WDGypl/2OFQEMll/zSFU8f/lfmfLXvTaORHF3cfXIQMw==", + "dev": true, + "dependencies": { + "@babel/code-frame": "^7.14.5", + "@babel/generator": "^7.15.0", + "@babel/helper-compilation-targets": "^7.15.0", + "@babel/helper-module-transforms": "^7.15.0", + "@babel/helpers": "^7.14.8", + "@babel/parser": "^7.15.0", + "@babel/template": "^7.14.5", + "@babel/traverse": "^7.15.0", + "@babel/types": "^7.15.0", + "convert-source-map": "^1.7.0", + "debug": "^4.1.0", + "gensync": "^1.0.0-beta.2", + "json5": "^2.1.2", + "semver": "^6.3.0", + "source-map": "^0.5.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/core/node_modules/debug": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz", + "integrity": "sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw==", + "dev": true, + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + } + }, + "node_modules/@babel/core/node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + }, + "node_modules/@babel/core/node_modules/semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "dev": true, + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/@babel/generator": { + "version": "7.15.0", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.15.0.tgz", + "integrity": "sha512-eKl4XdMrbpYvuB505KTta4AV9g+wWzmVBW69tX0H2NwKVKd2YJbKgyK6M8j/rgLbmHOYJn6rUklV677nOyJrEQ==", + "dev": true, + "dependencies": { + "@babel/types": "^7.15.0", + "jsesc": "^2.5.1", + "source-map": "^0.5.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-compilation-targets": { + "version": "7.15.0", + "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.15.0.tgz", + "integrity": "sha512-h+/9t0ncd4jfZ8wsdAsoIxSa61qhBYlycXiHWqJaQBCXAhDCMbPRSMTGnZIkkmt1u4ag+UQmuqcILwqKzZ4N2A==", + "dev": true, + "dependencies": { + "@babel/compat-data": "^7.15.0", + "@babel/helper-validator-option": "^7.14.5", + "browserslist": "^4.16.6", + "semver": "^6.3.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-compilation-targets/node_modules/semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "dev": true, + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/@babel/helper-function-name": { + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.14.5.tgz", + "integrity": "sha512-Gjna0AsXWfFvrAuX+VKcN/aNNWonizBj39yGwUzVDVTlMYJMK2Wp6xdpy72mfArFq5uK+NOuexfzZlzI1z9+AQ==", + "dev": true, + "dependencies": { + "@babel/helper-get-function-arity": "^7.14.5", + "@babel/template": "^7.14.5", + "@babel/types": "^7.14.5" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-get-function-arity": { + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/helper-get-function-arity/-/helper-get-function-arity-7.14.5.tgz", + "integrity": "sha512-I1Db4Shst5lewOM4V+ZKJzQ0JGGaZ6VY1jYvMghRjqs6DWgxLCIyFt30GlnKkfUeFLpJt2vzbMVEXVSXlIFYUg==", + "dev": true, + "dependencies": { + "@babel/types": "^7.14.5" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-hoist-variables": { + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/helper-hoist-variables/-/helper-hoist-variables-7.14.5.tgz", + "integrity": "sha512-R1PXiz31Uc0Vxy4OEOm07x0oSjKAdPPCh3tPivn/Eo8cvz6gveAeuyUUPB21Hoiif0uoPQSSdhIPS3352nvdyQ==", + "dev": true, + "dependencies": { + "@babel/types": "^7.14.5" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-member-expression-to-functions": { + "version": "7.15.0", + "resolved": "https://registry.npmjs.org/@babel/helper-member-expression-to-functions/-/helper-member-expression-to-functions-7.15.0.tgz", + "integrity": "sha512-Jq8H8U2kYiafuj2xMTPQwkTBnEEdGKpT35lJEQsRRjnG0LW3neucsaMWLgKcwu3OHKNeYugfw+Z20BXBSEs2Lg==", + "dev": true, + "dependencies": { + "@babel/types": "^7.15.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-imports": { + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.14.5.tgz", + "integrity": "sha512-SwrNHu5QWS84XlHwGYPDtCxcA0hrSlL2yhWYLgeOc0w7ccOl2qv4s/nARI0aYZW+bSwAL5CukeXA47B/1NKcnQ==", + "dev": true, + "dependencies": { + "@babel/types": "^7.14.5" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-transforms": { + "version": "7.15.0", + "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.15.0.tgz", + "integrity": "sha512-RkGiW5Rer7fpXv9m1B3iHIFDZdItnO2/BLfWVW/9q7+KqQSDY5kUfQEbzdXM1MVhJGcugKV7kRrNVzNxmk7NBg==", + "dev": true, + "dependencies": { + "@babel/helper-module-imports": "^7.14.5", + "@babel/helper-replace-supers": "^7.15.0", + "@babel/helper-simple-access": "^7.14.8", + "@babel/helper-split-export-declaration": "^7.14.5", + "@babel/helper-validator-identifier": "^7.14.9", + "@babel/template": "^7.14.5", + "@babel/traverse": "^7.15.0", + "@babel/types": "^7.15.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-optimise-call-expression": { + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/helper-optimise-call-expression/-/helper-optimise-call-expression-7.14.5.tgz", + "integrity": "sha512-IqiLIrODUOdnPU9/F8ib1Fx2ohlgDhxnIDU7OEVi+kAbEZcyiF7BLU8W6PfvPi9LzztjS7kcbzbmL7oG8kD6VA==", + "dev": true, + "dependencies": { + "@babel/types": "^7.14.5" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-replace-supers": { + "version": "7.15.0", + "resolved": "https://registry.npmjs.org/@babel/helper-replace-supers/-/helper-replace-supers-7.15.0.tgz", + "integrity": "sha512-6O+eWrhx+HEra/uJnifCwhwMd6Bp5+ZfZeJwbqUTuqkhIT6YcRhiZCOOFChRypOIe0cV46kFrRBlm+t5vHCEaA==", + "dev": true, + "dependencies": { + "@babel/helper-member-expression-to-functions": "^7.15.0", + "@babel/helper-optimise-call-expression": "^7.14.5", + "@babel/traverse": "^7.15.0", + "@babel/types": "^7.15.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-simple-access": { + "version": "7.14.8", + "resolved": "https://registry.npmjs.org/@babel/helper-simple-access/-/helper-simple-access-7.14.8.tgz", + "integrity": "sha512-TrFN4RHh9gnWEU+s7JloIho2T76GPwRHhdzOWLqTrMnlas8T9O7ec+oEDNsRXndOmru9ymH9DFrEOxpzPoSbdg==", + "dev": true, + "dependencies": { + "@babel/types": "^7.14.8" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-split-export-declaration": { + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.14.5.tgz", + "integrity": "sha512-hprxVPu6e5Kdp2puZUmvOGjaLv9TCe58E/Fl6hRq4YiVQxIcNvuq6uTM2r1mT/oPskuS9CgR+I94sqAYv0NGKA==", + "dev": true, + "dependencies": { + "@babel/types": "^7.14.5" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.14.9", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.14.9.tgz", + "integrity": "sha512-pQYxPY0UP6IHISRitNe8bsijHex4TWZXi2HwKVsjPiltzlhse2znVcm9Ace510VT1kxIHjGJCZZQBX2gJDbo0g==", + "dev": true, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-option": { + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.14.5.tgz", + "integrity": "sha512-OX8D5eeX4XwcroVW45NMvoYaIuFI+GQpA2a8Gi+X/U/cDUIRsV37qQfF905F0htTRCREQIB4KqPeaveRJUl3Ow==", + "dev": true, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helpers": { + "version": "7.15.3", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.15.3.tgz", + "integrity": "sha512-HwJiz52XaS96lX+28Tnbu31VeFSQJGOeKHJeaEPQlTl7PnlhFElWPj8tUXtqFIzeN86XxXoBr+WFAyK2PPVz6g==", + "dev": true, + "dependencies": { + "@babel/template": "^7.14.5", + "@babel/traverse": "^7.15.0", + "@babel/types": "^7.15.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/highlight": { + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.14.5.tgz", + "integrity": "sha512-qf9u2WFWVV0MppaL877j2dBtQIDgmidgjGk5VIMw3OadXvYaXn66U1BFlH2t4+t3i+8PhedppRv+i40ABzd+gg==", + "dev": true, + "dependencies": { + "@babel/helper-validator-identifier": "^7.14.5", + "chalk": "^2.0.0", + "js-tokens": "^4.0.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/highlight/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dev": true, + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/@babel/highlight/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dev": true, + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/@babel/highlight/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dev": true, + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/@babel/highlight/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=", + "dev": true + }, + "node_modules/@babel/highlight/node_modules/escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=", + "dev": true, + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/@babel/highlight/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/@babel/highlight/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/@babel/parser": { + "version": "7.15.3", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.15.3.tgz", + "integrity": "sha512-O0L6v/HvqbdJawj0iBEfVQMc3/6WP+AeOsovsIgBFyJaG+W2w7eqvZB7puddATmWuARlm1SX7DwxJ/JJUnDpEA==", + "dev": true, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/template": { + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.14.5.tgz", + "integrity": "sha512-6Z3Po85sfxRGachLULUhOmvAaOo7xCvqGQtxINai2mEGPFm6pQ4z5QInFnUrRpfoSV60BnjyF5F3c+15fxFV1g==", + "dev": true, + "dependencies": { + "@babel/code-frame": "^7.14.5", + "@babel/parser": "^7.14.5", + "@babel/types": "^7.14.5" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/traverse": { + "version": "7.15.0", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.15.0.tgz", + "integrity": "sha512-392d8BN0C9eVxVWd8H6x9WfipgVH5IaIoLp23334Sc1vbKKWINnvwRpb4us0xtPaCumlwbTtIYNA0Dv/32sVFw==", + "dev": true, + "dependencies": { + "@babel/code-frame": "^7.14.5", + "@babel/generator": "^7.15.0", + "@babel/helper-function-name": "^7.14.5", + "@babel/helper-hoist-variables": "^7.14.5", + "@babel/helper-split-export-declaration": "^7.14.5", + "@babel/parser": "^7.15.0", + "@babel/types": "^7.15.0", + "debug": "^4.1.0", + "globals": "^11.1.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/traverse/node_modules/debug": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz", + "integrity": "sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw==", + "dev": true, + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + } + }, + "node_modules/@babel/traverse/node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + }, + "node_modules/@babel/types": { + "version": "7.15.0", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.15.0.tgz", + "integrity": "sha512-OBvfqnllOIdX4ojTHpwZbpvz4j3EWyjkZEdmjH0/cgsd6QOdSgU8rLSk6ard/pcW7rlmjdVSX/AWOaORR1uNOQ==", + "dev": true, + "dependencies": { + "@babel/helper-validator-identifier": "^7.14.9", + "to-fast-properties": "^2.0.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@eslint/eslintrc": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-0.2.2.tgz", + "integrity": "sha512-EfB5OHNYp1F4px/LI/FEnGylop7nOqkQ1LRzCM0KccA2U8tvV8w01KBv37LbO7nW4H+YhKyo2LcJhRwjjV17QQ==", + "dev": true, + "dependencies": { + "ajv": "^6.12.4", + "debug": "^4.1.1", + "espree": "^7.3.0", + "globals": "^12.1.0", + "ignore": "^4.0.6", + "import-fresh": "^3.2.1", + "js-yaml": "^3.13.1", + "lodash": "^4.17.19", + "minimatch": "^3.0.4", + "strip-json-comments": "^3.1.1" + }, + "engines": { + "node": "^10.12.0 || >=12.0.0" + } + }, + "node_modules/@eslint/eslintrc/node_modules/argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "dev": true, + "dependencies": { + "sprintf-js": "~1.0.2" + } + }, + "node_modules/@eslint/eslintrc/node_modules/debug": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz", + "integrity": "sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw==", + "dev": true, + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + } + }, + "node_modules/@eslint/eslintrc/node_modules/globals": { + "version": "12.4.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-12.4.0.tgz", + "integrity": "sha512-BWICuzzDvDoH54NHKCseDanAhE3CeDorgDL5MT6LMXXj2WCnd9UC2szdk4AWLfjdgNBCXLUanXYcpBBKOSWGwg==", + "dev": true, + "dependencies": { + "type-fest": "^0.8.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/@eslint/eslintrc/node_modules/js-yaml": { + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", + "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "dev": true, + "dependencies": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/@eslint/eslintrc/node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + }, + "node_modules/@eslint/eslintrc/node_modules/strip-json-comments": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", + "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/@istanbuljs/load-nyc-config": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz", + "integrity": "sha512-VjeHSlIzpv/NyD3N0YuHfXOPDIixcA1q2ZV98wsMqcYlPmv2n3Yb2lYP9XMElnaFVXg5A7YLTeLu6V84uQDjmQ==", + "dev": true, + "dependencies": { + "camelcase": "^5.3.1", + "find-up": "^4.1.0", + "get-package-type": "^0.1.0", + "js-yaml": "^3.13.1", + "resolve-from": "^5.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/@istanbuljs/load-nyc-config/node_modules/argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "dev": true, + "dependencies": { + "sprintf-js": "~1.0.2" + } + }, + "node_modules/@istanbuljs/load-nyc-config/node_modules/camelcase": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", + "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/@istanbuljs/load-nyc-config/node_modules/find-up": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "dev": true, + "dependencies": { + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/@istanbuljs/load-nyc-config/node_modules/js-yaml": { + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", + "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "dev": true, + "dependencies": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/@istanbuljs/load-nyc-config/node_modules/locate-path": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "dev": true, + "dependencies": { + "p-locate": "^4.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/@istanbuljs/load-nyc-config/node_modules/p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dev": true, + "dependencies": { + "p-try": "^2.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/@istanbuljs/load-nyc-config/node_modules/p-locate": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "dev": true, + "dependencies": { + "p-limit": "^2.2.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/@istanbuljs/schema": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/@istanbuljs/schema/-/schema-0.1.3.tgz", + "integrity": "sha512-ZXRY4jNvVgSVQ8DL3LTcakaAtXwTVUxE81hslsyD2AtoXW/wVob10HkOJ1X/pAlcI7D+2YoZKg5do8G/w6RYgA==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/@types/json5": { + "version": "0.0.29", + "resolved": "https://registry.npmjs.org/@types/json5/-/json5-0.0.29.tgz", + "integrity": "sha1-7ihweulOEdK4J7y+UnC86n8+ce4=", + "dev": true + }, + "node_modules/@ungap/promise-all-settled": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@ungap/promise-all-settled/-/promise-all-settled-1.1.2.tgz", + "integrity": "sha512-sL/cEvJWAnClXw0wHk85/2L0G6Sj8UB0Ctc1TEMbKSsmpRosqhwj9gWgFRZSrBr2f9tiXISwNhCPmlfqUqyb9Q==", + "dev": true + }, + "node_modules/abort-controller": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "dependencies": { + "event-target-shim": "^5.0.0" + }, + "engines": { + "node": ">=6.5" + } + }, + "node_modules/accepts": { + "version": "1.3.7", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.7.tgz", + "integrity": "sha512-Il80Qs2WjYlJIBNzNkK6KYqlVMTbZLXgHx2oT0pU/fjRHyEp+PEfEPY0R3WCwAGVOtauxh1hOxNgIf5bv7dQpA==", + "dependencies": { + "mime-types": "~2.1.24", + "negotiator": "0.6.2" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/acorn": { + "version": "7.4.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.1.tgz", + "integrity": "sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A==", + "dev": true, + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-jsx": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", + "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", + "dev": true + }, + "node_modules/aes-js": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/aes-js/-/aes-js-3.1.2.tgz", + "integrity": "sha512-e5pEa2kBnBOgR4Y/p20pskXI74UEz7de8ZGVo58asOtvSVG5YAbJeELPZxOmt+Bnz3rX753YKhfIn4X4l1PPRQ==" + }, + "node_modules/aggregate-error": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.1.0.tgz", + "integrity": "sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA==", + "dev": true, + "dependencies": { + "clean-stack": "^2.0.0", + "indent-string": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dev": true, + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, + "node_modules/ansi-colors": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.1.tgz", + "integrity": "sha512-JoX0apGbHaUJBNl6yF+p6JAFYZ666/hhCGKN5t9QFjbJQKUU/g8MNbFDbvfrgKXvI1QpZplPOnwIo99lX/AAmA==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/anymatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.2.tgz", + "integrity": "sha512-P43ePfOAIupkguHUycrc4qJ9kz8ZiuOUijaETwX7THt0Y/GNK7v0aa8rY816xWjZ7rJdA5XdMcpVFTKMq+RvWg==", + "dev": true, + "dependencies": { + "normalize-path": "^3.0.0", + "picomatch": "^2.0.4" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/append-transform": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/append-transform/-/append-transform-2.0.0.tgz", + "integrity": "sha512-7yeyCEurROLQJFv5Xj4lEGTy0borxepjFv1g22oAdqFu//SrAlDl1O1Nxx15SH1RoliUml6p8dwJW9jvZughhg==", + "dev": true, + "dependencies": { + "default-require-extensions": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/aproba": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz", + "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==" + }, + "node_modules/archy": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/archy/-/archy-1.0.0.tgz", + "integrity": "sha1-+cjBN1fMHde8N5rHeyxipcKGjEA=", + "dev": true + }, + "node_modules/are-we-there-yet": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.5.tgz", + "integrity": "sha512-5hYdAkZlcG8tOLujVDTgCT+uPX0VnpAH28gWsLfzpXYm7wP6mp5Q/gYyR7YQ0cKVJcXJnl3j2kpBan13PtQf6w==", + "dependencies": { + "delegates": "^1.0.0", + "readable-stream": "^2.0.6" + } + }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "dev": true + }, + "node_modules/array-flatten": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", + "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=" + }, + "node_modules/array-includes": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/array-includes/-/array-includes-3.1.3.tgz", + "integrity": "sha512-gcem1KlBU7c9rB+Rq8/3PPKsK2kjqeEBa3bD5kkQo4nYlOHQCJqIJFqBXDEfwaRuYTT4E+FxA9xez7Gf/e3Q7A==", + "dev": true, + "dependencies": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.3", + "es-abstract": "^1.18.0-next.2", + "get-intrinsic": "^1.1.1", + "is-string": "^1.0.5" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/array.prototype.flat": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/array.prototype.flat/-/array.prototype.flat-1.2.4.tgz", + "integrity": "sha512-4470Xi3GAPAjZqFcljX2xzckv1qeKPizoNkiS0+O4IoPR2ZNpcjE0pkhdihlDouK+x6QOast26B4Q/O9DJnwSg==", + "dev": true, + "dependencies": { + "call-bind": "^1.0.0", + "define-properties": "^1.1.3", + "es-abstract": "^1.18.0-next.1" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/array.prototype.flatmap": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/array.prototype.flatmap/-/array.prototype.flatmap-1.2.4.tgz", + "integrity": "sha512-r9Z0zYoxqHz60vvQbWEdXIEtCwHF0yxaWfno9qzXeNHvfyl3BZqygmGzb84dsubyaXLH4husF+NFgMSdpZhk2Q==", + "dev": true, + "dependencies": { + "call-bind": "^1.0.0", + "define-properties": "^1.1.3", + "es-abstract": "^1.18.0-next.1", + "function-bind": "^1.1.1" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/assertion-error": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-1.1.0.tgz", + "integrity": "sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw==", + "dev": true, + "engines": { + "node": "*" + } + }, + "node_modules/astral-regex": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/astral-regex/-/astral-regex-1.0.0.tgz", + "integrity": "sha512-+Ryf6g3BKoRc7jfp7ad8tM4TtMiaWvbF/1/sQcZPkkS7ag3D5nMBCe2UfOTONtAkaG0tO0ij3C5Lwmf1EiyjHg==", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/axios": { + "version": "0.21.1", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.1.tgz", + "integrity": "sha512-dKQiRHxGD9PPRIUNIWvZhPTPpl1rf/OxTYKsqKUDjBwYylTvV7SjSHJb9ratfyzM6wCdLCOYLzs73qpg5c4iGA==", + "dependencies": { + "follow-redirects": "^1.10.0" + } + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true + }, + "node_modules/base-x": { + "version": "3.0.8", + "resolved": "https://registry.npmjs.org/base-x/-/base-x-3.0.8.tgz", + "integrity": "sha512-Rl/1AWP4J/zRrk54hhlxH4drNxPJXYUaKffODVI53/dAsV4t9fBxyxYKAVPU1XBHxYwOWP9h9H0hM2MVw4YfJA==", + "dependencies": { + "safe-buffer": "^5.0.1" + } + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" + }, + "node_modules/basic-auth": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/basic-auth/-/basic-auth-2.0.1.tgz", + "integrity": "sha512-NF+epuEdnUYVlGuhaxbbq+dvJttwLnGY+YixlXlME5KpQ5W3CnXA5cVTneY3SPbPDRkcjMbifrwmFYcClgOZeg==", + "dependencies": { + "safe-buffer": "5.1.2" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/better-sqlite3": { + "version": "7.4.3", + "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-7.4.3.tgz", + "integrity": "sha512-07bKjClZg/f4KMVRkzWtoIvazVPcF1gsvVKVIXlxwleC2DxuIhnra3KCMlUT1rFeRYXXckot2a46UciF2d9KLw==", + "hasInstallScript": true, + "dependencies": { + "bindings": "^1.5.0", + "prebuild-install": "^6.0.1", + "tar": "^6.1.0" + } + }, + "node_modules/binary-extensions": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", + "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/bindings": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", + "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", + "dependencies": { + "file-uri-to-path": "1.0.0" + } + }, + "node_modules/bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "dependencies": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, + "node_modules/bl/node_modules/readable-stream": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", + "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/bn.js": { + "version": "4.11.9", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.9.tgz", + "integrity": "sha512-E6QoYqCKZfgatHTdHzs1RRKP7ip4vvm+EyRUeE2RF0NblwVvb0p6jSVeNTOFxPn26QXN2o6SMfNxKp6kU8zQaw==" + }, + "node_modules/body-parser": { + "version": "1.19.0", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.0.tgz", + "integrity": "sha512-dhEPs72UPbDnAQJ9ZKMNTP6ptJaionhP5cBb541nXPlW60Jepo9RV/a4fX4XWW9CuFNK22krhrj1+rgzifNCsw==", + "dependencies": { + "bytes": "3.1.0", + "content-type": "~1.0.4", + "debug": "2.6.9", + "depd": "~1.1.2", + "http-errors": "1.7.2", + "iconv-lite": "0.4.24", + "on-finished": "~2.3.0", + "qs": "6.7.0", + "raw-body": "2.4.0", + "type-is": "~1.6.17" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/braces": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", + "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", + "dev": true, + "dependencies": { + "fill-range": "^7.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/brorand": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/brorand/-/brorand-1.1.0.tgz", + "integrity": "sha1-EsJe/kCkXjwyPrhnWgoM5XsiNx8=" + }, + "node_modules/browser-stdout": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/browser-stdout/-/browser-stdout-1.3.1.tgz", + "integrity": "sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw==", + "dev": true + }, + "node_modules/browserslist": { + "version": "4.16.8", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.16.8.tgz", + "integrity": "sha512-sc2m9ohR/49sWEbPj14ZSSZqp+kbi16aLao42Hmn3Z8FpjuMaq2xCA2l4zl9ITfyzvnvyE0hcg62YkIGKxgaNQ==", + "dev": true, + "dependencies": { + "caniuse-lite": "^1.0.30001251", + "colorette": "^1.3.0", + "electron-to-chromium": "^1.3.811", + "escalade": "^3.1.1", + "node-releases": "^1.1.75" + }, + "bin": { + "browserslist": "cli.js" + }, + "engines": { + "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" + } + }, + "node_modules/bs58": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/bs58/-/bs58-4.0.1.tgz", + "integrity": "sha1-vhYedsNU9veIrkBx9j806MTwpCo=", + "dependencies": { + "base-x": "^3.0.2" + } + }, + "node_modules/bsv": { + "version": "1.5.6", + "resolved": "https://registry.npmjs.org/bsv/-/bsv-1.5.6.tgz", + "integrity": "sha512-A0g36x63lVF9Ia6/z/RjcxaQMHE5cLl2rDxjUIKz0UTMLf5bPPyLI9yVyY2JkecF77MrU+MQdKVt0MSdU5abtw==", + "dependencies": { + "aes-js": "^3.1.2", + "bn.js": "=4.11.9", + "bs58": "=4.0.1", + "clone-deep": "^4.0.1", + "elliptic": "6.5.4", + "hash.js": "^1.1.7", + "inherits": "2.0.3", + "unorm": "1.4.1" + } + }, + "node_modules/bsv/node_modules/inherits": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", + "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" + }, + "node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "node_modules/bytes": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz", + "integrity": "sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg==", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/caching-transform": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/caching-transform/-/caching-transform-4.0.0.tgz", + "integrity": "sha512-kpqOvwXnjjN44D89K5ccQC+RUrsy7jB/XLlRrx0D7/2HNcTPqzsb6XgYoErwko6QsV184CA2YgS1fxDiiDZMWA==", + "dev": true, + "dependencies": { + "hasha": "^5.0.0", + "make-dir": "^3.0.0", + "package-hash": "^4.0.0", + "write-file-atomic": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/call-bind": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", + "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==", + "dev": true, + "dependencies": { + "function-bind": "^1.1.1", + "get-intrinsic": "^1.0.2" + } + }, + "node_modules/callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/camelcase": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.2.0.tgz", + "integrity": "sha512-c7wVvbw3f37nuobQNtgsgG9POC9qMbNuMQmTCqZv23b6MIz0fcYpBiOlv9gEN/hdLdnZTDQhg6e9Dq5M1vKvfg==", + "dev": true, + "engines": { + "node": ">=10" + } + }, + "node_modules/caniuse-lite": { + "version": "1.0.30001252", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001252.tgz", + "integrity": "sha512-I56jhWDGMtdILQORdusxBOH+Nl/KgQSdDmpJezYddnAkVOmnoU8zwjTV9xAjMIYxr0iPreEAVylCGcmHCjfaOw==", + "dev": true + }, + "node_modules/chai": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/chai/-/chai-4.3.4.tgz", + "integrity": "sha512-yS5H68VYOCtN1cjfwumDSuzn/9c+yza4f3reKXlE5rUg7SFcCEy90gJvydNgOYtblyf4Zi6jIWRnXOgErta0KA==", + "dev": true, + "dependencies": { + "assertion-error": "^1.1.0", + "check-error": "^1.0.2", + "deep-eql": "^3.0.1", + "get-func-name": "^2.0.0", + "pathval": "^1.1.1", + "type-detect": "^4.0.5" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/chai-as-promised": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/chai-as-promised/-/chai-as-promised-7.1.1.tgz", + "integrity": "sha512-azL6xMoi+uxu6z4rhWQ1jbdUhOMhis2PvscD/xjLqNMkv3BPPp2JyyuTHOrf9BOosGpNQ11v6BKv/g57RXbiaA==", + "dev": true, + "dependencies": { + "check-error": "^1.0.2" + } + }, + "node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/chalk/node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/check-error": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/check-error/-/check-error-1.0.2.tgz", + "integrity": "sha1-V00xLt2Iu13YkS6Sht1sCu1KrII=", + "dev": true, + "engines": { + "node": "*" + } + }, + "node_modules/chokidar": { + "version": "3.5.1", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.1.tgz", + "integrity": "sha512-9+s+Od+W0VJJzawDma/gvBNQqkTiqYTWLuZoyAsivsI4AaWTCzHG06/TMjsf1cYe9Cb97UCEhjz7HvnPk2p/tw==", + "dev": true, + "dependencies": { + "anymatch": "~3.1.1", + "braces": "~3.0.2", + "glob-parent": "~5.1.0", + "is-binary-path": "~2.1.0", + "is-glob": "~4.0.1", + "normalize-path": "~3.0.0", + "readdirp": "~3.5.0" + }, + "engines": { + "node": ">= 8.10.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.1" + } + }, + "node_modules/chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==" + }, + "node_modules/clean-stack": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-2.2.0.tgz", + "integrity": "sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/cliui": { + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", + "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==", + "dev": true, + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.0", + "wrap-ansi": "^7.0.0" + } + }, + "node_modules/cliui/node_modules/ansi-regex": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", + "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/cliui/node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/cliui/node_modules/string-width": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.2.tgz", + "integrity": "sha512-XBJbT3N4JhVumXE0eoLU9DCjcaF92KLNqTmFCnG1pf8duUxFGwtP6AD6nkjw9a3IdiRtL3E2w3JDiE/xi3vOeA==", + "dev": true, + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/cliui/node_modules/strip-ansi": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", + "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==", + "dev": true, + "dependencies": { + "ansi-regex": "^5.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/clone-deep": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/clone-deep/-/clone-deep-4.0.1.tgz", + "integrity": "sha512-neHB9xuzh/wk0dIHweyAXv2aPGZIVk3pLMe+/RNzINf17fe0OG96QroktYAUm7SM1PBnzTabaLboqqxDyMU+SQ==", + "dependencies": { + "is-plain-object": "^2.0.4", + "kind-of": "^6.0.2", + "shallow-clone": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/code-point-at": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", + "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "node_modules/colorette": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/colorette/-/colorette-1.3.0.tgz", + "integrity": "sha512-ecORCqbSFP7Wm8Y6lyqMJjexBQqXSF7SSeaTyGGphogUjBlFP9m9o08wy86HL2uB7fMTxtOUzLMk7ogKcxMg1w==", + "dev": true + }, + "node_modules/commondir": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/commondir/-/commondir-1.0.1.tgz", + "integrity": "sha1-3dgA2gxmEnOTzKWVDqloo6rxJTs=", + "dev": true + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", + "dev": true + }, + "node_modules/console-control-strings": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz", + "integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4=" + }, + "node_modules/contains-path": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/contains-path/-/contains-path-0.1.0.tgz", + "integrity": "sha1-/ozxhP9mcLa67wGp1IYaXL7EEgo=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/content-disposition": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.3.tgz", + "integrity": "sha512-ExO0774ikEObIAEV9kDo50o+79VCUdEB6n6lzKgGwupcVeRlhrj3qGAfwq8G6uBJjkqLrhT0qEYFcWng8z1z0g==", + "dependencies": { + "safe-buffer": "5.1.2" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/content-type": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz", + "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/convert-source-map": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.8.0.tgz", + "integrity": "sha512-+OQdjP49zViI/6i7nIJpA8rAl4sV/JdPfU9nZs3VqOwGIgizICvuN2ru6fMd+4llL0tar18UYJXfZ/TWtmhUjA==", + "dev": true, + "dependencies": { + "safe-buffer": "~5.1.1" + } + }, + "node_modules/cookie": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.0.tgz", + "integrity": "sha512-+Hp8fLp57wnUSt0tY0tHEXh4voZRDnoIrZPqlo3DPiI4y9lwg/jqx+1Om94/W6ZaPDOUbnjOt/99w66zk+l1Xg==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", + "integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw=" + }, + "node_modules/core-util-is": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", + "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" + }, + "node_modules/cors": { + "version": "2.8.5", + "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.5.tgz", + "integrity": "sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g==", + "dependencies": { + "object-assign": "^4", + "vary": "^1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", + "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "dev": true, + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/decamelize": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-4.0.0.tgz", + "integrity": "sha512-9iE1PgSik9HeIIw2JO94IidnE3eBoQrFJ3w7sFuzSX4DpmZ3v5sZpUiV5Swcf6mQEF+Y0ru8Neo+p+nyh2J+hQ==", + "dev": true, + "engines": { + "node": ">=10" + } + }, + "node_modules/decompress-response": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-4.2.1.tgz", + "integrity": "sha512-jOSne2qbyE+/r8G1VU+G/82LBs2Fs4LAsTiLSHOCOMZQl2OKZ6i8i4IyHemTe+/yIXOtTcRQMzPcgyhoFlqPkw==", + "dependencies": { + "mimic-response": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/deep-eql": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-3.0.1.tgz", + "integrity": "sha512-+QeIQyN5ZuO+3Uk5DYh6/1eKO0m0YmJFGNmFHGACpf1ClL1nmlV/p4gNgbl2pJGxgXb4faqo6UE+M5ACEMyVcw==", + "dev": true, + "dependencies": { + "type-detect": "^4.0.0" + }, + "engines": { + "node": ">=0.12" + } + }, + "node_modules/deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/deep-is": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz", + "integrity": "sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ=", + "dev": true + }, + "node_modules/default-require-extensions": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/default-require-extensions/-/default-require-extensions-3.0.0.tgz", + "integrity": "sha512-ek6DpXq/SCpvjhpFsLFRVtIxJCRw6fUR42lYMVZuUMK7n8eMz4Uh5clckdBjEpLhn/gEBZo7hDJnJcwdKLKQjg==", + "dev": true, + "dependencies": { + "strip-bom": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/define-properties": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz", + "integrity": "sha512-3MqfYKj2lLzdMSf8ZIZE/V+Zuy+BgD6f164e8K2w7dgnpKArBDerGYpM46IYYcjnkdPNMjPk9A6VFB8+3SKlXQ==", + "dev": true, + "dependencies": { + "object-keys": "^1.0.12" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/delegates": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", + "integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=" + }, + "node_modules/depd": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", + "integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak=", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/destroy": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz", + "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=" + }, + "node_modules/detect-libc": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-1.0.3.tgz", + "integrity": "sha1-+hN8S9aY7fVc1c0CrFWfkaTEups=", + "bin": { + "detect-libc": "bin/detect-libc.js" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/diff": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/diff/-/diff-5.0.0.tgz", + "integrity": "sha512-/VTCrvm5Z0JGty/BWHljh+BAiw3IK+2j87NGMu8Nwc/f48WoDAC395uomO9ZD117ZOBaHmkX1oyLvkVM/aIT3w==", + "dev": true, + "engines": { + "node": ">=0.3.1" + } + }, + "node_modules/doctrine": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", + "integrity": "sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==", + "dev": true, + "dependencies": { + "esutils": "^2.0.2" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/dotenv": { + "version": "8.6.0", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-8.6.0.tgz", + "integrity": "sha512-IrPdXQsk2BbzvCBGBOTmmSH5SodmqZNt4ERAZDmW4CT+tL8VtvinqywuANaFu4bOMWki16nqf0e4oC0QIaDr/g==", + "engines": { + "node": ">=10" + } + }, + "node_modules/duplexer": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/duplexer/-/duplexer-0.1.2.tgz", + "integrity": "sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg==" + }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" + }, + "node_modules/electron-to-chromium": { + "version": "1.3.818", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.3.818.tgz", + "integrity": "sha512-c/Z9gIr+jDZAR9q+mn40hEc1NharBT+8ejkarjbCDnBNFviI6hvcC5j2ezkAXru//bTnQp5n6iPi0JA83Tla1Q==", + "dev": true + }, + "node_modules/elliptic": { + "version": "6.5.4", + "resolved": "https://registry.npmjs.org/elliptic/-/elliptic-6.5.4.tgz", + "integrity": "sha512-iLhC6ULemrljPZb+QutR5TQGB+pdW6KGD5RSegS+8sorOZT+rdQFbsQFJgvN3eRqNALqJer4oQ16YvJHlU8hzQ==", + "dependencies": { + "bn.js": "^4.11.9", + "brorand": "^1.1.0", + "hash.js": "^1.0.0", + "hmac-drbg": "^1.0.1", + "inherits": "^2.0.4", + "minimalistic-assert": "^1.0.1", + "minimalistic-crypto-utils": "^1.0.1" + } + }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true + }, + "node_modules/encodeurl": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", + "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/end-of-stream": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", + "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "dependencies": { + "once": "^1.4.0" + } + }, + "node_modules/enquirer": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/enquirer/-/enquirer-2.3.6.tgz", + "integrity": "sha512-yjNnPr315/FjS4zIsUxYguYUPP2e1NK4d7E7ZOLiyYCcbFBiTMyID+2wvm2w6+pZ/odMA7cRkjhsPbltwBOrLg==", + "dev": true, + "dependencies": { + "ansi-colors": "^4.1.1" + }, + "engines": { + "node": ">=8.6" + } + }, + "node_modules/error-ex": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", + "integrity": "sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==", + "dev": true, + "dependencies": { + "is-arrayish": "^0.2.1" + } + }, + "node_modules/es-abstract": { + "version": "1.18.5", + "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.18.5.tgz", + "integrity": "sha512-DDggyJLoS91CkJjgauM5c0yZMjiD1uK3KcaCeAmffGwZ+ODWzOkPN4QwRbsK5DOFf06fywmyLci3ZD8jLGhVYA==", + "dev": true, + "dependencies": { + "call-bind": "^1.0.2", + "es-to-primitive": "^1.2.1", + "function-bind": "^1.1.1", + "get-intrinsic": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.2", + "internal-slot": "^1.0.3", + "is-callable": "^1.2.3", + "is-negative-zero": "^2.0.1", + "is-regex": "^1.1.3", + "is-string": "^1.0.6", + "object-inspect": "^1.11.0", + "object-keys": "^1.1.1", + "object.assign": "^4.1.2", + "string.prototype.trimend": "^1.0.4", + "string.prototype.trimstart": "^1.0.4", + "unbox-primitive": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-to-primitive": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.1.tgz", + "integrity": "sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==", + "dev": true, + "dependencies": { + "is-callable": "^1.1.4", + "is-date-object": "^1.0.1", + "is-symbol": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es6-error": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/es6-error/-/es6-error-4.1.1.tgz", + "integrity": "sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==", + "dev": true + }, + "node_modules/escalade": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", + "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg=" + }, + "node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "dev": true, + "engines": { + "node": ">=10" + } + }, + "node_modules/eslint": { + "version": "7.13.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-7.13.0.tgz", + "integrity": "sha512-uCORMuOO8tUzJmsdRtrvcGq5qposf7Rw0LwkTJkoDbOycVQtQjmnhZSuLQnozLE4TmAzlMVV45eCHmQ1OpDKUQ==", + "dev": true, + "dependencies": { + "@babel/code-frame": "^7.0.0", + "@eslint/eslintrc": "^0.2.1", + "ajv": "^6.10.0", + "chalk": "^4.0.0", + "cross-spawn": "^7.0.2", + "debug": "^4.0.1", + "doctrine": "^3.0.0", + "enquirer": "^2.3.5", + "eslint-scope": "^5.1.1", + "eslint-utils": "^2.1.0", + "eslint-visitor-keys": "^2.0.0", + "espree": "^7.3.0", + "esquery": "^1.2.0", + "esutils": "^2.0.2", + "file-entry-cache": "^5.0.1", + "functional-red-black-tree": "^1.0.1", + "glob-parent": "^5.0.0", + "globals": "^12.1.0", + "ignore": "^4.0.6", + "import-fresh": "^3.0.0", + "imurmurhash": "^0.1.4", + "is-glob": "^4.0.0", + "js-yaml": "^3.13.1", + "json-stable-stringify-without-jsonify": "^1.0.1", + "levn": "^0.4.1", + "lodash": "^4.17.19", + "minimatch": "^3.0.4", + "natural-compare": "^1.4.0", + "optionator": "^0.9.1", + "progress": "^2.0.0", + "regexpp": "^3.1.0", + "semver": "^7.2.1", + "strip-ansi": "^6.0.0", + "strip-json-comments": "^3.1.0", + "table": "^5.2.3", + "text-table": "^0.2.0", + "v8-compile-cache": "^2.0.3" + }, + "bin": { + "eslint": "bin/eslint.js" + }, + "engines": { + "node": "^10.12.0 || >=12.0.0" + } + }, + "node_modules/eslint-config-standard": { + "version": "16.0.2", + "resolved": "https://registry.npmjs.org/eslint-config-standard/-/eslint-config-standard-16.0.2.tgz", + "integrity": "sha512-fx3f1rJDsl9bY7qzyX8SAtP8GBSk6MfXFaTfaGgk12aAYW4gJSyRm7dM790L6cbXv63fvjY4XeSzXnb4WM+SKw==", + "dev": true + }, + "node_modules/eslint-config-standard-jsx": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/eslint-config-standard-jsx/-/eslint-config-standard-jsx-10.0.0.tgz", + "integrity": "sha512-hLeA2f5e06W1xyr/93/QJulN/rLbUVUmqTlexv9PRKHFwEC9ffJcH2LvJhMoEqYQBEYafedgGZXH2W8NUpt5lA==", + "dev": true + }, + "node_modules/eslint-import-resolver-node": { + "version": "0.3.6", + "resolved": "https://registry.npmjs.org/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.6.tgz", + "integrity": "sha512-0En0w03NRVMn9Uiyn8YRPDKvWjxCWkslUEhGNTdGx15RvPJYQ+lbOlqrlNI2vEAs4pDYK4f/HN2TbDmk5TP0iw==", + "dev": true, + "dependencies": { + "debug": "^3.2.7", + "resolve": "^1.20.0" + } + }, + "node_modules/eslint-import-resolver-node/node_modules/debug": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "dev": true, + "dependencies": { + "ms": "^2.1.1" + } + }, + "node_modules/eslint-import-resolver-node/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true + }, + "node_modules/eslint-module-utils": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/eslint-module-utils/-/eslint-module-utils-2.6.2.tgz", + "integrity": "sha512-QG8pcgThYOuqxupd06oYTZoNOGaUdTY1PqK+oS6ElF6vs4pBdk/aYxFVQQXzcrAqp9m7cl7lb2ubazX+g16k2Q==", + "dev": true, + "dependencies": { + "debug": "^3.2.7", + "pkg-dir": "^2.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/eslint-module-utils/node_modules/debug": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "dev": true, + "dependencies": { + "ms": "^2.1.1" + } + }, + "node_modules/eslint-module-utils/node_modules/find-up": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-2.1.0.tgz", + "integrity": "sha1-RdG35QbHF93UgndaK3eSCjwMV6c=", + "dev": true, + "dependencies": { + "locate-path": "^2.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/eslint-module-utils/node_modules/locate-path": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-2.0.0.tgz", + "integrity": "sha1-K1aLJl7slExtnA3pw9u7ygNUzY4=", + "dev": true, + "dependencies": { + "p-locate": "^2.0.0", + "path-exists": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/eslint-module-utils/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true + }, + "node_modules/eslint-module-utils/node_modules/p-limit": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-1.3.0.tgz", + "integrity": "sha512-vvcXsLAJ9Dr5rQOPk7toZQZJApBl2K4J6dANSsEuh6QI41JYcsS/qhTGa9ErIUUgK3WNQoJYvylxvjqmiqEA9Q==", + "dev": true, + "dependencies": { + "p-try": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/eslint-module-utils/node_modules/p-locate": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-2.0.0.tgz", + "integrity": "sha1-IKAQOyIqcMj9OcwuWAaA893l7EM=", + "dev": true, + "dependencies": { + "p-limit": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/eslint-module-utils/node_modules/p-try": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-1.0.0.tgz", + "integrity": "sha1-y8ec26+P1CKOE/Yh8rGiN8GyB7M=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/eslint-module-utils/node_modules/path-exists": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", + "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/eslint-module-utils/node_modules/pkg-dir": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-2.0.0.tgz", + "integrity": "sha1-9tXREJ4Z1j7fQo4L1X4Sd3YVM0s=", + "dev": true, + "dependencies": { + "find-up": "^2.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/eslint-plugin-es": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/eslint-plugin-es/-/eslint-plugin-es-3.0.1.tgz", + "integrity": "sha512-GUmAsJaN4Fc7Gbtl8uOBlayo2DqhwWvEzykMHSCZHU3XdJ+NSzzZcVhXh3VxX5icqQ+oQdIEawXX8xkR3mIFmQ==", + "dev": true, + "dependencies": { + "eslint-utils": "^2.0.0", + "regexpp": "^3.0.0" + }, + "engines": { + "node": ">=8.10.0" + } + }, + "node_modules/eslint-plugin-import": { + "version": "2.22.1", + "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.22.1.tgz", + "integrity": "sha512-8K7JjINHOpH64ozkAhpT3sd+FswIZTfMZTjdx052pnWrgRCVfp8op9tbjpAk3DdUeI/Ba4C8OjdC0r90erHEOw==", + "dev": true, + "dependencies": { + "array-includes": "^3.1.1", + "array.prototype.flat": "^1.2.3", + "contains-path": "^0.1.0", + "debug": "^2.6.9", + "doctrine": "1.5.0", + "eslint-import-resolver-node": "^0.3.4", + "eslint-module-utils": "^2.6.0", + "has": "^1.0.3", + "minimatch": "^3.0.4", + "object.values": "^1.1.1", + "read-pkg-up": "^2.0.0", + "resolve": "^1.17.0", + "tsconfig-paths": "^3.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/eslint-plugin-import/node_modules/doctrine": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-1.5.0.tgz", + "integrity": "sha1-N53Ocw9hZvds76TmcHoVmwLFpvo=", + "dev": true, + "dependencies": { + "esutils": "^2.0.2", + "isarray": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/eslint-plugin-node": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-node/-/eslint-plugin-node-11.1.0.tgz", + "integrity": "sha512-oUwtPJ1W0SKD0Tr+wqu92c5xuCeQqB3hSCHasn/ZgjFdA9iDGNkNf2Zi9ztY7X+hNuMib23LNGRm6+uN+KLE3g==", + "dev": true, + "dependencies": { + "eslint-plugin-es": "^3.0.0", + "eslint-utils": "^2.0.0", + "ignore": "^5.1.1", + "minimatch": "^3.0.4", + "resolve": "^1.10.1", + "semver": "^6.1.0" + }, + "engines": { + "node": ">=8.10.0" + } + }, + "node_modules/eslint-plugin-node/node_modules/ignore": { + "version": "5.1.8", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.1.8.tgz", + "integrity": "sha512-BMpfD7PpiETpBl/A6S498BaIJ6Y/ABT93ETbby2fP00v4EbvPBXWEoaR1UBPKs3iR53pJY7EtZk5KACI57i1Uw==", + "dev": true, + "engines": { + "node": ">= 4" + } + }, + "node_modules/eslint-plugin-node/node_modules/semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "dev": true, + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/eslint-plugin-promise": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/eslint-plugin-promise/-/eslint-plugin-promise-4.2.1.tgz", + "integrity": "sha512-VoM09vT7bfA7D+upt+FjeBO5eHIJQBUWki1aPvB+vbNiHS3+oGIJGIeyBtKQTME6UPXXy3vV07OL1tHd3ANuDw==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/eslint-plugin-react": { + "version": "7.21.5", + "resolved": "https://registry.npmjs.org/eslint-plugin-react/-/eslint-plugin-react-7.21.5.tgz", + "integrity": "sha512-8MaEggC2et0wSF6bUeywF7qQ46ER81irOdWS4QWxnnlAEsnzeBevk1sWh7fhpCghPpXb+8Ks7hvaft6L/xsR6g==", + "dev": true, + "dependencies": { + "array-includes": "^3.1.1", + "array.prototype.flatmap": "^1.2.3", + "doctrine": "^2.1.0", + "has": "^1.0.3", + "jsx-ast-utils": "^2.4.1 || ^3.0.0", + "object.entries": "^1.1.2", + "object.fromentries": "^2.0.2", + "object.values": "^1.1.1", + "prop-types": "^15.7.2", + "resolve": "^1.18.1", + "string.prototype.matchall": "^4.0.2" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/eslint-plugin-react/node_modules/doctrine": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-2.1.0.tgz", + "integrity": "sha512-35mSku4ZXK0vfCuHEDAwt55dg2jNajHZ1odvF+8SSr82EsZY4QmXfuWso8oEd8zRhVObSN18aM0CjSdoBX7zIw==", + "dev": true, + "dependencies": { + "esutils": "^2.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/eslint-scope": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-5.1.1.tgz", + "integrity": "sha512-2NxwbF/hZ0KpepYN0cNbo+FN6XoK7GaHlQhgx/hIZl6Va0bF45RQOOwhLIy8lQDbuCiadSLCBnH2CFYquit5bw==", + "dev": true, + "dependencies": { + "esrecurse": "^4.3.0", + "estraverse": "^4.1.1" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/eslint-utils": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/eslint-utils/-/eslint-utils-2.1.0.tgz", + "integrity": "sha512-w94dQYoauyvlDc43XnGB8lU3Zt713vNChgt4EWwhXAP2XkBvndfxF0AgIqKOOasjPIPzj9JqgwkwbCYD0/V3Zg==", + "dev": true, + "dependencies": { + "eslint-visitor-keys": "^1.1.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/eslint-utils/node_modules/eslint-visitor-keys": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz", + "integrity": "sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/eslint-visitor-keys": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-2.1.0.tgz", + "integrity": "sha512-0rSmRBzXgDzIsD6mGdJgevzgezI534Cer5L/vyMX0kHzT/jiB43jRhd9YUlMGYLQy2zprNmoT8qasCGtY+QaKw==", + "dev": true, + "engines": { + "node": ">=10" + } + }, + "node_modules/eslint/node_modules/ansi-regex": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", + "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/eslint/node_modules/argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "dev": true, + "dependencies": { + "sprintf-js": "~1.0.2" + } + }, + "node_modules/eslint/node_modules/debug": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz", + "integrity": "sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw==", + "dev": true, + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + } + }, + "node_modules/eslint/node_modules/globals": { + "version": "12.4.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-12.4.0.tgz", + "integrity": "sha512-BWICuzzDvDoH54NHKCseDanAhE3CeDorgDL5MT6LMXXj2WCnd9UC2szdk4AWLfjdgNBCXLUanXYcpBBKOSWGwg==", + "dev": true, + "dependencies": { + "type-fest": "^0.8.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/eslint/node_modules/js-yaml": { + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", + "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "dev": true, + "dependencies": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/eslint/node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + }, + "node_modules/eslint/node_modules/semver": { + "version": "7.3.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz", + "integrity": "sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ==", + "dev": true, + "dependencies": { + "lru-cache": "^6.0.0" + }, + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/eslint/node_modules/strip-ansi": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", + "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==", + "dev": true, + "dependencies": { + "ansi-regex": "^5.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/eslint/node_modules/strip-json-comments": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", + "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/espree": { + "version": "7.3.1", + "resolved": "https://registry.npmjs.org/espree/-/espree-7.3.1.tgz", + "integrity": "sha512-v3JCNCE64umkFpmkFGqzVKsOT0tN1Zr+ueqLZfpV1Ob8e+CEgPWa+OxCoGH3tnhimMKIaBm4m/vaRpJ/krRz2g==", + "dev": true, + "dependencies": { + "acorn": "^7.4.0", + "acorn-jsx": "^5.3.1", + "eslint-visitor-keys": "^1.3.0" + }, + "engines": { + "node": "^10.12.0 || >=12.0.0" + } + }, + "node_modules/espree/node_modules/eslint-visitor-keys": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz", + "integrity": "sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "dev": true, + "bin": { + "esparse": "bin/esparse.js", + "esvalidate": "bin/esvalidate.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/esquery": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.4.0.tgz", + "integrity": "sha512-cCDispWt5vHHtwMY2YrAQ4ibFkAL8RbH5YGBnZBc90MolvvfkkQcJro/aZiAQUlQ3qgrYS6D6v8Gc5G5CQsc9w==", + "dev": true, + "dependencies": { + "estraverse": "^5.1.0" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/esquery/node_modules/estraverse": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.2.0.tgz", + "integrity": "sha512-BxbNGGNm0RyRYvUdHpIwv9IWzeM9XClbOxwoATuFdOE7ZE6wHL+HQ5T8hoPM+zHvmKzzsEqhgy0GrQ5X13afiQ==", + "dev": true, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/esrecurse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", + "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", + "dev": true, + "dependencies": { + "estraverse": "^5.2.0" + }, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/esrecurse/node_modules/estraverse": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.2.0.tgz", + "integrity": "sha512-BxbNGGNm0RyRYvUdHpIwv9IWzeM9XClbOxwoATuFdOE7ZE6wHL+HQ5T8hoPM+zHvmKzzsEqhgy0GrQ5X13afiQ==", + "dev": true, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/estraverse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.3.0.tgz", + "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==", + "dev": true, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/event-stream": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/event-stream/-/event-stream-4.0.1.tgz", + "integrity": "sha512-qACXdu/9VHPBzcyhdOWR5/IahhGMf0roTeZJfzz077GwylcDd90yOHLouhmv7GJ5XzPi6ekaQWd8AvPP2nOvpA==", + "dependencies": { + "duplexer": "^0.1.1", + "from": "^0.1.7", + "map-stream": "0.0.7", + "pause-stream": "^0.0.11", + "split": "^1.0.1", + "stream-combiner": "^0.2.2", + "through": "^2.3.8" + } + }, + "node_modules/event-target-shim": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", + "engines": { + "node": ">=6" + } + }, + "node_modules/eventsource": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/eventsource/-/eventsource-1.1.0.tgz", + "integrity": "sha512-VSJjT5oCNrFvCS6igjzPAt5hBzQ2qPBFIbJ03zLI9SE0mxwZpMw6BfJrbFHm1a141AavMEB8JHmBhWAd66PfCg==", + "dependencies": { + "original": "^1.0.0" + }, + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/expand-template": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", + "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==", + "engines": { + "node": ">=6" + } + }, + "node_modules/express": { + "version": "4.17.1", + "resolved": "https://registry.npmjs.org/express/-/express-4.17.1.tgz", + "integrity": "sha512-mHJ9O79RqluphRrcw2X/GTh3k9tVv8YcoyY4Kkh4WDMUYKRZUq0h1o0w2rrrxBqM7VoeUVqgb27xlEMXTnYt4g==", + "dependencies": { + "accepts": "~1.3.7", + "array-flatten": "1.1.1", + "body-parser": "1.19.0", + "content-disposition": "0.5.3", + "content-type": "~1.0.4", + "cookie": "0.4.0", + "cookie-signature": "1.0.6", + "debug": "2.6.9", + "depd": "~1.1.2", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "finalhandler": "~1.1.2", + "fresh": "0.5.2", + "merge-descriptors": "1.0.1", + "methods": "~1.1.2", + "on-finished": "~2.3.0", + "parseurl": "~1.3.3", + "path-to-regexp": "0.1.7", + "proxy-addr": "~2.0.5", + "qs": "6.7.0", + "range-parser": "~1.2.1", + "safe-buffer": "5.1.2", + "send": "0.17.1", + "serve-static": "1.14.1", + "setprototypeof": "1.1.1", + "statuses": "~1.5.0", + "type-is": "~1.6.18", + "utils-merge": "1.0.1", + "vary": "~1.1.2" + }, + "engines": { + "node": ">= 0.10.0" + } + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "dev": true + }, + "node_modules/fast-json-stable-stringify": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", + "dev": true + }, + "node_modules/fast-levenshtein": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", + "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=", + "dev": true + }, + "node_modules/file-entry-cache": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-5.0.1.tgz", + "integrity": "sha512-bCg29ictuBaKUwwArK4ouCaqDgLZcysCFLmM/Yn/FDoqndh/9vNuQfXRDvTuXKLxfD/JtZQGKFT8MGcJBK644g==", + "dev": true, + "dependencies": { + "flat-cache": "^2.0.1" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/file-uri-to-path": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", + "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==" + }, + "node_modules/fill-range": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", + "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "dev": true, + "dependencies": { + "to-regex-range": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/finalhandler": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.2.tgz", + "integrity": "sha512-aAWcW57uxVNrQZqFXjITpW3sIUQmHGG3qSb9mUah9MgMC4NeWhNOlNjXEYq3HjRAvL6arUviZGGJsBg6z0zsWA==", + "dependencies": { + "debug": "2.6.9", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "on-finished": "~2.3.0", + "parseurl": "~1.3.3", + "statuses": "~1.5.0", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/find-cache-dir": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/find-cache-dir/-/find-cache-dir-3.3.1.tgz", + "integrity": "sha512-t2GDMt3oGC/v+BMwzmllWDuJF/xcDtE5j/fCGbqDD7OLuJkj0cfh1YSA5VKPvwMeLFLNDBkwOKZ2X85jGLVftQ==", + "dev": true, + "dependencies": { + "commondir": "^1.0.1", + "make-dir": "^3.0.2", + "pkg-dir": "^4.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/find-up": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", + "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", + "dev": true, + "dependencies": { + "locate-path": "^6.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/flat": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/flat/-/flat-5.0.2.tgz", + "integrity": "sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ==", + "dev": true, + "bin": { + "flat": "cli.js" + } + }, + "node_modules/flat-cache": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-2.0.1.tgz", + "integrity": "sha512-LoQe6yDuUMDzQAEH8sgmh4Md6oZnc/7PjtwjNFSzveXqSHt6ka9fPBuso7IGf9Rz4uqnSnWiFH2B/zj24a5ReA==", + "dev": true, + "dependencies": { + "flatted": "^2.0.0", + "rimraf": "2.6.3", + "write": "1.0.3" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/flat-cache/node_modules/rimraf": { + "version": "2.6.3", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz", + "integrity": "sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==", + "dev": true, + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + } + }, + "node_modules/flatted": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-2.0.2.tgz", + "integrity": "sha512-r5wGx7YeOwNWNlCA0wQ86zKyDLMQr+/RB8xy74M4hTphfmjlijTSSXGuH8rnvKZnfT9i+75zmd8jcKdMR4O6jA==", + "dev": true + }, + "node_modules/follow-redirects": { + "version": "1.14.2", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.2.tgz", + "integrity": "sha512-yLR6WaE2lbF0x4K2qE2p9PEXKLDjUjnR/xmjS3wHAYxtlsI9MLLBJUZirAHKzUZDGLxje7w/cXR49WOUo4rbsA==", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/foreground-child": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-2.0.0.tgz", + "integrity": "sha512-dCIq9FpEcyQyXKCkyzmlPTFNgrCzPudOe+mhvJU5zAtlBnGVy2yKxtfsxK2tQBThwq225jcvBjpw1Gr40uzZCA==", + "dev": true, + "dependencies": { + "cross-spawn": "^7.0.0", + "signal-exit": "^3.0.2" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fresh": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", + "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/from": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/from/-/from-0.1.7.tgz", + "integrity": "sha1-g8YK/Fi5xWmXAH7Rp2izqzA6RP4=" + }, + "node_modules/fromentries": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/fromentries/-/fromentries-1.3.2.tgz", + "integrity": "sha512-cHEpEQHUg0f8XdtZCc2ZAhrHzKzT0MrFUTcvx+hfxYu7rGMDc5SKoXFh+n4YigxsHXRzc6OrCshdR1bWH6HHyg==", + "dev": true + }, + "node_modules/fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" + }, + "node_modules/fs-minipass": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", + "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", + "dev": true + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/function-bind": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", + "dev": true + }, + "node_modules/functional-red-black-tree": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/functional-red-black-tree/-/functional-red-black-tree-1.0.1.tgz", + "integrity": "sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc=", + "dev": true + }, + "node_modules/gauge": { + "version": "2.7.4", + "resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.4.tgz", + "integrity": "sha1-LANAXHU4w51+s3sxcCLjJfsBi/c=", + "dependencies": { + "aproba": "^1.0.3", + "console-control-strings": "^1.0.0", + "has-unicode": "^2.0.0", + "object-assign": "^4.1.0", + "signal-exit": "^3.0.0", + "string-width": "^1.0.1", + "strip-ansi": "^3.0.1", + "wide-align": "^1.1.0" + } + }, + "node_modules/gensync": { + "version": "1.0.0-beta.2", + "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", + "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==", + "dev": true, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "dev": true, + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/get-func-name": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.0.tgz", + "integrity": "sha1-6td0q+5y4gQJQzoGY2YCPdaIekE=", + "dev": true, + "engines": { + "node": "*" + } + }, + "node_modules/get-intrinsic": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.1.tgz", + "integrity": "sha512-kWZrnVM42QCiEA2Ig1bG8zjoIMOgxWwYCEeNdwY6Tv/cOSeGpcoX4pXHfKUxNKVoArnrEr2e9srnAxxGIraS9Q==", + "dev": true, + "dependencies": { + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.1" + } + }, + "node_modules/get-package-type": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/get-package-type/-/get-package-type-0.1.0.tgz", + "integrity": "sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==", + "dev": true, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/get-stdin": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-8.0.0.tgz", + "integrity": "sha512-sY22aA6xchAzprjyqmSEQv4UbAAzRN0L2dQB0NlN5acTTK9Don6nhoc3eAbUnpZiCANAMfd/+40kVdKfFygohg==", + "dev": true, + "engines": { + "node": ">=10" + } + }, + "node_modules/github-from-package": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", + "integrity": "sha1-l/tdlr/eiXMxPyDoKI75oWf6ZM4=" + }, + "node_modules/glob": { + "version": "7.1.6", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", + "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", + "dev": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + } + }, + "node_modules/glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "dev": true, + "dependencies": { + "is-glob": "^4.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/globals": { + "version": "11.12.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz", + "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.8", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.8.tgz", + "integrity": "sha512-qkIilPUYcNhJpd33n0GBXTB1MMPp14TxEsEs0pTrsSVucApsYzW5V+Q8Qxhik6KU3evy+qkAAowTByymK0avdg==", + "dev": true + }, + "node_modules/growl": { + "version": "1.10.5", + "resolved": "https://registry.npmjs.org/growl/-/growl-1.10.5.tgz", + "integrity": "sha512-qBr4OuELkhPenW6goKVXiv47US3clb3/IbuWF9KNKEijAy9oeHxU9IgzjvJhHkUzhaj7rOUD7+YGWqUjLp5oSA==", + "dev": true, + "engines": { + "node": ">=4.x" + } + }, + "node_modules/has": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", + "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", + "dev": true, + "dependencies": { + "function-bind": "^1.1.1" + }, + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/has-bigints": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/has-bigints/-/has-bigints-1.0.1.tgz", + "integrity": "sha512-LSBS2LjbNBTf6287JEbEzvJgftkF5qFkmCo9hDRpAzKhUOlJ+hx8dd4USs00SgsUNwc4617J9ki5YtEClM2ffA==", + "dev": true + }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/has-symbols": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.2.tgz", + "integrity": "sha512-chXa79rL/UC2KlX17jo3vRGz0azaWEx5tGqZg5pO3NUyEJVB17dMruQlzCCOfUvElghKcm5194+BCRvi2Rv/Gw==", + "dev": true, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/has-tostringtag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.0.tgz", + "integrity": "sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ==", + "dev": true, + "dependencies": { + "has-symbols": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/has-unicode": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz", + "integrity": "sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk=" + }, + "node_modules/hash.js": { + "version": "1.1.7", + "resolved": "https://registry.npmjs.org/hash.js/-/hash.js-1.1.7.tgz", + "integrity": "sha512-taOaskGt4z4SOANNseOviYDvjEJinIkRgmp7LbKP2YTTmVxWBl87s/uzK9r+44BclBSp2X7K1hqeNfz9JbBeXA==", + "dependencies": { + "inherits": "^2.0.3", + "minimalistic-assert": "^1.0.1" + } + }, + "node_modules/hasha": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/hasha/-/hasha-5.2.2.tgz", + "integrity": "sha512-Hrp5vIK/xr5SkeN2onO32H0MgNZ0f17HRNH39WfL0SYUNOTZ5Lz1TJ8Pajo/87dYGEFlLMm7mIc/k/s6Bvz9HQ==", + "dev": true, + "dependencies": { + "is-stream": "^2.0.0", + "type-fest": "^0.8.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/he": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", + "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==", + "dev": true, + "bin": { + "he": "bin/he" + } + }, + "node_modules/hmac-drbg": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/hmac-drbg/-/hmac-drbg-1.0.1.tgz", + "integrity": "sha1-0nRXAQJabHdabFRXk+1QL8DGSaE=", + "dependencies": { + "hash.js": "^1.0.3", + "minimalistic-assert": "^1.0.0", + "minimalistic-crypto-utils": "^1.0.1" + } + }, + "node_modules/hosted-git-info": { + "version": "2.8.9", + "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.9.tgz", + "integrity": "sha512-mxIDAb9Lsm6DoOJ7xH+5+X4y1LU/4Hi50L9C5sIswK3JzULS4bwk1FvjdBgvYR4bzT4tuUQiC15FE2f5HbLvYw==", + "dev": true + }, + "node_modules/html-escaper": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", + "integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==", + "dev": true + }, + "node_modules/http-errors": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz", + "integrity": "sha512-uUQBt3H/cSIVfch6i1EuPNy/YsRSOUBXTVfZ+yR7Zjez3qjBz6i9+i4zjNaoqcoFVI4lQJ5plg63TvGfRSDCRg==", + "dependencies": { + "depd": "~1.1.2", + "inherits": "2.0.3", + "setprototypeof": "1.1.1", + "statuses": ">= 1.5.0 < 2", + "toidentifier": "1.0.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/http-errors/node_modules/inherits": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", + "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" + }, + "node_modules/iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==" + }, + "node_modules/ignore": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-4.0.6.tgz", + "integrity": "sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==", + "dev": true, + "engines": { + "node": ">= 4" + } + }, + "node_modules/import-fresh": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz", + "integrity": "sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw==", + "dev": true, + "dependencies": { + "parent-module": "^1.0.0", + "resolve-from": "^4.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/import-fresh/node_modules/resolve-from": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha1-khi5srkoojixPcT7a21XbyMUU+o=", + "dev": true, + "engines": { + "node": ">=0.8.19" + } + }, + "node_modules/indent-string": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-4.0.0.tgz", + "integrity": "sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", + "dev": true, + "dependencies": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "node_modules/ini": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==" + }, + "node_modules/internal-slot": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/internal-slot/-/internal-slot-1.0.3.tgz", + "integrity": "sha512-O0DB1JC/sPyZl7cIo78n5dR7eUSwwpYPiXRhTzNxZVAMUuB8vlnRFyLxdrVToks6XPLVnFfbzaVd5WLjhgg+vA==", + "dev": true, + "dependencies": { + "get-intrinsic": "^1.1.0", + "has": "^1.0.3", + "side-channel": "^1.0.4" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/is-arrayish": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", + "integrity": "sha1-d8mYQFJ6qOyxqLppe4BkWnqSap0=", + "dev": true + }, + "node_modules/is-bigint": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/is-bigint/-/is-bigint-1.0.4.tgz", + "integrity": "sha512-zB9CruMamjym81i2JZ3UMn54PKGsQzsJeo6xvN3HJJ4CAsQNB6iRutp2To77OfCNuoxspsIhzaPoO1zyCEhFOg==", + "dev": true, + "dependencies": { + "has-bigints": "^1.0.1" + } + }, + "node_modules/is-binary-path": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", + "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", + "dev": true, + "dependencies": { + "binary-extensions": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/is-boolean-object": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/is-boolean-object/-/is-boolean-object-1.1.2.tgz", + "integrity": "sha512-gDYaKHJmnj4aWxyj6YHyXVpdQawtVLHU5cb+eztPGczf6cjuTdwve5ZIEfgXqH4e57An1D1AKf8CZ3kYrQRqYA==", + "dev": true, + "dependencies": { + "call-bind": "^1.0.2", + "has-tostringtag": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/is-callable": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.4.tgz", + "integrity": "sha512-nsuwtxZfMX67Oryl9LCQ+upnC0Z0BgpwntpS89m1H/TLF0zNfzfLMV/9Wa/6MZsj0acpEjAO0KF1xT6ZdLl95w==", + "dev": true, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/is-core-module": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.6.0.tgz", + "integrity": "sha512-wShG8vs60jKfPWpF2KZRaAtvt3a20OAn7+IJ6hLPECpSABLcKtFKTTI4ZtH5QcBruBHlq+WsdHWyz0BCZW7svQ==", + "dev": true, + "dependencies": { + "has": "^1.0.3" + } + }, + "node_modules/is-date-object": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.5.tgz", + "integrity": "sha512-9YQaSxsAiSwcvS33MBk3wTCVnWK+HhF8VZR2jRxehM16QcVOdHqPn4VPHmRK4lSr38n9JriurInLcP90xsYNfQ==", + "dev": true, + "dependencies": { + "has-tostringtag": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha1-qIwCU1eR8C7TfHahueqXc8gz+MI=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz", + "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=", + "dependencies": { + "number-is-nan": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-glob": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.1.tgz", + "integrity": "sha512-5G0tKtBTFImOqDnLB2hG6Bp2qcKEFduo4tZu9MT/H6NQv/ghhy30o55ufafxJ/LdH79LLs2Kfrn85TLKyA7BUg==", + "dev": true, + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-negative-zero": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-negative-zero/-/is-negative-zero-2.0.1.tgz", + "integrity": "sha512-2z6JzQvZRa9A2Y7xC6dQQm4FSTSTNWjKIYYTt4246eMTJmIo0Q+ZyOsU66X8lxK1AbB92dFeglPLrhwpeRKO6w==", + "dev": true, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "dev": true, + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/is-number-object": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/is-number-object/-/is-number-object-1.0.6.tgz", + "integrity": "sha512-bEVOqiRcvo3zO1+G2lVMy+gkkEm9Yh7cDMRusKKu5ZJKPUYSJwICTKZrNKHA2EbSP0Tu0+6B/emsYNHZyn6K8g==", + "dev": true, + "dependencies": { + "has-tostringtag": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/is-plain-obj": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-2.1.0.tgz", + "integrity": "sha512-YWnfyRwxL/+SsrWYfOpUtz5b3YD+nyfkHvjbcanzk8zgyO4ASD67uVMRt8k5bM4lLMDnXfriRhOpemw+NfT1eA==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/is-plain-object": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz", + "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==", + "dependencies": { + "isobject": "^3.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-regex": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.4.tgz", + "integrity": "sha512-kvRdxDsxZjhzUX07ZnLydzS1TU/TJlTUHHY4YLL87e37oUA49DfkLqgy+VjFocowy29cKvcSiu+kIv728jTTVg==", + "dev": true, + "dependencies": { + "call-bind": "^1.0.2", + "has-tostringtag": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/is-string": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/is-string/-/is-string-1.0.7.tgz", + "integrity": "sha512-tE2UXzivje6ofPW7l23cjDOMa09gb7xlAqG6jG5ej6uPV32TlWP3NKPigtaGeHNu9fohccRYvIiZMfOOnOYUtg==", + "dev": true, + "dependencies": { + "has-tostringtag": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/is-symbol": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.4.tgz", + "integrity": "sha512-C/CPBqKWnvdcxqIARxyOh4v1UUEOCHpgDa0WYgpKDFMszcrPcffg5uhwSgPCLD2WWxmq6isisz87tzT01tuGhg==", + "dev": true, + "dependencies": { + "has-symbols": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/is-typedarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", + "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=", + "dev": true + }, + "node_modules/is-windows": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-windows/-/is-windows-1.0.2.tgz", + "integrity": "sha512-eXK1UInq2bPmjyX6e3VHIzMLobc4J94i4AWn+Hpq3OU5KkrRC96OAcR3PRJ/pGu6m8TRnBHP9dkXQVsT/COVIA==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=", + "dev": true + }, + "node_modules/isobject": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz", + "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/istanbul-lib-coverage": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.0.0.tgz", + "integrity": "sha512-UiUIqxMgRDET6eR+o5HbfRYP1l0hqkWOs7vNxC/mggutCMUIhWMm8gAHb8tHlyfD3/l6rlgNA5cKdDzEAf6hEg==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/istanbul-lib-hook": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/istanbul-lib-hook/-/istanbul-lib-hook-3.0.0.tgz", + "integrity": "sha512-Pt/uge1Q9s+5VAZ+pCo16TYMWPBIl+oaNIjgLQxcX0itS6ueeaA+pEfThZpH8WxhFgCiEb8sAJY6MdUKgiIWaQ==", + "dev": true, + "dependencies": { + "append-transform": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/istanbul-lib-instrument": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-4.0.3.tgz", + "integrity": "sha512-BXgQl9kf4WTCPCCpmFGoJkz/+uhvm7h7PFKUYxh7qarQd3ER33vHG//qaE8eN25l07YqZPpHXU9I09l/RD5aGQ==", + "dev": true, + "dependencies": { + "@babel/core": "^7.7.5", + "@istanbuljs/schema": "^0.1.2", + "istanbul-lib-coverage": "^3.0.0", + "semver": "^6.3.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/istanbul-lib-instrument/node_modules/semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "dev": true, + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/istanbul-lib-processinfo": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/istanbul-lib-processinfo/-/istanbul-lib-processinfo-2.0.2.tgz", + "integrity": "sha512-kOwpa7z9hme+IBPZMzQ5vdQj8srYgAtaRqeI48NGmAQ+/5yKiHLV0QbYqQpxsdEF0+w14SoB8YbnHKcXE2KnYw==", + "dev": true, + "dependencies": { + "archy": "^1.0.0", + "cross-spawn": "^7.0.0", + "istanbul-lib-coverage": "^3.0.0-alpha.1", + "make-dir": "^3.0.0", + "p-map": "^3.0.0", + "rimraf": "^3.0.0", + "uuid": "^3.3.3" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/istanbul-lib-report": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/istanbul-lib-report/-/istanbul-lib-report-3.0.0.tgz", + "integrity": "sha512-wcdi+uAKzfiGT2abPpKZ0hSU1rGQjUQnLvtY5MpQ7QCTahD3VODhcu4wcfY1YtkGaDD5yuydOLINXsfbus9ROw==", + "dev": true, + "dependencies": { + "istanbul-lib-coverage": "^3.0.0", + "make-dir": "^3.0.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/istanbul-lib-report/node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/istanbul-lib-source-maps": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/istanbul-lib-source-maps/-/istanbul-lib-source-maps-4.0.0.tgz", + "integrity": "sha512-c16LpFRkR8vQXyHZ5nLpY35JZtzj1PQY1iZmesUbf1FZHbIupcWfjgOXBY9YHkLEQ6puz1u4Dgj6qmU/DisrZg==", + "dev": true, + "dependencies": { + "debug": "^4.1.1", + "istanbul-lib-coverage": "^3.0.0", + "source-map": "^0.6.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/istanbul-lib-source-maps/node_modules/debug": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz", + "integrity": "sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw==", + "dev": true, + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + } + }, + "node_modules/istanbul-lib-source-maps/node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + }, + "node_modules/istanbul-lib-source-maps/node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/istanbul-reports": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.0.2.tgz", + "integrity": "sha512-9tZvz7AiR3PEDNGiV9vIouQ/EAcqMXFmkcA1CDFTwOB98OZVDL0PH9glHotf5Ugp6GCOTypfzGWI/OqjWNCRUw==", + "dev": true, + "dependencies": { + "html-escaper": "^2.0.0", + "istanbul-lib-report": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "dev": true + }, + "node_modules/js-yaml": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.0.0.tgz", + "integrity": "sha512-pqon0s+4ScYUvX30wxQi3PogGFAlUyH0awepWvwkj4jD4v+ova3RiYw8bmA6x2rDrEaj8i/oWKoRxpVNW+Re8Q==", + "dev": true, + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/jsesc": { + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-2.5.2.tgz", + "integrity": "sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA==", + "dev": true, + "bin": { + "jsesc": "bin/jsesc" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/json-parse-better-errors": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz", + "integrity": "sha512-mrqyZKfX5EhL7hvqcV6WG1yYjnjeuYDzDhhcAAUrq8Po85NBQBJP+ZDUT75qZQ98IkUoBqdkExkukOU7Ts2wrw==", + "dev": true + }, + "node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true + }, + "node_modules/json-stable-stringify-without-jsonify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", + "integrity": "sha1-nbe1lJatPzz+8wp1FC0tkwrXJlE=", + "dev": true + }, + "node_modules/json5": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.0.tgz", + "integrity": "sha512-f+8cldu7X/y7RAJurMEJmdoKXGB/X550w2Nr3tTbezL6RwEE/iMcm+tZnXeoZtKuOq6ft8+CqzEkrIgx1fPoQA==", + "dev": true, + "dependencies": { + "minimist": "^1.2.5" + }, + "bin": { + "json5": "lib/cli.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/jsx-ast-utils": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/jsx-ast-utils/-/jsx-ast-utils-3.2.0.tgz", + "integrity": "sha512-EIsmt3O3ljsU6sot/J4E1zDRxfBNrhjyf/OKjlydwgEimQuznlM4Wv7U+ueONJMyEn1WRE0K8dhi3dVAXYT24Q==", + "dev": true, + "dependencies": { + "array-includes": "^3.1.2", + "object.assign": "^4.1.2" + }, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/kind-of": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/levn": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", + "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==", + "dev": true, + "dependencies": { + "prelude-ls": "^1.2.1", + "type-check": "~0.4.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/load-json-file": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-2.0.0.tgz", + "integrity": "sha1-eUfkIUmvgNaWy/eXvKq8/h/inKg=", + "dev": true, + "dependencies": { + "graceful-fs": "^4.1.2", + "parse-json": "^2.2.0", + "pify": "^2.0.0", + "strip-bom": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/load-json-file/node_modules/strip-bom": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz", + "integrity": "sha1-IzTBjpx1n3vdVv3vfprj1YjmjtM=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/locate-path": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", + "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", + "dev": true, + "dependencies": { + "p-locate": "^5.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", + "dev": true + }, + "node_modules/lodash.flattendeep": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/lodash.flattendeep/-/lodash.flattendeep-4.4.0.tgz", + "integrity": "sha1-+wMJF/hqMTTlvJvsDWngAT3f7bI=", + "dev": true + }, + "node_modules/log-symbols": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.0.0.tgz", + "integrity": "sha512-FN8JBzLx6CzeMrB0tg6pqlGU1wCrXW+ZXGH481kfsBqer0hToTIiHdjH4Mq8xJUbvATujKCvaREGWpGUionraA==", + "dev": true, + "dependencies": { + "chalk": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/loose-envify": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", + "integrity": "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==", + "dev": true, + "dependencies": { + "js-tokens": "^3.0.0 || ^4.0.0" + }, + "bin": { + "loose-envify": "cli.js" + } + }, + "node_modules/lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "dev": true, + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/make-dir": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz", + "integrity": "sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==", + "dev": true, + "dependencies": { + "semver": "^6.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/make-dir/node_modules/semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "dev": true, + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/map-stream": { + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/map-stream/-/map-stream-0.0.7.tgz", + "integrity": "sha1-ih8HiW2CsQkmvTdEokIACfiJdKg=" + }, + "node_modules/media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/merge-descriptors": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", + "integrity": "sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E=" + }, + "node_modules/methods": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", + "integrity": "sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4=", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/mime-db": { + "version": "1.49.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.49.0.tgz", + "integrity": "sha512-CIc8j9URtOVApSFCQIF+VBkX1RwXp/oMMOrqdyXSBXq5RWNEsRfyj1kiRnQgmNXmHxPoFIxOroKA3zcU9P+nAA==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.32", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.32.tgz", + "integrity": "sha512-hJGaVS4G4c9TSMYh2n6SQAGrC4RnfU+daP8G7cSCmaqNjiOoUY0VHCMS42pxnQmVF1GWwFhbHWn3RIxCqTmZ9A==", + "dependencies": { + "mime-db": "1.49.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mimic-response": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-2.1.0.tgz", + "integrity": "sha512-wXqjST+SLt7R009ySCglWBCFpjUygmCIfD790/kVbiGmUgfYGuB14PiTd5DwVxSV4NcYHjzMkoj5LjQZwTQLEA==", + "engines": { + "node": ">=8" + } + }, + "node_modules/minimalistic-assert": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/minimalistic-assert/-/minimalistic-assert-1.0.1.tgz", + "integrity": "sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A==" + }, + "node_modules/minimalistic-crypto-utils": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/minimalistic-crypto-utils/-/minimalistic-crypto-utils-1.0.1.tgz", + "integrity": "sha1-9sAMHAsIIkblxNmd+4x8CDsrWCo=" + }, + "node_modules/minimatch": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", + "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "dev": true, + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/minimist": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", + "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==" + }, + "node_modules/minipass": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.1.3.tgz", + "integrity": "sha512-Mgd2GdMVzY+x3IJ+oHnVM+KG3lA5c8tnabyJKmHSaG2kAGpudxuOf8ToDkhumF7UzME7DecbQE9uOZhNm7PuJg==", + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/minizlib": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-2.1.2.tgz", + "integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==", + "dependencies": { + "minipass": "^3.0.0", + "yallist": "^4.0.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/mkdirp": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", + "bin": { + "mkdirp": "bin/cmd.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/mkdirp-classic": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==" + }, + "node_modules/mocha": { + "version": "8.4.0", + "resolved": "https://registry.npmjs.org/mocha/-/mocha-8.4.0.tgz", + "integrity": "sha512-hJaO0mwDXmZS4ghXsvPVriOhsxQ7ofcpQdm8dE+jISUOKopitvnXFQmpRR7jd2K6VBG6E26gU3IAbXXGIbu4sQ==", + "dev": true, + "dependencies": { + "@ungap/promise-all-settled": "1.1.2", + "ansi-colors": "4.1.1", + "browser-stdout": "1.3.1", + "chokidar": "3.5.1", + "debug": "4.3.1", + "diff": "5.0.0", + "escape-string-regexp": "4.0.0", + "find-up": "5.0.0", + "glob": "7.1.6", + "growl": "1.10.5", + "he": "1.2.0", + "js-yaml": "4.0.0", + "log-symbols": "4.0.0", + "minimatch": "3.0.4", + "ms": "2.1.3", + "nanoid": "3.1.20", + "serialize-javascript": "5.0.1", + "strip-json-comments": "3.1.1", + "supports-color": "8.1.1", + "which": "2.0.2", + "wide-align": "1.1.3", + "workerpool": "6.1.0", + "yargs": "16.2.0", + "yargs-parser": "20.2.4", + "yargs-unparser": "2.0.0" + }, + "bin": { + "_mocha": "bin/_mocha", + "mocha": "bin/mocha" + }, + "engines": { + "node": ">= 10.12.0" + } + }, + "node_modules/mocha/node_modules/debug": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz", + "integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==", + "dev": true, + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + } + }, + "node_modules/mocha/node_modules/debug/node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + }, + "node_modules/mocha/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true + }, + "node_modules/mocha/node_modules/strip-json-comments": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", + "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/morgan": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/morgan/-/morgan-1.10.0.tgz", + "integrity": "sha512-AbegBVI4sh6El+1gNwvD5YIck7nSA36weD7xvIxG4in80j/UoK8AEGaWnnz8v1GxonMCltmlNs5ZKbGvl9b1XQ==", + "dependencies": { + "basic-auth": "~2.0.1", + "debug": "2.6.9", + "depd": "~2.0.0", + "on-finished": "~2.3.0", + "on-headers": "~1.0.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/morgan/node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + }, + "node_modules/nan": { + "version": "2.14.2", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.2.tgz", + "integrity": "sha512-M2ufzIiINKCuDfBSAUr1vWQ+vuVcA9kqx8JJUsbQi6yf1uGRyb7HfpdfUr5qLXf3B/t8dPvcjhKMmlfnP47EzQ==", + "optional": true + }, + "node_modules/nanoid": { + "version": "3.1.20", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.1.20.tgz", + "integrity": "sha512-a1cQNyczgKbLX9jwbS/+d7W8fX/RfgYR7lVWwWOGIPNgK2m0MWvrGF6/m4kk6U3QcFMnZf3RIhL0v2Jgh/0Uxw==", + "dev": true, + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/napi-build-utils": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz", + "integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==" + }, + "node_modules/natural-compare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", + "integrity": "sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=", + "dev": true + }, + "node_modules/negotiator": { + "version": "0.6.2", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz", + "integrity": "sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/node-abi": { + "version": "2.30.0", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-2.30.0.tgz", + "integrity": "sha512-g6bZh3YCKQRdwuO/tSZZYJAw622SjsRfJ2X0Iy4sSOHZ34/sPPdVBn8fev2tj7njzLwuqPw9uMtGsGkO5kIQvg==", + "dependencies": { + "semver": "^5.4.1" + } + }, + "node_modules/node-fetch": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", + "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==", + "engines": { + "node": "4.x || >=6.0.0" + } + }, + "node_modules/node-gyp-build": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.2.3.tgz", + "integrity": "sha512-MN6ZpzmfNCRM+3t57PTJHgHyw/h4OWnZ6mR8P5j/uZtqQr46RRuDE/P+g3n0YR/AiYXeWixZZzaip77gdICfRg==", + "optional": true, + "bin": { + "node-gyp-build": "bin.js", + "node-gyp-build-optional": "optional.js", + "node-gyp-build-test": "build-test.js" + } + }, + "node_modules/node-preload": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/node-preload/-/node-preload-0.2.1.tgz", + "integrity": "sha512-RM5oyBy45cLEoHqCeh+MNuFAxO0vTFBLskvQbOKnEE7YTTSN4tbN8QWDIPQ6L+WvKsB/qLEGpYe2ZZ9d4W9OIQ==", + "dev": true, + "dependencies": { + "process-on-spawn": "^1.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/node-releases": { + "version": "1.1.75", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-1.1.75.tgz", + "integrity": "sha512-Qe5OUajvqrqDSy6wrWFmMwfJ0jVgwiw4T3KqmbTcZ62qW0gQkheXYhcFM1+lOVcGUoRxcEcfyvFMAnDgaF1VWw==", + "dev": true + }, + "node_modules/normalize-package-data": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.5.0.tgz", + "integrity": "sha512-/5CMN3T0R4XTj4DcGaexo+roZSdSFW/0AOOTROrjxzCG1wrWXEsGbRKevjlIL+ZDE4sZlJr5ED4YW0yqmkK+eA==", + "dev": true, + "dependencies": { + "hosted-git-info": "^2.1.4", + "resolve": "^1.10.0", + "semver": "2 || 3 || 4 || 5", + "validate-npm-package-license": "^3.0.1" + } + }, + "node_modules/normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/npmlog": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-4.1.2.tgz", + "integrity": "sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==", + "dependencies": { + "are-we-there-yet": "~1.1.2", + "console-control-strings": "~1.1.0", + "gauge": "~2.7.3", + "set-blocking": "~2.0.0" + } + }, + "node_modules/number-is-nan": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz", + "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/nyc": { + "version": "15.1.0", + "resolved": "https://registry.npmjs.org/nyc/-/nyc-15.1.0.tgz", + "integrity": "sha512-jMW04n9SxKdKi1ZMGhvUTHBN0EICCRkHemEoE5jm6mTYcqcdas0ATzgUgejlQUHMvpnOZqGB5Xxsv9KxJW1j8A==", + "dev": true, + "dependencies": { + "@istanbuljs/load-nyc-config": "^1.0.0", + "@istanbuljs/schema": "^0.1.2", + "caching-transform": "^4.0.0", + "convert-source-map": "^1.7.0", + "decamelize": "^1.2.0", + "find-cache-dir": "^3.2.0", + "find-up": "^4.1.0", + "foreground-child": "^2.0.0", + "get-package-type": "^0.1.0", + "glob": "^7.1.6", + "istanbul-lib-coverage": "^3.0.0", + "istanbul-lib-hook": "^3.0.0", + "istanbul-lib-instrument": "^4.0.0", + "istanbul-lib-processinfo": "^2.0.2", + "istanbul-lib-report": "^3.0.0", + "istanbul-lib-source-maps": "^4.0.0", + "istanbul-reports": "^3.0.2", + "make-dir": "^3.0.0", + "node-preload": "^0.2.1", + "p-map": "^3.0.0", + "process-on-spawn": "^1.0.0", + "resolve-from": "^5.0.0", + "rimraf": "^3.0.0", + "signal-exit": "^3.0.2", + "spawn-wrap": "^2.0.0", + "test-exclude": "^6.0.0", + "yargs": "^15.0.2" + }, + "bin": { + "nyc": "bin/nyc.js" + }, + "engines": { + "node": ">=8.9" + } + }, + "node_modules/nyc/node_modules/ansi-regex": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", + "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/nyc/node_modules/camelcase": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", + "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/nyc/node_modules/cliui": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-6.0.0.tgz", + "integrity": "sha512-t6wbgtoCXvAzst7QgXxJYqPt0usEfbgQdftEPbLL/cvv6HPE5VgvqCuAIDR0NgU52ds6rFwqrgakNLrHEjCbrQ==", + "dev": true, + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.0", + "wrap-ansi": "^6.2.0" + } + }, + "node_modules/nyc/node_modules/decamelize": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", + "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/nyc/node_modules/find-up": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "dev": true, + "dependencies": { + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/nyc/node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/nyc/node_modules/locate-path": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "dev": true, + "dependencies": { + "p-locate": "^4.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/nyc/node_modules/p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dev": true, + "dependencies": { + "p-try": "^2.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/nyc/node_modules/p-locate": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "dev": true, + "dependencies": { + "p-limit": "^2.2.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/nyc/node_modules/string-width": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.2.tgz", + "integrity": "sha512-XBJbT3N4JhVumXE0eoLU9DCjcaF92KLNqTmFCnG1pf8duUxFGwtP6AD6nkjw9a3IdiRtL3E2w3JDiE/xi3vOeA==", + "dev": true, + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/nyc/node_modules/strip-ansi": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", + "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==", + "dev": true, + "dependencies": { + "ansi-regex": "^5.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/nyc/node_modules/wrap-ansi": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz", + "integrity": "sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==", + "dev": true, + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/nyc/node_modules/y18n": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.3.tgz", + "integrity": "sha512-JKhqTOwSrqNA1NY5lSztJ1GrBiUodLMmIZuLiDaMRJ+itFd+ABVE8XBjOvIWL+rSqNDC74LCSFmlb/U4UZ4hJQ==", + "dev": true + }, + "node_modules/nyc/node_modules/yargs": { + "version": "15.4.1", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-15.4.1.tgz", + "integrity": "sha512-aePbxDmcYW++PaqBsJ+HYUFwCdv4LVvdnhBy78E57PIor8/OVvhMrADFFEDh8DHDFRv/O9i3lPhsENjO7QX0+A==", + "dev": true, + "dependencies": { + "cliui": "^6.0.0", + "decamelize": "^1.2.0", + "find-up": "^4.1.0", + "get-caller-file": "^2.0.1", + "require-directory": "^2.1.1", + "require-main-filename": "^2.0.0", + "set-blocking": "^2.0.0", + "string-width": "^4.2.0", + "which-module": "^2.0.0", + "y18n": "^4.0.0", + "yargs-parser": "^18.1.2" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/nyc/node_modules/yargs-parser": { + "version": "18.1.3", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-18.1.3.tgz", + "integrity": "sha512-o50j0JeToy/4K6OZcaQmW6lyXXKhq7csREXcDwk2omFPJEwUNOVtJKvmDr9EI1fAJZUyZcRF7kxGBWmRXudrCQ==", + "dev": true, + "dependencies": { + "camelcase": "^5.0.0", + "decamelize": "^1.2.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object-inspect": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.11.0.tgz", + "integrity": "sha512-jp7ikS6Sd3GxQfZJPyH3cjcbJF6GZPClgdV+EFygjFLQ5FmW/dRUnTd9PQ9k0JhoNDabWFbpF1yCdSWCC6gexg==", + "dev": true + }, + "node_modules/object-keys": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", + "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", + "dev": true, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/object.assign": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.2.tgz", + "integrity": "sha512-ixT2L5THXsApyiUPYKmW+2EHpXXe5Ii3M+f4e+aJFAHao5amFRW6J0OO6c/LU8Be47utCx2GL89hxGB6XSmKuQ==", + "dev": true, + "dependencies": { + "call-bind": "^1.0.0", + "define-properties": "^1.1.3", + "has-symbols": "^1.0.1", + "object-keys": "^1.1.1" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/object.entries": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/object.entries/-/object.entries-1.1.4.tgz", + "integrity": "sha512-h4LWKWE+wKQGhtMjZEBud7uLGhqyLwj8fpHOarZhD2uY3C9cRtk57VQ89ke3moByLXMedqs3XCHzyb4AmA2DjA==", + "dev": true, + "dependencies": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.3", + "es-abstract": "^1.18.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/object.fromentries": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/object.fromentries/-/object.fromentries-2.0.4.tgz", + "integrity": "sha512-EsFBshs5RUUpQEY1D4q/m59kMfz4YJvxuNCJcv/jWwOJr34EaVnG11ZrZa0UHB3wnzV1wx8m58T4hQL8IuNXlQ==", + "dev": true, + "dependencies": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.3", + "es-abstract": "^1.18.0-next.2", + "has": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/object.values": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/object.values/-/object.values-1.1.4.tgz", + "integrity": "sha512-TnGo7j4XSnKQoK3MfvkzqKCi0nVe/D9I9IjwTNYdb/fxYHpjrluHVOgw0AF6jrRFGMPHdfuidR09tIDiIvnaSg==", + "dev": true, + "dependencies": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.3", + "es-abstract": "^1.18.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/on-finished": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", + "integrity": "sha1-IPEzZIGwg811M3mSoWlxqi2QaUc=", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/on-headers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/on-headers/-/on-headers-1.0.2.tgz", + "integrity": "sha512-pZAE+FJLoyITytdqK0U5s+FIpjN0JP3OzFi/u8Rx+EV5/W+JTWGXG8xFzevE7AjBfDqHv/8vL8qQsIhHnqRkrA==", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/optionator": { + "version": "0.9.1", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.1.tgz", + "integrity": "sha512-74RlY5FCnhq4jRxVUPKDaRwrVNXMqsGsiW6AJw4XK8hmtm10wC0ypZBLw5IIp85NZMr91+qd1RvvENwg7jjRFw==", + "dev": true, + "dependencies": { + "deep-is": "^0.1.3", + "fast-levenshtein": "^2.0.6", + "levn": "^0.4.1", + "prelude-ls": "^1.2.1", + "type-check": "^0.4.0", + "word-wrap": "^1.2.3" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/original": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/original/-/original-1.0.2.tgz", + "integrity": "sha512-hyBVl6iqqUOJ8FqRe+l/gS8H+kKYjrEndd5Pm1MfBtsEKA038HkkdbAl/72EAXGyonD/PFsvmVG+EvcIpliMBg==", + "dependencies": { + "url-parse": "^1.4.3" + } + }, + "node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/p-locate": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", + "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", + "dev": true, + "dependencies": { + "p-limit": "^3.0.2" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/p-map": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/p-map/-/p-map-3.0.0.tgz", + "integrity": "sha512-d3qXVTF/s+W+CdJ5A29wywV2n8CQQYahlgz2bFiA+4eVNJbHJodPZ+/gXwPGh0bOqA+j8S+6+ckmvLGPk1QpxQ==", + "dev": true, + "dependencies": { + "aggregate-error": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/p-try": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/package-hash": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/package-hash/-/package-hash-4.0.0.tgz", + "integrity": "sha512-whdkPIooSu/bASggZ96BWVvZTRMOFxnyUG5PnTSGKoJE2gd5mbVNmR2Nj20QFzxYYgAXpoqC+AiXzl+UMRh7zQ==", + "dev": true, + "dependencies": { + "graceful-fs": "^4.1.15", + "hasha": "^5.0.0", + "lodash.flattendeep": "^4.4.0", + "release-zalgo": "^1.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/parent-module": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", + "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", + "dev": true, + "dependencies": { + "callsites": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/parse-json": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-2.2.0.tgz", + "integrity": "sha1-9ID0BDTvgHQfhGkJn43qGPVaTck=", + "dev": true, + "dependencies": { + "error-ex": "^1.2.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/path-parse": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", + "dev": true + }, + "node_modules/path-to-regexp": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", + "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" + }, + "node_modules/path-type": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-2.0.0.tgz", + "integrity": "sha1-8BLMuEFbcJb8LaoQVMPXI4lZTHM=", + "dev": true, + "dependencies": { + "pify": "^2.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/pathval": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/pathval/-/pathval-1.1.1.tgz", + "integrity": "sha512-Dp6zGqpTdETdR63lehJYPeIOqpiNBNtc7BpWSLrOje7UaIsE5aY92r/AunQA7rsXvet3lrJ3JnZX29UPTKXyKQ==", + "dev": true, + "engines": { + "node": "*" + } + }, + "node_modules/pause-stream": { + "version": "0.0.11", + "resolved": "https://registry.npmjs.org/pause-stream/-/pause-stream-0.0.11.tgz", + "integrity": "sha1-/lo0sMvOErWqaitAPuLnO2AvFEU=", + "dependencies": { + "through": "~2.3" + } + }, + "node_modules/picomatch": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.0.tgz", + "integrity": "sha512-lY1Q/PiJGC2zOv/z391WOTD+Z02bCgsFfvxoXXf6h7kv9o+WmsmzYqrAwY63sNgOxE4xEdq0WyUnXfKeBrSvYw==", + "dev": true, + "engines": { + "node": ">=8.6" + } + }, + "node_modules/pify": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/pkg-conf": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/pkg-conf/-/pkg-conf-3.1.0.tgz", + "integrity": "sha512-m0OTbR/5VPNPqO1ph6Fqbj7Hv6QU7gR/tQW40ZqrL1rjgCU85W6C1bJn0BItuJqnR98PWzw7Z8hHeChD1WrgdQ==", + "dev": true, + "dependencies": { + "find-up": "^3.0.0", + "load-json-file": "^5.2.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/pkg-conf/node_modules/find-up": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz", + "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==", + "dev": true, + "dependencies": { + "locate-path": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/pkg-conf/node_modules/load-json-file": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-5.3.0.tgz", + "integrity": "sha512-cJGP40Jc/VXUsp8/OrnyKyTZ1y6v/dphm3bioS+RrKXjK2BB6wHUd6JptZEFDGgGahMT+InnZO5i1Ei9mpC8Bw==", + "dev": true, + "dependencies": { + "graceful-fs": "^4.1.15", + "parse-json": "^4.0.0", + "pify": "^4.0.1", + "strip-bom": "^3.0.0", + "type-fest": "^0.3.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/pkg-conf/node_modules/locate-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz", + "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==", + "dev": true, + "dependencies": { + "p-locate": "^3.0.0", + "path-exists": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/pkg-conf/node_modules/p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dev": true, + "dependencies": { + "p-try": "^2.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/pkg-conf/node_modules/p-locate": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz", + "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==", + "dev": true, + "dependencies": { + "p-limit": "^2.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/pkg-conf/node_modules/parse-json": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-4.0.0.tgz", + "integrity": "sha1-vjX1Qlvh9/bHRxhPmKeIy5lHfuA=", + "dev": true, + "dependencies": { + "error-ex": "^1.3.1", + "json-parse-better-errors": "^1.0.1" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/pkg-conf/node_modules/path-exists": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", + "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/pkg-conf/node_modules/pify": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz", + "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/pkg-conf/node_modules/strip-bom": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz", + "integrity": "sha1-IzTBjpx1n3vdVv3vfprj1YjmjtM=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/pkg-conf/node_modules/type-fest": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.3.1.tgz", + "integrity": "sha512-cUGJnCdr4STbePCgqNFbpVNCepa+kAVohJs1sLhxzdH+gnEoOd8VhbYa7pD3zZYGiURWM2xzEII3fQcRizDkYQ==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/pkg-dir": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", + "integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==", + "dev": true, + "dependencies": { + "find-up": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/pkg-dir/node_modules/find-up": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "dev": true, + "dependencies": { + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/pkg-dir/node_modules/locate-path": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "dev": true, + "dependencies": { + "p-locate": "^4.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/pkg-dir/node_modules/p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dev": true, + "dependencies": { + "p-try": "^2.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/pkg-dir/node_modules/p-locate": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "dev": true, + "dependencies": { + "p-limit": "^2.2.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/prebuild-install": { + "version": "6.1.4", + "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-6.1.4.tgz", + "integrity": "sha512-Z4vpywnK1lBg+zdPCVCsKq0xO66eEV9rWo2zrROGGiRS4JtueBOdlB1FnY8lcy7JsUud/Q3ijUxyWN26Ika0vQ==", + "dependencies": { + "detect-libc": "^1.0.3", + "expand-template": "^2.0.3", + "github-from-package": "0.0.0", + "minimist": "^1.2.3", + "mkdirp-classic": "^0.5.3", + "napi-build-utils": "^1.0.1", + "node-abi": "^2.21.0", + "npmlog": "^4.0.1", + "pump": "^3.0.0", + "rc": "^1.2.7", + "simple-get": "^3.0.3", + "tar-fs": "^2.0.0", + "tunnel-agent": "^0.6.0" + }, + "bin": { + "prebuild-install": "bin.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/prelude-ls": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", + "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==", + "dev": true, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" + }, + "node_modules/process-on-spawn": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/process-on-spawn/-/process-on-spawn-1.0.0.tgz", + "integrity": "sha512-1WsPDsUSMmZH5LeMLegqkPDrsGgsWwk1Exipy2hvB0o/F0ASzbpIctSCcZIK1ykJvtTJULEH+20WOFjMvGnCTg==", + "dev": true, + "dependencies": { + "fromentries": "^1.2.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/progress": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", + "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", + "dev": true, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/prop-types": { + "version": "15.7.2", + "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.7.2.tgz", + "integrity": "sha512-8QQikdH7//R2vurIJSutZ1smHYTcLpRWEOlHnzcWHmBYrOGUysKwSsrC89BCiFj3CbrfJ/nXFdJepOVrY1GCHQ==", + "dev": true, + "dependencies": { + "loose-envify": "^1.4.0", + "object-assign": "^4.1.1", + "react-is": "^16.8.1" + } + }, + "node_modules/proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/pump": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", + "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, + "node_modules/punycode": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", + "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/qs": { + "version": "6.7.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz", + "integrity": "sha512-VCdBRNFTX1fyE7Nb6FYoURo/SPe62QCaAyzJvUjwRaIsc+NePBEniHlvxFmmX56+HZphIGtV0XeCirBtpDrTyQ==", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/querystringify": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz", + "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==" + }, + "node_modules/randombytes": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", + "integrity": "sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==", + "dev": true, + "dependencies": { + "safe-buffer": "^5.1.0" + } + }, + "node_modules/range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/raw-body": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.0.tgz", + "integrity": "sha512-4Oz8DUIwdvoa5qMJelxipzi/iJIi40O5cGV1wNYp5hvZP8ZN0T+jiNkL0QepXs+EsQ9XJ8ipEDoiH70ySUJP3Q==", + "dependencies": { + "bytes": "3.1.0", + "http-errors": "1.7.2", + "iconv-lite": "0.4.24", + "unpipe": "1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/rc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "dependencies": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + }, + "bin": { + "rc": "cli.js" + } + }, + "node_modules/react-is": { + "version": "16.13.1", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz", + "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==", + "dev": true + }, + "node_modules/read-pkg": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-2.0.0.tgz", + "integrity": "sha1-jvHAYjxqbbDcZxPEv6xGMysjaPg=", + "dev": true, + "dependencies": { + "load-json-file": "^2.0.0", + "normalize-package-data": "^2.3.2", + "path-type": "^2.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/read-pkg-up": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/read-pkg-up/-/read-pkg-up-2.0.0.tgz", + "integrity": "sha1-a3KoBImE4MQeeVEP1en6mbO1Sb4=", + "dev": true, + "dependencies": { + "find-up": "^2.0.0", + "read-pkg": "^2.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/read-pkg-up/node_modules/find-up": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-2.1.0.tgz", + "integrity": "sha1-RdG35QbHF93UgndaK3eSCjwMV6c=", + "dev": true, + "dependencies": { + "locate-path": "^2.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/read-pkg-up/node_modules/locate-path": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-2.0.0.tgz", + "integrity": "sha1-K1aLJl7slExtnA3pw9u7ygNUzY4=", + "dev": true, + "dependencies": { + "p-locate": "^2.0.0", + "path-exists": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/read-pkg-up/node_modules/p-limit": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-1.3.0.tgz", + "integrity": "sha512-vvcXsLAJ9Dr5rQOPk7toZQZJApBl2K4J6dANSsEuh6QI41JYcsS/qhTGa9ErIUUgK3WNQoJYvylxvjqmiqEA9Q==", + "dev": true, + "dependencies": { + "p-try": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/read-pkg-up/node_modules/p-locate": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-2.0.0.tgz", + "integrity": "sha1-IKAQOyIqcMj9OcwuWAaA893l7EM=", + "dev": true, + "dependencies": { + "p-limit": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/read-pkg-up/node_modules/p-try": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-1.0.0.tgz", + "integrity": "sha1-y8ec26+P1CKOE/Yh8rGiN8GyB7M=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/read-pkg-up/node_modules/path-exists": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", + "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/readdirp": { + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.5.0.tgz", + "integrity": "sha512-cMhu7c/8rdhkHXWsY+osBhfSy0JikwpHK/5+imo+LpeasTF8ouErHrlYkwT0++njiyuDvc7OFY5T3ukvZ8qmFQ==", + "dev": true, + "dependencies": { + "picomatch": "^2.2.1" + }, + "engines": { + "node": ">=8.10.0" + } + }, + "node_modules/reconnecting-eventsource": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/reconnecting-eventsource/-/reconnecting-eventsource-1.1.0.tgz", + "integrity": "sha512-KJbQG1iZOC4IRBccT1mOodzKp5YM9gC1H4L6LgDmmXx+4U3pabhb+UF0UI0u+Lck1cvb4oyAuihaQqTbKnslqQ==" + }, + "node_modules/regexp.prototype.flags": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.3.1.tgz", + "integrity": "sha512-JiBdRBq91WlY7uRJ0ds7R+dU02i6LKi8r3BuQhNXn+kmeLN+EfHhfjqMRis1zJxnlu88hq/4dx0P2OP3APRTOA==", + "dev": true, + "dependencies": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.3" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/regexpp": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/regexpp/-/regexpp-3.2.0.tgz", + "integrity": "sha512-pq2bWo9mVD43nbts2wGv17XLiNLya+GklZ8kaDLV2Z08gDCsGpnKn9BFMepvWuHCbyVvY7J5o5+BVvoQbmlJLg==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/release-zalgo": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/release-zalgo/-/release-zalgo-1.0.0.tgz", + "integrity": "sha1-CXALflB0Mpc5Mw5TXFqQ+2eFFzA=", + "dev": true, + "dependencies": { + "es6-error": "^4.0.1" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/require-main-filename": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz", + "integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==", + "dev": true + }, + "node_modules/requires-port": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", + "integrity": "sha1-kl0mAdOaxIXgkc8NpcbmlNw9yv8=" + }, + "node_modules/resolve": { + "version": "1.20.0", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.20.0.tgz", + "integrity": "sha512-wENBPt4ySzg4ybFQW2TT1zMQucPK95HSh/nq2CFTZVOGut2+pQvSsgtda4d26YrYcr067wjbmzOG8byDPBX63A==", + "dev": true, + "dependencies": { + "is-core-module": "^2.2.0", + "path-parse": "^1.0.6" + } + }, + "node_modules/resolve-from": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz", + "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "dev": true, + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + } + }, + "node_modules/run-sdk": { + "version": "0.6.29", + "resolved": "https://registry.npmjs.org/run-sdk/-/run-sdk-0.6.29.tgz", + "integrity": "sha512-rbSu7Uo6AIaTmhOpJrByAXa3GfCcGrZSFtSNTdya5yORuEwh9g2hefM6dDQxU/pFeADk+mC788XjOXZ6V1oyOA==", + "dependencies": { + "bsv": "^1.5.6" + } + }, + "node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + }, + "node_modules/semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "bin": { + "semver": "bin/semver" + } + }, + "node_modules/send": { + "version": "0.17.1", + "resolved": "https://registry.npmjs.org/send/-/send-0.17.1.tgz", + "integrity": "sha512-BsVKsiGcQMFwT8UxypobUKyv7irCNRHk1T0G680vk88yf6LBByGcZJOTJCrTP2xVN6yI+XjPJcNuE3V4fT9sAg==", + "dependencies": { + "debug": "2.6.9", + "depd": "~1.1.2", + "destroy": "~1.0.4", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "0.5.2", + "http-errors": "~1.7.2", + "mime": "1.6.0", + "ms": "2.1.1", + "on-finished": "~2.3.0", + "range-parser": "~1.2.1", + "statuses": "~1.5.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/send/node_modules/ms": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", + "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==" + }, + "node_modules/serialize-javascript": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-5.0.1.tgz", + "integrity": "sha512-SaaNal9imEO737H2c05Og0/8LUXG7EnsZyMa8MzkmuHoELfT6txuj0cMqRj6zfPKnmQ1yasR4PCJc8x+M4JSPA==", + "dev": true, + "dependencies": { + "randombytes": "^2.1.0" + } + }, + "node_modules/serve-static": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.1.tgz", + "integrity": "sha512-JMrvUwE54emCYWlTI+hGrGv5I8dEwmco/00EvkzIIsR7MqrHonbD9pO2MOfFnpFntl7ecpZs+3mW+XbQZu9QCg==", + "dependencies": { + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "parseurl": "~1.3.3", + "send": "0.17.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/set-blocking": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", + "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=" + }, + "node_modules/setprototypeof": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.1.tgz", + "integrity": "sha512-JvdAWfbXeIGaZ9cILp38HntZSFSo3mWg6xGcJJsd+d4aRMOqauag1C63dJfDw7OaMYwEbHMOxEZ1lqVRYP2OAw==" + }, + "node_modules/shallow-clone": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/shallow-clone/-/shallow-clone-3.0.1.tgz", + "integrity": "sha512-/6KqX+GVUdqPuPPd2LxDDxzX6CAbjJehAAOKlNpqqUpAqPM6HeL8f+o3a+JsyGjn2lv0WY8UsTgUJjU9Ok55NA==", + "dependencies": { + "kind-of": "^6.0.2" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/side-channel": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", + "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==", + "dev": true, + "dependencies": { + "call-bind": "^1.0.0", + "get-intrinsic": "^1.0.2", + "object-inspect": "^1.9.0" + } + }, + "node_modules/signal-exit": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.3.tgz", + "integrity": "sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA==" + }, + "node_modules/simple-concat": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", + "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==" + }, + "node_modules/simple-get": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-3.1.0.tgz", + "integrity": "sha512-bCR6cP+aTdScaQCnQKbPKtJOKDp/hj9EDLJo3Nw4y1QksqaovlW/bnptB6/c1e+qmNIDHRK+oXFDdEqBT8WzUA==", + "dependencies": { + "decompress-response": "^4.2.0", + "once": "^1.3.1", + "simple-concat": "^1.0.0" + } + }, + "node_modules/slice-ansi": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-2.1.0.tgz", + "integrity": "sha512-Qu+VC3EwYLldKa1fCxuuvULvSJOKEgk9pi8dZeCVK7TqBfUNTH4sFkk4joj8afVSfAYgJoSOetjx9QWOJ5mYoQ==", + "dev": true, + "dependencies": { + "ansi-styles": "^3.2.0", + "astral-regex": "^1.0.0", + "is-fullwidth-code-point": "^2.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/slice-ansi/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dev": true, + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/slice-ansi/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dev": true, + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/slice-ansi/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=", + "dev": true + }, + "node_modules/slice-ansi/node_modules/is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/source-map": { + "version": "0.5.7", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz", + "integrity": "sha1-igOdLRAh0i0eoUyA2OpGi6LvP8w=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/spawn-wrap": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/spawn-wrap/-/spawn-wrap-2.0.0.tgz", + "integrity": "sha512-EeajNjfN9zMnULLwhZZQU3GWBoFNkbngTUPfaawT4RkMiviTxcX0qfhVbGey39mfctfDHkWtuecgQ8NJcyQWHg==", + "dev": true, + "dependencies": { + "foreground-child": "^2.0.0", + "is-windows": "^1.0.2", + "make-dir": "^3.0.0", + "rimraf": "^3.0.0", + "signal-exit": "^3.0.2", + "which": "^2.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/spdx-correct": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.1.1.tgz", + "integrity": "sha512-cOYcUWwhCuHCXi49RhFRCyJEK3iPj1Ziz9DpViV3tbZOwXD49QzIN3MpOLJNxh2qwq2lJJZaKMVw9qNi4jTC0w==", + "dev": true, + "dependencies": { + "spdx-expression-parse": "^3.0.0", + "spdx-license-ids": "^3.0.0" + } + }, + "node_modules/spdx-exceptions": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/spdx-exceptions/-/spdx-exceptions-2.3.0.tgz", + "integrity": "sha512-/tTrYOC7PPI1nUAgx34hUpqXuyJG+DTHJTnIULG4rDygi4xu/tfgmq1e1cIRwRzwZgo4NLySi+ricLkZkw4i5A==", + "dev": true + }, + "node_modules/spdx-expression-parse": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/spdx-expression-parse/-/spdx-expression-parse-3.0.1.tgz", + "integrity": "sha512-cbqHunsQWnJNE6KhVSMsMeH5H/L9EpymbzqTQ3uLwNCLZ1Q481oWaofqH7nO6V07xlXwY6PhQdQ2IedWx/ZK4Q==", + "dev": true, + "dependencies": { + "spdx-exceptions": "^2.1.0", + "spdx-license-ids": "^3.0.0" + } + }, + "node_modules/spdx-license-ids": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.10.tgz", + "integrity": "sha512-oie3/+gKf7QtpitB0LYLETe+k8SifzsX4KixvpOsbI6S0kRiRQ5MKOio8eMSAKQ17N06+wdEOXRiId+zOxo0hA==", + "dev": true + }, + "node_modules/split": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/split/-/split-1.0.1.tgz", + "integrity": "sha512-mTyOoPbrivtXnwnIxZRFYRrPNtEFKlpB2fvjSnCQUiAA6qAZzqwna5envK4uk6OIeP17CsdF3rSBGYVBsU0Tkg==", + "dependencies": { + "through": "2" + }, + "engines": { + "node": "*" + } + }, + "node_modules/sprintf-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=", + "dev": true + }, + "node_modules/standard": { + "version": "16.0.3", + "resolved": "https://registry.npmjs.org/standard/-/standard-16.0.3.tgz", + "integrity": "sha512-70F7NH0hSkNXosXRltjSv6KpTAOkUkSfyu3ynyM5dtRUiLtR+yX9EGZ7RKwuGUqCJiX/cnkceVM6HTZ4JpaqDg==", + "dev": true, + "dependencies": { + "eslint": "~7.13.0", + "eslint-config-standard": "16.0.2", + "eslint-config-standard-jsx": "10.0.0", + "eslint-plugin-import": "~2.22.1", + "eslint-plugin-node": "~11.1.0", + "eslint-plugin-promise": "~4.2.1", + "eslint-plugin-react": "~7.21.5", + "standard-engine": "^14.0.1" + }, + "bin": { + "standard": "bin/cmd.js" + }, + "engines": { + "node": "^10.12.0 || >=12.0.0" + } + }, + "node_modules/standard-engine": { + "version": "14.0.1", + "resolved": "https://registry.npmjs.org/standard-engine/-/standard-engine-14.0.1.tgz", + "integrity": "sha512-7FEzDwmHDOGva7r9ifOzD3BGdTbA7ujJ50afLVdW/tK14zQEptJjbFuUfn50irqdHDcTbNh0DTIoMPynMCXb0Q==", + "dev": true, + "dependencies": { + "get-stdin": "^8.0.0", + "minimist": "^1.2.5", + "pkg-conf": "^3.1.0", + "xdg-basedir": "^4.0.0" + }, + "engines": { + "node": ">=8.10" + } + }, + "node_modules/statuses": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", + "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/stream-combiner": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/stream-combiner/-/stream-combiner-0.2.2.tgz", + "integrity": "sha1-rsjLrBd7Vrb0+kec7YwZEs7lKFg=", + "dependencies": { + "duplexer": "~0.1.1", + "through": "~2.3.4" + } + }, + "node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, + "node_modules/string-width": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", + "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", + "dependencies": { + "code-point-at": "^1.0.0", + "is-fullwidth-code-point": "^1.0.0", + "strip-ansi": "^3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/string.prototype.matchall": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/string.prototype.matchall/-/string.prototype.matchall-4.0.5.tgz", + "integrity": "sha512-Z5ZaXO0svs0M2xd/6By3qpeKpLKd9mO4v4q3oMEQrk8Ck4xOD5d5XeBOOjGrmVZZ/AHB1S0CgG4N5r1G9N3E2Q==", + "dev": true, + "dependencies": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.3", + "es-abstract": "^1.18.2", + "get-intrinsic": "^1.1.1", + "has-symbols": "^1.0.2", + "internal-slot": "^1.0.3", + "regexp.prototype.flags": "^1.3.1", + "side-channel": "^1.0.4" + } + }, + "node_modules/string.prototype.trimend": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/string.prototype.trimend/-/string.prototype.trimend-1.0.4.tgz", + "integrity": "sha512-y9xCjw1P23Awk8EvTpcyL2NIr1j7wJ39f+k6lvRnSMz+mz9CGz9NYPelDk42kOz6+ql8xjfK8oYzy3jAP5QU5A==", + "dev": true, + "dependencies": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.3" + } + }, + "node_modules/string.prototype.trimstart": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/string.prototype.trimstart/-/string.prototype.trimstart-1.0.4.tgz", + "integrity": "sha512-jh6e984OBfvxS50tdY2nRZnoC5/mLFKOREQfw8t5yytkoUsJRNxvI/E39qu1sD0OtWI3OC0XgKSmcWwziwYuZw==", + "dev": true, + "dependencies": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.3" + } + }, + "node_modules/strip-ansi": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", + "dependencies": { + "ansi-regex": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/strip-bom": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-4.0.0.tgz", + "integrity": "sha512-3xurFv5tEgii33Zi8Jtp55wEIILR9eh34FAW00PZf+JnSsTmV/ioewSgQl97JHvgjoRGwPShsWm+IdrxB35d0w==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/supports-color": { + "version": "8.1.1", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", + "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==", + "dev": true, + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/table": { + "version": "5.4.6", + "resolved": "https://registry.npmjs.org/table/-/table-5.4.6.tgz", + "integrity": "sha512-wmEc8m4fjnob4gt5riFRtTu/6+4rSe12TpAELNSqHMfF3IqnA+CH37USM6/YR3qRZv7e56kAEAtd6nKZaxe0Ug==", + "dev": true, + "dependencies": { + "ajv": "^6.10.2", + "lodash": "^4.17.14", + "slice-ansi": "^2.1.0", + "string-width": "^3.0.0" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/table/node_modules/ansi-regex": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", + "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/table/node_modules/emoji-regex": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz", + "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==", + "dev": true + }, + "node_modules/table/node_modules/is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/table/node_modules/string-width": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", + "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", + "dev": true, + "dependencies": { + "emoji-regex": "^7.0.1", + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^5.1.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/table/node_modules/strip-ansi": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", + "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", + "dev": true, + "dependencies": { + "ansi-regex": "^4.1.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/tar": { + "version": "6.1.11", + "resolved": "https://registry.npmjs.org/tar/-/tar-6.1.11.tgz", + "integrity": "sha512-an/KZQzQUkZCkuoAA64hM92X0Urb6VpRhAFllDzz44U2mcD5scmT3zBc4VgVpkugF580+DQn8eAFSyoQt0tznA==", + "dependencies": { + "chownr": "^2.0.0", + "fs-minipass": "^2.0.0", + "minipass": "^3.0.0", + "minizlib": "^2.1.1", + "mkdirp": "^1.0.3", + "yallist": "^4.0.0" + }, + "engines": { + "node": ">= 10" + } + }, + "node_modules/tar-fs": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz", + "integrity": "sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==", + "dependencies": { + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.1.4" + } + }, + "node_modules/tar-stream": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "dependencies": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/tar-stream/node_modules/readable-stream": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", + "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/tar/node_modules/chownr": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz", + "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==", + "engines": { + "node": ">=10" + } + }, + "node_modules/test-exclude": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-6.0.0.tgz", + "integrity": "sha512-cAGWPIyOHU6zlmg88jwm7VRyXnMN7iV68OGAbYDk/Mh/xC/pzVPlQtY6ngoIH/5/tciuhGfvESU8GrHrcxD56w==", + "dev": true, + "dependencies": { + "@istanbuljs/schema": "^0.1.2", + "glob": "^7.1.4", + "minimatch": "^3.0.4" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/text-table": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", + "integrity": "sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=", + "dev": true + }, + "node_modules/through": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", + "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=" + }, + "node_modules/to-fast-properties": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz", + "integrity": "sha1-3F5pjL0HkmW8c+A3doGk5Og/YW4=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dev": true, + "dependencies": { + "is-number": "^7.0.0" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/toidentifier": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz", + "integrity": "sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw==", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/tsconfig-paths": { + "version": "3.11.0", + "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.11.0.tgz", + "integrity": "sha512-7ecdYDnIdmv639mmDwslG6KQg1Z9STTz1j7Gcz0xa+nshh/gKDAHcPxRbWOsA3SPp0tXP2leTcY9Kw+NAkfZzA==", + "dev": true, + "dependencies": { + "@types/json5": "^0.0.29", + "json5": "^1.0.1", + "minimist": "^1.2.0", + "strip-bom": "^3.0.0" + } + }, + "node_modules/tsconfig-paths/node_modules/json5": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/json5/-/json5-1.0.1.tgz", + "integrity": "sha512-aKS4WQjPenRxiQsC93MNfjx+nbF4PAdYzmd/1JIj8HYzqfbu86beTuNgXDzPknWk0n0uARlyewZo4s++ES36Ow==", + "dev": true, + "dependencies": { + "minimist": "^1.2.0" + }, + "bin": { + "json5": "lib/cli.js" + } + }, + "node_modules/tsconfig-paths/node_modules/strip-bom": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz", + "integrity": "sha1-IzTBjpx1n3vdVv3vfprj1YjmjtM=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=", + "dependencies": { + "safe-buffer": "^5.0.1" + }, + "engines": { + "node": "*" + } + }, + "node_modules/type-check": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", + "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==", + "dev": true, + "dependencies": { + "prelude-ls": "^1.2.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/type-detect": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", + "integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/type-fest": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.8.1.tgz", + "integrity": "sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/type-is": { + "version": "1.6.18", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "dependencies": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/typedarray-to-buffer": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/typedarray-to-buffer/-/typedarray-to-buffer-3.1.5.tgz", + "integrity": "sha512-zdu8XMNEDepKKR+XYOXAVPtWui0ly0NtohUscw+UmaHiAWT8hrV1rr//H6V+0DvJ3OQ19S979M0laLfX8rm82Q==", + "dev": true, + "dependencies": { + "is-typedarray": "^1.0.0" + } + }, + "node_modules/unbox-primitive": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/unbox-primitive/-/unbox-primitive-1.0.1.tgz", + "integrity": "sha512-tZU/3NqK3dA5gpE1KtyiJUrEB0lxnGkMFHptJ7q6ewdZ8s12QrODwNbhIJStmJkd1QDXa1NRA8aF2A1zk/Ypyw==", + "dev": true, + "dependencies": { + "function-bind": "^1.1.1", + "has-bigints": "^1.0.1", + "has-symbols": "^1.0.2", + "which-boxed-primitive": "^1.0.2" + } + }, + "node_modules/unorm": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/unorm/-/unorm-1.4.1.tgz", + "integrity": "sha1-NkIA1fE2RsqLzURJAnEzVhR5IwA=", + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/uri-js": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", + "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", + "dev": true, + "dependencies": { + "punycode": "^2.1.0" + } + }, + "node_modules/url-parse": { + "version": "1.5.3", + "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.3.tgz", + "integrity": "sha512-IIORyIQD9rvj0A4CLWsHkBBJuNqWpFQe224b6j9t/ABmquIS0qDU2pY6kl6AuOrL5OkCXHMCFNe1jBcuAggjvQ==", + "dependencies": { + "querystringify": "^2.1.1", + "requires-port": "^1.0.0" + } + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" + }, + "node_modules/utils-merge": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", + "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=", + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/uuid": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", + "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==", + "dev": true, + "bin": { + "uuid": "bin/uuid" + } + }, + "node_modules/v8-compile-cache": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.3.0.tgz", + "integrity": "sha512-l8lCEmLcLYZh4nbunNZvQCJc5pv7+RCwa8q/LdUx8u7lsWvPDKmpodJAJNwkAhJC//dFY48KuIEmjtd4RViDrA==", + "dev": true + }, + "node_modules/validate-npm-package-license": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz", + "integrity": "sha512-DpKm2Ui/xN7/HQKCtpZxoRWBhZ9Z0kqtygG8XCgNQ8ZlDnxuQmWhj566j8fN4Cu3/JmbhsDo7fcAJq4s9h27Ew==", + "dev": true, + "dependencies": { + "spdx-correct": "^3.0.0", + "spdx-expression-parse": "^3.0.0" + } + }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/which-boxed-primitive": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz", + "integrity": "sha512-bwZdv0AKLpplFY2KZRX6TvyuN7ojjr7lwkg6ml0roIy9YeuSr7JS372qlNW18UQYzgYK9ziGcerWqZOmEn9VNg==", + "dev": true, + "dependencies": { + "is-bigint": "^1.0.1", + "is-boolean-object": "^1.1.0", + "is-number-object": "^1.0.4", + "is-string": "^1.0.5", + "is-symbol": "^1.0.3" + } + }, + "node_modules/which-module": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz", + "integrity": "sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=", + "dev": true + }, + "node_modules/wide-align": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.3.tgz", + "integrity": "sha512-QGkOQc8XL6Bt5PwnsExKBPuMKBxnGxWWW3fU55Xt4feHozMUhdUMaBCk290qpm/wG5u/RSKzwdAC4i51YigihA==", + "dependencies": { + "string-width": "^1.0.2 || 2" + } + }, + "node_modules/word-wrap": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz", + "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/workerpool": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-6.1.0.tgz", + "integrity": "sha512-toV7q9rWNYha963Pl/qyeZ6wG+3nnsyvolaNUS8+R5Wtw6qJPTxIlOP1ZSvcGhEJw+l3HMMmtiNo9Gl61G4GVg==", + "dev": true + }, + "node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/wrap-ansi/node_modules/ansi-regex": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", + "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi/node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi/node_modules/string-width": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.2.tgz", + "integrity": "sha512-XBJbT3N4JhVumXE0eoLU9DCjcaF92KLNqTmFCnG1pf8duUxFGwtP6AD6nkjw9a3IdiRtL3E2w3JDiE/xi3vOeA==", + "dev": true, + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi/node_modules/strip-ansi": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", + "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==", + "dev": true, + "dependencies": { + "ansi-regex": "^5.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" + }, + "node_modules/write": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/write/-/write-1.0.3.tgz", + "integrity": "sha512-/lg70HAjtkUgWPVZhZcm+T4hkL8Zbtp1nFNOn3lRrxnlv50SRBv7cR7RqR+GMsd3hUXy9hWBo4CHTbFTcOYwig==", + "dev": true, + "dependencies": { + "mkdirp": "^0.5.1" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/write-file-atomic": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-3.0.3.tgz", + "integrity": "sha512-AvHcyZ5JnSfq3ioSyjrBkH9yW4m7Ayk8/9My/DD9onKeu/94fwrMocemO2QAJFAlnnDN+ZDS+ZjAR5ua1/PV/Q==", + "dev": true, + "dependencies": { + "imurmurhash": "^0.1.4", + "is-typedarray": "^1.0.0", + "signal-exit": "^3.0.2", + "typedarray-to-buffer": "^3.1.5" + } + }, + "node_modules/write/node_modules/mkdirp": { + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz", + "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", + "dev": true, + "dependencies": { + "minimist": "^1.2.5" + }, + "bin": { + "mkdirp": "bin/cmd.js" + } + }, + "node_modules/xdg-basedir": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/xdg-basedir/-/xdg-basedir-4.0.0.tgz", + "integrity": "sha512-PSNhEJDejZYV7h50BohL09Er9VaIefr2LMAf3OEmpCkjOi34eYyQYAXUTjEQtZJTKcF0E2UKTh+osDLsgNim9Q==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "dev": true, + "engines": { + "node": ">=10" + } + }, + "node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + }, + "node_modules/yargs": { + "version": "16.2.0", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz", + "integrity": "sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==", + "dev": true, + "dependencies": { + "cliui": "^7.0.2", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.0", + "y18n": "^5.0.5", + "yargs-parser": "^20.2.2" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/yargs-parser": { + "version": "20.2.4", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.4.tgz", + "integrity": "sha512-WOkpgNhPTlE73h4VFAFsOnomJVaovO8VqLDzy5saChRBFQFBoMYirowyW+Q9HB4HFF4Z7VZTiG3iSzJJA29yRA==", + "dev": true, + "engines": { + "node": ">=10" + } + }, + "node_modules/yargs-unparser": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/yargs-unparser/-/yargs-unparser-2.0.0.tgz", + "integrity": "sha512-7pRTIA9Qc1caZ0bZ6RYRGbHJthJWuakf+WmHK0rVeLkNrrGhfoabBNdue6kdINI6r4if7ocq9aD/n7xwKOdzOA==", + "dev": true, + "dependencies": { + "camelcase": "^6.0.0", + "decamelize": "^4.0.0", + "flat": "^5.0.2", + "is-plain-obj": "^2.1.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/yargs/node_modules/ansi-regex": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", + "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/yargs/node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/yargs/node_modules/string-width": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.2.tgz", + "integrity": "sha512-XBJbT3N4JhVumXE0eoLU9DCjcaF92KLNqTmFCnG1pf8duUxFGwtP6AD6nkjw9a3IdiRtL3E2w3JDiE/xi3vOeA==", + "dev": true, + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/yargs/node_modules/strip-ansi": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", + "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==", + "dev": true, + "dependencies": { + "ansi-regex": "^5.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true, + "engines": { + "node": ">=10" + } + }, + "node_modules/zeromq": { + "version": "5.2.8", + "resolved": "https://registry.npmjs.org/zeromq/-/zeromq-5.2.8.tgz", + "integrity": "sha512-bXzsk7KOmgLSv1tC0Ms1VXBy90+Rz27ZYf27cLuldRYbpqYpuWJfxxHFhO710t22zgWBnmdUP0m3SKFpLI0u5g==", + "hasInstallScript": true, + "optional": true, + "dependencies": { + "nan": "2.14.2", + "node-gyp-build": "^4.2.3" + }, + "engines": { + "node": ">=6.0" + } + } + }, "dependencies": { "@babel/code-frame": { "version": "7.14.5", @@ -4262,6 +10324,14 @@ "through": "~2.3.4" } }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "requires": { + "safe-buffer": "~5.1.0" + } + }, "string-width": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", @@ -4308,14 +10378,6 @@ "define-properties": "^1.1.3" } }, - "string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "requires": { - "safe-buffer": "~5.1.0" - } - }, "strip-ansi": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", diff --git a/src/database.js b/src/database.js index 7463cba..108752a 100644 --- a/src/database.js +++ b/src/database.js @@ -468,11 +468,12 @@ class Database { async transaction (f) { if (!this.db) return try { - this.db.raw('beggin') + // this.db.exec('begin;') await f() - this.db.raw('commit') + // this.db.exec('commit;') } catch (e) { - this.db.raw('rollback') + // this.db.exec('rollback;') + console.error(e) throw e } // this.db.transaction(f)() @@ -484,18 +485,18 @@ class Database { async addBlock (txids, txhexs, height, hash, time) { await this.transaction(async () => { - const indexes = new Array(txids.length).map((_, i) => i) - for(const index of indexes) { + const indexes = new Array(txids.length).fill(null).map((_, i) => i) + for (const index of indexes) { const txid = txids[index] - const txHex = txhexs && txhexs[i] + const txHex = txhexs && txhexs[index] await this.addTransaction(txid, txHex, height, time) } // txids.forEach(async (txid, i) => { // const txhex = txhexs && txhexs[i] // await this.addTransaction(txid, txhex, height, time) // }) - this.setHeight(height) - this.setHash(hash) + await this.setHeight(height) + await this.setHash(hash) }) } @@ -517,7 +518,7 @@ class Database { } async parseAndStoreTransaction (txid, hex) { - if (this.isTransactionDownloaded(txid)) return + if (await this.isTransactionDownloaded(txid)) return let metadata = null let bsvtx = null @@ -542,7 +543,7 @@ class Database { metadata = Run.util.metadata(hex) } catch (e) { this.logger.error(`${txid} => ${e.message}`) - this.storeParsedNonExecutableTransaction(txid, hex, inputs, outputs) + await this.storeParsedNonExecutableTransaction(txid, hex, inputs, outputs) return } @@ -567,34 +568,34 @@ class Database { const hasCode = metadata.exec.some(cmd => cmd.op === 'DEPLOY' || cmd.op === 'UPGRADE') - this.storeParsedExecutableTransaction(txid, hex, hasCode, deps, inputs, outputs) + await this.storeParsedExecutableTransaction(txid, hex, hasCode, deps, inputs, outputs) for (const deptxid of deps) { - if (!this.isTransactionDownloaded(deptxid)) { + if (!await this.isTransactionDownloaded(deptxid)) { if (this.onRequestDownload) this.onRequestDownload(deptxid) } } } - addNewTransaction (txid) { - if (this.hasTransaction(txid)) return + async addNewTransaction (txid) { + if (await this.hasTransaction(txid)) return const time = Math.round(Date.now() / 1000) this.addNewTransactionStmt.run(txid, time) - if (this.onAddTransaction) this.onAddTransaction(txid) + if (this.onAddTransaction) { await this.onAddTransaction(txid) } } - setTransactionHeight (txid, height) { + async setTransactionHeight (txid, height) { this.setTransactionHeightStmt.run(height, txid) } - setTransactionTime (txid, time) { + async setTransactionTime (txid, time) { this.setTransactionTimeStmt.run(time, txid) } - storeParsedNonExecutableTransaction (txid, hex, inputs, outputs) { + async storeParsedNonExecutableTransaction (txid, hex, inputs, outputs) { await this.transaction(() => { const bytes = Buffer.from(hex, 'hex') this.setTransactionBytesStmt.run(bytes, txid) @@ -612,8 +613,8 @@ class Database { }) } - storeParsedExecutableTransaction (txid, hex, hasCode, deps, inputs, outputs) { - await this.transaction(() => { + async storeParsedExecutableTransaction (txid, hex, hasCode, deps, inputs, outputs) { + await this.transaction(async () => { const bytes = Buffer.from(hex, 'hex') this.setTransactionBytesStmt.run(bytes, txid) this.setTransactionExecutableStmt.run(1, txid) @@ -623,20 +624,20 @@ class Database { outputs.forEach(location => this.setUnspentStmt.run(location)) for (const deptxid of deps) { - this.addNewTransaction(deptxid) + await this.addNewTransaction(deptxid) this.addDepStmt.run(deptxid, txid) if (this.getTransactionFailedStmt.get(deptxid).failed) { - this.setTransactionExecutionFailed(txid) + await this.setTransactionExecutionFailed(txid) return } } }) - this._checkExecutability(txid) + await this._checkExecutability(txid) } - storeExecutedTransaction (txid, result) { + async storeExecutedTransaction (txid, result) { const { cache, classes, locks, scripthashes } = result await this.transaction(() => { @@ -678,7 +679,7 @@ class Database { }) } - setTransactionExecutionFailed (txid) { + async setTransactionExecutionFailed (txid) { await this.transaction(() => { this.setTransactionExecutableStmt.run(0, txid) this.setTransactionExecutedStmt.run(1, txid) @@ -691,33 +692,35 @@ class Database { let executable = false try { - const rawtx = this.getTransactionHex(txid) - Run.util.metadata(rawtx) + const rawTx = await this.getTransactionHex(txid) + Run.util.metadata(rawTx) executable = true } catch (e) { } if (!executable) { const downstream = this.getDownstreamStmt.raw(true).all(txid).map(x => x[0]) - downstream.forEach(downtxid => this._checkExecutability(downtxid)) + for (const downtxid of downstream) { + await this._checkExecutability(downtxid) + } } } - getTransactionHex (txid) { + async getTransactionHex (txid) { const row = this.getTransactionHexStmt.raw(true).get(txid) return row && row[0] } - getTransactionTime (txid) { + async getTransactionTime (txid) { const row = this.getTransactionTimeStmt.raw(true).get(txid) return row && row[0] } - getTransactionHeight (txid) { + async getTransactionHeight (txid) { const row = this.getTransactionHeightStmt.raw(true).get(txid) return row && row[0] } - deleteTransaction (txid, deleted = new Set()) { + async deleteTransaction (txid, deleted = new Set()) { if (deleted.has(txid)) return const txids = [txid] @@ -747,12 +750,12 @@ class Database { }) } - unconfirmTransaction (txid) { + async unconfirmTransaction (txid) { this.unconfirmTransactionStmt.run(txid) } - unindexTransaction (txid) { - await this.transaction(() => { + async unindexTransaction (txid) { + await this.transaction(async () => { if (this.getTransactionIndexedStmt.raw(true).get(txid)[0]) { this.setTransactionExecutedStmt.run(0, txid) this.setTransactionIndexedStmt.run(0, txid) @@ -760,31 +763,38 @@ class Database { this.deleteBerryStatesStmt.run(txid) this.unmarkExecutingStmt.run(txid) - const downtxids = this.getDownstreamStmt.raw(true).all(txid).map(row => row[0]) - downtxids.forEach(downtxid => this.unindexTransaction(downtxid)) + const downloadedTxids = this.getDownstreamStmt.raw(true).all(txid).map(row => row[0]) + for (const downloadedTxid of downloadedTxids) { + await this.unindexTransaction(downloadedTxid) + } - if (this.onUnindexTransaction) this.onUnindexTransaction(txid) + if (this.onUnindexTransaction) { await this.onUnindexTransaction(txid) } } }) } - hasTransaction (txid) { return !!this.hasTransactionStmt.get(txid) } - isTransactionDownloaded (txid) { + async hasTransaction (txid) { return !!this.hasTransactionStmt.get(txid) } + + async isTransactionDownloaded (txid) { const result = this.getTransactionDownloadedStmt.raw(true).get(txid) - return result && !!result[0] + return result ? !!result[0] : false } - getTransactionsAboveHeight (height) { return this.getTransactionsAboveHeightStmt.raw(true).all(height).map(row => row[0]) } - getMempoolTransactionsBeforeTime (time) { return this.getMempoolTransactionsBeforeTimeStmt.raw(true).all(time).map(row => row[0]) } - getTransactionsToDownload () { return this.getTransactionsToDownloadStmt.raw(true).all().map(row => row[0]) } - getDownloadedCount () { return this.getTransactionsDownloadedCountStmt.get().count } - getIndexedCount () { return this.getTransactionsIndexedCountStmt.get().count } + async getTransactionsAboveHeight (height) { return this.getTransactionsAboveHeightStmt.raw(true).all(height).map(row => row[0]) } + + async getMempoolTransactionsBeforeTime (time) { return this.getMempoolTransactionsBeforeTimeStmt.raw(true).all(time).map(row => row[0]) } + + async getTransactionsToDownload () { return this.getTransactionsToDownloadStmt.raw(true).all().map(row => row[0]) } + + async getDownloadedCount () { return this.getTransactionsDownloadedCountStmt.get().count } + + async getIndexedCount () { return this.getTransactionsIndexedCountStmt.get().count } // -------------------------------------------------------------------------- // spends // -------------------------------------------------------------------------- - getSpend (location) { + async getSpend (location) { const row = this.getSpendStmt.raw(true).get(location) return row && row[0] } @@ -793,27 +803,31 @@ class Database { // deps // -------------------------------------------------------------------------- - addDep (txid, deptxid) { - this.addNewTransaction(deptxid) + async addDep (txid, deptxid) { + await this.addNewTransaction(deptxid) this.addDepStmt.run(deptxid, txid) if (this.getTransactionFailedStmt.get(deptxid).failed) { - this.setTransactionExecutionFailed(deptxid) + await this.setTransactionExecutionFailed(deptxid) } } - addMissingDeps (txid, deptxids) { - await this.transaction(() => deptxids.forEach(deptxid => this.addDep(txid, deptxid))) + async addMissingDeps (txid, deptxids) { + await this.transaction(async () => { + for (const deptxid of deptxids) { + await this.addDep(txid, deptxid) + } + }) - this._checkExecutability(txid) + await this._checkExecutability(txid) } // -------------------------------------------------------------------------- // jig // -------------------------------------------------------------------------- - getJigState (location) { + async getJigState (location) { const row = this.getJigStateStmt.raw(true).get(location) return row && row[0] } @@ -822,39 +836,39 @@ class Database { // unspent // -------------------------------------------------------------------------- - getAllUnspent () { + async getAllUnspent () { return this.getAllUnspentStmt.raw(true).all().map(row => row[0]) } - getAllUnspentByClassOrigin (origin) { + async getAllUnspentByClassOrigin (origin) { return this.getAllUnspentByClassStmt.raw(true).all(origin).map(row => row[0]) } - getAllUnspentByLockOrigin (origin) { + async getAllUnspentByLockOrigin (origin) { return this.getAllUnspentByLockStmt.raw(true).all(origin).map(row => row[0]) } - getAllUnspentByScripthash (scripthash) { + async getAllUnspentByScripthash (scripthash) { return this.getAllUnspentByScripthashStmt.raw(true).all(scripthash).map(row => row[0]) } - getAllUnspentByClassOriginAndLockOrigin (clsOrigin, lockOrigin) { + async getAllUnspentByClassOriginAndLockOrigin (clsOrigin, lockOrigin) { return this.getAllUnspentByClassLockStmt.raw(true).all(clsOrigin, lockOrigin).map(row => row[0]) } - getAllUnspentByClassOriginAndScripthash (clsOrigin, scripthash) { + async getAllUnspentByClassOriginAndScripthash (clsOrigin, scripthash) { return this.getAllUnspentByClassScripthashStmt.raw(true).all(clsOrigin, scripthash).map(row => row[0]) } - getAllUnspentByLockOriginAndScripthash (lockOrigin, scripthash) { + async getAllUnspentByLockOriginAndScripthash (lockOrigin, scripthash) { return this.getAllUnspentByLockScripthashStmt.raw(true).all(lockOrigin, scripthash).map(row => row[0]) } - getAllUnspentByClassOriginAndLockOriginAndScripthash (clsOrigin, lockOrigin, scripthash) { + async getAllUnspentByClassOriginAndLockOriginAndScripthash (clsOrigin, lockOrigin, scripthash) { return this.getAllUnspentByClassLockScripthashStmt.raw(true).all(clsOrigin, lockOrigin, scripthash).map(row => row[0]) } - getNumUnspent () { + async getNumUnspent () { return this.getNumUnspentStmt.get().unspent } @@ -862,7 +876,7 @@ class Database { // berry // -------------------------------------------------------------------------- - getBerryState (location) { + async getBerryState (location) { const row = this.getBerryStateStmt.raw(true).get(location) return row && row[0] } @@ -871,13 +885,13 @@ class Database { // trust // -------------------------------------------------------------------------- - isTrusted (txid) { + async isTrusted (txid) { const row = this.isTrustedStmt.raw(true).get(txid) return !!row && !!row[0] } - trust (txid) { - if (this.isTrusted(txid)) return + async trust (txid) { + if (await this.isTrusted(txid)) return const trusted = [txid] @@ -887,7 +901,7 @@ class Database { while (queue.length) { const uptxid = queue.shift() if (visited.has(uptxid)) continue - if (this.isTrusted(uptxid)) continue + if (await this.isTrusted(uptxid)) continue visited.add(uptxid) trusted.push(txid) this.getUpstreamUnexecutedCodeStmt.raw(true).all(txid).forEach(x => queue.push(x[0])) @@ -895,21 +909,23 @@ class Database { await this.transaction(() => trusted.forEach(txid => this.setTrustedStmt.run(txid, 1))) - trusted.forEach(txid => this._checkExecutability(txid)) + for (const txid of trusted) { + await this._checkExecutability(txid) + } if (this.onTrustTransaction) trusted.forEach(txid => this.onTrustTransaction(txid)) } - untrust (txid) { - if (!this.isTrusted(txid)) return - await this.transaction(() => { - this.unindexTransaction(txid) + async untrust (txid) { + if (!await this.isTrusted(txid)) return + await this.transaction(async () => { + await this.unindexTransaction(txid) this.setTrustedStmt.run(txid, 0) }) if (this.onUntrustTransaction) this.onUntrustTransaction(txid) } - getTrustlist () { + async getTrustlist () { return this.getTrustlistStmt.raw(true).all().map(x => x[0]) } @@ -917,26 +933,26 @@ class Database { // ban // -------------------------------------------------------------------------- - isBanned (txid) { + async isBanned (txid) { const row = this.isBannedStmt.raw(true).get(txid) return !!row && !!row[0] } - ban (txid) { - await this.transaction(() => { - this.unindexTransaction(txid) + async ban (txid) { + await this.transaction(async () => { + await this.unindexTransaction(txid) this.banStmt.run(txid) }) if (this.onBanTransaction) this.onBanTransaction(txid) } - unban (txid) { + async unban (txid) { this.unbanStmt.run(txid) - this._checkExecutability(txid) + await this._checkExecutability(txid) if (this.onUnbanTransaction) this.onUnbanTransaction(txid) } - getBanlist () { + async getBanlist () { return this.getBanlistStmt.raw(true).all().map(x => x[0]) } @@ -944,21 +960,21 @@ class Database { // crawl // -------------------------------------------------------------------------- - getHeight () { + async getHeight () { const row = this.getHeightStmt.raw(true).all()[0] return row && parseInt(row[0]) } - getHash () { + async getHash () { const row = this.getHashStmt.raw(true).all()[0] return row && row[0] } - setHeight (height) { + async setHeight (height) { this.setHeightStmt.run(height.toString()) } - setHash (hash) { + async setHash (hash) { this.setHashStmt.run(hash) } @@ -966,13 +982,15 @@ class Database { // internal // -------------------------------------------------------------------------- - loadTransactionsToExecute () { + async loadTransactionsToExecute () { this.logger.debug('Loading transactions to execute') const txids = this.db.prepare('SELECT txid FROM executing').raw(true).all().map(x => x[0]) - txids.forEach(txid => this._checkExecutability(txid)) + for (const txid of txids) { + await this._checkExecutability(txid) + } } - _checkExecutability (txid) { + async _checkExecutability (txid) { const row = this.isReadyToExecuteStmt.get(txid) if (row && row.ready) { this.markExecutingStmt.run(txid) diff --git a/src/executor.js b/src/executor.js index 0d61590..a72959e 100644 --- a/src/executor.js +++ b/src/executor.js @@ -74,11 +74,11 @@ class Executor { worker.missingDeps = new Set() - const hex = this.database.getTransactionHex(txid) - const trustlist = this.database.getTrustlist() + const hex = await this.database.getTransactionHex(txid) + const trustList = await this.database.getTrustlist() try { - const result = await Bus.sendRequest(worker, 'execute', txid, hex, trustlist) + const result = await Bus.sendRequest(worker, 'execute', txid, hex, trustList) if (this.onIndexed) this.onIndexed(txid, result) } catch (e) { @@ -107,27 +107,27 @@ class Executor { return worker } - return new Promise((resolve, reject) => { + return new Promise((resolve, _reject) => { this.workerRequests.push(resolve) }) } - _onCacheGet (key) { + async _onCacheGet (key) { if (key.startsWith('jig://')) { - const state = this.database.getJigState(key.slice('jig://'.length)) + const state = await this.database.getJigState(key.slice('jig://'.length)) if (state) return JSON.parse(state) } if (key.startsWith('berry://')) { - const state = this.database.getBerryState(key.slice('berry://'.length)) + const state = await this.database.getBerryState(key.slice('berry://'.length)) if (state) return JSON.parse(state) } if (key.startsWith('tx://')) { - return this.database.getTransactionHex(key.slice('tx://'.length)) + return await this.database.getTransactionHex(key.slice('tx://'.length)) } } - _onBlockchainFetch (worker, txid) { - const hex = this.database.getTransactionHex(txid) + async _onBlockchainFetch (worker, txid) { + const hex = await this.database.getTransactionHex(txid) if (hex) return hex worker.missingDeps.add(txid) throw new Error(`Not found: ${txid}`) diff --git a/src/indexer.js b/src/indexer.js index 64f5ee2..d747288 100644 --- a/src/indexer.js +++ b/src/indexer.js @@ -62,14 +62,18 @@ class Indexer { this.logger.debug('Starting indexer') this.executor.start() - this.defaultTrustlist.forEach(txid => this.database.trust(txid)) - this.database.loadTransactionsToExecute() - const height = this.database.getHeight() || this.startHeight - const hash = this.database.getHash() + for (const txid of this.defaultTrustlist) { + await this.database.trust(txid) + } + + await this.database.loadTransactionsToExecute() + const height = await this.database.getHeight() || this.startHeight + const hash = await this.database.getHash() if (this.api.connect) await this.api.connect(height, this.network) this.logger.debug('Loading transactions to download') - this.database.getTransactionsToDownload().forEach(txid => this.downloader.add(txid)) + const txsToDownload = await this.database.getTransactionsToDownload() + txsToDownload.forEach(txid => this.downloader.add(txid)) this.crawler.start(height, hash) } @@ -81,12 +85,12 @@ class Indexer { await this.executor.stop() } - _onDownloadTransaction (txid, hex, height, time) { + async _onDownloadTransaction (txid, hex, height, time) { this.logger.info(`Downloaded ${txid} (${this.downloader.remaining()} remaining)`) - if (!this.database.hasTransaction(txid)) return - if (height) this.database.setTransactionHeight(txid, height) - if (time) this.database.setTransactionTime(txid, time) - this.database.parseAndStoreTransaction(txid, hex) + if (!await this.database.hasTransaction(txid)) return + if (height) { await this.database.setTransactionHeight(txid, height) } + if (time) { await this.database.setTransactionTime(txid, time) } + await this.database.parseAndStoreTransaction(txid, hex) if (this.onDownload) this.onDownload(txid) } @@ -99,16 +103,16 @@ class Indexer { this.logger.info('Retrying download', txid, 'after', secondsToRetry, 'seconds') } - _onIndexed (txid, result) { - if (!this.database.hasTransaction(txid)) return // Check not re-orged + async _onIndexed (txid, result) { + if (!await this.database.hasTransaction(txid)) return // Check not re-orged this.logger.info(`Executed ${txid}`) - this.database.storeExecutedTransaction(txid, result) + await this.database.storeExecutedTransaction(txid, result) if (this.onIndex) this.onIndex(txid) } - _onExecuteFailed (txid, e) { + async _onExecuteFailed (txid, e) { this.logger.error(`Failed to execute ${txid}: ${e.toString()}`) - this.database.setTransactionExecutionFailed(txid) + await this.database.setTransactionExecutionFailed(txid) if (this.onFailToIndex) this.onFailToIndex(txid, e) } @@ -149,9 +153,9 @@ class Indexer { this.downloader.add(txid) } - _onMissingDeps (txid, deptxids) { + async _onMissingDeps (txid, deptxids) { this.logger.debug(`Discovered ${deptxids.length} dep(s) for ${txid}`) - this.database.addMissingDeps(txid, deptxids) + await this.database.addMissingDeps(txid, deptxids) deptxids.forEach(deptxid => this.downloader.add(deptxid)) } @@ -161,38 +165,44 @@ class Indexer { async _onCrawlBlockTransactions (height, hash, time, txids, txhexs) { this.logger.info(`Crawled block ${height} for ${txids.length} transactions`) - this.database.addBlock(txids, txhexs, height, hash, time) + await this.database.addBlock(txids, txhexs, height, hash, time) if (this.onBlock) this.onBlock(height) } async _onRewindBlocks (newHeight) { this.logger.info(`Rewinding to block ${newHeight}`) - const txids = this.database.getTransactionsAboveHeight(newHeight) + const txids = await this.database.getTransactionsAboveHeight(newHeight) - await this.database.transaction(() => { + await this.database.transaction(async () => { // Put all transactions back into the mempool. This is better than deleting them, because // when we assume they will just go into a different block, we don't need to re-execute. // If they don't make it into a block, then they will be expired in time. - txids.forEach(txid => this.database.unconfirmTransaction(txid)) + for (const txid of txids) { + await this.database.unconfirmTransaction(txid) + } - this.database.setHeight(newHeight) - this.database.setHash(null) + await this.database.setHeight(newHeight) + await this.database.setHash(null) }) if (this.onReorg) this.onReorg(newHeight) } - _onMempoolTransaction (txid, hex) { - this.database.addTransaction(txid, hex, Database.HEIGHT_MEMPOOL, null) + async _onMempoolTransaction (txid, hex) { + await this.database.addTransaction(txid, hex, Database.HEIGHT_MEMPOOL, null) } async _onExpireMempoolTransactions () { const expirationTime = Math.round(Date.now() / 1000) - this.mempoolExpiration - const expired = this.database.getMempoolTransactionsBeforeTime(expirationTime) + const expired = await this.database.getMempoolTransactionsBeforeTime(expirationTime) const deleted = new Set() - await this.database.transaction(() => expired.forEach(txid => this.database.deleteTransaction(txid, deleted))) + await this.database.transaction(async () => { + for (const txid of expired) { + await this.database.deleteTransaction(txid, deleted) + } + }) } } diff --git a/src/server.js b/src/server.js index 7d6b01c..98bee31 100644 --- a/src/server.js +++ b/src/server.js @@ -67,8 +67,7 @@ class Server { app.post('/trust/:txid?', this.postTrust.bind(this)) app.post('/ban/:txid', this.postBan.bind(this)) - app.post('/tx', this.postTx.bind(this)) - app.post('/tx/:txid', this.postTx.bind(this)) // Keeping this for retro compatibility. + app.post('/tx/:txid', this.postTx.bind(this)) app.delete('/trust/:txid', this.deleteTrust.bind(this)) app.delete('/ban/:txid', this.deleteBan.bind(this)) @@ -94,7 +93,7 @@ class Server { async getJig (req, res, next) { try { - const state = this.database.getJigState(req.params.location) + const state = await this.database.getJigState(req.params.location) if (state) { res.setHeader('Content-Type', 'application/json') res.send(state) @@ -119,9 +118,9 @@ class Server { async getTx (req, res, next) { try { const txid = this._parseTxid(req.params.txid) - const rawtx = this.database.getTransactionHex(txid) - if (rawtx) { - res.send(rawtx) + const rawTx = await this.database.getTransactionHex(txid) + if (rawTx) { + res.send(rawTx) } else { res.status(404).send(`Not found: ${req.params.txid}\n`) } @@ -131,7 +130,7 @@ class Server { async getTime (req, res, next) { try { const txid = this._parseTxid(req.params.txid) - const time = this.database.getTransactionTime(txid) + const time = await this.database.getTransactionTime(txid) if (time) { res.json(time) } else { @@ -142,7 +141,7 @@ class Server { async getSpends (req, res, next) { try { - const txid = this.database.getSpend(req.params.location) + const txid = await this.database.getSpend(req.params.location) if (txid) { res.send(txid) } else { @@ -160,21 +159,21 @@ class Server { if (req.query.pubkey) scripthash = calculateScripthash(new Run.util.CommonLock(req.query.pubkey).script()) if (cls && lock && scripthash) { - res.json(this.database.getAllUnspentByClassOriginAndLockOriginAndScripthash(cls, lock, scripthash)) + res.json(await this.database.getAllUnspentByClassOriginAndLockOriginAndScripthash(cls, lock, scripthash)) } else if (cls && lock) { - res.json(this.database.getAllUnspentByClassOriginAndLockOrigin(cls, lock)) + res.json(await this.database.getAllUnspentByClassOriginAndLockOrigin(cls, lock)) } else if (cls && scripthash) { - res.json(this.database.getAllUnspentByClassOriginAndScripthash(cls, scripthash)) + res.json(await this.database.getAllUnspentByClassOriginAndScripthash(cls, scripthash)) } else if (lock && scripthash) { - res.json(this.database.getAllUnspentByLockOriginAndScripthash(lock, scripthash)) + res.json(await this.database.getAllUnspentByLockOriginAndScripthash(lock, scripthash)) } else if (scripthash) { - res.json(this.database.getAllUnspentByScripthash(scripthash)) + res.json(await this.database.getAllUnspentByScripthash(scripthash)) } else if (lock) { - res.json(this.database.getAllUnspentByLockOrigin(lock)) + res.json(await this.database.getAllUnspentByLockOrigin(lock)) } else if (cls) { - res.json(this.database.getAllUnspentByClassOrigin(cls)) + res.json(await this.database.getAllUnspentByClassOrigin(cls)) } else { - res.json(this.database.getAllUnspent()) + res.json(await this.database.getAllUnspent()) } } catch (e) { next(e) } } @@ -182,9 +181,9 @@ class Server { async getTrust (req, res, next) { try { if (req.params.txid) { - res.json(this.database.isTrusted(req.params.txid)) + res.json(await this.database.isTrusted(req.params.txid)) } else { - res.json(Array.from(this.database.getTrustlist())) + res.json(Array.from(await this.database.getTrustlist())) } } catch (e) { next(e) } } @@ -192,9 +191,9 @@ class Server { async getBan (req, res, next) { try { if (req.params.txid) { - res.json(this.database.isBanned(req.params.txid)) + res.json(await this.database.isBanned(req.params.txid)) } else { - res.json(Array.from(this.database.getBanlist())) + res.json(Array.from(await this.database.getBanlist())) } } catch (e) { next(e) } } @@ -202,8 +201,8 @@ class Server { async getStatus (req, res, next) { try { const status = { - height: this.database.getHeight(), - hash: this.database.getHash() + height: await this.database.getHeight(), + hash: await this.database.getHash() } res.json(status) } catch (e) { next(e) } @@ -212,14 +211,14 @@ class Server { async postTrust (req, res, next) { try { if (Array.isArray(req.body)) { - req.body.forEach(txid => { - txid = this._parseTxid(txid) - this.database.trust(txid) - }) + for (const maybeTxid of req.body) { + const txid = this._parseTxid(maybeTxid) + await this.database.trust(txid) + } res.send(`Trusted ${req.body.length} transactions\n`) } else { const txid = this._parseTxid(req.params.txid) - this.database.trust(txid) + await this.database.trust(txid) res.send(`Trusted ${req.params.txid}\n`) } } catch (e) { next(e) } @@ -228,28 +227,31 @@ class Server { async postBan (req, res, next) { try { const txid = this._parseTxid(req.params.txid) - this.database.ban(txid) + await this.database.ban(txid) res.send(`Banned ${req.params.txid}\n`) } catch (e) { next(e) } } async postTx (req, res, next) { try { - if (typeof req.body !== 'string') { - throw new Error('missing rawtx') + let txid = this._parseTxid(req.params.txid) + let hex = null + if (typeof req.body === 'string') { + hex = req.body + const bsvtx = new bsv.Transaction(hex) + if (!txid) txid = bsvtx.hash + if (txid && txid !== bsvtx.hash) throw new Error('txid does not match rawtx') } - const hex = req.body - const bsvtx = new bsv.Transaction(hex) - - this.database.addTransaction(bsvtx.hash, hex) - res.send(`Added ${bsvtx.hash}\n`) + if (!txid) throw new Error('Invalid request parameters') + await this.database.addTransaction(txid, hex) + res.send(`Added ${txid}\n`) } catch (e) { next(e) } } async deleteTrust (req, res, next) { try { const txid = this._parseTxid(req.params.txid) - this.database.untrust(txid) + await this.database.untrust(txid) res.send(`Untrusted ${req.params.txid}\n`) } catch (e) { next(e) } } @@ -257,7 +259,7 @@ class Server { async deleteBan (req, res, next) { try { const txid = this._parseTxid(req.params.txid) - this.database.unban(txid) + await this.database.unban(txid) res.send(`Unbanned ${req.params.txid}\n`) } catch (e) { next(e) } } @@ -265,7 +267,7 @@ class Server { async deleteTx (req, res, next) { try { const txid = this._parseTxid(req.params.txid) - this.database.deleteTransaction(txid) + await this.database.deleteTransaction(txid) res.send(`Removed ${req.params.txid}\n`) } catch (e) { next(e) } } diff --git a/test/indexer.js b/test/indexer.js index b5b04ed..0ee8c1b 100644 --- a/test/indexer.js +++ b/test/indexer.js @@ -182,7 +182,7 @@ describe('Indexer', () => { database.addTransaction(txid2, rawtx2) database.trust(txid1) await indexed(indexer, txid2) - expect(indexer.database.getNumUnspent()).to.equal(0) + expect(await indexer.database.getNumUnspent()).to.equal(0) await indexer.stop() }) }) From e876be2ed7c83d46194a5a186308f28fae3f73e7 Mon Sep 17 00:00:00 2001 From: hojarasca Date: Thu, 2 Dec 2021 12:24:52 -0300 Subject: [PATCH 03/82] more awaits --- src/database.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/database.js b/src/database.js index 108752a..527ff42 100644 --- a/src/database.js +++ b/src/database.js @@ -913,7 +913,11 @@ class Database { await this._checkExecutability(txid) } - if (this.onTrustTransaction) trusted.forEach(txid => this.onTrustTransaction(txid)) + if (this.onTrustTransaction) { + for (const txid of trusted) { + await this.onTrustTransaction(txid) + } + } } async untrust (txid) { From 8b07c637ee03ff36ef7d0d0c1300534af57afdfb Mon Sep 17 00:00:00 2001 From: Miguel Duarte Date: Thu, 2 Dec 2021 17:28:04 -0300 Subject: [PATCH 04/82] removed external transactions. --- src/database.js | 32 +++++++++++++++----------------- src/indexer.js | 27 +++++++++++---------------- 2 files changed, 26 insertions(+), 33 deletions(-) diff --git a/src/database.js b/src/database.js index 527ff42..9d5bfbb 100644 --- a/src/database.js +++ b/src/database.js @@ -468,11 +468,11 @@ class Database { async transaction (f) { if (!this.db) return try { - // this.db.exec('begin;') + this.db.exec('begin;') await f() - // this.db.exec('commit;') + this.db.exec('commit;') } catch (e) { - // this.db.exec('rollback;') + this.db.exec('rollback;') console.error(e) throw e } @@ -484,20 +484,18 @@ class Database { // -------------------------------------------------------------------------- async addBlock (txids, txhexs, height, hash, time) { - await this.transaction(async () => { - const indexes = new Array(txids.length).fill(null).map((_, i) => i) - for (const index of indexes) { - const txid = txids[index] - const txHex = txhexs && txhexs[index] - await this.addTransaction(txid, txHex, height, time) - } - // txids.forEach(async (txid, i) => { - // const txhex = txhexs && txhexs[i] - // await this.addTransaction(txid, txhex, height, time) - // }) - await this.setHeight(height) - await this.setHash(hash) - }) + const indexes = new Array(txids.length).fill(null).map((_, i) => i) + for (const index of indexes) { + const txid = txids[index] + const txHex = txhexs && txhexs[index] + await this.addTransaction(txid, txHex, height, time) + } + // txids.forEach(async (txid, i) => { + // const txhex = txhexs && txhexs[i] + // await this.addTransaction(txid, txhex, height, time) + // }) + await this.setHeight(height) + await this.setHash(hash) } async addTransaction (txid, txhex, height, time) { diff --git a/src/indexer.js b/src/indexer.js index d747288..b3a7666 100644 --- a/src/indexer.js +++ b/src/indexer.js @@ -173,18 +173,15 @@ class Indexer { this.logger.info(`Rewinding to block ${newHeight}`) const txids = await this.database.getTransactionsAboveHeight(newHeight) + // Put all transactions back into the mempool. This is better than deleting them, because + // when we assume they will just go into a different block, we don't need to re-execute. + // If they don't make it into a block, then they will be expired in time. + for (const txid of txids) { + await this.database.unconfirmTransaction(txid) + } - await this.database.transaction(async () => { - // Put all transactions back into the mempool. This is better than deleting them, because - // when we assume they will just go into a different block, we don't need to re-execute. - // If they don't make it into a block, then they will be expired in time. - for (const txid of txids) { - await this.database.unconfirmTransaction(txid) - } - - await this.database.setHeight(newHeight) - await this.database.setHash(null) - }) + await this.database.setHeight(newHeight) + await this.database.setHash(null) if (this.onReorg) this.onReorg(newHeight) } @@ -198,11 +195,9 @@ class Indexer { const expired = await this.database.getMempoolTransactionsBeforeTime(expirationTime) const deleted = new Set() - await this.database.transaction(async () => { - for (const txid of expired) { - await this.database.deleteTransaction(txid, deleted) - } - }) + for (const txid of expired) { + await this.database.deleteTransaction(txid, deleted) + } } } From 2aa9d2548687f7d05e3672f845cf29a732381d17 Mon Sep 17 00:00:00 2001 From: hojarasca Date: Fri, 3 Dec 2021 01:16:48 -0300 Subject: [PATCH 05/82] tests working again. --- src/database.js | 4 ++-- src/server.js | 2 +- test/crawler.js | 44 ++++++++++++++++++++++---------------------- test/indexer.js | 28 ++++++++++++++-------------- test/server.js | 10 ++++++---- 5 files changed, 45 insertions(+), 43 deletions(-) diff --git a/src/database.js b/src/database.js index 9d5bfbb..b544256 100644 --- a/src/database.js +++ b/src/database.js @@ -523,9 +523,9 @@ class Database { const inputs = [] const outputs = [] - try { - if (!hex) throw new Error('No hex') + if (!hex) { throw new Error('No hex') } + try { bsvtx = new bsv.Transaction(hex) bsvtx.inputs.forEach(input => { diff --git a/src/server.js b/src/server.js index 98bee31..08b17ba 100644 --- a/src/server.js +++ b/src/server.js @@ -105,7 +105,7 @@ class Server { async getBerry (req, res, next) { try { - const state = this.database.getBerryState(req.params.location) + const state = await this.database.getBerryState(req.params.location) if (state) { res.setHeader('Content-Type', 'application/json') res.send(state) diff --git a/test/crawler.js b/test/crawler.js index 64b0059..8201070 100644 --- a/test/crawler.js +++ b/test/crawler.js @@ -38,10 +38,10 @@ describe('Crawler', () => { const api = { getNextBlock, fetch } const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, []) await indexer.start() - database.trust(txid) + await database.trust(txid) await indexed(indexer, txid) - expect(database.getHeight()).to.equal(1) - expect(database.getHash()).to.equal('abc') + expect(await database.getHeight()).to.equal(1) + expect(await database.getHash()).to.equal('abc') await indexer.stop() }) @@ -55,10 +55,10 @@ describe('Crawler', () => { const api = { getNextBlock } const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, []) await indexer.start() - database.trust(txid) + await database.trust(txid) await indexed(indexer, txid) - expect(database.getHeight()).to.equal(1) - expect(database.getHash()).to.equal('abc') + expect(await database.getHeight()).to.equal(1) + expect(await database.getHash()).to.equal('abc') await indexer.stop() }) @@ -81,15 +81,15 @@ describe('Crawler', () => { indexer.crawler.pollForNewBlocksInterval = 10 await indexer.start() await database.addTransaction(txids[1]) - database.trust(txids[0]) - database.trust(txids[1]) - database.trust(txids[2]) + await database.trust(txids[0]) + await database.trust(txids[1]) + await database.trust(txids[2]) await indexed(indexer, txids[1]) indexedMiddleTransaction = true await indexed(indexer, txids[0]) - expect(database.getTransactionHex(txids[0])).to.equal(txns[txids[0]]) - expect(database.getTransactionHex(txids[1])).to.equal(txns[txids[1]]) - expect(database.getTransactionHex(txids[2])).to.equal(txns[txids[2]]) + expect(await database.getTransactionHex(txids[0])).to.equal(txns[txids[0]]) + expect(await database.getTransactionHex(txids[1])).to.equal(txns[txids[1]]) + expect(await database.getTransactionHex(txids[2])).to.equal(txns[txids[2]]) await indexer.stop() }) @@ -115,13 +115,13 @@ describe('Crawler', () => { await indexed(indexer, txid) didIndex = true await reorged(indexer) - expect(indexer.database.getHeight()).to.equal(2) + expect(await indexer.database.getHeight()).to.equal(2) await crawled(indexer, 3) - expect(database.getHeight()).to.equal(3) - expect(database.getHash()).to.equal('def') - expect(database.getTransactionHex(txid)).not.to.equal(undefined) - expect(database.getJigState(txid + '_o1')).not.to.equal(undefined) - expect(database.getTransactionHeight(txid)).to.equal(-1) + expect(await database.getHeight()).to.equal(3) + expect(await database.getHash()).to.equal('def') + expect(await database.getTransactionHex(txid)).not.to.equal(undefined) + expect(await database.getJigState(txid + '_o1')).not.to.equal(undefined) + expect(await database.getTransactionHeight(txid)).to.equal(-1) await indexer.stop() }) @@ -140,11 +140,11 @@ describe('Crawler', () => { const api = { getNextBlock, fetch } const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, []) await indexer.start() - database.trust(txid) + await database.trust(txid) await reorged(indexer) - expect(database.getTransactionHex(txid)).not.to.equal(undefined) - expect(database.getJigState(txid + '_o1')).to.equal(undefined) - expect(database.getTransactionHeight(txid)).to.equal(-1) + expect(await database.getTransactionHex(txid)).not.to.equal(undefined) + expect(await database.getJigState(txid + '_o1')).to.equal(undefined) + expect(await database.getTransactionHeight(txid)).to.equal(-1) await indexer.stop() }) }) diff --git a/test/indexer.js b/test/indexer.js index 0ee8c1b..6463847 100644 --- a/test/indexer.js +++ b/test/indexer.js @@ -41,10 +41,10 @@ describe('Indexer', () => { database.trust('9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102') await indexed(indexer, '9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102') const txid = '9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102' - expect(database.getTransactionHex(txid)).to.equal(fetch(txid).hex) - expect(database.getTransactionHeight(txid)).to.equal(null) - expect(database.getTransactionTime(txid)).to.be.greaterThan(new Date() / 1000 - 3) - expect(database.getTransactionTime(txid)).to.be.lessThan(new Date() / 1000 + 3) + expect(await database.getTransactionHex(txid)).to.equal(fetch(txid).hex) + expect(await database.getTransactionHeight(txid)).to.equal(null) + expect(await database.getTransactionTime(txid)).to.be.greaterThan(new Date() / 1000 - 3) + expect(await database.getTransactionTime(txid)).to.be.lessThan(new Date() / 1000 + 3) await indexer.stop() }) @@ -130,9 +130,9 @@ describe('Indexer', () => { database.addTransaction('bfa5180e601e92af23d80782bf625b102ac110105a392e376fe7607e4e87dc8d') // Class with berry image database.trust('bfa5180e601e92af23d80782bf625b102ac110105a392e376fe7607e4e87dc8d') await indexed(indexer, 'bfa5180e601e92af23d80782bf625b102ac110105a392e376fe7607e4e87dc8d') - expect(database.getTransactionHex('bfa5180e601e92af23d80782bf625b102ac110105a392e376fe7607e4e87dc8d')).not.to.equal(undefined) - database.deleteTransaction('2f3492ef5401d887a93ca09820dff952f355431cea306841a70d163e32b2acad') // Berry data - expect(database.getTransactionHex('bfa5180e601e92af23d80782bf625b102ac110105a392e376fe7607e4e87dc8d')).to.equal(undefined) + expect(await database.getTransactionHex('bfa5180e601e92af23d80782bf625b102ac110105a392e376fe7607e4e87dc8d')).not.to.equal(undefined) + await database.deleteTransaction('2f3492ef5401d887a93ca09820dff952f355431cea306841a70d163e32b2acad') // Berry data + expect(await database.getTransactionHex('bfa5180e601e92af23d80782bf625b102ac110105a392e376fe7607e4e87dc8d')).to.equal(undefined) await indexer.stop() }) @@ -142,11 +142,11 @@ describe('Indexer', () => { this.timeout(40000) const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, DEFAULT_TRUSTLIST) await indexer.start() - database.addTransaction('11f27cdad53128a4eb14c8328515dfab56b16ea5a71dd26abe9e9d7488f3ab83') + await database.addTransaction('11f27cdad53128a4eb14c8328515dfab56b16ea5a71dd26abe9e9d7488f3ab83') await indexed(indexer, '11f27cdad53128a4eb14c8328515dfab56b16ea5a71dd26abe9e9d7488f3ab83') - expect(database.getSpend('7fa1b0eb8408047e138aadf72ee0980e42afab2208181429b050ad495a384d39_o1')) + expect(await database.getSpend('7fa1b0eb8408047e138aadf72ee0980e42afab2208181429b050ad495a384d39_o1')) .to.equal('11f27cdad53128a4eb14c8328515dfab56b16ea5a71dd26abe9e9d7488f3ab83') - expect(database.getSpend('11f27cdad53128a4eb14c8328515dfab56b16ea5a71dd26abe9e9d7488f3ab83_o1')) + expect(await database.getSpend('11f27cdad53128a4eb14c8328515dfab56b16ea5a71dd26abe9e9d7488f3ab83_o1')) .to.equal(null) await indexer.stop() }) @@ -160,12 +160,12 @@ describe('Indexer', () => { const txid1 = new bsv.Transaction(rawtx1).hash const txid2 = new bsv.Transaction(rawtx2).hash await indexer.start() - database.addTransaction(txid1, rawtx1) - database.trust(txid1) + await database.addTransaction(txid1, rawtx1) + await database.trust(txid1) await indexed(indexer, txid1) - database.addTransaction(txid2, rawtx2) + await database.addTransaction(txid2, rawtx2) await failed(indexer, txid2) - expect(database.getSpend(txid1 + '_o1')).to.equal(txid2) + expect(await database.getSpend(txid1 + '_o1')).to.equal(txid2) await indexer.stop() }) diff --git a/test/server.js b/test/server.js index caf0d04..350c332 100644 --- a/test/server.js +++ b/test/server.js @@ -50,7 +50,7 @@ describe('Server', () => { await axios.post(`http://localhost:${server.port}/tx/${txid}`, txns[txid], options) await axios.post(`http://localhost:${server.port}/trust/${txid}`) await indexed(indexer, txid) - server.stop() + await server.stop() await indexer.stop() }) @@ -90,7 +90,9 @@ describe('Server', () => { ] const options = { headers: { 'Content-Type': 'application/json' } } await axios.post(`http://localhost:${server.port}/trust`, trustlist, options) - trustlist.forEach(txid => expect(indexer.database.isTrusted(txid)).to.equal(true)) + for (const txid of trustlist) { + expect(await indexer.database.isTrusted(txid)).to.equal(true) + } server.stop() await indexer.stop() }) @@ -146,9 +148,9 @@ describe('Server', () => { const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, DEFAULT_TRUSTLIST) const server = new Server(database, logger, null) await indexer.start() - server.start() + await server.start() await listening(server) - database.addTransaction('bfa5180e601e92af23d80782bf625b102ac110105a392e376fe7607e4e87dc8d') + await database.addTransaction('bfa5180e601e92af23d80782bf625b102ac110105a392e376fe7607e4e87dc8d') await indexed(indexer, 'bfa5180e601e92af23d80782bf625b102ac110105a392e376fe7607e4e87dc8d') const location = '24cde3638a444c8ad397536127833878ffdfe1b04d5595489bd294e50d77105a_o1?berry=2f3492ef5401d887a93ca09820dff952f355431cea306841a70d163e32b2acad&version=5' const state = (await axios.get(`http://localhost:${server.port}/berry/${encodeURIComponent(location)}`)).data From 6d9467e316b53ce5f834fc84df96decfe8985166 Mon Sep 17 00:00:00 2001 From: hojarasca Date: Fri, 3 Dec 2021 14:51:21 -0300 Subject: [PATCH 06/82] some tests passing.wq --- src/crawler.js | 24 +++++++++++----------- src/database.js | 34 ++++++++++++++----------------- src/downloader.js | 28 ++++++++++++------------- src/executor.js | 8 +++++--- src/indexer.js | 40 +++++++++++++++++++----------------- test/crawler.js | 8 ++++---- test/indexer.js | 52 +++++++++++++++++++++++------------------------ 7 files changed, 97 insertions(+), 97 deletions(-) diff --git a/src/crawler.js b/src/crawler.js index 860b78c..aaa1559 100644 --- a/src/crawler.js +++ b/src/crawler.js @@ -29,7 +29,7 @@ class Crawler { this.onExpireMempoolTransactions = null } - start (height, hash) { + async start (height, hash) { this.logger.debug('Starting crawler') if (this.started) return @@ -38,8 +38,8 @@ class Crawler { this.height = height this.hash = hash - this._pollForNewBlocks() - this._expireMempoolTransactions() + await this._pollForNewBlocks() + await this._expireMempoolTransactions() } stop () { @@ -51,12 +51,12 @@ class Crawler { this.expireMempoolTransactionsTimerId = null } - _expireMempoolTransactions () { + async _expireMempoolTransactions () { if (!this.started) return this.logger.debug('Expiring mempool transactions') - if (this.onExpireMempoolTransactions) this.onExpireMempoolTransactions() + if (this.onExpireMempoolTransactions) { await this.onExpireMempoolTransactions() } this.expireMempoolTransactionsTimerId = setTimeout( this._expireMempoolTransactions.bind(this), this.expireMempoolTransactionsInterval) @@ -68,7 +68,7 @@ class Crawler { try { await this._pollForNextBlock() } catch (e) { - if (this.onCrawlError) this.onCrawlError(e) + if (this.onCrawlError) { await this.onCrawlError(e) } // Swallow, we'll retry } @@ -97,7 +97,7 @@ class Crawler { // Case: reorg if (block && block.reorg) { this.logger.debug('Reorg detected') - this._rewindAfterReorg() + await this._rewindAfterReorg() setTimeout(() => this._pollForNextBlock(), 0) return } @@ -113,7 +113,7 @@ class Crawler { if (block) { this.logger.debug('Received new block at height', block.height) if (this.onCrawlBlockTransactions) { - this.onCrawlBlockTransactions(block.height, block.hash, block.time, block.txids, block.txhexs) + await this.onCrawlBlockTransactions(block.height, block.hash, block.time, block.txids, block.txhexs) } this.height = block.height this.hash = block.hash @@ -121,9 +121,9 @@ class Crawler { } } - _rewindAfterReorg () { + async _rewindAfterReorg () { const newHeight = this.height - this.rewindCount - if (this.onRewindBlocks) this.onRewindBlocks(newHeight) + if (this.onRewindBlocks) { await this.onRewindBlocks(newHeight) } this.height = newHeight this.hash = null } @@ -138,8 +138,8 @@ class Crawler { this.listeningForMempool = true } - _onMempoolRunTransaction (txid, rawtx) { - if (this.onMempoolTransaction) this.onMempoolTransaction(txid, rawtx) + async _onMempoolRunTransaction (txid, rawtx) { + if (this.onMempoolTransaction) await this.onMempoolTransaction(txid, rawtx) } } diff --git a/src/database.js b/src/database.js index b544256..64733bb 100644 --- a/src/database.js +++ b/src/database.js @@ -490,10 +490,6 @@ class Database { const txHex = txhexs && txhexs[index] await this.addTransaction(txid, txHex, height, time) } - // txids.forEach(async (txid, i) => { - // const txhex = txhexs && txhexs[i] - // await this.addTransaction(txid, txhex, height, time) - // }) await this.setHeight(height) await this.setHash(hash) } @@ -570,7 +566,7 @@ class Database { for (const deptxid of deps) { if (!await this.isTransactionDownloaded(deptxid)) { - if (this.onRequestDownload) this.onRequestDownload(deptxid) + if (this.onRequestDownload) await this.onRequestDownload(deptxid) } } } @@ -605,10 +601,10 @@ class Database { // Non-executable might be berry data. We execute once we receive them. const downstreamReadyToExecute = this.getDownstreamReadyToExecuteStmt.raw(true).all(txid).map(x => x[0]) - downstreamReadyToExecute.forEach(downtxid => { + for (const downtxid of downstreamReadyToExecute) { this.markExecutingStmt.run(downtxid) - if (this.onReadyToExecute) this.onReadyToExecute(downtxid) - }) + if (this.onReadyToExecute) { await this.onReadyToExecute(downtxid) } + } } async storeParsedExecutableTransaction (txid, hex, hasCode, deps, inputs, outputs) { @@ -638,7 +634,7 @@ class Database { async storeExecutedTransaction (txid, result) { const { cache, classes, locks, scripthashes } = result - await this.transaction(() => { + await this.transaction(async () => { this.setTransactionExecutedStmt.run(1, txid) this.setTransactionIndexedStmt.run(1, txid) this.unmarkExecutingStmt.run(txid) @@ -671,10 +667,10 @@ class Database { }) const downstreamReadyToExecute = this.getDownstreamReadyToExecuteStmt.raw(true).all(txid).map(x => x[0]) - downstreamReadyToExecute.forEach(downtxid => { + for (const downtxid of downstreamReadyToExecute) { this.markExecutingStmt.run(downtxid) - if (this.onReadyToExecute) this.onReadyToExecute(downtxid) - }) + if (this.onReadyToExecute) { await this.onReadyToExecute(downtxid) } + } } async setTransactionExecutionFailed (txid) { @@ -724,11 +720,11 @@ class Database { const txids = [txid] deleted.add(txid) - await this.transaction(() => { + await this.transaction(async () => { while (txids.length) { const txid = txids.shift() - if (this.onDeleteTransaction) this.onDeleteTransaction(txid) + if (this.onDeleteTransaction) { await this.onDeleteTransaction(txid) } this.deleteTransactionStmt.run(txid) this.deleteJigStatesStmt.run(txid) @@ -905,7 +901,7 @@ class Database { this.getUpstreamUnexecutedCodeStmt.raw(true).all(txid).forEach(x => queue.push(x[0])) } - await this.transaction(() => trusted.forEach(txid => this.setTrustedStmt.run(txid, 1))) + trusted.forEach(txid => this.setTrustedStmt.run(txid, 1)) for (const txid of trusted) { await this._checkExecutability(txid) @@ -924,7 +920,7 @@ class Database { await this.unindexTransaction(txid) this.setTrustedStmt.run(txid, 0) }) - if (this.onUntrustTransaction) this.onUntrustTransaction(txid) + if (this.onUntrustTransaction) await this.onUntrustTransaction(txid) } async getTrustlist () { @@ -945,13 +941,13 @@ class Database { await this.unindexTransaction(txid) this.banStmt.run(txid) }) - if (this.onBanTransaction) this.onBanTransaction(txid) + if (this.onBanTransaction) await this.onBanTransaction(txid) } async unban (txid) { this.unbanStmt.run(txid) await this._checkExecutability(txid) - if (this.onUnbanTransaction) this.onUnbanTransaction(txid) + if (this.onUnbanTransaction) await this.onUnbanTransaction(txid) } async getBanlist () { @@ -996,7 +992,7 @@ class Database { const row = this.isReadyToExecuteStmt.get(txid) if (row && row.ready) { this.markExecutingStmt.run(txid) - if (this.onReadyToExecute) this.onReadyToExecute(txid) + if (this.onReadyToExecute) { await this.onReadyToExecute(txid) } } } } diff --git a/src/downloader.js b/src/downloader.js index c3c626e..33c9d03 100644 --- a/src/downloader.js +++ b/src/downloader.js @@ -30,18 +30,18 @@ class Downloader { this.attempts = new Map() } - add (txid) { + async add (txid) { if (this.has(txid)) return if (!this.fetchFunction) return - this._enqueueFetch(txid) + await this._enqueueFetch(txid) } - _enqueueFetch (txid) { + async _enqueueFetch (txid) { if (this.fetching.size >= this.numParallelDownloads) { this.queued.add(txid) } else { - this._fetch(txid) + await this._fetch(txid) } } @@ -67,31 +67,31 @@ class Downloader { try { const { hex, height, time } = await this.fetchFunction(txid) - this._onFetchSucceed(txid, hex, height, time) + await this._onFetchSucceed(txid, hex, height, time) } catch (e) { - this._onFetchFailed(txid, e) + await this._onFetchFailed(txid, e) } finally { - this._fetchNextInQueue() + await this._fetchNextInQueue() } } - _onFetchSucceed (txid, hex, height, time) { + async _onFetchSucceed (txid, hex, height, time) { if (!this.fetching.delete(txid)) return this.attempts.delete(txid) - if (this.onDownloadTransaction) this.onDownloadTransaction(txid, hex, height, time) + if (this.onDownloadTransaction) { await this.onDownloadTransaction(txid, hex, height, time) } } - _onFetchFailed (txid, e) { + async _onFetchFailed (txid, e) { if (!this.fetching.delete(txid)) return - if (this.onFailedToDownloadTransaction) this.onFailedToDownloadTransaction(txid, e) + if (this.onFailedToDownloadTransaction) await this.onFailedToDownloadTransaction(txid, e) const attempts = (this.attempts.get(txid) || 0) + 1 const secondsToRetry = Math.pow(2, attempts) - if (this.onRetryingDownload) this.onRetryingDownload(txid, secondsToRetry) + if (this.onRetryingDownload) await this.onRetryingDownload(txid, secondsToRetry) this.attempts.set(txid, attempts) this.waitingToRetry.add(txid) @@ -103,13 +103,13 @@ class Downloader { }, secondsToRetry * 1000) } - _fetchNextInQueue () { + async _fetchNextInQueue () { if (!this.queued.size) return const txid = this.queued.keys().next().value this.queued.delete(txid) - this._fetch(txid) + await this._fetch(txid) } } diff --git a/src/executor.js b/src/executor.js index a72959e..dbecbcb 100644 --- a/src/executor.js +++ b/src/executor.js @@ -80,12 +80,14 @@ class Executor { try { const result = await Bus.sendRequest(worker, 'execute', txid, hex, trustList) - if (this.onIndexed) this.onIndexed(txid, result) + if (this.onIndexed) { + this.onIndexed(txid, result).catch(console.error) + } } catch (e) { if (worker.missingDeps.size) { - if (this.onMissingDeps) this.onMissingDeps(txid, Array.from(worker.missingDeps)) + if (this.onMissingDeps) await this.onMissingDeps(txid, Array.from(worker.missingDeps)) } else { - if (this.onExecuteFailed) this.onExecuteFailed(txid, e) + if (this.onExecuteFailed) await this.onExecuteFailed(txid, e) } } finally { this.executing.delete(txid) diff --git a/src/indexer.js b/src/indexer.js index b3a7666..fcbfd73 100644 --- a/src/indexer.js +++ b/src/indexer.js @@ -91,15 +91,15 @@ class Indexer { if (height) { await this.database.setTransactionHeight(txid, height) } if (time) { await this.database.setTransactionTime(txid, time) } await this.database.parseAndStoreTransaction(txid, hex) - if (this.onDownload) this.onDownload(txid) + if (this.onDownload) await this.onDownload(txid) } - _onFailedToDownloadTransaction (txid, e) { + async _onFailedToDownloadTransaction (txid, e) { this.logger.error('Failed to download', txid, e.toString()) - if (this.onFailToDownload) this.onFailToDownload(txid) + if (this.onFailToDownload) { await this.onFailToDownload(txid) } } - _onRetryingDownload (txid, secondsToRetry) { + async _onRetryingDownload (txid, secondsToRetry) { this.logger.info('Retrying download', txid, 'after', secondsToRetry, 'seconds') } @@ -107,7 +107,9 @@ class Indexer { if (!await this.database.hasTransaction(txid)) return // Check not re-orged this.logger.info(`Executed ${txid}`) await this.database.storeExecutedTransaction(txid, result) - if (this.onIndex) this.onIndex(txid) + if (this.onIndex) { + await this.onIndex(txid) + } } async _onExecuteFailed (txid, e) { @@ -116,41 +118,41 @@ class Indexer { if (this.onFailToIndex) this.onFailToIndex(txid, e) } - _onReadyToExecute (txid) { - this.executor.execute(txid) + async _onReadyToExecute (txid) { + await this.executor.execute(txid) } - _onAddTransaction (txid) { + async _onAddTransaction (txid) { this.logger.info('Added', txid) } - _onDeleteTransaction (txid) { + async _onDeleteTransaction (txid) { this.logger.info('Removed', txid) - this.downloader.remove(txid) + await this.downloader.remove(txid) } - _onTrustTransaction (txid) { + async _onTrustTransaction (txid) { this.logger.info('Trusted', txid) } - _onUntrustTransaction (txid) { + async _onUntrustTransaction (txid) { this.logger.info('Untrusted', txid) } - _onBanTransaction (txid) { + async _onBanTransaction (txid) { this.logger.info('Banned', txid) } - _onUnbanTransaction (txid) { + async _onUnbanTransaction (txid) { this.logger.info('Unbanned', txid) } - _onUnindexTransaction (txid) { + async _onUnindexTransaction (txid) { this.logger.info('Unindexed', txid) } - _onRequestDownload (txid) { - this.downloader.add(txid) + async _onRequestDownload (txid) { + await this.downloader.add(txid) } async _onMissingDeps (txid, deptxids) { @@ -159,14 +161,14 @@ class Indexer { deptxids.forEach(deptxid => this.downloader.add(deptxid)) } - _onCrawlError (e) { + async _onCrawlError (e) { this.logger.error(`Crawl error: ${e.toString()}`) } async _onCrawlBlockTransactions (height, hash, time, txids, txhexs) { this.logger.info(`Crawled block ${height} for ${txids.length} transactions`) await this.database.addBlock(txids, txhexs, height, hash, time) - if (this.onBlock) this.onBlock(height) + if (this.onBlock) await this.onBlock(height) } async _onRewindBlocks (newHeight) { diff --git a/test/crawler.js b/test/crawler.js index 8201070..630cdc6 100644 --- a/test/crawler.js +++ b/test/crawler.js @@ -40,9 +40,9 @@ describe('Crawler', () => { await indexer.start() await database.trust(txid) await indexed(indexer, txid) + await indexer.stop() expect(await database.getHeight()).to.equal(1) expect(await database.getHash()).to.equal('abc') - await indexer.stop() }) // -------------------------------------------------------------------------- @@ -57,9 +57,9 @@ describe('Crawler', () => { await indexer.start() await database.trust(txid) await indexed(indexer, txid) + await indexer.stop() expect(await database.getHeight()).to.equal(1) expect(await database.getHash()).to.equal('abc') - await indexer.stop() }) // -------------------------------------------------------------------------- @@ -87,10 +87,10 @@ describe('Crawler', () => { await indexed(indexer, txids[1]) indexedMiddleTransaction = true await indexed(indexer, txids[0]) + await indexer.stop() expect(await database.getTransactionHex(txids[0])).to.equal(txns[txids[0]]) expect(await database.getTransactionHex(txids[1])).to.equal(txns[txids[1]]) expect(await database.getTransactionHex(txids[2])).to.equal(txns[txids[2]]) - await indexer.stop() }) // -------------------------------------------------------------------------- @@ -142,10 +142,10 @@ describe('Crawler', () => { await indexer.start() await database.trust(txid) await reorged(indexer) + await indexer.stop() expect(await database.getTransactionHex(txid)).not.to.equal(undefined) expect(await database.getJigState(txid + '_o1')).to.equal(undefined) expect(await database.getTransactionHeight(txid)).to.equal(-1) - await indexer.stop() }) }) diff --git a/test/indexer.js b/test/indexer.js index 6463847..7ea3b78 100644 --- a/test/indexer.js +++ b/test/indexer.js @@ -35,10 +35,10 @@ describe('Indexer', () => { it('add and index', async () => { const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, []) await indexer.start() - database.addTransaction('3f9de452f0c3c96be737d42aa0941b27412211976688967adb3174ee18b04c64') - database.addTransaction('9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102') - database.trust('3f9de452f0c3c96be737d42aa0941b27412211976688967adb3174ee18b04c64') - database.trust('9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102') + await database.addTransaction('3f9de452f0c3c96be737d42aa0941b27412211976688967adb3174ee18b04c64') + await database.addTransaction('9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102') + await database.trust('3f9de452f0c3c96be737d42aa0941b27412211976688967adb3174ee18b04c64') + await database.trust('9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102') await indexed(indexer, '9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102') const txid = '9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102' expect(await database.getTransactionHex(txid)).to.equal(fetch(txid).hex) @@ -72,10 +72,10 @@ describe('Indexer', () => { it('add in reverse and index', async () => { const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, []) await indexer.start() - database.addTransaction('9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102') - database.addTransaction('3f9de452f0c3c96be737d42aa0941b27412211976688967adb3174ee18b04c64') - database.trust('3f9de452f0c3c96be737d42aa0941b27412211976688967adb3174ee18b04c64') - database.trust('9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102') + await database.addTransaction('9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102') + await database.addTransaction('3f9de452f0c3c96be737d42aa0941b27412211976688967adb3174ee18b04c64') + await database.trust('3f9de452f0c3c96be737d42aa0941b27412211976688967adb3174ee18b04c64') + await database.trust('9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102') await indexed(indexer, '9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102') await indexer.stop() }) @@ -85,10 +85,10 @@ describe('Indexer', () => { it('fail to index', async () => { const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, []) await indexer.start() - database.trust('b17a9af70ab0f46809f908b2e900e395ba40996000bf4f00e3b27a1e93280cf1') - database.trust('a5291157ab7a2d80d834bbe82c380ce3976f53990d20c62c477ca3a2ac93a7e9') - database.addTransaction('b17a9af70ab0f46809f908b2e900e395ba40996000bf4f00e3b27a1e93280cf1') - database.addTransaction('a5291157ab7a2d80d834bbe82c380ce3976f53990d20c62c477ca3a2ac93a7e9') + await database.trust('b17a9af70ab0f46809f908b2e900e395ba40996000bf4f00e3b27a1e93280cf1') + await database.trust('a5291157ab7a2d80d834bbe82c380ce3976f53990d20c62c477ca3a2ac93a7e9') + await database.addTransaction('b17a9af70ab0f46809f908b2e900e395ba40996000bf4f00e3b27a1e93280cf1') + await database.addTransaction('a5291157ab7a2d80d834bbe82c380ce3976f53990d20c62c477ca3a2ac93a7e9') await failed(indexer, 'a5291157ab7a2d80d834bbe82c380ce3976f53990d20c62c477ca3a2ac93a7e9') await indexer.stop() }) @@ -98,14 +98,14 @@ describe('Indexer', () => { it('discovered berry transaction', async () => { const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, []) await indexer.start() - database.addTransaction('bfa5180e601e92af23d80782bf625b102ac110105a392e376fe7607e4e87dc8d') // Class with berry image - database.addTransaction('24cde3638a444c8ad397536127833878ffdfe1b04d5595489bd294e50d77105a') // B (old) - database.addTransaction('312985bd960ae4c59856b3089b04017ede66506ea181333eec7c9bb88b11c490') // txo, Tx - database.addTransaction('727e7b423b7ee40c0b5be87fba7fa5673ea2d20a74259040a7295d9c32a90011') // Hex - database.trust('bfa5180e601e92af23d80782bf625b102ac110105a392e376fe7607e4e87dc8d') - database.trust('24cde3638a444c8ad397536127833878ffdfe1b04d5595489bd294e50d77105a') - database.trust('312985bd960ae4c59856b3089b04017ede66506ea181333eec7c9bb88b11c490') - database.trust('727e7b423b7ee40c0b5be87fba7fa5673ea2d20a74259040a7295d9c32a90011') + await database.addTransaction('bfa5180e601e92af23d80782bf625b102ac110105a392e376fe7607e4e87dc8d') // Class with berry image + await database.addTransaction('24cde3638a444c8ad397536127833878ffdfe1b04d5595489bd294e50d77105a') // B (old) + await database.addTransaction('312985bd960ae4c59856b3089b04017ede66506ea181333eec7c9bb88b11c490') // txo, Tx + await database.addTransaction('727e7b423b7ee40c0b5be87fba7fa5673ea2d20a74259040a7295d9c32a90011') // Hex + await database.trust('bfa5180e601e92af23d80782bf625b102ac110105a392e376fe7607e4e87dc8d') + await database.trust('24cde3638a444c8ad397536127833878ffdfe1b04d5595489bd294e50d77105a') + await database.trust('312985bd960ae4c59856b3089b04017ede66506ea181333eec7c9bb88b11c490') + await database.trust('727e7b423b7ee40c0b5be87fba7fa5673ea2d20a74259040a7295d9c32a90011') // Don't index the berry data, because it will be fetched automatically // database.addTransaction('2f3492ef5401d887a93ca09820dff952f355431cea306841a70d163e32b2acad') // Berry data await indexed(indexer, 'bfa5180e601e92af23d80782bf625b102ac110105a392e376fe7607e4e87dc8d') @@ -117,7 +117,7 @@ describe('Indexer', () => { it('add and download dependencies', async () => { const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, []) await indexer.start() - database.addTransaction('9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102') + await database.addTransaction('9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102') await new Promise((resolve, reject) => setTimeout(resolve, 1000)) await indexer.stop() }) @@ -127,8 +127,8 @@ describe('Indexer', () => { it('remove discovered dep', async () => { const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, DEFAULT_TRUSTLIST) await indexer.start() - database.addTransaction('bfa5180e601e92af23d80782bf625b102ac110105a392e376fe7607e4e87dc8d') // Class with berry image - database.trust('bfa5180e601e92af23d80782bf625b102ac110105a392e376fe7607e4e87dc8d') + await database.addTransaction('bfa5180e601e92af23d80782bf625b102ac110105a392e376fe7607e4e87dc8d') // Class with berry image + await database.trust('bfa5180e601e92af23d80782bf625b102ac110105a392e376fe7607e4e87dc8d') await indexed(indexer, 'bfa5180e601e92af23d80782bf625b102ac110105a392e376fe7607e4e87dc8d') expect(await database.getTransactionHex('bfa5180e601e92af23d80782bf625b102ac110105a392e376fe7607e4e87dc8d')).not.to.equal(undefined) await database.deleteTransaction('2f3492ef5401d887a93ca09820dff952f355431cea306841a70d163e32b2acad') // Berry data @@ -178,9 +178,9 @@ describe('Indexer', () => { const txid1 = new bsv.Transaction(rawtx1).hash const txid2 = new bsv.Transaction(rawtx2).hash await indexer.start() - database.addTransaction(txid1, rawtx1) - database.addTransaction(txid2, rawtx2) - database.trust(txid1) + await database.addTransaction(txid1, rawtx1) + await database.addTransaction(txid2, rawtx2) + await database.trust(txid1) await indexed(indexer, txid2) expect(await indexer.database.getNumUnspent()).to.equal(0) await indexer.stop() From fa0f664cafc65cbc6b5b0ee14bd15381ad045164 Mon Sep 17 00:00:00 2001 From: Miguel Duarte Date: Fri, 3 Dec 2021 17:06:57 -0300 Subject: [PATCH 07/82] 1 pending test. --- src/crawler.js | 4 ++-- src/server.js | 24 ++++++++++++------------ test/crawler.js | 4 ++-- test/indexer.js | 9 ++++++--- test/server.js | 7 ++++--- 5 files changed, 26 insertions(+), 22 deletions(-) diff --git a/src/crawler.js b/src/crawler.js index aaa1559..16e9ab6 100644 --- a/src/crawler.js +++ b/src/crawler.js @@ -38,8 +38,8 @@ class Crawler { this.height = height this.hash = hash - await this._pollForNewBlocks() - await this._expireMempoolTransactions() + this._pollForNewBlocks().catch(console.error) + this._expireMempoolTransactions().catch(console.error) } stop () { diff --git a/src/server.js b/src/server.js index 08b17ba..c1aa3c0 100644 --- a/src/server.js +++ b/src/server.js @@ -37,18 +37,18 @@ class Server { const app = express() - let buffer = '' - const write = (chunk, encoding, callback) => { - buffer = buffer + chunk.toString() - const lines = buffer.split(/\r\n|\n\r|\n|\r/) - for (let i = 0; i < lines.length - 1; i++) { - this.logger.info(lines[i]) - } - buffer = lines[lines.length - 1] - callback() - return true - } - app.use(morgan('tiny', { stream: new Writable({ write }) })) + // let buffer = '' + // const write = (chunk, encoding, callback) => { + // buffer = buffer + chunk.toString() + // const lines = buffer.split(/\r\n|\n\r|\n|\r/) + // for (let i = 0; i < lines.length - 1; i++) { + // this.logger.info(lines[i]) + // } + // buffer = lines[lines.length - 1] + // callback() + // return true + // } + // app.use(morgan('tiny', { stream: new Writable({ write }) })) app.use(bodyParser.text({ limit: '25mb' })) app.use(bodyParser.json({ limit: '10mb' })) diff --git a/test/crawler.js b/test/crawler.js index 630cdc6..4a08d9c 100644 --- a/test/crawler.js +++ b/test/crawler.js @@ -127,7 +127,7 @@ describe('Crawler', () => { // -------------------------------------------------------------------------- - it('reorg while executing', async () => { + it.skip('reorg while executing', async () => { const txid = '3f9de452f0c3c96be737d42aa0941b27412211976688967adb3174ee18b04c64' let didReorg = false function getNextBlock (height, hash) { @@ -144,7 +144,7 @@ describe('Crawler', () => { await reorged(indexer) await indexer.stop() expect(await database.getTransactionHex(txid)).not.to.equal(undefined) - expect(await database.getJigState(txid + '_o1')).to.equal(undefined) + expect(await database.getJigState(txid + '_o1')).to.equal() expect(await database.getTransactionHeight(txid)).to.equal(-1) }) }) diff --git a/test/indexer.js b/test/indexer.js index 7ea3b78..6e70570 100644 --- a/test/indexer.js +++ b/test/indexer.js @@ -85,11 +85,12 @@ describe('Indexer', () => { it('fail to index', async () => { const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, []) await indexer.start() + const promise = failed(indexer, 'a5291157ab7a2d80d834bbe82c380ce3976f53990d20c62c477ca3a2ac93a7e9') await database.trust('b17a9af70ab0f46809f908b2e900e395ba40996000bf4f00e3b27a1e93280cf1') await database.trust('a5291157ab7a2d80d834bbe82c380ce3976f53990d20c62c477ca3a2ac93a7e9') await database.addTransaction('b17a9af70ab0f46809f908b2e900e395ba40996000bf4f00e3b27a1e93280cf1') await database.addTransaction('a5291157ab7a2d80d834bbe82c380ce3976f53990d20c62c477ca3a2ac93a7e9') - await failed(indexer, 'a5291157ab7a2d80d834bbe82c380ce3976f53990d20c62c477ca3a2ac93a7e9') + await promise await indexer.stop() }) @@ -160,11 +161,13 @@ describe('Indexer', () => { const txid1 = new bsv.Transaction(rawtx1).hash const txid2 = new bsv.Transaction(rawtx2).hash await indexer.start() + const successPromise = indexed(indexer, txid1) + const failurePromise = failed(indexer, txid2) await database.addTransaction(txid1, rawtx1) await database.trust(txid1) - await indexed(indexer, txid1) + await successPromise await database.addTransaction(txid2, rawtx2) - await failed(indexer, txid2) + await failurePromise expect(await database.getSpend(txid1 + '_o1')).to.equal(txid2) await indexer.stop() }) diff --git a/test/server.js b/test/server.js index 350c332..b0f70e1 100644 --- a/test/server.js +++ b/test/server.js @@ -41,15 +41,16 @@ describe('Server', () => { describe('post tx', () => { it('add with body', async () => { const indexer = new Indexer(database, {}, 'main', 1, 1, logger, 0, Infinity, []) - const server = new Server(database, logger, null) + const server = new Server(database, logger, 52521) await indexer.start() - server.start() + await server.start() await listening(server) const txid = '3f9de452f0c3c96be737d42aa0941b27412211976688967adb3174ee18b04c64' const options = { headers: { 'Content-Type': 'text/plain' } } + const promise = indexed(indexer, txid) await axios.post(`http://localhost:${server.port}/tx/${txid}`, txns[txid], options) await axios.post(`http://localhost:${server.port}/trust/${txid}`) - await indexed(indexer, txid) + await promise await server.stop() await indexer.stop() }) From fcbb5302b5bc050c81314b98fd43446ecb653d8d Mon Sep 17 00:00:00 2001 From: hojarasca Date: Fri, 3 Dec 2021 17:55:48 -0300 Subject: [PATCH 08/82] all test passing --- run-db.sqlite3 | Bin 344064 -> 0 bytes test/crawler.js | 6 ++++-- 2 files changed, 4 insertions(+), 2 deletions(-) delete mode 100644 run-db.sqlite3 diff --git a/run-db.sqlite3 b/run-db.sqlite3 deleted file mode 100644 index 692789d59c327b1f19799112ae6a8ba19b4e0b26..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 344064 zcmeI)OYG*?eb@0BI~gZ#+|+TVLkcK=Rfu{*fOEgh#WPMa#0F!xjzbc0anA2t>>;l0 zddAH}P=P567F~eYu*iZ12nj*eMK%Z)2z7xFAQlK^38)K%sDwm8Y(RW|&okG=sVr5- zk^CNc{(kQG7K6CLeT>R+^|N7#;y%h61`Cq-;K)i_bdio2`T|2q^_Q{hs zZ(ScJFJ1fkODA7_{@Gvn>b3Lsr_Y?b_Sw^?uD?F!tsB4en!eWea(nyc8;9RtzI*4( z_U_BGf9;t^A368<UKIS7PM&`5YfpdU zD<@BX<>blZY@hwg$(_ZYKlwAyzWCBtpMLKA^PfC(?vI{3{mS7XH{W{u#?6;^tC#Qn z{!_PQpMCyI*S>!8gO{AV@ci+;syX zmv`&22gZ2d!^6UE-Cot@pHQ$rar$eAnf%4;uN-Hg7oT`wfV-dUH|v*^C;Myd#*OnA zzjx}hd%XVYjR)fO%-J)Clzrw?GJE{CnI>@z$J<+Q>xl;*@dF>a-I{>T}<@R7q7{OU&!Gg3O~pN~E; zn?s7-z5nFo_dJ*Tr+P@;!&L8o_eOu~`kOaZ=wJPzkM#6!9p1b)-(UX8#s8W9R{b~ii}6>k*h?2K zek(mv|HkEyhhNYC^~x`&|M1j5C^s+u;Vb_%fAgvT_S7G|{QRZAa{2q|xu>@Be=q;- zOTStEcr4+6T=>+5uf>0H;ky_9)5YJ&fB%D86VF8e0R#|0009ILKmY**5I_I{1Q0*~ z0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~fe%99OCNdXv5)&z z#|+Ue(|xaLo7v|PTx+HVySW^!bJo_JQYkg%Ha2f_nA4Wcgr34-kZxM8v$VD4-nJQ+ zi`!_;*OH1KX{?z_bJe=d)irAjjOfGI)NyAy= zTDR6%6E&`zKBqNvGI@qs^5|t8263T}>60%lgwRVHzUJtYb-8$7a#>50v6}jbvBps9 zHe3xenvwH9^kI+@H2BEg)Mn<~%2wUTHclUVnlAe`lJCa2xK`@DRhPG6mf2F6qv#Qb zLA;-OT+2Q9Esflg=`P!$dAJ%^(@M8I>*~|U+pcA|=u%15r_ImQ=3$U5qb|fXeGR_O zu|!#Al2V;{3&T(Ck*jmg++A|ByP)MvKfN`x*25rX8_WCVeb6MTsY^qSw)qgFF@4j9 z`n9bQ&1Od})*0vWY|~V?72ExBkkD+^#GS}v%v#Rojl{HD#%|T;8jK0fYRgI;K5QZu zvXn~{u+7Jrc<;=zF6`oU+jPTBON-AgY|Gl1Y;mP36mu$lwJ@7@uF0`h>xVlGlKeE& z%4(Fh89mbWnD^K#Q&Fq?yVlf#*Ra~{s$Co=Y`M&BZfVAJ7-Utcx9*z4bHzEN()31H zKGY@awq=sCCZ?@wwa$2jfbCk7qRuPyqE&!p zmqtnxN~UUiXo|eq+tS{ZGBuIW z^u&3L*cg1=FGxn?#})P%Gqxw)=cWrQwH!jxugrdDO5%UPixl9FVUYTFpTD~h#%!*Y}?O6U zt7=lCO~&UE#>%~97j5mPCM{9B6i406@S`gox9PXseCn+@g+SA`9Zb5i-If^4a3#eq znHCkqDMd4z;;vh>gyDW zOy{i%VeCr*syb4%$;yzbA}XrGAS2m*_hL)V1qISrNl#Z97J+K?rZ@Jgs{*0Qkhirt zziDpr(!-;P$3e1c;FfYVEsUwiY}O`iOIyoKg`Y20RKJZ1Slic|mcW7<$+1w! zRz^?((}LA5q^3Y?-K&^J1?yC`m4@;r^)DhRT0Yp;6;f_tLQ1Y}L8YRV&kK$6y;-86K?zb< zHDzg!$&j4|5^YkyB|sHtx&P>e$VV4`H0BC`tpJvD|3lAbZPz2w`dNoa>- zNIguYd(R;&lG56?yhoBV$C)VB_C}{T%t~L=%@#GuytbBgUP#%`rjjY}pfV{Y#Y|OB zcJG?zYL7Efr25&XO8e6OWN(8?>uNgh#H^fDBkO9@d2Z?$acx#vsou6!F1*TZxZR73 zG*s~vX+`Vw*(k{N2-j=1T@CMa_ULYD%p7#E$gR)06e4kEWy--MTg}`T1@hiC_p{Gb zJRzGsdnd~vsNAIj@l;eLTSei7}blrv0i7Lt|W3@@or5MSPDAcUg>EW1wD)FkirRpGJy*k-8qDrFLWSQz;`!ai8aX;fJX>3;1YMWfzN>ak> zV0BoIE5fsCjk;RVW{cn3*``*yXI@o&s9mX^*`+LynU0Y7x z<-1W`NxE+@+1(ChgL-dQdW%Y}Sv7c68~5-3pSt=%oaK2Y0tg_000IagfB*srAb+Y=cW0R#|0 z009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{ z1Q0*~0R#|0009ILKmY**5I_I{1m1(dqx&^i-@}Dmf&c;tAba0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{ z1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_Kd_bkBo|KBqe7b1WF z0tg_000IagfB*srAbi-j#6anTvlpeZKzP%byP4FCV*dEB)`M z{!#gx7t57@mjCKg|NW^ybouL-e(myqOm9B*YW;nezkBK5mFMC&;~%*2y#L9Wz9?O!vK}ZDyZGaIKjd?B;T? z&RJV?N~P44+t|F#VNP2%6M72!7~Qm7XK8E8y=^lt7q`)xuO$^f(pWQ<=Bjm@t83O! z=CIxj8>Vm07qgGCtrl%DGZ{OpYs>jjOfGI)NuycgTDR6%6E&=xKBqNvGI@qs^5|vk zW4O@A^vRbNLg=LpUvu=yx?H?3xvZtBSj~IHSYs%48?J^K&B%Elc^_j04L!0qwV64$ zvQ;;-jnl`TrpvyKW6 zTIQB#U40r++qLW#T`H;iwE3CZ+{eh`>GE6C*Wl|MONvz z3tG$c(_1rZ-N!K7Sl&19gQieTT^ez;&4(C`>6>=auWgNJHalvu&N!E6o2I(0*zONw zgl4N|?Svg;)><}igr$8lcB?+uU`%jUn^o%YVUw^3zN!rcv5v^g!EV-b1cT#j0-aT2KpK zqiUP0_HUT55Z^_u*ufZ6xFH7ZlR`mpH*4!kplW+)3b@(Z(yo;z|7tmXE;OlMmwt1qK?Rd_|?Y8y_GmX(>NRwbXyS@9l{ zGGml#+Zetpb~XKdElLn&@CYGpN=YSe&$@9GNvyezA;YB@O(um=Q*Be5TGrTHoYq%= zFty9bw2T@;&YSI9%*oZE*()HDO)E`hk2bSynw0`_H06m+RW{1vL%#T=q^QNXlGlFS zjB|#~^yrK8&g$uL(;~b{#U!cP8LjEnS{t@Lk1=LdJ!-Vc_*}wRxtHvst=-h5^=W_N zsGAvnbYLXwWj&&!M>|9)iuI&-m2=wz7&?K8%4XTw5Y0>V%o}cX)jI4sUrCd!5V=5k- zwMiS&)-qF3=ZjVT&lsIbwNbrl`(o1)R#3+{)a2O82r5@vtonk$6kn}-70##-ovN&o zP_Cq2Z0ib}+Vn`3sAN<`?V-PTTZU`K)ah#9)H94aNwY~ds%?T$p)V>d>Q8D?8dbZj zsusEmV_NMHVDC?k_C?ds5RT_5hnr3Pb^H6;H*{6#5(#~XWgUaY? zI?lwbG*rLpYQ=eOY7KF1R*9$1Hnp#@=N0$kn-axl#i};QrL82TxQ(+>bK3ta_qNN^>IHM<)mFxK&YWRRdBWct_a)fMOW6Vtu3$C zp^ByhqYf=|D1+9qzbILBa!@jC>g`^~t)(ksRe~GUk)+%9>fG&63aGz!CA65-epPQr zwQ>Lc|Ea4Vs{b@#u?Qf500IagfB*srAbjRy@csWEyDect2q1s} z0tg_000IagfB*srAbE5vb diff --git a/test/crawler.js b/test/crawler.js index 4a08d9c..4e73c65 100644 --- a/test/crawler.js +++ b/test/crawler.js @@ -127,7 +127,7 @@ describe('Crawler', () => { // -------------------------------------------------------------------------- - it.skip('reorg while executing', async () => { + it('keeps the states after a reorg', async () => { const txid = '3f9de452f0c3c96be737d42aa0941b27412211976688967adb3174ee18b04c64' let didReorg = false function getNextBlock (height, hash) { @@ -144,7 +144,9 @@ describe('Crawler', () => { await reorged(indexer) await indexer.stop() expect(await database.getTransactionHex(txid)).not.to.equal(undefined) - expect(await database.getJigState(txid + '_o1')).to.equal() + const state = JSON.parse(await database.getJigState(txid + '_o1')) + expect(state.props.origin).to.equal('_o1') + expect(state.src).to.match(/class Dragon/) expect(await database.getTransactionHeight(txid)).to.equal(-1) }) }) From 5ac0235643b846d09ecfed884992df4de6aafcfc Mon Sep 17 00:00:00 2001 From: hojarasca Date: Mon, 6 Dec 2021 15:29:19 -0300 Subject: [PATCH 09/82] wip --- src/constants.js | 7 + src/database.bak.js | 591 +++++++++++++++++++++++++++++++ src/database.js | 265 +++++++------- src/sqlite-datasource.js | 748 +++++++++++++++++++++++++++++++++++++++ test/crawler.js | 3 +- test/indexer.js | 5 +- 6 files changed, 1493 insertions(+), 126 deletions(-) create mode 100644 src/constants.js create mode 100644 src/database.bak.js create mode 100644 src/sqlite-datasource.js diff --git a/src/constants.js b/src/constants.js new file mode 100644 index 0000000..72f81f8 --- /dev/null +++ b/src/constants.js @@ -0,0 +1,7 @@ +const HEIGHT_MEMPOOL = -1 +const HEIGHT_UNKNOWN = null + +module.exports = { + HEIGHT_MEMPOOL, + HEIGHT_UNKNOWN +} \ No newline at end of file diff --git a/src/database.bak.js b/src/database.bak.js new file mode 100644 index 0000000..e085ab7 --- /dev/null +++ b/src/database.bak.js @@ -0,0 +1,591 @@ +/** + * database.js + * + * Layer between the database and the application + */ + +const Run = require('run-sdk') +const bsv = require('bsv') +const { SqliteDatasource } = require('./sqlite-datasource') +const { HEIGHT_MEMPOOL, HEIGHT_UNKNOWN } = require('./constants') + +// ------------------------------------------------------------------------------------------------ +// Database +// ------------------------------------------------------------------------------------------------ + +class Database { + constructor (path, logger, readonly = false) { + this.db = new SqliteDatasource(path, logger, readonly) + this.logger = logger + + this.onReadyToExecute = null + this.onAddTransaction = null + this.onDeleteTransaction = null + this.onTrustTransaction = null + this.onUntrustTransaction = null + this.onBanTransaction = null + this.onUnbanTransaction = null + this.onUntrustTransaction = null + this.onRequestDownload = null + this.onUnindexTransaction = null + } + + async open () { + await this.db.setUp() + } + + async close () { + await this.db.tearDown() + } + + async transaction (f) { + return this.db.transaction(f) + } + + // -------------------------------------------------------------------------- + // tx + // -------------------------------------------------------------------------- + + async addBlock (txids, txhexs, height, hash, time) { + const indexes = new Array(txids.length).fill(null).map((_, i) => i) + for (const index of indexes) { + const txid = txids[index] + const txHex = txhexs && txhexs[index] + await this.addTransaction(txid, txHex, height, time) + } + await this.setHeight(height) + await this.setHash(hash) + } + + async addTransaction (txid, txhex, height, time) { + await this.transaction(async () => { + await this.addNewTransaction(txid) + if (height) { await this.setTransactionHeight(txid, height) } + if (time) { await this.setTransactionTime(txid, time) } + }) + + const downloaded = await this.isTransactionDownloaded(txid) + if (downloaded) return + + if (txhex) { + await this.parseAndStoreTransaction(txid, txhex) + } else { + if (this.onRequestDownload) { await this.onRequestDownload(txid) } + } + } + + async parseAndStoreTransaction (txid, hex) { + if (await this.isTransactionDownloaded(txid)) return + + let metadata = null + let bsvtx = null + const inputs = [] + const outputs = [] + + if (!hex) { throw new Error('No hex') } + + try { + bsvtx = new bsv.Transaction(hex) + + bsvtx.inputs.forEach(input => { + const location = `${input.prevTxId.toString('hex')}_o${input.outputIndex}` + inputs.push(location) + }) + + bsvtx.outputs.forEach((output, n) => { + if (output.script.isDataOut() || output.script.isSafeDataOut()) return + outputs.push(`${txid}_o${n}`) + }) + + metadata = Run.util.metadata(hex) + } catch (e) { + this.logger.error(`${txid} => ${e.message}`) + await this.storeParsedNonExecutableTransaction(txid, hex, inputs, outputs) + return + } + + const deps = new Set() + + for (let i = 0; i < metadata.in; i++) { + const prevtxid = bsvtx.inputs[i].prevTxId.toString('hex') + deps.add(prevtxid) + } + + for (const ref of metadata.ref) { + if (ref.startsWith('native://')) { + continue + } else if (ref.includes('berry')) { + const reftxid = ref.slice(0, 64) + deps.add(reftxid) + } else { + const reftxid = ref.slice(0, 64) + deps.add(reftxid) + } + } + + const hasCode = metadata.exec.some(cmd => cmd.op === 'DEPLOY' || cmd.op === 'UPGRADE') + + await this.storeParsedExecutableTransaction(txid, hex, hasCode, deps, inputs, outputs) + + for (const deptxid of deps) { + if (!await this.isTransactionDownloaded(deptxid)) { + if (this.onRequestDownload) await this.onRequestDownload(deptxid) + } + } + } + + async addNewTransaction (txid) { + if (await this.hasTransaction(txid)) return + + const time = Math.round(Date.now() / 1000) + + await this.db.addNewTx(txid, time) + + if (this.onAddTransaction) { await this.onAddTransaction(txid) } + } + + async setTransactionHeight (txid, height) { + await this.db.setTxHeight(txid, height) + } + + async setTransactionTime (txid, time) { + await this.db.setTxTime(txid, time) + } + + async storeParsedNonExecutableTransaction (txid, hex, inputs, outputs) { + await this.transaction(async () => { + const bytes = Buffer.from(hex, 'hex') + + await this.db.setTxBytes(txid, bytes) + await this.db.setExecutableForTx(txid, 0) + + for (const location of inputs) { + await this.db.upsertSpend(location, txid) + } + for (const location of outputs) { + await this.db.setAsUnspend(location) + } + }) + + // Non-executable might be berry data. We execute once we receive them. + const downstreamReadyToExecute = this.db.searchDownstreamTxidsReadyToExecute(txid) + for (const downtxid of downstreamReadyToExecute) { + await this.db.markTxAsExecuting(downtxid) + if (this.onReadyToExecute) { await this.onReadyToExecute(downtxid) } + } + } + + async storeParsedExecutableTransaction (txid, hex, hasCode, deps, inputs, outputs) { + await this.transaction(async () => { + const bytes = Buffer.from(hex, 'hex') + await this.db.setTxBytes(txid, bytes) + await this.db.setExecutableForTx(txid, 1) + await this.db.setHasCodeForTx(txid, hasCode ? 1 : 0) + + for (const location of inputs) { + await this.db.upsertSpend(location, txid) + } + + for (const location of outputs) { + await this.db.setAsUnspend(location) + } + + for (const deptxid of deps) { + await this.addNewTransaction(deptxid) + await this.db.addDep(deptxid, txid) + + const failed = await this.db.getFailedTx(deptxid) + if (failed) { + await this.setTransactionExecutionFailed(txid) + return + } + } + }) + + await this._checkExecutability(txid) + } + + async storeExecutedTransaction (txid, result) { + const { cache, classes, locks, scripthashes } = result + + await this.transaction(async () => { + await this.db.setExecutedForTx(txid, 1) + await this.db.setIndexedForTx(txid, 1) + + await this.db.removeTxFromExecuting(txid) + + for (const key of Object.keys(cache)) { + if (key.startsWith('jig://')) { + const location = key.slice('jig://'.length) + await this.db.setJig(location, JSON.stringify(cache[key])) + continue + } + + if (key.startsWith('berry://')) { + const location = key.slice('berry://'.length) + await this.db.setBerry(location, JSON.stringify(cache[key])) + continue + } + } + + for (const [location, cls] of classes) { + await this.db.setJigClass(location, cls) + } + + for (const [location, lock] of locks) { + await this.db.setJigLock(location, lock) + } + + for (const [location, scripthash] of scripthashes) { + await this.db.setJigScriptHash(location, scripthash) + } + }) + + const downstreamReadyToExecute = await this.db.searchDownstreamTxidsReadyToExecute(txid) + for (const downtxid of downstreamReadyToExecute) { + await this.db.markTxAsExecuting(downtxid) + if (this.onReadyToExecute) { await this.onReadyToExecute(downtxid) } + } + } + + async setTransactionExecutionFailed (txid) { + // await this.transaction(async () => { + // }) + await this.db.setExecutableForTx(txid, 0) + await this.db.setExecutedForTx(txid, 1) + await this.db.setIndexedForTx(txid, 0) + await this.db.removeTxFromExecuting(txid) + + // We try executing downstream transactions if this was marked executable but it wasn't. + // This allows an admin to manually change executable status in the database. + + let executable = false + try { + const rawTx = await this.getTransactionHex(txid) + Run.util.metadata(rawTx) + executable = true + } catch (e) { } + + if (!executable) { + const downstream = await this.db.searchDownstreamForTxid(txid) + for (const downtxid of downstream) { + await this._checkExecutability(downtxid) + } + } + } + + async getTransactionHex (txid) { + return this.db.getTxHex(txid) + } + + async getTransactionTime (txid) { + return this.db.getTxTime(txid) + } + + async getTransactionHeight (txid) { + return this.db.getTxHeight(txid) + } + + async deleteTransaction (txid, deleted = new Set()) { + if (deleted.has(txid)) return + + const txids = [txid] + deleted.add(txid) + + await this.transaction(async () => { + while (txids.length) { + const txid = txids.shift() + + if (this.onDeleteTransaction) { await this.onDeleteTransaction(txid) } + + + await this.db.deleteTx(txid) + await this.db.deleteJigStatesForTxid(txid) + await this.db.deleteBerryStatesForTxid(txid) + await this.db.deleteSpendsForTxid(txid) + await this.db.unspendOutput(txid) + await this.db.deleteDepsForTxid(txid) + + const downtxids = this.db.searchDownstreamForTxid(txid) + + for (const downtxid of downtxids) { + if (deleted.has(downtxid)) continue + deleted.add(downtxid) + txids.push(downtxid) + } + } + }) + } + + async unconfirmTransaction (txid) { + await this.db.unconfirmTx(txid) + } + + async unindexTransaction (txid) { + await this.transaction(async () => { + + if (await this.txIsIndexed()) { + await this.db.setExecutedForTx(txid, 0) + await this.db.setIndexedForTx(txid, 0) + await this.db.deleteJigStatesForTxid(txid) + await this.db.deleteBerryStatesForTxid(txid) + await this.db.removeTxFromExecuting(txid) + + const downloadedTxids = await this.db.searchDownstreamForTxid(txid) + for (const downloadedTxid of downloadedTxids) { + await this.unindexTransaction(downloadedTxid) + } + + if (this.onUnindexTransaction) { await this.onUnindexTransaction(txid) } + } + }) + } + + async hasTransaction (txid) { + return this.db.txExists(txid) + } + + async isTransactionDownloaded (txid) { + const result = this.db.checkTxIsDownloaded(txid) + return result ? !!result[0] : false + } + + async getTransactionsAboveHeight (height) { + const txsOverHeight = await this.db.searchTxsAboveHeight(height) + return txsOverHeight.map(row => row[0]) + } + + async getMempoolTransactionsBeforeTime (time) { + const txs = await this.db.mempoolTxsPreviousToTime(time) + return txs.map(row => row[0]) + } + + async getTransactionsToDownload () { + const rows = await this.db.searchTxsToDownload() + return rows.map(row => row[0]) + } + + async getDownloadedCount () { + return this.db.countDownloadedTxs() + } + + async getIndexedCount () { + return this.countIndexedTxs() + } + + // -------------------------------------------------------------------------- + // spends + // -------------------------------------------------------------------------- + + async getSpend (location) { + return this.db.getSpendingTxid(location) + } + + // -------------------------------------------------------------------------- + // deps + // -------------------------------------------------------------------------- + + async addDep (txid, deptxid) { + await this.addNewTransaction(deptxid) + + await this.db.addDep(deptxid, txid) + + const failed = await this.db.getFailedTx(deptxid) + if (failed) { + await this.setTransactionExecutionFailed(deptxid) + } + } + + async addMissingDeps (txid, deptxids) { + await this.transaction(async () => { + for (const deptxid of deptxids) { + await this.addDep(txid, deptxid) + } + }) + + await this._checkExecutability(txid) + } + + // -------------------------------------------------------------------------- + // jig + // -------------------------------------------------------------------------- + + async getJigState (location) { + return this.db.getJigState(location) + } + + // -------------------------------------------------------------------------- + // unspent + // -------------------------------------------------------------------------- + + async getAllUnspent () { + return this.db.getAllUnspent() + } + + async getAllUnspentByClassOrigin (origin) { + return this.db.getAllUnspentByClassOrigin(origin) + } + + async getAllUnspentByLockOrigin (origin) { + return this.db.getAllUnspentByLockOrigin(origin) + } + + async getAllUnspentByScripthash (scripthash) { + return this.db.getAllUnspentByScripthash(scripthash) + } + + async getAllUnspentByClassOriginAndLockOrigin (clsOrigin, lockOrigin) { + return this.db.getAllUnspentByClassOriginAndLockOrigin(clsOrigin, lockOrigin) + } + + async getAllUnspentByClassOriginAndScripthash (clsOrigin, scripthash) { + return this.db.getAllUnspentByClassOriginAndScripthash(clsOrigin, scripthash) + } + + async getAllUnspentByLockOriginAndScripthash (lockOrigin, scripthash) { + return this.db.getAllUnspentByLockOriginAndScripthash(lockOrigin, scripthash) + } + + async getAllUnspentByClassOriginAndLockOriginAndScripthash (clsOrigin, lockOrigin, scripthash) { + return this.db.getAllUnspentByClassOriginAndLockOriginAndScripthash(clsOrigin, lockOrigin, scripthash) + } + + async getNumUnspent () { + return this.db.countTotalUnspent() + } + + // -------------------------------------------------------------------------- + // berry + // -------------------------------------------------------------------------- + + async getBerryState (location) { + return this.db.getBerry(location) + } + + // -------------------------------------------------------------------------- + // trust + // -------------------------------------------------------------------------- + + async isTrusted (txid) { + return this.db.isTrusted(txid) + } + + async trust (txid) { + if (await this.isTrusted(txid)) return + + const trusted = [txid] + + // Recursively trust code parents + const queue = await this.db.getNonExecutedUpstreamTxIds(txid) + const visited = new Set() + while (queue.length) { + const uptxid = queue.shift() + if (visited.has(uptxid)) continue + if (await this.isTrusted(uptxid)) continue + visited.add(uptxid) + trusted.push(txid) + const txids = await this.db.getNonExecutedUpstreamTxIds(uptxid) + txids.forEach(x => queue.push(x[0])) + } + + for (const trustedTxid of trusted) { + await this.db.setTrust(trustedTxid, 1) + } + + for (const txid of trusted) { + await this._checkExecutability(txid) + } + + if (this.onTrustTransaction) { + for (const txid of trusted) { + await this.onTrustTransaction(txid) + } + } + } + + async untrust (txid) { + if (!await this.isTrusted(txid)) return + await this.transaction(async () => { + await this.unindexTransaction(txid) + await this.db.setTrust(txid, 0) + + }) + if (this.onUntrustTransaction) await this.onUntrustTransaction(txid) + } + + async getTrustlist () { + return this.db.searchAllTrust() + } + + // -------------------------------------------------------------------------- + // ban + // -------------------------------------------------------------------------- + + async isBanned (txid) { + return this.db.checkIsBanned(txid) + } + + async ban (txid) { + await this.transaction(async () => { + await this.unindexTransaction(txid) + await this.db.saveBan(txid) + }) + if (this.onBanTransaction) await this.onBanTransaction(txid) + } + + async unban (txid) { + await this.db.removeBan(txid) + await this._checkExecutability(txid) + if (this.onUnbanTransaction) await this.onUnbanTransaction(txid) + } + + async getBanlist () { + return this.db.searchAllBans() + } + + // -------------------------------------------------------------------------- + // crawl + // -------------------------------------------------------------------------- + + async getHeight () { + return this.db.getCrawlHeight() + } + + async getHash () { + return this.db.getCrawlHash() + } + + async setHeight (height) { + await this.db.setCrawlHeight(height) + } + + async setHash (hash) { + await this.db.setCrawlHash(hash) + } + + // -------------------------------------------------------------------------- + // internal + // -------------------------------------------------------------------------- + + async loadTransactionsToExecute () { + this.logger.debug('Loading transactions to execute') + const txids = await this.db.findAllExecutingTxids() + for (const txid of txids) { + await this._checkExecutability(txid) + } + } + + async _checkExecutability (txid) { + const row = await this.db.txidReadyToExecute(txid) + if (row && row.ready) { + await this.db.markTxAsExecuting(txid) + if (this.onReadyToExecute) { await this.onReadyToExecute(txid) } + } + } +} + +// ------------------------------------------------------------------------------------------------ + +Database.HEIGHT_MEMPOOL = HEIGHT_MEMPOOL + +module.exports = Database diff --git a/src/database.js b/src/database.js index 64733bb..1decea7 100644 --- a/src/database.js +++ b/src/database.js @@ -4,9 +4,9 @@ * Layer between the database and the application */ -const Sqlite3Database = require('better-sqlite3') const Run = require('run-sdk') const bsv = require('bsv') +const { SqliteDatasource } = require('./sqlite-datasource') // ------------------------------------------------------------------------------------------------ // Globals @@ -65,9 +65,9 @@ const GET_DOWNSTREADM_READY_TO_EXECUTE_SQL = ` class Database { constructor (path, logger, readonly = false) { + this.ds = new SqliteDatasource(path, logger, readonly) this.path = path this.logger = logger - this.db = null this.readonly = readonly this.onReadyToExecute = null @@ -81,12 +81,17 @@ class Database { this.onRequestDownload = null } + get db () { + return this.ds.connection + } + async open () { this.logger.debug('Opening' + (this.readonly ? ' readonly' : '') + ' database') if (this.db) throw new Error('Database already open') - this.db = new Sqlite3Database(this.path, { readonly: this.readonly }) + await this.ds.setUp() + // this.db = new Sqlite3Database(this.path, { readonly: this.readonly }) // 100MB cache this.db.pragma('cache_size = 6400') @@ -97,59 +102,59 @@ class Database { // Synchronizes WAL at checkpoints this.db.pragma('synchronous = NORMAL') - - if (!this.readonly) { - // Initialise and perform upgrades - await this.initializeV1() - await this.initializeV2() - await this.initializeV3() - await this.initializeV4() - await this.initializeV5() - await this.initializeV6() - await this.initializeV7() - } - - this.addNewTransactionStmt = this.db.prepare('INSERT OR IGNORE INTO tx (txid, height, time, bytes, has_code, executable, executed, indexed) VALUES (?, null, ?, null, 0, 0, 0, 0)') - this.setTransactionBytesStmt = this.db.prepare('UPDATE tx SET bytes = ? WHERE txid = ?') - this.setTransactionExecutableStmt = this.db.prepare('UPDATE tx SET executable = ? WHERE txid = ?') - this.setTransactionTimeStmt = this.db.prepare('UPDATE tx SET time = ? WHERE txid = ?') - this.setTransactionHeightStmt = this.db.prepare(`UPDATE tx SET height = ? WHERE txid = ? AND (height IS NULL OR height = ${HEIGHT_MEMPOOL})`) - this.setTransactionHasCodeStmt = this.db.prepare('UPDATE tx SET has_code = ? WHERE txid = ?') - this.setTransactionExecutedStmt = this.db.prepare('UPDATE tx SET executed = ? WHERE txid = ?') - this.setTransactionIndexedStmt = this.db.prepare('UPDATE tx SET indexed = ? WHERE txid = ?') - this.hasTransactionStmt = this.db.prepare('SELECT txid FROM tx WHERE txid = ?') - this.getTransactionHexStmt = this.db.prepare('SELECT LOWER(HEX(bytes)) AS hex FROM tx WHERE txid = ?') - this.getTransactionTimeStmt = this.db.prepare('SELECT time FROM tx WHERE txid = ?') - this.getTransactionHeightStmt = this.db.prepare('SELECT height FROM tx WHERE txid = ?') - this.getTransactionHasCodeStmt = this.db.prepare('SELECT has_code FROM tx WHERE txid = ?') - this.getTransactionIndexedStmt = this.db.prepare('SELECT indexed FROM tx WHERE txid = ?') - this.getTransactionFailedStmt = this.db.prepare('SELECT (executed = 1 AND indexed = 0) AS failed FROM tx WHERE txid = ?') - this.getTransactionDownloadedStmt = this.db.prepare('SELECT downloaded FROM tx WHERE txid = ?') - this.deleteTransactionStmt = this.db.prepare('DELETE FROM tx WHERE txid = ?') - this.unconfirmTransactionStmt = this.db.prepare(`UPDATE tx SET height = ${HEIGHT_MEMPOOL} WHERE txid = ?`) - this.getTransactionsAboveHeightStmt = this.db.prepare('SELECT txid FROM tx WHERE height > ?') - this.getMempoolTransactionsBeforeTimeStmt = this.db.prepare(`SELECT txid FROM tx WHERE height = ${HEIGHT_MEMPOOL} AND time < ?`) - this.getTransactionsToDownloadStmt = this.db.prepare('SELECT txid FROM tx WHERE downloaded = 0') - this.getTransactionsDownloadedCountStmt = this.db.prepare('SELECT COUNT(*) AS count FROM tx WHERE downloaded = 1') - this.getTransactionsIndexedCountStmt = this.db.prepare('SELECT COUNT(*) AS count FROM tx WHERE indexed = 1') - this.isReadyToExecuteStmt = this.db.prepare(IS_READY_TO_EXECUTE_SQL) - this.getDownstreamReadyToExecuteStmt = this.db.prepare(GET_DOWNSTREADM_READY_TO_EXECUTE_SQL) - - this.setSpendStmt = this.db.prepare('INSERT OR REPLACE INTO spends (location, spend_txid) VALUES (?, ?)') - this.setUnspentStmt = this.db.prepare('INSERT OR IGNORE INTO spends (location, spend_txid) VALUES (?, null)') - this.getSpendStmt = this.db.prepare('SELECT spend_txid FROM spends WHERE location = ?') - this.unspendOutputsStmt = this.db.prepare('UPDATE spends SET spend_txid = null WHERE spend_txid = ?') - this.deleteSpendsStmt = this.db.prepare('DELETE FROM spends WHERE location LIKE ? || \'%\'') - - this.addDepStmt = this.db.prepare('INSERT OR IGNORE INTO deps (up, down) VALUES (?, ?)') - this.deleteDepsStmt = this.db.prepare('DELETE FROM deps WHERE down = ?') - this.getDownstreamStmt = this.db.prepare('SELECT down FROM deps WHERE up = ?') - this.getUpstreamUnexecutedCodeStmt = this.db.prepare(` - SELECT txdeps.txid as txid - FROM (SELECT up AS txid FROM deps WHERE down = ?) as txdeps - JOIN tx ON tx.txid = txdeps.txid - WHERE tx.executable = 1 AND tx.executed = 0 AND tx.has_code = 1 - `) + // + // if (!this.readonly) { + // // Initialise and perform upgrades + // await this.initializeV1() + // await this.initializeV2() + // await this.initializeV3() + // await this.initializeV4() + // await this.initializeV5() + // await this.initializeV6() + // await this.initializeV7() + // } + + // this.addNewTransactionStmt = this.db.prepare('INSERT OR IGNORE INTO tx (txid, height, time, bytes, has_code, executable, executed, indexed) VALUES (?, null, ?, null, 0, 0, 0, 0)') + // this.setTransactionBytesStmt = this.db.prepare('UPDATE tx SET bytes = ? WHERE txid = ?') + // this.setTransactionExecutableStmt = this.db.prepare('UPDATE tx SET executable = ? WHERE txid = ?') + // this.setTransactionTimeStmt = this.db.prepare('UPDATE tx SET time = ? WHERE txid = ?') + // this.setTransactionHeightStmt = this.db.prepare(`UPDATE tx SET height = ? WHERE txid = ? AND (height IS NULL OR height = ${HEIGHT_MEMPOOL})`) + // this.setTransactionHasCodeStmt = this.db.prepare('UPDATE tx SET has_code = ? WHERE txid = ?') + // this.setTransactionExecutedStmt = this.db.prepare('UPDATE tx SET executed = ? WHERE txid = ?') + // this.setTransactionIndexedStmt = this.db.prepare('UPDATE tx SET indexed = ? WHERE txid = ?') + // this.hasTransactionStmt = this.db.prepare('SELECT txid FROM tx WHERE txid = ?') + // this.getTransactionHexStmt = this.db.prepare('SELECT LOWER(HEX(bytes)) AS hex FROM tx WHERE txid = ?') + // this.getTransactionTimeStmt = this.db.prepare('SELECT time FROM tx WHERE txid = ?') + // this.getTransactionHeightStmt = this.db.prepare('SELECT height FROM tx WHERE txid = ?') + // this.getTransactionHasCodeStmt = this.db.prepare('SELECT has_code FROM tx WHERE txid = ?') + // this.getTransactionIndexedStmt = this.db.prepare('SELECT indexed FROM tx WHERE txid = ?') + // this.getTransactionFailedStmt = this.db.prepare('SELECT (executed = 1 AND indexed = 0) AS failed FROM tx WHERE txid = ?') + // this.getTransactionDownloadedStmt = this.db.prepare('SELECT downloaded FROM tx WHERE txid = ?') + // this.deleteTransactionStmt = this.db.prepare('DELETE FROM tx WHERE txid = ?') + // this.unconfirmTransactionStmt = this.db.prepare(`UPDATE tx SET height = ${HEIGHT_MEMPOOL} WHERE txid = ?`) + // this.getTransactionsAboveHeightStmt = this.db.prepare('SELECT txid FROM tx WHERE height > ?') + // this.getMempoolTransactionsBeforeTimeStmt = this.db.prepare(`SELECT txid FROM tx WHERE height = ${HEIGHT_MEMPOOL} AND time < ?`) + // this.getTransactionsToDownloadStmt = this.db.prepare('SELECT txid FROM tx WHERE downloaded = 0') + // this.getTransactionsDownloadedCountStmt = this.db.prepare('SELECT COUNT(*) AS count FROM tx WHERE downloaded = 1') + // this.getTransactionsIndexedCountStmt = this.db.prepare('SELECT COUNT(*) AS count FROM tx WHERE indexed = 1') + // this.isReadyToExecuteStmt = this.db.prepare(IS_READY_TO_EXECUTE_SQL) + // this.getDownstreamReadyToExecuteStmt = this.db.prepare(GET_DOWNSTREADM_READY_TO_EXECUTE_SQL) + + // this.setSpendStmt = this.db.prepare('INSERT OR REPLACE INTO spends (location, spend_txid) VALUES (?, ?)') + // this.setUnspentStmt = this.db.prepare('INSERT OR IGNORE INTO spends (location, spend_txid) VALUES (?, null)') + // this.getSpendStmt = this.db.prepare('SELECT spend_txid FROM spends WHERE location = ?') + // this.unspendOutputsStmt = this.db.prepare('UPDATE spends SET spend_txid = null WHERE spend_txid = ?') + // this.deleteSpendsStmt = this.db.prepare('DELETE FROM spends WHERE location LIKE ? || \'%\'') + + // this.addDepStmt = this.db.prepare('INSERT OR IGNORE INTO deps (up, down) VALUES (?, ?)') + // this.deleteDepsStmt = this.db.prepare('DELETE FROM deps WHERE down = ?') + // this.getDownstreamStmt = this.db.prepare('SELECT down FROM deps WHERE up = ?') + // this.getUpstreamUnexecutedCodeStmt = this.db.prepare(` + // SELECT txdeps.txid as txid + // FROM (SELECT up AS txid FROM deps WHERE down = ?) as txdeps + // JOIN tx ON tx.txid = txdeps.txid + // WHERE tx.executable = 1 AND tx.executed = 0 AND tx.has_code = 1 + // `) this.setJigStateStmt = this.db.prepare('INSERT OR IGNORE INTO jig (location, state, class, lock, scripthash) VALUES (?, ?, null, null, null)') this.setJigClassStmt = this.db.prepare('UPDATE jig SET class = ? WHERE location = ?') @@ -452,17 +457,7 @@ class Database { } async close () { - if (this.worker) { - this.logger.debug('Terminating background loader') - await this.worker.terminate() - this.worker = null - } - - if (this.db) { - this.logger.debug('Closing' + (this.readonly ? ' readonly' : '') + ' database') - this.db.close() - this.db = null - } + this.ds.tearDown() } async transaction (f) { @@ -576,31 +571,36 @@ class Database { const time = Math.round(Date.now() / 1000) - this.addNewTransactionStmt.run(txid, time) + // this.addNewTransactionStmt.run(txid, time) + await this.ds.addNewTx(txid, time) if (this.onAddTransaction) { await this.onAddTransaction(txid) } } async setTransactionHeight (txid, height) { - this.setTransactionHeightStmt.run(height, txid) + await this.ds.setTxHeight(txid, height) } async setTransactionTime (txid, time) { - this.setTransactionTimeStmt.run(time, txid) + await this.ds.setTxTime(txid, time) } async storeParsedNonExecutableTransaction (txid, hex, inputs, outputs) { - await this.transaction(() => { + await this.transaction(async () => { const bytes = Buffer.from(hex, 'hex') - this.setTransactionBytesStmt.run(bytes, txid) - this.setTransactionExecutableStmt.run(0, txid) + await this.ds.setTxBytes(txid, bytes) + await this.ds.setExecutableForTx(txid, 0) - inputs.forEach(location => this.setSpendStmt.run(location, txid)) - outputs.forEach(location => this.setUnspentStmt.run(location)) + for (const location of inputs) { + await this.ds.upsertSpend(location, txid) + } + for (const location of outputs) { + await this.ds.setAsUnspent(location) + } }) // Non-executable might be berry data. We execute once we receive them. - const downstreamReadyToExecute = this.getDownstreamReadyToExecuteStmt.raw(true).all(txid).map(x => x[0]) + const downstreamReadyToExecute = await this.ds.searchDownstreamTxidsReadyToExecute(txid) for (const downtxid of downstreamReadyToExecute) { this.markExecutingStmt.run(downtxid) if (this.onReadyToExecute) { await this.onReadyToExecute(downtxid) } @@ -610,18 +610,24 @@ class Database { async storeParsedExecutableTransaction (txid, hex, hasCode, deps, inputs, outputs) { await this.transaction(async () => { const bytes = Buffer.from(hex, 'hex') - this.setTransactionBytesStmt.run(bytes, txid) - this.setTransactionExecutableStmt.run(1, txid) - this.setTransactionHasCodeStmt.run(hasCode ? 1 : 0, txid) + await this.ds.setTxBytes(txid, bytes) + await this.ds.setExecutableForTx(txid, 1) - inputs.forEach(location => this.setSpendStmt.run(location, txid)) - outputs.forEach(location => this.setUnspentStmt.run(location)) + await this.ds.setHasCodeForTx(txid, hasCode ? 1 : 0) + + for (const location of inputs) { + await this.ds.upsertSpend(location, txid) + } + for (const location of outputs) { + await this.ds.setAsUnspent(location) + } for (const deptxid of deps) { await this.addNewTransaction(deptxid) - this.addDepStmt.run(deptxid, txid) + await this.ds.addDep(deptxid, txid) - if (this.getTransactionFailedStmt.get(deptxid).failed) { + const failed = await this.ds.getFailedTx(deptxid) + if (failed) { await this.setTransactionExecutionFailed(txid) return } @@ -635,14 +641,15 @@ class Database { const { cache, classes, locks, scripthashes } = result await this.transaction(async () => { - this.setTransactionExecutedStmt.run(1, txid) - this.setTransactionIndexedStmt.run(1, txid) + await this.ds.setExecutedForTx(txid, 1) + await this.ds.setIndexedForTx(txid, 1) this.unmarkExecutingStmt.run(txid) for (const key of Object.keys(cache)) { if (key.startsWith('jig://')) { const location = key.slice('jig://'.length) - this.setJigStateStmt.run(location, JSON.stringify(cache[key])) + await this.ds.setJig(location, JSON.stringify(cache[key])) + // this.setJigStateStmt.run(location, JSON.stringify(cache[key])) continue } @@ -666,7 +673,8 @@ class Database { } }) - const downstreamReadyToExecute = this.getDownstreamReadyToExecuteStmt.raw(true).all(txid).map(x => x[0]) + + const downstreamReadyToExecute = await this.ds.searchDownstreamForTxid(txid) for (const downtxid of downstreamReadyToExecute) { this.markExecutingStmt.run(downtxid) if (this.onReadyToExecute) { await this.onReadyToExecute(downtxid) } @@ -674,10 +682,10 @@ class Database { } async setTransactionExecutionFailed (txid) { - await this.transaction(() => { - this.setTransactionExecutableStmt.run(0, txid) - this.setTransactionExecutedStmt.run(1, txid) - this.setTransactionIndexedStmt.run(0, txid) + await this.transaction(async () => { + await this.ds.setExecutableForTx(txid, 0) + await this.ds.setExecutedForTx(txid, 1) + await this.ds.setIndexedForTx(txid, 0) this.unmarkExecutingStmt.run(txid) }) @@ -692,7 +700,8 @@ class Database { } catch (e) { } if (!executable) { - const downstream = this.getDownstreamStmt.raw(true).all(txid).map(x => x[0]) + + const downstream = await this.ds.searchDownstreamForTxid(txid) for (const downtxid of downstream) { await this._checkExecutability(downtxid) } @@ -700,18 +709,15 @@ class Database { } async getTransactionHex (txid) { - const row = this.getTransactionHexStmt.raw(true).get(txid) - return row && row[0] + return this.ds.getTxHex(txid) } async getTransactionTime (txid) { - const row = this.getTransactionTimeStmt.raw(true).get(txid) - return row && row[0] + return this.ds.getTxTime(txid) } async getTransactionHeight (txid) { - const row = this.getTransactionHeightStmt.raw(true).get(txid) - return row && row[0] + return this.ds.getTxHeight(txid) } async deleteTransaction (txid, deleted = new Set()) { @@ -726,14 +732,14 @@ class Database { if (this.onDeleteTransaction) { await this.onDeleteTransaction(txid) } - this.deleteTransactionStmt.run(txid) + await this.ds.deleteTx(txid) this.deleteJigStatesStmt.run(txid) this.deleteBerryStatesStmt.run(txid) - this.deleteSpendsStmt.run(txid) - this.unspendOutputsStmt.run(txid) - this.deleteDepsStmt.run(txid) + await this.ds.deleteSpendsForTxid(txid) + await this.ds.unspendOutput(txid) + await this.ds.deleteDepsForTxid(txid) - const downtxids = this.getDownstreamStmt.raw(true).all(txid).map(row => row[0]) + const downtxids = await this.ds.searchDownstreamForTxid(txid) for (const downtxid of downtxids) { if (deleted.has(downtxid)) continue @@ -745,19 +751,20 @@ class Database { } async unconfirmTransaction (txid) { - this.unconfirmTransactionStmt.run(txid) + await this.ds.unconfirmTx(txid) } async unindexTransaction (txid) { await this.transaction(async () => { - if (this.getTransactionIndexedStmt.raw(true).get(txid)[0]) { - this.setTransactionExecutedStmt.run(0, txid) - this.setTransactionIndexedStmt.run(0, txid) + const indexed = await this.ds.txIsIndexed(txid) + if (indexed) { + await this.ds.setExecutedForTx(txid, 0) + await this.ds.setIndexedForTx(txid, 0) this.deleteJigStatesStmt.run(txid) this.deleteBerryStatesStmt.run(txid) this.unmarkExecutingStmt.run(txid) - const downloadedTxids = this.getDownstreamStmt.raw(true).all(txid).map(row => row[0]) + const downloadedTxids = await this.ds.searchDownstreamForTxid(txid) for (const downloadedTxid of downloadedTxids) { await this.unindexTransaction(downloadedTxid) } @@ -767,30 +774,40 @@ class Database { }) } - async hasTransaction (txid) { return !!this.hasTransactionStmt.get(txid) } + async hasTransaction (txid) { + return this.ds.txExists(txid) + } async isTransactionDownloaded (txid) { - const result = this.getTransactionDownloadedStmt.raw(true).get(txid) - return result ? !!result[0] : false + return this.ds.checkTxIsDownloaded(txid) } - async getTransactionsAboveHeight (height) { return this.getTransactionsAboveHeightStmt.raw(true).all(height).map(row => row[0]) } + async getTransactionsAboveHeight (height) { + return this.ds.searchTxsAboveHeight(height) + } - async getMempoolTransactionsBeforeTime (time) { return this.getMempoolTransactionsBeforeTimeStmt.raw(true).all(time).map(row => row[0]) } + async getMempoolTransactionsBeforeTime (time) { + return this.ds.mempoolTxsPreviousToTime(time) + } - async getTransactionsToDownload () { return this.getTransactionsToDownloadStmt.raw(true).all().map(row => row[0]) } + async getTransactionsToDownload () { + return this.ds.searchTxsToDownload() + } - async getDownloadedCount () { return this.getTransactionsDownloadedCountStmt.get().count } + async getDownloadedCount () { + return this.ds.countDownloadedTxs() + } - async getIndexedCount () { return this.getTransactionsIndexedCountStmt.get().count } + async getIndexedCount () { + return this.ds.countIndexedTxs() + } // -------------------------------------------------------------------------- // spends // -------------------------------------------------------------------------- async getSpend (location) { - const row = this.getSpendStmt.raw(true).get(location) - return row && row[0] + return this.ds.getSpendingTxid(location) } // -------------------------------------------------------------------------- @@ -800,9 +817,10 @@ class Database { async addDep (txid, deptxid) { await this.addNewTransaction(deptxid) - this.addDepStmt.run(deptxid, txid) + await this.ds.addDep(deptxid, txid) - if (this.getTransactionFailedStmt.get(deptxid).failed) { + const failed = await this.ds.getFailedTx(deptxid) + if (failed) { await this.setTransactionExecutionFailed(deptxid) } } @@ -890,7 +908,7 @@ class Database { const trusted = [txid] // Recursively trust code parents - const queue = this.getUpstreamUnexecutedCodeStmt.raw(true).all(txid).map(x => x[0]) + const queue = await this.ds.getNonExecutedUpstreamTxIds(txid) const visited = new Set() while (queue.length) { const uptxid = queue.shift() @@ -898,7 +916,8 @@ class Database { if (await this.isTrusted(uptxid)) continue visited.add(uptxid) trusted.push(txid) - this.getUpstreamUnexecutedCodeStmt.raw(true).all(txid).forEach(x => queue.push(x[0])) + const nextTxids = await this.ds.getNonExecutedUpstreamTxIds(uptxid) + nextTxids.forEach(txid => queue.push(txid)) } trusted.forEach(txid => this.setTrustedStmt.run(txid, 1)) @@ -989,7 +1008,7 @@ class Database { } async _checkExecutability (txid) { - const row = this.isReadyToExecuteStmt.get(txid) + const row = await this.ds.txidReadyToExecute(txid) if (row && row.ready) { this.markExecutingStmt.run(txid) if (this.onReadyToExecute) { await this.onReadyToExecute(txid) } diff --git a/src/sqlite-datasource.js b/src/sqlite-datasource.js new file mode 100644 index 0000000..137b453 --- /dev/null +++ b/src/sqlite-datasource.js @@ -0,0 +1,748 @@ +/** + * database.js + * + * Layer between the database and the application + */ +const Sqlite3Database = require('better-sqlite3') +const { HEIGHT_MEMPOOL } = require('./constants') + +// The + in the following 2 queries before downloaded improves performance by NOT using the +// tx_downloaded index, which is rarely an improvement over a simple filter for single txns. +// See: https://www.sqlite.org/optoverview.html +const IS_READY_TO_EXECUTE_SQL = ` + SELECT ( + downloaded = 1 + AND executable = 1 + AND executed = 0 + AND (has_code = 0 OR (SELECT COUNT(*) FROM trust WHERE trust.txid = tx.txid AND trust.value = 1) = 1) + AND txid NOT IN ban + AND ( + SELECT COUNT(*) + FROM tx AS tx2 + JOIN deps + ON deps.up = tx2.txid + WHERE deps.down = tx.txid + AND (+tx2.downloaded = 0 OR (tx2.executable = 1 AND tx2.executed = 0)) + ) = 0 + ) AS ready + FROM tx + WHERE txid = ? + ` + +const GET_DOWNSTREAM_READY_TO_EXECUTE_SQL = ` + SELECT down + FROM deps + JOIN tx + ON tx.txid = deps.down + WHERE up = ? + AND +downloaded = 1 + AND executable = 1 + AND executed = 0 + AND (has_code = 0 OR (SELECT COUNT(*) FROM trust WHERE trust.txid = tx.txid AND trust.value = 1) = 1) + AND txid NOT IN ban + AND ( + SELECT COUNT(*) + FROM tx AS tx2 + JOIN deps + ON deps.up = tx2.txid + WHERE deps.down = tx.txid + AND (+tx2.downloaded = 0 OR (tx2.executable = 1 AND tx2.executed = 0)) + ) = 0 + ` + +// ------------------------------------------------------------------------------------------------ +// Database +// ------------------------------------------------------------------------------------------------ + +class SqliteDatasource { + constructor (path, logger, readonly = false) { + this.path = path + this.logger = logger + this.readonly = readonly + this.connection = null + } + + async setUp () { + this.logger.debug('Opening' + (this.readonly ? ' readonly' : '') + ' database') + if (this.connection) throw new Error('Database already open') + + this.connection = new Sqlite3Database(this.path, { readonly: this.readonly }) + + + // 100MB cache + this.connection.pragma('cache_size = 6400') + this.connection.pragma('page_size = 16384') + + // WAL mode allows simultaneous readers + this.connection.pragma('journal_mode = WAL') + + // Synchronizes WAL at checkpoints + this.connection.pragma('synchronous = NORMAL') + + if (!this.readonly) { + // Initialise and perform upgrades + await this.initializeV1() + await this.initializeV2() + await this.initializeV3() + await this.initializeV4() + await this.initializeV5() + await this.initializeV6() + await this.initializeV7() + } + // + this.addNewTransactionStmt = this.connection.prepare('INSERT OR IGNORE INTO tx (txid, height, time, bytes, has_code, executable, executed, indexed) VALUES (?, null, ?, null, 0, 0, 0, 0)') + this.setTransactionBytesStmt = this.connection.prepare('UPDATE tx SET bytes = ? WHERE txid = ?') + this.setTransactionExecutableStmt = this.connection.prepare('UPDATE tx SET executable = ? WHERE txid = ?') + this.setTransactionTimeStmt = this.connection.prepare('UPDATE tx SET time = ? WHERE txid = ?') + this.setTransactionHeightStmt = this.connection.prepare(`UPDATE tx SET height = ? WHERE txid = ? AND (height IS NULL OR height = ${HEIGHT_MEMPOOL})`) + this.setTransactionHasCodeStmt = this.connection.prepare('UPDATE tx SET has_code = ? WHERE txid = ?') + this.setTransactionExecutedStmt = this.connection.prepare('UPDATE tx SET executed = ? WHERE txid = ?') + this.setTransactionIndexedStmt = this.connection.prepare('UPDATE tx SET indexed = ? WHERE txid = ?') + this.txExistsStmt = this.connection.prepare('SELECT txid FROM tx WHERE txid = ?') + this.getTransactionHexStmt = this.connection.prepare('SELECT LOWER(HEX(bytes)) AS hex FROM tx WHERE txid = ?') + this.getTransactionTimeStmt = this.connection.prepare('SELECT time FROM tx WHERE txid = ?') + this.getTransactionHeightStmt = this.connection.prepare('SELECT height FROM tx WHERE txid = ?') + this.getTransactionHasCodeStmt = this.connection.prepare('SELECT has_code FROM tx WHERE txid = ?') + this.getTransactionIndexedStmt = this.connection.prepare('SELECT indexed FROM tx WHERE txid = ?') + this.getTransactionFailedStmt = this.connection.prepare('SELECT (executed = 1 AND indexed = 0) AS failed FROM tx WHERE txid = ?') + this.getTransactionDownloadedStmt = this.connection.prepare('SELECT downloaded FROM tx WHERE txid = ?') + this.deleteTransactionStmt = this.connection.prepare('DELETE FROM tx WHERE txid = ?') + this.unconfirmTransactionStmt = this.connection.prepare(`UPDATE tx SET height = ${HEIGHT_MEMPOOL} WHERE txid = ?`) + this.getTransactionsAboveHeightStmt = this.connection.prepare('SELECT txid FROM tx WHERE height > ?') + this.getMempoolTransactionsBeforeTimeStmt = this.connection.prepare(`SELECT txid FROM tx WHERE height = ${HEIGHT_MEMPOOL} AND time < ?`) + this.getTransactionsToDownloadStmt = this.connection.prepare('SELECT txid FROM tx WHERE downloaded = 0') + this.getTransactionsDownloadedCountStmt = this.connection.prepare('SELECT COUNT(*) AS count FROM tx WHERE downloaded = 1') + this.getTransactionsIndexedCountStmt = this.connection.prepare('SELECT COUNT(*) AS count FROM tx WHERE indexed = 1') + this.isReadyToExecuteStmt = this.connection.prepare(IS_READY_TO_EXECUTE_SQL) + this.getDownstreamReadyToExecuteStmt = this.connection.prepare(GET_DOWNSTREAM_READY_TO_EXECUTE_SQL) + + this.setSpendStmt = this.connection.prepare('INSERT OR REPLACE INTO spends (location, spend_txid) VALUES (?, ?)') + this.setUnspentStmt = this.connection.prepare('INSERT OR IGNORE INTO spends (location, spend_txid) VALUES (?, null)') + this.getSpendStmt = this.connection.prepare('SELECT spend_txid FROM spends WHERE location = ?') + this.unspendOutputsStmt = this.connection.prepare('UPDATE spends SET spend_txid = null WHERE spend_txid = ?') + this.deleteSpendsStmt = this.connection.prepare('DELETE FROM spends WHERE location LIKE ? || \'%\'') + // + this.addDepStmt = this.connection.prepare('INSERT OR IGNORE INTO deps (up, down) VALUES (?, ?)') + this.deleteDepsStmt = this.connection.prepare('DELETE FROM deps WHERE down = ?') + this.getDownstreamStmt = this.connection.prepare('SELECT down FROM deps WHERE up = ?') + this.getUpstreamUnexecutedCodeStmt = this.connection.prepare(` + SELECT txdeps.txid as txid + FROM (SELECT up AS txid FROM deps WHERE down = ?) as txdeps + JOIN tx ON tx.txid = txdeps.txid + WHERE tx.executable = 1 AND tx.executed = 0 AND tx.has_code = 1 + `) + // + this.setJigStateStmt = this.connection.prepare('INSERT OR IGNORE INTO jig (location, state, class, lock, scripthash) VALUES (?, ?, null, null, null)') + this.setJigClassStmt = this.connection.prepare('UPDATE jig SET class = ? WHERE location = ?') + this.setJigLockStmt = this.connection.prepare('UPDATE jig SET lock = ? WHERE location = ?') + this.setJigScripthashStmt = this.connection.prepare('UPDATE jig SET scripthash = ? WHERE location = ?') + this.getJigStateStmt = this.connection.prepare('SELECT state FROM jig WHERE location = ?') + this.deleteJigStatesStmt = this.connection.prepare('DELETE FROM jig WHERE location LIKE ? || \'%\'') + + const getAllUnspentSql = ` + SELECT spends.location AS location FROM spends + JOIN jig ON spends.location = jig.location + WHERE spends.spend_txid IS NULL` + this.getAllUnspentStmt = this.connection.prepare(getAllUnspentSql) + this.getAllUnspentByClassStmt = this.connection.prepare(`${getAllUnspentSql} AND jig.class = ?`) + this.getAllUnspentByLockStmt = this.connection.prepare(`${getAllUnspentSql} AND jig.lock = ?`) + this.getAllUnspentByScripthashStmt = this.connection.prepare(`${getAllUnspentSql} AND jig.scripthash = ?`) + this.getAllUnspentByClassLockStmt = this.connection.prepare(`${getAllUnspentSql} AND jig.class = ? AND lock = ?`) + this.getAllUnspentByClassScripthashStmt = this.connection.prepare(`${getAllUnspentSql} AND jig.class = ? AND scripthash = ?`) + this.getAllUnspentByLockScripthashStmt = this.connection.prepare(`${getAllUnspentSql} AND jig.lock = ? AND scripthash = ?`) + this.getAllUnspentByClassLockScripthashStmt = this.connection.prepare(`${getAllUnspentSql} AND jig.class = ? AND jig.lock = ? AND scripthash = ?`) + this.getNumUnspentStmt = this.connection.prepare('SELECT COUNT(*) as unspent FROM spends JOIN jig ON spends.location = jig.location WHERE spends.spend_txid IS NULL') + + this.setBerryStateStmt = this.connection.prepare('INSERT OR IGNORE INTO berry (location, state) VALUES (?, ?)') + this.getBerryStateStmt = this.connection.prepare('SELECT state FROM berry WHERE location = ?') + this.deleteBerryStatesStmt = this.connection.prepare('DELETE FROM berry WHERE location LIKE ? || \'%\'') + + this.setTrustedStmt = this.connection.prepare('INSERT OR REPLACE INTO trust (txid, value) VALUES (?, ?)') + this.getTrustlistStmt = this.connection.prepare('SELECT txid FROM trust WHERE value = 1') + this.isTrustedStmt = this.connection.prepare('SELECT COUNT(*) FROM trust WHERE txid = ? AND value = 1') + + this.banStmt = this.connection.prepare('INSERT OR REPLACE INTO ban (txid) VALUES (?)') + this.unbanStmt = this.connection.prepare('DELETE FROM ban WHERE txid = ?') + this.isBannedStmt = this.connection.prepare('SELECT COUNT(*) FROM ban WHERE txid = ?') + this.getBanlistStmt = this.connection.prepare('SELECT txid FROM ban') + + this.getHeightStmt = this.connection.prepare('SELECT value FROM crawl WHERE key = \'height\'') + this.getHashStmt = this.connection.prepare('SELECT value FROM crawl WHERE key = \'hash\'') + this.setHeightStmt = this.connection.prepare('UPDATE crawl SET value = ? WHERE key = \'height\'') + this.setHashStmt = this.connection.prepare('UPDATE crawl SET value = ? WHERE key = \'hash\'') + + this.markExecutingStmt = this.connection.prepare('INSERT OR IGNORE INTO executing (txid) VALUES (?)') + this.unmarkExecutingStmt = this.connection.prepare('DELETE FROM executing WHERE txid = ?') + this.findAllExecutingTxidsStmt = this.connection.prepare('SELECT txid FROM executing') + } + + async initializeV1 () { + if (this.connection.pragma('user_version')[0].user_version !== 0) return + + this.logger.info('Setting up database v1') + + await this.transaction(() => { + this.connection.pragma('user_version = 1') + + this.connection.prepare( + `CREATE TABLE IF NOT EXISTS tx ( + txid TEXT NOT NULL, + height INTEGER, + time INTEGER, + hex TEXT, + has_code INTEGER, + executable INTEGER, + executed INTEGER, + indexed INTEGER, + UNIQUE(txid) + )` + ).run() + + this.connection.prepare( + `CREATE TABLE IF NOT EXISTS spends ( + location TEXT NOT NULL PRIMARY KEY, + spend_txid TEXT + ) WITHOUT ROWID` + ).run() + + this.connection.prepare( + `CREATE TABLE IF NOT EXISTS deps ( + up TEXT NOT NULL, + down TEXT NOT NULL, + UNIQUE(up, down) + )` + ).run() + + this.connection.prepare( + `CREATE TABLE IF NOT EXISTS jig ( + location TEXT NOT NULL PRIMARY KEY, + state TEXT NOT NULL, + class TEXT, + scripthash TEXT, + lock TEXT + ) WITHOUT ROWID` + ).run() + + this.connection.prepare( + `CREATE TABLE IF NOT EXISTS berry ( + location TEXT NOT NULL PRIMARY KEY, + state TEXT NOT NULL + ) WITHOUT ROWID` + ).run() + + this.connection.prepare( + `CREATE TABLE IF NOT EXISTS trust ( + txid TEXT NOT NULL PRIMARY KEY, + value INTEGER + ) WITHOUT ROWID` + ).run() + + this.connection.prepare( + `CREATE TABLE IF NOT EXISTS ban ( + txid TEXT NOT NULL PRIMARY KEY + ) WITHOUT ROWID` + ).run() + + this.connection.prepare( + `CREATE TABLE IF NOT EXISTS crawl ( + role TEXT UNIQUE, + height INTEGER, + hash TEXT + )` + ).run() + + this.connection.prepare( + 'CREATE INDEX IF NOT EXISTS tx_txid_index ON tx (txid)' + ).run() + + this.connection.prepare( + 'CREATE INDEX IF NOT EXISTS jig_index ON jig (class)' + ).run() + + this.connection.prepare( + 'INSERT OR IGNORE INTO crawl (role, height, hash) VALUES (\'tip\', 0, NULL)' + ).run() + }) + } + + async initializeV2 () { + if (this.connection.pragma('user_version')[0].user_version !== 1) return + + this.logger.info('Setting up database v2') + + await this.transaction(() => { + this.connection.pragma('user_version = 2') + + this.connection.prepare( + `CREATE TABLE tx_v2 ( + txid TEXT NOT NULL, + height INTEGER, + time INTEGER, + bytes BLOB, + has_code INTEGER, + executable INTEGER, + executed INTEGER, + indexed INTEGER + )` + ).run() + + const txids = this.connection.prepare('SELECT txid FROM tx').all().map(row => row.txid) + const gettx = this.connection.prepare('SELECT * FROM tx WHERE txid = ?') + const insert = this.connection.prepare('INSERT INTO tx_v2 (txid, height, time, bytes, has_code, executable, executed, indexed) VALUES (?, ?, ?, ?, ?, ?, ?, ?)') + + this.logger.info('Migrating data') + for (const txid of txids) { + const row = gettx.get(txid) + const bytes = row.hex ? Buffer.from(row.hex, 'hex') : null + insert.run(row.txid, row.height, row.time, bytes, row.has_code, row.executable, row.executed, row.indexed) + } + + this.connection.prepare( + 'DROP INDEX tx_txid_index' + ).run() + + this.connection.prepare( + 'DROP TABLE tx' + ).run() + + this.connection.prepare( + 'ALTER TABLE tx_v2 RENAME TO tx' + ).run() + + this.connection.prepare( + 'CREATE INDEX IF NOT EXISTS tx_txid_index ON tx (txid)' + ).run() + + this.logger.info('Saving results') + }) + + this.logger.info('Optimizing database') + this.connection.prepare('VACUUM').run() + } + + async initializeV3 () { + if (this.connection.pragma('user_version')[0].user_version !== 2) return + + this.logger.info('Setting up database v3') + + await this.transaction(() => { + this.connection.pragma('user_version = 3') + + this.connection.prepare('CREATE INDEX IF NOT EXISTS deps_up_index ON deps (up)').run() + this.connection.prepare('CREATE INDEX IF NOT EXISTS deps_down_index ON deps (down)').run() + this.connection.prepare('CREATE INDEX IF NOT EXISTS trust_txid_index ON trust (txid)').run() + + this.logger.info('Saving results') + }) + } + + async initializeV4 () { + if (this.connection.pragma('user_version')[0].user_version !== 3) return + + this.logger.info('Setting up database v4') + + await this.transaction(() => { + this.connection.pragma('user_version = 4') + + this.connection.prepare('ALTER TABLE tx ADD COLUMN downloaded INTEGER GENERATED ALWAYS AS (bytes IS NOT NULL) VIRTUAL').run() + + this.connection.prepare('CREATE INDEX IF NOT EXISTS tx_downloaded_index ON tx (downloaded)').run() + + this.logger.info('Saving results') + }) + } + + async initializeV5 () { + if (this.connection.pragma('user_version')[0].user_version !== 4) return + + this.logger.info('Setting up database v5') + + await this.transaction(() => { + this.connection.pragma('user_version = 5') + + this.connection.prepare('CREATE INDEX IF NOT EXISTS ban_txid_index ON ban (txid)').run() + this.connection.prepare('CREATE INDEX IF NOT EXISTS tx_height_index ON tx (height)').run() + + this.logger.info('Saving results') + }) + } + + async initializeV6 () { + if (this.connection.pragma('user_version')[0].user_version !== 5) return + + this.logger.info('Setting up database v6') + + await this.transaction(() => { + this.connection.pragma('user_version = 6') + + const height = this.connection.prepare('SELECT height FROM crawl WHERE role = \'tip\'').raw(true).all()[0] + const hash = this.connection.prepare('SELECT hash FROM crawl WHERE role = \'tip\'').raw(true).all()[0] + + this.connection.prepare('DROP TABLE crawl').run() + + this.connection.prepare( + `CREATE TABLE IF NOT EXISTS crawl ( + key TEXT UNIQUE, + value TEXT + )` + ).run() + + this.connection.prepare('INSERT INTO crawl (key, value) VALUES (\'height\', ?)').run(height.toString()) + this.connection.prepare('INSERT INTO crawl (key, value) VALUES (\'hash\', ?)').run(hash) + + this.logger.info('Saving results') + }) + } + + async initializeV7 () { + if (this.connection.pragma('user_version')[0].user_version !== 6) return + + this.logger.info('Setting up database v7') + + await this.transaction(() => { + this.connection.pragma('user_version = 7') + + this.logger.info('Getting possible transactions to execute') + const stmt = this.connection.prepare(` + SELECT txid + FROM tx + WHERE downloaded = 1 + AND executable = 1 + AND executed = 0 + AND (has_code = 0 OR (SELECT COUNT(*) FROM trust WHERE trust.txid = tx.txid AND trust.value = 1) = 1) + AND txid NOT IN ban + `) + const txids = stmt.raw(true).all().map(x => x[0]) + + const isReadyToExecuteStmt = this.connection.prepare(IS_READY_TO_EXECUTE_SQL) + + const ready = [] + for (let i = 0; i < txids.length; i++) { + const txid = txids[i] + const row = isReadyToExecuteStmt.get(txid) + if (row && row.ready) ready.push(txid) + if (i % 1000 === 0) console.log('Checking to execute', i, 'of', txids.length) + } + + this.logger.info('Marking', ready.length, 'transactions to execute') + this.connection.prepare('CREATE TABLE IF NOT EXISTS executing (txid TEXT UNIQUE)').run() + const markExecutingStmt = this.connection.prepare('INSERT OR IGNORE INTO executing (txid) VALUES (?)') + ready.forEach(txid => markExecutingStmt.run(txid)) + + this.logger.info('Saving results') + }) + } + + async tearDown () { + if (this.connection) { + this.logger.debug('Closing' + (this.readonly ? ' readonly' : '') + ' database') + this.connection.close() + this.connection = null + } + } + + async transaction (f) { + if (!this.connection) return + try { + this.connection.exec('begin;') + await f() + this.connection.exec('commit;') + } catch (e) { + this.connection.exec('rollback;') + console.error(e) + throw e + } + // this.connection.transaction(f)() + } + + async txExists (txid) { + return !!this.txExistsStmt.get(txid) + } + + async checkTxIsDownloaded (txid) { + const result = this.getTransactionDownloadedStmt.raw(true).get(txid) + return result ? !!result[0] : false + } + + async searchTxsAboveHeight (height) { + return this.getTransactionsAboveHeightStmt.raw(true).all(height) + } + + async mempoolTxsPreviousToTime (time) { + return this.getMempoolTransactionsBeforeTimeStmt.raw(true).all(time) + } + + async searchTxsToDownload () { + return this.getTransactionsToDownloadStmt.raw(true).all() + } + + async countDownloadedTxs () { + return this.getTransactionsDownloadedCountStmt.get().count + } + + async countIndexedTxs () { + return this.getTransactionsIndexedCountStmt.get().count + } + + async getFailedTx (deptxid) { + return this.getTransactionFailedStmt.get(deptxid).failed + } + + async addNewTx (txid, time) { + await this.addNewTransactionStmt.run(txid, time) + } + + async setTxHeight (txid, height) { + this.setTransactionHeightStmt.run(height, txid) + } + + async setTxTime (txid, time) { + this.setTransactionTimeStmt.run(time, txid) + } + + async setTxBytes (txid, bytes) { + this.setTransactionBytesStmt.run(bytes, txid) + } + + async setExecutableForTx (txid, executable) { + this.setTransactionExecutableStmt.run(executable, txid) + } + + async setHasCodeForTx (txid, hasCode) { + this.setTransactionHasCodeStmt.run(hasCode, txid) + } + + async setExecutedForTx (txid, executed) { + this.setTransactionExecutedStmt.run(executed, txid) + } + + async setIndexedForTx (txid, indexed) { + this.setTransactionIndexedStmt.run(indexed, txid) + } + + async txIsIndexed (txid) { + return this.getTransactionIndexedStmt.raw(true).get(txid)[0] + } + + async getTxHex (txid) { + const row = this.getTransactionHexStmt.raw(true).get(txid) + return row && row[0] + } + + async getTxTime (txid) { + const row = this.getTransactionTimeStmt.raw(true).get(txid) + return row && row[0] + } + + async getTxHeight (txid) { + const row = this.getTransactionHeightStmt.raw(true).get(txid) + return row && row[0] + } + + async deleteTx (txid) { + this.deleteTransactionStmt.run(txid) + } + + async unconfirmTx (txid) { + this.unconfirmTransactionStmt.run(txid) + } + + // executing + + async markTxAsExecuting (txid) { + this.markExecutingStmt.run(txid) + } + + async removeTxFromExecuting (txid) { + this.unmarkExecutingStmt.run(txid) + } + + async findAllExecutingTxids () { + return this.findAllExecutingTxidsStmt.raw(true).all().map(x => x[0]) + } + + async txidReadyToExecute (txid) { + return this.isReadyToExecuteStmt.get(txid) + } + + // spends + + async getSpendingTxid (location) { + const row = this.getSpendStmt.raw(true).get(location) + return row && row[0] + } + + async upsertSpend (location, txid) { + await this.setSpendStmt.run(location, txid) + } + + async setAsUnspent (location) { + this.setUnspentStmt.run(location) + } + + async deleteSpendsForTxid (txid) { + this.deleteSpendsStmt.run(txid) + } + + async unspendOutput (txid) { + this.unspendOutputsStmt.run(txid) + } + + // deps + + async addDep (deptxid, txid) { + this.addDepStmt.run(deptxid, txid) + } + + async searchDownstreamTxidsReadyToExecute (txid) { + return this.getDownstreamReadyToExecuteStmt.raw(true).all(txid).map(x => x[0]) + } + + async searchDownstreamForTxid (txid) { + return this.getDownstreamStmt.raw(true).all(txid).map(x => x[0]) + } + + async deleteDepsForTxid (txid) { + this.deleteDepsStmt.run(txid) + } + + async getNonExecutedUpstreamTxIds (txid) { + return this.getUpstreamUnexecutedCodeStmt.raw(true).all(txid).map(x => x[0]) + } + + // jig + + async getJigState (location) { + const row = this.getJigStateStmt.raw(true).get(location) + return row && row[0] + } + + async setJig (location, jig) { + this.setJigStateStmt.run(location, jig) + } + + async setBerry (location, berry) { + this.setBerryStateStmt.run(location, berry) + } + + async getBerry (location) { + const row = this.getBerryStateStmt.raw(true).get(location) + return row && row[0] + } + + async setJigClass (location, cls) { + this.setJigClassStmt.run(cls, location) + } + + async setJigLock (location, lock) { + this.setJigLockStmt.run(lock, location) + } + + async setJigScriptHash (location, scriptHash) { + this.setJigScripthashStmt.run(scriptHash, location) + } + + async deleteJigStatesForTxid (txid) { + this.deleteJigStatesStmt.run(txid) + } + + async deleteBerryStatesForTxid (txid) { + this.deleteBerryStatesStmt.run(txid) + } + + // unspent + + async getAllUnspent () { + return this.getAllUnspentStmt.raw(true).all().map(row => row[0]) + } + + async getAllUnspentByClassOrigin (origin) { + return this.getAllUnspentByClassStmt.raw(true).all(origin).map(row => row[0]) + } + + async getAllUnspentByLockOrigin (origin) { + return this.getAllUnspentByLockStmt.raw(true).all(origin).map(row => row[0]) + } + + async getAllUnspentByScripthash (scripthash) { + return this.getAllUnspentByScripthashStmt.raw(true).all(scripthash).map(row => row[0]) + } + + async getAllUnspentByClassOriginAndLockOrigin (clsOrigin, lockOrigin) { + return this.getAllUnspentByClassLockStmt.raw(true).all(clsOrigin, lockOrigin).map(row => row[0]) + } + + async getAllUnspentByClassOriginAndScripthash (clsOrigin, scripthash) { + return this.getAllUnspentByClassScripthashStmt.raw(true).all(clsOrigin, scripthash).map(row => row[0]) + } + + async getAllUnspentByLockOriginAndScripthash (lockOrigin, scripthash) { + return this.getAllUnspentByLockScripthashStmt.raw(true).all(lockOrigin, scripthash).map(row => row[0]) + } + + async getAllUnspentByClassOriginAndLockOriginAndScripthash (clsOrigin, lockOrigin, scripthash) { + return this.getAllUnspentByClassLockScripthashStmt.raw(true).all(clsOrigin, lockOrigin, scripthash).map(row => row[0]) + } + + async countTotalUnspent () { + return this.getNumUnspentStmt.get().unspent + } + + // trust + + async isTrusted (txid) { + const row = this.isTrustedStmt.raw(true).get(txid) + return !!row && !!row[0] + } + + async setTrust (txid, trusted) { + this.setTrustedStmt.run(txid, trusted) + } + + async searchAllTrust () { + return this.getTrustlistStmt.raw(true).all().map(x => x[0]) + } + + // ban + + async checkIsBanned (txid) { + const row = this.isBannedStmt.raw(true).get(txid) + return !!row && !!row[0] + } + + async saveBan (txid) { + this.banStmt.run(txid) + } + + async removeBan (txid) { + this.unbanStmt.run(txid) + } + + async searchAllBans () { + return this.getBanlistStmt.raw(true).all().map(x => x[0]) + } + + // crawl + + async setCrawlHeight (heigth) { + this.setHeightStmt.run(heigth.toString()) + } + + async setCrawlHash (hash) { + this.setHashStmt.run(hash) + } + + async getCrawlHeight () { + const row = this.getHeightStmt.raw(true).all()[0] + return row && parseInt(row[0]) + } + + async getCrawlHash () { + const row = this.getHashStmt.raw(true).all()[0] + return row && row[0] + } +} + +// ------------------------------------------------------------------------------------------------ + +module.exports = { SqliteDatasource } diff --git a/test/crawler.js b/test/crawler.js index 4e73c65..09d3dae 100644 --- a/test/crawler.js +++ b/test/crawler.js @@ -37,9 +37,10 @@ describe('Crawler', () => { } const api = { getNextBlock, fetch } const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, []) + const promise = indexed(indexer, txid) await indexer.start() await database.trust(txid) - await indexed(indexer, txid) + await promise await indexer.stop() expect(await database.getHeight()).to.equal(1) expect(await database.getHash()).to.equal('abc') diff --git a/test/indexer.js b/test/indexer.js index 6e70570..6c12ab3 100644 --- a/test/indexer.js +++ b/test/indexer.js @@ -35,11 +35,12 @@ describe('Indexer', () => { it('add and index', async () => { const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, []) await indexer.start() + const promise = indexed(indexer, '9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102') await database.addTransaction('3f9de452f0c3c96be737d42aa0941b27412211976688967adb3174ee18b04c64') await database.addTransaction('9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102') await database.trust('3f9de452f0c3c96be737d42aa0941b27412211976688967adb3174ee18b04c64') await database.trust('9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102') - await indexed(indexer, '9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102') + await promise const txid = '9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102' expect(await database.getTransactionHex(txid)).to.equal(fetch(txid).hex) expect(await database.getTransactionHeight(txid)).to.equal(null) @@ -168,8 +169,8 @@ describe('Indexer', () => { await successPromise await database.addTransaction(txid2, rawtx2) await failurePromise - expect(await database.getSpend(txid1 + '_o1')).to.equal(txid2) await indexer.stop() + expect(await database.getSpend(txid1 + '_o1')).to.equal(txid2) }) // -------------------------------------------------------------------------- From 3aeb88797dcd61c926268a8635479f82da596ce0 Mon Sep 17 00:00:00 2001 From: Miguel Duarte Date: Tue, 7 Dec 2021 00:15:07 -0300 Subject: [PATCH 10/82] all tests working --- src/database.js | 512 ++++----------------------------------- src/executor.js | 12 +- src/sqlite-datasource.js | 3 +- test/indexer.js | 17 +- 4 files changed, 59 insertions(+), 485 deletions(-) diff --git a/src/database.js b/src/database.js index 1decea7..609902c 100644 --- a/src/database.js +++ b/src/database.js @@ -7,57 +7,7 @@ const Run = require('run-sdk') const bsv = require('bsv') const { SqliteDatasource } = require('./sqlite-datasource') - -// ------------------------------------------------------------------------------------------------ -// Globals -// ------------------------------------------------------------------------------------------------ - -const HEIGHT_MEMPOOL = -1 -const HEIGHT_UNKNOWN = null - -// The + in the following 2 queries before downloaded improves performance by NOT using the -// tx_downloaded index, which is rarely an improvement over a simple filter for single txns. -// See: https://www.sqlite.org/optoverview.html -const IS_READY_TO_EXECUTE_SQL = ` - SELECT ( - downloaded = 1 - AND executable = 1 - AND executed = 0 - AND (has_code = 0 OR (SELECT COUNT(*) FROM trust WHERE trust.txid = tx.txid AND trust.value = 1) = 1) - AND txid NOT IN ban - AND ( - SELECT COUNT(*) - FROM tx AS tx2 - JOIN deps - ON deps.up = tx2.txid - WHERE deps.down = tx.txid - AND (+tx2.downloaded = 0 OR (tx2.executable = 1 AND tx2.executed = 0)) - ) = 0 - ) AS ready - FROM tx - WHERE txid = ? - ` - -const GET_DOWNSTREADM_READY_TO_EXECUTE_SQL = ` - SELECT down - FROM deps - JOIN tx - ON tx.txid = deps.down - WHERE up = ? - AND +downloaded = 1 - AND executable = 1 - AND executed = 0 - AND (has_code = 0 OR (SELECT COUNT(*) FROM trust WHERE trust.txid = tx.txid AND trust.value = 1) = 1) - AND txid NOT IN ban - AND ( - SELECT COUNT(*) - FROM tx AS tx2 - JOIN deps - ON deps.up = tx2.txid - WHERE deps.down = tx.txid - AND (+tx2.downloaded = 0 OR (tx2.executable = 1 AND tx2.executed = 0)) - ) = 0 - ` +const { HEIGHT_MEMPOOL, HEIGHT_UNKNOWN } = require('./constants') // ------------------------------------------------------------------------------------------------ // Database @@ -86,378 +36,11 @@ class Database { } async open () { - this.logger.debug('Opening' + (this.readonly ? ' readonly' : '') + ' database') - - if (this.db) throw new Error('Database already open') - await this.ds.setUp() - // this.db = new Sqlite3Database(this.path, { readonly: this.readonly }) - - // 100MB cache - this.db.pragma('cache_size = 6400') - this.db.pragma('page_size = 16384') - - // WAL mode allows simultaneous readers - this.db.pragma('journal_mode = WAL') - - // Synchronizes WAL at checkpoints - this.db.pragma('synchronous = NORMAL') - // - // if (!this.readonly) { - // // Initialise and perform upgrades - // await this.initializeV1() - // await this.initializeV2() - // await this.initializeV3() - // await this.initializeV4() - // await this.initializeV5() - // await this.initializeV6() - // await this.initializeV7() - // } - - // this.addNewTransactionStmt = this.db.prepare('INSERT OR IGNORE INTO tx (txid, height, time, bytes, has_code, executable, executed, indexed) VALUES (?, null, ?, null, 0, 0, 0, 0)') - // this.setTransactionBytesStmt = this.db.prepare('UPDATE tx SET bytes = ? WHERE txid = ?') - // this.setTransactionExecutableStmt = this.db.prepare('UPDATE tx SET executable = ? WHERE txid = ?') - // this.setTransactionTimeStmt = this.db.prepare('UPDATE tx SET time = ? WHERE txid = ?') - // this.setTransactionHeightStmt = this.db.prepare(`UPDATE tx SET height = ? WHERE txid = ? AND (height IS NULL OR height = ${HEIGHT_MEMPOOL})`) - // this.setTransactionHasCodeStmt = this.db.prepare('UPDATE tx SET has_code = ? WHERE txid = ?') - // this.setTransactionExecutedStmt = this.db.prepare('UPDATE tx SET executed = ? WHERE txid = ?') - // this.setTransactionIndexedStmt = this.db.prepare('UPDATE tx SET indexed = ? WHERE txid = ?') - // this.hasTransactionStmt = this.db.prepare('SELECT txid FROM tx WHERE txid = ?') - // this.getTransactionHexStmt = this.db.prepare('SELECT LOWER(HEX(bytes)) AS hex FROM tx WHERE txid = ?') - // this.getTransactionTimeStmt = this.db.prepare('SELECT time FROM tx WHERE txid = ?') - // this.getTransactionHeightStmt = this.db.prepare('SELECT height FROM tx WHERE txid = ?') - // this.getTransactionHasCodeStmt = this.db.prepare('SELECT has_code FROM tx WHERE txid = ?') - // this.getTransactionIndexedStmt = this.db.prepare('SELECT indexed FROM tx WHERE txid = ?') - // this.getTransactionFailedStmt = this.db.prepare('SELECT (executed = 1 AND indexed = 0) AS failed FROM tx WHERE txid = ?') - // this.getTransactionDownloadedStmt = this.db.prepare('SELECT downloaded FROM tx WHERE txid = ?') - // this.deleteTransactionStmt = this.db.prepare('DELETE FROM tx WHERE txid = ?') - // this.unconfirmTransactionStmt = this.db.prepare(`UPDATE tx SET height = ${HEIGHT_MEMPOOL} WHERE txid = ?`) - // this.getTransactionsAboveHeightStmt = this.db.prepare('SELECT txid FROM tx WHERE height > ?') - // this.getMempoolTransactionsBeforeTimeStmt = this.db.prepare(`SELECT txid FROM tx WHERE height = ${HEIGHT_MEMPOOL} AND time < ?`) - // this.getTransactionsToDownloadStmt = this.db.prepare('SELECT txid FROM tx WHERE downloaded = 0') - // this.getTransactionsDownloadedCountStmt = this.db.prepare('SELECT COUNT(*) AS count FROM tx WHERE downloaded = 1') - // this.getTransactionsIndexedCountStmt = this.db.prepare('SELECT COUNT(*) AS count FROM tx WHERE indexed = 1') - // this.isReadyToExecuteStmt = this.db.prepare(IS_READY_TO_EXECUTE_SQL) - // this.getDownstreamReadyToExecuteStmt = this.db.prepare(GET_DOWNSTREADM_READY_TO_EXECUTE_SQL) - - // this.setSpendStmt = this.db.prepare('INSERT OR REPLACE INTO spends (location, spend_txid) VALUES (?, ?)') - // this.setUnspentStmt = this.db.prepare('INSERT OR IGNORE INTO spends (location, spend_txid) VALUES (?, null)') - // this.getSpendStmt = this.db.prepare('SELECT spend_txid FROM spends WHERE location = ?') - // this.unspendOutputsStmt = this.db.prepare('UPDATE spends SET spend_txid = null WHERE spend_txid = ?') - // this.deleteSpendsStmt = this.db.prepare('DELETE FROM spends WHERE location LIKE ? || \'%\'') - - // this.addDepStmt = this.db.prepare('INSERT OR IGNORE INTO deps (up, down) VALUES (?, ?)') - // this.deleteDepsStmt = this.db.prepare('DELETE FROM deps WHERE down = ?') - // this.getDownstreamStmt = this.db.prepare('SELECT down FROM deps WHERE up = ?') - // this.getUpstreamUnexecutedCodeStmt = this.db.prepare(` - // SELECT txdeps.txid as txid - // FROM (SELECT up AS txid FROM deps WHERE down = ?) as txdeps - // JOIN tx ON tx.txid = txdeps.txid - // WHERE tx.executable = 1 AND tx.executed = 0 AND tx.has_code = 1 - // `) - - this.setJigStateStmt = this.db.prepare('INSERT OR IGNORE INTO jig (location, state, class, lock, scripthash) VALUES (?, ?, null, null, null)') - this.setJigClassStmt = this.db.prepare('UPDATE jig SET class = ? WHERE location = ?') - this.setJigLockStmt = this.db.prepare('UPDATE jig SET lock = ? WHERE location = ?') - this.setJigScripthashStmt = this.db.prepare('UPDATE jig SET scripthash = ? WHERE location = ?') - this.getJigStateStmt = this.db.prepare('SELECT state FROM jig WHERE location = ?') - this.deleteJigStatesStmt = this.db.prepare('DELETE FROM jig WHERE location LIKE ? || \'%\'') - - const getAllUnspentSql = ` - SELECT spends.location AS location FROM spends - JOIN jig ON spends.location = jig.location - WHERE spends.spend_txid IS NULL` - this.getAllUnspentStmt = this.db.prepare(getAllUnspentSql) - this.getAllUnspentByClassStmt = this.db.prepare(`${getAllUnspentSql} AND jig.class = ?`) - this.getAllUnspentByLockStmt = this.db.prepare(`${getAllUnspentSql} AND jig.lock = ?`) - this.getAllUnspentByScripthashStmt = this.db.prepare(`${getAllUnspentSql} AND jig.scripthash = ?`) - this.getAllUnspentByClassLockStmt = this.db.prepare(`${getAllUnspentSql} AND jig.class = ? AND lock = ?`) - this.getAllUnspentByClassScripthashStmt = this.db.prepare(`${getAllUnspentSql} AND jig.class = ? AND scripthash = ?`) - this.getAllUnspentByLockScripthashStmt = this.db.prepare(`${getAllUnspentSql} AND jig.lock = ? AND scripthash = ?`) - this.getAllUnspentByClassLockScripthashStmt = this.db.prepare(`${getAllUnspentSql} AND jig.class = ? AND jig.lock = ? AND scripthash = ?`) - this.getNumUnspentStmt = this.db.prepare('SELECT COUNT(*) as unspent FROM spends JOIN jig ON spends.location = jig.location WHERE spends.spend_txid IS NULL') - - this.setBerryStateStmt = this.db.prepare('INSERT OR IGNORE INTO berry (location, state) VALUES (?, ?)') - this.getBerryStateStmt = this.db.prepare('SELECT state FROM berry WHERE location = ?') - this.deleteBerryStatesStmt = this.db.prepare('DELETE FROM berry WHERE location LIKE ? || \'%\'') - - this.setTrustedStmt = this.db.prepare('INSERT OR REPLACE INTO trust (txid, value) VALUES (?, ?)') - this.getTrustlistStmt = this.db.prepare('SELECT txid FROM trust WHERE value = 1') - this.isTrustedStmt = this.db.prepare('SELECT COUNT(*) FROM trust WHERE txid = ? AND value = 1') - - this.banStmt = this.db.prepare('INSERT OR REPLACE INTO ban (txid) VALUES (?)') - this.unbanStmt = this.db.prepare('DELETE FROM ban WHERE txid = ?') - this.isBannedStmt = this.db.prepare('SELECT COUNT(*) FROM ban WHERE txid = ?') - this.getBanlistStmt = this.db.prepare('SELECT txid FROM ban') - - this.getHeightStmt = this.db.prepare('SELECT value FROM crawl WHERE key = \'height\'') - this.getHashStmt = this.db.prepare('SELECT value FROM crawl WHERE key = \'hash\'') - this.setHeightStmt = this.db.prepare('UPDATE crawl SET value = ? WHERE key = \'height\'') - this.setHashStmt = this.db.prepare('UPDATE crawl SET value = ? WHERE key = \'hash\'') - - this.markExecutingStmt = this.db.prepare('INSERT OR IGNORE INTO executing (txid) VALUES (?)') - this.unmarkExecutingStmt = this.db.prepare('DELETE FROM executing WHERE txid = ?') - } - - async initializeV1 () { - if (this.db.pragma('user_version')[0].user_version !== 0) return - - this.logger.info('Setting up database v1') - - await this.transaction(() => { - this.db.pragma('user_version = 1') - - this.db.prepare( - `CREATE TABLE IF NOT EXISTS tx ( - txid TEXT NOT NULL, - height INTEGER, - time INTEGER, - hex TEXT, - has_code INTEGER, - executable INTEGER, - executed INTEGER, - indexed INTEGER, - UNIQUE(txid) - )` - ).run() - - this.db.prepare( - `CREATE TABLE IF NOT EXISTS spends ( - location TEXT NOT NULL PRIMARY KEY, - spend_txid TEXT - ) WITHOUT ROWID` - ).run() - - this.db.prepare( - `CREATE TABLE IF NOT EXISTS deps ( - up TEXT NOT NULL, - down TEXT NOT NULL, - UNIQUE(up, down) - )` - ).run() - - this.db.prepare( - `CREATE TABLE IF NOT EXISTS jig ( - location TEXT NOT NULL PRIMARY KEY, - state TEXT NOT NULL, - class TEXT, - scripthash TEXT, - lock TEXT - ) WITHOUT ROWID` - ).run() - - this.db.prepare( - `CREATE TABLE IF NOT EXISTS berry ( - location TEXT NOT NULL PRIMARY KEY, - state TEXT NOT NULL - ) WITHOUT ROWID` - ).run() - - this.db.prepare( - `CREATE TABLE IF NOT EXISTS trust ( - txid TEXT NOT NULL PRIMARY KEY, - value INTEGER - ) WITHOUT ROWID` - ).run() - - this.db.prepare( - `CREATE TABLE IF NOT EXISTS ban ( - txid TEXT NOT NULL PRIMARY KEY - ) WITHOUT ROWID` - ).run() - - this.db.prepare( - `CREATE TABLE IF NOT EXISTS crawl ( - role TEXT UNIQUE, - height INTEGER, - hash TEXT - )` - ).run() - - this.db.prepare( - 'CREATE INDEX IF NOT EXISTS tx_txid_index ON tx (txid)' - ).run() - - this.db.prepare( - 'CREATE INDEX IF NOT EXISTS jig_index ON jig (class)' - ).run() - - this.db.prepare( - 'INSERT OR IGNORE INTO crawl (role, height, hash) VALUES (\'tip\', 0, NULL)' - ).run() - }) - } - - async initializeV2 () { - if (this.db.pragma('user_version')[0].user_version !== 1) return - - this.logger.info('Setting up database v2') - - await this.transaction(() => { - this.db.pragma('user_version = 2') - - this.db.prepare( - `CREATE TABLE tx_v2 ( - txid TEXT NOT NULL, - height INTEGER, - time INTEGER, - bytes BLOB, - has_code INTEGER, - executable INTEGER, - executed INTEGER, - indexed INTEGER - )` - ).run() - - const txids = this.db.prepare('SELECT txid FROM tx').all().map(row => row.txid) - const gettx = this.db.prepare('SELECT * FROM tx WHERE txid = ?') - const insert = this.db.prepare('INSERT INTO tx_v2 (txid, height, time, bytes, has_code, executable, executed, indexed) VALUES (?, ?, ?, ?, ?, ?, ?, ?)') - - this.logger.info('Migrating data') - for (const txid of txids) { - const row = gettx.get(txid) - const bytes = row.hex ? Buffer.from(row.hex, 'hex') : null - insert.run(row.txid, row.height, row.time, bytes, row.has_code, row.executable, row.executed, row.indexed) - } - - this.db.prepare( - 'DROP INDEX tx_txid_index' - ).run() - - this.db.prepare( - 'DROP TABLE tx' - ).run() - - this.db.prepare( - 'ALTER TABLE tx_v2 RENAME TO tx' - ).run() - - this.db.prepare( - 'CREATE INDEX IF NOT EXISTS tx_txid_index ON tx (txid)' - ).run() - - this.logger.info('Saving results') - }) - - this.logger.info('Optimizing database') - this.db.prepare('VACUUM').run() - } - - async initializeV3 () { - if (this.db.pragma('user_version')[0].user_version !== 2) return - - this.logger.info('Setting up database v3') - - await this.transaction(() => { - this.db.pragma('user_version = 3') - - this.db.prepare('CREATE INDEX IF NOT EXISTS deps_up_index ON deps (up)').run() - this.db.prepare('CREATE INDEX IF NOT EXISTS deps_down_index ON deps (down)').run() - this.db.prepare('CREATE INDEX IF NOT EXISTS trust_txid_index ON trust (txid)').run() - - this.logger.info('Saving results') - }) - } - - async initializeV4 () { - if (this.db.pragma('user_version')[0].user_version !== 3) return - - this.logger.info('Setting up database v4') - - await this.transaction(() => { - this.db.pragma('user_version = 4') - - this.db.prepare('ALTER TABLE tx ADD COLUMN downloaded INTEGER GENERATED ALWAYS AS (bytes IS NOT NULL) VIRTUAL').run() - - this.db.prepare('CREATE INDEX IF NOT EXISTS tx_downloaded_index ON tx (downloaded)').run() - - this.logger.info('Saving results') - }) - } - - async initializeV5 () { - if (this.db.pragma('user_version')[0].user_version !== 4) return - - this.logger.info('Setting up database v5') - - await this.transaction(() => { - this.db.pragma('user_version = 5') - - this.db.prepare('CREATE INDEX IF NOT EXISTS ban_txid_index ON ban (txid)').run() - this.db.prepare('CREATE INDEX IF NOT EXISTS tx_height_index ON tx (height)').run() - - this.logger.info('Saving results') - }) - } - - async initializeV6 () { - if (this.db.pragma('user_version')[0].user_version !== 5) return - - this.logger.info('Setting up database v6') - - await this.transaction(() => { - this.db.pragma('user_version = 6') - - const height = this.db.prepare('SELECT height FROM crawl WHERE role = \'tip\'').raw(true).all()[0] - const hash = this.db.prepare('SELECT hash FROM crawl WHERE role = \'tip\'').raw(true).all()[0] - - this.db.prepare('DROP TABLE crawl').run() - - this.db.prepare( - `CREATE TABLE IF NOT EXISTS crawl ( - key TEXT UNIQUE, - value TEXT - )` - ).run() - - this.db.prepare('INSERT INTO crawl (key, value) VALUES (\'height\', ?)').run(height.toString()) - this.db.prepare('INSERT INTO crawl (key, value) VALUES (\'hash\', ?)').run(hash) - - this.logger.info('Saving results') - }) - } - - async initializeV7 () { - if (this.db.pragma('user_version')[0].user_version !== 6) return - - this.logger.info('Setting up database v7') - - await this.transaction(() => { - this.db.pragma('user_version = 7') - - this.logger.info('Getting possible transactions to execute') - const stmt = this.db.prepare(` - SELECT txid - FROM tx - WHERE downloaded = 1 - AND executable = 1 - AND executed = 0 - AND (has_code = 0 OR (SELECT COUNT(*) FROM trust WHERE trust.txid = tx.txid AND trust.value = 1) = 1) - AND txid NOT IN ban - `) - const txids = stmt.raw(true).all().map(x => x[0]) - - const isReadyToExecuteStmt = this.db.prepare(IS_READY_TO_EXECUTE_SQL) - - const ready = [] - for (let i = 0; i < txids.length; i++) { - const txid = txids[i] - const row = isReadyToExecuteStmt.get(txid) - if (row && row.ready) ready.push(txid) - if (i % 1000 === 0) console.log('Checking to execute', i, 'of', txids.length) - } - - this.logger.info('Marking', ready.length, 'transactions to execute') - this.db.prepare('CREATE TABLE IF NOT EXISTS executing (txid TEXT UNIQUE)').run() - const markExecutingStmt = this.db.prepare('INSERT OR IGNORE INTO executing (txid) VALUES (?)') - ready.forEach(txid => markExecutingStmt.run(txid)) - - this.logger.info('Saving results') - }) } async close () { - this.ds.tearDown() + await this.ds.tearDown() } async transaction (f) { @@ -602,7 +185,7 @@ class Database { // Non-executable might be berry data. We execute once we receive them. const downstreamReadyToExecute = await this.ds.searchDownstreamTxidsReadyToExecute(txid) for (const downtxid of downstreamReadyToExecute) { - this.markExecutingStmt.run(downtxid) + await this.ds.markTxAsExecuting(downtxid) if (this.onReadyToExecute) { await this.onReadyToExecute(downtxid) } } } @@ -643,40 +226,34 @@ class Database { await this.transaction(async () => { await this.ds.setExecutedForTx(txid, 1) await this.ds.setIndexedForTx(txid, 1) - this.unmarkExecutingStmt.run(txid) + await this.ds.removeTxFromExecuting(txid) for (const key of Object.keys(cache)) { if (key.startsWith('jig://')) { const location = key.slice('jig://'.length) await this.ds.setJig(location, JSON.stringify(cache[key])) - // this.setJigStateStmt.run(location, JSON.stringify(cache[key])) - continue - } - - if (key.startsWith('berry://')) { + } else if (key.startsWith('berry://')) { const location = key.slice('berry://'.length) - this.setBerryStateStmt.run(location, JSON.stringify(cache[key])) - continue + await this.ds.setBerry(location, JSON.stringify(cache[key])) } } for (const [location, cls] of classes) { - this.setJigClassStmt.run(cls, location) + await this.ds.setJigClass(location, cls) } for (const [location, lock] of locks) { - this.setJigLockStmt.run(lock, location) + await this.ds.setJigLockStmt(location, lock) } for (const [location, scripthash] of scripthashes) { - this.setJigScripthashStmt.run(scripthash, location) + await this.ds.setJigScriptHash(location, scripthash) } }) - const downstreamReadyToExecute = await this.ds.searchDownstreamForTxid(txid) for (const downtxid of downstreamReadyToExecute) { - this.markExecutingStmt.run(downtxid) + await this.ds.markTxAsExecuting(downtxid) if (this.onReadyToExecute) { await this.onReadyToExecute(downtxid) } } } @@ -686,7 +263,7 @@ class Database { await this.ds.setExecutableForTx(txid, 0) await this.ds.setExecutedForTx(txid, 1) await this.ds.setIndexedForTx(txid, 0) - this.unmarkExecutingStmt.run(txid) + await this.ds.removeTxFromExecuting(txid) }) // We try executing downstream transactions if this was marked executable but it wasn't. @@ -700,7 +277,6 @@ class Database { } catch (e) { } if (!executable) { - const downstream = await this.ds.searchDownstreamForTxid(txid) for (const downtxid of downstream) { await this._checkExecutability(downtxid) @@ -733,8 +309,8 @@ class Database { if (this.onDeleteTransaction) { await this.onDeleteTransaction(txid) } await this.ds.deleteTx(txid) - this.deleteJigStatesStmt.run(txid) - this.deleteBerryStatesStmt.run(txid) + await this.ds.deleteJigStatesForTxid(txid) + await this.ds.deleteBerryStatesForTxid(txid) await this.ds.deleteSpendsForTxid(txid) await this.ds.unspendOutput(txid) await this.ds.deleteDepsForTxid(txid) @@ -760,9 +336,9 @@ class Database { if (indexed) { await this.ds.setExecutedForTx(txid, 0) await this.ds.setIndexedForTx(txid, 0) - this.deleteJigStatesStmt.run(txid) - this.deleteBerryStatesStmt.run(txid) - this.unmarkExecutingStmt.run(txid) + await this.ds.deleteJigStatesStmt(txid) + await this.ds.deleteBerryStatesForTxid(txid) + await this.ds.removeTxFromExecuting(txid) const downloadedTxids = await this.ds.searchDownstreamForTxid(txid) for (const downloadedTxid of downloadedTxids) { @@ -840,8 +416,7 @@ class Database { // -------------------------------------------------------------------------- async getJigState (location) { - const row = this.getJigStateStmt.raw(true).get(location) - return row && row[0] + return this.ds.getJigState(location) } // -------------------------------------------------------------------------- @@ -849,39 +424,39 @@ class Database { // -------------------------------------------------------------------------- async getAllUnspent () { - return this.getAllUnspentStmt.raw(true).all().map(row => row[0]) + return this.ds.getAllUnspent() } async getAllUnspentByClassOrigin (origin) { - return this.getAllUnspentByClassStmt.raw(true).all(origin).map(row => row[0]) + return this.ds.getAllUnspentByClassOrigin(origin) } async getAllUnspentByLockOrigin (origin) { - return this.getAllUnspentByLockStmt.raw(true).all(origin).map(row => row[0]) + return this.ds.getAllUnspentByLockOrigin(origin) } async getAllUnspentByScripthash (scripthash) { - return this.getAllUnspentByScripthashStmt.raw(true).all(scripthash).map(row => row[0]) + return this.ds.getAllUnspentByScripthash(scripthash) } async getAllUnspentByClassOriginAndLockOrigin (clsOrigin, lockOrigin) { - return this.getAllUnspentByClassLockStmt.raw(true).all(clsOrigin, lockOrigin).map(row => row[0]) + return this.ds.getAllUnspentByClassOriginAndLockOrigin(clsOrigin, lockOrigin) } async getAllUnspentByClassOriginAndScripthash (clsOrigin, scripthash) { - return this.getAllUnspentByClassScripthashStmt.raw(true).all(clsOrigin, scripthash).map(row => row[0]) + return this.ds.getAllUnspentByClassOriginAndScripthash(clsOrigin, scripthash) } async getAllUnspentByLockOriginAndScripthash (lockOrigin, scripthash) { - return this.getAllUnspentByLockScripthashStmt.raw(true).all(lockOrigin, scripthash).map(row => row[0]) + return this.ds.getAllUnspentByLockOriginAndScripthash(lockOrigin, scripthash) } async getAllUnspentByClassOriginAndLockOriginAndScripthash (clsOrigin, lockOrigin, scripthash) { - return this.getAllUnspentByClassLockScripthashStmt.raw(true).all(clsOrigin, lockOrigin, scripthash).map(row => row[0]) + return this.ds.getAllUnspentByClassOriginAndLockOriginAndScriptHash(clsOrigin, lockOrigin, scripthash) } async getNumUnspent () { - return this.getNumUnspentStmt.get().unspent + return this.ds.countTotalUnspent() } // -------------------------------------------------------------------------- @@ -889,8 +464,7 @@ class Database { // -------------------------------------------------------------------------- async getBerryState (location) { - const row = this.getBerryStateStmt.raw(true).get(location) - return row && row[0] + return this.ds.getBerry(location) } // -------------------------------------------------------------------------- @@ -898,8 +472,7 @@ class Database { // -------------------------------------------------------------------------- async isTrusted (txid) { - const row = this.isTrustedStmt.raw(true).get(txid) - return !!row && !!row[0] + return this.ds.isTrusted(txid) } async trust (txid) { @@ -920,7 +493,9 @@ class Database { nextTxids.forEach(txid => queue.push(txid)) } - trusted.forEach(txid => this.setTrustedStmt.run(txid, 1)) + for (const trustedTxid of trusted) { + await this.ds.setTrust(trustedTxid, 1) + } for (const txid of trusted) { await this._checkExecutability(txid) @@ -937,13 +512,13 @@ class Database { if (!await this.isTrusted(txid)) return await this.transaction(async () => { await this.unindexTransaction(txid) - this.setTrustedStmt.run(txid, 0) + await this.ds.setTrust(txid, 0) }) if (this.onUntrustTransaction) await this.onUntrustTransaction(txid) } async getTrustlist () { - return this.getTrustlistStmt.raw(true).all().map(x => x[0]) + return this.ds.searchAllTrust() } // -------------------------------------------------------------------------- @@ -951,26 +526,25 @@ class Database { // -------------------------------------------------------------------------- async isBanned (txid) { - const row = this.isBannedStmt.raw(true).get(txid) - return !!row && !!row[0] + return this.ds.checkIsBanned(txid) } async ban (txid) { await this.transaction(async () => { await this.unindexTransaction(txid) - this.banStmt.run(txid) + await this.ds.saveBan(txid) }) if (this.onBanTransaction) await this.onBanTransaction(txid) } async unban (txid) { - this.unbanStmt.run(txid) + await this.ds.removeBan(txid) await this._checkExecutability(txid) if (this.onUnbanTransaction) await this.onUnbanTransaction(txid) } async getBanlist () { - return this.getBanlistStmt.raw(true).all().map(x => x[0]) + return this.ds.searchAllBans() } // -------------------------------------------------------------------------- @@ -978,21 +552,19 @@ class Database { // -------------------------------------------------------------------------- async getHeight () { - const row = this.getHeightStmt.raw(true).all()[0] - return row && parseInt(row[0]) + return this.ds.getCrawlHeight() } async getHash () { - const row = this.getHashStmt.raw(true).all()[0] - return row && row[0] + return this.ds.getCrawlHash() } async setHeight (height) { - this.setHeightStmt.run(height.toString()) + await this.ds.setCrawlHeight(height) } async setHash (hash) { - this.setHashStmt.run(hash) + await this.ds.setCrawlHash(hash) } // -------------------------------------------------------------------------- @@ -1010,7 +582,7 @@ class Database { async _checkExecutability (txid) { const row = await this.ds.txidReadyToExecute(txid) if (row && row.ready) { - this.markExecutingStmt.run(txid) + await this.ds.markTxAsExecuting(txid) if (this.onReadyToExecute) { await this.onReadyToExecute(txid) } } } diff --git a/src/executor.js b/src/executor.js index dbecbcb..f8fece3 100644 --- a/src/executor.js +++ b/src/executor.js @@ -77,12 +77,9 @@ class Executor { const hex = await this.database.getTransactionHex(txid) const trustList = await this.database.getTrustlist() + let result = null try { - const result = await Bus.sendRequest(worker, 'execute', txid, hex, trustList) - - if (this.onIndexed) { - this.onIndexed(txid, result).catch(console.error) - } + result = await Bus.sendRequest(worker, 'execute', txid, hex, trustList) } catch (e) { if (worker.missingDeps.size) { if (this.onMissingDeps) await this.onMissingDeps(txid, Array.from(worker.missingDeps)) @@ -99,6 +96,9 @@ class Executor { this.workerRequests.shift()(worker) } } + if (this.onIndexed && result !== null) { + await this.onIndexed(txid, result) + } } _requestWorker () { @@ -109,7 +109,7 @@ class Executor { return worker } - return new Promise((resolve, _reject) => { + return new Promise((resolve) => { this.workerRequests.push(resolve) }) } diff --git a/src/sqlite-datasource.js b/src/sqlite-datasource.js index 137b453..ae257e7 100644 --- a/src/sqlite-datasource.js +++ b/src/sqlite-datasource.js @@ -68,7 +68,6 @@ class SqliteDatasource { this.connection = new Sqlite3Database(this.path, { readonly: this.readonly }) - // 100MB cache this.connection.pragma('cache_size = 6400') this.connection.pragma('page_size = 16384') @@ -680,7 +679,7 @@ class SqliteDatasource { return this.getAllUnspentByLockScripthashStmt.raw(true).all(lockOrigin, scripthash).map(row => row[0]) } - async getAllUnspentByClassOriginAndLockOriginAndScripthash (clsOrigin, lockOrigin, scripthash) { + async getAllUnspentByClassOriginAndLockOriginAndScriptHash (clsOrigin, lockOrigin, scripthash) { return this.getAllUnspentByClassLockScripthashStmt.raw(true).all(clsOrigin, lockOrigin, scripthash).map(row => row[0]) } diff --git a/test/indexer.js b/test/indexer.js index 6c12ab3..686a274 100644 --- a/test/indexer.js +++ b/test/indexer.js @@ -19,8 +19,8 @@ const Database = require('../src/database') const fetch = txid => { return { hex: require('./txns.json')[txid] } } const api = { fetch } -const indexed = (indexer, txid) => new Promise((resolve, reject) => { indexer.onIndex = x => txid === x && resolve() }) -const failed = (indexer, txid) => new Promise((resolve, reject) => { indexer.onFailToIndex = x => txid === x && resolve() }) +const indexed = (indexer, txid) => new Promise((resolve) => { indexer.onIndex = x => txid === x && resolve() }) +const failed = (indexer, txid) => new Promise((resolve) => { indexer.onFailToIndex = x => txid === x && resolve() }) const logger = { info: () => {}, warn: () => {}, error: () => {}, debug: () => {} } const database = new Database(':memory:', logger, false) @@ -58,7 +58,7 @@ describe('Indexer', () => { const pubkey = new bsv.PrivateKey('testnet').toPublicKey().toString() tx.update(() => new A(pubkey)) const rawtx = await tx.export() - const api = { fetch: txid => { return { hex: rawtx } } } + const api = { fetch: _txid => { return { hex: rawtx } } } const indexer = new Indexer(database, api, 'test', 1, 1, logger, 0, Infinity, []) const txid = new bsv.Transaction(rawtx).hash await indexer.start() @@ -73,11 +73,12 @@ describe('Indexer', () => { it('add in reverse and index', async () => { const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, []) await indexer.start() + const promise = indexed(indexer, '9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102') await database.addTransaction('9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102') await database.addTransaction('3f9de452f0c3c96be737d42aa0941b27412211976688967adb3174ee18b04c64') await database.trust('3f9de452f0c3c96be737d42aa0941b27412211976688967adb3174ee18b04c64') await database.trust('9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102') - await indexed(indexer, '9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102') + await promise await indexer.stop() }) @@ -120,7 +121,7 @@ describe('Indexer', () => { const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, []) await indexer.start() await database.addTransaction('9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102') - await new Promise((resolve, reject) => setTimeout(resolve, 1000)) + await new Promise((resolve) => setTimeout(resolve, 1000)) await indexer.stop() }) @@ -144,8 +145,9 @@ describe('Indexer', () => { this.timeout(40000) const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, DEFAULT_TRUSTLIST) await indexer.start() + const promise = indexed(indexer, '11f27cdad53128a4eb14c8328515dfab56b16ea5a71dd26abe9e9d7488f3ab83') await database.addTransaction('11f27cdad53128a4eb14c8328515dfab56b16ea5a71dd26abe9e9d7488f3ab83') - await indexed(indexer, '11f27cdad53128a4eb14c8328515dfab56b16ea5a71dd26abe9e9d7488f3ab83') + await promise expect(await database.getSpend('7fa1b0eb8408047e138aadf72ee0980e42afab2208181429b050ad495a384d39_o1')) .to.equal('11f27cdad53128a4eb14c8328515dfab56b16ea5a71dd26abe9e9d7488f3ab83') expect(await database.getSpend('11f27cdad53128a4eb14c8328515dfab56b16ea5a71dd26abe9e9d7488f3ab83_o1')) @@ -182,10 +184,11 @@ describe('Indexer', () => { const txid1 = new bsv.Transaction(rawtx1).hash const txid2 = new bsv.Transaction(rawtx2).hash await indexer.start() + const promise = indexed(indexer, txid2) await database.addTransaction(txid1, rawtx1) await database.addTransaction(txid2, rawtx2) await database.trust(txid1) - await indexed(indexer, txid2) + await promise expect(await indexer.database.getNumUnspent()).to.equal(0) await indexer.stop() }) From ef56a64a6c3a286eda7b2d23e4109a9031c5ab7b Mon Sep 17 00:00:00 2001 From: hojarasca Date: Tue, 7 Dec 2021 00:51:48 -0300 Subject: [PATCH 11/82] sync working for testnet. --- src/database.js | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/database.js b/src/database.js index 609902c..666b1e9 100644 --- a/src/database.js +++ b/src/database.js @@ -259,12 +259,10 @@ class Database { } async setTransactionExecutionFailed (txid) { - await this.transaction(async () => { - await this.ds.setExecutableForTx(txid, 0) - await this.ds.setExecutedForTx(txid, 1) - await this.ds.setIndexedForTx(txid, 0) - await this.ds.removeTxFromExecuting(txid) - }) + await this.ds.setExecutableForTx(txid, 0) + await this.ds.setExecutedForTx(txid, 1) + await this.ds.setIndexedForTx(txid, 0) + await this.ds.removeTxFromExecuting(txid) // We try executing downstream transactions if this was marked executable but it wasn't. // This allows an admin to manually change executable status in the database. From 3e7a8ac6ccc6b94f001fafb5796fa0ea0e7c110c Mon Sep 17 00:00:00 2001 From: hojarasca Date: Tue, 7 Dec 2021 08:43:39 -0300 Subject: [PATCH 12/82] extracted db trasaction method to data source. --- src/constants.js | 2 +- src/database.bak.js | 591 --------------------------------------- src/database.js | 30 +- src/server.js | 24 +- src/sqlite-datasource.js | 13 + 5 files changed, 36 insertions(+), 624 deletions(-) delete mode 100644 src/database.bak.js diff --git a/src/constants.js b/src/constants.js index 72f81f8..119a91b 100644 --- a/src/constants.js +++ b/src/constants.js @@ -4,4 +4,4 @@ const HEIGHT_UNKNOWN = null module.exports = { HEIGHT_MEMPOOL, HEIGHT_UNKNOWN -} \ No newline at end of file +} diff --git a/src/database.bak.js b/src/database.bak.js deleted file mode 100644 index e085ab7..0000000 --- a/src/database.bak.js +++ /dev/null @@ -1,591 +0,0 @@ -/** - * database.js - * - * Layer between the database and the application - */ - -const Run = require('run-sdk') -const bsv = require('bsv') -const { SqliteDatasource } = require('./sqlite-datasource') -const { HEIGHT_MEMPOOL, HEIGHT_UNKNOWN } = require('./constants') - -// ------------------------------------------------------------------------------------------------ -// Database -// ------------------------------------------------------------------------------------------------ - -class Database { - constructor (path, logger, readonly = false) { - this.db = new SqliteDatasource(path, logger, readonly) - this.logger = logger - - this.onReadyToExecute = null - this.onAddTransaction = null - this.onDeleteTransaction = null - this.onTrustTransaction = null - this.onUntrustTransaction = null - this.onBanTransaction = null - this.onUnbanTransaction = null - this.onUntrustTransaction = null - this.onRequestDownload = null - this.onUnindexTransaction = null - } - - async open () { - await this.db.setUp() - } - - async close () { - await this.db.tearDown() - } - - async transaction (f) { - return this.db.transaction(f) - } - - // -------------------------------------------------------------------------- - // tx - // -------------------------------------------------------------------------- - - async addBlock (txids, txhexs, height, hash, time) { - const indexes = new Array(txids.length).fill(null).map((_, i) => i) - for (const index of indexes) { - const txid = txids[index] - const txHex = txhexs && txhexs[index] - await this.addTransaction(txid, txHex, height, time) - } - await this.setHeight(height) - await this.setHash(hash) - } - - async addTransaction (txid, txhex, height, time) { - await this.transaction(async () => { - await this.addNewTransaction(txid) - if (height) { await this.setTransactionHeight(txid, height) } - if (time) { await this.setTransactionTime(txid, time) } - }) - - const downloaded = await this.isTransactionDownloaded(txid) - if (downloaded) return - - if (txhex) { - await this.parseAndStoreTransaction(txid, txhex) - } else { - if (this.onRequestDownload) { await this.onRequestDownload(txid) } - } - } - - async parseAndStoreTransaction (txid, hex) { - if (await this.isTransactionDownloaded(txid)) return - - let metadata = null - let bsvtx = null - const inputs = [] - const outputs = [] - - if (!hex) { throw new Error('No hex') } - - try { - bsvtx = new bsv.Transaction(hex) - - bsvtx.inputs.forEach(input => { - const location = `${input.prevTxId.toString('hex')}_o${input.outputIndex}` - inputs.push(location) - }) - - bsvtx.outputs.forEach((output, n) => { - if (output.script.isDataOut() || output.script.isSafeDataOut()) return - outputs.push(`${txid}_o${n}`) - }) - - metadata = Run.util.metadata(hex) - } catch (e) { - this.logger.error(`${txid} => ${e.message}`) - await this.storeParsedNonExecutableTransaction(txid, hex, inputs, outputs) - return - } - - const deps = new Set() - - for (let i = 0; i < metadata.in; i++) { - const prevtxid = bsvtx.inputs[i].prevTxId.toString('hex') - deps.add(prevtxid) - } - - for (const ref of metadata.ref) { - if (ref.startsWith('native://')) { - continue - } else if (ref.includes('berry')) { - const reftxid = ref.slice(0, 64) - deps.add(reftxid) - } else { - const reftxid = ref.slice(0, 64) - deps.add(reftxid) - } - } - - const hasCode = metadata.exec.some(cmd => cmd.op === 'DEPLOY' || cmd.op === 'UPGRADE') - - await this.storeParsedExecutableTransaction(txid, hex, hasCode, deps, inputs, outputs) - - for (const deptxid of deps) { - if (!await this.isTransactionDownloaded(deptxid)) { - if (this.onRequestDownload) await this.onRequestDownload(deptxid) - } - } - } - - async addNewTransaction (txid) { - if (await this.hasTransaction(txid)) return - - const time = Math.round(Date.now() / 1000) - - await this.db.addNewTx(txid, time) - - if (this.onAddTransaction) { await this.onAddTransaction(txid) } - } - - async setTransactionHeight (txid, height) { - await this.db.setTxHeight(txid, height) - } - - async setTransactionTime (txid, time) { - await this.db.setTxTime(txid, time) - } - - async storeParsedNonExecutableTransaction (txid, hex, inputs, outputs) { - await this.transaction(async () => { - const bytes = Buffer.from(hex, 'hex') - - await this.db.setTxBytes(txid, bytes) - await this.db.setExecutableForTx(txid, 0) - - for (const location of inputs) { - await this.db.upsertSpend(location, txid) - } - for (const location of outputs) { - await this.db.setAsUnspend(location) - } - }) - - // Non-executable might be berry data. We execute once we receive them. - const downstreamReadyToExecute = this.db.searchDownstreamTxidsReadyToExecute(txid) - for (const downtxid of downstreamReadyToExecute) { - await this.db.markTxAsExecuting(downtxid) - if (this.onReadyToExecute) { await this.onReadyToExecute(downtxid) } - } - } - - async storeParsedExecutableTransaction (txid, hex, hasCode, deps, inputs, outputs) { - await this.transaction(async () => { - const bytes = Buffer.from(hex, 'hex') - await this.db.setTxBytes(txid, bytes) - await this.db.setExecutableForTx(txid, 1) - await this.db.setHasCodeForTx(txid, hasCode ? 1 : 0) - - for (const location of inputs) { - await this.db.upsertSpend(location, txid) - } - - for (const location of outputs) { - await this.db.setAsUnspend(location) - } - - for (const deptxid of deps) { - await this.addNewTransaction(deptxid) - await this.db.addDep(deptxid, txid) - - const failed = await this.db.getFailedTx(deptxid) - if (failed) { - await this.setTransactionExecutionFailed(txid) - return - } - } - }) - - await this._checkExecutability(txid) - } - - async storeExecutedTransaction (txid, result) { - const { cache, classes, locks, scripthashes } = result - - await this.transaction(async () => { - await this.db.setExecutedForTx(txid, 1) - await this.db.setIndexedForTx(txid, 1) - - await this.db.removeTxFromExecuting(txid) - - for (const key of Object.keys(cache)) { - if (key.startsWith('jig://')) { - const location = key.slice('jig://'.length) - await this.db.setJig(location, JSON.stringify(cache[key])) - continue - } - - if (key.startsWith('berry://')) { - const location = key.slice('berry://'.length) - await this.db.setBerry(location, JSON.stringify(cache[key])) - continue - } - } - - for (const [location, cls] of classes) { - await this.db.setJigClass(location, cls) - } - - for (const [location, lock] of locks) { - await this.db.setJigLock(location, lock) - } - - for (const [location, scripthash] of scripthashes) { - await this.db.setJigScriptHash(location, scripthash) - } - }) - - const downstreamReadyToExecute = await this.db.searchDownstreamTxidsReadyToExecute(txid) - for (const downtxid of downstreamReadyToExecute) { - await this.db.markTxAsExecuting(downtxid) - if (this.onReadyToExecute) { await this.onReadyToExecute(downtxid) } - } - } - - async setTransactionExecutionFailed (txid) { - // await this.transaction(async () => { - // }) - await this.db.setExecutableForTx(txid, 0) - await this.db.setExecutedForTx(txid, 1) - await this.db.setIndexedForTx(txid, 0) - await this.db.removeTxFromExecuting(txid) - - // We try executing downstream transactions if this was marked executable but it wasn't. - // This allows an admin to manually change executable status in the database. - - let executable = false - try { - const rawTx = await this.getTransactionHex(txid) - Run.util.metadata(rawTx) - executable = true - } catch (e) { } - - if (!executable) { - const downstream = await this.db.searchDownstreamForTxid(txid) - for (const downtxid of downstream) { - await this._checkExecutability(downtxid) - } - } - } - - async getTransactionHex (txid) { - return this.db.getTxHex(txid) - } - - async getTransactionTime (txid) { - return this.db.getTxTime(txid) - } - - async getTransactionHeight (txid) { - return this.db.getTxHeight(txid) - } - - async deleteTransaction (txid, deleted = new Set()) { - if (deleted.has(txid)) return - - const txids = [txid] - deleted.add(txid) - - await this.transaction(async () => { - while (txids.length) { - const txid = txids.shift() - - if (this.onDeleteTransaction) { await this.onDeleteTransaction(txid) } - - - await this.db.deleteTx(txid) - await this.db.deleteJigStatesForTxid(txid) - await this.db.deleteBerryStatesForTxid(txid) - await this.db.deleteSpendsForTxid(txid) - await this.db.unspendOutput(txid) - await this.db.deleteDepsForTxid(txid) - - const downtxids = this.db.searchDownstreamForTxid(txid) - - for (const downtxid of downtxids) { - if (deleted.has(downtxid)) continue - deleted.add(downtxid) - txids.push(downtxid) - } - } - }) - } - - async unconfirmTransaction (txid) { - await this.db.unconfirmTx(txid) - } - - async unindexTransaction (txid) { - await this.transaction(async () => { - - if (await this.txIsIndexed()) { - await this.db.setExecutedForTx(txid, 0) - await this.db.setIndexedForTx(txid, 0) - await this.db.deleteJigStatesForTxid(txid) - await this.db.deleteBerryStatesForTxid(txid) - await this.db.removeTxFromExecuting(txid) - - const downloadedTxids = await this.db.searchDownstreamForTxid(txid) - for (const downloadedTxid of downloadedTxids) { - await this.unindexTransaction(downloadedTxid) - } - - if (this.onUnindexTransaction) { await this.onUnindexTransaction(txid) } - } - }) - } - - async hasTransaction (txid) { - return this.db.txExists(txid) - } - - async isTransactionDownloaded (txid) { - const result = this.db.checkTxIsDownloaded(txid) - return result ? !!result[0] : false - } - - async getTransactionsAboveHeight (height) { - const txsOverHeight = await this.db.searchTxsAboveHeight(height) - return txsOverHeight.map(row => row[0]) - } - - async getMempoolTransactionsBeforeTime (time) { - const txs = await this.db.mempoolTxsPreviousToTime(time) - return txs.map(row => row[0]) - } - - async getTransactionsToDownload () { - const rows = await this.db.searchTxsToDownload() - return rows.map(row => row[0]) - } - - async getDownloadedCount () { - return this.db.countDownloadedTxs() - } - - async getIndexedCount () { - return this.countIndexedTxs() - } - - // -------------------------------------------------------------------------- - // spends - // -------------------------------------------------------------------------- - - async getSpend (location) { - return this.db.getSpendingTxid(location) - } - - // -------------------------------------------------------------------------- - // deps - // -------------------------------------------------------------------------- - - async addDep (txid, deptxid) { - await this.addNewTransaction(deptxid) - - await this.db.addDep(deptxid, txid) - - const failed = await this.db.getFailedTx(deptxid) - if (failed) { - await this.setTransactionExecutionFailed(deptxid) - } - } - - async addMissingDeps (txid, deptxids) { - await this.transaction(async () => { - for (const deptxid of deptxids) { - await this.addDep(txid, deptxid) - } - }) - - await this._checkExecutability(txid) - } - - // -------------------------------------------------------------------------- - // jig - // -------------------------------------------------------------------------- - - async getJigState (location) { - return this.db.getJigState(location) - } - - // -------------------------------------------------------------------------- - // unspent - // -------------------------------------------------------------------------- - - async getAllUnspent () { - return this.db.getAllUnspent() - } - - async getAllUnspentByClassOrigin (origin) { - return this.db.getAllUnspentByClassOrigin(origin) - } - - async getAllUnspentByLockOrigin (origin) { - return this.db.getAllUnspentByLockOrigin(origin) - } - - async getAllUnspentByScripthash (scripthash) { - return this.db.getAllUnspentByScripthash(scripthash) - } - - async getAllUnspentByClassOriginAndLockOrigin (clsOrigin, lockOrigin) { - return this.db.getAllUnspentByClassOriginAndLockOrigin(clsOrigin, lockOrigin) - } - - async getAllUnspentByClassOriginAndScripthash (clsOrigin, scripthash) { - return this.db.getAllUnspentByClassOriginAndScripthash(clsOrigin, scripthash) - } - - async getAllUnspentByLockOriginAndScripthash (lockOrigin, scripthash) { - return this.db.getAllUnspentByLockOriginAndScripthash(lockOrigin, scripthash) - } - - async getAllUnspentByClassOriginAndLockOriginAndScripthash (clsOrigin, lockOrigin, scripthash) { - return this.db.getAllUnspentByClassOriginAndLockOriginAndScripthash(clsOrigin, lockOrigin, scripthash) - } - - async getNumUnspent () { - return this.db.countTotalUnspent() - } - - // -------------------------------------------------------------------------- - // berry - // -------------------------------------------------------------------------- - - async getBerryState (location) { - return this.db.getBerry(location) - } - - // -------------------------------------------------------------------------- - // trust - // -------------------------------------------------------------------------- - - async isTrusted (txid) { - return this.db.isTrusted(txid) - } - - async trust (txid) { - if (await this.isTrusted(txid)) return - - const trusted = [txid] - - // Recursively trust code parents - const queue = await this.db.getNonExecutedUpstreamTxIds(txid) - const visited = new Set() - while (queue.length) { - const uptxid = queue.shift() - if (visited.has(uptxid)) continue - if (await this.isTrusted(uptxid)) continue - visited.add(uptxid) - trusted.push(txid) - const txids = await this.db.getNonExecutedUpstreamTxIds(uptxid) - txids.forEach(x => queue.push(x[0])) - } - - for (const trustedTxid of trusted) { - await this.db.setTrust(trustedTxid, 1) - } - - for (const txid of trusted) { - await this._checkExecutability(txid) - } - - if (this.onTrustTransaction) { - for (const txid of trusted) { - await this.onTrustTransaction(txid) - } - } - } - - async untrust (txid) { - if (!await this.isTrusted(txid)) return - await this.transaction(async () => { - await this.unindexTransaction(txid) - await this.db.setTrust(txid, 0) - - }) - if (this.onUntrustTransaction) await this.onUntrustTransaction(txid) - } - - async getTrustlist () { - return this.db.searchAllTrust() - } - - // -------------------------------------------------------------------------- - // ban - // -------------------------------------------------------------------------- - - async isBanned (txid) { - return this.db.checkIsBanned(txid) - } - - async ban (txid) { - await this.transaction(async () => { - await this.unindexTransaction(txid) - await this.db.saveBan(txid) - }) - if (this.onBanTransaction) await this.onBanTransaction(txid) - } - - async unban (txid) { - await this.db.removeBan(txid) - await this._checkExecutability(txid) - if (this.onUnbanTransaction) await this.onUnbanTransaction(txid) - } - - async getBanlist () { - return this.db.searchAllBans() - } - - // -------------------------------------------------------------------------- - // crawl - // -------------------------------------------------------------------------- - - async getHeight () { - return this.db.getCrawlHeight() - } - - async getHash () { - return this.db.getCrawlHash() - } - - async setHeight (height) { - await this.db.setCrawlHeight(height) - } - - async setHash (hash) { - await this.db.setCrawlHash(hash) - } - - // -------------------------------------------------------------------------- - // internal - // -------------------------------------------------------------------------- - - async loadTransactionsToExecute () { - this.logger.debug('Loading transactions to execute') - const txids = await this.db.findAllExecutingTxids() - for (const txid of txids) { - await this._checkExecutability(txid) - } - } - - async _checkExecutability (txid) { - const row = await this.db.txidReadyToExecute(txid) - if (row && row.ready) { - await this.db.markTxAsExecuting(txid) - if (this.onReadyToExecute) { await this.onReadyToExecute(txid) } - } - } -} - -// ------------------------------------------------------------------------------------------------ - -Database.HEIGHT_MEMPOOL = HEIGHT_MEMPOOL - -module.exports = Database diff --git a/src/database.js b/src/database.js index 666b1e9..6461ed0 100644 --- a/src/database.js +++ b/src/database.js @@ -44,17 +44,7 @@ class Database { } async transaction (f) { - if (!this.db) return - try { - this.db.exec('begin;') - await f() - this.db.exec('commit;') - } catch (e) { - this.db.exec('rollback;') - console.error(e) - throw e - } - // this.db.transaction(f)() + await this.ds.performOnTransaction(f) } // -------------------------------------------------------------------------- @@ -73,7 +63,7 @@ class Database { } async addTransaction (txid, txhex, height, time) { - await this.transaction(async () => { + await this.ds.performOnTransaction(async () => { await this.addNewTransaction(txid) if (height) { await this.setTransactionHeight(txid, height) } if (time) { await this.setTransactionTime(txid, time) } @@ -169,7 +159,7 @@ class Database { } async storeParsedNonExecutableTransaction (txid, hex, inputs, outputs) { - await this.transaction(async () => { + await this.ds.performOnTransaction(async () => { const bytes = Buffer.from(hex, 'hex') await this.ds.setTxBytes(txid, bytes) await this.ds.setExecutableForTx(txid, 0) @@ -191,7 +181,7 @@ class Database { } async storeParsedExecutableTransaction (txid, hex, hasCode, deps, inputs, outputs) { - await this.transaction(async () => { + await this.ds.performOnTransaction(async () => { const bytes = Buffer.from(hex, 'hex') await this.ds.setTxBytes(txid, bytes) await this.ds.setExecutableForTx(txid, 1) @@ -223,7 +213,7 @@ class Database { async storeExecutedTransaction (txid, result) { const { cache, classes, locks, scripthashes } = result - await this.transaction(async () => { + await this.ds.performOnTransaction(async () => { await this.ds.setExecutedForTx(txid, 1) await this.ds.setIndexedForTx(txid, 1) await this.ds.removeTxFromExecuting(txid) @@ -300,7 +290,7 @@ class Database { const txids = [txid] deleted.add(txid) - await this.transaction(async () => { + await this.ds.performOnTransaction(async () => { while (txids.length) { const txid = txids.shift() @@ -329,7 +319,7 @@ class Database { } async unindexTransaction (txid) { - await this.transaction(async () => { + await this.ds.performOnTransaction(async () => { const indexed = await this.ds.txIsIndexed(txid) if (indexed) { await this.ds.setExecutedForTx(txid, 0) @@ -400,7 +390,7 @@ class Database { } async addMissingDeps (txid, deptxids) { - await this.transaction(async () => { + await this.ds.performOnTransaction(async () => { for (const deptxid of deptxids) { await this.addDep(txid, deptxid) } @@ -508,7 +498,7 @@ class Database { async untrust (txid) { if (!await this.isTrusted(txid)) return - await this.transaction(async () => { + await this.ds.performOnTransaction(async () => { await this.unindexTransaction(txid) await this.ds.setTrust(txid, 0) }) @@ -528,7 +518,7 @@ class Database { } async ban (txid) { - await this.transaction(async () => { + await this.ds.performOnTransaction(async () => { await this.unindexTransaction(txid) await this.ds.saveBan(txid) }) diff --git a/src/server.js b/src/server.js index c1aa3c0..08b17ba 100644 --- a/src/server.js +++ b/src/server.js @@ -37,18 +37,18 @@ class Server { const app = express() - // let buffer = '' - // const write = (chunk, encoding, callback) => { - // buffer = buffer + chunk.toString() - // const lines = buffer.split(/\r\n|\n\r|\n|\r/) - // for (let i = 0; i < lines.length - 1; i++) { - // this.logger.info(lines[i]) - // } - // buffer = lines[lines.length - 1] - // callback() - // return true - // } - // app.use(morgan('tiny', { stream: new Writable({ write }) })) + let buffer = '' + const write = (chunk, encoding, callback) => { + buffer = buffer + chunk.toString() + const lines = buffer.split(/\r\n|\n\r|\n|\r/) + for (let i = 0; i < lines.length - 1; i++) { + this.logger.info(lines[i]) + } + buffer = lines[lines.length - 1] + callback() + return true + } + app.use(morgan('tiny', { stream: new Writable({ write }) })) app.use(bodyParser.text({ limit: '25mb' })) app.use(bodyParser.json({ limit: '10mb' })) diff --git a/src/sqlite-datasource.js b/src/sqlite-datasource.js index ae257e7..bda9ea3 100644 --- a/src/sqlite-datasource.js +++ b/src/sqlite-datasource.js @@ -440,6 +440,19 @@ class SqliteDatasource { } } + async performOnTransaction (fn) { + if (!this.connection) return + try { + this.connection.exec('begin;') + await fn() + this.connection.exec('commit;') + } catch (e) { + this.connection.exec('rollback;') + console.error(e) + throw e + } + } + async transaction (f) { if (!this.connection) return try { From ef62a697ce34eb41fc2f7309538d92a014014b82 Mon Sep 17 00:00:00 2001 From: hojarasca Date: Tue, 7 Dec 2021 09:14:21 -0300 Subject: [PATCH 13/82] removed non used data from database. --- src/database.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/database.js b/src/database.js index 6461ed0..16e522e 100644 --- a/src/database.js +++ b/src/database.js @@ -16,9 +16,7 @@ const { HEIGHT_MEMPOOL, HEIGHT_UNKNOWN } = require('./constants') class Database { constructor (path, logger, readonly = false) { this.ds = new SqliteDatasource(path, logger, readonly) - this.path = path this.logger = logger - this.readonly = readonly this.onReadyToExecute = null this.onAddTransaction = null From 9af198d5614ba1b3720fa313371df7d0d6d04b49 Mon Sep 17 00:00:00 2001 From: hojarasca Date: Tue, 7 Dec 2021 10:11:48 -0300 Subject: [PATCH 14/82] started adding knex --- package-lock.json | 249 ++++++- package.json | 1 + src/data-sources/knex-datasource.js | 744 ++++++++++++++++++++ src/{ => data-sources}/sqlite-datasource.js | 17 +- src/database.js | 2 +- 5 files changed, 979 insertions(+), 34 deletions(-) create mode 100644 src/data-sources/knex-datasource.js rename src/{ => data-sources}/sqlite-datasource.js (98%) diff --git a/package-lock.json b/package-lock.json index 2429375..06ccd6b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,6 +17,7 @@ "event-stream": "^4.0.1", "eventsource": "^1.1.0", "express": "^4.17.1", + "knex": "^0.95.14", "morgan": "^1.10.0", "node-fetch": "^2.6.1", "reconnecting-eventsource": "^1.1.0", @@ -1344,6 +1345,14 @@ "integrity": "sha512-ecORCqbSFP7Wm8Y6lyqMJjexBQqXSF7SSeaTyGGphogUjBlFP9m9o08wy86HL2uB7fMTxtOUzLMk7ogKcxMg1w==", "dev": true }, + "node_modules/commander": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz", + "integrity": "sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==", + "engines": { + "node": ">= 10" + } + }, "node_modules/commondir": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/commondir/-/commondir-1.0.1.tgz", @@ -1703,7 +1712,6 @@ "version": "3.1.1", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==", - "dev": true, "engines": { "node": ">=6" } @@ -2185,6 +2193,14 @@ "node": ">=8" } }, + "node_modules/esm": { + "version": "3.2.25", + "resolved": "https://registry.npmjs.org/esm/-/esm-3.2.25.tgz", + "integrity": "sha512-U1suiZ2oDVWv4zPO56S0NcR5QriEahGtdN2OR6FiOG4WJvcjBVFB0qI4+eKoWFH483PKGuLuu6V8Z4T5g63UVA==", + "engines": { + "node": ">=6" + } + }, "node_modules/espree": { "version": "7.3.1", "resolved": "https://registry.npmjs.org/espree/-/espree-7.3.1.tgz", @@ -2588,8 +2604,7 @@ "node_modules/function-bind": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", - "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", - "dev": true + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" }, "node_modules/functional-red-black-tree": { "version": "1.0.1", @@ -2668,6 +2683,11 @@ "node": ">=10" } }, + "node_modules/getopts": { + "version": "2.2.5", + "resolved": "https://registry.npmjs.org/getopts/-/getopts-2.2.5.tgz", + "integrity": "sha512-9jb7AW5p3in+IiJWhQiZmmwkpLaR/ccTWdWQCtZM66HJcHHLegowh4q4tSD7gouUyeNvFWRavfK9GXosQHDpFA==" + }, "node_modules/github-from-package": { "version": "0.0.0", "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", @@ -2730,7 +2750,6 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", - "dev": true, "dependencies": { "function-bind": "^1.1.1" }, @@ -2951,6 +2970,14 @@ "node": ">= 0.4" } }, + "node_modules/interpret": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/interpret/-/interpret-2.2.0.tgz", + "integrity": "sha512-Ju0Bz/cEia55xDwUWEa8+olFpCiQoypjnQySseKtmjNrnps3P+xfpUmGr90T7yjlVJmOtybRvPXhKMbHr+fWnw==", + "engines": { + "node": ">= 0.10" + } + }, "node_modules/ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -3012,7 +3039,6 @@ "version": "2.6.0", "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.6.0.tgz", "integrity": "sha512-wShG8vs60jKfPWpF2KZRaAtvt3a20OAn7+IJ6hLPECpSABLcKtFKTTI4ZtH5QcBruBHlq+WsdHWyz0BCZW7svQ==", - "dev": true, "dependencies": { "has": "^1.0.3" } @@ -3418,6 +3444,78 @@ "node": ">=0.10.0" } }, + "node_modules/knex": { + "version": "0.95.14", + "resolved": "https://registry.npmjs.org/knex/-/knex-0.95.14.tgz", + "integrity": "sha512-j4qLjWySrC/JRRVtOpoR2LcS1yBOsd7Krc6mEukPvmTDX/w11pD52Pq9FYR56/kLXGeAV8jFdWBjsZFi1mscWg==", + "dependencies": { + "colorette": "2.0.16", + "commander": "^7.1.0", + "debug": "4.3.2", + "escalade": "^3.1.1", + "esm": "^3.2.25", + "getopts": "2.2.5", + "interpret": "^2.2.0", + "lodash": "^4.17.21", + "pg-connection-string": "2.5.0", + "rechoir": "0.7.0", + "resolve-from": "^5.0.0", + "tarn": "^3.0.1", + "tildify": "2.0.0" + }, + "bin": { + "knex": "bin/cli.js" + }, + "engines": { + "node": ">=10" + }, + "peerDependenciesMeta": { + "mysql": { + "optional": true + }, + "mysql2": { + "optional": true + }, + "pg": { + "optional": true + }, + "pg-native": { + "optional": true + }, + "sqlite3": { + "optional": true + }, + "tedious": { + "optional": true + } + } + }, + "node_modules/knex/node_modules/colorette": { + "version": "2.0.16", + "resolved": "https://registry.npmjs.org/colorette/-/colorette-2.0.16.tgz", + "integrity": "sha512-hUewv7oMjCp+wkBv5Rm0v87eJhq4woh5rSR+42YSQJKecCqgIqNkZ6lAlQms/BwHPJA5NKMRlpxPRv0n8HQW6g==" + }, + "node_modules/knex/node_modules/debug": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz", + "integrity": "sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw==", + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/knex/node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, "node_modules/levn": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", @@ -3470,8 +3568,7 @@ "node_modules/lodash": { "version": "4.17.21", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", - "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", - "dev": true + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, "node_modules/lodash.flattendeep": { "version": "4.4.0", @@ -4362,8 +4459,7 @@ "node_modules/path-parse": { "version": "1.0.7", "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", - "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", - "dev": true + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==" }, "node_modules/path-to-regexp": { "version": "0.1.7", @@ -4399,6 +4495,11 @@ "through": "~2.3" } }, + "node_modules/pg-connection-string": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.5.0.tgz", + "integrity": "sha512-r5o/V/ORTA6TmUnyWZR9nCj1klXCO2CEKNRlVuJptZe85QuhFayC7WeMic7ndayT5IRIR0S0xFxFi2ousartlQ==" + }, "node_modules/picomatch": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.0.tgz", @@ -4891,6 +4992,17 @@ "node": ">=8.10.0" } }, + "node_modules/rechoir": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.7.0.tgz", + "integrity": "sha512-ADsDEH2bvbjltXEP+hTIAmeFekTFK0V2BTxMkok6qILyAJEXV0AFfoWcAq4yfll5VdIMd/RVXq0lR+wQi5ZU3Q==", + "dependencies": { + "resolve": "^1.9.0" + }, + "engines": { + "node": ">= 0.10" + } + }, "node_modules/reconnecting-eventsource": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/reconnecting-eventsource/-/reconnecting-eventsource-1.1.0.tgz", @@ -4954,7 +5066,6 @@ "version": "1.20.0", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.20.0.tgz", "integrity": "sha512-wENBPt4ySzg4ybFQW2TT1zMQucPK95HSh/nq2CFTZVOGut2+pQvSsgtda4d26YrYcr067wjbmzOG8byDPBX63A==", - "dev": true, "dependencies": { "is-core-module": "^2.2.0", "path-parse": "^1.0.6" @@ -4964,7 +5075,6 @@ "version": "5.0.0", "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz", "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", - "dev": true, "engines": { "node": ">=8" } @@ -5535,6 +5645,14 @@ "node": ">=10" } }, + "node_modules/tarn": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/tarn/-/tarn-3.0.2.tgz", + "integrity": "sha512-51LAVKUSZSVfI05vjPESNc5vwqqZpbXCsU+/+wxlOrUjk2SnFTt97v9ZgQrD4YmxYW1Px6w2KjaDitCfkvgxMQ==", + "engines": { + "node": ">=8.0.0" + } + }, "node_modules/test-exclude": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-6.0.0.tgz", @@ -5560,6 +5678,14 @@ "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=" }, + "node_modules/tildify": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/tildify/-/tildify-2.0.0.tgz", + "integrity": "sha512-Cc+OraorugtXNfs50hU9KS369rFXCfgGLpfCfvlc+Ud5u6VWmUQsOAa9HbTvheQdYnrdJqqv1e5oIqXppMYnSw==", + "engines": { + "node": ">=8" + } + }, "node_modules/to-fast-properties": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz", @@ -7139,6 +7265,11 @@ "integrity": "sha512-ecORCqbSFP7Wm8Y6lyqMJjexBQqXSF7SSeaTyGGphogUjBlFP9m9o08wy86HL2uB7fMTxtOUzLMk7ogKcxMg1w==", "dev": true }, + "commander": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz", + "integrity": "sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==" + }, "commondir": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/commondir/-/commondir-1.0.1.tgz", @@ -7431,8 +7562,7 @@ "escalade": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", - "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==", - "dev": true + "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==" }, "escape-html": { "version": "1.0.3", @@ -7829,6 +7959,11 @@ "integrity": "sha512-0rSmRBzXgDzIsD6mGdJgevzgezI534Cer5L/vyMX0kHzT/jiB43jRhd9YUlMGYLQy2zprNmoT8qasCGtY+QaKw==", "dev": true }, + "esm": { + "version": "3.2.25", + "resolved": "https://registry.npmjs.org/esm/-/esm-3.2.25.tgz", + "integrity": "sha512-U1suiZ2oDVWv4zPO56S0NcR5QriEahGtdN2OR6FiOG4WJvcjBVFB0qI4+eKoWFH483PKGuLuu6V8Z4T5g63UVA==" + }, "espree": { "version": "7.3.1", "resolved": "https://registry.npmjs.org/espree/-/espree-7.3.1.tgz", @@ -8149,8 +8284,7 @@ "function-bind": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", - "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", - "dev": true + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" }, "functional-red-black-tree": { "version": "1.0.1", @@ -8214,6 +8348,11 @@ "integrity": "sha512-sY22aA6xchAzprjyqmSEQv4UbAAzRN0L2dQB0NlN5acTTK9Don6nhoc3eAbUnpZiCANAMfd/+40kVdKfFygohg==", "dev": true }, + "getopts": { + "version": "2.2.5", + "resolved": "https://registry.npmjs.org/getopts/-/getopts-2.2.5.tgz", + "integrity": "sha512-9jb7AW5p3in+IiJWhQiZmmwkpLaR/ccTWdWQCtZM66HJcHHLegowh4q4tSD7gouUyeNvFWRavfK9GXosQHDpFA==" + }, "github-from-package": { "version": "0.0.0", "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", @@ -8264,7 +8403,6 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", - "dev": true, "requires": { "function-bind": "^1.1.1" } @@ -8447,6 +8585,11 @@ "side-channel": "^1.0.4" } }, + "interpret": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/interpret/-/interpret-2.2.0.tgz", + "integrity": "sha512-Ju0Bz/cEia55xDwUWEa8+olFpCiQoypjnQySseKtmjNrnps3P+xfpUmGr90T7yjlVJmOtybRvPXhKMbHr+fWnw==" + }, "ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -8496,7 +8639,6 @@ "version": "2.6.0", "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.6.0.tgz", "integrity": "sha512-wShG8vs60jKfPWpF2KZRaAtvt3a20OAn7+IJ6hLPECpSABLcKtFKTTI4ZtH5QcBruBHlq+WsdHWyz0BCZW7svQ==", - "dev": true, "requires": { "has": "^1.0.3" } @@ -8809,6 +8951,46 @@ "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==" }, + "knex": { + "version": "0.95.14", + "resolved": "https://registry.npmjs.org/knex/-/knex-0.95.14.tgz", + "integrity": "sha512-j4qLjWySrC/JRRVtOpoR2LcS1yBOsd7Krc6mEukPvmTDX/w11pD52Pq9FYR56/kLXGeAV8jFdWBjsZFi1mscWg==", + "requires": { + "colorette": "2.0.16", + "commander": "^7.1.0", + "debug": "4.3.2", + "escalade": "^3.1.1", + "esm": "^3.2.25", + "getopts": "2.2.5", + "interpret": "^2.2.0", + "lodash": "^4.17.21", + "pg-connection-string": "2.5.0", + "rechoir": "0.7.0", + "resolve-from": "^5.0.0", + "tarn": "^3.0.1", + "tildify": "2.0.0" + }, + "dependencies": { + "colorette": { + "version": "2.0.16", + "resolved": "https://registry.npmjs.org/colorette/-/colorette-2.0.16.tgz", + "integrity": "sha512-hUewv7oMjCp+wkBv5Rm0v87eJhq4woh5rSR+42YSQJKecCqgIqNkZ6lAlQms/BwHPJA5NKMRlpxPRv0n8HQW6g==" + }, + "debug": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz", + "integrity": "sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw==", + "requires": { + "ms": "2.1.2" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + } + } + }, "levn": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", @@ -8851,8 +9033,7 @@ "lodash": { "version": "4.17.21", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", - "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", - "dev": true + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, "lodash.flattendeep": { "version": "4.4.0", @@ -9552,8 +9733,7 @@ "path-parse": { "version": "1.0.7", "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", - "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", - "dev": true + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==" }, "path-to-regexp": { "version": "0.1.7", @@ -9583,6 +9763,11 @@ "through": "~2.3" } }, + "pg-connection-string": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.5.0.tgz", + "integrity": "sha512-r5o/V/ORTA6TmUnyWZR9nCj1klXCO2CEKNRlVuJptZe85QuhFayC7WeMic7ndayT5IRIR0S0xFxFi2ousartlQ==" + }, "picomatch": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.0.tgz", @@ -9967,6 +10152,14 @@ "picomatch": "^2.2.1" } }, + "rechoir": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.7.0.tgz", + "integrity": "sha512-ADsDEH2bvbjltXEP+hTIAmeFekTFK0V2BTxMkok6qILyAJEXV0AFfoWcAq4yfll5VdIMd/RVXq0lR+wQi5ZU3Q==", + "requires": { + "resolve": "^1.9.0" + } + }, "reconnecting-eventsource": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/reconnecting-eventsource/-/reconnecting-eventsource-1.1.0.tgz", @@ -10018,7 +10211,6 @@ "version": "1.20.0", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.20.0.tgz", "integrity": "sha512-wENBPt4ySzg4ybFQW2TT1zMQucPK95HSh/nq2CFTZVOGut2+pQvSsgtda4d26YrYcr067wjbmzOG8byDPBX63A==", - "dev": true, "requires": { "is-core-module": "^2.2.0", "path-parse": "^1.0.6" @@ -10027,8 +10219,7 @@ "resolve-from": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz", - "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", - "dev": true + "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==" }, "rimraf": { "version": "3.0.2", @@ -10513,6 +10704,11 @@ } } }, + "tarn": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/tarn/-/tarn-3.0.2.tgz", + "integrity": "sha512-51LAVKUSZSVfI05vjPESNc5vwqqZpbXCsU+/+wxlOrUjk2SnFTt97v9ZgQrD4YmxYW1Px6w2KjaDitCfkvgxMQ==" + }, "test-exclude": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-6.0.0.tgz", @@ -10535,6 +10731,11 @@ "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=" }, + "tildify": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/tildify/-/tildify-2.0.0.tgz", + "integrity": "sha512-Cc+OraorugtXNfs50hU9KS369rFXCfgGLpfCfvlc+Ud5u6VWmUQsOAa9HbTvheQdYnrdJqqv1e5oIqXppMYnSw==" + }, "to-fast-properties": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz", diff --git a/package.json b/package.json index 724d46e..688371f 100644 --- a/package.json +++ b/package.json @@ -39,6 +39,7 @@ "event-stream": "^4.0.1", "eventsource": "^1.1.0", "express": "^4.17.1", + "knex": "^0.95.14", "morgan": "^1.10.0", "node-fetch": "^2.6.1", "reconnecting-eventsource": "^1.1.0", diff --git a/src/data-sources/knex-datasource.js b/src/data-sources/knex-datasource.js new file mode 100644 index 0000000..597179d --- /dev/null +++ b/src/data-sources/knex-datasource.js @@ -0,0 +1,744 @@ +/** + * database.js + * + * Layer between the database and the application + */ +const knex = require('knex') +const { HEIGHT_MEMPOOL } = require('../constants') + +// The + in the following 2 queries before downloaded improves performance by NOT using the +// tx_downloaded index, which is rarely an improvement over a simple filter for single txns. +// See: https://www.sqlite.org/optoverview.html +const IS_READY_TO_EXECUTE_SQL = ` + SELECT ( + downloaded = 1 + AND executable = 1 + AND executed = 0 + AND (has_code = 0 OR (SELECT COUNT(*) FROM trust WHERE trust.txid = tx.txid AND trust.value = 1) = 1) + AND txid NOT IN ban + AND ( + SELECT COUNT(*) + FROM tx AS tx2 + JOIN deps + ON deps.up = tx2.txid + WHERE deps.down = tx.txid + AND (+tx2.downloaded = 0 OR (tx2.executable = 1 AND tx2.executed = 0)) + ) = 0 + ) AS ready + FROM tx + WHERE txid = ? + ` + +const GET_DOWNSTREAM_READY_TO_EXECUTE_SQL = ` + SELECT down + FROM deps + JOIN tx + ON tx.txid = deps.down + WHERE up = ? + AND +downloaded = 1 + AND executable = 1 + AND executed = 0 + AND (has_code = 0 OR (SELECT COUNT(*) FROM trust WHERE trust.txid = tx.txid AND trust.value = 1) = 1) + AND txid NOT IN ban + AND ( + SELECT COUNT(*) + FROM tx AS tx2 + JOIN deps + ON deps.up = tx2.txid + WHERE deps.down = tx.txid + AND (+tx2.downloaded = 0 OR (tx2.executable = 1 AND tx2.executed = 0)) + ) = 0 + ` + +// ------------------------------------------------------------------------------------------------ +// Database +// ------------------------------------------------------------------------------------------------ + +class KnexDataSource { + constructor (dbUri, logger, readonly = false) { + this.dbUri = dbUri + this.logger = logger + this.readonly = readonly + this.connection = null + } + + async setUp () { + this.logger.debug('Opening' + (this.readonly ? ' readonly' : '') + ' database') + if (this.connection) throw new Error('Database already open') + + this.connection = knex({ + client: 'pg', + connection: this.dbUri + }) + + // Initialise and perform upgrades + await this.initializeV1() + await this.initializeV2() + await this.initializeV3() + await this.initializeV4() + await this.initializeV5() + await this.initializeV6() + await this.initializeV7() + + this.addNewTransactionStmt = this.connection.prepare('INSERT OR IGNORE INTO tx (txid, height, time, bytes, has_code, executable, executed, indexed) VALUES (?, null, ?, null, 0, 0, 0, 0)') + this.setTransactionBytesStmt = this.connection.prepare('UPDATE tx SET bytes = ? WHERE txid = ?') + this.setTransactionExecutableStmt = this.connection.prepare('UPDATE tx SET executable = ? WHERE txid = ?') + this.setTransactionTimeStmt = this.connection.prepare('UPDATE tx SET time = ? WHERE txid = ?') + this.setTransactionHeightStmt = this.connection.prepare(`UPDATE tx SET height = ? WHERE txid = ? AND (height IS NULL OR height = ${HEIGHT_MEMPOOL})`) + this.setTransactionHasCodeStmt = this.connection.prepare('UPDATE tx SET has_code = ? WHERE txid = ?') + this.setTransactionExecutedStmt = this.connection.prepare('UPDATE tx SET executed = ? WHERE txid = ?') + this.setTransactionIndexedStmt = this.connection.prepare('UPDATE tx SET indexed = ? WHERE txid = ?') + this.txExistsStmt = this.connection.prepare('SELECT txid FROM tx WHERE txid = ?') + this.getTransactionHexStmt = this.connection.prepare('SELECT LOWER(HEX(bytes)) AS hex FROM tx WHERE txid = ?') + this.getTransactionTimeStmt = this.connection.prepare('SELECT time FROM tx WHERE txid = ?') + this.getTransactionHeightStmt = this.connection.prepare('SELECT height FROM tx WHERE txid = ?') + this.getTransactionHasCodeStmt = this.connection.prepare('SELECT has_code FROM tx WHERE txid = ?') + this.getTransactionIndexedStmt = this.connection.prepare('SELECT indexed FROM tx WHERE txid = ?') + this.getTransactionFailedStmt = this.connection.prepare('SELECT (executed = 1 AND indexed = 0) AS failed FROM tx WHERE txid = ?') + this.getTransactionDownloadedStmt = this.connection.prepare('SELECT downloaded FROM tx WHERE txid = ?') + this.deleteTransactionStmt = this.connection.prepare('DELETE FROM tx WHERE txid = ?') + this.unconfirmTransactionStmt = this.connection.prepare(`UPDATE tx SET height = ${HEIGHT_MEMPOOL} WHERE txid = ?`) + this.getTransactionsAboveHeightStmt = this.connection.prepare('SELECT txid FROM tx WHERE height > ?') + this.getMempoolTransactionsBeforeTimeStmt = this.connection.prepare(`SELECT txid FROM tx WHERE height = ${HEIGHT_MEMPOOL} AND time < ?`) + this.getTransactionsToDownloadStmt = this.connection.prepare('SELECT txid FROM tx WHERE downloaded = 0') + this.getTransactionsDownloadedCountStmt = this.connection.prepare('SELECT COUNT(*) AS count FROM tx WHERE downloaded = 1') + this.getTransactionsIndexedCountStmt = this.connection.prepare('SELECT COUNT(*) AS count FROM tx WHERE indexed = 1') + this.isReadyToExecuteStmt = this.connection.prepare(IS_READY_TO_EXECUTE_SQL) + this.getDownstreamReadyToExecuteStmt = this.connection.prepare(GET_DOWNSTREAM_READY_TO_EXECUTE_SQL) + + this.setSpendStmt = this.connection.prepare('INSERT OR REPLACE INTO spends (location, spend_txid) VALUES (?, ?)') + this.setUnspentStmt = this.connection.prepare('INSERT OR IGNORE INTO spends (location, spend_txid) VALUES (?, null)') + this.getSpendStmt = this.connection.prepare('SELECT spend_txid FROM spends WHERE location = ?') + this.unspendOutputsStmt = this.connection.prepare('UPDATE spends SET spend_txid = null WHERE spend_txid = ?') + this.deleteSpendsStmt = this.connection.prepare('DELETE FROM spends WHERE location LIKE ? || \'%\'') + // + this.addDepStmt = this.connection.prepare('INSERT OR IGNORE INTO deps (up, down) VALUES (?, ?)') + this.deleteDepsStmt = this.connection.prepare('DELETE FROM deps WHERE down = ?') + this.getDownstreamStmt = this.connection.prepare('SELECT down FROM deps WHERE up = ?') + this.getUpstreamUnexecutedCodeStmt = this.connection.prepare(` + SELECT txdeps.txid as txid + FROM (SELECT up AS txid FROM deps WHERE down = ?) as txdeps + JOIN tx ON tx.txid = txdeps.txid + WHERE tx.executable = 1 AND tx.executed = 0 AND tx.has_code = 1 + `) + // + this.setJigStateStmt = this.connection.prepare('INSERT OR IGNORE INTO jig (location, state, class, lock, scripthash) VALUES (?, ?, null, null, null)') + this.setJigClassStmt = this.connection.prepare('UPDATE jig SET class = ? WHERE location = ?') + this.setJigLockStmt = this.connection.prepare('UPDATE jig SET lock = ? WHERE location = ?') + this.setJigScripthashStmt = this.connection.prepare('UPDATE jig SET scripthash = ? WHERE location = ?') + this.getJigStateStmt = this.connection.prepare('SELECT state FROM jig WHERE location = ?') + this.deleteJigStatesStmt = this.connection.prepare('DELETE FROM jig WHERE location LIKE ? || \'%\'') + + const getAllUnspentSql = ` + SELECT spends.location AS location FROM spends + JOIN jig ON spends.location = jig.location + WHERE spends.spend_txid IS NULL` + this.getAllUnspentStmt = this.connection.prepare(getAllUnspentSql) + this.getAllUnspentByClassStmt = this.connection.prepare(`${getAllUnspentSql} AND jig.class = ?`) + this.getAllUnspentByLockStmt = this.connection.prepare(`${getAllUnspentSql} AND jig.lock = ?`) + this.getAllUnspentByScripthashStmt = this.connection.prepare(`${getAllUnspentSql} AND jig.scripthash = ?`) + this.getAllUnspentByClassLockStmt = this.connection.prepare(`${getAllUnspentSql} AND jig.class = ? AND lock = ?`) + this.getAllUnspentByClassScripthashStmt = this.connection.prepare(`${getAllUnspentSql} AND jig.class = ? AND scripthash = ?`) + this.getAllUnspentByLockScripthashStmt = this.connection.prepare(`${getAllUnspentSql} AND jig.lock = ? AND scripthash = ?`) + this.getAllUnspentByClassLockScripthashStmt = this.connection.prepare(`${getAllUnspentSql} AND jig.class = ? AND jig.lock = ? AND scripthash = ?`) + this.getNumUnspentStmt = this.connection.prepare('SELECT COUNT(*) as unspent FROM spends JOIN jig ON spends.location = jig.location WHERE spends.spend_txid IS NULL') + + this.setBerryStateStmt = this.connection.prepare('INSERT OR IGNORE INTO berry (location, state) VALUES (?, ?)') + this.getBerryStateStmt = this.connection.prepare('SELECT state FROM berry WHERE location = ?') + this.deleteBerryStatesStmt = this.connection.prepare('DELETE FROM berry WHERE location LIKE ? || \'%\'') + + this.setTrustedStmt = this.connection.prepare('INSERT OR REPLACE INTO trust (txid, value) VALUES (?, ?)') + this.getTrustlistStmt = this.connection.prepare('SELECT txid FROM trust WHERE value = 1') + this.isTrustedStmt = this.connection.prepare('SELECT COUNT(*) FROM trust WHERE txid = ? AND value = 1') + + this.banStmt = this.connection.prepare('INSERT OR REPLACE INTO ban (txid) VALUES (?)') + this.unbanStmt = this.connection.prepare('DELETE FROM ban WHERE txid = ?') + this.isBannedStmt = this.connection.prepare('SELECT COUNT(*) FROM ban WHERE txid = ?') + this.getBanlistStmt = this.connection.prepare('SELECT txid FROM ban') + + this.getHeightStmt = this.connection.prepare('SELECT value FROM crawl WHERE key = \'height\'') + this.getHashStmt = this.connection.prepare('SELECT value FROM crawl WHERE key = \'hash\'') + this.setHeightStmt = this.connection.prepare('UPDATE crawl SET value = ? WHERE key = \'height\'') + this.setHashStmt = this.connection.prepare('UPDATE crawl SET value = ? WHERE key = \'hash\'') + + this.markExecutingStmt = this.connection.prepare('INSERT OR IGNORE INTO executing (txid) VALUES (?)') + this.unmarkExecutingStmt = this.connection.prepare('DELETE FROM executing WHERE txid = ?') + this.findAllExecutingTxidsStmt = this.connection.prepare('SELECT txid FROM executing') + } + + async initializeV1 () { + if (this.connection.pragma('user_version')[0].user_version !== 0) return + + this.logger.info('Setting up database v1') + + await this.performOnTransaction(() => { + this.connection.pragma('user_version = 1') + + this.connection.prepare( + `CREATE TABLE IF NOT EXISTS tx ( + txid TEXT NOT NULL, + height INTEGER, + time INTEGER, + hex TEXT, + has_code INTEGER, + executable INTEGER, + executed INTEGER, + indexed INTEGER, + UNIQUE(txid) + )` + ).run() + + this.connection.prepare( + `CREATE TABLE IF NOT EXISTS spends ( + location TEXT NOT NULL PRIMARY KEY, + spend_txid TEXT + ) WITHOUT ROWID` + ).run() + + this.connection.prepare( + `CREATE TABLE IF NOT EXISTS deps ( + up TEXT NOT NULL, + down TEXT NOT NULL, + UNIQUE(up, down) + )` + ).run() + + this.connection.prepare( + `CREATE TABLE IF NOT EXISTS jig ( + location TEXT NOT NULL PRIMARY KEY, + state TEXT NOT NULL, + class TEXT, + scripthash TEXT, + lock TEXT + ) WITHOUT ROWID` + ).run() + + this.connection.prepare( + `CREATE TABLE IF NOT EXISTS berry ( + location TEXT NOT NULL PRIMARY KEY, + state TEXT NOT NULL + ) WITHOUT ROWID` + ).run() + + this.connection.prepare( + `CREATE TABLE IF NOT EXISTS trust ( + txid TEXT NOT NULL PRIMARY KEY, + value INTEGER + ) WITHOUT ROWID` + ).run() + + this.connection.prepare( + `CREATE TABLE IF NOT EXISTS ban ( + txid TEXT NOT NULL PRIMARY KEY + ) WITHOUT ROWID` + ).run() + + this.connection.prepare( + `CREATE TABLE IF NOT EXISTS crawl ( + role TEXT UNIQUE, + height INTEGER, + hash TEXT + )` + ).run() + + this.connection.prepare( + 'CREATE INDEX IF NOT EXISTS tx_txid_index ON tx (txid)' + ).run() + + this.connection.prepare( + 'CREATE INDEX IF NOT EXISTS jig_index ON jig (class)' + ).run() + + this.connection.prepare( + 'INSERT OR IGNORE INTO crawl (role, height, hash) VALUES (\'tip\', 0, NULL)' + ).run() + }) + } + + async initializeV2 () { + if (this.connection.pragma('user_version')[0].user_version !== 1) return + + this.logger.info('Setting up database v2') + + await this.transaction(() => { + this.connection.pragma('user_version = 2') + + this.connection.prepare( + `CREATE TABLE tx_v2 ( + txid TEXT NOT NULL, + height INTEGER, + time INTEGER, + bytes BLOB, + has_code INTEGER, + executable INTEGER, + executed INTEGER, + indexed INTEGER + )` + ).run() + + const txids = this.connection.prepare('SELECT txid FROM tx').all().map(row => row.txid) + const gettx = this.connection.prepare('SELECT * FROM tx WHERE txid = ?') + const insert = this.connection.prepare('INSERT INTO tx_v2 (txid, height, time, bytes, has_code, executable, executed, indexed) VALUES (?, ?, ?, ?, ?, ?, ?, ?)') + + this.logger.info('Migrating data') + for (const txid of txids) { + const row = gettx.get(txid) + const bytes = row.hex ? Buffer.from(row.hex, 'hex') : null + insert.run(row.txid, row.height, row.time, bytes, row.has_code, row.executable, row.executed, row.indexed) + } + + this.connection.prepare( + 'DROP INDEX tx_txid_index' + ).run() + + this.connection.prepare( + 'DROP TABLE tx' + ).run() + + this.connection.prepare( + 'ALTER TABLE tx_v2 RENAME TO tx' + ).run() + + this.connection.prepare( + 'CREATE INDEX IF NOT EXISTS tx_txid_index ON tx (txid)' + ).run() + + this.logger.info('Saving results') + }) + + this.logger.info('Optimizing database') + this.connection.prepare('VACUUM').run() + } + + async initializeV3 () { + if (this.connection.pragma('user_version')[0].user_version !== 2) return + + this.logger.info('Setting up database v3') + + await this.transaction(() => { + this.connection.pragma('user_version = 3') + + this.connection.prepare('CREATE INDEX IF NOT EXISTS deps_up_index ON deps (up)').run() + this.connection.prepare('CREATE INDEX IF NOT EXISTS deps_down_index ON deps (down)').run() + this.connection.prepare('CREATE INDEX IF NOT EXISTS trust_txid_index ON trust (txid)').run() + + this.logger.info('Saving results') + }) + } + + async initializeV4 () { + if (this.connection.pragma('user_version')[0].user_version !== 3) return + + this.logger.info('Setting up database v4') + + await this.transaction(() => { + this.connection.pragma('user_version = 4') + + this.connection.prepare('ALTER TABLE tx ADD COLUMN downloaded INTEGER GENERATED ALWAYS AS (bytes IS NOT NULL) VIRTUAL').run() + + this.connection.prepare('CREATE INDEX IF NOT EXISTS tx_downloaded_index ON tx (downloaded)').run() + + this.logger.info('Saving results') + }) + } + + async initializeV5 () { + if (this.connection.pragma('user_version')[0].user_version !== 4) return + + this.logger.info('Setting up database v5') + + await this.transaction(() => { + this.connection.pragma('user_version = 5') + + this.connection.prepare('CREATE INDEX IF NOT EXISTS ban_txid_index ON ban (txid)').run() + this.connection.prepare('CREATE INDEX IF NOT EXISTS tx_height_index ON tx (height)').run() + + this.logger.info('Saving results') + }) + } + + async initializeV6 () { + if (this.connection.pragma('user_version')[0].user_version !== 5) return + + this.logger.info('Setting up database v6') + + await this.transaction(() => { + this.connection.pragma('user_version = 6') + + const height = this.connection.prepare('SELECT height FROM crawl WHERE role = \'tip\'').raw(true).all()[0] + const hash = this.connection.prepare('SELECT hash FROM crawl WHERE role = \'tip\'').raw(true).all()[0] + + this.connection.prepare('DROP TABLE crawl').run() + + this.connection.prepare( + `CREATE TABLE IF NOT EXISTS crawl ( + key TEXT UNIQUE, + value TEXT + )` + ).run() + + this.connection.prepare('INSERT INTO crawl (key, value) VALUES (\'height\', ?)').run(height.toString()) + this.connection.prepare('INSERT INTO crawl (key, value) VALUES (\'hash\', ?)').run(hash) + + this.logger.info('Saving results') + }) + } + + async initializeV7 () { + if (this.connection.pragma('user_version')[0].user_version !== 6) return + + this.logger.info('Setting up database v7') + + await this.transaction(() => { + this.connection.pragma('user_version = 7') + + this.logger.info('Getting possible transactions to execute') + const stmt = this.connection.prepare(` + SELECT txid + FROM tx + WHERE downloaded = 1 + AND executable = 1 + AND executed = 0 + AND (has_code = 0 OR (SELECT COUNT(*) FROM trust WHERE trust.txid = tx.txid AND trust.value = 1) = 1) + AND txid NOT IN ban + `) + const txids = stmt.raw(true).all().map(x => x[0]) + + const isReadyToExecuteStmt = this.connection.prepare(IS_READY_TO_EXECUTE_SQL) + + const ready = [] + for (let i = 0; i < txids.length; i++) { + const txid = txids[i] + const row = isReadyToExecuteStmt.get(txid) + if (row && row.ready) ready.push(txid) + if (i % 1000 === 0) console.log('Checking to execute', i, 'of', txids.length) + } + + this.logger.info('Marking', ready.length, 'transactions to execute') + this.connection.prepare('CREATE TABLE IF NOT EXISTS executing (txid TEXT UNIQUE)').run() + const markExecutingStmt = this.connection.prepare('INSERT OR IGNORE INTO executing (txid) VALUES (?)') + ready.forEach(txid => markExecutingStmt.run(txid)) + + this.logger.info('Saving results') + }) + } + + async tearDown () { + if (this.connection) { + this.logger.debug('Closing' + (this.readonly ? ' readonly' : '') + ' database') + this.connection.close() + this.connection = null + } + } + + async performOnTransaction (fn) { + this.knex.transaction(async (trx) => { + await fn(trx) + }) + } + + async transaction (f) { + if (!this.connection) return + try { + this.connection.exec('begin;') + await f() + this.connection.exec('commit;') + } catch (e) { + this.connection.exec('rollback;') + console.error(e) + throw e + } + // this.connection.transaction(f)() + } + + async txExists (txid) { + return !!this.txExistsStmt.get(txid) + } + + async checkTxIsDownloaded (txid) { + const result = this.getTransactionDownloadedStmt.raw(true).get(txid) + return result ? !!result[0] : false + } + + async searchTxsAboveHeight (height) { + return this.getTransactionsAboveHeightStmt.raw(true).all(height) + } + + async mempoolTxsPreviousToTime (time) { + return this.getMempoolTransactionsBeforeTimeStmt.raw(true).all(time) + } + + async searchTxsToDownload () { + return this.getTransactionsToDownloadStmt.raw(true).all() + } + + async countDownloadedTxs () { + return this.getTransactionsDownloadedCountStmt.get().count + } + + async countIndexedTxs () { + return this.getTransactionsIndexedCountStmt.get().count + } + + async getFailedTx (deptxid) { + return this.getTransactionFailedStmt.get(deptxid).failed + } + + async addNewTx (txid, time) { + await this.addNewTransactionStmt.run(txid, time) + } + + async setTxHeight (txid, height) { + this.setTransactionHeightStmt.run(height, txid) + } + + async setTxTime (txid, time) { + this.setTransactionTimeStmt.run(time, txid) + } + + async setTxBytes (txid, bytes) { + this.setTransactionBytesStmt.run(bytes, txid) + } + + async setExecutableForTx (txid, executable) { + this.setTransactionExecutableStmt.run(executable, txid) + } + + async setHasCodeForTx (txid, hasCode) { + this.setTransactionHasCodeStmt.run(hasCode, txid) + } + + async setExecutedForTx (txid, executed) { + this.setTransactionExecutedStmt.run(executed, txid) + } + + async setIndexedForTx (txid, indexed) { + this.setTransactionIndexedStmt.run(indexed, txid) + } + + async txIsIndexed (txid) { + return this.getTransactionIndexedStmt.raw(true).get(txid)[0] + } + + async getTxHex (txid) { + const row = this.getTransactionHexStmt.raw(true).get(txid) + return row && row[0] + } + + async getTxTime (txid) { + const row = this.getTransactionTimeStmt.raw(true).get(txid) + return row && row[0] + } + + async getTxHeight (txid) { + const row = this.getTransactionHeightStmt.raw(true).get(txid) + return row && row[0] + } + + async deleteTx (txid) { + this.deleteTransactionStmt.run(txid) + } + + async unconfirmTx (txid) { + this.unconfirmTransactionStmt.run(txid) + } + + // executing + + async markTxAsExecuting (txid) { + this.markExecutingStmt.run(txid) + } + + async removeTxFromExecuting (txid) { + this.unmarkExecutingStmt.run(txid) + } + + async findAllExecutingTxids () { + return this.findAllExecutingTxidsStmt.raw(true).all().map(x => x[0]) + } + + async txidReadyToExecute (txid) { + return this.isReadyToExecuteStmt.get(txid) + } + + // spends + + async getSpendingTxid (location) { + const row = this.getSpendStmt.raw(true).get(location) + return row && row[0] + } + + async upsertSpend (location, txid) { + await this.setSpendStmt.run(location, txid) + } + + async setAsUnspent (location) { + this.setUnspentStmt.run(location) + } + + async deleteSpendsForTxid (txid) { + this.deleteSpendsStmt.run(txid) + } + + async unspendOutput (txid) { + this.unspendOutputsStmt.run(txid) + } + + // deps + + async addDep (deptxid, txid) { + this.addDepStmt.run(deptxid, txid) + } + + async searchDownstreamTxidsReadyToExecute (txid) { + return this.getDownstreamReadyToExecuteStmt.raw(true).all(txid).map(x => x[0]) + } + + async searchDownstreamForTxid (txid) { + return this.getDownstreamStmt.raw(true).all(txid).map(x => x[0]) + } + + async deleteDepsForTxid (txid) { + this.deleteDepsStmt.run(txid) + } + + async getNonExecutedUpstreamTxIds (txid) { + return this.getUpstreamUnexecutedCodeStmt.raw(true).all(txid).map(x => x[0]) + } + + // jig + + async getJigState (location) { + const row = this.getJigStateStmt.raw(true).get(location) + return row && row[0] + } + + async setJig (location, jig) { + this.setJigStateStmt.run(location, jig) + } + + async setBerry (location, berry) { + this.setBerryStateStmt.run(location, berry) + } + + async getBerry (location) { + const row = this.getBerryStateStmt.raw(true).get(location) + return row && row[0] + } + + async setJigClass (location, cls) { + this.setJigClassStmt.run(cls, location) + } + + async setJigLock (location, lock) { + this.setJigLockStmt.run(lock, location) + } + + async setJigScriptHash (location, scriptHash) { + this.setJigScripthashStmt.run(scriptHash, location) + } + + async deleteJigStatesForTxid (txid) { + this.deleteJigStatesStmt.run(txid) + } + + async deleteBerryStatesForTxid (txid) { + this.deleteBerryStatesStmt.run(txid) + } + + // unspent + + async getAllUnspent () { + return this.getAllUnspentStmt.raw(true).all().map(row => row[0]) + } + + async getAllUnspentByClassOrigin (origin) { + return this.getAllUnspentByClassStmt.raw(true).all(origin).map(row => row[0]) + } + + async getAllUnspentByLockOrigin (origin) { + return this.getAllUnspentByLockStmt.raw(true).all(origin).map(row => row[0]) + } + + async getAllUnspentByScripthash (scripthash) { + return this.getAllUnspentByScripthashStmt.raw(true).all(scripthash).map(row => row[0]) + } + + async getAllUnspentByClassOriginAndLockOrigin (clsOrigin, lockOrigin) { + return this.getAllUnspentByClassLockStmt.raw(true).all(clsOrigin, lockOrigin).map(row => row[0]) + } + + async getAllUnspentByClassOriginAndScripthash (clsOrigin, scripthash) { + return this.getAllUnspentByClassScripthashStmt.raw(true).all(clsOrigin, scripthash).map(row => row[0]) + } + + async getAllUnspentByLockOriginAndScripthash (lockOrigin, scripthash) { + return this.getAllUnspentByLockScripthashStmt.raw(true).all(lockOrigin, scripthash).map(row => row[0]) + } + + async getAllUnspentByClassOriginAndLockOriginAndScriptHash (clsOrigin, lockOrigin, scripthash) { + return this.getAllUnspentByClassLockScripthashStmt.raw(true).all(clsOrigin, lockOrigin, scripthash).map(row => row[0]) + } + + async countTotalUnspent () { + return this.getNumUnspentStmt.get().unspent + } + + // trust + + async isTrusted (txid) { + const row = this.isTrustedStmt.raw(true).get(txid) + return !!row && !!row[0] + } + + async setTrust (txid, trusted) { + this.setTrustedStmt.run(txid, trusted) + } + + async searchAllTrust () { + return this.getTrustlistStmt.raw(true).all().map(x => x[0]) + } + + // ban + + async checkIsBanned (txid) { + const row = this.isBannedStmt.raw(true).get(txid) + return !!row && !!row[0] + } + + async saveBan (txid) { + this.banStmt.run(txid) + } + + async removeBan (txid) { + this.unbanStmt.run(txid) + } + + async searchAllBans () { + return this.getBanlistStmt.raw(true).all().map(x => x[0]) + } + + // crawl + + async setCrawlHeight (heigth) { + this.setHeightStmt.run(heigth.toString()) + } + + async setCrawlHash (hash) { + this.setHashStmt.run(hash) + } + + async getCrawlHeight () { + const row = this.getHeightStmt.raw(true).all()[0] + return row && parseInt(row[0]) + } + + async getCrawlHash () { + const row = this.getHashStmt.raw(true).all()[0] + return row && row[0] + } +} + +// ------------------------------------------------------------------------------------------------ + +module.exports = { KnexDataSource } diff --git a/src/sqlite-datasource.js b/src/data-sources/sqlite-datasource.js similarity index 98% rename from src/sqlite-datasource.js rename to src/data-sources/sqlite-datasource.js index bda9ea3..77327c8 100644 --- a/src/sqlite-datasource.js +++ b/src/data-sources/sqlite-datasource.js @@ -4,7 +4,7 @@ * Layer between the database and the application */ const Sqlite3Database = require('better-sqlite3') -const { HEIGHT_MEMPOOL } = require('./constants') +const { HEIGHT_MEMPOOL } = require('../constants') // The + in the following 2 queries before downloaded improves performance by NOT using the // tx_downloaded index, which is rarely an improvement over a simple filter for single txns. @@ -180,7 +180,7 @@ class SqliteDatasource { this.logger.info('Setting up database v1') - await this.transaction(() => { + await this.performOnTransaction(() => { this.connection.pragma('user_version = 1') this.connection.prepare( @@ -269,7 +269,7 @@ class SqliteDatasource { this.logger.info('Setting up database v2') - await this.transaction(() => { + await this.performOnTransaction(() => { this.connection.pragma('user_version = 2') this.connection.prepare( @@ -324,7 +324,7 @@ class SqliteDatasource { this.logger.info('Setting up database v3') - await this.transaction(() => { + await this.performOnTransaction(() => { this.connection.pragma('user_version = 3') this.connection.prepare('CREATE INDEX IF NOT EXISTS deps_up_index ON deps (up)').run() @@ -340,7 +340,7 @@ class SqliteDatasource { this.logger.info('Setting up database v4') - await this.transaction(() => { + await this.performOnTransaction(() => { this.connection.pragma('user_version = 4') this.connection.prepare('ALTER TABLE tx ADD COLUMN downloaded INTEGER GENERATED ALWAYS AS (bytes IS NOT NULL) VIRTUAL').run() @@ -356,7 +356,7 @@ class SqliteDatasource { this.logger.info('Setting up database v5') - await this.transaction(() => { + await this.performOnTransaction(() => { this.connection.pragma('user_version = 5') this.connection.prepare('CREATE INDEX IF NOT EXISTS ban_txid_index ON ban (txid)').run() @@ -371,7 +371,7 @@ class SqliteDatasource { this.logger.info('Setting up database v6') - await this.transaction(() => { + await this.performOnTransaction(() => { this.connection.pragma('user_version = 6') const height = this.connection.prepare('SELECT height FROM crawl WHERE role = \'tip\'').raw(true).all()[0] @@ -398,7 +398,7 @@ class SqliteDatasource { this.logger.info('Setting up database v7') - await this.transaction(() => { + await this.performOnTransaction(() => { this.connection.pragma('user_version = 7') this.logger.info('Getting possible transactions to execute') @@ -464,7 +464,6 @@ class SqliteDatasource { console.error(e) throw e } - // this.connection.transaction(f)() } async txExists (txid) { diff --git a/src/database.js b/src/database.js index 16e522e..16ac9e4 100644 --- a/src/database.js +++ b/src/database.js @@ -6,7 +6,7 @@ const Run = require('run-sdk') const bsv = require('bsv') -const { SqliteDatasource } = require('./sqlite-datasource') +const { SqliteDatasource } = require('./data-sources/sqlite-datasource') const { HEIGHT_MEMPOOL, HEIGHT_UNKNOWN } = require('./constants') // ------------------------------------------------------------------------------------------------ From c7e66710c9204df9f90be89ad0eed25f82999a4a Mon Sep 17 00:00:00 2001 From: hojarasca Date: Tue, 7 Dec 2021 13:26:59 -0300 Subject: [PATCH 15/82] simplified get and set state --- src/data-sources/knex-datasource.js | 744 -------------------------- src/data-sources/sqlite-datasource.js | 22 +- src/database.js | 6 +- src/executor.js | 4 +- test/crawler.js | 18 +- test/indexer.js | 3 +- 6 files changed, 31 insertions(+), 766 deletions(-) delete mode 100644 src/data-sources/knex-datasource.js diff --git a/src/data-sources/knex-datasource.js b/src/data-sources/knex-datasource.js deleted file mode 100644 index 597179d..0000000 --- a/src/data-sources/knex-datasource.js +++ /dev/null @@ -1,744 +0,0 @@ -/** - * database.js - * - * Layer between the database and the application - */ -const knex = require('knex') -const { HEIGHT_MEMPOOL } = require('../constants') - -// The + in the following 2 queries before downloaded improves performance by NOT using the -// tx_downloaded index, which is rarely an improvement over a simple filter for single txns. -// See: https://www.sqlite.org/optoverview.html -const IS_READY_TO_EXECUTE_SQL = ` - SELECT ( - downloaded = 1 - AND executable = 1 - AND executed = 0 - AND (has_code = 0 OR (SELECT COUNT(*) FROM trust WHERE trust.txid = tx.txid AND trust.value = 1) = 1) - AND txid NOT IN ban - AND ( - SELECT COUNT(*) - FROM tx AS tx2 - JOIN deps - ON deps.up = tx2.txid - WHERE deps.down = tx.txid - AND (+tx2.downloaded = 0 OR (tx2.executable = 1 AND tx2.executed = 0)) - ) = 0 - ) AS ready - FROM tx - WHERE txid = ? - ` - -const GET_DOWNSTREAM_READY_TO_EXECUTE_SQL = ` - SELECT down - FROM deps - JOIN tx - ON tx.txid = deps.down - WHERE up = ? - AND +downloaded = 1 - AND executable = 1 - AND executed = 0 - AND (has_code = 0 OR (SELECT COUNT(*) FROM trust WHERE trust.txid = tx.txid AND trust.value = 1) = 1) - AND txid NOT IN ban - AND ( - SELECT COUNT(*) - FROM tx AS tx2 - JOIN deps - ON deps.up = tx2.txid - WHERE deps.down = tx.txid - AND (+tx2.downloaded = 0 OR (tx2.executable = 1 AND tx2.executed = 0)) - ) = 0 - ` - -// ------------------------------------------------------------------------------------------------ -// Database -// ------------------------------------------------------------------------------------------------ - -class KnexDataSource { - constructor (dbUri, logger, readonly = false) { - this.dbUri = dbUri - this.logger = logger - this.readonly = readonly - this.connection = null - } - - async setUp () { - this.logger.debug('Opening' + (this.readonly ? ' readonly' : '') + ' database') - if (this.connection) throw new Error('Database already open') - - this.connection = knex({ - client: 'pg', - connection: this.dbUri - }) - - // Initialise and perform upgrades - await this.initializeV1() - await this.initializeV2() - await this.initializeV3() - await this.initializeV4() - await this.initializeV5() - await this.initializeV6() - await this.initializeV7() - - this.addNewTransactionStmt = this.connection.prepare('INSERT OR IGNORE INTO tx (txid, height, time, bytes, has_code, executable, executed, indexed) VALUES (?, null, ?, null, 0, 0, 0, 0)') - this.setTransactionBytesStmt = this.connection.prepare('UPDATE tx SET bytes = ? WHERE txid = ?') - this.setTransactionExecutableStmt = this.connection.prepare('UPDATE tx SET executable = ? WHERE txid = ?') - this.setTransactionTimeStmt = this.connection.prepare('UPDATE tx SET time = ? WHERE txid = ?') - this.setTransactionHeightStmt = this.connection.prepare(`UPDATE tx SET height = ? WHERE txid = ? AND (height IS NULL OR height = ${HEIGHT_MEMPOOL})`) - this.setTransactionHasCodeStmt = this.connection.prepare('UPDATE tx SET has_code = ? WHERE txid = ?') - this.setTransactionExecutedStmt = this.connection.prepare('UPDATE tx SET executed = ? WHERE txid = ?') - this.setTransactionIndexedStmt = this.connection.prepare('UPDATE tx SET indexed = ? WHERE txid = ?') - this.txExistsStmt = this.connection.prepare('SELECT txid FROM tx WHERE txid = ?') - this.getTransactionHexStmt = this.connection.prepare('SELECT LOWER(HEX(bytes)) AS hex FROM tx WHERE txid = ?') - this.getTransactionTimeStmt = this.connection.prepare('SELECT time FROM tx WHERE txid = ?') - this.getTransactionHeightStmt = this.connection.prepare('SELECT height FROM tx WHERE txid = ?') - this.getTransactionHasCodeStmt = this.connection.prepare('SELECT has_code FROM tx WHERE txid = ?') - this.getTransactionIndexedStmt = this.connection.prepare('SELECT indexed FROM tx WHERE txid = ?') - this.getTransactionFailedStmt = this.connection.prepare('SELECT (executed = 1 AND indexed = 0) AS failed FROM tx WHERE txid = ?') - this.getTransactionDownloadedStmt = this.connection.prepare('SELECT downloaded FROM tx WHERE txid = ?') - this.deleteTransactionStmt = this.connection.prepare('DELETE FROM tx WHERE txid = ?') - this.unconfirmTransactionStmt = this.connection.prepare(`UPDATE tx SET height = ${HEIGHT_MEMPOOL} WHERE txid = ?`) - this.getTransactionsAboveHeightStmt = this.connection.prepare('SELECT txid FROM tx WHERE height > ?') - this.getMempoolTransactionsBeforeTimeStmt = this.connection.prepare(`SELECT txid FROM tx WHERE height = ${HEIGHT_MEMPOOL} AND time < ?`) - this.getTransactionsToDownloadStmt = this.connection.prepare('SELECT txid FROM tx WHERE downloaded = 0') - this.getTransactionsDownloadedCountStmt = this.connection.prepare('SELECT COUNT(*) AS count FROM tx WHERE downloaded = 1') - this.getTransactionsIndexedCountStmt = this.connection.prepare('SELECT COUNT(*) AS count FROM tx WHERE indexed = 1') - this.isReadyToExecuteStmt = this.connection.prepare(IS_READY_TO_EXECUTE_SQL) - this.getDownstreamReadyToExecuteStmt = this.connection.prepare(GET_DOWNSTREAM_READY_TO_EXECUTE_SQL) - - this.setSpendStmt = this.connection.prepare('INSERT OR REPLACE INTO spends (location, spend_txid) VALUES (?, ?)') - this.setUnspentStmt = this.connection.prepare('INSERT OR IGNORE INTO spends (location, spend_txid) VALUES (?, null)') - this.getSpendStmt = this.connection.prepare('SELECT spend_txid FROM spends WHERE location = ?') - this.unspendOutputsStmt = this.connection.prepare('UPDATE spends SET spend_txid = null WHERE spend_txid = ?') - this.deleteSpendsStmt = this.connection.prepare('DELETE FROM spends WHERE location LIKE ? || \'%\'') - // - this.addDepStmt = this.connection.prepare('INSERT OR IGNORE INTO deps (up, down) VALUES (?, ?)') - this.deleteDepsStmt = this.connection.prepare('DELETE FROM deps WHERE down = ?') - this.getDownstreamStmt = this.connection.prepare('SELECT down FROM deps WHERE up = ?') - this.getUpstreamUnexecutedCodeStmt = this.connection.prepare(` - SELECT txdeps.txid as txid - FROM (SELECT up AS txid FROM deps WHERE down = ?) as txdeps - JOIN tx ON tx.txid = txdeps.txid - WHERE tx.executable = 1 AND tx.executed = 0 AND tx.has_code = 1 - `) - // - this.setJigStateStmt = this.connection.prepare('INSERT OR IGNORE INTO jig (location, state, class, lock, scripthash) VALUES (?, ?, null, null, null)') - this.setJigClassStmt = this.connection.prepare('UPDATE jig SET class = ? WHERE location = ?') - this.setJigLockStmt = this.connection.prepare('UPDATE jig SET lock = ? WHERE location = ?') - this.setJigScripthashStmt = this.connection.prepare('UPDATE jig SET scripthash = ? WHERE location = ?') - this.getJigStateStmt = this.connection.prepare('SELECT state FROM jig WHERE location = ?') - this.deleteJigStatesStmt = this.connection.prepare('DELETE FROM jig WHERE location LIKE ? || \'%\'') - - const getAllUnspentSql = ` - SELECT spends.location AS location FROM spends - JOIN jig ON spends.location = jig.location - WHERE spends.spend_txid IS NULL` - this.getAllUnspentStmt = this.connection.prepare(getAllUnspentSql) - this.getAllUnspentByClassStmt = this.connection.prepare(`${getAllUnspentSql} AND jig.class = ?`) - this.getAllUnspentByLockStmt = this.connection.prepare(`${getAllUnspentSql} AND jig.lock = ?`) - this.getAllUnspentByScripthashStmt = this.connection.prepare(`${getAllUnspentSql} AND jig.scripthash = ?`) - this.getAllUnspentByClassLockStmt = this.connection.prepare(`${getAllUnspentSql} AND jig.class = ? AND lock = ?`) - this.getAllUnspentByClassScripthashStmt = this.connection.prepare(`${getAllUnspentSql} AND jig.class = ? AND scripthash = ?`) - this.getAllUnspentByLockScripthashStmt = this.connection.prepare(`${getAllUnspentSql} AND jig.lock = ? AND scripthash = ?`) - this.getAllUnspentByClassLockScripthashStmt = this.connection.prepare(`${getAllUnspentSql} AND jig.class = ? AND jig.lock = ? AND scripthash = ?`) - this.getNumUnspentStmt = this.connection.prepare('SELECT COUNT(*) as unspent FROM spends JOIN jig ON spends.location = jig.location WHERE spends.spend_txid IS NULL') - - this.setBerryStateStmt = this.connection.prepare('INSERT OR IGNORE INTO berry (location, state) VALUES (?, ?)') - this.getBerryStateStmt = this.connection.prepare('SELECT state FROM berry WHERE location = ?') - this.deleteBerryStatesStmt = this.connection.prepare('DELETE FROM berry WHERE location LIKE ? || \'%\'') - - this.setTrustedStmt = this.connection.prepare('INSERT OR REPLACE INTO trust (txid, value) VALUES (?, ?)') - this.getTrustlistStmt = this.connection.prepare('SELECT txid FROM trust WHERE value = 1') - this.isTrustedStmt = this.connection.prepare('SELECT COUNT(*) FROM trust WHERE txid = ? AND value = 1') - - this.banStmt = this.connection.prepare('INSERT OR REPLACE INTO ban (txid) VALUES (?)') - this.unbanStmt = this.connection.prepare('DELETE FROM ban WHERE txid = ?') - this.isBannedStmt = this.connection.prepare('SELECT COUNT(*) FROM ban WHERE txid = ?') - this.getBanlistStmt = this.connection.prepare('SELECT txid FROM ban') - - this.getHeightStmt = this.connection.prepare('SELECT value FROM crawl WHERE key = \'height\'') - this.getHashStmt = this.connection.prepare('SELECT value FROM crawl WHERE key = \'hash\'') - this.setHeightStmt = this.connection.prepare('UPDATE crawl SET value = ? WHERE key = \'height\'') - this.setHashStmt = this.connection.prepare('UPDATE crawl SET value = ? WHERE key = \'hash\'') - - this.markExecutingStmt = this.connection.prepare('INSERT OR IGNORE INTO executing (txid) VALUES (?)') - this.unmarkExecutingStmt = this.connection.prepare('DELETE FROM executing WHERE txid = ?') - this.findAllExecutingTxidsStmt = this.connection.prepare('SELECT txid FROM executing') - } - - async initializeV1 () { - if (this.connection.pragma('user_version')[0].user_version !== 0) return - - this.logger.info('Setting up database v1') - - await this.performOnTransaction(() => { - this.connection.pragma('user_version = 1') - - this.connection.prepare( - `CREATE TABLE IF NOT EXISTS tx ( - txid TEXT NOT NULL, - height INTEGER, - time INTEGER, - hex TEXT, - has_code INTEGER, - executable INTEGER, - executed INTEGER, - indexed INTEGER, - UNIQUE(txid) - )` - ).run() - - this.connection.prepare( - `CREATE TABLE IF NOT EXISTS spends ( - location TEXT NOT NULL PRIMARY KEY, - spend_txid TEXT - ) WITHOUT ROWID` - ).run() - - this.connection.prepare( - `CREATE TABLE IF NOT EXISTS deps ( - up TEXT NOT NULL, - down TEXT NOT NULL, - UNIQUE(up, down) - )` - ).run() - - this.connection.prepare( - `CREATE TABLE IF NOT EXISTS jig ( - location TEXT NOT NULL PRIMARY KEY, - state TEXT NOT NULL, - class TEXT, - scripthash TEXT, - lock TEXT - ) WITHOUT ROWID` - ).run() - - this.connection.prepare( - `CREATE TABLE IF NOT EXISTS berry ( - location TEXT NOT NULL PRIMARY KEY, - state TEXT NOT NULL - ) WITHOUT ROWID` - ).run() - - this.connection.prepare( - `CREATE TABLE IF NOT EXISTS trust ( - txid TEXT NOT NULL PRIMARY KEY, - value INTEGER - ) WITHOUT ROWID` - ).run() - - this.connection.prepare( - `CREATE TABLE IF NOT EXISTS ban ( - txid TEXT NOT NULL PRIMARY KEY - ) WITHOUT ROWID` - ).run() - - this.connection.prepare( - `CREATE TABLE IF NOT EXISTS crawl ( - role TEXT UNIQUE, - height INTEGER, - hash TEXT - )` - ).run() - - this.connection.prepare( - 'CREATE INDEX IF NOT EXISTS tx_txid_index ON tx (txid)' - ).run() - - this.connection.prepare( - 'CREATE INDEX IF NOT EXISTS jig_index ON jig (class)' - ).run() - - this.connection.prepare( - 'INSERT OR IGNORE INTO crawl (role, height, hash) VALUES (\'tip\', 0, NULL)' - ).run() - }) - } - - async initializeV2 () { - if (this.connection.pragma('user_version')[0].user_version !== 1) return - - this.logger.info('Setting up database v2') - - await this.transaction(() => { - this.connection.pragma('user_version = 2') - - this.connection.prepare( - `CREATE TABLE tx_v2 ( - txid TEXT NOT NULL, - height INTEGER, - time INTEGER, - bytes BLOB, - has_code INTEGER, - executable INTEGER, - executed INTEGER, - indexed INTEGER - )` - ).run() - - const txids = this.connection.prepare('SELECT txid FROM tx').all().map(row => row.txid) - const gettx = this.connection.prepare('SELECT * FROM tx WHERE txid = ?') - const insert = this.connection.prepare('INSERT INTO tx_v2 (txid, height, time, bytes, has_code, executable, executed, indexed) VALUES (?, ?, ?, ?, ?, ?, ?, ?)') - - this.logger.info('Migrating data') - for (const txid of txids) { - const row = gettx.get(txid) - const bytes = row.hex ? Buffer.from(row.hex, 'hex') : null - insert.run(row.txid, row.height, row.time, bytes, row.has_code, row.executable, row.executed, row.indexed) - } - - this.connection.prepare( - 'DROP INDEX tx_txid_index' - ).run() - - this.connection.prepare( - 'DROP TABLE tx' - ).run() - - this.connection.prepare( - 'ALTER TABLE tx_v2 RENAME TO tx' - ).run() - - this.connection.prepare( - 'CREATE INDEX IF NOT EXISTS tx_txid_index ON tx (txid)' - ).run() - - this.logger.info('Saving results') - }) - - this.logger.info('Optimizing database') - this.connection.prepare('VACUUM').run() - } - - async initializeV3 () { - if (this.connection.pragma('user_version')[0].user_version !== 2) return - - this.logger.info('Setting up database v3') - - await this.transaction(() => { - this.connection.pragma('user_version = 3') - - this.connection.prepare('CREATE INDEX IF NOT EXISTS deps_up_index ON deps (up)').run() - this.connection.prepare('CREATE INDEX IF NOT EXISTS deps_down_index ON deps (down)').run() - this.connection.prepare('CREATE INDEX IF NOT EXISTS trust_txid_index ON trust (txid)').run() - - this.logger.info('Saving results') - }) - } - - async initializeV4 () { - if (this.connection.pragma('user_version')[0].user_version !== 3) return - - this.logger.info('Setting up database v4') - - await this.transaction(() => { - this.connection.pragma('user_version = 4') - - this.connection.prepare('ALTER TABLE tx ADD COLUMN downloaded INTEGER GENERATED ALWAYS AS (bytes IS NOT NULL) VIRTUAL').run() - - this.connection.prepare('CREATE INDEX IF NOT EXISTS tx_downloaded_index ON tx (downloaded)').run() - - this.logger.info('Saving results') - }) - } - - async initializeV5 () { - if (this.connection.pragma('user_version')[0].user_version !== 4) return - - this.logger.info('Setting up database v5') - - await this.transaction(() => { - this.connection.pragma('user_version = 5') - - this.connection.prepare('CREATE INDEX IF NOT EXISTS ban_txid_index ON ban (txid)').run() - this.connection.prepare('CREATE INDEX IF NOT EXISTS tx_height_index ON tx (height)').run() - - this.logger.info('Saving results') - }) - } - - async initializeV6 () { - if (this.connection.pragma('user_version')[0].user_version !== 5) return - - this.logger.info('Setting up database v6') - - await this.transaction(() => { - this.connection.pragma('user_version = 6') - - const height = this.connection.prepare('SELECT height FROM crawl WHERE role = \'tip\'').raw(true).all()[0] - const hash = this.connection.prepare('SELECT hash FROM crawl WHERE role = \'tip\'').raw(true).all()[0] - - this.connection.prepare('DROP TABLE crawl').run() - - this.connection.prepare( - `CREATE TABLE IF NOT EXISTS crawl ( - key TEXT UNIQUE, - value TEXT - )` - ).run() - - this.connection.prepare('INSERT INTO crawl (key, value) VALUES (\'height\', ?)').run(height.toString()) - this.connection.prepare('INSERT INTO crawl (key, value) VALUES (\'hash\', ?)').run(hash) - - this.logger.info('Saving results') - }) - } - - async initializeV7 () { - if (this.connection.pragma('user_version')[0].user_version !== 6) return - - this.logger.info('Setting up database v7') - - await this.transaction(() => { - this.connection.pragma('user_version = 7') - - this.logger.info('Getting possible transactions to execute') - const stmt = this.connection.prepare(` - SELECT txid - FROM tx - WHERE downloaded = 1 - AND executable = 1 - AND executed = 0 - AND (has_code = 0 OR (SELECT COUNT(*) FROM trust WHERE trust.txid = tx.txid AND trust.value = 1) = 1) - AND txid NOT IN ban - `) - const txids = stmt.raw(true).all().map(x => x[0]) - - const isReadyToExecuteStmt = this.connection.prepare(IS_READY_TO_EXECUTE_SQL) - - const ready = [] - for (let i = 0; i < txids.length; i++) { - const txid = txids[i] - const row = isReadyToExecuteStmt.get(txid) - if (row && row.ready) ready.push(txid) - if (i % 1000 === 0) console.log('Checking to execute', i, 'of', txids.length) - } - - this.logger.info('Marking', ready.length, 'transactions to execute') - this.connection.prepare('CREATE TABLE IF NOT EXISTS executing (txid TEXT UNIQUE)').run() - const markExecutingStmt = this.connection.prepare('INSERT OR IGNORE INTO executing (txid) VALUES (?)') - ready.forEach(txid => markExecutingStmt.run(txid)) - - this.logger.info('Saving results') - }) - } - - async tearDown () { - if (this.connection) { - this.logger.debug('Closing' + (this.readonly ? ' readonly' : '') + ' database') - this.connection.close() - this.connection = null - } - } - - async performOnTransaction (fn) { - this.knex.transaction(async (trx) => { - await fn(trx) - }) - } - - async transaction (f) { - if (!this.connection) return - try { - this.connection.exec('begin;') - await f() - this.connection.exec('commit;') - } catch (e) { - this.connection.exec('rollback;') - console.error(e) - throw e - } - // this.connection.transaction(f)() - } - - async txExists (txid) { - return !!this.txExistsStmt.get(txid) - } - - async checkTxIsDownloaded (txid) { - const result = this.getTransactionDownloadedStmt.raw(true).get(txid) - return result ? !!result[0] : false - } - - async searchTxsAboveHeight (height) { - return this.getTransactionsAboveHeightStmt.raw(true).all(height) - } - - async mempoolTxsPreviousToTime (time) { - return this.getMempoolTransactionsBeforeTimeStmt.raw(true).all(time) - } - - async searchTxsToDownload () { - return this.getTransactionsToDownloadStmt.raw(true).all() - } - - async countDownloadedTxs () { - return this.getTransactionsDownloadedCountStmt.get().count - } - - async countIndexedTxs () { - return this.getTransactionsIndexedCountStmt.get().count - } - - async getFailedTx (deptxid) { - return this.getTransactionFailedStmt.get(deptxid).failed - } - - async addNewTx (txid, time) { - await this.addNewTransactionStmt.run(txid, time) - } - - async setTxHeight (txid, height) { - this.setTransactionHeightStmt.run(height, txid) - } - - async setTxTime (txid, time) { - this.setTransactionTimeStmt.run(time, txid) - } - - async setTxBytes (txid, bytes) { - this.setTransactionBytesStmt.run(bytes, txid) - } - - async setExecutableForTx (txid, executable) { - this.setTransactionExecutableStmt.run(executable, txid) - } - - async setHasCodeForTx (txid, hasCode) { - this.setTransactionHasCodeStmt.run(hasCode, txid) - } - - async setExecutedForTx (txid, executed) { - this.setTransactionExecutedStmt.run(executed, txid) - } - - async setIndexedForTx (txid, indexed) { - this.setTransactionIndexedStmt.run(indexed, txid) - } - - async txIsIndexed (txid) { - return this.getTransactionIndexedStmt.raw(true).get(txid)[0] - } - - async getTxHex (txid) { - const row = this.getTransactionHexStmt.raw(true).get(txid) - return row && row[0] - } - - async getTxTime (txid) { - const row = this.getTransactionTimeStmt.raw(true).get(txid) - return row && row[0] - } - - async getTxHeight (txid) { - const row = this.getTransactionHeightStmt.raw(true).get(txid) - return row && row[0] - } - - async deleteTx (txid) { - this.deleteTransactionStmt.run(txid) - } - - async unconfirmTx (txid) { - this.unconfirmTransactionStmt.run(txid) - } - - // executing - - async markTxAsExecuting (txid) { - this.markExecutingStmt.run(txid) - } - - async removeTxFromExecuting (txid) { - this.unmarkExecutingStmt.run(txid) - } - - async findAllExecutingTxids () { - return this.findAllExecutingTxidsStmt.raw(true).all().map(x => x[0]) - } - - async txidReadyToExecute (txid) { - return this.isReadyToExecuteStmt.get(txid) - } - - // spends - - async getSpendingTxid (location) { - const row = this.getSpendStmt.raw(true).get(location) - return row && row[0] - } - - async upsertSpend (location, txid) { - await this.setSpendStmt.run(location, txid) - } - - async setAsUnspent (location) { - this.setUnspentStmt.run(location) - } - - async deleteSpendsForTxid (txid) { - this.deleteSpendsStmt.run(txid) - } - - async unspendOutput (txid) { - this.unspendOutputsStmt.run(txid) - } - - // deps - - async addDep (deptxid, txid) { - this.addDepStmt.run(deptxid, txid) - } - - async searchDownstreamTxidsReadyToExecute (txid) { - return this.getDownstreamReadyToExecuteStmt.raw(true).all(txid).map(x => x[0]) - } - - async searchDownstreamForTxid (txid) { - return this.getDownstreamStmt.raw(true).all(txid).map(x => x[0]) - } - - async deleteDepsForTxid (txid) { - this.deleteDepsStmt.run(txid) - } - - async getNonExecutedUpstreamTxIds (txid) { - return this.getUpstreamUnexecutedCodeStmt.raw(true).all(txid).map(x => x[0]) - } - - // jig - - async getJigState (location) { - const row = this.getJigStateStmt.raw(true).get(location) - return row && row[0] - } - - async setJig (location, jig) { - this.setJigStateStmt.run(location, jig) - } - - async setBerry (location, berry) { - this.setBerryStateStmt.run(location, berry) - } - - async getBerry (location) { - const row = this.getBerryStateStmt.raw(true).get(location) - return row && row[0] - } - - async setJigClass (location, cls) { - this.setJigClassStmt.run(cls, location) - } - - async setJigLock (location, lock) { - this.setJigLockStmt.run(lock, location) - } - - async setJigScriptHash (location, scriptHash) { - this.setJigScripthashStmt.run(scriptHash, location) - } - - async deleteJigStatesForTxid (txid) { - this.deleteJigStatesStmt.run(txid) - } - - async deleteBerryStatesForTxid (txid) { - this.deleteBerryStatesStmt.run(txid) - } - - // unspent - - async getAllUnspent () { - return this.getAllUnspentStmt.raw(true).all().map(row => row[0]) - } - - async getAllUnspentByClassOrigin (origin) { - return this.getAllUnspentByClassStmt.raw(true).all(origin).map(row => row[0]) - } - - async getAllUnspentByLockOrigin (origin) { - return this.getAllUnspentByLockStmt.raw(true).all(origin).map(row => row[0]) - } - - async getAllUnspentByScripthash (scripthash) { - return this.getAllUnspentByScripthashStmt.raw(true).all(scripthash).map(row => row[0]) - } - - async getAllUnspentByClassOriginAndLockOrigin (clsOrigin, lockOrigin) { - return this.getAllUnspentByClassLockStmt.raw(true).all(clsOrigin, lockOrigin).map(row => row[0]) - } - - async getAllUnspentByClassOriginAndScripthash (clsOrigin, scripthash) { - return this.getAllUnspentByClassScripthashStmt.raw(true).all(clsOrigin, scripthash).map(row => row[0]) - } - - async getAllUnspentByLockOriginAndScripthash (lockOrigin, scripthash) { - return this.getAllUnspentByLockScripthashStmt.raw(true).all(lockOrigin, scripthash).map(row => row[0]) - } - - async getAllUnspentByClassOriginAndLockOriginAndScriptHash (clsOrigin, lockOrigin, scripthash) { - return this.getAllUnspentByClassLockScripthashStmt.raw(true).all(clsOrigin, lockOrigin, scripthash).map(row => row[0]) - } - - async countTotalUnspent () { - return this.getNumUnspentStmt.get().unspent - } - - // trust - - async isTrusted (txid) { - const row = this.isTrustedStmt.raw(true).get(txid) - return !!row && !!row[0] - } - - async setTrust (txid, trusted) { - this.setTrustedStmt.run(txid, trusted) - } - - async searchAllTrust () { - return this.getTrustlistStmt.raw(true).all().map(x => x[0]) - } - - // ban - - async checkIsBanned (txid) { - const row = this.isBannedStmt.raw(true).get(txid) - return !!row && !!row[0] - } - - async saveBan (txid) { - this.banStmt.run(txid) - } - - async removeBan (txid) { - this.unbanStmt.run(txid) - } - - async searchAllBans () { - return this.getBanlistStmt.raw(true).all().map(x => x[0]) - } - - // crawl - - async setCrawlHeight (heigth) { - this.setHeightStmt.run(heigth.toString()) - } - - async setCrawlHash (hash) { - this.setHashStmt.run(hash) - } - - async getCrawlHeight () { - const row = this.getHeightStmt.raw(true).all()[0] - return row && parseInt(row[0]) - } - - async getCrawlHash () { - const row = this.getHashStmt.raw(true).all()[0] - return row && row[0] - } -} - -// ------------------------------------------------------------------------------------------------ - -module.exports = { KnexDataSource } diff --git a/src/data-sources/sqlite-datasource.js b/src/data-sources/sqlite-datasource.js index 77327c8..09005d1 100644 --- a/src/data-sources/sqlite-datasource.js +++ b/src/data-sources/sqlite-datasource.js @@ -625,20 +625,28 @@ class SqliteDatasource { async getJigState (location) { const row = this.getJigStateStmt.raw(true).get(location) - return row && row[0] + if (row && row[0]) { + return JSON.parse(row[0]) + } else { + return null + } } - async setJig (location, jig) { - this.setJigStateStmt.run(location, jig) + async setJigState (location, stateObject) { + this.setJigStateStmt.run(location, JSON.stringify(stateObject)) } - async setBerry (location, berry) { - this.setBerryStateStmt.run(location, berry) + async setBerryState (location, stateObject) { + this.setBerryStateStmt.run(location, JSON.stringify(stateObject)) } - async getBerry (location) { + async getBerryState (location) { const row = this.getBerryStateStmt.raw(true).get(location) - return row && row[0] + if (row && row[0]) { + return JSON.parse(row[0]) + } else { + return null + } } async setJigClass (location, cls) { diff --git a/src/database.js b/src/database.js index 16ac9e4..84e0053 100644 --- a/src/database.js +++ b/src/database.js @@ -219,10 +219,10 @@ class Database { for (const key of Object.keys(cache)) { if (key.startsWith('jig://')) { const location = key.slice('jig://'.length) - await this.ds.setJig(location, JSON.stringify(cache[key])) + await this.ds.setJigState(location, cache[key]) } else if (key.startsWith('berry://')) { const location = key.slice('berry://'.length) - await this.ds.setBerry(location, JSON.stringify(cache[key])) + await this.ds.setBerryState(location, cache[key]) } } @@ -450,7 +450,7 @@ class Database { // -------------------------------------------------------------------------- async getBerryState (location) { - return this.ds.getBerry(location) + return this.ds.getBerryState(location) } // -------------------------------------------------------------------------- diff --git a/src/executor.js b/src/executor.js index f8fece3..cf34124 100644 --- a/src/executor.js +++ b/src/executor.js @@ -117,11 +117,11 @@ class Executor { async _onCacheGet (key) { if (key.startsWith('jig://')) { const state = await this.database.getJigState(key.slice('jig://'.length)) - if (state) return JSON.parse(state) + if (state) return state } if (key.startsWith('berry://')) { const state = await this.database.getBerryState(key.slice('berry://'.length)) - if (state) return JSON.parse(state) + if (state) return state } if (key.startsWith('tx://')) { return await this.database.getTransactionHex(key.slice('tx://'.length)) diff --git a/test/crawler.js b/test/crawler.js index 09d3dae..409ddf1 100644 --- a/test/crawler.js +++ b/test/crawler.js @@ -16,9 +16,9 @@ const Database = require('../src/database') // ------------------------------------------------------------------------------------------------ const fetch = txid => { return { hex: require('./txns.json')[txid] } } -const indexed = (indexer, txid) => new Promise((resolve, reject) => { indexer.onIndex = x => txid === x && resolve() }) -const crawled = (indexer) => new Promise((resolve, reject) => { indexer.onBlock = height => resolve(height) }) -const reorged = (indexer) => new Promise((resolve, reject) => { indexer.onReorg = newHeight => resolve(newHeight) }) +const indexed = (indexer, txid) => new Promise((resolve) => { indexer.onIndex = x => txid === x && resolve() }) +const crawled = (indexer) => new Promise((resolve) => { indexer.onBlock = height => resolve(height) }) +const reorged = (indexer) => new Promise((resolve) => { indexer.onReorg = newHeight => resolve(newHeight) }) const logger = { info: () => {}, warn: () => {}, error: () => {}, debug: () => {} } const database = new Database(':memory:', logger, false) @@ -32,7 +32,7 @@ afterEach(() => database.close()) describe('Crawler', () => { it('add txids', async () => { const txid = '3f9de452f0c3c96be737d42aa0941b27412211976688967adb3174ee18b04c64' - function getNextBlock (height, hash) { + function getNextBlock (_height, _hash) { return { height: 1, hash: 'abc', txids: [txid] } } const api = { getNextBlock, fetch } @@ -50,7 +50,7 @@ describe('Crawler', () => { it('add block', async () => { const txid = '3f9de452f0c3c96be737d42aa0941b27412211976688967adb3174ee18b04c64' - function getNextBlock (height, hash) { + function getNextBlock (_height, _hash) { return { height: 1, hash: 'abc', txids: [txid], txhexs: [txns[txid]] } } const api = { getNextBlock } @@ -72,7 +72,7 @@ describe('Crawler', () => { 'ca9555f54dd44457d7c912e8eea375a8ed6d8ea1806a206b43af5c7f94ea47e7' ] let indexedMiddleTransaction = false - function getNextBlock (height, hash) { + function getNextBlock (height, _hash) { if (!indexedMiddleTransaction) return null if (height === 1) return null return { height: 1, hash: 'abc', txids, txhexs: txids.map(txid => txns[txid]) } @@ -100,7 +100,7 @@ describe('Crawler', () => { const txid = '3f9de452f0c3c96be737d42aa0941b27412211976688967adb3174ee18b04c64' let didReorg = false let didIndex = false - function getNextBlock (height, hash) { + function getNextBlock (height, _hash) { if (didReorg) return { height: 3, hash: 'def', txids: [] } if (height < 5) return { height: height + 1, hash: 'abc' + height, txids: [] } if (height === 5) return { height: height + 1, hash: 'abc', txids: [txid] } @@ -131,7 +131,7 @@ describe('Crawler', () => { it('keeps the states after a reorg', async () => { const txid = '3f9de452f0c3c96be737d42aa0941b27412211976688967adb3174ee18b04c64' let didReorg = false - function getNextBlock (height, hash) { + function getNextBlock (height, _hash) { if (didReorg) return { height: 3, hash: 'def', txids: [] } if (height < 5) return { height: height + 1, hash: 'abc' + height, txids: [] } if (height === 5) return { height: height + 1, hash: 'abc', txids: [txid] } @@ -145,7 +145,7 @@ describe('Crawler', () => { await reorged(indexer) await indexer.stop() expect(await database.getTransactionHex(txid)).not.to.equal(undefined) - const state = JSON.parse(await database.getJigState(txid + '_o1')) + const state = await database.getJigState(txid + '_o1') expect(state.props.origin).to.equal('_o1') expect(state.src).to.match(/class Dragon/) expect(await database.getTransactionHeight(txid)).to.equal(-1) diff --git a/test/indexer.js b/test/indexer.js index 686a274..3c115ee 100644 --- a/test/indexer.js +++ b/test/indexer.js @@ -101,6 +101,7 @@ describe('Indexer', () => { it('discovered berry transaction', async () => { const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, []) await indexer.start() + const promise = indexed(indexer, 'bfa5180e601e92af23d80782bf625b102ac110105a392e376fe7607e4e87dc8d') await database.addTransaction('bfa5180e601e92af23d80782bf625b102ac110105a392e376fe7607e4e87dc8d') // Class with berry image await database.addTransaction('24cde3638a444c8ad397536127833878ffdfe1b04d5595489bd294e50d77105a') // B (old) await database.addTransaction('312985bd960ae4c59856b3089b04017ede66506ea181333eec7c9bb88b11c490') // txo, Tx @@ -111,7 +112,7 @@ describe('Indexer', () => { await database.trust('727e7b423b7ee40c0b5be87fba7fa5673ea2d20a74259040a7295d9c32a90011') // Don't index the berry data, because it will be fetched automatically // database.addTransaction('2f3492ef5401d887a93ca09820dff952f355431cea306841a70d163e32b2acad') // Berry data - await indexed(indexer, 'bfa5180e601e92af23d80782bf625b102ac110105a392e376fe7607e4e87dc8d') + await promise await indexer.stop() }) From 57f0194fd02d758e7cc6a54d564a2122a6f053ef Mon Sep 17 00:00:00 2001 From: hojarasca Date: Wed, 8 Dec 2021 09:55:36 -0300 Subject: [PATCH 16/82] Added new datasource --- src/data-sources/sqlite-datasource.js | 59 +++++++----- src/data-sources/sqlite-mixed-datasource.js | 100 ++++++++++++++++++++ src/database.js | 5 +- src/direct-server-worker.js | 4 +- src/index.js | 5 +- test/crawler.js | 4 +- test/indexer.js | 4 +- test/server.js | 44 +++++---- 8 files changed, 172 insertions(+), 53 deletions(-) create mode 100644 src/data-sources/sqlite-mixed-datasource.js diff --git a/src/data-sources/sqlite-datasource.js b/src/data-sources/sqlite-datasource.js index 09005d1..2798175 100644 --- a/src/data-sources/sqlite-datasource.js +++ b/src/data-sources/sqlite-datasource.js @@ -62,33 +62,17 @@ class SqliteDatasource { this.connection = null } - async setUp () { - this.logger.debug('Opening' + (this.readonly ? ' readonly' : '') + ' database') - if (this.connection) throw new Error('Database already open') - - this.connection = new Sqlite3Database(this.path, { readonly: this.readonly }) - - // 100MB cache - this.connection.pragma('cache_size = 6400') - this.connection.pragma('page_size = 16384') - - // WAL mode allows simultaneous readers - this.connection.pragma('journal_mode = WAL') - - // Synchronizes WAL at checkpoints - this.connection.pragma('synchronous = NORMAL') + async migrateSchema () { + await this.initializeV1() + await this.initializeV2() + await this.initializeV3() + await this.initializeV4() + await this.initializeV5() + await this.initializeV6() + await this.initializeV7() + } - if (!this.readonly) { - // Initialise and perform upgrades - await this.initializeV1() - await this.initializeV2() - await this.initializeV3() - await this.initializeV4() - await this.initializeV5() - await this.initializeV6() - await this.initializeV7() - } - // + prepareStatements () { this.addNewTransactionStmt = this.connection.prepare('INSERT OR IGNORE INTO tx (txid, height, time, bytes, has_code, executable, executed, indexed) VALUES (?, null, ?, null, 0, 0, 0, 0)') this.setTransactionBytesStmt = this.connection.prepare('UPDATE tx SET bytes = ? WHERE txid = ?') this.setTransactionExecutableStmt = this.connection.prepare('UPDATE tx SET executable = ? WHERE txid = ?') @@ -175,6 +159,29 @@ class SqliteDatasource { this.findAllExecutingTxidsStmt = this.connection.prepare('SELECT txid FROM executing') } + async setUp () { + this.logger.debug('Opening' + (this.readonly ? ' readonly' : '') + ' database') + if (this.connection) throw new Error('Database already open') + + this.connection = new Sqlite3Database(this.path, { readonly: this.readonly }) + + // 100MB cache + this.connection.pragma('cache_size = 6400') + this.connection.pragma('page_size = 16384') + + // WAL mode allows simultaneous readers + this.connection.pragma('journal_mode = WAL') + + // Synchronizes WAL at checkpoints + this.connection.pragma('synchronous = NORMAL') + + if (!this.readonly) { + await this.migrateSchema() + } + + this.prepareStatements() + } + async initializeV1 () { if (this.connection.pragma('user_version')[0].user_version !== 0) return diff --git a/src/data-sources/sqlite-mixed-datasource.js b/src/data-sources/sqlite-mixed-datasource.js new file mode 100644 index 0000000..1eaff50 --- /dev/null +++ b/src/data-sources/sqlite-mixed-datasource.js @@ -0,0 +1,100 @@ +/** + * database.js + * + * Layer between the database and the application + */ +const { SqliteDatasource } = require('./sqlite-datasource') +const fetch = require('node-fetch') + +// ------------------------------------------------------------------------------------------------ +// Database +// ------------------------------------------------------------------------------------------------ + +class SqliteMixedDatasource extends SqliteDatasource { + constructor (path, logger, readonly, baseApiUrl) { + super(path, logger, readonly) + this.baseApiUrl = baseApiUrl + } + + async extraSchemaMigrations () { + this.connection.exec(` + ALTER TABLE tx DROP COLUMN bytes; + `) + this.connection.exec(` + ALTER TABLE jig DROP COLUMN state; + `) + this.connection.exec(` + ALTER TABLE berry DROP COLUMN state; + `) + } + + async setUp () { + await super.setUp() + await this.extraSchemaMigrations() + this.getJigStateStmt = null + this.setJigStateStmt = null + this.getBerryStateStmt = null + this.setBerryStateStmt = null + } + + async getTxHex (txid) { + const row = this.getTransactionHexStmt.raw(true).get(txid) + return row && row[0] + } + + async setTxBytes (txid, bytes) { + this.setTransactionBytesStmt.run(bytes, txid) + } + + // jig + + async getJigState (location) { + return this._pullJigState(location) + } + + async setJigState (location, stateObject) { + await this._pushJigState(location, stateObject) + } + + async getBerryState (location) { + return this._pullJigState(location) + } + + async setBerryState (location, stateObject) { + return this._pushJigState(location, stateObject) + } + + async _pullJigState (location) { + const result = await fetch(`${this.baseApiUrl}/state/${location}`) + if (result.status === 404) { + return null + } + + if (!result.ok) { + throw new Error(`error fetching jig state ${location}`) + } + + return result.json() + } + + async _pushJigState (location, stateObject) { + const result = await fetch(`${this.baseApiUrl}/state`, { + method: 'POST', + body: JSON.stringify({ + location, + state: stateObject + }), + headers: { + 'Content-Type': 'application/json' + } + }) + + if (!result.ok) { + throw new Error(`Error saving jig state: ${location}`) + } + } +} + +// ------------------------------------------------------------------------------------------------ + +module.exports = { SqliteMixedDatasource } diff --git a/src/database.js b/src/database.js index 84e0053..56076db 100644 --- a/src/database.js +++ b/src/database.js @@ -6,7 +6,6 @@ const Run = require('run-sdk') const bsv = require('bsv') -const { SqliteDatasource } = require('./data-sources/sqlite-datasource') const { HEIGHT_MEMPOOL, HEIGHT_UNKNOWN } = require('./constants') // ------------------------------------------------------------------------------------------------ @@ -14,8 +13,8 @@ const { HEIGHT_MEMPOOL, HEIGHT_UNKNOWN } = require('./constants') // ------------------------------------------------------------------------------------------------ class Database { - constructor (path, logger, readonly = false) { - this.ds = new SqliteDatasource(path, logger, readonly) + constructor (datasource, logger) { + this.ds = datasource this.logger = logger this.onReadyToExecute = null diff --git a/src/direct-server-worker.js b/src/direct-server-worker.js index 7cc59c4..83c9134 100644 --- a/src/direct-server-worker.js +++ b/src/direct-server-worker.js @@ -8,6 +8,7 @@ const { parentPort, workerData } = require('worker_threads') const Server = require('./server') const Bus = require('./bus') const Database = require('./database') +const { SqliteDatasource } = require('./data-sources/sqlite-datasource') const logger = { info: (...args) => Bus.sendRequest(parentPort, 'info', ...args), @@ -17,7 +18,8 @@ const logger = { } const readonly = true -const database = new Database(workerData.dbPath, logger, readonly) +const ds = new SqliteDatasource(':memory:', logger, readonly) +const database = new Database(ds, logger) const server = new Server(database, logger, workerData.port) database.trust = (txid) => Bus.sendRequest(parentPort, 'trust', txid) diff --git a/src/index.js b/src/index.js index 7a32ce3..5691599 100644 --- a/src/index.js +++ b/src/index.js @@ -18,6 +18,7 @@ const BitcoinRpc = require('./bitcoin-rpc') const BitcoinZmq = require('./bitcoin-zmq') const Database = require('./database') const DirectServer = require('./direct-server') +const { SqliteDatasource } = require('./data-sources/sqlite-datasource') // ------------------------------------------------------------------------------------------------ // Globals @@ -49,7 +50,9 @@ switch (API) { } const readonly = !!SERVE_ONLY -const database = new Database(DB, logger, readonly) + +const dataSource = new SqliteDatasource(DB, logger, readonly) +const database = new Database(dataSource, logger) const indexer = new Indexer(database, api, NETWORK, FETCH_LIMIT, WORKERS, logger, START_HEIGHT, MEMPOOL_EXPIRATION, DEFAULT_TRUSTLIST) diff --git a/test/crawler.js b/test/crawler.js index 409ddf1..0124aa4 100644 --- a/test/crawler.js +++ b/test/crawler.js @@ -10,6 +10,7 @@ const Indexer = require('../src/indexer') const txns = require('./txns.json') const { DEFAULT_TRUSTLIST } = require('../src/config') const Database = require('../src/database') +const { SqliteDatasource } = require('../src/data-sources/sqlite-datasource') // ------------------------------------------------------------------------------------------------ // Globals @@ -20,7 +21,8 @@ const indexed = (indexer, txid) => new Promise((resolve) => { indexer.onIndex = const crawled = (indexer) => new Promise((resolve) => { indexer.onBlock = height => resolve(height) }) const reorged = (indexer) => new Promise((resolve) => { indexer.onReorg = newHeight => resolve(newHeight) }) const logger = { info: () => {}, warn: () => {}, error: () => {}, debug: () => {} } -const database = new Database(':memory:', logger, false) +const ds = new SqliteDatasource(':memory:', logger, false) +const database = new Database(ds, logger) beforeEach(() => database.open()) afterEach(() => database.close()) diff --git a/test/indexer.js b/test/indexer.js index 3c115ee..c2f4604 100644 --- a/test/indexer.js +++ b/test/indexer.js @@ -12,6 +12,7 @@ const Run = require('run-sdk') const { Jig } = Run const { DEFAULT_TRUSTLIST } = require('../src/config') const Database = require('../src/database') +const { SqliteDatasource } = require('../src/data-sources/sqlite-datasource') // ------------------------------------------------------------------------------------------------ // Globals @@ -22,7 +23,8 @@ const api = { fetch } const indexed = (indexer, txid) => new Promise((resolve) => { indexer.onIndex = x => txid === x && resolve() }) const failed = (indexer, txid) => new Promise((resolve) => { indexer.onFailToIndex = x => txid === x && resolve() }) const logger = { info: () => {}, warn: () => {}, error: () => {}, debug: () => {} } -const database = new Database(':memory:', logger, false) +const ds = new SqliteDatasource(':memory:', logger, false) +const database = new Database(ds, logger) beforeEach(() => database.open()) afterEach(() => database.close()) diff --git a/test/server.js b/test/server.js index b0f70e1..d612f56 100644 --- a/test/server.js +++ b/test/server.js @@ -13,6 +13,7 @@ const Server = require('../src/server') const txns = require('./txns.json') const { DEFAULT_TRUSTLIST } = require('../src/config') const Database = require('../src/database') +const { SqliteDatasource } = require('../src/data-sources/sqlite-datasource') // ------------------------------------------------------------------------------------------------ // Globals @@ -20,11 +21,12 @@ const Database = require('../src/database') const fetch = async txid => { return { hex: require('./txns.json')[txid] } } const api = { fetch } -const downloaded = (indexer, txid) => new Promise((resolve, reject) => { indexer.onDownload = x => txid === x && resolve() }) -const indexed = (indexer, txid) => new Promise((resolve, reject) => { indexer.onIndex = x => txid === x && resolve() }) -const listening = (server) => new Promise((resolve, reject) => { server.onListening = () => resolve() }) +const downloaded = (indexer, txid) => new Promise((resolve) => { indexer.onDownload = x => txid === x && resolve() }) +const indexed = (indexer, txid) => new Promise((resolve) => { indexer.onIndex = x => txid === x && resolve() }) +const listening = (server) => new Promise((resolve) => { server.onListening = () => resolve() }) const logger = { info: () => {}, warn: () => {}, error: () => {}, debug: () => {} } -const database = new Database(':memory:', logger, false) +const ds = new SqliteDatasource(':memory:', logger, false) +const database = new Database(ds, logger) beforeEach(() => database.open()) afterEach(() => database.close()) @@ -57,9 +59,7 @@ describe('Server', () => { // ------------------------------------------------------------------------ - it('does not throw if add with rawtx mismatch', async () => { - // Because the "POST /tx/:txid" endpoint is being deprecated we are not doing this - // checking anymore. The txid of the url is ignored. + it('throws if add with rawtx mismatch', async () => { const indexer = new Indexer(database, {}, 'main', 1, 1, logger, 0, Infinity, []) const server = new Server(database, logger, null) await indexer.start() @@ -68,7 +68,7 @@ describe('Server', () => { const txid = '3f9de452f0c3c96be737d42aa0941b27412211976688967adb3174ee18b04c64' const otherTxid = 'bfa5180e601e92af23d80782bf625b102ac110105a392e376fe7607e4e87dc8d' const options = { headers: { 'Content-Type': 'text/plain' } } - await expect(axios.post(`http://localhost:${server.port}/tx/${txid}`, txns[otherTxid], options)).to.be.fulfilled + await expect(axios.post(`http://localhost:${server.port}/tx/${txid}`, txns[otherTxid], options)).to.be.rejectedWith(Error) server.stop() await indexer.stop() }) @@ -110,8 +110,9 @@ describe('Server', () => { await indexer.start() server.start() await listening(server) - database.addTransaction('9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102') - await indexed(indexer, '9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102') + const promise = indexed(indexer, '9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102') + await database.addTransaction('9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102') + await promise const location = '9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102_o1' const state = (await axios.get(`http://localhost:${server.port}/jig/${location}`)).data expect(typeof state).to.equal('object') @@ -129,7 +130,7 @@ describe('Server', () => { server.start() await listening(server) const location = '9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102_o1' - await expect(axios.get(`http://localhost:${server.port}/jig/${location}`)).to.be.rejected + await expect(axios.get(`http://localhost:${server.port}/jig/${location}`)).to.be.rejectedWith(Error) try { await axios.get(`http://localhost:${server.port}/jig/${location}`) } catch (e) { @@ -170,7 +171,7 @@ describe('Server', () => { server.start() await listening(server) const location = '24cde3638a444c8ad397536127833878ffdfe1b04d5595489bd294e50d77105a_o1?berry=2f3492ef5401d887a93ca09820dff952f355431cea306841a70d163e32b2acad&version=5' - await expect(axios.get(`http://localhost:${server.port}/berry/${location}`)).to.be.rejected + await expect(axios.get(`http://localhost:${server.port}/berry/${location}`)).to.be.rejectedWith(Error) try { await axios.get(`http://localhost:${server.port}/berry/${location}`) } catch (e) { @@ -193,8 +194,9 @@ describe('Server', () => { server.start() await listening(server) const txid = '9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102' - database.addTransaction(txid) - await downloaded(indexer, txid) + const promise = downloaded(indexer, txid) + await database.addTransaction(txid) + await promise const rawtx = (await axios.get(`http://localhost:${server.port}/tx/${txid}`)).data expect(typeof rawtx).to.equal('string') expect(rawtx.length).to.equal(2074) @@ -211,7 +213,7 @@ describe('Server', () => { server.start() await listening(server) const txid = '9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102' - await expect(axios.get(`http://localhost:${server.port}/tx/${txid}`)).to.be.rejected + await expect(axios.get(`http://localhost:${server.port}/tx/${txid}`)).to.be.rejectedWith(Error) try { await axios.get(`http://localhost:${server.port}/tx/${txid}`) } catch (e) { @@ -231,7 +233,7 @@ describe('Server', () => { await listening(server) const txid = '1111111111111111111111111111111111111111111111111111111111111111' database.addTransaction(txid) - await expect(axios.get(`http://localhost:${server.port}/tx/${txid}`)).to.be.rejected + await expect(axios.get(`http://localhost:${server.port}/tx/${txid}`)).to.be.rejectedWith(Error) try { await axios.get(`http://localhost:${server.port}/tx/${txid}`) } catch (e) { @@ -253,8 +255,9 @@ describe('Server', () => { await indexer.start() server.start() await listening(server) - database.addTransaction('9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102') - await indexed(indexer, '9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102') + const promise = indexed(indexer, '9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102') + await database.addTransaction('9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102') + await promise const unspent = (await axios.get(`http://localhost:${server.port}/unspent`)).data expect(unspent.length).to.equal(3) expect(unspent.includes('3f9de452f0c3c96be737d42aa0941b27412211976688967adb3174ee18b04c64_o1')).to.equal(true) @@ -272,8 +275,9 @@ describe('Server', () => { await indexer.start() server.start() await listening(server) - database.addTransaction('9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102') - await indexed(indexer, '9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102') + const promise = indexed(indexer, '9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102') + await database.addTransaction('9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102') + await promise const address = '1Kc8XRNryDycwvfEQiFF2TZwD1CVhgwGy2' const unspent = (await axios.get(`http://localhost:${server.port}/unspent?address=${address}`)).data expect(unspent.length).to.equal(3) From c1736bbaabb34697117f17c800cb34f67fbc4513 Mon Sep 17 00:00:00 2001 From: hojarasca Date: Wed, 8 Dec 2021 11:41:43 -0300 Subject: [PATCH 17/82] added configuration to select data source. --- src/config.js | 6 ++- src/data-sources/sqlite-mixed-datasource.js | 42 +++++++++++++++++---- src/index.js | 14 ++++++- 3 files changed, 52 insertions(+), 10 deletions(-) diff --git a/src/config.js b/src/config.js index bc1a2fb..d798468 100644 --- a/src/config.js +++ b/src/config.js @@ -25,6 +25,8 @@ const ZMQ_URL = process.env.ZMQ_URL || null const RPC_URL = process.env.RPC_URL || null const DEBUG = process.env.DEBUG || false const SERVE_ONLY = process.env.SERVE_ONLY || false +const DATA_SOURCE = process.env.DATA_SOURCE || 'sqlite' +const DATA_API_ROOT = process.env.DATA_API_ROOT || null require('axios').default.defaults.timeout = TIMEOUT @@ -115,5 +117,7 @@ module.exports = { ZMQ_URL, RPC_URL, DEBUG, - SERVE_ONLY + SERVE_ONLY, + DATA_SOURCE, + DATA_API_ROOT } diff --git a/src/data-sources/sqlite-mixed-datasource.js b/src/data-sources/sqlite-mixed-datasource.js index 1eaff50..6c3078b 100644 --- a/src/data-sources/sqlite-mixed-datasource.js +++ b/src/data-sources/sqlite-mixed-datasource.js @@ -17,15 +17,29 @@ class SqliteMixedDatasource extends SqliteDatasource { } async extraSchemaMigrations () { - this.connection.exec(` - ALTER TABLE tx DROP COLUMN bytes; - `) - this.connection.exec(` + this.connection.prepare( + 'DROP INDEX IF EXISTS tx_downloaded_index;' + ).run() + + this.connection.prepare( + 'ALTER TABLE tx DROP COLUMN downloaded;' + ).run() + + this.connection.prepare( + 'ALTER TABLE tx DROP COLUMN bytes;' + ).run() + + this.connection.prepare( + 'ALTER TABLE tx ADD COLUMN downloaded INTEGER GENERATED ALWAYS AS (1) VIRTUAL' + ).run() + + this.connection.prepare(` ALTER TABLE jig DROP COLUMN state; - `) - this.connection.exec(` + `).run() + + this.connection.prepare(` ALTER TABLE berry DROP COLUMN state; - `) + `).run() } async setUp () { @@ -35,6 +49,10 @@ class SqliteMixedDatasource extends SqliteDatasource { this.setJigStateStmt = null this.getBerryStateStmt = null this.setBerryStateStmt = null + this.getTransactionsCountStmt = this.connection.prepare(` + select count(*) as count from tx; + `) + this.addNewTransactionStmt = this.connection.prepare('INSERT OR IGNORE INTO tx (txid, height, time, has_code, executable, executed, indexed) VALUES (?, null, ?, 0, 0, 0, 0)') } async getTxHex (txid) { @@ -93,6 +111,16 @@ class SqliteMixedDatasource extends SqliteDatasource { throw new Error(`Error saving jig state: ${location}`) } } + + // tx + + async checkTxIsDownloaded (_txid) { + return true + } + + async countDownloadedTxs () { + return this.getTransactionsCountStmt.get().count + } } // ------------------------------------------------------------------------------------------------ diff --git a/src/index.js b/src/index.js index 5691599..2b0f852 100644 --- a/src/index.js +++ b/src/index.js @@ -8,7 +8,7 @@ const Indexer = require('./indexer') const Server = require('./server') const { API, DB, NETWORK, PORT, FETCH_LIMIT, WORKERS, MATTERCLOUD_KEY, PLANARIA_TOKEN, START_HEIGHT, - MEMPOOL_EXPIRATION, ZMQ_URL, RPC_URL, DEFAULT_TRUSTLIST, DEBUG, SERVE_ONLY + MEMPOOL_EXPIRATION, ZMQ_URL, RPC_URL, DEFAULT_TRUSTLIST, DEBUG, SERVE_ONLY, DATA_SOURCE, DATA_API_ROOT } = require('./config') const MatterCloud = require('./mattercloud') const Planaria = require('./planaria') @@ -19,6 +19,8 @@ const BitcoinZmq = require('./bitcoin-zmq') const Database = require('./database') const DirectServer = require('./direct-server') const { SqliteDatasource } = require('./data-sources/sqlite-datasource') +const path = require('path') +const { SqliteMixedDatasource } = require('./data-sources/sqlite-mixed-datasource') // ------------------------------------------------------------------------------------------------ // Globals @@ -51,7 +53,15 @@ switch (API) { const readonly = !!SERVE_ONLY -const dataSource = new SqliteDatasource(DB, logger, readonly) +let dataSource +if (DATA_SOURCE === 'sqlite') { + dataSource = new SqliteDatasource(DB, logger, readonly) +} else if (DATA_SOURCE === 'mixed') { + dataSource = new SqliteMixedDatasource(DB, logger, readonly, DATA_API_ROOT) +} else { + throw new Error(`unknown datasource: ${DATA_SOURCE}. Please check "DATA_SOURCE" configuration.`) +} + const database = new Database(dataSource, logger) const indexer = new Indexer(database, api, NETWORK, FETCH_LIMIT, WORKERS, logger, From a2636556793299784222462ca2009879c696eb9c Mon Sep 17 00:00:00 2001 From: Miguel Duarte Date: Thu, 9 Dec 2021 19:43:56 -0300 Subject: [PATCH 18/82] Working with newer run plus some fixes. --- package-lock.json | 14 +- package.json | 2 +- src/data-sources/sqlite-mixed-datasource.js | 203 +++++++++++++++++--- src/database.js | 19 +- src/direct-server-worker.js | 15 +- src/index.js | 1 - src/worker.js | 1 + test/indexer.js | 3 +- 8 files changed, 207 insertions(+), 51 deletions(-) diff --git a/package-lock.json b/package-lock.json index 06ccd6b..b49dbb7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -21,7 +21,7 @@ "morgan": "^1.10.0", "node-fetch": "^2.6.1", "reconnecting-eventsource": "^1.1.0", - "run-sdk": "^0.6.18" + "run-sdk": "0.6.34" }, "devDependencies": { "chai": "^4.3.4", @@ -5092,9 +5092,9 @@ } }, "node_modules/run-sdk": { - "version": "0.6.29", - "resolved": "https://registry.npmjs.org/run-sdk/-/run-sdk-0.6.29.tgz", - "integrity": "sha512-rbSu7Uo6AIaTmhOpJrByAXa3GfCcGrZSFtSNTdya5yORuEwh9g2hefM6dDQxU/pFeADk+mC788XjOXZ6V1oyOA==", + "version": "0.6.34", + "resolved": "https://registry.npmjs.org/run-sdk/-/run-sdk-0.6.34.tgz", + "integrity": "sha512-deredB71NI/ndTCClj0OEnhZ9fiv+b8PToeJvPvkHSFeoYJBOQoe+EZwdceTCYFJ0XiScQnkpeiTqPXVgLDxyg==", "dependencies": { "bsv": "^1.5.6" } @@ -10231,9 +10231,9 @@ } }, "run-sdk": { - "version": "0.6.29", - "resolved": "https://registry.npmjs.org/run-sdk/-/run-sdk-0.6.29.tgz", - "integrity": "sha512-rbSu7Uo6AIaTmhOpJrByAXa3GfCcGrZSFtSNTdya5yORuEwh9g2hefM6dDQxU/pFeADk+mC788XjOXZ6V1oyOA==", + "version": "0.6.34", + "resolved": "https://registry.npmjs.org/run-sdk/-/run-sdk-0.6.34.tgz", + "integrity": "sha512-deredB71NI/ndTCClj0OEnhZ9fiv+b8PToeJvPvkHSFeoYJBOQoe+EZwdceTCYFJ0XiScQnkpeiTqPXVgLDxyg==", "requires": { "bsv": "^1.5.6" } diff --git a/package.json b/package.json index 688371f..b1fd725 100644 --- a/package.json +++ b/package.json @@ -43,7 +43,7 @@ "morgan": "^1.10.0", "node-fetch": "^2.6.1", "reconnecting-eventsource": "^1.1.0", - "run-sdk": "^0.6.18" + "run-sdk": "0.6.34" }, "optionalDependencies": { "zeromq": "^5.2.8" diff --git a/src/data-sources/sqlite-mixed-datasource.js b/src/data-sources/sqlite-mixed-datasource.js index 6c3078b..9a4b8c9 100644 --- a/src/data-sources/sqlite-mixed-datasource.js +++ b/src/data-sources/sqlite-mixed-datasource.js @@ -5,69 +5,204 @@ */ const { SqliteDatasource } = require('./sqlite-datasource') const fetch = require('node-fetch') +const { HEIGHT_MEMPOOL } = require('../constants') // ------------------------------------------------------------------------------------------------ // Database // ------------------------------------------------------------------------------------------------ +const IS_READY_TO_EXECUTE_SQL = ` + SELECT ( + downloaded = 1 + AND executable = 1 + AND executed = 0 + AND (has_code = 0 OR (SELECT COUNT(*) FROM trust WHERE trust.txid = tx.txid AND trust.value = 1) = 1) + AND txid NOT IN ban + AND ( + SELECT COUNT(*) + FROM tx AS tx2 + JOIN deps + ON deps.up = tx2.txid + WHERE deps.down = tx.txid + AND (+tx2.downloaded = 0 OR (tx2.executable = 1 AND tx2.executed = 0)) + ) = 0 + ) AS ready + FROM tx + WHERE txid = ? + ` + +const GET_DOWNSTREAM_READY_TO_EXECUTE_SQL = ` + SELECT down + FROM deps + JOIN tx + ON tx.txid = deps.down + WHERE up = ? + AND +downloaded = 1 + AND executable = 1 + AND executed = 0 + AND (has_code = 0 OR (SELECT COUNT(*) FROM trust WHERE trust.txid = tx.txid AND trust.value = 1) = 1) + AND txid NOT IN ban + AND ( + SELECT COUNT(*) + FROM tx AS tx2 + JOIN deps + ON deps.up = tx2.txid + WHERE deps.down = tx.txid + AND (+tx2.downloaded = 0 OR (tx2.executable = 1 AND tx2.executed = 0)) + ) = 0 + ` + class SqliteMixedDatasource extends SqliteDatasource { constructor (path, logger, readonly, baseApiUrl) { super(path, logger, readonly) this.baseApiUrl = baseApiUrl } - async extraSchemaMigrations () { - this.connection.prepare( - 'DROP INDEX IF EXISTS tx_downloaded_index;' - ).run() + prepareStatements () { + this.addNewTransactionStmt = this.connection.prepare('INSERT OR IGNORE INTO tx (txid, height, time, has_code, executable, executed, indexed) VALUES (?, null, ?, 0, 0, 0, 0)') + this.setTransactionExecutableStmt = this.connection.prepare('UPDATE tx SET executable = ? WHERE txid = ?') + this.setTransactionTimeStmt = this.connection.prepare('UPDATE tx SET time = ? WHERE txid = ?') + this.setTransactionHeightStmt = this.connection.prepare(`UPDATE tx SET height = ? WHERE txid = ? AND (height IS NULL OR height = ${HEIGHT_MEMPOOL})`) + this.setTransactionHasCodeStmt = this.connection.prepare('UPDATE tx SET has_code = ? WHERE txid = ?') + this.setTransactionExecutedStmt = this.connection.prepare('UPDATE tx SET executed = ? WHERE txid = ?') + this.setTransactionIndexedStmt = this.connection.prepare('UPDATE tx SET indexed = ? WHERE txid = ?') + this.txExistsStmt = this.connection.prepare('SELECT txid FROM tx WHERE txid = ?') + this.getTransactionTimeStmt = this.connection.prepare('SELECT time FROM tx WHERE txid = ?') + this.getTransactionHeightStmt = this.connection.prepare('SELECT height FROM tx WHERE txid = ?') + this.getTransactionHasCodeStmt = this.connection.prepare('SELECT has_code FROM tx WHERE txid = ?') + this.getTransactionIndexedStmt = this.connection.prepare('SELECT indexed FROM tx WHERE txid = ?') + this.getTransactionFailedStmt = this.connection.prepare('SELECT (executed = 1 AND indexed = 0) AS failed FROM tx WHERE txid = ?') + this.getTransactionDownloadedStmt = this.connection.prepare('SELECT downloaded FROM tx WHERE txid = ?') + this.deleteTransactionStmt = this.connection.prepare('DELETE FROM tx WHERE txid = ?') + this.unconfirmTransactionStmt = this.connection.prepare(`UPDATE tx SET height = ${HEIGHT_MEMPOOL} WHERE txid = ?`) + this.getTransactionsAboveHeightStmt = this.connection.prepare('SELECT txid FROM tx WHERE height > ?') + this.getMempoolTransactionsBeforeTimeStmt = this.connection.prepare(`SELECT txid FROM tx WHERE height = ${HEIGHT_MEMPOOL} AND time < ?`) + this.getTransactionsToDownloadStmt = this.connection.prepare('SELECT txid FROM tx WHERE downloaded = 0') + this.getTransactionsDownloadedCountStmt = this.connection.prepare('SELECT COUNT(*) AS count FROM tx WHERE downloaded = 1') + this.getTransactionsIndexedCountStmt = this.connection.prepare('SELECT COUNT(*) AS count FROM tx WHERE indexed = 1') + this.isReadyToExecuteStmt = this.connection.prepare(IS_READY_TO_EXECUTE_SQL) + this.getDownstreamReadyToExecuteStmt = this.connection.prepare(GET_DOWNSTREAM_READY_TO_EXECUTE_SQL) - this.connection.prepare( - 'ALTER TABLE tx DROP COLUMN downloaded;' - ).run() + this.setSpendStmt = this.connection.prepare('INSERT OR REPLACE INTO spends (location, spend_txid) VALUES (?, ?)') + this.setUnspentStmt = this.connection.prepare('INSERT OR IGNORE INTO spends (location, spend_txid) VALUES (?, null)') + this.getSpendStmt = this.connection.prepare('SELECT spend_txid FROM spends WHERE location = ?') + this.unspendOutputsStmt = this.connection.prepare('UPDATE spends SET spend_txid = null WHERE spend_txid = ?') + this.deleteSpendsStmt = this.connection.prepare('DELETE FROM spends WHERE location LIKE ? || \'%\'') + // + this.addDepStmt = this.connection.prepare('INSERT OR IGNORE INTO deps (up, down) VALUES (?, ?)') + this.deleteDepsStmt = this.connection.prepare('DELETE FROM deps WHERE down = ?') + this.getDownstreamStmt = this.connection.prepare('SELECT down FROM deps WHERE up = ?') + this.getUpstreamUnexecutedCodeStmt = this.connection.prepare(` + SELECT txdeps.txid as txid + FROM (SELECT up AS txid FROM deps WHERE down = ?) as txdeps + JOIN tx ON tx.txid = txdeps.txid + WHERE tx.executable = 1 AND tx.executed = 0 AND tx.has_code = 1 + `) - this.connection.prepare( - 'ALTER TABLE tx DROP COLUMN bytes;' - ).run() + this.setJigClassStmt = this.connection.prepare('UPDATE jig SET class = ? WHERE location = ?') + this.setJigLockStmt = this.connection.prepare('UPDATE jig SET lock = ? WHERE location = ?') + this.setJigScripthashStmt = this.connection.prepare('UPDATE jig SET scripthash = ? WHERE location = ?') + this.deleteJigStatesStmt = this.connection.prepare('DELETE FROM jig WHERE location LIKE ? || \'%\'') + + const getAllUnspentSql = ` + SELECT spends.location AS location FROM spends + JOIN jig ON spends.location = jig.location + WHERE spends.spend_txid IS NULL` + this.getAllUnspentStmt = this.connection.prepare(getAllUnspentSql) + this.getAllUnspentByClassStmt = this.connection.prepare(`${getAllUnspentSql} AND jig.class = ?`) + this.getAllUnspentByLockStmt = this.connection.prepare(`${getAllUnspentSql} AND jig.lock = ?`) + this.getAllUnspentByScripthashStmt = this.connection.prepare(`${getAllUnspentSql} AND jig.scripthash = ?`) + this.getAllUnspentByClassLockStmt = this.connection.prepare(`${getAllUnspentSql} AND jig.class = ? AND lock = ?`) + this.getAllUnspentByClassScripthashStmt = this.connection.prepare(`${getAllUnspentSql} AND jig.class = ? AND scripthash = ?`) + this.getAllUnspentByLockScripthashStmt = this.connection.prepare(`${getAllUnspentSql} AND jig.lock = ? AND scripthash = ?`) + this.getAllUnspentByClassLockScripthashStmt = this.connection.prepare(`${getAllUnspentSql} AND jig.class = ? AND jig.lock = ? AND scripthash = ?`) + this.getNumUnspentStmt = this.connection.prepare('SELECT COUNT(*) as unspent FROM spends JOIN jig ON spends.location = jig.location WHERE spends.spend_txid IS NULL') + + this.deleteBerryStatesStmt = this.connection.prepare('DELETE FROM berry WHERE location LIKE ? || \'%\'') + + this.setTrustedStmt = this.connection.prepare('INSERT OR REPLACE INTO trust (txid, value) VALUES (?, ?)') + this.getTrustlistStmt = this.connection.prepare('SELECT txid FROM trust WHERE value = 1') + this.isTrustedStmt = this.connection.prepare('SELECT COUNT(*) FROM trust WHERE txid = ? AND value = 1') + + this.banStmt = this.connection.prepare('INSERT OR REPLACE INTO ban (txid) VALUES (?)') + this.unbanStmt = this.connection.prepare('DELETE FROM ban WHERE txid = ?') + this.isBannedStmt = this.connection.prepare('SELECT COUNT(*) FROM ban WHERE txid = ?') + this.getBanlistStmt = this.connection.prepare('SELECT txid FROM ban') + + this.getHeightStmt = this.connection.prepare('SELECT value FROM crawl WHERE key = \'height\'') + this.getHashStmt = this.connection.prepare('SELECT value FROM crawl WHERE key = \'hash\'') + this.setHeightStmt = this.connection.prepare('UPDATE crawl SET value = ? WHERE key = \'height\'') + this.setHashStmt = this.connection.prepare('UPDATE crawl SET value = ? WHERE key = \'hash\'') + this.markExecutingStmt = this.connection.prepare('INSERT OR IGNORE INTO executing (txid) VALUES (?)') + this.unmarkExecutingStmt = this.connection.prepare('DELETE FROM executing WHERE txid = ?') + this.findAllExecutingTxidsStmt = this.connection.prepare('SELECT txid FROM executing') + + this.getTransactionsCountStmt = this.connection.prepare(` + select count(*) as count from tx; + `) + this.addNewTransactionStmt = this.connection.prepare('INSERT OR IGNORE INTO tx (txid, height, time, has_code, executable, executed, indexed) VALUES (?, null, ?, 0, 0, 0, 0)') + } + + async migrateSchema () { + await this.initializeV1() + await this.initializeV2() + await this.initializeV3() + await this.initializeV4() + await this.initializeV5() + await this.initializeV6() + await this.initializeV7() + await this.extraSchemaMigrations() + } + + async extraSchemaMigrations () { + this.logger.info('Setting up database extra migrations') this.connection.prepare( - 'ALTER TABLE tx ADD COLUMN downloaded INTEGER GENERATED ALWAYS AS (1) VIRTUAL' + 'DROP INDEX IF EXISTS tx_downloaded_index;' ).run() + const txColumns = this.connection.pragma('table_info(tx)') + + if (txColumns.some(column => column.name === 'bytes')) { + this.connection.prepare( + 'ALTER TABLE tx DROP COLUMN downloaded;' + ).run() + this.connection.prepare( + 'ALTER TABLE tx DROP COLUMN bytes;' + ).run() + this.connection.prepare( + 'ALTER TABLE tx ADD COLUMN downloaded INTEGER GENERATED ALWAYS AS (1) VIRTUAL' + ).run() + } - this.connection.prepare(` + const jigColumns = this.connection.pragma('table_info(jig)') + if (jigColumns.some(column => column.name === 'state')) { + this.connection.prepare(` ALTER TABLE jig DROP COLUMN state; `).run() + } - this.connection.prepare(` + const berryColumns = this.connection.pragma('table_info(berry)') + if (berryColumns.some(column => column.name === 'state')) { + this.connection.prepare(` ALTER TABLE berry DROP COLUMN state; `).run() - } - - async setUp () { - await super.setUp() - await this.extraSchemaMigrations() - this.getJigStateStmt = null - this.setJigStateStmt = null - this.getBerryStateStmt = null - this.setBerryStateStmt = null - this.getTransactionsCountStmt = this.connection.prepare(` - select count(*) as count from tx; - `) - this.addNewTransactionStmt = this.connection.prepare('INSERT OR IGNORE INTO tx (txid, height, time, has_code, executable, executed, indexed) VALUES (?, null, ?, 0, 0, 0, 0)') + } } async getTxHex (txid) { - const row = this.getTransactionHexStmt.raw(true).get(txid) - return row && row[0] + const buff = await this.fetchTx(txid) + return buff.toString('hex') } async setTxBytes (txid, bytes) { - this.setTransactionBytesStmt.run(bytes, txid) + return null } // jig async getJigState (location) { - return this._pullJigState(location) + const state = await this._pullJigState(location) + return state } async setJigState (location, stateObject) { @@ -92,7 +227,13 @@ class SqliteMixedDatasource extends SqliteDatasource { throw new Error(`error fetching jig state ${location}`) } - return result.json() + const json = await result.json() + return json.state + } + + async fetchTx (txid) { + const result = await fetch(`${this.baseApiUrl}/rawtx/${txid}`) + return result.buffer() } async _pushJigState (location, stateObject) { diff --git a/src/database.js b/src/database.js index 56076db..b83f4ba 100644 --- a/src/database.js +++ b/src/database.js @@ -66,18 +66,21 @@ class Database { if (time) { await this.setTransactionTime(txid, time) } }) - const downloaded = await this.isTransactionDownloaded(txid) - if (downloaded) return - - if (txhex) { - await this.parseAndStoreTransaction(txid, txhex) - } else { - if (this.onRequestDownload) { await this.onRequestDownload(txid) } + // const downloaded = await this.isTransactionIn(txid) + // if (downloaded) return + if (!txhex) { + txhex = await this.ds.getTxHex() } + await this.parseAndStoreTransaction(txid, txhex) + // if (txhex) { + // + // } else { + // if (this.onRequestDownload) { await this.onRequestDownload(txid) } + // } } async parseAndStoreTransaction (txid, hex) { - if (await this.isTransactionDownloaded(txid)) return + // if (await this.isTransactionDownloaded(txid)) return let metadata = null let bsvtx = null diff --git a/src/direct-server-worker.js b/src/direct-server-worker.js index 83c9134..4fe2e68 100644 --- a/src/direct-server-worker.js +++ b/src/direct-server-worker.js @@ -9,6 +9,8 @@ const Server = require('./server') const Bus = require('./bus') const Database = require('./database') const { SqliteDatasource } = require('./data-sources/sqlite-datasource') +const { DATA_SOURCE, DB, DATA_API_ROOT } = require('./config') +const { SqliteMixedDatasource } = require('./data-sources/sqlite-mixed-datasource') const logger = { info: (...args) => Bus.sendRequest(parentPort, 'info', ...args), @@ -18,8 +20,17 @@ const logger = { } const readonly = true -const ds = new SqliteDatasource(':memory:', logger, readonly) -const database = new Database(ds, logger) + +let dataSource +if (DATA_SOURCE === 'sqlite') { + dataSource = new SqliteDatasource(DB, logger, readonly) +} else if (DATA_SOURCE === 'mixed') { + dataSource = new SqliteMixedDatasource(DB, logger, readonly, DATA_API_ROOT) +} else { + throw new Error(`unknown datasource: ${DATA_SOURCE}. Please check "DATA_SOURCE" configuration.`) +} + +const database = new Database(dataSource, logger) const server = new Server(database, logger, workerData.port) database.trust = (txid) => Bus.sendRequest(parentPort, 'trust', txid) diff --git a/src/index.js b/src/index.js index 2b0f852..280973f 100644 --- a/src/index.js +++ b/src/index.js @@ -19,7 +19,6 @@ const BitcoinZmq = require('./bitcoin-zmq') const Database = require('./database') const DirectServer = require('./direct-server') const { SqliteDatasource } = require('./data-sources/sqlite-datasource') -const path = require('path') const { SqliteMixedDatasource } = require('./data-sources/sqlite-mixed-datasource') // ------------------------------------------------------------------------------------------------ diff --git a/src/worker.js b/src/worker.js index d89b31c..4f44b7b 100644 --- a/src/worker.js +++ b/src/worker.js @@ -75,6 +75,7 @@ const run = new Run({ network, logger: null }) async function execute (txid, hex, trustlist) { run.cache = new Cache() + run.state = new Run.plugins.LocalState() run.blockchain = new Blockchain(txid) run.timeout = 300000 run.client = false diff --git a/test/indexer.js b/test/indexer.js index c2f4604..51a3e4a 100644 --- a/test/indexer.js +++ b/test/indexer.js @@ -133,9 +133,10 @@ describe('Indexer', () => { it('remove discovered dep', async () => { const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, DEFAULT_TRUSTLIST) await indexer.start() + const promise = indexed(indexer, 'bfa5180e601e92af23d80782bf625b102ac110105a392e376fe7607e4e87dc8d') await database.addTransaction('bfa5180e601e92af23d80782bf625b102ac110105a392e376fe7607e4e87dc8d') // Class with berry image await database.trust('bfa5180e601e92af23d80782bf625b102ac110105a392e376fe7607e4e87dc8d') - await indexed(indexer, 'bfa5180e601e92af23d80782bf625b102ac110105a392e376fe7607e4e87dc8d') + await promise expect(await database.getTransactionHex('bfa5180e601e92af23d80782bf625b102ac110105a392e376fe7607e4e87dc8d')).not.to.equal(undefined) await database.deleteTransaction('2f3492ef5401d887a93ca09820dff952f355431cea306841a70d163e32b2acad') // Berry data expect(await database.getTransactionHex('bfa5180e601e92af23d80782bf625b102ac110105a392e376fe7607e4e87dc8d')).to.equal(undefined) From 6908164a9da778c27ac93a015a00d584b5df637e Mon Sep 17 00:00:00 2001 From: Miguel Duarte Date: Thu, 16 Dec 2021 12:59:35 -0300 Subject: [PATCH 19/82] Several little fixes --- package-lock.json | 31 ++++++++++ package.json | 1 + src/data-sources/sqlite-datasource.js | 68 ++++++++++++++------- src/data-sources/sqlite-mixed-datasource.js | 7 ++- src/database.js | 5 +- 5 files changed, 84 insertions(+), 28 deletions(-) diff --git a/package-lock.json b/package-lock.json index b49dbb7..623cd68 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,6 +17,7 @@ "event-stream": "^4.0.1", "eventsource": "^1.1.0", "express": "^4.17.1", + "fastq": "^1.13.0", "knex": "^0.95.14", "morgan": "^1.10.0", "node-fetch": "^2.6.1", @@ -2404,6 +2405,14 @@ "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=", "dev": true }, + "node_modules/fastq": { + "version": "1.13.0", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.13.0.tgz", + "integrity": "sha512-YpkpUnK8od0o1hmeSc7UUs/eB/vIPWJYjKck2QKIzAf71Vm1AAQ3EbuZB3g2JIy+pg+ERD0vqI79KyZiB2e2Nw==", + "dependencies": { + "reusify": "^1.0.4" + } + }, "node_modules/file-entry-cache": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-5.0.1.tgz", @@ -5079,6 +5088,15 @@ "node": ">=8" } }, + "node_modules/reusify": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", + "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==", + "engines": { + "iojs": ">=1.0.0", + "node": ">=0.10.0" + } + }, "node_modules/rimraf": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", @@ -8127,6 +8145,14 @@ "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=", "dev": true }, + "fastq": { + "version": "1.13.0", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.13.0.tgz", + "integrity": "sha512-YpkpUnK8od0o1hmeSc7UUs/eB/vIPWJYjKck2QKIzAf71Vm1AAQ3EbuZB3g2JIy+pg+ERD0vqI79KyZiB2e2Nw==", + "requires": { + "reusify": "^1.0.4" + } + }, "file-entry-cache": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-5.0.1.tgz", @@ -10221,6 +10247,11 @@ "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz", "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==" }, + "reusify": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", + "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==" + }, "rimraf": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", diff --git a/package.json b/package.json index b1fd725..303de25 100644 --- a/package.json +++ b/package.json @@ -39,6 +39,7 @@ "event-stream": "^4.0.1", "eventsource": "^1.1.0", "express": "^4.17.1", + "fastq": "^1.13.0", "knex": "^0.95.14", "morgan": "^1.10.0", "node-fetch": "^2.6.1", diff --git a/src/data-sources/sqlite-datasource.js b/src/data-sources/sqlite-datasource.js index 2798175..a63c1d7 100644 --- a/src/data-sources/sqlite-datasource.js +++ b/src/data-sources/sqlite-datasource.js @@ -5,6 +5,7 @@ */ const Sqlite3Database = require('better-sqlite3') const { HEIGHT_MEMPOOL } = require('../constants') +const fastq = require('fastq') // The + in the following 2 queries before downloaded improves performance by NOT using the // tx_downloaded index, which is rarely an improvement over a simple filter for single txns. @@ -60,6 +61,21 @@ class SqliteDatasource { this.logger = logger this.readonly = readonly this.connection = null + + const worker = async (fn, cb) => { + try { + this.connection.exec('begin;') + const result = await fn() + this.connection.exec('commit;') + cb(result) + } catch (e) { + this.connection.exec('rollback;') + console.error(e) + throw e + } + } + + this.txQueue = fastq(worker, 1) } async migrateSchema () { @@ -116,6 +132,7 @@ class SqliteDatasource { `) // this.setJigStateStmt = this.connection.prepare('INSERT OR IGNORE INTO jig (location, state, class, lock, scripthash) VALUES (?, ?, null, null, null)') + this.setJigMetadataStmt = this.connection.prepare('INSERT OR IGNORE INTO jig (location) VALUES (?)') this.setJigClassStmt = this.connection.prepare('UPDATE jig SET class = ? WHERE location = ?') this.setJigLockStmt = this.connection.prepare('UPDATE jig SET lock = ? WHERE location = ?') this.setJigScripthashStmt = this.connection.prepare('UPDATE jig SET scripthash = ? WHERE location = ?') @@ -137,6 +154,7 @@ class SqliteDatasource { this.getNumUnspentStmt = this.connection.prepare('SELECT COUNT(*) as unspent FROM spends JOIN jig ON spends.location = jig.location WHERE spends.spend_txid IS NULL') this.setBerryStateStmt = this.connection.prepare('INSERT OR IGNORE INTO berry (location, state) VALUES (?, ?)') + this.setBerryMetadataStmt = this.connection.prepare('INSERT OR IGNORE INTO berry (location) VALUES (?)') this.getBerryStateStmt = this.connection.prepare('SELECT state FROM berry WHERE location = ?') this.deleteBerryStatesStmt = this.connection.prepare('DELETE FROM berry WHERE location LIKE ? || \'%\'') @@ -349,7 +367,6 @@ class SqliteDatasource { await this.performOnTransaction(() => { this.connection.pragma('user_version = 4') - this.connection.prepare('ALTER TABLE tx ADD COLUMN downloaded INTEGER GENERATED ALWAYS AS (bytes IS NOT NULL) VIRTUAL').run() this.connection.prepare('CREATE INDEX IF NOT EXISTS tx_downloaded_index ON tx (downloaded)').run() @@ -449,28 +466,25 @@ class SqliteDatasource { async performOnTransaction (fn) { if (!this.connection) return - try { - this.connection.exec('begin;') - await fn() - this.connection.exec('commit;') - } catch (e) { - this.connection.exec('rollback;') - console.error(e) - throw e - } - } - - async transaction (f) { - if (!this.connection) return - try { - this.connection.exec('begin;') - await f() - this.connection.exec('commit;') - } catch (e) { - this.connection.exec('rollback;') - console.error(e) - throw e - } + return new Promise((resolve, reject) => { + this.txQueue.push(fn, (err, result) => { + if (err) { + reject(err) + } else { + resolve(result) + } + }) + }) + // if (!this.connection) return + // try { + // this.connection.exec('begin;') + // await fn() + // this.connection.exec('commit;') + // } catch (e) { + // this.connection.exec('rollback;') + // console.error(e) + // throw e + // } } async txExists (txid) { @@ -630,6 +644,10 @@ class SqliteDatasource { // jig + async setJigMetadata (location) { + this.setJigMetadataStmt.run(location) + } + async getJigState (location) { const row = this.getJigStateStmt.raw(true).get(location) if (row && row[0]) { @@ -647,6 +665,10 @@ class SqliteDatasource { this.setBerryStateStmt.run(location, JSON.stringify(stateObject)) } + async setBerryMetadata (location) { + this.setBerryMetadataStmt.run(location) + } + async getBerryState (location) { const row = this.getBerryStateStmt.raw(true).get(location) if (row && row[0]) { diff --git a/src/data-sources/sqlite-mixed-datasource.js b/src/data-sources/sqlite-mixed-datasource.js index 9a4b8c9..77788e1 100644 --- a/src/data-sources/sqlite-mixed-datasource.js +++ b/src/data-sources/sqlite-mixed-datasource.js @@ -72,7 +72,7 @@ class SqliteMixedDatasource extends SqliteDatasource { this.getTransactionHasCodeStmt = this.connection.prepare('SELECT has_code FROM tx WHERE txid = ?') this.getTransactionIndexedStmt = this.connection.prepare('SELECT indexed FROM tx WHERE txid = ?') this.getTransactionFailedStmt = this.connection.prepare('SELECT (executed = 1 AND indexed = 0) AS failed FROM tx WHERE txid = ?') - this.getTransactionDownloadedStmt = this.connection.prepare('SELECT downloaded FROM tx WHERE txid = ?') + // this.getTransactionDownloadedStmt = this.connection.prepare('SELECT downloaded FROM tx WHERE txid = ?') this.deleteTransactionStmt = this.connection.prepare('DELETE FROM tx WHERE txid = ?') this.unconfirmTransactionStmt = this.connection.prepare(`UPDATE tx SET height = ${HEIGHT_MEMPOOL} WHERE txid = ?`) this.getTransactionsAboveHeightStmt = this.connection.prepare('SELECT txid FROM tx WHERE height > ?') @@ -99,6 +99,7 @@ class SqliteMixedDatasource extends SqliteDatasource { WHERE tx.executable = 1 AND tx.executed = 0 AND tx.has_code = 1 `) + this.setJigMetadataStmt = this.connection.prepare('INSERT OR IGNORE INTO jig (location) VALUES (?)') this.setJigClassStmt = this.connection.prepare('UPDATE jig SET class = ? WHERE location = ?') this.setJigLockStmt = this.connection.prepare('UPDATE jig SET lock = ? WHERE location = ?') this.setJigScripthashStmt = this.connection.prepare('UPDATE jig SET scripthash = ? WHERE location = ?') @@ -119,6 +120,7 @@ class SqliteMixedDatasource extends SqliteDatasource { this.getNumUnspentStmt = this.connection.prepare('SELECT COUNT(*) as unspent FROM spends JOIN jig ON spends.location = jig.location WHERE spends.spend_txid IS NULL') this.deleteBerryStatesStmt = this.connection.prepare('DELETE FROM berry WHERE location LIKE ? || \'%\'') + this.setBerryMetadataStmt = this.connection.prepare('INSERT OR IGNORE INTO berry (location) VALUES (?)') this.setTrustedStmt = this.connection.prepare('INSERT OR REPLACE INTO trust (txid, value) VALUES (?, ?)') this.getTrustlistStmt = this.connection.prepare('SELECT txid FROM trust WHERE value = 1') @@ -201,8 +203,7 @@ class SqliteMixedDatasource extends SqliteDatasource { // jig async getJigState (location) { - const state = await this._pullJigState(location) - return state + return this._pullJigState(location) } async setJigState (location, stateObject) { diff --git a/src/database.js b/src/database.js index b83f4ba..fba446c 100644 --- a/src/database.js +++ b/src/database.js @@ -69,7 +69,7 @@ class Database { // const downloaded = await this.isTransactionIn(txid) // if (downloaded) return if (!txhex) { - txhex = await this.ds.getTxHex() + txhex = await this.ds.getTxHex(txid) } await this.parseAndStoreTransaction(txid, txhex) // if (txhex) { @@ -222,6 +222,7 @@ class Database { if (key.startsWith('jig://')) { const location = key.slice('jig://'.length) await this.ds.setJigState(location, cache[key]) + await this.ds.setJigMetadata(location) } else if (key.startsWith('berry://')) { const location = key.slice('berry://'.length) await this.ds.setBerryState(location, cache[key]) @@ -233,7 +234,7 @@ class Database { } for (const [location, lock] of locks) { - await this.ds.setJigLockStmt(location, lock) + await this.ds.setJigLock(location, lock) } for (const [location, scripthash] of scripthashes) { From bc7058b7dbc64e8b84b55b5874695abd318cbd49 Mon Sep 17 00:00:00 2001 From: Miguel Duarte Date: Fri, 17 Dec 2021 11:23:42 -0300 Subject: [PATCH 20/82] Removed start script to different file. --- package.json | 2 +- src/index.js | 123 ++++++++------------------------------------------ src/start.js | 118 ++++++++++++++++++++++++++++++++++++++++++++++++ src/worker.js | 67 ++++++++++++++++++++++++--- 4 files changed, 200 insertions(+), 110 deletions(-) create mode 100644 src/start.js diff --git a/package.json b/package.json index 303de25..b484f65 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,7 @@ "main": "src/index.js", "scripts": { "lint": "standard --fix", - "start": "node --experimental-worker .", + "start": "node --experimental-worker src/start.js", "test": "node --experimental-worker node_modules/mocha/bin/mocha", "test:cover": "nyc -r=text -r=lcovonly -x=test/** node --experimental-worker node_modules/mocha/bin/mocha", "bump": "npm version patch && git push --follow-tags && npm publish" diff --git a/src/index.js b/src/index.js index 280973f..159157b 100644 --- a/src/index.js +++ b/src/index.js @@ -4,115 +4,32 @@ * Entry point */ -const Indexer = require('./indexer') -const Server = require('./server') -const { - API, DB, NETWORK, PORT, FETCH_LIMIT, WORKERS, MATTERCLOUD_KEY, PLANARIA_TOKEN, START_HEIGHT, - MEMPOOL_EXPIRATION, ZMQ_URL, RPC_URL, DEFAULT_TRUSTLIST, DEBUG, SERVE_ONLY, DATA_SOURCE, DATA_API_ROOT -} = require('./config') -const MatterCloud = require('./mattercloud') -const Planaria = require('./planaria') -const RunConnectFetcher = require('./run-connect') const BitcoinNodeConnection = require('./bitcoin-node-connection') const BitcoinRpc = require('./bitcoin-rpc') const BitcoinZmq = require('./bitcoin-zmq') const Database = require('./database') const DirectServer = require('./direct-server') +const Indexer = require('./indexer') +const MatterCloud = require('./mattercloud') +const Planaria = require('./planaria') +const RunConnectFetcher = require('./run-connect') +const Server = require('./server') +const config = require('./config') const { SqliteDatasource } = require('./data-sources/sqlite-datasource') const { SqliteMixedDatasource } = require('./data-sources/sqlite-mixed-datasource') -// ------------------------------------------------------------------------------------------------ -// Globals -// ------------------------------------------------------------------------------------------------ - -const logger = {} -logger.info = console.info.bind(console) -logger.warn = console.warn.bind(console) -logger.error = console.error.bind(console) -logger.debug = DEBUG ? console.debug.bind(console) : () => {} - -let api = null -switch (API) { - case 'mattercloud': api = new MatterCloud(MATTERCLOUD_KEY, logger); break - case 'planaria': api = new Planaria(PLANARIA_TOKEN, logger); break - case 'bitcoin-node': - if (ZMQ_URL === null) { - throw new Error('please specify ZQM_URL when using bitcoin-node API') - } - - if (RPC_URL === null) { - throw new Error('please specify RPC_URL when using bitcoin-node API') - } - api = new BitcoinNodeConnection(new BitcoinZmq(ZMQ_URL), new BitcoinRpc(RPC_URL)) - break - case 'run': api = new RunConnectFetcher(); break - case 'none': api = {}; break - default: throw new Error(`Unknown API: ${API}`) -} - -const readonly = !!SERVE_ONLY - -let dataSource -if (DATA_SOURCE === 'sqlite') { - dataSource = new SqliteDatasource(DB, logger, readonly) -} else if (DATA_SOURCE === 'mixed') { - dataSource = new SqliteMixedDatasource(DB, logger, readonly, DATA_API_ROOT) -} else { - throw new Error(`unknown datasource: ${DATA_SOURCE}. Please check "DATA_SOURCE" configuration.`) -} - -const database = new Database(dataSource, logger) - -const indexer = new Indexer(database, api, NETWORK, FETCH_LIMIT, WORKERS, logger, - START_HEIGHT, MEMPOOL_EXPIRATION, DEFAULT_TRUSTLIST) - -const server = SERVE_ONLY - ? new Server(database, logger, PORT) - : new DirectServer(DB, PORT, logger, database) - -let started = false - -// ------------------------------------------------------------------------------------------------ -// main -// ------------------------------------------------------------------------------------------------ - -async function main () { - await database.open() - - if (!SERVE_ONLY) { - await indexer.start() - } - - await server.start() - - started = true +module.exports = { + config, + BitcoinNodeConnection, + BitcoinRpc, + BitcoinZmq, + Database, + DirectServer, + Indexer, + MatterCloud, + Planaria, + RunConnectFetcher, + Server, + SqliteDatasource, + SqliteMixedDatasource } - -// ------------------------------------------------------------------------------------------------ -// shutdown -// ------------------------------------------------------------------------------------------------ - -async function shutdown () { - if (!started) return - - logger.debug('Shutting down') - - started = false - - await server.stop() - - if (!SERVE_ONLY) { - await indexer.stop() - } - - await database.close() - - process.exit(0) -} - -// ------------------------------------------------------------------------------------------------ - -process.on('SIGTERM', shutdown) -process.on('SIGINT', shutdown) - -main() diff --git a/src/start.js b/src/start.js new file mode 100644 index 0000000..280973f --- /dev/null +++ b/src/start.js @@ -0,0 +1,118 @@ +/** + * index.js + * + * Entry point + */ + +const Indexer = require('./indexer') +const Server = require('./server') +const { + API, DB, NETWORK, PORT, FETCH_LIMIT, WORKERS, MATTERCLOUD_KEY, PLANARIA_TOKEN, START_HEIGHT, + MEMPOOL_EXPIRATION, ZMQ_URL, RPC_URL, DEFAULT_TRUSTLIST, DEBUG, SERVE_ONLY, DATA_SOURCE, DATA_API_ROOT +} = require('./config') +const MatterCloud = require('./mattercloud') +const Planaria = require('./planaria') +const RunConnectFetcher = require('./run-connect') +const BitcoinNodeConnection = require('./bitcoin-node-connection') +const BitcoinRpc = require('./bitcoin-rpc') +const BitcoinZmq = require('./bitcoin-zmq') +const Database = require('./database') +const DirectServer = require('./direct-server') +const { SqliteDatasource } = require('./data-sources/sqlite-datasource') +const { SqliteMixedDatasource } = require('./data-sources/sqlite-mixed-datasource') + +// ------------------------------------------------------------------------------------------------ +// Globals +// ------------------------------------------------------------------------------------------------ + +const logger = {} +logger.info = console.info.bind(console) +logger.warn = console.warn.bind(console) +logger.error = console.error.bind(console) +logger.debug = DEBUG ? console.debug.bind(console) : () => {} + +let api = null +switch (API) { + case 'mattercloud': api = new MatterCloud(MATTERCLOUD_KEY, logger); break + case 'planaria': api = new Planaria(PLANARIA_TOKEN, logger); break + case 'bitcoin-node': + if (ZMQ_URL === null) { + throw new Error('please specify ZQM_URL when using bitcoin-node API') + } + + if (RPC_URL === null) { + throw new Error('please specify RPC_URL when using bitcoin-node API') + } + api = new BitcoinNodeConnection(new BitcoinZmq(ZMQ_URL), new BitcoinRpc(RPC_URL)) + break + case 'run': api = new RunConnectFetcher(); break + case 'none': api = {}; break + default: throw new Error(`Unknown API: ${API}`) +} + +const readonly = !!SERVE_ONLY + +let dataSource +if (DATA_SOURCE === 'sqlite') { + dataSource = new SqliteDatasource(DB, logger, readonly) +} else if (DATA_SOURCE === 'mixed') { + dataSource = new SqliteMixedDatasource(DB, logger, readonly, DATA_API_ROOT) +} else { + throw new Error(`unknown datasource: ${DATA_SOURCE}. Please check "DATA_SOURCE" configuration.`) +} + +const database = new Database(dataSource, logger) + +const indexer = new Indexer(database, api, NETWORK, FETCH_LIMIT, WORKERS, logger, + START_HEIGHT, MEMPOOL_EXPIRATION, DEFAULT_TRUSTLIST) + +const server = SERVE_ONLY + ? new Server(database, logger, PORT) + : new DirectServer(DB, PORT, logger, database) + +let started = false + +// ------------------------------------------------------------------------------------------------ +// main +// ------------------------------------------------------------------------------------------------ + +async function main () { + await database.open() + + if (!SERVE_ONLY) { + await indexer.start() + } + + await server.start() + + started = true +} + +// ------------------------------------------------------------------------------------------------ +// shutdown +// ------------------------------------------------------------------------------------------------ + +async function shutdown () { + if (!started) return + + logger.debug('Shutting down') + + started = false + + await server.stop() + + if (!SERVE_ONLY) { + await indexer.stop() + } + + await database.close() + + process.exit(0) +} + +// ------------------------------------------------------------------------------------------------ + +process.on('SIGTERM', shutdown) +process.on('SIGINT', shutdown) + +main() diff --git a/src/worker.js b/src/worker.js index 4f44b7b..67f37d7 100644 --- a/src/worker.js +++ b/src/worker.js @@ -9,6 +9,10 @@ const crypto = require('crypto') const Run = require('run-sdk') const bsv = require('bsv') const Bus = require('./bus') +const config = require('./config') +const Database = require('./database') +const { SqliteMixedDatasource } = require('./data-sources/sqlite-mixed-datasource') +const { DEBUG } = require('./config') // ------------------------------------------------------------------------------------------------ // Startup @@ -47,6 +51,39 @@ class Cache { } } +class DirectCache { + constructor (db) { + this.db = db + this.state = new Map() + } + + async get (key) { + const value = this.state[key] + if (value) { return value } + + const [type, identifier] = key.split('://') + if (type === 'jig') { + const jig = await this.db.getJigState(identifier) + this.state[key] = jig + return jig + } else if (type === 'tx') { + const txHex = await this.db.getTxHex(identifier) + this.state[key] = txHex + return txHex + } else if (type === 'berry') { + const berry = await this.db.getBerryState(identifier) + this.state[key] = berry + return berry + } else { + return null + } + } + + async set () { + + } +} + // ------------------------------------------------------------------------------------------------ // Blockchain // ------------------------------------------------------------------------------------------------ @@ -54,11 +91,11 @@ class Cache { class Blockchain { constructor (txid) { this.txid = txid } get network () { return network } - async broadcast (hex) { return this.txid } + async broadcast (_hex) { return this.txid } async fetch (txid) { return await Bus.sendRequest(parentPort, 'blockchainFetch', txid) } - async utxos (script) { throw new Error('not implemented: utxos') } - async spends (txid, vout) { throw new Error('not implemented: spends') } - async time (txid) { throw new Error('not implemented: time') } + async utxos (_script) { throw new Error('not implemented: utxos') } + async spends (_txid, _vout) { throw new Error('not implemented: spends') } + async time (_txid) { throw new Error('not implemented: time') } } // ------------------------------------------------------------------------------------------------ @@ -73,14 +110,32 @@ const scripthash = x => crypto.createHash('sha256').update(Buffer.from(x, 'hex') const run = new Run({ network, logger: null }) +const logger = {} +logger.info = console.info.bind(console) +logger.warn = console.warn.bind(console) +logger.error = console.error.bind(console) +logger.debug = DEBUG ? console.debug.bind(console) : () => {} + async function execute (txid, hex, trustlist) { - run.cache = new Cache() + if (config.DATA_SOURCE === 'mixed') { + const ds = new SqliteMixedDatasource(config.DB, logger, false, config.DATA_API_ROOT) + const db = new Database(ds, logger) + run.cache = new DirectCache(db) + } else { + run.cache = new Cache() + } + run.state = new Run.plugins.LocalState() run.blockchain = new Blockchain(txid) run.timeout = 300000 run.client = false run.preverify = false - trustlist.forEach(txid => run.trust(txid)) + + if (config.DATA_SOURCE === 'mixed') { + run.trust('*') + } else { + trustlist.forEach(txid => run.trust(txid)) + } run.trust('cache') const tx = await run.import(hex, { txid }) From 97878c3b1431524e4045e470b9bb2d93a07c7fa0 Mon Sep 17 00:00:00 2001 From: Miguel Duarte Date: Mon, 20 Dec 2021 14:56:59 -0300 Subject: [PATCH 21/82] New logic working --- .mocharv.yaml | 3 + package-lock.json | 33 ++++++++--- package.json | 1 + src/clock.js | 21 +++++++ src/config.js | 6 +- src/crawler.js | 2 +- src/data-sources/sqlite-mixed-datasource.js | 2 +- src/database.js | 23 ++++---- src/direct-server.js | 2 +- src/executor.js | 8 ++- src/indexer.js | 6 +- src/server.js | 3 +- src/start.js | 8 ++- src/threading/parent-thread.js | 37 +++++++++++++ src/threading/sub-thread.js | 61 +++++++++++++++++++++ src/worker.js | 21 ++++--- test/{crawler.js => crawler.test.js} | 2 +- test/index.js | 6 +- test/{indexer.js => indexer.test.js} | 28 +++++----- test/{server.js => server.test.js} | 8 ++- test/thread.test.js | 7 +++ 21 files changed, 230 insertions(+), 58 deletions(-) create mode 100644 .mocharv.yaml create mode 100644 src/clock.js create mode 100644 src/threading/parent-thread.js create mode 100644 src/threading/sub-thread.js rename test/{crawler.js => crawler.test.js} (99%) rename test/{indexer.js => indexer.test.js} (92%) rename test/{server.js => server.test.js} (99%) create mode 100644 test/thread.test.js diff --git a/.mocharv.yaml b/.mocharv.yaml new file mode 100644 index 0000000..184f5e7 --- /dev/null +++ b/.mocharv.yaml @@ -0,0 +1,3 @@ +require: test/setup.js +recursive: true +extension: [".test.js"] diff --git a/package-lock.json b/package-lock.json index 623cd68..d9ccb31 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20,6 +20,7 @@ "fastq": "^1.13.0", "knex": "^0.95.14", "morgan": "^1.10.0", + "nanoid": "^3.1.30", "node-fetch": "^2.6.1", "reconnecting-eventsource": "^1.1.0", "run-sdk": "0.6.34" @@ -3836,6 +3837,18 @@ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "dev": true }, + "node_modules/mocha/node_modules/nanoid": { + "version": "3.1.20", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.1.20.tgz", + "integrity": "sha512-a1cQNyczgKbLX9jwbS/+d7W8fX/RfgYR7lVWwWOGIPNgK2m0MWvrGF6/m4kk6U3QcFMnZf3RIhL0v2Jgh/0Uxw==", + "dev": true, + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, "node_modules/mocha/node_modules/strip-json-comments": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", @@ -3880,10 +3893,9 @@ "optional": true }, "node_modules/nanoid": { - "version": "3.1.20", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.1.20.tgz", - "integrity": "sha512-a1cQNyczgKbLX9jwbS/+d7W8fX/RfgYR7lVWwWOGIPNgK2m0MWvrGF6/m4kk6U3QcFMnZf3RIhL0v2Jgh/0Uxw==", - "dev": true, + "version": "3.1.30", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.1.30.tgz", + "integrity": "sha512-zJpuPDwOv8D2zq2WRoMe1HsfZthVewpel9CAvTfc/2mBD1uUT/agc5f7GHGWXlYkFvi1mVxe4IjvP2HNrop7nQ==", "bin": { "nanoid": "bin/nanoid.cjs" }, @@ -9261,6 +9273,12 @@ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "dev": true }, + "nanoid": { + "version": "3.1.20", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.1.20.tgz", + "integrity": "sha512-a1cQNyczgKbLX9jwbS/+d7W8fX/RfgYR7lVWwWOGIPNgK2m0MWvrGF6/m4kk6U3QcFMnZf3RIhL0v2Jgh/0Uxw==", + "dev": true + }, "strip-json-comments": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", @@ -9300,10 +9318,9 @@ "optional": true }, "nanoid": { - "version": "3.1.20", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.1.20.tgz", - "integrity": "sha512-a1cQNyczgKbLX9jwbS/+d7W8fX/RfgYR7lVWwWOGIPNgK2m0MWvrGF6/m4kk6U3QcFMnZf3RIhL0v2Jgh/0Uxw==", - "dev": true + "version": "3.1.30", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.1.30.tgz", + "integrity": "sha512-zJpuPDwOv8D2zq2WRoMe1HsfZthVewpel9CAvTfc/2mBD1uUT/agc5f7GHGWXlYkFvi1mVxe4IjvP2HNrop7nQ==" }, "napi-build-utils": { "version": "1.0.2", diff --git a/package.json b/package.json index b484f65..4666cdf 100644 --- a/package.json +++ b/package.json @@ -42,6 +42,7 @@ "fastq": "^1.13.0", "knex": "^0.95.14", "morgan": "^1.10.0", + "nanoid": "^3.1.30", "node-fetch": "^2.6.1", "reconnecting-eventsource": "^1.1.0", "run-sdk": "0.6.34" diff --git a/src/clock.js b/src/clock.js new file mode 100644 index 0000000..018e1c3 --- /dev/null +++ b/src/clock.js @@ -0,0 +1,21 @@ +class DelayedTask { + constructor (fn, timeMs) { + this.timeout = setTimeout(fn, timeMs) + } + + cancel () { + clearTimeout(this.timeout) + } +} + +class Clock { + now () { + return new Date() + } + + delay (callback, timeMs) { + return new DelayedTask(callback, timeMs) + } +} + +module.exports = { Clock } \ No newline at end of file diff --git a/src/config.js b/src/config.js index d798468..075ab5a 100644 --- a/src/config.js +++ b/src/config.js @@ -27,6 +27,8 @@ const DEBUG = process.env.DEBUG || false const SERVE_ONLY = process.env.SERVE_ONLY || false const DATA_SOURCE = process.env.DATA_SOURCE || 'sqlite' const DATA_API_ROOT = process.env.DATA_API_ROOT || null +const WORKER_TRUST_SOURCE = process.env.WORKER_TRUST_SOURCE || 'db' +const WORKER_CACHE_TYPE = process.env.WORKER_CACHE_TYPE || 'parent' require('axios').default.defaults.timeout = TIMEOUT @@ -119,5 +121,7 @@ module.exports = { DEBUG, SERVE_ONLY, DATA_SOURCE, - DATA_API_ROOT + DATA_API_ROOT, + WORKER_TRUST_SOURCE, + WORKER_CACHE_TYPE } diff --git a/src/crawler.js b/src/crawler.js index 16e9ab6..8dc68b8 100644 --- a/src/crawler.js +++ b/src/crawler.js @@ -1,5 +1,5 @@ /** - * crawler.js + * crawler.test.js * * Generic blockchain crawler that adds and removes transactions to the db */ diff --git a/src/data-sources/sqlite-mixed-datasource.js b/src/data-sources/sqlite-mixed-datasource.js index 77788e1..5fed141 100644 --- a/src/data-sources/sqlite-mixed-datasource.js +++ b/src/data-sources/sqlite-mixed-datasource.js @@ -16,7 +16,6 @@ const IS_READY_TO_EXECUTE_SQL = ` downloaded = 1 AND executable = 1 AND executed = 0 - AND (has_code = 0 OR (SELECT COUNT(*) FROM trust WHERE trust.txid = tx.txid AND trust.value = 1) = 1) AND txid NOT IN ban AND ( SELECT COUNT(*) @@ -207,6 +206,7 @@ class SqliteMixedDatasource extends SqliteDatasource { } async setJigState (location, stateObject) { + // this.setJigMetadataStmt(location) await this._pushJigState(location, stateObject) } diff --git a/src/database.js b/src/database.js index fba446c..ff4da17 100644 --- a/src/database.js +++ b/src/database.js @@ -66,21 +66,20 @@ class Database { if (time) { await this.setTransactionTime(txid, time) } }) - // const downloaded = await this.isTransactionIn(txid) - // if (downloaded) return + const downloaded = await this.isTransactionIndexed(txid) + if (downloaded) return if (!txhex) { txhex = await this.ds.getTxHex(txid) } - await this.parseAndStoreTransaction(txid, txhex) - // if (txhex) { - // - // } else { - // if (this.onRequestDownload) { await this.onRequestDownload(txid) } - // } + if (txhex) { + await this.parseAndStoreTransaction(txid, txhex) + } else { + if (this.onRequestDownload) { await this.onRequestDownload(txid) } + } } async parseAndStoreTransaction (txid, hex) { - // if (await this.isTransactionDownloaded(txid)) return + if (await this.isTransactionIndexed(txid)) return let metadata = null let bsvtx = null @@ -221,8 +220,8 @@ class Database { for (const key of Object.keys(cache)) { if (key.startsWith('jig://')) { const location = key.slice('jig://'.length) - await this.ds.setJigState(location, cache[key]) await this.ds.setJigMetadata(location) + await this.ds.setJigState(location, cache[key]) } else if (key.startsWith('berry://')) { const location = key.slice('berry://'.length) await this.ds.setBerryState(location, cache[key]) @@ -347,6 +346,10 @@ class Database { return this.ds.checkTxIsDownloaded(txid) } + async isTransactionIndexed (txid) { + return this.ds.txIsIndexed(txid) + } + async getTransactionsAboveHeight (height) { return this.ds.searchTxsAboveHeight(height) } diff --git a/src/direct-server.js b/src/direct-server.js index 653f422..89872e1 100644 --- a/src/direct-server.js +++ b/src/direct-server.js @@ -1,5 +1,5 @@ /** - * direct-server.js + * direct-server.test.js * * Serves GET requets directly from the database and proxies other requests to the normal server */ diff --git a/src/executor.js b/src/executor.js index cf34124..10b6c29 100644 --- a/src/executor.js +++ b/src/executor.js @@ -12,11 +12,15 @@ const Bus = require('./bus') // ------------------------------------------------------------------------------------------------ class Executor { - constructor (network, numWorkers, database, logger) { + constructor (network, numWorkers, database, logger, opts = {}) { this.network = network this.numWorkers = numWorkers this.database = database this.logger = logger + this.workerOpts = { + cacheType: opts.cacheType || 'parentConnection', + trustSource: opts.trustSource || 'db' + } this.onIndexed = null this.onExecuteFailed = null @@ -33,7 +37,7 @@ class Executor { const path = require.resolve('./worker.js') - const worker = new Worker(path, { workerData: { id: i, network: this.network } }) + const worker = new Worker(path, { workerData: { id: i, network: this.network, ...this.workerOpts } }) worker.id = i worker.available = true diff --git a/src/indexer.js b/src/indexer.js index fcbfd73..1e6d91d 100644 --- a/src/indexer.js +++ b/src/indexer.js @@ -1,5 +1,5 @@ /** - * indexer.js + * indexer.test.js * * Main object that discovers, downloads, executes and stores RUN transactions */ @@ -14,7 +14,7 @@ const Crawler = require('./crawler') // ------------------------------------------------------------------------------------------------ class Indexer { - constructor (database, api, network, numParallelDownloads, numParallelExecutes, logger, startHeight, mempoolExpiration, defaultTrustlist) { + constructor (database, api, network, numParallelDownloads, numParallelExecutes, logger, startHeight, mempoolExpiration, defaultTrustlist, workerOpts = {}) { this.onDownload = null this.onFailToDownload = null this.onIndex = null @@ -33,7 +33,7 @@ class Indexer { const fetchFunction = this.api.fetch ? this.api.fetch.bind(this.api) : null this.downloader = new Downloader(fetchFunction, numParallelDownloads) - this.executor = new Executor(network, numParallelExecutes, this.database, this.logger) + this.executor = new Executor(network, numParallelExecutes, this.database, this.logger, workerOpts) this.crawler = new Crawler(api, this.logger) this.database.onReadyToExecute = this._onReadyToExecute.bind(this) diff --git a/src/server.js b/src/server.js index 08b17ba..2356089 100644 --- a/src/server.js +++ b/src/server.js @@ -1,5 +1,5 @@ /** - * server.js + * server.test.js * * Express server that exposes the Indexer */ @@ -76,6 +76,7 @@ class Server { app.use((err, req, res, next) => { if (this.logger) this.logger.error(err.stack) res.status(500).send('Something broke!') + next() }) this.listener = app.listen(this.port, () => { diff --git a/src/start.js b/src/start.js index 280973f..8237ab6 100644 --- a/src/start.js +++ b/src/start.js @@ -8,7 +8,8 @@ const Indexer = require('./indexer') const Server = require('./server') const { API, DB, NETWORK, PORT, FETCH_LIMIT, WORKERS, MATTERCLOUD_KEY, PLANARIA_TOKEN, START_HEIGHT, - MEMPOOL_EXPIRATION, ZMQ_URL, RPC_URL, DEFAULT_TRUSTLIST, DEBUG, SERVE_ONLY, DATA_SOURCE, DATA_API_ROOT + MEMPOOL_EXPIRATION, ZMQ_URL, RPC_URL, DEFAULT_TRUSTLIST, DEBUG, SERVE_ONLY, DATA_SOURCE, DATA_API_ROOT, + WORKER_TRUST_SOURCE, WORKER_CACHE_TYPE } = require('./config') const MatterCloud = require('./mattercloud') const Planaria = require('./planaria') @@ -64,7 +65,10 @@ if (DATA_SOURCE === 'sqlite') { const database = new Database(dataSource, logger) const indexer = new Indexer(database, api, NETWORK, FETCH_LIMIT, WORKERS, logger, - START_HEIGHT, MEMPOOL_EXPIRATION, DEFAULT_TRUSTLIST) + START_HEIGHT, MEMPOOL_EXPIRATION, DEFAULT_TRUSTLIST, { + trustSource: WORKER_TRUST_SOURCE, + cacheType: WORKER_CACHE_TYPE + }) const server = SERVE_ONLY ? new Server(database, logger, PORT) diff --git a/src/threading/parent-thread.js b/src/threading/parent-thread.js new file mode 100644 index 0000000..cd8e598 --- /dev/null +++ b/src/threading/parent-thread.js @@ -0,0 +1,37 @@ +const { parentPort } = require('worker_threads') + +class ParentThread { + constructor () { + this.parent = parentPort + this.parent.on('message', this._onMessage.bind(this)) + this.handlers = new Map() + } + + async _onMessage (msg) { + const { headers, body } = msg + const handler = this.handlers.get(headers.topic) + if (!handler) { + throw new Error(`unknown topic: ${headers.topic}`) + } + try { + const response = await handler(body) + this.parent.postMessage({ + headers: { + replyTo: headers.id, + type: 'response' + }, + body: response + }) + } catch (e) { + this.parent.postMessage({ + headers: { + replyTo: headers.id, + type: 'error' + }, + body: e + }) + } + } +} + +module.exports = { ParentThread } diff --git a/src/threading/sub-thread.js b/src/threading/sub-thread.js new file mode 100644 index 0000000..00f0f99 --- /dev/null +++ b/src/threading/sub-thread.js @@ -0,0 +1,61 @@ +const { Worker } = require('worker_threads') +const { Clock } = require('../clock') +const { nanoid } = require('nanoid') + +class TimeoutError extends Error {} + +class SubThread { + constructor (path, workerOpts = {}, clientOpts = {}) { + this.thread = new Worker(path, workerOpts) + this.thread.on('message', this._onMessage.bind(this)) + this.thread.on('exit', this._onExit.bind(this)) + this.thread.on('error', this._onError.bind(this)) + + this.pending = new Map() + this.clock = new Clock() + this.timeoutMs = clientOpts.timeoutMs || 10 * 1000 + } + + send (topic, body, opts) { + const id = nanoid() + const headers = { ...opts, id, topic } + const timeoutMs = opts.timeoutMs || this.timeoutMs + + this.thread.postMessage({ headers, body }) + return new Promise((resolve, reject) => { + const cancel = this.clock.delay(() => { + reject(new TimeoutError()) + this.pending.delete(id) + }, timeoutMs) + this.pending.set(id, { resolve, reject, timeout: cancel }) + }).finally( + (data) => { + const { timeout } = this.pending.get(id) + timeout.cancel() + this.pending.delete(id) + return data + }) + } + + async _onMessage (msg) { + const { header: { replyTo, type }, body } = msg + const { resolve, reject } = this.pending.get(replyTo) + if (type === 'response') { + resolve(body) + } else if (type === 'error') { + reject(body) + } else { + throw new Error(`unknown type: ${type}`) + } + } + + async _onExit () { + + } + + async _onError () { + + } +} + +module.exports = { SubThread } diff --git a/src/worker.js b/src/worker.js index 67f37d7..219ba30 100644 --- a/src/worker.js +++ b/src/worker.js @@ -19,6 +19,8 @@ const { DEBUG } = require('./config') // ------------------------------------------------------------------------------------------------ const network = workerData.network +const cacheType = workerData.cacheType +const trustSource = workerData.trustSource Bus.listen(parentPort, { execute }) @@ -54,7 +56,7 @@ class Cache { class DirectCache { constructor (db) { this.db = db - this.state = new Map() + this.state = {} } async get (key) { @@ -67,7 +69,7 @@ class DirectCache { this.state[key] = jig return jig } else if (type === 'tx') { - const txHex = await this.db.getTxHex(identifier) + const txHex = await this.db.getTransactionHex(identifier) this.state[key] = txHex return txHex } else if (type === 'berry') { @@ -79,8 +81,8 @@ class DirectCache { } } - async set () { - + async set (key, value) { + this.state[key] = value } } @@ -117,7 +119,7 @@ logger.error = console.error.bind(console) logger.debug = DEBUG ? console.debug.bind(console) : () => {} async function execute (txid, hex, trustlist) { - if (config.DATA_SOURCE === 'mixed') { + if (cacheType === 'direct') { const ds = new SqliteMixedDatasource(config.DB, logger, false, config.DATA_API_ROOT) const db = new Database(ds, logger) run.cache = new DirectCache(db) @@ -131,7 +133,7 @@ async function execute (txid, hex, trustlist) { run.client = false run.preverify = false - if (config.DATA_SOURCE === 'mixed') { + if (trustSource === 'all') { run.trust('*') } else { trustlist.forEach(txid => run.trust(txid)) @@ -140,7 +142,12 @@ async function execute (txid, hex, trustlist) { const tx = await run.import(hex, { txid }) - await tx.cache() + try { + await tx.cache() + } catch (e) { + console.error(e) + throw e + } const cache = run.cache.state const jigs = tx.outputs.filter(creation => creation instanceof Run.Jig) diff --git a/test/crawler.js b/test/crawler.test.js similarity index 99% rename from test/crawler.js rename to test/crawler.test.js index 0124aa4..eeed92e 100644 --- a/test/crawler.js +++ b/test/crawler.test.js @@ -1,5 +1,5 @@ /** - * crawler.js + * crawler.test.js * * Tests for the crawler and APIs it uses */ diff --git a/test/index.js b/test/index.js index f7fd486..97e8e7d 100644 --- a/test/index.js +++ b/test/index.js @@ -4,6 +4,6 @@ * Tests for run-db */ -require('./crawler') -require('./indexer') -require('./server') +require('./crawler.test') +require('./indexer.test') +require('./server.test') diff --git a/test/indexer.js b/test/indexer.test.js similarity index 92% rename from test/indexer.js rename to test/indexer.test.js index 51a3e4a..2c99feb 100644 --- a/test/indexer.js +++ b/test/indexer.test.js @@ -1,13 +1,13 @@ /** - * indexer.js + * indexer.test.js * - * Tests for the Indexer + * Tests for the IndexerTest */ const { describe, it, beforeEach, afterEach } = require('mocha') const { expect } = require('chai') const bsv = require('bsv') -const Indexer = require('../src/indexer') +const IndexerTest = require('../src/indexer') const Run = require('run-sdk') const { Jig } = Run const { DEFAULT_TRUSTLIST } = require('../src/config') @@ -30,12 +30,12 @@ beforeEach(() => database.open()) afterEach(() => database.close()) // ------------------------------------------------------------------------------------------------ -// Indexer +// IndexerTest // ------------------------------------------------------------------------------------------------ describe('Indexer', () => { it('add and index', async () => { - const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, []) + const indexer = new IndexerTest(database, api, 'main', 1, 1, logger, 0, Infinity, []) await indexer.start() const promise = indexed(indexer, '9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102') await database.addTransaction('3f9de452f0c3c96be737d42aa0941b27412211976688967adb3174ee18b04c64') @@ -61,7 +61,7 @@ describe('Indexer', () => { tx.update(() => new A(pubkey)) const rawtx = await tx.export() const api = { fetch: _txid => { return { hex: rawtx } } } - const indexer = new Indexer(database, api, 'test', 1, 1, logger, 0, Infinity, []) + const indexer = new IndexerTest(database, api, 'test', 1, 1, logger, 0, Infinity, []) const txid = new bsv.Transaction(rawtx).hash await indexer.start() database.addTransaction(txid) @@ -73,7 +73,7 @@ describe('Indexer', () => { // -------------------------------------------------------------------------- it('add in reverse and index', async () => { - const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, []) + const indexer = new IndexerTest(database, api, 'main', 1, 1, logger, 0, Infinity, []) await indexer.start() const promise = indexed(indexer, '9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102') await database.addTransaction('9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102') @@ -87,7 +87,7 @@ describe('Indexer', () => { // -------------------------------------------------------------------------- it('fail to index', async () => { - const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, []) + const indexer = new IndexerTest(database, api, 'main', 1, 1, logger, 0, Infinity, []) await indexer.start() const promise = failed(indexer, 'a5291157ab7a2d80d834bbe82c380ce3976f53990d20c62c477ca3a2ac93a7e9') await database.trust('b17a9af70ab0f46809f908b2e900e395ba40996000bf4f00e3b27a1e93280cf1') @@ -101,7 +101,7 @@ describe('Indexer', () => { // -------------------------------------------------------------------------- it('discovered berry transaction', async () => { - const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, []) + const indexer = new IndexerTest(database, api, 'main', 1, 1, logger, 0, Infinity, []) await indexer.start() const promise = indexed(indexer, 'bfa5180e601e92af23d80782bf625b102ac110105a392e376fe7607e4e87dc8d') await database.addTransaction('bfa5180e601e92af23d80782bf625b102ac110105a392e376fe7607e4e87dc8d') // Class with berry image @@ -121,7 +121,7 @@ describe('Indexer', () => { // -------------------------------------------------------------------------- it('add and download dependencies', async () => { - const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, []) + const indexer = new IndexerTest(database, api, 'main', 1, 1, logger, 0, Infinity, []) await indexer.start() await database.addTransaction('9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102') await new Promise((resolve) => setTimeout(resolve, 1000)) @@ -131,7 +131,7 @@ describe('Indexer', () => { // -------------------------------------------------------------------------- it('remove discovered dep', async () => { - const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, DEFAULT_TRUSTLIST) + const indexer = new IndexerTest(database, api, 'main', 1, 1, logger, 0, Infinity, DEFAULT_TRUSTLIST) await indexer.start() const promise = indexed(indexer, 'bfa5180e601e92af23d80782bf625b102ac110105a392e376fe7607e4e87dc8d') await database.addTransaction('bfa5180e601e92af23d80782bf625b102ac110105a392e376fe7607e4e87dc8d') // Class with berry image @@ -147,7 +147,7 @@ describe('Indexer', () => { it('get spent', async function () { this.timeout(40000) - const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, DEFAULT_TRUSTLIST) + const indexer = new IndexerTest(database, api, 'main', 1, 1, logger, 0, Infinity, DEFAULT_TRUSTLIST) await indexer.start() const promise = indexed(indexer, '11f27cdad53128a4eb14c8328515dfab56b16ea5a71dd26abe9e9d7488f3ab83') await database.addTransaction('11f27cdad53128a4eb14c8328515dfab56b16ea5a71dd26abe9e9d7488f3ab83') @@ -162,7 +162,7 @@ describe('Indexer', () => { // -------------------------------------------------------------------------- it('mark failed execute as melts', async () => { - const indexer = new Indexer(database, {}, 'test', 1, 1, logger, 0, Infinity, []) + const indexer = new IndexerTest(database, {}, 'test', 1, 1, logger, 0, Infinity, []) const rawtx1 = '0100000001a11d53c34263d1ea9dec40d3cc5beb7eb461a601d898a8337dea215cd90a9e4a010000006a47304402202f294c5ceca857cfc03e38b1a49a79d6c133e9e6b18047f0301f9f74bb2abdab022027aa6662cd24428106b9f8f2e38d2e5b8f0b7c30929eef6dbc1d013c43b0493f41210211f2cc632921525ec8650cb65c2ed520e400a2644010c1e794203d5823f604c0ffffffff030000000000000000fd0301006a0372756e0105004cf87b22696e223a302c22726566223a5b226e61746976653a2f2f4a6967225d2c226f7574223a5b2238396336653439636532653831373962653138383563396230653032343863363935666130373634343939656665626362363936623238323732366239666165225d2c2264656c223a5b5d2c22637265223a5b226d737138444642455777546166675a6173474c4a386f3338517a456367346267364a225d2c2265786563223a5b7b226f70223a224445504c4f59222c2264617461223a5b22636c617373204120657874656e6473204a6967207b207d222c7b2264657073223a7b224a6967223a7b22246a6967223a307d7d7d5d7d5d7d11010000000000001976a9148711466c1f8b5977cb788485fcb6cc1fb9d0407788acf6def505000000001976a9142208fb2364d1551e2dd26549d7c22eab613a207188ac00000000' const rawtx2 = '0100000002cb8c61b7d73cf14ed2526f2adcb0ef941563c69fb794a87eb39a94423886d273010000006a4730440220306a24e0464c90889d6fd1580db4420fe9ee1bd8f167ec793d40d2296ff0d8ea02202224f4f13e4c07354478983b2dc88170342a4f1ac3e6cacad8616a92348fc768412103a6fa27cfcda39be6ee9dc5dbd43a44c2c749ca136f7d41cd81468f72cc0fda59ffffffffcb8c61b7d73cf14ed2526f2adcb0ef941563c69fb794a87eb39a94423886d273020000006b483045022100c2b7a660b22dd2c3ac22d47ba16fa3f7df852f5a6cfdec5ce14c734517a0b1900220592da53a61ec1387aa96050c370b7c5ba162ee35e8d30b55d9999f1c2ba06ade41210211f2cc632921525ec8650cb65c2ed520e400a2644010c1e794203d5823f604c0ffffffff030000000000000000ae006a0372756e0105004ca37b22696e223a312c22726566223a5b5d2c226f7574223a5b2264633031326334616436346533626136373632383762323239623865306662303934326448626535303435393036363830616637633937663134666239663433225d2c2264656c223a5b5d2c22637265223a5b5d2c2265786563223a5b7b226f70223a2243414c4c222c2264617461223a5b7b22246a6967223a307d2c2261757468222c5b5d5d7d5d7d11010000000000001976a9148711466c1f8b5977cb788485fcb6cc1fb9d0407788acdeddf505000000001976a9142208fb2364d1551e2dd26549d7c22eab613a207188ac00000000' const txid1 = new bsv.Transaction(rawtx1).hash @@ -182,7 +182,7 @@ describe('Indexer', () => { // -------------------------------------------------------------------------- it('deletes are not included in unspent', async () => { - const indexer = new Indexer(database, {}, 'test', 1, 1, logger, 0, Infinity, []) + const indexer = new IndexerTest(database, {}, 'test', 1, 1, logger, 0, Infinity, []) const rawtx1 = '01000000016f4f66891029280028bce15768b3fdc385533b0bcc77a029add646176207e77f010000006b483045022100a76777ae759178595cb83ce9473699c9056e32faa8e0d07c2517918744fab9e90220369d7a6a2f52b5ddd9bff4ed659ef5a8e676397dac15e9c5dc6dad09e5eab85e412103ac8a61b3fb98161003daaaa63ec1983dc127f4f978a42f2eefd31a074a814345ffffffff030000000000000000fd0301006a0372756e0105004cf87b22696e223a302c22726566223a5b226e61746976653a2f2f4a6967225d2c226f7574223a5b2237373864313934336265613463353166356561313635666630346335613039323435356365386437343335623936336333613130623961343536633463623330225d2c2264656c223a5b5d2c22637265223a5b226d674671626e5254774c3155436d384a654e6e556d6b7a58665a6f3271385764364c225d2c2265786563223a5b7b226f70223a224445504c4f59222c2264617461223a5b22636c617373204120657874656e6473204a6967207b207d222c7b2264657073223a7b224a6967223a7b22246a6967223a307d7d7d5d7d5d7d11010000000000001976a914081c4c589c062b1b1d4e4b25a8b3096868059d7a88acf6def505000000001976a914146caf0030b67f3fae5d53b7c3fa7e1e6fcaaf3b88ac00000000' const rawtx2 = '01000000015991661ed379a0d12a68feacdbf7776d82bcffe1761f995cf0412c5ae2d25d28010000006a47304402203776f765d6915431388110a7f4645a61bd8d2f2ab00ade0049f0da95b5455c22022074ca4b6a87891ba852416bf08b64ad3db130a0b780e2a658c451ebacbbcffbf8412103646b0e969bd3825f781f39b737bdfed1e2cd63533301317099e5ac021b4826aaffffffff010000000000000000b1006a0372756e0105004ca67b22696e223a312c22726566223a5b5d2c226f7574223a5b5d2c2264656c223a5b2265386436393434613366383765323936663237326562656437663033623133323962653262313733653732376436623431643632616365343036656434373539225d2c22637265223a5b5d2c2265786563223a5b7b226f70223a2243414c4c222c2264617461223a5b7b22246a6967223a307d2c2264657374726f79222c5b5d5d7d5d7d00000000' const txid1 = new bsv.Transaction(rawtx1).hash diff --git a/test/server.js b/test/server.test.js similarity index 99% rename from test/server.js rename to test/server.test.js index d612f56..5b5c163 100644 --- a/test/server.js +++ b/test/server.test.js @@ -1,7 +1,7 @@ /** - * server.js + * server.test.js * - * Tests for src/server.js + * Tests for src/server.test.js */ const { describe, it, beforeEach, afterEach } = require('mocha') @@ -19,7 +19,9 @@ const { SqliteDatasource } = require('../src/data-sources/sqlite-datasource') // Globals // ------------------------------------------------------------------------------------------------ -const fetch = async txid => { return { hex: require('./txns.json')[txid] } } +const fetch = async txid => { + return { hex: require('./txns.json')[txid] } +} const api = { fetch } const downloaded = (indexer, txid) => new Promise((resolve) => { indexer.onDownload = x => txid === x && resolve() }) const indexed = (indexer, txid) => new Promise((resolve) => { indexer.onIndex = x => txid === x && resolve() }) diff --git a/test/thread.test.js b/test/thread.test.js new file mode 100644 index 0000000..2da1eb5 --- /dev/null +++ b/test/thread.test.js @@ -0,0 +1,7 @@ +const { expect } = require('chai') + +describe('SubThread', () => { + it('works', () => { + expect(1).to.eql(1) + }) +}) From f2e5a12133a7e97e5e19230232b434c6c682a60f Mon Sep 17 00:00:00 2001 From: Miguel Duarte Date: Tue, 21 Dec 2021 11:43:33 -0300 Subject: [PATCH 22/82] Fixed infinite loop problem --- src/database.js | 16 +++++++++++++--- src/indexer.js | 8 ++++++-- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/src/database.js b/src/database.js index ff4da17..3c3e0d7 100644 --- a/src/database.js +++ b/src/database.js @@ -7,7 +7,6 @@ const Run = require('run-sdk') const bsv = require('bsv') const { HEIGHT_MEMPOOL, HEIGHT_UNKNOWN } = require('./constants') - // ------------------------------------------------------------------------------------------------ // Database // ------------------------------------------------------------------------------------------------ @@ -174,6 +173,10 @@ class Database { // Non-executable might be berry data. We execute once we receive them. const downstreamReadyToExecute = await this.ds.searchDownstreamTxidsReadyToExecute(txid) for (const downtxid of downstreamReadyToExecute) { + const executed = await this.ds.txIsIndexed(downtxid) + if (executed) { + continue + } await this.ds.markTxAsExecuting(downtxid) if (this.onReadyToExecute) { await this.onReadyToExecute(downtxid) } } @@ -243,8 +246,12 @@ class Database { const downstreamReadyToExecute = await this.ds.searchDownstreamForTxid(txid) for (const downtxid of downstreamReadyToExecute) { + const executed = await this.ds.txIsIndexed(downtxid) + if (executed) { + continue + } await this.ds.markTxAsExecuting(downtxid) - if (this.onReadyToExecute) { await this.onReadyToExecute(downtxid) } + if (this.onReadyToExecute) { this.onReadyToExecute(downtxid) } } } @@ -565,7 +572,8 @@ class Database { async loadTransactionsToExecute () { this.logger.debug('Loading transactions to execute') - const txids = this.db.prepare('SELECT txid FROM executing').raw(true).all().map(x => x[0]) + + const txids = await this.ds.findAllExecutingTxids() for (const txid of txids) { await this._checkExecutability(txid) } @@ -576,6 +584,8 @@ class Database { if (row && row.ready) { await this.ds.markTxAsExecuting(txid) if (this.onReadyToExecute) { await this.onReadyToExecute(txid) } + } else { + await this.ds.removeTxFromExecuting(txid) } } } diff --git a/src/indexer.js b/src/indexer.js index 1e6d91d..63974ac 100644 --- a/src/indexer.js +++ b/src/indexer.js @@ -106,7 +106,8 @@ class Indexer { async _onIndexed (txid, result) { if (!await this.database.hasTransaction(txid)) return // Check not re-orged this.logger.info(`Executed ${txid}`) - await this.database.storeExecutedTransaction(txid, result) + this.database.storeExecutedTransaction(txid, result) + .catch(console.error) if (this.onIndex) { await this.onIndex(txid) } @@ -119,7 +120,10 @@ class Indexer { } async _onReadyToExecute (txid) { - await this.executor.execute(txid) + this.executor.execute(txid) + .catch((e) => + console.warn(`error executing tx ${txid}: ${e.message}`) + ) } async _onAddTransaction (txid) { From d238674a0e60fd297edb7f6a7a660df82c9b8e39 Mon Sep 17 00:00:00 2001 From: Miguel Duarte Date: Tue, 28 Dec 2021 01:28:13 -0300 Subject: [PATCH 23/82] refactor wip --- deleteme.js | 49 ++++++++++++ src/data-sources/api-blob-storage.js | 45 +++++++++++ src/data-sources/sqlite-datasource.js | 17 +++++ src/data-sources/sqlite-mixed-datasource.js | 84 ++++++++------------- src/database.js | 67 ++++++++-------- src/worker.js | 28 +++++-- 6 files changed, 199 insertions(+), 91 deletions(-) create mode 100644 deleteme.js create mode 100644 src/data-sources/api-blob-storage.js diff --git a/deleteme.js b/deleteme.js new file mode 100644 index 0000000..c2f0ff4 --- /dev/null +++ b/deleteme.js @@ -0,0 +1,49 @@ +const Run = require('run-sdk') + +const owner = 'cPepyUbXNEmJFqBcmrxFxuYr69v4Jn5qL9NtdA1wqch7GNZTBowE' +const purse = 'cVcVT1hpsJh5gvtgVWUH93HchkhtteHdWftqkzncwMtc1iXR7eU3' + +const run = new Run({ + cache: new Run.plugins.LocalCache(), + state: new Run.plugins.RunDB('http://localhost:3101'), + network: 'test', + owner, + purse +}) +run.trust('23c615454eaa787db6de70d1f61e8f1212c213278a4f6e031b9ca50010b8c283') +run.trust('8e76e558ea4e5383c22875d2eb442f9189cda9b60a187ec3cd307755903cda46') +run.blockchain.host = 'http://localhost:3000' + +class AThing extends Run.Jig { + init (initialThing) { + this.thing = initialThing + } + + change (newThing) { + this.thing = newThing + } +} + +const main = async () => { + // await run.transaction(() => { + // run.deploy(AThing) + // }) + // await run.sync() + // console.log(AThing.location) + // const Klass = await run.load('8e76e558ea4e5383c22875d2eb442f9189cda9b60a187ec3cd307755903cda46_o1') + // const instance = new Klass('a thing') + // await instance.sync() + // instance.change('shinier thing') + // await instance.sync() + // console.log(instance.location) + const instance = await run.load('97b4149941677ba2da82de7c994dc7eb0d3fabef7efbf53ae9173dd592626d68_o1') + await instance.sync() + // instance.change('shinier thing' + instance.location) + // await instance.sync() + console.log(instance.location) + console.log(instance.constructor.location) + + // console.log(await run.blockchain.utxos('n2nRMck2LEP7FBmWzWfS6T1EMzuSULnnsz')) +} + +main() \ No newline at end of file diff --git a/src/data-sources/api-blob-storage.js b/src/data-sources/api-blob-storage.js new file mode 100644 index 0000000..7459ca4 --- /dev/null +++ b/src/data-sources/api-blob-storage.js @@ -0,0 +1,45 @@ +const fetch = require('node-fetch') + +class ApiBlobStorage { + constructor (baseUri) { + this.baseUri = baseUri + } + + async pushJigState (location, stateObject) { + const result = await fetch(`${this.baseApiUrl}/state`, { + method: 'POST', + body: JSON.stringify({ + location, + state: stateObject + }), + headers: { + 'Content-Type': 'application/json' + } + }) + + if (!result.ok) { + throw new Error(`Error saving jig state: ${location}`) + } + } + + async pullJigState (location) { + const result = await fetch(`${this.baseApiUrl}/state/${location}`) + if (result.status === 404) { + return null + } + + if (!result.ok) { + throw new Error(`error fetching jig state ${location}`) + } + + const json = await result.json() + return json.state + } + + async fetchTx (txid) { + const result = await fetch(`${this.baseApiUrl}/rawtx/${txid}`) + return result.buffer() + } +} + +module.exports = { ApiBlobStorage } diff --git a/src/data-sources/sqlite-datasource.js b/src/data-sources/sqlite-datasource.js index a63c1d7..cdf5357 100644 --- a/src/data-sources/sqlite-datasource.js +++ b/src/data-sources/sqlite-datasource.js @@ -103,6 +103,7 @@ class SqliteDatasource { this.getTransactionHeightStmt = this.connection.prepare('SELECT height FROM tx WHERE txid = ?') this.getTransactionHasCodeStmt = this.connection.prepare('SELECT has_code FROM tx WHERE txid = ?') this.getTransactionIndexedStmt = this.connection.prepare('SELECT indexed FROM tx WHERE txid = ?') + this.getTransactionWasExecutedStmt = this.connection.prepare('SELECT executed FROM tx WHERE txid = ?') this.getTransactionFailedStmt = this.connection.prepare('SELECT (executed = 1 AND indexed = 0) AS failed FROM tx WHERE txid = ?') this.getTransactionDownloadedStmt = this.connection.prepare('SELECT downloaded FROM tx WHERE txid = ?') this.deleteTransactionStmt = this.connection.prepare('DELETE FROM tx WHERE txid = ?') @@ -130,6 +131,14 @@ class SqliteDatasource { JOIN tx ON tx.txid = txdeps.txid WHERE tx.executable = 1 AND tx.executed = 0 AND tx.has_code = 1 `) + this.hasFailedDepStmt = this.connection.prepare(` + SELECT count(*) > 0 FROM tx + JOIN deps ON deps.up = tx.txid + WHERE + tx.txid = ? AND + tx.executed = 1 AND + tx.indexed = 0; + `) // this.setJigStateStmt = this.connection.prepare('INSERT OR IGNORE INTO jig (location, state, class, lock, scripthash) VALUES (?, ?, null, null, null)') this.setJigMetadataStmt = this.connection.prepare('INSERT OR IGNORE INTO jig (location) VALUES (?)') @@ -556,6 +565,14 @@ class SqliteDatasource { return this.getTransactionIndexedStmt.raw(true).get(txid)[0] } + async hasFailedDep (txid) { + return this.hasFailedDepStmt.raw(true).get(txid)[0] + } + + async checkTxWasExecuted (txid) { + return this.getTransactionWasExecutedStmt.raw(true).get(txid)[0] + } + async getTxHex (txid) { const row = this.getTransactionHexStmt.raw(true).get(txid) return row && row[0] diff --git a/src/data-sources/sqlite-mixed-datasource.js b/src/data-sources/sqlite-mixed-datasource.js index 5fed141..c9feb7a 100644 --- a/src/data-sources/sqlite-mixed-datasource.js +++ b/src/data-sources/sqlite-mixed-datasource.js @@ -13,21 +13,20 @@ const { HEIGHT_MEMPOOL } = require('../constants') const IS_READY_TO_EXECUTE_SQL = ` SELECT ( - downloaded = 1 - AND executable = 1 - AND executed = 0 - AND txid NOT IN ban - AND ( - SELECT COUNT(*) - FROM tx AS tx2 + executable = 1 AND + executed = 0 AND + ( + SELECT count(*) + FROM tx AS txInner JOIN deps - ON deps.up = tx2.txid - WHERE deps.down = tx.txid - AND (+tx2.downloaded = 0 OR (tx2.executable = 1 AND tx2.executed = 0)) + ON deps.up = txInner.txid + WHERE deps.down = txOuter.txid AND + txInner.executable = 1 AND + txInner.indexed = 0 ) = 0 ) AS ready - FROM tx - WHERE txid = ? + FROM tx as txOuter + WHERE txOuter.txid = ?; ` const GET_DOWNSTREAM_READY_TO_EXECUTE_SQL = ` @@ -52,9 +51,9 @@ const GET_DOWNSTREAM_READY_TO_EXECUTE_SQL = ` ` class SqliteMixedDatasource extends SqliteDatasource { - constructor (path, logger, readonly, baseApiUrl) { + constructor (path, logger, readonly, blobStorage) { super(path, logger, readonly) - this.baseApiUrl = baseApiUrl + this.blobStorage = blobStorage } prepareStatements () { @@ -65,6 +64,7 @@ class SqliteMixedDatasource extends SqliteDatasource { this.setTransactionHasCodeStmt = this.connection.prepare('UPDATE tx SET has_code = ? WHERE txid = ?') this.setTransactionExecutedStmt = this.connection.prepare('UPDATE tx SET executed = ? WHERE txid = ?') this.setTransactionIndexedStmt = this.connection.prepare('UPDATE tx SET indexed = ? WHERE txid = ?') + this.getTransactionWasExecutedStmt = this.connection.prepare('SELECT executed FROM tx WHERE txid = ?') this.txExistsStmt = this.connection.prepare('SELECT txid FROM tx WHERE txid = ?') this.getTransactionTimeStmt = this.connection.prepare('SELECT time FROM tx WHERE txid = ?') this.getTransactionHeightStmt = this.connection.prepare('SELECT height FROM tx WHERE txid = ?') @@ -97,6 +97,14 @@ class SqliteMixedDatasource extends SqliteDatasource { JOIN tx ON tx.txid = txdeps.txid WHERE tx.executable = 1 AND tx.executed = 0 AND tx.has_code = 1 `) + this.hasFailedDepStmt = this.connection.prepare(` + SELECT count(*) > 0 FROM tx + JOIN deps ON deps.up = tx.txid + WHERE + tx.txid = ? AND + tx.executed = 1 AND + tx.indexed = 0; + `) this.setJigMetadataStmt = this.connection.prepare('INSERT OR IGNORE INTO jig (location) VALUES (?)') this.setJigClassStmt = this.connection.prepare('UPDATE jig SET class = ? WHERE location = ?') @@ -191,7 +199,7 @@ class SqliteMixedDatasource extends SqliteDatasource { } async getTxHex (txid) { - const buff = await this.fetchTx(txid) + const buff = await this.blobStorage.fetchTx(txid) return buff.toString('hex') } @@ -202,56 +210,24 @@ class SqliteMixedDatasource extends SqliteDatasource { // jig async getJigState (location) { - return this._pullJigState(location) + return this.blobStorage.pullJigState(location) } async setJigState (location, stateObject) { // this.setJigMetadataStmt(location) - await this._pushJigState(location, stateObject) + await this.blobStorage.pushJigState(location, stateObject) } async getBerryState (location) { - return this._pullJigState(location) - } - - async setBerryState (location, stateObject) { - return this._pushJigState(location, stateObject) - } - - async _pullJigState (location) { - const result = await fetch(`${this.baseApiUrl}/state/${location}`) - if (result.status === 404) { - return null - } - - if (!result.ok) { - throw new Error(`error fetching jig state ${location}`) - } - - const json = await result.json() - return json.state + return this.blobStorage.pullJigState(location) } - async fetchTx (txid) { - const result = await fetch(`${this.baseApiUrl}/rawtx/${txid}`) - return result.buffer() + async setBerryMetadata (location) { + this.setBerryMetadataStmt.run(location) } - async _pushJigState (location, stateObject) { - const result = await fetch(`${this.baseApiUrl}/state`, { - method: 'POST', - body: JSON.stringify({ - location, - state: stateObject - }), - headers: { - 'Content-Type': 'application/json' - } - }) - - if (!result.ok) { - throw new Error(`Error saving jig state: ${location}`) - } + async setBerryState (location, stateObject) { + return this.blobStorage.pushJigState(location, stateObject) } // tx diff --git a/src/database.js b/src/database.js index 3c3e0d7..c68509d 100644 --- a/src/database.js +++ b/src/database.js @@ -78,7 +78,7 @@ class Database { } async parseAndStoreTransaction (txid, hex) { - if (await this.isTransactionIndexed(txid)) return + if (await this.ds.checkTxWasExecuted(txid)) return let metadata = null let bsvtx = null @@ -141,8 +141,6 @@ class Database { if (await this.hasTransaction(txid)) return const time = Math.round(Date.now() / 1000) - - // this.addNewTransactionStmt.run(txid, time) await this.ds.addNewTx(txid, time) if (this.onAddTransaction) { await this.onAddTransaction(txid) } @@ -173,12 +171,7 @@ class Database { // Non-executable might be berry data. We execute once we receive them. const downstreamReadyToExecute = await this.ds.searchDownstreamTxidsReadyToExecute(txid) for (const downtxid of downstreamReadyToExecute) { - const executed = await this.ds.txIsIndexed(downtxid) - if (executed) { - continue - } - await this.ds.markTxAsExecuting(downtxid) - if (this.onReadyToExecute) { await this.onReadyToExecute(downtxid) } + await this._checkExecutability(downtxid) } } @@ -224,10 +217,11 @@ class Database { if (key.startsWith('jig://')) { const location = key.slice('jig://'.length) await this.ds.setJigMetadata(location) - await this.ds.setJigState(location, cache[key]) + // await this.ds.setJigState(location, cache[key]) } else if (key.startsWith('berry://')) { const location = key.slice('berry://'.length) - await this.ds.setBerryState(location, cache[key]) + await this.ds.setBerryMetadata(location) + // await this.ds.setBerryState(location, cache[key]) } } @@ -246,12 +240,7 @@ class Database { const downstreamReadyToExecute = await this.ds.searchDownstreamForTxid(txid) for (const downtxid of downstreamReadyToExecute) { - const executed = await this.ds.txIsIndexed(downtxid) - if (executed) { - continue - } - await this.ds.markTxAsExecuting(downtxid) - if (this.onReadyToExecute) { this.onReadyToExecute(downtxid) } + await this._checkExecutability(downtxid) } } @@ -264,19 +253,19 @@ class Database { // We try executing downstream transactions if this was marked executable but it wasn't. // This allows an admin to manually change executable status in the database. - let executable = false - try { - const rawTx = await this.getTransactionHex(txid) - Run.util.metadata(rawTx) - executable = true - } catch (e) { } - - if (!executable) { - const downstream = await this.ds.searchDownstreamForTxid(txid) - for (const downtxid of downstream) { - await this._checkExecutability(downtxid) - } + // let executable = false + // try { + // const rawTx = await this.getTransactionHex(txid) + // Run.util.metadata(rawTx) + // executable = true + // } catch (e) { } + + // if (!executable) { + const downstream = await this.ds.searchDownstreamForTxid(txid) + for (const downtxid of downstream) { + await this._checkExecutability(downtxid) } + // } } async getTransactionHex (txid) { @@ -580,12 +569,28 @@ class Database { } async _checkExecutability (txid) { + const executed = await this.ds.checkTxWasExecuted(txid) + if (executed) { + await this.ds.removeTxFromExecuting(txid) + return + } + const someDepFailed = await this.ds.hasFailedDep(txid) + if (someDepFailed) { + await this.ds.setExecutedForTx(txid, 1) + await this.ds.setIndexedForTx(txid, 0) + await this.ds.removeTxFromExecuting(txid) + const downstreamReadyToExecute = await this.ds.searchDownstreamTxidsReadyToExecute(txid) + for (const downtxid of downstreamReadyToExecute) { + await this._checkExecutability(downtxid) + } + return + } + const row = await this.ds.txidReadyToExecute(txid) + if (row && row.ready) { await this.ds.markTxAsExecuting(txid) if (this.onReadyToExecute) { await this.onReadyToExecute(txid) } - } else { - await this.ds.removeTxFromExecuting(txid) } } } diff --git a/src/worker.js b/src/worker.js index 219ba30..4727bd1 100644 --- a/src/worker.js +++ b/src/worker.js @@ -13,6 +13,7 @@ const config = require('./config') const Database = require('./database') const { SqliteMixedDatasource } = require('./data-sources/sqlite-mixed-datasource') const { DEBUG } = require('./config') +const { ApiBlobStorage } = require('./data-sources/api-blob-storage') // ------------------------------------------------------------------------------------------------ // Startup @@ -21,6 +22,7 @@ const { DEBUG } = require('./config') const network = workerData.network const cacheType = workerData.cacheType const trustSource = workerData.trustSource +const id = workerData.id Bus.listen(parentPort, { execute }) @@ -54,8 +56,8 @@ class Cache { } class DirectCache { - constructor (db) { - this.db = db + constructor (blobStorage) { + this.blobs = blobStorage this.state = {} } @@ -65,15 +67,16 @@ class DirectCache { const [type, identifier] = key.split('://') if (type === 'jig') { - const jig = await this.db.getJigState(identifier) + const jig = await this.blobs.getJigState(identifier) this.state[key] = jig return jig } else if (type === 'tx') { - const txHex = await this.db.getTransactionHex(identifier) + const rawTx = await this.blobs.fetchTx(identifier) + const txHex = rawTx.toString('hex') this.state[key] = txHex return txHex } else if (type === 'berry') { - const berry = await this.db.getBerryState(identifier) + const berry = await this.blobs.getBerryState(identifier) this.state[key] = berry return berry } else { @@ -82,7 +85,16 @@ class DirectCache { } async set (key, value) { + const existedBefore = !!this.state[key] this.state[key] = value + if (existedBefore) { + return null + } + + const [type, identifier] = key.split('://') + if (type === 'jig' || type === 'berry') { + await this.blobs.pushJigState(identifier, value) + } } } @@ -119,8 +131,11 @@ logger.error = console.error.bind(console) logger.debug = DEBUG ? console.debug.bind(console) : () => {} async function execute (txid, hex, trustlist) { + const back = console.log + console.log = function () {} + console.log() if (cacheType === 'direct') { - const ds = new SqliteMixedDatasource(config.DB, logger, false, config.DATA_API_ROOT) + const ds = new ApiBlobStorage(config.DATA_API_ROOT) const db = new Database(ds, logger) run.cache = new DirectCache(db) } else { @@ -162,6 +177,7 @@ async function execute (txid, hex, trustlist) { const scripts = customLocks.concat(commonLocks).map(([location, lock]) => [location, lock.script()]) const scripthashes = scripts.map(([location, script]) => [location, scripthash(script)]) + console.log = back return { cache, classes, locks, scripthashes } } From 3c4c28cb4bc325d001629c333b21058f8193f710 Mon Sep 17 00:00:00 2001 From: Miguel Duarte Date: Tue, 11 Jan 2022 18:10:56 -0300 Subject: [PATCH 24/82] little refactors. --- .gitignore | 3 ++- src/data-sources/sqlite-datasource.js | 3 ++- src/data-sources/sqlite-mixed-datasource.js | 4 ++-- src/database.js | 6 ++---- test/crawler.test.js | 3 +-- 5 files changed, 9 insertions(+), 10 deletions(-) diff --git a/.gitignore b/.gitignore index 47d272d..602daf2 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ node_modules coverage run.db* .idea -*.sqlite3 +*.sqlite3* +.runcache \ No newline at end of file diff --git a/src/data-sources/sqlite-datasource.js b/src/data-sources/sqlite-datasource.js index cdf5357..bc4ed5d 100644 --- a/src/data-sources/sqlite-datasource.js +++ b/src/data-sources/sqlite-datasource.js @@ -570,7 +570,8 @@ class SqliteDatasource { } async checkTxWasExecuted (txid) { - return this.getTransactionWasExecutedStmt.raw(true).get(txid)[0] + const queryResult = this.getTransactionWasExecutedStmt.raw(true).get(txid) + return queryResult && !!queryResult[0] } async getTxHex (txid) { diff --git a/src/data-sources/sqlite-mixed-datasource.js b/src/data-sources/sqlite-mixed-datasource.js index c9feb7a..848f96b 100644 --- a/src/data-sources/sqlite-mixed-datasource.js +++ b/src/data-sources/sqlite-mixed-datasource.js @@ -4,7 +4,6 @@ * Layer between the database and the application */ const { SqliteDatasource } = require('./sqlite-datasource') -const fetch = require('node-fetch') const { HEIGHT_MEMPOOL } = require('../constants') // ------------------------------------------------------------------------------------------------ @@ -214,7 +213,7 @@ class SqliteMixedDatasource extends SqliteDatasource { } async setJigState (location, stateObject) { - // this.setJigMetadataStmt(location) + this.setJigMetadataStmt(location) await this.blobStorage.pushJigState(location, stateObject) } @@ -227,6 +226,7 @@ class SqliteMixedDatasource extends SqliteDatasource { } async setBerryState (location, stateObject) { + await this.setBerryMetadata(location) return this.blobStorage.pushJigState(location, stateObject) } diff --git a/src/database.js b/src/database.js index c68509d..0655024 100644 --- a/src/database.js +++ b/src/database.js @@ -216,12 +216,10 @@ class Database { for (const key of Object.keys(cache)) { if (key.startsWith('jig://')) { const location = key.slice('jig://'.length) - await this.ds.setJigMetadata(location) - // await this.ds.setJigState(location, cache[key]) + await this.ds.setJigState(location, cache[key]) } else if (key.startsWith('berry://')) { const location = key.slice('berry://'.length) - await this.ds.setBerryMetadata(location) - // await this.ds.setBerryState(location, cache[key]) + await this.ds.setBerryState(location, cache[key]) } } diff --git a/test/crawler.test.js b/test/crawler.test.js index eeed92e..b6424e3 100644 --- a/test/crawler.test.js +++ b/test/crawler.test.js @@ -145,6 +145,7 @@ describe('Crawler', () => { await indexer.start() await database.trust(txid) await reorged(indexer) + await indexed(indexer, txid) await indexer.stop() expect(await database.getTransactionHex(txid)).not.to.equal(undefined) const state = await database.getJigState(txid + '_o1') @@ -153,5 +154,3 @@ describe('Crawler', () => { expect(await database.getTransactionHeight(txid)).to.equal(-1) }) }) - -// ------------------------------------------------------------------------------------------------ From b2f5eda9b6f9bdbc38159975055b7d0e708ddbfe Mon Sep 17 00:00:00 2001 From: Miguel Duarte Date: Wed, 12 Jan 2022 15:22:52 -0300 Subject: [PATCH 25/82] Blob storage working --- .mocharv.yaml => .mocharc.yaml | 0 deleteme.js | 49 -------------- package.json | 3 + src/clock.js | 2 +- src/data-sources/api-blob-storage.js | 8 ++- src/data-sources/memory-blob-storage.js | 34 ++++++++++ src/data-sources/sqlite-mixed-datasource.js | 15 ++--- src/start.js | 4 +- src/worker.js | 19 ++---- test/setup.js | 5 ++ test/sqlite-mixed-datasource-test.js | 75 +++++++++++++++++++++ 11 files changed, 139 insertions(+), 75 deletions(-) rename .mocharv.yaml => .mocharc.yaml (100%) delete mode 100644 deleteme.js create mode 100644 src/data-sources/memory-blob-storage.js create mode 100644 test/setup.js create mode 100644 test/sqlite-mixed-datasource-test.js diff --git a/.mocharv.yaml b/.mocharc.yaml similarity index 100% rename from .mocharv.yaml rename to .mocharc.yaml diff --git a/deleteme.js b/deleteme.js deleted file mode 100644 index c2f0ff4..0000000 --- a/deleteme.js +++ /dev/null @@ -1,49 +0,0 @@ -const Run = require('run-sdk') - -const owner = 'cPepyUbXNEmJFqBcmrxFxuYr69v4Jn5qL9NtdA1wqch7GNZTBowE' -const purse = 'cVcVT1hpsJh5gvtgVWUH93HchkhtteHdWftqkzncwMtc1iXR7eU3' - -const run = new Run({ - cache: new Run.plugins.LocalCache(), - state: new Run.plugins.RunDB('http://localhost:3101'), - network: 'test', - owner, - purse -}) -run.trust('23c615454eaa787db6de70d1f61e8f1212c213278a4f6e031b9ca50010b8c283') -run.trust('8e76e558ea4e5383c22875d2eb442f9189cda9b60a187ec3cd307755903cda46') -run.blockchain.host = 'http://localhost:3000' - -class AThing extends Run.Jig { - init (initialThing) { - this.thing = initialThing - } - - change (newThing) { - this.thing = newThing - } -} - -const main = async () => { - // await run.transaction(() => { - // run.deploy(AThing) - // }) - // await run.sync() - // console.log(AThing.location) - // const Klass = await run.load('8e76e558ea4e5383c22875d2eb442f9189cda9b60a187ec3cd307755903cda46_o1') - // const instance = new Klass('a thing') - // await instance.sync() - // instance.change('shinier thing') - // await instance.sync() - // console.log(instance.location) - const instance = await run.load('97b4149941677ba2da82de7c994dc7eb0d3fabef7efbf53ae9173dd592626d68_o1') - await instance.sync() - // instance.change('shinier thing' + instance.location) - // await instance.sync() - console.log(instance.location) - console.log(instance.constructor.location) - - // console.log(await run.blockchain.utxos('n2nRMck2LEP7FBmWzWfS6T1EMzuSULnnsz')) -} - -main() \ No newline at end of file diff --git a/package.json b/package.json index 4666cdf..a2ea036 100644 --- a/package.json +++ b/package.json @@ -56,5 +56,8 @@ "mocha": "^8.3.2", "nyc": "^15.1.0", "standard": "^16.0.3" + }, + "standard": { + "env": "mocha" } } diff --git a/src/clock.js b/src/clock.js index 018e1c3..68a8379 100644 --- a/src/clock.js +++ b/src/clock.js @@ -18,4 +18,4 @@ class Clock { } } -module.exports = { Clock } \ No newline at end of file +module.exports = { Clock } diff --git a/src/data-sources/api-blob-storage.js b/src/data-sources/api-blob-storage.js index 7459ca4..80fd3b3 100644 --- a/src/data-sources/api-blob-storage.js +++ b/src/data-sources/api-blob-storage.js @@ -2,7 +2,7 @@ const fetch = require('node-fetch') class ApiBlobStorage { constructor (baseUri) { - this.baseUri = baseUri + this.baseApiUrl = baseUri } async pushJigState (location, stateObject) { @@ -36,10 +36,14 @@ class ApiBlobStorage { return json.state } - async fetchTx (txid) { + async pullTx (txid) { const result = await fetch(`${this.baseApiUrl}/rawtx/${txid}`) return result.buffer() } + + async pushTx (_rawTx) { + // do nothing + } } module.exports = { ApiBlobStorage } diff --git a/src/data-sources/memory-blob-storage.js b/src/data-sources/memory-blob-storage.js new file mode 100644 index 0000000..ef52324 --- /dev/null +++ b/src/data-sources/memory-blob-storage.js @@ -0,0 +1,34 @@ +const returnNull = () => null + +class MemoryBlobStorage { + constructor () { + this._states = new Map() + this._txs = new Map() + } + + async pushJigState (location, stateObject) { + this._states.set(location, stateObject) + } + + async pullJigState (location, ifNone = returnNull) { + const state = this._states.get(location) + if (!state) { + return ifNone(location) + } + return state + } + + async pushTx (txid, txBuff) { + this._txs.set(txid, txBuff) + } + + async pullTx (txid, ifNone = returnNull) { + const txBuff = this._txs.get(txid) + if (!txBuff) { + return ifNone(txid) + } + return txBuff + } +} + +module.exports = { MemoryBlobStorage } diff --git a/src/data-sources/sqlite-mixed-datasource.js b/src/data-sources/sqlite-mixed-datasource.js index 848f96b..4c1c28a 100644 --- a/src/data-sources/sqlite-mixed-datasource.js +++ b/src/data-sources/sqlite-mixed-datasource.js @@ -198,7 +198,10 @@ class SqliteMixedDatasource extends SqliteDatasource { } async getTxHex (txid) { - const buff = await this.blobStorage.fetchTx(txid) + const buff = await this.blobStorage.pullTx(txid, () => null) + if (buff === null) { + return null + } return buff.toString('hex') } @@ -213,21 +216,15 @@ class SqliteMixedDatasource extends SqliteDatasource { } async setJigState (location, stateObject) { - this.setJigMetadataStmt(location) - await this.blobStorage.pushJigState(location, stateObject) + this.setJigMetadataStmt.run(location) } async getBerryState (location) { return this.blobStorage.pullJigState(location) } - async setBerryMetadata (location) { - this.setBerryMetadataStmt.run(location) - } - async setBerryState (location, stateObject) { - await this.setBerryMetadata(location) - return this.blobStorage.pushJigState(location, stateObject) + this.setBerryMetadataStmt.run(location) } // tx diff --git a/src/start.js b/src/start.js index 8237ab6..48d73c7 100644 --- a/src/start.js +++ b/src/start.js @@ -21,6 +21,7 @@ const Database = require('./database') const DirectServer = require('./direct-server') const { SqliteDatasource } = require('./data-sources/sqlite-datasource') const { SqliteMixedDatasource } = require('./data-sources/sqlite-mixed-datasource') +const { ApiBlobStorage } = require('./data-sources/api-blob-storage') // ------------------------------------------------------------------------------------------------ // Globals @@ -57,7 +58,8 @@ let dataSource if (DATA_SOURCE === 'sqlite') { dataSource = new SqliteDatasource(DB, logger, readonly) } else if (DATA_SOURCE === 'mixed') { - dataSource = new SqliteMixedDatasource(DB, logger, readonly, DATA_API_ROOT) + const blobStorage = new ApiBlobStorage(DATA_API_ROOT) + dataSource = new SqliteMixedDatasource(DB, logger, readonly, blobStorage) } else { throw new Error(`unknown datasource: ${DATA_SOURCE}. Please check "DATA_SOURCE" configuration.`) } diff --git a/src/worker.js b/src/worker.js index 4727bd1..cb7b597 100644 --- a/src/worker.js +++ b/src/worker.js @@ -10,8 +10,6 @@ const Run = require('run-sdk') const bsv = require('bsv') const Bus = require('./bus') const config = require('./config') -const Database = require('./database') -const { SqliteMixedDatasource } = require('./data-sources/sqlite-mixed-datasource') const { DEBUG } = require('./config') const { ApiBlobStorage } = require('./data-sources/api-blob-storage') @@ -22,7 +20,7 @@ const { ApiBlobStorage } = require('./data-sources/api-blob-storage') const network = workerData.network const cacheType = workerData.cacheType const trustSource = workerData.trustSource -const id = workerData.id +// const id = workerData.id Bus.listen(parentPort, { execute }) @@ -66,19 +64,15 @@ class DirectCache { if (value) { return value } const [type, identifier] = key.split('://') - if (type === 'jig') { - const jig = await this.blobs.getJigState(identifier) + if (type === 'jig' || type === 'berry') { + const jig = await this.blobs.pullJigState(identifier) this.state[key] = jig return jig } else if (type === 'tx') { - const rawTx = await this.blobs.fetchTx(identifier) + const rawTx = await this.blobs.pullTx(identifier) const txHex = rawTx.toString('hex') this.state[key] = txHex return txHex - } else if (type === 'berry') { - const berry = await this.blobs.getBerryState(identifier) - this.state[key] = berry - return berry } else { return null } @@ -135,9 +129,8 @@ async function execute (txid, hex, trustlist) { console.log = function () {} console.log() if (cacheType === 'direct') { - const ds = new ApiBlobStorage(config.DATA_API_ROOT) - const db = new Database(ds, logger) - run.cache = new DirectCache(db) + const bs = new ApiBlobStorage(config.DATA_API_ROOT) + run.cache = new DirectCache(bs) } else { run.cache = new Cache() } diff --git a/test/setup.js b/test/setup.js new file mode 100644 index 0000000..9ce3c55 --- /dev/null +++ b/test/setup.js @@ -0,0 +1,5 @@ +const chai = require('chai') +const chaiAsPromised = require('chai-as-promised') + +chai.expect() +chai.use(chaiAsPromised) diff --git a/test/sqlite-mixed-datasource-test.js b/test/sqlite-mixed-datasource-test.js new file mode 100644 index 0000000..fec0e76 --- /dev/null +++ b/test/sqlite-mixed-datasource-test.js @@ -0,0 +1,75 @@ +const { SqliteMixedDatasource } = require('../src/data-sources/sqlite-mixed-datasource') +const { MemoryBlobStorage } = require('../src/data-sources/memory-blob-storage') +const { expect } = require('chai') +const txns = require('./txns.json') + +const logger = { info: () => {}, warn: () => {}, error: () => {}, debug: () => {} } + +describe('SqliteMixedDataSource', () => { + let blobStorage = null + let ds + + beforeEach(async () => { + blobStorage = new MemoryBlobStorage() + ds = new SqliteMixedDatasource(':memory:', logger, false, blobStorage) + await ds.setUp() + }) + + describe('tx management', () => { + it('returns null for a tx that is not in the blob storage', async () => { + const result = await ds.getTxHex('doesnotexists') + expect(result).to.eql(null) + }) + + it('returns a tx that was defined in the storage', async () => { + const txid = 'bfa5180e601e92af23d80782bf625b102ac110105a392e376fe7607e4e87dc8d' + const txHex = txns[txid] + blobStorage.pushTx(txid, Buffer.from(txHex, 'hex')) + + const result = await ds.getTxHex(txid) + expect(result).to.eql(txHex) + }) + + it('does not send the tx hex when registering the tx', async () => { + const txid = 'bfa5180e601e92af23d80782bf625b102ac110105a392e376fe7607e4e87dc8d' + const txHex = txns[txid] + await ds.setTxBytes(txid, Buffer.from(txHex, 'hex')) + expect(await blobStorage.pullTx(txid, () => null)).to.eql(null) + }) + }) + + describe('state management', () => { + it('returns null for a state that is not in the blob storage', async () => { + const result = await ds.getJigState('doesNotExists') + expect(result).to.eql(null) + }) + + it('returns a state that is present in the blob storage.', async () => { + const location = 'someLocation' + await blobStorage.pushJigState(location, { state: true }) + const result = await ds.getJigState(location) + expect(result).to.eql({ state: true }) + }) + + it('returns a berry state that is present in the blob storage.', async () => { + const location = 'someLocation' + await blobStorage.pushJigState(location, { state: true }) + const result = await ds.getBerryState(location) + expect(result).to.eql({ state: true }) + }) + + it('does not push the state on the publishing', async () => { + const location = 'somelocation' + const state = { state: true } + await ds.setJigState(location, state) + expect(await blobStorage.pullJigState(location, () => null)).to.eql(null) + }) + + it('does not push berry state on set', async () => { + const location = 'somelocation' + const state = { state: true } + await ds.setBerryState(location, state) + expect(await blobStorage.pullJigState(location, () => null)).to.eql(null) + }) + }) +}) From f333cebaa070b49a0127c52e7638b0a4ecafd247 Mon Sep 17 00:00:00 2001 From: Miguel Duarte Date: Thu, 13 Jan 2022 17:30:25 -0300 Subject: [PATCH 26/82] trustlist working. --- src/config.js | 4 +- src/data-sources/sqlite-datasource.js | 52 ++++++++++++++-- src/data-sources/sqlite-mixed-datasource.js | 59 +++++++++++++------ src/database.js | 54 ++++------------- src/executor.js | 7 +-- src/start.js | 13 +++- src/trust-list/base-trust-list.js | 19 ++++++ src/trust-list/db-trust-list.js | 43 ++++++++++++++ src/trust-list/trust-all-trust-list.js | 23 ++++++++ src/worker.js | 7 +-- test/crawler.test.js | 4 +- test/indexer.test.js | 42 ++++++++++++- test/server.test.js | 4 +- ...est.js => sqlite-mixed-datasource.test.js} | 0 14 files changed, 247 insertions(+), 84 deletions(-) create mode 100644 src/trust-list/base-trust-list.js create mode 100644 src/trust-list/db-trust-list.js create mode 100644 src/trust-list/trust-all-trust-list.js rename test/{sqlite-mixed-datasource-test.js => sqlite-mixed-datasource.test.js} (100%) diff --git a/src/config.js b/src/config.js index 075ab5a..5211f8f 100644 --- a/src/config.js +++ b/src/config.js @@ -29,6 +29,7 @@ const DATA_SOURCE = process.env.DATA_SOURCE || 'sqlite' const DATA_API_ROOT = process.env.DATA_API_ROOT || null const WORKER_TRUST_SOURCE = process.env.WORKER_TRUST_SOURCE || 'db' const WORKER_CACHE_TYPE = process.env.WORKER_CACHE_TYPE || 'parent' +const TRUST_LIST = process.env.TRUST_LIST || 'db' require('axios').default.defaults.timeout = TIMEOUT @@ -123,5 +124,6 @@ module.exports = { DATA_SOURCE, DATA_API_ROOT, WORKER_TRUST_SOURCE, - WORKER_CACHE_TYPE + WORKER_CACHE_TYPE, + TRUST_LIST } diff --git a/src/data-sources/sqlite-datasource.js b/src/data-sources/sqlite-datasource.js index bc4ed5d..1bea489 100644 --- a/src/data-sources/sqlite-datasource.js +++ b/src/data-sources/sqlite-datasource.js @@ -10,7 +10,7 @@ const fastq = require('fastq') // The + in the following 2 queries before downloaded improves performance by NOT using the // tx_downloaded index, which is rarely an improvement over a simple filter for single txns. // See: https://www.sqlite.org/optoverview.html -const IS_READY_TO_EXECUTE_SQL = ` +const TRUSTED_AND_READY_TO_EXECUTE_SQL = ` SELECT ( downloaded = 1 AND executable = 1 @@ -30,6 +30,24 @@ const IS_READY_TO_EXECUTE_SQL = ` WHERE txid = ? ` +const READY_TO_EXECUTE_SQL = ` + SELECT ( + downloaded = 1 + AND executable = 1 + AND executed = 0 + AND ( + SELECT COUNT(*) + FROM tx AS tx2 + JOIN deps + ON deps.up = tx2.txid + WHERE deps.down = tx.txid + AND (+tx2.downloaded = 0 OR (tx2.executable = 1 AND tx2.executed = 0)) + ) = 0 + ) AS ready + FROM tx + WHERE txid = ? +` + const GET_DOWNSTREAM_READY_TO_EXECUTE_SQL = ` SELECT down FROM deps @@ -113,8 +131,17 @@ class SqliteDatasource { this.getTransactionsToDownloadStmt = this.connection.prepare('SELECT txid FROM tx WHERE downloaded = 0') this.getTransactionsDownloadedCountStmt = this.connection.prepare('SELECT COUNT(*) AS count FROM tx WHERE downloaded = 1') this.getTransactionsIndexedCountStmt = this.connection.prepare('SELECT COUNT(*) AS count FROM tx WHERE indexed = 1') - this.isReadyToExecuteStmt = this.connection.prepare(IS_READY_TO_EXECUTE_SQL) + this.isTrustedAndReadyToExecuteStmt = this.connection.prepare(TRUSTED_AND_READY_TO_EXECUTE_SQL) + this.isReadyToExecuteStmt = this.connection.prepare(READY_TO_EXECUTE_SQL) + this.depsExecutedOkStmt = this.connection.prepare(` + SELECT COUNT(*) = 0 as ok + FROM tx + JOIN deps ON deps.up = tx.txid + WHERE deps.down = ? + AND (+tx.downloaded = 0 OR (tx.executable = 1 AND tx.executed = 0)) + `) this.getDownstreamReadyToExecuteStmt = this.connection.prepare(GET_DOWNSTREAM_READY_TO_EXECUTE_SQL) + this.getTxMetadataStmt = this.connection.prepare('SELECT * FROM tx WHERE txid = ?') this.setSpendStmt = this.connection.prepare('INSERT OR REPLACE INTO spends (location, spend_txid) VALUES (?, ?)') this.setUnspentStmt = this.connection.prepare('INSERT OR IGNORE INTO spends (location, spend_txid) VALUES (?, null)') @@ -446,7 +473,7 @@ class SqliteDatasource { `) const txids = stmt.raw(true).all().map(x => x[0]) - const isReadyToExecuteStmt = this.connection.prepare(IS_READY_TO_EXECUTE_SQL) + const isReadyToExecuteStmt = this.connection.prepare(TRUSTED_AND_READY_TO_EXECUTE_SQL) const ready = [] for (let i = 0; i < txids.length; i++) { @@ -597,6 +624,10 @@ class SqliteDatasource { this.unconfirmTransactionStmt.run(txid) } + async getTxMetadata (txid) { + return this.getTxMetadataStmt.get(txid) + } + // executing async markTxAsExecuting (txid) { @@ -611,8 +642,19 @@ class SqliteDatasource { return this.findAllExecutingTxidsStmt.raw(true).all().map(x => x[0]) } - async txidReadyToExecute (txid) { - return this.isReadyToExecuteStmt.get(txid) + async txidTrustedAndReadyToExecute (txid) { + const row = this.isTrustedAndReadyToExecuteStmt.get(txid) + return row && row.ready + } + + async txidIsReadyToExecute (txid) { + const row = this.isReadyToExecuteStmt.get(txid) + return row && row.ready + } + + async checkDependenciesWereExecutedOk (txid) { + const row = this.depsExecutedOkStmt.get(txid) + return row && row.ok } // spends diff --git a/src/data-sources/sqlite-mixed-datasource.js b/src/data-sources/sqlite-mixed-datasource.js index 4c1c28a..403cef0 100644 --- a/src/data-sources/sqlite-mixed-datasource.js +++ b/src/data-sources/sqlite-mixed-datasource.js @@ -10,23 +10,44 @@ const { HEIGHT_MEMPOOL } = require('../constants') // Database // ------------------------------------------------------------------------------------------------ -const IS_READY_TO_EXECUTE_SQL = ` - SELECT ( - executable = 1 AND - executed = 0 AND - ( - SELECT count(*) - FROM tx AS txInner - JOIN deps - ON deps.up = txInner.txid - WHERE deps.down = txOuter.txid AND - txInner.executable = 1 AND - txInner.indexed = 0 - ) = 0 - ) AS ready - FROM tx as txOuter - WHERE txOuter.txid = ?; - ` +const TRUSTED_AND_READY_TO_EXECUTE_SQL = ` + SELECT ( + executable = 1 + AND executed = 0 + AND (has_code = 0 OR (SELECT COUNT(*) FROM trust WHERE trust.txid = txOuter.txid AND trust.value = 1) = 1) + AND txid NOT IN ban + AND ( + SELECT count(*) + FROM tx AS txDeps + JOIN deps + ON deps.up = txDeps.txid + WHERE deps.down = txOuter.txid AND + txDeps.executable = 1 AND + txDeps.indexed = 0 + ) = 0 + ) AS ready + FROM tx as txOuter + WHERE txid = ? +` + +const READY_TO_EXECUTE_SQL = ` + SELECT ( + executable = 1 + AND executed = 0 + AND ( + SELECT count(*) + FROM tx AS txDeps + JOIN deps + ON deps.up = txDeps.txid + WHERE deps + .down = txOuter.txid AND + txDeps.executable = 1 AND + txDeps.indexed = 0 + ) = 0 + ) AS ready + FROM tx as txOuter + WHERE txid = ? +` const GET_DOWNSTREAM_READY_TO_EXECUTE_SQL = ` SELECT down @@ -78,8 +99,10 @@ class SqliteMixedDatasource extends SqliteDatasource { this.getTransactionsToDownloadStmt = this.connection.prepare('SELECT txid FROM tx WHERE downloaded = 0') this.getTransactionsDownloadedCountStmt = this.connection.prepare('SELECT COUNT(*) AS count FROM tx WHERE downloaded = 1') this.getTransactionsIndexedCountStmt = this.connection.prepare('SELECT COUNT(*) AS count FROM tx WHERE indexed = 1') - this.isReadyToExecuteStmt = this.connection.prepare(IS_READY_TO_EXECUTE_SQL) + this.isTrustedAndReadyToExecuteStmt = this.connection.prepare(TRUSTED_AND_READY_TO_EXECUTE_SQL) + this.isReadyToExecuteStmt = this.connection.prepare(READY_TO_EXECUTE_SQL) this.getDownstreamReadyToExecuteStmt = this.connection.prepare(GET_DOWNSTREAM_READY_TO_EXECUTE_SQL) + this.getTxMetadataStmt = this.connection.prepare('SELECT * FROM tx WHERE txid = ?') this.setSpendStmt = this.connection.prepare('INSERT OR REPLACE INTO spends (location, spend_txid) VALUES (?, ?)') this.setUnspentStmt = this.connection.prepare('INSERT OR IGNORE INTO spends (location, spend_txid) VALUES (?, null)') diff --git a/src/database.js b/src/database.js index 0655024..ce5a088 100644 --- a/src/database.js +++ b/src/database.js @@ -12,8 +12,9 @@ const { HEIGHT_MEMPOOL, HEIGHT_UNKNOWN } = require('./constants') // ------------------------------------------------------------------------------------------------ class Database { - constructor (datasource, logger) { + constructor (datasource, trustList, logger) { this.ds = datasource + this.trustList = trustList this.logger = logger this.onReadyToExecute = null @@ -356,6 +357,10 @@ class Database { return this.ds.searchTxsToDownload() } + async getTxMetadata (txid) { + return this.ds.getTxMetadata(txid) + } + async getDownloadedCount () { return this.ds.countDownloadedTxs() } @@ -462,26 +467,7 @@ class Database { } async trust (txid) { - if (await this.isTrusted(txid)) return - - const trusted = [txid] - - // Recursively trust code parents - const queue = await this.ds.getNonExecutedUpstreamTxIds(txid) - const visited = new Set() - while (queue.length) { - const uptxid = queue.shift() - if (visited.has(uptxid)) continue - if (await this.isTrusted(uptxid)) continue - visited.add(uptxid) - trusted.push(txid) - const nextTxids = await this.ds.getNonExecutedUpstreamTxIds(uptxid) - nextTxids.forEach(txid => queue.push(txid)) - } - - for (const trustedTxid of trusted) { - await this.ds.setTrust(trustedTxid, 1) - } + const trusted = await this.trustList.trust(txid) for (const txid of trusted) { await this._checkExecutability(txid) @@ -495,16 +481,15 @@ class Database { } async untrust (txid) { - if (!await this.isTrusted(txid)) return await this.ds.performOnTransaction(async () => { await this.unindexTransaction(txid) - await this.ds.setTrust(txid, 0) + await this.trustList.untrust(txid) }) if (this.onUntrustTransaction) await this.onUntrustTransaction(txid) } async getTrustlist () { - return this.ds.searchAllTrust() + return this.trustList.executionTrustList() } // -------------------------------------------------------------------------- @@ -567,26 +552,9 @@ class Database { } async _checkExecutability (txid) { - const executed = await this.ds.checkTxWasExecuted(txid) - if (executed) { - await this.ds.removeTxFromExecuting(txid) - return - } - const someDepFailed = await this.ds.hasFailedDep(txid) - if (someDepFailed) { - await this.ds.setExecutedForTx(txid, 1) - await this.ds.setIndexedForTx(txid, 0) - await this.ds.removeTxFromExecuting(txid) - const downstreamReadyToExecute = await this.ds.searchDownstreamTxidsReadyToExecute(txid) - for (const downtxid of downstreamReadyToExecute) { - await this._checkExecutability(downtxid) - } - return - } - - const row = await this.ds.txidReadyToExecute(txid) + const executable = await this.trustList.checkExecutability(txid) - if (row && row.ready) { + if (executable) { await this.ds.markTxAsExecuting(txid) if (this.onReadyToExecute) { await this.onReadyToExecute(txid) } } diff --git a/src/executor.js b/src/executor.js index 10b6c29..54fbf51 100644 --- a/src/executor.js +++ b/src/executor.js @@ -18,8 +18,7 @@ class Executor { this.database = database this.logger = logger this.workerOpts = { - cacheType: opts.cacheType || 'parentConnection', - trustSource: opts.trustSource || 'db' + cacheType: opts.cacheType || 'parentConnection' } this.onIndexed = null @@ -138,10 +137,6 @@ class Executor { worker.missingDeps.add(txid) throw new Error(`Not found: ${txid}`) } - - _onTrustlistGet () { - return this.database.getTrustlist() - } } // ------------------------------------------------------------------------------------------------ diff --git a/src/start.js b/src/start.js index 48d73c7..c45b262 100644 --- a/src/start.js +++ b/src/start.js @@ -9,7 +9,7 @@ const Server = require('./server') const { API, DB, NETWORK, PORT, FETCH_LIMIT, WORKERS, MATTERCLOUD_KEY, PLANARIA_TOKEN, START_HEIGHT, MEMPOOL_EXPIRATION, ZMQ_URL, RPC_URL, DEFAULT_TRUSTLIST, DEBUG, SERVE_ONLY, DATA_SOURCE, DATA_API_ROOT, - WORKER_TRUST_SOURCE, WORKER_CACHE_TYPE + WORKER_TRUST_SOURCE, WORKER_CACHE_TYPE, TRUST_LIST } = require('./config') const MatterCloud = require('./mattercloud') const Planaria = require('./planaria') @@ -22,6 +22,8 @@ const DirectServer = require('./direct-server') const { SqliteDatasource } = require('./data-sources/sqlite-datasource') const { SqliteMixedDatasource } = require('./data-sources/sqlite-mixed-datasource') const { ApiBlobStorage } = require('./data-sources/api-blob-storage') +const { DbTrustList } = require('./trust-list/db-trust-list') +const { TrustAllTrustList } = require('./trust-list/trust-all-trust-list') // ------------------------------------------------------------------------------------------------ // Globals @@ -64,7 +66,14 @@ if (DATA_SOURCE === 'sqlite') { throw new Error(`unknown datasource: ${DATA_SOURCE}. Please check "DATA_SOURCE" configuration.`) } -const database = new Database(dataSource, logger) +let trustList +if (TRUST_LIST === 'db') { + trustList = new DbTrustList(dataSource) +} else if (TRUST_LIST === 'all') { + trustList = new TrustAllTrustList(dataSource) +} + +const database = new Database(dataSource, trustList, logger) const indexer = new Indexer(database, api, NETWORK, FETCH_LIMIT, WORKERS, logger, START_HEIGHT, MEMPOOL_EXPIRATION, DEFAULT_TRUSTLIST, { diff --git a/src/trust-list/base-trust-list.js b/src/trust-list/base-trust-list.js new file mode 100644 index 0000000..319cca8 --- /dev/null +++ b/src/trust-list/base-trust-list.js @@ -0,0 +1,19 @@ +class BaseTrustList { + async executionTrustList () { + throw new Error('subclass responsibility') + } + + async checkExecutability (_txid) { + throw new Error('subclass responsibility') + } + + async trust (_txid) { + throw new Error('subclass responsibility') + } + + async untrust (_txid) { + throw new Error('subclass responsibility') + } +} + +module.exports = { BaseTrustList } \ No newline at end of file diff --git a/src/trust-list/db-trust-list.js b/src/trust-list/db-trust-list.js new file mode 100644 index 0000000..00f3a57 --- /dev/null +++ b/src/trust-list/db-trust-list.js @@ -0,0 +1,43 @@ +class DbTrustList { + constructor (ds) { + this.ds = ds + } + + async executionTrustList () { + return this.ds.searchAllTrust() + } + + async checkExecutability (txid) { + return this.ds.txidTrustedAndReadyToExecute(txid) + } + + async trust (txid) { + if (await this.ds.isTrusted(txid)) return [] + + const trusted = [txid] + + // Recursively trust code parents + const queue = await this.ds.getNonExecutedUpstreamTxIds(txid) + const visited = new Set() + while (queue.length) { + const uptxid = queue.shift() + if (visited.has(uptxid)) continue + if (await this.ds.isTrusted(uptxid)) continue + visited.add(uptxid) + trusted.push(txid) + const nextTxids = await this.ds.getNonExecutedUpstreamTxIds(uptxid) + nextTxids.forEach(txid => queue.push(txid)) + } + + for (const trustedTxid of trusted) { + await this.ds.setTrust(trustedTxid, 1) + } + return trusted + } + + async untrust (txid) { + await this.ds.setTrust(txid, 0) + } +} + +module.exports = { DbTrustList } \ No newline at end of file diff --git a/src/trust-list/trust-all-trust-list.js b/src/trust-list/trust-all-trust-list.js new file mode 100644 index 0000000..1d4c8bb --- /dev/null +++ b/src/trust-list/trust-all-trust-list.js @@ -0,0 +1,23 @@ +class TrustAllTrustList { + constructor (ds) { + this.ds = ds + } + + async executionTrustList () { + return ['*'] + } + + async checkExecutability (txid) { + return this.ds.txidIsReadyToExecute(txid) + } + + async trust (txid) { + return [txid] + } + + async untrust (_txid) { + // do nothing + } +} + +module.exports = { TrustAllTrustList } \ No newline at end of file diff --git a/src/worker.js b/src/worker.js index cb7b597..ecd545d 100644 --- a/src/worker.js +++ b/src/worker.js @@ -19,7 +19,6 @@ const { ApiBlobStorage } = require('./data-sources/api-blob-storage') const network = workerData.network const cacheType = workerData.cacheType -const trustSource = workerData.trustSource // const id = workerData.id Bus.listen(parentPort, { execute }) @@ -141,11 +140,7 @@ async function execute (txid, hex, trustlist) { run.client = false run.preverify = false - if (trustSource === 'all') { - run.trust('*') - } else { - trustlist.forEach(txid => run.trust(txid)) - } + trustlist.forEach(txid => run.trust(txid)) run.trust('cache') const tx = await run.import(hex, { txid }) diff --git a/test/crawler.test.js b/test/crawler.test.js index b6424e3..d4d1cb4 100644 --- a/test/crawler.test.js +++ b/test/crawler.test.js @@ -11,6 +11,7 @@ const txns = require('./txns.json') const { DEFAULT_TRUSTLIST } = require('../src/config') const Database = require('../src/database') const { SqliteDatasource } = require('../src/data-sources/sqlite-datasource') +const { DbTrustList } = require('../src/trust-list/db-trust-list') // ------------------------------------------------------------------------------------------------ // Globals @@ -22,7 +23,8 @@ const crawled = (indexer) => new Promise((resolve) => { indexer.onBlock = height const reorged = (indexer) => new Promise((resolve) => { indexer.onReorg = newHeight => resolve(newHeight) }) const logger = { info: () => {}, warn: () => {}, error: () => {}, debug: () => {} } const ds = new SqliteDatasource(':memory:', logger, false) -const database = new Database(ds, logger) +const trustList = new DbTrustList(ds) +const database = new Database(ds, trustList, logger) beforeEach(() => database.open()) afterEach(() => database.close()) diff --git a/test/indexer.test.js b/test/indexer.test.js index 2c99feb..056601c 100644 --- a/test/indexer.test.js +++ b/test/indexer.test.js @@ -13,6 +13,7 @@ const { Jig } = Run const { DEFAULT_TRUSTLIST } = require('../src/config') const Database = require('../src/database') const { SqliteDatasource } = require('../src/data-sources/sqlite-datasource') +const { DbTrustList } = require('../src/trust-list/db-trust-list') // ------------------------------------------------------------------------------------------------ // Globals @@ -24,7 +25,8 @@ const indexed = (indexer, txid) => new Promise((resolve) => { indexer.onIndex = const failed = (indexer, txid) => new Promise((resolve) => { indexer.onFailToIndex = x => txid === x && resolve() }) const logger = { info: () => {}, warn: () => {}, error: () => {}, debug: () => {} } const ds = new SqliteDatasource(':memory:', logger, false) -const database = new Database(ds, logger) +const trustList = new DbTrustList(ds) +const database = new Database(ds, trustList, logger) beforeEach(() => database.open()) afterEach(() => database.close()) @@ -196,6 +198,44 @@ describe('Indexer', () => { expect(await indexer.database.getNumUnspent()).to.equal(0) await indexer.stop() }) + + it('mark a transaction as failed when a dependency already failed', async () => { + const run = new Run({ network: 'mock' }) + class Counter extends Run.Jig { + init () { this.count = 0 } + inc () { this.count += 1 } + } + run.deploy(Counter) + await run.sync() + const instance = new Counter() + await run.sync() + instance.inc() + await run.sync() + + const txid1 = Counter.location.split('_')[0] + const txid2 = instance.origin.split('_')[0] + const txid3 = instance.location.split('_')[0] + + const txHex1 = await run.blockchain.fetch(txid1) + const txHex2 = await run.blockchain.fetch(txid2) + const txHex3 = await run.blockchain.fetch(txid3) + + const indexer = new IndexerTest(database, run.blockchain, 'test', 1, 1, logger, 0, Infinity, []) + await indexer.start() + const promise = indexed(indexer, txid2) + await database.trust(txid1) + await database.addTransaction(txid1, txHex1) + await database.addTransaction(txid2, txHex2) + await promise + await database.setTransactionExecutionFailed(txid2) + + await database.addTransaction(txid3, txHex3) + + const metadata = await database.getTxMetadata(txid3) + expect(metadata.executable).to.eql(0) + + await indexer.stop() + }) }) // ------------------------------------------------------------------------------------------------ diff --git a/test/server.test.js b/test/server.test.js index 5b5c163..8817e61 100644 --- a/test/server.test.js +++ b/test/server.test.js @@ -14,6 +14,7 @@ const txns = require('./txns.json') const { DEFAULT_TRUSTLIST } = require('../src/config') const Database = require('../src/database') const { SqliteDatasource } = require('../src/data-sources/sqlite-datasource') +const { DbTrustList } = require('../src/trust-list/db-trust-list') // ------------------------------------------------------------------------------------------------ // Globals @@ -28,7 +29,8 @@ const indexed = (indexer, txid) => new Promise((resolve) => { indexer.onIndex = const listening = (server) => new Promise((resolve) => { server.onListening = () => resolve() }) const logger = { info: () => {}, warn: () => {}, error: () => {}, debug: () => {} } const ds = new SqliteDatasource(':memory:', logger, false) -const database = new Database(ds, logger) +const trustList = new DbTrustList(ds) +const database = new Database(ds, trustList, logger) beforeEach(() => database.open()) afterEach(() => database.close()) diff --git a/test/sqlite-mixed-datasource-test.js b/test/sqlite-mixed-datasource.test.js similarity index 100% rename from test/sqlite-mixed-datasource-test.js rename to test/sqlite-mixed-datasource.test.js From 6a337958a068e9ed1ae7e1984943d11437f9c147 Mon Sep 17 00:00:00 2001 From: Miguel Duarte Date: Thu, 13 Jan 2022 17:32:56 -0300 Subject: [PATCH 27/82] lint fixes. --- src/trust-list/base-trust-list.js | 2 +- src/trust-list/db-trust-list.js | 2 +- src/trust-list/trust-all-trust-list.js | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/trust-list/base-trust-list.js b/src/trust-list/base-trust-list.js index 319cca8..a45fbd1 100644 --- a/src/trust-list/base-trust-list.js +++ b/src/trust-list/base-trust-list.js @@ -16,4 +16,4 @@ class BaseTrustList { } } -module.exports = { BaseTrustList } \ No newline at end of file +module.exports = { BaseTrustList } diff --git a/src/trust-list/db-trust-list.js b/src/trust-list/db-trust-list.js index 00f3a57..ddc7f41 100644 --- a/src/trust-list/db-trust-list.js +++ b/src/trust-list/db-trust-list.js @@ -40,4 +40,4 @@ class DbTrustList { } } -module.exports = { DbTrustList } \ No newline at end of file +module.exports = { DbTrustList } diff --git a/src/trust-list/trust-all-trust-list.js b/src/trust-list/trust-all-trust-list.js index 1d4c8bb..5c9c46e 100644 --- a/src/trust-list/trust-all-trust-list.js +++ b/src/trust-list/trust-all-trust-list.js @@ -20,4 +20,4 @@ class TrustAllTrustList { } } -module.exports = { TrustAllTrustList } \ No newline at end of file +module.exports = { TrustAllTrustList } From feb799c6dbe09784d74935816894001d967839c6 Mon Sep 17 00:00:00 2001 From: hojarasca Date: Mon, 17 Jan 2022 16:24:36 -0300 Subject: [PATCH 28/82] some change in package.lock. --- package-lock.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package-lock.json b/package-lock.json index d9ccb31..81e94f6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5,6 +5,7 @@ "requires": true, "packages": { "": { + "name": "run-db", "version": "1.0.28", "license": "MIT", "dependencies": { From 1ef4ec0380fb59723e409895b4761eeb91b64ba6 Mon Sep 17 00:00:00 2001 From: hojarasca Date: Tue, 18 Jan 2022 12:03:55 -0300 Subject: [PATCH 29/82] fix api blob storage for berries. --- src/data-sources/api-blob-storage.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/data-sources/api-blob-storage.js b/src/data-sources/api-blob-storage.js index 80fd3b3..fc97242 100644 --- a/src/data-sources/api-blob-storage.js +++ b/src/data-sources/api-blob-storage.js @@ -23,7 +23,7 @@ class ApiBlobStorage { } async pullJigState (location) { - const result = await fetch(`${this.baseApiUrl}/state/${location}`) + const result = await fetch(`${this.baseApiUrl}/state/${encodeURIComponent(location)}`) if (result.status === 404) { return null } From 74d6c080ae722543041a01dd2315ee9aa9be0945 Mon Sep 17 00:00:00 2001 From: hojarasca Date: Tue, 18 Jan 2022 14:53:01 -0300 Subject: [PATCH 30/82] generic server. --- package-lock.json | 25 ++++ package.json | 2 + src/build-server.js | 182 ++++++++++++++++++++++++++++++ src/execution/execution-server.js | 5 + src/http/api-server.js | 72 ++++++++++++ src/start.js | 11 +- 6 files changed, 292 insertions(+), 5 deletions(-) create mode 100644 src/build-server.js create mode 100644 src/execution/execution-server.js create mode 100644 src/http/api-server.js diff --git a/package-lock.json b/package-lock.json index 81e94f6..4872228 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,7 +18,9 @@ "event-stream": "^4.0.1", "eventsource": "^1.1.0", "express": "^4.17.1", + "express-async-handler": "^1.2.0", "fastq": "^1.13.0", + "helmet": "^5.0.1", "knex": "^0.95.14", "morgan": "^1.10.0", "nanoid": "^3.1.30", @@ -2389,6 +2391,11 @@ "node": ">= 0.10.0" } }, + "node_modules/express-async-handler": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/express-async-handler/-/express-async-handler-1.2.0.tgz", + "integrity": "sha512-rCSVtPXRmQSW8rmik/AIb2P0op6l7r1fMW538yyvTMltCO4xQEWMmobfrIxN2V1/mVrgxB8Az3reYF6yUZw37w==" + }, "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -2840,6 +2847,14 @@ "he": "bin/he" } }, + "node_modules/helmet": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/helmet/-/helmet-5.0.1.tgz", + "integrity": "sha512-iyYpGYH2nbQVaQtauYDnemWg45S2RyGvJ+iKj+V9jp7Dc1NTtAJHmD+hFOSYS7Xdwe1GeyVEYSydggXLOg6TKQ==", + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/hmac-drbg": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/hmac-drbg/-/hmac-drbg-1.0.1.tgz", @@ -8140,6 +8155,11 @@ "vary": "~1.1.2" } }, + "express-async-handler": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/express-async-handler/-/express-async-handler-1.2.0.tgz", + "integrity": "sha512-rCSVtPXRmQSW8rmik/AIb2P0op6l7r1fMW538yyvTMltCO4xQEWMmobfrIxN2V1/mVrgxB8Az3reYF6yUZw37w==" + }, "fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -8503,6 +8523,11 @@ "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==", "dev": true }, + "helmet": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/helmet/-/helmet-5.0.1.tgz", + "integrity": "sha512-iyYpGYH2nbQVaQtauYDnemWg45S2RyGvJ+iKj+V9jp7Dc1NTtAJHmD+hFOSYS7Xdwe1GeyVEYSydggXLOg6TKQ==" + }, "hmac-drbg": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/hmac-drbg/-/hmac-drbg-1.0.1.tgz", diff --git a/package.json b/package.json index a2ea036..91d7ea8 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,9 @@ "event-stream": "^4.0.1", "eventsource": "^1.1.0", "express": "^4.17.1", + "express-async-handler": "^1.2.0", "fastq": "^1.13.0", + "helmet": "^5.0.1", "knex": "^0.95.14", "morgan": "^1.10.0", "nanoid": "^3.1.30", diff --git a/src/build-server.js b/src/build-server.js new file mode 100644 index 0000000..3c0c0b3 --- /dev/null +++ b/src/build-server.js @@ -0,0 +1,182 @@ +/** + * server.test.js + * + * Express server that exposes the Indexer + */ + +const bsv = require('bsv') +const crypto = require('crypto') +const Run = require('run-sdk') +const { ApiServer } = require('./http/api-server') + +// ------------------------------------------------------------------------------------------------ +// Globals +// ------------------------------------------------------------------------------------------------ + +const calculateScripthash = x => crypto.createHash('sha256').update(Buffer.from(x, 'hex')).digest().reverse().toString('hex') + +const parseTxid = (txid) => { + txid = txid.trim().toLowerCase() + if (!/^[0-9a-f]{64}$/.test(txid)) throw new Error('Not a txid: ' + txid) + return txid +} + +// ------------------------------------------------------------------------------------------------ +// Server +// ------------------------------------------------------------------------------------------------ + +const buildServer = (database, logger) => { + const server = new ApiServer(logger) + + server.get('/jig/:location', async (req, res) => { + const state = await database.getJigState(req.params.location) + if (state) { + res.setHeader('Content-Type', 'application/json') + res.send(state) + } else { + res.status(404).send(`Not found: ${req.params.location}\n`) + } + }) + + server.get('/berry/:location', async (req, res) => { + const state = await database.getBerryState(req.params.location) + if (state) { + res.setHeader('Content-Type', 'application/json') + res.send(state) + } else { + res.status(404).send(`Not found: ${req.params.location}\n`) + } + }) + + server.get('/tx/:txid', async (req, res) => { + const txid = parseTxid(req.params.txid) + const rawTx = await database.getTransactionHex(txid) + if (rawTx) { + res.send(rawTx) + } else { + res.status(404).send(`Not found: ${req.params.txid}\n`) + } + }) + + server.get('/time/:txid', async (req, res) => { + const txid = parseTxid(req.params.txid) + const time = await database.getTransactionTime(txid) + if (time) { + res.json(time) + } else { + res.status(404).send(`Not found: ${req.params.txid}\n`) + } + }) + + server.get('/spends/:location', async (req, res, next) => { + const txid = await database.getSpend(req.params.location) + if (txid) { + res.send(txid) + } else { + res.status(404).send(`Not spent: ${req.params.location}\n`) + } + }) + + server.get('/unspent', async (req, res) => { + const cls = req.query.class + const lock = req.query.lock + let scripthash = req.query.scripthash + if (req.query.address) scripthash = calculateScripthash(new Run.util.CommonLock(req.query.address).script()) + if (req.query.pubkey) scripthash = calculateScripthash(new Run.util.CommonLock(req.query.pubkey).script()) + + if (cls && lock && scripthash) { + res.json(await database.getAllUnspentByClassOriginAndLockOriginAndScripthash(cls, lock, scripthash)) + } else if (cls && lock) { + res.json(await database.getAllUnspentByClassOriginAndLockOrigin(cls, lock)) + } else if (cls && scripthash) { + res.json(await database.getAllUnspentByClassOriginAndScripthash(cls, scripthash)) + } else if (lock && scripthash) { + res.json(await database.getAllUnspentByLockOriginAndScripthash(lock, scripthash)) + } else if (scripthash) { + res.json(await database.getAllUnspentByScripthash(scripthash)) + } else if (lock) { + res.json(await database.getAllUnspentByLockOrigin(lock)) + } else if (cls) { + res.json(await database.getAllUnspentByClassOrigin(cls)) + } else { + res.json(await database.getAllUnspent()) + } + }) + + server.get('/trust/:txid?', async (req, res, next) => { + if (req.params.txid) { + res.json(await database.isTrusted(req.params.txid)) + } else { + res.json(Array.from(await database.getTrustlist())) + } + }) + + server.get('/ban/:txid?', async (req, res, next) => { + if (req.params.txid) { + res.json(await database.isBanned(req.params.txid)) + } else { + res.json(Array.from(await database.getBanlist())) + } + }) + + server.get('/status', async (req, res) => { + const status = { + height: await database.getHeight(), + hash: await database.getHash() + } + res.json(status) + }) + + server.post('/trust/:txid?', async (req, res) => { + if (Array.isArray(req.body)) { + for (const maybeTxid of req.body) { + const txid = parseTxid(maybeTxid) + await database.trust(txid) + } + res.send(`Trusted ${req.body.length} transactions\n`) + } else { + const txid = parseTxid(req.params.txid) + await database.trust(txid) + res.send(`Trusted ${req.params.txid}\n`) + } + }) + + server.post('/ban/:txid', async (req, res) => { + const txid = parseTxid(req.params.txid) + await database.ban(txid) + res.send(`Banned ${req.params.txid}\n`) + }) + + server.post('/tx/:txid?', async (req, res) => { + if (!(typeof req.body === 'string')) { + Error('Invalid request parameters') + } + + const hex = req.body + const bsvtx = new bsv.Transaction(hex) + await database.addTransaction(bsvtx.hash, hex) + res.send(`Added ${(bsvtx.hash)}\n`) + }) + + server.delete('/trust/:txid', async (req, res) => { + const txid = parseTxid(req.params.txid) + await database.untrust(txid) + res.send(`Untrusted ${req.params.txid}\n`) + }) + + server.delete('/ban/:txid', async (req, res) => { + const txid = parseTxid(req.params.txid) + await database.unban(txid) + res.send(`Unbanned ${req.params.txid}\n`) + }) + + server.delete('/tx/:txid', async (req, res) => { + const txid = parseTxid(req.params.txid) + await database.deleteTransaction(txid) + res.send(`Removed ${req.params.txid}\n`) + }) + + return server +} + +module.exports = { buildServer } diff --git a/src/execution/execution-server.js b/src/execution/execution-server.js new file mode 100644 index 0000000..647cb7d --- /dev/null +++ b/src/execution/execution-server.js @@ -0,0 +1,5 @@ +class ExecutionServer { + +} + +module.exports = { ExecutionServer } \ No newline at end of file diff --git a/src/http/api-server.js b/src/http/api-server.js new file mode 100644 index 0000000..128b1e0 --- /dev/null +++ b/src/http/api-server.js @@ -0,0 +1,72 @@ +const express = require('express') +const morgan = require('morgan') +const bodyParser = require('body-parser') +const cors = require('cors') +const { Writable } = require('stream') +const asyncHandler = require('express-async-handler') +const helmet = require('helmet') + +class ApiServer { + constructor (logger) { + this.logger = logger + this.logger.debug('Starting server') + this.listener = null + this.onListening = null + + const app = express() + this.app = app + + this.app.use(helmet()) + let buffer = '' + const write = (chunk, encoding, callback) => { + buffer = buffer + chunk.toString() + const lines = buffer.split(/\r\n|\n\r|\n|\r/) + for (let i = 0; i < lines.length - 1; i++) { + this.logger.info(lines[i]) + } + buffer = lines[lines.length - 1] + callback() + return true + } + app.use(morgan('tiny', { stream: new Writable({ write }) })) + + app.use(bodyParser.text({ limit: '25mb' })) + app.use(bodyParser.json({ limit: '10mb' })) + + app.use(cors()) + } + + async start (port = null) { + this.app.use((err, req, res, next) => { + if (this.logger) this.logger.error(err.stack) + res.status(500).send('Something broke!') + next() + }) + + this.port = port + this.listener = this.app.listen(port, () => { + if (this.logger) this.logger.info(`Listening at http://localhost:${port}`) + if (this.onListening) this.onListening() + }) + } + + stop () { + if (!this.listener) return + this.listener.close() + this.listener = null + } + + get (url, handler) { + this.app.get(url, asyncHandler(handler)) + } + + post (url, handler) { + this.app.post(url, asyncHandler(handler)) + } + + delete (url, handler) { + this.app.delete(url, asyncHandler(handler)) + } +} + +module.exports = { ApiServer } \ No newline at end of file diff --git a/src/start.js b/src/start.js index c45b262..669c5c5 100644 --- a/src/start.js +++ b/src/start.js @@ -5,7 +5,6 @@ */ const Indexer = require('./indexer') -const Server = require('./server') const { API, DB, NETWORK, PORT, FETCH_LIMIT, WORKERS, MATTERCLOUD_KEY, PLANARIA_TOKEN, START_HEIGHT, MEMPOOL_EXPIRATION, ZMQ_URL, RPC_URL, DEFAULT_TRUSTLIST, DEBUG, SERVE_ONLY, DATA_SOURCE, DATA_API_ROOT, @@ -24,6 +23,7 @@ const { SqliteMixedDatasource } = require('./data-sources/sqlite-mixed-datasourc const { ApiBlobStorage } = require('./data-sources/api-blob-storage') const { DbTrustList } = require('./trust-list/db-trust-list') const { TrustAllTrustList } = require('./trust-list/trust-all-trust-list') +const { buildServer } = require('./build-server') // ------------------------------------------------------------------------------------------------ // Globals @@ -81,9 +81,10 @@ const indexer = new Indexer(database, api, NETWORK, FETCH_LIMIT, WORKERS, logger cacheType: WORKER_CACHE_TYPE }) -const server = SERVE_ONLY - ? new Server(database, logger, PORT) - : new DirectServer(DB, PORT, logger, database) +// const server = SERVE_ONLY +// ? buildServer(database, logger) +// : new DirectServer(DB, PORT, logger, database) +const server = buildServer(database, logger) let started = false @@ -98,7 +99,7 @@ async function main () { await indexer.start() } - await server.start() + await server.start(PORT) started = true } From fb72d9eafe1d0fe31757dac8537235cef4b90470 Mon Sep 17 00:00:00 2001 From: hojarasca Date: Tue, 18 Jan 2022 14:53:43 -0300 Subject: [PATCH 31/82] created generic server 2 --- src/execution/execution-server.js | 2 +- src/http/api-server.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/execution/execution-server.js b/src/execution/execution-server.js index 647cb7d..5a75e3d 100644 --- a/src/execution/execution-server.js +++ b/src/execution/execution-server.js @@ -2,4 +2,4 @@ class ExecutionServer { } -module.exports = { ExecutionServer } \ No newline at end of file +module.exports = { ExecutionServer } diff --git a/src/http/api-server.js b/src/http/api-server.js index 128b1e0..63a7b86 100644 --- a/src/http/api-server.js +++ b/src/http/api-server.js @@ -69,4 +69,4 @@ class ApiServer { } } -module.exports = { ApiServer } \ No newline at end of file +module.exports = { ApiServer } From b9a29db6c815ce5eefec0b40709f2e463defee83 Mon Sep 17 00:00:00 2001 From: hojarasca Date: Tue, 18 Jan 2022 15:06:51 -0300 Subject: [PATCH 32/82] generic server working on tests. --- src/build-server.js | 6 ++++- src/direct-server.js | 51 ------------------------------------------ src/http/api-server.js | 1 + src/index.js | 2 -- src/start.js | 6 +---- test/server.test.js | 41 ++++++++++++++++----------------- 6 files changed, 28 insertions(+), 79 deletions(-) delete mode 100644 src/direct-server.js diff --git a/src/build-server.js b/src/build-server.js index 3c0c0b3..8f44403 100644 --- a/src/build-server.js +++ b/src/build-server.js @@ -25,7 +25,7 @@ const parseTxid = (txid) => { // Server // ------------------------------------------------------------------------------------------------ -const buildServer = (database, logger) => { +const buildServer = (database, logger, readonly = false) => { const server = new ApiServer(logger) server.get('/jig/:location', async (req, res) => { @@ -127,6 +127,10 @@ const buildServer = (database, logger) => { res.json(status) }) + if (readonly) { + return server + } + server.post('/trust/:txid?', async (req, res) => { if (Array.isArray(req.body)) { for (const maybeTxid of req.body) { diff --git a/src/direct-server.js b/src/direct-server.js deleted file mode 100644 index 89872e1..0000000 --- a/src/direct-server.js +++ /dev/null @@ -1,51 +0,0 @@ -/** - * direct-server.test.js - * - * Serves GET requets directly from the database and proxies other requests to the normal server - */ - -const { Worker } = require('worker_threads') -const Bus = require('./bus') - -class DirectServer { - constructor (dbPath, port, logger, database) { - this.dbPath = dbPath - this.port = port - this.logger = logger - this.database = database - this.worker = null - } - - async start () { - if (this.worker) return - const path = require.resolve('./direct-server-worker.js') - const workerData = { dbPath: this.dbPath, port: this.port } - this.worker = new Worker(path, { workerData }) - - const handlers = { - info: this.logger.info.bind(this.logger), - warn: this.logger.warn.bind(this.logger), - error: this.logger.error.bind(this.logger), - debug: this.logger.debug.bind(this.logger), - trust: this.database.trust.bind(this.database), - ban: this.database.ban.bind(this.database), - addTransaction: this.database.addTransaction.bind(this.database), - untrust: this.database.untrust.bind(this.database), - unban: this.database.unban.bind(this.database), - deleteTransaction: this.database.deleteTransaction.bind(this.database) - } - - Bus.listen(this.worker, handlers) - - await Bus.sendRequest(this.worker, 'start') - } - - async stop () { - if (!this.worker) return - await Bus.sendRequest(this.worker, 'stop') - await this.worker.terminate() - this.worker = null - } -} - -module.exports = DirectServer diff --git a/src/http/api-server.js b/src/http/api-server.js index 63a7b86..f6c607a 100644 --- a/src/http/api-server.js +++ b/src/http/api-server.js @@ -45,6 +45,7 @@ class ApiServer { this.port = port this.listener = this.app.listen(port, () => { + this.port = this.listener.address().port if (this.logger) this.logger.info(`Listening at http://localhost:${port}`) if (this.onListening) this.onListening() }) diff --git a/src/index.js b/src/index.js index 159157b..221cf24 100644 --- a/src/index.js +++ b/src/index.js @@ -8,7 +8,6 @@ const BitcoinNodeConnection = require('./bitcoin-node-connection') const BitcoinRpc = require('./bitcoin-rpc') const BitcoinZmq = require('./bitcoin-zmq') const Database = require('./database') -const DirectServer = require('./direct-server') const Indexer = require('./indexer') const MatterCloud = require('./mattercloud') const Planaria = require('./planaria') @@ -24,7 +23,6 @@ module.exports = { BitcoinRpc, BitcoinZmq, Database, - DirectServer, Indexer, MatterCloud, Planaria, diff --git a/src/start.js b/src/start.js index 669c5c5..0239c21 100644 --- a/src/start.js +++ b/src/start.js @@ -17,7 +17,6 @@ const BitcoinNodeConnection = require('./bitcoin-node-connection') const BitcoinRpc = require('./bitcoin-rpc') const BitcoinZmq = require('./bitcoin-zmq') const Database = require('./database') -const DirectServer = require('./direct-server') const { SqliteDatasource } = require('./data-sources/sqlite-datasource') const { SqliteMixedDatasource } = require('./data-sources/sqlite-mixed-datasource') const { ApiBlobStorage } = require('./data-sources/api-blob-storage') @@ -81,10 +80,7 @@ const indexer = new Indexer(database, api, NETWORK, FETCH_LIMIT, WORKERS, logger cacheType: WORKER_CACHE_TYPE }) -// const server = SERVE_ONLY -// ? buildServer(database, logger) -// : new DirectServer(DB, PORT, logger, database) -const server = buildServer(database, logger) +const server = buildServer(database, logger, readonly) let started = false diff --git a/test/server.test.js b/test/server.test.js index 8817e61..3bf4eff 100644 --- a/test/server.test.js +++ b/test/server.test.js @@ -9,12 +9,12 @@ require('chai').use(require('chai-as-promised')) const { expect } = require('chai') const axios = require('axios') const Indexer = require('../src/indexer') -const Server = require('../src/server') const txns = require('./txns.json') const { DEFAULT_TRUSTLIST } = require('../src/config') const Database = require('../src/database') const { SqliteDatasource } = require('../src/data-sources/sqlite-datasource') const { DbTrustList } = require('../src/trust-list/db-trust-list') +const { buildServer } = require('../src/build-server') // ------------------------------------------------------------------------------------------------ // Globals @@ -47,15 +47,16 @@ describe('Server', () => { describe('post tx', () => { it('add with body', async () => { const indexer = new Indexer(database, {}, 'main', 1, 1, logger, 0, Infinity, []) - const server = new Server(database, logger, 52521) + const server = buildServer(database, logger) await indexer.start() - await server.start() + const port = 52521 + await server.start(port) await listening(server) const txid = '3f9de452f0c3c96be737d42aa0941b27412211976688967adb3174ee18b04c64' const options = { headers: { 'Content-Type': 'text/plain' } } const promise = indexed(indexer, txid) - await axios.post(`http://localhost:${server.port}/tx/${txid}`, txns[txid], options) - await axios.post(`http://localhost:${server.port}/trust/${txid}`) + await axios.post(`http://localhost:${port}/tx/${txid}`, txns[txid], options) + await axios.post(`http://localhost:${port}/trust/${txid}`) await promise await server.stop() await indexer.stop() @@ -63,16 +64,16 @@ describe('Server', () => { // ------------------------------------------------------------------------ - it('throws if add with rawtx mismatch', async () => { + it('does not throw if add with rawtx mismatch', async () => { const indexer = new Indexer(database, {}, 'main', 1, 1, logger, 0, Infinity, []) - const server = new Server(database, logger, null) + const server = buildServer(database, logger) await indexer.start() - server.start() + await server.start() await listening(server) const txid = '3f9de452f0c3c96be737d42aa0941b27412211976688967adb3174ee18b04c64' const otherTxid = 'bfa5180e601e92af23d80782bf625b102ac110105a392e376fe7607e4e87dc8d' const options = { headers: { 'Content-Type': 'text/plain' } } - await expect(axios.post(`http://localhost:${server.port}/tx/${txid}`, txns[otherTxid], options)).to.be.rejectedWith(Error) + await expect(axios.post(`http://localhost:${server.port}/tx/${txid}`, txns[otherTxid], options)).to.be.fulfilled server.stop() await indexer.stop() }) @@ -85,7 +86,7 @@ describe('Server', () => { describe('post trust', () => { it('trust multiple', async () => { const indexer = new Indexer(database, {}, 'main', 1, 1, logger, 0, Infinity, []) - const server = new Server(database, logger, null) + const server = buildServer(database, logger) await indexer.start() server.start() await listening(server) @@ -110,7 +111,7 @@ describe('Server', () => { describe('get jig', () => { it('returns state if exists', async () => { const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, DEFAULT_TRUSTLIST) - const server = new Server(database, logger, null) + const server = buildServer(database, logger) await indexer.start() server.start() await listening(server) @@ -129,7 +130,7 @@ describe('Server', () => { it('returns 404 if missing', async () => { const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, []) - const server = new Server(database, logger, null) + const server = buildServer(database, logger) await indexer.start() server.start() await listening(server) @@ -152,7 +153,7 @@ describe('Server', () => { describe('get berry', () => { it('returns state if exists', async () => { const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, DEFAULT_TRUSTLIST) - const server = new Server(database, logger, null) + const server = buildServer(database, logger) await indexer.start() await server.start() await listening(server) @@ -170,7 +171,7 @@ describe('Server', () => { it('returns 404 if missing', async () => { const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, []) - const server = new Server(database, logger, null) + const server = buildServer(database, logger) await indexer.start() server.start() await listening(server) @@ -193,7 +194,7 @@ describe('Server', () => { describe('get tx', () => { it('returns rawtx if downloaded', async () => { const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, []) - const server = new Server(database, logger, null) + const server = buildServer(database, logger) await indexer.start() server.start() await listening(server) @@ -212,7 +213,7 @@ describe('Server', () => { it('returns 404 if missing', async () => { const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, []) - const server = new Server(database, logger, null) + const server = buildServer(database, logger) await indexer.start() server.start() await listening(server) @@ -231,7 +232,7 @@ describe('Server', () => { it('returns 404 if not downloaded', async () => { const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, []) - const server = new Server(database, logger, null) + const server = buildServer(database, logger) await indexer.start() server.start() await listening(server) @@ -255,7 +256,7 @@ describe('Server', () => { describe('get unspent', () => { it('query all unspent', async () => { const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, DEFAULT_TRUSTLIST) - const server = new Server(database, logger, null) + const server = buildServer(database, logger) await indexer.start() server.start() await listening(server) @@ -275,7 +276,7 @@ describe('Server', () => { it('query unspent by address', async () => { const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, DEFAULT_TRUSTLIST) - const server = new Server(database, logger, null) + const server = buildServer(database, logger) await indexer.start() server.start() await listening(server) @@ -297,7 +298,7 @@ describe('Server', () => { describe('misc', () => { it('cors', async () => { const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, []) - const server = new Server(database, logger, null) + const server = buildServer(database, logger) await indexer.start() server.start() await listening(server) From ffdcf6d72648ea4a7812008f2993cea52f363e93 Mon Sep 17 00:00:00 2001 From: hojarasca Date: Tue, 18 Jan 2022 15:09:16 -0300 Subject: [PATCH 33/82] added missing await. --- test/server.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/server.test.js b/test/server.test.js index 3bf4eff..391bdd7 100644 --- a/test/server.test.js +++ b/test/server.test.js @@ -300,7 +300,7 @@ describe('Server', () => { const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, []) const server = buildServer(database, logger) await indexer.start() - server.start() + await server.start() await listening(server) const opts = { headers: { Origin: 'https://www.google.com' } } const resp = (await axios.get(`http://localhost:${server.port}/status`, opts)) From 6497aff85e7fa6ba8ca3cffdf2e0374e292f323c Mon Sep 17 00:00:00 2001 From: hojarasca Date: Wed, 19 Jan 2022 05:46:05 -0300 Subject: [PATCH 34/82] execution server tests working fine. --- package-lock.json | 14 ++++++++ package.json | 1 + src/execution/build-execution-server.js | 44 +++++++++++++++++++++++++ src/execution/execution-server.js | 5 --- src/http/api-server.js | 17 ++++++---- src/worker.js | 1 - test/execution-server.test.js | 44 +++++++++++++++++++++++++ test/server.test.js | 32 +++++------------- test/test-worker.js | 8 +++++ 9 files changed, 131 insertions(+), 35 deletions(-) create mode 100644 src/execution/build-execution-server.js delete mode 100644 src/execution/execution-server.js create mode 100644 test/execution-server.test.js create mode 100644 test/test-worker.js diff --git a/package-lock.json b/package-lock.json index 4872228..73a5f3e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20,6 +20,7 @@ "express": "^4.17.1", "express-async-handler": "^1.2.0", "fastq": "^1.13.0", + "generic-pool": "^3.8.2", "helmet": "^5.0.1", "knex": "^0.95.14", "morgan": "^1.10.0", @@ -2645,6 +2646,14 @@ "wide-align": "^1.1.0" } }, + "node_modules/generic-pool": { + "version": "3.8.2", + "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.8.2.tgz", + "integrity": "sha512-nGToKy6p3PAbYQ7p1UlWl6vSPwfwU6TMSWK7TTu+WUY4ZjyZQGniGGt2oNVvyNSpyZYSB43zMXVLcBm08MTMkg==", + "engines": { + "node": ">= 4" + } + }, "node_modules/gensync": { "version": "1.0.0-beta.2", "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", @@ -8366,6 +8375,11 @@ "wide-align": "^1.1.0" } }, + "generic-pool": { + "version": "3.8.2", + "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.8.2.tgz", + "integrity": "sha512-nGToKy6p3PAbYQ7p1UlWl6vSPwfwU6TMSWK7TTu+WUY4ZjyZQGniGGt2oNVvyNSpyZYSB43zMXVLcBm08MTMkg==" + }, "gensync": { "version": "1.0.0-beta.2", "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", diff --git a/package.json b/package.json index 91d7ea8..46c1525 100644 --- a/package.json +++ b/package.json @@ -41,6 +41,7 @@ "express": "^4.17.1", "express-async-handler": "^1.2.0", "fastq": "^1.13.0", + "generic-pool": "^3.8.2", "helmet": "^5.0.1", "knex": "^0.95.14", "morgan": "^1.10.0", diff --git a/src/execution/build-execution-server.js b/src/execution/build-execution-server.js new file mode 100644 index 0000000..e37a8d4 --- /dev/null +++ b/src/execution/build-execution-server.js @@ -0,0 +1,44 @@ +const { ApiServer } = require('../http/api-server') +const genericPool = require('generic-pool') +const { Worker } = require('worker_threads') +const Bus = require('../bus') + +const buildExecutionServer = (logger, count, blobStorage, workerPath, network, workerOpts = {}) => { + const factory = { + create: () => { + const worker = new Worker(workerPath, { workerData: { network: network, ...workerOpts, cacheType: 'direct' } }) + Bus.listen(worker, {}) + return worker + }, + + destroy: (worker) => { + worker.terminate() + } + } + + const opts = { + max: count, + min: 1 + } + + const pool = genericPool.createPool(factory, opts) + const server = new ApiServer(logger, { + onStop: async () => { + await pool.drain() + await pool.clear() + } + }) + + server.post('/execute', async (req, res) => { + const worker = await pool.acquire() + const { txid, trustList } = req.body + const hex = await blobStorage.pullTx(txid) + const response = await Bus.sendRequest(worker, 'execute', txid, hex, trustList) + pool.release(worker) + res.json(response) + }) + + return server +} + +module.exports = { buildExecutionServer } diff --git a/src/execution/execution-server.js b/src/execution/execution-server.js deleted file mode 100644 index 5a75e3d..0000000 --- a/src/execution/execution-server.js +++ /dev/null @@ -1,5 +0,0 @@ -class ExecutionServer { - -} - -module.exports = { ExecutionServer } diff --git a/src/http/api-server.js b/src/http/api-server.js index f6c607a..0cc9b2b 100644 --- a/src/http/api-server.js +++ b/src/http/api-server.js @@ -7,8 +7,9 @@ const asyncHandler = require('express-async-handler') const helmet = require('helmet') class ApiServer { - constructor (logger) { + constructor (logger, opts = {}) { this.logger = logger + this.onStop = opts.onStop || function () {} this.logger.debug('Starting server') this.listener = null this.onListening = null @@ -44,15 +45,19 @@ class ApiServer { }) this.port = port - this.listener = this.app.listen(port, () => { - this.port = this.listener.address().port - if (this.logger) this.logger.info(`Listening at http://localhost:${port}`) - if (this.onListening) this.onListening() + return new Promise(resolve => { + this.listener = this.app.listen(port, () => { + this.port = this.listener.address().port + if (this.logger) this.logger.info(`Listening at http://localhost:${this.port}`) + if (this.onListening) this.onListening() + resolve() + }) }) } - stop () { + async stop () { if (!this.listener) return + await this.onStop() this.listener.close() this.listener = null } diff --git a/src/worker.js b/src/worker.js index ecd545d..c9c9ef8 100644 --- a/src/worker.js +++ b/src/worker.js @@ -19,7 +19,6 @@ const { ApiBlobStorage } = require('./data-sources/api-blob-storage') const network = workerData.network const cacheType = workerData.cacheType -// const id = workerData.id Bus.listen(parentPort, { execute }) diff --git a/test/execution-server.test.js b/test/execution-server.test.js new file mode 100644 index 0000000..b4c9c6c --- /dev/null +++ b/test/execution-server.test.js @@ -0,0 +1,44 @@ +const { buildExecutionServer } = require('../src/execution/build-execution-server') +const { MemoryBlobStorage } = require('../src/data-sources/memory-blob-storage') +const txs = require('./txns.json') +const fetch = require('node-fetch') +const { expect } = require('chai') + +const logger = { + debug: () => {}, + info: () => {}, + log: () => {}, + warn: () => {}, + error: () => {} +} + +describe('execution-server', () => { + let bs + let server + + beforeEach(async () => { + bs = new MemoryBlobStorage() + server = buildExecutionServer(logger, 1, bs, require.resolve('./test-worker.js'), 'test') + await server.start() + }) + + afterEach(async () => { + await server.stop() + }) + + it('calls the worker', async () => { + const txid = 'bfa5180e601e92af23d80782bf625b102ac110105a392e376fe7607e4e87dc8d' + await bs.pushTx(txid, Buffer.from(txs[txid], 'hex')) + + const response = await fetch(`http://localhost:${server.port}/execute`, { + method: 'POST', + body: JSON.stringify({ + txid, trustList: '*' + }) + }) + + expect(response.status).to.eql(200) + const jsonResponse = await response.json() + expect(jsonResponse.solution).to.eql(42) + }) +}) diff --git a/test/server.test.js b/test/server.test.js index 391bdd7..c28ac01 100644 --- a/test/server.test.js +++ b/test/server.test.js @@ -26,7 +26,6 @@ const fetch = async txid => { const api = { fetch } const downloaded = (indexer, txid) => new Promise((resolve) => { indexer.onDownload = x => txid === x && resolve() }) const indexed = (indexer, txid) => new Promise((resolve) => { indexer.onIndex = x => txid === x && resolve() }) -const listening = (server) => new Promise((resolve) => { server.onListening = () => resolve() }) const logger = { info: () => {}, warn: () => {}, error: () => {}, debug: () => {} } const ds = new SqliteDatasource(':memory:', logger, false) const trustList = new DbTrustList(ds) @@ -51,7 +50,6 @@ describe('Server', () => { await indexer.start() const port = 52521 await server.start(port) - await listening(server) const txid = '3f9de452f0c3c96be737d42aa0941b27412211976688967adb3174ee18b04c64' const options = { headers: { 'Content-Type': 'text/plain' } } const promise = indexed(indexer, txid) @@ -69,7 +67,6 @@ describe('Server', () => { const server = buildServer(database, logger) await indexer.start() await server.start() - await listening(server) const txid = '3f9de452f0c3c96be737d42aa0941b27412211976688967adb3174ee18b04c64' const otherTxid = 'bfa5180e601e92af23d80782bf625b102ac110105a392e376fe7607e4e87dc8d' const options = { headers: { 'Content-Type': 'text/plain' } } @@ -88,8 +85,7 @@ describe('Server', () => { const indexer = new Indexer(database, {}, 'main', 1, 1, logger, 0, Infinity, []) const server = buildServer(database, logger) await indexer.start() - server.start() - await listening(server) + await server.start() const trustlist = [ '3f9de452f0c3c96be737d42aa0941b27412211976688967adb3174ee18b04c64', 'bfa5180e601e92af23d80782bf625b102ac110105a392e376fe7607e4e87dc8d' @@ -113,8 +109,7 @@ describe('Server', () => { const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, DEFAULT_TRUSTLIST) const server = buildServer(database, logger) await indexer.start() - server.start() - await listening(server) + await server.start() const promise = indexed(indexer, '9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102') await database.addTransaction('9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102') await promise @@ -132,8 +127,7 @@ describe('Server', () => { const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, []) const server = buildServer(database, logger) await indexer.start() - server.start() - await listening(server) + await server.start() const location = '9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102_o1' await expect(axios.get(`http://localhost:${server.port}/jig/${location}`)).to.be.rejectedWith(Error) try { @@ -156,7 +150,6 @@ describe('Server', () => { const server = buildServer(database, logger) await indexer.start() await server.start() - await listening(server) await database.addTransaction('bfa5180e601e92af23d80782bf625b102ac110105a392e376fe7607e4e87dc8d') await indexed(indexer, 'bfa5180e601e92af23d80782bf625b102ac110105a392e376fe7607e4e87dc8d') const location = '24cde3638a444c8ad397536127833878ffdfe1b04d5595489bd294e50d77105a_o1?berry=2f3492ef5401d887a93ca09820dff952f355431cea306841a70d163e32b2acad&version=5' @@ -173,8 +166,7 @@ describe('Server', () => { const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, []) const server = buildServer(database, logger) await indexer.start() - server.start() - await listening(server) + await server.start() const location = '24cde3638a444c8ad397536127833878ffdfe1b04d5595489bd294e50d77105a_o1?berry=2f3492ef5401d887a93ca09820dff952f355431cea306841a70d163e32b2acad&version=5' await expect(axios.get(`http://localhost:${server.port}/berry/${location}`)).to.be.rejectedWith(Error) try { @@ -196,8 +188,7 @@ describe('Server', () => { const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, []) const server = buildServer(database, logger) await indexer.start() - server.start() - await listening(server) + await server.start() const txid = '9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102' const promise = downloaded(indexer, txid) await database.addTransaction(txid) @@ -215,8 +206,7 @@ describe('Server', () => { const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, []) const server = buildServer(database, logger) await indexer.start() - server.start() - await listening(server) + await server.start() const txid = '9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102' await expect(axios.get(`http://localhost:${server.port}/tx/${txid}`)).to.be.rejectedWith(Error) try { @@ -234,8 +224,7 @@ describe('Server', () => { const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, []) const server = buildServer(database, logger) await indexer.start() - server.start() - await listening(server) + await server.start() const txid = '1111111111111111111111111111111111111111111111111111111111111111' database.addTransaction(txid) await expect(axios.get(`http://localhost:${server.port}/tx/${txid}`)).to.be.rejectedWith(Error) @@ -258,8 +247,7 @@ describe('Server', () => { const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, DEFAULT_TRUSTLIST) const server = buildServer(database, logger) await indexer.start() - server.start() - await listening(server) + await server.start() const promise = indexed(indexer, '9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102') await database.addTransaction('9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102') await promise @@ -278,8 +266,7 @@ describe('Server', () => { const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, DEFAULT_TRUSTLIST) const server = buildServer(database, logger) await indexer.start() - server.start() - await listening(server) + await server.start() const promise = indexed(indexer, '9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102') await database.addTransaction('9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102') await promise @@ -301,7 +288,6 @@ describe('Server', () => { const server = buildServer(database, logger) await indexer.start() await server.start() - await listening(server) const opts = { headers: { Origin: 'https://www.google.com' } } const resp = (await axios.get(`http://localhost:${server.port}/status`, opts)) expect(resp.headers['access-control-allow-origin']).to.equal('*') diff --git a/test/test-worker.js b/test/test-worker.js new file mode 100644 index 0000000..3ff10b9 --- /dev/null +++ b/test/test-worker.js @@ -0,0 +1,8 @@ +const Bus = require('../src/bus') +const { parentPort } = require('worker_threads') + +const execute = (...params) => { + return { solution: 42, params } +} + +Bus.listen(parentPort, { execute }) From c954c1faebec3094340e2df9eed9499e60bc142b Mon Sep 17 00:00:00 2001 From: hojarasca Date: Wed, 19 Jan 2022 08:51:38 -0300 Subject: [PATCH 35/82] added some validatiosn to executon server. --- src/data-sources/api-blob-storage.js | 2 +- src/execution/build-execution-server.js | 31 +++++++++++---- src/http/api-error.js | 10 +++++ src/http/api-server.js | 9 ++++- src/util/parse-txid.js | 7 ++++ test/execution-server.test.js | 53 ++++++++++++++++++++++--- 6 files changed, 96 insertions(+), 16 deletions(-) create mode 100644 src/http/api-error.js create mode 100644 src/util/parse-txid.js diff --git a/src/data-sources/api-blob-storage.js b/src/data-sources/api-blob-storage.js index fc97242..187d879 100644 --- a/src/data-sources/api-blob-storage.js +++ b/src/data-sources/api-blob-storage.js @@ -36,7 +36,7 @@ class ApiBlobStorage { return json.state } - async pullTx (txid) { + async pullTx (txid, _ifNone) { const result = await fetch(`${this.baseApiUrl}/rawtx/${txid}`) return result.buffer() } diff --git a/src/execution/build-execution-server.js b/src/execution/build-execution-server.js index e37a8d4..3d8ca16 100644 --- a/src/execution/build-execution-server.js +++ b/src/execution/build-execution-server.js @@ -2,6 +2,8 @@ const { ApiServer } = require('../http/api-server') const genericPool = require('generic-pool') const { Worker } = require('worker_threads') const Bus = require('../bus') +const { ApiError } = require('../http/api-error') +const { parseTxid } = require('../util/parse-txid') const buildExecutionServer = (logger, count, blobStorage, workerPath, network, workerOpts = {}) => { const factory = { @@ -11,8 +13,8 @@ const buildExecutionServer = (logger, count, blobStorage, workerPath, network, w return worker }, - destroy: (worker) => { - worker.terminate() + destroy: async (worker) => { + await worker.terminate() } } @@ -31,11 +33,26 @@ const buildExecutionServer = (logger, count, blobStorage, workerPath, network, w server.post('/execute', async (req, res) => { const worker = await pool.acquire() - const { txid, trustList } = req.body - const hex = await blobStorage.pullTx(txid) - const response = await Bus.sendRequest(worker, 'execute', txid, hex, trustList) - pool.release(worker) - res.json(response) + try { + const { txid: rawTxid, trustList } = req.body + if (!Array.isArray(trustList)) { + throw new ApiError('wrong parameter: trustList', 'wrong-arguments', 400, { trustList }) + } + const txid = parseTxid(rawTxid, () => { + throw new ApiError( + 'wrong parameter: txid', + 'wrong-arguments', + 400, + { txid: rawTxid } + ) + }) + const buff = await blobStorage.pullTx(txid, () => { throw new Error('not found') }) + const hex = buff.toString('hex') + const response = await Bus.sendRequest(worker, 'execute', txid, hex, trustList) + res.json(response) + } finally { + pool.release(worker).catch(() => {}) + } }) return server diff --git a/src/http/api-error.js b/src/http/api-error.js new file mode 100644 index 0000000..b089c43 --- /dev/null +++ b/src/http/api-error.js @@ -0,0 +1,10 @@ +class ApiError extends Error { + constructor (msg, errorCode, httpCode, extraData = {}) { + super(msg) + this.errorCode = errorCode + this.httpCode = httpCode + this.extraData = extraData + } +} + +module.exports = { ApiError } diff --git a/src/http/api-server.js b/src/http/api-server.js index 0cc9b2b..781637e 100644 --- a/src/http/api-server.js +++ b/src/http/api-server.js @@ -5,6 +5,7 @@ const cors = require('cors') const { Writable } = require('stream') const asyncHandler = require('express-async-handler') const helmet = require('helmet') +const { ApiError } = require('./api-error') class ApiServer { constructor (logger, opts = {}) { @@ -39,8 +40,12 @@ class ApiServer { async start (port = null) { this.app.use((err, req, res, next) => { - if (this.logger) this.logger.error(err.stack) - res.status(500).send('Something broke!') + if (this.logger) { this.logger.error(err.stack) } + if (err instanceof ApiError) { + res.status(err.httpCode).json({ code: err.errorCode, message: err.message, data: err.extraData }) + } else { + res.status(500).send('Something broke!') + } next() }) diff --git a/src/util/parse-txid.js b/src/util/parse-txid.js new file mode 100644 index 0000000..88738b4 --- /dev/null +++ b/src/util/parse-txid.js @@ -0,0 +1,7 @@ +const parseTxid = (aString, ifNone) => { + const txid = aString.trim().toLowerCase() + if (!/^[0-9a-f]{64}$/.test(txid)) { return ifNone(txid) } + return txid +} + +module.exports = { parseTxid } diff --git a/test/execution-server.test.js b/test/execution-server.test.js index b4c9c6c..77020ee 100644 --- a/test/execution-server.test.js +++ b/test/execution-server.test.js @@ -12,6 +12,16 @@ const logger = { error: () => {} } +async function fetchExecution (server, txid, params) { + return await fetch(`http://localhost:${server.port}/execute`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify(params) + }) +} + describe('execution-server', () => { let bs let server @@ -30,15 +40,46 @@ describe('execution-server', () => { const txid = 'bfa5180e601e92af23d80782bf625b102ac110105a392e376fe7607e4e87dc8d' await bs.pushTx(txid, Buffer.from(txs[txid], 'hex')) - const response = await fetch(`http://localhost:${server.port}/execute`, { - method: 'POST', - body: JSON.stringify({ - txid, trustList: '*' - }) - }) + const response = await fetchExecution(server, txid, { txid, trustList: ['*'] }) expect(response.status).to.eql(200) const jsonResponse = await response.json() expect(jsonResponse.solution).to.eql(42) }) + + it('sends the right parameters to the worker', async () => { + const txid = 'bfa5180e601e92af23d80782bf625b102ac110105a392e376fe7607e4e87dc8d' + const txHex = txs[txid] + await bs.pushTx(txid, Buffer.from(txHex, 'hex')) + + const response = await fetchExecution(server, txid, { txid, trustList: ['*'] }) + + const jsonResponse = await response.json() + expect(jsonResponse.params).to.have.length(3) + expect(jsonResponse.params[0]).to.eql(txid) + expect(jsonResponse.params[1]).to.eql(txHex) + expect(jsonResponse.params[2]).to.eql(['*']) + }) + + it('returns 400 when trustlist is not a list', async () => { + const txid = 'bfa5180e601e92af23d80782bf625b102ac110105a392e376fe7607e4e87dc8d' + + const response = await fetchExecution(server, txid, { txid, trustList: '*' }) + expect(response.status).to.eql(400) + const jsonResponse = await response.json() + expect(jsonResponse.code).to.eql('wrong-arguments') + expect(jsonResponse.message).to.eql('wrong parameter: trustList') + expect(jsonResponse.data).to.eql({ trustList: '*' }) + }) + + it('returns 400 when txid is not a txid', async () => { + const txid = 'notatxid' + + const response = await fetchExecution(server, txid, { txid, trustList: ['*'] }) + expect(response.status).to.eql(400) + const jsonResponse = await response.json() + expect(jsonResponse.code).to.eql('wrong-arguments') + expect(jsonResponse.message).to.eql('wrong parameter: txid') + expect(jsonResponse.data).to.eql({ txid }) + }) }) From 4c61cb41d1768b0c8fab757ead39b624bc98d0f4 Mon Sep 17 00:00:00 2001 From: hojarasca Date: Wed, 19 Jan 2022 11:15:13 -0300 Subject: [PATCH 36/82] handling execution errors. --- src/bus.js | 9 ++-- src/execution/build-execution-server.js | 56 +++++++++++++++++-------- src/execution/execution-error.js | 3 ++ test/execution-server.test.js | 29 ++++++++++--- test/test-worker.js | 3 ++ 5 files changed, 74 insertions(+), 26 deletions(-) create mode 100644 src/execution/execution-error.js diff --git a/src/bus.js b/src/bus.js index 7020d75..c1e1a74 100644 --- a/src/bus.js +++ b/src/bus.js @@ -15,9 +15,9 @@ const messageCallbacks = {} // sendRequest // ------------------------------------------------------------------------------------------------ -async function sendRequest (port, func, ...args) { +async function sendRequest (port, func, errorClass = Error, args = []) { return await new Promise((resolve, reject) => { - messageCallbacks[messageId] = { resolve, reject } + messageCallbacks[messageId] = { resolve, reject, ErrorClass: errorClass } port.postMessage({ id: messageId, func, args }) messageId++ }) @@ -30,10 +30,11 @@ async function sendRequest (port, func, ...args) { function listen (port, handlers) { port.on('message', async msg => { if (msg.response) { + const { resolve, reject, ErrorClass } = messageCallbacks[msg.id] if (msg.err) { - messageCallbacks[msg.id].reject(msg.err) + reject(new ErrorClass(msg.err)) } else { - messageCallbacks[msg.id].resolve(msg.ret) + resolve(msg.ret) } delete messageCallbacks[msg.id] return diff --git a/src/execution/build-execution-server.js b/src/execution/build-execution-server.js index 3d8ca16..00fb7a0 100644 --- a/src/execution/build-execution-server.js +++ b/src/execution/build-execution-server.js @@ -4,6 +4,7 @@ const { Worker } = require('worker_threads') const Bus = require('../bus') const { ApiError } = require('../http/api-error') const { parseTxid } = require('../util/parse-txid') +const { ExecutionError } = require('./execution-error') const buildExecutionServer = (logger, count, blobStorage, workerPath, network, workerOpts = {}) => { const factory = { @@ -32,26 +33,47 @@ const buildExecutionServer = (logger, count, blobStorage, workerPath, network, w }) server.post('/execute', async (req, res) => { + const { txid: rawTxid, trustList } = req.body + if (!Array.isArray(trustList)) { + throw new ApiError('wrong parameter: trustList', 'wrong-arguments', 400, { trustList }) + } + const txid = parseTxid(rawTxid, () => { + throw new ApiError( + 'wrong parameter: txid', + 'wrong-arguments', + 400, + { txid: rawTxid } + ) + }) + const buff = await blobStorage.pullTx(txid, () => { throw new Error('not found') }) + const hex = buff.toString('hex') + const worker = await pool.acquire() try { - const { txid: rawTxid, trustList } = req.body - if (!Array.isArray(trustList)) { - throw new ApiError('wrong parameter: trustList', 'wrong-arguments', 400, { trustList }) - } - const txid = parseTxid(rawTxid, () => { - throw new ApiError( - 'wrong parameter: txid', - 'wrong-arguments', - 400, - { txid: rawTxid } - ) + const response = await Bus.sendRequest(worker, 'execute', ExecutionError, [txid, hex, trustList]) + pool.release(worker).catch(logger.error) + res.json({ + ok: true, + error: null, + response + }) + } catch (e) { + pool.destroy(worker).catch(logger.error) + const error = e instanceof ExecutionError + ? { + type: e.constructor.name, + message: e.message + } + : { + type: 'Error', + message: 'unexpected error' + } + + res.json({ + ok: false, + error, + result: null }) - const buff = await blobStorage.pullTx(txid, () => { throw new Error('not found') }) - const hex = buff.toString('hex') - const response = await Bus.sendRequest(worker, 'execute', txid, hex, trustList) - res.json(response) - } finally { - pool.release(worker).catch(() => {}) } }) diff --git a/src/execution/execution-error.js b/src/execution/execution-error.js new file mode 100644 index 0000000..5d2a69c --- /dev/null +++ b/src/execution/execution-error.js @@ -0,0 +1,3 @@ +class ExecutionError extends Error {} + +module.exports = { ExecutionError } diff --git a/test/execution-server.test.js b/test/execution-server.test.js index 77020ee..fe3e214 100644 --- a/test/execution-server.test.js +++ b/test/execution-server.test.js @@ -44,7 +44,9 @@ describe('execution-server', () => { expect(response.status).to.eql(200) const jsonResponse = await response.json() - expect(jsonResponse.solution).to.eql(42) + expect(jsonResponse.ok).to.eql(true) + expect(jsonResponse.response.solution).to.eql(42) + expect(jsonResponse.error).to.eql(null) }) it('sends the right parameters to the worker', async () => { @@ -55,10 +57,11 @@ describe('execution-server', () => { const response = await fetchExecution(server, txid, { txid, trustList: ['*'] }) const jsonResponse = await response.json() - expect(jsonResponse.params).to.have.length(3) - expect(jsonResponse.params[0]).to.eql(txid) - expect(jsonResponse.params[1]).to.eql(txHex) - expect(jsonResponse.params[2]).to.eql(['*']) + const params = jsonResponse.response.params + expect(params).to.have.length(3) + expect(params[0]).to.eql(txid) + expect(params[1]).to.eql(txHex) + expect(params[2]).to.eql(['*']) }) it('returns 400 when trustlist is not a list', async () => { @@ -82,4 +85,20 @@ describe('execution-server', () => { expect(jsonResponse.message).to.eql('wrong parameter: txid') expect(jsonResponse.data).to.eql({ txid }) }) + + it('returns 200 and fail details when execution fails', async () => { + const txid = Buffer.alloc(32).fill(1).toString('hex') + const txHex = 'sometxhex' + await bs.pushTx(txid, Buffer.from(txHex, 'hex')) + + const response = await fetchExecution(server, txid, { txid, trustList: ['*'] }) + expect(response.status).to.eql(200) + const jsonResponse = await response.json() + expect(jsonResponse.ok).to.eql(false) + expect(jsonResponse.error).to.eql({ + type: 'ExecutionError', + message: 'execution failed' + }) + expect(jsonResponse.result).to.eql(null) + }) }) diff --git a/test/test-worker.js b/test/test-worker.js index 3ff10b9..9d6b7fd 100644 --- a/test/test-worker.js +++ b/test/test-worker.js @@ -2,6 +2,9 @@ const Bus = require('../src/bus') const { parentPort } = require('worker_threads') const execute = (...params) => { + if (params[0] === Buffer.alloc(32).fill(1).toString('hex')) { + throw new Error('execution failed') + } return { solution: 42, params } } From 15797e8bc41d04e84958c3d24ce767f5127ad936 Mon Sep 17 00:00:00 2001 From: hojarasca Date: Wed, 19 Jan 2022 11:20:30 -0300 Subject: [PATCH 37/82] spread refactor on bus helper. --- src/bus.js | 2 +- src/direct-server-worker.js | 53 ------------------------- src/execution/build-execution-server.js | 2 +- src/executor.js | 4 +- src/worker.js | 4 +- 5 files changed, 6 insertions(+), 59 deletions(-) delete mode 100644 src/direct-server-worker.js diff --git a/src/bus.js b/src/bus.js index c1e1a74..cb42c67 100644 --- a/src/bus.js +++ b/src/bus.js @@ -15,7 +15,7 @@ const messageCallbacks = {} // sendRequest // ------------------------------------------------------------------------------------------------ -async function sendRequest (port, func, errorClass = Error, args = []) { +async function sendRequest (port, func, args = [], errorClass = Error) { return await new Promise((resolve, reject) => { messageCallbacks[messageId] = { resolve, reject, ErrorClass: errorClass } port.postMessage({ id: messageId, func, args }) diff --git a/src/direct-server-worker.js b/src/direct-server-worker.js deleted file mode 100644 index 4fe2e68..0000000 --- a/src/direct-server-worker.js +++ /dev/null @@ -1,53 +0,0 @@ -/** - * direct-server-worker.js - * - * Internal worker thread that runs the user server - */ - -const { parentPort, workerData } = require('worker_threads') -const Server = require('./server') -const Bus = require('./bus') -const Database = require('./database') -const { SqliteDatasource } = require('./data-sources/sqlite-datasource') -const { DATA_SOURCE, DB, DATA_API_ROOT } = require('./config') -const { SqliteMixedDatasource } = require('./data-sources/sqlite-mixed-datasource') - -const logger = { - info: (...args) => Bus.sendRequest(parentPort, 'info', ...args), - warn: (...args) => Bus.sendRequest(parentPort, 'warn', ...args), - error: (...args) => Bus.sendRequest(parentPort, 'error', ...args), - debug: (...args) => Bus.sendRequest(parentPort, 'debug', ...args) -} - -const readonly = true - -let dataSource -if (DATA_SOURCE === 'sqlite') { - dataSource = new SqliteDatasource(DB, logger, readonly) -} else if (DATA_SOURCE === 'mixed') { - dataSource = new SqliteMixedDatasource(DB, logger, readonly, DATA_API_ROOT) -} else { - throw new Error(`unknown datasource: ${DATA_SOURCE}. Please check "DATA_SOURCE" configuration.`) -} - -const database = new Database(dataSource, logger) -const server = new Server(database, logger, workerData.port) - -database.trust = (txid) => Bus.sendRequest(parentPort, 'trust', txid) -database.ban = (txid) => Bus.sendRequest(parentPort, 'ban', txid) -database.addTransaction = (txid, hex) => Bus.sendRequest(parentPort, 'addTransaction', txid, hex) -database.untrust = (txid) => Bus.sendRequest(parentPort, 'untrust', txid) -database.unban = (txid) => Bus.sendRequest(parentPort, 'unban', txid) -database.deleteTransaction = (txid) => Bus.sendRequest(parentPort, 'deleteTransaction', txid) - -Bus.listen(parentPort, { start, stop }) - -async function start () { - await database.open() - await server.start() -} - -async function stop () { - await server.stop() - await database.close() -} diff --git a/src/execution/build-execution-server.js b/src/execution/build-execution-server.js index 00fb7a0..6fb6b89 100644 --- a/src/execution/build-execution-server.js +++ b/src/execution/build-execution-server.js @@ -50,7 +50,7 @@ const buildExecutionServer = (logger, count, blobStorage, workerPath, network, w const worker = await pool.acquire() try { - const response = await Bus.sendRequest(worker, 'execute', ExecutionError, [txid, hex, trustList]) + const response = await Bus.sendRequest(worker, 'execute', [txid, hex, trustList], ExecutionError) pool.release(worker).catch(logger.error) res.json({ ok: true, diff --git a/src/executor.js b/src/executor.js index 54fbf51..2f49f09 100644 --- a/src/executor.js +++ b/src/executor.js @@ -82,12 +82,12 @@ class Executor { let result = null try { - result = await Bus.sendRequest(worker, 'execute', txid, hex, trustList) + result = await Bus.sendRequest(worker, 'execute', [txid, hex, trustList]) } catch (e) { if (worker.missingDeps.size) { if (this.onMissingDeps) await this.onMissingDeps(txid, Array.from(worker.missingDeps)) } else { - if (this.onExecuteFailed) await this.onExecuteFailed(txid, e) + if (this.onExecuteFailed) await this.onExecuteFailed(txid, e.message) } } finally { this.executing.delete(txid) diff --git a/src/worker.js b/src/worker.js index c9c9ef8..c495b71 100644 --- a/src/worker.js +++ b/src/worker.js @@ -43,7 +43,7 @@ class Cache { return this.state[key] } - return await Bus.sendRequest(parentPort, 'cacheGet', key) + return await Bus.sendRequest(parentPort, 'cacheGet', [key]) } async set (key, value) { @@ -98,7 +98,7 @@ class Blockchain { constructor (txid) { this.txid = txid } get network () { return network } async broadcast (_hex) { return this.txid } - async fetch (txid) { return await Bus.sendRequest(parentPort, 'blockchainFetch', txid) } + async fetch (txid) { return await Bus.sendRequest(parentPort, 'blockchainFetch', [txid]) } async utxos (_script) { throw new Error('not implemented: utxos') } async spends (_txid, _vout) { throw new Error('not implemented: spends') } async time (_txid) { throw new Error('not implemented: time') } From 8d1dbf6d5d9c71c296bfb4ec7af9f8d3126f7eec Mon Sep 17 00:00:00 2001 From: hojarasca Date: Thu, 20 Jan 2022 08:31:37 -0300 Subject: [PATCH 38/82] move configuration to top level. --- src/executor.js | 3 ++- src/start.js | 3 ++- src/worker.js | 8 ++++++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/executor.js b/src/executor.js index 2f49f09..4f4d3d0 100644 --- a/src/executor.js +++ b/src/executor.js @@ -18,7 +18,8 @@ class Executor { this.database = database this.logger = logger this.workerOpts = { - cacheType: opts.cacheType || 'parentConnection' + cacheType: opts.cacheType || 'parentConnection', + dataApiRoot: opts.dataApiRoot || null } this.onIndexed = null diff --git a/src/start.js b/src/start.js index 0239c21..6651ad7 100644 --- a/src/start.js +++ b/src/start.js @@ -77,7 +77,8 @@ const database = new Database(dataSource, trustList, logger) const indexer = new Indexer(database, api, NETWORK, FETCH_LIMIT, WORKERS, logger, START_HEIGHT, MEMPOOL_EXPIRATION, DEFAULT_TRUSTLIST, { trustSource: WORKER_TRUST_SOURCE, - cacheType: WORKER_CACHE_TYPE + cacheType: WORKER_CACHE_TYPE, + dataApiRoot: DATA_API_ROOT }) const server = buildServer(database, logger, readonly) diff --git a/src/worker.js b/src/worker.js index c495b71..d752f7c 100644 --- a/src/worker.js +++ b/src/worker.js @@ -9,7 +9,6 @@ const crypto = require('crypto') const Run = require('run-sdk') const bsv = require('bsv') const Bus = require('./bus') -const config = require('./config') const { DEBUG } = require('./config') const { ApiBlobStorage } = require('./data-sources/api-blob-storage') @@ -19,6 +18,11 @@ const { ApiBlobStorage } = require('./data-sources/api-blob-storage') const network = workerData.network const cacheType = workerData.cacheType +const blobApiRoot = workerData.dataApiRoot + +if (cacheType === 'direct' && !blobApiRoot) { + throw new Error('missing api root for direct cache') +} Bus.listen(parentPort, { execute }) @@ -127,7 +131,7 @@ async function execute (txid, hex, trustlist) { console.log = function () {} console.log() if (cacheType === 'direct') { - const bs = new ApiBlobStorage(config.DATA_API_ROOT) + const bs = new ApiBlobStorage(blobApiRoot) run.cache = new DirectCache(bs) } else { run.cache = new Cache() From cbbcb683ba17f9566c6d282b1037456302130fef Mon Sep 17 00:00:00 2001 From: hojarasca Date: Thu, 20 Jan 2022 08:41:10 -0300 Subject: [PATCH 39/82] move configuration to top level. --- .../build-main-server.js} | 12 +- src/index.js | 2 - src/server.js | 285 ------------------ src/start.js | 4 +- test/server.test.js | 2 +- 5 files changed, 9 insertions(+), 296 deletions(-) rename src/{build-server.js => http/build-main-server.js} (94%) delete mode 100644 src/server.js diff --git a/src/build-server.js b/src/http/build-main-server.js similarity index 94% rename from src/build-server.js rename to src/http/build-main-server.js index 8f44403..2e6ea60 100644 --- a/src/build-server.js +++ b/src/http/build-main-server.js @@ -7,7 +7,7 @@ const bsv = require('bsv') const crypto = require('crypto') const Run = require('run-sdk') -const { ApiServer } = require('./http/api-server') +const { ApiServer } = require('./api-server') // ------------------------------------------------------------------------------------------------ // Globals @@ -25,7 +25,7 @@ const parseTxid = (txid) => { // Server // ------------------------------------------------------------------------------------------------ -const buildServer = (database, logger, readonly = false) => { +const buildMainServer = (database, logger, readonly = false) => { const server = new ApiServer(logger) server.get('/jig/:location', async (req, res) => { @@ -68,7 +68,7 @@ const buildServer = (database, logger, readonly = false) => { } }) - server.get('/spends/:location', async (req, res, next) => { + server.get('/spends/:location', async (req, res) => { const txid = await database.getSpend(req.params.location) if (txid) { res.send(txid) @@ -103,7 +103,7 @@ const buildServer = (database, logger, readonly = false) => { } }) - server.get('/trust/:txid?', async (req, res, next) => { + server.get('/trust/:txid?', async (req, res) => { if (req.params.txid) { res.json(await database.isTrusted(req.params.txid)) } else { @@ -111,7 +111,7 @@ const buildServer = (database, logger, readonly = false) => { } }) - server.get('/ban/:txid?', async (req, res, next) => { + server.get('/ban/:txid?', async (req, res) => { if (req.params.txid) { res.json(await database.isBanned(req.params.txid)) } else { @@ -183,4 +183,4 @@ const buildServer = (database, logger, readonly = false) => { return server } -module.exports = { buildServer } +module.exports = { buildMainServer } diff --git a/src/index.js b/src/index.js index 221cf24..9e587d0 100644 --- a/src/index.js +++ b/src/index.js @@ -12,7 +12,6 @@ const Indexer = require('./indexer') const MatterCloud = require('./mattercloud') const Planaria = require('./planaria') const RunConnectFetcher = require('./run-connect') -const Server = require('./server') const config = require('./config') const { SqliteDatasource } = require('./data-sources/sqlite-datasource') const { SqliteMixedDatasource } = require('./data-sources/sqlite-mixed-datasource') @@ -27,7 +26,6 @@ module.exports = { MatterCloud, Planaria, RunConnectFetcher, - Server, SqliteDatasource, SqliteMixedDatasource } diff --git a/src/server.js b/src/server.js deleted file mode 100644 index 2356089..0000000 --- a/src/server.js +++ /dev/null @@ -1,285 +0,0 @@ -/** - * server.test.js - * - * Express server that exposes the Indexer - */ - -const express = require('express') -const morgan = require('morgan') -const bodyParser = require('body-parser') -const bsv = require('bsv') -const crypto = require('crypto') -const cors = require('cors') -const { Writable } = require('stream') -const Run = require('run-sdk') - -// ------------------------------------------------------------------------------------------------ -// Globals -// ------------------------------------------------------------------------------------------------ - -const calculateScripthash = x => crypto.createHash('sha256').update(Buffer.from(x, 'hex')).digest().reverse().toString('hex') - -// ------------------------------------------------------------------------------------------------ -// Server -// ------------------------------------------------------------------------------------------------ - -class Server { - constructor (database, logger, port) { - this.database = database - this.logger = logger - this.port = port - this.listener = null - this.onListening = null - } - - start () { - this.logger.debug('Starting server') - - const app = express() - - let buffer = '' - const write = (chunk, encoding, callback) => { - buffer = buffer + chunk.toString() - const lines = buffer.split(/\r\n|\n\r|\n|\r/) - for (let i = 0; i < lines.length - 1; i++) { - this.logger.info(lines[i]) - } - buffer = lines[lines.length - 1] - callback() - return true - } - app.use(morgan('tiny', { stream: new Writable({ write }) })) - - app.use(bodyParser.text({ limit: '25mb' })) - app.use(bodyParser.json({ limit: '10mb' })) - - app.use(cors()) - - app.get('/jig/:location', this.getJig.bind(this)) - app.get('/berry/:location', this.getBerry.bind(this)) - app.get('/tx/:txid', this.getTx.bind(this)) - app.get('/time/:txid', this.getTime.bind(this)) - app.get('/spends/:location', this.getSpends.bind(this)) - app.get('/unspent', this.getUnspent.bind(this)) - app.get('/trust/:txid?', this.getTrust.bind(this)) - app.get('/ban/:txid?', this.getBan.bind(this)) - app.get('/status', this.getStatus.bind(this)) - - app.post('/trust/:txid?', this.postTrust.bind(this)) - app.post('/ban/:txid', this.postBan.bind(this)) - app.post('/tx/:txid', this.postTx.bind(this)) - - app.delete('/trust/:txid', this.deleteTrust.bind(this)) - app.delete('/ban/:txid', this.deleteBan.bind(this)) - app.delete('/tx/:txid', this.deleteTx.bind(this)) - - app.use((err, req, res, next) => { - if (this.logger) this.logger.error(err.stack) - res.status(500).send('Something broke!') - next() - }) - - this.listener = app.listen(this.port, () => { - if (this.logger) this.logger.info(`Listening at http://localhost:${this.listener.address().port}`) - this.port = this.listener.address().port - if (this.onListening) this.onListening() - }) - } - - stop () { - if (!this.listener) return - this.listener.close() - this.listener = null - } - - async getJig (req, res, next) { - try { - const state = await this.database.getJigState(req.params.location) - if (state) { - res.setHeader('Content-Type', 'application/json') - res.send(state) - } else { - res.status(404).send(`Not found: ${req.params.location}\n`) - } - } catch (e) { next(e) } - } - - async getBerry (req, res, next) { - try { - const state = await this.database.getBerryState(req.params.location) - if (state) { - res.setHeader('Content-Type', 'application/json') - res.send(state) - } else { - res.status(404).send(`Not found: ${req.params.location}\n`) - } - } catch (e) { next(e) } - } - - async getTx (req, res, next) { - try { - const txid = this._parseTxid(req.params.txid) - const rawTx = await this.database.getTransactionHex(txid) - if (rawTx) { - res.send(rawTx) - } else { - res.status(404).send(`Not found: ${req.params.txid}\n`) - } - } catch (e) { next(e) } - } - - async getTime (req, res, next) { - try { - const txid = this._parseTxid(req.params.txid) - const time = await this.database.getTransactionTime(txid) - if (time) { - res.json(time) - } else { - res.status(404).send(`Not found: ${req.params.txid}\n`) - } - } catch (e) { next(e) } - } - - async getSpends (req, res, next) { - try { - const txid = await this.database.getSpend(req.params.location) - if (txid) { - res.send(txid) - } else { - res.status(404).send(`Not spent: ${req.params.location}\n`) - } - } catch (e) { next(e) } - } - - async getUnspent (req, res, next) { - try { - const cls = req.query.class - const lock = req.query.lock - let scripthash = req.query.scripthash - if (req.query.address) scripthash = calculateScripthash(new Run.util.CommonLock(req.query.address).script()) - if (req.query.pubkey) scripthash = calculateScripthash(new Run.util.CommonLock(req.query.pubkey).script()) - - if (cls && lock && scripthash) { - res.json(await this.database.getAllUnspentByClassOriginAndLockOriginAndScripthash(cls, lock, scripthash)) - } else if (cls && lock) { - res.json(await this.database.getAllUnspentByClassOriginAndLockOrigin(cls, lock)) - } else if (cls && scripthash) { - res.json(await this.database.getAllUnspentByClassOriginAndScripthash(cls, scripthash)) - } else if (lock && scripthash) { - res.json(await this.database.getAllUnspentByLockOriginAndScripthash(lock, scripthash)) - } else if (scripthash) { - res.json(await this.database.getAllUnspentByScripthash(scripthash)) - } else if (lock) { - res.json(await this.database.getAllUnspentByLockOrigin(lock)) - } else if (cls) { - res.json(await this.database.getAllUnspentByClassOrigin(cls)) - } else { - res.json(await this.database.getAllUnspent()) - } - } catch (e) { next(e) } - } - - async getTrust (req, res, next) { - try { - if (req.params.txid) { - res.json(await this.database.isTrusted(req.params.txid)) - } else { - res.json(Array.from(await this.database.getTrustlist())) - } - } catch (e) { next(e) } - } - - async getBan (req, res, next) { - try { - if (req.params.txid) { - res.json(await this.database.isBanned(req.params.txid)) - } else { - res.json(Array.from(await this.database.getBanlist())) - } - } catch (e) { next(e) } - } - - async getStatus (req, res, next) { - try { - const status = { - height: await this.database.getHeight(), - hash: await this.database.getHash() - } - res.json(status) - } catch (e) { next(e) } - } - - async postTrust (req, res, next) { - try { - if (Array.isArray(req.body)) { - for (const maybeTxid of req.body) { - const txid = this._parseTxid(maybeTxid) - await this.database.trust(txid) - } - res.send(`Trusted ${req.body.length} transactions\n`) - } else { - const txid = this._parseTxid(req.params.txid) - await this.database.trust(txid) - res.send(`Trusted ${req.params.txid}\n`) - } - } catch (e) { next(e) } - } - - async postBan (req, res, next) { - try { - const txid = this._parseTxid(req.params.txid) - await this.database.ban(txid) - res.send(`Banned ${req.params.txid}\n`) - } catch (e) { next(e) } - } - - async postTx (req, res, next) { - try { - let txid = this._parseTxid(req.params.txid) - let hex = null - if (typeof req.body === 'string') { - hex = req.body - const bsvtx = new bsv.Transaction(hex) - if (!txid) txid = bsvtx.hash - if (txid && txid !== bsvtx.hash) throw new Error('txid does not match rawtx') - } - if (!txid) throw new Error('Invalid request parameters') - await this.database.addTransaction(txid, hex) - res.send(`Added ${txid}\n`) - } catch (e) { next(e) } - } - - async deleteTrust (req, res, next) { - try { - const txid = this._parseTxid(req.params.txid) - await this.database.untrust(txid) - res.send(`Untrusted ${req.params.txid}\n`) - } catch (e) { next(e) } - } - - async deleteBan (req, res, next) { - try { - const txid = this._parseTxid(req.params.txid) - await this.database.unban(txid) - res.send(`Unbanned ${req.params.txid}\n`) - } catch (e) { next(e) } - } - - async deleteTx (req, res, next) { - try { - const txid = this._parseTxid(req.params.txid) - await this.database.deleteTransaction(txid) - res.send(`Removed ${req.params.txid}\n`) - } catch (e) { next(e) } - } - - _parseTxid (txid) { - txid = txid.trim().toLowerCase() - if (!/^[0-9a-f]{64}$/.test(txid)) throw new Error('Not a txid: ' + txid) - return txid - } -} - -// ------------------------------------------------------------------------------------------------ - -module.exports = Server diff --git a/src/start.js b/src/start.js index 6651ad7..16582b6 100644 --- a/src/start.js +++ b/src/start.js @@ -22,7 +22,7 @@ const { SqliteMixedDatasource } = require('./data-sources/sqlite-mixed-datasourc const { ApiBlobStorage } = require('./data-sources/api-blob-storage') const { DbTrustList } = require('./trust-list/db-trust-list') const { TrustAllTrustList } = require('./trust-list/trust-all-trust-list') -const { buildServer } = require('./build-server') +const { buildMainServer } = require('./http/build-main-server') // ------------------------------------------------------------------------------------------------ // Globals @@ -81,7 +81,7 @@ const indexer = new Indexer(database, api, NETWORK, FETCH_LIMIT, WORKERS, logger dataApiRoot: DATA_API_ROOT }) -const server = buildServer(database, logger, readonly) +const server = buildMainServer(database, logger, readonly) let started = false diff --git a/test/server.test.js b/test/server.test.js index c28ac01..083658c 100644 --- a/test/server.test.js +++ b/test/server.test.js @@ -14,7 +14,7 @@ const { DEFAULT_TRUSTLIST } = require('../src/config') const Database = require('../src/database') const { SqliteDatasource } = require('../src/data-sources/sqlite-datasource') const { DbTrustList } = require('../src/trust-list/db-trust-list') -const { buildServer } = require('../src/build-server') +const { buildServer } = require('../src/http/build-main-server') // ------------------------------------------------------------------------------------------------ // Globals From 1f948e71456fc06dbe942f21bdd19105f16eec81 Mon Sep 17 00:00:00 2001 From: hojarasca Date: Thu, 20 Jan 2022 08:54:40 -0300 Subject: [PATCH 40/82] using express js paramenters --- src/http/api-server.js | 4 ++++ src/http/build-main-server.js | 34 +++++++++++++++++++++------------- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/src/http/api-server.js b/src/http/api-server.js index 781637e..532f1b4 100644 --- a/src/http/api-server.js +++ b/src/http/api-server.js @@ -78,6 +78,10 @@ class ApiServer { delete (url, handler) { this.app.delete(url, asyncHandler(handler)) } + + param (name, fn) { + this.app.param(name, fn) + } } module.exports = { ApiServer } diff --git a/src/http/build-main-server.js b/src/http/build-main-server.js index 2e6ea60..6066e30 100644 --- a/src/http/build-main-server.js +++ b/src/http/build-main-server.js @@ -8,6 +8,8 @@ const bsv = require('bsv') const crypto = require('crypto') const Run = require('run-sdk') const { ApiServer } = require('./api-server') +const { parseTxid } = require('../util/parse-txid') +const { ApiError } = require('./api-error') // ------------------------------------------------------------------------------------------------ // Globals @@ -15,11 +17,12 @@ const { ApiServer } = require('./api-server') const calculateScripthash = x => crypto.createHash('sha256').update(Buffer.from(x, 'hex')).digest().reverse().toString('hex') -const parseTxid = (txid) => { - txid = txid.trim().toLowerCase() - if (!/^[0-9a-f]{64}$/.test(txid)) throw new Error('Not a txid: ' + txid) - return txid -} +const validateTxid = (aString) => parseTxid( + aString, + () => { + throw new ApiError('wrong argument: txid', 'wrong-arguments', 400, { txid: aString }) + } +) // ------------------------------------------------------------------------------------------------ // Server @@ -28,6 +31,11 @@ const parseTxid = (txid) => { const buildMainServer = (database, logger, readonly = false) => { const server = new ApiServer(logger) + server.param('txid', (req, res, next, value) => { + req.params.txid = validateTxid(value) + next() + }) + server.get('/jig/:location', async (req, res) => { const state = await database.getJigState(req.params.location) if (state) { @@ -49,7 +57,7 @@ const buildMainServer = (database, logger, readonly = false) => { }) server.get('/tx/:txid', async (req, res) => { - const txid = parseTxid(req.params.txid) + const txid = req.params.txid const rawTx = await database.getTransactionHex(txid) if (rawTx) { res.send(rawTx) @@ -59,7 +67,7 @@ const buildMainServer = (database, logger, readonly = false) => { }) server.get('/time/:txid', async (req, res) => { - const txid = parseTxid(req.params.txid) + const txid = req.params.txid const time = await database.getTransactionTime(txid) if (time) { res.json(time) @@ -134,19 +142,19 @@ const buildMainServer = (database, logger, readonly = false) => { server.post('/trust/:txid?', async (req, res) => { if (Array.isArray(req.body)) { for (const maybeTxid of req.body) { - const txid = parseTxid(maybeTxid) + const txid = validateTxid(maybeTxid) await database.trust(txid) } res.send(`Trusted ${req.body.length} transactions\n`) } else { - const txid = parseTxid(req.params.txid) + const txid = req.params.txid await database.trust(txid) res.send(`Trusted ${req.params.txid}\n`) } }) server.post('/ban/:txid', async (req, res) => { - const txid = parseTxid(req.params.txid) + const txid = req.params.txid await database.ban(txid) res.send(`Banned ${req.params.txid}\n`) }) @@ -163,19 +171,19 @@ const buildMainServer = (database, logger, readonly = false) => { }) server.delete('/trust/:txid', async (req, res) => { - const txid = parseTxid(req.params.txid) + const txid = req.params.txid await database.untrust(txid) res.send(`Untrusted ${req.params.txid}\n`) }) server.delete('/ban/:txid', async (req, res) => { - const txid = parseTxid(req.params.txid) + const txid = req.params.txid await database.unban(txid) res.send(`Unbanned ${req.params.txid}\n`) }) server.delete('/tx/:txid', async (req, res) => { - const txid = parseTxid(req.params.txid) + const txid = req.params.txid await database.deleteTransaction(txid) res.send(`Removed ${req.params.txid}\n`) }) From 30948b360cd42d48143593fa9208865624ab69df Mon Sep 17 00:00:00 2001 From: hojarasca Date: Thu, 20 Jan 2022 09:28:28 -0300 Subject: [PATCH 41/82] created binary for execution server --- package.json | 3 +- src/bin/start-execution-server.js | 35 +++++++++++++++++++ src/{ => bin}/start.js | 30 ++++++++-------- src/database.js | 4 +-- src/{ => execution}/executor.js | 4 +-- .../build-execution-server.js | 6 ++-- src/indexer.js | 2 +- test/execution-server.test.js | 2 +- 8 files changed, 61 insertions(+), 25 deletions(-) create mode 100644 src/bin/start-execution-server.js rename src/{ => bin}/start.js (79%) rename src/{ => execution}/executor.js (97%) rename src/{execution => http}/build-execution-server.js (92%) diff --git a/package.json b/package.json index 46c1525..5df5a33 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,8 @@ "main": "src/index.js", "scripts": { "lint": "standard --fix", - "start": "node --experimental-worker src/start.js", + "start": "node --experimental-worker src/bin/start.js", + "start-execution-server": "node --experimental-worker src/bin/start-execution-server.js", "test": "node --experimental-worker node_modules/mocha/bin/mocha", "test:cover": "nyc -r=text -r=lcovonly -x=test/** node --experimental-worker node_modules/mocha/bin/mocha", "bump": "npm version patch && git push --follow-tags && npm publish" diff --git a/src/bin/start-execution-server.js b/src/bin/start-execution-server.js new file mode 100644 index 0000000..25cf8a5 --- /dev/null +++ b/src/bin/start-execution-server.js @@ -0,0 +1,35 @@ +const { PORT, DEBUG, WORKERS, DATA_API_ROOT, NETWORK } = require('../config') +const { buildExecutionServer } = require('../http/build-execution-server') +const { ApiBlobStorage } = require('../data-sources/api-blob-storage') + +const logger = {} +logger.info = console.info.bind(console) +logger.warn = console.warn.bind(console) +logger.error = console.error.bind(console) +logger.debug = DEBUG ? console.debug.bind(console) : () => {} + +const blobStorage = new ApiBlobStorage(DATA_API_ROOT) + +const server = buildExecutionServer( + logger, + WORKERS, + blobStorage, + require.resolve('../worker.js'), + NETWORK, + { + dataApiRoot: DATA_API_ROOT + } +) + +async function main () { + await server.start(PORT) +} + +const shutdown = async () => { + await server.stop() +} + +process.on('SIGTERM', shutdown) +process.on('SIGINT', shutdown) + +main() diff --git a/src/start.js b/src/bin/start.js similarity index 79% rename from src/start.js rename to src/bin/start.js index 16582b6..cc2d2d0 100644 --- a/src/start.js +++ b/src/bin/start.js @@ -4,25 +4,25 @@ * Entry point */ -const Indexer = require('./indexer') +const Indexer = require('../indexer') const { API, DB, NETWORK, PORT, FETCH_LIMIT, WORKERS, MATTERCLOUD_KEY, PLANARIA_TOKEN, START_HEIGHT, MEMPOOL_EXPIRATION, ZMQ_URL, RPC_URL, DEFAULT_TRUSTLIST, DEBUG, SERVE_ONLY, DATA_SOURCE, DATA_API_ROOT, WORKER_TRUST_SOURCE, WORKER_CACHE_TYPE, TRUST_LIST -} = require('./config') -const MatterCloud = require('./mattercloud') -const Planaria = require('./planaria') -const RunConnectFetcher = require('./run-connect') -const BitcoinNodeConnection = require('./bitcoin-node-connection') -const BitcoinRpc = require('./bitcoin-rpc') -const BitcoinZmq = require('./bitcoin-zmq') -const Database = require('./database') -const { SqliteDatasource } = require('./data-sources/sqlite-datasource') -const { SqliteMixedDatasource } = require('./data-sources/sqlite-mixed-datasource') -const { ApiBlobStorage } = require('./data-sources/api-blob-storage') -const { DbTrustList } = require('./trust-list/db-trust-list') -const { TrustAllTrustList } = require('./trust-list/trust-all-trust-list') -const { buildMainServer } = require('./http/build-main-server') +} = require('../config') +const MatterCloud = require('../mattercloud') +const Planaria = require('../planaria') +const RunConnectFetcher = require('../run-connect') +const BitcoinNodeConnection = require('../bitcoin-node-connection') +const BitcoinRpc = require('../bitcoin-rpc') +const BitcoinZmq = require('../bitcoin-zmq') +const Database = require('../database') +const { SqliteDatasource } = require('../data-sources/sqlite-datasource') +const { SqliteMixedDatasource } = require('../data-sources/sqlite-mixed-datasource') +const { ApiBlobStorage } = require('../data-sources/api-blob-storage') +const { DbTrustList } = require('../trust-list/db-trust-list') +const { TrustAllTrustList } = require('../trust-list/trust-all-trust-list') +const { buildMainServer } = require('../http/build-main-server') // ------------------------------------------------------------------------------------------------ // Globals diff --git a/src/database.js b/src/database.js index ce5a088..dfb299a 100644 --- a/src/database.js +++ b/src/database.js @@ -66,8 +66,8 @@ class Database { if (time) { await this.setTransactionTime(txid, time) } }) - const downloaded = await this.isTransactionIndexed(txid) - if (downloaded) return + const indexed = await this.isTransactionIndexed(txid) + if (indexed) return if (!txhex) { txhex = await this.ds.getTxHex(txid) } diff --git a/src/executor.js b/src/execution/executor.js similarity index 97% rename from src/executor.js rename to src/execution/executor.js index 4f4d3d0..4546464 100644 --- a/src/executor.js +++ b/src/execution/executor.js @@ -5,7 +5,7 @@ */ const { Worker } = require('worker_threads') -const Bus = require('./bus') +const Bus = require('../bus') // ------------------------------------------------------------------------------------------------ // Executor @@ -35,7 +35,7 @@ class Executor { for (let i = 0; i < this.numWorkers; i++) { this.logger.debug('Starting worker', i) - const path = require.resolve('./worker.js') + const path = require.resolve('../worker.js') const worker = new Worker(path, { workerData: { id: i, network: this.network, ...this.workerOpts } }) diff --git a/src/execution/build-execution-server.js b/src/http/build-execution-server.js similarity index 92% rename from src/execution/build-execution-server.js rename to src/http/build-execution-server.js index 6fb6b89..a5c1d21 100644 --- a/src/execution/build-execution-server.js +++ b/src/http/build-execution-server.js @@ -1,10 +1,10 @@ -const { ApiServer } = require('../http/api-server') +const { ApiServer } = require('./api-server') const genericPool = require('generic-pool') const { Worker } = require('worker_threads') const Bus = require('../bus') -const { ApiError } = require('../http/api-error') +const { ApiError } = require('./api-error') const { parseTxid } = require('../util/parse-txid') -const { ExecutionError } = require('./execution-error') +const { ExecutionError } = require('../execution/execution-error') const buildExecutionServer = (logger, count, blobStorage, workerPath, network, workerOpts = {}) => { const factory = { diff --git a/src/indexer.js b/src/indexer.js index 63974ac..74b091f 100644 --- a/src/indexer.js +++ b/src/indexer.js @@ -6,7 +6,7 @@ const Database = require('./database') const Downloader = require('./downloader') -const Executor = require('./executor') +const Executor = require('./execution/executor') const Crawler = require('./crawler') // ------------------------------------------------------------------------------------------------ diff --git a/test/execution-server.test.js b/test/execution-server.test.js index fe3e214..e6e1126 100644 --- a/test/execution-server.test.js +++ b/test/execution-server.test.js @@ -1,4 +1,4 @@ -const { buildExecutionServer } = require('../src/execution/build-execution-server') +const { buildExecutionServer } = require('../src/http/build-execution-server') const { MemoryBlobStorage } = require('../src/data-sources/memory-blob-storage') const txs = require('./txns.json') const fetch = require('node-fetch') From d0ea22a51976ba891b0ded5a8e315e67cd887d41 Mon Sep 17 00:00:00 2001 From: hojarasca Date: Thu, 20 Jan 2022 09:29:39 -0300 Subject: [PATCH 42/82] fixed tests. --- test/server.test.js | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/test/server.test.js b/test/server.test.js index 083658c..c96167e 100644 --- a/test/server.test.js +++ b/test/server.test.js @@ -14,7 +14,7 @@ const { DEFAULT_TRUSTLIST } = require('../src/config') const Database = require('../src/database') const { SqliteDatasource } = require('../src/data-sources/sqlite-datasource') const { DbTrustList } = require('../src/trust-list/db-trust-list') -const { buildServer } = require('../src/http/build-main-server') +const { buildMainServer } = require('../src/http/build-main-server') // ------------------------------------------------------------------------------------------------ // Globals @@ -46,7 +46,7 @@ describe('Server', () => { describe('post tx', () => { it('add with body', async () => { const indexer = new Indexer(database, {}, 'main', 1, 1, logger, 0, Infinity, []) - const server = buildServer(database, logger) + const server = buildMainServer(database, logger) await indexer.start() const port = 52521 await server.start(port) @@ -64,7 +64,7 @@ describe('Server', () => { it('does not throw if add with rawtx mismatch', async () => { const indexer = new Indexer(database, {}, 'main', 1, 1, logger, 0, Infinity, []) - const server = buildServer(database, logger) + const server = buildMainServer(database, logger) await indexer.start() await server.start() const txid = '3f9de452f0c3c96be737d42aa0941b27412211976688967adb3174ee18b04c64' @@ -83,7 +83,7 @@ describe('Server', () => { describe('post trust', () => { it('trust multiple', async () => { const indexer = new Indexer(database, {}, 'main', 1, 1, logger, 0, Infinity, []) - const server = buildServer(database, logger) + const server = buildMainServer(database, logger) await indexer.start() await server.start() const trustlist = [ @@ -107,7 +107,7 @@ describe('Server', () => { describe('get jig', () => { it('returns state if exists', async () => { const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, DEFAULT_TRUSTLIST) - const server = buildServer(database, logger) + const server = buildMainServer(database, logger) await indexer.start() await server.start() const promise = indexed(indexer, '9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102') @@ -125,7 +125,7 @@ describe('Server', () => { it('returns 404 if missing', async () => { const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, []) - const server = buildServer(database, logger) + const server = buildMainServer(database, logger) await indexer.start() await server.start() const location = '9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102_o1' @@ -147,7 +147,7 @@ describe('Server', () => { describe('get berry', () => { it('returns state if exists', async () => { const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, DEFAULT_TRUSTLIST) - const server = buildServer(database, logger) + const server = buildMainServer(database, logger) await indexer.start() await server.start() await database.addTransaction('bfa5180e601e92af23d80782bf625b102ac110105a392e376fe7607e4e87dc8d') @@ -164,7 +164,7 @@ describe('Server', () => { it('returns 404 if missing', async () => { const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, []) - const server = buildServer(database, logger) + const server = buildMainServer(database, logger) await indexer.start() await server.start() const location = '24cde3638a444c8ad397536127833878ffdfe1b04d5595489bd294e50d77105a_o1?berry=2f3492ef5401d887a93ca09820dff952f355431cea306841a70d163e32b2acad&version=5' @@ -186,7 +186,7 @@ describe('Server', () => { describe('get tx', () => { it('returns rawtx if downloaded', async () => { const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, []) - const server = buildServer(database, logger) + const server = buildMainServer(database, logger) await indexer.start() await server.start() const txid = '9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102' @@ -204,7 +204,7 @@ describe('Server', () => { it('returns 404 if missing', async () => { const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, []) - const server = buildServer(database, logger) + const server = buildMainServer(database, logger) await indexer.start() await server.start() const txid = '9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102' @@ -222,7 +222,7 @@ describe('Server', () => { it('returns 404 if not downloaded', async () => { const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, []) - const server = buildServer(database, logger) + const server = buildMainServer(database, logger) await indexer.start() await server.start() const txid = '1111111111111111111111111111111111111111111111111111111111111111' @@ -245,7 +245,7 @@ describe('Server', () => { describe('get unspent', () => { it('query all unspent', async () => { const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, DEFAULT_TRUSTLIST) - const server = buildServer(database, logger) + const server = buildMainServer(database, logger) await indexer.start() await server.start() const promise = indexed(indexer, '9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102') @@ -264,7 +264,7 @@ describe('Server', () => { it('query unspent by address', async () => { const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, DEFAULT_TRUSTLIST) - const server = buildServer(database, logger) + const server = buildMainServer(database, logger) await indexer.start() await server.start() const promise = indexed(indexer, '9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102') @@ -285,7 +285,7 @@ describe('Server', () => { describe('misc', () => { it('cors', async () => { const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, []) - const server = buildServer(database, logger) + const server = buildMainServer(database, logger) await indexer.start() await server.start() const opts = { headers: { Origin: 'https://www.google.com' } } From 727c3cfca9634b19c76ab700537228bcc1e56850 Mon Sep 17 00:00:00 2001 From: hojarasca Date: Thu, 20 Jan 2022 09:31:02 -0300 Subject: [PATCH 43/82] removed weird file --- test/index.js | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 test/index.js diff --git a/test/index.js b/test/index.js deleted file mode 100644 index 97e8e7d..0000000 --- a/test/index.js +++ /dev/null @@ -1,9 +0,0 @@ -/** - * index.js - * - * Tests for run-db - */ - -require('./crawler.test') -require('./indexer.test') -require('./server.test') From 497f9903017a0b37691f9dfd0816940137804afc Mon Sep 17 00:00:00 2001 From: hojarasca Date: Thu, 20 Jan 2022 11:23:37 -0300 Subject: [PATCH 44/82] basic external execution working. --- src/bin/start.js | 44 ++++++++++++++----- src/config.js | 4 +- src/execution/api-executor.js | 83 +++++++++++++++++++++++++++++++++++ src/indexer.js | 5 +-- 4 files changed, 120 insertions(+), 16 deletions(-) create mode 100644 src/execution/api-executor.js diff --git a/src/bin/start.js b/src/bin/start.js index cc2d2d0..df2f7da 100644 --- a/src/bin/start.js +++ b/src/bin/start.js @@ -8,7 +8,7 @@ const Indexer = require('../indexer') const { API, DB, NETWORK, PORT, FETCH_LIMIT, WORKERS, MATTERCLOUD_KEY, PLANARIA_TOKEN, START_HEIGHT, MEMPOOL_EXPIRATION, ZMQ_URL, RPC_URL, DEFAULT_TRUSTLIST, DEBUG, SERVE_ONLY, DATA_SOURCE, DATA_API_ROOT, - WORKER_TRUST_SOURCE, WORKER_CACHE_TYPE, TRUST_LIST + WORKER_TRUST_SOURCE, WORKER_CACHE_TYPE, TRUST_LIST, EXECUTOR } = require('../config') const MatterCloud = require('../mattercloud') const Planaria = require('../planaria') @@ -23,6 +23,8 @@ const { ApiBlobStorage } = require('../data-sources/api-blob-storage') const { DbTrustList } = require('../trust-list/db-trust-list') const { TrustAllTrustList } = require('../trust-list/trust-all-trust-list') const { buildMainServer } = require('../http/build-main-server') +const Executor = require('../execution/executor') +const ApiExecutor = require('../execution/api-executor') // ------------------------------------------------------------------------------------------------ // Globals @@ -36,8 +38,12 @@ logger.debug = DEBUG ? console.debug.bind(console) : () => {} let api = null switch (API) { - case 'mattercloud': api = new MatterCloud(MATTERCLOUD_KEY, logger); break - case 'planaria': api = new Planaria(PLANARIA_TOKEN, logger); break + case 'mattercloud': + api = new MatterCloud(MATTERCLOUD_KEY, logger) + break + case 'planaria': + api = new Planaria(PLANARIA_TOKEN, logger) + break case 'bitcoin-node': if (ZMQ_URL === null) { throw new Error('please specify ZQM_URL when using bitcoin-node API') @@ -48,9 +54,14 @@ switch (API) { } api = new BitcoinNodeConnection(new BitcoinZmq(ZMQ_URL), new BitcoinRpc(RPC_URL)) break - case 'run': api = new RunConnectFetcher(); break - case 'none': api = {}; break - default: throw new Error(`Unknown API: ${API}`) + case 'run': + api = new RunConnectFetcher() + break + case 'none': + api = {} + break + default: + throw new Error(`Unknown API: ${API}`) } const readonly = !!SERVE_ONLY @@ -74,12 +85,21 @@ if (TRUST_LIST === 'db') { const database = new Database(dataSource, trustList, logger) -const indexer = new Indexer(database, api, NETWORK, FETCH_LIMIT, WORKERS, logger, - START_HEIGHT, MEMPOOL_EXPIRATION, DEFAULT_TRUSTLIST, { - trustSource: WORKER_TRUST_SOURCE, - cacheType: WORKER_CACHE_TYPE, - dataApiRoot: DATA_API_ROOT - }) +const executor = EXECUTOR === 'local' + ? new Executor(NETWORK, WORKERS, database, logger, { trustSource: WORKER_TRUST_SOURCE, cacheType: WORKER_CACHE_TYPE, dataApiRoot: DATA_API_ROOT }) + : new ApiExecutor('http://localhost:3200/execute', trustList, NETWORK, WORKERS, logger) + +const indexer = new Indexer( + database, + api, + executor, + NETWORK, + FETCH_LIMIT, + logger, + START_HEIGHT, + MEMPOOL_EXPIRATION, + DEFAULT_TRUSTLIST +) const server = buildMainServer(database, logger, readonly) diff --git a/src/config.js b/src/config.js index 5211f8f..6ef7936 100644 --- a/src/config.js +++ b/src/config.js @@ -30,6 +30,7 @@ const DATA_API_ROOT = process.env.DATA_API_ROOT || null const WORKER_TRUST_SOURCE = process.env.WORKER_TRUST_SOURCE || 'db' const WORKER_CACHE_TYPE = process.env.WORKER_CACHE_TYPE || 'parent' const TRUST_LIST = process.env.TRUST_LIST || 'db' +const EXECUTOR = process.env.EXECUTOR || 'local' require('axios').default.defaults.timeout = TIMEOUT @@ -125,5 +126,6 @@ module.exports = { DATA_API_ROOT, WORKER_TRUST_SOURCE, WORKER_CACHE_TYPE, - TRUST_LIST + TRUST_LIST, + EXECUTOR } diff --git a/src/execution/api-executor.js b/src/execution/api-executor.js new file mode 100644 index 0000000..d3e88e0 --- /dev/null +++ b/src/execution/api-executor.js @@ -0,0 +1,83 @@ +/** + * executor.js + * + * Executes RUN transactions and calculates state + */ +const fetch = require('node-fetch') +const genericPool = require('generic-pool') + +class ApiExecutor { + constructor (endpoint, trustList, network, concurrencyNumber, logger) { + this.endpoint = endpoint + this.trustList = trustList + this.network = network + this.logger = logger + + this.onIndexed = null + this.onExecuteFailed = null + + this.executing = new Set() + + const factory = { + create: () => ({}), + destroy: () => {} + } + const opts = { + min: 1, + max: concurrencyNumber + } + this.pool = genericPool.createPool(factory, opts) + } + + start () { + + } + + async stop () { + this.logger.debug('Stopping all workers') + + await Promise.all(this.workers.map(worker => worker.terminate())) + + this.workers = [] + this.workerRequests = [] + } + + async execute (txid) { + if (this.executing.has(txid)) return + this.logger.debug('Enqueueing', txid, 'for execution') + const token = await this.pool.acquire() + try { + this.executing.add(txid) + + const trustList = await this.trustList.executionTrustList() + + const httpResponse = await fetch(this.endpoint, { + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ + txid, + trustList + }) + }) + if (!httpResponse.ok) { + await this.onExecuteFailed(txid, 'execution error') // TOOD: this should actually be a retry. + this.executing.delete(txid) + return + } + const json = await httpResponse.json() + const { response } = json + await this.onIndexed(txid, response) + } catch (e) { + await this.onExecuteFailed(txid, 'execution error') // TOOD: this should actually be a retry. + } finally { + this.executing.delete(txid) + await this.pool.release(token) + } + } +} + +// ------------------------------------------------------------------------------------------------ + +module.exports = ApiExecutor diff --git a/src/indexer.js b/src/indexer.js index 74b091f..4801b67 100644 --- a/src/indexer.js +++ b/src/indexer.js @@ -6,7 +6,6 @@ const Database = require('./database') const Downloader = require('./downloader') -const Executor = require('./execution/executor') const Crawler = require('./crawler') // ------------------------------------------------------------------------------------------------ @@ -14,7 +13,7 @@ const Crawler = require('./crawler') // ------------------------------------------------------------------------------------------------ class Indexer { - constructor (database, api, network, numParallelDownloads, numParallelExecutes, logger, startHeight, mempoolExpiration, defaultTrustlist, workerOpts = {}) { + constructor (database, api, executor, network, numParallelDownloads, logger, startHeight, mempoolExpiration, defaultTrustlist) { this.onDownload = null this.onFailToDownload = null this.onIndex = null @@ -33,7 +32,7 @@ class Indexer { const fetchFunction = this.api.fetch ? this.api.fetch.bind(this.api) : null this.downloader = new Downloader(fetchFunction, numParallelDownloads) - this.executor = new Executor(network, numParallelExecutes, this.database, this.logger, workerOpts) + this.executor = executor this.crawler = new Crawler(api, this.logger) this.database.onReadyToExecute = this._onReadyToExecute.bind(this) From a52544a9e7c1852e5f60098fe02acbcb8d028f4b Mon Sep 17 00:00:00 2001 From: hojarasca Date: Thu, 20 Jan 2022 13:18:57 -0300 Subject: [PATCH 45/82] extracted execution url config. --- src/bin/start.js | 4 ++-- src/config.js | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/bin/start.js b/src/bin/start.js index df2f7da..6e66b69 100644 --- a/src/bin/start.js +++ b/src/bin/start.js @@ -8,7 +8,7 @@ const Indexer = require('../indexer') const { API, DB, NETWORK, PORT, FETCH_LIMIT, WORKERS, MATTERCLOUD_KEY, PLANARIA_TOKEN, START_HEIGHT, MEMPOOL_EXPIRATION, ZMQ_URL, RPC_URL, DEFAULT_TRUSTLIST, DEBUG, SERVE_ONLY, DATA_SOURCE, DATA_API_ROOT, - WORKER_TRUST_SOURCE, WORKER_CACHE_TYPE, TRUST_LIST, EXECUTOR + WORKER_TRUST_SOURCE, WORKER_CACHE_TYPE, TRUST_LIST, EXECUTOR, EXECUTE_ENDPOINT } = require('../config') const MatterCloud = require('../mattercloud') const Planaria = require('../planaria') @@ -87,7 +87,7 @@ const database = new Database(dataSource, trustList, logger) const executor = EXECUTOR === 'local' ? new Executor(NETWORK, WORKERS, database, logger, { trustSource: WORKER_TRUST_SOURCE, cacheType: WORKER_CACHE_TYPE, dataApiRoot: DATA_API_ROOT }) - : new ApiExecutor('http://localhost:3200/execute', trustList, NETWORK, WORKERS, logger) + : new ApiExecutor(EXECUTE_ENDPOINT, trustList, NETWORK, WORKERS, logger) const indexer = new Indexer( database, diff --git a/src/config.js b/src/config.js index 6ef7936..71a12f9 100644 --- a/src/config.js +++ b/src/config.js @@ -31,6 +31,7 @@ const WORKER_TRUST_SOURCE = process.env.WORKER_TRUST_SOURCE || 'db' const WORKER_CACHE_TYPE = process.env.WORKER_CACHE_TYPE || 'parent' const TRUST_LIST = process.env.TRUST_LIST || 'db' const EXECUTOR = process.env.EXECUTOR || 'local' +const EXECUTE_ENDPOINT = process.env.EXECUTE_ENDPOINT || null require('axios').default.defaults.timeout = TIMEOUT @@ -127,5 +128,6 @@ module.exports = { WORKER_TRUST_SOURCE, WORKER_CACHE_TYPE, TRUST_LIST, - EXECUTOR + EXECUTOR, + EXECUTE_ENDPOINT } From d866bf316eaacdac3543f3942e3a0ad3154bb846 Mon Sep 17 00:00:00 2001 From: hojarasca Date: Thu, 20 Jan 2022 15:12:47 -0300 Subject: [PATCH 46/82] fixed api server close. --- src/http/api-server.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/http/api-server.js b/src/http/api-server.js index 532f1b4..e2fcb98 100644 --- a/src/http/api-server.js +++ b/src/http/api-server.js @@ -63,7 +63,7 @@ class ApiServer { async stop () { if (!this.listener) return await this.onStop() - this.listener.close() + await this.listener.close() this.listener = null } From 3fe3ac1440aa80de211067b2ed93c353c4509228 Mon Sep 17 00:00:00 2001 From: hojarasca Date: Thu, 20 Jan 2022 15:14:25 -0300 Subject: [PATCH 47/82] fixed vulneratiblities. --- package-lock.json | 1831 ++++++++++++++++++++++++++++----------------- package.json | 2 +- 2 files changed, 1126 insertions(+), 707 deletions(-) diff --git a/package-lock.json b/package-lock.json index 73a5f3e..e73b7ab 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,7 @@ "license": "MIT", "dependencies": { "abort-controller": "^3.0.0", - "axios": "^0.21.1", + "axios": "^0.21.2", "better-sqlite3": "^7.4.1", "body-parser": "^1.19.0", "cors": "^2.8.5", @@ -479,9 +479,9 @@ } }, "node_modules/@eslint/eslintrc": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-0.2.2.tgz", - "integrity": "sha512-EfB5OHNYp1F4px/LI/FEnGylop7nOqkQ1LRzCM0KccA2U8tvV8w01KBv37LbO7nW4H+YhKyo2LcJhRwjjV17QQ==", + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-0.3.0.tgz", + "integrity": "sha512-1JTKgrOKAHVivSvOYw+sJOunkBjUOvjqWk1DPja7ZFhIS2mX/4EgTT8M7eTK9jrKhL/FvXXEbQwIs3pg1xp3dg==", "dev": true, "dependencies": { "ajv": "^6.12.4", @@ -491,7 +491,7 @@ "ignore": "^4.0.6", "import-fresh": "^3.2.1", "js-yaml": "^3.13.1", - "lodash": "^4.17.19", + "lodash": "^4.17.20", "minimatch": "^3.0.4", "strip-json-comments": "^3.1.1" }, @@ -509,15 +509,20 @@ } }, "node_modules/@eslint/eslintrc/node_modules/debug": { - "version": "4.3.2", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz", - "integrity": "sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw==", + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.3.tgz", + "integrity": "sha512-/zxw5+vh1Tfv+4Qn7a5nsbcJKPaSvCDhojn6FEl9vupwK2VCSDtEiEtqr8DFtzYFOdz63LBkxec7DYuc2jon6Q==", "dev": true, "dependencies": { "ms": "2.1.2" }, "engines": { "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } } }, "node_modules/@eslint/eslintrc/node_modules/globals": { @@ -530,6 +535,9 @@ }, "engines": { "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/@eslint/eslintrc/node_modules/js-yaml": { @@ -558,6 +566,9 @@ "dev": true, "engines": { "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/@istanbuljs/load-nyc-config": { @@ -716,7 +727,10 @@ "version": "5.3.2", "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", - "dev": true + "dev": true, + "peerDependencies": { + "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" + } }, "node_modules/aes-js": { "version": "3.1.2", @@ -746,6 +760,10 @@ "fast-json-stable-stringify": "^2.0.0", "json-schema-traverse": "^0.4.1", "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" } }, "node_modules/ansi-colors": { @@ -834,48 +852,56 @@ "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=" }, "node_modules/array-includes": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/array-includes/-/array-includes-3.1.3.tgz", - "integrity": "sha512-gcem1KlBU7c9rB+Rq8/3PPKsK2kjqeEBa3bD5kkQo4nYlOHQCJqIJFqBXDEfwaRuYTT4E+FxA9xez7Gf/e3Q7A==", + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/array-includes/-/array-includes-3.1.4.tgz", + "integrity": "sha512-ZTNSQkmWumEbiHO2GF4GmWxYVTiQyJy2XOTa15sdQSrvKn7l+180egQMqlrMOUMCyLMD7pmyQe4mMDUT6Behrw==", "dev": true, "dependencies": { "call-bind": "^1.0.2", "define-properties": "^1.1.3", - "es-abstract": "^1.18.0-next.2", + "es-abstract": "^1.19.1", "get-intrinsic": "^1.1.1", - "is-string": "^1.0.5" + "is-string": "^1.0.7" }, "engines": { "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, "node_modules/array.prototype.flat": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/array.prototype.flat/-/array.prototype.flat-1.2.4.tgz", - "integrity": "sha512-4470Xi3GAPAjZqFcljX2xzckv1qeKPizoNkiS0+O4IoPR2ZNpcjE0pkhdihlDouK+x6QOast26B4Q/O9DJnwSg==", + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/array.prototype.flat/-/array.prototype.flat-1.2.5.tgz", + "integrity": "sha512-KaYU+S+ndVqyUnignHftkwc58o3uVU1jzczILJ1tN2YaIZpFIKBiP/x/j97E5MVPsaCloPbqWLB/8qCTVvT2qg==", "dev": true, "dependencies": { - "call-bind": "^1.0.0", + "call-bind": "^1.0.2", "define-properties": "^1.1.3", - "es-abstract": "^1.18.0-next.1" + "es-abstract": "^1.19.0" }, "engines": { "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, "node_modules/array.prototype.flatmap": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/array.prototype.flatmap/-/array.prototype.flatmap-1.2.4.tgz", - "integrity": "sha512-r9Z0zYoxqHz60vvQbWEdXIEtCwHF0yxaWfno9qzXeNHvfyl3BZqygmGzb84dsubyaXLH4husF+NFgMSdpZhk2Q==", + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/array.prototype.flatmap/-/array.prototype.flatmap-1.2.5.tgz", + "integrity": "sha512-08u6rVyi1Lj7oqWbS9nUxliETrtIROT4XGTA4D/LWGten6E3ocm7cy9SIrmNHOL5XVbVuckUp3X6Xyg8/zpvHA==", "dev": true, "dependencies": { "call-bind": "^1.0.0", "define-properties": "^1.1.3", - "es-abstract": "^1.18.0-next.1", - "function-bind": "^1.1.1" + "es-abstract": "^1.19.0" }, "engines": { "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, "node_modules/assertion-error": { @@ -888,20 +914,20 @@ } }, "node_modules/astral-regex": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/astral-regex/-/astral-regex-1.0.0.tgz", - "integrity": "sha512-+Ryf6g3BKoRc7jfp7ad8tM4TtMiaWvbF/1/sQcZPkkS7ag3D5nMBCe2UfOTONtAkaG0tO0ij3C5Lwmf1EiyjHg==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/astral-regex/-/astral-regex-2.0.0.tgz", + "integrity": "sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ==", "dev": true, "engines": { - "node": ">=4" + "node": ">=8" } }, "node_modules/axios": { - "version": "0.21.1", - "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.1.tgz", - "integrity": "sha512-dKQiRHxGD9PPRIUNIWvZhPTPpl1rf/OxTYKsqKUDjBwYylTvV7SjSHJb9ratfyzM6wCdLCOYLzs73qpg5c4iGA==", + "version": "0.21.4", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.4.tgz", + "integrity": "sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==", "dependencies": { - "follow-redirects": "^1.10.0" + "follow-redirects": "^1.14.0" } }, "node_modules/balanced-match": { @@ -1130,6 +1156,9 @@ "dependencies": { "function-bind": "^1.1.1", "get-intrinsic": "^1.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, "node_modules/callsites": { @@ -1263,9 +1292,9 @@ } }, "node_modules/cliui/node_modules/ansi-regex": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", - "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", "dev": true, "engines": { "node": ">=8" @@ -1376,15 +1405,6 @@ "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz", "integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4=" }, - "node_modules/contains-path": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/contains-path/-/contains-path-0.1.0.tgz", - "integrity": "sha1-/ozxhP9mcLa67wGp1IYaXL7EEgo=", - "dev": true, - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/content-disposition": { "version": "0.5.3", "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.3.tgz", @@ -1506,9 +1526,9 @@ } }, "node_modules/deep-is": { - "version": "0.1.3", - "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz", - "integrity": "sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ=", + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", + "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", "dev": true }, "node_modules/default-require-extensions": { @@ -1667,22 +1687,25 @@ } }, "node_modules/es-abstract": { - "version": "1.18.5", - "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.18.5.tgz", - "integrity": "sha512-DDggyJLoS91CkJjgauM5c0yZMjiD1uK3KcaCeAmffGwZ+ODWzOkPN4QwRbsK5DOFf06fywmyLci3ZD8jLGhVYA==", + "version": "1.19.1", + "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.19.1.tgz", + "integrity": "sha512-2vJ6tjA/UfqLm2MPs7jxVybLoB8i1t1Jd9R3kISld20sIxPcTbLuggQOUxeWeAvIUkduv/CfMjuh4WmiXr2v9w==", "dev": true, "dependencies": { "call-bind": "^1.0.2", "es-to-primitive": "^1.2.1", "function-bind": "^1.1.1", "get-intrinsic": "^1.1.1", + "get-symbol-description": "^1.0.0", "has": "^1.0.3", "has-symbols": "^1.0.2", "internal-slot": "^1.0.3", - "is-callable": "^1.2.3", + "is-callable": "^1.2.4", "is-negative-zero": "^2.0.1", - "is-regex": "^1.1.3", - "is-string": "^1.0.6", + "is-regex": "^1.1.4", + "is-shared-array-buffer": "^1.0.1", + "is-string": "^1.0.7", + "is-weakref": "^1.0.1", "object-inspect": "^1.11.0", "object-keys": "^1.1.1", "object.assign": "^4.1.2", @@ -1692,6 +1715,9 @@ }, "engines": { "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, "node_modules/es-to-primitive": { @@ -1706,6 +1732,9 @@ }, "engines": { "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, "node_modules/es6-error": { @@ -1737,13 +1766,13 @@ } }, "node_modules/eslint": { - "version": "7.13.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-7.13.0.tgz", - "integrity": "sha512-uCORMuOO8tUzJmsdRtrvcGq5qposf7Rw0LwkTJkoDbOycVQtQjmnhZSuLQnozLE4TmAzlMVV45eCHmQ1OpDKUQ==", + "version": "7.18.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-7.18.0.tgz", + "integrity": "sha512-fbgTiE8BfUJZuBeq2Yi7J3RB3WGUQ9PNuNbmgi6jt9Iv8qrkxfy19Ds3OpL1Pm7zg3BtTVhvcUZbIRQ0wmSjAQ==", "dev": true, "dependencies": { "@babel/code-frame": "^7.0.0", - "@eslint/eslintrc": "^0.2.1", + "@eslint/eslintrc": "^0.3.0", "ajv": "^6.10.0", "chalk": "^4.0.0", "cross-spawn": "^7.0.2", @@ -1753,10 +1782,10 @@ "eslint-scope": "^5.1.1", "eslint-utils": "^2.1.0", "eslint-visitor-keys": "^2.0.0", - "espree": "^7.3.0", + "espree": "^7.3.1", "esquery": "^1.2.0", "esutils": "^2.0.2", - "file-entry-cache": "^5.0.1", + "file-entry-cache": "^6.0.0", "functional-red-black-tree": "^1.0.1", "glob-parent": "^5.0.0", "globals": "^12.1.0", @@ -1767,7 +1796,7 @@ "js-yaml": "^3.13.1", "json-stable-stringify-without-jsonify": "^1.0.1", "levn": "^0.4.1", - "lodash": "^4.17.19", + "lodash": "^4.17.20", "minimatch": "^3.0.4", "natural-compare": "^1.4.0", "optionator": "^0.9.1", @@ -1776,7 +1805,7 @@ "semver": "^7.2.1", "strip-ansi": "^6.0.0", "strip-json-comments": "^3.1.0", - "table": "^5.2.3", + "table": "^6.0.4", "text-table": "^0.2.0", "v8-compile-cache": "^2.0.3" }, @@ -1785,13 +1814,36 @@ }, "engines": { "node": "^10.12.0 || >=12.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" } }, "node_modules/eslint-config-standard": { - "version": "16.0.2", - "resolved": "https://registry.npmjs.org/eslint-config-standard/-/eslint-config-standard-16.0.2.tgz", - "integrity": "sha512-fx3f1rJDsl9bY7qzyX8SAtP8GBSk6MfXFaTfaGgk12aAYW4gJSyRm7dM790L6cbXv63fvjY4XeSzXnb4WM+SKw==", - "dev": true + "version": "16.0.3", + "resolved": "https://registry.npmjs.org/eslint-config-standard/-/eslint-config-standard-16.0.3.tgz", + "integrity": "sha512-x4fmJL5hGqNJKGHSjnLdgA6U6h1YW/G2dW9fA+cyVur4SK6lyue8+UgNKWlZtUDTXvgKDD/Oa3GQjmB5kjtVvg==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "peerDependencies": { + "eslint": "^7.12.1", + "eslint-plugin-import": "^2.22.1", + "eslint-plugin-node": "^11.1.0", + "eslint-plugin-promise": "^4.2.1 || ^5.0.0" + } }, "node_modules/eslint-config-standard-jsx": { "version": "10.0.0", @@ -1825,13 +1877,13 @@ "dev": true }, "node_modules/eslint-module-utils": { - "version": "2.6.2", - "resolved": "https://registry.npmjs.org/eslint-module-utils/-/eslint-module-utils-2.6.2.tgz", - "integrity": "sha512-QG8pcgThYOuqxupd06oYTZoNOGaUdTY1PqK+oS6ElF6vs4pBdk/aYxFVQQXzcrAqp9m7cl7lb2ubazX+g16k2Q==", + "version": "2.7.2", + "resolved": "https://registry.npmjs.org/eslint-module-utils/-/eslint-module-utils-2.7.2.tgz", + "integrity": "sha512-zquepFnWCY2ISMFwD/DqzaM++H+7PDzOpUvotJWm/y1BAFt5R4oeULgdrTejKqLkz7MA/tgstsUMNYc7wNdTrg==", "dev": true, "dependencies": { "debug": "^3.2.7", - "pkg-dir": "^2.0.0" + "find-up": "^2.1.0" }, "engines": { "node": ">=4" @@ -1919,18 +1971,6 @@ "node": ">=4" } }, - "node_modules/eslint-module-utils/node_modules/pkg-dir": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-2.0.0.tgz", - "integrity": "sha1-9tXREJ4Z1j7fQo4L1X4Sd3YVM0s=", - "dev": true, - "dependencies": { - "find-up": "^2.1.0" - }, - "engines": { - "node": ">=4" - } - }, "node_modules/eslint-plugin-es": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/eslint-plugin-es/-/eslint-plugin-es-3.0.1.tgz", @@ -1945,42 +1985,113 @@ } }, "node_modules/eslint-plugin-import": { - "version": "2.22.1", - "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.22.1.tgz", - "integrity": "sha512-8K7JjINHOpH64ozkAhpT3sd+FswIZTfMZTjdx052pnWrgRCVfp8op9tbjpAk3DdUeI/Ba4C8OjdC0r90erHEOw==", + "version": "2.24.2", + "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.24.2.tgz", + "integrity": "sha512-hNVtyhiEtZmpsabL4neEj+6M5DCLgpYyG9nzJY8lZQeQXEn5UPW1DpUdsMHMXsq98dbNm7nt1w9ZMSVpfJdi8Q==", "dev": true, "dependencies": { - "array-includes": "^3.1.1", - "array.prototype.flat": "^1.2.3", - "contains-path": "^0.1.0", + "array-includes": "^3.1.3", + "array.prototype.flat": "^1.2.4", "debug": "^2.6.9", - "doctrine": "1.5.0", - "eslint-import-resolver-node": "^0.3.4", - "eslint-module-utils": "^2.6.0", + "doctrine": "^2.1.0", + "eslint-import-resolver-node": "^0.3.6", + "eslint-module-utils": "^2.6.2", + "find-up": "^2.0.0", "has": "^1.0.3", + "is-core-module": "^2.6.0", "minimatch": "^3.0.4", - "object.values": "^1.1.1", - "read-pkg-up": "^2.0.0", - "resolve": "^1.17.0", - "tsconfig-paths": "^3.9.0" + "object.values": "^1.1.4", + "pkg-up": "^2.0.0", + "read-pkg-up": "^3.0.0", + "resolve": "^1.20.0", + "tsconfig-paths": "^3.11.0" }, "engines": { "node": ">=4" + }, + "peerDependencies": { + "eslint": "^2 || ^3 || ^4 || ^5 || ^6 || ^7.2.0" } }, "node_modules/eslint-plugin-import/node_modules/doctrine": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-1.5.0.tgz", - "integrity": "sha1-N53Ocw9hZvds76TmcHoVmwLFpvo=", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-2.1.0.tgz", + "integrity": "sha512-35mSku4ZXK0vfCuHEDAwt55dg2jNajHZ1odvF+8SSr82EsZY4QmXfuWso8oEd8zRhVObSN18aM0CjSdoBX7zIw==", "dev": true, "dependencies": { - "esutils": "^2.0.2", - "isarray": "^1.0.0" + "esutils": "^2.0.2" }, "engines": { "node": ">=0.10.0" } }, + "node_modules/eslint-plugin-import/node_modules/find-up": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-2.1.0.tgz", + "integrity": "sha1-RdG35QbHF93UgndaK3eSCjwMV6c=", + "dev": true, + "dependencies": { + "locate-path": "^2.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/eslint-plugin-import/node_modules/locate-path": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-2.0.0.tgz", + "integrity": "sha1-K1aLJl7slExtnA3pw9u7ygNUzY4=", + "dev": true, + "dependencies": { + "p-locate": "^2.0.0", + "path-exists": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/eslint-plugin-import/node_modules/p-limit": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-1.3.0.tgz", + "integrity": "sha512-vvcXsLAJ9Dr5rQOPk7toZQZJApBl2K4J6dANSsEuh6QI41JYcsS/qhTGa9ErIUUgK3WNQoJYvylxvjqmiqEA9Q==", + "dev": true, + "dependencies": { + "p-try": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/eslint-plugin-import/node_modules/p-locate": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-2.0.0.tgz", + "integrity": "sha1-IKAQOyIqcMj9OcwuWAaA893l7EM=", + "dev": true, + "dependencies": { + "p-limit": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/eslint-plugin-import/node_modules/p-try": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-1.0.0.tgz", + "integrity": "sha1-y8ec26+P1CKOE/Yh8rGiN8GyB7M=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/eslint-plugin-import/node_modules/path-exists": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", + "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=", + "dev": true, + "engines": { + "node": ">=4" + } + }, "node_modules/eslint-plugin-node": { "version": "11.1.0", "resolved": "https://registry.npmjs.org/eslint-plugin-node/-/eslint-plugin-node-11.1.0.tgz", @@ -2017,34 +2128,42 @@ } }, "node_modules/eslint-plugin-promise": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/eslint-plugin-promise/-/eslint-plugin-promise-4.2.1.tgz", - "integrity": "sha512-VoM09vT7bfA7D+upt+FjeBO5eHIJQBUWki1aPvB+vbNiHS3+oGIJGIeyBtKQTME6UPXXy3vV07OL1tHd3ANuDw==", + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/eslint-plugin-promise/-/eslint-plugin-promise-5.1.1.tgz", + "integrity": "sha512-XgdcdyNzHfmlQyweOPTxmc7pIsS6dE4MvwhXWMQ2Dxs1XAL2GJDilUsjWen6TWik0aSI+zD/PqocZBblcm9rdA==", "dev": true, "engines": { - "node": ">=6" + "node": "^10.12.0 || >=12.0.0" + }, + "peerDependencies": { + "eslint": "^7.0.0" } }, "node_modules/eslint-plugin-react": { - "version": "7.21.5", - "resolved": "https://registry.npmjs.org/eslint-plugin-react/-/eslint-plugin-react-7.21.5.tgz", - "integrity": "sha512-8MaEggC2et0wSF6bUeywF7qQ46ER81irOdWS4QWxnnlAEsnzeBevk1sWh7fhpCghPpXb+8Ks7hvaft6L/xsR6g==", + "version": "7.25.3", + "resolved": "https://registry.npmjs.org/eslint-plugin-react/-/eslint-plugin-react-7.25.3.tgz", + "integrity": "sha512-ZMbFvZ1WAYSZKY662MBVEWR45VaBT6KSJCiupjrNlcdakB90juaZeDCbJq19e73JZQubqFtgETohwgAt8u5P6w==", "dev": true, "dependencies": { - "array-includes": "^3.1.1", - "array.prototype.flatmap": "^1.2.3", + "array-includes": "^3.1.3", + "array.prototype.flatmap": "^1.2.4", "doctrine": "^2.1.0", - "has": "^1.0.3", + "estraverse": "^5.2.0", "jsx-ast-utils": "^2.4.1 || ^3.0.0", - "object.entries": "^1.1.2", - "object.fromentries": "^2.0.2", - "object.values": "^1.1.1", + "minimatch": "^3.0.4", + "object.entries": "^1.1.4", + "object.fromentries": "^2.0.4", + "object.hasown": "^1.0.0", + "object.values": "^1.1.4", "prop-types": "^15.7.2", - "resolve": "^1.18.1", - "string.prototype.matchall": "^4.0.2" + "resolve": "^2.0.0-next.3", + "string.prototype.matchall": "^4.0.5" }, "engines": { "node": ">=4" + }, + "peerDependencies": { + "eslint": "^3 || ^4 || ^5 || ^6 || ^7" } }, "node_modules/eslint-plugin-react/node_modules/doctrine": { @@ -2059,6 +2178,19 @@ "node": ">=0.10.0" } }, + "node_modules/eslint-plugin-react/node_modules/resolve": { + "version": "2.0.0-next.3", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-2.0.0-next.3.tgz", + "integrity": "sha512-W8LucSynKUIDu9ylraa7ueVZ7hc0uAgJBxVsQSKOXOyle8a93qXhcz+XAXZ8bIq2d6i4Ehddn6Evt+0/UwKk6Q==", + "dev": true, + "dependencies": { + "is-core-module": "^2.2.0", + "path-parse": "^1.0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/eslint-scope": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-5.1.1.tgz", @@ -2072,6 +2204,15 @@ "node": ">=8.0.0" } }, + "node_modules/eslint-scope/node_modules/estraverse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.3.0.tgz", + "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==", + "dev": true, + "engines": { + "node": ">=4.0" + } + }, "node_modules/eslint-utils": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/eslint-utils/-/eslint-utils-2.1.0.tgz", @@ -2103,9 +2244,9 @@ } }, "node_modules/eslint/node_modules/ansi-regex": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", - "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", "dev": true, "engines": { "node": ">=8" @@ -2121,15 +2262,20 @@ } }, "node_modules/eslint/node_modules/debug": { - "version": "4.3.2", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz", - "integrity": "sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw==", + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.3.tgz", + "integrity": "sha512-/zxw5+vh1Tfv+4Qn7a5nsbcJKPaSvCDhojn6FEl9vupwK2VCSDtEiEtqr8DFtzYFOdz63LBkxec7DYuc2jon6Q==", "dev": true, "dependencies": { "ms": "2.1.2" }, "engines": { "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } } }, "node_modules/eslint/node_modules/globals": { @@ -2142,6 +2288,9 @@ }, "engines": { "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/eslint/node_modules/js-yaml": { @@ -2179,12 +2328,12 @@ } }, "node_modules/eslint/node_modules/strip-ansi": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", - "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", "dev": true, "dependencies": { - "ansi-regex": "^5.0.0" + "ansi-regex": "^5.0.1" }, "engines": { "node": ">=8" @@ -2197,6 +2346,9 @@ "dev": true, "engines": { "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/esm": { @@ -2255,15 +2407,6 @@ "node": ">=0.10" } }, - "node_modules/esquery/node_modules/estraverse": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.2.0.tgz", - "integrity": "sha512-BxbNGGNm0RyRYvUdHpIwv9IWzeM9XClbOxwoATuFdOE7ZE6wHL+HQ5T8hoPM+zHvmKzzsEqhgy0GrQ5X13afiQ==", - "dev": true, - "engines": { - "node": ">=4.0" - } - }, "node_modules/esrecurse": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", @@ -2276,19 +2419,10 @@ "node": ">=4.0" } }, - "node_modules/esrecurse/node_modules/estraverse": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.2.0.tgz", - "integrity": "sha512-BxbNGGNm0RyRYvUdHpIwv9IWzeM9XClbOxwoATuFdOE7ZE6wHL+HQ5T8hoPM+zHvmKzzsEqhgy0GrQ5X13afiQ==", - "dev": true, - "engines": { - "node": ">=4.0" - } - }, "node_modules/estraverse": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.3.0.tgz", - "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==", + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", "dev": true, "engines": { "node": ">=4.0" @@ -2424,15 +2558,15 @@ } }, "node_modules/file-entry-cache": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-5.0.1.tgz", - "integrity": "sha512-bCg29ictuBaKUwwArK4ouCaqDgLZcysCFLmM/Yn/FDoqndh/9vNuQfXRDvTuXKLxfD/JtZQGKFT8MGcJBK644g==", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz", + "integrity": "sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==", "dev": true, "dependencies": { - "flat-cache": "^2.0.1" + "flat-cache": "^3.0.4" }, "engines": { - "node": ">=4" + "node": "^10.12.0 || >=12.0.0" } }, "node_modules/file-uri-to-path": { @@ -2506,43 +2640,41 @@ } }, "node_modules/flat-cache": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-2.0.1.tgz", - "integrity": "sha512-LoQe6yDuUMDzQAEH8sgmh4Md6oZnc/7PjtwjNFSzveXqSHt6ka9fPBuso7IGf9Rz4uqnSnWiFH2B/zj24a5ReA==", + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-3.0.4.tgz", + "integrity": "sha512-dm9s5Pw7Jc0GvMYbshN6zchCA9RgQlzzEZX3vylR9IqFfS8XciblUXOKfW6SiuJ0e13eDYZoZV5wdrev7P3Nwg==", "dev": true, "dependencies": { - "flatted": "^2.0.0", - "rimraf": "2.6.3", - "write": "1.0.3" + "flatted": "^3.1.0", + "rimraf": "^3.0.2" }, "engines": { - "node": ">=4" - } - }, - "node_modules/flat-cache/node_modules/rimraf": { - "version": "2.6.3", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz", - "integrity": "sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==", - "dev": true, - "dependencies": { - "glob": "^7.1.3" - }, - "bin": { - "rimraf": "bin.js" + "node": "^10.12.0 || >=12.0.0" } }, "node_modules/flatted": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/flatted/-/flatted-2.0.2.tgz", - "integrity": "sha512-r5wGx7YeOwNWNlCA0wQ86zKyDLMQr+/RB8xy74M4hTphfmjlijTSSXGuH8rnvKZnfT9i+75zmd8jcKdMR4O6jA==", + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.2.4.tgz", + "integrity": "sha512-8/sOawo8tJ4QOBX8YlQBMxL8+RLZfxMQOif9o0KUKTNTjMYElWPE0r/m5VNFxTRd0NSw8qSy8dajrwX4RYI1Hw==", "dev": true }, "node_modules/follow-redirects": { - "version": "1.14.2", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.2.tgz", - "integrity": "sha512-yLR6WaE2lbF0x4K2qE2p9PEXKLDjUjnR/xmjS3wHAYxtlsI9MLLBJUZirAHKzUZDGLxje7w/cXR49WOUo4rbsA==", + "version": "1.14.7", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.7.tgz", + "integrity": "sha512-+hbxoLbFMbRKDwohX8GkTataGqO6Jb7jGwpAlwgy2bIz25XtRm7KEzJM76R1WiNT5SwZkX4Y75SwBolkpmE7iQ==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], "engines": { "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } } }, "node_modules/foreground-child": { @@ -2690,6 +2822,9 @@ "function-bind": "^1.1.1", "has": "^1.0.3", "has-symbols": "^1.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, "node_modules/get-package-type": { @@ -2710,6 +2845,22 @@ "node": ">=10" } }, + "node_modules/get-symbol-description": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/get-symbol-description/-/get-symbol-description-1.0.0.tgz", + "integrity": "sha512-2EmdH1YvIQiZpltCNgkuiUnyukzxM/R6NDJX31Ke3BG1Nq5b0S2PhX59UKi9vZpPDQVdqn+1IcaAwnzTT5vCjw==", + "dev": true, + "dependencies": { + "call-bind": "^1.0.2", + "get-intrinsic": "^1.1.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/getopts": { "version": "2.2.5", "resolved": "https://registry.npmjs.org/getopts/-/getopts-2.2.5.tgz", @@ -2788,7 +2939,10 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/has-bigints/-/has-bigints-1.0.1.tgz", "integrity": "sha512-LSBS2LjbNBTf6287JEbEzvJgftkF5qFkmCo9hDRpAzKhUOlJ+hx8dd4USs00SgsUNwc4617J9ki5YtEClM2ffA==", - "dev": true + "dev": true, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } }, "node_modules/has-flag": { "version": "4.0.0", @@ -2806,6 +2960,9 @@ "dev": true, "engines": { "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, "node_modules/has-tostringtag": { @@ -2818,6 +2975,9 @@ }, "engines": { "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, "node_modules/has-unicode": { @@ -2942,6 +3102,9 @@ }, "engines": { "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/import-fresh/node_modules/resolve-from": { @@ -3034,6 +3197,9 @@ "dev": true, "dependencies": { "has-bigints": "^1.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, "node_modules/is-binary-path": { @@ -3059,6 +3225,9 @@ }, "engines": { "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, "node_modules/is-callable": { @@ -3068,6 +3237,9 @@ "dev": true, "engines": { "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, "node_modules/is-core-module": { @@ -3088,6 +3260,9 @@ }, "engines": { "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, "node_modules/is-extglob": { @@ -3123,12 +3298,15 @@ } }, "node_modules/is-negative-zero": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/is-negative-zero/-/is-negative-zero-2.0.1.tgz", - "integrity": "sha512-2z6JzQvZRa9A2Y7xC6dQQm4FSTSTNWjKIYYTt4246eMTJmIo0Q+ZyOsU66X8lxK1AbB92dFeglPLrhwpeRKO6w==", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/is-negative-zero/-/is-negative-zero-2.0.2.tgz", + "integrity": "sha512-dqJvarLawXsFbNDeJW7zAz8ItJ9cd28YufuuFzh0G8pNHjJMnY08Dv7sYX2uF5UpQOwieAeOExEYAWWfu7ZZUA==", "dev": true, "engines": { "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, "node_modules/is-number": { @@ -3150,6 +3328,9 @@ }, "engines": { "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, "node_modules/is-plain-obj": { @@ -3183,6 +3364,18 @@ }, "engines": { "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-shared-array-buffer": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-shared-array-buffer/-/is-shared-array-buffer-1.0.1.tgz", + "integrity": "sha512-IU0NmyknYZN0rChcKhRO1X8LYz5Isj/Fsqh8NJOSf+N/hCOTwy29F32Ik7a+QszE63IdvmwdTPDd6cZ5pg4cwA==", + "dev": true, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, "node_modules/is-stream": { @@ -3204,6 +3397,9 @@ }, "engines": { "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, "node_modules/is-symbol": { @@ -3216,6 +3412,9 @@ }, "engines": { "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, "node_modules/is-typedarray": { @@ -3224,6 +3423,18 @@ "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=", "dev": true }, + "node_modules/is-weakref": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-weakref/-/is-weakref-1.0.2.tgz", + "integrity": "sha512-qctsuLZmIQ0+vSSMfoVvyFe2+GSEvnmZ2ezTup1SBse9+twCCeial6EEi3Nc2KFcf6+qz2FBPnjXsk8xhKSaPQ==", + "dev": true, + "dependencies": { + "call-bind": "^1.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/is-windows": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-windows/-/is-windows-1.0.2.tgz", @@ -3459,12 +3670,12 @@ } }, "node_modules/jsx-ast-utils": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/jsx-ast-utils/-/jsx-ast-utils-3.2.0.tgz", - "integrity": "sha512-EIsmt3O3ljsU6sot/J4E1zDRxfBNrhjyf/OKjlydwgEimQuznlM4Wv7U+ueONJMyEn1WRE0K8dhi3dVAXYT24Q==", + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/jsx-ast-utils/-/jsx-ast-utils-3.2.1.tgz", + "integrity": "sha512-uP5vu8xfy2F9A6LGC22KO7e2/vGTS1MhP+18f++ZNlf0Ohaxbc9nIEwHAsejlJKyzfZzU5UIhe5ItYkitcZnZA==", "dev": true, "dependencies": { - "array-includes": "^3.1.2", + "array-includes": "^3.1.3", "object.assign": "^4.1.2" }, "engines": { @@ -3565,14 +3776,14 @@ } }, "node_modules/load-json-file": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-2.0.0.tgz", - "integrity": "sha1-eUfkIUmvgNaWy/eXvKq8/h/inKg=", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-4.0.0.tgz", + "integrity": "sha1-L19Fq5HjMhYjT9U62rZo607AmTs=", "dev": true, "dependencies": { "graceful-fs": "^4.1.2", - "parse-json": "^2.2.0", - "pify": "^2.0.0", + "parse-json": "^4.0.0", + "pify": "^3.0.0", "strip-bom": "^3.0.0" }, "engines": { @@ -3611,6 +3822,12 @@ "integrity": "sha1-+wMJF/hqMTTlvJvsDWngAT3f7bI=", "dev": true }, + "node_modules/lodash.truncate": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/lodash.truncate/-/lodash.truncate-4.4.2.tgz", + "integrity": "sha1-WjUNoLERO4N+z//VgSy+WNbq4ZM=", + "dev": true + }, "node_modules/log-symbols": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.0.0.tgz", @@ -4074,9 +4291,9 @@ } }, "node_modules/nyc/node_modules/ansi-regex": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", - "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", "dev": true, "engines": { "node": ">=8" @@ -4259,10 +4476,13 @@ } }, "node_modules/object-inspect": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.11.0.tgz", - "integrity": "sha512-jp7ikS6Sd3GxQfZJPyH3cjcbJF6GZPClgdV+EFygjFLQ5FmW/dRUnTd9PQ9k0JhoNDabWFbpF1yCdSWCC6gexg==", - "dev": true + "version": "1.12.0", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.0.tgz", + "integrity": "sha512-Ho2z80bVIvJloH+YzRmpZVQe87+qASmBUKZDWgx9cu+KDrX2ZDH/3tMy+gXbZETVGs2M8YdxObOh7XAtim9Y0g==", + "dev": true, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } }, "node_modules/object-keys": { "version": "1.1.1", @@ -4286,49 +4506,70 @@ }, "engines": { "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, "node_modules/object.entries": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/object.entries/-/object.entries-1.1.4.tgz", - "integrity": "sha512-h4LWKWE+wKQGhtMjZEBud7uLGhqyLwj8fpHOarZhD2uY3C9cRtk57VQ89ke3moByLXMedqs3XCHzyb4AmA2DjA==", + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/object.entries/-/object.entries-1.1.5.tgz", + "integrity": "sha512-TyxmjUoZggd4OrrU1W66FMDG6CuqJxsFvymeyXI51+vQLN67zYfZseptRge703kKQdo4uccgAKebXFcRCzk4+g==", "dev": true, "dependencies": { "call-bind": "^1.0.2", "define-properties": "^1.1.3", - "es-abstract": "^1.18.2" + "es-abstract": "^1.19.1" }, "engines": { "node": ">= 0.4" } }, "node_modules/object.fromentries": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/object.fromentries/-/object.fromentries-2.0.4.tgz", - "integrity": "sha512-EsFBshs5RUUpQEY1D4q/m59kMfz4YJvxuNCJcv/jWwOJr34EaVnG11ZrZa0UHB3wnzV1wx8m58T4hQL8IuNXlQ==", + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/object.fromentries/-/object.fromentries-2.0.5.tgz", + "integrity": "sha512-CAyG5mWQRRiBU57Re4FKoTBjXfDoNwdFVH2Y1tS9PqCsfUTymAohOkEMSG3aRNKmv4lV3O7p1et7c187q6bynw==", "dev": true, "dependencies": { "call-bind": "^1.0.2", "define-properties": "^1.1.3", - "es-abstract": "^1.18.0-next.2", - "has": "^1.0.3" + "es-abstract": "^1.19.1" }, "engines": { "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/object.hasown": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/object.hasown/-/object.hasown-1.1.0.tgz", + "integrity": "sha512-MhjYRfj3GBlhSkDHo6QmvgjRLXQ2zndabdf3nX0yTyZK9rPfxb6uRpAac8HXNLy1GpqWtZ81Qh4v3uOls2sRAg==", + "dev": true, + "dependencies": { + "define-properties": "^1.1.3", + "es-abstract": "^1.19.1" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, "node_modules/object.values": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/object.values/-/object.values-1.1.4.tgz", - "integrity": "sha512-TnGo7j4XSnKQoK3MfvkzqKCi0nVe/D9I9IjwTNYdb/fxYHpjrluHVOgw0AF6jrRFGMPHdfuidR09tIDiIvnaSg==", + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/object.values/-/object.values-1.1.5.tgz", + "integrity": "sha512-QUZRW0ilQ3PnPpbNtgdNV1PDbEqLIiSFB3l+EnGtBQ/8SUTLj1PZwtQHABZtLgwpJZTSZhuGLOGk57Drx2IvYg==", "dev": true, "dependencies": { "call-bind": "^1.0.2", "define-properties": "^1.1.3", - "es-abstract": "^1.18.2" + "es-abstract": "^1.19.1" }, "engines": { "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, "node_modules/on-finished": { @@ -4456,15 +4697,16 @@ } }, "node_modules/parse-json": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-2.2.0.tgz", - "integrity": "sha1-9ID0BDTvgHQfhGkJn43qGPVaTck=", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-4.0.0.tgz", + "integrity": "sha1-vjX1Qlvh9/bHRxhPmKeIy5lHfuA=", "dev": true, "dependencies": { - "error-ex": "^1.2.0" + "error-ex": "^1.3.1", + "json-parse-better-errors": "^1.0.1" }, "engines": { - "node": ">=0.10.0" + "node": ">=4" } }, "node_modules/parseurl": { @@ -4513,12 +4755,12 @@ "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" }, "node_modules/path-type": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/path-type/-/path-type-2.0.0.tgz", - "integrity": "sha1-8BLMuEFbcJb8LaoQVMPXI4lZTHM=", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-3.0.0.tgz", + "integrity": "sha512-T2ZUsdZFHgA3u4e5PfPbjd7HDDpxPnQb5jN0SrDsjNSuVXHJqtwTnWqG0B1jZrgmJ/7lj1EmVIByWt1gxGkWvg==", "dev": true, "dependencies": { - "pify": "^2.0.0" + "pify": "^3.0.0" }, "engines": { "node": ">=4" @@ -4556,12 +4798,12 @@ } }, "node_modules/pify": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", - "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", + "integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=", "dev": true, "engines": { - "node": ">=0.10.0" + "node": ">=4" } }, "node_modules/pkg-conf": { @@ -4642,19 +4884,6 @@ "node": ">=6" } }, - "node_modules/pkg-conf/node_modules/parse-json": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-4.0.0.tgz", - "integrity": "sha1-vjX1Qlvh9/bHRxhPmKeIy5lHfuA=", - "dev": true, - "dependencies": { - "error-ex": "^1.3.1", - "json-parse-better-errors": "^1.0.1" - }, - "engines": { - "node": ">=4" - } - }, "node_modules/pkg-conf/node_modules/path-exists": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", @@ -4752,6 +4981,85 @@ "node": ">=8" } }, + "node_modules/pkg-up": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/pkg-up/-/pkg-up-2.0.0.tgz", + "integrity": "sha1-yBmscoBZpGHKscOImivjxJoATX8=", + "dev": true, + "dependencies": { + "find-up": "^2.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/pkg-up/node_modules/find-up": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-2.1.0.tgz", + "integrity": "sha1-RdG35QbHF93UgndaK3eSCjwMV6c=", + "dev": true, + "dependencies": { + "locate-path": "^2.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/pkg-up/node_modules/locate-path": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-2.0.0.tgz", + "integrity": "sha1-K1aLJl7slExtnA3pw9u7ygNUzY4=", + "dev": true, + "dependencies": { + "p-locate": "^2.0.0", + "path-exists": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/pkg-up/node_modules/p-limit": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-1.3.0.tgz", + "integrity": "sha512-vvcXsLAJ9Dr5rQOPk7toZQZJApBl2K4J6dANSsEuh6QI41JYcsS/qhTGa9ErIUUgK3WNQoJYvylxvjqmiqEA9Q==", + "dev": true, + "dependencies": { + "p-try": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/pkg-up/node_modules/p-locate": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-2.0.0.tgz", + "integrity": "sha1-IKAQOyIqcMj9OcwuWAaA893l7EM=", + "dev": true, + "dependencies": { + "p-limit": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/pkg-up/node_modules/p-try": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-1.0.0.tgz", + "integrity": "sha1-y8ec26+P1CKOE/Yh8rGiN8GyB7M=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/pkg-up/node_modules/path-exists": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", + "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=", + "dev": true, + "engines": { + "node": ">=4" + } + }, "node_modules/prebuild-install": { "version": "6.1.4", "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-6.1.4.tgz", @@ -4814,14 +5122,14 @@ } }, "node_modules/prop-types": { - "version": "15.7.2", - "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.7.2.tgz", - "integrity": "sha512-8QQikdH7//R2vurIJSutZ1smHYTcLpRWEOlHnzcWHmBYrOGUysKwSsrC89BCiFj3CbrfJ/nXFdJepOVrY1GCHQ==", + "version": "15.8.1", + "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz", + "integrity": "sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==", "dev": true, "dependencies": { "loose-envify": "^1.4.0", "object-assign": "^4.1.1", - "react-is": "^16.8.1" + "react-is": "^16.13.1" } }, "node_modules/proxy-addr": { @@ -4919,27 +5227,27 @@ "dev": true }, "node_modules/read-pkg": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-2.0.0.tgz", - "integrity": "sha1-jvHAYjxqbbDcZxPEv6xGMysjaPg=", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-3.0.0.tgz", + "integrity": "sha1-nLxoaXj+5l0WwA4rGcI3/Pbjg4k=", "dev": true, "dependencies": { - "load-json-file": "^2.0.0", + "load-json-file": "^4.0.0", "normalize-package-data": "^2.3.2", - "path-type": "^2.0.0" + "path-type": "^3.0.0" }, "engines": { "node": ">=4" } }, "node_modules/read-pkg-up": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/read-pkg-up/-/read-pkg-up-2.0.0.tgz", - "integrity": "sha1-a3KoBImE4MQeeVEP1en6mbO1Sb4=", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/read-pkg-up/-/read-pkg-up-3.0.0.tgz", + "integrity": "sha1-PtSWaF26D4/hGNBpHcUfSh/5bwc=", "dev": true, "dependencies": { "find-up": "^2.0.0", - "read-pkg": "^2.0.0" + "read-pkg": "^3.0.0" }, "engines": { "node": ">=4" @@ -5055,9 +5363,9 @@ "integrity": "sha512-KJbQG1iZOC4IRBccT1mOodzKp5YM9gC1H4L6LgDmmXx+4U3pabhb+UF0UI0u+Lck1cvb4oyAuihaQqTbKnslqQ==" }, "node_modules/regexp.prototype.flags": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.3.1.tgz", - "integrity": "sha512-JiBdRBq91WlY7uRJ0ds7R+dU02i6LKi8r3BuQhNXn+kmeLN+EfHhfjqMRis1zJxnlu88hq/4dx0P2OP3APRTOA==", + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.4.1.tgz", + "integrity": "sha512-pMR7hBVUUGI7PMA37m2ofIdQCsomVnas+Jn5UPGAHQ+/LlwKm/aTLJHdasmHRzlfeZwHiAOaRSo2rbBDm3nNUQ==", "dev": true, "dependencies": { "call-bind": "^1.0.2", @@ -5065,6 +5373,9 @@ }, "engines": { "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, "node_modules/regexpp": { @@ -5097,6 +5408,15 @@ "node": ">=0.10.0" } }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/require-main-filename": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz", @@ -5274,6 +5594,9 @@ "call-bind": "^1.0.0", "get-intrinsic": "^1.0.2", "object-inspect": "^1.9.0" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, "node_modules/signal-exit": { @@ -5297,53 +5620,29 @@ } }, "node_modules/slice-ansi": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-2.1.0.tgz", - "integrity": "sha512-Qu+VC3EwYLldKa1fCxuuvULvSJOKEgk9pi8dZeCVK7TqBfUNTH4sFkk4joj8afVSfAYgJoSOetjx9QWOJ5mYoQ==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-4.0.0.tgz", + "integrity": "sha512-qMCMfhY040cVHT43K9BFygqYbUPFZKHOg7K73mtTWJRb8pyP3fzf4Ixd5SzdEJQ6MRUg/WBnOLxghZtKKurENQ==", "dev": true, "dependencies": { - "ansi-styles": "^3.2.0", - "astral-regex": "^1.0.0", - "is-fullwidth-code-point": "^2.0.0" + "ansi-styles": "^4.0.0", + "astral-regex": "^2.0.0", + "is-fullwidth-code-point": "^3.0.0" }, "engines": { - "node": ">=6" - } - }, - "node_modules/slice-ansi/node_modules/ansi-styles": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", - "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", - "dev": true, - "dependencies": { - "color-convert": "^1.9.0" + "node": ">=10" }, - "engines": { - "node": ">=4" - } - }, - "node_modules/slice-ansi/node_modules/color-convert": { - "version": "1.9.3", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", - "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", - "dev": true, - "dependencies": { - "color-name": "1.1.3" + "funding": { + "url": "https://github.com/chalk/slice-ansi?sponsor=1" } }, - "node_modules/slice-ansi/node_modules/color-name": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=", - "dev": true - }, "node_modules/slice-ansi/node_modules/is-fullwidth-code-point": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", - "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", "dev": true, "engines": { - "node": ">=4" + "node": ">=8" } }, "node_modules/source-map": { @@ -5399,9 +5698,9 @@ } }, "node_modules/spdx-license-ids": { - "version": "3.0.10", - "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.10.tgz", - "integrity": "sha512-oie3/+gKf7QtpitB0LYLETe+k8SifzsX4KixvpOsbI6S0kRiRQ5MKOio8eMSAKQ17N06+wdEOXRiId+zOxo0hA==", + "version": "3.0.11", + "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.11.tgz", + "integrity": "sha512-Ctl2BrFiM0X3MANYgj3CkygxhRmr9mi6xhejbdO960nF6EDJApTYpn0BQnDKlnNBULKiCN1n3w9EBkHK8ZWg+g==", "dev": true }, "node_modules/split": { @@ -5422,25 +5721,39 @@ "dev": true }, "node_modules/standard": { - "version": "16.0.3", - "resolved": "https://registry.npmjs.org/standard/-/standard-16.0.3.tgz", - "integrity": "sha512-70F7NH0hSkNXosXRltjSv6KpTAOkUkSfyu3ynyM5dtRUiLtR+yX9EGZ7RKwuGUqCJiX/cnkceVM6HTZ4JpaqDg==", - "dev": true, + "version": "16.0.4", + "resolved": "https://registry.npmjs.org/standard/-/standard-16.0.4.tgz", + "integrity": "sha512-2AGI874RNClW4xUdM+bg1LRXVlYLzTNEkHmTG5mhyn45OhbgwA+6znowkOGYy+WMb5HRyELvtNy39kcdMQMcYQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], "dependencies": { - "eslint": "~7.13.0", - "eslint-config-standard": "16.0.2", + "eslint": "~7.18.0", + "eslint-config-standard": "16.0.3", "eslint-config-standard-jsx": "10.0.0", - "eslint-plugin-import": "~2.22.1", + "eslint-plugin-import": "~2.24.2", "eslint-plugin-node": "~11.1.0", - "eslint-plugin-promise": "~4.2.1", - "eslint-plugin-react": "~7.21.5", + "eslint-plugin-promise": "~5.1.0", + "eslint-plugin-react": "~7.25.1", "standard-engine": "^14.0.1" }, "bin": { "standard": "bin/cmd.js" }, "engines": { - "node": "^10.12.0 || >=12.0.0" + "node": ">=10.12.0" } }, "node_modules/standard-engine": { @@ -5497,19 +5810,22 @@ } }, "node_modules/string.prototype.matchall": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/string.prototype.matchall/-/string.prototype.matchall-4.0.5.tgz", - "integrity": "sha512-Z5ZaXO0svs0M2xd/6By3qpeKpLKd9mO4v4q3oMEQrk8Ck4xOD5d5XeBOOjGrmVZZ/AHB1S0CgG4N5r1G9N3E2Q==", + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/string.prototype.matchall/-/string.prototype.matchall-4.0.6.tgz", + "integrity": "sha512-6WgDX8HmQqvEd7J+G6VtAahhsQIssiZ8zl7zKh1VDMFyL3hRTJP4FTNA3RbIp2TOQ9AYNDcc7e3fH0Qbup+DBg==", "dev": true, "dependencies": { "call-bind": "^1.0.2", "define-properties": "^1.1.3", - "es-abstract": "^1.18.2", + "es-abstract": "^1.19.1", "get-intrinsic": "^1.1.1", "has-symbols": "^1.0.2", "internal-slot": "^1.0.3", "regexp.prototype.flags": "^1.3.1", "side-channel": "^1.0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, "node_modules/string.prototype.trimend": { @@ -5520,6 +5836,9 @@ "dependencies": { "call-bind": "^1.0.2", "define-properties": "^1.1.3" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, "node_modules/string.prototype.trimstart": { @@ -5530,6 +5849,9 @@ "dependencies": { "call-bind": "^1.0.2", "define-properties": "^1.1.3" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, "node_modules/strip-ansi": { @@ -5573,68 +5895,85 @@ } }, "node_modules/table": { - "version": "5.4.6", - "resolved": "https://registry.npmjs.org/table/-/table-5.4.6.tgz", - "integrity": "sha512-wmEc8m4fjnob4gt5riFRtTu/6+4rSe12TpAELNSqHMfF3IqnA+CH37USM6/YR3qRZv7e56kAEAtd6nKZaxe0Ug==", + "version": "6.8.0", + "resolved": "https://registry.npmjs.org/table/-/table-6.8.0.tgz", + "integrity": "sha512-s/fitrbVeEyHKFa7mFdkuQMWlH1Wgw/yEXMt5xACT4ZpzWFluehAxRtUUQKPuWhaLAWhFcVx6w3oC8VKaUfPGA==", "dev": true, "dependencies": { - "ajv": "^6.10.2", - "lodash": "^4.17.14", - "slice-ansi": "^2.1.0", - "string-width": "^3.0.0" + "ajv": "^8.0.1", + "lodash.truncate": "^4.4.2", + "slice-ansi": "^4.0.0", + "string-width": "^4.2.3", + "strip-ansi": "^6.0.1" }, "engines": { - "node": ">=6.0.0" + "node": ">=10.0.0" + } + }, + "node_modules/table/node_modules/ajv": { + "version": "8.9.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.9.0.tgz", + "integrity": "sha512-qOKJyNj/h+OWx7s5DePL6Zu1KeM9jPZhwBqs+7DzP6bGOvqzVCSf0xueYmVuaC/oQ/VtS2zLMLHdQFbkka+XDQ==", + "dev": true, + "dependencies": { + "fast-deep-equal": "^3.1.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" } }, "node_modules/table/node_modules/ansi-regex": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", - "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", "dev": true, "engines": { - "node": ">=6" + "node": ">=8" } }, - "node_modules/table/node_modules/emoji-regex": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz", - "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==", - "dev": true - }, "node_modules/table/node_modules/is-fullwidth-code-point": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", - "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", "dev": true, "engines": { - "node": ">=4" + "node": ">=8" } }, + "node_modules/table/node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "dev": true + }, "node_modules/table/node_modules/string-width": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", - "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", "dev": true, "dependencies": { - "emoji-regex": "^7.0.1", - "is-fullwidth-code-point": "^2.0.0", - "strip-ansi": "^5.1.0" + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" }, "engines": { - "node": ">=6" + "node": ">=8" } }, "node_modules/table/node_modules/strip-ansi": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", - "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", "dev": true, "dependencies": { - "ansi-regex": "^4.1.0" + "ansi-regex": "^5.0.1" }, "engines": { - "node": ">=6" + "node": ">=8" } }, "node_modules/tar": { @@ -5771,9 +6110,9 @@ } }, "node_modules/tsconfig-paths": { - "version": "3.11.0", - "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.11.0.tgz", - "integrity": "sha512-7ecdYDnIdmv639mmDwslG6KQg1Z9STTz1j7Gcz0xa+nshh/gKDAHcPxRbWOsA3SPp0tXP2leTcY9Kw+NAkfZzA==", + "version": "3.12.0", + "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.12.0.tgz", + "integrity": "sha512-e5adrnOYT6zqVnWqZu7i/BQ3BnhzvGbjEjejFXO20lKIKpwTaupkCPgEfv4GZK1IBciJUEhYs3J3p75FdaTFVg==", "dev": true, "dependencies": { "@types/json5": "^0.0.29", @@ -5875,6 +6214,9 @@ "has-bigints": "^1.0.1", "has-symbols": "^1.0.2", "which-boxed-primitive": "^1.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, "node_modules/unorm": { @@ -5983,6 +6325,9 @@ "is-number-object": "^1.0.4", "is-string": "^1.0.5", "is-symbol": "^1.0.3" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, "node_modules/which-module": { @@ -6029,9 +6374,9 @@ } }, "node_modules/wrap-ansi/node_modules/ansi-regex": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", - "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", "dev": true, "engines": { "node": ">=8" @@ -6077,18 +6422,6 @@ "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" }, - "node_modules/write": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/write/-/write-1.0.3.tgz", - "integrity": "sha512-/lg70HAjtkUgWPVZhZcm+T4hkL8Zbtp1nFNOn3lRrxnlv50SRBv7cR7RqR+GMsd3hUXy9hWBo4CHTbFTcOYwig==", - "dev": true, - "dependencies": { - "mkdirp": "^0.5.1" - }, - "engines": { - "node": ">=4" - } - }, "node_modules/write-file-atomic": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-3.0.3.tgz", @@ -6101,18 +6434,6 @@ "typedarray-to-buffer": "^3.1.5" } }, - "node_modules/write/node_modules/mkdirp": { - "version": "0.5.5", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz", - "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", - "dev": true, - "dependencies": { - "minimist": "^1.2.5" - }, - "bin": { - "mkdirp": "bin/cmd.js" - } - }, "node_modules/xdg-basedir": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/xdg-basedir/-/xdg-basedir-4.0.0.tgz", @@ -6179,9 +6500,9 @@ } }, "node_modules/yargs/node_modules/ansi-regex": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", - "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", "dev": true, "engines": { "node": ">=8" @@ -6595,9 +6916,9 @@ } }, "@eslint/eslintrc": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-0.2.2.tgz", - "integrity": "sha512-EfB5OHNYp1F4px/LI/FEnGylop7nOqkQ1LRzCM0KccA2U8tvV8w01KBv37LbO7nW4H+YhKyo2LcJhRwjjV17QQ==", + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-0.3.0.tgz", + "integrity": "sha512-1JTKgrOKAHVivSvOYw+sJOunkBjUOvjqWk1DPja7ZFhIS2mX/4EgTT8M7eTK9jrKhL/FvXXEbQwIs3pg1xp3dg==", "dev": true, "requires": { "ajv": "^6.12.4", @@ -6607,7 +6928,7 @@ "ignore": "^4.0.6", "import-fresh": "^3.2.1", "js-yaml": "^3.13.1", - "lodash": "^4.17.19", + "lodash": "^4.17.20", "minimatch": "^3.0.4", "strip-json-comments": "^3.1.1" }, @@ -6622,9 +6943,9 @@ } }, "debug": { - "version": "4.3.2", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz", - "integrity": "sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw==", + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.3.tgz", + "integrity": "sha512-/zxw5+vh1Tfv+4Qn7a5nsbcJKPaSvCDhojn6FEl9vupwK2VCSDtEiEtqr8DFtzYFOdz63LBkxec7DYuc2jon6Q==", "dev": true, "requires": { "ms": "2.1.2" @@ -6785,7 +7106,8 @@ "version": "5.3.2", "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", - "dev": true + "dev": true, + "requires": {} }, "aes-js": { "version": "3.1.2", @@ -6885,39 +7207,38 @@ "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=" }, "array-includes": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/array-includes/-/array-includes-3.1.3.tgz", - "integrity": "sha512-gcem1KlBU7c9rB+Rq8/3PPKsK2kjqeEBa3bD5kkQo4nYlOHQCJqIJFqBXDEfwaRuYTT4E+FxA9xez7Gf/e3Q7A==", + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/array-includes/-/array-includes-3.1.4.tgz", + "integrity": "sha512-ZTNSQkmWumEbiHO2GF4GmWxYVTiQyJy2XOTa15sdQSrvKn7l+180egQMqlrMOUMCyLMD7pmyQe4mMDUT6Behrw==", "dev": true, "requires": { "call-bind": "^1.0.2", "define-properties": "^1.1.3", - "es-abstract": "^1.18.0-next.2", + "es-abstract": "^1.19.1", "get-intrinsic": "^1.1.1", - "is-string": "^1.0.5" + "is-string": "^1.0.7" } }, "array.prototype.flat": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/array.prototype.flat/-/array.prototype.flat-1.2.4.tgz", - "integrity": "sha512-4470Xi3GAPAjZqFcljX2xzckv1qeKPizoNkiS0+O4IoPR2ZNpcjE0pkhdihlDouK+x6QOast26B4Q/O9DJnwSg==", + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/array.prototype.flat/-/array.prototype.flat-1.2.5.tgz", + "integrity": "sha512-KaYU+S+ndVqyUnignHftkwc58o3uVU1jzczILJ1tN2YaIZpFIKBiP/x/j97E5MVPsaCloPbqWLB/8qCTVvT2qg==", "dev": true, "requires": { - "call-bind": "^1.0.0", + "call-bind": "^1.0.2", "define-properties": "^1.1.3", - "es-abstract": "^1.18.0-next.1" + "es-abstract": "^1.19.0" } }, "array.prototype.flatmap": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/array.prototype.flatmap/-/array.prototype.flatmap-1.2.4.tgz", - "integrity": "sha512-r9Z0zYoxqHz60vvQbWEdXIEtCwHF0yxaWfno9qzXeNHvfyl3BZqygmGzb84dsubyaXLH4husF+NFgMSdpZhk2Q==", + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/array.prototype.flatmap/-/array.prototype.flatmap-1.2.5.tgz", + "integrity": "sha512-08u6rVyi1Lj7oqWbS9nUxliETrtIROT4XGTA4D/LWGten6E3ocm7cy9SIrmNHOL5XVbVuckUp3X6Xyg8/zpvHA==", "dev": true, "requires": { "call-bind": "^1.0.0", "define-properties": "^1.1.3", - "es-abstract": "^1.18.0-next.1", - "function-bind": "^1.1.1" + "es-abstract": "^1.19.0" } }, "assertion-error": { @@ -6927,17 +7248,17 @@ "dev": true }, "astral-regex": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/astral-regex/-/astral-regex-1.0.0.tgz", - "integrity": "sha512-+Ryf6g3BKoRc7jfp7ad8tM4TtMiaWvbF/1/sQcZPkkS7ag3D5nMBCe2UfOTONtAkaG0tO0ij3C5Lwmf1EiyjHg==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/astral-regex/-/astral-regex-2.0.0.tgz", + "integrity": "sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ==", "dev": true }, "axios": { - "version": "0.21.1", - "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.1.tgz", - "integrity": "sha512-dKQiRHxGD9PPRIUNIWvZhPTPpl1rf/OxTYKsqKUDjBwYylTvV7SjSHJb9ratfyzM6wCdLCOYLzs73qpg5c4iGA==", + "version": "0.21.4", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.4.tgz", + "integrity": "sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==", "requires": { - "follow-redirects": "^1.10.0" + "follow-redirects": "^1.14.0" } }, "balanced-match": { @@ -7251,9 +7572,9 @@ }, "dependencies": { "ansi-regex": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", - "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", "dev": true }, "is-fullwidth-code-point": { @@ -7342,12 +7663,6 @@ "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz", "integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4=" }, - "contains-path": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/contains-path/-/contains-path-0.1.0.tgz", - "integrity": "sha1-/ozxhP9mcLa67wGp1IYaXL7EEgo=", - "dev": true - }, "content-disposition": { "version": "0.5.3", "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.3.tgz", @@ -7442,9 +7757,9 @@ "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==" }, "deep-is": { - "version": "0.1.3", - "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz", - "integrity": "sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ=", + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", + "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", "dev": true }, "default-require-extensions": { @@ -7573,22 +7888,25 @@ } }, "es-abstract": { - "version": "1.18.5", - "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.18.5.tgz", - "integrity": "sha512-DDggyJLoS91CkJjgauM5c0yZMjiD1uK3KcaCeAmffGwZ+ODWzOkPN4QwRbsK5DOFf06fywmyLci3ZD8jLGhVYA==", + "version": "1.19.1", + "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.19.1.tgz", + "integrity": "sha512-2vJ6tjA/UfqLm2MPs7jxVybLoB8i1t1Jd9R3kISld20sIxPcTbLuggQOUxeWeAvIUkduv/CfMjuh4WmiXr2v9w==", "dev": true, "requires": { "call-bind": "^1.0.2", "es-to-primitive": "^1.2.1", "function-bind": "^1.1.1", "get-intrinsic": "^1.1.1", + "get-symbol-description": "^1.0.0", "has": "^1.0.3", "has-symbols": "^1.0.2", "internal-slot": "^1.0.3", - "is-callable": "^1.2.3", + "is-callable": "^1.2.4", "is-negative-zero": "^2.0.1", - "is-regex": "^1.1.3", - "is-string": "^1.0.6", + "is-regex": "^1.1.4", + "is-shared-array-buffer": "^1.0.1", + "is-string": "^1.0.7", + "is-weakref": "^1.0.1", "object-inspect": "^1.11.0", "object-keys": "^1.1.1", "object.assign": "^4.1.2", @@ -7631,13 +7949,13 @@ "dev": true }, "eslint": { - "version": "7.13.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-7.13.0.tgz", - "integrity": "sha512-uCORMuOO8tUzJmsdRtrvcGq5qposf7Rw0LwkTJkoDbOycVQtQjmnhZSuLQnozLE4TmAzlMVV45eCHmQ1OpDKUQ==", + "version": "7.18.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-7.18.0.tgz", + "integrity": "sha512-fbgTiE8BfUJZuBeq2Yi7J3RB3WGUQ9PNuNbmgi6jt9Iv8qrkxfy19Ds3OpL1Pm7zg3BtTVhvcUZbIRQ0wmSjAQ==", "dev": true, "requires": { "@babel/code-frame": "^7.0.0", - "@eslint/eslintrc": "^0.2.1", + "@eslint/eslintrc": "^0.3.0", "ajv": "^6.10.0", "chalk": "^4.0.0", "cross-spawn": "^7.0.2", @@ -7647,10 +7965,10 @@ "eslint-scope": "^5.1.1", "eslint-utils": "^2.1.0", "eslint-visitor-keys": "^2.0.0", - "espree": "^7.3.0", + "espree": "^7.3.1", "esquery": "^1.2.0", "esutils": "^2.0.2", - "file-entry-cache": "^5.0.1", + "file-entry-cache": "^6.0.0", "functional-red-black-tree": "^1.0.1", "glob-parent": "^5.0.0", "globals": "^12.1.0", @@ -7661,7 +7979,7 @@ "js-yaml": "^3.13.1", "json-stable-stringify-without-jsonify": "^1.0.1", "levn": "^0.4.1", - "lodash": "^4.17.19", + "lodash": "^4.17.20", "minimatch": "^3.0.4", "natural-compare": "^1.4.0", "optionator": "^0.9.1", @@ -7670,15 +7988,15 @@ "semver": "^7.2.1", "strip-ansi": "^6.0.0", "strip-json-comments": "^3.1.0", - "table": "^5.2.3", + "table": "^6.0.4", "text-table": "^0.2.0", "v8-compile-cache": "^2.0.3" }, "dependencies": { "ansi-regex": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", - "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", "dev": true }, "argparse": { @@ -7691,9 +8009,9 @@ } }, "debug": { - "version": "4.3.2", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz", - "integrity": "sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw==", + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.3.tgz", + "integrity": "sha512-/zxw5+vh1Tfv+4Qn7a5nsbcJKPaSvCDhojn6FEl9vupwK2VCSDtEiEtqr8DFtzYFOdz63LBkxec7DYuc2jon6Q==", "dev": true, "requires": { "ms": "2.1.2" @@ -7734,12 +8052,12 @@ } }, "strip-ansi": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", - "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", "dev": true, "requires": { - "ansi-regex": "^5.0.0" + "ansi-regex": "^5.0.1" } }, "strip-json-comments": { @@ -7751,10 +8069,11 @@ } }, "eslint-config-standard": { - "version": "16.0.2", - "resolved": "https://registry.npmjs.org/eslint-config-standard/-/eslint-config-standard-16.0.2.tgz", - "integrity": "sha512-fx3f1rJDsl9bY7qzyX8SAtP8GBSk6MfXFaTfaGgk12aAYW4gJSyRm7dM790L6cbXv63fvjY4XeSzXnb4WM+SKw==", - "dev": true + "version": "16.0.3", + "resolved": "https://registry.npmjs.org/eslint-config-standard/-/eslint-config-standard-16.0.3.tgz", + "integrity": "sha512-x4fmJL5hGqNJKGHSjnLdgA6U6h1YW/G2dW9fA+cyVur4SK6lyue8+UgNKWlZtUDTXvgKDD/Oa3GQjmB5kjtVvg==", + "dev": true, + "requires": {} }, "eslint-config-standard-jsx": { "version": "10.0.0", @@ -7790,13 +8109,13 @@ } }, "eslint-module-utils": { - "version": "2.6.2", - "resolved": "https://registry.npmjs.org/eslint-module-utils/-/eslint-module-utils-2.6.2.tgz", - "integrity": "sha512-QG8pcgThYOuqxupd06oYTZoNOGaUdTY1PqK+oS6ElF6vs4pBdk/aYxFVQQXzcrAqp9m7cl7lb2ubazX+g16k2Q==", + "version": "2.7.2", + "resolved": "https://registry.npmjs.org/eslint-module-utils/-/eslint-module-utils-2.7.2.tgz", + "integrity": "sha512-zquepFnWCY2ISMFwD/DqzaM++H+7PDzOpUvotJWm/y1BAFt5R4oeULgdrTejKqLkz7MA/tgstsUMNYc7wNdTrg==", "dev": true, "requires": { "debug": "^3.2.7", - "pkg-dir": "^2.0.0" + "find-up": "^2.1.0" }, "dependencies": { "debug": { @@ -7862,15 +8181,6 @@ "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=", "dev": true - }, - "pkg-dir": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-2.0.0.tgz", - "integrity": "sha1-9tXREJ4Z1j7fQo4L1X4Sd3YVM0s=", - "dev": true, - "requires": { - "find-up": "^2.1.0" - } } } }, @@ -7885,35 +8195,85 @@ } }, "eslint-plugin-import": { - "version": "2.22.1", - "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.22.1.tgz", - "integrity": "sha512-8K7JjINHOpH64ozkAhpT3sd+FswIZTfMZTjdx052pnWrgRCVfp8op9tbjpAk3DdUeI/Ba4C8OjdC0r90erHEOw==", + "version": "2.24.2", + "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.24.2.tgz", + "integrity": "sha512-hNVtyhiEtZmpsabL4neEj+6M5DCLgpYyG9nzJY8lZQeQXEn5UPW1DpUdsMHMXsq98dbNm7nt1w9ZMSVpfJdi8Q==", "dev": true, "requires": { - "array-includes": "^3.1.1", - "array.prototype.flat": "^1.2.3", - "contains-path": "^0.1.0", + "array-includes": "^3.1.3", + "array.prototype.flat": "^1.2.4", "debug": "^2.6.9", - "doctrine": "1.5.0", - "eslint-import-resolver-node": "^0.3.4", - "eslint-module-utils": "^2.6.0", + "doctrine": "^2.1.0", + "eslint-import-resolver-node": "^0.3.6", + "eslint-module-utils": "^2.6.2", + "find-up": "^2.0.0", "has": "^1.0.3", + "is-core-module": "^2.6.0", "minimatch": "^3.0.4", - "object.values": "^1.1.1", - "read-pkg-up": "^2.0.0", - "resolve": "^1.17.0", - "tsconfig-paths": "^3.9.0" + "object.values": "^1.1.4", + "pkg-up": "^2.0.0", + "read-pkg-up": "^3.0.0", + "resolve": "^1.20.0", + "tsconfig-paths": "^3.11.0" }, "dependencies": { "doctrine": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-1.5.0.tgz", - "integrity": "sha1-N53Ocw9hZvds76TmcHoVmwLFpvo=", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-2.1.0.tgz", + "integrity": "sha512-35mSku4ZXK0vfCuHEDAwt55dg2jNajHZ1odvF+8SSr82EsZY4QmXfuWso8oEd8zRhVObSN18aM0CjSdoBX7zIw==", + "dev": true, + "requires": { + "esutils": "^2.0.2" + } + }, + "find-up": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-2.1.0.tgz", + "integrity": "sha1-RdG35QbHF93UgndaK3eSCjwMV6c=", + "dev": true, + "requires": { + "locate-path": "^2.0.0" + } + }, + "locate-path": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-2.0.0.tgz", + "integrity": "sha1-K1aLJl7slExtnA3pw9u7ygNUzY4=", + "dev": true, + "requires": { + "p-locate": "^2.0.0", + "path-exists": "^3.0.0" + } + }, + "p-limit": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-1.3.0.tgz", + "integrity": "sha512-vvcXsLAJ9Dr5rQOPk7toZQZJApBl2K4J6dANSsEuh6QI41JYcsS/qhTGa9ErIUUgK3WNQoJYvylxvjqmiqEA9Q==", + "dev": true, + "requires": { + "p-try": "^1.0.0" + } + }, + "p-locate": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-2.0.0.tgz", + "integrity": "sha1-IKAQOyIqcMj9OcwuWAaA893l7EM=", "dev": true, "requires": { - "esutils": "^2.0.2", - "isarray": "^1.0.0" + "p-limit": "^1.1.0" } + }, + "p-try": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-1.0.0.tgz", + "integrity": "sha1-y8ec26+P1CKOE/Yh8rGiN8GyB7M=", + "dev": true + }, + "path-exists": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", + "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=", + "dev": true } } }, @@ -7946,28 +8306,31 @@ } }, "eslint-plugin-promise": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/eslint-plugin-promise/-/eslint-plugin-promise-4.2.1.tgz", - "integrity": "sha512-VoM09vT7bfA7D+upt+FjeBO5eHIJQBUWki1aPvB+vbNiHS3+oGIJGIeyBtKQTME6UPXXy3vV07OL1tHd3ANuDw==", - "dev": true + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/eslint-plugin-promise/-/eslint-plugin-promise-5.1.1.tgz", + "integrity": "sha512-XgdcdyNzHfmlQyweOPTxmc7pIsS6dE4MvwhXWMQ2Dxs1XAL2GJDilUsjWen6TWik0aSI+zD/PqocZBblcm9rdA==", + "dev": true, + "requires": {} }, "eslint-plugin-react": { - "version": "7.21.5", - "resolved": "https://registry.npmjs.org/eslint-plugin-react/-/eslint-plugin-react-7.21.5.tgz", - "integrity": "sha512-8MaEggC2et0wSF6bUeywF7qQ46ER81irOdWS4QWxnnlAEsnzeBevk1sWh7fhpCghPpXb+8Ks7hvaft6L/xsR6g==", + "version": "7.25.3", + "resolved": "https://registry.npmjs.org/eslint-plugin-react/-/eslint-plugin-react-7.25.3.tgz", + "integrity": "sha512-ZMbFvZ1WAYSZKY662MBVEWR45VaBT6KSJCiupjrNlcdakB90juaZeDCbJq19e73JZQubqFtgETohwgAt8u5P6w==", "dev": true, "requires": { - "array-includes": "^3.1.1", - "array.prototype.flatmap": "^1.2.3", + "array-includes": "^3.1.3", + "array.prototype.flatmap": "^1.2.4", "doctrine": "^2.1.0", - "has": "^1.0.3", + "estraverse": "^5.2.0", "jsx-ast-utils": "^2.4.1 || ^3.0.0", - "object.entries": "^1.1.2", - "object.fromentries": "^2.0.2", - "object.values": "^1.1.1", + "minimatch": "^3.0.4", + "object.entries": "^1.1.4", + "object.fromentries": "^2.0.4", + "object.hasown": "^1.0.0", + "object.values": "^1.1.4", "prop-types": "^15.7.2", - "resolve": "^1.18.1", - "string.prototype.matchall": "^4.0.2" + "resolve": "^2.0.0-next.3", + "string.prototype.matchall": "^4.0.5" }, "dependencies": { "doctrine": { @@ -7978,6 +8341,16 @@ "requires": { "esutils": "^2.0.2" } + }, + "resolve": { + "version": "2.0.0-next.3", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-2.0.0-next.3.tgz", + "integrity": "sha512-W8LucSynKUIDu9ylraa7ueVZ7hc0uAgJBxVsQSKOXOyle8a93qXhcz+XAXZ8bIq2d6i4Ehddn6Evt+0/UwKk6Q==", + "dev": true, + "requires": { + "is-core-module": "^2.2.0", + "path-parse": "^1.0.6" + } } } }, @@ -7989,6 +8362,14 @@ "requires": { "esrecurse": "^4.3.0", "estraverse": "^4.1.1" + }, + "dependencies": { + "estraverse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.3.0.tgz", + "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==", + "dev": true + } } }, "eslint-utils": { @@ -8051,14 +8432,6 @@ "dev": true, "requires": { "estraverse": "^5.1.0" - }, - "dependencies": { - "estraverse": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.2.0.tgz", - "integrity": "sha512-BxbNGGNm0RyRYvUdHpIwv9IWzeM9XClbOxwoATuFdOE7ZE6wHL+HQ5T8hoPM+zHvmKzzsEqhgy0GrQ5X13afiQ==", - "dev": true - } } }, "esrecurse": { @@ -8068,20 +8441,12 @@ "dev": true, "requires": { "estraverse": "^5.2.0" - }, - "dependencies": { - "estraverse": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.2.0.tgz", - "integrity": "sha512-BxbNGGNm0RyRYvUdHpIwv9IWzeM9XClbOxwoATuFdOE7ZE6wHL+HQ5T8hoPM+zHvmKzzsEqhgy0GrQ5X13afiQ==", - "dev": true - } } }, "estraverse": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.3.0.tgz", - "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==", + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", "dev": true }, "esutils": { @@ -8196,12 +8561,12 @@ } }, "file-entry-cache": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-5.0.1.tgz", - "integrity": "sha512-bCg29ictuBaKUwwArK4ouCaqDgLZcysCFLmM/Yn/FDoqndh/9vNuQfXRDvTuXKLxfD/JtZQGKFT8MGcJBK644g==", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz", + "integrity": "sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==", "dev": true, "requires": { - "flat-cache": "^2.0.1" + "flat-cache": "^3.0.4" } }, "file-uri-to-path": { @@ -8260,37 +8625,25 @@ "dev": true }, "flat-cache": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-2.0.1.tgz", - "integrity": "sha512-LoQe6yDuUMDzQAEH8sgmh4Md6oZnc/7PjtwjNFSzveXqSHt6ka9fPBuso7IGf9Rz4uqnSnWiFH2B/zj24a5ReA==", + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-3.0.4.tgz", + "integrity": "sha512-dm9s5Pw7Jc0GvMYbshN6zchCA9RgQlzzEZX3vylR9IqFfS8XciblUXOKfW6SiuJ0e13eDYZoZV5wdrev7P3Nwg==", "dev": true, "requires": { - "flatted": "^2.0.0", - "rimraf": "2.6.3", - "write": "1.0.3" - }, - "dependencies": { - "rimraf": { - "version": "2.6.3", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz", - "integrity": "sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==", - "dev": true, - "requires": { - "glob": "^7.1.3" - } - } + "flatted": "^3.1.0", + "rimraf": "^3.0.2" } }, "flatted": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/flatted/-/flatted-2.0.2.tgz", - "integrity": "sha512-r5wGx7YeOwNWNlCA0wQ86zKyDLMQr+/RB8xy74M4hTphfmjlijTSSXGuH8rnvKZnfT9i+75zmd8jcKdMR4O6jA==", + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.2.4.tgz", + "integrity": "sha512-8/sOawo8tJ4QOBX8YlQBMxL8+RLZfxMQOif9o0KUKTNTjMYElWPE0r/m5VNFxTRd0NSw8qSy8dajrwX4RYI1Hw==", "dev": true }, "follow-redirects": { - "version": "1.14.2", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.2.tgz", - "integrity": "sha512-yLR6WaE2lbF0x4K2qE2p9PEXKLDjUjnR/xmjS3wHAYxtlsI9MLLBJUZirAHKzUZDGLxje7w/cXR49WOUo4rbsA==" + "version": "1.14.7", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.7.tgz", + "integrity": "sha512-+hbxoLbFMbRKDwohX8GkTataGqO6Jb7jGwpAlwgy2bIz25XtRm7KEzJM76R1WiNT5SwZkX4Y75SwBolkpmE7iQ==" }, "foreground-child": { "version": "2.0.0", @@ -8421,6 +8774,16 @@ "integrity": "sha512-sY22aA6xchAzprjyqmSEQv4UbAAzRN0L2dQB0NlN5acTTK9Don6nhoc3eAbUnpZiCANAMfd/+40kVdKfFygohg==", "dev": true }, + "get-symbol-description": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/get-symbol-description/-/get-symbol-description-1.0.0.tgz", + "integrity": "sha512-2EmdH1YvIQiZpltCNgkuiUnyukzxM/R6NDJX31Ke3BG1Nq5b0S2PhX59UKi9vZpPDQVdqn+1IcaAwnzTT5vCjw==", + "dev": true, + "requires": { + "call-bind": "^1.0.2", + "get-intrinsic": "^1.1.1" + } + }, "getopts": { "version": "2.2.5", "resolved": "https://registry.npmjs.org/getopts/-/getopts-2.2.5.tgz", @@ -8754,9 +9117,9 @@ } }, "is-negative-zero": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/is-negative-zero/-/is-negative-zero-2.0.1.tgz", - "integrity": "sha512-2z6JzQvZRa9A2Y7xC6dQQm4FSTSTNWjKIYYTt4246eMTJmIo0Q+ZyOsU66X8lxK1AbB92dFeglPLrhwpeRKO6w==", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/is-negative-zero/-/is-negative-zero-2.0.2.tgz", + "integrity": "sha512-dqJvarLawXsFbNDeJW7zAz8ItJ9cd28YufuuFzh0G8pNHjJMnY08Dv7sYX2uF5UpQOwieAeOExEYAWWfu7ZZUA==", "dev": true }, "is-number": { @@ -8798,6 +9161,12 @@ "has-tostringtag": "^1.0.0" } }, + "is-shared-array-buffer": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-shared-array-buffer/-/is-shared-array-buffer-1.0.1.tgz", + "integrity": "sha512-IU0NmyknYZN0rChcKhRO1X8LYz5Isj/Fsqh8NJOSf+N/hCOTwy29F32Ik7a+QszE63IdvmwdTPDd6cZ5pg4cwA==", + "dev": true + }, "is-stream": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", @@ -8828,6 +9197,15 @@ "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=", "dev": true }, + "is-weakref": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-weakref/-/is-weakref-1.0.2.tgz", + "integrity": "sha512-qctsuLZmIQ0+vSSMfoVvyFe2+GSEvnmZ2ezTup1SBse9+twCCeial6EEi3Nc2KFcf6+qz2FBPnjXsk8xhKSaPQ==", + "dev": true, + "requires": { + "call-bind": "^1.0.2" + } + }, "is-windows": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-windows/-/is-windows-1.0.2.tgz", @@ -9015,12 +9393,12 @@ } }, "jsx-ast-utils": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/jsx-ast-utils/-/jsx-ast-utils-3.2.0.tgz", - "integrity": "sha512-EIsmt3O3ljsU6sot/J4E1zDRxfBNrhjyf/OKjlydwgEimQuznlM4Wv7U+ueONJMyEn1WRE0K8dhi3dVAXYT24Q==", + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/jsx-ast-utils/-/jsx-ast-utils-3.2.1.tgz", + "integrity": "sha512-uP5vu8xfy2F9A6LGC22KO7e2/vGTS1MhP+18f++ZNlf0Ohaxbc9nIEwHAsejlJKyzfZzU5UIhe5ItYkitcZnZA==", "dev": true, "requires": { - "array-includes": "^3.1.2", + "array-includes": "^3.1.3", "object.assign": "^4.1.2" } }, @@ -9080,14 +9458,14 @@ } }, "load-json-file": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-2.0.0.tgz", - "integrity": "sha1-eUfkIUmvgNaWy/eXvKq8/h/inKg=", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-4.0.0.tgz", + "integrity": "sha1-L19Fq5HjMhYjT9U62rZo607AmTs=", "dev": true, "requires": { "graceful-fs": "^4.1.2", - "parse-json": "^2.2.0", - "pify": "^2.0.0", + "parse-json": "^4.0.0", + "pify": "^3.0.0", "strip-bom": "^3.0.0" }, "dependencies": { @@ -9119,6 +9497,12 @@ "integrity": "sha1-+wMJF/hqMTTlvJvsDWngAT3f7bI=", "dev": true }, + "lodash.truncate": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/lodash.truncate/-/lodash.truncate-4.4.2.tgz", + "integrity": "sha1-WjUNoLERO4N+z//VgSy+WNbq4ZM=", + "dev": true + }, "log-symbols": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.0.0.tgz", @@ -9482,9 +9866,9 @@ }, "dependencies": { "ansi-regex": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", - "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", "dev": true }, "camelcase": { @@ -9627,9 +10011,9 @@ "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=" }, "object-inspect": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.11.0.tgz", - "integrity": "sha512-jp7ikS6Sd3GxQfZJPyH3cjcbJF6GZPClgdV+EFygjFLQ5FmW/dRUnTd9PQ9k0JhoNDabWFbpF1yCdSWCC6gexg==", + "version": "1.12.0", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.0.tgz", + "integrity": "sha512-Ho2z80bVIvJloH+YzRmpZVQe87+qASmBUKZDWgx9cu+KDrX2ZDH/3tMy+gXbZETVGs2M8YdxObOh7XAtim9Y0g==", "dev": true }, "object-keys": { @@ -9651,37 +10035,46 @@ } }, "object.entries": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/object.entries/-/object.entries-1.1.4.tgz", - "integrity": "sha512-h4LWKWE+wKQGhtMjZEBud7uLGhqyLwj8fpHOarZhD2uY3C9cRtk57VQ89ke3moByLXMedqs3XCHzyb4AmA2DjA==", + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/object.entries/-/object.entries-1.1.5.tgz", + "integrity": "sha512-TyxmjUoZggd4OrrU1W66FMDG6CuqJxsFvymeyXI51+vQLN67zYfZseptRge703kKQdo4uccgAKebXFcRCzk4+g==", "dev": true, "requires": { "call-bind": "^1.0.2", "define-properties": "^1.1.3", - "es-abstract": "^1.18.2" + "es-abstract": "^1.19.1" } }, "object.fromentries": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/object.fromentries/-/object.fromentries-2.0.4.tgz", - "integrity": "sha512-EsFBshs5RUUpQEY1D4q/m59kMfz4YJvxuNCJcv/jWwOJr34EaVnG11ZrZa0UHB3wnzV1wx8m58T4hQL8IuNXlQ==", + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/object.fromentries/-/object.fromentries-2.0.5.tgz", + "integrity": "sha512-CAyG5mWQRRiBU57Re4FKoTBjXfDoNwdFVH2Y1tS9PqCsfUTymAohOkEMSG3aRNKmv4lV3O7p1et7c187q6bynw==", "dev": true, "requires": { "call-bind": "^1.0.2", "define-properties": "^1.1.3", - "es-abstract": "^1.18.0-next.2", - "has": "^1.0.3" + "es-abstract": "^1.19.1" + } + }, + "object.hasown": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/object.hasown/-/object.hasown-1.1.0.tgz", + "integrity": "sha512-MhjYRfj3GBlhSkDHo6QmvgjRLXQ2zndabdf3nX0yTyZK9rPfxb6uRpAac8HXNLy1GpqWtZ81Qh4v3uOls2sRAg==", + "dev": true, + "requires": { + "define-properties": "^1.1.3", + "es-abstract": "^1.19.1" } }, "object.values": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/object.values/-/object.values-1.1.4.tgz", - "integrity": "sha512-TnGo7j4XSnKQoK3MfvkzqKCi0nVe/D9I9IjwTNYdb/fxYHpjrluHVOgw0AF6jrRFGMPHdfuidR09tIDiIvnaSg==", + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/object.values/-/object.values-1.1.5.tgz", + "integrity": "sha512-QUZRW0ilQ3PnPpbNtgdNV1PDbEqLIiSFB3l+EnGtBQ/8SUTLj1PZwtQHABZtLgwpJZTSZhuGLOGk57Drx2IvYg==", "dev": true, "requires": { "call-bind": "^1.0.2", "define-properties": "^1.1.3", - "es-abstract": "^1.18.2" + "es-abstract": "^1.19.1" } }, "on-finished": { @@ -9782,12 +10175,13 @@ } }, "parse-json": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-2.2.0.tgz", - "integrity": "sha1-9ID0BDTvgHQfhGkJn43qGPVaTck=", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-4.0.0.tgz", + "integrity": "sha1-vjX1Qlvh9/bHRxhPmKeIy5lHfuA=", "dev": true, "requires": { - "error-ex": "^1.2.0" + "error-ex": "^1.3.1", + "json-parse-better-errors": "^1.0.1" } }, "parseurl": { @@ -9824,12 +10218,12 @@ "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" }, "path-type": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/path-type/-/path-type-2.0.0.tgz", - "integrity": "sha1-8BLMuEFbcJb8LaoQVMPXI4lZTHM=", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-3.0.0.tgz", + "integrity": "sha512-T2ZUsdZFHgA3u4e5PfPbjd7HDDpxPnQb5jN0SrDsjNSuVXHJqtwTnWqG0B1jZrgmJ/7lj1EmVIByWt1gxGkWvg==", "dev": true, "requires": { - "pify": "^2.0.0" + "pify": "^3.0.0" } }, "pathval": { @@ -9858,9 +10252,9 @@ "dev": true }, "pify": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", - "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", + "integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=", "dev": true }, "pkg-conf": { @@ -9923,16 +10317,6 @@ "p-limit": "^2.0.0" } }, - "parse-json": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-4.0.0.tgz", - "integrity": "sha1-vjX1Qlvh9/bHRxhPmKeIy5lHfuA=", - "dev": true, - "requires": { - "error-ex": "^1.3.1", - "json-parse-better-errors": "^1.0.1" - } - }, "path-exists": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", @@ -10007,6 +10391,66 @@ } } }, + "pkg-up": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/pkg-up/-/pkg-up-2.0.0.tgz", + "integrity": "sha1-yBmscoBZpGHKscOImivjxJoATX8=", + "dev": true, + "requires": { + "find-up": "^2.1.0" + }, + "dependencies": { + "find-up": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-2.1.0.tgz", + "integrity": "sha1-RdG35QbHF93UgndaK3eSCjwMV6c=", + "dev": true, + "requires": { + "locate-path": "^2.0.0" + } + }, + "locate-path": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-2.0.0.tgz", + "integrity": "sha1-K1aLJl7slExtnA3pw9u7ygNUzY4=", + "dev": true, + "requires": { + "p-locate": "^2.0.0", + "path-exists": "^3.0.0" + } + }, + "p-limit": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-1.3.0.tgz", + "integrity": "sha512-vvcXsLAJ9Dr5rQOPk7toZQZJApBl2K4J6dANSsEuh6QI41JYcsS/qhTGa9ErIUUgK3WNQoJYvylxvjqmiqEA9Q==", + "dev": true, + "requires": { + "p-try": "^1.0.0" + } + }, + "p-locate": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-2.0.0.tgz", + "integrity": "sha1-IKAQOyIqcMj9OcwuWAaA893l7EM=", + "dev": true, + "requires": { + "p-limit": "^1.1.0" + } + }, + "p-try": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-1.0.0.tgz", + "integrity": "sha1-y8ec26+P1CKOE/Yh8rGiN8GyB7M=", + "dev": true + }, + "path-exists": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", + "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=", + "dev": true + } + } + }, "prebuild-install": { "version": "6.1.4", "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-6.1.4.tgz", @@ -10054,14 +10498,14 @@ "dev": true }, "prop-types": { - "version": "15.7.2", - "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.7.2.tgz", - "integrity": "sha512-8QQikdH7//R2vurIJSutZ1smHYTcLpRWEOlHnzcWHmBYrOGUysKwSsrC89BCiFj3CbrfJ/nXFdJepOVrY1GCHQ==", + "version": "15.8.1", + "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz", + "integrity": "sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==", "dev": true, "requires": { "loose-envify": "^1.4.0", "object-assign": "^4.1.1", - "react-is": "^16.8.1" + "react-is": "^16.13.1" } }, "proxy-addr": { @@ -10141,24 +10585,24 @@ "dev": true }, "read-pkg": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-2.0.0.tgz", - "integrity": "sha1-jvHAYjxqbbDcZxPEv6xGMysjaPg=", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-3.0.0.tgz", + "integrity": "sha1-nLxoaXj+5l0WwA4rGcI3/Pbjg4k=", "dev": true, "requires": { - "load-json-file": "^2.0.0", + "load-json-file": "^4.0.0", "normalize-package-data": "^2.3.2", - "path-type": "^2.0.0" + "path-type": "^3.0.0" } }, "read-pkg-up": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/read-pkg-up/-/read-pkg-up-2.0.0.tgz", - "integrity": "sha1-a3KoBImE4MQeeVEP1en6mbO1Sb4=", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/read-pkg-up/-/read-pkg-up-3.0.0.tgz", + "integrity": "sha1-PtSWaF26D4/hGNBpHcUfSh/5bwc=", "dev": true, "requires": { "find-up": "^2.0.0", - "read-pkg": "^2.0.0" + "read-pkg": "^3.0.0" }, "dependencies": { "find-up": { @@ -10249,9 +10693,9 @@ "integrity": "sha512-KJbQG1iZOC4IRBccT1mOodzKp5YM9gC1H4L6LgDmmXx+4U3pabhb+UF0UI0u+Lck1cvb4oyAuihaQqTbKnslqQ==" }, "regexp.prototype.flags": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.3.1.tgz", - "integrity": "sha512-JiBdRBq91WlY7uRJ0ds7R+dU02i6LKi8r3BuQhNXn+kmeLN+EfHhfjqMRis1zJxnlu88hq/4dx0P2OP3APRTOA==", + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.4.1.tgz", + "integrity": "sha512-pMR7hBVUUGI7PMA37m2ofIdQCsomVnas+Jn5UPGAHQ+/LlwKm/aTLJHdasmHRzlfeZwHiAOaRSo2rbBDm3nNUQ==", "dev": true, "requires": { "call-bind": "^1.0.2", @@ -10279,6 +10723,12 @@ "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=", "dev": true }, + "require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "dev": true + }, "require-main-filename": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz", @@ -10453,44 +10903,20 @@ } }, "slice-ansi": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-2.1.0.tgz", - "integrity": "sha512-Qu+VC3EwYLldKa1fCxuuvULvSJOKEgk9pi8dZeCVK7TqBfUNTH4sFkk4joj8afVSfAYgJoSOetjx9QWOJ5mYoQ==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-4.0.0.tgz", + "integrity": "sha512-qMCMfhY040cVHT43K9BFygqYbUPFZKHOg7K73mtTWJRb8pyP3fzf4Ixd5SzdEJQ6MRUg/WBnOLxghZtKKurENQ==", "dev": true, "requires": { - "ansi-styles": "^3.2.0", - "astral-regex": "^1.0.0", - "is-fullwidth-code-point": "^2.0.0" + "ansi-styles": "^4.0.0", + "astral-regex": "^2.0.0", + "is-fullwidth-code-point": "^3.0.0" }, "dependencies": { - "ansi-styles": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", - "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", - "dev": true, - "requires": { - "color-convert": "^1.9.0" - } - }, - "color-convert": { - "version": "1.9.3", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", - "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", - "dev": true, - "requires": { - "color-name": "1.1.3" - } - }, - "color-name": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=", - "dev": true - }, "is-fullwidth-code-point": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", - "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", "dev": true } } @@ -10542,9 +10968,9 @@ } }, "spdx-license-ids": { - "version": "3.0.10", - "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.10.tgz", - "integrity": "sha512-oie3/+gKf7QtpitB0LYLETe+k8SifzsX4KixvpOsbI6S0kRiRQ5MKOio8eMSAKQ17N06+wdEOXRiId+zOxo0hA==", + "version": "3.0.11", + "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.11.tgz", + "integrity": "sha512-Ctl2BrFiM0X3MANYgj3CkygxhRmr9mi6xhejbdO960nF6EDJApTYpn0BQnDKlnNBULKiCN1n3w9EBkHK8ZWg+g==", "dev": true }, "split": { @@ -10562,18 +10988,18 @@ "dev": true }, "standard": { - "version": "16.0.3", - "resolved": "https://registry.npmjs.org/standard/-/standard-16.0.3.tgz", - "integrity": "sha512-70F7NH0hSkNXosXRltjSv6KpTAOkUkSfyu3ynyM5dtRUiLtR+yX9EGZ7RKwuGUqCJiX/cnkceVM6HTZ4JpaqDg==", + "version": "16.0.4", + "resolved": "https://registry.npmjs.org/standard/-/standard-16.0.4.tgz", + "integrity": "sha512-2AGI874RNClW4xUdM+bg1LRXVlYLzTNEkHmTG5mhyn45OhbgwA+6znowkOGYy+WMb5HRyELvtNy39kcdMQMcYQ==", "dev": true, "requires": { - "eslint": "~7.13.0", - "eslint-config-standard": "16.0.2", + "eslint": "~7.18.0", + "eslint-config-standard": "16.0.3", "eslint-config-standard-jsx": "10.0.0", - "eslint-plugin-import": "~2.22.1", + "eslint-plugin-import": "~2.24.2", "eslint-plugin-node": "~11.1.0", - "eslint-plugin-promise": "~4.2.1", - "eslint-plugin-react": "~7.21.5", + "eslint-plugin-promise": "~5.1.0", + "eslint-plugin-react": "~7.25.1", "standard-engine": "^14.0.1" } }, @@ -10622,14 +11048,14 @@ } }, "string.prototype.matchall": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/string.prototype.matchall/-/string.prototype.matchall-4.0.5.tgz", - "integrity": "sha512-Z5ZaXO0svs0M2xd/6By3qpeKpLKd9mO4v4q3oMEQrk8Ck4xOD5d5XeBOOjGrmVZZ/AHB1S0CgG4N5r1G9N3E2Q==", + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/string.prototype.matchall/-/string.prototype.matchall-4.0.6.tgz", + "integrity": "sha512-6WgDX8HmQqvEd7J+G6VtAahhsQIssiZ8zl7zKh1VDMFyL3hRTJP4FTNA3RbIp2TOQ9AYNDcc7e3fH0Qbup+DBg==", "dev": true, "requires": { "call-bind": "^1.0.2", "define-properties": "^1.1.3", - "es-abstract": "^1.18.2", + "es-abstract": "^1.19.1", "get-intrinsic": "^1.1.1", "has-symbols": "^1.0.2", "internal-slot": "^1.0.3", @@ -10686,53 +11112,66 @@ } }, "table": { - "version": "5.4.6", - "resolved": "https://registry.npmjs.org/table/-/table-5.4.6.tgz", - "integrity": "sha512-wmEc8m4fjnob4gt5riFRtTu/6+4rSe12TpAELNSqHMfF3IqnA+CH37USM6/YR3qRZv7e56kAEAtd6nKZaxe0Ug==", + "version": "6.8.0", + "resolved": "https://registry.npmjs.org/table/-/table-6.8.0.tgz", + "integrity": "sha512-s/fitrbVeEyHKFa7mFdkuQMWlH1Wgw/yEXMt5xACT4ZpzWFluehAxRtUUQKPuWhaLAWhFcVx6w3oC8VKaUfPGA==", "dev": true, "requires": { - "ajv": "^6.10.2", - "lodash": "^4.17.14", - "slice-ansi": "^2.1.0", - "string-width": "^3.0.0" + "ajv": "^8.0.1", + "lodash.truncate": "^4.4.2", + "slice-ansi": "^4.0.0", + "string-width": "^4.2.3", + "strip-ansi": "^6.0.1" }, "dependencies": { + "ajv": { + "version": "8.9.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.9.0.tgz", + "integrity": "sha512-qOKJyNj/h+OWx7s5DePL6Zu1KeM9jPZhwBqs+7DzP6bGOvqzVCSf0xueYmVuaC/oQ/VtS2zLMLHdQFbkka+XDQ==", + "dev": true, + "requires": { + "fast-deep-equal": "^3.1.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2", + "uri-js": "^4.2.2" + } + }, "ansi-regex": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", - "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", "dev": true }, - "emoji-regex": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz", - "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==", + "is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", "dev": true }, - "is-fullwidth-code-point": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", - "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=", + "json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", "dev": true }, "string-width": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", - "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", "dev": true, "requires": { - "emoji-regex": "^7.0.1", - "is-fullwidth-code-point": "^2.0.0", - "strip-ansi": "^5.1.0" + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" } }, "strip-ansi": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", - "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", "dev": true, "requires": { - "ansi-regex": "^4.1.0" + "ansi-regex": "^5.0.1" } } } @@ -10845,9 +11284,9 @@ "integrity": "sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw==" }, "tsconfig-paths": { - "version": "3.11.0", - "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.11.0.tgz", - "integrity": "sha512-7ecdYDnIdmv639mmDwslG6KQg1Z9STTz1j7Gcz0xa+nshh/gKDAHcPxRbWOsA3SPp0tXP2leTcY9Kw+NAkfZzA==", + "version": "3.12.0", + "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.12.0.tgz", + "integrity": "sha512-e5adrnOYT6zqVnWqZu7i/BQ3BnhzvGbjEjejFXO20lKIKpwTaupkCPgEfv4GZK1IBciJUEhYs3J3p75FdaTFVg==", "dev": true, "requires": { "@types/json5": "^0.0.29", @@ -11057,9 +11496,9 @@ }, "dependencies": { "ansi-regex": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", - "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", "dev": true }, "is-fullwidth-code-point": { @@ -11095,26 +11534,6 @@ "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" }, - "write": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/write/-/write-1.0.3.tgz", - "integrity": "sha512-/lg70HAjtkUgWPVZhZcm+T4hkL8Zbtp1nFNOn3lRrxnlv50SRBv7cR7RqR+GMsd3hUXy9hWBo4CHTbFTcOYwig==", - "dev": true, - "requires": { - "mkdirp": "^0.5.1" - }, - "dependencies": { - "mkdirp": { - "version": "0.5.5", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz", - "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", - "dev": true, - "requires": { - "minimist": "^1.2.5" - } - } - } - }, "write-file-atomic": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-3.0.3.tgz", @@ -11160,9 +11579,9 @@ }, "dependencies": { "ansi-regex": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", - "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", "dev": true }, "is-fullwidth-code-point": { diff --git a/package.json b/package.json index 5df5a33..968dc06 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,7 @@ }, "dependencies": { "abort-controller": "^3.0.0", - "axios": "^0.21.1", + "axios": "^0.21.2", "better-sqlite3": "^7.4.1", "body-parser": "^1.19.0", "cors": "^2.8.5", From 11b0a7670d347af8595de92d6a833f8999bf1a24 Mon Sep 17 00:00:00 2001 From: hojarasca Date: Thu, 20 Jan 2022 15:27:42 -0300 Subject: [PATCH 48/82] fix stop of api executor. --- src/execution/api-executor.js | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/src/execution/api-executor.js b/src/execution/api-executor.js index d3e88e0..b83a7af 100644 --- a/src/execution/api-executor.js +++ b/src/execution/api-executor.js @@ -29,18 +29,9 @@ class ApiExecutor { this.pool = genericPool.createPool(factory, opts) } - start () { + async start () {} - } - - async stop () { - this.logger.debug('Stopping all workers') - - await Promise.all(this.workers.map(worker => worker.terminate())) - - this.workers = [] - this.workerRequests = [] - } + async stop () {} async execute (txid) { if (this.executing.has(txid)) return From 3f06382e7bfb39bdf056d6324486f4b775f87633 Mon Sep 17 00:00:00 2001 From: hojarasca Date: Thu, 20 Jan 2022 16:15:04 -0300 Subject: [PATCH 49/82] fixed tests. --- test/crawler.test.js | 16 +++++++++++----- test/indexer.test.js | 36 +++++++++++++++++++++++------------- test/server.test.js | 40 +++++++++++++++++++++++++++------------- 3 files changed, 61 insertions(+), 31 deletions(-) diff --git a/test/crawler.test.js b/test/crawler.test.js index d4d1cb4..ad992c8 100644 --- a/test/crawler.test.js +++ b/test/crawler.test.js @@ -12,6 +12,7 @@ const { DEFAULT_TRUSTLIST } = require('../src/config') const Database = require('../src/database') const { SqliteDatasource } = require('../src/data-sources/sqlite-datasource') const { DbTrustList } = require('../src/trust-list/db-trust-list') +const Executor = require('../src/execution/executor') // ------------------------------------------------------------------------------------------------ // Globals @@ -40,7 +41,8 @@ describe('Crawler', () => { return { height: 1, hash: 'abc', txids: [txid] } } const api = { getNextBlock, fetch } - const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, []) + const executor = new Executor('main', 1, database, logger) + const indexer = new Indexer(database, api, executor, 1, 1, logger, 0, Infinity, []) const promise = indexed(indexer, txid) await indexer.start() await database.trust(txid) @@ -58,7 +60,8 @@ describe('Crawler', () => { return { height: 1, hash: 'abc', txids: [txid], txhexs: [txns[txid]] } } const api = { getNextBlock } - const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, []) + const executor = new Executor('main', 1, database, logger) + const indexer = new Indexer(database, api, executor, 1, 1, logger, 0, Infinity, []) await indexer.start() await database.trust(txid) await indexed(indexer, txid) @@ -82,7 +85,8 @@ describe('Crawler', () => { return { height: 1, hash: 'abc', txids, txhexs: txids.map(txid => txns[txid]) } } const api = { getNextBlock, fetch } - const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, DEFAULT_TRUSTLIST) + const executor = new Executor('main', 1, database, logger) + const indexer = new Indexer(database, api, executor, 1, 1, logger, 0, Infinity, DEFAULT_TRUSTLIST) indexer.crawler.pollForNewBlocksInterval = 10 await indexer.start() await database.addTransaction(txids[1]) @@ -113,7 +117,8 @@ describe('Crawler', () => { if (height === 12) { didReorg = true; return { reorg: true } } } const api = { getNextBlock, fetch } - const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, []) + const executor = new Executor('main', 1, database, logger) + const indexer = new Indexer(database, api, executor, 1, 1, logger, 0, Infinity, []) indexer.crawler.pollForNewBlocksInterval = 10 await indexer.start() database.trust(txid) @@ -143,7 +148,8 @@ describe('Crawler', () => { if (height === 12) { didReorg = true; return { reorg: true } } } const api = { getNextBlock, fetch } - const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, []) + const executor = new Executor('main', 1, database, logger) + const indexer = new Indexer(database, api, executor, 1, 1, logger, 0, Infinity, []) await indexer.start() await database.trust(txid) await reorged(indexer) diff --git a/test/indexer.test.js b/test/indexer.test.js index 056601c..f9f2e88 100644 --- a/test/indexer.test.js +++ b/test/indexer.test.js @@ -14,6 +14,7 @@ const { DEFAULT_TRUSTLIST } = require('../src/config') const Database = require('../src/database') const { SqliteDatasource } = require('../src/data-sources/sqlite-datasource') const { DbTrustList } = require('../src/trust-list/db-trust-list') +const Executor = require('../src/execution/executor') // ------------------------------------------------------------------------------------------------ // Globals @@ -37,7 +38,8 @@ afterEach(() => database.close()) describe('Indexer', () => { it('add and index', async () => { - const indexer = new IndexerTest(database, api, 'main', 1, 1, logger, 0, Infinity, []) + const executor = new Executor('main', 1, database, logger) + const indexer = new IndexerTest(database, api, executor, 1, 1, logger, 0, Infinity, []) await indexer.start() const promise = indexed(indexer, '9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102') await database.addTransaction('3f9de452f0c3c96be737d42aa0941b27412211976688967adb3174ee18b04c64') @@ -63,7 +65,8 @@ describe('Indexer', () => { tx.update(() => new A(pubkey)) const rawtx = await tx.export() const api = { fetch: _txid => { return { hex: rawtx } } } - const indexer = new IndexerTest(database, api, 'test', 1, 1, logger, 0, Infinity, []) + const executor = new Executor('test', 1, database, logger) + const indexer = new IndexerTest(database, api, executor, 1, 1, logger, 0, Infinity, []) const txid = new bsv.Transaction(rawtx).hash await indexer.start() database.addTransaction(txid) @@ -75,7 +78,8 @@ describe('Indexer', () => { // -------------------------------------------------------------------------- it('add in reverse and index', async () => { - const indexer = new IndexerTest(database, api, 'main', 1, 1, logger, 0, Infinity, []) + const executor = new Executor('main', 1, database, logger) + const indexer = new IndexerTest(database, api, executor, 1, 1, logger, 0, Infinity, []) await indexer.start() const promise = indexed(indexer, '9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102') await database.addTransaction('9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102') @@ -89,7 +93,8 @@ describe('Indexer', () => { // -------------------------------------------------------------------------- it('fail to index', async () => { - const indexer = new IndexerTest(database, api, 'main', 1, 1, logger, 0, Infinity, []) + const executor = new Executor('main', 1, database, logger) + const indexer = new IndexerTest(database, api, executor, 1, 1, logger, 0, Infinity, []) await indexer.start() const promise = failed(indexer, 'a5291157ab7a2d80d834bbe82c380ce3976f53990d20c62c477ca3a2ac93a7e9') await database.trust('b17a9af70ab0f46809f908b2e900e395ba40996000bf4f00e3b27a1e93280cf1') @@ -103,7 +108,8 @@ describe('Indexer', () => { // -------------------------------------------------------------------------- it('discovered berry transaction', async () => { - const indexer = new IndexerTest(database, api, 'main', 1, 1, logger, 0, Infinity, []) + const executor = new Executor('main', 1, database, logger) + const indexer = new IndexerTest(database, api, executor, 1, 1, logger, 0, Infinity, []) await indexer.start() const promise = indexed(indexer, 'bfa5180e601e92af23d80782bf625b102ac110105a392e376fe7607e4e87dc8d') await database.addTransaction('bfa5180e601e92af23d80782bf625b102ac110105a392e376fe7607e4e87dc8d') // Class with berry image @@ -123,7 +129,8 @@ describe('Indexer', () => { // -------------------------------------------------------------------------- it('add and download dependencies', async () => { - const indexer = new IndexerTest(database, api, 'main', 1, 1, logger, 0, Infinity, []) + const executor = new Executor('main', 1, database, logger) + const indexer = new IndexerTest(database, api, executor, 1, 1, logger, 0, Infinity, []) await indexer.start() await database.addTransaction('9bb02c2f34817fec181dcf3f8f7556232d3fac9ef76660326f0583d57bf0d102') await new Promise((resolve) => setTimeout(resolve, 1000)) @@ -133,7 +140,8 @@ describe('Indexer', () => { // -------------------------------------------------------------------------- it('remove discovered dep', async () => { - const indexer = new IndexerTest(database, api, 'main', 1, 1, logger, 0, Infinity, DEFAULT_TRUSTLIST) + const executor = new Executor('main', 1, database, logger) + const indexer = new IndexerTest(database, api, executor, 1, 1, logger, 0, Infinity, DEFAULT_TRUSTLIST) await indexer.start() const promise = indexed(indexer, 'bfa5180e601e92af23d80782bf625b102ac110105a392e376fe7607e4e87dc8d') await database.addTransaction('bfa5180e601e92af23d80782bf625b102ac110105a392e376fe7607e4e87dc8d') // Class with berry image @@ -149,7 +157,8 @@ describe('Indexer', () => { it('get spent', async function () { this.timeout(40000) - const indexer = new IndexerTest(database, api, 'main', 1, 1, logger, 0, Infinity, DEFAULT_TRUSTLIST) + const executor = new Executor('main', 1, database, logger) + const indexer = new IndexerTest(database, api, executor, 1, 1, logger, 0, Infinity, DEFAULT_TRUSTLIST) await indexer.start() const promise = indexed(indexer, '11f27cdad53128a4eb14c8328515dfab56b16ea5a71dd26abe9e9d7488f3ab83') await database.addTransaction('11f27cdad53128a4eb14c8328515dfab56b16ea5a71dd26abe9e9d7488f3ab83') @@ -164,7 +173,8 @@ describe('Indexer', () => { // -------------------------------------------------------------------------- it('mark failed execute as melts', async () => { - const indexer = new IndexerTest(database, {}, 'test', 1, 1, logger, 0, Infinity, []) + const executor = new Executor('test', 1, database, logger) + const indexer = new IndexerTest(database, {}, executor, 1, 1, logger, 0, Infinity, []) const rawtx1 = '0100000001a11d53c34263d1ea9dec40d3cc5beb7eb461a601d898a8337dea215cd90a9e4a010000006a47304402202f294c5ceca857cfc03e38b1a49a79d6c133e9e6b18047f0301f9f74bb2abdab022027aa6662cd24428106b9f8f2e38d2e5b8f0b7c30929eef6dbc1d013c43b0493f41210211f2cc632921525ec8650cb65c2ed520e400a2644010c1e794203d5823f604c0ffffffff030000000000000000fd0301006a0372756e0105004cf87b22696e223a302c22726566223a5b226e61746976653a2f2f4a6967225d2c226f7574223a5b2238396336653439636532653831373962653138383563396230653032343863363935666130373634343939656665626362363936623238323732366239666165225d2c2264656c223a5b5d2c22637265223a5b226d737138444642455777546166675a6173474c4a386f3338517a456367346267364a225d2c2265786563223a5b7b226f70223a224445504c4f59222c2264617461223a5b22636c617373204120657874656e6473204a6967207b207d222c7b2264657073223a7b224a6967223a7b22246a6967223a307d7d7d5d7d5d7d11010000000000001976a9148711466c1f8b5977cb788485fcb6cc1fb9d0407788acf6def505000000001976a9142208fb2364d1551e2dd26549d7c22eab613a207188ac00000000' const rawtx2 = '0100000002cb8c61b7d73cf14ed2526f2adcb0ef941563c69fb794a87eb39a94423886d273010000006a4730440220306a24e0464c90889d6fd1580db4420fe9ee1bd8f167ec793d40d2296ff0d8ea02202224f4f13e4c07354478983b2dc88170342a4f1ac3e6cacad8616a92348fc768412103a6fa27cfcda39be6ee9dc5dbd43a44c2c749ca136f7d41cd81468f72cc0fda59ffffffffcb8c61b7d73cf14ed2526f2adcb0ef941563c69fb794a87eb39a94423886d273020000006b483045022100c2b7a660b22dd2c3ac22d47ba16fa3f7df852f5a6cfdec5ce14c734517a0b1900220592da53a61ec1387aa96050c370b7c5ba162ee35e8d30b55d9999f1c2ba06ade41210211f2cc632921525ec8650cb65c2ed520e400a2644010c1e794203d5823f604c0ffffffff030000000000000000ae006a0372756e0105004ca37b22696e223a312c22726566223a5b5d2c226f7574223a5b2264633031326334616436346533626136373632383762323239623865306662303934326448626535303435393036363830616637633937663134666239663433225d2c2264656c223a5b5d2c22637265223a5b5d2c2265786563223a5b7b226f70223a2243414c4c222c2264617461223a5b7b22246a6967223a307d2c2261757468222c5b5d5d7d5d7d11010000000000001976a9148711466c1f8b5977cb788485fcb6cc1fb9d0407788acdeddf505000000001976a9142208fb2364d1551e2dd26549d7c22eab613a207188ac00000000' const txid1 = new bsv.Transaction(rawtx1).hash @@ -184,7 +194,8 @@ describe('Indexer', () => { // -------------------------------------------------------------------------- it('deletes are not included in unspent', async () => { - const indexer = new IndexerTest(database, {}, 'test', 1, 1, logger, 0, Infinity, []) + const executor = new Executor('test', 1, database, logger) + const indexer = new IndexerTest(database, {}, executor, 1, 1, logger, 0, Infinity, []) const rawtx1 = '01000000016f4f66891029280028bce15768b3fdc385533b0bcc77a029add646176207e77f010000006b483045022100a76777ae759178595cb83ce9473699c9056e32faa8e0d07c2517918744fab9e90220369d7a6a2f52b5ddd9bff4ed659ef5a8e676397dac15e9c5dc6dad09e5eab85e412103ac8a61b3fb98161003daaaa63ec1983dc127f4f978a42f2eefd31a074a814345ffffffff030000000000000000fd0301006a0372756e0105004cf87b22696e223a302c22726566223a5b226e61746976653a2f2f4a6967225d2c226f7574223a5b2237373864313934336265613463353166356561313635666630346335613039323435356365386437343335623936336333613130623961343536633463623330225d2c2264656c223a5b5d2c22637265223a5b226d674671626e5254774c3155436d384a654e6e556d6b7a58665a6f3271385764364c225d2c2265786563223a5b7b226f70223a224445504c4f59222c2264617461223a5b22636c617373204120657874656e6473204a6967207b207d222c7b2264657073223a7b224a6967223a7b22246a6967223a307d7d7d5d7d5d7d11010000000000001976a914081c4c589c062b1b1d4e4b25a8b3096868059d7a88acf6def505000000001976a914146caf0030b67f3fae5d53b7c3fa7e1e6fcaaf3b88ac00000000' const rawtx2 = '01000000015991661ed379a0d12a68feacdbf7776d82bcffe1761f995cf0412c5ae2d25d28010000006a47304402203776f765d6915431388110a7f4645a61bd8d2f2ab00ade0049f0da95b5455c22022074ca4b6a87891ba852416bf08b64ad3db130a0b780e2a658c451ebacbbcffbf8412103646b0e969bd3825f781f39b737bdfed1e2cd63533301317099e5ac021b4826aaffffffff010000000000000000b1006a0372756e0105004ca67b22696e223a312c22726566223a5b5d2c226f7574223a5b5d2c2264656c223a5b2265386436393434613366383765323936663237326562656437663033623133323962653262313733653732376436623431643632616365343036656434373539225d2c22637265223a5b5d2c2265786563223a5b7b226f70223a2243414c4c222c2264617461223a5b7b22246a6967223a307d2c2264657374726f79222c5b5d5d7d5d7d00000000' const txid1 = new bsv.Transaction(rawtx1).hash @@ -220,7 +231,8 @@ describe('Indexer', () => { const txHex2 = await run.blockchain.fetch(txid2) const txHex3 = await run.blockchain.fetch(txid3) - const indexer = new IndexerTest(database, run.blockchain, 'test', 1, 1, logger, 0, Infinity, []) + const executor = new Executor('test', 1, database, logger) + const indexer = new IndexerTest(database, run.blockchain, executor, 1, 1, logger, 0, Infinity, []) await indexer.start() const promise = indexed(indexer, txid2) await database.trust(txid1) @@ -237,5 +249,3 @@ describe('Indexer', () => { await indexer.stop() }) }) - -// ------------------------------------------------------------------------------------------------ diff --git a/test/server.test.js b/test/server.test.js index c96167e..c6dd5fc 100644 --- a/test/server.test.js +++ b/test/server.test.js @@ -15,6 +15,7 @@ const Database = require('../src/database') const { SqliteDatasource } = require('../src/data-sources/sqlite-datasource') const { DbTrustList } = require('../src/trust-list/db-trust-list') const { buildMainServer } = require('../src/http/build-main-server') +const Executor = require('../src/execution/executor') // ------------------------------------------------------------------------------------------------ // Globals @@ -45,7 +46,8 @@ describe('Server', () => { describe('post tx', () => { it('add with body', async () => { - const indexer = new Indexer(database, {}, 'main', 1, 1, logger, 0, Infinity, []) + const executor = new Executor('main', 1, database, logger) + const indexer = new Indexer(database, {}, executor, 1, 1, logger, 0, Infinity, DEFAULT_TRUSTLIST) const server = buildMainServer(database, logger) await indexer.start() const port = 52521 @@ -63,7 +65,8 @@ describe('Server', () => { // ------------------------------------------------------------------------ it('does not throw if add with rawtx mismatch', async () => { - const indexer = new Indexer(database, {}, 'main', 1, 1, logger, 0, Infinity, []) + const executor = new Executor('main', 1, database, logger) + const indexer = new Indexer(database, {}, executor, 1, 1, logger, 0, Infinity, DEFAULT_TRUSTLIST) const server = buildMainServer(database, logger) await indexer.start() await server.start() @@ -82,7 +85,8 @@ describe('Server', () => { describe('post trust', () => { it('trust multiple', async () => { - const indexer = new Indexer(database, {}, 'main', 1, 1, logger, 0, Infinity, []) + const executor = new Executor('main', 1, database, logger) + const indexer = new Indexer(database, {}, executor, 1, 1, logger, 0, Infinity, DEFAULT_TRUSTLIST) const server = buildMainServer(database, logger) await indexer.start() await server.start() @@ -106,7 +110,8 @@ describe('Server', () => { describe('get jig', () => { it('returns state if exists', async () => { - const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, DEFAULT_TRUSTLIST) + const executor = new Executor('main', 1, database, logger) + const indexer = new Indexer(database, api, executor, 1, 1, logger, 0, Infinity, DEFAULT_TRUSTLIST) const server = buildMainServer(database, logger) await indexer.start() await server.start() @@ -124,7 +129,8 @@ describe('Server', () => { // ------------------------------------------------------------------------ it('returns 404 if missing', async () => { - const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, []) + const executor = new Executor('main', 1, database, logger) + const indexer = new Indexer(database, api, executor, 1, 1, logger, 0, Infinity, DEFAULT_TRUSTLIST) const server = buildMainServer(database, logger) await indexer.start() await server.start() @@ -146,7 +152,8 @@ describe('Server', () => { describe('get berry', () => { it('returns state if exists', async () => { - const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, DEFAULT_TRUSTLIST) + const executor = new Executor('main', 1, database, logger) + const indexer = new Indexer(database, api, executor, 1, 1, logger, 0, Infinity, DEFAULT_TRUSTLIST) const server = buildMainServer(database, logger) await indexer.start() await server.start() @@ -163,7 +170,8 @@ describe('Server', () => { // ------------------------------------------------------------------------ it('returns 404 if missing', async () => { - const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, []) + const executor = new Executor('main', 1, database, logger) + const indexer = new Indexer(database, api, executor, 1, 1, logger, 0, Infinity, DEFAULT_TRUSTLIST) const server = buildMainServer(database, logger) await indexer.start() await server.start() @@ -185,7 +193,8 @@ describe('Server', () => { describe('get tx', () => { it('returns rawtx if downloaded', async () => { - const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, []) + const executor = new Executor('main', 1, database, logger) + const indexer = new Indexer(database, api, executor, 1, 1, logger, 0, Infinity, DEFAULT_TRUSTLIST) const server = buildMainServer(database, logger) await indexer.start() await server.start() @@ -203,7 +212,8 @@ describe('Server', () => { // ------------------------------------------------------------------------ it('returns 404 if missing', async () => { - const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, []) + const executor = new Executor('main', 1, database, logger) + const indexer = new Indexer(database, api, executor, 1, 1, logger, 0, Infinity, DEFAULT_TRUSTLIST) const server = buildMainServer(database, logger) await indexer.start() await server.start() @@ -221,7 +231,8 @@ describe('Server', () => { // ------------------------------------------------------------------------ it('returns 404 if not downloaded', async () => { - const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, []) + const executor = new Executor('main', 1, database, logger) + const indexer = new Indexer(database, api, executor, 1, 1, logger, 0, Infinity, DEFAULT_TRUSTLIST) const server = buildMainServer(database, logger) await indexer.start() await server.start() @@ -244,7 +255,8 @@ describe('Server', () => { describe('get unspent', () => { it('query all unspent', async () => { - const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, DEFAULT_TRUSTLIST) + const executor = new Executor('main', 1, database, logger) + const indexer = new Indexer(database, api, executor, 1, 1, logger, 0, Infinity, DEFAULT_TRUSTLIST) const server = buildMainServer(database, logger) await indexer.start() await server.start() @@ -263,7 +275,8 @@ describe('Server', () => { // ------------------------------------------------------------------------ it('query unspent by address', async () => { - const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, DEFAULT_TRUSTLIST) + const executor = new Executor('main', 1, database, logger) + const indexer = new Indexer(database, api, executor, 1, 1, logger, 0, Infinity, DEFAULT_TRUSTLIST) const server = buildMainServer(database, logger) await indexer.start() await server.start() @@ -284,7 +297,8 @@ describe('Server', () => { describe('misc', () => { it('cors', async () => { - const indexer = new Indexer(database, api, 'main', 1, 1, logger, 0, Infinity, []) + const executor = new Executor('main', 1, database, logger) + const indexer = new Indexer(database, api, executor, 'main', 1, logger, 0, Infinity, []) const server = buildMainServer(database, logger) await indexer.start() await server.start() From ae0928ee1d7e54aa9abd86a1433269422faa5555 Mon Sep 17 00:00:00 2001 From: hojarasca Date: Thu, 20 Jan 2022 17:10:07 -0300 Subject: [PATCH 50/82] Added simple retry mechanism. --- src/execution/api-executor.js | 12 ++++++++---- src/indexer.js | 15 ++++++++++++--- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/src/execution/api-executor.js b/src/execution/api-executor.js index b83a7af..69a6605 100644 --- a/src/execution/api-executor.js +++ b/src/execution/api-executor.js @@ -53,15 +53,19 @@ class ApiExecutor { }) }) if (!httpResponse.ok) { - await this.onExecuteFailed(txid, 'execution error') // TOOD: this should actually be a retry. + await this.onExecuteFailed(txid, 'execution error', true) this.executing.delete(txid) return } const json = await httpResponse.json() - const { response } = json - await this.onIndexed(txid, response) + if (json.ok) { + const { response } = json + await this.onIndexed(txid, response) + } else { + await this.onExecuteFailed(txid, json.errors, false) + } } catch (e) { - await this.onExecuteFailed(txid, 'execution error') // TOOD: this should actually be a retry. + await this.onExecuteFailed(txid, 'execution error', true) } finally { this.executing.delete(txid) await this.pool.release(token) diff --git a/src/indexer.js b/src/indexer.js index 4801b67..c6d7909 100644 --- a/src/indexer.js +++ b/src/indexer.js @@ -20,6 +20,7 @@ class Indexer { this.onFailToIndex = null this.onBlock = null this.onReorg = null + this.pendingRetries = new Map() this.logger = logger this.database = database @@ -79,6 +80,9 @@ class Indexer { async stop () { this.crawler.stop() + for (const entry of this.pendingRetries.entries()) { + clearTimeout(entry[1]) + } if (this.api.disconnect) await this.api.disconnect() this.downloader.stop() await this.executor.stop() @@ -112,9 +116,14 @@ class Indexer { } } - async _onExecuteFailed (txid, e) { - this.logger.error(`Failed to execute ${txid}: ${e.toString()}`) - await this.database.setTransactionExecutionFailed(txid) + async _onExecuteFailed (txid, e, shouldRetry = false) { + if (shouldRetry) { + const timeout = setTimeout(() => { this._onReadyToExecute(txid) }, 10000) + this.pendingRetries.set(txid, timeout) + } else { + this.logger.error(`Failed to execute ${txid}: ${e.toString()}`) + await this.database.setTransactionExecutionFailed(txid) + } if (this.onFailToIndex) this.onFailToIndex(txid, e) } From 694e1ab4078720bd52cd10babed1dd224a3de12f Mon Sep 17 00:00:00 2001 From: hojarasca Date: Fri, 21 Jan 2022 09:18:23 -0300 Subject: [PATCH 51/82] prevent memory leak. --- src/indexer.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/indexer.js b/src/indexer.js index c6d7909..f10d14a 100644 --- a/src/indexer.js +++ b/src/indexer.js @@ -107,6 +107,7 @@ class Indexer { } async _onIndexed (txid, result) { + this.pendingRetries.delete(txid) if (!await this.database.hasTransaction(txid)) return // Check not re-orged this.logger.info(`Executed ${txid}`) this.database.storeExecutedTransaction(txid, result) @@ -121,6 +122,7 @@ class Indexer { const timeout = setTimeout(() => { this._onReadyToExecute(txid) }, 10000) this.pendingRetries.set(txid, timeout) } else { + this.pendingRetries.delete(txid) this.logger.error(`Failed to execute ${txid}: ${e.toString()}`) await this.database.setTransactionExecutionFailed(txid) } From ed9005873949128139bd24aaa031da55b91f1d92 Mon Sep 17 00:00:00 2001 From: hojarasca Date: Fri, 21 Jan 2022 10:53:11 -0300 Subject: [PATCH 52/82] splitted url config for blob api. --- src/bin/start-execution-server.js | 8 ++-- src/bin/start.js | 8 ++-- src/config.js | 69 ++++++++++++++++------------ src/data-sources/api-blob-storage.js | 11 +++-- src/execution/executor.js | 6 ++- src/worker.js | 8 ++-- 6 files changed, 63 insertions(+), 47 deletions(-) diff --git a/src/bin/start-execution-server.js b/src/bin/start-execution-server.js index 25cf8a5..ac1d88c 100644 --- a/src/bin/start-execution-server.js +++ b/src/bin/start-execution-server.js @@ -1,4 +1,4 @@ -const { PORT, DEBUG, WORKERS, DATA_API_ROOT, NETWORK } = require('../config') +const { PORT, DEBUG, WORKERS, DATA_API_ROOT, NETWORK, DATA_API_STATE_ROOT, DATA_API_TX_ROOT } = require('../config') const { buildExecutionServer } = require('../http/build-execution-server') const { ApiBlobStorage } = require('../data-sources/api-blob-storage') @@ -8,8 +8,9 @@ logger.warn = console.warn.bind(console) logger.error = console.error.bind(console) logger.debug = DEBUG ? console.debug.bind(console) : () => {} -const blobStorage = new ApiBlobStorage(DATA_API_ROOT) +const blobStorage = new ApiBlobStorage(DATA_API_TX_ROOT, DATA_API_STATE_ROOT) +console.log(DATA_API_TX_ROOT, DATA_API_STATE_ROOT) const server = buildExecutionServer( logger, WORKERS, @@ -17,7 +18,8 @@ const server = buildExecutionServer( require.resolve('../worker.js'), NETWORK, { - dataApiRoot: DATA_API_ROOT + txApiRoot: DATA_API_TX_ROOT, + stateApiRoot: DATA_API_STATE_ROOT } ) diff --git a/src/bin/start.js b/src/bin/start.js index 6e66b69..4b3766c 100644 --- a/src/bin/start.js +++ b/src/bin/start.js @@ -7,8 +7,8 @@ const Indexer = require('../indexer') const { API, DB, NETWORK, PORT, FETCH_LIMIT, WORKERS, MATTERCLOUD_KEY, PLANARIA_TOKEN, START_HEIGHT, - MEMPOOL_EXPIRATION, ZMQ_URL, RPC_URL, DEFAULT_TRUSTLIST, DEBUG, SERVE_ONLY, DATA_SOURCE, DATA_API_ROOT, - WORKER_TRUST_SOURCE, WORKER_CACHE_TYPE, TRUST_LIST, EXECUTOR, EXECUTE_ENDPOINT + MEMPOOL_EXPIRATION, ZMQ_URL, RPC_URL, DEFAULT_TRUSTLIST, DEBUG, SERVE_ONLY, DATA_SOURCE, + WORKER_TRUST_SOURCE, WORKER_CACHE_TYPE, TRUST_LIST, EXECUTOR, EXECUTE_ENDPOINT, DATA_API_TX_ROOT, DATA_API_STATE_ROOT } = require('../config') const MatterCloud = require('../mattercloud') const Planaria = require('../planaria') @@ -70,7 +70,7 @@ let dataSource if (DATA_SOURCE === 'sqlite') { dataSource = new SqliteDatasource(DB, logger, readonly) } else if (DATA_SOURCE === 'mixed') { - const blobStorage = new ApiBlobStorage(DATA_API_ROOT) + const blobStorage = new ApiBlobStorage(DATA_API_TX_ROOT, DATA_API_STATE_ROOT) dataSource = new SqliteMixedDatasource(DB, logger, readonly, blobStorage) } else { throw new Error(`unknown datasource: ${DATA_SOURCE}. Please check "DATA_SOURCE" configuration.`) @@ -86,7 +86,7 @@ if (TRUST_LIST === 'db') { const database = new Database(dataSource, trustList, logger) const executor = EXECUTOR === 'local' - ? new Executor(NETWORK, WORKERS, database, logger, { trustSource: WORKER_TRUST_SOURCE, cacheType: WORKER_CACHE_TYPE, dataApiRoot: DATA_API_ROOT }) + ? new Executor(NETWORK, WORKERS, database, logger, { trustSource: WORKER_TRUST_SOURCE, cacheType: WORKER_CACHE_TYPE, txApiRoot: DATA_API_TX_ROOT, stateApiRoot: DATA_API_STATE_ROOT }) : new ApiExecutor(EXECUTE_ENDPOINT, trustList, NETWORK, WORKERS, logger) const indexer = new Indexer( diff --git a/src/config.js b/src/config.js index 71a12f9..2789171 100644 --- a/src/config.js +++ b/src/config.js @@ -11,27 +11,35 @@ require('dotenv').config() // ------------------------------------------------------------------------------------------------ const API = process.env.API || 'mattercloud' -const MATTERCLOUD_KEY = process.env.MATTERCLOUD_KEY -const PLANARIA_TOKEN = process.env.PLANARIA_TOKEN -const NETWORK = process.env.NETWORK || 'main' +const DATA_API_ROOT = process.env.DATA_API_ROOT || null +let DATA_API_STATE_ROOT = process.env.DATA_API_STATE_ROOT || null +let DATA_API_TX_ROOT = process.env.DATA_API_TX_ROOT || null +const DATA_SOURCE = process.env.DATA_SOURCE || 'sqlite' const DB = process.env.DB || 'run.db' -const PORT = typeof process.env.PORT !== 'undefined' ? parseInt(process.env.PORT) : 0 -const WORKERS = typeof process.env.WORKERS !== 'undefined' ? parseInt(process.env.WORKERS) : 4 +const DEBUG = process.env.DEBUG || false +const EXECUTE_ENDPOINT = process.env.EXECUTE_ENDPOINT || null +const EXECUTOR = process.env.EXECUTOR || 'local' const FETCH_LIMIT = typeof process.env.FETCH_LIMIT !== 'undefined' ? parseInt(process.env.FETCH_LIMIT) : 20 -const START_HEIGHT = process.env.START_HEIGHT || (NETWORK === 'test' ? 1382000 : 650000) -const TIMEOUT = typeof process.env.TIMEOUT !== 'undefined' ? parseInt(process.env.TIMEOUT) : 10000 +const MATTERCLOUD_KEY = process.env.MATTERCLOUD_KEY const MEMPOOL_EXPIRATION = typeof process.env.MEMPOOL_EXPIRATION !== 'undefined' ? parseInt(process.env.MEMPOOL_EXPIRATION) : 60 * 60 * 24 -const ZMQ_URL = process.env.ZMQ_URL || null +const NETWORK = process.env.NETWORK || 'main' +const PLANARIA_TOKEN = process.env.PLANARIA_TOKEN +const PORT = typeof process.env.PORT !== 'undefined' ? parseInt(process.env.PORT) : 0 const RPC_URL = process.env.RPC_URL || null -const DEBUG = process.env.DEBUG || false const SERVE_ONLY = process.env.SERVE_ONLY || false -const DATA_SOURCE = process.env.DATA_SOURCE || 'sqlite' -const DATA_API_ROOT = process.env.DATA_API_ROOT || null -const WORKER_TRUST_SOURCE = process.env.WORKER_TRUST_SOURCE || 'db' -const WORKER_CACHE_TYPE = process.env.WORKER_CACHE_TYPE || 'parent' +const START_HEIGHT = process.env.START_HEIGHT || (NETWORK === 'test' ? 1382000 : 650000) +const TIMEOUT = typeof process.env.TIMEOUT !== 'undefined' ? parseInt(process.env.TIMEOUT) : 10000 const TRUST_LIST = process.env.TRUST_LIST || 'db' -const EXECUTOR = process.env.EXECUTOR || 'local' -const EXECUTE_ENDPOINT = process.env.EXECUTE_ENDPOINT || null +const WORKERS = typeof process.env.WORKERS !== 'undefined' ? parseInt(process.env.WORKERS) : 4 +const WORKER_CACHE_TYPE = process.env.WORKER_CACHE_TYPE || 'parent' +const WORKER_TRUST_SOURCE = process.env.WORKER_TRUST_SOURCE || 'db' +const ZMQ_URL = process.env.ZMQ_URL || null + +if (!DATA_API_TX_ROOT && !DATA_API_STATE_ROOT && DATA_API_ROOT) { + console.log() + DATA_API_TX_ROOT = `${DATA_API_ROOT}/tx` + DATA_API_STATE_ROOT = `${DATA_API_ROOT}/state` +} require('axios').default.defaults.timeout = TIMEOUT @@ -109,25 +117,26 @@ const DEFAULT_TRUSTLIST = ENV_VAR_DEFAULT_TRUSTLIST || [ module.exports = { API, - MATTERCLOUD_KEY, - PLANARIA_TOKEN, - NETWORK, + DATA_API_STATE_ROOT, + DATA_API_TX_ROOT, + DATA_SOURCE, DB, - PORT, - WORKERS, + DEBUG, + DEFAULT_TRUSTLIST, + EXECUTE_ENDPOINT, + EXECUTOR, FETCH_LIMIT, - START_HEIGHT, + MATTERCLOUD_KEY, MEMPOOL_EXPIRATION, - DEFAULT_TRUSTLIST, - ZMQ_URL, + NETWORK, + PLANARIA_TOKEN, + PORT, RPC_URL, - DEBUG, SERVE_ONLY, - DATA_SOURCE, - DATA_API_ROOT, - WORKER_TRUST_SOURCE, - WORKER_CACHE_TYPE, + START_HEIGHT, TRUST_LIST, - EXECUTOR, - EXECUTE_ENDPOINT + WORKERS, + WORKER_CACHE_TYPE, + WORKER_TRUST_SOURCE, + ZMQ_URL } diff --git a/src/data-sources/api-blob-storage.js b/src/data-sources/api-blob-storage.js index 187d879..b94d126 100644 --- a/src/data-sources/api-blob-storage.js +++ b/src/data-sources/api-blob-storage.js @@ -1,12 +1,13 @@ const fetch = require('node-fetch') class ApiBlobStorage { - constructor (baseUri) { - this.baseApiUrl = baseUri + constructor (baseTxUrl, baseStateUrl) { + this.txUrl = baseTxUrl + this.stateUrl = baseStateUrl } async pushJigState (location, stateObject) { - const result = await fetch(`${this.baseApiUrl}/state`, { + const result = await fetch(this.stateUrl, { method: 'POST', body: JSON.stringify({ location, @@ -23,7 +24,7 @@ class ApiBlobStorage { } async pullJigState (location) { - const result = await fetch(`${this.baseApiUrl}/state/${encodeURIComponent(location)}`) + const result = await fetch(`${this.stateUrl}/${encodeURIComponent(location)}`) if (result.status === 404) { return null } @@ -37,7 +38,7 @@ class ApiBlobStorage { } async pullTx (txid, _ifNone) { - const result = await fetch(`${this.baseApiUrl}/rawtx/${txid}`) + const result = await fetch(`${this.txUrl}/${txid}`) return result.buffer() } diff --git a/src/execution/executor.js b/src/execution/executor.js index 4546464..10b632a 100644 --- a/src/execution/executor.js +++ b/src/execution/executor.js @@ -19,7 +19,9 @@ class Executor { this.logger = logger this.workerOpts = { cacheType: opts.cacheType || 'parentConnection', - dataApiRoot: opts.dataApiRoot || null + dataApiRoot: opts.dataApiRoot || null, + txApiRoot: opts.txApiRoot || null, + stateApiRoot: opts.stateApiRoot || null } this.onIndexed = null @@ -36,7 +38,7 @@ class Executor { this.logger.debug('Starting worker', i) const path = require.resolve('../worker.js') - + console.log(this.workerOpts) const worker = new Worker(path, { workerData: { id: i, network: this.network, ...this.workerOpts } }) worker.id = i diff --git a/src/worker.js b/src/worker.js index d752f7c..117472e 100644 --- a/src/worker.js +++ b/src/worker.js @@ -18,9 +18,11 @@ const { ApiBlobStorage } = require('./data-sources/api-blob-storage') const network = workerData.network const cacheType = workerData.cacheType -const blobApiRoot = workerData.dataApiRoot +const txApiRoot = workerData.txApiRoot +const stateApiRoot = workerData.stateApiRoot -if (cacheType === 'direct' && !blobApiRoot) { +if (cacheType === 'direct' && (!txApiRoot || !stateApiRoot)) { + console.log(txApiRoot, stateApiRoot) throw new Error('missing api root for direct cache') } @@ -131,7 +133,7 @@ async function execute (txid, hex, trustlist) { console.log = function () {} console.log() if (cacheType === 'direct') { - const bs = new ApiBlobStorage(blobApiRoot) + const bs = new ApiBlobStorage(txApiRoot, stateApiRoot) run.cache = new DirectCache(bs) } else { run.cache = new Cache() From ff75c97cc4aab2fc21a378660159dedfd879e74a Mon Sep 17 00:00:00 2001 From: Miguel Duarte Date: Tue, 1 Feb 2022 15:03:07 -0300 Subject: [PATCH 53/82] fixed errors --- src/execution/api-executor.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/execution/api-executor.js b/src/execution/api-executor.js index 69a6605..aac8727 100644 --- a/src/execution/api-executor.js +++ b/src/execution/api-executor.js @@ -62,7 +62,7 @@ class ApiExecutor { const { response } = json await this.onIndexed(txid, response) } else { - await this.onExecuteFailed(txid, json.errors, false) + await this.onExecuteFailed(txid, json.error && json.error.message, false) } } catch (e) { await this.onExecuteFailed(txid, 'execution error', true) From ce26cb60e698d37b743178ef07769beebd95ff81 Mon Sep 17 00:00:00 2001 From: hojarasca Date: Tue, 1 Feb 2022 16:38:02 -0300 Subject: [PATCH 54/82] exporting everything on index.js --- src/data-sources/index.js | 11 +++++++++++ src/execution/index.js | 9 +++++++++ src/http/index.js | 11 +++++++++++ src/index.js | 18 ++++++++++++++++-- 4 files changed, 47 insertions(+), 2 deletions(-) create mode 100644 src/data-sources/index.js create mode 100644 src/execution/index.js create mode 100644 src/http/index.js diff --git a/src/data-sources/index.js b/src/data-sources/index.js new file mode 100644 index 0000000..baa5aab --- /dev/null +++ b/src/data-sources/index.js @@ -0,0 +1,11 @@ +const { ApiBlobStorage } = require('./api-blob-storage') +const { MemoryBlobStorage } = require('./memory-blob-storage') +const { SqliteDatasource } = require('./sqlite-datasource') +const { SqliteMixedDatasource } = require('./sqlite-mixed-datasource') + +module.exports = { + ApiBlobStorage, + MemoryBlobStorage, + SqliteDatasource, + SqliteMixedDatasource +} diff --git a/src/execution/index.js b/src/execution/index.js new file mode 100644 index 0000000..ff31b9e --- /dev/null +++ b/src/execution/index.js @@ -0,0 +1,9 @@ +const { ApiExecutor } = require('./api-executor') +const { ExecutionError } = require('./execution-error') +const { Executor } = require('./executor') + +module.exports = { + ApiExecutor, + ExecutionError, + Executor +} diff --git a/src/http/index.js b/src/http/index.js new file mode 100644 index 0000000..f28d301 --- /dev/null +++ b/src/http/index.js @@ -0,0 +1,11 @@ +const { ApiError } = require('./api-error') +const { ApiServer } = require('./api-server') +const { buildExecutionServer } = require('./build-execution-server') +const { buildMainServer } = require('./build-main-server') + +module.exports = { + ApiError, + ApiServer, + buildExecutionServer, + buildMainServer +} diff --git a/src/index.js b/src/index.js index 9e587d0..9a39c87 100644 --- a/src/index.js +++ b/src/index.js @@ -7,25 +7,39 @@ const BitcoinNodeConnection = require('./bitcoin-node-connection') const BitcoinRpc = require('./bitcoin-rpc') const BitcoinZmq = require('./bitcoin-zmq') +const Bus = require('./bus') +const Crawler = require('./crawler') const Database = require('./database') +const Downloader = require('./downloader') const Indexer = require('./indexer') const MatterCloud = require('./mattercloud') const Planaria = require('./planaria') const RunConnectFetcher = require('./run-connect') const config = require('./config') +const { Clock } = require('./clock') const { SqliteDatasource } = require('./data-sources/sqlite-datasource') const { SqliteMixedDatasource } = require('./data-sources/sqlite-mixed-datasource') +const dataSources = require('./data-sources') +const execution = require('./execution') +const http = require('./http') module.exports = { - config, BitcoinNodeConnection, BitcoinRpc, BitcoinZmq, + Bus, + Clock, + Crawler, Database, + Downloader, Indexer, MatterCloud, Planaria, RunConnectFetcher, SqliteDatasource, - SqliteMixedDatasource + SqliteMixedDatasource, + config, + dataSources, + execution, + http } From 55bb3cd984aa263bf739caf2e65144b610960b8d Mon Sep 17 00:00:00 2001 From: hojarasca Date: Tue, 1 Feb 2022 17:09:41 -0300 Subject: [PATCH 55/82] Added a little bit of readme --- README.md | 66 +++++++++++++++++++++++++++++++++--------------- src/bin/start.js | 4 +-- src/config.js | 2 -- 3 files changed, 47 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 7bada4e..bdae3a9 100644 --- a/README.md +++ b/README.md @@ -52,28 +52,52 @@ You may also want to run additional instance of Run-DB in `SERVE_ONLY` mode. Tha The same approach taken for servers can be used to improve performance of client `Run` instances. You should expose your Run-DB endpoints on a public or private domain rather than connect to `localhost`. If your client connections are not authenticated, be sure to only expose the GET endpoints and never the POST or DELETE endpoints, and use HTTPS to prevent MITM attacks. +## Blob storage + +Run-db needs to handle multiple kinds of raw data in order to work. There is 2 ways to manage that data +at the moment: +- Save them inside the main db. +- Save them externally using a data api. + +The blob storage implementation can be configured using env variables. + +## Use execution server + +Run-db allows to separate the execution of run transactions into different machines. The way to do that is trough +execution servers. The execution server is a standalone process used to execute run transactions. +The main process communicates with the execution server using http. + +In order to run the execution server it's required to use an external blob storage. + ## Configuration Create a .env file or set the following environment variables before running to configure the DB. -| Name | Description | Default | -| ---- | ----------- | ------- | -| **API**| mattercloud, planaria, bitcoin-node, run, or none | mattercloud -| **MATTERCLOUD_KEY** | Mattercloud API key | undefined -| **PLANARIA_TOKEN** | Planaria API key | undefined -| **ZMQ_URL** | Only for bitcoin-node. ZMQ tcp url | null -| **RPC_URL** | Only for bitcoin-node. bitcoin RPC http url | null -| **NETWORK** | Bitcoin network (main or test) | main -| **DB** | Database file | run.db -| **PORT** | Port used for the REST server | randomly generated -| **WORKERS** | Number of threads used to index | 4 -| **FETCH_LIMIT** | Number of parallel downloads | 20 -| **START_HEIGHT** | Block height to start indexing | block shortly before sep 2020 -| **TIMEOUT** | Network timeout in milliseconds | 10000 -| **MEMPOOL_EXPIRATION** | Seconds until transactions are removed from the mempool | 86400 -| **DEFAULT_TRUSTLIST** | Comma-separated values of trusted txids | predefined trustlist -| **SERVE_ONLY** | Whether to only serve data and not index transactions | false - +| Name | Description | Default | +|-------------------------|-------------------------------------------------------------------------|-------------------------------| +| **API** | mattercloud, planaria, bitcoin-node, run, or none | mattercloud | +| **MATTERCLOUD_KEY** | Mattercloud API key | undefined | +| **PLANARIA_TOKEN** | Planaria API key | undefined | +| **ZMQ_URL** | Only for bitcoin-node. ZMQ tcp url | null | +| **RPC_URL** | Only for bitcoin-node. bitcoin RPC http url | null | +| **NETWORK** | Bitcoin network (main or test) | main | +| **DB** | Database file | run.db | +| **PORT** | Port used for the REST server | randomly generated | +| **WORKERS** | Number of threads used to index | 4 | +| **FETCH_LIMIT** | Number of parallel downloads | 20 | +| **START_HEIGHT** | Block height to start indexing | block shortly before sep 2020 | +| **TIMEOUT** | Network timeout in milliseconds | 10000 | +| **MEMPOOL_EXPIRATION** | Seconds until transactions are removed from the mempool | 86400 | +| **DEFAULT_TRUSTLIST** | Comma-separated values of trusted txids | predefined trustlist | +| **SERVE_ONLY** | Whether to only serve data and not index transactions | false | +| **DATA_SOURCE** | Blob storage implementaiton. Either `sqlite` or `mixed` | sqlite | +| **DATA_API_ROOT** | If present this value is used as based for all blob storage requests | null | +| **DATA_API_TX_ROOT** | Base path for tx api for blob storage | null | +| **DATA_API_STATE_ROOT** | Base path for state api for blob storage | null | +| **EXECUTE_ENDPOINT** | Endpoint where external execution servers can be reached | null | +| **EXECUTOR** | Executor implementation. Valid values: `local` or `api` | 'local' | +| **WORKER_CACHE_TYPE** | 'parent' or 'direct'. Defines how workers get and save generated state. | 'parent' | +| **TRUST_LIST** | Trust list implementation. `all` or `db` | `db` | ### Connecting with a bitcoin node During development is useful to connect to a local node. In order @@ -197,8 +221,8 @@ Stores spend information about transaction outputs. | Column | Type | Description | | ------ | ---- | ----------- | -| location | TEXT | \_o\ string describing an output -| spend_txid| TXID | Hex txid that spent this output, or `NULL` if unspent +| location | TEXT | \_o\ string describing an output| +| spend_txid| TXID | Hex txid that spent this output, or `NULL` if unspent| #### deps @@ -242,5 +266,5 @@ Stores the crawled block tip height and hash for data in the database. | Column | Type | Description | | ------ | ---- | ----------- | -| key | TEXT | 'height' or 'hash' +| key | TEXT | 'height' or 'hash'| | value | TEXT | String value for the key | diff --git a/src/bin/start.js b/src/bin/start.js index 4b3766c..dac7429 100644 --- a/src/bin/start.js +++ b/src/bin/start.js @@ -8,7 +8,7 @@ const Indexer = require('../indexer') const { API, DB, NETWORK, PORT, FETCH_LIMIT, WORKERS, MATTERCLOUD_KEY, PLANARIA_TOKEN, START_HEIGHT, MEMPOOL_EXPIRATION, ZMQ_URL, RPC_URL, DEFAULT_TRUSTLIST, DEBUG, SERVE_ONLY, DATA_SOURCE, - WORKER_TRUST_SOURCE, WORKER_CACHE_TYPE, TRUST_LIST, EXECUTOR, EXECUTE_ENDPOINT, DATA_API_TX_ROOT, DATA_API_STATE_ROOT + WORKER_CACHE_TYPE, TRUST_LIST, EXECUTOR, EXECUTE_ENDPOINT, DATA_API_TX_ROOT, DATA_API_STATE_ROOT } = require('../config') const MatterCloud = require('../mattercloud') const Planaria = require('../planaria') @@ -86,7 +86,7 @@ if (TRUST_LIST === 'db') { const database = new Database(dataSource, trustList, logger) const executor = EXECUTOR === 'local' - ? new Executor(NETWORK, WORKERS, database, logger, { trustSource: WORKER_TRUST_SOURCE, cacheType: WORKER_CACHE_TYPE, txApiRoot: DATA_API_TX_ROOT, stateApiRoot: DATA_API_STATE_ROOT }) + ? new Executor(NETWORK, WORKERS, database, logger, { cacheType: WORKER_CACHE_TYPE, txApiRoot: DATA_API_TX_ROOT, stateApiRoot: DATA_API_STATE_ROOT }) : new ApiExecutor(EXECUTE_ENDPOINT, trustList, NETWORK, WORKERS, logger) const indexer = new Indexer( diff --git a/src/config.js b/src/config.js index 2789171..6aea43f 100644 --- a/src/config.js +++ b/src/config.js @@ -32,7 +32,6 @@ const TIMEOUT = typeof process.env.TIMEOUT !== 'undefined' ? parseInt(process.en const TRUST_LIST = process.env.TRUST_LIST || 'db' const WORKERS = typeof process.env.WORKERS !== 'undefined' ? parseInt(process.env.WORKERS) : 4 const WORKER_CACHE_TYPE = process.env.WORKER_CACHE_TYPE || 'parent' -const WORKER_TRUST_SOURCE = process.env.WORKER_TRUST_SOURCE || 'db' const ZMQ_URL = process.env.ZMQ_URL || null if (!DATA_API_TX_ROOT && !DATA_API_STATE_ROOT && DATA_API_ROOT) { @@ -137,6 +136,5 @@ module.exports = { TRUST_LIST, WORKERS, WORKER_CACHE_TYPE, - WORKER_TRUST_SOURCE, ZMQ_URL } From 9fb992466a77d5b45d201283928f51bea8047b5b Mon Sep 17 00:00:00 2001 From: hojarasca Date: Tue, 1 Feb 2022 17:15:42 -0300 Subject: [PATCH 56/82] Improved readme tables --- README.md | 80 +++++++++++++++++++++++++++---------------------------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index bdae3a9..27de498 100644 --- a/README.md +++ b/README.md @@ -192,79 +192,79 @@ There are currently 8 tables updated by Run-DB. Stores jig and code states at output locations or destroyed locations. -| Column | Type | Description | -| ------ | ---- | ----------- | -| location | TEXT | Jig or code location | -| state | TEXT | JSON string describing the object state | -| class | TEXT | Contract origin if this state is a jig | +| Column | Type | Description | +|------------| ---- |-------------------------------------------------------| +| location | TEXT | Jig or code location | +| state | TEXT | JSON string describing the object state | +| class | TEXT | Contract origin if this state is a jig | | scripthash | TEXT | Hex string of the reversed sha256 of the owner script | -| lock | TEXT | Lock class origin if this state has a custom lock | +| lock | TEXT | Lock class origin if this state has a custom lock | #### tx Stores all transactions known by Run-DB and their indexing state. -| Column | Type | Description | -| ------ | ---- | ----------- | -| txid | TEXT | Hex string for the transaction hash | -| height | INTEGER | Block height for this transaction, or `-1` for mempool, or `NULL` for unknown | -| time | INTEGER | Transaction or bock time in seconds since the unix epoch | -| bytes | BLOB | Raw transaction data, or `NULL` if not downloaded | -| has_code | INTEGER | `1` if this transaction deployed or upgraded code and requires trust, `0` otherwise | -| executable | INTEGER | `1` if this transaction is a valid RUN transaction, `0` otherwise | -| executed | INTEGER | `1` if this transaction was executed, even if it failed, `0` otherwise | -| indexed | INTEGER | `1` if this transaction's jig states were calculated successfully, `0` otherwise | +| Column | Type | Description | +|------------|----------|-------------------------------------------------------------------------------------| +| txid | TEXT | Hex string for the transaction hash | +| height | INTEGER | Block height for this transaction, or `-1` for mempool, or `NULL` for unknown | +| time | INTEGER | Transaction or bock time in seconds since the unix epoch | +| bytes | BLOB | Raw transaction data, or `NULL` if not downloaded | +| has_code | INTEGER | `1` if this transaction deployed or upgraded code and requires trust, `0` otherwise | +| executable | INTEGER | `1` if this transaction is a valid RUN transaction, `0` otherwise | +| executed | INTEGER | `1` if this transaction was executed, even if it failed, `0` otherwise | +| indexed | INTEGER | `1` if this transaction's jig states were calculated successfully, `0` otherwise | #### spends Stores spend information about transaction outputs. -| Column | Type | Description | -| ------ | ---- | ----------- | -| location | TEXT | \_o\ string describing an output| -| spend_txid| TXID | Hex txid that spent this output, or `NULL` if unspent| +| Column | Type | Description | +|------------| ---- |--------------------------------------------------------| +| location | TEXT | \_o\ string describing an output | +| spend_txid | TXID | Hex txid that spent this output, or `NULL` if unspent | #### deps Stores the transaction needed to load a RUN transaction. -| Column | Type | Description | -| ------ | ---- | ----------- | -| up | TEXT | A transaction ID in hex | -| down | TEXT | Hex txid for a transaction that depends on `up` | +| Column | Type | Description | +|--------| ---- |-------------------------------------------------| +| up | TEXT | A transaction ID in hex | +| down | TEXT | Hex txid for a transaction that depends on `up` | #### berry Stores berry states for third-party protocol data. -| Column | Type | Description | -| ------ | ---- | ----------- | -| location | TEXT | Berry location without the &hash query param | -| state | TEXT | JSON string describing the object state | +| Column | Type | Description | +|-----------| ---- |----------------------------------------------| +| location | TEXT | Berry location without the &hash query param | +| state | TEXT | JSON string describing the object state | #### trust Stores the transactions which have been trusted and whose code will be executed. -| Column | Type | Description | -| ------ | ---- | ----------- | -| txid | TEXT | Hex string txid | -| value | INTEGER | `1` if trusted, `0` if untrusted | +| Column | Type | Description | +|--------|---------|----------------------------------| +| txid | TEXT | Hex string txid | +| value | INTEGER | `1` if trusted, `0` if untrusted | #### ban Stores the transactions which have been blacklisted. -| Column | Type | Description | -| ------ | ---- | ----------- | -| txid | TEXT | Hex string txid | -| value | INTEGER | `1` if blacklisted, `0` otherwise | +| Column | Type | Description | +|---------|---------|-----------------------------------| +| txid | TEXT | Hex string txid | +| value | INTEGER | `1` if blacklisted, `0` otherwise | #### crawl Stores the crawled block tip height and hash for data in the database. -| Column | Type | Description | -| ------ | ---- | ----------- | -| key | TEXT | 'height' or 'hash'| -| value | TEXT | String value for the key | +| Column | Type | Description | +|--------|-------|--------------------------| +| key | TEXT | 'height' or 'hash' | +| value | TEXT | String value for the key | From 1ea332de54f2a4f28613d67c542799ee25906bd7 Mon Sep 17 00:00:00 2001 From: hojarasca Date: Tue, 1 Feb 2022 17:20:23 -0300 Subject: [PATCH 57/82] 1.0.32-0 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index e73b7ab..0dcef44 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "run-db", - "version": "1.0.28", + "version": "1.0.32-0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "run-db", - "version": "1.0.28", + "version": "1.0.32-0", "license": "MIT", "dependencies": { "abort-controller": "^3.0.0", diff --git a/package.json b/package.json index 968dc06..f8fb6b9 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "run-db", - "version": "1.0.31", + "version": "1.0.32-0", "description": "A local database that indexes jig states from RUN transactions", "keywords": [ "run", From 619b621f2161d1e5482d3493f8642f0754e23a00 Mon Sep 17 00:00:00 2001 From: hojarasca Date: Tue, 1 Feb 2022 17:21:19 -0300 Subject: [PATCH 58/82] bump version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index f8fb6b9..9aedcfe 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "run-db", - "version": "1.0.32-0", + "version": "1.1.0-beta.1", "description": "A local database that indexes jig states from RUN transactions", "keywords": [ "run", From 2aac2d3b62ea35ebe2421ab5df529b6aa50771b7 Mon Sep 17 00:00:00 2001 From: Miguel Duarte Date: Thu, 3 Feb 2022 09:53:06 -0300 Subject: [PATCH 59/82] better export things. --- src/bin/start.js | 2 +- src/execution/api-executor.js | 2 +- src/index.js | 4 +++- src/trust-list/index.js | 9 +++++++++ 4 files changed, 14 insertions(+), 3 deletions(-) create mode 100644 src/trust-list/index.js diff --git a/src/bin/start.js b/src/bin/start.js index dac7429..3a638fd 100644 --- a/src/bin/start.js +++ b/src/bin/start.js @@ -24,7 +24,7 @@ const { DbTrustList } = require('../trust-list/db-trust-list') const { TrustAllTrustList } = require('../trust-list/trust-all-trust-list') const { buildMainServer } = require('../http/build-main-server') const Executor = require('../execution/executor') -const ApiExecutor = require('../execution/api-executor') +const { ApiExecutor } = require('../execution/api-executor') // ------------------------------------------------------------------------------------------------ // Globals diff --git a/src/execution/api-executor.js b/src/execution/api-executor.js index aac8727..9a706e6 100644 --- a/src/execution/api-executor.js +++ b/src/execution/api-executor.js @@ -75,4 +75,4 @@ class ApiExecutor { // ------------------------------------------------------------------------------------------------ -module.exports = ApiExecutor +module.exports = { ApiExecutor } diff --git a/src/index.js b/src/index.js index 9a39c87..e40c51a 100644 --- a/src/index.js +++ b/src/index.js @@ -22,6 +22,7 @@ const { SqliteMixedDatasource } = require('./data-sources/sqlite-mixed-datasourc const dataSources = require('./data-sources') const execution = require('./execution') const http = require('./http') +const trustList = require('./trust-list') module.exports = { BitcoinNodeConnection, @@ -41,5 +42,6 @@ module.exports = { config, dataSources, execution, - http + http, + trustList } diff --git a/src/trust-list/index.js b/src/trust-list/index.js new file mode 100644 index 0000000..b21026f --- /dev/null +++ b/src/trust-list/index.js @@ -0,0 +1,9 @@ +const { BaseTrustList } = require('./base-trust-list') +const { DbTrustList } = require('./db-trust-list') +const { TrustAllTrustList } = require('./trust-all-trust-list') + +module.exports = { + BaseTrustList, + DbTrustList, + TrustAllTrustList +} From 2571edd7311fa88a962c1618c959f887b05478c4 Mon Sep 17 00:00:00 2001 From: hojarasca Date: Thu, 3 Feb 2022 11:59:56 -0300 Subject: [PATCH 60/82] remove non used var. --- src/bin/start-execution-server.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bin/start-execution-server.js b/src/bin/start-execution-server.js index ac1d88c..da37308 100644 --- a/src/bin/start-execution-server.js +++ b/src/bin/start-execution-server.js @@ -1,4 +1,4 @@ -const { PORT, DEBUG, WORKERS, DATA_API_ROOT, NETWORK, DATA_API_STATE_ROOT, DATA_API_TX_ROOT } = require('../config') +const { PORT, DEBUG, WORKERS, NETWORK, DATA_API_STATE_ROOT, DATA_API_TX_ROOT } = require('../config') const { buildExecutionServer } = require('../http/build-execution-server') const { ApiBlobStorage } = require('../data-sources/api-blob-storage') From 9b4bdc214885a38409b798ab1ccfc6a908f4641d Mon Sep 17 00:00:00 2001 From: hojarasca Date: Thu, 3 Feb 2022 12:01:28 -0300 Subject: [PATCH 61/82] removed debbug lines. --- src/bin/start-execution-server.js | 1 - src/config.js | 1 - src/execution/executor.js | 1 - src/worker.js | 1 - 4 files changed, 4 deletions(-) diff --git a/src/bin/start-execution-server.js b/src/bin/start-execution-server.js index da37308..d51c50e 100644 --- a/src/bin/start-execution-server.js +++ b/src/bin/start-execution-server.js @@ -10,7 +10,6 @@ logger.debug = DEBUG ? console.debug.bind(console) : () => {} const blobStorage = new ApiBlobStorage(DATA_API_TX_ROOT, DATA_API_STATE_ROOT) -console.log(DATA_API_TX_ROOT, DATA_API_STATE_ROOT) const server = buildExecutionServer( logger, WORKERS, diff --git a/src/config.js b/src/config.js index 6aea43f..bb7cc20 100644 --- a/src/config.js +++ b/src/config.js @@ -35,7 +35,6 @@ const WORKER_CACHE_TYPE = process.env.WORKER_CACHE_TYPE || 'parent' const ZMQ_URL = process.env.ZMQ_URL || null if (!DATA_API_TX_ROOT && !DATA_API_STATE_ROOT && DATA_API_ROOT) { - console.log() DATA_API_TX_ROOT = `${DATA_API_ROOT}/tx` DATA_API_STATE_ROOT = `${DATA_API_ROOT}/state` } diff --git a/src/execution/executor.js b/src/execution/executor.js index 10b632a..797b700 100644 --- a/src/execution/executor.js +++ b/src/execution/executor.js @@ -38,7 +38,6 @@ class Executor { this.logger.debug('Starting worker', i) const path = require.resolve('../worker.js') - console.log(this.workerOpts) const worker = new Worker(path, { workerData: { id: i, network: this.network, ...this.workerOpts } }) worker.id = i diff --git a/src/worker.js b/src/worker.js index 117472e..6ec0b06 100644 --- a/src/worker.js +++ b/src/worker.js @@ -22,7 +22,6 @@ const txApiRoot = workerData.txApiRoot const stateApiRoot = workerData.stateApiRoot if (cacheType === 'direct' && (!txApiRoot || !stateApiRoot)) { - console.log(txApiRoot, stateApiRoot) throw new Error('missing api root for direct cache') } From df85af670171952f8c5098be8a6ae2477fa6eb18 Mon Sep 17 00:00:00 2001 From: hojarasca Date: Thu, 3 Feb 2022 12:31:33 -0300 Subject: [PATCH 62/82] avoid chrashing when zeromq is not available. --- package.json | 2 +- src/bitcoin-zmq.js | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 9aedcfe..dc47b02 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "run-db", - "version": "1.1.0-beta.1", + "version": "1.1.0-beta.2", "description": "A local database that indexes jig states from RUN transactions", "keywords": [ "run", diff --git a/src/bitcoin-zmq.js b/src/bitcoin-zmq.js index c455634..f85fce6 100644 --- a/src/bitcoin-zmq.js +++ b/src/bitcoin-zmq.js @@ -1,4 +1,7 @@ -const zmq = require('zeromq') +let zmq +try { + zmq = require('zeromq') +} catch (e) {} class BitcoinZmq { constructor (url) { From b8f5bbc14728da8bfe7fb22d9d1f3fd78fe12acd Mon Sep 17 00:00:00 2001 From: Miguel Duarte Date: Mon, 7 Feb 2022 11:32:36 -0300 Subject: [PATCH 63/82] added status endpoint. --- src/http/build-execution-server.js | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/http/build-execution-server.js b/src/http/build-execution-server.js index a5c1d21..2418a4d 100644 --- a/src/http/build-execution-server.js +++ b/src/http/build-execution-server.js @@ -32,6 +32,15 @@ const buildExecutionServer = (logger, count, blobStorage, workerPath, network, w } }) + server.get('/status', async (req, res) => { + res.status(200).json({ + ok: true, + available: pool.spareResourceCapacity, + active: pool.borrowed, + pending: pool.pending + }) + }) + server.post('/execute', async (req, res) => { const { txid: rawTxid, trustList } = req.body if (!Array.isArray(trustList)) { From ccfe69d77a089131558b70e4d7119b8fb7b819a4 Mon Sep 17 00:00:00 2001 From: Miguel Duarte Date: Mon, 7 Feb 2022 11:35:00 -0300 Subject: [PATCH 64/82] bump version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index dc47b02..ea9f11b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "run-db", - "version": "1.1.0-beta.2", + "version": "1.1.0-beta.3", "description": "A local database that indexes jig states from RUN transactions", "keywords": [ "run", From b67024a139cc971c172375d986c256c5eb3e20b2 Mon Sep 17 00:00:00 2001 From: Miguel Duarte Date: Fri, 11 Feb 2022 10:25:04 -0300 Subject: [PATCH 65/82] logs and external cache implementation. --- src/direct-cache.js | 51 ++++++++++++++++++++++++++++++ src/http/build-execution-server.js | 14 +++++++- src/worker.js | 43 ++----------------------- 3 files changed, 67 insertions(+), 41 deletions(-) create mode 100644 src/direct-cache.js diff --git a/src/direct-cache.js b/src/direct-cache.js new file mode 100644 index 0000000..744761b --- /dev/null +++ b/src/direct-cache.js @@ -0,0 +1,51 @@ + +const withTimeMeasure = async (label, fn, logger) => { + const start = process.hrtime.bigint() + const result = await fn() + const end = process.hrtime.bigint() + const diff = end - start + logger.log(`[time] ${label}: ${Number(diff / 1000n) / 1000}ms`) + return result +} + +class DirectCache { + constructor (blobStorage, logger) { + this.blobs = blobStorage + this.logger = logger + this.state = {} + } + + async get (key) { + const value = this.state[key] + if (value) { return value } + + const [type, identifier] = key.split('://') + if (type === 'jig' || type === 'berry') { + const jig = await withTimeMeasure(`fetch state ${identifier}`, async () => this.blobs.pullJigState(identifier), this.logger) + this.state[key] = jig + return jig + } else if (type === 'tx') { + const rawTx = await withTimeMeasure(`fetch rawtx ${identifier}`, async () => this.blobs.pullTx(identifier)) + const txHex = rawTx.toString('hex') + this.state[key] = txHex + return txHex + } else { + return null + } + } + + async set (key, value) { + const existedBefore = !!this.state[key] + this.state[key] = value + if (existedBefore) { + return null + } + + const [type, identifier] = key.split('://') + if (type === 'jig' || type === 'berry') { + await this.blobs.pushJigState(identifier, value) + } + } +} + +module.exports = { DirectCache } \ No newline at end of file diff --git a/src/http/build-execution-server.js b/src/http/build-execution-server.js index 2418a4d..41591dd 100644 --- a/src/http/build-execution-server.js +++ b/src/http/build-execution-server.js @@ -9,7 +9,14 @@ const { ExecutionError } = require('../execution/execution-error') const buildExecutionServer = (logger, count, blobStorage, workerPath, network, workerOpts = {}) => { const factory = { create: () => { - const worker = new Worker(workerPath, { workerData: { network: network, ...workerOpts, cacheType: 'direct' } }) + const worker = new Worker(workerPath, { + workerData: { + network: network, + directCachePath: require.resolve('../direct-cache.js'), + ...workerOpts, + cacheType: 'direct' + } + }) Bus.listen(worker, {}) return worker }, @@ -43,6 +50,7 @@ const buildExecutionServer = (logger, count, blobStorage, workerPath, network, w server.post('/execute', async (req, res) => { const { txid: rawTxid, trustList } = req.body + console.log(`Received tx to execute: ${rawTxid}`) if (!Array.isArray(trustList)) { throw new ApiError('wrong parameter: trustList', 'wrong-arguments', 400, { trustList }) } @@ -59,7 +67,11 @@ const buildExecutionServer = (logger, count, blobStorage, workerPath, network, w const worker = await pool.acquire() try { + console.log(`executing: ${txid}`) + const start = process.hrtime.bigint() const response = await Bus.sendRequest(worker, 'execute', [txid, hex, trustList], ExecutionError) + const end = process.hrtime.bigint() + console.log(`finished: ${txid}. ${Number((end - start) / 1000n) / 1000}ms`) pool.release(worker).catch(logger.error) res.json({ ok: true, diff --git a/src/worker.js b/src/worker.js index 6ec0b06..1d56935 100644 --- a/src/worker.js +++ b/src/worker.js @@ -20,6 +20,8 @@ const network = workerData.network const cacheType = workerData.cacheType const txApiRoot = workerData.txApiRoot const stateApiRoot = workerData.stateApiRoot +const originalConsole = console.log +const { DirectCache } = require(workerData.directCachePath || './direct-cache') if (cacheType === 'direct' && (!txApiRoot || !stateApiRoot)) { throw new Error('missing api root for direct cache') @@ -56,45 +58,6 @@ class Cache { } } -class DirectCache { - constructor (blobStorage) { - this.blobs = blobStorage - this.state = {} - } - - async get (key) { - const value = this.state[key] - if (value) { return value } - - const [type, identifier] = key.split('://') - if (type === 'jig' || type === 'berry') { - const jig = await this.blobs.pullJigState(identifier) - this.state[key] = jig - return jig - } else if (type === 'tx') { - const rawTx = await this.blobs.pullTx(identifier) - const txHex = rawTx.toString('hex') - this.state[key] = txHex - return txHex - } else { - return null - } - } - - async set (key, value) { - const existedBefore = !!this.state[key] - this.state[key] = value - if (existedBefore) { - return null - } - - const [type, identifier] = key.split('://') - if (type === 'jig' || type === 'berry') { - await this.blobs.pushJigState(identifier, value) - } - } -} - // ------------------------------------------------------------------------------------------------ // Blockchain // ------------------------------------------------------------------------------------------------ @@ -133,7 +96,7 @@ async function execute (txid, hex, trustlist) { console.log() if (cacheType === 'direct') { const bs = new ApiBlobStorage(txApiRoot, stateApiRoot) - run.cache = new DirectCache(bs) + run.cache = new DirectCache(bs, originalConsole) } else { run.cache = new Cache() } From 0a1e5c95c60b2ef45f7df16caf06a64092fd9a93 Mon Sep 17 00:00:00 2001 From: Miguel Duarte Date: Fri, 11 Feb 2022 10:25:33 -0300 Subject: [PATCH 66/82] bump version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index ea9f11b..0912079 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "run-db", - "version": "1.1.0-beta.3", + "version": "1.1.0-beta.4", "description": "A local database that indexes jig states from RUN transactions", "keywords": [ "run", From 6208c0e02219a52936af9bb5ca5ffc190c795bc6 Mon Sep 17 00:00:00 2001 From: Miguel Duarte Date: Fri, 11 Feb 2022 10:26:58 -0300 Subject: [PATCH 67/82] lint fixes. --- src/direct-cache.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/direct-cache.js b/src/direct-cache.js index 744761b..2c593cf 100644 --- a/src/direct-cache.js +++ b/src/direct-cache.js @@ -48,4 +48,4 @@ class DirectCache { } } -module.exports = { DirectCache } \ No newline at end of file +module.exports = { DirectCache } From af0d7ee293aabd1b27d096426ef19896d6326923 Mon Sep 17 00:00:00 2001 From: Miguel Duarte Date: Fri, 11 Feb 2022 12:56:10 -0300 Subject: [PATCH 68/82] Created cache provider abstraction for worker. --- src/bin/start-execution-server.js | 2 +- src/execution/executor.js | 4 +-- src/http/build-execution-server.js | 5 ++- src/index.js | 8 +++-- src/worker/cache-provider.js | 21 ++++++++++++ src/worker/direct-cache-provider.js | 14 ++++++++ src/{ => worker}/direct-cache.js | 2 +- src/worker/parent-port-cache-provider.js | 14 ++++++++ src/worker/parent-port-cache.js | 22 +++++++++++++ src/{ => worker}/worker.js | 42 +++++------------------- 10 files changed, 91 insertions(+), 43 deletions(-) create mode 100644 src/worker/cache-provider.js create mode 100644 src/worker/direct-cache-provider.js rename src/{ => worker}/direct-cache.js (95%) create mode 100644 src/worker/parent-port-cache-provider.js create mode 100644 src/worker/parent-port-cache.js rename src/{ => worker}/worker.js (82%) diff --git a/src/bin/start-execution-server.js b/src/bin/start-execution-server.js index d51c50e..679ef70 100644 --- a/src/bin/start-execution-server.js +++ b/src/bin/start-execution-server.js @@ -14,7 +14,7 @@ const server = buildExecutionServer( logger, WORKERS, blobStorage, - require.resolve('../worker.js'), + require.resolve('../worker/worker.js'), NETWORK, { txApiRoot: DATA_API_TX_ROOT, diff --git a/src/execution/executor.js b/src/execution/executor.js index 797b700..79f5a70 100644 --- a/src/execution/executor.js +++ b/src/execution/executor.js @@ -18,10 +18,10 @@ class Executor { this.database = database this.logger = logger this.workerOpts = { - cacheType: opts.cacheType || 'parentConnection', dataApiRoot: opts.dataApiRoot || null, txApiRoot: opts.txApiRoot || null, - stateApiRoot: opts.stateApiRoot || null + stateApiRoot: opts.stateApiRoot || null, + cacheProviderPath: opts.cacheProviderPath || null } this.onIndexed = null diff --git a/src/http/build-execution-server.js b/src/http/build-execution-server.js index 41591dd..71a88c6 100644 --- a/src/http/build-execution-server.js +++ b/src/http/build-execution-server.js @@ -12,9 +12,8 @@ const buildExecutionServer = (logger, count, blobStorage, workerPath, network, w const worker = new Worker(workerPath, { workerData: { network: network, - directCachePath: require.resolve('../direct-cache.js'), - ...workerOpts, - cacheType: 'direct' + cacheProviderPath: require.resolve('../worker/direct-cache-provider.js'), + ...workerOpts } }) Bus.listen(worker, {}) diff --git a/src/index.js b/src/index.js index e40c51a..e2a8da0 100644 --- a/src/index.js +++ b/src/index.js @@ -16,19 +16,21 @@ const MatterCloud = require('./mattercloud') const Planaria = require('./planaria') const RunConnectFetcher = require('./run-connect') const config = require('./config') -const { Clock } = require('./clock') -const { SqliteDatasource } = require('./data-sources/sqlite-datasource') -const { SqliteMixedDatasource } = require('./data-sources/sqlite-mixed-datasource') const dataSources = require('./data-sources') const execution = require('./execution') const http = require('./http') const trustList = require('./trust-list') +const { CacheProvider } = require('./worker/cache-provider') +const { Clock } = require('./clock') +const { SqliteDatasource } = require('./data-sources/sqlite-datasource') +const { SqliteMixedDatasource } = require('./data-sources/sqlite-mixed-datasource') module.exports = { BitcoinNodeConnection, BitcoinRpc, BitcoinZmq, Bus, + CacheProvider, Clock, Crawler, Database, diff --git a/src/worker/cache-provider.js b/src/worker/cache-provider.js new file mode 100644 index 0000000..06ccbf6 --- /dev/null +++ b/src/worker/cache-provider.js @@ -0,0 +1,21 @@ +class CacheProvider { + constructor (blobStorage, logger, opts) { + this.blob = blobStorage + this.logger = logger + this.opts = opts + } + + async setUp () { + throw new Error('subclass responsibility') + } + + async get () { + throw new Error('subclass responsibility') + } + + async tearDown () { + throw new Error('subclass responsibility') + } +} + +module.exports = { CacheProvider } diff --git a/src/worker/direct-cache-provider.js b/src/worker/direct-cache-provider.js new file mode 100644 index 0000000..b81b532 --- /dev/null +++ b/src/worker/direct-cache-provider.js @@ -0,0 +1,14 @@ +const { CacheProvider } = require('./cache-provider') +const { DirectCache } = require('./direct-cache') + +class ParentPortCacheProvider extends CacheProvider { + async setUp () {} + + async tearDown () {} + + async get () { + return new DirectCache(this.blob, this.logger) + } +} + +module.exports = ParentPortCacheProvider diff --git a/src/direct-cache.js b/src/worker/direct-cache.js similarity index 95% rename from src/direct-cache.js rename to src/worker/direct-cache.js index 2c593cf..ae7625f 100644 --- a/src/direct-cache.js +++ b/src/worker/direct-cache.js @@ -4,7 +4,7 @@ const withTimeMeasure = async (label, fn, logger) => { const result = await fn() const end = process.hrtime.bigint() const diff = end - start - logger.log(`[time] ${label}: ${Number(diff / 1000n) / 1000}ms`) + logger(`[time] ${label}: ${Number(diff / 1000n) / 1000}ms`) return result } diff --git a/src/worker/parent-port-cache-provider.js b/src/worker/parent-port-cache-provider.js new file mode 100644 index 0000000..7355ca9 --- /dev/null +++ b/src/worker/parent-port-cache-provider.js @@ -0,0 +1,14 @@ +const { CacheProvider } = require('./cache-provider') +const { ParentPortCache } = require('./parent-port-cache') + +class ParentPortCacheProvider extends CacheProvider { + async setUp () {} + + async tearDown () {} + + async get () { + return new ParentPortCache() + } +} + +module.exports = ParentPortCacheProvider diff --git a/src/worker/parent-port-cache.js b/src/worker/parent-port-cache.js new file mode 100644 index 0000000..e5305e9 --- /dev/null +++ b/src/worker/parent-port-cache.js @@ -0,0 +1,22 @@ +const Bus = require('../bus') +const { parentPort } = require('worker_threads') + +class ParentPortCache { + constructor () { + this.state = {} + } + + async get (key) { + if (key in this.state) { + return this.state[key] + } + + return await Bus.sendRequest(parentPort, 'cacheGet', [key]) + } + + async set (key, value) { + this.state[key] = value + } +} + +module.exports = { ParentPortCache } diff --git a/src/worker.js b/src/worker/worker.js similarity index 82% rename from src/worker.js rename to src/worker/worker.js index 1d56935..d452789 100644 --- a/src/worker.js +++ b/src/worker/worker.js @@ -8,9 +8,9 @@ const { parentPort, workerData } = require('worker_threads') const crypto = require('crypto') const Run = require('run-sdk') const bsv = require('bsv') -const Bus = require('./bus') -const { DEBUG } = require('./config') -const { ApiBlobStorage } = require('./data-sources/api-blob-storage') +const Bus = require('../bus') +const { DEBUG } = require('../config') +const { ApiBlobStorage } = require('../data-sources/api-blob-storage') // ------------------------------------------------------------------------------------------------ // Startup @@ -20,8 +20,8 @@ const network = workerData.network const cacheType = workerData.cacheType const txApiRoot = workerData.txApiRoot const stateApiRoot = workerData.stateApiRoot -const originalConsole = console.log -const { DirectCache } = require(workerData.directCachePath || './direct-cache') +const originalLog = console.log +const CacheProvider = require(workerData.cacheProviderPath || './parent-port-cache-provider.js') if (cacheType === 'direct' && (!txApiRoot || !stateApiRoot)) { throw new Error('missing api root for direct cache') @@ -36,28 +36,6 @@ process.on('unhandledRejection', (e) => { console.warn('Unhandled promise rejection', e) }) -// ------------------------------------------------------------------------------------------------ -// Cache -// ------------------------------------------------------------------------------------------------ - -class Cache { - constructor () { - this.state = {} - } - - async get (key) { - if (key in this.state) { - return this.state[key] - } - - return await Bus.sendRequest(parentPort, 'cacheGet', [key]) - } - - async set (key, value) { - this.state[key] = value - } -} - // ------------------------------------------------------------------------------------------------ // Blockchain // ------------------------------------------------------------------------------------------------ @@ -90,16 +68,14 @@ logger.warn = console.warn.bind(console) logger.error = console.error.bind(console) logger.debug = DEBUG ? console.debug.bind(console) : () => {} +const bs = new ApiBlobStorage(txApiRoot, stateApiRoot) +const cacheProvider = new CacheProvider(bs, originalLog) + async function execute (txid, hex, trustlist) { const back = console.log console.log = function () {} console.log() - if (cacheType === 'direct') { - const bs = new ApiBlobStorage(txApiRoot, stateApiRoot) - run.cache = new DirectCache(bs, originalConsole) - } else { - run.cache = new Cache() - } + run.cache = await cacheProvider.get() run.state = new Run.plugins.LocalState() run.blockchain = new Blockchain(txid) From bbf57d16d5234e000115abde37791b37308dfceb Mon Sep 17 00:00:00 2001 From: Miguel Duarte Date: Fri, 11 Feb 2022 12:56:28 -0300 Subject: [PATCH 69/82] bump version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 0912079..5b396b2 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "run-db", - "version": "1.1.0-beta.4", + "version": "1.1.0-beta.5", "description": "A local database that indexes jig states from RUN transactions", "keywords": [ "run", From 15c1c6a9b83582a27fcb06418d9269e80e8643e0 Mon Sep 17 00:00:00 2001 From: Miguel Duarte Date: Mon, 14 Feb 2022 09:47:02 -0300 Subject: [PATCH 70/82] added missing logger. --- src/worker/direct-cache.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/worker/direct-cache.js b/src/worker/direct-cache.js index ae7625f..c575b9c 100644 --- a/src/worker/direct-cache.js +++ b/src/worker/direct-cache.js @@ -25,7 +25,7 @@ class DirectCache { this.state[key] = jig return jig } else if (type === 'tx') { - const rawTx = await withTimeMeasure(`fetch rawtx ${identifier}`, async () => this.blobs.pullTx(identifier)) + const rawTx = await withTimeMeasure(`fetch rawtx ${identifier}`, async () => this.blobs.pullTx(identifier), this.logger) const txHex = rawTx.toString('hex') this.state[key] = txHex return txHex From f393fced7e6e62d98ba61afbcd6a4482d10a7f59 Mon Sep 17 00:00:00 2001 From: Miguel Duarte Date: Mon, 14 Feb 2022 13:31:34 -0300 Subject: [PATCH 71/82] added missing async --- package.json | 2 +- src/http/build-execution-server.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 5b396b2..498f9e8 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "run-db", - "version": "1.1.0-beta.5", + "version": "1.1.0-beta.6", "description": "A local database that indexes jig states from RUN transactions", "keywords": [ "run", diff --git a/src/http/build-execution-server.js b/src/http/build-execution-server.js index 71a88c6..3272729 100644 --- a/src/http/build-execution-server.js +++ b/src/http/build-execution-server.js @@ -8,7 +8,7 @@ const { ExecutionError } = require('../execution/execution-error') const buildExecutionServer = (logger, count, blobStorage, workerPath, network, workerOpts = {}) => { const factory = { - create: () => { + create: async () => { const worker = new Worker(workerPath, { workerData: { network: network, From 141ce26de564bb0f251f4c16980deed31d547792 Mon Sep 17 00:00:00 2001 From: Miguel Duarte Date: Mon, 14 Feb 2022 15:34:38 -0300 Subject: [PATCH 72/82] better logging. --- src/http/build-execution-server.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/http/build-execution-server.js b/src/http/build-execution-server.js index 3272729..ee210f4 100644 --- a/src/http/build-execution-server.js +++ b/src/http/build-execution-server.js @@ -49,7 +49,7 @@ const buildExecutionServer = (logger, count, blobStorage, workerPath, network, w server.post('/execute', async (req, res) => { const { txid: rawTxid, trustList } = req.body - console.log(`Received tx to execute: ${rawTxid}`) + logger.info(`Received tx to execute: ${rawTxid}`) if (!Array.isArray(trustList)) { throw new ApiError('wrong parameter: trustList', 'wrong-arguments', 400, { trustList }) } @@ -66,11 +66,11 @@ const buildExecutionServer = (logger, count, blobStorage, workerPath, network, w const worker = await pool.acquire() try { - console.log(`executing: ${txid}`) + logger.info(`executing: ${txid}`) const start = process.hrtime.bigint() const response = await Bus.sendRequest(worker, 'execute', [txid, hex, trustList], ExecutionError) const end = process.hrtime.bigint() - console.log(`finished: ${txid}. ${Number((end - start) / 1000n) / 1000}ms`) + logger.info(`finished: ${txid}. ${Number((end - start) / 1000n) / 1000}ms`) pool.release(worker).catch(logger.error) res.json({ ok: true, @@ -78,6 +78,7 @@ const buildExecutionServer = (logger, count, blobStorage, workerPath, network, w response }) } catch (e) { + logger.info(`failure executing tx ${txid}: ${e.message}`) pool.destroy(worker).catch(logger.error) const error = e instanceof ExecutionError ? { From 7dde6ee8792d8447c9f45a49f612b0270e0599cc Mon Sep 17 00:00:00 2001 From: Miguel Duarte Date: Mon, 14 Feb 2022 15:35:14 -0300 Subject: [PATCH 73/82] bump version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 498f9e8..bdf9137 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "run-db", - "version": "1.1.0-beta.6", + "version": "1.1.0-beta.7", "description": "A local database that indexes jig states from RUN transactions", "keywords": [ "run", From ebaeb5328d053df561be5256c24fedbb42d8841f Mon Sep 17 00:00:00 2001 From: Miguel Duarte Date: Mon, 14 Feb 2022 15:59:59 -0300 Subject: [PATCH 74/82] bump version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index bdf9137..3f95a3b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "run-db", - "version": "1.1.0-beta.7", + "version": "1.1.0-beta.8", "description": "A local database that indexes jig states from RUN transactions", "keywords": [ "run", From 06f838f7a835233b91ae4c7f14d3c3b067198f4b Mon Sep 17 00:00:00 2001 From: Miguel Duarte Date: Tue, 15 Feb 2022 09:21:17 -0300 Subject: [PATCH 75/82] better error handling on execution api. --- src/execution/executor.js | 2 +- src/http/build-execution-server.js | 26 +++++++++++++------------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/execution/executor.js b/src/execution/executor.js index 79f5a70..a5e96da 100644 --- a/src/execution/executor.js +++ b/src/execution/executor.js @@ -37,7 +37,7 @@ class Executor { for (let i = 0; i < this.numWorkers; i++) { this.logger.debug('Starting worker', i) - const path = require.resolve('../worker.js') + const path = require.resolve('../worker/worker.js') const worker = new Worker(path, { workerData: { id: i, network: this.network, ...this.workerOpts } }) worker.id = i diff --git a/src/http/build-execution-server.js b/src/http/build-execution-server.js index ee210f4..138f9c9 100644 --- a/src/http/build-execution-server.js +++ b/src/http/build-execution-server.js @@ -80,21 +80,21 @@ const buildExecutionServer = (logger, count, blobStorage, workerPath, network, w } catch (e) { logger.info(`failure executing tx ${txid}: ${e.message}`) pool.destroy(worker).catch(logger.error) - const error = e instanceof ExecutionError - ? { + if (e instanceof ExecutionError) { + return res.status(200).json({ + ok: false, + error: { type: e.constructor.name, message: e.message - } - : { - type: 'Error', - message: 'unexpected error' - } - - res.json({ - ok: false, - error, - result: null - }) + }, + result: null + }) + } else { + res.status(500).send({ + type: 'Error', + message: 'unexpected error' + }) + } } }) From 4542f23eeb6b99c168832c7d9e774782c6a130fc Mon Sep 17 00:00:00 2001 From: Miguel Duarte Date: Tue, 15 Feb 2022 09:21:47 -0300 Subject: [PATCH 76/82] bump versionbump version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 3f95a3b..1da4f7d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "run-db", - "version": "1.1.0-beta.8", + "version": "1.1.0-beta.9", "description": "A local database that indexes jig states from RUN transactions", "keywords": [ "run", From 45547d4514c5263df8f1df9815a2b6974cfd290c Mon Sep 17 00:00:00 2001 From: Miguel Duarte Date: Wed, 16 Feb 2022 17:50:55 -0300 Subject: [PATCH 77/82] consider failed transactions for executability. --- package-lock.json | 4 +- src/data-sources/sqlite-datasource.js | 10 +- src/data-sources/sqlite-mixed-datasource.js | 33 ++-- test/sqlite-mixed-datasource.test.js | 163 ++++++++++++++++++++ 4 files changed, 196 insertions(+), 14 deletions(-) diff --git a/package-lock.json b/package-lock.json index 0dcef44..8c84406 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "run-db", - "version": "1.0.32-0", + "version": "1.1.0-beta.9", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "run-db", - "version": "1.0.32-0", + "version": "1.1.0-beta.9", "license": "MIT", "dependencies": { "abort-controller": "^3.0.0", diff --git a/src/data-sources/sqlite-datasource.js b/src/data-sources/sqlite-datasource.js index 1bea489..e3d3178 100644 --- a/src/data-sources/sqlite-datasource.js +++ b/src/data-sources/sqlite-datasource.js @@ -40,8 +40,12 @@ const READY_TO_EXECUTE_SQL = ` FROM tx AS tx2 JOIN deps ON deps.up = tx2.txid - WHERE deps.down = tx.txid - AND (+tx2.downloaded = 0 OR (tx2.executable = 1 AND tx2.executed = 0)) + WHERE deps.down = tx.txid AND + ( + +tx2.downloaded = 0 OR + (tx2.executable = 1 AND tx2.executed = 0) OR + (tx2.executed = 1 AND tx2.indexed = 0) + ) ) = 0 ) AS ready FROM tx @@ -649,7 +653,7 @@ class SqliteDatasource { async txidIsReadyToExecute (txid) { const row = this.isReadyToExecuteStmt.get(txid) - return row && row.ready + return !!(row && row.ready) } async checkDependenciesWereExecutedOk (txid) { diff --git a/src/data-sources/sqlite-mixed-datasource.js b/src/data-sources/sqlite-mixed-datasource.js index 403cef0..26898b7 100644 --- a/src/data-sources/sqlite-mixed-datasource.js +++ b/src/data-sources/sqlite-mixed-datasource.js @@ -21,9 +21,14 @@ const TRUSTED_AND_READY_TO_EXECUTE_SQL = ` FROM tx AS txDeps JOIN deps ON deps.up = txDeps.txid - WHERE deps.down = txOuter.txid AND - txDeps.executable = 1 AND - txDeps.indexed = 0 + WHERE deps.down = txOuter.txid AND + (( + txDeps.executable = 1 AND + txDeps.indexed = 0 + ) OR ( + txDeps.executed = 1 AND + txDeps.indexed = 0 + )) ) = 0 ) AS ready FROM tx as txOuter @@ -39,10 +44,14 @@ const READY_TO_EXECUTE_SQL = ` FROM tx AS txDeps JOIN deps ON deps.up = txDeps.txid - WHERE deps - .down = txOuter.txid AND - txDeps.executable = 1 AND - txDeps.indexed = 0 + WHERE deps.down = txOuter.txid AND + (( + txDeps.executable = 1 AND + txDeps.indexed = 0 + ) OR ( + txDeps.executed = 1 AND + txDeps.indexed = 0 + )) ) = 0 ) AS ready FROM tx as txOuter @@ -65,8 +74,14 @@ const GET_DOWNSTREAM_READY_TO_EXECUTE_SQL = ` FROM tx AS tx2 JOIN deps ON deps.up = tx2.txid - WHERE deps.down = tx.txid - AND (+tx2.downloaded = 0 OR (tx2.executable = 1 AND tx2.executed = 0)) + WHERE deps.down = tx.txid AND + (( + tx2.executable = 1 AND + tx2.indexed = 0 + ) OR ( + tx2.executed = 1 AND + tx2.indexed = 0 + )) ) = 0 ` diff --git a/test/sqlite-mixed-datasource.test.js b/test/sqlite-mixed-datasource.test.js index fec0e76..ff5c1dd 100644 --- a/test/sqlite-mixed-datasource.test.js +++ b/test/sqlite-mixed-datasource.test.js @@ -72,4 +72,167 @@ describe('SqliteMixedDataSource', () => { expect(await blobStorage.pullJigState(location, () => null)).to.eql(null) }) }) + + describe('#txidIsReadyToExecute', () => { + it('returns false when tx does not exists', async () => { + const result = await ds.txidIsReadyToExecute('doesnotexists') + expect(result).to.eql(false) + }) + + it('returns true when tx exists and has no deps when tx does not exists', async () => { + const txid = 'sometxid' + await ds.addNewTx(txid, new Date().valueOf()) + await ds.setExecutableForTx(txid, 1) + const result = await ds.txidIsReadyToExecute(txid) + expect(result).to.eql(true) + }) + + it('returns false when tx exists has no dependencies but is not executable', async () => { + const txid = 'sometxid' + await ds.addNewTx(txid, new Date().valueOf()) + await ds.setExecutableForTx(txid, 0) + const result = await ds.txidIsReadyToExecute(txid) + expect(result).to.eql(false) + }) + + it('returns true when tx exists has has dependencies and the where executed ok', async () => { + const dep = 'deptxid' + const main = 'sometxid' + + await ds.addNewTx(dep, new Date().valueOf()) + await ds.addNewTx(main, new Date().valueOf()) + await ds.addDep(dep, main) + + await ds.setExecutableForTx(dep, 1) + await ds.setExecutableForTx(dep, 1) + + await ds.setExecutableForTx(main, 1) + + const result = await ds.txidIsReadyToExecute(main) + expect(result).to.eql(false) + }) + + it('returns false when tx exists has has a dependency but the dependency was not executed yet', async () => { + const dep = 'deptxid' + const main = 'sometxid' + + await ds.addNewTx(dep, new Date().valueOf()) + await ds.addNewTx(main, new Date().valueOf()) + await ds.addDep(dep, main) + + await ds.setExecutableForTx(dep, 1) + await ds.setExecutableForTx(dep, 1) + + await ds.setExecutableForTx(main, 1) + + const result = await ds.txidIsReadyToExecute(main) + expect(result).to.eql(false) + }) + + it('returns false when tx exists has has a dependency but the dependency failed on the execution', async () => { + const dep = 'deptxid' + const main = 'sometxid' + + await ds.addNewTx(dep, new Date().valueOf()) + await ds.addNewTx(main, new Date().valueOf()) + await ds.addDep(dep, main) + + // Mark as failed + await ds.setExecutableForTx(dep, 0) + await ds.setExecutedForTx(dep, 1) + await ds.setIndexedForTx(dep, 0) + + await ds.setExecutableForTx(main, 1) + + const result = await ds.txidIsReadyToExecute(main) + expect(result).to.eql(false) + }) + + it('returns true when tx exists and several has a dependencies all ok', async () => { + const dep1 = 'deptxid1' + const dep2 = 'deptxid2' + const dep3 = 'deptxid3' + const main = 'sometxid' + + await ds.addNewTx(dep1, new Date().valueOf()) + await ds.addNewTx(dep2, new Date().valueOf()) + await ds.addNewTx(dep3, new Date().valueOf()) + await ds.addNewTx(main, new Date().valueOf()) + await ds.addDep(dep1, main) + await ds.addDep(dep2, main) + await ds.addDep(dep3, main) + + // deps are ok + await ds.setExecutableForTx(dep1, 1) + await ds.setIndexedForTx(dep1, 1) + await ds.setExecutableForTx(dep2, 1) + await ds.setIndexedForTx(dep2, 1) + await ds.setExecutableForTx(dep3, 1) + await ds.setIndexedForTx(dep3, 1) + + await ds.setExecutableForTx(main, 1) + + const result = await ds.txidIsReadyToExecute(main) + expect(result).to.eql(true) + }) + + it('returns false when tx exists and one was not indexed', async () => { + const dep1 = 'deptxid1' + const dep2 = 'deptxid2' + const dep3 = 'deptxid3' + const main = 'sometxid' + + await ds.addNewTx(dep1, new Date().valueOf()) + await ds.addNewTx(dep2, new Date().valueOf()) + await ds.addNewTx(dep3, new Date().valueOf()) + await ds.addNewTx(main, new Date().valueOf()) + await ds.addDep(dep1, main) + await ds.addDep(dep2, main) + await ds.addDep(dep3, main) + + // deps are ok + await ds.setExecutableForTx(dep1, 1) + await ds.setIndexedForTx(dep1, 1) + await ds.setExecutableForTx(dep2, 1) + await ds.setIndexedForTx(dep2, 1) + await ds.setExecutableForTx(dep3, 1) + await ds.setIndexedForTx(dep3, 0) + + await ds.setExecutableForTx(main, 1) + + const result = await ds.txidIsReadyToExecute(main) + expect(result).to.eql(false) + }) + + it('returns false when tx exists and one failed', async () => { + const dep1 = 'deptxid1' + const dep2 = 'deptxid2' + const dep3 = 'deptxid3' + const main = 'sometxid' + + await ds.addNewTx(dep1, new Date().valueOf()) + await ds.addNewTx(dep2, new Date().valueOf()) + await ds.addNewTx(dep3, new Date().valueOf()) + await ds.addNewTx(main, new Date().valueOf()) + await ds.addDep(dep1, main) + await ds.addDep(dep2, main) + await ds.addDep(dep3, main) + + // deps are ok + await ds.setExecutableForTx(dep1, 1) + await ds.setIndexedForTx(dep1, 1) + await ds.setExecutableForTx(dep2, 1) + await ds.setIndexedForTx(dep2, 1) + + // one dep failed + await ds.setExecutableForTx(dep3, 0) + await ds.setExecutedForTx(dep3, 1) + await ds.setIndexedForTx(dep3, 0) + + await ds.setExecutableForTx(main, 1) + + const result = await ds.txidIsReadyToExecute(main) + expect(result).to.eql(false) + }) + }) }) From b4fdd90829127a6f60557f8cffecab9d75b74317 Mon Sep 17 00:00:00 2001 From: Miguel Duarte Date: Thu, 17 Feb 2022 10:12:49 -0300 Subject: [PATCH 78/82] bump version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 1da4f7d..8de25bd 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "run-db", - "version": "1.1.0-beta.9", + "version": "1.1.0-beta.10", "description": "A local database that indexes jig states from RUN transactions", "keywords": [ "run", From 14871257c4e8760d904a142393e33e2370193e01 Mon Sep 17 00:00:00 2001 From: Miguel Duarte Date: Mon, 7 Mar 2022 12:23:49 -0300 Subject: [PATCH 79/82] better logging exclussion --- README.md | 56 ++++++++++++++++++++++++++++---------------- src/bin/start.js | 35 +++++++++++++++++++++++---- src/config.js | 8 +++++-- src/worker/worker.js | 22 ++++++++++++----- 4 files changed, 89 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 27de498..25a3e6f 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ Crawls the blockchain and indexes RUN state. Using Run-DB, you can self-host the State APIs that Run uses to work well. Use Run-DB to: + - Operate a State Server to improve RUN performance by pre-loading jigs - Query balances, volume, history, and other information across many users and contracts - Blacklist individual transactions and their descendants in your app @@ -23,7 +24,9 @@ Node 10+ 1. Install `npm run install` 2. Download a db snapshot: `wget https://run.network/run-db-snapshots/main/latest -O run.db` (*optional*) 3. Run `npm run start` -4. Install a trustlist: `curl -s https://api.run.network/v1/main/trust | curl -H "Content-Type: application/json" -X POST -d @- http://localhost:8000/trust` (*optional*) +4. Install a + trustlist: `curl -s https://api.run.network/v1/main/trust | curl -H "Content-Type: application/json" -X POST -d @- http://localhost:8000/trust` (* + optional*) **Note**: For testnet, you may use `test` in place of `main` in the above commands. @@ -38,24 +41,30 @@ const trust = ['state'] const run = new Run({ client, state, trust }) ``` -Client mode makes Run-DB the source of truth for your server for all jig information. RUN will not load jigs that are not in your database, and your inventory will only be populated by jig UTXOs known to your database. +Client mode makes Run-DB the source of truth for your server for all jig information. RUN will not load jigs that are +not in your database, and your inventory will only be populated by jig UTXOs known to your database. -Setting trust to `'state'` makes Run use your database for its trustlist too. This means you only have to setup trust in one place using: +Setting trust to `'state'` makes Run use your database for its trustlist too. This means you only have to setup trust in +one place using: ``` curl -X POST localhost:8000/trust/ ``` -You may also want to run additional instance of Run-DB in `SERVE_ONLY` mode. That allows you to have an writer that crawls transactions and puts data into the database, and multiple readers that serve your application servers. +You may also want to run additional instance of Run-DB in `SERVE_ONLY` mode. That allows you to have an writer that +crawls transactions and puts data into the database, and multiple readers that serve your application servers. ## Use with a Browser or Mobile Client -The same approach taken for servers can be used to improve performance of client `Run` instances. You should expose your Run-DB endpoints on a public or private domain rather than connect to `localhost`. If your client connections are not authenticated, be sure to only expose the GET endpoints and never the POST or DELETE endpoints, and use HTTPS to prevent MITM attacks. +The same approach taken for servers can be used to improve performance of client `Run` instances. You should expose your +Run-DB endpoints on a public or private domain rather than connect to `localhost`. If your client connections are not +authenticated, be sure to only expose the GET endpoints and never the POST or DELETE endpoints, and use HTTPS to prevent +MITM attacks. ## Blob storage -Run-db needs to handle multiple kinds of raw data in order to work. There is 2 ways to manage that data -at the moment: +Run-db needs to handle multiple kinds of raw data in order to work. There is 2 ways to manage that data at the moment: + - Save them inside the main db. - Save them externally using a data api. @@ -64,8 +73,8 @@ The blob storage implementation can be configured using env variables. ## Use execution server Run-db allows to separate the execution of run transactions into different machines. The way to do that is trough -execution servers. The execution server is a standalone process used to execute run transactions. -The main process communicates with the execution server using http. +execution servers. The execution server is a standalone process used to execute run transactions. The main process +communicates with the execution server using http. In order to run the execution server it's required to use an external blob storage. @@ -98,11 +107,13 @@ Create a .env file or set the following environment variables before running to | **EXECUTOR** | Executor implementation. Valid values: `local` or `api` | 'local' | | **WORKER_CACHE_TYPE** | 'parent' or 'direct'. Defines how workers get and save generated state. | 'parent' | | **TRUST_LIST** | Trust list implementation. `all` or `db` | `db` | +| **PRESERVE_STDERR** | Actually preserve stdout on jig executions. | false | +| **PRESERVE_STDOUT** | Actually preserve stderr on jig executions. | false | + ### Connecting with a bitcoin node -During development is useful to connect to a local node. In order -to do this you need to provide RUN-db with access to a bitcoin node -trough RPC and ZMQ. +During development is useful to connect to a local node. In order to do this you need to provide RUN-db with access to a +bitcoin node trough RPC and ZMQ. ``` export API="bitcoin-node" @@ -110,11 +121,10 @@ export ZMQ_URL="tcp://your-node-uri:port" export RPC_URL="http://user:password@your-node-uri:port" ``` -The only zmq message is needed is `rawtx`. ZMQ is only used to get -the new transactions in the mempool. +The only zmq message is needed is `rawtx`. ZMQ is only used to get the new transactions in the mempool. -Direct connection with the node is tested in regtest and testnet, but -it's not recommeded for production environments in mainnet at the moment. +Direct connection with the node is tested in regtest and testnet, but it's not recommeded for production environments in +mainnet at the moment. ## Endpoints @@ -123,14 +133,18 @@ it's not recommeded for production environments in mainnet at the moment. * `GET /tx/:txid` - Gets the raw transaction hex for an added transaction * `GET /time/:txid` - Gets the block or mempool time of a transaction in seconds since unix epoch * `GET /spends/:location` - Gets the spending txid for an output at a particular location -* `GET /unspent` - Gets the locations of all unspent jigs that are trusted. You may optionally pass in the following query params: `class` to filter by contract origin, `address` to filter by owner address, `pubkey` to filter by owner pubkey, `scripthash` to filter by hash of the owner script, `lock` to filter by lock class origin. +* `GET /unspent` - Gets the locations of all unspent jigs that are trusted. You may optionally pass in the following + query params: `class` to filter by contract origin, `address` to filter by owner address, `pubkey` to filter by owner + pubkey, `scripthash` to filter by hash of the owner script, `lock` to filter by lock class origin. * `GET /trust/:txid?` - Gets whether a particular txid is trusted, or the entire trust list * `GET /ban/:txid?` - Gets whether a particular txid is banned, or the entire ban list * `GET /status` - Prints status information -* `POST /trust/:txid?` - Trusts a transaction to execute its code, as well as any untrusted ancestors. To trust multiple transactions at once, you may add an array of txids in the body as application/json. +* `POST /trust/:txid?` - Trusts a transaction to execute its code, as well as any untrusted ancestors. To trust multiple + transactions at once, you may add an array of txids in the body as application/json. * `POST /ban/:txid` - Bans a transaction from being executed, and unindexes it and its descendents -* `POST /tx/:txid?` - Indexes a transaction and any ancestors. You may optionally add the raw hex data for the transaction in the body as text/plain. +* `POST /tx/:txid?` - Indexes a transaction and any ancestors. You may optionally add the raw hex data for the + transaction in the body as text/plain. * `DELETE /trust/:txid` - Removes trust for a transaction, and unindexes it and its descendents * `DELETE /ban/:txid` - Removes a transaction ban, and reindexes it and its descendents @@ -138,7 +152,9 @@ it's not recommeded for production environments in mainnet at the moment. ## Performing Custom Queries -Run-DB uses SQLite as its underlying database in [WAL](https://sqlite.org/wal.html) mode. SQLite and WAL allows multiple connections to the database so long as there is only one writer, which should be Run-DB. Alternatively, forking Run-DB to create new endpoints for your application may be simpler. +Run-DB uses SQLite as its underlying database in [WAL](https://sqlite.org/wal.html) mode. SQLite and WAL allows multiple +connections to the database so long as there is only one writer, which should be Run-DB. Alternatively, forking Run-DB +to create new endpoints for your application may be simpler. ### Example Queries diff --git a/src/bin/start.js b/src/bin/start.js index 3a638fd..c203e2f 100644 --- a/src/bin/start.js +++ b/src/bin/start.js @@ -6,9 +6,30 @@ const Indexer = require('../indexer') const { - API, DB, NETWORK, PORT, FETCH_LIMIT, WORKERS, MATTERCLOUD_KEY, PLANARIA_TOKEN, START_HEIGHT, - MEMPOOL_EXPIRATION, ZMQ_URL, RPC_URL, DEFAULT_TRUSTLIST, DEBUG, SERVE_ONLY, DATA_SOURCE, - WORKER_CACHE_TYPE, TRUST_LIST, EXECUTOR, EXECUTE_ENDPOINT, DATA_API_TX_ROOT, DATA_API_STATE_ROOT + API, + DB, + NETWORK, + PORT, + FETCH_LIMIT, + WORKERS, + MATTERCLOUD_KEY, + PLANARIA_TOKEN, + START_HEIGHT, + MEMPOOL_EXPIRATION, + ZMQ_URL, + RPC_URL, + DEFAULT_TRUSTLIST, + DEBUG, + SERVE_ONLY, + DATA_SOURCE, + WORKER_CACHE_TYPE, + TRUST_LIST, + EXECUTOR, + EXECUTE_ENDPOINT, + DATA_API_TX_ROOT, + DATA_API_STATE_ROOT, + PRESERVE_STDOUT, + PRESERVE_STDERR } = require('../config') const MatterCloud = require('../mattercloud') const Planaria = require('../planaria') @@ -86,7 +107,13 @@ if (TRUST_LIST === 'db') { const database = new Database(dataSource, trustList, logger) const executor = EXECUTOR === 'local' - ? new Executor(NETWORK, WORKERS, database, logger, { cacheType: WORKER_CACHE_TYPE, txApiRoot: DATA_API_TX_ROOT, stateApiRoot: DATA_API_STATE_ROOT }) + ? new Executor(NETWORK, WORKERS, database, logger, { + cacheType: WORKER_CACHE_TYPE, + txApiRoot: DATA_API_TX_ROOT, + stateApiRoot: DATA_API_STATE_ROOT, + preserveStdout: PRESERVE_STDOUT, + preserveStdErr: PRESERVE_STDERR + }) : new ApiExecutor(EXECUTE_ENDPOINT, trustList, NETWORK, WORKERS, logger) const indexer = new Indexer( diff --git a/src/config.js b/src/config.js index bb7cc20..db5a563 100644 --- a/src/config.js +++ b/src/config.js @@ -12,8 +12,6 @@ require('dotenv').config() const API = process.env.API || 'mattercloud' const DATA_API_ROOT = process.env.DATA_API_ROOT || null -let DATA_API_STATE_ROOT = process.env.DATA_API_STATE_ROOT || null -let DATA_API_TX_ROOT = process.env.DATA_API_TX_ROOT || null const DATA_SOURCE = process.env.DATA_SOURCE || 'sqlite' const DB = process.env.DB || 'run.db' const DEBUG = process.env.DEBUG || false @@ -25,6 +23,8 @@ const MEMPOOL_EXPIRATION = typeof process.env.MEMPOOL_EXPIRATION !== 'undefined' const NETWORK = process.env.NETWORK || 'main' const PLANARIA_TOKEN = process.env.PLANARIA_TOKEN const PORT = typeof process.env.PORT !== 'undefined' ? parseInt(process.env.PORT) : 0 +const PRESERVE_STDERR = process.env.PRESERVE_STDERR || false +const PRESERVE_STDOUT = process.env.PRESERVE_STDOUT || false const RPC_URL = process.env.RPC_URL || null const SERVE_ONLY = process.env.SERVE_ONLY || false const START_HEIGHT = process.env.START_HEIGHT || (NETWORK === 'test' ? 1382000 : 650000) @@ -33,6 +33,8 @@ const TRUST_LIST = process.env.TRUST_LIST || 'db' const WORKERS = typeof process.env.WORKERS !== 'undefined' ? parseInt(process.env.WORKERS) : 4 const WORKER_CACHE_TYPE = process.env.WORKER_CACHE_TYPE || 'parent' const ZMQ_URL = process.env.ZMQ_URL || null +let DATA_API_STATE_ROOT = process.env.DATA_API_STATE_ROOT || null +let DATA_API_TX_ROOT = process.env.DATA_API_TX_ROOT || null if (!DATA_API_TX_ROOT && !DATA_API_STATE_ROOT && DATA_API_ROOT) { DATA_API_TX_ROOT = `${DATA_API_ROOT}/tx` @@ -129,6 +131,8 @@ module.exports = { NETWORK, PLANARIA_TOKEN, PORT, + PRESERVE_STDERR, + PRESERVE_STDOUT, RPC_URL, SERVE_ONLY, START_HEIGHT, diff --git a/src/worker/worker.js b/src/worker/worker.js index d452789..e818798 100644 --- a/src/worker/worker.js +++ b/src/worker/worker.js @@ -11,6 +11,7 @@ const bsv = require('bsv') const Bus = require('../bus') const { DEBUG } = require('../config') const { ApiBlobStorage } = require('../data-sources/api-blob-storage') +const stream = require('stream') // ------------------------------------------------------------------------------------------------ // Startup @@ -20,9 +21,23 @@ const network = workerData.network const cacheType = workerData.cacheType const txApiRoot = workerData.txApiRoot const stateApiRoot = workerData.stateApiRoot +const preserveStdout = workerData.preserveStdout || false +const preserveStdErr = workerData.preserveStdErr || false const originalLog = console.log const CacheProvider = require(workerData.cacheProviderPath || './parent-port-cache-provider.js') +// Prepare console + +const outConsole = global.console +const executorStdout = preserveStdout + ? process.stdout + : new stream.Writable({ write: (_a, _b, cb) => cb() }) +const executorStdErr = preserveStdErr + ? process.stderr + : new stream.Writable({ write: (_a, _b, cb) => cb() }) +const nullConsole = new console.Console(executorStdout, executorStdErr) +global.console = nullConsole + if (cacheType === 'direct' && (!txApiRoot || !stateApiRoot)) { throw new Error('missing api root for direct cache') } @@ -33,7 +48,7 @@ Bus.listen(parentPort, { execute }) // and unhandled promise rejection error. However, it can't reproduce outside of Run-DB. // This needs investigation. Perhaps it's related to the worker thread. Perhaps something else. process.on('unhandledRejection', (e) => { - console.warn('Unhandled promise rejection', e) + outConsole.warn('Unhandled promise rejection', e) }) // ------------------------------------------------------------------------------------------------ @@ -72,9 +87,6 @@ const bs = new ApiBlobStorage(txApiRoot, stateApiRoot) const cacheProvider = new CacheProvider(bs, originalLog) async function execute (txid, hex, trustlist) { - const back = console.log - console.log = function () {} - console.log() run.cache = await cacheProvider.get() run.state = new Run.plugins.LocalState() @@ -107,8 +119,6 @@ async function execute (txid, hex, trustlist) { const commonLocks = addresses.map(([location, address]) => [location, new Run.util.CommonLock(address)]) const scripts = customLocks.concat(commonLocks).map(([location, lock]) => [location, lock.script()]) const scripthashes = scripts.map(([location, script]) => [location, scripthash(script)]) - - console.log = back return { cache, classes, locks, scripthashes } } From 076b4c8f80b4584698a60acc521e80f281e117ca Mon Sep 17 00:00:00 2001 From: Miguel Duarte Date: Wed, 9 Mar 2022 14:47:25 -0300 Subject: [PATCH 80/82] removed threading files. --- src/threading/parent-thread.js | 37 --------------------- src/threading/sub-thread.js | 61 ---------------------------------- 2 files changed, 98 deletions(-) delete mode 100644 src/threading/parent-thread.js delete mode 100644 src/threading/sub-thread.js diff --git a/src/threading/parent-thread.js b/src/threading/parent-thread.js deleted file mode 100644 index cd8e598..0000000 --- a/src/threading/parent-thread.js +++ /dev/null @@ -1,37 +0,0 @@ -const { parentPort } = require('worker_threads') - -class ParentThread { - constructor () { - this.parent = parentPort - this.parent.on('message', this._onMessage.bind(this)) - this.handlers = new Map() - } - - async _onMessage (msg) { - const { headers, body } = msg - const handler = this.handlers.get(headers.topic) - if (!handler) { - throw new Error(`unknown topic: ${headers.topic}`) - } - try { - const response = await handler(body) - this.parent.postMessage({ - headers: { - replyTo: headers.id, - type: 'response' - }, - body: response - }) - } catch (e) { - this.parent.postMessage({ - headers: { - replyTo: headers.id, - type: 'error' - }, - body: e - }) - } - } -} - -module.exports = { ParentThread } diff --git a/src/threading/sub-thread.js b/src/threading/sub-thread.js deleted file mode 100644 index 00f0f99..0000000 --- a/src/threading/sub-thread.js +++ /dev/null @@ -1,61 +0,0 @@ -const { Worker } = require('worker_threads') -const { Clock } = require('../clock') -const { nanoid } = require('nanoid') - -class TimeoutError extends Error {} - -class SubThread { - constructor (path, workerOpts = {}, clientOpts = {}) { - this.thread = new Worker(path, workerOpts) - this.thread.on('message', this._onMessage.bind(this)) - this.thread.on('exit', this._onExit.bind(this)) - this.thread.on('error', this._onError.bind(this)) - - this.pending = new Map() - this.clock = new Clock() - this.timeoutMs = clientOpts.timeoutMs || 10 * 1000 - } - - send (topic, body, opts) { - const id = nanoid() - const headers = { ...opts, id, topic } - const timeoutMs = opts.timeoutMs || this.timeoutMs - - this.thread.postMessage({ headers, body }) - return new Promise((resolve, reject) => { - const cancel = this.clock.delay(() => { - reject(new TimeoutError()) - this.pending.delete(id) - }, timeoutMs) - this.pending.set(id, { resolve, reject, timeout: cancel }) - }).finally( - (data) => { - const { timeout } = this.pending.get(id) - timeout.cancel() - this.pending.delete(id) - return data - }) - } - - async _onMessage (msg) { - const { header: { replyTo, type }, body } = msg - const { resolve, reject } = this.pending.get(replyTo) - if (type === 'response') { - resolve(body) - } else if (type === 'error') { - reject(body) - } else { - throw new Error(`unknown type: ${type}`) - } - } - - async _onExit () { - - } - - async _onError () { - - } -} - -module.exports = { SubThread } From 0c95dbcead1ff85240dd37e0fb4e25df454d43e8 Mon Sep 17 00:00:00 2001 From: Miguel Duarte Date: Wed, 9 Mar 2022 14:58:04 -0300 Subject: [PATCH 81/82] added dockerfile. --- .dockerignore | 5 + Dockerfile | 13 ++ package-lock.json | 351 +++++++++++++++++++++++++++++----------------- package.json | 2 +- 4 files changed, 242 insertions(+), 129 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..5493cc6 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,5 @@ +node_modules +*.sqlite3* +.runcache +.github +.git \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..a482d0a --- /dev/null +++ b/Dockerfile @@ -0,0 +1,13 @@ +FROM node:16-slim + +RUN mkdir /app +WORKDIR /app + +COPY package.json . +COPY package-lock.json . + +RUN npm install --production + +COPY . . + +CMD ["npm", "start"] \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 8c84406..b29dd52 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "run-db", - "version": "1.1.0-beta.9", + "version": "1.1.0-beta.10", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "run-db", - "version": "1.1.0-beta.9", + "version": "1.1.0-beta.10", "license": "MIT", "dependencies": { "abort-controller": "^3.0.0", @@ -32,7 +32,7 @@ "devDependencies": { "chai": "^4.3.4", "chai-as-promised": "^7.1.1", - "mocha": "^8.3.2", + "mocha": "^9.2.1", "nyc": "^15.1.0", "standard": "^16.0.3" }, @@ -1246,24 +1246,30 @@ } }, "node_modules/chokidar": { - "version": "3.5.1", - "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.1.tgz", - "integrity": "sha512-9+s+Od+W0VJJzawDma/gvBNQqkTiqYTWLuZoyAsivsI4AaWTCzHG06/TMjsf1cYe9Cb97UCEhjz7HvnPk2p/tw==", + "version": "3.5.3", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz", + "integrity": "sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==", "dev": true, + "funding": [ + { + "type": "individual", + "url": "https://paulmillr.com/funding/" + } + ], "dependencies": { - "anymatch": "~3.1.1", + "anymatch": "~3.1.2", "braces": "~3.0.2", - "glob-parent": "~5.1.0", + "glob-parent": "~5.1.2", "is-binary-path": "~2.1.0", "is-glob": "~4.0.1", "normalize-path": "~3.0.0", - "readdirp": "~3.5.0" + "readdirp": "~3.6.0" }, "engines": { "node": ">= 8.10.0" }, "optionalDependencies": { - "fsevents": "~2.3.1" + "fsevents": "~2.3.2" } }, "node_modules/chownr": { @@ -2659,9 +2665,9 @@ "dev": true }, "node_modules/follow-redirects": { - "version": "1.14.7", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.7.tgz", - "integrity": "sha512-+hbxoLbFMbRKDwohX8GkTataGqO6Jb7jGwpAlwgy2bIz25XtRm7KEzJM76R1WiNT5SwZkX4Y75SwBolkpmE7iQ==", + "version": "1.14.9", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.9.tgz", + "integrity": "sha512-MQDfihBQYMcyy5dhRDJUHcw7lb2Pv/TuE6xP1vyraLukNDHKbDxDNaOE3NbCAdKQApno+GPRyo1YAp89yCjK4w==", "funding": [ { "type": "individual", @@ -2744,6 +2750,7 @@ "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", "dev": true, + "hasInstallScript": true, "optional": true, "os": [ "darwin" @@ -2872,9 +2879,9 @@ "integrity": "sha1-l/tdlr/eiXMxPyDoKI75oWf6ZM4=" }, "node_modules/glob": { - "version": "7.1.6", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", - "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.0.tgz", + "integrity": "sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q==", "dev": true, "dependencies": { "fs.realpath": "^1.0.0", @@ -2886,6 +2893,9 @@ }, "engines": { "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" } }, "node_modules/glob-parent": { @@ -3423,6 +3433,18 @@ "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=", "dev": true }, + "node_modules/is-unicode-supported": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-0.1.0.tgz", + "integrity": "sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw==", + "dev": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/is-weakref": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-weakref/-/is-weakref-1.0.2.tgz", @@ -3613,9 +3635,9 @@ "dev": true }, "node_modules/js-yaml": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.0.0.tgz", - "integrity": "sha512-pqon0s+4ScYUvX30wxQi3PogGFAlUyH0awepWvwkj4jD4v+ova3RiYw8bmA6x2rDrEaj8i/oWKoRxpVNW+Re8Q==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", "dev": true, "dependencies": { "argparse": "^2.0.1" @@ -3829,15 +3851,19 @@ "dev": true }, "node_modules/log-symbols": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.0.0.tgz", - "integrity": "sha512-FN8JBzLx6CzeMrB0tg6pqlGU1wCrXW+ZXGH481kfsBqer0hToTIiHdjH4Mq8xJUbvATujKCvaREGWpGUionraA==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.1.0.tgz", + "integrity": "sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg==", "dev": true, "dependencies": { - "chalk": "^4.0.0" + "chalk": "^4.1.0", + "is-unicode-supported": "^0.1.0" }, "engines": { "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/loose-envify": { @@ -4016,33 +4042,32 @@ "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==" }, "node_modules/mocha": { - "version": "8.4.0", - "resolved": "https://registry.npmjs.org/mocha/-/mocha-8.4.0.tgz", - "integrity": "sha512-hJaO0mwDXmZS4ghXsvPVriOhsxQ7ofcpQdm8dE+jISUOKopitvnXFQmpRR7jd2K6VBG6E26gU3IAbXXGIbu4sQ==", + "version": "9.2.1", + "resolved": "https://registry.npmjs.org/mocha/-/mocha-9.2.1.tgz", + "integrity": "sha512-T7uscqjJVS46Pq1XDXyo9Uvey9gd3huT/DD9cYBb4K2Xc/vbKRPUWK067bxDQRK0yIz6Jxk73IrnimvASzBNAQ==", "dev": true, "dependencies": { "@ungap/promise-all-settled": "1.1.2", "ansi-colors": "4.1.1", "browser-stdout": "1.3.1", - "chokidar": "3.5.1", - "debug": "4.3.1", + "chokidar": "3.5.3", + "debug": "4.3.3", "diff": "5.0.0", "escape-string-regexp": "4.0.0", "find-up": "5.0.0", - "glob": "7.1.6", + "glob": "7.2.0", "growl": "1.10.5", "he": "1.2.0", - "js-yaml": "4.0.0", - "log-symbols": "4.0.0", + "js-yaml": "4.1.0", + "log-symbols": "4.1.0", "minimatch": "3.0.4", "ms": "2.1.3", - "nanoid": "3.1.20", - "serialize-javascript": "5.0.1", + "nanoid": "3.2.0", + "serialize-javascript": "6.0.0", "strip-json-comments": "3.1.1", "supports-color": "8.1.1", "which": "2.0.2", - "wide-align": "1.1.3", - "workerpool": "6.1.0", + "workerpool": "6.2.0", "yargs": "16.2.0", "yargs-parser": "20.2.4", "yargs-unparser": "2.0.0" @@ -4052,19 +4077,28 @@ "mocha": "bin/mocha" }, "engines": { - "node": ">= 10.12.0" + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/mochajs" } }, "node_modules/mocha/node_modules/debug": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz", - "integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==", + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.3.tgz", + "integrity": "sha512-/zxw5+vh1Tfv+4Qn7a5nsbcJKPaSvCDhojn6FEl9vupwK2VCSDtEiEtqr8DFtzYFOdz63LBkxec7DYuc2jon6Q==", "dev": true, "dependencies": { "ms": "2.1.2" }, "engines": { "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } } }, "node_modules/mocha/node_modules/debug/node_modules/ms": { @@ -4080,9 +4114,9 @@ "dev": true }, "node_modules/mocha/node_modules/nanoid": { - "version": "3.1.20", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.1.20.tgz", - "integrity": "sha512-a1cQNyczgKbLX9jwbS/+d7W8fX/RfgYR7lVWwWOGIPNgK2m0MWvrGF6/m4kk6U3QcFMnZf3RIhL0v2Jgh/0Uxw==", + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.2.0.tgz", + "integrity": "sha512-fmsZYa9lpn69Ad5eDn7FMcnnSR+8R34W9qJEijxYhTbfOWzr22n1QxCMzXLK+ODyW2973V3Fux959iQoUxzUIA==", "dev": true, "bin": { "nanoid": "bin/nanoid.cjs" @@ -4135,9 +4169,9 @@ "optional": true }, "node_modules/nanoid": { - "version": "3.1.30", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.1.30.tgz", - "integrity": "sha512-zJpuPDwOv8D2zq2WRoMe1HsfZthVewpel9CAvTfc/2mBD1uUT/agc5f7GHGWXlYkFvi1mVxe4IjvP2HNrop7nQ==", + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.1.tgz", + "integrity": "sha512-n6Vs/3KGyxPQd6uO0eH4Bv0ojGSUvuLlIHtC3Y0kEO23YRge8H9x1GCzLn28YX0H66pMkxuaeESFq4tKISKwdw==", "bin": { "nanoid": "bin/nanoid.cjs" }, @@ -4173,11 +4207,22 @@ } }, "node_modules/node-fetch": { - "version": "2.6.1", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", - "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==", + "version": "2.6.7", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", + "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", + "dependencies": { + "whatwg-url": "^5.0.0" + }, "engines": { "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } } }, "node_modules/node-gyp-build": { @@ -4789,12 +4834,15 @@ "integrity": "sha512-r5o/V/ORTA6TmUnyWZR9nCj1klXCO2CEKNRlVuJptZe85QuhFayC7WeMic7ndayT5IRIR0S0xFxFi2ousartlQ==" }, "node_modules/picomatch": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.0.tgz", - "integrity": "sha512-lY1Q/PiJGC2zOv/z391WOTD+Z02bCgsFfvxoXXf6h7kv9o+WmsmzYqrAwY63sNgOxE4xEdq0WyUnXfKeBrSvYw==", + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", "dev": true, "engines": { "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" } }, "node_modules/pify": { @@ -5335,9 +5383,9 @@ } }, "node_modules/readdirp": { - "version": "3.5.0", - "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.5.0.tgz", - "integrity": "sha512-cMhu7c/8rdhkHXWsY+osBhfSy0JikwpHK/5+imo+LpeasTF8ouErHrlYkwT0++njiyuDvc7OFY5T3ukvZ8qmFQ==", + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", + "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", "dev": true, "dependencies": { "picomatch": "^2.2.1" @@ -5521,9 +5569,9 @@ "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==" }, "node_modules/serialize-javascript": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-5.0.1.tgz", - "integrity": "sha512-SaaNal9imEO737H2c05Og0/8LUXG7EnsZyMa8MzkmuHoELfT6txuj0cMqRj6zfPKnmQ1yasR4PCJc8x+M4JSPA==", + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-6.0.0.tgz", + "integrity": "sha512-Qr3TosvguFt8ePWqsvRfrKyQXIiW+nGbYpy8XK24NQHE83caxWt+mIymTT19DGFbNWNLfEwsrkSmN64lVWB9ag==", "dev": true, "dependencies": { "randombytes": "^2.1.0" @@ -5610,9 +5658,9 @@ "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==" }, "node_modules/simple-get": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-3.1.0.tgz", - "integrity": "sha512-bCR6cP+aTdScaQCnQKbPKtJOKDp/hj9EDLJo3Nw4y1QksqaovlW/bnptB6/c1e+qmNIDHRK+oXFDdEqBT8WzUA==", + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-3.1.1.tgz", + "integrity": "sha512-CQ5LTKGfCpvE1K0n2us+kuMPbk/q0EKl82s4aheV9oXjFEz6W/Y7oQFVJuU6QG77hRT4Ghb5RURteF5vnWjupA==", "dependencies": { "decompress-response": "^4.2.0", "once": "^1.3.1", @@ -6109,6 +6157,11 @@ "node": ">=0.6" } }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o=" + }, "node_modules/tsconfig-paths": { "version": "3.12.0", "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.12.0.tgz", @@ -6245,9 +6298,9 @@ } }, "node_modules/url-parse": { - "version": "1.5.3", - "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.3.tgz", - "integrity": "sha512-IIORyIQD9rvj0A4CLWsHkBBJuNqWpFQe224b6j9t/ABmquIS0qDU2pY6kl6AuOrL5OkCXHMCFNe1jBcuAggjvQ==", + "version": "1.5.10", + "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz", + "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==", "dependencies": { "querystringify": "^2.1.1", "requires-port": "^1.0.0" @@ -6299,6 +6352,20 @@ "node": ">= 0.8" } }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE=" + }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha1-lmRU6HZUYuN2RNNib2dCzotwll0=", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", @@ -6354,9 +6421,9 @@ } }, "node_modules/workerpool": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-6.1.0.tgz", - "integrity": "sha512-toV7q9rWNYha963Pl/qyeZ6wG+3nnsyvolaNUS8+R5Wtw6qJPTxIlOP1ZSvcGhEJw+l3HMMmtiNo9Gl61G4GVg==", + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-6.2.0.tgz", + "integrity": "sha512-Rsk5qQHJ9eowMH28Jwhe8HEbmdYDX4lwoMWshiCXugjtHqMD9ZbiqSDLxcsfdqsETPzVUtX5s1Z5kStiIM6l4A==", "dev": true }, "node_modules/wrap-ansi": { @@ -7534,19 +7601,19 @@ "dev": true }, "chokidar": { - "version": "3.5.1", - "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.1.tgz", - "integrity": "sha512-9+s+Od+W0VJJzawDma/gvBNQqkTiqYTWLuZoyAsivsI4AaWTCzHG06/TMjsf1cYe9Cb97UCEhjz7HvnPk2p/tw==", + "version": "3.5.3", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz", + "integrity": "sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==", "dev": true, "requires": { - "anymatch": "~3.1.1", + "anymatch": "~3.1.2", "braces": "~3.0.2", - "fsevents": "~2.3.1", - "glob-parent": "~5.1.0", + "fsevents": "~2.3.2", + "glob-parent": "~5.1.2", "is-binary-path": "~2.1.0", "is-glob": "~4.0.1", "normalize-path": "~3.0.0", - "readdirp": "~3.5.0" + "readdirp": "~3.6.0" } }, "chownr": { @@ -8641,9 +8708,9 @@ "dev": true }, "follow-redirects": { - "version": "1.14.7", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.7.tgz", - "integrity": "sha512-+hbxoLbFMbRKDwohX8GkTataGqO6Jb7jGwpAlwgy2bIz25XtRm7KEzJM76R1WiNT5SwZkX4Y75SwBolkpmE7iQ==" + "version": "1.14.9", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.9.tgz", + "integrity": "sha512-MQDfihBQYMcyy5dhRDJUHcw7lb2Pv/TuE6xP1vyraLukNDHKbDxDNaOE3NbCAdKQApno+GPRyo1YAp89yCjK4w==" }, "foreground-child": { "version": "2.0.0", @@ -8795,9 +8862,9 @@ "integrity": "sha1-l/tdlr/eiXMxPyDoKI75oWf6ZM4=" }, "glob": { - "version": "7.1.6", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", - "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.0.tgz", + "integrity": "sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q==", "dev": true, "requires": { "fs.realpath": "^1.0.0", @@ -9197,6 +9264,12 @@ "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=", "dev": true }, + "is-unicode-supported": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-0.1.0.tgz", + "integrity": "sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw==", + "dev": true + }, "is-weakref": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-weakref/-/is-weakref-1.0.2.tgz", @@ -9351,9 +9424,9 @@ "dev": true }, "js-yaml": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.0.0.tgz", - "integrity": "sha512-pqon0s+4ScYUvX30wxQi3PogGFAlUyH0awepWvwkj4jD4v+ova3RiYw8bmA6x2rDrEaj8i/oWKoRxpVNW+Re8Q==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", "dev": true, "requires": { "argparse": "^2.0.1" @@ -9504,12 +9577,13 @@ "dev": true }, "log-symbols": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.0.0.tgz", - "integrity": "sha512-FN8JBzLx6CzeMrB0tg6pqlGU1wCrXW+ZXGH481kfsBqer0hToTIiHdjH4Mq8xJUbvATujKCvaREGWpGUionraA==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.1.0.tgz", + "integrity": "sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg==", "dev": true, "requires": { - "chalk": "^4.0.0" + "chalk": "^4.1.0", + "is-unicode-supported": "^0.1.0" } }, "loose-envify": { @@ -9642,42 +9716,41 @@ "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==" }, "mocha": { - "version": "8.4.0", - "resolved": "https://registry.npmjs.org/mocha/-/mocha-8.4.0.tgz", - "integrity": "sha512-hJaO0mwDXmZS4ghXsvPVriOhsxQ7ofcpQdm8dE+jISUOKopitvnXFQmpRR7jd2K6VBG6E26gU3IAbXXGIbu4sQ==", + "version": "9.2.1", + "resolved": "https://registry.npmjs.org/mocha/-/mocha-9.2.1.tgz", + "integrity": "sha512-T7uscqjJVS46Pq1XDXyo9Uvey9gd3huT/DD9cYBb4K2Xc/vbKRPUWK067bxDQRK0yIz6Jxk73IrnimvASzBNAQ==", "dev": true, "requires": { "@ungap/promise-all-settled": "1.1.2", "ansi-colors": "4.1.1", "browser-stdout": "1.3.1", - "chokidar": "3.5.1", - "debug": "4.3.1", + "chokidar": "3.5.3", + "debug": "4.3.3", "diff": "5.0.0", "escape-string-regexp": "4.0.0", "find-up": "5.0.0", - "glob": "7.1.6", + "glob": "7.2.0", "growl": "1.10.5", "he": "1.2.0", - "js-yaml": "4.0.0", - "log-symbols": "4.0.0", + "js-yaml": "4.1.0", + "log-symbols": "4.1.0", "minimatch": "3.0.4", "ms": "2.1.3", - "nanoid": "3.1.20", - "serialize-javascript": "5.0.1", + "nanoid": "3.2.0", + "serialize-javascript": "6.0.0", "strip-json-comments": "3.1.1", "supports-color": "8.1.1", "which": "2.0.2", - "wide-align": "1.1.3", - "workerpool": "6.1.0", + "workerpool": "6.2.0", "yargs": "16.2.0", "yargs-parser": "20.2.4", "yargs-unparser": "2.0.0" }, "dependencies": { "debug": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz", - "integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==", + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.3.tgz", + "integrity": "sha512-/zxw5+vh1Tfv+4Qn7a5nsbcJKPaSvCDhojn6FEl9vupwK2VCSDtEiEtqr8DFtzYFOdz63LBkxec7DYuc2jon6Q==", "dev": true, "requires": { "ms": "2.1.2" @@ -9698,9 +9771,9 @@ "dev": true }, "nanoid": { - "version": "3.1.20", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.1.20.tgz", - "integrity": "sha512-a1cQNyczgKbLX9jwbS/+d7W8fX/RfgYR7lVWwWOGIPNgK2m0MWvrGF6/m4kk6U3QcFMnZf3RIhL0v2Jgh/0Uxw==", + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.2.0.tgz", + "integrity": "sha512-fmsZYa9lpn69Ad5eDn7FMcnnSR+8R34W9qJEijxYhTbfOWzr22n1QxCMzXLK+ODyW2973V3Fux959iQoUxzUIA==", "dev": true }, "strip-json-comments": { @@ -9742,9 +9815,9 @@ "optional": true }, "nanoid": { - "version": "3.1.30", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.1.30.tgz", - "integrity": "sha512-zJpuPDwOv8D2zq2WRoMe1HsfZthVewpel9CAvTfc/2mBD1uUT/agc5f7GHGWXlYkFvi1mVxe4IjvP2HNrop7nQ==" + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.1.tgz", + "integrity": "sha512-n6Vs/3KGyxPQd6uO0eH4Bv0ojGSUvuLlIHtC3Y0kEO23YRge8H9x1GCzLn28YX0H66pMkxuaeESFq4tKISKwdw==" }, "napi-build-utils": { "version": "1.0.2", @@ -9771,9 +9844,12 @@ } }, "node-fetch": { - "version": "2.6.1", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", - "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==" + "version": "2.6.7", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", + "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", + "requires": { + "whatwg-url": "^5.0.0" + } }, "node-gyp-build": { "version": "4.2.3", @@ -10246,9 +10322,9 @@ "integrity": "sha512-r5o/V/ORTA6TmUnyWZR9nCj1klXCO2CEKNRlVuJptZe85QuhFayC7WeMic7ndayT5IRIR0S0xFxFi2ousartlQ==" }, "picomatch": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.0.tgz", - "integrity": "sha512-lY1Q/PiJGC2zOv/z391WOTD+Z02bCgsFfvxoXXf6h7kv9o+WmsmzYqrAwY63sNgOxE4xEdq0WyUnXfKeBrSvYw==", + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", "dev": true }, "pify": { @@ -10671,9 +10747,9 @@ } }, "readdirp": { - "version": "3.5.0", - "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.5.0.tgz", - "integrity": "sha512-cMhu7c/8rdhkHXWsY+osBhfSy0JikwpHK/5+imo+LpeasTF8ouErHrlYkwT0++njiyuDvc7OFY5T3ukvZ8qmFQ==", + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", + "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", "dev": true, "requires": { "picomatch": "^2.2.1" @@ -10819,9 +10895,9 @@ } }, "serialize-javascript": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-5.0.1.tgz", - "integrity": "sha512-SaaNal9imEO737H2c05Og0/8LUXG7EnsZyMa8MzkmuHoELfT6txuj0cMqRj6zfPKnmQ1yasR4PCJc8x+M4JSPA==", + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-6.0.0.tgz", + "integrity": "sha512-Qr3TosvguFt8ePWqsvRfrKyQXIiW+nGbYpy8XK24NQHE83caxWt+mIymTT19DGFbNWNLfEwsrkSmN64lVWB9ag==", "dev": true, "requires": { "randombytes": "^2.1.0" @@ -10893,9 +10969,9 @@ "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==" }, "simple-get": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-3.1.0.tgz", - "integrity": "sha512-bCR6cP+aTdScaQCnQKbPKtJOKDp/hj9EDLJo3Nw4y1QksqaovlW/bnptB6/c1e+qmNIDHRK+oXFDdEqBT8WzUA==", + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-3.1.1.tgz", + "integrity": "sha512-CQ5LTKGfCpvE1K0n2us+kuMPbk/q0EKl82s4aheV9oXjFEz6W/Y7oQFVJuU6QG77hRT4Ghb5RURteF5vnWjupA==", "requires": { "decompress-response": "^4.2.0", "once": "^1.3.1", @@ -11283,6 +11359,11 @@ "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz", "integrity": "sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw==" }, + "tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o=" + }, "tsconfig-paths": { "version": "3.12.0", "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.12.0.tgz", @@ -11391,9 +11472,9 @@ } }, "url-parse": { - "version": "1.5.3", - "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.3.tgz", - "integrity": "sha512-IIORyIQD9rvj0A4CLWsHkBBJuNqWpFQe224b6j9t/ABmquIS0qDU2pY6kl6AuOrL5OkCXHMCFNe1jBcuAggjvQ==", + "version": "1.5.10", + "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz", + "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==", "requires": { "querystringify": "^2.1.1", "requires-port": "^1.0.0" @@ -11436,6 +11517,20 @@ "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", "integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=" }, + "webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE=" + }, + "whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha1-lmRU6HZUYuN2RNNib2dCzotwll0=", + "requires": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", @@ -11479,9 +11574,9 @@ "dev": true }, "workerpool": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-6.1.0.tgz", - "integrity": "sha512-toV7q9rWNYha963Pl/qyeZ6wG+3nnsyvolaNUS8+R5Wtw6qJPTxIlOP1ZSvcGhEJw+l3HMMmtiNo9Gl61G4GVg==", + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-6.2.0.tgz", + "integrity": "sha512-Rsk5qQHJ9eowMH28Jwhe8HEbmdYDX4lwoMWshiCXugjtHqMD9ZbiqSDLxcsfdqsETPzVUtX5s1Z5kStiIM6l4A==", "dev": true }, "wrap-ansi": { diff --git a/package.json b/package.json index 8de25bd..285073c 100644 --- a/package.json +++ b/package.json @@ -57,7 +57,7 @@ "devDependencies": { "chai": "^4.3.4", "chai-as-promised": "^7.1.1", - "mocha": "^8.3.2", + "mocha": "^9.2.1", "nyc": "^15.1.0", "standard": "^16.0.3" }, From 64065acc043ec38bfd50602ee874f4d75c54a7dd Mon Sep 17 00:00:00 2001 From: Miguel Duarte Date: Wed, 9 Mar 2022 15:37:39 -0300 Subject: [PATCH 82/82] docker-compose. --- docker-compose.yml | 50 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 docker-compose.yml diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..b66c589 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,50 @@ +version: '3' +services: + bitcoind: + image: hojarasca/bitcoinsv-regtest:1.0.10 + volumes: + - bitcoind-data:/bitcoin/data + environment: + RPCUSER: rundb + RPCPASSWORD: rundb + ports: + - 18332:18332 + - 28332:28332 + + main-run-db: + build: + context: . + command: ["npm", "start"] + volumes: + - ./src:/app/src + - run-db-data:/data + environment: + DB: /data/run-db.sqlite3 + BITCOIN_RPC_URL: http://rundb:rundb@bitcoind:18332 + ZMQ_URL: tcp://bitcoind:28332 + START_HEIGHT: 0 + WORKERS: 1 + PORT: 3000 + NETWORK: test + API: ${API} + EXECUTOR: ${EXECUTOR} + EXECUTE_ENDPOINT: + + execution-server: + build: + context: . + command: [ "npm", "start-execution-server" ] + volumes: + - ./src:/app/src + - run-db-data:/data + environment: + WORKERS: 2 + PORT: 3001 + NETWORK: test + DATA_API_TX_ROOT: ${DATA_API_TX_ROOT} + DATA_API_STATE_ROOT: ${DATA_API_STATE_ROOT} + +volumes: + bitcoind-data: +# rabbitmq-data: + run-db-data: \ No newline at end of file