From b05e514582a2c19b9a24cd01756894f04a361c91 Mon Sep 17 00:00:00 2001 From: Toni Sevener Date: Fri, 10 Jul 2020 11:31:07 -0500 Subject: [PATCH 01/47] initial commit from gerrit patch --- lib/mobile-util.js | 4 +- lib/mwapi.js | 16 ++ package-lock.json | 115 +++++++++++++- package.json | 1 + routes/page/history-batch.js | 287 +++++++++++++++++++++++++++++++++++ 5 files changed, 415 insertions(+), 8 deletions(-) create mode 100644 routes/page/history-batch.js diff --git a/lib/mobile-util.js b/lib/mobile-util.js index 278a5c7f..8a0d6f0c 100644 --- a/lib/mobile-util.js +++ b/lib/mobile-util.js @@ -14,6 +14,7 @@ mUtil.CONTENT_TYPES = { mobileSections: { name: 'mobile-sections', version: '0.14.5', type: 'application/json' }, media: { name: 'Media', version: '1.4.5', type: 'application/json' }, mobileHtml: { name: 'Mobile-HTML', version: '1.2.1', type: 'text/html' }, + references: { name: 'References', version: '2.0.0', type: 'application/json' }, metadata: { name: 'Metadata', version: '1.3.0', type: 'application/json' }, summary: { name: 'Summary', version: '1.4.2', type: 'application/json' }, definition: { name: 'definition', version: '0.8.1', type: 'application/json' }, @@ -22,10 +23,11 @@ mUtil.CONTENT_TYPES = { unpublished: { name: 'Unpublished', version: '0.0.0', type: 'application/json' }, mobileHtmlOfflineResources: { name: 'Mobile-HTML-Offline-Resources', - version: '1.2.1', + version: '1.2.0', type: 'application/json' }, talk: { name: 'Talk', version: '0.1.1', type: 'application/json' }, + historyBatch: { name: 'HistoryBatch', version: '0.0.1', type: 'application/json' }, mediaList: { name: 'MediaList', version: '1.1.0', type: 'application/json' }, i18n: { name: 'i18n', version: '0.0.1', type: 'application/json' } }; diff --git a/lib/mwapi.js b/lib/mwapi.js index 2346b742..70033543 100644 --- a/lib/mwapi.js +++ b/lib/mwapi.js @@ -126,6 +126,22 @@ mwapi.getSiteInfo = function(req) { return siteInfoCache[rp.domain]; }; +mwapi.queryForRevisions = function(req) { + const query = apiParams({ + action: 'query', + prop: 'revisions', + titles: req.params.title, + rvslots: 'main', + rvprop: 'ids|timestamp|user|userid|size|parsedcomment|comment|tags|flags|size', + rvdir: 'older', + format: 'json', + rvlimit: '21', + rvstartid: req.query.rvstartid + }); + + return api.mwApiGet(req, query); +}; + /** * Given protection status for an article simplify it to allow easy reference * @param {!Array} mwApiProtectionObj e.g. diff --git a/package-lock.json b/package-lock.json index a73dbb44..a9773582 100644 --- a/package-lock.json +++ b/package-lock.json @@ -171,6 +171,32 @@ "integrity": "sha512-+iTbntw2IZPb/anVDbypzfQa+ay64MW0Zo8aJ8gZPWMMK6/OubMVb6lUPMagqjOPnmtauXnFCACVl3O7ogjeqQ==", "dev": true }, + "@types/concat-stream": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@types/concat-stream/-/concat-stream-1.6.0.tgz", + "integrity": "sha1-OU2+C7X+5Gs42JZzXoto7yOQ0A0=", + "requires": { + "@types/node": "*" + } + }, + "@types/form-data": { + "version": "0.0.33", + "resolved": "https://registry.npmjs.org/@types/form-data/-/form-data-0.0.33.tgz", + "integrity": "sha1-yayFsqX9GENbjIXZ7LUObWyJP/g=", + "requires": { + "@types/node": "*" + } + }, + "@types/node": { + "version": "10.17.27", + "resolved": "https://registry.npmjs.org/@types/node/-/node-10.17.27.tgz", + "integrity": "sha512-J0oqm9ZfAXaPdwNXMMgAhylw5fhmXkToJd06vuDUSAgEDZ/n/69/69UmyBZbc+zT34UnShuDSBqvim3SPnozJg==" + }, + "@types/qs": { + "version": "6.9.3", + "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.3.tgz", + "integrity": "sha512-7s9EQWupR1fTc2pSMtXRQ9w9gLOcrJn+h7HOXw4evxyvVqMi4f+q7d2tnFe3ng3SNHjtK+0EzGMGFUQX4/AQRA==" + }, "@wikimedia/less-plugin-clean-css": { "version": "1.5.2", "resolved": "https://registry.npmjs.org/@wikimedia/less-plugin-clean-css/-/less-plugin-clean-css-1.5.2.tgz", @@ -534,8 +560,7 @@ "buffer-from": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", - "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==", - "dev": true + "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==" }, "bunyan": { "version": "1.8.12", @@ -824,7 +849,6 @@ "version": "1.6.2", "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==", - "dev": true, "requires": { "buffer-from": "^1.0.0", "inherits": "^2.0.3", @@ -836,7 +860,6 @@ "version": "2.3.7", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", - "dev": true, "requires": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", @@ -851,7 +874,6 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "dev": true, "requires": { "safe-buffer": "~5.1.0" } @@ -2369,6 +2391,11 @@ "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==" }, + "get-port": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/get-port/-/get-port-3.2.0.tgz", + "integrity": "sha1-3Xzn3hh8Bsi/NTeWrHHgmfCYDrw=" + }, "get-stream": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", @@ -2547,6 +2574,17 @@ "readable-stream": "^3.1.1" } }, + "http-basic": { + "version": "8.1.3", + "resolved": "https://registry.npmjs.org/http-basic/-/http-basic-8.1.3.tgz", + "integrity": "sha512-/EcDMwJZh3mABI2NhGfHOGOeOZITqfkEO4p/xK+l3NpyncIHUQBoMvCSF/b5GqvKtySC2srL/GGG3+EtlqlmCw==", + "requires": { + "caseless": "^0.12.0", + "concat-stream": "^1.6.2", + "http-response-object": "^3.0.1", + "parse-cache-control": "^1.0.1" + } + }, "http-errors": { "version": "1.7.2", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz", @@ -2559,6 +2597,14 @@ "toidentifier": "1.0.0" } }, + "http-response-object": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/http-response-object/-/http-response-object-3.0.2.tgz", + "integrity": "sha512-bqX0XTF6fnXSQcEJ2Iuyr75yVakyjIDCqroJQ/aHfSdlM743Cwqoi2nDYMzLGWUcuTWGWy8AAvOKXTfiv6q9RA==", + "requires": { + "@types/node": "^10.0.3" + } + }, "http-shutdown": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/http-shutdown/-/http-shutdown-1.2.2.tgz", @@ -4404,6 +4450,11 @@ "callsites": "^3.0.0" } }, + "parse-cache-control": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/parse-cache-control/-/parse-cache-control-1.0.1.tgz", + "integrity": "sha1-juqz5U+laSD+Fro493+iGqzC104=" + }, "parse-json": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-4.0.0.tgz", @@ -5367,6 +5418,24 @@ "resolved": "https://registry.npmjs.org/swagger-ui-dist/-/swagger-ui-dist-3.25.0.tgz", "integrity": "sha512-vwvJPPbdooTvDwLGzjIXinOXizDJJ6U1hxnJL3y6U3aL1d2MSXDmKg2139XaLBhsVZdnQJV2bOkX4reB+RXamg==" }, + "sync-request": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/sync-request/-/sync-request-6.1.0.tgz", + "integrity": "sha512-8fjNkrNlNCrVc/av+Jn+xxqfCjYaBoHqCsDz6mt030UMxJGr+GSfCV1dQt2gRtlL63+VPidwDVLr7V2OcTSdRw==", + "requires": { + "http-response-object": "^3.0.1", + "sync-rpc": "^1.2.1", + "then-request": "^6.0.0" + } + }, + "sync-rpc": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/sync-rpc/-/sync-rpc-1.3.6.tgz", + "integrity": "sha512-J8jTXuZzRlvU7HemDgHi3pGnh/rkoqR/OZSjhTyyZrEkkYQbk7Z33AXp37mkPfPpfdOuj7Ex3H/TJM1z48uPQw==", + "requires": { + "get-port": "^3.1.0" + } + }, "table": { "version": "5.4.6", "resolved": "https://registry.npmjs.org/table/-/table-5.4.6.tgz", @@ -5432,6 +5501,39 @@ "integrity": "sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=", "dev": true }, + "then-request": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/then-request/-/then-request-6.0.2.tgz", + "integrity": "sha512-3ZBiG7JvP3wbDzA9iNY5zJQcHL4jn/0BWtXIkagfz7QgOL/LqjCEOBQuJNZfu0XYnv5JhKh+cDxCPM4ILrqruA==", + "requires": { + "@types/concat-stream": "^1.6.0", + "@types/form-data": "0.0.33", + "@types/node": "^8.0.0", + "@types/qs": "^6.2.31", + "caseless": "~0.12.0", + "concat-stream": "^1.6.0", + "form-data": "^2.2.0", + "http-basic": "^8.1.1", + "http-response-object": "^3.0.1", + "promise": "^8.0.0", + "qs": "^6.4.0" + }, + "dependencies": { + "@types/node": { + "version": "8.10.61", + "resolved": "https://registry.npmjs.org/@types/node/-/node-8.10.61.tgz", + "integrity": "sha512-l+zSbvT8TPRaCxL1l9cwHCb0tSqGAGcjPJFItGGYat5oCTiq1uQQKYg5m7AF1mgnEBzFXGLJ2LRmNjtreRX76Q==" + }, + "promise": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/promise/-/promise-8.1.0.tgz", + "integrity": "sha512-W04AqnILOL/sPRXziNicCjSNRruLAuIHEOVBazepu0545DDNGYHz7ar9ZgZ1fMU8/MA4mVxp5rkBWRi6OXIy3Q==", + "requires": { + "asap": "~2.0.6" + } + } + } + }, "through": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", @@ -5548,8 +5650,7 @@ "typedarray": { "version": "0.0.6", "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", - "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=", - "dev": true + "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=" }, "undefsafe": { "version": "2.0.3", diff --git a/package.json b/package.json index d3ba9c78..9ae3cdbc 100644 --- a/package.json +++ b/package.json @@ -54,6 +54,7 @@ }, "homepage": "https://www.mediawiki.org/wiki/RESTBase_services_for_apps", "dependencies": { + "sync-request": "^6.1.0", "banana-i18n": "^1.2.1", "bluebird": "^3.7.2", "body-parser": "^1.19.0", diff --git a/routes/page/history-batch.js b/routes/page/history-batch.js new file mode 100644 index 00000000..d3c5063d --- /dev/null +++ b/routes/page/history-batch.js @@ -0,0 +1,287 @@ +const router = require('../../lib/util').router(); +const mUtil = require('../../lib/mobile-util'); +const mwapi = require('../../lib/mwapi'); +const BBPromise = require('bluebird'); +const api = require('../../lib/api-util'); + +let app; + +function getHistoryBatch(req, res) { + return BBPromise.props({ + mw: mwapi.queryForRevisions(req) + }).then((response) => { + return response.mw; + }).then((response) => { + res.status(response.status); + mUtil.setContentType(res, mUtil.CONTENT_TYPES.historyBatch); + // mUtil.setETag(res, mobileHTML.metadata.revision); + // mUtil.setLanguageHeaders(res, mobileHTML.metadata._headers); + // mUtil.setContentSecurityPolicy(res, app.conf.mobile_html_csp); + + // BEGIN - Mark byte delta on each revision + let nextRevision; + const revisionsWithDelta = []; + const revisions = response.body.query.pages[0].revisions; + revisions.forEach(function (revision) { + if (!(nextRevision === undefined || nextRevision === null)) { + nextRevision.delta = nextRevision.size - revision.size; + revisionsWithDelta.push(nextRevision); + } + nextRevision = revision; + }); + const nextRevId = nextRevision.revid; + const beginningDate = revisions[0].timestamp; + const endDate = revisions[revisions.length - 1].timestamp; + // END - Mark byte delta on each revision + + // BEGIN - Gather talk page new discussions + const talkPageURL = `https://en.wikipedia.org/w/api.php?action=query&prop=revisions&titles=Talk:${req.params.title}&rvslots=main&rvprop=ids|timestamp|user|userid|size|parsedcomment|comment|tags|flags&rvdir=older&format=json&rvlimit=51&rvstart=${beginningDate}&rvend=${endDate}`; + const syncRequest = require('sync-request'); + const talkPageResponse = syncRequest('GET', talkPageURL, { + headers: { + 'Content-Type': 'application/json; charset=utf-8', + }, + }); + const talkPageBody = JSON.parse(talkPageResponse.getBody('utf8')); + + const newSectionTalkPageRevisions = []; + const talkPage = talkPageBody.query.pages; + const talkPageObject = talkPage[Object.keys(talkPage)[0]]; + const talkPageRevisions = talkPageObject.revisions; + talkPageRevisions.forEach(function (revision, index) { + if (revision.comment.toLowerCase().includes('new section') + && revision.userid !== 4936590) { // don't show signbot topics + // see if this section was reverted in previous iterations + var wasReverted = false; + if (index - 1 > 0) { + var nextRevision = talkPageRevisions[index - 1]; + if (nextRevision.userid === 4936590) { // signbot, look for next revision + if (index - 2 > 0) { + nextRevision = talkPageRevisions[index - 2]; + } + } + + if (nextRevision.tags.includes('mw-undo') || + nextRevision.tags.includes('mw-rollback')) { + wasReverted = true; + } + } + + if (wasReverted === false) { + newSectionTalkPageRevisions.push(revision); + } + } + }); + // END - Gather talk page new discussions + + // BEGIN - Break it down into types + const significantChangeObjects = []; + revisionsWithDelta.forEach(function (revision, index) { + + // BEGIN - Shuffle in new section talk pages on each iteration + for (let i = 0; i < newSectionTalkPageRevisions.length; ++i) { + const sectionTalkPageRevision = newSectionTalkPageRevisions[i]; + const talkPageDate = new Date(sectionTalkPageRevision.timestamp); + const revisionDate = new Date(revision.timestamp); + if (talkPageDate > revisionDate) { + // todo: why doesn't parsedComment show in response? + const newTopic = Object.assign({ + type: 'new-talk-page-topic', + revid: sectionTalkPageRevision.revid, + user: sectionTalkPageRevision.user, + userid: sectionTalkPageRevision.userid, + timestamp: sectionTalkPageRevision.timestamp, + comment: sectionTalkPageRevision.comment, + parsedComment: sectionTalkPageRevision.parsedComment }); + significantChangeObjects.push(newTopic); + newSectionTalkPageRevisions.splice(i, 1); + --i; // Correct the index value + } + } + // END - Shuffle in new section talk pages on each iteration + + const threshold = req.query.threshold === null || req.query.threshold === undefined ? + 100 : req.query.threshold; + if (revision.tags.includes('mw-rollback') && + revision.comment.toLowerCase().includes('revert') && + revision.comment.toLowerCase().includes('vandalism') && revision.delta < 0 ) { + // Add vandalism + const vandalism = Object.assign({ type: 'vandalism-revert' } ); + significantChangeObjects.push(vandalism); + } else if (revision.delta >= -threshold && revision.delta <= threshold) { + // Add small changes + const smallChange = Object.assign({ type: 'small-change' } ); + significantChangeObjects.push(smallChange); + } else { + // BEGIN - Add large changes + const largeChange = Object.assign({ + type: 'large-change', + delta: revision.delta, + timestamp: revision.timestamp, + revid: revision.revid, + user: revision.user, + userid: revision.userid + }); + + // get snippet of change + // https://en.wikipedia.org/w/rest.php/v1/revision/847170467/compare/851733941 + // const restParams = Object.assign({domain: 'en.wikipedia.org', + // path: 'w/rest.php/v1/revision/847170467/compare/851733941'} ); + // const restReq = Object.assign({method: 'get', query: null, + // headers: null, params: restParams} ); + + // BBPromise.props({ + // // rest: api.restApiGet(req, restReq) + // // }).then((response) => { + // // return response.rest; + // // }).then((response) => { + // // console.log(response); + // // console.log(revision); + // // }); + + // BEGIN: For each large change, hit the diff endpoint and pull a snippet + const syncRequest = require('sync-request'); + const url = `https://en.wikipedia.org/w/rest.php/v1/revision/${revision.parentid}/compare/${revision.revid}`; + const diffResponse = syncRequest('GET', url, { + headers: { + 'Content-Type': 'application/json; charset=utf-8', + }, + }); + const body = JSON.parse(diffResponse.getBody('utf8')); + const changedLines = body.diff.filter(diffItem => diffItem.type >= 1 && + diffItem.type <= 3); + + changedLines.forEach(function (diff) { + if (diff.type === 1 || diff.type === 2) { + diff.byteChange = diff.text.length; + } else { + var byteChange = 0; + diff.highlightRanges.forEach(function (range) { + byteChange += range.length; + }); + + diff.byteChange = byteChange; + } + }); + changedLines.sort(function(a, b) { + return b.byteChange - a.byteChange; + }); + if (changedLines.length > 0) { + var changeType; + switch (changedLines[0].type) { + case 1: + changeType = 'added-line'; + break; + case 2: + changeType = 'deleted-line'; + break; + case 3: + changeType = 'added-and-or-deleted-words-in-line'; + break; + default: break; + } + if (changedLines[0].highlightRanges !== undefined && + changedLines[0].highlightRanges.length > 0) { + changedLines[0].highlightRanges.forEach( function (range) { + switch (range.type) { + case 0: + range.type = 'added'; + break; + case 1: + range.type = 'deleted'; + break; + default: + break; + } + }); + } + + const contentSnippet = Object.assign({ + changeType: changeType, + text: changedLines[0].text, + highlightRanges: changedLines[0].highlightRanges + }); + largeChange.contentSnippet = contentSnippet; + } + // END: For each large change, hit the diff endpoint and pull a snippet + + significantChangeObjects.push(largeChange); + // END - Add large changes + } + + // BEGIN - Shuffle in new section talk pages on each iteration + // (this catches any straggler new talk pages at the end) + if (index === revisionsWithDelta.length - 1) { + for (let i = 0; i < newSectionTalkPageRevisions.length; ++i) { + const sectionTalkPageRevision = newSectionTalkPageRevisions[i]; + const talkPageDate = new Date(sectionTalkPageRevision.timestamp); + const revisionDate = new Date(revision.timestamp); + if (talkPageDate < revisionDate) { + const newTopic = Object.assign({ + type: 'new-talk-page-topic', + revid: sectionTalkPageRevision.revid, + user: sectionTalkPageRevision.user, + userid: sectionTalkPageRevision.userid, + timestamp: sectionTalkPageRevision.timestamp, + comment: sectionTalkPageRevision.comment, + parsedComment: sectionTalkPageRevision.parsedComment }); + significantChangeObjects.push(sectionTalkPageRevision); + newSectionTalkPageRevisions.splice(i, 1); + --i; // Correct the index value + } + } + } + // END - Shuffle in new section talk pages on each iteration + }); + + // BEGIN - Collapse small changes + const collapsedSignificantChanges = []; + let numSmallChanges = 0; + significantChangeObjects.forEach(function (revision) { + if (revision.type === 'small-change') { + numSmallChanges++; + return; + } else if (numSmallChanges > 0) { + const consolidatedSmallChange = Object.assign({ + type: 'small-change', + count: numSmallChanges } ); + collapsedSignificantChanges.push(consolidatedSmallChange); + numSmallChanges = 0; + } + + collapsedSignificantChanges.push(revision); + }); + if (numSmallChanges > 0) { + const consolidatedSmallChange = Object.assign({ + type: 'small-change', + count: numSmallChanges + }); + collapsedSignificantChanges.push(consolidatedSmallChange); + numSmallChanges = 0; + } + // END - Collapse small changes + + const result = Object.assign({ + nextRvStartId: nextRevId, + revisions: collapsedSignificantChanges } ); + res.send(result).end(); + // res.send(response.body.query.pages[0].revisions).end(); + }); +} + +router.get('/page/history-batch/:title', (req, res) => { + // res.status(200); + return getHistoryBatch(req, res); + // const result = Object.assign({ result: "What up new endpoint."}); + // mUtil.setContentType(res, mUtil.CONTENT_TYPES.talk); + // res.json(result).end(); +}); + +module.exports = function(appObj) { + app = appObj; + return { + path: '/', + api_version: 1, + router + }; +}; From 682bc4073cd8022d12d259f642a8ff4cd9f3d65b Mon Sep 17 00:00:00 2001 From: Toni Sevener Date: Fri, 10 Jul 2020 11:34:57 -0500 Subject: [PATCH 02/47] rename history-batch to significant-changes --- lib/mobile-util.js | 5 ++--- routes/page/{history-batch.js => significant-changes.js} | 8 ++++---- 2 files changed, 6 insertions(+), 7 deletions(-) rename routes/page/{history-batch.js => significant-changes.js} (98%) diff --git a/lib/mobile-util.js b/lib/mobile-util.js index 8a0d6f0c..3e919e58 100644 --- a/lib/mobile-util.js +++ b/lib/mobile-util.js @@ -14,7 +14,6 @@ mUtil.CONTENT_TYPES = { mobileSections: { name: 'mobile-sections', version: '0.14.5', type: 'application/json' }, media: { name: 'Media', version: '1.4.5', type: 'application/json' }, mobileHtml: { name: 'Mobile-HTML', version: '1.2.1', type: 'text/html' }, - references: { name: 'References', version: '2.0.0', type: 'application/json' }, metadata: { name: 'Metadata', version: '1.3.0', type: 'application/json' }, summary: { name: 'Summary', version: '1.4.2', type: 'application/json' }, definition: { name: 'definition', version: '0.8.1', type: 'application/json' }, @@ -23,11 +22,11 @@ mUtil.CONTENT_TYPES = { unpublished: { name: 'Unpublished', version: '0.0.0', type: 'application/json' }, mobileHtmlOfflineResources: { name: 'Mobile-HTML-Offline-Resources', - version: '1.2.0', + version: '1.2.1', type: 'application/json' }, talk: { name: 'Talk', version: '0.1.1', type: 'application/json' }, - historyBatch: { name: 'HistoryBatch', version: '0.0.1', type: 'application/json' }, + significantChanges: { name: 'SignificantChanges', version: '0.0.1', type: 'application/json' }, mediaList: { name: 'MediaList', version: '1.1.0', type: 'application/json' }, i18n: { name: 'i18n', version: '0.0.1', type: 'application/json' } }; diff --git a/routes/page/history-batch.js b/routes/page/significant-changes.js similarity index 98% rename from routes/page/history-batch.js rename to routes/page/significant-changes.js index d3c5063d..b8d033be 100644 --- a/routes/page/history-batch.js +++ b/routes/page/significant-changes.js @@ -6,14 +6,14 @@ const api = require('../../lib/api-util'); let app; -function getHistoryBatch(req, res) { +function getSignificantChanges(req, res) { return BBPromise.props({ mw: mwapi.queryForRevisions(req) }).then((response) => { return response.mw; }).then((response) => { res.status(response.status); - mUtil.setContentType(res, mUtil.CONTENT_TYPES.historyBatch); + mUtil.setContentType(res, mUtil.CONTENT_TYPES.significantChanges); // mUtil.setETag(res, mobileHTML.metadata.revision); // mUtil.setLanguageHeaders(res, mobileHTML.metadata._headers); // mUtil.setContentSecurityPolicy(res, app.conf.mobile_html_csp); @@ -269,9 +269,9 @@ function getHistoryBatch(req, res) { }); } -router.get('/page/history-batch/:title', (req, res) => { +router.get('/page/significant-changes/:title', (req, res) => { // res.status(200); - return getHistoryBatch(req, res); + return getSignificantChanges(req, res); // const result = Object.assign({ result: "What up new endpoint."}); // mUtil.setContentType(res, mUtil.CONTENT_TYPES.talk); // res.json(result).end(); From c29af0fd9ad84072266c3ecf2f8ba18fe12f4547 Mon Sep 17 00:00:00 2001 From: Toni Sevener Date: Fri, 10 Jul 2020 17:19:27 -0500 Subject: [PATCH 03/47] progress converting to promises --- config.apps.yaml | 8 ++++++ config.dev.yaml | 8 ++++++ config.labs.yaml | 8 ++++++ config.prod.yaml | 8 ++++++ dist/config.yaml | 7 +++++ lib/api-util.js | 44 +++++++++++++++++++++++++++++- lib/mwrestapi.js | 10 +++++++ routes/page/significant-changes.js | 35 +++++++++++++++++++++++- 8 files changed, 126 insertions(+), 2 deletions(-) create mode 100644 lib/mwrestapi.js diff --git a/config.apps.yaml b/config.apps.yaml index d05ece2f..6f03d196 100644 --- a/config.apps.yaml +++ b/config.apps.yaml @@ -86,6 +86,14 @@ services: user-agent: '{{user-agent}}' accept-language: '{{accept-language}}' body: '{{ default(request.query, {}) }}' + # the template used for contacting MW Rest API + mwrestapi_req: + method: '{{request.method}}' + uri: https://{{domain}}/w/rest.php/v1/{+path} + query: '{{ default(request.query, {}) }}' + headers: '{{request.headers}}' + body: '{{request.body}}' + timeout: 60000 # 60 * 1000 # the template used for contacting RESTBase restbase_req: method: '{{request.method}}' diff --git a/config.dev.yaml b/config.dev.yaml index 82feebb6..35914eab 100644 --- a/config.dev.yaml +++ b/config.dev.yaml @@ -87,6 +87,14 @@ services: user-agent: '{{user-agent}}' accept-language: '{{accept-language}}' body: '{{ default(request.query, {}) }}' + # the template used for contacting MW Rest API + mwrestapi_req: + method: '{{request.method}}' + uri: https://{{domain}}/w/rest.php/v1/{+path} + query: '{{ default(request.query, {}) }}' + headers: '{{request.headers}}' + body: '{{request.body}}' + timeout: 60000 # 60 * 1000 # the template used for contacting RESTBase restbase_req: method: '{{request.method}}' diff --git a/config.labs.yaml b/config.labs.yaml index 87511986..50a62715 100644 --- a/config.labs.yaml +++ b/config.labs.yaml @@ -86,6 +86,14 @@ services: user-agent: '{{user-agent}}' accept-language: '{{accept-language}}' body: '{{ default(request.query, {}) }}' + # the template used for contacting MW Rest API + mwrestapi_req: + method: '{{request.method}}' + uri: https://{{domain}}/w/rest.php/v1/{+path} + query: '{{ default(request.query, {}) }}' + headers: '{{request.headers}}' + body: '{{request.body}}' + timeout: 60000 # 60 * 1000 # the template used for contacting RESTBase restbase_req: method: '{{request.method}}' diff --git a/config.prod.yaml b/config.prod.yaml index 6d474f60..5257f9a6 100644 --- a/config.prod.yaml +++ b/config.prod.yaml @@ -99,6 +99,14 @@ services: user-agent: '{{user-agent}}' accept-language: '{{accept-language}}' body: '{{ default(request.query, {}) }}' + # the template used for contacting MW Rest API + mwrestapi_req: + method: '{{request.method}}' + uri: https://{{domain}}/w/rest.php/v1/{+path} + query: '{{ default(request.query, {}) }}' + headers: '{{request.headers}}' + body: '{{request.body}}' + timeout: 60000 # 60 * 1000 # the template used for contacting RESTBase restbase_req: method: '{{request.method}}' diff --git a/dist/config.yaml b/dist/config.yaml index 708bde99..c62cc54d 100644 --- a/dist/config.yaml +++ b/dist/config.yaml @@ -68,6 +68,13 @@ services: headers: user-agent: '{{user-agent}}' body: '{{ default(request.query, {}) }}' + # the template used for contacting MW Rest API + mwrestapi_req: + method: '{{request.method}}' + uri: https://{{domain}}/w/rest.php/v1/{+path} + query: '{{ default(request.query, {}) }}' + headers: '{{request.headers}}' + body: '{{request.body}}' # the template used for contacting RESTBase restbase_req: method: '{{request.method}}' diff --git a/lib/api-util.js b/lib/api-util.js index 3401b41b..59982a16 100644 --- a/lib/api-util.js +++ b/lib/api-util.js @@ -14,6 +14,47 @@ function prettyMwApiReq(request) { return `${request.uri}?${querystring.stringify(request.body)}`; } +/** + * Calls the MW REST API with the supplied domain, path and request parameters + * @param {!Object} req the incoming request object + * @param {?string} path the REST API path to contact without the leading slash + * @param {?Object} [restReq={}] the object containing the REST request details + * @param {?string} [restReq.method=get] the request method + * @param {?Object} [restReq.query={}] the query string to send, if any + * @param {?Object} [restReq.headers={}] the request headers to send + * @param {?Object} [restReq.body=null] the body of the request, if any + * @return {!Promise} a promise resolving as the response object from the REST API + * + */ +function mwRestApiGet(req, path, restReq) { + + const app = req.app; + if (path.constructor === Object) { + restReq = path; + path = undefined; + } + restReq = restReq || {}; + restReq.method = restReq.method || 'get'; + restReq.query = restReq.query || {}; + restReq.headers = restReq.headers || {}; + restReq.params = restReq.params || {}; + restReq.params.path = path || restReq.params.path; + restReq.params.domain = restReq.params.domain || req.params.domain; + if (!restReq.params.path || !restReq.params.domain) { + return BBPromise.reject(new HTTPError({ + status: 500, + type: 'internal_error', + title: 'Invalid internal call', + detail: 'domain and path need to be defined for the REST API call' + })); + } + restReq.params.path = restReq.params.path[0] === '/' ? + restReq.params.path.slice(1) : restReq.params.path; + + return req.issueRequest(app.mwrestapi_tbl.expand({ request: restReq })); + +} + /** * Calls the MW API with the supplied query as its body * @param {!Object} req the incoming request object @@ -199,7 +240,7 @@ function setupApiTemplates(app) { }; } app.restbase_tpl = new Template(app.conf.restbase_req); - + app.mwrestapi_tbl = new Template(app.conf.mwrestapi_req); } /** @@ -234,6 +275,7 @@ function getCommonsDomain(domain) { module.exports = { // Shared with service-template-node mwApiGet, + mwRestApiGet, restApiGet, setupApiTemplates, diff --git a/lib/mwrestapi.js b/lib/mwrestapi.js new file mode 100644 index 00000000..53765ccd --- /dev/null +++ b/lib/mwrestapi.js @@ -0,0 +1,10 @@ +const api = require('./api-util'); +const mwrestapi = {}; + +mwrestapi.queryForDiff = function(req, fromRevID, toRevID) { + const path = `revision/${fromRevID}/compare/${toRevID}`; + + return api.mwRestApiGet(req, path); +}; + +module.exports = mwrestapi; diff --git a/routes/page/significant-changes.js b/routes/page/significant-changes.js index b8d033be..e8b9210d 100644 --- a/routes/page/significant-changes.js +++ b/routes/page/significant-changes.js @@ -1,11 +1,44 @@ const router = require('../../lib/util').router(); const mUtil = require('../../lib/mobile-util'); const mwapi = require('../../lib/mwapi'); +const mwrestapi = require('../../lib/mwrestapi'); const BBPromise = require('bluebird'); const api = require('../../lib/api-util'); +const express = require('express'); let app; +const diffPromise = (req, revid, parentid) => { + return mwrestapi.queryForDiff(req, revid, parentid) + .then( (response) => { + return Object.assign({ + revID: revid, + body: response.body + }); + }); +}; + +function getSignificantChanges2(req, res) { + return mwapi.queryForRevisions(req) + .then( (response) => { + // eslint-disable-next-line no-console + console.log(response); + + // hit compare endpoint for each revision + const revisions = response.body.query.pages[0].revisions; + + // may be able to clean this up with map somehow http://bluebirdjs.com/docs/api/promise.map.html + + return BBPromise.map(revisions, function(revision) { + return diffPromise(req, revision.revid, revision.parentid); + }); + }) + .then( (response) => { + // eslint-disable-next-line no-console + console.log(response); + }); +} + function getSignificantChanges(req, res) { return BBPromise.props({ mw: mwapi.queryForRevisions(req) @@ -271,7 +304,7 @@ function getSignificantChanges(req, res) { router.get('/page/significant-changes/:title', (req, res) => { // res.status(200); - return getSignificantChanges(req, res); + return getSignificantChanges2(req, res); // const result = Object.assign({ result: "What up new endpoint."}); // mUtil.setContentType(res, mUtil.CONTENT_TYPES.talk); // res.json(result).end(); From 51f3e703e7fb129afe8d7df3e7fdbd2e3044ea3f Mon Sep 17 00:00:00 2001 From: Toni Sevener Date: Mon, 13 Jul 2020 17:28:14 -0500 Subject: [PATCH 04/47] progress processing data --- lib/api-util.js | 2 + lib/mwapi.js | 10 +- routes/page/significant-changes.js | 252 +++++++++++++++++++++++++++-- 3 files changed, 250 insertions(+), 14 deletions(-) diff --git a/lib/api-util.js b/lib/api-util.js index 59982a16..5ed850df 100644 --- a/lib/api-util.js +++ b/lib/api-util.js @@ -51,6 +51,8 @@ function mwRestApiGet(req, path, restReq) { restReq.params.path = restReq.params.path[0] === '/' ? restReq.params.path.slice(1) : restReq.params.path; + var wat = app.mwrestapi_tbl.expand({ request: restReq }); + console.log(wat); return req.issueRequest(app.mwrestapi_tbl.expand({ request: restReq })); } diff --git a/lib/mwapi.js b/lib/mwapi.js index 70033543..bd7041e5 100644 --- a/lib/mwapi.js +++ b/lib/mwapi.js @@ -126,17 +126,19 @@ mwapi.getSiteInfo = function(req) { return siteInfoCache[rp.domain]; }; -mwapi.queryForRevisions = function(req) { +mwapi.queryForRevisions = function(req, title, pageSize, rvStart, rvEnd) { const query = apiParams({ action: 'query', prop: 'revisions', - titles: req.params.title, + titles: title || req.params.title, rvslots: 'main', rvprop: 'ids|timestamp|user|userid|size|parsedcomment|comment|tags|flags|size', rvdir: 'older', format: 'json', - rvlimit: '21', - rvstartid: req.query.rvstartid + rvlimit: pageSize || '20', + rvstartid: req.query.rvstartid, + rvstart: rvStart, + rvend: rvEnd }); return api.mwApiGet(req, query); diff --git a/routes/page/significant-changes.js b/routes/page/significant-changes.js index e8b9210d..53fb5402 100644 --- a/routes/page/significant-changes.js +++ b/routes/page/significant-changes.js @@ -5,40 +5,272 @@ const mwrestapi = require('../../lib/mwrestapi'); const BBPromise = require('bluebird'); const api = require('../../lib/api-util'); const express = require('express'); +const parsoidApi = require('../../lib/parsoid-access'); let app; -const diffPromise = (req, revid, parentid) => { - return mwrestapi.queryForDiff(req, revid, parentid) +const significantChangesCache = {}; + +const diffAndRevisionPromise = (req, revision) => { + return mwrestapi.queryForDiff(req, revision.revid, revision.parentid) .then( (response) => { return Object.assign({ - revID: revid, + revision: revision, body: response.body }); }); }; +//curl -X POST "https://en.wikipedia.org/api/rest_v1/transform/wikitext/to/html/Dog" -H "accept: text/html; charset=utf-8; profile="https://www.mediawiki.org/wiki/Specs/HTML/2.1.0"" -H "Content-Type: multipart/form-data" -F "wikitext='testing'" -F "body_only=true" -F "stash=true" +//"https://en.wikipedia.org/api/rest_v1/transform/wikitext/to/html/Dog" -H "accept: text/html; charset=utf-8; profile="https://www.mediawiki.org/wiki/Specs/HTML/2.1.0"" -H "Content-Type: multipart/form-data" -F "wikitext='testing'" -F "body_only=true" -F "stash=true" +//wat console write +/* +{ method: 'get', + uri: + "https://en.wikipedia.org/w/rest.php/v1/revision/967289946/compare/967285956", + query: {}, + headers: {}, + body: undefined, + timeout: 60000 } + */ + +//call like req.issueRequest(wat); +//then chain parsoidApi.mobileHTMLPromiseFromHTML +// const snippetAndRevisionPromise = (app, req, res, html, revision) => { +// return mwrestapi.queryForDiff(req, revision.revid, revision.parentid) +// .then( (response) => { +// return Object.assign({ +// revision: revision, +// body: response.body +// }); +// }); +// }; + +const diffAndRevisionPromises = (req, revisions) => { + return BBPromise.map(revisions, function(revision) { + return diffAndRevisionPromise(req, revision); + }); +}; + +const talkPageTitle = (req) => { + return `Talk:${req.params.title}` +}; + +const talkPageRevisionsPromise = (req, rvStart, rvEnd) => { + return mwapi.queryForRevisions(req, talkPageTitle(req), 100, rvStart, rvEnd ); +}; + +const significantChangesCacheKey = (req, title, revision) => { + const keyTitle = title || req.params.title; + return `${req.params.domain}-${title}-${revision.revid}`; +}; + +function getCachedAndUncachedItems(revisions, req, title) { + // add cache to output and filter out of processing flow + const uncachedRevisions = []; + const cachedOutput = []; + revisions.forEach(function (revision) { + + const cacheKey = significantChangesCacheKey(req, title, revision); + const cacheItem = significantChangesCache[cacheKey]; + if (cacheItem) { + cachedOutput.push(cacheItem); + } else { + uncachedRevisions.push(revision); + } + }); + + return Object.assign({ + uncachedRevisions: uncachedRevisions, + cachedOutput: cachedOutput + }); +} + +class ByteChange { + constructor(addedCount, deletedCount) { + this.addedCount = addedCount; + this.deletedCount = deletedCount; + } + + totalCount() { + return this.addedCount + this.deletedCount; + } +} + +class SmallOutput { + constructor(revid, timestamp, outputType) { + this.revid = revid; + this.timestamp = timestamp; + this.outputType = outputType; + } +} + +class LargeOutput { + constructor(revid, timestamp, outputType, snippet) { + this.revid = revid; + this.timestamp = timestamp; + this.outputType = outputType; + this.snippet = snippet; + } +} + +function updateDiffAndRevisionsWithByteCount(diffAndRevisions) { + console.log(diffAndRevisions); + + // Loop through diffs, filter out type 0 (context type) and assign byte change properties to the remaining + + diffAndRevisions.forEach(function (diffAndRevision) { + + var filteredDiffs = []; + + var aggregateAddedCount = 0; + var aggregateDeletedCount = 0; + diffAndRevision.body.diff.forEach(function (diff) { + var lineAddedCount = 0; + var lineDeletedCount = 0; + switch (diff.type) { + case 0: // Context line type + return; + case 1: // Add complete line type + lineAddedCount = diff.text.length; + break; + case 2: // Delete complete line type + lineDeletedCount = diff.text.length; + break; + case 3: // Change line type (add and deleted ranges in line) + //todo: there's something funky with added and deleted types. see United_States revid 966656680, says added 172 bytes (type 0 highlighted range) but UI shows deleted in desktop and app. + diff.highlightRanges.forEach(function (range) { + switch (range.type) { + case 0: // Add range type + lineAddedCount += range.length; + break; + case 1: // Delete range type + lineDeletedCount += range.length; + break; + default: + break; + } + }); + break; + default: + break; + } + + aggregateAddedCount += lineAddedCount; + aggregateDeletedCount += lineDeletedCount; + + diff.byteChange = new ByteChange(lineAddedCount, lineDeletedCount); + filteredDiffs.push(diff); + }); + + diffAndRevision.byteChange = new ByteChange(aggregateAddedCount, aggregateDeletedCount) + diffAndRevision.body.diff = filteredDiffs; + }); + + console.log(diffAndRevisions); +} + function getSignificantChanges2(req, res) { + + const output = []; + + // STEP 1: Gather list of article revisions return mwapi.queryForRevisions(req) .then( (response) => { - // eslint-disable-next-line no-console - console.log(response); - // hit compare endpoint for each revision + // STEP 2: All at once gather diffs for each uncached revision and list of talk page revisions const revisions = response.body.query.pages[0].revisions; + // todo: length error checking here, parentid check here + const nextRvStartId = revisions[revisions.length - 1].parentid; + + const articleEvalResults = getCachedAndUncachedItems(revisions, req, talkPageTitle(req)); + + // save cached article revisions to output + output.concat(articleEvalResults.cachedOutput); - // may be able to clean this up with map somehow http://bluebirdjs.com/docs/api/promise.map.html + const rvStart = revisions[0].timestamp; + // todo: length error checking here + const rvEnd = revisions[revisions.length - 1].timestamp; - return BBPromise.map(revisions, function(revision) { - return diffPromise(req, revision.revid, revision.parentid); + return BBPromise.props({ + articleDiffAndRevisions: diffAndRevisionPromises(req, articleEvalResults.uncachedRevisions), + talkPageRevisions: talkPageRevisionsPromise(req, rvStart, rvEnd), + nextRvStartId: nextRvStartId }); }) .then( (response) => { - // eslint-disable-next-line no-console + + // STEP 3: All at once gather diffs for uncached talk page revisions + + const talkPageRevisions = response.talkPageRevisions.body.query.pages[0].revisions; + const articleDiffAndRevisions = response.articleDiffAndRevisions; + const nextRvStartId = response.nextRvStartId; + + const talkPageEvalResults = getCachedAndUncachedItems(talkPageRevisions, req, null); + + // save cached talk page revisions to output + output.concat(talkPageEvalResults.cachedOutput); + + // for each uncached talk page revision, gather diffs + return diffAndRevisionPromises(req, talkPageEvalResults.uncachedRevisions) + .then( (response) => { + return Object.assign({ + articleDiffAndRevisions: articleDiffAndRevisions, + talkDiffAndRevisions: response, + nextRvStartId: nextRvStartId + }); + }); + }) + .then( (response) => { + + // Determine byte size of change for every diff line and aggregate for every revision + updateDiffAndRevisionsWithByteCount(response.articleDiffAndRevisions); + + return response; + }) + .then( (response) => { + console.log(response); + + const threshold = req.query.threshold === null || req.query.threshold === undefined ? + 100 : req.query.threshold; + + //segment off large changes and small changes + response.articleDiffAndRevisions.forEach(function (diffAndRevision) { + if (diffAndRevision.byteChange.totalCount() <= threshold) { + const revision = diffAndRevision.revision; + const smallOutputObject = new SmallOutput(revision.revid, revision.timestamp, "small-change"); + output.push(smallOutputObject); + const cacheKey = significantChangesCacheKey(req, null, revision.revid); + significantChangesCache[cacheKey] = smallOutputObject; + } else { + const revision = diffAndRevision.revision; + + //get largest diff + diffAndRevision.body.diff.sort(function(a, b) { + return b.byteChange.totalCount() - a.byteChange.totalCount(); + }); + + //todo: safety + const largestDiffLine = diffAndRevision.body.diff[0]; + + const largeOutputObject = new LargeOutput(revision.revid, revision.timestamp, "large-change", largestDiffLine.text); + output.push(largeOutputObject); + const cacheKey = significantChangesCacheKey(req, null, revision.revid); + significantChangesCache[cacheKey] = largeOutputObject; + } + }); + + return output; + }) + .then( (output) => { + + console.log(output); }); } + + function getSignificantChanges(req, res) { return BBPromise.props({ mw: mwapi.queryForRevisions(req) From 8c0b26865239709a5af1d90ff96ecb97686da4f1 Mon Sep 17 00:00:00 2001 From: Toni Sevener Date: Mon, 13 Jul 2020 20:33:24 -0500 Subject: [PATCH 05/47] more progress processing data --- routes/page/significant-changes.js | 118 +++++++++++++++++++++-------- 1 file changed, 85 insertions(+), 33 deletions(-) diff --git a/routes/page/significant-changes.js b/routes/page/significant-changes.js index 53fb5402..b56a53d3 100644 --- a/routes/page/significant-changes.js +++ b/routes/page/significant-changes.js @@ -21,30 +21,36 @@ const diffAndRevisionPromise = (req, revision) => { }); }; -//curl -X POST "https://en.wikipedia.org/api/rest_v1/transform/wikitext/to/html/Dog" -H "accept: text/html; charset=utf-8; profile="https://www.mediawiki.org/wiki/Specs/HTML/2.1.0"" -H "Content-Type: multipart/form-data" -F "wikitext='testing'" -F "body_only=true" -F "stash=true" -//"https://en.wikipedia.org/api/rest_v1/transform/wikitext/to/html/Dog" -H "accept: text/html; charset=utf-8; profile="https://www.mediawiki.org/wiki/Specs/HTML/2.1.0"" -H "Content-Type: multipart/form-data" -F "wikitext='testing'" -F "body_only=true" -F "stash=true" -//wat console write -/* -{ method: 'get', - uri: - "https://en.wikipedia.org/w/rest.php/v1/revision/967289946/compare/967285956", - query: {}, - headers: {}, - body: undefined, - timeout: 60000 } - */ - -//call like req.issueRequest(wat); -//then chain parsoidApi.mobileHTMLPromiseFromHTML -// const snippetAndRevisionPromise = (app, req, res, html, revision) => { -// return mwrestapi.queryForDiff(req, revision.revid, revision.parentid) -// .then( (response) => { -// return Object.assign({ -// revision: revision, -// body: response.body -// }); -// }); -// }; +const snippetAndRevisionPromise = (req, largeChange) => { + const headers = Object.assign( { + accept: 'text/html', + profile: 'https://www.mediawiki.org/wiki/Specs/Mobile-HTML/1.0.0', + 'content-type': 'multipart/form-data' + }); + const formData = Object.assign({ + wikitext: largeChange.snippet + }); + + const request = Object.assign({ + method: 'post', + uri: `https://en.wikipedia.org/api/rest_v1/transform/wikitext/to/mobile-html/${req.params.title}`, + query: {}, + headers: headers, + body: formData + }); + + return req.issueRequest(request) + .then( (response) => { + largeChange.snippet = response.body; + return largeChange; + }); +}; + +const snippetAndRevisionPromises = (req, largeChanges) => { + return BBPromise.map(largeChanges, function(largeChange) { + return snippetAndRevisionPromise(req, largeChange); + }); +}; const diffAndRevisionPromises = (req, revisions) => { return BBPromise.map(revisions, function(revision) { @@ -98,18 +104,25 @@ class ByteChange { } class SmallOutput { - constructor(revid, timestamp, outputType) { + constructor(revid, timestamp) { this.revid = revid; this.timestamp = timestamp; - this.outputType = outputType; + this.outputType = "small-change"; + } +} + +class ConsolidatedSmallOutput { + constructor(count) { + this.count = count; + this.outputType = "small-change"; } } class LargeOutput { - constructor(revid, timestamp, outputType, snippet) { + constructor(revid, timestamp, snippet) { this.revid = revid; this.timestamp = timestamp; - this.outputType = outputType; + this.outputType = "large-change"; this.snippet = snippet; } } @@ -170,6 +183,32 @@ function updateDiffAndRevisionsWithByteCount(diffAndRevisions) { console.log(diffAndRevisions); } +function collapseOutput(output) { + + const collapsedOutput = []; + let numSmallChanges = 0; + output.forEach(function (revision) { + if (revision.outputType === 'small-change') { + numSmallChanges++; + return; + } else if (numSmallChanges > 0) { + const change = new ConsolidatedSmallOutput(numSmallChanges); + collapsedOutput.push(change); + numSmallChanges = 0; + } + + collapsedOutput.push(revision); + }); + + if (numSmallChanges > 0) { + const change = new ConsolidatedSmallOutput(numSmallChanges); + collapsedOutput.push(change); + numSmallChanges = 0; + } + + return collapsedOutput; +} + function getSignificantChanges2(req, res) { const output = []; @@ -239,7 +278,7 @@ function getSignificantChanges2(req, res) { response.articleDiffAndRevisions.forEach(function (diffAndRevision) { if (diffAndRevision.byteChange.totalCount() <= threshold) { const revision = diffAndRevision.revision; - const smallOutputObject = new SmallOutput(revision.revid, revision.timestamp, "small-change"); + const smallOutputObject = new SmallOutput(revision.revid, revision.timestamp); output.push(smallOutputObject); const cacheKey = significantChangesCacheKey(req, null, revision.revid); significantChangesCache[cacheKey] = smallOutputObject; @@ -254,7 +293,7 @@ function getSignificantChanges2(req, res) { //todo: safety const largestDiffLine = diffAndRevision.body.diff[0]; - const largeOutputObject = new LargeOutput(revision.revid, revision.timestamp, "large-change", largestDiffLine.text); + const largeOutputObject = new LargeOutput(revision.revid, revision.timestamp, largestDiffLine.text); output.push(largeOutputObject); const cacheKey = significantChangesCacheKey(req, null, revision.revid); significantChangesCache[cacheKey] = largeOutputObject; @@ -265,11 +304,24 @@ function getSignificantChanges2(req, res) { }) .then( (output) => { - console.log(output); - }); -} + //convert large snippets from wikitext to mobile-html + const largeOutputs = output.filter(item => item.outputType === 'large-change'); + return snippetAndRevisionPromises(req, largeOutputs) + .then( (response) => { + return output; + }); + }) + .then( (convertedSnippetOutput) => { + console.log(convertedSnippetOutput); + //todo: should we sort output by timestamp here? + + const collapsedOutput = collapseOutput(output); + res.send(collapsedOutput).end(); + + }); +} function getSignificantChanges(req, res) { return BBPromise.props({ From c9b299152a6f0b68ffc99e630d271acc71c73bb7 Mon Sep 17 00:00:00 2001 From: Toni Sevener Date: Wed, 15 Jul 2020 17:32:44 -0500 Subject: [PATCH 06/47] more progress processing data (trimming snippets, fixing bugs, starting section detection) --- lib/mwapi.js | 13 +- package-lock.json | 5 + package.json | 1 + routes/page/significant-changes.js | 424 +++++++++++++++++++++-------- 4 files changed, 329 insertions(+), 114 deletions(-) diff --git a/lib/mwapi.js b/lib/mwapi.js index bd7041e5..c6d526f3 100644 --- a/lib/mwapi.js +++ b/lib/mwapi.js @@ -135,12 +135,17 @@ mwapi.queryForRevisions = function(req, title, pageSize, rvStart, rvEnd) { rvprop: 'ids|timestamp|user|userid|size|parsedcomment|comment|tags|flags|size', rvdir: 'older', format: 'json', - rvlimit: pageSize || '20', - rvstartid: req.query.rvstartid, - rvstart: rvStart, - rvend: rvEnd + rvlimit: pageSize || '20' }); + //to get around "parameters rvstartid & rvstart cannot be used together" error + if (rvStart && rvEnd) { + query.rvstart = rvStart; + query.rvend = rvEnd; + } else { + query.rvstartid = req.query.rvstartid; + } + return api.mwApiGet(req, query); }; diff --git a/package-lock.json b/package-lock.json index a9773582..21a2439a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -135,6 +135,11 @@ "to-fast-properties": "^2.0.0" } }, + "@root/encoding": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@root/encoding/-/encoding-1.0.1.tgz", + "integrity": "sha512-OaEub02ufoU038gy6bsNHQOjIn8nUjGiLcaRmJ40IUykneJkIW5fxDqKxQx48cszuNflYldsJLPPXCrGfHs8yQ==" + }, "@sinonjs/commons": { "version": "1.7.1", "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-1.7.1.tgz", diff --git a/package.json b/package.json index 9ae3cdbc..e0faaf95 100644 --- a/package.json +++ b/package.json @@ -54,6 +54,7 @@ }, "homepage": "https://www.mediawiki.org/wiki/RESTBase_services_for_apps", "dependencies": { + "@root/encoding": "^1.0.1", "sync-request": "^6.1.0", "banana-i18n": "^1.2.1", "bluebird": "^3.7.2", diff --git a/routes/page/significant-changes.js b/routes/page/significant-changes.js index b56a53d3..81636135 100644 --- a/routes/page/significant-changes.js +++ b/routes/page/significant-changes.js @@ -6,13 +6,86 @@ const BBPromise = require('bluebird'); const api = require('../../lib/api-util'); const express = require('express'); const parsoidApi = require('../../lib/parsoid-access'); +const encoding = require('@root/encoding'); let app; const significantChangesCache = {}; +String.prototype.insert = function(index, string) { + if (index > 0) { + return this.substring(0, index) + string + this.substring(index, this.length); + } + + return string + this; +}; + +class CharacterChange { + constructor(addedCount, deletedCount) { + this.addedCount = addedCount; + this.deletedCount = deletedCount; + } + + totalCount() { + return this.addedCount + this.deletedCount; + } +} + +class SmallOutput { + constructor(revid, timestamp, user, userid, characterChange) { + this.revid = revid; + this.timestamp = timestamp; + this.outputType = 'small-change'; + this.user = user; + this.userid = userid; + this.characterChange = characterChange; + } +} + +class ConsolidatedSmallOutput { + constructor(count) { + this.count = count; + this.outputType = 'small-change'; + } +} + +class LargeOutput { + constructor(largeOutputExpanded) { + this.revid = largeOutputExpanded.revid; + this.timestamp = largeOutputExpanded.timestamp; + this.outputType = 'large-change'; + this.snippet = largeOutputExpanded.snippet; + this.user = largeOutputExpanded.user; + this.userid = largeOutputExpanded.userid; + this.characterChange = largeOutputExpanded.characterChange; + this.section = largeOutputExpanded.section; + } +} + +class LargeOutputExpanded { + constructor(revid, timestamp, user, userid, snippet, type, highlightRanges, characterChange, section) { + this.revid = revid; + this.timestamp = timestamp; + this.outputType = 'large-change'; + this.snippet = snippet; + this.type = type; + this.highlightRanges = highlightRanges; + this.user = user; + this.userid = userid; + this.characterChange = characterChange; + this.section = section; + } +} + +class Section { + constructor(fromSection, toSection) { + this.fromSection = fromSection; + this.toSection = toSection; + } +} + const diffAndRevisionPromise = (req, revision) => { - return mwrestapi.queryForDiff(req, revision.revid, revision.parentid) + return mwrestapi.queryForDiff(req, revision.parentid, revision.revid) .then( (response) => { return Object.assign({ revision: revision, @@ -21,14 +94,67 @@ const diffAndRevisionPromise = (req, revision) => { }); }; -const snippetAndRevisionPromise = (req, largeChange) => { +const snippetPromise = (req, largeChange) => { const headers = Object.assign( { accept: 'text/html', profile: 'https://www.mediawiki.org/wiki/Specs/Mobile-HTML/1.0.0', - 'content-type': 'multipart/form-data' + 'content-type': 'multipart/form-data', + 'output-mode': 'editPreview' }); + + // add highlight delimiters first + var snippetBinary = encoding.strToBin(largeChange.snippet); + + //todo: it looks like parsoid *sometimes* strips these spans out depending on their placement. we need some token that parsoid won't touch. + //see results for this revision, it's missing an add-highlight. + //https://en.wikipedia.org/w/index.php?title=United_States&type=revision&diff=965295364&oldid=965071033 + const addHighlightStart = ''; + const deleteHighlightStart = ''; + const highlightEnd = '<\/span>'; + + const addHighlightStartBin = encoding.strToBin(addHighlightStart); + const deleteHighlightStartBin = encoding.strToBin(deleteHighlightStart); + const highlightEndBin = encoding.strToBin(highlightEnd); + + switch (largeChange.type) { + case 1: // Added complete line + snippetBinary = snippetBinary.insert(0, addHighlightStartBin); + snippetBinary = snippetBinary.insert(snippetBinary.length, highlightEndBin); + break; + case 2: // Deleted complete line + snippetBinary = snippetBinary.insert(0, deleteHighlightStart); + snippetBinary = snippetBinary.insert(snippetBinary.length, highlightEndBin); + break; + case 5: + case 3: // Added and deleted words in line + var offset = 0; + largeChange.highlightRanges.forEach(function (range) { + switch (range.type) { + case 0: // Added + snippetBinary = snippetBinary.insert(range.start + offset, addHighlightStartBin); + offset += addHighlightStartBin.length; + break; + case 1: // Deleted + snippetBinary = snippetBinary.insert(range.start + offset, deleteHighlightStartBin); + offset += deleteHighlightStartBin.length; + break; + default: + return; + } + snippetBinary = snippetBinary.insert(range.start + offset + range.length, highlightEndBin); + offset += highlightEndBin.length; + }); + break; + default: + break; + } + + const newSnippet = encoding.binToStr(snippetBinary); + + // make request to format to mobile-html, reassign result back to snippet + const formData = Object.assign({ - wikitext: largeChange.snippet + wikitext: newSnippet }); const request = Object.assign({ @@ -41,14 +167,44 @@ const snippetAndRevisionPromise = (req, largeChange) => { return req.issueRequest(request) .then( (response) => { - largeChange.snippet = response.body; + return mUtil.createDocument(response.body); + }).then((response) => { + + //removing script tags here + //todo: try to remove first section that is inserted too + const scripts = response.body.getElementsByTagName('script'); + const scriptsList = Array.prototype.slice.call(scripts); + const references = response.body.getElementsByClassName('mw-references-wrap'); + const referencesList = Array.prototype.slice.call(references); + const finalListToStrip = scriptsList.concat(referencesList); + finalListToStrip.forEach( (script) => { + script.parentNode.removeChild(script); + }); + + //mobile-html endpoint seems to return a wrapper pcs, section and paragraph element. strip these out as well. + + var strippedSnippet = response.body.innerHTML; + const pcsElement = response.getElementById('pcs'); + if (pcsElement) { + const sectionWrapper = pcsElement.firstChild; + if (sectionWrapper.tagName === 'SECTION') { + strippedSnippet = sectionWrapper.innerHTML; + } + + const paragraphWrapper = sectionWrapper.firstChild; + if (paragraphWrapper.tagName === 'P') { + strippedSnippet = paragraphWrapper.innerHTML; + } + } + + largeChange.snippet = strippedSnippet; return largeChange; }); }; -const snippetAndRevisionPromises = (req, largeChanges) => { +const snippetPromises = (req, largeChanges) => { return BBPromise.map(largeChanges, function(largeChange) { - return snippetAndRevisionPromise(req, largeChange); + return snippetPromise(req, largeChange); }); }; @@ -59,7 +215,7 @@ const diffAndRevisionPromises = (req, revisions) => { }; const talkPageTitle = (req) => { - return `Talk:${req.params.title}` + return `Talk:${req.params.title}`; }; const talkPageRevisionsPromise = (req, rvStart, rvEnd) => { @@ -68,7 +224,7 @@ const talkPageRevisionsPromise = (req, rvStart, rvEnd) => { const significantChangesCacheKey = (req, title, revision) => { const keyTitle = title || req.params.title; - return `${req.params.domain}-${title}-${revision.revid}`; + return `${req.params.domain}-${keyTitle}-${revision.revid}`; }; function getCachedAndUncachedItems(revisions, req, title) { @@ -92,43 +248,7 @@ function getCachedAndUncachedItems(revisions, req, title) { }); } -class ByteChange { - constructor(addedCount, deletedCount) { - this.addedCount = addedCount; - this.deletedCount = deletedCount; - } - - totalCount() { - return this.addedCount + this.deletedCount; - } -} - -class SmallOutput { - constructor(revid, timestamp) { - this.revid = revid; - this.timestamp = timestamp; - this.outputType = "small-change"; - } -} - -class ConsolidatedSmallOutput { - constructor(count) { - this.count = count; - this.outputType = "small-change"; - } -} - -class LargeOutput { - constructor(revid, timestamp, snippet) { - this.revid = revid; - this.timestamp = timestamp; - this.outputType = "large-change"; - this.snippet = snippet; - } -} - -function updateDiffAndRevisionsWithByteCount(diffAndRevisions) { - console.log(diffAndRevisions); +function updateDiffAndRevisionsWithCharacterCount(diffAndRevisions) { // Loop through diffs, filter out type 0 (context type) and assign byte change properties to the remaining @@ -150,15 +270,21 @@ function updateDiffAndRevisionsWithByteCount(diffAndRevisions) { case 2: // Delete complete line type lineDeletedCount = diff.text.length; break; + case 5: //Move paragraph destination type (also has added and deleted ranges in line) case 3: // Change line type (add and deleted ranges in line) - //todo: there's something funky with added and deleted types. see United_States revid 966656680, says added 172 bytes (type 0 highlighted range) but UI shows deleted in desktop and app. diff.highlightRanges.forEach(function (range) { + + const binaryText = encoding.strToBin(diff.text); + const binaryRangeText = binaryText.substring(range.start, range.start + range.length); + const rangeText = encoding.binToStr(binaryRangeText); + const lengthToCalculate = rangeText.length; + switch (range.type) { case 0: // Add range type - lineAddedCount += range.length; + lineAddedCount += lengthToCalculate; break; case 1: // Delete range type - lineDeletedCount += range.length; + lineDeletedCount += lengthToCalculate; break; default: break; @@ -172,47 +298,121 @@ function updateDiffAndRevisionsWithByteCount(diffAndRevisions) { aggregateAddedCount += lineAddedCount; aggregateDeletedCount += lineDeletedCount; - diff.byteChange = new ByteChange(lineAddedCount, lineDeletedCount); + diff.characterChange = new CharacterChange(lineAddedCount, lineDeletedCount); filteredDiffs.push(diff); }); - diffAndRevision.byteChange = new ByteChange(aggregateAddedCount, aggregateDeletedCount) + diffAndRevision.characterChange = new CharacterChange(aggregateAddedCount, aggregateDeletedCount); diffAndRevision.body.diff = filteredDiffs; }); +} + +function getSectionForDiffLine(diffBody, diffLine) { + + var fromSection = null; + var toSection = null; + + //capture intro + if (!diffBody.from.sections || + diffBody.from.sections.length === 0 || + !diffBody.to.sections || + diffBody.to.sections.length === 0) { + return null; + } + + if (diffLine.offset.from < diffBody.from.sections[0].offset) { + fromSection = 'Intro'; + } + + if (diffLine.offset.to < diffBody.to.sections[0].offset) { + toSection = 'Intro'; + } + + if (fromSection && toSection) { + return new Section(fromSection, toSection); + } + + var prevSection = null; + if (!fromSection && diffLine.offset.from) { + for (let i = 0; i < diffBody.from.sections.length; i++) { + const section = diffBody.from.sections[i]; + + if (diffLine.offset.from < section.offset && prevSection) { + fromSection = prevSection.title; + break; + } + + prevSection = section; + } - console.log(diffAndRevisions); + if (!fromSection && diffLine.offset.from > 0) { + fromSection = prevSection.title; + } + } + + + if (!toSection && diffLine.offset.to) { + prevSection = null; + for (let i = 0; i < diffBody.to.sections.length; i++) { + + const section = diffBody.to.sections[i]; + if (diffLine.offset.to < section.offset && prevSection) { + toSection = prevSection.title; + break; + } + + prevSection = section; + } + + if (!toSection && diffLine.offset.to > 0) { + toSection = prevSection.title; + } + } + + return new Section(fromSection, toSection); } -function collapseOutput(output) { +function cleanOutput(output) { + + //collapses small changes, converts large changes only to info needed - const collapsedOutput = []; + //sort by date first + output = output.sort(function(a, b) { + return new Date(b.timestamp) - new Date(a.timestamp); + }); + + const cleanedOutput = []; let numSmallChanges = 0; - output.forEach(function (revision) { - if (revision.outputType === 'small-change') { + output.forEach(function (item) { + if (item.outputType === 'small-change') { numSmallChanges++; return; - } else if (numSmallChanges > 0) { - const change = new ConsolidatedSmallOutput(numSmallChanges); - collapsedOutput.push(change); - numSmallChanges = 0; - } + } else { + if (numSmallChanges > 0) { + const change = new ConsolidatedSmallOutput(numSmallChanges); + cleanedOutput.push(change); + numSmallChanges = 0; + } - collapsedOutput.push(revision); + if (item.outputType === 'large-change') { + cleanedOutput.push(new LargeOutput(item)); + } else { + cleanedOutput.push(item); + } + } }); if (numSmallChanges > 0) { const change = new ConsolidatedSmallOutput(numSmallChanges); - collapsedOutput.push(change); + cleanedOutput.push(change); numSmallChanges = 0; } - return collapsedOutput; + return cleanedOutput; } function getSignificantChanges2(req, res) { - const output = []; - // STEP 1: Gather list of article revisions return mwapi.queryForRevisions(req) .then( (response) => { @@ -222,10 +422,11 @@ function getSignificantChanges2(req, res) { // todo: length error checking here, parentid check here const nextRvStartId = revisions[revisions.length - 1].parentid; - const articleEvalResults = getCachedAndUncachedItems(revisions, req, talkPageTitle(req)); + const articleEvalResults = getCachedAndUncachedItems(revisions, req, null); - // save cached article revisions to output - output.concat(articleEvalResults.cachedOutput); + // save cached article revisions to finalOutput + var finalOutput = []; + finalOutput = finalOutput.concat(articleEvalResults.cachedOutput); const rvStart = revisions[0].timestamp; // todo: length error checking here @@ -234,7 +435,8 @@ function getSignificantChanges2(req, res) { return BBPromise.props({ articleDiffAndRevisions: diffAndRevisionPromises(req, articleEvalResults.uncachedRevisions), talkPageRevisions: talkPageRevisionsPromise(req, rvStart, rvEnd), - nextRvStartId: nextRvStartId + nextRvStartId: nextRvStartId, + finalOutput: finalOutput }); }) .then( (response) => { @@ -245,10 +447,10 @@ function getSignificantChanges2(req, res) { const articleDiffAndRevisions = response.articleDiffAndRevisions; const nextRvStartId = response.nextRvStartId; - const talkPageEvalResults = getCachedAndUncachedItems(talkPageRevisions, req, null); + const talkPageEvalResults = getCachedAndUncachedItems(talkPageRevisions, req, talkPageTitle(req)); // save cached talk page revisions to output - output.concat(talkPageEvalResults.cachedOutput); + const finalOutput = response.finalOutput.concat(talkPageEvalResults.cachedOutput); // for each uncached talk page revision, gather diffs return diffAndRevisionPromises(req, talkPageEvalResults.uncachedRevisions) @@ -256,69 +458,71 @@ function getSignificantChanges2(req, res) { return Object.assign({ articleDiffAndRevisions: articleDiffAndRevisions, talkDiffAndRevisions: response, - nextRvStartId: nextRvStartId + nextRvStartId: nextRvStartId, + finalOutput: finalOutput }); }); }) .then( (response) => { - // Determine byte size of change for every diff line and aggregate for every revision - updateDiffAndRevisionsWithByteCount(response.articleDiffAndRevisions); + // Determine character size of change for every diff line and aggregate for every revision + updateDiffAndRevisionsWithCharacterCount(response.articleDiffAndRevisions); return response; }) .then( (response) => { - console.log(response); - const threshold = req.query.threshold === null || req.query.threshold === undefined ? 100 : req.query.threshold; - //segment off large changes and small changes + // segment off large changes and small changes + var uncachedOutput = []; response.articleDiffAndRevisions.forEach(function (diffAndRevision) { - if (diffAndRevision.byteChange.totalCount() <= threshold) { + if (diffAndRevision.characterChange.totalCount() <= threshold) { const revision = diffAndRevision.revision; - const smallOutputObject = new SmallOutput(revision.revid, revision.timestamp); - output.push(smallOutputObject); - const cacheKey = significantChangesCacheKey(req, null, revision.revid); - significantChangesCache[cacheKey] = smallOutputObject; + const smallOutputObject = new SmallOutput(revision.revid, revision.timestamp, revision.user, revision.userid, diffAndRevision.characterChange); + uncachedOutput.push(smallOutputObject); } else { const revision = diffAndRevision.revision; - //get largest diff + // get largest diff diffAndRevision.body.diff.sort(function(a, b) { - return b.byteChange.totalCount() - a.byteChange.totalCount(); + return b.characterChange.totalCount() - a.characterChange.totalCount(); }); - //todo: safety + // todo: safety const largestDiffLine = diffAndRevision.body.diff[0]; - - const largeOutputObject = new LargeOutput(revision.revid, revision.timestamp, largestDiffLine.text); - output.push(largeOutputObject); - const cacheKey = significantChangesCacheKey(req, null, revision.revid); - significantChangesCache[cacheKey] = largeOutputObject; + const diffSection = getSectionForDiffLine(diffAndRevision.body, largestDiffLine); + const largeOutputObject = new LargeOutputExpanded(revision.revid, revision.timestamp, revision.user, revision.userid, largestDiffLine.text, largestDiffLine.type, largestDiffLine.highlightRanges, diffAndRevision.characterChange, diffSection); + uncachedOutput.push(largeOutputObject); } }); - return output; + return Object.assign({ nextRvStartId: response.nextRvStartId, uncachedOutput: uncachedOutput, finalOutput: response.finalOutput } ); }) - .then( (output) => { + .then( (response) => { - //convert large snippets from wikitext to mobile-html - const largeOutputs = output.filter(item => item.outputType === 'large-change'); - return snippetAndRevisionPromises(req, largeOutputs) - .then( (response) => { - return output; + // convert large snippets from wikitext to mobile-html + const largeOutputs = response.uncachedOutput.filter(item => item.outputType === 'large-change'); + return snippetPromises(req, largeOutputs) + .then( (snippetResponse) => { + + //push to final output and cache + //note we are using original response list, not snippet response (snippet only contains large) + response.uncachedOutput.forEach((item) => { + response.finalOutput.push(item); + const cacheKey = significantChangesCacheKey(req, null, item); + significantChangesCache[cacheKey] = item; + }); + + return Object.assign({ nextRvStartId: response.nextRvStartId, finalOutput: response.finalOutput } ); }); }) - .then( (convertedSnippetOutput) => { - - console.log(convertedSnippetOutput); - - //todo: should we sort output by timestamp here? + .then( (response) => { - const collapsedOutput = collapseOutput(output); - res.send(collapsedOutput).end(); + const cleanedOutput = cleanOutput(response.finalOutput); + const result = Object.assign({ nextRvStartId: response.nextRvStartId, significantChanges: cleanedOutput } ); + res.send(result).end(); }); } @@ -470,18 +674,18 @@ function getSignificantChanges(req, res) { changedLines.forEach(function (diff) { if (diff.type === 1 || diff.type === 2) { - diff.byteChange = diff.text.length; + diff.characterChange = diff.text.length; } else { - var byteChange = 0; + var characterChange = 0; diff.highlightRanges.forEach(function (range) { - byteChange += range.length; + characterChange += range.length; }); - diff.byteChange = byteChange; + diff.characterChange = characterChange; } }); changedLines.sort(function(a, b) { - return b.byteChange - a.byteChange; + return b.characterChange - a.characterChange; }); if (changedLines.length > 0) { var changeType; From d6b09b3f1891b4fab082b9a35b964c468a25f927 Mon Sep 17 00:00:00 2001 From: Toni Sevener Date: Thu, 16 Jul 2020 09:57:52 -0500 Subject: [PATCH 07/47] fix some section bugs, fix linter warnings --- lib/api-util.js | 2 - lib/mwapi.js | 2 +- routes/page/significant-changes.js | 135 ++++++++++++++++++----------- 3 files changed, 84 insertions(+), 55 deletions(-) diff --git a/lib/api-util.js b/lib/api-util.js index 5ed850df..59982a16 100644 --- a/lib/api-util.js +++ b/lib/api-util.js @@ -51,8 +51,6 @@ function mwRestApiGet(req, path, restReq) { restReq.params.path = restReq.params.path[0] === '/' ? restReq.params.path.slice(1) : restReq.params.path; - var wat = app.mwrestapi_tbl.expand({ request: restReq }); - console.log(wat); return req.issueRequest(app.mwrestapi_tbl.expand({ request: restReq })); } diff --git a/lib/mwapi.js b/lib/mwapi.js index c6d526f3..0db7b8c9 100644 --- a/lib/mwapi.js +++ b/lib/mwapi.js @@ -138,7 +138,7 @@ mwapi.queryForRevisions = function(req, title, pageSize, rvStart, rvEnd) { rvlimit: pageSize || '20' }); - //to get around "parameters rvstartid & rvstart cannot be used together" error + // to get around "parameters rvstartid & rvstart cannot be used together" error if (rvStart && rvEnd) { query.rvstart = rvStart; query.rvend = rvEnd; diff --git a/routes/page/significant-changes.js b/routes/page/significant-changes.js index 81636135..7a2ae5fc 100644 --- a/routes/page/significant-changes.js +++ b/routes/page/significant-changes.js @@ -12,14 +12,6 @@ let app; const significantChangesCache = {}; -String.prototype.insert = function(index, string) { - if (index > 0) { - return this.substring(0, index) + string + this.substring(index, this.length); - } - - return string + this; -}; - class CharacterChange { constructor(addedCount, deletedCount) { this.addedCount = addedCount; @@ -63,7 +55,8 @@ class LargeOutput { } class LargeOutputExpanded { - constructor(revid, timestamp, user, userid, snippet, type, highlightRanges, characterChange, section) { + constructor(revid, timestamp, user, userid, snippet, type, highlightRanges, characterChange, + section) { this.revid = revid; this.timestamp = timestamp; this.outputType = 'large-change'; @@ -84,6 +77,15 @@ class Section { } } +function insertSubstringInString(originalString, substring, index) { + if (index > 0) { + return originalString.substring(0, index) + substring + originalString.substring(index, + originalString.length); + } + + return substring + originalString; +} + const diffAndRevisionPromise = (req, revision) => { return mwrestapi.queryForDiff(req, revision.parentid, revision.revid) .then( (response) => { @@ -105,12 +107,14 @@ const snippetPromise = (req, largeChange) => { // add highlight delimiters first var snippetBinary = encoding.strToBin(largeChange.snippet); - //todo: it looks like parsoid *sometimes* strips these spans out depending on their placement. we need some token that parsoid won't touch. - //see results for this revision, it's missing an add-highlight. - //https://en.wikipedia.org/w/index.php?title=United_States&type=revision&diff=965295364&oldid=965071033 - const addHighlightStart = ''; - const deleteHighlightStart = ''; - const highlightEnd = '<\/span>'; + // todo: it looks like parsoid *sometimes* strips these spans out depending on their placement. + // we need some token that parsoid won't touch. + // see results for this revision, it's missing an add-highlight. + // https://en.wikipedia.org/w/index.php?title=United_States + // &type=revision&diff=965295364&oldid=965071033 + const addHighlightStart = ''; + const deleteHighlightStart = ''; + const highlightEnd = ''; const addHighlightStartBin = encoding.strToBin(addHighlightStart); const deleteHighlightStartBin = encoding.strToBin(deleteHighlightStart); @@ -118,12 +122,16 @@ const snippetPromise = (req, largeChange) => { switch (largeChange.type) { case 1: // Added complete line - snippetBinary = snippetBinary.insert(0, addHighlightStartBin); - snippetBinary = snippetBinary.insert(snippetBinary.length, highlightEndBin); + + snippetBinary = insertSubstringInString(snippetBinary, addHighlightStartBin, 0); + snippetBinary = insertSubstringInString(snippetBinary, highlightEndBin, + snippetBinary.length); break; case 2: // Deleted complete line - snippetBinary = snippetBinary.insert(0, deleteHighlightStart); - snippetBinary = snippetBinary.insert(snippetBinary.length, highlightEndBin); + + snippetBinary = insertSubstringInString(snippetBinary, deleteHighlightStart, 0); + snippetBinary = insertSubstringInString(snippetBinary, highlightEndBin, + snippetBinary.length); break; case 5: case 3: // Added and deleted words in line @@ -131,17 +139,21 @@ const snippetPromise = (req, largeChange) => { largeChange.highlightRanges.forEach(function (range) { switch (range.type) { case 0: // Added - snippetBinary = snippetBinary.insert(range.start + offset, addHighlightStartBin); + snippetBinary = insertSubstringInString(snippetBinary, + addHighlightStartBin, range.start + offset); offset += addHighlightStartBin.length; break; case 1: // Deleted - snippetBinary = snippetBinary.insert(range.start + offset, deleteHighlightStartBin); + snippetBinary = insertSubstringInString(snippetBinary, + deleteHighlightStartBin, range.start + offset); offset += deleteHighlightStartBin.length; break; default: return; } - snippetBinary = snippetBinary.insert(range.start + offset + range.length, highlightEndBin); + + snippetBinary = insertSubstringInString(snippetBinary, highlightEndBin, + range.start + offset + range.length); offset += highlightEndBin.length; }); break; @@ -170,8 +182,8 @@ const snippetPromise = (req, largeChange) => { return mUtil.createDocument(response.body); }).then((response) => { - //removing script tags here - //todo: try to remove first section that is inserted too + // removing script tags here + // todo: try to remove first section that is inserted too const scripts = response.body.getElementsByTagName('script'); const scriptsList = Array.prototype.slice.call(scripts); const references = response.body.getElementsByClassName('mw-references-wrap'); @@ -181,7 +193,8 @@ const snippetPromise = (req, largeChange) => { script.parentNode.removeChild(script); }); - //mobile-html endpoint seems to return a wrapper pcs, section and paragraph element. strip these out as well. + // mobile-html endpoint seems to return a wrapper pcs, section and paragraph element. + // strip these out as well. var strippedSnippet = response.body.innerHTML; const pcsElement = response.getElementById('pcs'); @@ -250,7 +263,8 @@ function getCachedAndUncachedItems(revisions, req, title) { function updateDiffAndRevisionsWithCharacterCount(diffAndRevisions) { - // Loop through diffs, filter out type 0 (context type) and assign byte change properties to the remaining + // Loop through diffs, filter out type 0 (context type) and assign byte change properties + // to the remaining diffAndRevisions.forEach(function (diffAndRevision) { @@ -270,12 +284,15 @@ function updateDiffAndRevisionsWithCharacterCount(diffAndRevisions) { case 2: // Delete complete line type lineDeletedCount = diff.text.length; break; - case 5: //Move paragraph destination type (also has added and deleted ranges in line) + case 5: // Move paragraph destination type (also has added and deleted + // ranges in line) + // eslint-disable-next-line no-fallthrough case 3: // Change line type (add and deleted ranges in line) diff.highlightRanges.forEach(function (range) { const binaryText = encoding.strToBin(diff.text); - const binaryRangeText = binaryText.substring(range.start, range.start + range.length); + const binaryRangeText = binaryText.substring(range.start, + range.start + range.length); const rangeText = encoding.binToStr(binaryRangeText); const lengthToCalculate = rangeText.length; @@ -302,7 +319,8 @@ function updateDiffAndRevisionsWithCharacterCount(diffAndRevisions) { filteredDiffs.push(diff); }); - diffAndRevision.characterChange = new CharacterChange(aggregateAddedCount, aggregateDeletedCount); + diffAndRevision.characterChange = new CharacterChange(aggregateAddedCount, + aggregateDeletedCount); diffAndRevision.body.diff = filteredDiffs; }); } @@ -312,7 +330,7 @@ function getSectionForDiffLine(diffBody, diffLine) { var fromSection = null; var toSection = null; - //capture intro + // capture intro if (!diffBody.from.sections || diffBody.from.sections.length === 0 || !diffBody.to.sections || @@ -320,11 +338,11 @@ function getSectionForDiffLine(diffBody, diffLine) { return null; } - if (diffLine.offset.from < diffBody.from.sections[0].offset) { + if (diffLine.offset.from && diffLine.offset.from < diffBody.from.sections[0].offset) { fromSection = 'Intro'; } - if (diffLine.offset.to < diffBody.to.sections[0].offset) { + if (diffLine.offset.to && diffLine.offset.to < diffBody.to.sections[0].offset) { toSection = 'Intro'; } @@ -338,7 +356,7 @@ function getSectionForDiffLine(diffBody, diffLine) { const section = diffBody.from.sections[i]; if (diffLine.offset.from < section.offset && prevSection) { - fromSection = prevSection.title; + fromSection = prevSection.heading; break; } @@ -346,18 +364,17 @@ function getSectionForDiffLine(diffBody, diffLine) { } if (!fromSection && diffLine.offset.from > 0) { - fromSection = prevSection.title; + fromSection = prevSection.heading; } } - if (!toSection && diffLine.offset.to) { prevSection = null; for (let i = 0; i < diffBody.to.sections.length; i++) { const section = diffBody.to.sections[i]; if (diffLine.offset.to < section.offset && prevSection) { - toSection = prevSection.title; + toSection = prevSection.heading; break; } @@ -365,7 +382,7 @@ function getSectionForDiffLine(diffBody, diffLine) { } if (!toSection && diffLine.offset.to > 0) { - toSection = prevSection.title; + toSection = prevSection.heading; } } @@ -374,9 +391,9 @@ function getSectionForDiffLine(diffBody, diffLine) { function cleanOutput(output) { - //collapses small changes, converts large changes only to info needed + // collapses small changes, converts large changes only to info needed - //sort by date first + // sort by date first output = output.sort(function(a, b) { return new Date(b.timestamp) - new Date(a.timestamp); }); @@ -417,7 +434,8 @@ function getSignificantChanges2(req, res) { return mwapi.queryForRevisions(req) .then( (response) => { - // STEP 2: All at once gather diffs for each uncached revision and list of talk page revisions + // STEP 2: All at once gather diffs for each uncached revision and list of + // talk page revisions const revisions = response.body.query.pages[0].revisions; // todo: length error checking here, parentid check here const nextRvStartId = revisions[revisions.length - 1].parentid; @@ -433,7 +451,8 @@ function getSignificantChanges2(req, res) { const rvEnd = revisions[revisions.length - 1].timestamp; return BBPromise.props({ - articleDiffAndRevisions: diffAndRevisionPromises(req, articleEvalResults.uncachedRevisions), + articleDiffAndRevisions: diffAndRevisionPromises(req, + articleEvalResults.uncachedRevisions), talkPageRevisions: talkPageRevisionsPromise(req, rvStart, rvEnd), nextRvStartId: nextRvStartId, finalOutput: finalOutput @@ -447,7 +466,8 @@ function getSignificantChanges2(req, res) { const articleDiffAndRevisions = response.articleDiffAndRevisions; const nextRvStartId = response.nextRvStartId; - const talkPageEvalResults = getCachedAndUncachedItems(talkPageRevisions, req, talkPageTitle(req)); + const talkPageEvalResults = getCachedAndUncachedItems(talkPageRevisions, + req, talkPageTitle(req)); // save cached talk page revisions to output const finalOutput = response.finalOutput.concat(talkPageEvalResults.cachedOutput); @@ -465,7 +485,8 @@ function getSignificantChanges2(req, res) { }) .then( (response) => { - // Determine character size of change for every diff line and aggregate for every revision + // Determine character size of change for every diff line and aggregate + // for every revision updateDiffAndRevisionsWithCharacterCount(response.articleDiffAndRevisions); return response; @@ -480,7 +501,8 @@ function getSignificantChanges2(req, res) { response.articleDiffAndRevisions.forEach(function (diffAndRevision) { if (diffAndRevision.characterChange.totalCount() <= threshold) { const revision = diffAndRevision.revision; - const smallOutputObject = new SmallOutput(revision.revid, revision.timestamp, revision.user, revision.userid, diffAndRevision.characterChange); + const smallOutputObject = new SmallOutput(revision.revid, revision.timestamp, + revision.user, revision.userid, diffAndRevision.characterChange); uncachedOutput.push(smallOutputObject); } else { const revision = diffAndRevision.revision; @@ -492,36 +514,45 @@ function getSignificantChanges2(req, res) { // todo: safety const largestDiffLine = diffAndRevision.body.diff[0]; - const diffSection = getSectionForDiffLine(diffAndRevision.body, largestDiffLine); - const largeOutputObject = new LargeOutputExpanded(revision.revid, revision.timestamp, revision.user, revision.userid, largestDiffLine.text, largestDiffLine.type, largestDiffLine.highlightRanges, diffAndRevision.characterChange, diffSection); + const diffSection = getSectionForDiffLine(diffAndRevision.body, + largestDiffLine); + const largeOutputObject = new LargeOutputExpanded(revision.revid, + revision.timestamp, revision.user, revision.userid, largestDiffLine.text, + largestDiffLine.type, largestDiffLine.highlightRanges, + diffAndRevision.characterChange, diffSection); uncachedOutput.push(largeOutputObject); } }); - return Object.assign({ nextRvStartId: response.nextRvStartId, uncachedOutput: uncachedOutput, finalOutput: response.finalOutput } ); + return Object.assign({ nextRvStartId: response.nextRvStartId, + uncachedOutput: uncachedOutput, finalOutput: response.finalOutput } ); }) .then( (response) => { // convert large snippets from wikitext to mobile-html - const largeOutputs = response.uncachedOutput.filter(item => item.outputType === 'large-change'); + const largeOutputs = response.uncachedOutput.filter(item => + item.outputType === 'large-change'); return snippetPromises(req, largeOutputs) .then( (snippetResponse) => { - //push to final output and cache - //note we are using original response list, not snippet response (snippet only contains large) + // push to final output and cache + // note we are using original response list, not snippet response + // (snippet only contains large) response.uncachedOutput.forEach((item) => { response.finalOutput.push(item); const cacheKey = significantChangesCacheKey(req, null, item); significantChangesCache[cacheKey] = item; }); - return Object.assign({ nextRvStartId: response.nextRvStartId, finalOutput: response.finalOutput } ); + return Object.assign({ nextRvStartId: response.nextRvStartId, + finalOutput: response.finalOutput } ); }); }) .then( (response) => { const cleanedOutput = cleanOutput(response.finalOutput); - const result = Object.assign({ nextRvStartId: response.nextRvStartId, significantChanges: cleanedOutput } ); + const result = Object.assign({ nextRvStartId: response.nextRvStartId, + significantChanges: cleanedOutput } ); res.send(result).end(); }); From 931b5b2373509bb5c892fb631ae8c6ded5e761fe Mon Sep 17 00:00:00 2001 From: Toni Sevener Date: Thu, 16 Jul 2020 11:23:28 -0500 Subject: [PATCH 08/47] fix threshold caching issue and some snippet stripping issue --- routes/page/significant-changes.js | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/routes/page/significant-changes.js b/routes/page/significant-changes.js index 7a2ae5fc..b28810e3 100644 --- a/routes/page/significant-changes.js +++ b/routes/page/significant-changes.js @@ -77,6 +77,11 @@ class Section { } } +function getThreshold(req) { + return req.query.threshold === null || req.query.threshold === undefined ? + 100 : req.query.threshold; +} + function insertSubstringInString(originalString, substring, index) { if (index > 0) { return originalString.substring(0, index) + substring + originalString.substring(index, @@ -120,6 +125,14 @@ const snippetPromise = (req, largeChange) => { const deleteHighlightStartBin = encoding.strToBin(deleteHighlightStart); const highlightEndBin = encoding.strToBin(highlightEnd); + // const addHighlightTokenStart = 'ios-add-token'; + // const deleteHighlightTokenStart = 'ios-delete-token'; + // const highlightTokenEnd = 'ios-end-token'; + // + // const addHighlightTokenStartBin = encoding.strToBin(addHighlightTokenStart); + // const deleteHighlightTokenStartBin = encoding.strToBin(deleteHighlightTokenStart); + // const highlightTokenEndBin = encoding.strToBin(highlightTokenEnd); + switch (largeChange.type) { case 1: // Added complete line @@ -129,7 +142,7 @@ const snippetPromise = (req, largeChange) => { break; case 2: // Deleted complete line - snippetBinary = insertSubstringInString(snippetBinary, deleteHighlightStart, 0); + snippetBinary = insertSubstringInString(snippetBinary, deleteHighlightStartBin, 0); snippetBinary = insertSubstringInString(snippetBinary, highlightEndBin, snippetBinary.length); break; @@ -206,7 +219,14 @@ const snippetPromise = (req, largeChange) => { const paragraphWrapper = sectionWrapper.firstChild; if (paragraphWrapper.tagName === 'P') { - strippedSnippet = paragraphWrapper.innerHTML; + + var parent = paragraphWrapper.parentNode; + while ( paragraphWrapper.firstChild ) { + parent.insertBefore( paragraphWrapper.firstChild, paragraphWrapper ); + } + parent.removeChild( paragraphWrapper ); + + strippedSnippet = sectionWrapper.innerHTML; } } @@ -236,8 +256,9 @@ const talkPageRevisionsPromise = (req, rvStart, rvEnd) => { }; const significantChangesCacheKey = (req, title, revision) => { + const threshold = getThreshold(req); const keyTitle = title || req.params.title; - return `${req.params.domain}-${keyTitle}-${revision.revid}`; + return `${req.params.domain}-${keyTitle}-${revision.revid}-${threshold}`; }; function getCachedAndUncachedItems(revisions, req, title) { @@ -493,8 +514,7 @@ function getSignificantChanges2(req, res) { }) .then( (response) => { - const threshold = req.query.threshold === null || req.query.threshold === undefined ? - 100 : req.query.threshold; + const threshold = getThreshold(req); // segment off large changes and small changes var uncachedOutput = []; From 9720591658c9c010cbee65faca4423dcc1828af0 Mon Sep 17 00:00:00 2001 From: Toni Sevener Date: Sat, 18 Jul 2020 13:05:46 -0500 Subject: [PATCH 09/47] added vandalism reverts and new talk page topics --- routes/page/significant-changes.js | 314 ++++++++++++++++++++--------- 1 file changed, 222 insertions(+), 92 deletions(-) diff --git a/routes/page/significant-changes.js b/routes/page/significant-changes.js index b28810e3..7246f8a8 100644 --- a/routes/page/significant-changes.js +++ b/routes/page/significant-changes.js @@ -41,6 +41,17 @@ class ConsolidatedSmallOutput { } } +class VandalismOutput { + constructor(revid, timestamp, user, userid, section) { + this.revid = revid; + this.timestamp = timestamp; + this.outputType = 'vandalism-revert'; + this.user = user; + this.userid = userid; + this.section = section; + } +} + class LargeOutput { constructor(largeOutputExpanded) { this.revid = largeOutputExpanded.revid; @@ -70,10 +81,31 @@ class LargeOutputExpanded { } } -class Section { - constructor(fromSection, toSection) { - this.fromSection = fromSection; - this.toSection = toSection; +class NewTalkPageTopicExtended { + constructor(revid, timestamp, user, userid, snippet, type, highlightRanges, characterChange, + section) { + this.revid = revid; + this.timestamp = timestamp; + this.outputType = 'new-talk-page-topic'; + this.snippet = snippet; + this.type = type; + this.highlightRanges = highlightRanges; + this.user = user; + this.userid = userid; + this.characterChange = characterChange; + this.section = section; + } +} + +class NewTalkPageTopic { + constructor(newTalkPageTopicExpanded) { + this.revid = newTalkPageTopicExpanded.revid; + this.timestamp = newTalkPageTopicExpanded.timestamp; + this.outputType = 'new-talk-page-topic'; + this.snippet = newTalkPageTopicExpanded.snippet; + this.user = newTalkPageTopicExpanded.user; + this.userid = newTalkPageTopicExpanded.userid; + this.section = newTalkPageTopicExpanded.section; } } @@ -109,72 +141,71 @@ const snippetPromise = (req, largeChange) => { 'output-mode': 'editPreview' }); - // add highlight delimiters first - var snippetBinary = encoding.strToBin(largeChange.snippet); - - // todo: it looks like parsoid *sometimes* strips these spans out depending on their placement. - // we need some token that parsoid won't touch. - // see results for this revision, it's missing an add-highlight. - // https://en.wikipedia.org/w/index.php?title=United_States - // &type=revision&diff=965295364&oldid=965071033 - const addHighlightStart = ''; - const deleteHighlightStart = ''; - const highlightEnd = ''; - - const addHighlightStartBin = encoding.strToBin(addHighlightStart); - const deleteHighlightStartBin = encoding.strToBin(deleteHighlightStart); - const highlightEndBin = encoding.strToBin(highlightEnd); - - // const addHighlightTokenStart = 'ios-add-token'; - // const deleteHighlightTokenStart = 'ios-delete-token'; - // const highlightTokenEnd = 'ios-end-token'; - // - // const addHighlightTokenStartBin = encoding.strToBin(addHighlightTokenStart); - // const deleteHighlightTokenStartBin = encoding.strToBin(deleteHighlightTokenStart); - // const highlightTokenEndBin = encoding.strToBin(highlightTokenEnd); - - switch (largeChange.type) { - case 1: // Added complete line - - snippetBinary = insertSubstringInString(snippetBinary, addHighlightStartBin, 0); - snippetBinary = insertSubstringInString(snippetBinary, highlightEndBin, - snippetBinary.length); - break; - case 2: // Deleted complete line - - snippetBinary = insertSubstringInString(snippetBinary, deleteHighlightStartBin, 0); - snippetBinary = insertSubstringInString(snippetBinary, highlightEndBin, - snippetBinary.length); - break; - case 5: - case 3: // Added and deleted words in line - var offset = 0; - largeChange.highlightRanges.forEach(function (range) { - switch (range.type) { - case 0: // Added - snippetBinary = insertSubstringInString(snippetBinary, - addHighlightStartBin, range.start + offset); - offset += addHighlightStartBin.length; - break; - case 1: // Deleted - snippetBinary = insertSubstringInString(snippetBinary, - deleteHighlightStartBin, range.start + offset); - offset += deleteHighlightStartBin.length; - break; - default: - return; - } + var newSnippet; + + if (largeChange.outputType === 'large-change') { + // add highlight delimiters first + var snippetBinary = encoding.strToBin(largeChange.snippet); + // todo: it looks like parsoid *sometimes* strips these spans out + // depending on their placement. + // we need some token that parsoid won't touch. + // see results for this revision, it's missing an add-highlight. + // https://en.wikipedia.org/w/index.php?title=United_States + // &type=revision&diff=965295364&oldid=965071033 + const addHighlightStart = ''; + const deleteHighlightStart = ''; + const highlightEnd = ''; + + const addHighlightStartBin = encoding.strToBin(addHighlightStart); + const deleteHighlightStartBin = encoding.strToBin(deleteHighlightStart); + const highlightEndBin = encoding.strToBin(highlightEnd); + + switch (largeChange.type) { + case 1: // Added complete line + + snippetBinary = insertSubstringInString(snippetBinary, addHighlightStartBin, 0); snippetBinary = insertSubstringInString(snippetBinary, highlightEndBin, - range.start + offset + range.length); - offset += highlightEndBin.length; - }); - break; - default: - break; - } + snippetBinary.length); + break; + case 2: // Deleted complete line + + snippetBinary = insertSubstringInString(snippetBinary, deleteHighlightStartBin, 0); + snippetBinary = insertSubstringInString(snippetBinary, highlightEndBin, + snippetBinary.length); + break; + case 5: + case 3: // Added and deleted words in line + var offset = 0; + largeChange.highlightRanges.forEach(function (range) { + switch (range.type) { + case 0: // Added + snippetBinary = insertSubstringInString(snippetBinary, + addHighlightStartBin, range.start + offset); + offset += addHighlightStartBin.length; + break; + case 1: // Deleted + snippetBinary = insertSubstringInString(snippetBinary, + deleteHighlightStartBin, range.start + offset); + offset += deleteHighlightStartBin.length; + break; + default: + return; + } + + snippetBinary = insertSubstringInString(snippetBinary, highlightEndBin, + range.start + offset + range.length); + offset += highlightEndBin.length; + }); + break; + default: + break; + } - const newSnippet = encoding.binToStr(snippetBinary); + newSnippet = encoding.binToStr(snippetBinary); + } else { // new talk page topic + newSnippet = largeChange.snippet; + } // make request to format to mobile-html, reassign result back to snippet @@ -218,7 +249,7 @@ const snippetPromise = (req, largeChange) => { } const paragraphWrapper = sectionWrapper.firstChild; - if (paragraphWrapper.tagName === 'P') { + if (paragraphWrapper && paragraphWrapper.tagName === 'P') { var parent = paragraphWrapper.parentNode; while ( paragraphWrapper.firstChild ) { @@ -265,6 +296,14 @@ function getCachedAndUncachedItems(revisions, req, title) { // add cache to output and filter out of processing flow const uncachedRevisions = []; const cachedOutput = []; + + if (!revisions) { + return Object.assign({ + uncachedRevisions: [], + cachedOutput: [] + }); + } + revisions.forEach(function (revision) { const cacheKey = significantChangesCacheKey(req, title, revision); @@ -346,29 +385,81 @@ function updateDiffAndRevisionsWithCharacterCount(diffAndRevisions) { }); } +function getNewTopicDiffAndRevisions(talkDiffAndRevisions) { + + const newSectionTalkPageDiffAndRevisions = []; + // const talkPage = talkPageBody.query.pages; + // const talkPageObject = talkPage[Object.keys(talkPage)[0]]; + // const talkPageRevisions = talkPageObject.revisions; + talkDiffAndRevisions.forEach(function (diffAndRevision, index) { + if (diffAndRevision.revision.comment.toLowerCase().includes('new section') + && diffAndRevision.revision.userid !== 4936590) { // don't show signbot topics + // see if this section was reverted in previous iterations + var wasReverted = false; + if (index - 1 > 0) { + var nextDiffAndRevision = talkDiffAndRevisions[index - 1]; + if (nextDiffAndRevision.revision.userid === 4936590) { // signbot, + // look for next revision + if (index - 2 > 0) { + nextDiffAndRevision = talkDiffAndRevisions[index - 2]; + } + } + if (nextDiffAndRevision.revision.tags.includes('mw-undo') || + nextDiffAndRevision.revision.tags.includes('mw-rollback')) { + wasReverted = true; + } + } + if (wasReverted === false) { + newSectionTalkPageDiffAndRevisions.push(diffAndRevision); + } + } + }); + + return newSectionTalkPageDiffAndRevisions; +} + +function getLargestDiffLine(diffBody) { + diffBody.diff.sort(function(a, b) { + return b.characterChange.totalCount() - a.characterChange.totalCount(); + }); + + // todo: safety + const largestDiffLine = diffBody.diff[0]; + return largestDiffLine; +} + function getSectionForDiffLine(diffBody, diffLine) { var fromSection = null; var toSection = null; // capture intro - if (!diffBody.from.sections || - diffBody.from.sections.length === 0 || - !diffBody.to.sections || - diffBody.to.sections.length === 0) { + if ((!diffBody.from.sections || + diffBody.from.sections.length === 0) && + (!diffBody.to.sections || + diffBody.to.sections.length === 0)) { return null; } - if (diffLine.offset.from && diffLine.offset.from < diffBody.from.sections[0].offset) { + // diffLine.offset.from of = is still valid if it's at the very beginning of the article. + // In this case javascript evaluates diffLine.offset.from to false, + // hence the need for the separate check. + if ((diffLine.offset.from || diffLine.offset.from === 0) && + diffLine.offset.from < diffBody.from.sections[0].offset) { fromSection = 'Intro'; } - if (diffLine.offset.to && diffLine.offset.to < diffBody.to.sections[0].offset) { + if ((diffLine.offset.to || diffLine.offset.to === 0) && + diffLine.offset.to < diffBody.to.sections[0].offset) { toSection = 'Intro'; } if (fromSection && toSection) { - return new Section(fromSection, toSection); + if (diffLine.offset.to && diffLine.offset.to.length > 0) { + return toSection; + } else { + return fromSection; + } } var prevSection = null; @@ -407,7 +498,18 @@ function getSectionForDiffLine(diffBody, diffLine) { } } - return new Section(fromSection, toSection); + if (diffLine.offset.to) { + return toSection; + } else { + return fromSection; + } +} + +function getSectionForLargestDiffLine(diffBody, largestDiffLine) { + // get largest diff + const diffSection = getSectionForDiffLine(diffBody, + largestDiffLine); + return diffSection; } function cleanOutput(output) { @@ -434,6 +536,8 @@ function cleanOutput(output) { if (item.outputType === 'large-change') { cleanedOutput.push(new LargeOutput(item)); + } else if (item.outputType === 'new-talk-page-topic') { + cleanedOutput.push(new NewTalkPageTopic(item)); } else { cleanedOutput.push(item); } @@ -467,6 +571,8 @@ function getSignificantChanges2(req, res) { var finalOutput = []; finalOutput = finalOutput.concat(articleEvalResults.cachedOutput); + // todo: unfortunately this cuts out new talk page topics that appeared after + // the latest article revision. rethink this piece. const rvStart = revisions[0].timestamp; // todo: length error checking here const rvEnd = revisions[revisions.length - 1].timestamp; @@ -509,6 +615,7 @@ function getSignificantChanges2(req, res) { // Determine character size of change for every diff line and aggregate // for every revision updateDiffAndRevisionsWithCharacterCount(response.articleDiffAndRevisions); + updateDiffAndRevisionsWithCharacterCount(response.talkDiffAndRevisions); return response; }) @@ -516,34 +623,51 @@ function getSignificantChanges2(req, res) { const threshold = getThreshold(req); - // segment off large changes and small changes + // segment off into types var uncachedOutput = []; response.articleDiffAndRevisions.forEach(function (diffAndRevision) { - if (diffAndRevision.characterChange.totalCount() <= threshold) { - const revision = diffAndRevision.revision; + const revision = diffAndRevision.revision; + if (revision.tags.includes('mw-rollback') && + revision.comment.toLowerCase().includes('revert') && + revision.comment.toLowerCase().includes('vandalism')) { + const largestDiffLine = getLargestDiffLine(diffAndRevision.body); + const section = getSectionForLargestDiffLine(diffAndRevision.body, + largestDiffLine); + const vandalismRevertOutputObject = new VandalismOutput(revision.revid, + revision.timestamp, revision.user, revision.userid, section); + uncachedOutput.push(vandalismRevertOutputObject); + } else if (diffAndRevision.characterChange.totalCount() <= threshold) { const smallOutputObject = new SmallOutput(revision.revid, revision.timestamp, revision.user, revision.userid, diffAndRevision.characterChange); uncachedOutput.push(smallOutputObject); } else { - const revision = diffAndRevision.revision; - - // get largest diff - diffAndRevision.body.diff.sort(function(a, b) { - return b.characterChange.totalCount() - a.characterChange.totalCount(); - }); - - // todo: safety - const largestDiffLine = diffAndRevision.body.diff[0]; - const diffSection = getSectionForDiffLine(diffAndRevision.body, + const largestDiffLine = getLargestDiffLine(diffAndRevision.body); + const section = getSectionForLargestDiffLine(diffAndRevision.body, largestDiffLine); const largeOutputObject = new LargeOutputExpanded(revision.revid, revision.timestamp, revision.user, revision.userid, largestDiffLine.text, largestDiffLine.type, largestDiffLine.highlightRanges, - diffAndRevision.characterChange, diffSection); + diffAndRevision.characterChange, section); uncachedOutput.push(largeOutputObject); } }); + // get new talk page revisions, add to uncachedOutput. it will be sorted later. + const newTopicDiffAndRevisions = getNewTopicDiffAndRevisions( + response.talkDiffAndRevisions); + newTopicDiffAndRevisions.forEach(function (diffAndRevision) { + const revision = diffAndRevision.revision; + // todo: better check might be something like get first diff line + // that doesn't have a section title or empty line. + const largestDiffLine = getLargestDiffLine(diffAndRevision.body); + const section = getSectionForLargestDiffLine(diffAndRevision.body, largestDiffLine); + const newTalkPageTopicOutputObject = new NewTalkPageTopicExtended(revision.revid, + revision.timestamp, revision.user, revision.userid, largestDiffLine.text, + largestDiffLine.type, largestDiffLine.highlightRanges, + diffAndRevision.characterChange, section); + uncachedOutput.push(newTalkPageTopicOutputObject); + }); + return Object.assign({ nextRvStartId: response.nextRvStartId, uncachedOutput: uncachedOutput, finalOutput: response.finalOutput } ); }) @@ -551,7 +675,7 @@ function getSignificantChanges2(req, res) { // convert large snippets from wikitext to mobile-html const largeOutputs = response.uncachedOutput.filter(item => - item.outputType === 'large-change'); + item.outputType === 'large-change' || item.outputType === 'new-talk-page-topic'); return snippetPromises(req, largeOutputs) .then( (snippetResponse) => { @@ -560,7 +684,13 @@ function getSignificantChanges2(req, res) { // (snippet only contains large) response.uncachedOutput.forEach((item) => { response.finalOutput.push(item); - const cacheKey = significantChangesCacheKey(req, null, item); + var cacheKey; + if (item.outputType === 'new-talk-page-topic') { + const title = talkPageTitle(req); + cacheKey = significantChangesCacheKey(req, title, item); + } else { + cacheKey = significantChangesCacheKey(req, null, item); + } significantChangesCache[cacheKey] = item; }); From 7220953f14c7c2d8fc0917f0e71093c02c7b8951 Mon Sep 17 00:00:00 2001 From: Toni Sevener Date: Sat, 18 Jul 2020 13:07:41 -0500 Subject: [PATCH 10/47] remove old prototype code --- routes/page/significant-changes.js | 267 +---------------------------- 1 file changed, 2 insertions(+), 265 deletions(-) diff --git a/routes/page/significant-changes.js b/routes/page/significant-changes.js index 7246f8a8..b0bff38d 100644 --- a/routes/page/significant-changes.js +++ b/routes/page/significant-changes.js @@ -553,7 +553,7 @@ function cleanOutput(output) { return cleanedOutput; } -function getSignificantChanges2(req, res) { +function getSignificantChanges(req, res) { // STEP 1: Gather list of article revisions return mwapi.queryForRevisions(req) @@ -708,272 +708,9 @@ function getSignificantChanges2(req, res) { }); } -function getSignificantChanges(req, res) { - return BBPromise.props({ - mw: mwapi.queryForRevisions(req) - }).then((response) => { - return response.mw; - }).then((response) => { - res.status(response.status); - mUtil.setContentType(res, mUtil.CONTENT_TYPES.significantChanges); - // mUtil.setETag(res, mobileHTML.metadata.revision); - // mUtil.setLanguageHeaders(res, mobileHTML.metadata._headers); - // mUtil.setContentSecurityPolicy(res, app.conf.mobile_html_csp); - - // BEGIN - Mark byte delta on each revision - let nextRevision; - const revisionsWithDelta = []; - const revisions = response.body.query.pages[0].revisions; - revisions.forEach(function (revision) { - if (!(nextRevision === undefined || nextRevision === null)) { - nextRevision.delta = nextRevision.size - revision.size; - revisionsWithDelta.push(nextRevision); - } - nextRevision = revision; - }); - const nextRevId = nextRevision.revid; - const beginningDate = revisions[0].timestamp; - const endDate = revisions[revisions.length - 1].timestamp; - // END - Mark byte delta on each revision - - // BEGIN - Gather talk page new discussions - const talkPageURL = `https://en.wikipedia.org/w/api.php?action=query&prop=revisions&titles=Talk:${req.params.title}&rvslots=main&rvprop=ids|timestamp|user|userid|size|parsedcomment|comment|tags|flags&rvdir=older&format=json&rvlimit=51&rvstart=${beginningDate}&rvend=${endDate}`; - const syncRequest = require('sync-request'); - const talkPageResponse = syncRequest('GET', talkPageURL, { - headers: { - 'Content-Type': 'application/json; charset=utf-8', - }, - }); - const talkPageBody = JSON.parse(talkPageResponse.getBody('utf8')); - - const newSectionTalkPageRevisions = []; - const talkPage = talkPageBody.query.pages; - const talkPageObject = talkPage[Object.keys(talkPage)[0]]; - const talkPageRevisions = talkPageObject.revisions; - talkPageRevisions.forEach(function (revision, index) { - if (revision.comment.toLowerCase().includes('new section') - && revision.userid !== 4936590) { // don't show signbot topics - // see if this section was reverted in previous iterations - var wasReverted = false; - if (index - 1 > 0) { - var nextRevision = talkPageRevisions[index - 1]; - if (nextRevision.userid === 4936590) { // signbot, look for next revision - if (index - 2 > 0) { - nextRevision = talkPageRevisions[index - 2]; - } - } - - if (nextRevision.tags.includes('mw-undo') || - nextRevision.tags.includes('mw-rollback')) { - wasReverted = true; - } - } - - if (wasReverted === false) { - newSectionTalkPageRevisions.push(revision); - } - } - }); - // END - Gather talk page new discussions - - // BEGIN - Break it down into types - const significantChangeObjects = []; - revisionsWithDelta.forEach(function (revision, index) { - - // BEGIN - Shuffle in new section talk pages on each iteration - for (let i = 0; i < newSectionTalkPageRevisions.length; ++i) { - const sectionTalkPageRevision = newSectionTalkPageRevisions[i]; - const talkPageDate = new Date(sectionTalkPageRevision.timestamp); - const revisionDate = new Date(revision.timestamp); - if (talkPageDate > revisionDate) { - // todo: why doesn't parsedComment show in response? - const newTopic = Object.assign({ - type: 'new-talk-page-topic', - revid: sectionTalkPageRevision.revid, - user: sectionTalkPageRevision.user, - userid: sectionTalkPageRevision.userid, - timestamp: sectionTalkPageRevision.timestamp, - comment: sectionTalkPageRevision.comment, - parsedComment: sectionTalkPageRevision.parsedComment }); - significantChangeObjects.push(newTopic); - newSectionTalkPageRevisions.splice(i, 1); - --i; // Correct the index value - } - } - // END - Shuffle in new section talk pages on each iteration - - const threshold = req.query.threshold === null || req.query.threshold === undefined ? - 100 : req.query.threshold; - if (revision.tags.includes('mw-rollback') && - revision.comment.toLowerCase().includes('revert') && - revision.comment.toLowerCase().includes('vandalism') && revision.delta < 0 ) { - // Add vandalism - const vandalism = Object.assign({ type: 'vandalism-revert' } ); - significantChangeObjects.push(vandalism); - } else if (revision.delta >= -threshold && revision.delta <= threshold) { - // Add small changes - const smallChange = Object.assign({ type: 'small-change' } ); - significantChangeObjects.push(smallChange); - } else { - // BEGIN - Add large changes - const largeChange = Object.assign({ - type: 'large-change', - delta: revision.delta, - timestamp: revision.timestamp, - revid: revision.revid, - user: revision.user, - userid: revision.userid - }); - - // get snippet of change - // https://en.wikipedia.org/w/rest.php/v1/revision/847170467/compare/851733941 - // const restParams = Object.assign({domain: 'en.wikipedia.org', - // path: 'w/rest.php/v1/revision/847170467/compare/851733941'} ); - // const restReq = Object.assign({method: 'get', query: null, - // headers: null, params: restParams} ); - - // BBPromise.props({ - // // rest: api.restApiGet(req, restReq) - // // }).then((response) => { - // // return response.rest; - // // }).then((response) => { - // // console.log(response); - // // console.log(revision); - // // }); - - // BEGIN: For each large change, hit the diff endpoint and pull a snippet - const syncRequest = require('sync-request'); - const url = `https://en.wikipedia.org/w/rest.php/v1/revision/${revision.parentid}/compare/${revision.revid}`; - const diffResponse = syncRequest('GET', url, { - headers: { - 'Content-Type': 'application/json; charset=utf-8', - }, - }); - const body = JSON.parse(diffResponse.getBody('utf8')); - const changedLines = body.diff.filter(diffItem => diffItem.type >= 1 && - diffItem.type <= 3); - - changedLines.forEach(function (diff) { - if (diff.type === 1 || diff.type === 2) { - diff.characterChange = diff.text.length; - } else { - var characterChange = 0; - diff.highlightRanges.forEach(function (range) { - characterChange += range.length; - }); - - diff.characterChange = characterChange; - } - }); - changedLines.sort(function(a, b) { - return b.characterChange - a.characterChange; - }); - if (changedLines.length > 0) { - var changeType; - switch (changedLines[0].type) { - case 1: - changeType = 'added-line'; - break; - case 2: - changeType = 'deleted-line'; - break; - case 3: - changeType = 'added-and-or-deleted-words-in-line'; - break; - default: break; - } - if (changedLines[0].highlightRanges !== undefined && - changedLines[0].highlightRanges.length > 0) { - changedLines[0].highlightRanges.forEach( function (range) { - switch (range.type) { - case 0: - range.type = 'added'; - break; - case 1: - range.type = 'deleted'; - break; - default: - break; - } - }); - } - - const contentSnippet = Object.assign({ - changeType: changeType, - text: changedLines[0].text, - highlightRanges: changedLines[0].highlightRanges - }); - largeChange.contentSnippet = contentSnippet; - } - // END: For each large change, hit the diff endpoint and pull a snippet - - significantChangeObjects.push(largeChange); - // END - Add large changes - } - - // BEGIN - Shuffle in new section talk pages on each iteration - // (this catches any straggler new talk pages at the end) - if (index === revisionsWithDelta.length - 1) { - for (let i = 0; i < newSectionTalkPageRevisions.length; ++i) { - const sectionTalkPageRevision = newSectionTalkPageRevisions[i]; - const talkPageDate = new Date(sectionTalkPageRevision.timestamp); - const revisionDate = new Date(revision.timestamp); - if (talkPageDate < revisionDate) { - const newTopic = Object.assign({ - type: 'new-talk-page-topic', - revid: sectionTalkPageRevision.revid, - user: sectionTalkPageRevision.user, - userid: sectionTalkPageRevision.userid, - timestamp: sectionTalkPageRevision.timestamp, - comment: sectionTalkPageRevision.comment, - parsedComment: sectionTalkPageRevision.parsedComment }); - significantChangeObjects.push(sectionTalkPageRevision); - newSectionTalkPageRevisions.splice(i, 1); - --i; // Correct the index value - } - } - } - // END - Shuffle in new section talk pages on each iteration - }); - - // BEGIN - Collapse small changes - const collapsedSignificantChanges = []; - let numSmallChanges = 0; - significantChangeObjects.forEach(function (revision) { - if (revision.type === 'small-change') { - numSmallChanges++; - return; - } else if (numSmallChanges > 0) { - const consolidatedSmallChange = Object.assign({ - type: 'small-change', - count: numSmallChanges } ); - collapsedSignificantChanges.push(consolidatedSmallChange); - numSmallChanges = 0; - } - - collapsedSignificantChanges.push(revision); - }); - if (numSmallChanges > 0) { - const consolidatedSmallChange = Object.assign({ - type: 'small-change', - count: numSmallChanges - }); - collapsedSignificantChanges.push(consolidatedSmallChange); - numSmallChanges = 0; - } - // END - Collapse small changes - - const result = Object.assign({ - nextRvStartId: nextRevId, - revisions: collapsedSignificantChanges } ); - res.send(result).end(); - // res.send(response.body.query.pages[0].revisions).end(); - }); -} - router.get('/page/significant-changes/:title', (req, res) => { // res.status(200); - return getSignificantChanges2(req, res); + return getSignificantChanges(req, res); // const result = Object.assign({ result: "What up new endpoint."}); // mUtil.setContentType(res, mUtil.CONTENT_TYPES.talk); // res.json(result).end(); From 53e76d40652e117034e75b9e49480e277c3645ca Mon Sep 17 00:00:00 2001 From: Toni Sevener Date: Mon, 20 Jul 2020 19:27:17 -0500 Subject: [PATCH 11/47] progress structuring templates --- package-lock.json | 3852 +++++++++++++++++++++++++--- package.json | 3 +- routes/page/significant-changes.js | 103 +- 3 files changed, 3669 insertions(+), 289 deletions(-) diff --git a/package-lock.json b/package-lock.json index 21a2439a..eb2ccb6d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -176,32 +176,6 @@ "integrity": "sha512-+iTbntw2IZPb/anVDbypzfQa+ay64MW0Zo8aJ8gZPWMMK6/OubMVb6lUPMagqjOPnmtauXnFCACVl3O7ogjeqQ==", "dev": true }, - "@types/concat-stream": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/@types/concat-stream/-/concat-stream-1.6.0.tgz", - "integrity": "sha1-OU2+C7X+5Gs42JZzXoto7yOQ0A0=", - "requires": { - "@types/node": "*" - } - }, - "@types/form-data": { - "version": "0.0.33", - "resolved": "https://registry.npmjs.org/@types/form-data/-/form-data-0.0.33.tgz", - "integrity": "sha1-yayFsqX9GENbjIXZ7LUObWyJP/g=", - "requires": { - "@types/node": "*" - } - }, - "@types/node": { - "version": "10.17.27", - "resolved": "https://registry.npmjs.org/@types/node/-/node-10.17.27.tgz", - "integrity": "sha512-J0oqm9ZfAXaPdwNXMMgAhylw5fhmXkToJd06vuDUSAgEDZ/n/69/69UmyBZbc+zT34UnShuDSBqvim3SPnozJg==" - }, - "@types/qs": { - "version": "6.9.3", - "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.3.tgz", - "integrity": "sha512-7s9EQWupR1fTc2pSMtXRQ9w9gLOcrJn+h7HOXw4evxyvVqMi4f+q7d2tnFe3ng3SNHjtK+0EzGMGFUQX4/AQRA==" - }, "@wikimedia/less-plugin-clean-css": { "version": "1.5.2", "resolved": "https://registry.npmjs.org/@wikimedia/less-plugin-clean-css/-/less-plugin-clean-css-1.5.2.tgz", @@ -249,6 +223,17 @@ "uri-js": "^4.2.2" } }, + "align-text": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/align-text/-/align-text-0.1.4.tgz", + "integrity": "sha1-DNkKVhCT810KmSVsIrcGlDP60Rc=", + "optional": true, + "requires": { + "kind-of": "^3.0.2", + "longest": "^1.0.1", + "repeat-string": "^1.5.2" + } + }, "ansi-align": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ansi-align/-/ansi-align-2.0.0.tgz", @@ -565,7 +550,8 @@ "buffer-from": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", - "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==" + "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==", + "dev": true }, "bunyan": { "version": "1.8.12", @@ -660,6 +646,16 @@ "long": "^4.0.0" } }, + "center-align": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/center-align/-/center-align-0.1.3.tgz", + "integrity": "sha1-qg0yYptu6XIgBBHL1EYckHvCt60=", + "optional": true, + "requires": { + "align-text": "^0.1.3", + "lazy-cache": "^1.0.3" + } + }, "chalk": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", @@ -854,6 +850,7 @@ "version": "1.6.2", "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==", + "dev": true, "requires": { "buffer-from": "^1.0.0", "inherits": "^2.0.3", @@ -865,6 +862,7 @@ "version": "2.3.7", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "dev": true, "requires": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", @@ -879,6 +877,7 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dev": true, "requires": { "safe-buffer": "~5.1.0" } @@ -941,6 +940,12 @@ "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", "integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw=" }, + "core-js": { + "version": "2.6.11", + "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.11.tgz", + "integrity": "sha512-5wjnpaT/3dV+XB4borEsnAYQchn00XSgTAWKDkEqv+K8KevjbzmofK6hfJ9TZIlpj2N0xQpazy7PiRQiWHqzWg==", + "optional": true + }, "core-util-is": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", @@ -2396,11 +2401,6 @@ "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==" }, - "get-port": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/get-port/-/get-port-3.2.0.tgz", - "integrity": "sha1-3Xzn3hh8Bsi/NTeWrHHgmfCYDrw=" - }, "get-stream": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", @@ -2579,17 +2579,6 @@ "readable-stream": "^3.1.1" } }, - "http-basic": { - "version": "8.1.3", - "resolved": "https://registry.npmjs.org/http-basic/-/http-basic-8.1.3.tgz", - "integrity": "sha512-/EcDMwJZh3mABI2NhGfHOGOeOZITqfkEO4p/xK+l3NpyncIHUQBoMvCSF/b5GqvKtySC2srL/GGG3+EtlqlmCw==", - "requires": { - "caseless": "^0.12.0", - "concat-stream": "^1.6.2", - "http-response-object": "^3.0.1", - "parse-cache-control": "^1.0.1" - } - }, "http-errors": { "version": "1.7.2", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz", @@ -2602,14 +2591,6 @@ "toidentifier": "1.0.0" } }, - "http-response-object": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/http-response-object/-/http-response-object-3.0.2.tgz", - "integrity": "sha512-bqX0XTF6fnXSQcEJ2Iuyr75yVakyjIDCqroJQ/aHfSdlM743Cwqoi2nDYMzLGWUcuTWGWy8AAvOKXTfiv6q9RA==", - "requires": { - "@types/node": "^10.0.3" - } - }, "http-shutdown": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/http-shutdown/-/http-shutdown-1.2.2.tgz", @@ -3069,6 +3050,15 @@ "html-escaper": "^2.0.0" } }, + "jodid25519": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/jodid25519/-/jodid25519-1.0.2.tgz", + "integrity": "sha1-BtSRIlUJNBlHfUJWM2BuDpB4KWc=", + "optional": true, + "requires": { + "jsbn": "~0.1.0" + } + }, "js-beautify": { "version": "1.10.3", "resolved": "https://registry.npmjs.org/js-beautify/-/js-beautify-1.10.3.tgz", @@ -3315,6 +3305,23 @@ } } }, + "kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=", + "optional": true, + "requires": { + "is-buffer": "^1.1.5" + }, + "dependencies": { + "is-buffer": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", + "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==", + "optional": true + } + } + }, "latest-version": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/latest-version/-/latest-version-3.1.0.tgz", @@ -3324,6 +3331,12 @@ "package-json": "^4.0.0" } }, + "lazy-cache": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/lazy-cache/-/lazy-cache-1.0.4.tgz", + "integrity": "sha1-odePw6UEdMuAhF07O24dpJpEbo4=", + "optional": true + }, "less": { "version": "3.11.1", "resolved": "https://registry.npmjs.org/less/-/less-3.11.1.tgz", @@ -3534,6 +3547,12 @@ "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz", "integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==" }, + "longest": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/longest/-/longest-1.0.1.tgz", + "integrity": "sha1-MKCy2jj3N3DoKUoNIuZiXtd9AJc=", + "optional": true + }, "lowercase-keys": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-1.0.1.tgz", @@ -4455,11 +4474,6 @@ "callsites": "^3.0.0" } }, - "parse-cache-control": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/parse-cache-control/-/parse-cache-control-1.0.1.tgz", - "integrity": "sha1-juqz5U+laSD+Fro493+iGqzC104=" - }, "parse-json": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-4.0.0.tgz", @@ -4475,193 +4489,3428 @@ "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==" }, - "path-exists": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", - "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=" - }, - "path-is-absolute": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", - "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=" - }, - "path-is-inside": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/path-is-inside/-/path-is-inside-1.0.2.tgz", - "integrity": "sha1-NlQX3t5EQw0cEa9hAn+s8HS9/FM=", - "dev": true - }, - "path-key": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/path-key/-/path-key-2.0.1.tgz", - "integrity": "sha1-QRyttXTFoUDTpLGRDUDYDMn0C0A=", - "dev": true - }, - "path-parse": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.6.tgz", - "integrity": "sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==", - "dev": true - }, - "path-to-regexp": { - "version": "0.1.7", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", - "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" - }, - "path-type": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/path-type/-/path-type-3.0.0.tgz", - "integrity": "sha512-T2ZUsdZFHgA3u4e5PfPbjd7HDDpxPnQb5jN0SrDsjNSuVXHJqtwTnWqG0B1jZrgmJ/7lj1EmVIByWt1gxGkWvg==", - "dev": true, - "requires": { - "pify": "^3.0.0" - } - }, - "performance-now": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", - "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=" - }, - "picomatch": { - "version": "2.2.2", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.2.2.tgz", - "integrity": "sha512-q0M/9eZHzmr0AulXyPwNfZjtwZ/RBZlbN3K3CErVrk50T2ASYI7Bye0EvekFY3IP1Nt2DHu0re+V2ZHIpMkuWg==", - "dev": true - }, - "pify": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", - "integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=", - "dev": true - }, - "pkg-dir": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-3.0.0.tgz", - "integrity": "sha512-/E57AYkoeQ25qkxMj5PBOVgF8Kiu/h7cYS30Z5+R7WaiCCBfLq58ZI/dSeaEKb9WVJV5n/03QwrN3IeWIFllvw==", - "dev": true, - "requires": { - "find-up": "^3.0.0" - } - }, - "postcss": { - "version": "7.0.27", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-7.0.27.tgz", - "integrity": "sha512-WuQETPMcW9Uf1/22HWUWP9lgsIC+KEHg2kozMflKjbeUtw9ujvFX6QmIfozaErDkmLWS9WEnEdEe6Uo9/BNTdQ==", - "requires": { - "chalk": "^2.4.2", - "source-map": "^0.6.1", - "supports-color": "^6.1.0" + "parsoid": { + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/parsoid/-/parsoid-0.7.1.tgz", + "integrity": "sha1-Eh3PWdC6zZJHQFmIUliI+aa6Iyg=", + "requires": { + "async": "^0.9.2", + "babybird": "^0.0.1", + "body-parser": "^1.16.0", + "compression": "^1.6.2", + "connect-busboy": "^0.0.2", + "content-type": "git+https://github.com/wikimedia/content-type.git#master", + "core-js": "^2.4.1", + "diff": "^1.0.7", + "domino": "^1.0.28", + "entities": "^1.1.1", + "express": "^4.14.0", + "express-handlebars": "^3.0.0", + "finalhandler": "^0.5.0", + "js-yaml": "^3.6.1", + "mediawiki-title": "^0.5.6", + "negotiator": "git+https://github.com/arlolra/negotiator.git#full-parse-access", + "node-uuid": "^1.4.7", + "pegjs": "git+https://github.com/tstarling/pegjs.git#fork", + "prfun": "^2.1.4", + "request": "^2.79.0", + "semver": "^5.3.0", + "serve-favicon": "^2.3.2", + "service-runner": "^2.1.13", + "simplediff": "^0.1.1", + "yargs": "^5.0.0" }, "dependencies": { - "supports-color": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-6.1.0.tgz", - "integrity": "sha512-qe1jfm1Mg7Nq/NSh6XE24gPXROEVsWHxC1LIx//XNlD9iw7YZQGjZNjYN7xGaEG6iKdA8EtNFW6R0gjnVXp+wQ==", + "JSV": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/JSV/-/JSV-4.0.2.tgz", + "integrity": "sha1-0Hf2glVx+CEy+d/67Vh7QCn+/1c=" + }, + "abbrev": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.0.9.tgz", + "integrity": "sha1-kbR5JYinc4wl813W9jdSovh3YTU=" + }, + "accepts": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.3.tgz", + "integrity": "sha1-w8p0NJOGSMPg2cHjKN1otiLChMo=", "requires": { - "has-flag": "^3.0.0" + "mime-types": "~2.1.11", + "negotiator": "0.6.1" } - } - } - }, - "pre-commit": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/pre-commit/-/pre-commit-1.2.2.tgz", - "integrity": "sha1-287g7p3nI15X95xW186UZBpp7sY=", - "dev": true, - "requires": { - "cross-spawn": "^5.0.1", - "spawn-sync": "^1.0.15", - "which": "1.2.x" - }, - "dependencies": { - "cross-spawn": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-5.1.0.tgz", - "integrity": "sha1-6L0O/uWPz/b4+UUQoKVUu/ojVEk=", - "dev": true, + }, + "alea": { + "version": "0.0.9", + "resolved": "https://registry.npmjs.org/alea/-/alea-0.0.9.tgz", + "integrity": "sha1-9zjLRfg0MAafRc9pzL8xLdV6nho=" + }, + "amdefine": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/amdefine/-/amdefine-1.0.1.tgz", + "integrity": "sha1-SlKCrBZHKek2Gbz9OtFR+BfOkfU=" + }, + "ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=" + }, + "ansi-styles": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz", + "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=" + }, + "argparse": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.9.tgz", + "integrity": "sha1-c9g7wmP4bpf4zE9rrhsOkKfSLIY=", "requires": { - "lru-cache": "^4.0.1", - "shebang-command": "^1.2.0", - "which": "^1.2.9" + "sprintf-js": "~1.0.2" } }, - "which": { - "version": "1.2.14", - "resolved": "https://registry.npmjs.org/which/-/which-1.2.14.tgz", - "integrity": "sha1-mofEN48D6CfOyvGs31bHNsAcFOU=", - "dev": true, + "array-flatten": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", + "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=" + }, + "asap": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.5.tgz", + "integrity": "sha1-UidltQw1EEkOUtfc/ghe+bqWlY8=" + }, + "asn1": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.3.tgz", + "integrity": "sha1-2sh4dxPJlmhJ/IGAd36+nB3fO4Y=" + }, + "assert-plus": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-0.2.0.tgz", + "integrity": "sha1-104bh+ev/A24qttwIfP+SBAasjQ=" + }, + "assertion-error": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-1.0.2.tgz", + "integrity": "sha1-E8pRXYYgbaC6xm6DTdOX2HWBCUw=" + }, + "async": { + "version": "0.9.2", + "resolved": "https://registry.npmjs.org/async/-/async-0.9.2.tgz", + "integrity": "sha1-rqdNXmHB+JlhO/ZL2mbUx48v0X0=" + }, + "asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" + }, + "aws-sign2": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.6.0.tgz", + "integrity": "sha1-FDQt0428yU0OW4fXY81jYSwOeU8=" + }, + "aws4": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.5.0.tgz", + "integrity": "sha1-Cin/t5wxyecS7rCH6OemS0pW11U=" + }, + "babel-runtime": { + "version": "6.20.0", + "resolved": "https://registry.npmjs.org/babel-runtime/-/babel-runtime-6.20.0.tgz", + "integrity": "sha1-hzAL3PTNdw8JvwBIxkIE4XgG0W8=", "requires": { - "isexe": "^2.0.0" + "core-js": "^2.4.0", + "regenerator-runtime": "^0.10.0" } - } - } - }, - "prelude-ls": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", - "integrity": "sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ=", - "dev": true - }, - "prepend-http": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/prepend-http/-/prepend-http-1.0.4.tgz", - "integrity": "sha1-1PRWKwzjaW5BrFLQ4ALlemNdxtw=", - "dev": true - }, - "preq": { - "version": "0.5.14", - "resolved": "https://registry.npmjs.org/preq/-/preq-0.5.14.tgz", - "integrity": "sha512-kuJ5ndEgjs27kTTQ/P2ipPQoHeCJcAI4i97mU3xSjkjx6CsuQOsCe2l5twTGC0SCB5UkzRpmrpXmvN0Ip4ZCxA==", - "requires": { - "bluebird": "^3.5.5", - "request": "^2.88.0", - "requestretry": "4.0.2" - } - }, - "process-nextick-args": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", - "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" - }, - "progress": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", - "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", - "dev": true - }, - "promise": { - "version": "7.3.1", - "resolved": "https://registry.npmjs.org/promise/-/promise-7.3.1.tgz", - "integrity": "sha512-nolQXZ/4L+bP/UGlkfaIujX9BKxGwmQ9OT4mOt5yvy8iK1h3wqTEJCijzGANTCCl9nWjY41juyAn2K3Q1hLLTg==", - "dev": true, - "optional": true, - "requires": { - "asap": "~2.0.3" - } - }, - "proto-list": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/proto-list/-/proto-list-1.2.4.tgz", - "integrity": "sha1-IS1b/hMYMGpCD2QCuOJv85ZHqEk=", - "dev": true - }, - "proxy-addr": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.6.tgz", - "integrity": "sha512-dh/frvCBVmSsDYzw6n926jv974gddhkFPfiN8hPOi30Wax25QZyZEGveluCgliBnqmuM+UJmBErbAUFIoDbjOw==", - "requires": { - "forwarded": "~0.1.2", - "ipaddr.js": "1.9.1" - } - }, - "prr": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/prr/-/prr-1.0.1.tgz", - "integrity": "sha1-0/wRS6BplaRexok/SEzrHXj19HY=", - "dev": true, - "optional": true - }, - "pseudomap": { + }, + "babybird": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/babybird/-/babybird-0.0.1.tgz", + "integrity": "sha1-2oDHnG10Qc3+x8L/LcvXwT69vqI=", + "requires": { + "asap": "^2.0.3", + "is-arguments": "^1.0.2" + } + }, + "babylon": { + "version": "6.15.0", + "resolved": "https://registry.npmjs.org/babylon/-/babylon-6.15.0.tgz", + "integrity": "sha1-umXPoagOF1mw6J+1YuJ9zK5wNI4=" + }, + "balanced-match": { + "version": "0.4.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-0.4.2.tgz", + "integrity": "sha1-yz8+PHMtwPAe5wtAPzAuYddwmDg=" + }, + "bl": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-1.2.0.tgz", + "integrity": "sha1-E5fn7ELF9dw4dHDFAONKn2vp6pg=", + "requires": { + "readable-stream": "^2.0.5" + }, + "dependencies": { + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "readable-stream": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.2.2.tgz", + "integrity": "sha1-qeb+w8fdqF+LsbO6cChgRVb8gl4=", + "requires": { + "buffer-shims": "^1.0.0", + "core-util-is": "~1.0.0", + "inherits": "~2.0.1", + "isarray": "~1.0.0", + "process-nextick-args": "~1.0.6", + "string_decoder": "~0.10.x", + "util-deprecate": "~1.0.1" + } + } + } + }, + "bluebird": { + "version": "3.4.7", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.4.7.tgz", + "integrity": "sha1-9y12C+Cbf3bQjtj66Ysomo0F+rM=" + }, + "body-parser": { + "version": "1.16.0", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.16.0.tgz", + "integrity": "sha1-kkpeRyxiKfudabhaINXyUy3seIs=", + "requires": { + "bytes": "2.4.0", + "content-type": "~1.0.2", + "debug": "2.6.0", + "depd": "~1.1.0", + "http-errors": "~1.5.1", + "iconv-lite": "0.4.15", + "on-finished": "~2.3.0", + "qs": "6.2.1", + "raw-body": "~2.2.0", + "type-is": "~1.6.14" + }, + "dependencies": { + "content-type": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.2.tgz", + "integrity": "sha1-t9ETrueo3Se9IRM8TcJSnfFyHu0=" + } + } + }, + "boom": { + "version": "2.10.1", + "resolved": "https://registry.npmjs.org/boom/-/boom-2.10.1.tgz", + "integrity": "sha1-OciRjO/1eZ+D+UkqhI9iWt0Mdm8=", + "requires": { + "hoek": "2.x.x" + } + }, + "brace-expansion": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.6.tgz", + "integrity": "sha1-cZfX6qm4fmSDkOph/GbIRCdCDfk=", + "requires": { + "balanced-match": "^0.4.1", + "concat-map": "0.0.1" + } + }, + "buffer-shims": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/buffer-shims/-/buffer-shims-1.0.0.tgz", + "integrity": "sha1-mXjOMXOIxkmth5MCjDR37wRKi1E=" + }, + "builtin-modules": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-1.1.1.tgz", + "integrity": "sha1-Jw8HbFpywC9bZaR9+Uxf46J4iS8=" + }, + "bunyan": { + "version": "1.8.5", + "resolved": "https://registry.npmjs.org/bunyan/-/bunyan-1.8.5.tgz", + "integrity": "sha1-DWGegwBfuJBw9fR5gvwb8AYAh4o=", + "requires": { + "dtrace-provider": "~0.8", + "moment": "^2.10.6", + "mv": "~2", + "safe-json-stringify": "~1" + } + }, + "bunyan-syslog-udp": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/bunyan-syslog-udp/-/bunyan-syslog-udp-0.1.0.tgz", + "integrity": "sha1-+/ruA6gc0qlavBj5LJnyu4fiQpw=" + }, + "busboy": { + "version": "0.2.14", + "resolved": "https://registry.npmjs.org/busboy/-/busboy-0.2.14.tgz", + "integrity": "sha1-bCpiLvz0fFe7vh4qnDetNseSVFM=", + "requires": { + "dicer": "0.2.5", + "readable-stream": "1.1.x" + } + }, + "bytes": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-2.4.0.tgz", + "integrity": "sha1-fZcZb51br39pNeJZhVSe3SpsIzk=" + }, + "camelcase": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-3.0.0.tgz", + "integrity": "sha1-MvxLn82vhF/N9+c7uXysImHwqwo=" + }, + "caseless": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.11.0.tgz", + "integrity": "sha1-cVuW6phBWTzDMGeSP17GDr2k99c=" + }, + "chai": { + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/chai/-/chai-3.5.0.tgz", + "integrity": "sha1-TQJjewZ/6Vi9v906QOxW/vc3Mkc=", + "requires": { + "assertion-error": "^1.0.1", + "deep-eql": "^0.1.3", + "type-detect": "^1.0.0" + } + }, + "chalk": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", + "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=", + "requires": { + "ansi-styles": "^2.2.1", + "escape-string-regexp": "^1.0.2", + "has-ansi": "^2.0.0", + "strip-ansi": "^3.0.0", + "supports-color": "^2.0.0" + } + }, + "clarinet": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/clarinet/-/clarinet-0.11.0.tgz", + "integrity": "sha1-bMkSuTE43IZ/wnPNNOqQ6D4FRxk=" + }, + "cli": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/cli/-/cli-1.0.1.tgz", + "integrity": "sha1-IoF1NPJL+klQw01TLUjsvGIbjBQ=", + "requires": { + "exit": "0.1.2", + "glob": "^7.1.1" + }, + "dependencies": { + "glob": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.1.tgz", + "integrity": "sha1-gFIR3wT6rxxjo2ADBs31reULLsg=", + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.2", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + } + } + }, + "cli-table": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/cli-table/-/cli-table-0.3.1.tgz", + "integrity": "sha1-9TsFJmqLGguTSz0IIebi3FkUriM=", + "requires": { + "colors": "1.0.3" + }, + "dependencies": { + "colors": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/colors/-/colors-1.0.3.tgz", + "integrity": "sha1-BDP0TYCWgP3rYO0mDxsMJi6CpAs=" + } + } + }, + "cliui": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-3.2.0.tgz", + "integrity": "sha1-EgYBU3qRbSmUD5NNo7SNWFo5IT0=", + "requires": { + "string-width": "^1.0.1", + "strip-ansi": "^3.0.1", + "wrap-ansi": "^2.0.0" + } + }, + "code-point-at": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", + "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=" + }, + "colors": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/colors/-/colors-1.1.2.tgz", + "integrity": "sha1-FopHAXVran9RoSzgyXv6KMCE7WM=" + }, + "combined-stream": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.5.tgz", + "integrity": "sha1-k4NwpXtKUd6ix3wV1cX9+JUWQAk=", + "requires": { + "delayed-stream": "~1.0.0" + } + }, + "commander": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.9.0.tgz", + "integrity": "sha1-nJkJQXbhIkDLItbFFGCYQA/g99Q=", + "requires": { + "graceful-readlink": ">= 1.0.0" + } + }, + "comment-parser": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/comment-parser/-/comment-parser-0.3.1.tgz", + "integrity": "sha1-/WV6rIwUktMIyaYQD8m0nSQ1q6E=", + "requires": { + "readable-stream": "^2.0.4" + }, + "dependencies": { + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "readable-stream": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.2.2.tgz", + "integrity": "sha1-qeb+w8fdqF+LsbO6cChgRVb8gl4=", + "requires": { + "buffer-shims": "^1.0.0", + "core-util-is": "~1.0.0", + "inherits": "~2.0.1", + "isarray": "~1.0.0", + "process-nextick-args": "~1.0.6", + "string_decoder": "~0.10.x", + "util-deprecate": "~1.0.1" + } + } + } + }, + "component-emitter": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.2.1.tgz", + "integrity": "sha1-E3kY1teCg/ffemt8WmPhQOaUJeY=" + }, + "compressible": { + "version": "2.0.9", + "resolved": "https://registry.npmjs.org/compressible/-/compressible-2.0.9.tgz", + "integrity": "sha1-baq04rWZwncN2eIeeokbHFp1VCU=", + "requires": { + "mime-db": ">= 1.24.0 < 2" + } + }, + "compression": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/compression/-/compression-1.6.2.tgz", + "integrity": "sha1-zOsSHsydCcUtetDDNQ6pPd1AK8M=", + "requires": { + "accepts": "~1.3.3", + "bytes": "2.3.0", + "compressible": "~2.0.8", + "debug": "~2.2.0", + "on-headers": "~1.0.1", + "vary": "~1.1.0" + }, + "dependencies": { + "bytes": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-2.3.0.tgz", + "integrity": "sha1-1baAoWW2IBc5rLYRVCqrwtjOsHA=" + }, + "debug": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.2.0.tgz", + "integrity": "sha1-+HBX6ZWxofauaklgZkE3vFbwOdo=", + "requires": { + "ms": "0.7.1" + } + }, + "ms": { + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/ms/-/ms-0.7.1.tgz", + "integrity": "sha1-nNE8A62/8ltl7/3nzoZO6VIBcJg=" + } + } + }, + "concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" + }, + "connect-busboy": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/connect-busboy/-/connect-busboy-0.0.2.tgz", + "integrity": "sha1-rFyclmchcYheV2xmsr/ZXTuxEJc=", + "requires": { + "busboy": "*" + } + }, + "console-browserify": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/console-browserify/-/console-browserify-1.1.0.tgz", + "integrity": "sha1-8CQcRXMKn8YyOyBtvzjtx0HQuxA=", + "requires": { + "date-now": "^0.1.4" + } + }, + "content-disposition": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.1.tgz", + "integrity": "sha1-h0dsamfI2qh+Muh2Ft+IO6f7Bxs=" + }, + "content-type": { + "version": "git+https://github.com/wikimedia/content-type.git#47b2632d0a2ee79a7d67268e2f6621becd95d05b", + "from": "git+https://github.com/wikimedia/content-type.git#master" + }, + "cookie": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.3.1.tgz", + "integrity": "sha1-5+Ch+e9DtMi6klxcWpboBtFoc7s=" + }, + "cookie-signature": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", + "integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw=" + }, + "cookiejar": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/cookiejar/-/cookiejar-2.0.6.tgz", + "integrity": "sha1-Cr81atANHFohnYjURRgEbdAmrP4=" + }, + "core-js": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.4.1.tgz", + "integrity": "sha1-TekR5mew6ukSTjQlS1OupvxhjT4=" + }, + "core-util-is": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", + "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" + }, + "coveralls": { + "version": "2.11.15", + "resolved": "https://registry.npmjs.org/coveralls/-/coveralls-2.11.15.tgz", + "integrity": "sha1-N9NHQ2nWbBTzP6c6nSXO5uCZ/KA=", + "requires": { + "js-yaml": "3.6.1", + "lcov-parse": "0.0.10", + "log-driver": "1.2.5", + "minimist": "1.2.0", + "request": "2.75.0" + }, + "dependencies": { + "bl": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/bl/-/bl-1.1.2.tgz", + "integrity": "sha1-/cqHGplxOqANGeO7ukHER4emU5g=", + "requires": { + "readable-stream": "~2.0.5" + } + }, + "form-data": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.0.0.tgz", + "integrity": "sha1-bwrrrcxdoWwT4ezBETfYX5uIOyU=", + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.5", + "mime-types": "^2.1.11" + } + }, + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "js-yaml": { + "version": "3.6.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.6.1.tgz", + "integrity": "sha1-bl/mfYsgXOTSL60Ft3geja3MSzA=", + "requires": { + "argparse": "^1.0.7", + "esprima": "^2.6.0" + } + }, + "minimist": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", + "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=" + }, + "readable-stream": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.0.6.tgz", + "integrity": "sha1-j5A0HmilPMySh4jaz80Rs265t44=", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.1", + "isarray": "~1.0.0", + "process-nextick-args": "~1.0.6", + "string_decoder": "~0.10.x", + "util-deprecate": "~1.0.1" + } + }, + "request": { + "version": "2.75.0", + "resolved": "https://registry.npmjs.org/request/-/request-2.75.0.tgz", + "integrity": "sha1-0rgmiihtoT6qXQGt9dGMyQ9lfZM=", + "requires": { + "aws-sign2": "~0.6.0", + "aws4": "^1.2.1", + "bl": "~1.1.2", + "caseless": "~0.11.0", + "combined-stream": "~1.0.5", + "extend": "~3.0.0", + "forever-agent": "~0.6.1", + "form-data": "~2.0.0", + "har-validator": "~2.0.6", + "hawk": "~3.1.3", + "http-signature": "~1.1.0", + "is-typedarray": "~1.0.0", + "isstream": "~0.1.2", + "json-stringify-safe": "~5.0.1", + "mime-types": "~2.1.7", + "node-uuid": "~1.4.7", + "oauth-sign": "~0.8.1", + "qs": "~6.2.0", + "stringstream": "~0.0.4", + "tough-cookie": "~2.3.0", + "tunnel-agent": "~0.4.1" + } + } + } + }, + "cryptiles": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/cryptiles/-/cryptiles-2.0.5.tgz", + "integrity": "sha1-O9/s3GCBR8HGcgL6KR59ylnqo7g=", + "requires": { + "boom": "2.x.x" + } + }, + "cst": { + "version": "0.4.9", + "resolved": "https://registry.npmjs.org/cst/-/cst-0.4.9.tgz", + "integrity": "sha1-Ua8UITv1+OjnFZZqxkXh4qVsaDQ=", + "requires": { + "babel-runtime": "^6.9.2", + "babylon": "^6.8.1", + "source-map-support": "^0.4.0" + } + }, + "cycle": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/cycle/-/cycle-1.0.3.tgz", + "integrity": "sha1-IegLK+hYD5i0aPN5QwZisEbDStI=" + }, + "dashdash": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", + "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=", + "requires": { + "assert-plus": "^1.0.0" + }, + "dependencies": { + "assert-plus": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", + "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" + } + } + }, + "date-now": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/date-now/-/date-now-0.1.4.tgz", + "integrity": "sha1-6vQ5/U1ISK105cx9vvIAZyueNFs=" + }, + "debug": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.0.tgz", + "integrity": "sha1-vFlryr52F/Edn6FTYe3tVgi4SZs=", + "requires": { + "ms": "0.7.2" + } + }, + "decamelize": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", + "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=" + }, + "deep-eql": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-0.1.3.tgz", + "integrity": "sha1-71WKyrjeJSBs1xOQbXTlaTDrafI=", + "requires": { + "type-detect": "0.1.1" + }, + "dependencies": { + "type-detect": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-0.1.1.tgz", + "integrity": "sha1-C6XsKohWQORw6k6FBZcZANrFiCI=" + } + } + }, + "deep-equal": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-1.0.1.tgz", + "integrity": "sha1-9dJgKStmDghO/0zbyfCK0yR0SLU=" + }, + "deep-is": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz", + "integrity": "sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ=" + }, + "define-properties": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.2.tgz", + "integrity": "sha1-g6c/L+pWmJj7c3GTyPhzyvbUXJQ=", + "requires": { + "foreach": "^2.0.5", + "object-keys": "^1.0.8" + } + }, + "delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" + }, + "depd": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.0.tgz", + "integrity": "sha1-4b2Cxqq2ztlluXuIsX7T5SjKGMM=" + }, + "destroy": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz", + "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=" + }, + "dicer": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/dicer/-/dicer-0.2.5.tgz", + "integrity": "sha1-WZbAhrszIYyBLAkL3cCc0S+stw8=", + "requires": { + "readable-stream": "1.1.x", + "streamsearch": "0.1.2" + } + }, + "diff": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/diff/-/diff-1.4.0.tgz", + "integrity": "sha1-fyjS657nsVqX79ic5j3P2qPMur8=" + }, + "dom-serializer": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-0.1.0.tgz", + "integrity": "sha1-BzxpdUbOB4DOI75KKOKT5AvDDII=", + "requires": { + "domelementtype": "~1.1.1", + "entities": "~1.1.1" + }, + "dependencies": { + "domelementtype": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.1.3.tgz", + "integrity": "sha1-vSh3PiZCiBrsUVRJJCmcXNgiGFs=" + } + } + }, + "dom-storage": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/dom-storage/-/dom-storage-2.0.2.tgz", + "integrity": "sha1-7RfL9oq9EOCu+BgnE+KXxeS1ALA=" + }, + "domelementtype": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.0.tgz", + "integrity": "sha1-sXrtguirWeUt2cGbF1bg/BhyBMI=" + }, + "domhandler": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-2.3.0.tgz", + "integrity": "sha1-LeWaCCLVAn+r/28DLCsloqir5zg=", + "requires": { + "domelementtype": "1" + } + }, + "domino": { + "version": "1.0.28", + "resolved": "https://registry.npmjs.org/domino/-/domino-1.0.28.tgz", + "integrity": "sha1-nOP2qSIaLDKImEsU6hkc0ns5L4c=" + }, + "domutils": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.5.1.tgz", + "integrity": "sha1-3NhIiib1Y9YQeeSMn3t+Mjc2gs8=", + "requires": { + "dom-serializer": "0", + "domelementtype": "1" + } + }, + "ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" + }, + "encodeurl": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.1.tgz", + "integrity": "sha1-eePVhlU0aQn+bw9Fpd5oEDspTSA=" + }, + "entities": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.1.tgz", + "integrity": "sha1-blwtClYhtdra7O+AuQ7ftc13cvA=" + }, + "error-ex": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.0.tgz", + "integrity": "sha1-5ntD8+gsluo6WE/+4Ln8MyXYAtk=", + "requires": { + "is-arrayish": "^0.2.1" + } + }, + "escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg=" + }, + "escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=" + }, + "escodegen": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-1.8.1.tgz", + "integrity": "sha1-WltTr0aTEQvrsIZ6o0MN07cKEBg=", + "requires": { + "esprima": "^2.7.1", + "estraverse": "^1.9.1", + "esutils": "^2.0.2", + "optionator": "^0.8.1" + } + }, + "esprima": { + "version": "2.7.3", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-2.7.3.tgz", + "integrity": "sha1-luO3DVd59q1JzQMmc9HDEnZ7pYE=" + }, + "estraverse": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-1.9.3.tgz", + "integrity": "sha1-r2fy3JIlgkFZUJJgkaQAXSnJu0Q=" + }, + "esutils": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.2.tgz", + "integrity": "sha1-Cr9PHKpbyx96nYrMbepPqqBLrJs=" + }, + "etag": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.7.0.tgz", + "integrity": "sha1-A9MLX2fdbmMtKUXTDWZScxo01dg=" + }, + "exit": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/exit/-/exit-0.1.2.tgz", + "integrity": "sha1-BjJjj42HfMghB9MKD/8aF8uhzQw=" + }, + "express": { + "version": "4.14.0", + "resolved": "https://registry.npmjs.org/express/-/express-4.14.0.tgz", + "integrity": "sha1-we4/Qs3Ikfs9xlCoki1R7IR9DWY=", + "requires": { + "accepts": "~1.3.3", + "array-flatten": "1.1.1", + "content-disposition": "0.5.1", + "content-type": "~1.0.2", + "cookie": "0.3.1", + "cookie-signature": "1.0.6", + "debug": "~2.2.0", + "depd": "~1.1.0", + "encodeurl": "~1.0.1", + "escape-html": "~1.0.3", + "etag": "~1.7.0", + "finalhandler": "0.5.0", + "fresh": "0.3.0", + "merge-descriptors": "1.0.1", + "methods": "~1.1.2", + "on-finished": "~2.3.0", + "parseurl": "~1.3.1", + "path-to-regexp": "0.1.7", + "proxy-addr": "~1.1.2", + "qs": "6.2.0", + "range-parser": "~1.2.0", + "send": "0.14.1", + "serve-static": "~1.11.1", + "type-is": "~1.6.13", + "utils-merge": "1.0.0", + "vary": "~1.1.0" + }, + "dependencies": { + "content-type": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.2.tgz", + "integrity": "sha1-t9ETrueo3Se9IRM8TcJSnfFyHu0=" + }, + "debug": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.2.0.tgz", + "integrity": "sha1-+HBX6ZWxofauaklgZkE3vFbwOdo=", + "requires": { + "ms": "0.7.1" + } + }, + "finalhandler": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-0.5.0.tgz", + "integrity": "sha1-6VCKvs6bbbqHGmlCodeRG5GRGsc=", + "requires": { + "debug": "~2.2.0", + "escape-html": "~1.0.3", + "on-finished": "~2.3.0", + "statuses": "~1.3.0", + "unpipe": "~1.0.0" + } + }, + "ms": { + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/ms/-/ms-0.7.1.tgz", + "integrity": "sha1-nNE8A62/8ltl7/3nzoZO6VIBcJg=" + }, + "qs": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.2.0.tgz", + "integrity": "sha1-O3hIwDwt7OaalSKw+ujEEm10Xzs=" + } + } + }, + "express-handlebars": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/express-handlebars/-/express-handlebars-3.0.0.tgz", + "integrity": "sha1-gKBwu4GbCeSvLKbQeA91zgXnXC8=", + "requires": { + "glob": "^6.0.4", + "graceful-fs": "^4.1.2", + "handlebars": "^4.0.5", + "object.assign": "^4.0.3", + "promise": "^7.0.0" + } + }, + "extend": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.0.tgz", + "integrity": "sha1-WkdDU7nzNT3dgXbf03uRyDpG8dQ=" + }, + "extsprintf": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.0.2.tgz", + "integrity": "sha1-4QgOBljjALBilJkMxw4VAiNf1VA=" + }, + "eyes": { + "version": "0.1.8", + "resolved": "https://registry.npmjs.org/eyes/-/eyes-0.1.8.tgz", + "integrity": "sha1-Ys8SAjTGg3hdkCNIqADvPgzCC8A=" + }, + "fast-levenshtein": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", + "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=" + }, + "finalhandler": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-0.5.1.tgz", + "integrity": "sha1-LEANjUUwk1vCMlScX6OF7Afeb80=", + "requires": { + "debug": "~2.2.0", + "escape-html": "~1.0.3", + "on-finished": "~2.3.0", + "statuses": "~1.3.1", + "unpipe": "~1.0.0" + }, + "dependencies": { + "debug": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.2.0.tgz", + "integrity": "sha1-+HBX6ZWxofauaklgZkE3vFbwOdo=", + "requires": { + "ms": "0.7.1" + } + }, + "ms": { + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/ms/-/ms-0.7.1.tgz", + "integrity": "sha1-nNE8A62/8ltl7/3nzoZO6VIBcJg=" + } + } + }, + "find-up": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-1.1.2.tgz", + "integrity": "sha1-ay6YIrGizgpgq2TWEOzK1TyyTQ8=", + "requires": { + "path-exists": "^2.0.0", + "pinkie-promise": "^2.0.0" + } + }, + "foreach": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/foreach/-/foreach-2.0.5.tgz", + "integrity": "sha1-C+4AUBiusmDQo6865ljdATbsG5k=" + }, + "forever-agent": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", + "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=" + }, + "form-data": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.1.2.tgz", + "integrity": "sha1-icNTQAi5fq2ky7FX1Y9vXfAl6uQ=", + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.5", + "mime-types": "^2.1.12" + } + }, + "formidable": { + "version": "1.0.17", + "resolved": "https://registry.npmjs.org/formidable/-/formidable-1.0.17.tgz", + "integrity": "sha1-71SRSQ+UM7cF+qdyScmQKa40hVk=" + }, + "forwarded": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.0.tgz", + "integrity": "sha1-Ge+YdMSuHCl7zweP3mOgm2aoQ2M=" + }, + "fresh": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.3.0.tgz", + "integrity": "sha1-ZR+DjiJCTnVm3hYdg1jKoZn4PU8=" + }, + "fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" + }, + "function-bind": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.0.tgz", + "integrity": "sha1-FhdnFMgBeY5Ojyz391KUZ7tKV3E=" + }, + "gelf-stream": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/gelf-stream/-/gelf-stream-1.1.1.tgz", + "integrity": "sha1-nOqbY4asMBx0GDjKPLkeZtv79mk=", + "requires": { + "gelfling": "^0.3.0" + } + }, + "gelfling": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/gelfling/-/gelfling-0.3.1.tgz", + "integrity": "sha1-M2qY+BUQ+a4K8qSU4XRooRap3AQ=" + }, + "generate-function": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/generate-function/-/generate-function-2.0.0.tgz", + "integrity": "sha1-aFj+fAlpt9TpCTM3ZHrHn2DfvnQ=" + }, + "generate-object-property": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/generate-object-property/-/generate-object-property-1.2.0.tgz", + "integrity": "sha1-nA4cQDCM6AT0eDYYuTf6iPmdUNA=", + "requires": { + "is-property": "^1.0.0" + } + }, + "get-caller-file": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-1.0.2.tgz", + "integrity": "sha1-9wLmMSfn4jHBYKgMFVSstw1QR+U=" + }, + "getpass": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.6.tgz", + "integrity": "sha1-KD/9n8ElaECHUxHBtg6MQBhxEOY=", + "requires": { + "assert-plus": "^1.0.0" + }, + "dependencies": { + "assert-plus": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", + "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" + } + } + }, + "glob": { + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/glob/-/glob-6.0.4.tgz", + "integrity": "sha1-DwiGD2oVUSey+t1PnOJLGqtuTSI=", + "requires": { + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "2 || 3", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, + "graceful-fs": { + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.11.tgz", + "integrity": "sha1-Dovf5NHduIVNZOBOp8AOKgJuVlg=" + }, + "graceful-readlink": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/graceful-readlink/-/graceful-readlink-1.0.1.tgz", + "integrity": "sha1-TK+tdrxi8C+gObL5Tpo906ORpyU=" + }, + "growl": { + "version": "1.9.2", + "resolved": "https://registry.npmjs.org/growl/-/growl-1.9.2.tgz", + "integrity": "sha1-Dqd0NxXbjY3ixe3hd14bRayFwC8=" + }, + "handlebars": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.0.6.tgz", + "integrity": "sha1-LORISFBTf5yXqAJtU5m5NcTtTtc=", + "requires": { + "async": "^1.4.0", + "optimist": "^0.6.1", + "source-map": "^0.4.4", + "uglify-js": "^2.6" + }, + "dependencies": { + "async": { + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz", + "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=" + } + } + }, + "har-validator": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-2.0.6.tgz", + "integrity": "sha1-zcvAgYgmWtEZtqWnyKtw7s+10n0=", + "requires": { + "chalk": "^1.1.1", + "commander": "^2.9.0", + "is-my-json-valid": "^2.12.4", + "pinkie-promise": "^2.0.0" + } + }, + "has-ansi": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz", + "integrity": "sha1-NPUEnOHs3ysGSa8+8k5F7TVBbZE=", + "requires": { + "ansi-regex": "^2.0.0" + } + }, + "has-color": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/has-color/-/has-color-0.1.7.tgz", + "integrity": "sha1-ZxRKUmDDT8PMpnfQQdr1L+e3iy8=" + }, + "has-flag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-1.0.0.tgz", + "integrity": "sha1-nZ55MWXOAXoA8AQYxD+UKnsdEfo=" + }, + "hat": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/hat/-/hat-0.0.3.tgz", + "integrity": "sha1-uwFKnmSzeIrtgAWRdBPU/z1QLYo=" + }, + "hawk": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/hawk/-/hawk-3.1.3.tgz", + "integrity": "sha1-B4REvXwWQLD+VA0sm3PVlnjo4cQ=", + "requires": { + "boom": "2.x.x", + "cryptiles": "2.x.x", + "hoek": "2.x.x", + "sntp": "1.x.x" + } + }, + "hoek": { + "version": "2.16.3", + "resolved": "https://registry.npmjs.org/hoek/-/hoek-2.16.3.tgz", + "integrity": "sha1-ILt0A9POo5jpHcRxCo/xuCdKJe0=" + }, + "hosted-git-info": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.1.5.tgz", + "integrity": "sha1-C6gdkNouJas0ozLm7HeTbhWYEYs=" + }, + "hot-shots": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/hot-shots/-/hot-shots-4.3.1.tgz", + "integrity": "sha1-WKbB/3F/JWc75NL3NtHJTV154jk=" + }, + "htmlparser2": { + "version": "3.8.3", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.8.3.tgz", + "integrity": "sha1-mWwosZFRaovoZQGn15dX5ccMEGg=", + "requires": { + "domelementtype": "1", + "domhandler": "2.3", + "domutils": "1.5", + "entities": "1.0", + "readable-stream": "1.1" + }, + "dependencies": { + "entities": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-1.0.0.tgz", + "integrity": "sha1-sph6o4ITR/zeZCsk/fyeT7cSvyY=" + } + } + }, + "http-errors": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.5.1.tgz", + "integrity": "sha1-eIwNLB3iyBuebowBhDtrl+uSB1A=", + "requires": { + "inherits": "2.0.3", + "setprototypeof": "1.0.2", + "statuses": ">= 1.3.1 < 2" + } + }, + "http-signature": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.1.1.tgz", + "integrity": "sha1-33LiZwZs0Kxn+3at+OE0qPvPkb8=", + "requires": { + "assert-plus": "^0.2.0", + "jsprim": "^1.2.2", + "sshpk": "^1.7.0" + } + }, + "i": { + "version": "0.3.5", + "resolved": "https://registry.npmjs.org/i/-/i-0.3.5.tgz", + "integrity": "sha1-HSuFQVjsgWkRPGy39raAHpniEdU=" + }, + "iconv-lite": { + "version": "0.4.15", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.15.tgz", + "integrity": "sha1-/iZaIYrGpXz+hUkn6dBMGYJe3es=" + }, + "inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", + "requires": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "inherit": { + "version": "2.2.6", + "resolved": "https://registry.npmjs.org/inherit/-/inherit-2.2.6.tgz", + "integrity": "sha1-8WFLBshUToEo5CKchjR9tzrZeI0=" + }, + "inherits": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", + "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" + }, + "invert-kv": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/invert-kv/-/invert-kv-1.0.0.tgz", + "integrity": "sha1-EEqOSqym09jNFXqO+L+rLXo//bY=" + }, + "ipaddr.js": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.2.0.tgz", + "integrity": "sha1-irpJyRknmVhb3WQ+DMtQ6K53e6Q=" + }, + "is-arguments": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.0.2.tgz", + "integrity": "sha1-B+MK15UxhEF5tkLS2DmUNRgshyc=" + }, + "is-arrayish": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", + "integrity": "sha1-d8mYQFJ6qOyxqLppe4BkWnqSap0=" + }, + "is-builtin-module": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-builtin-module/-/is-builtin-module-1.0.0.tgz", + "integrity": "sha1-VAVy0096wxGfj3bDDLwbHgN6/74=", + "requires": { + "builtin-modules": "^1.0.0" + } + }, + "is-fullwidth-code-point": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz", + "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=", + "requires": { + "number-is-nan": "^1.0.0" + } + }, + "is-my-json-valid": { + "version": "2.15.0", + "resolved": "https://registry.npmjs.org/is-my-json-valid/-/is-my-json-valid-2.15.0.tgz", + "integrity": "sha1-k27do8o8IR/ZjzstPgjaQ/eykVs=", + "requires": { + "generate-function": "^2.0.0", + "generate-object-property": "^1.1.0", + "jsonpointer": "^4.0.0", + "xtend": "^4.0.0" + } + }, + "is-property": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-property/-/is-property-1.0.2.tgz", + "integrity": "sha1-V/4cTkhHTt1lsJkR8msc1Ald2oQ=" + }, + "is-typedarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", + "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=" + }, + "is-utf8": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/is-utf8/-/is-utf8-0.2.1.tgz", + "integrity": "sha1-Sw2hRCEE0bM2NA6AeX6GXPOffXI=" + }, + "isarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=" + }, + "isexe": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-1.1.2.tgz", + "integrity": "sha1-NvPiLmB1CSD15yQaR2qMakInWtA=" + }, + "isstream": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", + "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" + }, + "istanbul": { + "version": "0.4.5", + "resolved": "https://registry.npmjs.org/istanbul/-/istanbul-0.4.5.tgz", + "integrity": "sha1-ZcfXPUxNqE1POsMQuRj7C4Azczs=", + "requires": { + "abbrev": "1.0.x", + "async": "1.x", + "escodegen": "1.8.x", + "esprima": "2.7.x", + "glob": "^5.0.15", + "handlebars": "^4.0.1", + "js-yaml": "3.x", + "mkdirp": "0.5.x", + "nopt": "3.x", + "once": "1.x", + "resolve": "1.1.x", + "supports-color": "^3.1.0", + "which": "^1.1.1", + "wordwrap": "^1.0.0" + }, + "dependencies": { + "async": { + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz", + "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=" + }, + "glob": { + "version": "5.0.15", + "resolved": "https://registry.npmjs.org/glob/-/glob-5.0.15.tgz", + "integrity": "sha1-G8k2ueAvSmA/zCIuz3Yz0wuLk7E=", + "requires": { + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "2 || 3", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, + "supports-color": { + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-3.2.3.tgz", + "integrity": "sha1-ZawFBLOVQXHYpklGsq48u4pfVPY=", + "requires": { + "has-flag": "^1.0.0" + } + }, + "wordwrap": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz", + "integrity": "sha1-J1hIEIkUVqQXHI0CJkQa3pDLyus=" + } + } + }, + "jade": { + "version": "0.26.3", + "resolved": "https://registry.npmjs.org/jade/-/jade-0.26.3.tgz", + "integrity": "sha1-jxDXl32NefL2/4YqgbBRPMslaGw=", + "requires": { + "commander": "0.6.1", + "mkdirp": "0.3.0" + }, + "dependencies": { + "commander": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-0.6.1.tgz", + "integrity": "sha1-+mihT2qUXVTbvlDYzbMyDp47GgY=" + }, + "mkdirp": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.3.0.tgz", + "integrity": "sha1-G79asbqCevI1dRQ0kEJkVfSB/h4=" + } + } + }, + "js-yaml": { + "version": "3.7.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.7.0.tgz", + "integrity": "sha1-XJZ93YN6m/3KXy3oQlOr6KHAO4A=", + "requires": { + "argparse": "^1.0.7", + "esprima": "^2.6.0" + } + }, + "jscs": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/jscs/-/jscs-3.0.7.tgz", + "integrity": "sha1-cUG03/W4bjLQ6Z12S4NnZ8MNIBo=", + "requires": { + "chalk": "~1.1.0", + "cli-table": "~0.3.1", + "commander": "~2.9.0", + "cst": "^0.4.3", + "estraverse": "^4.1.0", + "exit": "~0.1.2", + "glob": "^5.0.1", + "htmlparser2": "3.8.3", + "js-yaml": "~3.4.0", + "jscs-jsdoc": "^2.0.0", + "jscs-preset-wikimedia": "~1.0.0", + "jsonlint": "~1.6.2", + "lodash": "~3.10.0", + "minimatch": "~3.0.0", + "natural-compare": "~1.2.2", + "pathval": "~0.1.1", + "prompt": "~0.2.14", + "reserved-words": "^0.1.1", + "resolve": "^1.1.6", + "strip-bom": "^2.0.0", + "strip-json-comments": "~1.0.2", + "to-double-quotes": "^2.0.0", + "to-single-quotes": "^2.0.0", + "vow": "~0.4.8", + "vow-fs": "~0.3.4", + "xmlbuilder": "^3.1.0" + }, + "dependencies": { + "estraverse": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.2.0.tgz", + "integrity": "sha1-De4/7TH81GlhjOc0IJn8GvoL2xM=" + }, + "glob": { + "version": "5.0.15", + "resolved": "https://registry.npmjs.org/glob/-/glob-5.0.15.tgz", + "integrity": "sha1-G8k2ueAvSmA/zCIuz3Yz0wuLk7E=", + "requires": { + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "2 || 3", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, + "js-yaml": { + "version": "3.4.6", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.4.6.tgz", + "integrity": "sha1-a+GyP2JJ9T0pM3D9TRqqY84bTrA=", + "requires": { + "argparse": "^1.0.2", + "esprima": "^2.6.0", + "inherit": "^2.2.2" + } + } + } + }, + "jscs-jsdoc": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/jscs-jsdoc/-/jscs-jsdoc-2.0.0.tgz", + "integrity": "sha1-9T684CmqMSW9iCkLpQ1k1FEKSHE=", + "requires": { + "comment-parser": "^0.3.1", + "jsdoctypeparser": "~1.2.0" + } + }, + "jscs-preset-wikimedia": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/jscs-preset-wikimedia/-/jscs-preset-wikimedia-1.0.0.tgz", + "integrity": "sha1-//VjNCA4/C6IJre7cwnDrjQG/H4=" + }, + "jsdoctypeparser": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/jsdoctypeparser/-/jsdoctypeparser-1.2.0.tgz", + "integrity": "sha1-597cFToRhJ/8UUEUSuhqfvDCU5I=", + "requires": { + "lodash": "^3.7.0" + } + }, + "jshint": { + "version": "2.9.4", + "resolved": "https://registry.npmjs.org/jshint/-/jshint-2.9.4.tgz", + "integrity": "sha1-XjupeEjVKQJz21FK7kf+JM9ZKTQ=", + "requires": { + "cli": "~1.0.0", + "console-browserify": "1.1.x", + "exit": "0.1.x", + "htmlparser2": "3.8.x", + "lodash": "3.7.x", + "minimatch": "~3.0.2", + "shelljs": "0.3.x", + "strip-json-comments": "1.0.x" + }, + "dependencies": { + "lodash": { + "version": "3.7.0", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-3.7.0.tgz", + "integrity": "sha1-Nni9irmVBXwHreg27S7wh9qBHUU=" + } + } + }, + "json-schema": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz", + "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=" + }, + "json-stringify-safe": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", + "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=" + }, + "jsonlint": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/jsonlint/-/jsonlint-1.6.2.tgz", + "integrity": "sha1-VzcEUIX1XrRVxosf9OvAG9UOiDA=", + "requires": { + "JSV": ">= 4.0.x", + "nomnom": ">= 1.5.x" + } + }, + "jsonpointer": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/jsonpointer/-/jsonpointer-4.0.1.tgz", + "integrity": "sha1-T9kss04OnbPInIYi7PUfm5eMbLk=" + }, + "jsprim": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.3.1.tgz", + "integrity": "sha1-KnJW9wQSop7jZwqspiWZTE3P8lI=", + "requires": { + "extsprintf": "1.0.2", + "json-schema": "0.2.3", + "verror": "1.3.6" + } + }, + "kad": { + "version": "git+https://github.com/gwicke/kad.git#f35971036f43814043245da82b12d035b7bbfd16", + "from": "git+https://github.com/gwicke/kad.git#master", + "requires": { + "async": "^0.9.0", + "clarinet": "^0.11.0", + "colors": "^1.0.3", + "hat": "0.0.3", + "kad-fs": "0.0.4", + "kad-localstorage": "0.0.7", + "kad-memstore": "0.0.1", + "lodash": "^3.6.0", + "merge": "^1.2.0", + "ms": "^0.7.0", + "msgpack5": "^3.3.0" + } + }, + "kad-fs": { + "version": "0.0.4", + "resolved": "https://registry.npmjs.org/kad-fs/-/kad-fs-0.0.4.tgz", + "integrity": "sha1-Aupapc8iIlclV5YnzP1tJmNyKJo=", + "requires": { + "readable-stream": "^2.0.4" + }, + "dependencies": { + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "readable-stream": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.2.2.tgz", + "integrity": "sha1-qeb+w8fdqF+LsbO6cChgRVb8gl4=", + "requires": { + "buffer-shims": "^1.0.0", + "core-util-is": "~1.0.0", + "inherits": "~2.0.1", + "isarray": "~1.0.0", + "process-nextick-args": "~1.0.6", + "string_decoder": "~0.10.x", + "util-deprecate": "~1.0.1" + } + } + } + }, + "kad-localstorage": { + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/kad-localstorage/-/kad-localstorage-0.0.7.tgz", + "integrity": "sha1-96LngNpT+yi5Q8LFqJTCeaqBDxc=", + "requires": { + "dom-storage": "^2.0.1" + } + }, + "kad-memstore": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/kad-memstore/-/kad-memstore-0.0.1.tgz", + "integrity": "sha1-g8t0hJasSRxxNRBMvla4jKc5JHc=", + "requires": { + "readable-stream": "^2.0.5" + }, + "dependencies": { + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "readable-stream": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.2.2.tgz", + "integrity": "sha1-qeb+w8fdqF+LsbO6cChgRVb8gl4=", + "requires": { + "buffer-shims": "^1.0.0", + "core-util-is": "~1.0.0", + "inherits": "~2.0.1", + "isarray": "~1.0.0", + "process-nextick-args": "~1.0.6", + "string_decoder": "~0.10.x", + "util-deprecate": "~1.0.1" + } + } + } + }, + "lcid": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/lcid/-/lcid-1.0.0.tgz", + "integrity": "sha1-MIrMr6C8SDo4Z7S28rlQYlHRuDU=", + "requires": { + "invert-kv": "^1.0.0" + } + }, + "lcov-parse": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/lcov-parse/-/lcov-parse-0.0.10.tgz", + "integrity": "sha1-GwuP+ayceIklBYK3C3ExXZ2m2aM=" + }, + "levn": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz", + "integrity": "sha1-OwmSTt+fCDwEkP3UwLxEIeBHZO4=", + "requires": { + "prelude-ls": "~1.1.2", + "type-check": "~0.3.2" + } + }, + "limitation": { + "version": "0.1.9", + "resolved": "https://registry.npmjs.org/limitation/-/limitation-0.1.9.tgz", + "integrity": "sha1-ugVf9906JnplzGvi3spOpr672wM=", + "requires": { + "bluebird": "^3.3.1", + "kad": "git+https://github.com/gwicke/kad.git#master", + "readable-stream": "^2.0.5" + }, + "dependencies": { + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "readable-stream": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.2.2.tgz", + "integrity": "sha1-qeb+w8fdqF+LsbO6cChgRVb8gl4=", + "requires": { + "buffer-shims": "^1.0.0", + "core-util-is": "~1.0.0", + "inherits": "~2.0.1", + "isarray": "~1.0.0", + "process-nextick-args": "~1.0.6", + "string_decoder": "~0.10.x", + "util-deprecate": "~1.0.1" + } + } + } + }, + "load-json-file": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-1.1.0.tgz", + "integrity": "sha1-lWkFcI1YtLq0wiYbBPWfMcmTdMA=", + "requires": { + "graceful-fs": "^4.1.2", + "parse-json": "^2.2.0", + "pify": "^2.0.0", + "pinkie-promise": "^2.0.0", + "strip-bom": "^2.0.0" + } + }, + "lodash": { + "version": "3.10.1", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-3.10.1.tgz", + "integrity": "sha1-W/Rejkm6QYnhfUgnid/RW9FAt7Y=" + }, + "lodash.assign": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.assign/-/lodash.assign-4.2.0.tgz", + "integrity": "sha1-DZnzzNem0mHRm9rrkkUAXShYCOc=" + }, + "log-driver": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/log-driver/-/log-driver-1.2.5.tgz", + "integrity": "sha1-euTsJXMC/XkNVXyxDJcQDYV7AFY=" + }, + "lru-cache": { + "version": "2.7.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-2.7.3.tgz", + "integrity": "sha1-bUUk6LlV+V1PW1iFHOId1y+06VI=" + }, + "media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=" + }, + "mediawiki-title": { + "version": "0.5.6", + "resolved": "https://registry.npmjs.org/mediawiki-title/-/mediawiki-title-0.5.6.tgz", + "integrity": "sha1-VJBpKU4ncoofE77T1wXWvuz06iQ=" + }, + "merge": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/merge/-/merge-1.2.0.tgz", + "integrity": "sha1-dTHjnUlJwoGma4xabgJl6LBYlNo=" + }, + "merge-descriptors": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", + "integrity": "sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E=" + }, + "methods": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", + "integrity": "sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4=" + }, + "mime": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.3.4.tgz", + "integrity": "sha1-EV+eO2s9rylZmDyzjxSaLUDrXVM=" + }, + "mime-db": { + "version": "1.26.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.26.0.tgz", + "integrity": "sha1-6v/NDk/Gk1z4E02iRuLmw1MFrf8=" + }, + "mime-types": { + "version": "2.1.14", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.14.tgz", + "integrity": "sha1-9+99l1g/yvO30oK2+LVnnaselO4=", + "requires": { + "mime-db": "~1.26.0" + } + }, + "minimatch": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.3.tgz", + "integrity": "sha1-Kk5AkLlrLbBqnX3wEFWmKnfJt3Q=", + "requires": { + "brace-expansion": "^1.0.0" + } + }, + "minimist": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.10.tgz", + "integrity": "sha1-3j+YVD2/lggr5IrRoMfNqDYwHc8=" + }, + "mkdirp": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", + "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", + "requires": { + "minimist": "0.0.8" + }, + "dependencies": { + "minimist": { + "version": "0.0.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", + "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=" + } + } + }, + "mocha": { + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/mocha/-/mocha-2.5.3.tgz", + "integrity": "sha1-FhvlvetJZ3HrmzV0UFC2IrWu/Fg=", + "requires": { + "commander": "2.3.0", + "debug": "2.2.0", + "diff": "1.4.0", + "escape-string-regexp": "1.0.2", + "glob": "3.2.11", + "growl": "1.9.2", + "jade": "0.26.3", + "mkdirp": "0.5.1", + "supports-color": "1.2.0", + "to-iso-string": "0.0.2" + }, + "dependencies": { + "commander": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.3.0.tgz", + "integrity": "sha1-/UMOiJgy7DU7ms0d4hfBHLPu+HM=" + }, + "debug": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.2.0.tgz", + "integrity": "sha1-+HBX6ZWxofauaklgZkE3vFbwOdo=", + "requires": { + "ms": "0.7.1" + } + }, + "escape-string-regexp": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.2.tgz", + "integrity": "sha1-Tbwv5nTnGUnK8/smlc5/LcHZqNE=" + }, + "glob": { + "version": "3.2.11", + "resolved": "https://registry.npmjs.org/glob/-/glob-3.2.11.tgz", + "integrity": "sha1-Spc/Y1uRkPcV0QmH1cAP0oFevj0=", + "requires": { + "inherits": "2", + "minimatch": "0.3" + } + }, + "minimatch": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-0.3.0.tgz", + "integrity": "sha1-J12O2qxPG7MyZHIInnlJyDlGmd0=", + "requires": { + "lru-cache": "2", + "sigmund": "~1.0.0" + } + }, + "ms": { + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/ms/-/ms-0.7.1.tgz", + "integrity": "sha1-nNE8A62/8ltl7/3nzoZO6VIBcJg=" + }, + "supports-color": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-1.2.0.tgz", + "integrity": "sha1-/x7R5hFp0Gs88tWI4YixjYhH4X4=" + } + } + }, + "ms": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-0.7.2.tgz", + "integrity": "sha1-riXPJRKziFodldfwN4aNhDESR2U=" + }, + "msgpack5": { + "version": "3.4.1", + "resolved": "https://registry.npmjs.org/msgpack5/-/msgpack5-3.4.1.tgz", + "integrity": "sha1-NQ7zWJnGyHc3EP2E2IHd0zQKgRQ=", + "requires": { + "bl": "^1.0.0", + "inherits": "^2.0.1", + "readable-stream": "^2.0.1" + }, + "dependencies": { + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "readable-stream": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.2.2.tgz", + "integrity": "sha1-qeb+w8fdqF+LsbO6cChgRVb8gl4=", + "requires": { + "buffer-shims": "^1.0.0", + "core-util-is": "~1.0.0", + "inherits": "~2.0.1", + "isarray": "~1.0.0", + "process-nextick-args": "~1.0.6", + "string_decoder": "~0.10.x", + "util-deprecate": "~1.0.1" + } + } + } + }, + "mute-stream": { + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.7.tgz", + "integrity": "sha1-MHXOk7whuPq0PhvE2n6BFe0ee6s=" + }, + "natural-compare": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.2.2.tgz", + "integrity": "sha1-H5bWDjFBysG20FZTzg2urHY69qo=" + }, + "ncp": { + "version": "0.4.2", + "resolved": "https://registry.npmjs.org/ncp/-/ncp-0.4.2.tgz", + "integrity": "sha1-q8xsvT7C7Spyn/bnwfqPAXhKhXQ=" + }, + "negotiator": { + "version": "git+https://github.com/arlolra/negotiator.git#0418ab4e9a665772b7e233564a4525c9d9a8ec3a", + "from": "git+https://github.com/arlolra/negotiator.git#full-parse-access" + }, + "nock": { + "version": "8.2.1", + "resolved": "https://registry.npmjs.org/nock/-/nock-8.2.1.tgz", + "integrity": "sha1-ZMxl4b3TiT9Yy6fhq/3Dj0DwNko=", + "requires": { + "chai": ">=1.9.2 <4.0.0", + "debug": "^2.2.0", + "deep-equal": "^1.0.0", + "json-stringify-safe": "^5.0.1", + "lodash": "~4.9.0", + "mkdirp": "^0.5.0", + "propagate": "0.4.0", + "qs": "^6.0.2" + }, + "dependencies": { + "lodash": { + "version": "4.9.0", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.9.0.tgz", + "integrity": "sha1-TCDXQvA86F3HAODderm8q4Xm/BQ=" + } + } + }, + "node-uuid": { + "version": "1.4.7", + "resolved": "https://registry.npmjs.org/node-uuid/-/node-uuid-1.4.7.tgz", + "integrity": "sha1-baWhdmjEs91ZYjvaEc9/pMH2Cm8=" + }, + "nomnom": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/nomnom/-/nomnom-1.8.1.tgz", + "integrity": "sha1-IVH3Ikcrp55Qp2/BJbuMjy5Nwqc=", + "requires": { + "chalk": "~0.4.0", + "underscore": "~1.6.0" + }, + "dependencies": { + "ansi-styles": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-1.0.0.tgz", + "integrity": "sha1-yxAt8cVvUSPquLZ817mAJ6AnkXg=" + }, + "chalk": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-0.4.0.tgz", + "integrity": "sha1-UZmj3c0MHv4jvAjBsCewYXbgxk8=", + "requires": { + "ansi-styles": "~1.0.0", + "has-color": "~0.1.0", + "strip-ansi": "~0.1.0" + } + }, + "strip-ansi": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-0.1.1.tgz", + "integrity": "sha1-OeipjQRNFQZgq+SmgIrPcLt7yZE=" + } + } + }, + "nopt": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-3.0.6.tgz", + "integrity": "sha1-xkZdvwirzU2zWTF/eaxopkayj/k=", + "requires": { + "abbrev": "1" + } + }, + "normalize-package-data": { + "version": "2.3.5", + "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.3.5.tgz", + "integrity": "sha1-jZJPFClg4Xd+f/4XBUNjHMfLAt8=", + "requires": { + "hosted-git-info": "^2.1.4", + "is-builtin-module": "^1.0.0", + "semver": "2 || 3 || 4 || 5", + "validate-npm-package-license": "^3.0.1" + } + }, + "nsp": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/nsp/-/nsp-2.6.2.tgz", + "integrity": "sha1-k9+0xbKIXMNU2MoYtz8J5SucixY=", + "requires": { + "chalk": "^1.1.1", + "cli-table": "^0.3.1", + "https-proxy-agent": "^1.0.0", + "joi": "^6.9.1", + "nodesecurity-npm-utils": "^5.0.0", + "path-is-absolute": "^1.0.0", + "rc": "^1.1.2", + "semver": "^5.0.3", + "subcommand": "^2.0.3", + "wreck": "^6.3.0" + }, + "dependencies": { + "chalk": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", + "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=", + "requires": { + "ansi-styles": "^2.2.1", + "escape-string-regexp": "^1.0.2", + "has-ansi": "^2.0.0", + "strip-ansi": "^3.0.0", + "supports-color": "^2.0.0" + }, + "dependencies": { + "ansi-styles": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz", + "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=" + }, + "escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=" + }, + "has-ansi": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz", + "integrity": "sha1-NPUEnOHs3ysGSa8+8k5F7TVBbZE=", + "requires": { + "ansi-regex": "^2.0.0" + }, + "dependencies": { + "ansi-regex": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.0.0.tgz", + "integrity": "sha1-xQYbbg74qBd15Q9dZhUb9r83EQc=" + } + } + }, + "strip-ansi": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", + "requires": { + "ansi-regex": "^2.0.0" + }, + "dependencies": { + "ansi-regex": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.0.0.tgz", + "integrity": "sha1-xQYbbg74qBd15Q9dZhUb9r83EQc=" + } + } + }, + "supports-color": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", + "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=" + } + } + }, + "cli-table": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/cli-table/-/cli-table-0.3.1.tgz", + "integrity": "sha1-9TsFJmqLGguTSz0IIebi3FkUriM=", + "requires": { + "colors": "1.0.3" + }, + "dependencies": { + "colors": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/colors/-/colors-1.0.3.tgz", + "integrity": "sha1-BDP0TYCWgP3rYO0mDxsMJi6CpAs=" + } + } + }, + "https-proxy-agent": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-1.0.0.tgz", + "integrity": "sha1-NffabEjOTdv6JkiRrFk+5f+GceY=", + "requires": { + "agent-base": "2", + "debug": "2", + "extend": "3" + }, + "dependencies": { + "agent-base": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-2.0.1.tgz", + "integrity": "sha1-vY+ehqjrIh//oHvRS+/VXfFCgV4=", + "requires": { + "extend": "~3.0.0", + "semver": "~5.0.1" + }, + "dependencies": { + "semver": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.0.3.tgz", + "integrity": "sha1-d0Zt5YnNXTyV8TiqeLxWmjy10no=" + } + } + }, + "debug": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.2.0.tgz", + "integrity": "sha1-+HBX6ZWxofauaklgZkE3vFbwOdo=", + "requires": { + "ms": "0.7.1" + }, + "dependencies": { + "ms": { + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/ms/-/ms-0.7.1.tgz", + "integrity": "sha1-nNE8A62/8ltl7/3nzoZO6VIBcJg=" + } + } + }, + "extend": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.0.tgz", + "integrity": "sha1-WkdDU7nzNT3dgXbf03uRyDpG8dQ=" + } + } + }, + "joi": { + "version": "6.10.1", + "resolved": "https://registry.npmjs.org/joi/-/joi-6.10.1.tgz", + "integrity": "sha1-TVDDGAeRIgAP5fFq8f+OGRe3fgY=", + "requires": { + "hoek": "2.x.x", + "isemail": "1.x.x", + "moment": "2.x.x", + "topo": "1.x.x" + }, + "dependencies": { + "hoek": { + "version": "2.16.3", + "resolved": "https://registry.npmjs.org/hoek/-/hoek-2.16.3.tgz", + "integrity": "sha1-ILt0A9POo5jpHcRxCo/xuCdKJe0=" + }, + "isemail": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/isemail/-/isemail-1.2.0.tgz", + "integrity": "sha1-vgPfjMPineTSxd9lASY/H6RZXpo=" + }, + "moment": { + "version": "2.12.0", + "resolved": "https://registry.npmjs.org/moment/-/moment-2.12.0.tgz", + "integrity": "sha1-3CVg0Zg41sBzGxpq+gRnUmTTYNY=" + }, + "topo": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/topo/-/topo-1.1.0.tgz", + "integrity": "sha1-6ddRYV0buH3IZdsYL6HKCl71NtU=", + "requires": { + "hoek": "2.x.x" + } + } + } + }, + "nodesecurity-npm-utils": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/nodesecurity-npm-utils/-/nodesecurity-npm-utils-5.0.0.tgz", + "integrity": "sha1-Baow3jDKjIRcQEjpT9eOXgi1Xtk=" + }, + "path-is-absolute": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.0.tgz", + "integrity": "sha1-Jj2tpmqz8vsQv3+dJN2PPlcO+RI=" + }, + "rc": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.1.6.tgz", + "integrity": "sha1-Q2UbdrauU7XIAvEVH6P8OwWZack=", + "requires": { + "deep-extend": "~0.4.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~1.0.4" + }, + "dependencies": { + "deep-extend": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.4.1.tgz", + "integrity": "sha1-7+QRPQgIX05vlod1mBD4B0aeIlM=" + }, + "ini": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.4.tgz", + "integrity": "sha1-BTfLedr1m1mhpRff9wbIbsA5Fi4=" + }, + "minimist": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", + "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=" + }, + "strip-json-comments": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-1.0.4.tgz", + "integrity": "sha1-HhX7ysl9Pumb8tc7TGVrCCu6+5E=" + } + } + }, + "semver": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.1.0.tgz", + "integrity": "sha1-hfLPhVBGXE3wAM99hvawVBBqueU=" + }, + "subcommand": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/subcommand/-/subcommand-2.0.3.tgz", + "integrity": "sha1-mz/Rp1PjxEHwBBDLRBMdhlX1LDI=", + "requires": { + "cliclopts": "^1.1.0", + "debug": "^2.1.3", + "minimist": "^1.2.0", + "xtend": "^4.0.0" + }, + "dependencies": { + "cliclopts": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/cliclopts/-/cliclopts-1.1.1.tgz", + "integrity": "sha1-aUMcfLWvcjd0sNORG0w3USQxkQ8=" + }, + "debug": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.2.0.tgz", + "integrity": "sha1-+HBX6ZWxofauaklgZkE3vFbwOdo=", + "requires": { + "ms": "0.7.1" + }, + "dependencies": { + "ms": { + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/ms/-/ms-0.7.1.tgz", + "integrity": "sha1-nNE8A62/8ltl7/3nzoZO6VIBcJg=" + } + } + }, + "minimist": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", + "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=" + }, + "xtend": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz", + "integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68=" + } + } + }, + "wreck": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/wreck/-/wreck-6.3.0.tgz", + "integrity": "sha1-oTaXafB7u2LWo3gzanhx/Hc8dAs=", + "requires": { + "boom": "2.x.x", + "hoek": "2.x.x" + }, + "dependencies": { + "boom": { + "version": "2.10.1", + "resolved": "https://registry.npmjs.org/boom/-/boom-2.10.1.tgz", + "integrity": "sha1-OciRjO/1eZ+D+UkqhI9iWt0Mdm8=", + "requires": { + "hoek": "2.x.x" + } + }, + "hoek": { + "version": "2.16.3", + "resolved": "https://registry.npmjs.org/hoek/-/hoek-2.16.3.tgz", + "integrity": "sha1-ILt0A9POo5jpHcRxCo/xuCdKJe0=" + } + } + } + } + }, + "number-is-nan": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz", + "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=" + }, + "oauth-sign": { + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.8.2.tgz", + "integrity": "sha1-Rqarfwrq2N6unsBWV4C31O/rnUM=" + }, + "object-keys": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.0.11.tgz", + "integrity": "sha1-xUYBd4rVYPEULODgG8yotW0TQm0=" + }, + "object.assign": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.0.4.tgz", + "integrity": "sha1-scnMBE7xuf5jYG/BQau7MuFHMMw=", + "requires": { + "define-properties": "^1.1.2", + "function-bind": "^1.1.0", + "object-keys": "^1.0.10" + } + }, + "on-finished": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", + "integrity": "sha1-IPEzZIGwg811M3mSoWlxqi2QaUc=", + "requires": { + "ee-first": "1.1.1" + } + }, + "on-headers": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/on-headers/-/on-headers-1.0.1.tgz", + "integrity": "sha1-ko9dD0cNSTQmUepnlLCFfBAGk/c=" + }, + "once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", + "requires": { + "wrappy": "1" + } + }, + "optimist": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/optimist/-/optimist-0.6.1.tgz", + "integrity": "sha1-2j6nRob6IaGaERwybpDrFaAZZoY=", + "requires": { + "minimist": "~0.0.1", + "wordwrap": "~0.0.2" + } + }, + "optionator": { + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.2.tgz", + "integrity": "sha1-NkxeQJ0/TWMB1sC0wFu6UBgK62Q=", + "requires": { + "deep-is": "~0.1.3", + "fast-levenshtein": "~2.0.4", + "levn": "~0.3.0", + "prelude-ls": "~1.1.2", + "type-check": "~0.3.2", + "wordwrap": "~1.0.0" + }, + "dependencies": { + "wordwrap": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz", + "integrity": "sha1-J1hIEIkUVqQXHI0CJkQa3pDLyus=" + } + } + }, + "os-locale": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/os-locale/-/os-locale-1.4.0.tgz", + "integrity": "sha1-IPnxeuKe00XoveWDsT0gCYA8FNk=", + "requires": { + "lcid": "^1.0.0" + } + }, + "parse-json": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-2.2.0.tgz", + "integrity": "sha1-9ID0BDTvgHQfhGkJn43qGPVaTck=", + "requires": { + "error-ex": "^1.2.0" + } + }, + "parseurl": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.1.tgz", + "integrity": "sha1-yKuMkiO6NIiKpkopeyiFO+wY2lY=" + }, + "path-exists": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-2.1.0.tgz", + "integrity": "sha1-D+tsZPD8UY2adU3V77YscCJ2H0s=", + "requires": { + "pinkie-promise": "^2.0.0" + } + }, + "path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=" + }, + "path-to-regexp": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", + "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" + }, + "path-type": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-1.1.0.tgz", + "integrity": "sha1-WcRPfuSR2nBNpBXaWkBwuk+P5EE=", + "requires": { + "graceful-fs": "^4.1.2", + "pify": "^2.0.0", + "pinkie-promise": "^2.0.0" + } + }, + "pathval": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/pathval/-/pathval-0.1.1.tgz", + "integrity": "sha1-CPkRzcqczllCiA2ngXvAtyO2bYI=" + }, + "pegjs": { + "version": "git+https://github.com/tstarling/pegjs.git#36d584bd7bbc564c86c058c5dfe8053b1fe1d584", + "from": "git+https://github.com/tstarling/pegjs.git#fork" + }, + "pify": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=" + }, + "pinkie": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/pinkie/-/pinkie-2.0.4.tgz", + "integrity": "sha1-clVrgM+g1IqXToDnckjoDtT3+HA=" + }, + "pinkie-promise": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/pinkie-promise/-/pinkie-promise-2.0.1.tgz", + "integrity": "sha1-ITXW36ejWMBprJsXh3YogihFD/o=", + "requires": { + "pinkie": "^2.0.0" + } + }, + "pkginfo": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/pkginfo/-/pkginfo-0.4.0.tgz", + "integrity": "sha1-NJ27f/04CB/K3AhT32h/DHdEzWU=" + }, + "prelude-ls": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", + "integrity": "sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ=" + }, + "prfun": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/prfun/-/prfun-2.1.4.tgz", + "integrity": "sha1-eHF9m3GM58q1XiC58kOI1fpR1cA=", + "requires": { + "core-js": "^2.3.0" + } + }, + "process-nextick-args": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.7.tgz", + "integrity": "sha1-FQ4gt1ZZCtP5EJPyWk8q2L/zC6M=" + }, + "promise": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/promise/-/promise-7.1.1.tgz", + "integrity": "sha1-SJZUxpJha4qlWwck+oCbt9tJxb8=", + "requires": { + "asap": "~2.0.3" + } + }, + "prompt": { + "version": "0.2.14", + "resolved": "https://registry.npmjs.org/prompt/-/prompt-0.2.14.tgz", + "integrity": "sha1-V3VPZPVD/XsIRXB8gY7OYY8F/9w=", + "requires": { + "pkginfo": "0.x.x", + "read": "1.0.x", + "revalidator": "0.1.x", + "utile": "0.2.x", + "winston": "0.8.x" + } + }, + "propagate": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/propagate/-/propagate-0.4.0.tgz", + "integrity": "sha1-8/zKCm/gZzanulcpZgaWF8EwtIE=" + }, + "proxy-addr": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-1.1.3.tgz", + "integrity": "sha1-3JdQL1ci6IhGez+iKXp7H/R98HQ=", + "requires": { + "forwarded": "~0.1.0", + "ipaddr.js": "1.2.0" + } + }, + "punycode": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", + "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=" + }, + "qs": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.2.1.tgz", + "integrity": "sha1-zgPF/wk1vB2daanxTL0Y5WjWdiU=" + }, + "range-parser": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.0.tgz", + "integrity": "sha1-9JvmtIeJTdxA3MlKMi9hEJLgDV4=" + }, + "raw-body": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.2.0.tgz", + "integrity": "sha1-mUl2z2pQlqQRYoQEkvC9xdbn+5Y=", + "requires": { + "bytes": "2.4.0", + "iconv-lite": "0.4.15", + "unpipe": "1.0.0" + } + }, + "read": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/read/-/read-1.0.7.tgz", + "integrity": "sha1-s9oZvQUkMal2cdRKQmNK33ELQMQ=", + "requires": { + "mute-stream": "~0.0.4" + } + }, + "read-pkg": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-1.1.0.tgz", + "integrity": "sha1-9f+qXs0pyzHAR0vKfXVra7KePyg=", + "requires": { + "load-json-file": "^1.0.0", + "normalize-package-data": "^2.3.2", + "path-type": "^1.0.0" + } + }, + "read-pkg-up": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/read-pkg-up/-/read-pkg-up-1.0.1.tgz", + "integrity": "sha1-nWPBMnbAZZGNV/ACpX9AobZD+wI=", + "requires": { + "find-up": "^1.0.0", + "read-pkg": "^1.0.0" + } + }, + "readable-stream": { + "version": "1.1.14", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", + "integrity": "sha1-fPTFTvZI44EwhMY23SB54WbAgdk=", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.1", + "isarray": "0.0.1", + "string_decoder": "~0.10.x" + } + }, + "reduce-component": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/reduce-component/-/reduce-component-1.0.1.tgz", + "integrity": "sha1-4Mk1QsV0UhvqE98PlIjtgqt3xdo=" + }, + "regenerator-runtime": { + "version": "0.10.1", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.10.1.tgz", + "integrity": "sha1-JX9BlhzkRVixj3gUr0jBdVn5+us=" + }, + "request": { + "version": "2.79.0", + "resolved": "https://registry.npmjs.org/request/-/request-2.79.0.tgz", + "integrity": "sha1-Tf5b9r6LjNw3/Pk+BLZVd3InEN4=", + "requires": { + "aws-sign2": "~0.6.0", + "aws4": "^1.2.1", + "caseless": "~0.11.0", + "combined-stream": "~1.0.5", + "extend": "~3.0.0", + "forever-agent": "~0.6.1", + "form-data": "~2.1.1", + "har-validator": "~2.0.6", + "hawk": "~3.1.3", + "http-signature": "~1.1.0", + "is-typedarray": "~1.0.0", + "isstream": "~0.1.2", + "json-stringify-safe": "~5.0.1", + "mime-types": "~2.1.7", + "oauth-sign": "~0.8.1", + "qs": "~6.3.0", + "stringstream": "~0.0.4", + "tough-cookie": "~2.3.0", + "tunnel-agent": "~0.4.1", + "uuid": "^3.0.0" + }, + "dependencies": { + "qs": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.3.0.tgz", + "integrity": "sha1-9AOyZPI7wBIox0ExtAfxjV6l1EI=" + }, + "uuid": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.0.1.tgz", + "integrity": "sha1-ZUS7ot/ajBzxfmKaOjBeK7H+5sE=" + } + } + }, + "require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=" + }, + "require-main-filename": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-1.0.1.tgz", + "integrity": "sha1-l/cXtp1IeE9fUmpsWqj/3aBVpNE=" + }, + "reserved-words": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/reserved-words/-/reserved-words-0.1.1.tgz", + "integrity": "sha1-b3wV5eVhTFDalhYw2kat3IfAzvI=" + }, + "resolve": { + "version": "1.1.7", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.1.7.tgz", + "integrity": "sha1-IDEU2CrSxe2ejgQRs5ModeiJ6Xs=" + }, + "revalidator": { + "version": "0.1.8", + "resolved": "https://registry.npmjs.org/revalidator/-/revalidator-0.1.8.tgz", + "integrity": "sha1-/s5hv6DBtSoga9axgZgYS91SOjs=" + }, + "rimraf": { + "version": "2.5.4", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.5.4.tgz", + "integrity": "sha1-loAAk8vxoMhr2VtGJUZ1NcKd+gQ=", + "requires": { + "glob": "^7.0.5" + }, + "dependencies": { + "glob": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.1.tgz", + "integrity": "sha1-gFIR3wT6rxxjo2ADBs31reULLsg=", + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.2", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + } + } + }, + "semver": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.3.0.tgz", + "integrity": "sha1-myzl094C0XxgEq0yaqa00M9U+U8=" + }, + "send": { + "version": "0.14.1", + "resolved": "https://registry.npmjs.org/send/-/send-0.14.1.tgz", + "integrity": "sha1-qVSYQyU5L1FTKndgdg5FlZjIn3o=", + "requires": { + "debug": "~2.2.0", + "depd": "~1.1.0", + "destroy": "~1.0.4", + "encodeurl": "~1.0.1", + "escape-html": "~1.0.3", + "etag": "~1.7.0", + "fresh": "0.3.0", + "http-errors": "~1.5.0", + "mime": "1.3.4", + "ms": "0.7.1", + "on-finished": "~2.3.0", + "range-parser": "~1.2.0", + "statuses": "~1.3.0" + }, + "dependencies": { + "debug": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.2.0.tgz", + "integrity": "sha1-+HBX6ZWxofauaklgZkE3vFbwOdo=", + "requires": { + "ms": "0.7.1" + } + }, + "ms": { + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/ms/-/ms-0.7.1.tgz", + "integrity": "sha1-nNE8A62/8ltl7/3nzoZO6VIBcJg=" + } + } + }, + "serve-favicon": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/serve-favicon/-/serve-favicon-2.3.2.tgz", + "integrity": "sha1-3UGeJo3gEqtysxnTN/IQUBP5OB8=", + "requires": { + "etag": "~1.7.0", + "fresh": "0.3.0", + "ms": "0.7.2", + "parseurl": "~1.3.1" + } + }, + "serve-static": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.11.1.tgz", + "integrity": "sha1-1sznaTUF9zPHWd5Xvvwa92wPCAU=", + "requires": { + "encodeurl": "~1.0.1", + "escape-html": "~1.0.3", + "parseurl": "~1.3.1", + "send": "0.14.1" + } + }, + "service-runner": { + "version": "2.1.13", + "resolved": "https://registry.npmjs.org/service-runner/-/service-runner-2.1.13.tgz", + "integrity": "sha1-6P94uTIw19gx6j7VWHqiKSuCnOs=", + "requires": { + "bluebird": "^3.4.6", + "bunyan": "^1.8.1", + "bunyan-syslog-udp": "^0.1.0", + "extend": "^3.0.0", + "gelf-stream": "^1.1.1", + "hot-shots": "^4.2.0", + "js-yaml": "^3.6.1", + "limitation": "^0.1.9", + "semver": "^5.3.0", + "yargs": "^5.0.0" + } + }, + "set-blocking": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", + "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=" + }, + "setprototypeof": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.0.2.tgz", + "integrity": "sha1-gaVSFB7BBLiOic44MQOtXGZWTQg=" + }, + "shelljs": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.3.0.tgz", + "integrity": "sha1-NZbmMHp4FUT1kfN9phg2DzHbV7E=" + }, + "sigmund": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/sigmund/-/sigmund-1.0.1.tgz", + "integrity": "sha1-P/IfGYytIXX587eBhT/ZTQ0ZtZA=" + }, + "simplediff": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/simplediff/-/simplediff-0.1.1.tgz", + "integrity": "sha1-sMrusJMiM3ADPGw6oRMNyGxqCHw=" + }, + "sntp": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/sntp/-/sntp-1.0.9.tgz", + "integrity": "sha1-ZUEYTMkK7qbG57NeJlkIJEPGYZg=", + "requires": { + "hoek": "2.x.x" + } + }, + "source-map": { + "version": "0.4.4", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.4.4.tgz", + "integrity": "sha1-66T12pwNyZneaAMti092FzZSA2s=", + "requires": { + "amdefine": ">=0.0.4" + } + }, + "source-map-support": { + "version": "0.4.10", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.4.10.tgz", + "integrity": "sha1-17GQOAQKFMCDehjmMKGWRTlSs3g=", + "requires": { + "source-map": "^0.5.3" + }, + "dependencies": { + "source-map": { + "version": "0.5.6", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.6.tgz", + "integrity": "sha1-dc449SvwczxafwwRjYEzSiu19BI=" + } + } + }, + "spdx-correct": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-1.0.2.tgz", + "integrity": "sha1-SzBz2TP/UfORLwOsVRlJikFQ20A=", + "requires": { + "spdx-license-ids": "^1.0.2" + } + }, + "spdx-expression-parse": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/spdx-expression-parse/-/spdx-expression-parse-1.0.4.tgz", + "integrity": "sha1-m98vIOH0DtRH++JzJmGR/O1RYmw=" + }, + "spdx-license-ids": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-1.2.2.tgz", + "integrity": "sha1-yd96NCRZSt5r0RkA1ZZpbcBrrFc=" + }, + "sprintf-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=" + }, + "sshpk": { + "version": "1.10.2", + "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.10.2.tgz", + "integrity": "sha1-1agEziJpVRVjjnmNviMnPeBwpfo=", + "requires": { + "asn1": "~0.2.3", + "assert-plus": "^1.0.0", + "bcrypt-pbkdf": "^1.0.0", + "dashdash": "^1.12.0", + "ecc-jsbn": "~0.1.1", + "getpass": "^0.1.1", + "jodid25519": "^1.0.0", + "jsbn": "~0.1.0", + "tweetnacl": "~0.14.0" + }, + "dependencies": { + "assert-plus": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", + "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" + } + } + }, + "stack-trace": { + "version": "0.0.9", + "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.9.tgz", + "integrity": "sha1-qPbq7KkGdMMz58Q5U/J1tFFRBpU=" + }, + "statuses": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.3.1.tgz", + "integrity": "sha1-+vUbnrdKrvOzrPStX2Gr8ky3uT4=" + }, + "streamsearch": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-0.1.2.tgz", + "integrity": "sha1-gIudDlb8Jz2Am6VzOOkpkZoanxo=" + }, + "string-width": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", + "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", + "requires": { + "code-point-at": "^1.0.0", + "is-fullwidth-code-point": "^1.0.0", + "strip-ansi": "^3.0.0" + } + }, + "string_decoder": { + "version": "0.10.31", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", + "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" + }, + "stringstream": { + "version": "0.0.5", + "resolved": "https://registry.npmjs.org/stringstream/-/stringstream-0.0.5.tgz", + "integrity": "sha1-TkhM1N5aC7vuGORjB3EKioFiGHg=" + }, + "strip-ansi": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", + "requires": { + "ansi-regex": "^2.0.0" + } + }, + "strip-bom": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-2.0.0.tgz", + "integrity": "sha1-YhmoVhZSBJHzV4i9vxRHqZx+aw4=", + "requires": { + "is-utf8": "^0.2.0" + } + }, + "strip-json-comments": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-1.0.4.tgz", + "integrity": "sha1-HhX7ysl9Pumb8tc7TGVrCCu6+5E=" + }, + "superagent": { + "version": "1.8.5", + "resolved": "https://registry.npmjs.org/superagent/-/superagent-1.8.5.tgz", + "integrity": "sha1-HA3cOvMOgOuE68BcshItqP6UC1U=", + "requires": { + "component-emitter": "~1.2.0", + "cookiejar": "2.0.6", + "debug": "2", + "extend": "3.0.0", + "form-data": "1.0.0-rc3", + "formidable": "~1.0.14", + "methods": "~1.1.1", + "mime": "1.3.4", + "qs": "2.3.3", + "readable-stream": "1.0.27-1", + "reduce-component": "1.0.1" + }, + "dependencies": { + "async": { + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz", + "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=" + }, + "form-data": { + "version": "1.0.0-rc3", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-1.0.0-rc3.tgz", + "integrity": "sha1-01vGLn+8KTeuePlIqqDTjZBgdXc=", + "requires": { + "async": "^1.4.0", + "combined-stream": "^1.0.5", + "mime-types": "^2.1.3" + } + }, + "qs": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/qs/-/qs-2.3.3.tgz", + "integrity": "sha1-6eha2+ddoLvkyOBHaghikPhjtAQ=" + }, + "readable-stream": { + "version": "1.0.27-1", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.27-1.tgz", + "integrity": "sha1-a2eYPCA1fO/QfwFlABoW1xDZEHg=", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.1", + "isarray": "0.0.1", + "string_decoder": "~0.10.x" + } + } + } + }, + "supertest": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/supertest/-/supertest-1.2.0.tgz", + "integrity": "sha1-hQp5X5Bo0vrxngF5n/CZYuDOQ74=", + "requires": { + "methods": "1.x", + "superagent": "^1.7.2" + } + }, + "supports-color": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", + "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=" + }, + "to-double-quotes": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/to-double-quotes/-/to-double-quotes-2.0.0.tgz", + "integrity": "sha1-qvIx1vqUiUn4GTAburRITYWI5Kc=" + }, + "to-iso-string": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/to-iso-string/-/to-iso-string-0.0.2.tgz", + "integrity": "sha1-TcGeZk38y+Jb2NtQiwDG2hWCVdE=" + }, + "to-single-quotes": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/to-single-quotes/-/to-single-quotes-2.0.1.tgz", + "integrity": "sha1-fMKRUfD18sQZRvEZ9ZMv5VQXASU=" + }, + "tough-cookie": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.3.2.tgz", + "integrity": "sha1-8IH3bkyFcg5sN6X6ztc3FQ2EByo=", + "requires": { + "punycode": "^1.4.1" + } + }, + "tunnel-agent": { + "version": "0.4.3", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.4.3.tgz", + "integrity": "sha1-Y3PbdpCf5XDgjXNYM2Xtgop07us=" + }, + "type-check": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz", + "integrity": "sha1-WITKtRLPHTVeP7eE8wgEsrUg23I=", + "requires": { + "prelude-ls": "~1.1.2" + } + }, + "type-detect": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-1.0.0.tgz", + "integrity": "sha1-diIXzAbbJY7EiQihKY6LlRIejqI=" + }, + "type-is": { + "version": "1.6.14", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.14.tgz", + "integrity": "sha1-4hljnBfe0coHiQkt1UoDgmuBfLI=", + "requires": { + "media-typer": "0.3.0", + "mime-types": "~2.1.13" + } + }, + "underscore": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.6.0.tgz", + "integrity": "sha1-izixDKze9jM3uLJOT/htRa6lKag=" + }, + "unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=" + }, + "util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" + }, + "utile": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/utile/-/utile-0.2.1.tgz", + "integrity": "sha1-kwyI6ZCY1iIINMNWy9mncFItkNc=", + "requires": { + "async": "~0.2.9", + "deep-equal": "*", + "i": "0.3.x", + "mkdirp": "0.x.x", + "ncp": "0.4.x", + "rimraf": "2.x.x" + }, + "dependencies": { + "async": { + "version": "0.2.10", + "resolved": "https://registry.npmjs.org/async/-/async-0.2.10.tgz", + "integrity": "sha1-trvgsGdLnXGXCMo43owjfLUmw9E=" + } + } + }, + "utils-merge": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.0.tgz", + "integrity": "sha1-ApT7kiu5N1FTVBxPcJYjHyh8ivg=" + }, + "uuid": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-2.0.3.tgz", + "integrity": "sha1-Z+LoY3lyFVMN/zGOW/nc6/1Hsho=" + }, + "validate-npm-package-license": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/validate-npm-package-license/-/validate-npm-package-license-3.0.1.tgz", + "integrity": "sha1-KAS6vnEq0zeUWaz74kdGqywwP7w=", + "requires": { + "spdx-correct": "~1.0.0", + "spdx-expression-parse": "~1.0.0" + } + }, + "vary": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.0.tgz", + "integrity": "sha1-4eWv+70WrnaN0mdDlLmtMCJlMUA=" + }, + "verror": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/verror/-/verror-1.3.6.tgz", + "integrity": "sha1-z/XfEpRtKX0rqu+qJoniW+AcAFw=", + "requires": { + "extsprintf": "1.0.2" + } + }, + "vow": { + "version": "0.4.13", + "resolved": "https://registry.npmjs.org/vow/-/vow-0.4.13.tgz", + "integrity": "sha1-58FPG9nIvg5zWaRZf+LR720afog=" + }, + "vow-fs": { + "version": "0.3.6", + "resolved": "https://registry.npmjs.org/vow-fs/-/vow-fs-0.3.6.tgz", + "integrity": "sha1-LUxZviLivyYY3fWXq0uqkjvnIA0=", + "requires": { + "glob": "^7.0.5", + "uuid": "^2.0.2", + "vow": "^0.4.7", + "vow-queue": "^0.4.1" + }, + "dependencies": { + "glob": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.1.tgz", + "integrity": "sha1-gFIR3wT6rxxjo2ADBs31reULLsg=", + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.2", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + } + } + }, + "vow-queue": { + "version": "0.4.2", + "resolved": "https://registry.npmjs.org/vow-queue/-/vow-queue-0.4.2.tgz", + "integrity": "sha1-5/4XFg4Vx8QYTRtmapvGThjjAYQ=", + "requires": { + "vow": "~0.4.0" + } + }, + "which": { + "version": "1.2.12", + "resolved": "https://registry.npmjs.org/which/-/which-1.2.12.tgz", + "integrity": "sha1-3me15FAmnxlJCe8j7OTr5Bb6EZI=", + "requires": { + "isexe": "^1.1.1" + } + }, + "which-module": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/which-module/-/which-module-1.0.0.tgz", + "integrity": "sha1-u6Y8qGGUiZT/MHc2CJ47lgJsKk8=" + }, + "window-size": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/window-size/-/window-size-0.2.0.tgz", + "integrity": "sha1-tDFbtCFKPXBY6+7okuE/ok2YsHU=" + }, + "winston": { + "version": "0.8.3", + "resolved": "https://registry.npmjs.org/winston/-/winston-0.8.3.tgz", + "integrity": "sha1-ZLar9M0Brcrv1QCTk7HY6L7BnbA=", + "requires": { + "async": "0.2.x", + "colors": "0.6.x", + "cycle": "1.0.x", + "eyes": "0.1.x", + "isstream": "0.1.x", + "pkginfo": "0.3.x", + "stack-trace": "0.0.x" + }, + "dependencies": { + "async": { + "version": "0.2.10", + "resolved": "https://registry.npmjs.org/async/-/async-0.2.10.tgz", + "integrity": "sha1-trvgsGdLnXGXCMo43owjfLUmw9E=" + }, + "colors": { + "version": "0.6.2", + "resolved": "https://registry.npmjs.org/colors/-/colors-0.6.2.tgz", + "integrity": "sha1-JCP+ZnisDF2uiFLl0OW+CMmXq8w=" + }, + "pkginfo": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/pkginfo/-/pkginfo-0.3.1.tgz", + "integrity": "sha1-Wyn2qB9wcXFC4J52W76rl7T4HiE=" + } + } + }, + "wordwrap": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-0.0.3.tgz", + "integrity": "sha1-o9XabNXAvAAI03I0u68b7WMFkQc=" + }, + "wrap-ansi": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.1.0.tgz", + "integrity": "sha1-2Pw9KE3QV5T+hJc8rs3Rz4JP3YU=", + "requires": { + "string-width": "^1.0.1", + "strip-ansi": "^3.0.1" + } + }, + "wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" + }, + "xmlbuilder": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-3.1.0.tgz", + "integrity": "sha1-LIaIjy1OrehQ+jjKf3Ij9yCVFuE=", + "requires": { + "lodash": "^3.5.0" + } + }, + "xtend": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz", + "integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68=" + }, + "y18n": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-3.2.1.tgz", + "integrity": "sha1-bRX7qITAhnnA136I53WegR4H+kE=" + }, + "yargs": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-5.0.0.tgz", + "integrity": "sha1-M1UUSXfQV1fbuG1uOOwFYSOzpm4=", + "requires": { + "cliui": "^3.2.0", + "decamelize": "^1.1.1", + "get-caller-file": "^1.0.1", + "lodash.assign": "^4.2.0", + "os-locale": "^1.4.0", + "read-pkg-up": "^1.0.1", + "require-directory": "^2.1.1", + "require-main-filename": "^1.0.1", + "set-blocking": "^2.0.0", + "string-width": "^1.0.2", + "which-module": "^1.0.0", + "window-size": "^0.2.0", + "y18n": "^3.2.1", + "yargs-parser": "^3.2.0" + } + }, + "yargs-parser": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-3.2.0.tgz", + "integrity": "sha1-UIE1XRnZ0MjF2BrakIy05tGGZk8=", + "requires": { + "camelcase": "^3.0.0", + "lodash.assign": "^4.1.0" + } + } + } + }, + "parsoid-jsapi": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/parsoid-jsapi/-/parsoid-jsapi-0.0.1.tgz", + "integrity": "sha1-AFiGuWCiP+0rcBogMH9rOr/6A7k=", + "requires": { + "domino": "^1.0.28", + "parsoid": "^0.7.1" + }, + "dependencies": { + "domino": { + "version": "1.0.30", + "resolved": "https://registry.npmjs.org/domino/-/domino-1.0.30.tgz", + "integrity": "sha512-ikq8WiDSkICdkElud317F2Sigc6A3EDpWsxWBwIZqOl95km4p/Vc9Rj98id7qKgsjDmExj0AVM7JOd4bb647Xg==" + } + } + }, + "path-exists": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", + "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=" + }, + "path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=" + }, + "path-is-inside": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/path-is-inside/-/path-is-inside-1.0.2.tgz", + "integrity": "sha1-NlQX3t5EQw0cEa9hAn+s8HS9/FM=", + "dev": true + }, + "path-key": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-2.0.1.tgz", + "integrity": "sha1-QRyttXTFoUDTpLGRDUDYDMn0C0A=", + "dev": true + }, + "path-parse": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.6.tgz", + "integrity": "sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==", + "dev": true + }, + "path-to-regexp": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", + "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" + }, + "path-type": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-3.0.0.tgz", + "integrity": "sha512-T2ZUsdZFHgA3u4e5PfPbjd7HDDpxPnQb5jN0SrDsjNSuVXHJqtwTnWqG0B1jZrgmJ/7lj1EmVIByWt1gxGkWvg==", + "dev": true, + "requires": { + "pify": "^3.0.0" + } + }, + "performance-now": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", + "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=" + }, + "picomatch": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.2.2.tgz", + "integrity": "sha512-q0M/9eZHzmr0AulXyPwNfZjtwZ/RBZlbN3K3CErVrk50T2ASYI7Bye0EvekFY3IP1Nt2DHu0re+V2ZHIpMkuWg==", + "dev": true + }, + "pify": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", + "integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=", + "dev": true + }, + "pkg-dir": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-3.0.0.tgz", + "integrity": "sha512-/E57AYkoeQ25qkxMj5PBOVgF8Kiu/h7cYS30Z5+R7WaiCCBfLq58ZI/dSeaEKb9WVJV5n/03QwrN3IeWIFllvw==", + "dev": true, + "requires": { + "find-up": "^3.0.0" + } + }, + "postcss": { + "version": "7.0.27", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-7.0.27.tgz", + "integrity": "sha512-WuQETPMcW9Uf1/22HWUWP9lgsIC+KEHg2kozMflKjbeUtw9ujvFX6QmIfozaErDkmLWS9WEnEdEe6Uo9/BNTdQ==", + "requires": { + "chalk": "^2.4.2", + "source-map": "^0.6.1", + "supports-color": "^6.1.0" + }, + "dependencies": { + "supports-color": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-6.1.0.tgz", + "integrity": "sha512-qe1jfm1Mg7Nq/NSh6XE24gPXROEVsWHxC1LIx//XNlD9iw7YZQGjZNjYN7xGaEG6iKdA8EtNFW6R0gjnVXp+wQ==", + "requires": { + "has-flag": "^3.0.0" + } + } + } + }, + "pre-commit": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/pre-commit/-/pre-commit-1.2.2.tgz", + "integrity": "sha1-287g7p3nI15X95xW186UZBpp7sY=", + "dev": true, + "requires": { + "cross-spawn": "^5.0.1", + "spawn-sync": "^1.0.15", + "which": "1.2.x" + }, + "dependencies": { + "cross-spawn": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-5.1.0.tgz", + "integrity": "sha1-6L0O/uWPz/b4+UUQoKVUu/ojVEk=", + "dev": true, + "requires": { + "lru-cache": "^4.0.1", + "shebang-command": "^1.2.0", + "which": "^1.2.9" + } + }, + "which": { + "version": "1.2.14", + "resolved": "https://registry.npmjs.org/which/-/which-1.2.14.tgz", + "integrity": "sha1-mofEN48D6CfOyvGs31bHNsAcFOU=", + "dev": true, + "requires": { + "isexe": "^2.0.0" + } + } + } + }, + "prelude-ls": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", + "integrity": "sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ=", + "dev": true + }, + "prepend-http": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/prepend-http/-/prepend-http-1.0.4.tgz", + "integrity": "sha1-1PRWKwzjaW5BrFLQ4ALlemNdxtw=", + "dev": true + }, + "preq": { + "version": "0.5.14", + "resolved": "https://registry.npmjs.org/preq/-/preq-0.5.14.tgz", + "integrity": "sha512-kuJ5ndEgjs27kTTQ/P2ipPQoHeCJcAI4i97mU3xSjkjx6CsuQOsCe2l5twTGC0SCB5UkzRpmrpXmvN0Ip4ZCxA==", + "requires": { + "bluebird": "^3.5.5", + "request": "^2.88.0", + "requestretry": "4.0.2" + } + }, + "prfun": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/prfun/-/prfun-2.1.5.tgz", + "integrity": "sha512-UCDQscAfQ1HArwvSUobJWbc3sTGLqGpYkRqXUpBZgf+zOWpOjz2dxnpRsOu+qxIj1K0n5UT1wgbCCgetsIwiug==", + "requires": { + "core-js": "^2.5.3" + } + }, + "process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" + }, + "progress": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", + "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", + "dev": true + }, + "promise": { + "version": "7.3.1", + "resolved": "https://registry.npmjs.org/promise/-/promise-7.3.1.tgz", + "integrity": "sha512-nolQXZ/4L+bP/UGlkfaIujX9BKxGwmQ9OT4mOt5yvy8iK1h3wqTEJCijzGANTCCl9nWjY41juyAn2K3Q1hLLTg==", + "dev": true, + "optional": true, + "requires": { + "asap": "~2.0.3" + } + }, + "proto-list": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/proto-list/-/proto-list-1.2.4.tgz", + "integrity": "sha1-IS1b/hMYMGpCD2QCuOJv85ZHqEk=", + "dev": true + }, + "proxy-addr": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.6.tgz", + "integrity": "sha512-dh/frvCBVmSsDYzw6n926jv974gddhkFPfiN8hPOi30Wax25QZyZEGveluCgliBnqmuM+UJmBErbAUFIoDbjOw==", + "requires": { + "forwarded": "~0.1.2", + "ipaddr.js": "1.9.1" + } + }, + "prr": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/prr/-/prr-1.0.1.tgz", + "integrity": "sha1-0/wRS6BplaRexok/SEzrHXj19HY=", + "dev": true, + "optional": true + }, + "pseudomap": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", "integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM=", @@ -4796,6 +8045,12 @@ "es6-error": "^4.0.1" } }, + "repeat-string": { + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/repeat-string/-/repeat-string-1.6.1.tgz", + "integrity": "sha1-jcrkcOHIirwtYA//Sndihtp15jc=", + "optional": true + }, "request": { "version": "2.88.2", "resolved": "https://registry.npmjs.org/request/-/request-2.88.2.tgz", @@ -4875,6 +8130,15 @@ "signal-exit": "^3.0.2" } }, + "right-align": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/right-align/-/right-align-0.1.3.tgz", + "integrity": "sha1-YTObci/mo1FWiSENJOFMlhSGE+8=", + "optional": true, + "requires": { + "align-text": "^0.1.1" + } + }, "rimraf": { "version": "2.4.5", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.4.5.tgz", @@ -5423,24 +8687,6 @@ "resolved": "https://registry.npmjs.org/swagger-ui-dist/-/swagger-ui-dist-3.25.0.tgz", "integrity": "sha512-vwvJPPbdooTvDwLGzjIXinOXizDJJ6U1hxnJL3y6U3aL1d2MSXDmKg2139XaLBhsVZdnQJV2bOkX4reB+RXamg==" }, - "sync-request": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/sync-request/-/sync-request-6.1.0.tgz", - "integrity": "sha512-8fjNkrNlNCrVc/av+Jn+xxqfCjYaBoHqCsDz6mt030UMxJGr+GSfCV1dQt2gRtlL63+VPidwDVLr7V2OcTSdRw==", - "requires": { - "http-response-object": "^3.0.1", - "sync-rpc": "^1.2.1", - "then-request": "^6.0.0" - } - }, - "sync-rpc": { - "version": "1.3.6", - "resolved": "https://registry.npmjs.org/sync-rpc/-/sync-rpc-1.3.6.tgz", - "integrity": "sha512-J8jTXuZzRlvU7HemDgHi3pGnh/rkoqR/OZSjhTyyZrEkkYQbk7Z33AXp37mkPfPpfdOuj7Ex3H/TJM1z48uPQw==", - "requires": { - "get-port": "^3.1.0" - } - }, "table": { "version": "5.4.6", "resolved": "https://registry.npmjs.org/table/-/table-5.4.6.tgz", @@ -5506,39 +8752,6 @@ "integrity": "sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=", "dev": true }, - "then-request": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/then-request/-/then-request-6.0.2.tgz", - "integrity": "sha512-3ZBiG7JvP3wbDzA9iNY5zJQcHL4jn/0BWtXIkagfz7QgOL/LqjCEOBQuJNZfu0XYnv5JhKh+cDxCPM4ILrqruA==", - "requires": { - "@types/concat-stream": "^1.6.0", - "@types/form-data": "0.0.33", - "@types/node": "^8.0.0", - "@types/qs": "^6.2.31", - "caseless": "~0.12.0", - "concat-stream": "^1.6.0", - "form-data": "^2.2.0", - "http-basic": "^8.1.1", - "http-response-object": "^3.0.1", - "promise": "^8.0.0", - "qs": "^6.4.0" - }, - "dependencies": { - "@types/node": { - "version": "8.10.61", - "resolved": "https://registry.npmjs.org/@types/node/-/node-8.10.61.tgz", - "integrity": "sha512-l+zSbvT8TPRaCxL1l9cwHCb0tSqGAGcjPJFItGGYat5oCTiq1uQQKYg5m7AF1mgnEBzFXGLJ2LRmNjtreRX76Q==" - }, - "promise": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/promise/-/promise-8.1.0.tgz", - "integrity": "sha512-W04AqnILOL/sPRXziNicCjSNRruLAuIHEOVBazepu0545DDNGYHz7ar9ZgZ1fMU8/MA4mVxp5rkBWRi6OXIy3Q==", - "requires": { - "asap": "~2.0.6" - } - } - } - }, "through": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", @@ -5655,7 +8868,62 @@ "typedarray": { "version": "0.0.6", "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", - "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=" + "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=", + "dev": true + }, + "uglify-js": { + "version": "2.8.29", + "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-2.8.29.tgz", + "integrity": "sha1-KcVzMUgFe7Th913zW3qcty5qWd0=", + "optional": true, + "requires": { + "source-map": "~0.5.1", + "uglify-to-browserify": "~1.0.0", + "yargs": "~3.10.0" + }, + "dependencies": { + "camelcase": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-1.2.1.tgz", + "integrity": "sha1-m7UwTS4LVmmLLHWLCKPqqdqlijk=", + "optional": true + }, + "cliui": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-2.1.0.tgz", + "integrity": "sha1-S0dXYP+AJkx2LDoXGQMukcf+oNE=", + "optional": true, + "requires": { + "center-align": "^0.1.1", + "right-align": "^0.1.1", + "wordwrap": "0.0.2" + } + }, + "source-map": { + "version": "0.5.7", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz", + "integrity": "sha1-igOdLRAh0i0eoUyA2OpGi6LvP8w=", + "optional": true + }, + "yargs": { + "version": "3.10.0", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-3.10.0.tgz", + "integrity": "sha1-9+572FfdfB0tOMDnTvvWgdFDH9E=", + "optional": true, + "requires": { + "camelcase": "^1.0.2", + "cliui": "^2.1.0", + "decamelize": "^1.0.0", + "window-size": "0.1.0" + } + } + } + }, + "uglify-to-browserify": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/uglify-to-browserify/-/uglify-to-browserify-1.0.2.tgz", + "integrity": "sha1-bgkk1r2mta/jSeOabWMoUKD4grc=", + "optional": true }, "undefsafe": { "version": "2.0.3", @@ -5904,12 +9172,24 @@ } } }, + "window-size": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/window-size/-/window-size-0.1.0.tgz", + "integrity": "sha1-VDjNLqk7IC76Ohn+iIeu58lPnJ0=", + "optional": true + }, "word-wrap": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz", "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==", "dev": true }, + "wordwrap": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-0.0.2.tgz", + "integrity": "sha1-t5Zpu0LstAn4PVg8rVLKF+qhZD8=", + "optional": true + }, "wrap-ansi": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-5.1.0.tgz", diff --git a/package.json b/package.json index e0faaf95..47c03ce1 100644 --- a/package.json +++ b/package.json @@ -55,7 +55,8 @@ "homepage": "https://www.mediawiki.org/wiki/RESTBase_services_for_apps", "dependencies": { "@root/encoding": "^1.0.1", - "sync-request": "^6.1.0", + "parsoid-jsapi": "^0.0.1", + "prfun": "^2.1.4", "banana-i18n": "^1.2.1", "bluebird": "^3.7.2", "body-parser": "^1.19.0", diff --git a/routes/page/significant-changes.js b/routes/page/significant-changes.js index b0bff38d..9a775230 100644 --- a/routes/page/significant-changes.js +++ b/routes/page/significant-changes.js @@ -7,7 +7,8 @@ const api = require('../../lib/api-util'); const express = require('express'); const parsoidApi = require('../../lib/parsoid-access'); const encoding = require('@root/encoding'); - +const ParsoidJS = require('parsoid-jsapi'); +const PRFunPromise = require('prfun'); let app; const significantChangesCache = {}; @@ -385,6 +386,100 @@ function updateDiffAndRevisionsWithCharacterCount(diffAndRevisions) { }); } +function structuredParsoidResultPromise(text, revision) { + return new BBPromise((resolve) => { + var main = PRFunPromise.async(function*() { + var pdoc = yield ParsoidJS.parse(text, { pdoc: true }); + const splitTemplates = yield PRFunPromise.map(pdoc.filterTemplates(), ParsoidJS.toWikitext); + var templateObjects = []; + + for (var s = 0; s < splitTemplates.length; s++) { + const splitTemplateText = splitTemplates[s]; + + const innerPdoc = yield ParsoidJS.parse(splitTemplateText, { pdoc: true }); + const individualTemplates = innerPdoc.filterTemplates(); + for (var i = 0; i < individualTemplates.length; i++) { + const template = individualTemplates[i]; + + var dict = {}; + dict['name'] = template.name; + for (var p = 0; p < template.params.length; p++) { + const param = template.params[p].name; + const value = yield template.get(param).value.toWikitext(); + dict[param] = value; + } + templateObjects.push(dict); + } + } + + resolve(templateObjects); + }); + + main().done(); + }); +} + +function needsToParseForAddedTemplates(text) { + return text.includes('{{'); +} + +function structuredParsoidResultPromises(diffAndRevisions) { + // Loop through added sections in diffs and detect template types + + var promises = []; + diffAndRevisions.forEach(function (diffAndRevision) { + diffAndRevision.body.diff.forEach(function (diff) { + + switch (diff.type) { + case 1: // Add complete line type + if (needsToParseForAddedTemplates(diff.text)) { + //{{cite web |url=http://www.mla.org/map_data |title=United States |publisher=[[Modern Language Association]]|access-date=September 2, 2013}} + //{{cite web |url=http://factfinder.census.gov/faces/tableservices/jsf/pages/productview.xhtml?pid=ACS_10_1YR_B16001&prodType=table |title=American FactFinder—Results |first=U.S. Census |last=Bureau |publisher= |access-date=May 29, 2017 |archive-url=https://archive.today/20200212213140/http://factfinder.census.gov/faces/tableservices/jsf/pages/productview.xhtml?pid=ACS_10_1YR_B16001&prodType=table |archive-date=February 12, 2020 |url-status=dead }} + //{{efn|Source: 2015 [[American Community Survey]], [[U.S. Census Bureau]]. Most respondents who speak a language other than English at home also report speaking English "well" or "very well". For the language groups listed above, the strongest English-language proficiency is among speakers of German (96% report that they speak English "well" or "very well"), followed by speakers of French (93.5%), Tagalog (92.8%), Spanish (74.1%), Korean (71.5%), Chinese (70.4%), and Vietnamese (66.9%).}} + promises.push(structuredParsoidResultPromise(diff.text, diffAndRevision.revision)); + } + break; + case 5: + case 3: + diff.highlightRanges.forEach(function (range) { + + const binaryText = encoding.strToBin(diff.text); + const binaryRangeText = binaryText.substring(range.start, + range.start + range.length); + const rangeText = encoding.binToStr(binaryRangeText); + + switch (range.type) { + case 0: // Add range type + if (needsToParseForAddedTemplates(rangeText)) { + promises.push(structuredParsoidResultPromise(rangeText, diffAndRevision.revision)); + } + break; + default: + break; + } + }); + default: + break; + } + }); + }); + + return Promise.all(promises) + .then( (response) => { + + //assign pdoc to diffAndRevision; + diffAndRevisions.forEach(function (diffAndRevision) { + response.forEach(function (parseResponse) { + if (diffAndRevision.revision.revid === parseResponse.revision.revid) { + diffAndRevision.pdoc = parseResponse.pdoc; + } + }); + }); + + return diffAndRevisions; + }); +} + function getNewTopicDiffAndRevisions(talkDiffAndRevisions) { const newSectionTalkPageDiffAndRevisions = []; @@ -617,7 +712,11 @@ function getSignificantChanges(req, res) { updateDiffAndRevisionsWithCharacterCount(response.articleDiffAndRevisions); updateDiffAndRevisionsWithCharacterCount(response.talkDiffAndRevisions); - return response; + // Flag added template types + return structuredParsoidResultPromises(response.articleDiffAndRevisions) + .then( (articleDiffAndRevisions) => { + return response; + }); }) .then( (response) => { From ac624163a916a94b577587b883feeddcca4596ee Mon Sep 17 00:00:00 2001 From: Toni Sevener Date: Mon, 20 Jul 2020 23:47:16 -0500 Subject: [PATCH 12/47] finish adding new reference type --- routes/page/significant-changes.js | 129 ++++++++++++++++++++--------- 1 file changed, 88 insertions(+), 41 deletions(-) diff --git a/routes/page/significant-changes.js b/routes/page/significant-changes.js index 9a775230..2732062c 100644 --- a/routes/page/significant-changes.js +++ b/routes/page/significant-changes.js @@ -53,6 +53,18 @@ class VandalismOutput { } } +class NewReference { + constructor(revid, timestamp, user, userid, section, templates) { + this.revid = revid; + this.timestamp = timestamp; + this.outputType = 'new-reference'; + this.user = user; + this.userid = userid; + this.section = section; + this.templates = templates; + } +} + class LargeOutput { constructor(largeOutputExpanded) { this.revid = largeOutputExpanded.revid; @@ -134,7 +146,7 @@ const diffAndRevisionPromise = (req, revision) => { }); }; -const snippetPromise = (req, largeChange) => { +const snippetPromise = (req, snippetOutput) => { const headers = Object.assign( { accept: 'text/html', profile: 'https://www.mediawiki.org/wiki/Specs/Mobile-HTML/1.0.0', @@ -144,9 +156,9 @@ const snippetPromise = (req, largeChange) => { var newSnippet; - if (largeChange.outputType === 'large-change') { + if (snippetOutput.outputType === 'large-change') { // add highlight delimiters first - var snippetBinary = encoding.strToBin(largeChange.snippet); + var snippetBinary = encoding.strToBin(snippetOutput.snippet); // todo: it looks like parsoid *sometimes* strips these spans out // depending on their placement. @@ -162,7 +174,7 @@ const snippetPromise = (req, largeChange) => { const deleteHighlightStartBin = encoding.strToBin(deleteHighlightStart); const highlightEndBin = encoding.strToBin(highlightEnd); - switch (largeChange.type) { + switch (snippetOutput.type) { case 1: // Added complete line snippetBinary = insertSubstringInString(snippetBinary, addHighlightStartBin, 0); @@ -178,7 +190,7 @@ const snippetPromise = (req, largeChange) => { case 5: case 3: // Added and deleted words in line var offset = 0; - largeChange.highlightRanges.forEach(function (range) { + snippetOutput.highlightRanges.forEach(function (range) { switch (range.type) { case 0: // Added snippetBinary = insertSubstringInString(snippetBinary, @@ -205,7 +217,7 @@ const snippetPromise = (req, largeChange) => { newSnippet = encoding.binToStr(snippetBinary); } else { // new talk page topic - newSnippet = largeChange.snippet; + newSnippet = snippetOutput.snippet; } // make request to format to mobile-html, reassign result back to snippet @@ -262,14 +274,14 @@ const snippetPromise = (req, largeChange) => { } } - largeChange.snippet = strippedSnippet; - return largeChange; + snippetOutput.snippet = strippedSnippet; + return snippetOutput; }); }; -const snippetPromises = (req, largeChanges) => { - return BBPromise.map(largeChanges, function(largeChange) { - return snippetPromise(req, largeChange); +const snippetPromises = (req, snippetOutputs) => { + return BBPromise.map(snippetOutputs, function(snippetOutput) { + return snippetPromise(req, snippetOutput); }); }; @@ -386,11 +398,29 @@ function updateDiffAndRevisionsWithCharacterCount(diffAndRevisions) { }); } -function structuredParsoidResultPromise(text, revision) { +function templateNamesToCallOut() { + return ['cite']; +} + +function needsToParseForAddedTemplates(text, includeOpeningBraces) { + const names = templateNamesToCallOut(); + for (var n = 0; n < names.length; n++) { + const name = names[n]; + + if ((text.includes(`{{${name}`) && includeOpeningBraces) || (text.includes(`${name}`) && !includeOpeningBraces)) { + return true; + } + } + + return false; +} + +function structuredTemplatePromise(text, diff, revision) { return new BBPromise((resolve) => { var main = PRFunPromise.async(function*() { var pdoc = yield ParsoidJS.parse(text, { pdoc: true }); - const splitTemplates = yield PRFunPromise.map(pdoc.filterTemplates(), ParsoidJS.toWikitext); + const splitTemplates = yield PRFunPromise.map(pdoc.filterTemplates(), + ParsoidJS.toWikitext); var templateObjects = []; for (var s = 0; s < splitTemplates.length; s++) { @@ -401,8 +431,12 @@ function structuredParsoidResultPromise(text, revision) { for (var i = 0; i < individualTemplates.length; i++) { const template = individualTemplates[i]; + if (!needsToParseForAddedTemplates(template.name, false)) { + continue; + } + var dict = {}; - dict['name'] = template.name; + dict.name = template.name; for (var p = 0; p < template.params.length; p++) { const param = template.params[p].name; const value = yield template.get(param).value.toWikitext(); @@ -411,32 +445,28 @@ function structuredParsoidResultPromise(text, revision) { templateObjects.push(dict); } } - - resolve(templateObjects); + const result = Object.assign( { + revision: revision, + diff: diff, + templates: templateObjects + }); + resolve(result); }); main().done(); }); } -function needsToParseForAddedTemplates(text) { - return text.includes('{{'); -} - -function structuredParsoidResultPromises(diffAndRevisions) { - // Loop through added sections in diffs and detect template types - +function addStructuredTemplates(diffAndRevisions) { var promises = []; diffAndRevisions.forEach(function (diffAndRevision) { diffAndRevision.body.diff.forEach(function (diff) { switch (diff.type) { case 1: // Add complete line type - if (needsToParseForAddedTemplates(diff.text)) { - //{{cite web |url=http://www.mla.org/map_data |title=United States |publisher=[[Modern Language Association]]|access-date=September 2, 2013}} - //{{cite web |url=http://factfinder.census.gov/faces/tableservices/jsf/pages/productview.xhtml?pid=ACS_10_1YR_B16001&prodType=table |title=American FactFinder—Results |first=U.S. Census |last=Bureau |publisher= |access-date=May 29, 2017 |archive-url=https://archive.today/20200212213140/http://factfinder.census.gov/faces/tableservices/jsf/pages/productview.xhtml?pid=ACS_10_1YR_B16001&prodType=table |archive-date=February 12, 2020 |url-status=dead }} - //{{efn|Source: 2015 [[American Community Survey]], [[U.S. Census Bureau]]. Most respondents who speak a language other than English at home also report speaking English "well" or "very well". For the language groups listed above, the strongest English-language proficiency is among speakers of German (96% report that they speak English "well" or "very well"), followed by speakers of French (93.5%), Tagalog (92.8%), Spanish (74.1%), Korean (71.5%), Chinese (70.4%), and Vietnamese (66.9%).}} - promises.push(structuredParsoidResultPromise(diff.text, diffAndRevision.revision)); + if (needsToParseForAddedTemplates(diff.text, true)) { + promises.push(structuredTemplatePromise(diff.text, diff, + diffAndRevision.revision)); } break; case 5: @@ -450,14 +480,16 @@ function structuredParsoidResultPromises(diffAndRevisions) { switch (range.type) { case 0: // Add range type - if (needsToParseForAddedTemplates(rangeText)) { - promises.push(structuredParsoidResultPromise(rangeText, diffAndRevision.revision)); + if (needsToParseForAddedTemplates(rangeText, true)) { + promises.push(structuredTemplatePromise(rangeText, diff, + diffAndRevision.revision)); } break; default: break; } }); + break; default: break; } @@ -467,13 +499,15 @@ function structuredParsoidResultPromises(diffAndRevisions) { return Promise.all(promises) .then( (response) => { - //assign pdoc to diffAndRevision; - diffAndRevisions.forEach(function (diffAndRevision) { - response.forEach(function (parseResponse) { - if (diffAndRevision.revision.revid === parseResponse.revision.revid) { - diffAndRevision.pdoc = parseResponse.pdoc; - } - }); + // loop through responses, add to revision. + + response.forEach( (item) => { + diffAndRevisions.forEach( (diffAndRevision) => { + if (item.revision.revid === diffAndRevision.revision.revid) { + diffAndRevision.templates = item.templates; + diffAndRevision.templateDiffLine = item.diff; + } + }); }); return diffAndRevisions; @@ -713,8 +747,9 @@ function getSignificantChanges(req, res) { updateDiffAndRevisionsWithCharacterCount(response.talkDiffAndRevisions); // Flag added template types - return structuredParsoidResultPromises(response.articleDiffAndRevisions) + return addStructuredTemplates(response.articleDiffAndRevisions) .then( (articleDiffAndRevisions) => { + response.articleDiffAndRevisions = articleDiffAndRevisions; return response; }); }) @@ -726,7 +761,19 @@ function getSignificantChanges(req, res) { var uncachedOutput = []; response.articleDiffAndRevisions.forEach(function (diffAndRevision) { const revision = diffAndRevision.revision; - if (revision.tags.includes('mw-rollback') && + if (diffAndRevision.templates && diffAndRevision.templates.length > 0) { + + var section = null; + if (diffAndRevision.templateDiffLine) { + section = getSectionForDiffLine(diffAndRevision.body, + diffAndRevision.templateDiffLine); + } + const newReferenceOutputObject = new NewReference(revision.revid, + revision.timestamp, revision.user, revision.userid, section, + diffAndRevision.templates); + + uncachedOutput.push(newReferenceOutputObject); + } else if (revision.tags.includes('mw-rollback') && revision.comment.toLowerCase().includes('revert') && revision.comment.toLowerCase().includes('vandalism')) { const largestDiffLine = getLargestDiffLine(diffAndRevision.body); @@ -773,9 +820,9 @@ function getSignificantChanges(req, res) { .then( (response) => { // convert large snippets from wikitext to mobile-html - const largeOutputs = response.uncachedOutput.filter(item => + const snippetOutputs = response.uncachedOutput.filter(item => item.outputType === 'large-change' || item.outputType === 'new-talk-page-topic'); - return snippetPromises(req, largeOutputs) + return snippetPromises(req, snippetOutputs) .then( (snippetResponse) => { // push to final output and cache From 45b2c00f89166a1eef2b06099f0d25595fd82200 Mon Sep 17 00:00:00 2001 From: Toni Sevener Date: Tue, 21 Jul 2020 15:49:30 -0500 Subject: [PATCH 13/47] quick added-templates endpoint for evaluating all templates --- routes/page/significant-changes.js | 118 +++++++++++++++++++++++++++-- 1 file changed, 110 insertions(+), 8 deletions(-) diff --git a/routes/page/significant-changes.js b/routes/page/significant-changes.js index 2732062c..5f5c2fff 100644 --- a/routes/page/significant-changes.js +++ b/routes/page/significant-changes.js @@ -65,6 +65,18 @@ class NewReference { } } +class AddedTemplate { + constructor(revid, timestamp, user, userid, section, templates) { + this.revid = revid; + this.timestamp = timestamp; + this.outputType = 'added-template'; + this.user = user; + this.userid = userid; + this.section = section; + this.templates = templates; + } +} + class LargeOutput { constructor(largeOutputExpanded) { this.revid = largeOutputExpanded.revid; @@ -402,8 +414,16 @@ function templateNamesToCallOut() { return ['cite']; } -function needsToParseForAddedTemplates(text, includeOpeningBraces) { +function needsToParseForAddedTemplates(text, includeOpeningBraces, includeAll) { const names = templateNamesToCallOut(); + + if (includeAll) { + if ((text.includes('{{') && includeOpeningBraces) || (!includeOpeningBraces)) { + return true; + } + return false; + } + for (var n = 0; n < names.length; n++) { const name = names[n]; @@ -415,7 +435,7 @@ function needsToParseForAddedTemplates(text, includeOpeningBraces) { return false; } -function structuredTemplatePromise(text, diff, revision) { +function structuredTemplatePromise(text, diff, revision, includeAll) { return new BBPromise((resolve) => { var main = PRFunPromise.async(function*() { var pdoc = yield ParsoidJS.parse(text, { pdoc: true }); @@ -431,7 +451,7 @@ function structuredTemplatePromise(text, diff, revision) { for (var i = 0; i < individualTemplates.length; i++) { const template = individualTemplates[i]; - if (!needsToParseForAddedTemplates(template.name, false)) { + if (!needsToParseForAddedTemplates(template.name, false, includeAll)) { continue; } @@ -457,14 +477,14 @@ function structuredTemplatePromise(text, diff, revision) { }); } -function addStructuredTemplates(diffAndRevisions) { +function addStructuredTemplates(diffAndRevisions, includeAll) { var promises = []; diffAndRevisions.forEach(function (diffAndRevision) { diffAndRevision.body.diff.forEach(function (diff) { switch (diff.type) { case 1: // Add complete line type - if (needsToParseForAddedTemplates(diff.text, true)) { + if (needsToParseForAddedTemplates(diff.text, true, includeAll)) { promises.push(structuredTemplatePromise(diff.text, diff, diffAndRevision.revision)); } @@ -480,9 +500,9 @@ function addStructuredTemplates(diffAndRevisions) { switch (range.type) { case 0: // Add range type - if (needsToParseForAddedTemplates(rangeText, true)) { + if (needsToParseForAddedTemplates(rangeText, true, includeAll)) { promises.push(structuredTemplatePromise(rangeText, diff, - diffAndRevision.revision)); + diffAndRevision.revision, includeAll)); } break; default: @@ -747,7 +767,7 @@ function getSignificantChanges(req, res) { updateDiffAndRevisionsWithCharacterCount(response.talkDiffAndRevisions); // Flag added template types - return addStructuredTemplates(response.articleDiffAndRevisions) + return addStructuredTemplates(response.articleDiffAndRevisions, false) .then( (articleDiffAndRevisions) => { response.articleDiffAndRevisions = articleDiffAndRevisions; return response; @@ -854,6 +874,80 @@ function getSignificantChanges(req, res) { }); } +function getAddedTemplates(req, res) { + + // STEP 1: Gather list of article revisions + return mwapi.queryForRevisions(req, null, req.query.pageSize) + .then( (response) => { + + // STEP 2: All at once gather diffs for each uncached revision and list of + // talk page revisions + const revisions = response.body.query.pages[0].revisions; + const nextRvStartId = revisions[revisions.length - 1].parentid; + var finalOutput = []; + + const articleEvalResults = Object.assign({ + uncachedRevisions: revisions, + cachedOutput: [] + }); + + return BBPromise.props({ + articleDiffAndRevisions: diffAndRevisionPromises(req, + articleEvalResults.uncachedRevisions), + nextRvStartId: nextRvStartId, + finalOutput: finalOutput + }); + }) + .then( (response) => { + + // Flag added template types + return addStructuredTemplates(response.articleDiffAndRevisions, true) + .then( (articleDiffAndRevisions) => { + response.articleDiffAndRevisions = articleDiffAndRevisions; + return response; + }); + }) + .then( (response) => { + + // segment off into types + var uncachedOutput = []; + response.articleDiffAndRevisions.forEach(function (diffAndRevision) { + const revision = diffAndRevision.revision; + if (diffAndRevision.templates && diffAndRevision.templates.length > 0) { + + var section = null; + if (diffAndRevision.templateDiffLine) { + section = getSectionForDiffLine(diffAndRevision.body, + diffAndRevision.templateDiffLine); + } + const addedTemplateOutputObject = new AddedTemplate(revision.revid, + revision.timestamp, revision.user, revision.userid, section, + diffAndRevision.templates); + + uncachedOutput.push(addedTemplateOutputObject); + } + }); + return Object.assign({ nextRvStartId: response.nextRvStartId, + uncachedOutput: uncachedOutput, finalOutput: response.finalOutput } ); + }) + .then( (response) => { + + response.uncachedOutput.forEach((item) => { + response.finalOutput.push(item); + }); + + return Object.assign({ nextRvStartId: response.nextRvStartId, + finalOutput: response.finalOutput } ); + }) + .then( (response) => { + + const cleanedOutput = cleanOutput(response.finalOutput); + const result = Object.assign({ nextRvStartId: response.nextRvStartId, + significantChanges: cleanedOutput } ); + res.send(result).end(); + }); +} + router.get('/page/significant-changes/:title', (req, res) => { // res.status(200); return getSignificantChanges(req, res); @@ -862,6 +956,14 @@ router.get('/page/significant-changes/:title', (req, res) => { // res.json(result).end(); }); +router.get('/page/added-templates/:title', (req, res) => { + // res.status(200); + return getAddedTemplates(req, res); + // const result = Object.assign({ result: "What up new endpoint."}); + // mUtil.setContentType(res, mUtil.CONTENT_TYPES.talk); + // res.json(result).end(); +}); + module.exports = function(appObj) { app = appObj; return { From f612aa279d4c29833193d615204897368b9cea56 Mon Sep 17 00:00:00 2001 From: Toni Sevener Date: Tue, 21 Jul 2020 16:16:44 -0500 Subject: [PATCH 14/47] quick new-talk-topics endpoint for evaluating new talk page data --- routes/page/significant-changes.js | 114 +++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) diff --git a/routes/page/significant-changes.js b/routes/page/significant-changes.js index 5f5c2fff..124aa1d3 100644 --- a/routes/page/significant-changes.js +++ b/routes/page/significant-changes.js @@ -106,6 +106,28 @@ class LargeOutputExpanded { } } +class NewTalkPageTopicExtraExtended { + constructor(revision, snippet, type, highlightRanges, characterChange, + section) { + this.revision = revision; + this.outputType = 'new-talk-page-topic-extra'; + this.snippet = snippet; + this.type = type; + this.highlightRanges = highlightRanges; + this.characterChange = characterChange; + this.section = section; + } +} + +class NewTalkPageTopicExtra { + constructor(item) { + this.revision = item.revision; + this.outputType = 'new-talk-page-topic-extra'; + this.snippet = item.snippet; + this.section = item.section; + } +} + class NewTalkPageTopicExtended { constructor(revid, timestamp, user, userid, snippet, type, highlightRanges, characterChange, section) { @@ -667,6 +689,10 @@ function cleanOutput(output) { // sort by date first output = output.sort(function(a, b) { + if (a.outputType === 'new-talk-page-topic-extra') { + return new Date(b.revision.timestamp) - new Date(a.revision.timestamp); + } + return new Date(b.timestamp) - new Date(a.timestamp); }); @@ -687,6 +713,8 @@ function cleanOutput(output) { cleanedOutput.push(new LargeOutput(item)); } else if (item.outputType === 'new-talk-page-topic') { cleanedOutput.push(new NewTalkPageTopic(item)); + } else if (item.outputType === 'new-talk-page-topic-extra') { + cleanedOutput.push(new NewTalkPageTopicExtra(item)); } else { cleanedOutput.push(item); } @@ -948,6 +976,84 @@ function getAddedTemplates(req, res) { }); } +function getNewTalkTopics(req, res) { + + // STEP 1: Gather list of article revisions + return mwapi.queryForRevisions(req, talkPageTitle(req), req.query.pageSize) + .then( (response) => { + + const talkPageRevisions = response.body.query.pages[0].revisions; + const nextRvStartId = talkPageRevisions[talkPageRevisions.length - 1].parentid; + + var finalOutput = []; + + const talkEvalResults = Object.assign({ + uncachedRevisions: talkPageRevisions, + cachedOutput: [] + }); + + return BBPromise.props({ + talkDiffAndRevisions: diffAndRevisionPromises(req, + talkEvalResults.uncachedRevisions), + nextRvStartId: nextRvStartId, + finalOutput: finalOutput + }); + }) + .then( (response) => { + + updateDiffAndRevisionsWithCharacterCount(response.talkDiffAndRevisions); + + // segment off into types + var uncachedOutput = []; + + // get new talk page revisions, add to uncachedOutput. it will be sorted later. + const newTopicDiffAndRevisions = getNewTopicDiffAndRevisions( + response.talkDiffAndRevisions); + newTopicDiffAndRevisions.forEach(function (diffAndRevision) { + const revision = diffAndRevision.revision; + // todo: better check might be something like get first diff line + // that doesn't have a section title or empty line. + const largestDiffLine = getLargestDiffLine(diffAndRevision.body); + const section = getSectionForLargestDiffLine(diffAndRevision.body, largestDiffLine); + const newTalkPageTopicOutputObject = new NewTalkPageTopicExtraExtended(revision, + largestDiffLine.text, + largestDiffLine.type, largestDiffLine.highlightRanges, + diffAndRevision.characterChange, section); + uncachedOutput.push(newTalkPageTopicOutputObject); + }); + + return Object.assign({ nextRvStartId: response.nextRvStartId, + uncachedOutput: uncachedOutput, finalOutput: response.finalOutput } ); + }) + .then( (response) => { + + // convert large snippets from wikitext to mobile-html + const snippetOutputs = response.uncachedOutput.filter(item => + item.outputType === 'new-talk-page-topic-extra'); + return snippetPromises(req, snippetOutputs) + .then( (snippetResponse) => { + + // push to final output and cache + // note we are using original response list, not snippet response + // (snippet only contains large) + response.uncachedOutput.forEach((item) => { + response.finalOutput.push(item); + }); + + return Object.assign({ nextRvStartId: response.nextRvStartId, + finalOutput: response.finalOutput } ); + }); + }) + .then( (response) => { + + const cleanedOutput = cleanOutput(response.finalOutput); + const result = Object.assign({ nextRvStartId: response.nextRvStartId, + significantChanges: cleanedOutput } ); + res.send(result).end(); + + }); +} + router.get('/page/significant-changes/:title', (req, res) => { // res.status(200); return getSignificantChanges(req, res); @@ -964,6 +1070,14 @@ router.get('/page/added-templates/:title', (req, res) => { // res.json(result).end(); }); +router.get('/page/new-talk-topics/:title', (req, res) => { + // res.status(200); + return getNewTalkTopics(req, res); + // const result = Object.assign({ result: "What up new endpoint."}); + // mUtil.setContentType(res, mUtil.CONTENT_TYPES.talk); + // res.json(result).end(); +}); + module.exports = function(appObj) { app = appObj; return { From 3f24a7b04d38871249d750b7d4761382b38e25c4 Mon Sep 17 00:00:00 2001 From: Toni Sevener Date: Mon, 10 Aug 2020 15:13:15 -0500 Subject: [PATCH 15/47] rename to significant-events, remove debug endpoints --- ...icant-changes.js => significant-events.js} | 174 +----------------- 1 file changed, 3 insertions(+), 171 deletions(-) rename routes/page/{significant-changes.js => significant-events.js} (82%) diff --git a/routes/page/significant-changes.js b/routes/page/significant-events.js similarity index 82% rename from routes/page/significant-changes.js rename to routes/page/significant-events.js index 124aa1d3..0548feb1 100644 --- a/routes/page/significant-changes.js +++ b/routes/page/significant-events.js @@ -730,7 +730,7 @@ function cleanOutput(output) { return cleanedOutput; } -function getSignificantChanges(req, res) { +function getSignificantEvents(req, res) { // STEP 1: Gather list of article revisions return mwapi.queryForRevisions(req) @@ -902,177 +902,9 @@ function getSignificantChanges(req, res) { }); } -function getAddedTemplates(req, res) { - - // STEP 1: Gather list of article revisions - return mwapi.queryForRevisions(req, null, req.query.pageSize) - .then( (response) => { - - // STEP 2: All at once gather diffs for each uncached revision and list of - // talk page revisions - const revisions = response.body.query.pages[0].revisions; - const nextRvStartId = revisions[revisions.length - 1].parentid; - var finalOutput = []; - - const articleEvalResults = Object.assign({ - uncachedRevisions: revisions, - cachedOutput: [] - }); - - return BBPromise.props({ - articleDiffAndRevisions: diffAndRevisionPromises(req, - articleEvalResults.uncachedRevisions), - nextRvStartId: nextRvStartId, - finalOutput: finalOutput - }); - }) - .then( (response) => { - - // Flag added template types - return addStructuredTemplates(response.articleDiffAndRevisions, true) - .then( (articleDiffAndRevisions) => { - response.articleDiffAndRevisions = articleDiffAndRevisions; - return response; - }); - }) - .then( (response) => { - - // segment off into types - var uncachedOutput = []; - response.articleDiffAndRevisions.forEach(function (diffAndRevision) { - const revision = diffAndRevision.revision; - if (diffAndRevision.templates && diffAndRevision.templates.length > 0) { - - var section = null; - if (diffAndRevision.templateDiffLine) { - section = getSectionForDiffLine(diffAndRevision.body, - diffAndRevision.templateDiffLine); - } - const addedTemplateOutputObject = new AddedTemplate(revision.revid, - revision.timestamp, revision.user, revision.userid, section, - diffAndRevision.templates); - - uncachedOutput.push(addedTemplateOutputObject); - } - }); - return Object.assign({ nextRvStartId: response.nextRvStartId, - uncachedOutput: uncachedOutput, finalOutput: response.finalOutput } ); - }) - .then( (response) => { - - response.uncachedOutput.forEach((item) => { - response.finalOutput.push(item); - }); - - return Object.assign({ nextRvStartId: response.nextRvStartId, - finalOutput: response.finalOutput } ); - }) - .then( (response) => { - - const cleanedOutput = cleanOutput(response.finalOutput); - const result = Object.assign({ nextRvStartId: response.nextRvStartId, - significantChanges: cleanedOutput } ); - res.send(result).end(); - }); -} - -function getNewTalkTopics(req, res) { - - // STEP 1: Gather list of article revisions - return mwapi.queryForRevisions(req, talkPageTitle(req), req.query.pageSize) - .then( (response) => { - - const talkPageRevisions = response.body.query.pages[0].revisions; - const nextRvStartId = talkPageRevisions[talkPageRevisions.length - 1].parentid; - - var finalOutput = []; - - const talkEvalResults = Object.assign({ - uncachedRevisions: talkPageRevisions, - cachedOutput: [] - }); - - return BBPromise.props({ - talkDiffAndRevisions: diffAndRevisionPromises(req, - talkEvalResults.uncachedRevisions), - nextRvStartId: nextRvStartId, - finalOutput: finalOutput - }); - }) - .then( (response) => { - - updateDiffAndRevisionsWithCharacterCount(response.talkDiffAndRevisions); - - // segment off into types - var uncachedOutput = []; - - // get new talk page revisions, add to uncachedOutput. it will be sorted later. - const newTopicDiffAndRevisions = getNewTopicDiffAndRevisions( - response.talkDiffAndRevisions); - newTopicDiffAndRevisions.forEach(function (diffAndRevision) { - const revision = diffAndRevision.revision; - // todo: better check might be something like get first diff line - // that doesn't have a section title or empty line. - const largestDiffLine = getLargestDiffLine(diffAndRevision.body); - const section = getSectionForLargestDiffLine(diffAndRevision.body, largestDiffLine); - const newTalkPageTopicOutputObject = new NewTalkPageTopicExtraExtended(revision, - largestDiffLine.text, - largestDiffLine.type, largestDiffLine.highlightRanges, - diffAndRevision.characterChange, section); - uncachedOutput.push(newTalkPageTopicOutputObject); - }); - - return Object.assign({ nextRvStartId: response.nextRvStartId, - uncachedOutput: uncachedOutput, finalOutput: response.finalOutput } ); - }) - .then( (response) => { - - // convert large snippets from wikitext to mobile-html - const snippetOutputs = response.uncachedOutput.filter(item => - item.outputType === 'new-talk-page-topic-extra'); - return snippetPromises(req, snippetOutputs) - .then( (snippetResponse) => { - - // push to final output and cache - // note we are using original response list, not snippet response - // (snippet only contains large) - response.uncachedOutput.forEach((item) => { - response.finalOutput.push(item); - }); - - return Object.assign({ nextRvStartId: response.nextRvStartId, - finalOutput: response.finalOutput } ); - }); - }) - .then( (response) => { - - const cleanedOutput = cleanOutput(response.finalOutput); - const result = Object.assign({ nextRvStartId: response.nextRvStartId, - significantChanges: cleanedOutput } ); - res.send(result).end(); - - }); -} - -router.get('/page/significant-changes/:title', (req, res) => { - // res.status(200); - return getSignificantChanges(req, res); - // const result = Object.assign({ result: "What up new endpoint."}); - // mUtil.setContentType(res, mUtil.CONTENT_TYPES.talk); - // res.json(result).end(); -}); - -router.get('/page/added-templates/:title', (req, res) => { - // res.status(200); - return getAddedTemplates(req, res); - // const result = Object.assign({ result: "What up new endpoint."}); - // mUtil.setContentType(res, mUtil.CONTENT_TYPES.talk); - // res.json(result).end(); -}); - -router.get('/page/new-talk-topics/:title', (req, res) => { +router.get('/page/significant-events/:title', (req, res) => { // res.status(200); - return getNewTalkTopics(req, res); + return getSignificantEvents(req, res); // const result = Object.assign({ result: "What up new endpoint."}); // mUtil.setContentType(res, mUtil.CONTENT_TYPES.talk); // res.json(result).end(); From 1ea504927f972af48e88a786184127e85e9e0ebb Mon Sep 17 00:00:00 2001 From: Toni Sevener Date: Mon, 10 Aug 2020 23:30:06 -0500 Subject: [PATCH 16/47] progress showing new reference, added text, and deleted text all together in one timeline event --- routes/page/significant-events.js | 468 ++++++++++++++++++------------ 1 file changed, 279 insertions(+), 189 deletions(-) diff --git a/routes/page/significant-events.js b/routes/page/significant-events.js index 0548feb1..01f1b5ea 100644 --- a/routes/page/significant-events.js +++ b/routes/page/significant-events.js @@ -13,6 +13,14 @@ let app; const significantChangesCache = {}; +class CharacterChangeWithSections { + constructor(counts, addedSections, deletedSections) { + this.counts = counts; + this.addedSections = addedSections; + this.deletedSections = deletedSections; + } +} + class CharacterChange { constructor(addedCount, deletedCount) { this.addedCount = addedCount; @@ -25,13 +33,12 @@ class CharacterChange { } class SmallOutput { - constructor(revid, timestamp, user, userid, characterChange) { + constructor(revid, timestamp, user, userid) { this.revid = revid; this.timestamp = timestamp; this.outputType = 'small-change'; this.user = user; this.userid = userid; - this.characterChange = characterChange; } } @@ -53,27 +60,39 @@ class VandalismOutput { } } -class NewReference { - constructor(revid, timestamp, user, userid, section, templates) { - this.revid = revid; - this.timestamp = timestamp; +class NewReferenceOutput { + constructor(sections, templates) { this.outputType = 'new-reference'; - this.user = user; - this.userid = userid; - this.section = section; + this.sections = sections; this.templates = templates; } } -class AddedTemplate { - constructor(revid, timestamp, user, userid, section, templates) { - this.revid = revid; - this.timestamp = timestamp; - this.outputType = 'added-template'; - this.user = user; - this.userid = userid; - this.section = section; - this.templates = templates; +class AddedTextOutputExpanded { + constructor(characterChangeWithSections, snippet, snippetType, snippetHighlightRanges) { + this.outputType = 'added-text'; + this.snippet = snippet; + this.snippetType = snippetType; + this.snippetHighlightRanges = snippetHighlightRanges; + this.characterCount = characterChangeWithSections.counts.addedCount; + this.sections = characterChangeWithSections.addedSections; + } +} + +class AddedTextOutput { + constructor(addedTextOutputExpanded) { + this.outputType = addedTextOutputExpanded.outputType; + this.snippet = addedTextOutputExpanded.snippet; + this.characterCount = addedTextOutputExpanded.characterCount; + this.sections = addedTextOutputExpanded.sections; + } +} + +class DeletedTextOutput { + constructor(characterChangeWithSections) { + this.outputType = 'deleted-text'; + this.characterCount = characterChangeWithSections.counts.deletedCount; + this.sections = characterChangeWithSections.deletedSections; } } @@ -82,49 +101,20 @@ class LargeOutput { this.revid = largeOutputExpanded.revid; this.timestamp = largeOutputExpanded.timestamp; this.outputType = 'large-change'; - this.snippet = largeOutputExpanded.snippet; this.user = largeOutputExpanded.user; this.userid = largeOutputExpanded.userid; - this.characterChange = largeOutputExpanded.characterChange; - this.section = largeOutputExpanded.section; + this.significantChanges = largeOutputExpanded.significantChanges; } } class LargeOutputExpanded { - constructor(revid, timestamp, user, userid, snippet, type, highlightRanges, characterChange, - section) { + constructor(revid, timestamp, user, userid, significantChanges) { this.revid = revid; this.timestamp = timestamp; this.outputType = 'large-change'; - this.snippet = snippet; - this.type = type; - this.highlightRanges = highlightRanges; this.user = user; this.userid = userid; - this.characterChange = characterChange; - this.section = section; - } -} - -class NewTalkPageTopicExtraExtended { - constructor(revision, snippet, type, highlightRanges, characterChange, - section) { - this.revision = revision; - this.outputType = 'new-talk-page-topic-extra'; - this.snippet = snippet; - this.type = type; - this.highlightRanges = highlightRanges; - this.characterChange = characterChange; - this.section = section; - } -} - -class NewTalkPageTopicExtra { - constructor(item) { - this.revision = item.revision; - this.outputType = 'new-talk-page-topic-extra'; - this.snippet = item.snippet; - this.section = item.section; + this.significantChanges = significantChanges; } } @@ -156,6 +146,18 @@ class NewTalkPageTopic { } } +class PreformattedSnippet { + constructor(revid, outputType, snippet, snippetType, snippetHighlightRanges, + indexOfSignificantChanges) { + this.revid = revid; + this.outputType = outputType; + this.snippet = snippet; + this.snippetType = snippetType; + this.snippetHighlightRanges = snippetHighlightRanges; + this.indexOfSignificantChanges = indexOfSignificantChanges; + } +} + function getThreshold(req) { return req.query.threshold === null || req.query.threshold === undefined ? 100 : req.query.threshold; @@ -180,7 +182,7 @@ const diffAndRevisionPromise = (req, revision) => { }); }; -const snippetPromise = (req, snippetOutput) => { +const snippetPromise = (req, preformattedSnippet) => { const headers = Object.assign( { accept: 'text/html', profile: 'https://www.mediawiki.org/wiki/Specs/Mobile-HTML/1.0.0', @@ -190,9 +192,9 @@ const snippetPromise = (req, snippetOutput) => { var newSnippet; - if (snippetOutput.outputType === 'large-change') { + if (preformattedSnippet.outputType === 'large-change') { // add highlight delimiters first - var snippetBinary = encoding.strToBin(snippetOutput.snippet); + var snippetBinary = encoding.strToBin(preformattedSnippet.snippet); // todo: it looks like parsoid *sometimes* strips these spans out // depending on their placement. @@ -200,31 +202,37 @@ const snippetPromise = (req, snippetOutput) => { // see results for this revision, it's missing an add-highlight. // https://en.wikipedia.org/w/index.php?title=United_States // &type=revision&diff=965295364&oldid=965071033 - const addHighlightStart = ''; - const deleteHighlightStart = ''; - const highlightEnd = ''; + // after looking at this, highlighted text that was added were references / new citations. + // Parsoid turns it into a basic superscript citation number and strips out any highlighting + // but using ~~~addhighlightstart~~~ instead of does seem to + // shift the delimiters less. still doesn't solve above revision 965295364 issue though. + const addHighlightStart = '~~~addhighlightstart~~~'; + const deleteHighlightStart = '~~~deletehighlightstart~~~'; + const highlightEnd = '~~~highlightend~~~'; const addHighlightStartBin = encoding.strToBin(addHighlightStart); const deleteHighlightStartBin = encoding.strToBin(deleteHighlightStart); const highlightEndBin = encoding.strToBin(highlightEnd); - switch (snippetOutput.type) { + switch (preformattedSnippet.snippetType) { case 1: // Added complete line - snippetBinary = insertSubstringInString(snippetBinary, addHighlightStartBin, 0); + snippetBinary = insertSubstringInString(snippetBinary, addHighlightStartBin, + 0); snippetBinary = insertSubstringInString(snippetBinary, highlightEndBin, snippetBinary.length); break; case 2: // Deleted complete line - snippetBinary = insertSubstringInString(snippetBinary, deleteHighlightStartBin, 0); + snippetBinary = insertSubstringInString(snippetBinary, deleteHighlightStartBin, + 0); snippetBinary = insertSubstringInString(snippetBinary, highlightEndBin, snippetBinary.length); break; case 5: case 3: // Added and deleted words in line var offset = 0; - snippetOutput.highlightRanges.forEach(function (range) { + preformattedSnippet.snippetHighlightRanges.forEach(function (range) { switch (range.type) { case 0: // Added snippetBinary = insertSubstringInString(snippetBinary, @@ -251,7 +259,7 @@ const snippetPromise = (req, snippetOutput) => { newSnippet = encoding.binToStr(snippetBinary); } else { // new talk page topic - newSnippet = snippetOutput.snippet; + newSnippet = preformattedSnippet.snippet; } // make request to format to mobile-html, reassign result back to snippet @@ -308,14 +316,14 @@ const snippetPromise = (req, snippetOutput) => { } } - snippetOutput.snippet = strippedSnippet; - return snippetOutput; + preformattedSnippet.snippet = strippedSnippet; + return preformattedSnippet; }); }; -const snippetPromises = (req, snippetOutputs) => { - return BBPromise.map(snippetOutputs, function(snippetOutput) { - return snippetPromise(req, snippetOutput); +const snippetPromises = (req, preformattedSnippets) => { + return BBPromise.map(preformattedSnippets, function(preformattedSnippet) { + return snippetPromise(req, preformattedSnippet); }); }; @@ -368,6 +376,83 @@ function getCachedAndUncachedItems(revisions, req, title) { }); } +function getSectionForDiffLine(diffBody, diffLine) { + + var fromSection = null; + var toSection = null; + + // capture intro + if ((!diffBody.from.sections || + diffBody.from.sections.length === 0) && + (!diffBody.to.sections || + diffBody.to.sections.length === 0)) { + return null; + } + + // diffLine.offset.from = 0 is still valid if it's at the very beginning of the article. + // In this case javascript evaluates diffLine.offset.from to false, + // hence the need for the separate check. + if ((diffLine.offset.from || diffLine.offset.from === 0) && + diffLine.offset.from < diffBody.from.sections[0].offset) { + fromSection = 'Intro'; + } + + if ((diffLine.offset.to || diffLine.offset.to === 0) && + diffLine.offset.to < diffBody.to.sections[0].offset) { + toSection = 'Intro'; + } + + if (fromSection && toSection) { + if (diffLine.offset.to && diffLine.offset.to.length > 0) { + return toSection; + } else { + return fromSection; + } + } + + var prevSection = null; + if (!fromSection && diffLine.offset.from) { + for (let i = 0; i < diffBody.from.sections.length; i++) { + const section = diffBody.from.sections[i]; + + if (diffLine.offset.from < section.offset && prevSection) { + fromSection = prevSection.heading; + break; + } + + prevSection = section; + } + + if (!fromSection && diffLine.offset.from > 0) { + fromSection = prevSection.heading; + } + } + + if (!toSection && diffLine.offset.to) { + prevSection = null; + for (let i = 0; i < diffBody.to.sections.length; i++) { + + const section = diffBody.to.sections[i]; + if (diffLine.offset.to < section.offset && prevSection) { + toSection = prevSection.heading; + break; + } + + prevSection = section; + } + + if (!toSection && diffLine.offset.to > 0) { + toSection = prevSection.heading; + } + } + + if (diffLine.offset.to) { + return toSection; + } else { + return fromSection; + } +} + function updateDiffAndRevisionsWithCharacterCount(diffAndRevisions) { // Loop through diffs, filter out type 0 (context type) and assign byte change properties @@ -379,6 +464,8 @@ function updateDiffAndRevisionsWithCharacterCount(diffAndRevisions) { var aggregateAddedCount = 0; var aggregateDeletedCount = 0; + var aggregateAddedSections = new Set(); + var aggregateDeletedSections = new Set(); diffAndRevision.body.diff.forEach(function (diff) { var lineAddedCount = 0; var lineDeletedCount = 0; @@ -422,12 +509,28 @@ function updateDiffAndRevisionsWithCharacterCount(diffAndRevisions) { aggregateAddedCount += lineAddedCount; aggregateDeletedCount += lineDeletedCount; + if (lineAddedCount > 0 ) { + const section = getSectionForDiffLine(diffAndRevision.body, diff); + if (section) { + aggregateAddedSections.add(section); + } + } + + if (lineDeletedCount > 0) { + const section = getSectionForDiffLine(diffAndRevision.body, diff); + if (section) { + aggregateDeletedSections.add(section); + } + } + diff.characterChange = new CharacterChange(lineAddedCount, lineDeletedCount); filteredDiffs.push(diff); }); - diffAndRevision.characterChange = new CharacterChange(aggregateAddedCount, - aggregateDeletedCount); + const aggregateCounts = new CharacterChange(aggregateAddedCount, aggregateDeletedCount); + diffAndRevision.characterChangeWithSections = new CharacterChangeWithSections( + aggregateCounts, Array.from(aggregateAddedSections), + Array.from(aggregateAddedSections)); diffAndRevision.body.diff = filteredDiffs; }); } @@ -449,7 +552,8 @@ function needsToParseForAddedTemplates(text, includeOpeningBraces, includeAll) { for (var n = 0; n < names.length; n++) { const name = names[n]; - if ((text.includes(`{{${name}`) && includeOpeningBraces) || (text.includes(`${name}`) && !includeOpeningBraces)) { + if ((text.includes(`{{${name}`) && includeOpeningBraces) || (text.includes(`${name}`) && + !includeOpeningBraces)) { return true; } } @@ -473,7 +577,8 @@ function structuredTemplatePromise(text, diff, revision, includeAll) { for (var i = 0; i < individualTemplates.length; i++) { const template = individualTemplates[i]; - if (!needsToParseForAddedTemplates(template.name, false, includeAll)) { + if (!needsToParseForAddedTemplates(template.name, false, + includeAll)) { continue; } @@ -506,7 +611,8 @@ function addStructuredTemplates(diffAndRevisions, includeAll) { switch (diff.type) { case 1: // Add complete line type - if (needsToParseForAddedTemplates(diff.text, true, includeAll)) { + if (needsToParseForAddedTemplates(diff.text, true, + includeAll)) { promises.push(structuredTemplatePromise(diff.text, diff, diffAndRevision.revision)); } @@ -522,7 +628,8 @@ function addStructuredTemplates(diffAndRevisions, includeAll) { switch (range.type) { case 0: // Add range type - if (needsToParseForAddedTemplates(rangeText, true, includeAll)) { + if (needsToParseForAddedTemplates(rangeText, true, + includeAll)) { promises.push(structuredTemplatePromise(rangeText, diff, diffAndRevision.revision, includeAll)); } @@ -599,83 +706,6 @@ function getLargestDiffLine(diffBody) { return largestDiffLine; } -function getSectionForDiffLine(diffBody, diffLine) { - - var fromSection = null; - var toSection = null; - - // capture intro - if ((!diffBody.from.sections || - diffBody.from.sections.length === 0) && - (!diffBody.to.sections || - diffBody.to.sections.length === 0)) { - return null; - } - - // diffLine.offset.from of = is still valid if it's at the very beginning of the article. - // In this case javascript evaluates diffLine.offset.from to false, - // hence the need for the separate check. - if ((diffLine.offset.from || diffLine.offset.from === 0) && - diffLine.offset.from < diffBody.from.sections[0].offset) { - fromSection = 'Intro'; - } - - if ((diffLine.offset.to || diffLine.offset.to === 0) && - diffLine.offset.to < diffBody.to.sections[0].offset) { - toSection = 'Intro'; - } - - if (fromSection && toSection) { - if (diffLine.offset.to && diffLine.offset.to.length > 0) { - return toSection; - } else { - return fromSection; - } - } - - var prevSection = null; - if (!fromSection && diffLine.offset.from) { - for (let i = 0; i < diffBody.from.sections.length; i++) { - const section = diffBody.from.sections[i]; - - if (diffLine.offset.from < section.offset && prevSection) { - fromSection = prevSection.heading; - break; - } - - prevSection = section; - } - - if (!fromSection && diffLine.offset.from > 0) { - fromSection = prevSection.heading; - } - } - - if (!toSection && diffLine.offset.to) { - prevSection = null; - for (let i = 0; i < diffBody.to.sections.length; i++) { - - const section = diffBody.to.sections[i]; - if (diffLine.offset.to < section.offset && prevSection) { - toSection = prevSection.heading; - break; - } - - prevSection = section; - } - - if (!toSection && diffLine.offset.to > 0) { - toSection = prevSection.heading; - } - } - - if (diffLine.offset.to) { - return toSection; - } else { - return fromSection; - } -} - function getSectionForLargestDiffLine(diffBody, largestDiffLine) { // get largest diff const diffSection = getSectionForDiffLine(diffBody, @@ -689,10 +719,6 @@ function cleanOutput(output) { // sort by date first output = output.sort(function(a, b) { - if (a.outputType === 'new-talk-page-topic-extra') { - return new Date(b.revision.timestamp) - new Date(a.revision.timestamp); - } - return new Date(b.timestamp) - new Date(a.timestamp); }); @@ -713,8 +739,6 @@ function cleanOutput(output) { cleanedOutput.push(new LargeOutput(item)); } else if (item.outputType === 'new-talk-page-topic') { cleanedOutput.push(new NewTalkPageTopic(item)); - } else if (item.outputType === 'new-talk-page-topic-extra') { - cleanedOutput.push(new NewTalkPageTopicExtra(item)); } else { cleanedOutput.push(item); } @@ -745,8 +769,7 @@ function getSignificantEvents(req, res) { const articleEvalResults = getCachedAndUncachedItems(revisions, req, null); // save cached article revisions to finalOutput - var finalOutput = []; - finalOutput = finalOutput.concat(articleEvalResults.cachedOutput); + const finalOutput = articleEvalResults.cachedOutput; // todo: unfortunately this cuts out new talk page topics that appeared after // the latest article revision. rethink this piece. @@ -773,7 +796,7 @@ function getSignificantEvents(req, res) { const talkPageEvalResults = getCachedAndUncachedItems(talkPageRevisions, req, talkPageTitle(req)); - // save cached talk page revisions to output + // save cached talk page revisions to finalOutput const finalOutput = response.finalOutput.concat(talkPageEvalResults.cachedOutput); // for each uncached talk page revision, gather diffs @@ -809,19 +832,7 @@ function getSignificantEvents(req, res) { var uncachedOutput = []; response.articleDiffAndRevisions.forEach(function (diffAndRevision) { const revision = diffAndRevision.revision; - if (diffAndRevision.templates && diffAndRevision.templates.length > 0) { - - var section = null; - if (diffAndRevision.templateDiffLine) { - section = getSectionForDiffLine(diffAndRevision.body, - diffAndRevision.templateDiffLine); - } - const newReferenceOutputObject = new NewReference(revision.revid, - revision.timestamp, revision.user, revision.userid, section, - diffAndRevision.templates); - - uncachedOutput.push(newReferenceOutputObject); - } else if (revision.tags.includes('mw-rollback') && + if (revision.tags.includes('mw-rollback') && revision.comment.toLowerCase().includes('revert') && revision.comment.toLowerCase().includes('vandalism')) { const largestDiffLine = getLargestDiffLine(diffAndRevision.body); @@ -830,19 +841,53 @@ function getSignificantEvents(req, res) { const vandalismRevertOutputObject = new VandalismOutput(revision.revid, revision.timestamp, revision.user, revision.userid, section); uncachedOutput.push(vandalismRevertOutputObject); - } else if (diffAndRevision.characterChange.totalCount() <= threshold) { - const smallOutputObject = new SmallOutput(revision.revid, revision.timestamp, - revision.user, revision.userid, diffAndRevision.characterChange); - uncachedOutput.push(smallOutputObject); } else { - const largestDiffLine = getLargestDiffLine(diffAndRevision.body); - const section = getSectionForLargestDiffLine(diffAndRevision.body, - largestDiffLine); - const largeOutputObject = new LargeOutputExpanded(revision.revid, - revision.timestamp, revision.user, revision.userid, largestDiffLine.text, - largestDiffLine.type, largestDiffLine.highlightRanges, - diffAndRevision.characterChange, section); - uncachedOutput.push(largeOutputObject); + + var significantChanges = []; + if (diffAndRevision.templates && diffAndRevision.templates.length > 0) { + var sections = []; + // todo: this section determination seems broken. + // multiple templates can occur on multiple lines. + if (diffAndRevision.templateDiffLine) { + const section = getSectionForDiffLine(diffAndRevision.body, + diffAndRevision.templateDiffLine); + if (section) { + sections.push(section); + } + } + const newReferenceOutputObject = new NewReferenceOutput(sections, + diffAndRevision.templates); + significantChanges.push(newReferenceOutputObject); + } + + if (diffAndRevision.characterChangeWithSections.counts.totalCount() > + threshold) { + + if (diffAndRevision.characterChangeWithSections.counts.addedCount > 0) { + const largestDiffLine = getLargestDiffLine(diffAndRevision.body); + // todo: get largest diff line of ADDED, do not include delete + const addedTextOutputObject = new AddedTextOutputExpanded( + diffAndRevision.characterChangeWithSections, largestDiffLine.text, + largestDiffLine.type, largestDiffLine.highlightRanges); + significantChanges.push(addedTextOutputObject); + } + + if (diffAndRevision.characterChangeWithSections.counts.deletedCount > 0) { + const deletedTextOutputObject = new DeletedTextOutput( + diffAndRevision.characterChangeWithSections); + significantChanges.push(deletedTextOutputObject); + } + } + + if (significantChanges.length > 0) { + const largeOutputObject = new LargeOutputExpanded(revision.revid, + revision.timestamp, revision.user, revision.userid, significantChanges); + uncachedOutput.push(largeOutputObject); + } else { + const smallOutputObject = new SmallOutput(revision.revid, + revision.timestamp, revision.user, revision.userid); + uncachedOutput.push(smallOutputObject); + } } }); @@ -858,7 +903,7 @@ function getSignificantEvents(req, res) { const newTalkPageTopicOutputObject = new NewTalkPageTopicExtended(revision.revid, revision.timestamp, revision.user, revision.userid, largestDiffLine.text, largestDiffLine.type, largestDiffLine.highlightRanges, - diffAndRevision.characterChange, section); + diffAndRevision.characterChangeWithSections.counts, section); uncachedOutput.push(newTalkPageTopicOutputObject); }); @@ -867,11 +912,56 @@ function getSignificantEvents(req, res) { }) .then( (response) => { + var preformattedSnippets = []; + response.uncachedOutput.forEach(function (item) { + if (item.outputType === 'new-talk-page-topic') { + const snippet = new PreformattedSnippet(item.revid, item.outputType, + item.snippet,1, null, + null); + preformattedSnippets.push(snippet); + } else if (item.outputType === 'large-change') { + + for (let i = 0; i < item.significantChanges.length; i++) { + const significantChange = item.significantChanges[i]; + if (significantChange.outputType === 'added-text') { + const snippet = new PreformattedSnippet(item.revid, item.outputType, + significantChange.snippet, significantChange.snippetType, + significantChange.snippetHighlightRanges, i); + preformattedSnippets.push(snippet); + } + } + } + }); + // convert large snippets from wikitext to mobile-html - const snippetOutputs = response.uncachedOutput.filter(item => - item.outputType === 'large-change' || item.outputType === 'new-talk-page-topic'); - return snippetPromises(req, snippetOutputs) - .then( (snippetResponse) => { + return snippetPromises(req, preformattedSnippets) + .then( (formattedSnippets) => { + + // reassign formattedSnippets to snippet output + formattedSnippets.forEach((formattedSnippet) => { + response.uncachedOutput.forEach((item) => { + if (item.revid === formattedSnippet.revid && + item.outputType === formattedSnippet.outputType) { + if (formattedSnippet.outputType === 'new-talk-page-topic') { + item.snippet = formattedSnippet.snippet; + } else if (formattedSnippet.outputType === 'large-change') { + if (item.significantChanges.length > + formattedSnippet.indexOfSignificantChanges) { + var significantChange = + item.significantChanges[ + formattedSnippet.indexOfSignificantChanges + ]; + significantChange.snippet = formattedSnippet.snippet; + if (significantChange.outputType === 'added-text') { + item.significantChanges[ + formattedSnippet.indexOfSignificantChanges + ] = new AddedTextOutput(significantChange); + } + } + } + } + }); + }); // push to final output and cache // note we are using original response list, not snippet response @@ -896,7 +986,7 @@ function getSignificantEvents(req, res) { const cleanedOutput = cleanOutput(response.finalOutput); const result = Object.assign({ nextRvStartId: response.nextRvStartId, - significantChanges: cleanedOutput } ); + timeline: cleanedOutput } ); res.send(result).end(); }); From dd4c5ffc61a39dc50d9227576fd3dee220c0d54c Mon Sep 17 00:00:00 2001 From: Toni Sevener Date: Mon, 10 Aug 2020 23:39:19 -0500 Subject: [PATCH 17/47] fixed a bug, found another bug --- routes/page/significant-events.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/routes/page/significant-events.js b/routes/page/significant-events.js index 01f1b5ea..347a0073 100644 --- a/routes/page/significant-events.js +++ b/routes/page/significant-events.js @@ -530,7 +530,7 @@ function updateDiffAndRevisionsWithCharacterCount(diffAndRevisions) { const aggregateCounts = new CharacterChange(aggregateAddedCount, aggregateDeletedCount); diffAndRevision.characterChangeWithSections = new CharacterChangeWithSections( aggregateCounts, Array.from(aggregateAddedSections), - Array.from(aggregateAddedSections)); + Array.from(aggregateDeletedSections)); diffAndRevision.body.diff = filteredDiffs; }); } @@ -561,6 +561,10 @@ function needsToParseForAddedTemplates(text, includeOpeningBraces, includeAll) { return false; } +// BUG: https://en.wikipedia.org/w/index.php?title=United_States&type=revision +// &diff=965295364&oldid=965071033 +// We are missing some an added reference from line 722. We also aren't +// catching the tags in line 579 function structuredTemplatePromise(text, diff, revision, includeAll) { return new BBPromise((resolve) => { var main = PRFunPromise.async(function*() { From 71f8ad68a7146efe708c88961328a877f9f4c725 Mon Sep 17 00:00:00 2001 From: Toni Sevener Date: Thu, 13 Aug 2020 17:29:05 -0500 Subject: [PATCH 18/47] fixing talk page bugs, attempt at truncating snippets --- lib/mwapi.js | 2 + routes/page/significant-events.js | 288 ++++++++++++++++++++---------- 2 files changed, 194 insertions(+), 96 deletions(-) diff --git a/lib/mwapi.js b/lib/mwapi.js index 0db7b8c9..a7f71d51 100644 --- a/lib/mwapi.js +++ b/lib/mwapi.js @@ -142,6 +142,8 @@ mwapi.queryForRevisions = function(req, title, pageSize, rvStart, rvEnd) { if (rvStart && rvEnd) { query.rvstart = rvStart; query.rvend = rvEnd; + } else if (rvEnd) { + query.rvend = rvEnd; } else { query.rvstartid = req.query.rvstartid; } diff --git a/routes/page/significant-events.js b/routes/page/significant-events.js index 347a0073..b7b51e02 100644 --- a/routes/page/significant-events.js +++ b/routes/page/significant-events.js @@ -83,6 +83,7 @@ class AddedTextOutput { constructor(addedTextOutputExpanded) { this.outputType = addedTextOutputExpanded.outputType; this.snippet = addedTextOutputExpanded.snippet; + this.snippetType = addedTextOutputExpanded.snippetType; this.characterCount = addedTextOutputExpanded.characterCount; this.sections = addedTextOutputExpanded.sections; } @@ -172,6 +173,12 @@ function insertSubstringInString(originalString, substring, index) { return substring + originalString; } +function stringByRemovingSubstring(originalString, beginningIndex, endIndex) { + const firstSubstring = originalString.substring(0,beginningIndex); + const secondSubstring = originalString.substring(endIndex); + return firstSubstring + secondSubstring; +} + const diffAndRevisionPromise = (req, revision) => { return mwrestapi.queryForDiff(req, revision.parentid, revision.revid) .then( (response) => { @@ -182,7 +189,8 @@ const diffAndRevisionPromise = (req, revision) => { }); }; -const snippetPromise = (req, preformattedSnippet) => { +const formattedSnippetFromTextPromise = (req, text) => { + const headers = Object.assign( { accept: 'text/html', profile: 'https://www.mediawiki.org/wiki/Specs/Mobile-HTML/1.0.0', @@ -190,82 +198,8 @@ const snippetPromise = (req, preformattedSnippet) => { 'output-mode': 'editPreview' }); - var newSnippet; - - if (preformattedSnippet.outputType === 'large-change') { - // add highlight delimiters first - var snippetBinary = encoding.strToBin(preformattedSnippet.snippet); - - // todo: it looks like parsoid *sometimes* strips these spans out - // depending on their placement. - // we need some token that parsoid won't touch. - // see results for this revision, it's missing an add-highlight. - // https://en.wikipedia.org/w/index.php?title=United_States - // &type=revision&diff=965295364&oldid=965071033 - // after looking at this, highlighted text that was added were references / new citations. - // Parsoid turns it into a basic superscript citation number and strips out any highlighting - // but using ~~~addhighlightstart~~~ instead of does seem to - // shift the delimiters less. still doesn't solve above revision 965295364 issue though. - const addHighlightStart = '~~~addhighlightstart~~~'; - const deleteHighlightStart = '~~~deletehighlightstart~~~'; - const highlightEnd = '~~~highlightend~~~'; - - const addHighlightStartBin = encoding.strToBin(addHighlightStart); - const deleteHighlightStartBin = encoding.strToBin(deleteHighlightStart); - const highlightEndBin = encoding.strToBin(highlightEnd); - - switch (preformattedSnippet.snippetType) { - case 1: // Added complete line - - snippetBinary = insertSubstringInString(snippetBinary, addHighlightStartBin, - 0); - snippetBinary = insertSubstringInString(snippetBinary, highlightEndBin, - snippetBinary.length); - break; - case 2: // Deleted complete line - - snippetBinary = insertSubstringInString(snippetBinary, deleteHighlightStartBin, - 0); - snippetBinary = insertSubstringInString(snippetBinary, highlightEndBin, - snippetBinary.length); - break; - case 5: - case 3: // Added and deleted words in line - var offset = 0; - preformattedSnippet.snippetHighlightRanges.forEach(function (range) { - switch (range.type) { - case 0: // Added - snippetBinary = insertSubstringInString(snippetBinary, - addHighlightStartBin, range.start + offset); - offset += addHighlightStartBin.length; - break; - case 1: // Deleted - snippetBinary = insertSubstringInString(snippetBinary, - deleteHighlightStartBin, range.start + offset); - offset += deleteHighlightStartBin.length; - break; - default: - return; - } - - snippetBinary = insertSubstringInString(snippetBinary, highlightEndBin, - range.start + offset + range.length); - offset += highlightEndBin.length; - }); - break; - default: - break; - } - - newSnippet = encoding.binToStr(snippetBinary); - } else { // new talk page topic - newSnippet = preformattedSnippet.snippet; - } - - // make request to format to mobile-html, reassign result back to snippet - const formData = Object.assign({ - wikitext: newSnippet + wikitext: text }); const request = Object.assign({ @@ -314,9 +248,141 @@ const snippetPromise = (req, preformattedSnippet) => { strippedSnippet = sectionWrapper.innerHTML; } + + // todo: remove monte tags from sectionWrapper + } else { + // todo: remove monte tags from body } - preformattedSnippet.snippet = strippedSnippet; + return strippedSnippet; + }); +}; + +const snippetPromise = (req, preformattedSnippet) => { + + if (preformattedSnippet.snippetType === 2) { + // deleted complete line type. + // we shouldn't get here but gracefully return null for snippet without trying to + // format + preformattedSnippet.snippet = null; + return preformattedSnippet; + } + + var strippedSnippet; + const addHighlightStart = 'ioshighlightstart'; + const highlightEnd = 'ioshighlightend'; + + const addHighlightStartBin = encoding.strToBin(addHighlightStart); + const highlightEndBin = encoding.strToBin(highlightEnd); + + // strip deleted ranges from snippet + // add added delimiters to type 3 & 5 snippet (Added and deleted words in line) + // delimiters are later used for truncation starting point + if (preformattedSnippet.outputType === 'large-change') { + var snippetBinary = encoding.strToBin(preformattedSnippet.snippet); + switch (preformattedSnippet.snippetType) { + case 1: // Added complete line + break; + case 2: // Deleted complete line + // should be caught earlier + break; + case 5: + case 3: // Added and deleted words in line + + // first strip deleted text + for (var d = preformattedSnippet.snippetHighlightRanges.length - 1; d >= 0; d-- ) { + const range = preformattedSnippet.snippetHighlightRanges[d]; + + switch (range.type) { + case 0: // Added + break; + case 1: // Deleted + snippetBinary = stringByRemovingSubstring(snippetBinary, range.start, + range.start + range.length); + + for (var i = d; i < preformattedSnippet.snippetHighlightRanges.length; + i++) { + const iRange = preformattedSnippet.snippetHighlightRanges[i]; + switch (iRange.type) { + case 0: // Added + iRange.start -= range.length; + preformattedSnippet.snippetHighlightRanges[i] = iRange; + break; + case 1: // Deleted + break; + default: + break; + } + } + break; + default: + break; + } + } + + // filter out deleted ranges + // eslint-disable-next-line no-case-declarations + const addedHighlightRanges = preformattedSnippet.snippetHighlightRanges + .filter(range => range.type === 0); + + // then add added text delimiters + var addOffset = 0; + addedHighlightRanges.forEach(function (range) { + switch (range.type) { + case 0: // Added + snippetBinary = insertSubstringInString(snippetBinary, + addHighlightStartBin, range.start + addOffset); + addOffset += addHighlightStartBin.length; + break; + case 1: // Deleted + break; + default: + break; + } + + snippetBinary = insertSubstringInString(snippetBinary, highlightEndBin, + range.start + addOffset + range.length); + addOffset += highlightEndBin.length; + }); + break; + default: + break; + } + + strippedSnippet = encoding.binToStr(snippetBinary); + } else { // new talk page topic + strippedSnippet = preformattedSnippet.snippet; + } + + // make request to format to mobile-html, reassign result back to snippet + return formattedSnippetFromTextPromise(req, strippedSnippet) + .then( (response) => { + // truncate snippet if needed + // note: there's an interesting case where the existence of the delimiters changes the + // output. In this case being next to + // a wikitext link caused Parsoid to think that this was a new wikitext article + // rather than an existing one. I'm forging ahead as I think the worst that could + // happen is a missing link + // but if other issues occur, we should remove delimiters. With that we would lose + // all possibility of stretch goal highlighting + // and truncation + var truncatedSnippet = response; + if (preformattedSnippet.snippetType === 5 || preformattedSnippet.snippetType === 3) { + // try to trim the areas before the first ioshighlightstart and after the last + // ioshighlight start so snippet is + // focused on area that changed + const firstStart = response.indexOf('ioshighlightstart'); + const lastStart = response.lastIndexOf('ioshighlightend'); + if (firstStart >= 0 && lastStart >= 0) { + truncatedSnippet = truncatedSnippet.slice(firstStart); + truncatedSnippet = truncatedSnippet.slice(0, lastStart); + // UNCOMMENT THIS LINE IF HIGHLIGHTING IS NOT WORTH IT + // truncatedSnippet = truncatedSnippet.replace(/ioshighlightstart/g, '') + // .replace(/ioshighlightend/g, ''); + truncatedSnippet = '...'.concat(truncatedSnippet).concat('...'); + } + } + preformattedSnippet.snippet = truncatedSnippet; return preformattedSnippet; }); }; @@ -675,6 +741,8 @@ function getNewTopicDiffAndRevisions(talkDiffAndRevisions) { // const talkPageRevisions = talkPageObject.revisions; talkDiffAndRevisions.forEach(function (diffAndRevision, index) { if (diffAndRevision.revision.comment.toLowerCase().includes('new section') + && !diffAndRevision.revision.comment.toLowerCase() + .includes('semi-protected edit request') // don't show semi-protected edit requests && diffAndRevision.revision.userid !== 4936590) { // don't show signbot topics // see if this section was reverted in previous iterations var wasReverted = false; @@ -710,11 +778,39 @@ function getLargestDiffLine(diffBody) { return largestDiffLine; } -function getSectionForLargestDiffLine(diffBody, largestDiffLine) { - // get largest diff - const diffSection = getSectionForDiffLine(diffBody, - largestDiffLine); - return diffSection; +function getLargestDiffLineOfAdded(diffBody) { + diffBody.diff.sort(function(a, b) { + return b.characterChange.addedCount - a.characterChange.addedCount; + }); + + // todo: safety + const largestDiffLine = diffBody.diff[0]; + return largestDiffLine; +} + +function textContainsEmptyLineOrSection(text) { + const trimmedText = text.trim(); + return (trimmedText.length === 0 || text.includes('==')); +} + +function getFirstDiffLineWithContent(diffBody) { + + for (let i = 0; i < diffBody.diff.length; i++) { + const diff = diffBody.diff[i]; + + switch (diff.type) { + case 0: // Context line type + continue; + default: + if (textContainsEmptyLineOrSection(diff.text)) { + continue; + } else { + return diff; + } + } + } + + return null; } function cleanOutput(output) { @@ -775,10 +871,12 @@ function getSignificantEvents(req, res) { // save cached article revisions to finalOutput const finalOutput = articleEvalResults.cachedOutput; - // todo: unfortunately this cuts out new talk page topics that appeared after - // the latest article revision. rethink this piece. - const rvStart = revisions[0].timestamp; - // todo: length error checking here + const rvStart = (req.query.rvstartid !== undefined && req.query.rvstartid !== null) ? + revisions[0].timestamp : null; + // if rvstartid is missing from query, they are fetching the first page + // if they are fetching the first page, we don't want to block of + // talk page revision fetching at the start, in case talk page topics came in + // after the latest article revision const rvEnd = revisions[revisions.length - 1].timestamp; return BBPromise.props({ @@ -840,8 +938,9 @@ function getSignificantEvents(req, res) { revision.comment.toLowerCase().includes('revert') && revision.comment.toLowerCase().includes('vandalism')) { const largestDiffLine = getLargestDiffLine(diffAndRevision.body); - const section = getSectionForLargestDiffLine(diffAndRevision.body, + const section = getSectionForDiffLine(diffAndRevision.body, largestDiffLine); + // todo: vandalism type can have reversions in multiple sections const vandalismRevertOutputObject = new VandalismOutput(revision.revid, revision.timestamp, revision.user, revision.userid, section); uncachedOutput.push(vandalismRevertOutputObject); @@ -868,8 +967,7 @@ function getSignificantEvents(req, res) { threshold) { if (diffAndRevision.characterChangeWithSections.counts.addedCount > 0) { - const largestDiffLine = getLargestDiffLine(diffAndRevision.body); - // todo: get largest diff line of ADDED, do not include delete + const largestDiffLine = getLargestDiffLineOfAdded(diffAndRevision.body); const addedTextOutputObject = new AddedTextOutputExpanded( diffAndRevision.characterChangeWithSections, largestDiffLine.text, largestDiffLine.type, largestDiffLine.highlightRanges); @@ -900,13 +998,12 @@ function getSignificantEvents(req, res) { response.talkDiffAndRevisions); newTopicDiffAndRevisions.forEach(function (diffAndRevision) { const revision = diffAndRevision.revision; - // todo: better check might be something like get first diff line - // that doesn't have a section title or empty line. - const largestDiffLine = getLargestDiffLine(diffAndRevision.body); - const section = getSectionForLargestDiffLine(diffAndRevision.body, largestDiffLine); + const firstDiffLine = getFirstDiffLineWithContent(diffAndRevision.body); + const section = getSectionForDiffLine(diffAndRevision.body, + firstDiffLine); const newTalkPageTopicOutputObject = new NewTalkPageTopicExtended(revision.revid, - revision.timestamp, revision.user, revision.userid, largestDiffLine.text, - largestDiffLine.type, largestDiffLine.highlightRanges, + revision.timestamp, revision.user, revision.userid, firstDiffLine.text, + firstDiffLine.type, firstDiffLine.highlightRanges, diffAndRevision.characterChangeWithSections.counts, section); uncachedOutput.push(newTalkPageTopicOutputObject); }); @@ -924,7 +1021,6 @@ function getSignificantEvents(req, res) { null); preformattedSnippets.push(snippet); } else if (item.outputType === 'large-change') { - for (let i = 0; i < item.significantChanges.length; i++) { const significantChange = item.significantChanges[i]; if (significantChange.outputType === 'added-text') { From 18e46b7538dc1b52098de50aa4aef5560d72f170 Mon Sep 17 00:00:00 2001 From: Toni Sevener Date: Mon, 17 Aug 2020 16:45:38 -0500 Subject: [PATCH 19/47] add article description template detection, editor counts, editor groups, sha --- lib/mwapi.js | 22 +++++ routes/page/significant-events.js | 129 +++++++++++++++++++++++------- 2 files changed, 123 insertions(+), 28 deletions(-) diff --git a/lib/mwapi.js b/lib/mwapi.js index a7f71d51..0a465757 100644 --- a/lib/mwapi.js +++ b/lib/mwapi.js @@ -151,6 +151,28 @@ mwapi.queryForRevisions = function(req, title, pageSize, rvStart, rvEnd) { return api.mwApiGet(req, query); }; +mwapi.queryForUsers = function(req, userids) { + + var useridstring = ''; + for (let i = 0; i < userids.length; i++) { + const userid = userids[i]; + if (i === 0) { + useridstring += userid; + } else { + useridstring += `|${userid}`; + } + } + + const query = apiParams({ + action: 'query', + list: 'users', + ususerids: useridstring, + usprop: 'groups|editcount', + }); + + return api.mwApiGet(req, query); +}; + /** * Given protection status for an article simplify it to allow easy reference * @param {!Array} mwApiProtectionObj e.g. diff --git a/routes/page/significant-events.js b/routes/page/significant-events.js index b7b51e02..8c5066ab 100644 --- a/routes/page/significant-events.js +++ b/routes/page/significant-events.js @@ -9,6 +9,7 @@ const parsoidApi = require('../../lib/parsoid-access'); const encoding = require('@root/encoding'); const ParsoidJS = require('parsoid-jsapi'); const PRFunPromise = require('prfun'); +const tUtil = require('../../lib/talk/TalkPageTopicUtilities'); let app; const significantChangesCache = {}; @@ -408,9 +409,8 @@ const talkPageRevisionsPromise = (req, rvStart, rvEnd) => { }; const significantChangesCacheKey = (req, title, revision) => { - const threshold = getThreshold(req); const keyTitle = title || req.params.title; - return `${req.params.domain}-${keyTitle}-${revision.revid}-${threshold}`; + return `${req.params.domain}-${keyTitle}-${revision.revid}`; }; function getCachedAndUncachedItems(revisions, req, title) { @@ -602,19 +602,12 @@ function updateDiffAndRevisionsWithCharacterCount(diffAndRevisions) { } function templateNamesToCallOut() { - return ['cite']; + return ['cite', 'citation', 'short description']; } -function needsToParseForAddedTemplates(text, includeOpeningBraces, includeAll) { +function needsToParseForAddedTemplates(text, includeOpeningBraces) { const names = templateNamesToCallOut(); - if (includeAll) { - if ((text.includes('{{') && includeOpeningBraces) || (!includeOpeningBraces)) { - return true; - } - return false; - } - for (var n = 0; n < names.length; n++) { const name = names[n]; @@ -631,7 +624,7 @@ function needsToParseForAddedTemplates(text, includeOpeningBraces, includeAll) { // &diff=965295364&oldid=965071033 // We are missing some an added reference from line 722. We also aren't // catching the tags in line 579 -function structuredTemplatePromise(text, diff, revision, includeAll) { +function structuredTemplatePromise(text, diff, revision) { return new BBPromise((resolve) => { var main = PRFunPromise.async(function*() { var pdoc = yield ParsoidJS.parse(text, { pdoc: true }); @@ -647,8 +640,7 @@ function structuredTemplatePromise(text, diff, revision, includeAll) { for (var i = 0; i < individualTemplates.length; i++) { const template = individualTemplates[i]; - if (!needsToParseForAddedTemplates(template.name, false, - includeAll)) { + if (!needsToParseForAddedTemplates(template.name, false)) { continue; } @@ -674,15 +666,14 @@ function structuredTemplatePromise(text, diff, revision, includeAll) { }); } -function addStructuredTemplates(diffAndRevisions, includeAll) { +function addStructuredTemplates(diffAndRevisions) { var promises = []; diffAndRevisions.forEach(function (diffAndRevision) { diffAndRevision.body.diff.forEach(function (diff) { switch (diff.type) { case 1: // Add complete line type - if (needsToParseForAddedTemplates(diff.text, true, - includeAll)) { + if (needsToParseForAddedTemplates(diff.text, true)) { promises.push(structuredTemplatePromise(diff.text, diff, diffAndRevision.revision)); } @@ -698,10 +689,9 @@ function addStructuredTemplates(diffAndRevisions, includeAll) { switch (range.type) { case 0: // Add range type - if (needsToParseForAddedTemplates(rangeText, true, - includeAll)) { + if (needsToParseForAddedTemplates(rangeText, true)) { promises.push(structuredTemplatePromise(rangeText, diff, - diffAndRevision.revision, includeAll)); + diffAndRevision.revision)); } break; default: @@ -813,14 +803,17 @@ function getFirstDiffLineWithContent(diffBody) { return null; } -function cleanOutput(output) { - - // collapses small changes, converts large changes only to info needed +function sortOutput(output) { - // sort by date first - output = output.sort(function(a, b) { + // sort by date + return output.sort(function(a, b) { return new Date(b.timestamp) - new Date(a.timestamp); }); +} + +function cleanOutput(output) { + + // collapses small changes, converts large changes only to info needed const cleanedOutput = []; let numSmallChanges = 0; @@ -854,6 +847,75 @@ function cleanOutput(output) { return cleanedOutput; } +function editCountsAndGroupsPromise(req, cleanedOutput) { + + // gather unique userids + var userids = []; + cleanedOutput.forEach( (outputItem) => { + if (outputItem.userid !== undefined && outputItem.userid !== null && + outputItem.userid !== 0) { + userids.push(outputItem.userid); + } + }); + var dedupedIdsSet = new Set(userids); + var dedupedIds = Array.from(dedupedIdsSet); + // fetch counts and groups + return mwapi.queryForUsers(req, dedupedIds) + .then( (response) => { + // distribute results back into cleanedOutput + const users = response.body.query.users; + users.forEach( (user) => { + cleanedOutput.forEach( (outputItem) => { + if (outputItem.userid === user.userid) { + outputItem.userGroups = user.groups; + outputItem.userEditCount = user.editcount; + } + }); + }); + + return cleanedOutput; + }); +} + +function shaFromSortedOutput(req, sortedOutput) { + var shaTitle; + var shaRevID; + for (var i = 0; i < sortedOutput.length; i++) { + const output = sortedOutput[i]; + if (output.outputType === 'large-change') { + shaTitle = req.params.title; + shaRevID = output.revid; + break; + } + + if (output.outputType === 'new-talk-page-topic') { + shaTitle = talkPageTitle(req); + shaRevID = output.revid; + break; + } + } + + if (shaTitle === undefined || shaTitle === null || shaRevID === undefined || + shaRevID === null) { + // grab first item, even if it's a small type + for (var s = 0; s < sortedOutput.length; s++) { + const output = sortedOutput[s]; + if (output.outputType === 'small-change') { + shaTitle = req.params.title; + shaRevID = output.revid; + break; + } + } + } + + if (shaTitle === undefined || shaTitle === null || shaRevID === undefined || + shaRevID === null) { + return null; + } + + return tUtil.createSha256(`${shaTitle}${shaRevID}`); +} + function getSignificantEvents(req, res) { // STEP 1: Gather list of article revisions @@ -1084,11 +1146,22 @@ function getSignificantEvents(req, res) { }) .then( (response) => { - const cleanedOutput = cleanOutput(response.finalOutput); + const sortedOutput = sortOutput((response.finalOutput)); + const cleanedOutput = cleanOutput(sortedOutput); + const sha = shaFromSortedOutput(req, sortedOutput); + + return editCountsAndGroupsPromise(req, cleanedOutput) + .then( (editCountsAndGroupsResponse) => { + return Object.assign({ nextRvStartId: response.nextRvStartId, + cleanedOutput: editCountsAndGroupsResponse, sha: sha }); + }); + }) + .then( (response) => { + const result = Object.assign({ nextRvStartId: response.nextRvStartId, - timeline: cleanedOutput } ); + sha: response.sha, + timeline: response.cleanedOutput } ); res.send(result).end(); - }); } From a27092483a1e8b87a3756f60ffeb20b60cb72f78 Mon Sep 17 00:00:00 2001 From: Toni Sevener Date: Tue, 18 Aug 2020 00:22:54 -0500 Subject: [PATCH 20/47] attempt at reworking cache, still needs testing --- routes/page/significant-events.js | 273 +++++++++++++++++++++++++++--- 1 file changed, 250 insertions(+), 23 deletions(-) diff --git a/routes/page/significant-events.js b/routes/page/significant-events.js index 8c5066ab..95c0705e 100644 --- a/routes/page/significant-events.js +++ b/routes/page/significant-events.js @@ -13,6 +13,7 @@ const tUtil = require('../../lib/talk/TalkPageTopicUtilities'); let app; const significantChangesCache = {}; +const maxAllowedCachedArticleSignificantEvents = 10; class CharacterChangeWithSections { constructor(counts, addedSections, deletedSections) { @@ -408,12 +409,8 @@ const talkPageRevisionsPromise = (req, rvStart, rvEnd) => { return mwapi.queryForRevisions(req, talkPageTitle(req), 100, rvStart, rvEnd ); }; -const significantChangesCacheKey = (req, title, revision) => { - const keyTitle = title || req.params.title; - return `${req.params.domain}-${keyTitle}-${revision.revid}`; -}; - function getCachedAndUncachedItems(revisions, req, title) { + // add cache to output and filter out of processing flow const uncachedRevisions = []; const cachedOutput = []; @@ -425,16 +422,23 @@ function getCachedAndUncachedItems(revisions, req, title) { }); } - revisions.forEach(function (revision) { - - const cacheKey = significantChangesCacheKey(req, title, revision); - const cacheItem = significantChangesCache[cacheKey]; - if (cacheItem) { - cachedOutput.push(cacheItem); - } else { - uncachedRevisions.push(revision); + const keyTitle = title || req.params.title; + for (var i = 0; i < revisions.length; i++) { + const revision = revisions[i]; + const domainDict = significantChangesCache[req.params.domain]; + if (domainDict) { + const titleDict = domainDict[keyTitle]; + if (titleDict) { + const cacheItem = titleDict[revision.revid]; + if (cacheItem) { + cachedOutput.push(cacheItem); + continue; + } + } } - }); + + uncachedRevisions.push(revision); + } return Object.assign({ uncachedRevisions: uncachedRevisions, @@ -442,6 +446,184 @@ function getCachedAndUncachedItems(revisions, req, title) { }); } +function outputTypeCountsTowardsCache(outputType) { + return outputType === 'large-change' || outputType === 'vandalism-revert'; +} + +function calculateCacheForTitleIsMaxedOut(titleDict) { + // eslint-disable-next-line no-restricted-properties + const titleArray = Object.values(titleDict); + const filteredObjects = titleArray.filter(outputObject => + outputTypeCountsTowardsCache(outputObject.outputType)); + return filteredObjects.length >= maxAllowedCachedArticleSignificantEvents; +} + +function cacheForTitleIsMaxedOut(req, title) { + const keyTitle = title || req.params.title; + var domainDict = significantChangesCache[req.params.domain]; + if (domainDict) { + var titleDict = domainDict[keyTitle]; + if (titleDict) { + if (titleDict.maxedOut) { + return true; + } + } + } + + return false; +} + +function latestAndEarliestCachedRevisionTimestamp(req, title) { + var domainDict = significantChangesCache[req.params.domain]; + const keyTitle = title || req.params.title; + if (domainDict) { + var titleDict = domainDict[keyTitle]; + if (titleDict) { + // eslint-disable-next-line no-restricted-properties + const titleArray = Object.values(titleDict); + const sortedTitleObjects = titleArray.sort(function (a, b) { + return new Date(b.timestamp) - new Date(a.timestamp); + }); + var latestTimestamp = null; + var earliestTimestamp = null; + if (sortedTitleObjects.length > 0) { + latestTimestamp = sortedTitleObjects[0].timestamp; + earliestTimestamp = sortedTitleObjects[sortedTitleObjects.length - 1].timestamp; + } + + return Object.assign( { + latestTimestamp: latestTimestamp, + earliestTimestamp: earliestTimestamp + }); + } + } +} + +function setSignificantChangesCache(req, title, item) { + var domainDict = significantChangesCache[req.params.domain]; + const keyTitle = title || req.params.title; + if (domainDict) { + var titleDict = domainDict[keyTitle]; + if (titleDict) { + titleDict[item.revid] = item; + titleDict.maxedOut = calculateCacheForTitleIsMaxedOut(titleDict); + } else { + titleDict = {}; + titleDict[item.revid] = item; + titleDict.maxedOut = calculateCacheForTitleIsMaxedOut(titleDict); + domainDict[keyTitle] = titleDict; + } + } else { + titleDict = {}; + titleDict[item.revid] = item; + titleDict.maxedOut = calculateCacheForTitleIsMaxedOut(titleDict); + domainDict = {}; + domainDict[keyTitle] = titleDict; + significantChangesCache[req.params.domain] = domainDict; + } +} + +function cleanupCache(req) { + + // cleanup article cache + var domainDict = significantChangesCache[req.params.domain]; + const articleTitle = req.params.title; + if (domainDict) { + var titleDict = domainDict[articleTitle]; + if (titleDict) { + // eslint-disable-next-line no-restricted-properties + const titleArray = Object.values(titleDict); + const sortedTitleArray = titleArray.sort(function(a, b) { + return new Date(b.timestamp) - new Date(a.timestamp); + }); + const significantCachedObjects = sortedTitleArray.filter(outputObject => + outputTypeCountsTowardsCache(outputObject.outputType)); + const delta = significantCachedObjects.length - + maxAllowedCachedArticleSignificantEvents; + + var timestampCutoff = null; + for (var i = significantCachedObjects.length - 1; + i > significantCachedObjects.length - delta; i--) { + const significantCachedObject = significantCachedObjects[i]; + timestampCutoff = significantCachedObject.timestamp; + } + + // clean out all cached article and cached talk page revisions + // after the 100th significant event + if (timestampCutoff) { + const cutoffDate = new Date(timestampCutoff); + + // clean out from article cache + for (var s = sortedTitleArray.length - 1; s >= 0; s--) { + const sortedObjectToConsider = sortedTitleArray[s]; + const objectDate = new Date(sortedObjectToConsider.timestamp); + if (objectDate < cutoffDate) { + titleDict.delete(sortedObjectToConsider.revid); + } else { + break; + } + } + + // clean out from talk page cache + var talkPageTitle = talkPageTitle(req); + var talkPageTitleDict = domainDict[talkPageTitle]; + if (talkPageTitleDict) { + // eslint-disable-next-line no-restricted-properties + const talkPageTitleArray = Object.values(talkPageTitleDict); + const sortedTalkPageTitleArray = talkPageTitleArray.sort(function(a, b) { + return new Date(b.timestamp) - new Date(a.timestamp); + }); + + for (var t = sortedTalkPageTitleArray.length - 1; t >= 0; t--) { + const sortedTalkObjectToConsider = sortedTalkPageTitleArray[t]; + const talkObjectDate = new Date(sortedTalkObjectToConsider.timestamp); + if (talkObjectDate < cutoffDate) { + talkPageTitleDict.delete(sortedTalkObjectToConsider.revid); + } else { + break; + } + } + } + } + } + } +} + +function getSummaryText(req) { + var domainDict = significantChangesCache[req.params.domain]; + const articleTitle = req.params.title; + if (domainDict) { + var titleDict = domainDict[articleTitle]; + if (titleDict) { + // eslint-disable-next-line no-restricted-properties + const titleArray = Object.values(titleDict); + const sortedTitleArray = titleArray.sort(function (a, b) { + return new Date(b.timestamp) - new Date(a.timestamp); + }); + const earliestTimestamp = sortedTitleArray[sortedTitleArray.length - 1].timestamp; + var userids = []; + for (var i = 0; i <= titleArray.length - 1; i++) { + const object = titleArray[i]; + userids.push(object.userid); + } + var dedupedUserIds = new Set(userids); + const numUsers = dedupedUserIds.size; + + return Object.assign({ + earliestTimestamp: earliestTimestamp, + dedupedUserIds: dedupedUserIds, + numUsers: numUsers + }); + } + } + + return Object.assign( { + earliestTimestamp: null, + dedupedUserIds: null, + numUsers: null + }); +} + function getSectionForDiffLine(diffBody, diffLine) { var fromSection = null; @@ -925,27 +1107,59 @@ function getSignificantEvents(req, res) { // STEP 2: All at once gather diffs for each uncached revision and list of // talk page revisions const revisions = response.body.query.pages[0].revisions; + + // BEGIN: PAGE CUTOFF LOGIC + // page cutoff: if cache is maxed out, filter out any revisions + // that occur before the cutoff + const cutoff = latestAndEarliestCachedRevisionTimestamp(req); + var filteredRevisions; + const isMaxedOut = cacheForTitleIsMaxedOut(req); + if (isMaxedOut && cutoff.earliestTimestamp) { + filteredRevisions = revisions.filter(revision => new Date(revision.timestamp) > + new Date(cutoff.earliestTimestamp)); + } else { + filteredRevisions = revisions; + } + + // if cache is maxed out and filteredRevisions contains + // revisions from after the latest cached revision, + // propagate flag for needing cleanup later + var needsCacheCleanup = false; + if (isMaxedOut) { + for (var i = 0; i < filteredRevisions.length; i++) { + const revision = filteredRevisions[i]; + const timestamp = new Date(revision.timestamp); + const latestCacheTimestamp = new Date(cutoff.latestTimestamp); + if (timestamp > latestCacheTimestamp) { + needsCacheCleanup = true; + break; + } + } + } + // END: PAGE CUTOFF LOGIC + // todo: length error checking here, parentid check here - const nextRvStartId = revisions[revisions.length - 1].parentid; + const nextRvStartId = filteredRevisions[filteredRevisions.length - 1].parentid; - const articleEvalResults = getCachedAndUncachedItems(revisions, req, null); + const articleEvalResults = getCachedAndUncachedItems(filteredRevisions, req, null); // save cached article revisions to finalOutput const finalOutput = articleEvalResults.cachedOutput; const rvStart = (req.query.rvstartid !== undefined && req.query.rvstartid !== null) ? - revisions[0].timestamp : null; + filteredRevisions[0].timestamp : null; // if rvstartid is missing from query, they are fetching the first page // if they are fetching the first page, we don't want to block of // talk page revision fetching at the start, in case talk page topics came in // after the latest article revision - const rvEnd = revisions[revisions.length - 1].timestamp; + const rvEnd = filteredRevisions[filteredRevisions.length - 1].timestamp; return BBPromise.props({ articleDiffAndRevisions: diffAndRevisionPromises(req, articleEvalResults.uncachedRevisions), talkPageRevisions: talkPageRevisionsPromise(req, rvStart, rvEnd), nextRvStartId: nextRvStartId, + needsCacheCleanup: needsCacheCleanup, finalOutput: finalOutput }); }) @@ -956,6 +1170,7 @@ function getSignificantEvents(req, res) { const talkPageRevisions = response.talkPageRevisions.body.query.pages[0].revisions; const articleDiffAndRevisions = response.articleDiffAndRevisions; const nextRvStartId = response.nextRvStartId; + const needsCacheCleanup = response.needsCacheCleanup; const talkPageEvalResults = getCachedAndUncachedItems(talkPageRevisions, req, talkPageTitle(req)); @@ -970,6 +1185,7 @@ function getSignificantEvents(req, res) { articleDiffAndRevisions: articleDiffAndRevisions, talkDiffAndRevisions: response, nextRvStartId: nextRvStartId, + needsCacheCleanup: needsCacheCleanup, finalOutput: finalOutput }); }); @@ -1071,7 +1287,9 @@ function getSignificantEvents(req, res) { }); return Object.assign({ nextRvStartId: response.nextRvStartId, - uncachedOutput: uncachedOutput, finalOutput: response.finalOutput } ); + needsCacheCleanup: response.needsCacheCleanup, + uncachedOutput: uncachedOutput, + finalOutput: response.finalOutput } ); }) .then( (response) => { @@ -1133,14 +1351,14 @@ function getSignificantEvents(req, res) { var cacheKey; if (item.outputType === 'new-talk-page-topic') { const title = talkPageTitle(req); - cacheKey = significantChangesCacheKey(req, title, item); + setSignificantChangesCache(req, title, item); } else { - cacheKey = significantChangesCacheKey(req, null, item); + setSignificantChangesCache(req, null, item); } - significantChangesCache[cacheKey] = item; }); return Object.assign({ nextRvStartId: response.nextRvStartId, + needsCacheCleanup: response.needsCacheCleanup, finalOutput: response.finalOutput } ); }); }) @@ -1153,14 +1371,23 @@ function getSignificantEvents(req, res) { return editCountsAndGroupsPromise(req, cleanedOutput) .then( (editCountsAndGroupsResponse) => { return Object.assign({ nextRvStartId: response.nextRvStartId, + needsCacheCleanup: response.needsCacheCleanup, cleanedOutput: editCountsAndGroupsResponse, sha: sha }); }); }) .then( (response) => { + if (response.needsCacheCleanup) { + cleanupCache(req); + } + + const summary = getSummaryText(req); + const result = Object.assign({ nextRvStartId: response.nextRvStartId, sha: response.sha, - timeline: response.cleanedOutput } ); + timeline: response.cleanedOutput, + cache: significantChangesCache, + summary: summary }); res.send(result).end(); }); } From 8f2fe8dfe8c5420f87e03782b5c232b378a5ca7e Mon Sep 17 00:00:00 2001 From: Toni Sevener Date: Tue, 18 Aug 2020 20:10:43 -0500 Subject: [PATCH 21/47] stripping additional tags from snippets, inspired by talk pages --- lib/snippet/Snippet.js | 102 +++++++++++++++++++++++ routes/page/significant-events.js | 132 ++++++++++++++++++++++++------ 2 files changed, 209 insertions(+), 25 deletions(-) create mode 100644 lib/snippet/Snippet.js diff --git a/lib/snippet/Snippet.js b/lib/snippet/Snippet.js new file mode 100644 index 00000000..9434ce99 --- /dev/null +++ b/lib/snippet/Snippet.js @@ -0,0 +1,102 @@ +// const P = require('bluebird'); +// const Chunk = require('../html/Chunk'); +// const DocumentWorker = require('../html/DocumentWorker'); +// const tagsToRemove = new Set(['STYLE', 'SCRIPT']); +// const sectionTagNames = new Set(['SECTION']); +// /** +// * TalkPage represents a structured version of a talk page. +// * @param {!Document} doc Parsoid document +// * @param {!String} lang the language of the document +// * @param {?boolean} immediate whether or not to process the document immediately +// */ +// class Snippet extends DocumentWorker { +// constructor(doc, immediate = true) { +// super(doc, doc.body); +// this.chunks = []; +// this.firstSectionNode = undefined; +// this.firstDivNode = undefined; +// this.firstParagraphNode = undefined; +// if (!immediate) { +// return; +// } +// this.workSync(); +// this.finalizeSync(); +// } +// +// /** +// * Returns a promise that is fufilled by a TalkPage +// * @param {!Document} doc Parsoid document +// * @param {!String} lang the language of the document +// * @param {?integer} limit the limit in ms for each processing chunk +// */ +// static promise(doc) { +// const snippet = new Snippet(doc, false); +// return snippet.promise; +// } +// +// removeNodeButPreserveContents(node) { +// while (node.childNodes.length > 0) { +// const firstChild = node.firstChild; +// if (firstChild) { +// node.parentNode.insertBefore(firstChild, node); +// } +// } +// node.parentNode.removeChild(node); +// } +// /** +// * Process the document +// * @param {?integer} limit the max number of DOMNodes to process +// */ +// process(node) { +// +// if (tagsToRemove.has(node.tagName)) { +// node.parentNode.removeChild(node); +// return; +// } +// +// if (sectionTagNames.has(node.tagName) && this.firstSectionNode === undefined) { +// this.firstSectionNode = node; +// removeNodeButPreserveContents(node); +// return; +// } +// +// if (node.tagName === 'DIV' && this.firstDivNode === undefined) { +// this.firstDivNode = node; +// removeNodeButPreserveContents(node); +// return; +// } +// +// if (node.tagName === 'P' && this.firstParagraphNode === undefined) { +// this.firstParagraphNode = node; +// removeNodeButPreserveContents(node); +// return; +// } +// +// console.log(node.tagName); +// console.log(doc.body.innerHTML); +// // if (tagsToRemove.has(node.tagName)) { +// // node.remove(); +// // return; +// // } +// // const chunk = new Chunk(node, false); +// // +// // while (this.ancestor && this.ancestor !== node.parentNode) { +// // const endChunk = new Chunk(this.ancestor, true); +// // this.chunks.push(endChunk); +// // this.ancestor = this.ancestor.parentNode; +// // } +// // +// // if (chunk.isTag) { +// // this.ancestor = node; +// // this.chunks.push(chunk); +// // } else if (chunk.isText) { +// // this.chunks.push(chunk); +// // } +// } +// +// finalizeStep() { +// return false; +// } +// } +// +// module.exports = Snippet; diff --git a/routes/page/significant-events.js b/routes/page/significant-events.js index 95c0705e..be438fe6 100644 --- a/routes/page/significant-events.js +++ b/routes/page/significant-events.js @@ -10,11 +10,44 @@ const encoding = require('@root/encoding'); const ParsoidJS = require('parsoid-jsapi'); const PRFunPromise = require('prfun'); const tUtil = require('../../lib/talk/TalkPageTopicUtilities'); +const Snippet = require('../../lib/snippet/Snippet'); +const NodeType = require('../../lib/nodeType'); let app; const significantChangesCache = {}; const maxAllowedCachedArticleSignificantEvents = 10; +const tagReplacements = { + A: 'a', + B: 'b', + I: 'i', + SUP: 'sup', + SUB: 'sub', + DT: 'b', + CODE: 'b', + BIG: 'b', + LI: 'li', + OL: 'ol', + UL: 'ul', + DL: 'ul', + DD: 'li' +}; + +const tagsToRemoveEntirely = 'script,style'; +const tagsToRemoveButKeepContents = new Set(['P', 'DIV', 'SPAN']); +const escapeRegex = /[<>]/g; +const escapeMap = { + '<': '<', + '>': '>', + '&': '&' +}; + +const escape = text => { + return text.replace(escapeRegex, (match) => { + return escapeMap[match] || ''; + }); +}; + class CharacterChangeWithSections { constructor(counts, addedSections, deletedSections) { this.counts = counts; @@ -191,6 +224,56 @@ const diffAndRevisionPromise = (req, revision) => { }); }; +function removeNodeButPreserveContents(node) { + while (node.childNodes.length > 0) { + const firstChild = node.firstChild; + if (firstChild) { + node.parentNode.insertBefore(firstChild, node); + } + } + node.parentNode.removeChild(node); +} + +function renameNode(doc, node, newNodeName) { + + // todo: I think leaving this commented out will clean out attributes. + // Not sure if this is good or bad. + // if (node.tagName.toLowerCase() === newNodeName.toLowerCase()) { + // return; + // } + + var newNode = doc.createElement(newNodeName); + newNode.innerHTML = node.innerHTML; + node.parentNode.replaceChild(newNode, node); +} + +function recursivelyEvaluateNode(doc, node) { + + if (node.nodeType === NodeType.ELEMENT_NODE) { + const sub = tagReplacements[node.tagName]; + + if (!sub) { + if (tagsToRemoveButKeepContents.has(node.tagName)) { + removeNodeButPreserveContents(node); + } else { + if (node.tagName !== 'BODY') { + node.parentNode.removeChild(node); + } + } + } else { + renameNode(doc, node, sub); + } + } else if (node.nodeType === NodeType.TEXT_NODE) { + node.textContent = escape(node.textContent); + } + + if (node.childElementCount > 0) { + Array.from(node.children).forEach(child => { + recursivelyEvaluateNode(doc, child); + }); + } +} + const formattedSnippetFromTextPromise = (req, text) => { const headers = Object.assign( { @@ -215,15 +298,14 @@ const formattedSnippetFromTextPromise = (req, text) => { return req.issueRequest(request) .then( (response) => { return mUtil.createDocument(response.body); - }).then((response) => { + }).then( (response) => { - // removing script tags here - // todo: try to remove first section that is inserted too - const scripts = response.body.getElementsByTagName('script'); - const scriptsList = Array.prototype.slice.call(scripts); + // removing tags here + const elementsToRemove = response.querySelectorAll(tagsToRemoveEntirely); + const elementsToRemoveList = Array.prototype.slice.call(elementsToRemove); const references = response.body.getElementsByClassName('mw-references-wrap'); const referencesList = Array.prototype.slice.call(references); - const finalListToStrip = scriptsList.concat(referencesList); + const finalListToStrip = elementsToRemoveList.concat(referencesList); finalListToStrip.forEach( (script) => { script.parentNode.removeChild(script); }); @@ -231,32 +313,31 @@ const formattedSnippetFromTextPromise = (req, text) => { // mobile-html endpoint seems to return a wrapper pcs, section and paragraph element. // strip these out as well. - var strippedSnippet = response.body.innerHTML; const pcsElement = response.getElementById('pcs'); if (pcsElement) { - const sectionWrapper = pcsElement.firstChild; - if (sectionWrapper.tagName === 'SECTION') { - strippedSnippet = sectionWrapper.innerHTML; + const section = pcsElement.firstChild; + if (section.tagName === 'SECTION') { + removeNodeButPreserveContents(section); } - const paragraphWrapper = sectionWrapper.firstChild; - if (paragraphWrapper && paragraphWrapper.tagName === 'P') { - - var parent = paragraphWrapper.parentNode; - while ( paragraphWrapper.firstChild ) { - parent.insertBefore( paragraphWrapper.firstChild, paragraphWrapper ); - } - parent.removeChild( paragraphWrapper ); - - strippedSnippet = sectionWrapper.innerHTML; + const paragraph = pcsElement.firstChild; + if (paragraph && paragraph.tagName === 'P') { + removeNodeButPreserveContents(paragraph); } - - // todo: remove monte tags from sectionWrapper } else { - // todo: remove monte tags from body + // todo: bail with empty snippet. } - return strippedSnippet; + removeNodeButPreserveContents(pcsElement); + + // now walk remaining doc elements and strip unsupported elements + + recursivelyEvaluateNode(response, response.body); + + return response; + }) + .then((response) => { + return response.body.innerHTML; }); }; @@ -384,6 +465,7 @@ const snippetPromise = (req, preformattedSnippet) => { truncatedSnippet = '...'.concat(truncatedSnippet).concat('...'); } } + truncatedSnippet = truncatedSnippet.replace(/ioshighlightstart/g, '').replace(/ioshighlightend/g, ''); preformattedSnippet.snippet = truncatedSnippet; return preformattedSnippet; }); @@ -1386,7 +1468,7 @@ function getSignificantEvents(req, res) { const result = Object.assign({ nextRvStartId: response.nextRvStartId, sha: response.sha, timeline: response.cleanedOutput, - cache: significantChangesCache, + debugCache: significantChangesCache, summary: summary }); res.send(result).end(); }); From 6b6f7943c1ea5f372592ed35b35c134c6225a7e6 Mon Sep 17 00:00:00 2001 From: Toni Sevener Date: Wed, 19 Aug 2020 16:43:46 -0500 Subject: [PATCH 22/47] bug fixes --- routes/page/significant-events.js | 218 ++++++++++++++++++++---------- 1 file changed, 147 insertions(+), 71 deletions(-) diff --git a/routes/page/significant-events.js b/routes/page/significant-events.js index be438fe6..a28eea14 100644 --- a/routes/page/significant-events.js +++ b/routes/page/significant-events.js @@ -15,7 +15,7 @@ const NodeType = require('../../lib/nodeType'); let app; const significantChangesCache = {}; -const maxAllowedCachedArticleSignificantEvents = 10; +const maxAllowedCachedArticleSignificantEvents = 100; const tagReplacements = { A: 'a', @@ -41,6 +41,15 @@ const escapeMap = { '>': '>', '&': '&' }; +const attributesToRemoveMap = { + style: true, + id: true, + class: true, + rel: true, + about: true, + 'data-mw': true, + typeof: true +}; const escape = text => { return text.replace(escapeRegex, (match) => { @@ -215,6 +224,9 @@ function stringByRemovingSubstring(originalString, beginningIndex, endIndex) { } const diffAndRevisionPromise = (req, revision) => { + if (revision.parentid === 0) { + return null; + } return mwrestapi.queryForDiff(req, revision.parentid, revision.revid) .then( (response) => { return Object.assign({ @@ -234,13 +246,20 @@ function removeNodeButPreserveContents(node) { node.parentNode.removeChild(node); } -function renameNode(doc, node, newNodeName) { +function renameNodeAndClearOutAttributes(doc, node, newNodeName) { - // todo: I think leaving this commented out will clean out attributes. - // Not sure if this is good or bad. - // if (node.tagName.toLowerCase() === newNodeName.toLowerCase()) { - // return; - // } + // first clear out attributes + for (var i = 0; i < node.attributes.length; i++) { + var attrib = node.attributes[i]; + if (attributesToRemoveMap[attrib] === true) { + node.removeAttribute(attrib); + } + } + + // then rename node if necessary + if (node.tagName.toLowerCase() === newNodeName.toLowerCase()) { + return; + } var newNode = doc.createElement(newNodeName); newNode.innerHTML = node.innerHTML; @@ -261,7 +280,7 @@ function recursivelyEvaluateNode(doc, node) { } } } else { - renameNode(doc, node, sub); + renameNodeAndClearOutAttributes(doc, node, sub); } } else if (node.nodeType === NodeType.TEXT_NODE) { node.textContent = escape(node.textContent); @@ -456,15 +475,25 @@ const snippetPromise = (req, preformattedSnippet) => { // focused on area that changed const firstStart = response.indexOf('ioshighlightstart'); const lastStart = response.lastIndexOf('ioshighlightend'); - if (firstStart >= 0 && lastStart >= 0) { + + // If highlighting starts at the beginning of the line, don't + // truncate or prepend ... + // Otherwise truncate and prepend ... + // all lines truncate at the last highlight end line and suffix with ... + // regardless if + // it's the last thing in the snippet. + truncatedSnippet = truncatedSnippet.slice(0, lastStart + 'ioshighlightend'.length); + truncatedSnippet = truncatedSnippet.concat('...'); + if (firstStart > 0) { truncatedSnippet = truncatedSnippet.slice(firstStart); - truncatedSnippet = truncatedSnippet.slice(0, lastStart); - // UNCOMMENT THIS LINE IF HIGHLIGHTING IS NOT WORTH IT - // truncatedSnippet = truncatedSnippet.replace(/ioshighlightstart/g, '') - // .replace(/ioshighlightend/g, ''); - truncatedSnippet = '...'.concat(truncatedSnippet).concat('...'); + truncatedSnippet = '...'.concat(truncatedSnippet); } + + // UNCOMMENT THIS LINE IF HIGHLIGHTING IS NOT WORTH IT + // truncatedSnippet = truncatedSnippet.replace(/ioshighlightstart/g, '') + // .replace(/ioshighlightend/g, ''); } + truncatedSnippet = truncatedSnippet.replace(/ioshighlightstart/g, '').replace(/ioshighlightend/g, ''); preformattedSnippet.snippet = truncatedSnippet; return preformattedSnippet; @@ -483,9 +512,9 @@ const diffAndRevisionPromises = (req, revisions) => { }); }; -const talkPageTitle = (req) => { +function talkPageTitle(req) { return `Talk:${req.params.title}`; -}; +} const talkPageRevisionsPromise = (req, rvStart, rvEnd) => { return mwapi.queryForRevisions(req, talkPageTitle(req), 100, rvStart, rvEnd ); @@ -534,7 +563,8 @@ function outputTypeCountsTowardsCache(outputType) { function calculateCacheForTitleIsMaxedOut(titleDict) { // eslint-disable-next-line no-restricted-properties - const titleArray = Object.values(titleDict); + var titleArray = Object.values(titleDict); + titleArray.pop(); // remove maxedOut element const filteredObjects = titleArray.filter(outputObject => outputTypeCountsTowardsCache(outputObject.outputType)); return filteredObjects.length >= maxAllowedCachedArticleSignificantEvents; @@ -562,7 +592,8 @@ function latestAndEarliestCachedRevisionTimestamp(req, title) { var titleDict = domainDict[keyTitle]; if (titleDict) { // eslint-disable-next-line no-restricted-properties - const titleArray = Object.values(titleDict); + var titleArray = Object.values(titleDict); + titleArray.pop(); // remove maxedOut element const sortedTitleObjects = titleArray.sort(function (a, b) { return new Date(b.timestamp) - new Date(a.timestamp); }); @@ -610,11 +641,13 @@ function cleanupCache(req) { // cleanup article cache var domainDict = significantChangesCache[req.params.domain]; const articleTitle = req.params.title; - if (domainDict) { + if (domainDict) { var titleDict = domainDict[articleTitle]; if (titleDict) { // eslint-disable-next-line no-restricted-properties - const titleArray = Object.values(titleDict); + var titleArray = Object.values(titleDict); + titleArray.pop(); // remove maxedOut element + const sortedTitleArray = titleArray.sort(function(a, b) { return new Date(b.timestamp) - new Date(a.timestamp); }); @@ -624,9 +657,9 @@ function cleanupCache(req) { maxAllowedCachedArticleSignificantEvents; var timestampCutoff = null; - for (var i = significantCachedObjects.length - 1; - i > significantCachedObjects.length - delta; i--) { - const significantCachedObject = significantCachedObjects[i]; + if (delta > 0) { + const significantCachedObject = + significantCachedObjects[significantCachedObjects.length - delta - 1]; timestampCutoff = significantCachedObject.timestamp; } @@ -640,18 +673,20 @@ function cleanupCache(req) { const sortedObjectToConsider = sortedTitleArray[s]; const objectDate = new Date(sortedObjectToConsider.timestamp); if (objectDate < cutoffDate) { - titleDict.delete(sortedObjectToConsider.revid); + delete titleDict[sortedObjectToConsider.revid]; } else { break; } } // clean out from talk page cache - var talkPageTitle = talkPageTitle(req); + // todo: why on earth do we get a talkPageTitle is not a function error here? + var talkPageTitle = `Talk:${req.params.title}`; // talkPageTitle(req); var talkPageTitleDict = domainDict[talkPageTitle]; if (talkPageTitleDict) { // eslint-disable-next-line no-restricted-properties - const talkPageTitleArray = Object.values(talkPageTitleDict); + var talkPageTitleArray = Object.values(talkPageTitleDict); + talkPageTitleArray.pop(); // remove maxedOut element const sortedTalkPageTitleArray = talkPageTitleArray.sort(function(a, b) { return new Date(b.timestamp) - new Date(a.timestamp); }); @@ -660,7 +695,7 @@ function cleanupCache(req) { const sortedTalkObjectToConsider = sortedTalkPageTitleArray[t]; const talkObjectDate = new Date(sortedTalkObjectToConsider.timestamp); if (talkObjectDate < cutoffDate) { - talkPageTitleDict.delete(sortedTalkObjectToConsider.revid); + delete talkPageTitleDict[sortedTalkObjectToConsider.revid]; } else { break; } @@ -678,7 +713,8 @@ function getSummaryText(req) { var titleDict = domainDict[articleTitle]; if (titleDict) { // eslint-disable-next-line no-restricted-properties - const titleArray = Object.values(titleDict); + var titleArray = Object.values(titleDict); + titleArray.pop(); // remove maxedOut element const sortedTitleArray = titleArray.sort(function (a, b) { return new Date(b.timestamp) - new Date(a.timestamp); }); @@ -693,7 +729,7 @@ function getSummaryText(req) { return Object.assign({ earliestTimestamp: earliestTimestamp, - dedupedUserIds: dedupedUserIds, + numChanges: titleArray.length, numUsers: numUsers }); } @@ -1141,7 +1177,16 @@ function editCountsAndGroupsPromise(req, cleanedOutput) { }); } +function isRequestingFirstPage(req) { + return req.query.rvstartid === undefined || req.query.rvstartid === null; +} + function shaFromSortedOutput(req, sortedOutput) { + + if (!isRequestingFirstPage(req)) { + return null; + } + var shaTitle; var shaRevID; for (var i = 0; i < sortedOutput.length; i++) { @@ -1220,26 +1265,42 @@ function getSignificantEvents(req, res) { } // END: PAGE CUTOFF LOGIC - // todo: length error checking here, parentid check here - const nextRvStartId = filteredRevisions[filteredRevisions.length - 1].parentid; + var nextRvStartId; + var talkPageRvStartId; + var talkPageRvEndId; + if (filteredRevisions && filteredRevisions.length > 0) { + const earliestRevision = filteredRevisions[filteredRevisions.length - 1]; + const latestRevision = filteredRevisions[0]; + if (earliestRevision.parentid) { + nextRvStartId = earliestRevision.parentid; + // if rvstartid is missing from query, they are fetching the first page + // if they are fetching the first page, we don't want to block of + // talk page revision fetching at the start, in case talk page topics came in + // after the latest article revision + talkPageRvStartId = isRequestingFirstPage(req) ? null : + latestRevision.timestamp; + talkPageRvEndId = earliestRevision.timestamp; + } + + } else { + nextRvStartId = null; + talkPageRvStartId = null; + talkPageRvEndId = null; + } const articleEvalResults = getCachedAndUncachedItems(filteredRevisions, req, null); // save cached article revisions to finalOutput const finalOutput = articleEvalResults.cachedOutput; - const rvStart = (req.query.rvstartid !== undefined && req.query.rvstartid !== null) ? - filteredRevisions[0].timestamp : null; - // if rvstartid is missing from query, they are fetching the first page - // if they are fetching the first page, we don't want to block of - // talk page revision fetching at the start, in case talk page topics came in - // after the latest article revision - const rvEnd = filteredRevisions[filteredRevisions.length - 1].timestamp; + const talkPageRevisionPromise = (talkPageRvEndId !== null) ? + talkPageRevisionsPromise(req, talkPageRvStartId, talkPageRvEndId) + : null; return BBPromise.props({ articleDiffAndRevisions: diffAndRevisionPromises(req, articleEvalResults.uncachedRevisions), - talkPageRevisions: talkPageRevisionsPromise(req, rvStart, rvEnd), + talkPageRevisions: talkPageRevisionPromise, nextRvStartId: nextRvStartId, needsCacheCleanup: needsCacheCleanup, finalOutput: finalOutput @@ -1249,35 +1310,48 @@ function getSignificantEvents(req, res) { // STEP 3: All at once gather diffs for uncached talk page revisions - const talkPageRevisions = response.talkPageRevisions.body.query.pages[0].revisions; - const articleDiffAndRevisions = response.articleDiffAndRevisions; + const articleDiffAndRevisions = response.articleDiffAndRevisions + .filter(x => x !== null); const nextRvStartId = response.nextRvStartId; const needsCacheCleanup = response.needsCacheCleanup; - const talkPageEvalResults = getCachedAndUncachedItems(talkPageRevisions, - req, talkPageTitle(req)); - - // save cached talk page revisions to finalOutput - const finalOutput = response.finalOutput.concat(talkPageEvalResults.cachedOutput); - - // for each uncached talk page revision, gather diffs - return diffAndRevisionPromises(req, talkPageEvalResults.uncachedRevisions) - .then( (response) => { - return Object.assign({ - articleDiffAndRevisions: articleDiffAndRevisions, - talkDiffAndRevisions: response, - nextRvStartId: nextRvStartId, - needsCacheCleanup: needsCacheCleanup, - finalOutput: finalOutput + if (response.talkPageRevisions) { + const talkPageRevisions = response.talkPageRevisions.body.query.pages[0].revisions; + const talkPageEvalResults = getCachedAndUncachedItems(talkPageRevisions, + req, talkPageTitle(req)); + + // save cached talk page revisions to finalOutput + const finalOutput = response.finalOutput.concat(talkPageEvalResults.cachedOutput); + + // for each uncached talk page revision, gather diffs + return diffAndRevisionPromises(req, talkPageEvalResults.uncachedRevisions) + .then( (response) => { + return Object.assign({ + articleDiffAndRevisions: articleDiffAndRevisions, + talkDiffAndRevisions: response.filter(x => x !== null), + nextRvStartId: nextRvStartId, + needsCacheCleanup: needsCacheCleanup, + finalOutput: finalOutput + }); }); + } else { + return Object.assign({ + articleDiffAndRevisions: articleDiffAndRevisions, + talkDiffAndRevisions: null, + nextRvStartId: nextRvStartId, + needsCacheCleanup: needsCacheCleanup, + finalOutput: response.finalOutput }); + } }) .then( (response) => { // Determine character size of change for every diff line and aggregate // for every revision updateDiffAndRevisionsWithCharacterCount(response.articleDiffAndRevisions); - updateDiffAndRevisionsWithCharacterCount(response.talkDiffAndRevisions); + if (response.talkDiffAndRevisions) { + updateDiffAndRevisionsWithCharacterCount(response.talkDiffAndRevisions); + } // Flag added template types return addStructuredTemplates(response.articleDiffAndRevisions, false) @@ -1354,19 +1428,21 @@ function getSignificantEvents(req, res) { }); // get new talk page revisions, add to uncachedOutput. it will be sorted later. - const newTopicDiffAndRevisions = getNewTopicDiffAndRevisions( - response.talkDiffAndRevisions); - newTopicDiffAndRevisions.forEach(function (diffAndRevision) { - const revision = diffAndRevision.revision; - const firstDiffLine = getFirstDiffLineWithContent(diffAndRevision.body); - const section = getSectionForDiffLine(diffAndRevision.body, - firstDiffLine); - const newTalkPageTopicOutputObject = new NewTalkPageTopicExtended(revision.revid, - revision.timestamp, revision.user, revision.userid, firstDiffLine.text, - firstDiffLine.type, firstDiffLine.highlightRanges, - diffAndRevision.characterChangeWithSections.counts, section); - uncachedOutput.push(newTalkPageTopicOutputObject); - }); + if (response.talkDiffAndRevisions) { + const newTopicDiffAndRevisions = getNewTopicDiffAndRevisions( + response.talkDiffAndRevisions); + newTopicDiffAndRevisions.forEach(function (diffAndRevision) { + const revision = diffAndRevision.revision; + const firstDiffLine = getFirstDiffLineWithContent(diffAndRevision.body); + const section = getSectionForDiffLine(diffAndRevision.body, + firstDiffLine); + const newTalkPageTopicOutputObject = new NewTalkPageTopicExtended( + revision.revid,revision.timestamp, revision.user, revision.userid, + firstDiffLine.text,firstDiffLine.type, firstDiffLine.highlightRanges, + diffAndRevision.characterChangeWithSections.counts, section); + uncachedOutput.push(newTalkPageTopicOutputObject); + }); + } return Object.assign({ nextRvStartId: response.nextRvStartId, needsCacheCleanup: response.needsCacheCleanup, @@ -1465,10 +1541,10 @@ function getSignificantEvents(req, res) { const summary = getSummaryText(req); + // todo: sometimes nextRvStartId doesn't show at all in response. const result = Object.assign({ nextRvStartId: response.nextRvStartId, sha: response.sha, timeline: response.cleanedOutput, - debugCache: significantChangesCache, summary: summary }); res.send(result).end(); }); From 812ee82f0cbb7a94bc400a1eb684866c12a0c780 Mon Sep 17 00:00:00 2001 From: Toni Sevener Date: Mon, 24 Aug 2020 17:49:57 -0500 Subject: [PATCH 23/47] attempt to add some safety --- routes/page/significant-events.js | 902 +++++++++++++++++++++++------- 1 file changed, 687 insertions(+), 215 deletions(-) diff --git a/routes/page/significant-events.js b/routes/page/significant-events.js index a28eea14..2c51dad2 100644 --- a/routes/page/significant-events.js +++ b/routes/page/significant-events.js @@ -224,8 +224,11 @@ function stringByRemovingSubstring(originalString, beginningIndex, endIndex) { } const diffAndRevisionPromise = (req, revision) => { - if (revision.parentid === 0) { - return null; + if (revision.parentid === null || revision.parentid === undefined || revision.parentid === 0) { + return Object.assign({ + revision: revision, + body: null + }); } return mwrestapi.queryForDiff(req, revision.parentid, revision.revid) .then( (response) => { @@ -233,6 +236,12 @@ const diffAndRevisionPromise = (req, revision) => { revision: revision, body: response.body }); + }) + .catch(e => { + return Object.assign({ + revision: revision, + body: null + }); }); }; @@ -293,6 +302,13 @@ function recursivelyEvaluateNode(doc, node) { } } +class UnexpectedSnippetFormatError extends Error { + constructor(message) { + super(message); + this.name = 'UnexpectedSnippetFormatError'; + } +} + const formattedSnippetFromTextPromise = (req, text) => { const headers = Object.assign( { @@ -344,7 +360,7 @@ const formattedSnippetFromTextPromise = (req, text) => { removeNodeButPreserveContents(paragraph); } } else { - // todo: bail with empty snippet. + throw new UnexpectedSnippetFormatError(); } removeNodeButPreserveContents(pcsElement); @@ -357,12 +373,19 @@ const formattedSnippetFromTextPromise = (req, text) => { }) .then((response) => { return response.body.innerHTML; + }) + .catch( (e) => { + return null; }); }; const snippetPromise = (req, preformattedSnippet) => { - if (preformattedSnippet.snippetType === 2) { + if (preformattedSnippet.snippet === null || + preformattedSnippet.snippet === undefined || + preformattedSnippet.snippetType === null || + preformattedSnippet.snippetType === undefined || + preformattedSnippet.snippetType === 2) { // deleted complete line type. // we shouldn't get here but gracefully return null for snippet without trying to // format @@ -391,6 +414,11 @@ const snippetPromise = (req, preformattedSnippet) => { case 5: case 3: // Added and deleted words in line + if (preformattedSnippet.snippetHighlightRanges === null || + preformattedSnippet.snippetHighlightRanges === undefined) { + break; + } + // first strip deleted text for (var d = preformattedSnippet.snippetHighlightRanges.length - 1; d >= 0; d-- ) { const range = preformattedSnippet.snippetHighlightRanges[d]; @@ -425,13 +453,17 @@ const snippetPromise = (req, preformattedSnippet) => { // filter out deleted ranges // eslint-disable-next-line no-case-declarations const addedHighlightRanges = preformattedSnippet.snippetHighlightRanges - .filter(range => range.type === 0); + .filter(range => range.type !== null && range.type !== undefined && + range.type === 0); // then add added text delimiters var addOffset = 0; addedHighlightRanges.forEach(function (range) { switch (range.type) { case 0: // Added + if (range.start === null || range.start === undefined) { + break; + } snippetBinary = insertSubstringInString(snippetBinary, addHighlightStartBin, range.start + addOffset); addOffset += addHighlightStartBin.length; @@ -468,8 +500,16 @@ const snippetPromise = (req, preformattedSnippet) => { // but if other issues occur, we should remove delimiters. With that we would lose // all possibility of stretch goal highlighting // and truncation + + if (!response) { + preformattedSnippet.snippet = null; + return preformattedSnippet; + } + var truncatedSnippet = response; - if (preformattedSnippet.snippetType === 5 || preformattedSnippet.snippetType === 3) { + if (preformattedSnippet.snippetType !== undefined && + preformattedSnippet.snippetType !== null && + preformattedSnippet.snippetType === 5 || preformattedSnippet.snippetType === 3) { // try to trim the areas before the first ioshighlightstart and after the last // ioshighlight start so snippet is // focused on area that changed @@ -517,7 +557,10 @@ function talkPageTitle(req) { } const talkPageRevisionsPromise = (req, rvStart, rvEnd) => { - return mwapi.queryForRevisions(req, talkPageTitle(req), 100, rvStart, rvEnd ); + return mwapi.queryForRevisions(req, talkPageTitle(req), 100, rvStart, rvEnd ) + .catch(e => { + return null; + }); }; function getCachedAndUncachedItems(revisions, req, title) { @@ -564,10 +607,14 @@ function outputTypeCountsTowardsCache(outputType) { function calculateCacheForTitleIsMaxedOut(titleDict) { // eslint-disable-next-line no-restricted-properties var titleArray = Object.values(titleDict); - titleArray.pop(); // remove maxedOut element - const filteredObjects = titleArray.filter(outputObject => - outputTypeCountsTowardsCache(outputObject.outputType)); - return filteredObjects.length >= maxAllowedCachedArticleSignificantEvents; + if (titleArray.length > 0) { + titleArray.pop(); // remove maxedOut element + const filteredObjects = titleArray.filter(outputObject => + outputTypeCountsTowardsCache(outputObject.outputType)); + return filteredObjects.length >= maxAllowedCachedArticleSignificantEvents; + } + + return false; } function cacheForTitleIsMaxedOut(req, title) { @@ -593,23 +640,27 @@ function latestAndEarliestCachedRevisionTimestamp(req, title) { if (titleDict) { // eslint-disable-next-line no-restricted-properties var titleArray = Object.values(titleDict); - titleArray.pop(); // remove maxedOut element - const sortedTitleObjects = titleArray.sort(function (a, b) { - return new Date(b.timestamp) - new Date(a.timestamp); - }); - var latestTimestamp = null; - var earliestTimestamp = null; - if (sortedTitleObjects.length > 0) { - latestTimestamp = sortedTitleObjects[0].timestamp; - earliestTimestamp = sortedTitleObjects[sortedTitleObjects.length - 1].timestamp; - } + if (titleArray.length > 0) { + titleArray.pop(); // remove maxedOut element + const sortedTitleObjects = titleArray.sort(function (a, b) { + return new Date(b.timestamp) - new Date(a.timestamp); + }); + var latestTimestamp = null; + var earliestTimestamp = null; + if (sortedTitleObjects.length > 0) { + latestTimestamp = sortedTitleObjects[0].timestamp; + earliestTimestamp = sortedTitleObjects[sortedTitleObjects.length - 1].timestamp; + } - return Object.assign( { - latestTimestamp: latestTimestamp, - earliestTimestamp: earliestTimestamp - }); + return Object.assign( { + latestTimestamp: latestTimestamp, + earliestTimestamp: earliestTimestamp + }); + } } } + + return null; } function setSignificantChangesCache(req, title, item) { @@ -646,62 +697,82 @@ function cleanupCache(req) { if (titleDict) { // eslint-disable-next-line no-restricted-properties var titleArray = Object.values(titleDict); - titleArray.pop(); // remove maxedOut element + if (titleArray.length > 0) { + titleArray.pop(); // remove maxedOut element - const sortedTitleArray = titleArray.sort(function(a, b) { - return new Date(b.timestamp) - new Date(a.timestamp); - }); - const significantCachedObjects = sortedTitleArray.filter(outputObject => - outputTypeCountsTowardsCache(outputObject.outputType)); - const delta = significantCachedObjects.length - - maxAllowedCachedArticleSignificantEvents; - - var timestampCutoff = null; - if (delta > 0) { - const significantCachedObject = - significantCachedObjects[significantCachedObjects.length - delta - 1]; - timestampCutoff = significantCachedObject.timestamp; - } - - // clean out all cached article and cached talk page revisions - // after the 100th significant event - if (timestampCutoff) { - const cutoffDate = new Date(timestampCutoff); - - // clean out from article cache - for (var s = sortedTitleArray.length - 1; s >= 0; s--) { - const sortedObjectToConsider = sortedTitleArray[s]; - const objectDate = new Date(sortedObjectToConsider.timestamp); - if (objectDate < cutoffDate) { - delete titleDict[sortedObjectToConsider.revid]; - } else { - break; + const sortedTitleArray = titleArray.sort(function(a, b) { + if (a.timestamp && b.timestamp) { + return new Date(b.timestamp) - new Date(a.timestamp); + } + return 0; + }); + const significantCachedObjects = sortedTitleArray.filter(outputObject => { + if (outputObject.outputType) { + return outputTypeCountsTowardsCache(outputObject.outputType); } + + return false; + }); + + const delta = significantCachedObjects.length - + maxAllowedCachedArticleSignificantEvents; + + var timestampCutoff = null; + if (delta > 0 && delta < significantCachedObjects.length && + significantCachedObjects.length > 0) { + const significantCachedObject = + significantCachedObjects[significantCachedObjects.length - delta - 1]; + timestampCutoff = significantCachedObject.timestamp; } - // clean out from talk page cache - // todo: why on earth do we get a talkPageTitle is not a function error here? - var talkPageTitle = `Talk:${req.params.title}`; // talkPageTitle(req); - var talkPageTitleDict = domainDict[talkPageTitle]; - if (talkPageTitleDict) { - // eslint-disable-next-line no-restricted-properties - var talkPageTitleArray = Object.values(talkPageTitleDict); - talkPageTitleArray.pop(); // remove maxedOut element - const sortedTalkPageTitleArray = talkPageTitleArray.sort(function(a, b) { - return new Date(b.timestamp) - new Date(a.timestamp); - }); + // clean out all cached article and cached talk page revisions + // after the 100th significant event + if (timestampCutoff) { + const cutoffDate = new Date(timestampCutoff); + + // clean out from article cache + for (var s = sortedTitleArray.length - 1; s >= 0; s--) { + const sortedObjectToConsider = sortedTitleArray[s]; + if (sortedObjectToConsider.timestamp !== null && + sortedObjectToConsider.timestamp !== undefined) { + const objectDate = new Date(sortedObjectToConsider.timestamp); + if (objectDate < cutoffDate) { + delete titleDict[sortedObjectToConsider.revid]; + } else { + break; + } + } + } - for (var t = sortedTalkPageTitleArray.length - 1; t >= 0; t--) { - const sortedTalkObjectToConsider = sortedTalkPageTitleArray[t]; - const talkObjectDate = new Date(sortedTalkObjectToConsider.timestamp); - if (talkObjectDate < cutoffDate) { - delete talkPageTitleDict[sortedTalkObjectToConsider.revid]; - } else { - break; + // clean out from talk page cache + // todo: why on earth do we get a talkPageTitle is not a function error here? + var talkPageTitle = `Talk:${req.params.title}`; // talkPageTitle(req); + var talkPageTitleDict = domainDict[talkPageTitle]; + if (talkPageTitleDict) { + // eslint-disable-next-line no-restricted-properties + var talkPageTitleArray = Object.values(talkPageTitleDict); + if (talkPageTitleArray.length > 0) { + talkPageTitleArray.pop(); // remove maxedOut element + const sortedTalkPageTitleArray = + talkPageTitleArray.sort(function(a, b) { + return new Date(b.timestamp) - new Date(a.timestamp); + }); + + for (var t = sortedTalkPageTitleArray.length - 1; t >= 0; t--) { + const sortedTalkObjectToConsider = sortedTalkPageTitleArray[t]; + const talkObjectDate = new + Date(sortedTalkObjectToConsider.timestamp); + if (talkObjectDate < cutoffDate) { + delete talkPageTitleDict[sortedTalkObjectToConsider.revid]; + } else { + break; + } + } } } } } + } } } @@ -714,24 +785,29 @@ function getSummaryText(req) { if (titleDict) { // eslint-disable-next-line no-restricted-properties var titleArray = Object.values(titleDict); - titleArray.pop(); // remove maxedOut element - const sortedTitleArray = titleArray.sort(function (a, b) { - return new Date(b.timestamp) - new Date(a.timestamp); - }); - const earliestTimestamp = sortedTitleArray[sortedTitleArray.length - 1].timestamp; - var userids = []; - for (var i = 0; i <= titleArray.length - 1; i++) { - const object = titleArray[i]; - userids.push(object.userid); - } - var dedupedUserIds = new Set(userids); - const numUsers = dedupedUserIds.size; + if (titleArray.length > 0) { + titleArray.pop(); // remove maxedOut element + const sortedTitleArray = titleArray.sort(function (a, b) { + return new Date(b.timestamp) - new Date(a.timestamp); + }); + const earliestItem = sortedTitleArray[sortedTitleArray.length - 1]; + if (earliestItem.timestamp !== undefined && earliestItem.timestamp !== null) { + const earliestTimestamp = earliestItem.timestamp; + var userids = []; + for (var i = 0; i <= titleArray.length - 1; i++) { + const object = titleArray[i]; + userids.push(object.userid); + } + var dedupedUserIds = new Set(userids); + const numUsers = dedupedUserIds.size; - return Object.assign({ - earliestTimestamp: earliestTimestamp, - numChanges: titleArray.length, - numUsers: numUsers - }); + return Object.assign({ + earliestTimestamp: earliestTimestamp, + numChanges: titleArray.length, + numUsers: numUsers + }); + } + } } } @@ -744,9 +820,24 @@ function getSummaryText(req) { function getSectionForDiffLine(diffBody, diffLine) { + if (!diffBody || !diffLine) { + return null; + } + var fromSection = null; var toSection = null; + // safety + if (!diffBody.from || + !diffBody.from.sections || + !diffBody.to || + !diffBody.to.sections || + !diffLine.offset || + !diffLine.offset.from || + !diffLine.offset.to) { + return null; + } + // capture intro if ((!diffBody.from.sections || diffBody.from.sections.length === 0) && @@ -782,15 +873,20 @@ function getSectionForDiffLine(diffBody, diffLine) { const section = diffBody.from.sections[i]; if (diffLine.offset.from < section.offset && prevSection) { - fromSection = prevSection.heading; - break; + if (prevSection.heading) { + fromSection = prevSection.heading; + break; + } } prevSection = section; } if (!fromSection && diffLine.offset.from > 0) { - fromSection = prevSection.heading; + if (prevSection.heading) { + fromSection = prevSection.heading; + } + } } @@ -800,15 +896,19 @@ function getSectionForDiffLine(diffBody, diffLine) { const section = diffBody.to.sections[i]; if (diffLine.offset.to < section.offset && prevSection) { - toSection = prevSection.heading; - break; + if (prevSection.heading) { + toSection = prevSection.heading; + break; + } } prevSection = section; } if (!toSection && diffLine.offset.to > 0) { - toSection = prevSection.heading; + if (prevSection.heading) { + toSection = prevSection.heading; + } } } @@ -821,10 +921,19 @@ function getSectionForDiffLine(diffBody, diffLine) { function updateDiffAndRevisionsWithCharacterCount(diffAndRevisions) { + if (!diffAndRevisions) { + return; + } + // Loop through diffs, filter out type 0 (context type) and assign byte change properties // to the remaining - diffAndRevisions.forEach(function (diffAndRevision) { + for (var i = 0; i < diffAndRevisions.length; i++) { + const diffAndRevision = diffAndRevisions[i]; + + if (!diffAndRevision.body || !diffAndRevision.body.diff) { + continue; + } var filteredDiffs = []; @@ -832,12 +941,22 @@ function updateDiffAndRevisionsWithCharacterCount(diffAndRevisions) { var aggregateDeletedCount = 0; var aggregateAddedSections = new Set(); var aggregateDeletedSections = new Set(); - diffAndRevision.body.diff.forEach(function (diff) { + + for (var d = 0; d < diffAndRevision.body.diff.length; d++) { + const diff = diffAndRevision.body.diff[d]; + + if (diff.type === undefined || + diff.type === null || + diff.text === undefined || + diff.text === null) { + continue; + } + var lineAddedCount = 0; var lineDeletedCount = 0; switch (diff.type) { case 0: // Context line type - return; + continue; case 1: // Add complete line type lineAddedCount = diff.text.length; break; @@ -848,7 +967,21 @@ function updateDiffAndRevisionsWithCharacterCount(diffAndRevisions) { // ranges in line) // eslint-disable-next-line no-fallthrough case 3: // Change line type (add and deleted ranges in line) - diff.highlightRanges.forEach(function (range) { + + if (diff.highlightRanges === undefined || + diff.highlightRanges === null) { + break; + } + + for (var h = 0; h < diff.highlightRanges.length; h++) { + const range = diff.highlightRanges[h]; + + if (range.start === null || + range.start === undefined || + range.length === null || + range.length === undefined) { + continue; + } const binaryText = encoding.strToBin(diff.text); const binaryRangeText = binaryText.substring(range.start, @@ -866,7 +999,7 @@ function updateDiffAndRevisionsWithCharacterCount(diffAndRevisions) { default: break; } - }); + } break; default: break; @@ -891,14 +1024,14 @@ function updateDiffAndRevisionsWithCharacterCount(diffAndRevisions) { diff.characterChange = new CharacterChange(lineAddedCount, lineDeletedCount); filteredDiffs.push(diff); - }); + } const aggregateCounts = new CharacterChange(aggregateAddedCount, aggregateDeletedCount); diffAndRevision.characterChangeWithSections = new CharacterChangeWithSections( aggregateCounts, Array.from(aggregateAddedSections), Array.from(aggregateDeletedSections)); diffAndRevision.body.diff = filteredDiffs; - }); + } } function templateNamesToCallOut() { @@ -906,6 +1039,14 @@ function templateNamesToCallOut() { } function needsToParseForAddedTemplates(text, includeOpeningBraces) { + + if (text === undefined || + text === null || + includeOpeningBraces === undefined || + includeOpeningBraces === null) { + return false; + } + const names = templateNamesToCallOut(); for (var n = 0; n < names.length; n++) { @@ -924,6 +1065,15 @@ function needsToParseForAddedTemplates(text, includeOpeningBraces) { // &diff=965295364&oldid=965071033 // We are missing some an added reference from line 722. We also aren't // catching the tags in line 579 +// ANOTHER BUG: +// Sometimes new citations have spaces within deemed the same, so only part of it +// is sent in as text whereas we need all of cite. +// See here +// https://en.wikipedia.org/w/index.php?title=United_States&type=revision&diff=971349395&oldid=971332725 +// Also this one +// https://en.wikipedia.org/w/index.php?title=United_States&type=revision&diff=971260075&oldid=971259267 +// Definitely some mangled 'added-text' snippets going on here. +// https://en.wikipedia.org/w/index.php?title=United_States&type=revision&diff=970665187&oldid=970577931 function structuredTemplatePromise(text, diff, revision) { return new BBPromise((resolve) => { var main = PRFunPromise.async(function*() { @@ -937,6 +1087,9 @@ function structuredTemplatePromise(text, diff, revision) { const innerPdoc = yield ParsoidJS.parse(splitTemplateText, { pdoc: true }); const individualTemplates = innerPdoc.filterTemplates(); + if (individualTemplates === undefined || individualTemplates === null) { + throw new Error('Unexpected result from filterTemplates'); + } for (var i = 0; i < individualTemplates.length; i++) { const template = individualTemplates[i]; @@ -944,12 +1097,21 @@ function structuredTemplatePromise(text, diff, revision) { continue; } + if (template.name === undefined || + template.name === null || + template.params === undefined || + template.params === null) { + continue; + } + var dict = {}; dict.name = template.name; for (var p = 0; p < template.params.length; p++) { const param = template.params[p].name; - const value = yield template.get(param).value.toWikitext(); - dict[param] = value; + if (param !== undefined && param !== null) { + const value = yield template.get(param).value.toWikitext(); + dict[param] = value; + } } templateObjects.push(dict); } @@ -967,9 +1129,35 @@ function structuredTemplatePromise(text, diff, revision) { } function addStructuredTemplates(diffAndRevisions) { + var promises = []; - diffAndRevisions.forEach(function (diffAndRevision) { - diffAndRevision.body.diff.forEach(function (diff) { + + if (!diffAndRevisions) { + return Promise.all(promises) + .then( (response) => { + return diffAndRevisions; + }); + } + + for (var i = 0; i < diffAndRevisions.length; i++) { + const diffAndRevision = diffAndRevisions[i]; + + if (diffAndRevision.body === null || + diffAndRevision.body === undefined || + diffAndRevision.body.diff === null || + diffAndRevision.body.diff === undefined || + diffAndRevision.revision === null || + diffAndRevision.revision === undefined) { + continue; + } + + for (var d = 0; d < diffAndRevision.body.diff.length; d++) { + const diff = diffAndRevision.body.diff[d]; + + if (diff.text === undefined || + diff.text === null) { + continue; + } switch (diff.type) { case 1: // Add complete line type @@ -980,7 +1168,16 @@ function addStructuredTemplates(diffAndRevisions) { break; case 5: case 3: - diff.highlightRanges.forEach(function (range) { + + for (var h = 0; h < diff.highlightRanges.length; h++) { + const range = diff.highlightRanges[h]; + + if (range.start === undefined || + range.start === null || + range.length === undefined || + range.length === null) { + continue; + } const binaryText = encoding.strToBin(diff.text); const binaryRangeText = binaryText.substring(range.start, @@ -997,13 +1194,13 @@ function addStructuredTemplates(diffAndRevisions) { default: break; } - }); + } break; default: break; } - }); - }); + } + } return Promise.all(promises) .then( (response) => { @@ -1012,10 +1209,16 @@ function addStructuredTemplates(diffAndRevisions) { response.forEach( (item) => { diffAndRevisions.forEach( (diffAndRevision) => { - if (item.revision.revid === diffAndRevision.revision.revid) { - diffAndRevision.templates = item.templates; - diffAndRevision.templateDiffLine = item.diff; - } + + if (item.revision !== null && + item.revision !== undefined && + diffAndRevision.revision !== null && + diffAndRevision.revision !== undefined) { + if (item.revision.revid === diffAndRevision.revision.revid) { + diffAndRevision.templates = item.templates; + diffAndRevision.templateDiffLine = item.diff; + } + } }); }); @@ -1025,69 +1228,145 @@ function addStructuredTemplates(diffAndRevisions) { function getNewTopicDiffAndRevisions(talkDiffAndRevisions) { + if (talkDiffAndRevisions === null || + talkDiffAndRevisions === undefined) { + return null; + } + const newSectionTalkPageDiffAndRevisions = []; // const talkPage = talkPageBody.query.pages; // const talkPageObject = talkPage[Object.keys(talkPage)[0]]; // const talkPageRevisions = talkPageObject.revisions; - talkDiffAndRevisions.forEach(function (diffAndRevision, index) { - if (diffAndRevision.revision.comment.toLowerCase().includes('new section') - && !diffAndRevision.revision.comment.toLowerCase() - .includes('semi-protected edit request') // don't show semi-protected edit requests - && diffAndRevision.revision.userid !== 4936590) { // don't show signbot topics - // see if this section was reverted in previous iterations - var wasReverted = false; - if (index - 1 > 0) { - var nextDiffAndRevision = talkDiffAndRevisions[index - 1]; - if (nextDiffAndRevision.revision.userid === 4936590) { // signbot, - // look for next revision - if (index - 2 > 0) { - nextDiffAndRevision = talkDiffAndRevisions[index - 2]; - } - } - if (nextDiffAndRevision.revision.tags.includes('mw-undo') || - nextDiffAndRevision.revision.tags.includes('mw-rollback')) { - wasReverted = true; - } - } - if (wasReverted === false) { - newSectionTalkPageDiffAndRevisions.push(diffAndRevision); - } - } - }); + for (var index = 0; index < talkDiffAndRevisions.length; index++) { + const diffAndRevision = talkDiffAndRevisions[index]; + + if (diffAndRevision.revision === null || + diffAndRevision.revision === undefined || + diffAndRevision.revision.comment === null || + diffAndRevision.revision.comment === undefined || + diffAndRevision.revision.userid === null || + diffAndRevision.revision.userid === undefined || + diffAndRevision.revision.tags === null || + diffAndRevision.revision.tags === undefined) { + continue; + } + + if (diffAndRevision.revision.comment.toLowerCase().includes('new section') + && !diffAndRevision.revision.comment.toLowerCase() + .includes('semi-protected edit request') // don't show semi-protected edit requests + && diffAndRevision.revision.userid !== 4936590) { // don't show signbot topics + // see if this section was reverted in previous iterations + var wasReverted = false; + if (index - 1 > 0) { + var nextDiffAndRevision = talkDiffAndRevisions[index - 1]; + if (nextDiffAndRevision.revision.userid === 4936590) { // signbot, + // look for next revision + if (index - 2 > 0) { + nextDiffAndRevision = talkDiffAndRevisions[index - 2]; + } + } + if (nextDiffAndRevision.revision.tags.includes('mw-undo') || + nextDiffAndRevision.revision.tags.includes('mw-rollback')) { + wasReverted = true; + } + } + if (wasReverted === false) { + newSectionTalkPageDiffAndRevisions.push(diffAndRevision); + } + } + } return newSectionTalkPageDiffAndRevisions; } function getLargestDiffLine(diffBody) { + + if (diffBody === null || + diffBody === undefined || + diffBody.diff === null || + diffBody.diff === undefined) { + return null; + } + diffBody.diff.sort(function(a, b) { + + if (b.characterChange === null || + b.characterChange === undefined || + a.characterChange === null || + a.characterChange === undefined) { + return 0; + } + return b.characterChange.totalCount() - a.characterChange.totalCount(); }); - // todo: safety - const largestDiffLine = diffBody.diff[0]; - return largestDiffLine; + if (diffBody.diff.length > 0) { + const largestDiffLine = diffBody.diff[0]; + return largestDiffLine; + } + + return null; } function getLargestDiffLineOfAdded(diffBody) { + + if (diffBody === null || + diffBody === undefined || + diffBody.diff === null || + diffBody.diff === undefined) { + return null; + } + diffBody.diff.sort(function(a, b) { + + if (b.characterChange === null || + b.characterChange === undefined || + a.characterChange === null || + a.characterChange === undefined) { + return 0; + } + return b.characterChange.addedCount - a.characterChange.addedCount; }); - // todo: safety - const largestDiffLine = diffBody.diff[0]; - return largestDiffLine; + if (diffBody.diff.length > 0) { + const largestDiffLine = diffBody.diff[0]; + return largestDiffLine; + } + + return null; } function textContainsEmptyLineOrSection(text) { + + if (text === null || + text === undefined) { + return false; + } + const trimmedText = text.trim(); return (trimmedText.length === 0 || text.includes('==')); } function getFirstDiffLineWithContent(diffBody) { + if (diffBody === null || + diffBody === undefined || + diffBody.diff === null || + diffBody.diff === undefined) { + return null; + } + for (let i = 0; i < diffBody.diff.length; i++) { const diff = diffBody.diff[i]; + if (diff.type === null || + diff.type === undefined || + diff.text === null || + diff.text === undefined) { + continue; + } + switch (diff.type) { case 0: // Context line type continue; @@ -1105,22 +1384,47 @@ function getFirstDiffLineWithContent(diffBody) { function sortOutput(output) { + if (output === null || + output === undefined) { + return null; + } + // sort by date return output.sort(function(a, b) { + + if (b.timestamp === null || + b.timestamp === undefined || + a.timestamp === null || + a.timestamp === undefined) { + return 0; + } + return new Date(b.timestamp) - new Date(a.timestamp); }); } function cleanOutput(output) { + if (output === null || + output === undefined) { + return null; + } + // collapses small changes, converts large changes only to info needed const cleanedOutput = []; let numSmallChanges = 0; - output.forEach(function (item) { + for (var i = 0; i < output.length; i++) { + const item = output[i]; + + if (item.outputType === null || + item.outputType === undefined) { + continue; + } + if (item.outputType === 'small-change') { numSmallChanges++; - return; + continue; } else { if (numSmallChanges > 0) { const change = new ConsolidatedSmallOutput(numSmallChanges); @@ -1136,7 +1440,7 @@ function cleanOutput(output) { cleanedOutput.push(item); } } - }); + } if (numSmallChanges > 0) { const change = new ConsolidatedSmallOutput(numSmallChanges); @@ -1149,6 +1453,11 @@ function cleanOutput(output) { function editCountsAndGroupsPromise(req, cleanedOutput) { + if (cleanedOutput === undefined || + cleanedOutput === null) { + return null; + } + // gather unique userids var userids = []; cleanedOutput.forEach( (outputItem) => { @@ -1163,16 +1472,35 @@ function editCountsAndGroupsPromise(req, cleanedOutput) { return mwapi.queryForUsers(req, dedupedIds) .then( (response) => { // distribute results back into cleanedOutput + + if (response.body === null || + response.body === undefined || + response.body.query === null || + response.body.query === undefined || + response.body.query.users === null || + response.body.query.users === undefined) { + return cleanedOutput; + } + const users = response.body.query.users; users.forEach( (user) => { cleanedOutput.forEach( (outputItem) => { - if (outputItem.userid === user.userid) { - outputItem.userGroups = user.groups; - outputItem.userEditCount = user.editcount; + + if (outputItem.userid !== null && + outputItem.userid !== undefined && + user.userid !== null && + user.userid !== undefined) { + if (outputItem.userid === user.userid) { + outputItem.userGroups = user.groups; + outputItem.userEditCount = user.editcount; + } } }); }); + return cleanedOutput; + }) + .catch( (e) => { return cleanedOutput; }); } @@ -1183,6 +1511,11 @@ function isRequestingFirstPage(req) { function shaFromSortedOutput(req, sortedOutput) { + if (sortedOutput === null || + sortedOutput === undefined) { + return null; + } + if (!isRequestingFirstPage(req)) { return null; } @@ -1191,6 +1524,14 @@ function shaFromSortedOutput(req, sortedOutput) { var shaRevID; for (var i = 0; i < sortedOutput.length; i++) { const output = sortedOutput[i]; + + if (output.outputType === undefined || + output.outputType === null || + output.revid === undefined || + output.revid === null) { + continue; + } + if (output.outputType === 'large-change') { shaTitle = req.params.title; shaRevID = output.revid; @@ -1209,6 +1550,13 @@ function shaFromSortedOutput(req, sortedOutput) { // grab first item, even if it's a small type for (var s = 0; s < sortedOutput.length; s++) { const output = sortedOutput[s]; + if (output.outputType === undefined || + output.outputType === null || + output.revid === undefined || + output.revid === null) { + continue; + } + if (output.outputType === 'small-change') { shaTitle = req.params.title; shaRevID = output.revid; @@ -1225,6 +1573,13 @@ function shaFromSortedOutput(req, sortedOutput) { return tUtil.createSha256(`${shaTitle}${shaRevID}`); } +class MalformedArticleRevisionResponse extends Error { + constructor(message) { + super(message); + this.name = 'MalformedArticleRevisionResponse'; + } +} + function getSignificantEvents(req, res) { // STEP 1: Gather list of article revisions @@ -1233,6 +1588,20 @@ function getSignificantEvents(req, res) { // STEP 2: All at once gather diffs for each uncached revision and list of // talk page revisions + + if (response.body === undefined || + response.body === null || + response.body.query === undefined || + response.body.query === null || + response.body.query.pages === undefined || + response.body.query.pages === null || + response.body.query.pages[0] === undefined || + response.body.query.pages[0] === null || + response.body.query.pages[0].revisions === undefined || + response.body.query.pages[0].revisions === null) { + throw new MalformedArticleRevisionResponse(); + } + const revisions = response.body.query.pages[0].revisions; // BEGIN: PAGE CUTOFF LOGIC @@ -1241,9 +1610,14 @@ function getSignificantEvents(req, res) { const cutoff = latestAndEarliestCachedRevisionTimestamp(req); var filteredRevisions; const isMaxedOut = cacheForTitleIsMaxedOut(req); - if (isMaxedOut && cutoff.earliestTimestamp) { - filteredRevisions = revisions.filter(revision => new Date(revision.timestamp) > - new Date(cutoff.earliestTimestamp)); + if (isMaxedOut && cutoff && cutoff.earliestTimestamp) { + filteredRevisions = revisions.filter((revision) => { + if (revision.timestamp === undefined || revision.timestamp === null || + cutoff.earliestTimestamp === undefined || cutoff.earliestTimestamp === null) { + return 0; + } + return new Date(revision.timestamp) > new Date(cutoff.earliestTimestamp); + }); } else { filteredRevisions = revisions; } @@ -1254,7 +1628,15 @@ function getSignificantEvents(req, res) { var needsCacheCleanup = false; if (isMaxedOut) { for (var i = 0; i < filteredRevisions.length; i++) { + const revision = filteredRevisions[i]; + + if (revision.timestamp === undefined || revision.timestamp === null || + cutoff.earliestTimestamp === undefined || + cutoff.earliestTimestamp === null) { + continue; + } + const timestamp = new Date(revision.timestamp); const latestCacheTimestamp = new Date(cutoff.latestTimestamp); if (timestamp > latestCacheTimestamp) { @@ -1265,13 +1647,14 @@ function getSignificantEvents(req, res) { } // END: PAGE CUTOFF LOGIC - var nextRvStartId; - var talkPageRvStartId; - var talkPageRvEndId; + var nextRvStartId = null; + var talkPageRvStartId = null; + var talkPageRvEndId = null; if (filteredRevisions && filteredRevisions.length > 0) { const earliestRevision = filteredRevisions[filteredRevisions.length - 1]; const latestRevision = filteredRevisions[0]; - if (earliestRevision.parentid) { + if (earliestRevision.parentid !== null && earliestRevision.parentid !== undefined + && latestRevision.timestamp !== null && latestRevision.timestamp !== undefined) { nextRvStartId = earliestRevision.parentid; // if rvstartid is missing from query, they are fetching the first page // if they are fetching the first page, we don't want to block of @@ -1281,11 +1664,6 @@ function getSignificantEvents(req, res) { latestRevision.timestamp; talkPageRvEndId = earliestRevision.timestamp; } - - } else { - nextRvStartId = null; - talkPageRvStartId = null; - talkPageRvEndId = null; } const articleEvalResults = getCachedAndUncachedItems(filteredRevisions, req, null); @@ -1315,7 +1693,15 @@ function getSignificantEvents(req, res) { const nextRvStartId = response.nextRvStartId; const needsCacheCleanup = response.needsCacheCleanup; - if (response.talkPageRevisions) { + if (response.talkPageRevisions !== undefined && response.talkPageRevisions !== null && + response.talkPageRevisions.body !== undefined && + response.talkPageRevisions.body !== null && + response.talkPageRevisions.body.query !== undefined && + response.talkPageRevisions.body.query !== null && + response.talkPageRevisions.body.query.pages[0] !== undefined && + response.talkPageRevisions.body.query.pages[0] !== null && + response.talkPageRevisions.body.query.pages[0].revisions !== undefined && + response.talkPageRevisions.body.query.pages[0].revisions !== null) { const talkPageRevisions = response.talkPageRevisions.body.query.pages[0].revisions; const talkPageEvalResults = getCachedAndUncachedItems(talkPageRevisions, req, talkPageTitle(req)); @@ -1328,7 +1714,7 @@ function getSignificantEvents(req, res) { .then( (response) => { return Object.assign({ articleDiffAndRevisions: articleDiffAndRevisions, - talkDiffAndRevisions: response.filter(x => x !== null), + talkDiffAndRevisions: response, nextRvStartId: nextRvStartId, needsCacheCleanup: needsCacheCleanup, finalOutput: finalOutput @@ -1366,11 +1752,30 @@ function getSignificantEvents(req, res) { // segment off into types var uncachedOutput = []; - response.articleDiffAndRevisions.forEach(function (diffAndRevision) { + + for (var i = 0; i < response.articleDiffAndRevisions.length; i++) { + const diffAndRevision = response.articleDiffAndRevisions[i]; + + if (diffAndRevision.revision === undefined || diffAndRevision.revision === null) { + continue; + } + const revision = diffAndRevision.revision; - if (revision.tags.includes('mw-rollback') && + + // edge case in case one of the diff endpoints fail...fallback to small type + if (diffAndRevision.body === undefined || diffAndRevision.body === null) { + const smallOutputObject = new SmallOutput(revision.revid, + revision.timestamp, revision.user, revision.userid); + uncachedOutput.push(smallOutputObject); + continue; + } + + if (revision.tags !== undefined && revision.tags !== null && + revision.tags.includes('mw-rollback') && + revision.comment !== undefined || revision.comment !== null && revision.comment.toLowerCase().includes('revert') && - revision.comment.toLowerCase().includes('vandalism')) { + revision.comment.toLowerCase().includes('vandalism') && + diffAndRevision.body !== null && diffAndRevision.body !== undefined) { const largestDiffLine = getLargestDiffLine(diffAndRevision.body); const section = getSectionForDiffLine(diffAndRevision.body, largestDiffLine); @@ -1381,11 +1786,14 @@ function getSignificantEvents(req, res) { } else { var significantChanges = []; - if (diffAndRevision.templates && diffAndRevision.templates.length > 0) { + if (diffAndRevision.templates !== undefined && + diffAndRevision.templates !== null && + diffAndRevision.templates.length > 0) { var sections = []; // todo: this section determination seems broken. // multiple templates can occur on multiple lines. - if (diffAndRevision.templateDiffLine) { + if (diffAndRevision.templateDiffLine !== undefined && + diffAndRevision.templateDiffLine !== null) { const section = getSectionForDiffLine(diffAndRevision.body, diffAndRevision.templateDiffLine); if (section) { @@ -1397,15 +1805,22 @@ function getSignificantEvents(req, res) { significantChanges.push(newReferenceOutputObject); } - if (diffAndRevision.characterChangeWithSections.counts.totalCount() > + if (diffAndRevision.characterChangeWithSections !== undefined && + diffAndRevision.characterChangeWithSections !== null && + diffAndRevision.characterChangeWithSections.counts !== undefined && + diffAndRevision.characterChangeWithSections.counts !== null && + diffAndRevision.characterChangeWithSections.counts.totalCount() > threshold) { if (diffAndRevision.characterChangeWithSections.counts.addedCount > 0) { const largestDiffLine = getLargestDiffLineOfAdded(diffAndRevision.body); - const addedTextOutputObject = new AddedTextOutputExpanded( - diffAndRevision.characterChangeWithSections, largestDiffLine.text, - largestDiffLine.type, largestDiffLine.highlightRanges); - significantChanges.push(addedTextOutputObject); + if (largestDiffLine !== undefined && largestDiffLine !== null) { + const addedTextOutputObject = new AddedTextOutputExpanded( + diffAndRevision.characterChangeWithSections, + largestDiffLine.text, + largestDiffLine.type, largestDiffLine.highlightRanges); + significantChanges.push(addedTextOutputObject); + } } if (diffAndRevision.characterChangeWithSections.counts.deletedCount > 0) { @@ -1425,23 +1840,43 @@ function getSignificantEvents(req, res) { uncachedOutput.push(smallOutputObject); } } - }); + } // get new talk page revisions, add to uncachedOutput. it will be sorted later. - if (response.talkDiffAndRevisions) { + if (response.talkDiffAndRevisions !== undefined && + response.talkDiffAndRevisions !== null) { const newTopicDiffAndRevisions = getNewTopicDiffAndRevisions( response.talkDiffAndRevisions); - newTopicDiffAndRevisions.forEach(function (diffAndRevision) { - const revision = diffAndRevision.revision; - const firstDiffLine = getFirstDiffLineWithContent(diffAndRevision.body); - const section = getSectionForDiffLine(diffAndRevision.body, - firstDiffLine); - const newTalkPageTopicOutputObject = new NewTalkPageTopicExtended( - revision.revid,revision.timestamp, revision.user, revision.userid, - firstDiffLine.text,firstDiffLine.type, firstDiffLine.highlightRanges, - diffAndRevision.characterChangeWithSections.counts, section); - uncachedOutput.push(newTalkPageTopicOutputObject); - }); + if (newTopicDiffAndRevisions !== undefined && newTopicDiffAndRevisions !== null) { + newTopicDiffAndRevisions.forEach(function (diffAndRevision) { + const revision = diffAndRevision.revision; + + const nullSnippetTalkPageObject = new NewTalkPageTopicExtended( + revision.revid,revision.timestamp, revision.user, revision.userid, + null,null, null, null); + if (diffAndRevision.body !== undefined && diffAndRevision.body !== null) { + const firstDiffLine = getFirstDiffLineWithContent(diffAndRevision.body); + if (firstDiffLine !== undefined && firstDiffLine !== null) { + const section = getSectionForDiffLine(diffAndRevision.body, + firstDiffLine); + const newTalkPageTopicOutputObject = new NewTalkPageTopicExtended( + revision.revid,revision.timestamp, revision.user, + revision.userid, + firstDiffLine.text,firstDiffLine.type, + firstDiffLine.highlightRanges, + diffAndRevision.characterChangeWithSections.counts, section); + uncachedOutput.push(newTalkPageTopicOutputObject); + } else { + uncachedOutput.push(nullSnippetTalkPageObject); + } + } else { + const newTalkPageTopicOutputObject = new NewTalkPageTopicExtended( + revision.revid,revision.timestamp, revision.user, revision.userid, + null,null, null, null); + uncachedOutput.push(nullSnippetTalkPageObject); + } + }); + } } return Object.assign({ nextRvStartId: response.nextRvStartId, @@ -1452,8 +1887,14 @@ function getSignificantEvents(req, res) { .then( (response) => { var preformattedSnippets = []; - response.uncachedOutput.forEach(function (item) { - if (item.outputType === 'new-talk-page-topic') { + for (var o = 0; o < response.uncachedOutput.length; o++) { + const item = response.uncachedOutput[o]; + + if (item.outputType === undefined || item.outputType === null) { + continue; + } + + if (item.outputType === 'new-talk-page-topic' && item.snippet !== null) { const snippet = new PreformattedSnippet(item.revid, item.outputType, item.snippet,1, null, null); @@ -1461,6 +1902,16 @@ function getSignificantEvents(req, res) { } else if (item.outputType === 'large-change') { for (let i = 0; i < item.significantChanges.length; i++) { const significantChange = item.significantChanges[i]; + + if (significantChange.outputType === undefined || + significantChange.outputType === null || + significantChange.snippet === undefined || + significantChange.snippet === null || + significantChange.snippetType === undefined || + significantChange.snippetType === null) { + continue; + } + if (significantChange.outputType === 'added-text') { const snippet = new PreformattedSnippet(item.revid, item.outputType, significantChange.snippet, significantChange.snippetType, @@ -1469,37 +1920,58 @@ function getSignificantEvents(req, res) { } } } - }); + } // convert large snippets from wikitext to mobile-html return snippetPromises(req, preformattedSnippets) .then( (formattedSnippets) => { // reassign formattedSnippets to snippet output - formattedSnippets.forEach((formattedSnippet) => { - response.uncachedOutput.forEach((item) => { - if (item.revid === formattedSnippet.revid && - item.outputType === formattedSnippet.outputType) { - if (formattedSnippet.outputType === 'new-talk-page-topic') { - item.snippet = formattedSnippet.snippet; - } else if (formattedSnippet.outputType === 'large-change') { - if (item.significantChanges.length > - formattedSnippet.indexOfSignificantChanges) { - var significantChange = - item.significantChanges[ - formattedSnippet.indexOfSignificantChanges - ]; - significantChange.snippet = formattedSnippet.snippet; - if (significantChange.outputType === 'added-text') { - item.significantChanges[ - formattedSnippet.indexOfSignificantChanges - ] = new AddedTextOutput(significantChange); - } - } - } - } - }); - }); + for (var fs = 0; fs < formattedSnippets.length; fs++) { + const formattedSnippet = formattedSnippets[fs]; + + for (var i = 0; i < response.uncachedOutput.length; i++) { + const item = response.uncachedOutput[i]; + + if (item.revid === undefined || item.revid === null || + formattedSnippet.revid === undefined || + formattedSnippet.revid === null || + item.outputType === undefined || item.outputType === null || + formattedSnippet.outputType === undefined || + formattedSnippet.outputType === null) { + continue; + } + + if (item.revid === formattedSnippet.revid && + item.outputType === formattedSnippet.outputType) { + if (formattedSnippet.outputType === 'new-talk-page-topic') { + item.snippet = formattedSnippet.snippet; + } else if (formattedSnippet.outputType === 'large-change') { + + if (item.significantChanges === undefined || + item.significantChanges === null || + formattedSnippet.indexOfSignificantChanges === undefined || + formattedSnippet.indexOfSignificantChanges === null) { + continue; + } + + if (item.significantChanges.length > + formattedSnippet.indexOfSignificantChanges) { + var significantChange = + item.significantChanges[ + formattedSnippet.indexOfSignificantChanges + ]; + significantChange.snippet = formattedSnippet.snippet; + if (significantChange.outputType === 'added-text') { + item.significantChanges[ + formattedSnippet.indexOfSignificantChanges + ] = new AddedTextOutput(significantChange); + } + } + } + } + } + } // push to final output and cache // note we are using original response list, not snippet response From 00da92653c75c9d6edf3c0a7a6ad81194001f94b Mon Sep 17 00:00:00 2001 From: Toni Sevener Date: Mon, 24 Aug 2020 20:29:21 -0500 Subject: [PATCH 24/47] some bug fixes --- routes/page/significant-events.js | 79 ++++++++++++++++++++----------- 1 file changed, 52 insertions(+), 27 deletions(-) diff --git a/routes/page/significant-events.js b/routes/page/significant-events.js index 2c51dad2..de5ab4d3 100644 --- a/routes/page/significant-events.js +++ b/routes/page/significant-events.js @@ -832,9 +832,7 @@ function getSectionForDiffLine(diffBody, diffLine) { !diffBody.from.sections || !diffBody.to || !diffBody.to.sections || - !diffLine.offset || - !diffLine.offset.from || - !diffLine.offset.to) { + !diffLine.offset) { return null; } @@ -1087,33 +1085,32 @@ function structuredTemplatePromise(text, diff, revision) { const innerPdoc = yield ParsoidJS.parse(splitTemplateText, { pdoc: true }); const individualTemplates = innerPdoc.filterTemplates(); - if (individualTemplates === undefined || individualTemplates === null) { - throw new Error('Unexpected result from filterTemplates'); - } - for (var i = 0; i < individualTemplates.length; i++) { - const template = individualTemplates[i]; + if (individualTemplates !== undefined && individualTemplates !== null) { + for (var i = 0; i < individualTemplates.length; i++) { + const template = individualTemplates[i]; - if (!needsToParseForAddedTemplates(template.name, false)) { - continue; - } + if (!needsToParseForAddedTemplates(template.name, false)) { + continue; + } - if (template.name === undefined || - template.name === null || - template.params === undefined || - template.params === null) { - continue; - } + if (template.name === undefined || + template.name === null || + template.params === undefined || + template.params === null) { + continue; + } - var dict = {}; - dict.name = template.name; - for (var p = 0; p < template.params.length; p++) { - const param = template.params[p].name; - if (param !== undefined && param !== null) { - const value = yield template.get(param).value.toWikitext(); - dict[param] = value; + var dict = {}; + dict.name = template.name; + for (var p = 0; p < template.params.length; p++) { + const param = template.params[p].name; + if (param !== undefined && param !== null) { + const value = yield template.get(param).value.toWikitext(); + dict[param] = value; + } } + templateObjects.push(dict); } - templateObjects.push(dict); } } const result = Object.assign( { @@ -1479,6 +1476,12 @@ function editCountsAndGroupsPromise(req, cleanedOutput) { response.body.query === undefined || response.body.query.users === null || response.body.query.users === undefined) { + cleanedOutput.forEach( (outputItem) => { + if (outputItem.outputType !== 'small-change') { + outputItem.userGroups = null; + outputItem.userEditCount = null; + } + }); return cleanedOutput; } @@ -1580,6 +1583,13 @@ class MalformedArticleRevisionResponse extends Error { } } +class AllArticleDiffCallsFailed extends Error { + constructor(message) { + super(message); + this.name = 'AllArticleDiffCallsFailed'; + } +} + function getSignificantEvents(req, res) { // STEP 1: Gather list of article revisions @@ -1686,6 +1696,22 @@ function getSignificantEvents(req, res) { }) .then( (response) => { + // if all articleDiffAndRevisions fail, return error + if (response.articleDiffAndRevisions !== undefined && + response.articleDiffAndRevisions !== null) { + const articleDiffAndRevisionsNullDiff = + response.articleDiffAndRevisions.filter(diffAndRevision => { + return diffAndRevision.body === undefined || diffAndRevision.body === null; + }); + + if (articleDiffAndRevisionsNullDiff.length === + response.articleDiffAndRevisions.length && + articleDiffAndRevisionsNullDiff.length > 0 && + response.articleDiffAndRevisions.length > 0) { // all diff calls failed + throw new AllArticleDiffCallsFailed(); + } + } + // STEP 3: All at once gather diffs for uncached talk page revisions const articleDiffAndRevisions = response.articleDiffAndRevisions @@ -1853,7 +1879,7 @@ function getSignificantEvents(req, res) { const nullSnippetTalkPageObject = new NewTalkPageTopicExtended( revision.revid,revision.timestamp, revision.user, revision.userid, - null,null, null, null); + null,null, null, null, null); if (diffAndRevision.body !== undefined && diffAndRevision.body !== null) { const firstDiffLine = getFirstDiffLineWithContent(diffAndRevision.body); if (firstDiffLine !== undefined && firstDiffLine !== null) { @@ -2013,7 +2039,6 @@ function getSignificantEvents(req, res) { const summary = getSummaryText(req); - // todo: sometimes nextRvStartId doesn't show at all in response. const result = Object.assign({ nextRvStartId: response.nextRvStartId, sha: response.sha, timeline: response.cleanedOutput, From 60ad6f54d48c106aa4f902e9d20b109362c60b2d Mon Sep 17 00:00:00 2001 From: Toni Sevener Date: Tue, 25 Aug 2020 00:21:53 -0500 Subject: [PATCH 25/47] more bug fixes --- routes/page/significant-events.js | 95 ++++++++++++++++++++----------- 1 file changed, 61 insertions(+), 34 deletions(-) diff --git a/routes/page/significant-events.js b/routes/page/significant-events.js index de5ab4d3..b26a134c 100644 --- a/routes/page/significant-events.js +++ b/routes/page/significant-events.js @@ -94,13 +94,13 @@ class ConsolidatedSmallOutput { } class VandalismOutput { - constructor(revid, timestamp, user, userid, section) { + constructor(revid, timestamp, user, userid, sections) { this.revid = revid; this.timestamp = timestamp; this.outputType = 'vandalism-revert'; this.user = user; this.userid = userid; - this.section = section; + this.sections = sections; } } @@ -1072,7 +1072,7 @@ function needsToParseForAddedTemplates(text, includeOpeningBraces) { // https://en.wikipedia.org/w/index.php?title=United_States&type=revision&diff=971260075&oldid=971259267 // Definitely some mangled 'added-text' snippets going on here. // https://en.wikipedia.org/w/index.php?title=United_States&type=revision&diff=970665187&oldid=970577931 -function structuredTemplatePromise(text, diff, revision) { +function structuredTemplatePromise(text, diffItem, revision) { return new BBPromise((resolve) => { var main = PRFunPromise.async(function*() { var pdoc = yield ParsoidJS.parse(text, { pdoc: true }); @@ -1115,7 +1115,7 @@ function structuredTemplatePromise(text, diff, revision) { } const result = Object.assign( { revision: revision, - diff: diff, + diffItem: diffItem, templates: templateObjects }); resolve(result); @@ -1149,25 +1149,25 @@ function addStructuredTemplates(diffAndRevisions) { } for (var d = 0; d < diffAndRevision.body.diff.length; d++) { - const diff = diffAndRevision.body.diff[d]; + const diffItem = diffAndRevision.body.diff[d]; - if (diff.text === undefined || - diff.text === null) { + if (diffItem.text === undefined || + diffItem.text === null) { continue; } - switch (diff.type) { + switch (diffItem.type) { case 1: // Add complete line type - if (needsToParseForAddedTemplates(diff.text, true)) { - promises.push(structuredTemplatePromise(diff.text, diff, + if (needsToParseForAddedTemplates(diffItem.text, true)) { + promises.push(structuredTemplatePromise(diffItem.text, diffItem, diffAndRevision.revision)); } break; case 5: case 3: - for (var h = 0; h < diff.highlightRanges.length; h++) { - const range = diff.highlightRanges[h]; + for (var h = 0; h < diffItem.highlightRanges.length; h++) { + const range = diffItem.highlightRanges[h]; if (range.start === undefined || range.start === null || @@ -1176,7 +1176,7 @@ function addStructuredTemplates(diffAndRevisions) { continue; } - const binaryText = encoding.strToBin(diff.text); + const binaryText = encoding.strToBin(diffItem.text); const binaryRangeText = binaryText.substring(range.start, range.start + range.length); const rangeText = encoding.binToStr(binaryRangeText); @@ -1184,7 +1184,7 @@ function addStructuredTemplates(diffAndRevisions) { switch (range.type) { case 0: // Add range type if (needsToParseForAddedTemplates(rangeText, true)) { - promises.push(structuredTemplatePromise(rangeText, diff, + promises.push(structuredTemplatePromise(rangeText, diffItem, diffAndRevision.revision)); } break; @@ -1204,19 +1204,30 @@ function addStructuredTemplates(diffAndRevisions) { // loop through responses, add to revision. - response.forEach( (item) => { - diffAndRevisions.forEach( (diffAndRevision) => { + diffAndRevisions.forEach( (diffAndRevision) => { + + var templatesAndDiffItems = []; + response.forEach( (item) => { if (item.revision !== null && item.revision !== undefined && diffAndRevision.revision !== null && - diffAndRevision.revision !== undefined) { + diffAndRevision.revision !== undefined && + item.templates !== null && + item.templates !== undefined && + item.templates.length > 0) { if (item.revision.revid === diffAndRevision.revision.revid) { - diffAndRevision.templates = item.templates; - diffAndRevision.templateDiffLine = item.diff; + const templateAndDiffItem = Object.assign({ + templates: item.templates, + diffItem: item.diffItem + }); + + templatesAndDiffItems.push(templateAndDiffItem); } } }); + + diffAndRevision.templatesAndDiffItems = templatesAndDiffItems; }); return diffAndRevisions; @@ -1276,6 +1287,13 @@ function getNewTopicDiffAndRevisions(talkDiffAndRevisions) { return newSectionTalkPageDiffAndRevisions; } +function getAllChangedDiffLines(diffBody) { + const allChangedDiffLines = diffBody.diff.filter((item) => { + return item.type !== 0; + }); + return allChangedDiffLines; +} + function getLargestDiffLine(diffBody) { if (diffBody === null || @@ -1802,32 +1820,41 @@ function getSignificantEvents(req, res) { revision.comment.toLowerCase().includes('revert') && revision.comment.toLowerCase().includes('vandalism') && diffAndRevision.body !== null && diffAndRevision.body !== undefined) { - const largestDiffLine = getLargestDiffLine(diffAndRevision.body); - const section = getSectionForDiffLine(diffAndRevision.body, - largestDiffLine); - // todo: vandalism type can have reversions in multiple sections + const allChangedDiffLines = getAllChangedDiffLines(diffAndRevision.body); + const sections = allChangedDiffLines.map(diffLine => + getSectionForDiffLine(diffAndRevision.body, + diffLine)); const vandalismRevertOutputObject = new VandalismOutput(revision.revid, - revision.timestamp, revision.user, revision.userid, section); + revision.timestamp, revision.user, revision.userid, sections); uncachedOutput.push(vandalismRevertOutputObject); } else { var significantChanges = []; - if (diffAndRevision.templates !== undefined && - diffAndRevision.templates !== null && - diffAndRevision.templates.length > 0) { + if (diffAndRevision.templatesAndDiffItems !== undefined && + diffAndRevision.templatesAndDiffItems !== null && + diffAndRevision.templatesAndDiffItems.length > 0) { + var sections = []; - // todo: this section determination seems broken. - // multiple templates can occur on multiple lines. - if (diffAndRevision.templateDiffLine !== undefined && - diffAndRevision.templateDiffLine !== null) { + var combinedTemplates = []; + + for (var t = 0; t < diffAndRevision.templatesAndDiffItems.length; t++) { + const templatesAndDiffItem = diffAndRevision.templatesAndDiffItems[t]; + const templates = templatesAndDiffItem.templates; + const diffItem = templatesAndDiffItem.diffItem; const section = getSectionForDiffLine(diffAndRevision.body, - diffAndRevision.templateDiffLine); + diffItem); if (section) { sections.push(section); } + if (templates) { + combinedTemplates.push(templates); + } } - const newReferenceOutputObject = new NewReferenceOutput(sections, - diffAndRevision.templates); + + const dedupedSections = new Set(sections); + const newReferenceOutputObject = + new NewReferenceOutput(Array.from(dedupedSections), + combinedTemplates); significantChanges.push(newReferenceOutputObject); } From fabfc5fb91d86fe458acab61a9710312cea113b3 Mon Sep 17 00:00:00 2001 From: Toni Sevener Date: Tue, 25 Aug 2020 00:50:38 -0500 Subject: [PATCH 26/47] still more bug fixes --- routes/page/significant-events.js | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/routes/page/significant-events.js b/routes/page/significant-events.js index b26a134c..7d725e39 100644 --- a/routes/page/significant-events.js +++ b/routes/page/significant-events.js @@ -516,6 +516,21 @@ const snippetPromise = (req, preformattedSnippet) => { const firstStart = response.indexOf('ioshighlightstart'); const lastStart = response.lastIndexOf('ioshighlightend'); + // early exit here - some highlighting was inadvertantly pruned out, + // so don't attempt to truncate or preserve highlight delimiters + if (firstStart === -1 || lastStart === -1) { + if (firstStart === -1 && lastStart !== -1) { + truncatedSnippet = truncatedSnippet.replace(/ioshighlightend/g, ''); + } + + if (firstStart !== -1 && lastStart === -1) { + truncatedSnippet = truncatedSnippet.replace(/ioshighlightstart/g, ''); + } + + preformattedSnippet.snippet = truncatedSnippet; + return preformattedSnippet; + } + // If highlighting starts at the beginning of the line, don't // truncate or prepend ... // Otherwise truncate and prepend ... @@ -848,11 +863,13 @@ function getSectionForDiffLine(diffBody, diffLine) { // In this case javascript evaluates diffLine.offset.from to false, // hence the need for the separate check. if ((diffLine.offset.from || diffLine.offset.from === 0) && + diffBody.from.sections && diffBody.from.sections.length > 0 && diffLine.offset.from < diffBody.from.sections[0].offset) { fromSection = 'Intro'; } if ((diffLine.offset.to || diffLine.offset.to === 0) && + diffBody.to.sections && diffBody.to.sections.length > 0 && diffLine.offset.to < diffBody.to.sections[0].offset) { toSection = 'Intro'; } @@ -1824,8 +1841,10 @@ function getSignificantEvents(req, res) { const sections = allChangedDiffLines.map(diffLine => getSectionForDiffLine(diffAndRevision.body, diffLine)); + const dedupedSections = new Set(sections); const vandalismRevertOutputObject = new VandalismOutput(revision.revid, - revision.timestamp, revision.user, revision.userid, sections); + revision.timestamp, revision.user, revision.userid, + Array.from(dedupedSections)); uncachedOutput.push(vandalismRevertOutputObject); } else { From 88f66646e0a1c78299a69e42e0a58c6abff29d63 Mon Sep 17 00:00:00 2001 From: Toni Sevener Date: Tue, 25 Aug 2020 11:44:47 -0500 Subject: [PATCH 27/47] some rework to catch addtional missed templates --- routes/page/significant-events.js | 125 +++++++++++++++++++++++++----- 1 file changed, 107 insertions(+), 18 deletions(-) diff --git a/routes/page/significant-events.js b/routes/page/significant-events.js index 7d725e39..f5a071ff 100644 --- a/routes/page/significant-events.js +++ b/routes/page/significant-events.js @@ -1078,8 +1078,7 @@ function needsToParseForAddedTemplates(text, includeOpeningBraces) { // BUG: https://en.wikipedia.org/w/index.php?title=United_States&type=revision // &diff=965295364&oldid=965071033 -// We are missing some an added reference from line 722. We also aren't -// catching the tags in line 579 +// Should we be catching added tags? // ANOTHER BUG: // Sometimes new citations have spaces within deemed the same, so only part of it // is sent in as text whereas we need all of cite. @@ -1097,7 +1096,21 @@ function structuredTemplatePromise(text, diffItem, revision) { ParsoidJS.toWikitext); var templateObjects = []; - for (var s = 0; s < splitTemplates.length; s++) { + // Note: for now just grabbing the first split template + // (which includes only parent templates and not nested) + // to avoid duplicate template problem. If we need nested support we can loop through + // all split templates, then uniquely identify and dedupe templates later. + if (splitTemplates.length === 0) { + // bail early + const result = Object.assign( { + revision: revision, + diffItem: diffItem, + templates: templateObjects + }); + resolve(result); + return; + } + for (var s = 0; s < 1; s++) { const splitTemplateText = splitTemplates[s]; const innerPdoc = yield ParsoidJS.parse(splitTemplateText, { pdoc: true }); @@ -1181,35 +1194,111 @@ function addStructuredTemplates(diffAndRevisions) { } break; case 5: - case 3: + case 3: { + const binaryText = encoding.strToBin(diffItem.text); + var previousBinaryRangeText = null; + var previousRangeEndIndex = null; for (var h = 0; h < diffItem.highlightRanges.length; h++) { const range = diffItem.highlightRanges[h]; if (range.start === undefined || - range.start === null || - range.length === undefined || - range.length === null) { + range.start === null || + range.length === undefined || + range.length === null) { continue; } - const binaryText = encoding.strToBin(diffItem.text); - const binaryRangeText = binaryText.substring(range.start, - range.start + range.length); - const rangeText = encoding.binToStr(binaryRangeText); - switch (range.type) { - case 0: // Add range type - if (needsToParseForAddedTemplates(rangeText, true)) { - promises.push(structuredTemplatePromise(rangeText, diffItem, - diffAndRevision.revision)); + case 0: { // Add range type + + const binaryRangeText = binaryText.substring(range.start, + range.start + range.length); + if (previousBinaryRangeText !== null && + previousRangeEndIndex !== null) { + + const inBetweenText = binaryText.substring( + previousRangeEndIndex, + range.start); + if (inBetweenText === ' ') { + previousBinaryRangeText += inBetweenText; + previousBinaryRangeText += binaryRangeText; + previousRangeEndIndex = range.start + range.length; + continue; + } else { + // before attempting to send off for template parsing, + // grab following '}}' to capture edge case templates + const nextTwoCharacters = binaryText + .substring(previousRangeEndIndex, + previousRangeEndIndex + 2); + if (nextTwoCharacters === '}}') { + previousBinaryRangeText += '}}'; + } + const rangeText = encoding + .binToStr(previousBinaryRangeText); + if (needsToParseForAddedTemplates(rangeText, + true)) { + promises.push(structuredTemplatePromise(rangeText, + diffItem, + diffAndRevision.revision)); + } + + previousBinaryRangeText = binaryRangeText; + previousRangeEndIndex = range.start + range.length; + } + + } else { + previousBinaryRangeText = binaryRangeText; + previousRangeEndIndex = range.start + range.length; + continue; } + } break; - default: - break; + case 1: // Deleted + // if there's a space before deleted text, include it + // so it gets picked up in inBetweenText check up there + // const textBeforeDeleted = binaryText.substring(range.start - 1, + // (range.start - 1) + 1); + // if (textBeforeDeleted === ' ') { + // previousRangeEndIndex = range.start - 1; + // } else { + // previousRangeEndIndex = range.start + range.length; + // } + + previousRangeEndIndex = range.start + range.length; + + } + } + + // there's probably one last straggler range that we haven't checked + // due to how we're doing things + + if (previousBinaryRangeText !== null && previousRangeEndIndex !== null) { + + // before attempting to send off for template parsing, + // grab following '}}' to capture edge case templates + const nextTwoCharacters = binaryText.substring(previousRangeEndIndex, + previousRangeEndIndex + 2); + if (nextTwoCharacters === '}}') { + previousBinaryRangeText += '}}'; + } + + // also if first 2 characters start with '}}', it might throw things off + const firstTwoCharacters = previousBinaryRangeText.substring(0, + 2); + if (firstTwoCharacters === '}}') { + previousBinaryRangeText = previousBinaryRangeText.substring(2, + previousBinaryRangeText.length); + } + + const rangeText = encoding.binToStr(previousBinaryRangeText); + if (needsToParseForAddedTemplates(rangeText, true)) { + promises.push(structuredTemplatePromise(rangeText, diffItem, + diffAndRevision.revision)); } } break; + } default: break; } From f719ae18b987e349a3f1349df394504384a3dca6 Mon Sep 17 00:00:00 2001 From: Toni Sevener Date: Tue, 25 Aug 2020 17:10:56 -0500 Subject: [PATCH 28/47] fixed split citation bug --- routes/page/significant-events.js | 128 ++++++++++++++++-------------- 1 file changed, 69 insertions(+), 59 deletions(-) diff --git a/routes/page/significant-events.js b/routes/page/significant-events.js index f5a071ff..19515fb5 100644 --- a/routes/page/significant-events.js +++ b/routes/page/significant-events.js @@ -379,6 +379,50 @@ const formattedSnippetFromTextPromise = (req, text) => { }); }; +function stripDeletedRangesFromBinaryText(binaryText, highlightedRanges) { + // first strip deleted text + for (var d = highlightedRanges.length - 1; d >= 0; d-- ) { + const range = highlightedRanges[d]; + + switch (range.type) { + case 0: // Added + break; + case 1: // Deleted + binaryText = stringByRemovingSubstring(binaryText, range.start, + range.start + range.length); + + for (var i = d; i < highlightedRanges.length; + i++) { + const iRange = highlightedRanges[i]; + switch (iRange.type) { + case 0: // Added + iRange.start -= range.length; + highlightedRanges[i] = iRange; + break; + case 1: // Deleted + break; + default: + break; + } + } + break; + default: + break; + } + } + + // filter out deleted ranges + // eslint-disable-next-line no-case-declarations + const addedHighlightRanges = highlightedRanges + .filter(range => range.type !== null && range.type !== undefined && + range.type === 0); + + return Object.assign({ + strippedBinaryText: binaryText, + addedHighlightRanges: addedHighlightRanges + }); +} + const snippetPromise = (req, preformattedSnippet) => { if (preformattedSnippet.snippet === null || @@ -412,49 +456,17 @@ const snippetPromise = (req, preformattedSnippet) => { // should be caught earlier break; case 5: - case 3: // Added and deleted words in line + case 3: { // Added and deleted words in line if (preformattedSnippet.snippetHighlightRanges === null || - preformattedSnippet.snippetHighlightRanges === undefined) { + preformattedSnippet.snippetHighlightRanges === undefined) { break; } - // first strip deleted text - for (var d = preformattedSnippet.snippetHighlightRanges.length - 1; d >= 0; d-- ) { - const range = preformattedSnippet.snippetHighlightRanges[d]; - - switch (range.type) { - case 0: // Added - break; - case 1: // Deleted - snippetBinary = stringByRemovingSubstring(snippetBinary, range.start, - range.start + range.length); - - for (var i = d; i < preformattedSnippet.snippetHighlightRanges.length; - i++) { - const iRange = preformattedSnippet.snippetHighlightRanges[i]; - switch (iRange.type) { - case 0: // Added - iRange.start -= range.length; - preformattedSnippet.snippetHighlightRanges[i] = iRange; - break; - case 1: // Deleted - break; - default: - break; - } - } - break; - default: - break; - } - } - - // filter out deleted ranges - // eslint-disable-next-line no-case-declarations - const addedHighlightRanges = preformattedSnippet.snippetHighlightRanges - .filter(range => range.type !== null && range.type !== undefined && - range.type === 0); + const strippedResult = stripDeletedRangesFromBinaryText(snippetBinary, + preformattedSnippet.snippetHighlightRanges); + snippetBinary = strippedResult.strippedBinaryText; + const addedHighlightRanges = strippedResult.addedHighlightRanges; // then add added text delimiters var addOffset = 0; @@ -479,6 +491,7 @@ const snippetPromise = (req, preformattedSnippet) => { addOffset += highlightEndBin.length; }); break; + } default: break; } @@ -1079,15 +1092,8 @@ function needsToParseForAddedTemplates(text, includeOpeningBraces) { // BUG: https://en.wikipedia.org/w/index.php?title=United_States&type=revision // &diff=965295364&oldid=965071033 // Should we be catching added tags? -// ANOTHER BUG: -// Sometimes new citations have spaces within deemed the same, so only part of it -// is sent in as text whereas we need all of cite. -// See here -// https://en.wikipedia.org/w/index.php?title=United_States&type=revision&diff=971349395&oldid=971332725 -// Also this one +// This one just seems to have moved paragraphs. Seems like we shouldn't return a new citation here. // https://en.wikipedia.org/w/index.php?title=United_States&type=revision&diff=971260075&oldid=971259267 -// Definitely some mangled 'added-text' snippets going on here. -// https://en.wikipedia.org/w/index.php?title=United_States&type=revision&diff=970665187&oldid=970577931 function structuredTemplatePromise(text, diffItem, revision) { return new BBPromise((resolve) => { var main = PRFunPromise.async(function*() { @@ -1196,9 +1202,20 @@ function addStructuredTemplates(diffAndRevisions) { case 5: case 3: { - const binaryText = encoding.strToBin(diffItem.text); + var binaryText = encoding.strToBin(diffItem.text); + + // strip deleted ranges from text, this simplifies template detection later + const strippedResult = stripDeletedRangesFromBinaryText(binaryText, + diffItem.highlightRanges); + binaryText = strippedResult.strippedBinaryText; + diffItem.highlightRanges = strippedResult.addedHighlightRanges; + + // now loop through added highlighted ranges, + // fuzzying things to capture full citations + // and generate templates. var previousBinaryRangeText = null; var previousRangeEndIndex = null; + for (var h = 0; h < diffItem.highlightRanges.length; h++) { const range = diffItem.highlightRanges[h]; @@ -1220,6 +1237,9 @@ function addStructuredTemplates(diffAndRevisions) { const inBetweenText = binaryText.substring( previousRangeEndIndex, range.start); + // if an empty space is all that exists between added ranges, + // lump it in with previous range in case it is splitting + // up a new citation if (inBetweenText === ' ') { previousBinaryRangeText += inBetweenText; previousBinaryRangeText += binaryRangeText; @@ -1253,19 +1273,9 @@ function addStructuredTemplates(diffAndRevisions) { continue; } } + break; + default: break; - case 1: // Deleted - // if there's a space before deleted text, include it - // so it gets picked up in inBetweenText check up there - // const textBeforeDeleted = binaryText.substring(range.start - 1, - // (range.start - 1) + 1); - // if (textBeforeDeleted === ' ') { - // previousRangeEndIndex = range.start - 1; - // } else { - // previousRangeEndIndex = range.start + range.length; - // } - - previousRangeEndIndex = range.start + range.length; } } From c5745359bb50a5218d50f2468d2eb788e13c8c66 Mon Sep 17 00:00:00 2001 From: Toni Sevener Date: Tue, 25 Aug 2020 17:16:02 -0500 Subject: [PATCH 29/47] rename new reference type --- routes/page/significant-events.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/page/significant-events.js b/routes/page/significant-events.js index 19515fb5..91fcc063 100644 --- a/routes/page/significant-events.js +++ b/routes/page/significant-events.js @@ -106,7 +106,7 @@ class VandalismOutput { class NewReferenceOutput { constructor(sections, templates) { - this.outputType = 'new-reference'; + this.outputType = 'new-template'; this.sections = sections; this.templates = templates; } From cf93d117f6592905d89f8f56e18a708113599009 Mon Sep 17 00:00:00 2001 From: Toni Sevener Date: Fri, 28 Aug 2020 08:34:15 -0500 Subject: [PATCH 30/47] allowing cache to handle changing thresholds --- routes/page/significant-events.js | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/routes/page/significant-events.js b/routes/page/significant-events.js index 91fcc063..59873a30 100644 --- a/routes/page/significant-events.js +++ b/routes/page/significant-events.js @@ -591,6 +591,14 @@ const talkPageRevisionsPromise = (req, rvStart, rvEnd) => { }); }; +function keyTitleForCache(req, title) { + var keyTitle = title || req.params.title; + const threshold = getThreshold(req); + keyTitle = `${threshold}-${keyTitle}`; + + return keyTitle; +} + function getCachedAndUncachedItems(revisions, req, title) { // add cache to output and filter out of processing flow @@ -604,7 +612,7 @@ function getCachedAndUncachedItems(revisions, req, title) { }); } - const keyTitle = title || req.params.title; + const keyTitle = keyTitleForCache(req, title); for (var i = 0; i < revisions.length; i++) { const revision = revisions[i]; const domainDict = significantChangesCache[req.params.domain]; @@ -646,7 +654,7 @@ function calculateCacheForTitleIsMaxedOut(titleDict) { } function cacheForTitleIsMaxedOut(req, title) { - const keyTitle = title || req.params.title; + const keyTitle = keyTitleForCache(req, title); var domainDict = significantChangesCache[req.params.domain]; if (domainDict) { var titleDict = domainDict[keyTitle]; @@ -662,7 +670,7 @@ function cacheForTitleIsMaxedOut(req, title) { function latestAndEarliestCachedRevisionTimestamp(req, title) { var domainDict = significantChangesCache[req.params.domain]; - const keyTitle = title || req.params.title; + const keyTitle = keyTitleForCache(req, title); if (domainDict) { var titleDict = domainDict[keyTitle]; if (titleDict) { @@ -693,7 +701,7 @@ function latestAndEarliestCachedRevisionTimestamp(req, title) { function setSignificantChangesCache(req, title, item) { var domainDict = significantChangesCache[req.params.domain]; - const keyTitle = title || req.params.title; + const keyTitle = keyTitleForCache(req, title); if (domainDict) { var titleDict = domainDict[keyTitle]; if (titleDict) { @@ -719,7 +727,7 @@ function cleanupCache(req) { // cleanup article cache var domainDict = significantChangesCache[req.params.domain]; - const articleTitle = req.params.title; + const articleTitle = keyTitleForCache(req, null); if (domainDict) { var titleDict = domainDict[articleTitle]; if (titleDict) { @@ -774,8 +782,9 @@ function cleanupCache(req) { // clean out from talk page cache // todo: why on earth do we get a talkPageTitle is not a function error here? - var talkPageTitle = `Talk:${req.params.title}`; // talkPageTitle(req); - var talkPageTitleDict = domainDict[talkPageTitle]; + var talkPageTitle = `Talk:${req.params.title}`; + const keyTalkPageTitle = keyTitleForCache(req, talkPageTitle); + var talkPageTitleDict = domainDict[keyTalkPageTitle]; if (talkPageTitleDict) { // eslint-disable-next-line no-restricted-properties var talkPageTitleArray = Object.values(talkPageTitleDict); @@ -807,7 +816,7 @@ function cleanupCache(req) { function getSummaryText(req) { var domainDict = significantChangesCache[req.params.domain]; - const articleTitle = req.params.title; + const articleTitle = keyTitleForCache(req, null); if (domainDict) { var titleDict = domainDict[articleTitle]; if (titleDict) { From f510075e1cc3daaa433f36cd79dd2a0e720186c7 Mon Sep 17 00:00:00 2001 From: Toni Sevener Date: Fri, 28 Aug 2020 16:06:20 -0500 Subject: [PATCH 31/47] Revert "fixed split citation bug" This reverts commit f719ae18b987e349a3f1349df394504384a3dca6. --- routes/page/significant-events.js | 128 ++++++++++++++---------------- 1 file changed, 59 insertions(+), 69 deletions(-) diff --git a/routes/page/significant-events.js b/routes/page/significant-events.js index 59873a30..a5f5c5fd 100644 --- a/routes/page/significant-events.js +++ b/routes/page/significant-events.js @@ -379,50 +379,6 @@ const formattedSnippetFromTextPromise = (req, text) => { }); }; -function stripDeletedRangesFromBinaryText(binaryText, highlightedRanges) { - // first strip deleted text - for (var d = highlightedRanges.length - 1; d >= 0; d-- ) { - const range = highlightedRanges[d]; - - switch (range.type) { - case 0: // Added - break; - case 1: // Deleted - binaryText = stringByRemovingSubstring(binaryText, range.start, - range.start + range.length); - - for (var i = d; i < highlightedRanges.length; - i++) { - const iRange = highlightedRanges[i]; - switch (iRange.type) { - case 0: // Added - iRange.start -= range.length; - highlightedRanges[i] = iRange; - break; - case 1: // Deleted - break; - default: - break; - } - } - break; - default: - break; - } - } - - // filter out deleted ranges - // eslint-disable-next-line no-case-declarations - const addedHighlightRanges = highlightedRanges - .filter(range => range.type !== null && range.type !== undefined && - range.type === 0); - - return Object.assign({ - strippedBinaryText: binaryText, - addedHighlightRanges: addedHighlightRanges - }); -} - const snippetPromise = (req, preformattedSnippet) => { if (preformattedSnippet.snippet === null || @@ -456,17 +412,49 @@ const snippetPromise = (req, preformattedSnippet) => { // should be caught earlier break; case 5: - case 3: { // Added and deleted words in line + case 3: // Added and deleted words in line if (preformattedSnippet.snippetHighlightRanges === null || - preformattedSnippet.snippetHighlightRanges === undefined) { + preformattedSnippet.snippetHighlightRanges === undefined) { break; } - const strippedResult = stripDeletedRangesFromBinaryText(snippetBinary, - preformattedSnippet.snippetHighlightRanges); - snippetBinary = strippedResult.strippedBinaryText; - const addedHighlightRanges = strippedResult.addedHighlightRanges; + // first strip deleted text + for (var d = preformattedSnippet.snippetHighlightRanges.length - 1; d >= 0; d-- ) { + const range = preformattedSnippet.snippetHighlightRanges[d]; + + switch (range.type) { + case 0: // Added + break; + case 1: // Deleted + snippetBinary = stringByRemovingSubstring(snippetBinary, range.start, + range.start + range.length); + + for (var i = d; i < preformattedSnippet.snippetHighlightRanges.length; + i++) { + const iRange = preformattedSnippet.snippetHighlightRanges[i]; + switch (iRange.type) { + case 0: // Added + iRange.start -= range.length; + preformattedSnippet.snippetHighlightRanges[i] = iRange; + break; + case 1: // Deleted + break; + default: + break; + } + } + break; + default: + break; + } + } + + // filter out deleted ranges + // eslint-disable-next-line no-case-declarations + const addedHighlightRanges = preformattedSnippet.snippetHighlightRanges + .filter(range => range.type !== null && range.type !== undefined && + range.type === 0); // then add added text delimiters var addOffset = 0; @@ -491,7 +479,6 @@ const snippetPromise = (req, preformattedSnippet) => { addOffset += highlightEndBin.length; }); break; - } default: break; } @@ -1101,8 +1088,15 @@ function needsToParseForAddedTemplates(text, includeOpeningBraces) { // BUG: https://en.wikipedia.org/w/index.php?title=United_States&type=revision // &diff=965295364&oldid=965071033 // Should we be catching added tags? -// This one just seems to have moved paragraphs. Seems like we shouldn't return a new citation here. +// ANOTHER BUG: +// Sometimes new citations have spaces within deemed the same, so only part of it +// is sent in as text whereas we need all of cite. +// See here +// https://en.wikipedia.org/w/index.php?title=United_States&type=revision&diff=971349395&oldid=971332725 +// Also this one // https://en.wikipedia.org/w/index.php?title=United_States&type=revision&diff=971260075&oldid=971259267 +// Definitely some mangled 'added-text' snippets going on here. +// https://en.wikipedia.org/w/index.php?title=United_States&type=revision&diff=970665187&oldid=970577931 function structuredTemplatePromise(text, diffItem, revision) { return new BBPromise((resolve) => { var main = PRFunPromise.async(function*() { @@ -1211,20 +1205,9 @@ function addStructuredTemplates(diffAndRevisions) { case 5: case 3: { - var binaryText = encoding.strToBin(diffItem.text); - - // strip deleted ranges from text, this simplifies template detection later - const strippedResult = stripDeletedRangesFromBinaryText(binaryText, - diffItem.highlightRanges); - binaryText = strippedResult.strippedBinaryText; - diffItem.highlightRanges = strippedResult.addedHighlightRanges; - - // now loop through added highlighted ranges, - // fuzzying things to capture full citations - // and generate templates. + const binaryText = encoding.strToBin(diffItem.text); var previousBinaryRangeText = null; var previousRangeEndIndex = null; - for (var h = 0; h < diffItem.highlightRanges.length; h++) { const range = diffItem.highlightRanges[h]; @@ -1246,9 +1229,6 @@ function addStructuredTemplates(diffAndRevisions) { const inBetweenText = binaryText.substring( previousRangeEndIndex, range.start); - // if an empty space is all that exists between added ranges, - // lump it in with previous range in case it is splitting - // up a new citation if (inBetweenText === ' ') { previousBinaryRangeText += inBetweenText; previousBinaryRangeText += binaryRangeText; @@ -1282,9 +1262,19 @@ function addStructuredTemplates(diffAndRevisions) { continue; } } - break; - default: break; + case 1: // Deleted + // if there's a space before deleted text, include it + // so it gets picked up in inBetweenText check up there + // const textBeforeDeleted = binaryText.substring(range.start - 1, + // (range.start - 1) + 1); + // if (textBeforeDeleted === ' ') { + // previousRangeEndIndex = range.start - 1; + // } else { + // previousRangeEndIndex = range.start + range.length; + // } + + previousRangeEndIndex = range.start + range.length; } } From d776684dffa8a99a9fecd52b3d4020e2160e954f Mon Sep 17 00:00:00 2001 From: Toni Sevener Date: Fri, 28 Aug 2020 17:30:25 -0500 Subject: [PATCH 32/47] fix nested array issue with templates --- routes/page/significant-events.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/page/significant-events.js b/routes/page/significant-events.js index a5f5c5fd..f6f26a39 100644 --- a/routes/page/significant-events.js +++ b/routes/page/significant-events.js @@ -1964,7 +1964,7 @@ function getSignificantEvents(req, res) { sections.push(section); } if (templates) { - combinedTemplates.push(templates); + combinedTemplates = combinedTemplates.concat(templates); } } From 567021c991c72ee7a6576f8613a10fe109e59a89 Mon Sep 17 00:00:00 2001 From: Toni Sevener Date: Tue, 8 Sep 2020 15:14:15 -0500 Subject: [PATCH 33/47] delete unused code --- lib/snippet/Snippet.js | 102 ------------------------------ routes/page/significant-events.js | 5 -- 2 files changed, 107 deletions(-) delete mode 100644 lib/snippet/Snippet.js diff --git a/lib/snippet/Snippet.js b/lib/snippet/Snippet.js deleted file mode 100644 index 9434ce99..00000000 --- a/lib/snippet/Snippet.js +++ /dev/null @@ -1,102 +0,0 @@ -// const P = require('bluebird'); -// const Chunk = require('../html/Chunk'); -// const DocumentWorker = require('../html/DocumentWorker'); -// const tagsToRemove = new Set(['STYLE', 'SCRIPT']); -// const sectionTagNames = new Set(['SECTION']); -// /** -// * TalkPage represents a structured version of a talk page. -// * @param {!Document} doc Parsoid document -// * @param {!String} lang the language of the document -// * @param {?boolean} immediate whether or not to process the document immediately -// */ -// class Snippet extends DocumentWorker { -// constructor(doc, immediate = true) { -// super(doc, doc.body); -// this.chunks = []; -// this.firstSectionNode = undefined; -// this.firstDivNode = undefined; -// this.firstParagraphNode = undefined; -// if (!immediate) { -// return; -// } -// this.workSync(); -// this.finalizeSync(); -// } -// -// /** -// * Returns a promise that is fufilled by a TalkPage -// * @param {!Document} doc Parsoid document -// * @param {!String} lang the language of the document -// * @param {?integer} limit the limit in ms for each processing chunk -// */ -// static promise(doc) { -// const snippet = new Snippet(doc, false); -// return snippet.promise; -// } -// -// removeNodeButPreserveContents(node) { -// while (node.childNodes.length > 0) { -// const firstChild = node.firstChild; -// if (firstChild) { -// node.parentNode.insertBefore(firstChild, node); -// } -// } -// node.parentNode.removeChild(node); -// } -// /** -// * Process the document -// * @param {?integer} limit the max number of DOMNodes to process -// */ -// process(node) { -// -// if (tagsToRemove.has(node.tagName)) { -// node.parentNode.removeChild(node); -// return; -// } -// -// if (sectionTagNames.has(node.tagName) && this.firstSectionNode === undefined) { -// this.firstSectionNode = node; -// removeNodeButPreserveContents(node); -// return; -// } -// -// if (node.tagName === 'DIV' && this.firstDivNode === undefined) { -// this.firstDivNode = node; -// removeNodeButPreserveContents(node); -// return; -// } -// -// if (node.tagName === 'P' && this.firstParagraphNode === undefined) { -// this.firstParagraphNode = node; -// removeNodeButPreserveContents(node); -// return; -// } -// -// console.log(node.tagName); -// console.log(doc.body.innerHTML); -// // if (tagsToRemove.has(node.tagName)) { -// // node.remove(); -// // return; -// // } -// // const chunk = new Chunk(node, false); -// // -// // while (this.ancestor && this.ancestor !== node.parentNode) { -// // const endChunk = new Chunk(this.ancestor, true); -// // this.chunks.push(endChunk); -// // this.ancestor = this.ancestor.parentNode; -// // } -// // -// // if (chunk.isTag) { -// // this.ancestor = node; -// // this.chunks.push(chunk); -// // } else if (chunk.isText) { -// // this.chunks.push(chunk); -// // } -// } -// -// finalizeStep() { -// return false; -// } -// } -// -// module.exports = Snippet; diff --git a/routes/page/significant-events.js b/routes/page/significant-events.js index f6f26a39..825d3737 100644 --- a/routes/page/significant-events.js +++ b/routes/page/significant-events.js @@ -10,7 +10,6 @@ const encoding = require('@root/encoding'); const ParsoidJS = require('parsoid-jsapi'); const PRFunPromise = require('prfun'); const tUtil = require('../../lib/talk/TalkPageTopicUtilities'); -const Snippet = require('../../lib/snippet/Snippet'); const NodeType = require('../../lib/nodeType'); let app; @@ -2192,11 +2191,7 @@ function getSignificantEvents(req, res) { } router.get('/page/significant-events/:title', (req, res) => { - // res.status(200); return getSignificantEvents(req, res); - // const result = Object.assign({ result: "What up new endpoint."}); - // mUtil.setContentType(res, mUtil.CONTENT_TYPES.talk); - // res.json(result).end(); }); module.exports = function(appObj) { From 99a70d468323f69ed1de828a5bdec57a25e4e9d2 Mon Sep 17 00:00:00 2001 From: Toni Sevener Date: Tue, 8 Sep 2020 15:22:44 -0500 Subject: [PATCH 34/47] delete more unused code --- routes/page/significant-events.js | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/routes/page/significant-events.js b/routes/page/significant-events.js index 825d3737..5a3b69fa 100644 --- a/routes/page/significant-events.js +++ b/routes/page/significant-events.js @@ -1263,16 +1263,6 @@ function addStructuredTemplates(diffAndRevisions) { } break; case 1: // Deleted - // if there's a space before deleted text, include it - // so it gets picked up in inBetweenText check up there - // const textBeforeDeleted = binaryText.substring(range.start - 1, - // (range.start - 1) + 1); - // if (textBeforeDeleted === ' ') { - // previousRangeEndIndex = range.start - 1; - // } else { - // previousRangeEndIndex = range.start + range.length; - // } - previousRangeEndIndex = range.start + range.length; } @@ -1355,10 +1345,7 @@ function getNewTopicDiffAndRevisions(talkDiffAndRevisions) { return null; } - const newSectionTalkPageDiffAndRevisions = []; - // const talkPage = talkPageBody.query.pages; - // const talkPageObject = talkPage[Object.keys(talkPage)[0]]; - // const talkPageRevisions = talkPageObject.revisions; + const newSectionTalkPageDiffAndRevisions = []; for (var index = 0; index < talkDiffAndRevisions.length; index++) { const diffAndRevision = talkDiffAndRevisions[index]; From 7a35df8bdf7d4d691ac4f5907ab26ea5f83c6ec5 Mon Sep 17 00:00:00 2001 From: Joe Walsh Date: Wed, 9 Sep 2020 11:38:04 -0400 Subject: [PATCH 35/47] Update talkPageTitle function to getTalkPageTitle, utilize it in cleanupCache Change-Id: Ia2b76a7bfc63f34dccfb678a27ae42287c6d0c96 --- routes/page/significant-events.js | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/routes/page/significant-events.js b/routes/page/significant-events.js index 5a3b69fa..9c975899 100644 --- a/routes/page/significant-events.js +++ b/routes/page/significant-events.js @@ -566,12 +566,12 @@ const diffAndRevisionPromises = (req, revisions) => { }); }; -function talkPageTitle(req) { +function getTalkPageTitle(req) { return `Talk:${req.params.title}`; } const talkPageRevisionsPromise = (req, rvStart, rvEnd) => { - return mwapi.queryForRevisions(req, talkPageTitle(req), 100, rvStart, rvEnd ) + return mwapi.queryForRevisions(req, getTalkPageTitle(req), 100, rvStart, rvEnd ) .catch(e => { return null; }); @@ -710,7 +710,6 @@ function setSignificantChangesCache(req, title, item) { } function cleanupCache(req) { - // cleanup article cache var domainDict = significantChangesCache[req.params.domain]; const articleTitle = keyTitleForCache(req, null); @@ -767,8 +766,7 @@ function cleanupCache(req) { } // clean out from talk page cache - // todo: why on earth do we get a talkPageTitle is not a function error here? - var talkPageTitle = `Talk:${req.params.title}`; + var talkPageTitle = getTalkPageTitle(req); const keyTalkPageTitle = keyTitleForCache(req, talkPageTitle); var talkPageTitleDict = domainDict[keyTalkPageTitle]; if (talkPageTitleDict) { @@ -1661,7 +1659,7 @@ function shaFromSortedOutput(req, sortedOutput) { } if (output.outputType === 'new-talk-page-topic') { - shaTitle = talkPageTitle(req); + shaTitle = getTalkPageTitle(req); shaRevID = output.revid; break; } @@ -1849,7 +1847,7 @@ function getSignificantEvents(req, res) { response.talkPageRevisions.body.query.pages[0].revisions !== null) { const talkPageRevisions = response.talkPageRevisions.body.query.pages[0].revisions; const talkPageEvalResults = getCachedAndUncachedItems(talkPageRevisions, - req, talkPageTitle(req)); + req, getTalkPageTitle(req)); // save cached talk page revisions to finalOutput const finalOutput = response.finalOutput.concat(talkPageEvalResults.cachedOutput); @@ -2136,7 +2134,7 @@ function getSignificantEvents(req, res) { response.finalOutput.push(item); var cacheKey; if (item.outputType === 'new-talk-page-topic') { - const title = talkPageTitle(req); + const title = getTalkPageTitle(req); setSignificantChangesCache(req, title, item); } else { setSignificantChangesCache(req, null, item); From f7c292a247c7f68b3e5d0ecddc2d2de75a70bd6b Mon Sep 17 00:00:00 2001 From: Joe Walsh Date: Wed, 9 Sep 2020 11:38:26 -0400 Subject: [PATCH 36/47] Ensure significantChangesCache is updated Change-Id: I9c5f795e76d4564022cdd79a53162e5672d16bd8 --- routes/page/significant-events.js | 26 ++++++-------------------- 1 file changed, 6 insertions(+), 20 deletions(-) diff --git a/routes/page/significant-events.js b/routes/page/significant-events.js index 9c975899..54f2c11d 100644 --- a/routes/page/significant-events.js +++ b/routes/page/significant-events.js @@ -686,27 +686,13 @@ function latestAndEarliestCachedRevisionTimestamp(req, title) { } function setSignificantChangesCache(req, title, item) { - var domainDict = significantChangesCache[req.params.domain]; const keyTitle = keyTitleForCache(req, title); - if (domainDict) { - var titleDict = domainDict[keyTitle]; - if (titleDict) { - titleDict[item.revid] = item; - titleDict.maxedOut = calculateCacheForTitleIsMaxedOut(titleDict); - } else { - titleDict = {}; - titleDict[item.revid] = item; - titleDict.maxedOut = calculateCacheForTitleIsMaxedOut(titleDict); - domainDict[keyTitle] = titleDict; - } - } else { - titleDict = {}; - titleDict[item.revid] = item; - titleDict.maxedOut = calculateCacheForTitleIsMaxedOut(titleDict); - domainDict = {}; - domainDict[keyTitle] = titleDict; - significantChangesCache[req.params.domain] = domainDict; - } + var domainDict = significantChangesCache[req.params.domain] || {}; + var titleDict = domainDict[keyTitle] || {}; + titleDict[item.revid] = item; + titleDict.maxedOut = calculateCacheForTitleIsMaxedOut(titleDict); + domainDict[keyTitle] = titleDict; + significantChangesCache[req.params.domain] = domainDict; } function cleanupCache(req) { From 1c59ff761a84cded7ff818a3c068aa4fd5d5214d Mon Sep 17 00:00:00 2001 From: Toni Sevener Date: Wed, 9 Sep 2020 17:50:12 -0500 Subject: [PATCH 37/47] outputting expanded small objects instead of collapsed --- routes/page/significant-events.js | 45 +++++++++++-------------------- 1 file changed, 15 insertions(+), 30 deletions(-) diff --git a/routes/page/significant-events.js b/routes/page/significant-events.js index 54f2c11d..b8aee75d 100644 --- a/routes/page/significant-events.js +++ b/routes/page/significant-events.js @@ -75,7 +75,7 @@ class CharacterChange { } } -class SmallOutput { +class SmallOutputExtended { constructor(revid, timestamp, user, userid) { this.revid = revid; this.timestamp = timestamp; @@ -85,9 +85,10 @@ class SmallOutput { } } -class ConsolidatedSmallOutput { - constructor(count) { - this.count = count; +class SmallOutput { + constructor(smallOutputExtended) { + this.revid = smallOutputExtended.revid; + this.timestamp = smallOutputExtended.timestamp; this.outputType = 'small-change'; } } @@ -1510,10 +1511,9 @@ function cleanOutput(output) { return null; } - // collapses small changes, converts large changes only to info needed + // convert large and small changes only to info needed const cleanedOutput = []; - let numSmallChanges = 0; for (var i = 0; i < output.length; i++) { const item = output[i]; @@ -1522,32 +1522,17 @@ function cleanOutput(output) { continue; } - if (item.outputType === 'small-change') { - numSmallChanges++; - continue; + if (item.outputType === 'large-change') { + cleanedOutput.push(new LargeOutput(item)); + } else if (item.outputType === 'new-talk-page-topic') { + cleanedOutput.push(new NewTalkPageTopic(item)); + } else if (item.outputType === 'small-change') { + cleanedOutput.push(new SmallOutput(item)); } else { - if (numSmallChanges > 0) { - const change = new ConsolidatedSmallOutput(numSmallChanges); - cleanedOutput.push(change); - numSmallChanges = 0; - } - - if (item.outputType === 'large-change') { - cleanedOutput.push(new LargeOutput(item)); - } else if (item.outputType === 'new-talk-page-topic') { - cleanedOutput.push(new NewTalkPageTopic(item)); - } else { - cleanedOutput.push(item); - } + cleanedOutput.push(item); } } - if (numSmallChanges > 0) { - const change = new ConsolidatedSmallOutput(numSmallChanges); - cleanedOutput.push(change); - numSmallChanges = 0; - } - return cleanedOutput; } @@ -1893,7 +1878,7 @@ function getSignificantEvents(req, res) { // edge case in case one of the diff endpoints fail...fallback to small type if (diffAndRevision.body === undefined || diffAndRevision.body === null) { - const smallOutputObject = new SmallOutput(revision.revid, + const smallOutputObject = new SmallOutputExtended(revision.revid, revision.timestamp, revision.user, revision.userid); uncachedOutput.push(smallOutputObject); continue; @@ -1975,7 +1960,7 @@ function getSignificantEvents(req, res) { revision.timestamp, revision.user, revision.userid, significantChanges); uncachedOutput.push(largeOutputObject); } else { - const smallOutputObject = new SmallOutput(revision.revid, + const smallOutputObject = new SmallOutputExtended(revision.revid, revision.timestamp, revision.user, revision.userid); uncachedOutput.push(smallOutputObject); } From 859e4f55d9a1b09e32229fe7197e30717c988200 Mon Sep 17 00:00:00 2001 From: Toni Sevener Date: Fri, 25 Sep 2020 16:01:27 -0500 Subject: [PATCH 38/47] small section determination bugfix --- routes/page/significant-events.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/routes/page/significant-events.js b/routes/page/significant-events.js index b8aee75d..7d6c6a16 100644 --- a/routes/page/significant-events.js +++ b/routes/page/significant-events.js @@ -920,7 +920,8 @@ function getSectionForDiffLine(diffBody, diffLine) { } } - if (diffLine.offset.to) { + if (diffLine.offset.to || diffLine.offset.to === 0) { + // 0 is a valid value, but without explicit check JS sees it as invalid return toSection; } else { return fromSection; From f49da1baac90485db017addfb56c684f4774656d Mon Sep 17 00:00:00 2001 From: Toni Sevener Date: Fri, 2 Oct 2020 17:29:07 -0500 Subject: [PATCH 39/47] do not allow citation needed templates through --- routes/page/significant-events.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/routes/page/significant-events.js b/routes/page/significant-events.js index 7d6c6a16..f6cb4496 100644 --- a/routes/page/significant-events.js +++ b/routes/page/significant-events.js @@ -1063,7 +1063,10 @@ function needsToParseForAddedTemplates(text, includeOpeningBraces) { if ((text.includes(`{{${name}`) && includeOpeningBraces) || (text.includes(`${name}`) && !includeOpeningBraces)) { - return true; + + if (!text.includes('citation needed')) { + return true; + } } } From 097f8af021fcc4cc8ac0fdae0fceee1e037573a6 Mon Sep 17 00:00:00 2001 From: Toni Sevener Date: Mon, 5 Oct 2020 17:49:30 -0500 Subject: [PATCH 40/47] fix some talk page bugs causing null snippets --- routes/page/significant-events.js | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/routes/page/significant-events.js b/routes/page/significant-events.js index f6cb4496..1549497f 100644 --- a/routes/page/significant-events.js +++ b/routes/page/significant-events.js @@ -1352,9 +1352,10 @@ function getNewTopicDiffAndRevisions(talkDiffAndRevisions) { if (diffAndRevision.revision.comment.toLowerCase().includes('new section') && !diffAndRevision.revision.comment.toLowerCase() .includes('semi-protected edit request') // don't show semi-protected edit requests - && diffAndRevision.revision.userid !== 4936590) { // don't show signbot topics + && diffAndRevision.revision.userid !== 4936590 // don't show signbot topics + && !diffAndRevision.revision.tags.includes('mw-reverted')) { // don't show mw-reverted posts // see if this section was reverted in previous iterations - var wasReverted = false; + var wasUndoneOrRolledBackLater = false; if (index - 1 > 0) { var nextDiffAndRevision = talkDiffAndRevisions[index - 1]; if (nextDiffAndRevision.revision.userid === 4936590) { // signbot, @@ -1365,10 +1366,10 @@ function getNewTopicDiffAndRevisions(talkDiffAndRevisions) { } if (nextDiffAndRevision.revision.tags.includes('mw-undo') || nextDiffAndRevision.revision.tags.includes('mw-rollback')) { - wasReverted = true; + wasUndoneOrRolledBackLater = true; } } - if (wasReverted === false) { + if (wasUndoneOrRolledBackLater === false) { newSectionTalkPageDiffAndRevisions.push(diffAndRevision); } } @@ -1442,7 +1443,7 @@ function getLargestDiffLineOfAdded(diffBody) { return null; } -function textContainsEmptyLineOrSection(text) { +function textContainsEmptyLineOrSectionOrTemplate(text) { if (text === null || text === undefined) { @@ -1450,7 +1451,7 @@ function textContainsEmptyLineOrSection(text) { } const trimmedText = text.trim(); - return (trimmedText.length === 0 || text.includes('==')); + return (trimmedText.length === 0 || text.includes('==') || text.includes('{{')); } function getFirstDiffLineWithContent(diffBody) { @@ -1476,7 +1477,7 @@ function getFirstDiffLineWithContent(diffBody) { case 0: // Context line type continue; default: - if (textContainsEmptyLineOrSection(diff.text)) { + if (textContainsEmptyLineOrSectionOrTemplate(diff.text)) { continue; } else { return diff; From 9744e3b9356781a04426fddd1849d06e3f6a66db Mon Sep 17 00:00:00 2001 From: Toni Sevener Date: Tue, 6 Oct 2020 16:56:58 -0500 Subject: [PATCH 41/47] Better talk page fix to allow templates into talk page snippets if absolutely necessary --- routes/page/significant-events.js | 53 +++++++++++++++++-------------- 1 file changed, 30 insertions(+), 23 deletions(-) diff --git a/routes/page/significant-events.js b/routes/page/significant-events.js index 1549497f..70f047cf 100644 --- a/routes/page/significant-events.js +++ b/routes/page/significant-events.js @@ -1443,15 +1443,24 @@ function getLargestDiffLineOfAdded(diffBody) { return null; } -function textContainsEmptyLineOrSectionOrTemplate(text) { +function textContainsEmptyLineOrSection(text) { + if (text === null || + text === undefined) { + return false; + } + + const trimmedText = text.trim(); + return (trimmedText.length === 0 || text.includes('==')); +} + +function textContainsTemplate(text) { if (text === null || text === undefined) { return false; } - const trimmedText = text.trim(); - return (trimmedText.length === 0 || text.includes('==') || text.includes('{{')); + return text.includes('{{'); } function getFirstDiffLineWithContent(diffBody) { @@ -1463,29 +1472,27 @@ function getFirstDiffLineWithContent(diffBody) { return null; } - for (let i = 0; i < diffBody.diff.length; i++) { - const diff = diffBody.diff[i]; - - if (diff.type === null || - diff.type === undefined || - diff.text === null || - diff.text === undefined) { - continue; - } + const nonContextDiffLines = diffBody.diff.filter(diff => (diff.type !== null && + diff.type !== undefined && diff.text !== null && diff.text !== undefined + && diff.type > 0)); + const nonContextDiffLinesWithoutEmptyLineOrSectionsOrTemplates = + nonContextDiffLines.filter(diff => + !textContainsEmptyLineOrSection(diff.text) && !textContainsTemplate(diff.text)); + const nonContextDiffLinesWithoutEmptyLineOrSections = + nonContextDiffLines.filter(diff => !textContainsEmptyLineOrSection(diff.text)); - switch (diff.type) { - case 0: // Context line type - continue; - default: - if (textContainsEmptyLineOrSectionOrTemplate(diff.text)) { - continue; - } else { - return diff; - } - } + if (nonContextDiffLines.length === 0) { + return null; } - return null; + // prefer to return a line without templates, otherwise try line with templates + if (nonContextDiffLinesWithoutEmptyLineOrSectionsOrTemplates.length > 0) { + return nonContextDiffLinesWithoutEmptyLineOrSectionsOrTemplates[0]; + } else if (nonContextDiffLinesWithoutEmptyLineOrSections.length > 0) { + return nonContextDiffLinesWithoutEmptyLineOrSections[0]; + } else { + return null; + } } function sortOutput(output) { From 8ec4a48707c55e8b3e75f6f13eccaf5c69a2d10b Mon Sep 17 00:00:00 2001 From: Toni Sevener Date: Wed, 21 Oct 2020 17:59:19 -0500 Subject: [PATCH 42/47] return parentID for pushing directly to diff screen --- routes/page/significant-events.js | 38 ++++++++++++++++++++----------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/routes/page/significant-events.js b/routes/page/significant-events.js index 70f047cf..4fffeaff 100644 --- a/routes/page/significant-events.js +++ b/routes/page/significant-events.js @@ -76,8 +76,9 @@ class CharacterChange { } class SmallOutputExtended { - constructor(revid, timestamp, user, userid) { + constructor(revid, parentid, timestamp, user, userid) { this.revid = revid; + this.parentid = parentid; this.timestamp = timestamp; this.outputType = 'small-change'; this.user = user; @@ -88,14 +89,16 @@ class SmallOutputExtended { class SmallOutput { constructor(smallOutputExtended) { this.revid = smallOutputExtended.revid; + this.parentid = smallOutputExtended.parentid; this.timestamp = smallOutputExtended.timestamp; this.outputType = 'small-change'; } } class VandalismOutput { - constructor(revid, timestamp, user, userid, sections) { + constructor(revid, parentid, timestamp, user, userid, sections) { this.revid = revid; + this.parentid = parentid; this.timestamp = timestamp; this.outputType = 'vandalism-revert'; this.user = user; @@ -144,6 +147,7 @@ class DeletedTextOutput { class LargeOutput { constructor(largeOutputExpanded) { this.revid = largeOutputExpanded.revid; + this.parentid = largeOutputExpanded.parentid; this.timestamp = largeOutputExpanded.timestamp; this.outputType = 'large-change'; this.user = largeOutputExpanded.user; @@ -153,8 +157,9 @@ class LargeOutput { } class LargeOutputExpanded { - constructor(revid, timestamp, user, userid, significantChanges) { + constructor(revid, parentid, timestamp, user, userid, significantChanges) { this.revid = revid; + this.parentid = parentid; this.timestamp = timestamp; this.outputType = 'large-change'; this.user = user; @@ -164,9 +169,10 @@ class LargeOutputExpanded { } class NewTalkPageTopicExtended { - constructor(revid, timestamp, user, userid, snippet, type, highlightRanges, characterChange, - section) { + constructor(revid, parentid, timestamp, user, userid, snippet, type, highlightRanges, + characterChange, section) { this.revid = revid; + this.parentid = parentid; this.timestamp = timestamp; this.outputType = 'new-talk-page-topic'; this.snippet = snippet; @@ -182,6 +188,7 @@ class NewTalkPageTopicExtended { class NewTalkPageTopic { constructor(newTalkPageTopicExpanded) { this.revid = newTalkPageTopicExpanded.revid; + this.parentid = newTalkPageTopicExpanded.parentid; this.timestamp = newTalkPageTopicExpanded.timestamp; this.outputType = 'new-talk-page-topic'; this.snippet = newTalkPageTopicExpanded.snippet; @@ -1891,7 +1898,7 @@ function getSignificantEvents(req, res) { // edge case in case one of the diff endpoints fail...fallback to small type if (diffAndRevision.body === undefined || diffAndRevision.body === null) { const smallOutputObject = new SmallOutputExtended(revision.revid, - revision.timestamp, revision.user, revision.userid); + revision.parentid, revision.timestamp, revision.user, revision.userid); uncachedOutput.push(smallOutputObject); continue; } @@ -1908,7 +1915,7 @@ function getSignificantEvents(req, res) { diffLine)); const dedupedSections = new Set(sections); const vandalismRevertOutputObject = new VandalismOutput(revision.revid, - revision.timestamp, revision.user, revision.userid, + revision.parentid, revision.timestamp, revision.user, revision.userid, Array.from(dedupedSections)); uncachedOutput.push(vandalismRevertOutputObject); } else { @@ -1969,11 +1976,13 @@ function getSignificantEvents(req, res) { if (significantChanges.length > 0) { const largeOutputObject = new LargeOutputExpanded(revision.revid, - revision.timestamp, revision.user, revision.userid, significantChanges); + revision.parentid, revision.timestamp, revision.user, + revision.userid, significantChanges); uncachedOutput.push(largeOutputObject); } else { const smallOutputObject = new SmallOutputExtended(revision.revid, - revision.timestamp, revision.user, revision.userid); + revision.parentid, revision.timestamp, revision.user, + revision.userid); uncachedOutput.push(smallOutputObject); } } @@ -1989,15 +1998,17 @@ function getSignificantEvents(req, res) { const revision = diffAndRevision.revision; const nullSnippetTalkPageObject = new NewTalkPageTopicExtended( - revision.revid,revision.timestamp, revision.user, revision.userid, - null,null, null, null, null); + revision.revid, revision.parentid, revision.timestamp, + revision.user, revision.userid,null,null, + null, null, null); if (diffAndRevision.body !== undefined && diffAndRevision.body !== null) { const firstDiffLine = getFirstDiffLineWithContent(diffAndRevision.body); if (firstDiffLine !== undefined && firstDiffLine !== null) { const section = getSectionForDiffLine(diffAndRevision.body, firstDiffLine); const newTalkPageTopicOutputObject = new NewTalkPageTopicExtended( - revision.revid,revision.timestamp, revision.user, + revision.revid, revision.parentid, + revision.timestamp, revision.user, revision.userid, firstDiffLine.text,firstDiffLine.type, firstDiffLine.highlightRanges, @@ -2008,7 +2019,8 @@ function getSignificantEvents(req, res) { } } else { const newTalkPageTopicOutputObject = new NewTalkPageTopicExtended( - revision.revid,revision.timestamp, revision.user, revision.userid, + revision.revid, revision.parentid, + revision.timestamp, revision.user, revision.userid, null,null, null, null); uncachedOutput.push(nullSnippetTalkPageObject); } From f5c0326521d0ad674b51a73c8c970f611390f020 Mon Sep 17 00:00:00 2001 From: Toni Sevener Date: Mon, 26 Oct 2020 16:47:47 -0500 Subject: [PATCH 43/47] various bug fixes discovered from working on counts report --- routes/page/significant-events.js | 178 ++++++++++++++++-------------- 1 file changed, 94 insertions(+), 84 deletions(-) diff --git a/routes/page/significant-events.js b/routes/page/significant-events.js index 4fffeaff..df293591 100644 --- a/routes/page/significant-events.js +++ b/routes/page/significant-events.js @@ -422,7 +422,7 @@ const snippetPromise = (req, preformattedSnippet) => { case 3: // Added and deleted words in line if (preformattedSnippet.snippetHighlightRanges === null || - preformattedSnippet.snippetHighlightRanges === undefined) { + preformattedSnippet.snippetHighlightRanges === undefined) { break; } @@ -707,7 +707,7 @@ function cleanupCache(req) { // cleanup article cache var domainDict = significantChangesCache[req.params.domain]; const articleTitle = keyTitleForCache(req, null); - if (domainDict) { + if (domainDict) { var titleDict = domainDict[articleTitle]; if (titleDict) { // eslint-disable-next-line no-restricted-properties @@ -726,7 +726,7 @@ function cleanupCache(req) { return outputTypeCountsTowardsCache(outputObject.outputType); } - return false; + return false; }); const delta = significantCachedObjects.length - @@ -770,8 +770,8 @@ function cleanupCache(req) { talkPageTitleArray.pop(); // remove maxedOut element const sortedTalkPageTitleArray = talkPageTitleArray.sort(function(a, b) { - return new Date(b.timestamp) - new Date(a.timestamp); - }); + return new Date(b.timestamp) - new Date(a.timestamp); + }); for (var t = sortedTalkPageTitleArray.length - 1; t >= 0; t--) { const sortedTalkObjectToConsider = sortedTalkPageTitleArray[t]; @@ -844,10 +844,10 @@ function getSectionForDiffLine(diffBody, diffLine) { // safety if (!diffBody.from || - !diffBody.from.sections || - !diffBody.to || - !diffBody.to.sections || - !diffLine.offset) { + !diffBody.from.sections || + !diffBody.to || + !diffBody.to.sections || + !diffLine.offset) { return null; } @@ -897,7 +897,7 @@ function getSectionForDiffLine(diffBody, diffLine) { prevSection = section; } - if (!fromSection && diffLine.offset.from > 0) { + if (!fromSection && diffLine.offset.from > 0 && diffBody.from.sections.length > 0) { if (prevSection.heading) { fromSection = prevSection.heading; } @@ -920,7 +920,7 @@ function getSectionForDiffLine(diffBody, diffLine) { prevSection = section; } - if (!toSection && diffLine.offset.to > 0) { + if (!toSection && diffLine.offset.to > 0 && diffBody.to.sections.length > 0) { if (prevSection.heading) { toSection = prevSection.heading; } @@ -962,9 +962,9 @@ function updateDiffAndRevisionsWithCharacterCount(diffAndRevisions) { const diff = diffAndRevision.body.diff[d]; if (diff.type === undefined || - diff.type === null || - diff.text === undefined || - diff.text === null) { + diff.type === null || + diff.text === undefined || + diff.text === null) { continue; } @@ -994,8 +994,8 @@ function updateDiffAndRevisionsWithCharacterCount(diffAndRevisions) { if (range.start === null || range.start === undefined || - range.length === null || - range.length === undefined) { + range.length === null || + range.length === undefined) { continue; } @@ -1057,9 +1057,9 @@ function templateNamesToCallOut() { function needsToParseForAddedTemplates(text, includeOpeningBraces) { if (text === undefined || - text === null || - includeOpeningBraces === undefined || - includeOpeningBraces === null) { + text === null || + includeOpeningBraces === undefined || + includeOpeningBraces === null) { return false; } @@ -1148,9 +1148,9 @@ function structuredTemplatePromise(text, diffItem, revision) { } } const result = Object.assign( { - revision: revision, + revision: revision, diffItem: diffItem, - templates: templateObjects + templates: templateObjects }); resolve(result); }); @@ -1174,14 +1174,24 @@ function addStructuredTemplates(diffAndRevisions) { const diffAndRevision = diffAndRevisions[i]; if (diffAndRevision.body === null || - diffAndRevision.body === undefined || - diffAndRevision.body.diff === null || - diffAndRevision.body.diff === undefined || + diffAndRevision.body === undefined || + diffAndRevision.body.diff === null || + diffAndRevision.body.diff === undefined || diffAndRevision.revision === null || diffAndRevision.revision === undefined) { continue; } + const addedLinesCount = diffAndRevision.body.diff.filter(diffItem => diffItem.type === 1) + .length; + if (addedLinesCount > 500) { + // This suggests some sort of mass change that would be too much work on the server + // to parse for templates. + // bypassing template detection in this case. + // this may be a good metric to tweak to boost performance + continue; + } + for (var d = 0; d < diffAndRevision.body.diff.length; d++) { const diffItem = diffAndRevision.body.diff[d]; @@ -1234,7 +1244,7 @@ function addStructuredTemplates(diffAndRevisions) { // grab following '}}' to capture edge case templates const nextTwoCharacters = binaryText .substring(previousRangeEndIndex, - previousRangeEndIndex + 2); + previousRangeEndIndex + 2); if (nextTwoCharacters === '}}') { previousBinaryRangeText += '}}'; } @@ -1302,30 +1312,30 @@ function addStructuredTemplates(diffAndRevisions) { return Promise.all(promises) .then( (response) => { - // loop through responses, add to revision. + // loop through responses, add to revision. diffAndRevisions.forEach( (diffAndRevision) => { var templatesAndDiffItems = []; response.forEach( (item) => { - if (item.revision !== null && - item.revision !== undefined && - diffAndRevision.revision !== null && - diffAndRevision.revision !== undefined && - item.templates !== null && - item.templates !== undefined && - item.templates.length > 0) { - if (item.revision.revid === diffAndRevision.revision.revid) { - const templateAndDiffItem = Object.assign({ - templates: item.templates, - diffItem: item.diffItem - }); - - templatesAndDiffItems.push(templateAndDiffItem); - } - } - }); + if (item.revision !== null && + item.revision !== undefined && + diffAndRevision.revision !== null && + diffAndRevision.revision !== undefined && + item.templates !== null && + item.templates !== undefined && + item.templates.length > 0) { + if (item.revision.revid === diffAndRevision.revision.revid) { + const templateAndDiffItem = Object.assign({ + templates: item.templates, + diffItem: item.diffItem + }); + + templatesAndDiffItems.push(templateAndDiffItem); + } + } + }); diffAndRevision.templatesAndDiffItems = templatesAndDiffItems; }); @@ -1346,9 +1356,9 @@ function getNewTopicDiffAndRevisions(talkDiffAndRevisions) { const diffAndRevision = talkDiffAndRevisions[index]; if (diffAndRevision.revision === null || - diffAndRevision.revision === undefined || - diffAndRevision.revision.comment === null || - diffAndRevision.revision.comment === undefined || + diffAndRevision.revision === undefined || + diffAndRevision.revision.comment === null || + diffAndRevision.revision.comment === undefined || diffAndRevision.revision.userid === null || diffAndRevision.revision.userid === undefined || diffAndRevision.revision.tags === null || @@ -1387,7 +1397,7 @@ function getNewTopicDiffAndRevisions(talkDiffAndRevisions) { function getAllChangedDiffLines(diffBody) { const allChangedDiffLines = diffBody.diff.filter((item) => { - return item.type !== 0; + return item.type !== 0; }); return allChangedDiffLines; } @@ -1404,7 +1414,7 @@ function getLargestDiffLine(diffBody) { diffBody.diff.sort(function(a, b) { if (b.characterChange === null || - b.characterChange === undefined || + b.characterChange === undefined || a.characterChange === null || a.characterChange === undefined) { return 0; @@ -1463,7 +1473,7 @@ function textContainsEmptyLineOrSection(text) { function textContainsTemplate(text) { if (text === null || - text === undefined) { + text === undefined) { return false; } @@ -1505,7 +1515,7 @@ function getFirstDiffLineWithContent(diffBody) { function sortOutput(output) { if (output === null || - output === undefined) { + output === undefined) { return null; } @@ -1513,9 +1523,9 @@ function sortOutput(output) { return output.sort(function(a, b) { if (b.timestamp === null || - b.timestamp === undefined || - a.timestamp === null || - a.timestamp === undefined) { + b.timestamp === undefined || + a.timestamp === null || + a.timestamp === undefined) { return 0; } @@ -1526,7 +1536,7 @@ function sortOutput(output) { function cleanOutput(output) { if (output === null || - output === undefined) { + output === undefined) { return null; } @@ -1537,7 +1547,7 @@ function cleanOutput(output) { const item = output[i]; if (item.outputType === null || - item.outputType === undefined) { + item.outputType === undefined) { continue; } @@ -1558,7 +1568,7 @@ function cleanOutput(output) { function editCountsAndGroupsPromise(req, cleanedOutput) { if (cleanedOutput === undefined || - cleanedOutput === null) { + cleanedOutput === null) { return null; } @@ -1578,11 +1588,11 @@ function editCountsAndGroupsPromise(req, cleanedOutput) { // distribute results back into cleanedOutput if (response.body === null || - response.body === undefined || - response.body.query === null || - response.body.query === undefined || - response.body.query.users === null || - response.body.query.users === undefined) { + response.body === undefined || + response.body.query === null || + response.body.query === undefined || + response.body.query.users === null || + response.body.query.users === undefined) { cleanedOutput.forEach( (outputItem) => { if (outputItem.outputType !== 'small-change') { outputItem.userGroups = null; @@ -1597,9 +1607,9 @@ function editCountsAndGroupsPromise(req, cleanedOutput) { cleanedOutput.forEach( (outputItem) => { if (outputItem.userid !== null && - outputItem.userid !== undefined && - user.userid !== null && - user.userid !== undefined) { + outputItem.userid !== undefined && + user.userid !== null && + user.userid !== undefined) { if (outputItem.userid === user.userid) { outputItem.userGroups = user.groups; outputItem.userEditCount = user.editcount; @@ -1622,7 +1632,7 @@ function isRequestingFirstPage(req) { function shaFromSortedOutput(req, sortedOutput) { if (sortedOutput === null || - sortedOutput === undefined) { + sortedOutput === undefined) { return null; } @@ -1707,13 +1717,13 @@ function getSignificantEvents(req, res) { // talk page revisions if (response.body === undefined || - response.body === null || - response.body.query === undefined || - response.body.query === null || - response.body.query.pages === undefined || - response.body.query.pages === null || - response.body.query.pages[0] === undefined || - response.body.query.pages[0] === null || + response.body === null || + response.body.query === undefined || + response.body.query === null || + response.body.query.pages === undefined || + response.body.query.pages === null || + response.body.query.pages[0] === undefined || + response.body.query.pages[0] === null || response.body.query.pages[0].revisions === undefined || response.body.query.pages[0].revisions === null) { throw new MalformedArticleRevisionResponse(); @@ -1730,7 +1740,8 @@ function getSignificantEvents(req, res) { if (isMaxedOut && cutoff && cutoff.earliestTimestamp) { filteredRevisions = revisions.filter((revision) => { if (revision.timestamp === undefined || revision.timestamp === null || - cutoff.earliestTimestamp === undefined || cutoff.earliestTimestamp === null) { + cutoff.earliestTimestamp === undefined || + cutoff.earliestTimestamp === null) { return 0; } return new Date(revision.timestamp) > new Date(cutoff.earliestTimestamp); @@ -1771,7 +1782,8 @@ function getSignificantEvents(req, res) { const earliestRevision = filteredRevisions[filteredRevisions.length - 1]; const latestRevision = filteredRevisions[0]; if (earliestRevision.parentid !== null && earliestRevision.parentid !== undefined - && latestRevision.timestamp !== null && latestRevision.timestamp !== undefined) { + && latestRevision.timestamp !== null && + latestRevision.timestamp !== undefined) { nextRvStartId = earliestRevision.parentid; // if rvstartid is missing from query, they are fetching the first page // if they are fetching the first page, we don't want to block of @@ -1808,8 +1820,8 @@ function getSignificantEvents(req, res) { response.articleDiffAndRevisions !== null) { const articleDiffAndRevisionsNullDiff = response.articleDiffAndRevisions.filter(diffAndRevision => { - return diffAndRevision.body === undefined || diffAndRevision.body === null; - }); + return diffAndRevision.body === undefined || diffAndRevision.body === null; + }); if (articleDiffAndRevisionsNullDiff.length === response.articleDiffAndRevisions.length && @@ -1905,14 +1917,12 @@ function getSignificantEvents(req, res) { if (revision.tags !== undefined && revision.tags !== null && revision.tags.includes('mw-rollback') && - revision.comment !== undefined || revision.comment !== null && - revision.comment.toLowerCase().includes('revert') && - revision.comment.toLowerCase().includes('vandalism') && + (revision.comment !== undefined && revision.comment !== null) && + (revision.comment.toLowerCase().includes('revert') || + revision.comment.toLowerCase().includes('vandalism')) && diffAndRevision.body !== null && diffAndRevision.body !== undefined) { - const allChangedDiffLines = getAllChangedDiffLines(diffAndRevision.body); - const sections = allChangedDiffLines.map(diffLine => - getSectionForDiffLine(diffAndRevision.body, - diffLine)); + const sections = diffAndRevision.characterChangeWithSections.addedSections + .concat(diffAndRevision.characterChangeWithSections.deletedSections); const dedupedSections = new Set(sections); const vandalismRevertOutputObject = new VandalismOutput(revision.revid, revision.parentid, revision.timestamp, revision.user, revision.userid, @@ -1945,7 +1955,7 @@ function getSignificantEvents(req, res) { const dedupedSections = new Set(sections); const newReferenceOutputObject = new NewReferenceOutput(Array.from(dedupedSections), - combinedTemplates); + combinedTemplates); significantChanges.push(newReferenceOutputObject); } From e6f6f50ff58ee492f25922d1bed4af5256994747 Mon Sep 17 00:00:00 2001 From: Toni Sevener Date: Wed, 11 Nov 2020 10:06:15 -0600 Subject: [PATCH 44/47] text now meets a minimum length for truncating to avoid single space snippets --- routes/page/significant-events.js | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/routes/page/significant-events.js b/routes/page/significant-events.js index df293591..c8fce679 100644 --- a/routes/page/significant-events.js +++ b/routes/page/significant-events.js @@ -544,11 +544,16 @@ const snippetPromise = (req, preformattedSnippet) => { // all lines truncate at the last highlight end line and suffix with ... // regardless if // it's the last thing in the snippet. - truncatedSnippet = truncatedSnippet.slice(0, lastStart + 'ioshighlightend'.length); - truncatedSnippet = truncatedSnippet.concat('...'); + var newTruncatedSnippet = truncatedSnippet.slice(0, lastStart + 'ioshighlightend'.length); + newTruncatedSnippet = newTruncatedSnippet.concat('...'); if (firstStart > 0) { - truncatedSnippet = truncatedSnippet.slice(firstStart); - truncatedSnippet = '...'.concat(truncatedSnippet); + newTruncatedSnippet = newTruncatedSnippet.slice(firstStart); + newTruncatedSnippet = '...'.concat(newTruncatedSnippet); + } + + const minLengthForTruncation = 'ioshighlightstart'.length + 'ioshighlightend'.length + '...'.length + '...'.length + 20; + if (newTruncatedSnippet.length > minLengthForTruncation) { + truncatedSnippet = newTruncatedSnippet; } // UNCOMMENT THIS LINE IF HIGHLIGHTING IS NOT WORTH IT From 6a400064666ebfea1740c9b9b3dadf328cb55b4b Mon Sep 17 00:00:00 2001 From: Toni Sevener Date: Thu, 19 Nov 2020 18:12:34 -0600 Subject: [PATCH 45/47] strip out comments from section titles --- routes/page/significant-events.js | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/routes/page/significant-events.js b/routes/page/significant-events.js index c8fce679..7eb9d119 100644 --- a/routes/page/significant-events.js +++ b/routes/page/significant-events.js @@ -838,6 +838,10 @@ function getSummaryText(req) { }); } +function textWithHtmlCommentsStripped(text) { + return text.replace(/)/g, ''); +} + function getSectionForDiffLine(diffBody, diffLine) { if (!diffBody || !diffLine) { @@ -881,9 +885,9 @@ function getSectionForDiffLine(diffBody, diffLine) { if (fromSection && toSection) { if (diffLine.offset.to && diffLine.offset.to.length > 0) { - return toSection; + return textWithHtmlCommentsStripped(toSection); } else { - return fromSection; + return textWithHtmlCommentsStripped(fromSection); } } @@ -934,9 +938,9 @@ function getSectionForDiffLine(diffBody, diffLine) { if (diffLine.offset.to || diffLine.offset.to === 0) { // 0 is a valid value, but without explicit check JS sees it as invalid - return toSection; + return textWithHtmlCommentsStripped(toSection); } else { - return fromSection; + return textWithHtmlCommentsStripped(fromSection); } } From 2ba0f82d9742094ce693bc2541741b4308bb2b7b Mon Sep 17 00:00:00 2001 From: Toni Sevener Date: Thu, 19 Nov 2020 18:18:22 -0600 Subject: [PATCH 46/47] fix null break --- routes/page/significant-events.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/routes/page/significant-events.js b/routes/page/significant-events.js index 7eb9d119..f995130b 100644 --- a/routes/page/significant-events.js +++ b/routes/page/significant-events.js @@ -839,6 +839,11 @@ function getSummaryText(req) { } function textWithHtmlCommentsStripped(text) { + + if (text === null || text === undefined) { + return null; + } + return text.replace(/)/g, ''); } From 8e40440b72cb95f6f22c16b229b1cb44eee828a2 Mon Sep 17 00:00:00 2001 From: Toni Sevener Date: Mon, 30 Nov 2020 18:08:13 -0600 Subject: [PATCH 47/47] add significant-events-warmup endpoint --- routes/page/significant-events.js | 34 +++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/routes/page/significant-events.js b/routes/page/significant-events.js index f995130b..1ec68e79 100644 --- a/routes/page/significant-events.js +++ b/routes/page/significant-events.js @@ -211,8 +211,7 @@ class PreformattedSnippet { } function getThreshold(req) { - return req.query.threshold === null || req.query.threshold === undefined ? - 100 : req.query.threshold; + return 100; } function insertSubstringInString(originalString, substring, index) { @@ -2190,12 +2189,39 @@ function getSignificantEvents(req, res) { sha: response.sha, timeline: response.cleanedOutput, summary: summary }); - res.send(result).end(); + return result; }); } router.get('/page/significant-events/:title', (req, res) => { - return getSignificantEvents(req, res); + return getSignificantEvents(req, res).then( (response) => { + res.send(response).end(); + }); +}); + +var countCacheWarmupPages = 0; + +function recursiveGetSignificantEvents(req, res) { + countCacheWarmupPages++; + return getSignificantEvents(req, res).then( (response) => { + + if (!response.nextRvStartId) { + const finalWarmupCount = countCacheWarmupPages; + countCacheWarmupPages = 0; + return Object.assign({ warmedPages: finalWarmupCount, + significantChangesCache: significantChangesCache }); + } + + req.query.rvstartid = response.nextRvStartId; + return recursiveGetSignificantEvents(req, res); + }); +} + +router.get('/page/significant-events-warmup/:title', (req, res) => { + return recursiveGetSignificantEvents(req, res) + .then( (response) => { + res.send(response).end(); + }); }); module.exports = function(appObj) {