From 750033fe0c1708706159900ae5dcf4207340991d Mon Sep 17 00:00:00 2001 From: steely_phil Date: Thu, 29 Apr 2021 11:22:33 -0500 Subject: [PATCH 1/4] paginate to return items, getAll to use concurrency --- .gitignore | 3 ++- index.js | 32 ++++++++++++++++---------------- package-lock.json | 32 ++++++++++++++++++++++++++++++-- package.json | 11 ++++++++--- 4 files changed, 56 insertions(+), 22 deletions(-) diff --git a/.gitignore b/.gitignore index 40b878d..d18d285 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ -node_modules/ \ No newline at end of file +node_modules/ +test.js diff --git a/index.js b/index.js index 6e22394..fbd9f49 100644 --- a/index.js +++ b/index.js @@ -1,3 +1,4 @@ +const BlueBirdProm = require('bluebird') const fetch = require('node-fetch') const querystring = require('querystring') @@ -51,19 +52,22 @@ module.exports = class { * @param {eachPage} eachPage Callback for each page provided by endpoint * @param {object} queries Object w/ keys & string values of each url query parameter (example: {sku:'10205'}). Page & limit can be passed to control start & page size. */ - async paginate(endpoint, eachPage, queries={}){ - const page = await this.get(endpoint, queries) - const current = this.meta.pagination.current_page - let total = this.meta.pagination.total_pages - await eachPage(page) + async paginate(endpoint, queries={}){ + await this.get(endpoint, queries) + let current = this.meta.pagination.current_page + const total = this.meta.pagination.total_pages - if (this.debug) console.log('CURRENT PAGE:', current, 'TOTAL PAGES:', total) - if (current === total) return - for (let current_page = current + 1; current_page <= total; current_page++){ - queries.page = current_page - await eachPage(await this.get(endpoint, queries)) - if (this.debug) console.log('CURRENT PAGE:', current_page, 'TOTAL PAGES:', total) + if (this.debug) console.log('CURRENT PAGE:', current, 'TOTAL PAGES:', total); + + let res = []; + while(current < total) { + current++; + queries.page = current; + res.push(this.get(endpoint, queries)); + if (this.debug) console.log('CURRENT PAGE:', current, 'TOTAL PAGES:', total); } + + return res; } /** @@ -74,11 +78,7 @@ module.exports = class { * @param {object} queries Object w/ keys & string values of each url query parameter (example: {sku:'10205'}) */ async getAll(endpoint, queries={}){ - let result = [] - await this.paginate(endpoint, (items)=>{ - result = result.concat(items) - }, queries) - return result + return BlueBirdProm.map(this.paginate(endpoint, queries), (prom) => prom, { concurrency: 3 }) } /** diff --git a/package-lock.json b/package-lock.json index 473b076..3ca2c72 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,9 +1,37 @@ { "name": "bigcommerce-client", - "version": "1.0.0", - "lockfileVersion": 1, + "version": "1.2.2", + "lockfileVersion": 2, "requires": true, + "packages": { + "": { + "version": "1.2.2", + "license": "ISC", + "dependencies": { + "bluebird": "^3.7.2", + "node-fetch": "^2.6.1" + } + }, + "node_modules/bluebird": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", + "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==" + }, + "node_modules/node-fetch": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", + "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==", + "engines": { + "node": "4.x || >=6.0.0" + } + } + }, "dependencies": { + "bluebird": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", + "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==" + }, "node-fetch": { "version": "2.6.1", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", diff --git a/package.json b/package.json index 15bca83..f096747 100644 --- a/package.json +++ b/package.json @@ -2,11 +2,15 @@ "name": "bigcommerce-client", "version": "1.2.2", "description": "Simple API client for the BigCommerce Management API", - "keywords":["BigCommerce", "API", "Client"], + "keywords": [ + "BigCommerce", + "API", + "Client" + ], "main": "index.js", "repository": { - "type" : "git", - "url" : "https://github.com/hunterashaw/bigcommerce-client.git" + "type": "git", + "url": "https://github.com/hunterashaw/bigcommerce-client.git" }, "scripts": { "test": "echo \"Error: no test specified\" && exit 1" @@ -14,6 +18,7 @@ "author": "Hunter Shaw", "license": "ISC", "dependencies": { + "bluebird": "^3.7.2", "node-fetch": "^2.6.1" } } From 84a570c2916f36fee553709d7fb5393d46d2be03 Mon Sep 17 00:00:00 2001 From: steely_phil Date: Thu, 29 Apr 2021 11:41:30 -0500 Subject: [PATCH 2/4] Paginate with almost no state mutation --- index.js | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/index.js b/index.js index fbd9f49..116b210 100644 --- a/index.js +++ b/index.js @@ -53,21 +53,17 @@ module.exports = class { * @param {object} queries Object w/ keys & string values of each url query parameter (example: {sku:'10205'}). Page & limit can be passed to control start & page size. */ async paginate(endpoint, queries={}){ - await this.get(endpoint, queries) - let current = this.meta.pagination.current_page - const total = this.meta.pagination.total_pages - - if (this.debug) console.log('CURRENT PAGE:', current, 'TOTAL PAGES:', total); - - let res = []; - while(current < total) { - current++; + const _paginate = (accum = [], current) => { + const total = this.meta.pagination.total_pages; queries.page = current; - res.push(this.get(endpoint, queries)); + if (this.debug) console.log('CURRENT PAGE:', current, 'TOTAL PAGES:', total); - } - return res; + if(current < total) { + return _paginate([...accum, this.get(endpoint, queries)], current + 1) + } else return accum; + } + return _paginate([await this.get(endpoint, queries)], 2); } /** From 9e947e5c1b1f3d88783e8694864ce915332a385e Mon Sep 17 00:00:00 2001 From: steely_phil Date: Thu, 29 Apr 2021 11:43:33 -0500 Subject: [PATCH 3/4] Paginate with almost no state mutation --- index.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/index.js b/index.js index 116b210..573715b 100644 --- a/index.js +++ b/index.js @@ -55,7 +55,7 @@ module.exports = class { async paginate(endpoint, queries={}){ const _paginate = (accum = [], current) => { const total = this.meta.pagination.total_pages; - queries.page = current; + queries.page = current + 1; if (this.debug) console.log('CURRENT PAGE:', current, 'TOTAL PAGES:', total); @@ -63,7 +63,7 @@ module.exports = class { return _paginate([...accum, this.get(endpoint, queries)], current + 1) } else return accum; } - return _paginate([await this.get(endpoint, queries)], 2); + return _paginate([await this.get(endpoint, queries)], 1); } /** From 1ac8eb5664406a7cb3257e7efb67b8afbc054945 Mon Sep 17 00:00:00 2001 From: steely_phil Date: Thu, 29 Apr 2021 12:11:52 -0500 Subject: [PATCH 4/4] Paginate with almost no state mutation --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index 573715b..4407db2 100644 --- a/index.js +++ b/index.js @@ -52,7 +52,7 @@ module.exports = class { * @param {eachPage} eachPage Callback for each page provided by endpoint * @param {object} queries Object w/ keys & string values of each url query parameter (example: {sku:'10205'}). Page & limit can be passed to control start & page size. */ - async paginate(endpoint, queries={}){ + async paginate(endpoint, queries={}) { const _paginate = (accum = [], current) => { const total = this.meta.pagination.total_pages; queries.page = current + 1;