Skip to content

Commit 9cf8e5d

Browse files
dszakallasdszakallas
committed
feat: add exponential backoff to API client
1 parent b463c22 commit 9cf8e5d

File tree

13 files changed

+422
-51
lines changed

13 files changed

+422
-51
lines changed

‎lib/agent/api/httpError.js‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
'use strict'
2+
3+
var http = require('http')
4+
var inherits = require('util').inherits
5+
6+
function HttpError (statusCode, response) {
7+
Error.captureStackTrace && Error.captureStackTrace(this, this.constructor)
8+
this.message = String(statusCode) + ' - ' + http.STATUS_CODES[statusCode]
9+
this.statusCode = statusCode
10+
this.response = response
11+
}
12+
inherits(HttpError, Error)
13+
14+
module.exports = HttpError

‎lib/agent/api/httpRetry.js‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
'use strict'
2+
3+
var HttpError = require('./httpError')
4+
var exponentialRetry = require('../../utils/exponentialRetry')
5+
6+
var DEFAULT_MAX_RETRIES = Infinity
7+
var DEFAULT_MAX_WAIT = 10 * 60 * 1000
8+
var DEFAULT_EXP_SCALE = 0.828
9+
var DEFAULT_LIN_SCALE = 150
10+
var DEFAULT_ERR_SCALE = 0.24 // +-12% error
11+
12+
function httpRetry (opts, cb) {
13+
opts = opts || {}
14+
var client = opts.client
15+
var payload = opts.payload
16+
var reqOpts = opts.reqOpts
17+
var errorFilter = opts.errorFilter
18+
var maxRetries = opts.maxRetries != null ? opts.maxRetries : DEFAULT_MAX_RETRIES
19+
var maxWait = opts.maxWait != null ? opts.maxWait : DEFAULT_MAX_WAIT
20+
21+
function httpRequest (cb) {
22+
var completed = false
23+
var req = client.request(reqOpts, function (response) {
24+
completed = true
25+
if (response.statusCode >= 400) {
26+
return cb(new HttpError(response.statusCode), response)
27+
}
28+
return cb(null, response)
29+
})
30+
req.on('error', function (err) {
31+
if (!completed) {
32+
completed = true
33+
return cb(err)
34+
}
35+
})
36+
if (payload) {
37+
req.write(payload)
38+
}
39+
req.end()
40+
}
41+
return exponentialRetry({
42+
maxRetries: maxRetries,
43+
maxWait: maxWait,
44+
expScale: DEFAULT_EXP_SCALE,
45+
linScale: DEFAULT_LIN_SCALE,
46+
errScale: DEFAULT_ERR_SCALE,
47+
errorFilter: errorFilter
48+
}, httpRequest, cb)
49+
}
50+
51+
module.exports = httpRetry

‎lib/agent/api/httpRetry.spec.js‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
'use strict'
2+
3+
var http = require('http')
4+
var HttpError = require('./httpError')
5+
var httpRetry = require('./httpRetry')
6+
var expect = require('chai').expect
7+
var nock = require('nock')
8+
var bl = require('bl')
9+
10+
describe('httpRetry', function (done) {
11+
it('retries', function (done) {
12+
nock('http://something.com')
13+
.post('/', 'data')
14+
.reply(500)
15+
nock('http://something.com')
16+
.post('/', 'data')
17+
.reply(200, 'response')
18+
19+
this.sandbox.stub(global, 'setTimeout').callsFake(function (cb, int) {
20+
return process.nextTick(cb)
21+
})
22+
23+
httpRetry({
24+
client: http,
25+
maxRetries: 1,
26+
reqOpts: {
27+
hostname: 'something.com',
28+
method: 'POST',
29+
path: '/'
30+
},
31+
payload: 'data'
32+
}, function (err, data) {
33+
expect(err).to.not.exist
34+
data.pipe(bl(function (err, data) {
35+
expect(err).not.to.exist
36+
expect(data.toString()).to.eql('response')
37+
done()
38+
}))
39+
})
40+
})
41+
it('returns error', function (done) {
42+
nock('http://something.com')
43+
.post('/', 'data')
44+
.reply(500, 'bad')
45+
46+
this.sandbox.stub(global, 'setTimeout').callsFake(function (cb, int) {
47+
return process.nextTick(cb)
48+
})
49+
50+
httpRetry({
51+
client: http,
52+
maxRetries: 0,
53+
reqOpts: {
54+
hostname: 'something.com',
55+
method: 'POST',
56+
path: '/'
57+
},
58+
payload: 'data'
59+
}, function (err, data) {
60+
expect(err).to.be.instanceof(HttpError)
61+
data.pipe(bl(function (err, data) {
62+
expect(err).to.not.exist
63+
expect(data.toString()).to.eql('bad')
64+
done()
65+
}))
66+
})
67+
})
68+
})

‎lib/agent/api/index.js‎

Lines changed: 30 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,12 @@ var util = require('util')
55
var requestSync = require('sync-request')
66
var isNumber = require('lodash.isnumber')
77
var debug = require('../../utils/debug')('api')
8-
var format = require('util').format
98
var assign = require('lodash.assign')
109
var HttpsProxyAgent = require('https-proxy-agent')
1110
var stringify = require('json-stringify-safe')
1211
var BufferStream = require('./bufferStream')
12+
var httpRetry = require('./httpRetry')
13+
var CompositeError = require('../../utils/compositeError')
1314

1415
var bl = require('bl')
1516
var libPackage = require('../../../package')
@@ -34,6 +35,7 @@ function CollectorApi (options) {
3435
this.serviceName = options.serviceName
3536
this.baseRetryInterval = 1000 * 60 * 30 // 30 minutes
3637
this.serviceKey = null
38+
this.getServiceMaxRetries = Infinity
3739

3840
if (options.proxy) {
3941
this.proxyAgent = new HttpsProxyAgent(options.proxy)
@@ -270,7 +272,8 @@ CollectorApi.prototype.getService = function (cb) {
270272
cpus: self.system.cpus
271273
}
272274
})
273-
var req = https.request({
275+
276+
var reqOpts = {
274277
hostname: opts.hostname,
275278
port: opts.port,
276279
path: opts.path,
@@ -283,51 +286,41 @@ CollectorApi.prototype.getService = function (cb) {
283286
'X-Reporter-Language': this.collectorLanguage,
284287
'Content-Length': Buffer.byteLength(payload)
285288
}
286-
}, function (res) {
287-
res.setEncoding('utf8')
288-
res.pipe(bl(function (err, resBuffer) {
289-
var response
290-
291-
var retryInterval = self.baseRetryInterval
289+
}
292290

293-
if (err) {
294-
debug.error('getService', err)
295-
return setTimeout(function () {
296-
debug.warn('getService', format('Retrying with %d ms', retryInterval))
297-
self.getService()
298-
}, retryInterval)
291+
httpRetry({
292+
client: https,
293+
reqOpts: reqOpts,
294+
payload: payload,
295+
errorFilter: function shouldContinueRetrying (err) {
296+
if (err.statusCode === 401) {
297+
return false
299298
}
300-
301-
var resText = resBuffer.toString('utf8')
302-
303-
if (res.statusCode === 401) {
304-
debug.error('getService', 'Api key rejected')
305-
return
299+
return true
300+
},
301+
maxRetries: this.getServiceMaxRetries
302+
},
303+
function done (err, result) {
304+
if (err) {
305+
if (err.statusCode === 401) {
306+
debug.error('getService', 'API key rejected')
306307
}
307-
if (res.statusCode > 399) {
308-
debug.error('getService', 'Service responded with ' + res.statusCode)
309-
return setTimeout(function () {
310-
debug.warn('getService', format('Retrying with %d ms', retryInterval))
311-
self.getService(cb)
312-
}, retryInterval)
308+
return cb(new CompositeError('Could not get service key', err))
309+
}
310+
var response
311+
result.pipe(bl(function (err, data) {
312+
if (err) {
313+
cb(err)
313314
}
314-
315315
try {
316-
response = JSON.parse(resText)
317-
} catch (ex) {
318-
return
316+
response = JSON.parse(data.toString('utf8'))
317+
} catch (err) {
318+
return cb(err)
319319
}
320-
321320
self.serviceKey = response.key
322321
cb(null, response.key)
323322
}))
324323
})
325-
326-
req.on('error', function (error) {
327-
debug.error('getService', error)
328-
})
329-
req.write(payload)
330-
req.end()
331324
}
332325

333326
function logServiceKeyError (method) {

‎lib/agent/api/index.spec.js‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -275,16 +275,18 @@ describe('The Trace CollectorApi module', function () {
275275
}
276276
})
277277
.post(defaultConfig.collectorApiServiceEndpoint, JSON.stringify(data))
278-
.times(2)
278+
.times(100)
279279
.reply(409, {})
280280

281-
collectorApi.getService()
282-
283-
collectorApi.baseRetryInterval = 1
281+
collectorApi.getServiceMaxRetries = 100
284282

285-
this.timeout(500)
283+
global.setTimeout = this.sandbox.stub(global, 'setTimeout').callsFake(function (cb, int) {
284+
return process.nextTick(cb)
285+
})
286286

287-
this.sandbox.stub(collectorApi, 'getService').callsFake(function () {
287+
collectorApi.getService(function (err) {
288+
expect(setTimeout).to.have.callCount(100)
289+
expect(err).to.exist
288290
done()
289291
})
290292
})

‎lib/agent/index.js‎

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -116,17 +116,12 @@ Agent.prototype.stop = function (callback) {
116116
debug.info('stop', 'Stopping agents...')
117117
var agents = this.agents
118118
var counter = 1
119-
var error
120119

121120
agents.forEach(function (agent) {
122-
agent.stop(function (err) {
123-
if (!error && err) {
124-
error = err
125-
}
126-
121+
agent.stop(function () {
127122
if (counter >= agents.length) {
128123
if (callback && typeof callback === 'function') {
129-
callback(error)
124+
callback()
130125
}
131126
} else {
132127
counter++

‎lib/utils/compositeError.js‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
'use strict'
2+
3+
var inherits = require('util').inherits
4+
5+
function CompositeError (message, cause) {
6+
if (message instanceof Error) {
7+
message = ''
8+
cause = message
9+
}
10+
this.message = message ? message.toString() : ''
11+
this.cause = cause
12+
Error.captureStackTrace && Error.captureStackTrace(this, this.constructor)
13+
if (this.stack != null && this.cause instanceof Error && this.cause.stack != null) {
14+
this.stack += '\nCaused by: ' + this.cause.stack
15+
}
16+
}
17+
inherits(CompositeError, Error)
18+
19+
module.exports = CompositeError

‎lib/utils/exponentialRetry.js‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
'use strict'
2+
3+
var inherits = require('util').inherits
4+
var retry = require('async/retry')
5+
6+
var DEFAULT_MAX_RETRIES = Infinity
7+
var DEFAULT_MAX_WAIT = Infinity
8+
var DEFAULT_EXP_SCALE = 1
9+
var DEFAULT_LIN_SCALE = 1
10+
var DEFAULT_TRANS = 0
11+
var DEFAULT_ERR_SCALE = 0
12+
var DEFAULT_ERR_TRANS = 0
13+
14+
function MaxRetriesExceededError (n, last) {
15+
Error.captureStackTrace && Error.captureStackTrace(this, this.constructor)
16+
this.message = 'Network request max retry limit reached after ' + n + ' attempts. Last error message was: ' + last.message
17+
if (this.stack && last.stack) {
18+
this.stack += '\nCaused by: ' + last.stack
19+
}
20+
}
21+
inherits(MaxRetriesExceededError, Error)
22+
23+
function exponentialRetry (opts, task, cb) {
24+
if (typeof opts === 'function') {
25+
cb = task
26+
task = opts
27+
opts = {}
28+
}
29+
opts = opts || {}
30+
var maxRetries = opts.maxRetries != null ? opts.maxRetries : DEFAULT_MAX_RETRIES
31+
var maxWait = opts.maxWait != null ? opts.maxWait : DEFAULT_MAX_WAIT
32+
var expScale = opts.expScale != null ? opts.expScale : DEFAULT_EXP_SCALE
33+
var linScale = opts.linScale != null ? opts.linScale : DEFAULT_LIN_SCALE
34+
var trans = opts.trans != null ? opts.trans : DEFAULT_TRANS
35+
var errScale = opts.errScale != null ? opts.errScale : DEFAULT_ERR_SCALE
36+
var errTrans = opts.errTrans != null ? opts.errTrans : DEFAULT_ERR_TRANS
37+
var errorFilter = opts.errorFilter
38+
39+
return retry({
40+
times: maxRetries + 1,
41+
errorFilter: errorFilter,
42+
interval: function (i) {
43+
var wait = Math.exp((i - 1) * expScale) * linScale + trans
44+
if (wait > maxWait) {
45+
wait = maxWait
46+
}
47+
var rnd = 0.5 - Math.random()
48+
wait = wait + (wait * rnd * errScale) + errTrans
49+
var res = Math.floor(wait)
50+
return res
51+
}
52+
}, task, cb)
53+
}
54+
55+
module.exports = exponentialRetry
56+
module.exports.MaxRetriesExceededError = MaxRetriesExceededError

0 commit comments

Comments
 (0)