From da6263aa0d2f846c3ede04e3b143765c01df1102 Mon Sep 17 00:00:00 2001 From: Teemu Ikonen Date: Wed, 28 Sep 2011 00:23:03 +0300 Subject: [PATCH 01/10] Should not be global --- lib/staticGzip.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/staticGzip.js b/lib/staticGzip.js index 2dd3f64..f481b6e 100644 --- a/lib/staticGzip.js +++ b/lib/staticGzip.js @@ -37,7 +37,7 @@ exports = module.exports = function staticGzip(root, options) { if (!matchType.test) throw new Error('option matchType must be a regular expression'); options.root = root; - rootLength = root.length; + var rootLength = root.length; return function(req, res, next) { var url, filename, type, acceptEncoding, ua; From 00f86b3f848934e7992073f4e5422d826f815b6d Mon Sep 17 00:00:00 2001 From: Chad Weider Date: Wed, 25 Jan 2012 17:15:29 -0800 Subject: [PATCH 02/10] Use builtin Zlib library. --- lib/gzip.js | 63 +++++++++++++++++++++++++++++++++++++++++----------- package.json | 2 +- 2 files changed, 51 insertions(+), 14 deletions(-) diff --git a/lib/gzip.js b/lib/gzip.js index 0e31877..e2a7746 100644 --- a/lib/gzip.js +++ b/lib/gzip.js @@ -4,8 +4,6 @@ * MIT Licensed */ -var spawn = require('child_process').spawn; - /** * Connect middleware providing gzip compression on the fly. By default, it * compresses requests with mime types that match the expression @@ -14,23 +12,58 @@ var spawn = require('child_process').spawn; * Options: * * - `matchType` Regular expression matching mime types to be compressed - * - `flags` String of flags passed to the binary. Nothing by default - * - `bin` Binary executable defaulting to "gzip" + * - `flags` DEPRECATED: String of flags passed to the binary. Nothing + * by default * * @param {Object} options * @api public */ +var zlib = require('zlib'); + +function hasOwnProperty(o, key) { + return Object.prototype.hasOwnProperty.call(o, key); +} +function selectProperties(o, key1, key2) { + var object = {}; + for (var i = 1, ii = arguments.length; i < ii; i++) { + var key = arguments[i]; + if (hasOwnProperty(o, key)) { + object[key] = o[key]; + } + } + return object; +} + +// Do some cheap parsing of flags to ease deprecation. +function optionsFromGzipFlags(flags) { + var options = {}; + flags = (flags || '').split(' '); + if (flags.indexOf('--best') > 0) { + options.level = 9; + } + if (flags.indexOf('--fast') > 0) { + options.level = 1; + } + for (var i = 0, ii = 9; i < ii; i++) { + if (flags.indexOf('-' + i) > 0) { + options.level = i; + } + } + return i; +} + exports = module.exports = function gzip(options) { var options = options || {}, matchType = options.matchType || /text|javascript|json/, - bin = options.bin || 'gzip', - flags = options.flags || ''; + gzipOptions = selectProperties(options, + 'chunkSize', 'windowBits', 'level', 'memLevel', 'strategy'); if (!matchType.test) throw new Error('option matchType must be a regular expression'); - flags = (flags) ? '-c ' + flags : '-c'; - flags = flags.split(' '); + if (options.flags) { + options = optionsFromGzipFlags(options.flags); + } return function gzip(req, res, next) { var writeHead = res.writeHead, @@ -76,28 +109,32 @@ exports = module.exports = function gzip(options) { res.setHeader('Vary', 'Accept-Encoding'); res.removeHeader('Content-Length'); - gzip = spawn(bin, flags); + var gzip = zlib.createGzip(gzipOptions); res.write = function(chunk, encoding) { - gzip.stdin.write(chunk, encoding); + gzip.write(chunk, encoding); }; res.end = function(chunk, encoding) { if (chunk) { res.write(chunk, encoding); } - gzip.stdin.end(); + gzip.end(); }; - gzip.stdout.addListener('data', function(chunk) { + gzip.addListener('data', function(chunk) { write.call(res, chunk); }); - gzip.addListener('exit', function(code) { + gzip.addListener('end', function() { res.write = write; res.end = end; res.end(); }); + + gzip.addListener('error', function(error) { + res.close(); + }); finish(); diff --git a/package.json b/package.json index a460efe..e1b9baf 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "mime": ">=0.0.1" }, "engines": { - "node": "*" + "node": ">=0.6.0" }, "devDependencies": { "expresso": ">=0.9", From 2da263c873e1b3ff13e0e3fc72e1a331a61df066 Mon Sep 17 00:00:00 2001 From: Teemu Ikonen Date: Sun, 22 Apr 2012 11:40:13 +0800 Subject: [PATCH 03/10] Reverted own change after upstream merge --- lib/staticGzip.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/staticGzip.js b/lib/staticGzip.js index 2ca1d02..43573c0 100644 --- a/lib/staticGzip.js +++ b/lib/staticGzip.js @@ -38,7 +38,7 @@ exports = module.exports = function staticGzip(root, options) { if (!matchType.test) throw new Error('option matchType must be a regular expression'); options.root = root; - var rootLength = root.length; + rootLength = root.length; return function(req, res, next) { var url, filename, type, acceptEncoding, ua; From c89de692bf9824dc35a8d972c4fb01be66b20d8d Mon Sep 17 00:00:00 2001 From: Teemu Ikonen Date: Wed, 9 May 2012 23:27:56 +0800 Subject: [PATCH 04/10] Custom version number --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index e1b9baf..a571087 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "connect-gzip", "description": "Gzip middleware for Connect. Based on implementation in Connect 0.5.9. Original source: https://github.com/senchalabs/connect/tree/c9a0c1e0e98451bb5fffb70c622b827a11bf4fc7", - "version": "0.1.5", + "version": "0.1.5-f", "author": "Nate Smith", "main": "./index.js", "dependencies": { @@ -15,4 +15,4 @@ "expresso": ">=0.9", "should": ">=0.3.1" } -} \ No newline at end of file +} From 34a291af90e5d34efd683a82950dca4524971690 Mon Sep 17 00:00:00 2001 From: Teemu Ikonen Date: Sun, 17 Jun 2012 16:15:48 +0800 Subject: [PATCH 05/10] Use internal zlib and fix race condition on multinode installations --- lib/staticGzip.js | 75 +++++++++++++++++++++++++---------------------- package.json | 2 +- 2 files changed, 41 insertions(+), 36 deletions(-) diff --git a/lib/staticGzip.js b/lib/staticGzip.js index 43573c0..127144c 100644 --- a/lib/staticGzip.js +++ b/lib/staticGzip.js @@ -8,7 +8,7 @@ var fs = require('fs'), parse = require('url').parse, path = require('path'), mime = require('mime'), - exec = require('child_process').exec, + zlib = require('zlib'), staticSend = require('connect').static.send; /** @@ -19,8 +19,8 @@ var fs = require('fs'), * Options: * * - `matchType` Regular expression matching mime types to be compressed - * - `flags` String of flags passed to the binary. Defaults to "--best" - * - `bin` Binary executable defaulting to "gzip" + * - `flags` DEPRECATED: String of flags passed to the binary. Nothing + * by default * * @param {String} root * @param {Object} options @@ -30,30 +30,28 @@ var fs = require('fs'), exports = module.exports = function staticGzip(root, options) { var options = options || {}, matchType = options.matchType || /text|javascript|json/, - bin = options.bin || 'gzip', - flags = options.flags || '--best', rootLength; if (!root) throw new Error('staticGzip root must be set'); if (!matchType.test) throw new Error('option matchType must be a regular expression'); - + options.root = root; rootLength = root.length; - + return function(req, res, next) { var url, filename, type, acceptEncoding, ua; - + if (req.method !== 'GET') return next(); - + url = parse(req.url); filename = path.join(root, url.pathname); if ('/' == filename[filename.length - 1]) filename += 'index.html'; - + type = mime.lookup(filename); if (!matchType.test(type)) { return passToStatic(filename); } - + acceptEncoding = req.headers['accept-encoding'] || ''; if (!~acceptEncoding.indexOf('gzip')) { return passToStatic(filename); @@ -63,38 +61,55 @@ exports = module.exports = function staticGzip(root, options) { if (~ua.indexOf('MSIE 6') && !~ua.indexOf('SV1')) { return passToStatic(filename); } - + // Potentially malicious path if (~filename.indexOf('..')) { return passToStatic(filename); } - + // Check for requested file fs.stat(filename, function(err, stat) { if (err || stat.isDirectory()) { return passToStatic(filename); } - + // Check for compressed file var base = path.basename(filename), dir = path.dirname(filename), gzipname = path.join(dir, base + '.' + Number(stat.mtime) + '.gz'); fs.stat(gzipname, function(err) { if (err && err.code === 'ENOENT') { - // Remove any old gz files - exec('rm ' + path.join(dir, base + '.*.gz'), function(err) { - // Gzipped file doesn't exist, so make it then send - gzip(bin, flags, filename, gzipname, function(err) { - return sendGzip(); - }); - }); + + var gzip = zlib.createGzip() + // Gzipped file doesn't exist, so make it and then send + + // First write compressed data to a temporary file. This + // avoids race condition when several node instances are + // competing for the same file and other node instance + // would try to send half-done file. + var tmpname = gzipname + '.' + process.pid + '.tmp'; + var outfile = fs.createWriteStream( tmpname ); + var infile = fs.createReadStream( filename ); + + outfile.on('close', function() { + // compressed data has been written to the temporary + // file and file descriptor is closed. + fs.rename( tmpname, gzipname, function() { + // temporary file renamed to final file + return sendGzip(); + }); + }); + + // pipe compressed data to file + infile.pipe(gzip).pipe(outfile); + } else if (err) { - return passToStatic(filename); + return passToStatic(filename); } else { - return sendGzip(); + return sendGzip(); } }); - + function sendGzip() { var charset = mime.charsets.lookup(type), contentType = type + (charset ? '; charset=' + charset : ''); @@ -104,7 +119,7 @@ exports = module.exports = function staticGzip(root, options) { passToStatic(gzipname); } }); - + function passToStatic(name) { var o = Object.create(options); o.path = name.substr(rootLength); @@ -113,13 +128,3 @@ exports = module.exports = function staticGzip(root, options) { }; }; -function gzip(bin, flags, src, dest, callback) { - var cmd = bin + ' ' + flags + ' -c ' + src + ' > ' + dest; - exec(cmd, function(err, stdout, stderr) { - if (err) { - console.error('\n' + err.stack); - fs.unlink(dest); - } - callback(err); - }); -} diff --git a/package.json b/package.json index a571087..60159a0 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "connect-gzip", "description": "Gzip middleware for Connect. Based on implementation in Connect 0.5.9. Original source: https://github.com/senchalabs/connect/tree/c9a0c1e0e98451bb5fffb70c622b827a11bf4fc7", - "version": "0.1.5-f", + "version": "0.1.5-f1", "author": "Nate Smith", "main": "./index.js", "dependencies": { From b8642dcc2a08baf814144a1c192685c550c98e60 Mon Sep 17 00:00:00 2001 From: Teemu Ikonen Date: Sun, 28 Oct 2012 14:10:44 +0800 Subject: [PATCH 06/10] Use send instead of connect --- lib/staticGzip.js | 15 +++++++++------ package.json | 4 ++-- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/lib/staticGzip.js b/lib/staticGzip.js index 127144c..7accd4b 100644 --- a/lib/staticGzip.js +++ b/lib/staticGzip.js @@ -9,7 +9,7 @@ var fs = require('fs'), path = require('path'), mime = require('mime'), zlib = require('zlib'), - staticSend = require('connect').static.send; + send = require('send'); /** * staticGzip gzips statics and then serves them with the regular Connect @@ -70,7 +70,7 @@ exports = module.exports = function staticGzip(root, options) { // Check for requested file fs.stat(filename, function(err, stat) { if (err || stat.isDirectory()) { - return passToStatic(filename); + return next(); } // Check for compressed file @@ -120,11 +120,14 @@ exports = module.exports = function staticGzip(root, options) { } }); + // send file function passToStatic(name) { - var o = Object.create(options); - o.path = name.substr(rootLength); - staticSend(req, res, next, o); + send(req, name.substr(rootLength)) + .root(options.root) + .maxage(options.maxAge) + // in case of error just pass through + .on('error', function() { next(); }) + .pipe(res); } }; }; - diff --git a/package.json b/package.json index 60159a0..8afd436 100644 --- a/package.json +++ b/package.json @@ -1,11 +1,11 @@ { "name": "connect-gzip", "description": "Gzip middleware for Connect. Based on implementation in Connect 0.5.9. Original source: https://github.com/senchalabs/connect/tree/c9a0c1e0e98451bb5fffb70c622b827a11bf4fc7", - "version": "0.1.5-f1", + "version": "0.1.5-f2", "author": "Nate Smith", "main": "./index.js", "dependencies": { - "connect": ">=1 <2", + "send": ">=0.1", "mime": ">=0.0.1" }, "engines": { From 1d0afc85d45de9106251d9b0ba4434196db55288 Mon Sep 17 00:00:00 2001 From: Teemu Ikonen Date: Tue, 29 Jan 2013 10:46:05 +0800 Subject: [PATCH 07/10] Handle HEAD requests properly --- lib/staticGzip.js | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/staticGzip.js b/lib/staticGzip.js index 7accd4b..dcfd917 100644 --- a/lib/staticGzip.js +++ b/lib/staticGzip.js @@ -41,7 +41,7 @@ exports = module.exports = function staticGzip(root, options) { return function(req, res, next) { var url, filename, type, acceptEncoding, ua; - if (req.method !== 'GET') return next(); + if (req.method !== 'GET' && req.method !== 'HEAD' ) return next(); url = parse(req.url); filename = path.join(root, url.pathname); diff --git a/package.json b/package.json index 8afd436..9f91a54 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "connect-gzip", "description": "Gzip middleware for Connect. Based on implementation in Connect 0.5.9. Original source: https://github.com/senchalabs/connect/tree/c9a0c1e0e98451bb5fffb70c622b827a11bf4fc7", - "version": "0.1.5-f2", + "version": "0.1.5-f3", "author": "Nate Smith", "main": "./index.js", "dependencies": { From 7537cfc463a95b9b80971b30fa4fa90caa78318e Mon Sep 17 00:00:00 2001 From: Teemu Ikonen Date: Tue, 19 Feb 2013 14:10:39 +0800 Subject: [PATCH 08/10] Fix: Max cache age was set to NaN in HTTP response header if not defined --- lib/staticGzip.js | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/staticGzip.js b/lib/staticGzip.js index dcfd917..d52b505 100644 --- a/lib/staticGzip.js +++ b/lib/staticGzip.js @@ -124,7 +124,7 @@ exports = module.exports = function staticGzip(root, options) { function passToStatic(name) { send(req, name.substr(rootLength)) .root(options.root) - .maxage(options.maxAge) + .maxage(options.maxAge || 0) // in case of error just pass through .on('error', function() { next(); }) .pipe(res); diff --git a/package.json b/package.json index 9f91a54..021654b 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "connect-gzip", "description": "Gzip middleware for Connect. Based on implementation in Connect 0.5.9. Original source: https://github.com/senchalabs/connect/tree/c9a0c1e0e98451bb5fffb70c622b827a11bf4fc7", - "version": "0.1.5-f3", + "version": "0.1.5-f4", "author": "Nate Smith", "main": "./index.js", "dependencies": { From 44bccfe4b3368bc6dc378a607e92ba82ff8b33dc Mon Sep 17 00:00:00 2001 From: Teemu Ikonen Date: Sat, 22 Jun 2013 14:22:06 +0800 Subject: [PATCH 09/10] Removed obsolete code --- lib/gzip.js | 47 ++++++++++++----------------------------------- package.json | 2 +- 2 files changed, 13 insertions(+), 36 deletions(-) diff --git a/lib/gzip.js b/lib/gzip.js index 9fba4f9..be65272 100644 --- a/lib/gzip.js +++ b/lib/gzip.js @@ -12,8 +12,7 @@ * Options: * * - `matchType` Regular expression matching mime types to be compressed - * - `flags` DEPRECATED: String of flags passed to the binary. Nothing - * by default + * - gzip options chunkSize, windowBits, level, memLevel, strategy, flush * * @param {Object} options * @api public @@ -35,40 +34,18 @@ function selectProperties(o, key1, key2) { return object; } -// Do some cheap parsing of flags to ease deprecation. -function optionsFromGzipFlags(flags) { - var options = {}; - flags = (flags || '').split(' '); - if (flags.indexOf('--best') > 0) { - options.level = 9; - } - if (flags.indexOf('--fast') > 0) { - options.level = 1; - } - for (var i = 0, ii = 9; i < ii; i++) { - if (flags.indexOf('-' + i) > 0) { - options.level = i; - } - } - return i; -} - exports = module.exports = function gzip(options) { var options = options || {}, matchType = options.matchType || /text|javascript|json/, gzipOptions = selectProperties(options, - 'chunkSize', 'windowBits', 'level', 'memLevel', 'strategy'); + 'chunkSize', 'windowBits', 'level', 'memLevel', 'strategy', 'flush'); if (!matchType.test) throw new Error('option matchType must be a regular expression'); - - if (options.flags) { - options = optionsFromGzipFlags(options.flags); - } - + return function gzip(req, res, next) { var writeHead = res.writeHead, defaults = {}; - + ['write', 'end'].forEach(function(name) { defaults[name] = res[name]; res[name] = function() { @@ -79,7 +56,7 @@ exports = module.exports = function gzip(options) { res[name].apply(this, arguments); }; }); - + res.writeHead = function(code) { var args = Array.prototype.slice.call(arguments, 0), write = defaults.write, @@ -91,12 +68,12 @@ exports = module.exports = function gzip(options) { res.setHeader(key, headers[key]); } } - + ua = req.headers['user-agent'] || ''; accept = req.headers['accept-encoding'] || ''; type = res.getHeader('content-type') || ''; encoding = res.getHeader('content-encoding'); - + if (req.method === 'HEAD' || code !== 200 || !~accept.indexOf('gzip') || !matchType.test(type) || encoding || (~ua.indexOf('MSIE 6') && !~ua.indexOf('SV1'))) { @@ -104,13 +81,13 @@ exports = module.exports = function gzip(options) { res.end = end; return finish(); } - + res.setHeader('Content-Encoding', 'gzip'); res.setHeader('Vary', 'Accept-Encoding'); res.removeHeader('Content-Length'); - + var gzip = zlib.createGzip(gzipOptions); - + res.write = function(chunk, encoding) { gzip.write(chunk, encoding); }; @@ -135,9 +112,9 @@ exports = module.exports = function gzip(options) { gzip.addListener('error', function(error) { res.close(); }); - + finish(); - + function finish() { res.writeHead = writeHead; res.writeHead.apply(res, args); diff --git a/package.json b/package.json index 021654b..c5315e6 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "connect-gzip", "description": "Gzip middleware for Connect. Based on implementation in Connect 0.5.9. Original source: https://github.com/senchalabs/connect/tree/c9a0c1e0e98451bb5fffb70c622b827a11bf4fc7", - "version": "0.1.5-f4", + "version": "0.1.5-f5", "author": "Nate Smith", "main": "./index.js", "dependencies": { From 4a4393a98d406fd9530df36482747a8e75b306c2 Mon Sep 17 00:00:00 2001 From: Teemu Ikonen Date: Sat, 11 Jan 2014 15:16:42 +0800 Subject: [PATCH 10/10] Send compressed length with content-length if original response has it --- lib/gzip.js | 71 +++++++++++++++++++++++++++++++++++++--------------- package.json | 2 +- 2 files changed, 52 insertions(+), 21 deletions(-) diff --git a/lib/gzip.js b/lib/gzip.js index be65272..71487b4 100644 --- a/lib/gzip.js +++ b/lib/gzip.js @@ -38,7 +38,7 @@ exports = module.exports = function gzip(options) { var options = options || {}, matchType = options.matchType || /text|javascript|json/, gzipOptions = selectProperties(options, - 'chunkSize', 'windowBits', 'level', 'memLevel', 'strategy', 'flush'); + 'chunkSize', 'windowBits', 'level', 'memLevel', 'strategy', 'flush'); if (!matchType.test) throw new Error('option matchType must be a regular expression'); @@ -84,6 +84,7 @@ exports = module.exports = function gzip(options) { res.setHeader('Content-Encoding', 'gzip'); res.setHeader('Vary', 'Accept-Encoding'); + var hasLength = res.get('Content-Length'); res.removeHeader('Content-Length'); var gzip = zlib.createGzip(gzipOptions); @@ -99,26 +100,56 @@ exports = module.exports = function gzip(options) { gzip.end(); }; - gzip.addListener('data', function(chunk) { - write.call(res, chunk); - }); - - gzip.addListener('end', function() { - res.write = write; - res.end = end; - res.end(); - }); - - gzip.addListener('error', function(error) { - res.close(); - }); - - finish(); - - function finish() { - res.writeHead = writeHead; - res.writeHead.apply(res, args); + if(hasLength) { + // if length is defined, send the compressed content as whole + var chunks = []; + gzip.addListener('data', function(chunk) { + chunks.push(chunk); + }); + + gzip.addListener('end', function() { + res.write = write; + res.end = end; + + var l = 0; + for(var i=0; i < chunks.length; i++) { + l += chunks[i].length; + } + res.setHeader('Content-Length', l); + finish(); + + for(var i=0; i < chunks.length; i++) { + res.write( chunks[i] ); + } + res.end(); + }); + + gzip.addListener('error', function(error) { + finish(); + res.close(); + }); + + } else { + gzip.addListener('data', function(chunk) { + write.call(res, chunk); + }); + + gzip.addListener('end', function() { + res.write = write; + res.end = end; + res.end(); + }); + + gzip.addListener('error', function(error) { + res.close(); + }); + + finish(); } + function finish() { + res.writeHead = writeHead; + res.writeHead.apply(res, args); + } }; next(); diff --git a/package.json b/package.json index c5315e6..6476251 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "connect-gzip", "description": "Gzip middleware for Connect. Based on implementation in Connect 0.5.9. Original source: https://github.com/senchalabs/connect/tree/c9a0c1e0e98451bb5fffb70c622b827a11bf4fc7", - "version": "0.1.5-f5", + "version": "0.1.5-f6", "author": "Nate Smith", "main": "./index.js", "dependencies": {