fixed bug with unicode multibyte characters via StringDecoder

lib/byline.js

@@ -20,7 +20,9 @@
 
 var stream = require('stream'),
     util = require('util'),
-    timers = require('timers');
+    timers = require('timers'),
+    StringDecoder = require('string_decoder').StringDecoder;
+
 
 // convinience API
 module.exports = function(readStream, options) {
@@ -90,12 +92,13 @@ LineStream.prototype._transform = function(chunk, encoding, done) {
 
   if (Buffer.isBuffer(chunk)) {
     if (encoding == 'buffer') {
-      chunk = chunk.toString(); // utf8
       encoding = 'utf8';
     }
-    else {
-     chunk = chunk.toString(encoding);
+    if(!this.decoder){
+      this.decoder = new StringDecoder(encoding)
     }
+    chunk = this.decoder.write(chunk);
+   // chunk = chunk.toString(encoding);
   }
   this._chunkEncoding = encoding;
 

test/tests.js

@@ -117,6 +117,27 @@ describe('byline', function() {
     readFile('test/rfc_huge.txt', done);
   });
 
+  it('should not break multibyte unicode characters spanning chunks', function(done) {
+    var direct = fs.readFileSync('test/unicode.txt', 'utf8').split('\n')
+    var input = fs.createReadStream('test/unicode.txt');
+    var lineStream = byline(input, { keepEmptyLines: true });
+    lineStream.setEncoding('utf8');
+
+    var lines = [];
+    lineStream.on('data', function(line) {
+      lines.push(line);
+    });
+
+    lineStream.on('end', function() {
+      assert.equal(direct.length, lines.length);
+      lines.forEach((line, i) => {
+        var other = direct[i];
+        assert.equal(other, line)
+      })
+      done();
+    });
+  });
+
   function readFile(filename, done) {
     var input = fs.createReadStream(filename);
     var lineStream = byline(input);