Fixing tokenizers to correctly handle linux line endings (\n)

src/Base/CasedTokenizer.cs

@@ -1,18 +1,7 @@
-using BERTTokenizers.Extensions;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-
-namespace BERTTokenizers.Base
+namespace BERTTokenizers.Base
 {
     public abstract class CasedTokenizer : TokenizerBase
     {
         protected CasedTokenizer(string vocabularyFilePath) : base(vocabularyFilePath) { }
-
-        protected override IEnumerable<string> TokenizeSentence(string text)
-        {
-            return text.Split(new string[] { " ", "   ", "\r\n" }, StringSplitOptions.None)
-                .SelectMany(o => o.SplitAndKeep(".,;:\\/?!#$%()=+-*\"'–_`<>&^@{}[]|~'".ToArray()));
-        }
     }
 }

src/Base/TokenizerBase.cs

@@ -1,4 +1,5 @@
-using BERTTokenizers.Helpers;
+using BERTTokenizers.Extensions;
+using BERTTokenizers.Helpers;
 using System;
 using System.Collections.Generic;
 using System.Linq;
@@ -87,6 +88,12 @@ public List<string> Untokenize(List<string> tokens)
                                 => (tokenindex.Token, tokenindex.VocabularyIndex, segmentindex)).ToList();
         }
 
+        protected virtual IEnumerable<string> TokenizeSentence(string text)
+        {
+            return text.Split(new char[0], StringSplitOptions.RemoveEmptyEntries)
+                .SelectMany(o => o.SplitAndKeep(".,;:\\/?!#$%()=+-*\"'–_`<>&^@{}[]|~'".ToArray()));
+        }
+
         private IEnumerable<long> SegmentIndex(List<(string token, int index)> tokens)
         {
             var segmentIndex = 0;
@@ -152,7 +159,5 @@ private IEnumerable<long> SegmentIndex(List<(string token, int index)> tokens)
 
             return tokens;
         }
-
-        protected abstract IEnumerable<string> TokenizeSentence(string text);
     }
 }

src/Base/UncasedTokenizer.cs

@@ -1,6 +1,4 @@
-using BERTTokenizers.Extensions;
-using System;
-using System.Collections.Generic;
+using System.Collections.Generic;
 using System.Linq;
 
 namespace BERTTokenizers.Base
@@ -13,9 +11,7 @@ protected UncasedTokenizer(string vocabularyFilePath) : base(vocabularyFilePath)
 
         protected override IEnumerable<string> TokenizeSentence(string text)
         {
-            return text.Split(new string[] { " ", "   ", "\r\n" }, StringSplitOptions.None)
-                .SelectMany(o => o.SplitAndKeep(".,;:\\/?!#$%()=+-*\"'–_`<>&^@{}[]|~'".ToArray()))
-                .Select(o => o.ToLower());
+            return base.TokenizeSentence(text).Select(o => o.ToLower());
         }
     }
 }

tests/BERTTokenizers.Tests.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>

tests/BertBaseTokenizerUncasedShould.cs

@@ -25,7 +25,20 @@ public void Tokenize_sentence()
             Assert.Equal(("love", 2293, 0), tokens[2]);
             Assert.Equal(("you", 2017, 0), tokens[3]);
             Assert.Equal(("[SEP]", 102, 0), tokens[4]);
+        }
+
+        [Fact]
+        public void Tokenize_text_with_linux_line_endings()
+        {
+            var sentence = "Linux\nline\nendings";
 
+            var tokens = _tokenizer.Tokenize(sentence);
+            Assert.Equal(5, tokens.Count);
+            Assert.Equal(("[CLS]", 101, 0), tokens[0]);
+            Assert.Equal(("linux", 11603, 0), tokens[1]);
+            Assert.Equal(("line", 2240, 0), tokens[2]);
+            Assert.Equal(("endings", 21306, 0), tokens[3]);
+            Assert.Equal(("[SEP]", 102, 0), tokens[4]);
         }
 
         [Fact]
@@ -61,7 +74,7 @@ public void Encode_sentence()
         }
 
         [Fact]
-        public void Unokenize_sentence()
+        public void Untokenize_sentence()
         {
             var tokens = new List<string>(){ "she", "##s" };