scribble lexer returns an empty token #14
Description
This was found by random testing.
Here's the (non-random) test case that illustrates the bug. The function try should never raise an error (it is a simplification of the checks done by the option in the contract). The string big-s is the one that drdr found and it shrinks to "^|{|\\".
#lang racket
(require syntax-color/scribble-lexer)
(define (try s)
(define 3ary-lexer (make-scribble-inside-lexer #:command-char #\^))
(define in (open-input-string s))
(port-count-lines! in)
(let loop ([mode #f][offset 0])
(define-values (txt type paren start end backup new-mode)
(3ary-lexer in offset mode))
(cond
[(equal? type 'eof) #t]
[else
(unless (< start end)
(error 'whoops! "start ~s end ~s" start end))
(loop new-mode end)])))
;; returns #t if the test case failed, #f if it didn't fail
;; so `s` is helpful when this returns #t
(define (try/bool s)
(with-handlers ([(λ (x) (and (exn:fail? x)
(regexp-match #rx"whoops" (exn-message x))))
(λ (x) #t)])
(try s)
#f))
(define big-s
"\"\u0099\"\u009E\u0095Ýäλ|Á5|\u008D)è_)4\u0097(¬{\u009D })Ó|@^|{E cTuÐÆ}\u009E|\\|\u0017)\u0099j\u009A")
(define (get-candidates s)
(append
(for/list ([i (in-range (string-length s))])
(string-append (substring s 0 i)
(substring s (+ i 1) (string-length s))))
(for/list ([i (in-range (string-length s))]
#:unless (equal? (string-ref s i) #\a))
(string-append (substring s 0 i)
"a"
(substring s (+ i 1) (string-length s))))))
(define (shrink s)
(let loop ([s s])
(cond
[(try/bool s)
(define candidates (get-candidates s))
(cond
[(null? candidates) s]
[else
(or (for/or ([candidate (in-list candidates)])
(loop candidate))
s)])]
[else #f])))
(shrink big-s)