Correctly look for end delimiter dollar quoted string

[hansott]

Currently the tokenizer throws an error for

SELECT $abc$x$ab$abc$

The logic is also quite difficult to read so I made it a bit simpler.

Before

'searching_for_end: loop {
    s.push_str(&peeking_take_while(chars, |ch| ch != '$'));
    match chars.peek() {
        Some('$') => {
            chars.next();
            let mut maybe_s = String::from("$");
            for c in value.chars() {
                if let Some(next_char) = chars.next() {
                    maybe_s.push(next_char);
                    if next_char != c {
                        // This doesn't match the dollar quote delimiter so this
                        // is not the end of the string.
                        s.push_str(&maybe_s);
                        continue 'searching_for_end;
                    }
                } else {
                    return self.tokenizer_error(
                        chars.location(),
                        "Unterminated dollar-quoted, expected $",
                    );
                }
            }
            if chars.peek() == Some(&'$') {
                chars.next();
                maybe_s.push('$');
                // maybe_s matches the end delimiter
                break 'searching_for_end;
            } else {
                // This also doesn't match the dollar quote delimiter as there are
                // more characters before the second dollar so this is not the end
                // of the string.
                s.push_str(&maybe_s);
                continue 'searching_for_end;
            }
        }
        _ => {
            return self.tokenizer_error(
                chars.location(),
                "Unterminated dollar-quoted, expected $",
            )
        }
    }
}

After

let mut temp = String::new();
let end_delimiter = format!("${}$", value);

loop {
    match chars.next() {
        Some(ch) => {
            temp.push(ch);

            if temp.ends_with(&end_delimiter) {
                s.push_str(&temp[..temp.len() - end_delimiter.len()]);
                break;
            }
        }
        None => {
            if temp.ends_with(&end_delimiter) {
                s.push_str(&temp[..temp.len() - end_delimiter.len()]);
                break;
            }

            return self.tokenizer_error(
                chars.location(),
                "Unterminated dollar-quoted, expected $",
            );
        }
    }
}

[hansott]

Adding some more tests first

[hansott]

@iffyio let me know if see some improvements

[iffyio]

Looks good to me overall thanks @hansott! Left one comment regarding the suffix matching

[iffyio]

ah I think indexing based on String::len won't necessarily produce correct results on arbitrary UTF8 strings, we could something like strip_suffix that doesn't rely on the byte count? e.g

if let Some(temp) = temp.strip_suffix(&end_delimiter) {
    s.push_str(temp);
    break
}

[hansott]

@iffyio Updated, thanks for your feedback! 🥇

[iffyio]

LGTM! Thanks @hansott!
cc @alamb