Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Gemfile
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ source 'https://rubygems.org'
gemspec

gem 'json', '~>2.0', platforms: %i[mri_18 jruby]
gem 'logger'
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Added to fix this warning:

warning: logger was loaded from the standard library, but will no longer be part of the default gems starting from Ruby 3.5.0.


gem 'rdf', '~>3.0'
gem 'rdf-vocab', '~>3.0'
Expand Down
2 changes: 2 additions & 0 deletions Gemfile.lock
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ GEM
json (2.7.1-java)
latex-decode (0.4.0)
link_header (0.0.8)
logger (1.7.0)
mini_mime (1.1.5)
minitest (5.20.0)
multi_test (1.1.0)
Expand Down Expand Up @@ -99,6 +100,7 @@ DEPENDENCIES
gnuplot
iconv
json (~> 2.0)
logger
minitest
rake
rdf (~> 3.0)
Expand Down
16 changes: 11 additions & 5 deletions lib/bibtex/lexer.rb
Original file line number Diff line number Diff line change
Expand Up @@ -58,8 +58,8 @@ class Lexer
string: /string/io,
comment: /comment\b/io,
preamble: /preamble\b/io,
key: %r{\s*[[:alpha:][:digit:] /:_!$\?\.%+;&\*'"-]+,}io,
optional_key: %r{\s*[[:alpha:][:digit:] /:_!$\?\.%+;&\*'"-]*,}io
key: %r{\s*[[:alpha:][:digit:] /:_!$\?\.%+;&\*'"{}-]+,}io,
optional_key: %r{\s*[[:alpha:][:digit:] /:_!$\?\.%+;&\*'"{}-]*,}io
}

MODE = Hash.new(:meta).merge(
Expand Down Expand Up @@ -120,7 +120,7 @@ def next_token
@stack.shift
end

# Returns true if the lexer is currenty parsing a BibTeX object.
# Returns true if the lexer is currently parsing a BibTeX object.
def bibtex_mode?
MODE[@mode] == :bibtex
end
Expand Down Expand Up @@ -302,20 +302,26 @@ def enter_object
@mode = @active_object = :entry
push [:NAME, @scanner.matched]

# TODO: DRY - try to parse key
if @scanner.scan(Lexer.patterns[:lbrace])
@brace_level += 1
push([:LBRACE, '{'])
@mode = :content if @brace_level > 1 || @brace_level == 1 && active?(:comment)

push [:KEY, @scanner.matched.chop.strip] if @scanner.scan(Lexer.patterns[allow_missing_keys? ? :optional_key : :key])
parse_key
end

else
error_unexpected_object
end
end

def parse_key
return unless @scanner.scan(Lexer.patterns[allow_missing_keys? ? :optional_key : :key])

key = @scanner.matched.chop.strip
push [:KEY, key]
end

# Called when parser leaves a BibTeX object.
def leave_object
@mode = :meta
Expand Down
18 changes: 11 additions & 7 deletions test/bibtex/test_lexer.rb
Original file line number Diff line number Diff line change
Expand Up @@ -7,33 +7,37 @@ class LexerTest < Minitest::Spec
end

it 'strips line breaks by default' do
Lexer.new.analyse(%(@string{ x = "foo\nbar" })).stack[-3].must_be :==,
_(Lexer.new.analyse(%(@string{ x = "foo\nbar" })).stack[-3]).must_be :==,
[:STRING_LITERAL, 'foo bar']
end

it 'strips whitespace after line breaks by default' do
Lexer.new.analyse(%(@string{ x = "foo\n bar" })).stack[-3].must_be :==,
_(Lexer.new.analyse(%(@string{ x = "foo\n bar" })).stack[-3]).must_be :==,
[:STRING_LITERAL, 'foo bar']
end

it 'matches KEY tokens' do
Lexer.new.analyse('@misc{foo, }').symbols.must_be :==, [:AT, :NAME, :LBRACE, :KEY, :RBRACE, false]
_(Lexer.new.analyse('@misc{foo, }').symbols).must_be :==, [:AT, :NAME, :LBRACE, :KEY, :RBRACE, false]
end

it 'matches KEY tokens with non-ascii characters' do
Lexer.new.analyse('@misc{löwe, }').symbols.must_be :==, [:AT, :NAME, :LBRACE, :KEY, :RBRACE, false]
_(Lexer.new.analyse('@misc{löwe, }').symbols).must_be :==, [:AT, :NAME, :LBRACE, :KEY, :RBRACE, false]
end

it 'matches KEY tokens after whitespace' do
Lexer.new.analyse('@misc{ foo, }').symbols.must_be :==, [:AT, :NAME, :LBRACE, :KEY, :RBRACE, false]
_(Lexer.new.analyse('@misc{ foo, }').symbols).must_be :==, [:AT, :NAME, :LBRACE, :KEY, :RBRACE, false]
end

it 'matches KEY tokens with braces' do
_(Lexer.new.analyse('@misc{foo:{123}, }').symbols).must_be :==, [:AT, :NAME, :LBRACE, :KEY, :RBRACE, false]
end

it "doesn't start a comment for types starting with but not equal @comment" do
Lexer.new.analyse('@commentary{staudinger, }').symbols.must_be :==, [:AT, :NAME, :LBRACE, :KEY, :RBRACE, false]
_(Lexer.new.analyse('@commentary{staudinger, }').symbols).must_be :==, [:AT, :NAME, :LBRACE, :KEY, :RBRACE, false]
end

it "doesn't start a preamble for types starting with but not equal @preamble" do
Lexer.new.analyse('@preamblestring{ preamble }').symbols.must_be :==, [:AT, :NAME, :LBRACE, :NAME, :RBRACE, false]
_(Lexer.new.analyse('@preamblestring{ preamble }').symbols).must_be :==, [:AT, :NAME, :LBRACE, :NAME, :RBRACE, false]
end
end
end
7 changes: 5 additions & 2 deletions test/test_bibtex.rb
Original file line number Diff line number Diff line change
Expand Up @@ -101,19 +101,22 @@ def test_logger_can_be_assigned
end

def test_missing_key
assert_raises(BibTeX::ParseError) do
error = assert_raises(BibTeX::ParseError) do
BibTeX.parse(<<EOF)
@article{}
EOF
end
assert_equal('Failed to parse BibTeX entry: cite-key missing', error.message)

assert(
BibTeX.parse(<<EOF, allow_missing_keys: true)
@article{}
EOF
)

Timeout.timeout(2) do
BibTeX.parse(<<EOF, allow_missing_keys: true)
@article{},
@article{foo, },
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is the test that was failing, I couldn't find a proper way of fixing. Using an alternative test case here works.

@article{}
EOF
end
Expand Down
Loading