diff --git a/lib/net/imap/response_data.rb b/lib/net/imap/response_data.rb index 950205b3..41040889 100644 --- a/lib/net/imap/response_data.rb +++ b/lib/net/imap/response_data.rb @@ -75,9 +75,18 @@ class IgnoredResponse < UntaggedResponse # # Net::IMAP::UnparsedData represents data for unknown response types or # unknown extensions to response types without a well-defined extension - # grammar. + # grammar. UnparsedData represents the portion of the response which the + # parser has skipped over, without attempting to parse it. # - # See also: UnparsedNumericResponseData, ExtensionData, IgnoredResponse + # parser = Net::IMAP::ResponseParser.new + # response = parser.parse "* X-UNKNOWN-TYPE can't parse this\r\n" + # response => Net::IMAP::UntaggedResponse( + # name: "X-UNKNOWN-TYPE", + # data: Net::IMAP::UnparsedData(unparsed_data: "can't parse this"), + # ) + # + # See also: UnparsedNumericResponseData, ExtensionData, IgnoredResponse, + # InvalidParseData. class UnparsedData < Struct.new(:unparsed_data) ## # method: unparsed_data @@ -86,6 +95,61 @@ class UnparsedData < Struct.new(:unparsed_data) # The unparsed data end + # **Note:** This represents an intentionally _unstable_ API. Where + # instances of this class are returned, future releases may return a + # different (incompatible) object without deprecation or warning. + # + # When the response parser encounters a recoverable error, + # Net::IMAP::InvalidParseData represents that portion of the response which + # could not be parsed, allowing the parser to parse the remainder of the + # response. InvalidParseData is always associated with a ResponseParseError + # which has been rescued. + # + # This could be caused by a malformed server response, by a bug in + # Net::IMAP::ResponseParser, or by an unsupported extension to the response + # syntax. For example, if a server supports +UIDPLUS+, but sends an invalid + # +COPYUID+ response code: + # + # parser = Net::IMAP::ResponseParser.new + # parsed = parser.parse "* OK [COPYUID 701 ] copied one message\r\n" + # parsed => { + # data: Net::IMAP::ResponseText( + # code: Net::IMAP::ResponseCode( + # name: "COPYUID", + # data: Net::IMAP::InvalidParseData( + # parse_error: Net::IMAP::ResponseParseError, + # unparsed_data: "701 ", + # parsed_data: nil, + # ) + # ) + # ) + # } + # + # In this example, although [COPYUID 701 ] uses valid syntax for a + # _generic_ ResponseCode, it is _invalid_ syntax for a +COPYUID+ response + # code. + # + # See also: UnparsedData, ExtensionData + class InvalidParseData < Data.define(:parse_error, :unparsed_data, :parsed_data) + ## + # method: parse_error + # :call-seq: parse_error -> ResponseParseError + # + # Returns the rescued ResponseParseError. + + ## + # method: unparsed_data + # :call-seq: unparsed_data -> string + # + # Returns the raw string which was skipped over by the parser. + + ## + # method: parsed_data + # + # May return a partial parse result for unparsed_data, which had already + # been parsed before the parse_error. + end + # **Note:** This represents an intentionally _unstable_ API. Where # instances of this class are returned, future releases may return a # different (incompatible) object without deprecation or warning. @@ -93,7 +157,17 @@ class UnparsedData < Struct.new(:unparsed_data) # Net::IMAP::UnparsedNumericResponseData represents data for unhandled # response types with a numeric prefix. See the documentation for #number. # - # See also: UnparsedData, ExtensionData, IgnoredResponse + # parser = Net::IMAP::ResponseParser.new + # response = parser.parse "* 123 X-UNKNOWN-TYPE can't parse this\r\n" + # response => Net::IMAP::UntaggedResponse( + # name: "X-UNKNOWN-TYPE", + # data: Net::IMAP::UnparsedNumericData( + # number: 123, + # unparsed_data: "can't parse this" + # ), + # ) + # + # See also: UnparsedData, ExtensionData, IgnoredResponse, InvalidParseData class UnparsedNumericResponseData < Struct.new(:number, :unparsed_data) ## # method: number @@ -324,9 +398,10 @@ class ResponseText < Struct.new(:code, :text) # # Response codes are backwards compatible: Servers are allowed to send new # response codes even if the client has not enabled the extension that - # defines them. When Net::IMAP does not know how to parse response - # code text, #data returns the unparsed string. - # + # defines them. When ResponseParser does not know how to parse the response + # code data, #data may return the unparsed string, ExtensionData, or + # UnparsedData. When ResponseParser attempts but fails to parse the + # response code data, #data returns InvalidParseData. class ResponseCode < Struct.new(:name, :data) ## # method: name @@ -341,8 +416,13 @@ class ResponseCode < Struct.new(:name, :data) # # Returns the parsed response code data, e.g: an array of capabilities # strings, an array of character set strings, a list of permanent flags, - # an Integer, etc. The response #code determines what form the response - # code data can take. + # an Integer, etc. The response #name determines what form the response + # code #data can take. + # + # When ResponseParser does not know how to parse the response code data, + # #data may return the unparsed string, ExtensionData, or UnparsedData. + # When ResponseParser attempts but fails to parse the response code data, + # #data returns InvalidParseData. end # MailboxList represents the data of an untagged +LIST+ response, for a diff --git a/lib/net/imap/response_parser.rb b/lib/net/imap/response_parser.rb index ef240f7d..4a80fb64 100644 --- a/lib/net/imap/response_parser.rb +++ b/lib/net/imap/response_parser.rb @@ -1961,6 +1961,7 @@ def resp_text # resp-text-code =/ "UIDREQUIRED" def resp_text_code name = resp_text_code__name + state = current_state data = case name when "CAPABILITY" then resp_code__capability @@ -1983,8 +1984,18 @@ def resp_text_code when "MAILBOXID" then SP!; parens__objectid # RFC8474: OBJECTID when "UIDREQUIRED" then # RFC9586: UIDONLY else + state = nil # don't backtrack SP? and text_chars_except_rbra end + peek_rbra? or + parse_error("expected resp-text-code %p to be complete", name) + ResponseCode.new(name, data) + rescue Net::IMAP::ResponseParseError => parse_error + raise unless state + raise if parse_error.message.include?("uid-set") + restore_state state + unparsed_data = SP? && text_chars_except_rbra + data = InvalidParseData[parse_error:, unparsed_data:, parsed_data: data] ResponseCode.new(name, data) end diff --git a/test/net/imap/fixtures/response_parser/quirky_behaviors.yml b/test/net/imap/fixtures/response_parser/quirky_behaviors.yml index 1bef93aa..9d626fb5 100644 --- a/test/net/imap/fixtures/response_parser/quirky_behaviors.yml +++ b/test/net/imap/fixtures/response_parser/quirky_behaviors.yml @@ -57,6 +57,160 @@ MailboxBE=XXXXXXXXXXXXX.EURXXXX.PROD.OUTLOOK.COM Service=Imap4] AUTHENTICATE completed.\r\n" + "Outlook.com and Microsoft 365 can send an invalid COPYUID response code": + comment: | + Although this is a buggy COPYUID response from the server, it's still a + valid *generic* `resp-text-code`. We should always successfully parse + `resp-text-code` when it begins with a valid `atom SP`, even if that means + using UnparsedData or some other similar class to wrap the invalid or + unparsable payload that follows. + :response: "* OK [COPYUID 701 ]\r\n" + :expected: !ruby/struct:Net::IMAP::UntaggedResponse + name: OK + data: !ruby/struct:Net::IMAP::ResponseText + code: !ruby/struct:Net::IMAP::ResponseCode + name: COPYUID + data: !ruby/data:Net::IMAP::InvalidParseData + parse_error: !ruby/exception:Net::IMAP::ResponseParseError + message: unexpected token SPACE (expected ATOM or NUMBER or STAR) + parser_class: !ruby/class 'Net::IMAP::ResponseParser' + string: "* OK [COPYUID 701 ]\r\n" + pos: 19 + lex_state: :EXPR_BEG + token: !ruby/struct:Net::IMAP::ResponseParser::Token + symbol: :SPACE + value: " " + backtrace: + - "lib/net/imap/response_parser/parser_utils.rb:218:in `parse_error'" + - "lib/net/imap/response_parser/parser_utils.rb:127:in `combine_adjacent'" + - "lib/net/imap/response_parser.rb:499:in `sequence_set'" + - "lib/net/imap/response_parser.rb:2180:in `uid_set'" + - "lib/net/imap/response_parser.rb:2034:in `resp_code_copy__data'" + - "lib/net/imap/response_parser.rb:1973:in `resp_text_code'" + - "lib/net/imap/response_parser.rb:1894:in `resp_text'" + - "lib/net/imap/response_parser.rb:820:in `resp_cond_state'" + - "lib/net/imap/response_parser.rb:824:in `resp_cond_state__untagged'" + - "lib/net/imap/response_parser.rb:740:in `response_data'" + - "lib/net/imap/response_parser.rb:687:in `response'" + - "lib/net/imap/response_parser.rb:40:in `parse'" + - "test/net/imap/net_imap_test_helpers.rb ... ignoring 3 frames" + - "/gems/test-unit-3.7.3/lib/test/unit/... ignoring 42 frames" + unparsed_data: '701 ' + parsed_data: + text: "" + raw_data: "* OK [COPYUID 701 ]\r\n" + + "Extra resp-text-code payload": + comment: | + `resp-text-code` should parse when its payload continues unexpectedly. + :response: "* OK [COPYUID 1 1:10 101:110 extra junk] bad\r\n" + :expected: !ruby/struct:Net::IMAP::UntaggedResponse + name: OK + data: !ruby/struct:Net::IMAP::ResponseText + code: !ruby/struct:Net::IMAP::ResponseCode + name: COPYUID + data: !ruby/data:Net::IMAP::InvalidParseData + parse_error: !ruby/exception:Net::IMAP::ResponseParseError + message: expected resp-text-code "COPYUID" to be complete + backtrace: + - "lib/net/imap/response_parser/parser_utils.rb:218:in 'Net::IMAP::ResponseParser::ParserUtils#parse_error'" + - "lib/net/imap/response_parser.rb:1981:in 'Net::IMAP::ResponseParser#resp_text_code'" + - "lib/net/imap/response_parser.rb:1894:in 'Net::IMAP::ResponseParser#resp_text'" + - "lib/net/imap/response_parser.rb:820:in 'Net::IMAP::ResponseParser#resp_cond_state'" + - "lib/net/imap/response_parser.rb:824:in 'Net::IMAP::ResponseParser#resp_cond_state__untagged'" + - "lib/net/imap/response_parser.rb:740:in 'Net::IMAP::ResponseParser#response_data'" + - "lib/net/imap/response_parser.rb:687:in 'Net::IMAP::ResponseParser#response'" + - "lib/net/imap/response_parser.rb:40:in 'Net::IMAP::ResponseParser#parse'" + - "test/net/imap/net_imap_test_helpers.rb ... ignoring 3 frames" + - "/gems/test-unit-3.7.3/lib/test/unit/... ignoring 42 frames" + parser_class: !ruby/class 'Net::IMAP::ResponseParser' + string: "* OK [COPYUID 1 1:10 101:110 extra junk] bad\r\n" + pos: 29 + lex_state: :EXPR_BEG + token: !ruby/struct:Net::IMAP::ResponseParser::Token + symbol: :SPACE + value: " " + unparsed_data: 1 1:10 101:110 extra junk + parsed_data: !ruby/data:Net::IMAP::CopyUIDData + uidvalidity: 1 + source_uids: !ruby/object:Net::IMAP::SequenceSet + string: '1:10' + assigned_uids: !ruby/object:Net::IMAP::SequenceSet + string: 101:110 + text: bad + raw_data: "* OK [COPYUID 1 1:10 101:110 extra junk] bad\r\n" + + "Missing resp-text-code payload": + comment: | + `resp-text-code` should parse even when its expected payload is missing. + :response: "* OK [COPYUID] missing\r\n" + :expected: !ruby/struct:Net::IMAP::UntaggedResponse + name: OK + data: !ruby/struct:Net::IMAP::ResponseText + code: !ruby/struct:Net::IMAP::ResponseCode + name: COPYUID + data: !ruby/data:Net::IMAP::InvalidParseData + parse_error: !ruby/exception:Net::IMAP::ResponseParseError + message: unexpected RBRA (expected " ") + backtrace: + - "lib/net/imap/response_parser/parser_utils.rb:218:in 'Net::IMAP::ResponseParser::ParserUtils#parse_error'" + - "lib/net/imap/response_parser/parser_utils.rb:54:in 'Net::IMAP::ResponseParser#SP!'" + - "lib/net/imap/response_parser.rb:1973:in 'Net::IMAP::ResponseParser#resp_text_code'" + - "lib/net/imap/response_parser.rb:1894:in 'Net::IMAP::ResponseParser#resp_text'" + - "lib/net/imap/response_parser.rb:820:in 'Net::IMAP::ResponseParser#resp_cond_state'" + - "lib/net/imap/response_parser.rb:824:in 'Net::IMAP::ResponseParser#resp_cond_state__untagged'" + - "lib/net/imap/response_parser.rb:740:in 'Net::IMAP::ResponseParser#response_data'" + - "lib/net/imap/response_parser.rb:687:in 'Net::IMAP::ResponseParser#response'" + - "lib/net/imap/response_parser.rb:40:in 'Net::IMAP::ResponseParser#parse'" + - "test/net/imap/net_imap_test_helpers.rb ... ignoring 3 frames" + - "/gems/test-unit-3.7.3/lib/test/unit/... ignoring 42 frames" + parser_class: !ruby/class 'Net::IMAP::ResponseParser' + string: "* OK [COPYUID] missing\r\n" + pos: 14 + lex_state: :EXPR_BEG + token: !ruby/struct:Net::IMAP::ResponseParser::Token + symbol: :RBRA + value: "]" + unparsed_data: + parsed_data: + text: missing + raw_data: "* OK [COPYUID] missing\r\n" + + "Unexpected resp-text-code payload": + comment: | + `resp-text-code` should parse even when a payload is unexpectedly present. + :response: "* OK [CLOSED shouldn't be anything here] mailbox closed\r\n" + :expected: !ruby/struct:Net::IMAP::UntaggedResponse + name: OK + data: !ruby/struct:Net::IMAP::ResponseText + code: !ruby/struct:Net::IMAP::ResponseCode + name: CLOSED + data: !ruby/data:Net::IMAP::InvalidParseData + parse_error: !ruby/exception:Net::IMAP::ResponseParseError + message: expected resp-text-code "CLOSED" to be complete + backtrace: + - "lib/net/imap/response_parser/parser_utils.rb:218:in 'Net::IMAP::ResponseParser::ParserUtils#parse_error'" + - "lib/net/imap/response_parser.rb:1981:in 'Net::IMAP::ResponseParser#resp_text_code'" + - "lib/net/imap/response_parser.rb:1894:in 'Net::IMAP::ResponseParser#resp_text'" + - "lib/net/imap/response_parser.rb:820:in 'Net::IMAP::ResponseParser#resp_cond_state'" + - "lib/net/imap/response_parser.rb:824:in 'Net::IMAP::ResponseParser#resp_cond_state__untagged'" + - "lib/net/imap/response_parser.rb:740:in 'Net::IMAP::ResponseParser#response_data'" + - "lib/net/imap/response_parser.rb:687:in 'Net::IMAP::ResponseParser#response'" + - "lib/net/imap/response_parser.rb:40:in 'Net::IMAP::ResponseParser#parse'" + - "test/net/imap/net_imap_test_helpers.rb ... ignoring 3 frames" + - "/gems/test-unit-3.7.3/lib/test/unit/... ignoring 42 frames" + parser_class: !ruby/class 'Net::IMAP::ResponseParser' + string: "* OK [CLOSED shouldn't be anything here] mailbox closed\r\n" + pos: 13 + lex_state: :EXPR_BEG + token: !ruby/struct:Net::IMAP::ResponseParser::Token + symbol: :SPACE + value: " " + unparsed_data: shouldn't be anything here + parsed_data: + text: mailbox closed + raw_data: "* OK [CLOSED shouldn't be anything here] mailbox closed\r\n" + outlook.com puts an extra SP in ENVELOPE address lists: comment: | An annoying bug from outlook.com. They've had the bug for years, and