From 73121cf7719a19fc3a12b47c183a9184e49a4daf Mon Sep 17 00:00:00 2001 From: nick evans Date: Tue, 17 Feb 2026 10:46:46 -0500 Subject: [PATCH 1/2] =?UTF-8?q?=F0=9F=93=9A=20Add=20rdoc=20examples=20to?= =?UTF-8?q?=20UnparsedData,=20UnparsedNumericData?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/net/imap/response_data.rb | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/lib/net/imap/response_data.rb b/lib/net/imap/response_data.rb index 950205b3..c0795560 100644 --- a/lib/net/imap/response_data.rb +++ b/lib/net/imap/response_data.rb @@ -75,7 +75,15 @@ class IgnoredResponse < UntaggedResponse # # Net::IMAP::UnparsedData represents data for unknown response types or # unknown extensions to response types without a well-defined extension - # grammar. + # grammar. UnparsedData represents the portion of the response which the + # parser has skipped over, without attempting to parse it. + # + # parser = Net::IMAP::ResponseParser.new + # response = parser.parse "* X-UNKNOWN-TYPE can't parse this\r\n" + # response => Net::IMAP::UntaggedResponse( + # name: "X-UNKNOWN-TYPE", + # data: Net::IMAP::UnparsedData(unparsed_data: "can't parse this"), + # ) # # See also: UnparsedNumericResponseData, ExtensionData, IgnoredResponse class UnparsedData < Struct.new(:unparsed_data) @@ -93,7 +101,16 @@ class UnparsedData < Struct.new(:unparsed_data) # Net::IMAP::UnparsedNumericResponseData represents data for unhandled # response types with a numeric prefix. See the documentation for #number. # - # See also: UnparsedData, ExtensionData, IgnoredResponse + # parser = Net::IMAP::ResponseParser.new + # response = parser.parse "* 123 X-UNKNOWN-TYPE can't parse this\r\n" + # response => Net::IMAP::UntaggedResponse( + # name: "X-UNKNOWN-TYPE", + # data: Net::IMAP::UnparsedNumericData( + # number: 123, + # unparsed_data: "can't parse this" + # ), + # ) + # class UnparsedNumericResponseData < Struct.new(:number, :unparsed_data) ## # method: number From 7f0e115465e3a1d676ee77b70aa74cd5fbd8583e Mon Sep 17 00:00:00 2001 From: nick evans Date: Thu, 19 Feb 2026 12:19:26 -0500 Subject: [PATCH 2/2] =?UTF-8?q?=F0=9F=A5=85=20Successfully=20parse=20inval?= =?UTF-8?q?id=20response=20code=20data?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the parser encounters a recoverable error in `resp-text-code`, it now returns `InvalidParseData` to represent the data that we've skipped over. `InvalidParseData` can be used for similar recoverable parse errors in the future (for example, many servers respond with invalid `BODYSTRUCTURE` or incorrectly escaped quoted strings). The specific example I have encountered the most is when Microsoft's IMAP servers send an invalid COPYUID response code. Although it is invalid for `resp-code-copy`, it's still a valid `resp-text-code` because it does match `atom [SP 1*]`. This creates some minor differences for invalid `resp-text-code` data: * <= v0.6.2: raises ResponseParseError (this is a bug). * == v0.6.3: returns ResponseText with no ResponseCode (also a bug). * >= v0.6.4: returns ResponseText with code with InvalidParseData. Although this is a bugfix, it has a minor incompatibility for response handlers which assume that a particular `ResponseCode#name` always results in the same type of `ResponseCode#data`. ```ruby # It was previously safe to assume the class of #data, based on #name: imap.add_response_handler do |resp| if resp in {data: {code: {name: "COPYUID", data: opyuid}}} copyuid => Net::IMAP::CopyUIDData end end # With this change, ResponseCode#data could also be InvalidParseData imap.add_response_handler do |resp| if resp in {data: {code: {name: "COPYUID", data: copyuid}}} copyuid => Net::IMAP::CopyUIDData | Net::IMAP::InvalidParseData end end ``` Prior to v0.6.3, these responses would raise a ResponseParseError and the response handler would not have been called. --- lib/net/imap/response_data.rb | 75 ++++++++- lib/net/imap/response_parser.rb | 11 ++ .../response_parser/quirky_behaviors.yml | 154 ++++++++++++++++++ 3 files changed, 234 insertions(+), 6 deletions(-) diff --git a/lib/net/imap/response_data.rb b/lib/net/imap/response_data.rb index c0795560..41040889 100644 --- a/lib/net/imap/response_data.rb +++ b/lib/net/imap/response_data.rb @@ -85,7 +85,8 @@ class IgnoredResponse < UntaggedResponse # data: Net::IMAP::UnparsedData(unparsed_data: "can't parse this"), # ) # - # See also: UnparsedNumericResponseData, ExtensionData, IgnoredResponse + # See also: UnparsedNumericResponseData, ExtensionData, IgnoredResponse, + # InvalidParseData. class UnparsedData < Struct.new(:unparsed_data) ## # method: unparsed_data @@ -94,6 +95,61 @@ class UnparsedData < Struct.new(:unparsed_data) # The unparsed data end + # **Note:** This represents an intentionally _unstable_ API. Where + # instances of this class are returned, future releases may return a + # different (incompatible) object without deprecation or warning. + # + # When the response parser encounters a recoverable error, + # Net::IMAP::InvalidParseData represents that portion of the response which + # could not be parsed, allowing the parser to parse the remainder of the + # response. InvalidParseData is always associated with a ResponseParseError + # which has been rescued. + # + # This could be caused by a malformed server response, by a bug in + # Net::IMAP::ResponseParser, or by an unsupported extension to the response + # syntax. For example, if a server supports +UIDPLUS+, but sends an invalid + # +COPYUID+ response code: + # + # parser = Net::IMAP::ResponseParser.new + # parsed = parser.parse "* OK [COPYUID 701 ] copied one message\r\n" + # parsed => { + # data: Net::IMAP::ResponseText( + # code: Net::IMAP::ResponseCode( + # name: "COPYUID", + # data: Net::IMAP::InvalidParseData( + # parse_error: Net::IMAP::ResponseParseError, + # unparsed_data: "701 ", + # parsed_data: nil, + # ) + # ) + # ) + # } + # + # In this example, although [COPYUID 701 ] uses valid syntax for a + # _generic_ ResponseCode, it is _invalid_ syntax for a +COPYUID+ response + # code. + # + # See also: UnparsedData, ExtensionData + class InvalidParseData < Data.define(:parse_error, :unparsed_data, :parsed_data) + ## + # method: parse_error + # :call-seq: parse_error -> ResponseParseError + # + # Returns the rescued ResponseParseError. + + ## + # method: unparsed_data + # :call-seq: unparsed_data -> string + # + # Returns the raw string which was skipped over by the parser. + + ## + # method: parsed_data + # + # May return a partial parse result for unparsed_data, which had already + # been parsed before the parse_error. + end + # **Note:** This represents an intentionally _unstable_ API. Where # instances of this class are returned, future releases may return a # different (incompatible) object without deprecation or warning. @@ -111,6 +167,7 @@ class UnparsedData < Struct.new(:unparsed_data) # ), # ) # + # See also: UnparsedData, ExtensionData, IgnoredResponse, InvalidParseData class UnparsedNumericResponseData < Struct.new(:number, :unparsed_data) ## # method: number @@ -341,9 +398,10 @@ class ResponseText < Struct.new(:code, :text) # # Response codes are backwards compatible: Servers are allowed to send new # response codes even if the client has not enabled the extension that - # defines them. When Net::IMAP does not know how to parse response - # code text, #data returns the unparsed string. - # + # defines them. When ResponseParser does not know how to parse the response + # code data, #data may return the unparsed string, ExtensionData, or + # UnparsedData. When ResponseParser attempts but fails to parse the + # response code data, #data returns InvalidParseData. class ResponseCode < Struct.new(:name, :data) ## # method: name @@ -358,8 +416,13 @@ class ResponseCode < Struct.new(:name, :data) # # Returns the parsed response code data, e.g: an array of capabilities # strings, an array of character set strings, a list of permanent flags, - # an Integer, etc. The response #code determines what form the response - # code data can take. + # an Integer, etc. The response #name determines what form the response + # code #data can take. + # + # When ResponseParser does not know how to parse the response code data, + # #data may return the unparsed string, ExtensionData, or UnparsedData. + # When ResponseParser attempts but fails to parse the response code data, + # #data returns InvalidParseData. end # MailboxList represents the data of an untagged +LIST+ response, for a diff --git a/lib/net/imap/response_parser.rb b/lib/net/imap/response_parser.rb index ef240f7d..4a80fb64 100644 --- a/lib/net/imap/response_parser.rb +++ b/lib/net/imap/response_parser.rb @@ -1961,6 +1961,7 @@ def resp_text # resp-text-code =/ "UIDREQUIRED" def resp_text_code name = resp_text_code__name + state = current_state data = case name when "CAPABILITY" then resp_code__capability @@ -1983,8 +1984,18 @@ def resp_text_code when "MAILBOXID" then SP!; parens__objectid # RFC8474: OBJECTID when "UIDREQUIRED" then # RFC9586: UIDONLY else + state = nil # don't backtrack SP? and text_chars_except_rbra end + peek_rbra? or + parse_error("expected resp-text-code %p to be complete", name) + ResponseCode.new(name, data) + rescue Net::IMAP::ResponseParseError => parse_error + raise unless state + raise if parse_error.message.include?("uid-set") + restore_state state + unparsed_data = SP? && text_chars_except_rbra + data = InvalidParseData[parse_error:, unparsed_data:, parsed_data: data] ResponseCode.new(name, data) end diff --git a/test/net/imap/fixtures/response_parser/quirky_behaviors.yml b/test/net/imap/fixtures/response_parser/quirky_behaviors.yml index 1bef93aa..9d626fb5 100644 --- a/test/net/imap/fixtures/response_parser/quirky_behaviors.yml +++ b/test/net/imap/fixtures/response_parser/quirky_behaviors.yml @@ -57,6 +57,160 @@ MailboxBE=XXXXXXXXXXXXX.EURXXXX.PROD.OUTLOOK.COM Service=Imap4] AUTHENTICATE completed.\r\n" + "Outlook.com and Microsoft 365 can send an invalid COPYUID response code": + comment: | + Although this is a buggy COPYUID response from the server, it's still a + valid *generic* `resp-text-code`. We should always successfully parse + `resp-text-code` when it begins with a valid `atom SP`, even if that means + using UnparsedData or some other similar class to wrap the invalid or + unparsable payload that follows. + :response: "* OK [COPYUID 701 ]\r\n" + :expected: !ruby/struct:Net::IMAP::UntaggedResponse + name: OK + data: !ruby/struct:Net::IMAP::ResponseText + code: !ruby/struct:Net::IMAP::ResponseCode + name: COPYUID + data: !ruby/data:Net::IMAP::InvalidParseData + parse_error: !ruby/exception:Net::IMAP::ResponseParseError + message: unexpected token SPACE (expected ATOM or NUMBER or STAR) + parser_class: !ruby/class 'Net::IMAP::ResponseParser' + string: "* OK [COPYUID 701 ]\r\n" + pos: 19 + lex_state: :EXPR_BEG + token: !ruby/struct:Net::IMAP::ResponseParser::Token + symbol: :SPACE + value: " " + backtrace: + - "lib/net/imap/response_parser/parser_utils.rb:218:in `parse_error'" + - "lib/net/imap/response_parser/parser_utils.rb:127:in `combine_adjacent'" + - "lib/net/imap/response_parser.rb:499:in `sequence_set'" + - "lib/net/imap/response_parser.rb:2180:in `uid_set'" + - "lib/net/imap/response_parser.rb:2034:in `resp_code_copy__data'" + - "lib/net/imap/response_parser.rb:1973:in `resp_text_code'" + - "lib/net/imap/response_parser.rb:1894:in `resp_text'" + - "lib/net/imap/response_parser.rb:820:in `resp_cond_state'" + - "lib/net/imap/response_parser.rb:824:in `resp_cond_state__untagged'" + - "lib/net/imap/response_parser.rb:740:in `response_data'" + - "lib/net/imap/response_parser.rb:687:in `response'" + - "lib/net/imap/response_parser.rb:40:in `parse'" + - "test/net/imap/net_imap_test_helpers.rb ... ignoring 3 frames" + - "/gems/test-unit-3.7.3/lib/test/unit/... ignoring 42 frames" + unparsed_data: '701 ' + parsed_data: + text: "" + raw_data: "* OK [COPYUID 701 ]\r\n" + + "Extra resp-text-code payload": + comment: | + `resp-text-code` should parse when its payload continues unexpectedly. + :response: "* OK [COPYUID 1 1:10 101:110 extra junk] bad\r\n" + :expected: !ruby/struct:Net::IMAP::UntaggedResponse + name: OK + data: !ruby/struct:Net::IMAP::ResponseText + code: !ruby/struct:Net::IMAP::ResponseCode + name: COPYUID + data: !ruby/data:Net::IMAP::InvalidParseData + parse_error: !ruby/exception:Net::IMAP::ResponseParseError + message: expected resp-text-code "COPYUID" to be complete + backtrace: + - "lib/net/imap/response_parser/parser_utils.rb:218:in 'Net::IMAP::ResponseParser::ParserUtils#parse_error'" + - "lib/net/imap/response_parser.rb:1981:in 'Net::IMAP::ResponseParser#resp_text_code'" + - "lib/net/imap/response_parser.rb:1894:in 'Net::IMAP::ResponseParser#resp_text'" + - "lib/net/imap/response_parser.rb:820:in 'Net::IMAP::ResponseParser#resp_cond_state'" + - "lib/net/imap/response_parser.rb:824:in 'Net::IMAP::ResponseParser#resp_cond_state__untagged'" + - "lib/net/imap/response_parser.rb:740:in 'Net::IMAP::ResponseParser#response_data'" + - "lib/net/imap/response_parser.rb:687:in 'Net::IMAP::ResponseParser#response'" + - "lib/net/imap/response_parser.rb:40:in 'Net::IMAP::ResponseParser#parse'" + - "test/net/imap/net_imap_test_helpers.rb ... ignoring 3 frames" + - "/gems/test-unit-3.7.3/lib/test/unit/... ignoring 42 frames" + parser_class: !ruby/class 'Net::IMAP::ResponseParser' + string: "* OK [COPYUID 1 1:10 101:110 extra junk] bad\r\n" + pos: 29 + lex_state: :EXPR_BEG + token: !ruby/struct:Net::IMAP::ResponseParser::Token + symbol: :SPACE + value: " " + unparsed_data: 1 1:10 101:110 extra junk + parsed_data: !ruby/data:Net::IMAP::CopyUIDData + uidvalidity: 1 + source_uids: !ruby/object:Net::IMAP::SequenceSet + string: '1:10' + assigned_uids: !ruby/object:Net::IMAP::SequenceSet + string: 101:110 + text: bad + raw_data: "* OK [COPYUID 1 1:10 101:110 extra junk] bad\r\n" + + "Missing resp-text-code payload": + comment: | + `resp-text-code` should parse even when its expected payload is missing. + :response: "* OK [COPYUID] missing\r\n" + :expected: !ruby/struct:Net::IMAP::UntaggedResponse + name: OK + data: !ruby/struct:Net::IMAP::ResponseText + code: !ruby/struct:Net::IMAP::ResponseCode + name: COPYUID + data: !ruby/data:Net::IMAP::InvalidParseData + parse_error: !ruby/exception:Net::IMAP::ResponseParseError + message: unexpected RBRA (expected " ") + backtrace: + - "lib/net/imap/response_parser/parser_utils.rb:218:in 'Net::IMAP::ResponseParser::ParserUtils#parse_error'" + - "lib/net/imap/response_parser/parser_utils.rb:54:in 'Net::IMAP::ResponseParser#SP!'" + - "lib/net/imap/response_parser.rb:1973:in 'Net::IMAP::ResponseParser#resp_text_code'" + - "lib/net/imap/response_parser.rb:1894:in 'Net::IMAP::ResponseParser#resp_text'" + - "lib/net/imap/response_parser.rb:820:in 'Net::IMAP::ResponseParser#resp_cond_state'" + - "lib/net/imap/response_parser.rb:824:in 'Net::IMAP::ResponseParser#resp_cond_state__untagged'" + - "lib/net/imap/response_parser.rb:740:in 'Net::IMAP::ResponseParser#response_data'" + - "lib/net/imap/response_parser.rb:687:in 'Net::IMAP::ResponseParser#response'" + - "lib/net/imap/response_parser.rb:40:in 'Net::IMAP::ResponseParser#parse'" + - "test/net/imap/net_imap_test_helpers.rb ... ignoring 3 frames" + - "/gems/test-unit-3.7.3/lib/test/unit/... ignoring 42 frames" + parser_class: !ruby/class 'Net::IMAP::ResponseParser' + string: "* OK [COPYUID] missing\r\n" + pos: 14 + lex_state: :EXPR_BEG + token: !ruby/struct:Net::IMAP::ResponseParser::Token + symbol: :RBRA + value: "]" + unparsed_data: + parsed_data: + text: missing + raw_data: "* OK [COPYUID] missing\r\n" + + "Unexpected resp-text-code payload": + comment: | + `resp-text-code` should parse even when a payload is unexpectedly present. + :response: "* OK [CLOSED shouldn't be anything here] mailbox closed\r\n" + :expected: !ruby/struct:Net::IMAP::UntaggedResponse + name: OK + data: !ruby/struct:Net::IMAP::ResponseText + code: !ruby/struct:Net::IMAP::ResponseCode + name: CLOSED + data: !ruby/data:Net::IMAP::InvalidParseData + parse_error: !ruby/exception:Net::IMAP::ResponseParseError + message: expected resp-text-code "CLOSED" to be complete + backtrace: + - "lib/net/imap/response_parser/parser_utils.rb:218:in 'Net::IMAP::ResponseParser::ParserUtils#parse_error'" + - "lib/net/imap/response_parser.rb:1981:in 'Net::IMAP::ResponseParser#resp_text_code'" + - "lib/net/imap/response_parser.rb:1894:in 'Net::IMAP::ResponseParser#resp_text'" + - "lib/net/imap/response_parser.rb:820:in 'Net::IMAP::ResponseParser#resp_cond_state'" + - "lib/net/imap/response_parser.rb:824:in 'Net::IMAP::ResponseParser#resp_cond_state__untagged'" + - "lib/net/imap/response_parser.rb:740:in 'Net::IMAP::ResponseParser#response_data'" + - "lib/net/imap/response_parser.rb:687:in 'Net::IMAP::ResponseParser#response'" + - "lib/net/imap/response_parser.rb:40:in 'Net::IMAP::ResponseParser#parse'" + - "test/net/imap/net_imap_test_helpers.rb ... ignoring 3 frames" + - "/gems/test-unit-3.7.3/lib/test/unit/... ignoring 42 frames" + parser_class: !ruby/class 'Net::IMAP::ResponseParser' + string: "* OK [CLOSED shouldn't be anything here] mailbox closed\r\n" + pos: 13 + lex_state: :EXPR_BEG + token: !ruby/struct:Net::IMAP::ResponseParser::Token + symbol: :SPACE + value: " " + unparsed_data: shouldn't be anything here + parsed_data: + text: mailbox closed + raw_data: "* OK [CLOSED shouldn't be anything here] mailbox closed\r\n" + outlook.com puts an extra SP in ENVELOPE address lists: comment: | An annoying bug from outlook.com. They've had the bug for years, and