🥅 Successfully parse invalid response code data

lib/net/imap/response_data.rb

@@ -75,9 +75,18 @@ class IgnoredResponse < UntaggedResponse
     #
     # Net::IMAP::UnparsedData represents data for unknown response types or
     # unknown extensions to response types without a well-defined extension
-    # grammar.
+    # grammar.  UnparsedData represents the portion of the response which the
+    # parser has skipped over, without attempting to parse it.
     #
-    # See also: UnparsedNumericResponseData, ExtensionData, IgnoredResponse
+    #    parser = Net::IMAP::ResponseParser.new
+    #    response = parser.parse "* X-UNKNOWN-TYPE can't parse this\r\n"
+    #    response => Net::IMAP::UntaggedResponse(
+    #      name: "X-UNKNOWN-TYPE",
+    #      data: Net::IMAP::UnparsedData(unparsed_data: "can't parse this"),
+    #    )
+    #
+    # See also: UnparsedNumericResponseData, ExtensionData, IgnoredResponse,
+    # InvalidParseData.
     class UnparsedData < Struct.new(:unparsed_data)
       ##
       # method: unparsed_data
@@ -86,14 +95,79 @@ class UnparsedData < Struct.new(:unparsed_data)
       # The unparsed data
     end
 
+    # **Note:** This represents an intentionally _unstable_ API.  Where
+    # instances of this class are returned, future releases may return a
+    # different (incompatible) object <em>without deprecation or warning</em>.
+    #
+    # When the response parser encounters a recoverable error,
+    # Net::IMAP::InvalidParseData represents that portion of the response which
+    # could not be parsed, allowing the parser to parse the remainder of the
+    # response.  InvalidParseData is always associated with a ResponseParseError
+    # which has been rescued.
+    #
+    # This could be caused by a malformed server response, by a bug in
+    # Net::IMAP::ResponseParser, or by an unsupported extension to the response
+    # syntax.  For example, if a server supports +UIDPLUS+, but sends an invalid
+    # +COPYUID+ response code:
+    #
+    #    parser = Net::IMAP::ResponseParser.new
+    #    parsed = parser.parse "* OK [COPYUID 701  ] copied one message\r\n"
+    #    parsed => {
+    #      data: Net::IMAP::ResponseText(
+    #        code: Net::IMAP::ResponseCode(
+    #          name: "COPYUID",
+    #          data: Net::IMAP::InvalidParseData(
+    #            parse_error: Net::IMAP::ResponseParseError,
+    #            unparsed_data: "701  ",
+    #            parsed_data: nil,
+    #          )
+    #        )
+    #      )
+    #    }
+    #
+    # In this example, although <tt>[COPYUID 701  ]</tt> uses valid syntax for a
+    # _generic_ ResponseCode, it is _invalid_ syntax for a +COPYUID+ response
+    # code.
+    #
+    # See also: UnparsedData, ExtensionData
+    class InvalidParseData < Data.define(:parse_error, :unparsed_data, :parsed_data)
+      ##
+      # method: parse_error
+      # :call-seq: parse_error -> ResponseParseError
+      #
+      # Returns the rescued ResponseParseError.
+
+      ##
+      # method: unparsed_data
+      # :call-seq: unparsed_data -> string
+      #
+      # Returns the raw string which was skipped over by the parser.
+
+      ##
+      # method: parsed_data
+      #
+      # May return a partial parse result for unparsed_data, which had already
+      # been parsed before the parse_error.
+    end
+
     # **Note:** This represents an intentionally _unstable_ API.  Where
     # instances of this class are returned, future releases may return a
     # different (incompatible) object <em>without deprecation or warning</em>.
     #
     # Net::IMAP::UnparsedNumericResponseData represents data for unhandled
     # response types with a numeric prefix.  See the documentation for #number.
     #
-    # See also: UnparsedData, ExtensionData, IgnoredResponse
+    #    parser = Net::IMAP::ResponseParser.new
+    #    response = parser.parse "* 123 X-UNKNOWN-TYPE can't parse this\r\n"
+    #    response => Net::IMAP::UntaggedResponse(
+    #      name: "X-UNKNOWN-TYPE",
+    #      data: Net::IMAP::UnparsedNumericData(
+    #        number: 123,
+    #        unparsed_data: "can't parse this"
+    #      ),
+    #    )
+    #
+    # See also: UnparsedData, ExtensionData, IgnoredResponse, InvalidParseData
     class UnparsedNumericResponseData < Struct.new(:number, :unparsed_data)
       ##
       # method: number
@@ -324,9 +398,10 @@ class ResponseText < Struct.new(:code, :text)
     #
     # Response codes are backwards compatible:  Servers are allowed to send new
     # response codes even if the client has not enabled the extension that
-    # defines them.  When Net::IMAP does not know how to parse response
-    # code text, #data returns the unparsed string.
-    #
+    # defines them.  When ResponseParser does not know how to parse the response
+    # code data, #data may return the unparsed string, ExtensionData, or
+    # UnparsedData.  When ResponseParser attempts but fails to parse the
+    # response code data, #data returns InvalidParseData.
     class ResponseCode < Struct.new(:name, :data)
       ##
       # method: name
@@ -341,8 +416,13 @@ class ResponseCode < Struct.new(:name, :data)
       #
       # Returns the parsed response code data, e.g: an array of capabilities
       # strings, an array of character set strings, a list of permanent flags,
-      # an Integer, etc.  The response #code determines what form the response
-      # code data can take.
+      # an Integer, etc.  The response #name determines what form the response
+      # code #data can take.
+      #
+      # When ResponseParser does not know how to parse the response code data,
+      # #data may return the unparsed string, ExtensionData, or UnparsedData.
+      # When ResponseParser attempts but fails to parse the response code data,
+      # #data returns InvalidParseData.
     end
 
     # MailboxList represents the data of an untagged +LIST+ response, for a

lib/net/imap/response_parser.rb

@@ -1961,6 +1961,7 @@ def resp_text
       #   resp-text-code   =/ "UIDREQUIRED"
       def resp_text_code
         name = resp_text_code__name
+        state = current_state
         data =
           case name
           when "CAPABILITY"         then resp_code__capability
@@ -1983,8 +1984,18 @@ def resp_text_code
           when "MAILBOXID"          then SP!; parens__objectid     # RFC8474: OBJECTID
           when "UIDREQUIRED"        then                           # RFC9586: UIDONLY
           else
+            state = nil # don't backtrack
             SP? and text_chars_except_rbra
           end
+        peek_rbra? or
+          parse_error("expected resp-text-code %p to be complete", name)
+        ResponseCode.new(name, data)
+      rescue Net::IMAP::ResponseParseError => parse_error
+        raise unless state
+        raise if parse_error.message.include?("uid-set")
+        restore_state state
+        unparsed_data = SP? && text_chars_except_rbra
+        data = InvalidParseData[parse_error:, unparsed_data:, parsed_data: data]
         ResponseCode.new(name, data)
       end
 

test/net/imap/fixtures/response_parser/quirky_behaviors.yml

@@ -57,6 +57,160 @@
         MailboxBE=XXXXXXXXXXXXX.EURXXXX.PROD.OUTLOOK.COM Service=Imap4] AUTHENTICATE
         completed.\r\n"
 
+  "Outlook.com and Microsoft 365 can send an invalid COPYUID response code":
+    comment: |
+      Although this is a buggy COPYUID response from the server, it's still a
+      valid *generic* `resp-text-code`.  We should always successfully parse
+      `resp-text-code` when it begins with a valid `atom SP`, even if that means
+      using UnparsedData or some other similar class to wrap the invalid or
+      unparsable payload that follows.
+    :response: "* OK [COPYUID 701  ]\r\n"
+    :expected: !ruby/struct:Net::IMAP::UntaggedResponse
+      name: OK
+      data: !ruby/struct:Net::IMAP::ResponseText
+        code: !ruby/struct:Net::IMAP::ResponseCode
+          name: COPYUID
+          data: !ruby/data:Net::IMAP::InvalidParseData
+            parse_error: !ruby/exception:Net::IMAP::ResponseParseError
+              message: unexpected token SPACE (expected ATOM or NUMBER or STAR)
+              parser_class: !ruby/class 'Net::IMAP::ResponseParser'
+              string: "* OK [COPYUID 701  ]\r\n"
+              pos: 19
+              lex_state: :EXPR_BEG
+              token: !ruby/struct:Net::IMAP::ResponseParser::Token
+                symbol: :SPACE
+                value: " "
+              backtrace:
+              - "lib/net/imap/response_parser/parser_utils.rb:218:in `parse_error'"
+              - "lib/net/imap/response_parser/parser_utils.rb:127:in `combine_adjacent'"
+              - "lib/net/imap/response_parser.rb:499:in `sequence_set'"
+              - "lib/net/imap/response_parser.rb:2180:in `uid_set'"
+              - "lib/net/imap/response_parser.rb:2034:in `resp_code_copy__data'"
+              - "lib/net/imap/response_parser.rb:1973:in `resp_text_code'"
+              - "lib/net/imap/response_parser.rb:1894:in `resp_text'"
+              - "lib/net/imap/response_parser.rb:820:in `resp_cond_state'"
+              - "lib/net/imap/response_parser.rb:824:in `resp_cond_state__untagged'"
+              - "lib/net/imap/response_parser.rb:740:in `response_data'"
+              - "lib/net/imap/response_parser.rb:687:in `response'"
+              - "lib/net/imap/response_parser.rb:40:in `parse'"
+              - "test/net/imap/net_imap_test_helpers.rb ... ignoring 3 frames"
+              - "/gems/test-unit-3.7.3/lib/test/unit/... ignoring 42 frames"
+            unparsed_data: '701  '
+            parsed_data:
+        text: ""
+      raw_data: "* OK [COPYUID 701  ]\r\n"
+
+  "Extra resp-text-code payload":
+    comment: |
+      `resp-text-code` should parse when its payload continues unexpectedly.
+    :response: "* OK [COPYUID 1 1:10 101:110 extra junk] bad\r\n"
+    :expected: !ruby/struct:Net::IMAP::UntaggedResponse
+      name: OK
+      data: !ruby/struct:Net::IMAP::ResponseText
+        code: !ruby/struct:Net::IMAP::ResponseCode
+          name: COPYUID
+          data: !ruby/data:Net::IMAP::InvalidParseData
+            parse_error: !ruby/exception:Net::IMAP::ResponseParseError
+              message: expected resp-text-code "COPYUID" to be complete
+              backtrace:
+              - "lib/net/imap/response_parser/parser_utils.rb:218:in 'Net::IMAP::ResponseParser::ParserUtils#parse_error'"
+              - "lib/net/imap/response_parser.rb:1981:in 'Net::IMAP::ResponseParser#resp_text_code'"
+              - "lib/net/imap/response_parser.rb:1894:in 'Net::IMAP::ResponseParser#resp_text'"
+              - "lib/net/imap/response_parser.rb:820:in 'Net::IMAP::ResponseParser#resp_cond_state'"
+              - "lib/net/imap/response_parser.rb:824:in 'Net::IMAP::ResponseParser#resp_cond_state__untagged'"
+              - "lib/net/imap/response_parser.rb:740:in 'Net::IMAP::ResponseParser#response_data'"
+              - "lib/net/imap/response_parser.rb:687:in 'Net::IMAP::ResponseParser#response'"
+              - "lib/net/imap/response_parser.rb:40:in 'Net::IMAP::ResponseParser#parse'"
+              - "test/net/imap/net_imap_test_helpers.rb ... ignoring 3 frames"
+              - "/gems/test-unit-3.7.3/lib/test/unit/... ignoring 42 frames"
+              parser_class: !ruby/class 'Net::IMAP::ResponseParser'
+              string: "* OK [COPYUID 1 1:10 101:110 extra junk] bad\r\n"
+              pos: 29
+              lex_state: :EXPR_BEG
+              token: !ruby/struct:Net::IMAP::ResponseParser::Token
+                symbol: :SPACE
+                value: " "
+            unparsed_data: 1 1:10 101:110 extra junk
+            parsed_data: !ruby/data:Net::IMAP::CopyUIDData
+              uidvalidity: 1
+              source_uids: !ruby/object:Net::IMAP::SequenceSet
+                string: '1:10'
+              assigned_uids: !ruby/object:Net::IMAP::SequenceSet
+                string: 101:110
+        text: bad
+      raw_data: "* OK [COPYUID 1 1:10 101:110 extra junk] bad\r\n"
+
+  "Missing resp-text-code payload":
+    comment: |
+      `resp-text-code` should parse even when its expected payload is missing.
+    :response: "* OK [COPYUID] missing\r\n"
+    :expected: !ruby/struct:Net::IMAP::UntaggedResponse
+      name: OK
+      data: !ruby/struct:Net::IMAP::ResponseText
+        code: !ruby/struct:Net::IMAP::ResponseCode
+          name: COPYUID
+          data: !ruby/data:Net::IMAP::InvalidParseData
+            parse_error: !ruby/exception:Net::IMAP::ResponseParseError
+              message: unexpected RBRA (expected " ")
+              backtrace:
+              - "lib/net/imap/response_parser/parser_utils.rb:218:in 'Net::IMAP::ResponseParser::ParserUtils#parse_error'"
+              - "lib/net/imap/response_parser/parser_utils.rb:54:in 'Net::IMAP::ResponseParser#SP!'"
+              - "lib/net/imap/response_parser.rb:1973:in 'Net::IMAP::ResponseParser#resp_text_code'"
+              - "lib/net/imap/response_parser.rb:1894:in 'Net::IMAP::ResponseParser#resp_text'"
+              - "lib/net/imap/response_parser.rb:820:in 'Net::IMAP::ResponseParser#resp_cond_state'"
+              - "lib/net/imap/response_parser.rb:824:in 'Net::IMAP::ResponseParser#resp_cond_state__untagged'"
+              - "lib/net/imap/response_parser.rb:740:in 'Net::IMAP::ResponseParser#response_data'"
+              - "lib/net/imap/response_parser.rb:687:in 'Net::IMAP::ResponseParser#response'"
+              - "lib/net/imap/response_parser.rb:40:in 'Net::IMAP::ResponseParser#parse'"
+              - "test/net/imap/net_imap_test_helpers.rb ... ignoring 3 frames"
+              - "/gems/test-unit-3.7.3/lib/test/unit/... ignoring 42 frames"
+              parser_class: !ruby/class 'Net::IMAP::ResponseParser'
+              string: "* OK [COPYUID] missing\r\n"
+              pos: 14
+              lex_state: :EXPR_BEG
+              token: !ruby/struct:Net::IMAP::ResponseParser::Token
+                symbol: :RBRA
+                value: "]"
+            unparsed_data:
+            parsed_data:
+        text: missing
+      raw_data: "* OK [COPYUID] missing\r\n"
+
+  "Unexpected resp-text-code payload":
+    comment: |
+      `resp-text-code` should parse even when a payload is unexpectedly present.
+    :response: "* OK [CLOSED shouldn't be anything here] mailbox closed\r\n"
+    :expected: !ruby/struct:Net::IMAP::UntaggedResponse
+      name: OK
+      data: !ruby/struct:Net::IMAP::ResponseText
+        code: !ruby/struct:Net::IMAP::ResponseCode
+          name: CLOSED
+          data: !ruby/data:Net::IMAP::InvalidParseData
+            parse_error: !ruby/exception:Net::IMAP::ResponseParseError
+              message: expected resp-text-code "CLOSED" to be complete
+              backtrace:
+              - "lib/net/imap/response_parser/parser_utils.rb:218:in 'Net::IMAP::ResponseParser::ParserUtils#parse_error'"
+              - "lib/net/imap/response_parser.rb:1981:in 'Net::IMAP::ResponseParser#resp_text_code'"
+              - "lib/net/imap/response_parser.rb:1894:in 'Net::IMAP::ResponseParser#resp_text'"
+              - "lib/net/imap/response_parser.rb:820:in 'Net::IMAP::ResponseParser#resp_cond_state'"
+              - "lib/net/imap/response_parser.rb:824:in 'Net::IMAP::ResponseParser#resp_cond_state__untagged'"
+              - "lib/net/imap/response_parser.rb:740:in 'Net::IMAP::ResponseParser#response_data'"
+              - "lib/net/imap/response_parser.rb:687:in 'Net::IMAP::ResponseParser#response'"
+              - "lib/net/imap/response_parser.rb:40:in 'Net::IMAP::ResponseParser#parse'"
+              - "test/net/imap/net_imap_test_helpers.rb ... ignoring 3 frames"
+              - "/gems/test-unit-3.7.3/lib/test/unit/... ignoring 42 frames"
+              parser_class: !ruby/class 'Net::IMAP::ResponseParser'
+              string: "* OK [CLOSED shouldn't be anything here] mailbox closed\r\n"
+              pos: 13
+              lex_state: :EXPR_BEG
+              token: !ruby/struct:Net::IMAP::ResponseParser::Token
+                symbol: :SPACE
+                value: " "
+            unparsed_data: shouldn't be anything here
+            parsed_data:
+        text: mailbox closed
+      raw_data: "* OK [CLOSED shouldn't be anything here] mailbox closed\r\n"
+
   outlook.com puts an extra SP in ENVELOPE address lists:
     comment: |
       An annoying bug from outlook.com.  They've had the bug for years, and