🔀 Merge pull request #329 from nevans/vanished

✨ Add support for VANISHED responses

Author: nevans

lib/net/imap.rb

@@ -1889,48 +1889,64 @@ def unselect
       send_command("UNSELECT")
     end
 
+    # call-seq:
+    #   expunge -> array of message sequence numbers
+    #   expunge -> VanishedData of UIDs
+    #
     # Sends an {EXPUNGE command [IMAP4rev1 §6.4.3]}[https://www.rfc-editor.org/rfc/rfc3501#section-6.4.3]
-    # Sends a EXPUNGE command to permanently remove from the currently
-    # selected mailbox all messages that have the \Deleted flag set.
+    # to permanently remove all messages with the +\Deleted+ flag from the
+    # currently selected mailbox.
+    #
+    # Returns either an array of expunged message <em>sequence numbers</em> or
+    # (when the appropriate capability is enabled) VanishedData of expunged
+    # UIDs.  Previously unhandled +EXPUNGE+ or +VANISHED+ responses are merged
+    # with the direct response to this command.  <tt>VANISHED (EARLIER)</tt>
+    # responses will _not_ be merged.
+    #
+    # When no messages have been expunged, an empty array is returned,
+    # regardless of which extensions are enabled.  In a future release, an empty
+    # VanishedData may be returned, based on the currently enabled extensions.
     #
     # Related: #uid_expunge
+    #
+    # ==== Capabilities
+    #
+    # When either QRESYNC[https://tools.ietf.org/html/rfc7162] or
+    # UIDONLY[https://tools.ietf.org/html/rfc9586] are enabled, #expunge
+    # returns VanishedData, which contains UIDs---<em>not message sequence
+    # numbers</em>.
     def expunge
-      synchronize do
-        send_command("EXPUNGE")
-        clear_responses("EXPUNGE")
-      end
+      expunge_internal("EXPUNGE")
     end
 
+    # call-seq:
+    #   uid_expunge{uid_set) -> array of message sequence numbers
+    #   uid_expunge{uid_set) -> VanishedData of UIDs
+    #
     # Sends a {UID EXPUNGE command [RFC4315 §2.1]}[https://www.rfc-editor.org/rfc/rfc4315#section-2.1]
     # {[IMAP4rev2 §6.4.9]}[https://www.rfc-editor.org/rfc/rfc9051#section-6.4.9]
     # to permanently remove all messages that have both the <tt>\\Deleted</tt>
     # flag set and a UID that is included in +uid_set+.
     #
+    # Returns the same result type as #expunge.
+    #
     # By using #uid_expunge instead of #expunge when resynchronizing with
     # the server, the client can ensure that it does not inadvertantly
     # remove any messages that have been marked as <tt>\\Deleted</tt> by other
     # clients between the time that the client was last connected and
     # the time the client resynchronizes.
     #
-    # *Note:*
-    # >>>
-    #        Although the command takes a set of UIDs for its argument, the
-    #        server still returns regular EXPUNGE responses, which contain
-    #        a <em>sequence number</em>. These will be deleted from
-    #        #responses and this method returns them as an array of
-    #        <em>sequence number</em> integers.
-    #
     # Related: #expunge
     #
     # ==== Capabilities
     #
-    # The server's capabilities must include +UIDPLUS+
+    # The server's capabilities must include either +IMAP4rev2+ or +UIDPLUS+
     # [RFC4315[https://www.rfc-editor.org/rfc/rfc4315.html]].
+    #
+    # Otherwise, #uid_expunge is updated by extensions in the same way as
+    # #expunge.
     def uid_expunge(uid_set)
-      synchronize do
-        send_command("UID EXPUNGE", SequenceSet.new(uid_set))
-        clear_responses("EXPUNGE")
-      end
+      expunge_internal("UID EXPUNGE", SequenceSet.new(uid_set))
     end
 
     # :call-seq:
@@ -3261,6 +3277,22 @@ def enforce_logindisabled?
       end
     end
 
+    def expunge_internal(...)
+      synchronize do
+        send_command(...)
+        expunged_array = clear_responses("EXPUNGE")
+        vanished_array = extract_responses("VANISHED") { !_1.earlier? }
+        if vanished_array.empty?
+          expunged_array
+        elsif vanished_array.length == 1
+          vanished_array.first
+        else
+          merged_uids = SequenceSet[*vanished_array.map(&:uids)]
+          VanishedData[uids: merged_uids, earlier: false]
+        end
+      end
+    end
+
     RETURN_WHOLE = /\ARETURN\z/i
     RETURN_START = /\ARETURN\b/i
     private_constant :RETURN_WHOLE, :RETURN_START

lib/net/imap/response_data.rb

@@ -6,6 +6,7 @@ class IMAP < Protocol
     autoload :FetchData,        "#{__dir__}/fetch_data"
     autoload :SearchResult,     "#{__dir__}/search_result"
     autoload :SequenceSet,      "#{__dir__}/sequence_set"
+    autoload :VanishedData,     "#{__dir__}/vanished_data"
 
     # Net::IMAP::ContinuationRequest represents command continuation requests.
     #

lib/net/imap/response_parser.rb

@@ -769,7 +769,6 @@ def remaining_unparsed
       def response_data__ignored; response_data__unhandled(IgnoredResponse) end
       alias response_data__noop     response_data__ignored
 
-      alias expunged_resp           response_data__unhandled
       alias uidfetch_resp           response_data__unhandled
       alias listrights_data         response_data__unhandled
       alias myrights_data           response_data__unhandled
@@ -841,6 +840,20 @@ def response_data__simple_numeric
       alias mailbox_data__exists  response_data__simple_numeric
       alias mailbox_data__recent  response_data__simple_numeric
 
+      # The name for this is confusing, because it *replaces* EXPUNGE
+      # >>>
+      #   expunged-resp       =  "VANISHED" [SP "(EARLIER)"] SP known-uids
+      def expunged_resp
+        name    = label "VANISHED"; SP!
+        earlier = if lpar? then label("EARLIER"); rpar; SP!; true else false end
+        uids    = known_uids
+        data    = VanishedData[uids, earlier]
+        UntaggedResponse.new name, data, @str
+      end
+
+      # TODO: replace with uid_set
+      alias known_uids sequence_set
+
       # RFC3501 & RFC9051:
       #   msg-att         = "(" (msg-att-dynamic / msg-att-static)
       #                      *(SP (msg-att-dynamic / msg-att-static)) ")"

lib/net/imap/vanished_data.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Net
+  class IMAP < Protocol
+
+    # Net::IMAP::VanishedData represents the contents of a +VANISHED+ response,
+    # which is described by the
+    # {QRESYNC}[https://www.rfc-editor.org/rfc/rfc7162.html] extension.
+    # [{RFC7162 §3.2.10}[https://www.rfc-editor.org/rfc/rfc7162.html#section-3.2.10]].
+    #
+    # +VANISHED+ responses replace +EXPUNGE+ responses when either the
+    # {QRESYNC}[https://www.rfc-editor.org/rfc/rfc7162.html] or the
+    # {UIDONLY}[https://www.rfc-editor.org/rfc/rfc9586.html] extension has been
+    # enabled.
+    class VanishedData < Data.define(:uids, :earlier)
+
+      # Returns a new VanishedData object.
+      #
+      # * +uids+ will be converted by SequenceSet.[].
+      # * +earlier+ will be converted to +true+ or +false+
+      def initialize(uids:, earlier:)
+        uids    = SequenceSet[uids]
+        earlier = !!earlier
+        super
+      end
+
+      ##
+      # :attr_reader: uids
+      #
+      # SequenceSet of UIDs that have been permanently removed from the mailbox.
+
+      ##
+      # :attr_reader: earlier
+      #
+      # +true+ when the response was caused by Net::IMAP#uid_fetch with
+      # <tt>vanished: true</tt> or Net::IMAP#select/Net::IMAP#examine with
+      # <tt>qresync: true</tt>.
+      #
+      # +false+ when the response is used to announce message removals within an
+      # already selected mailbox.
+
+      # rdoc doesn't handle attr aliases nicely. :(
+      alias earlier? earlier # :nodoc:
+      ##
+      # :attr_reader: earlier?
+      #
+      # Alias for #earlier.
+
+      # Returns an Array of all of the UIDs in #uids.
+      #
+      # See SequenceSet#numbers.
+      def to_a; uids.numbers end
+
+    end
+  end
+end

test/net/imap/fixtures/response_parser/rfc7162_condstore_qresync_responses.yml

@@ -142,18 +142,38 @@
     :response: "* VANISHED (EARLIER) 41,43:116,118,120:211,214:540\r\n"
     :expected: !ruby/struct:Net::IMAP::UntaggedResponse
       name: VANISHED
-      data: !ruby/struct:Net::IMAP::UnparsedData
-        unparsed_data: "(EARLIER) 41,43:116,118,120:211,214:540"
+      data: !ruby/object:Net::IMAP::VanishedData
+        uids: !ruby/object:Net::IMAP::SequenceSet
+          string: 41,43:116,118,120:211,214:540
+          tuples:
+          - - 41
+            - 41
+          - - 43
+            - 116
+          - - 118
+            - 118
+          - - 120
+            - 211
+          - - 214
+            - 540
+        earlier: true
       raw_data: "* VANISHED (EARLIER) 41,43:116,118,120:211,214:540\r\n"
-    comment: |
-      Note that QRESYNC isn't supported yet, so the data is unparsed.
 
   "RFC7162 QRESYNC 3.2.7. EXPUNGE Command":
     :response: "* VANISHED 405,407,410,425\r\n"
     :expected: !ruby/struct:Net::IMAP::UntaggedResponse
       name: VANISHED
-      data: !ruby/struct:Net::IMAP::UnparsedData
-        unparsed_data: '405,407,410,425'
+      data: !ruby/object:Net::IMAP::VanishedData
+        uids: !ruby/object:Net::IMAP::SequenceSet
+          string: '405,407,410,425'
+          tuples:
+          - - 405
+            - 405
+          - - 407
+            - 407
+          - - 410
+            - 410
+          - - 425
+            - 425
+        earlier: false
       raw_data: "* VANISHED 405,407,410,425\r\n"
-    comment: |
-      Note that QRESYNC isn't supported yet, so the data is unparsed.

test/net/imap/test_imap.rb

@@ -1016,7 +1016,7 @@ def test_id
     end
   end
 
-  def test_uidplus_uid_expunge
+  test "#uid_expunge with EXPUNGE responses" do
     with_fake_server(select: "INBOX",
                      extensions: %i[UIDPLUS]) do |server, imap|
       server.on "UID EXPUNGE" do |resp|
@@ -1032,6 +1032,24 @@ def test_uidplus_uid_expunge
     end
   end
 
+  test "#uid_expunge with VANISHED response" do
+    with_fake_server(select: "INBOX",
+                     extensions: %i[UIDPLUS]) do |server, imap|
+      server.on "UID EXPUNGE" do |resp|
+        resp.untagged("VANISHED 1001,1003")
+        resp.done_ok
+      end
+      response = imap.uid_expunge(1000..1003)
+      cmd = server.commands.pop
+      assert_equal ["UID EXPUNGE", "1000:1003"], [cmd.name, cmd.args]
+      assert_equal(
+        Net::IMAP::VanishedData[uids: [1001, 1003], earlier: false],
+        response
+      )
+      assert_equal([], imap.clear_responses("VANISHED"))
+    end
+  end
+
   def test_uidplus_appenduid
     with_fake_server(select: "INBOX",
                      extensions: %i[UIDPLUS]) do |server, imap|
@@ -1168,6 +1186,65 @@ def test_enable
     end
   end
 
+  test "#expunge with EXPUNGE responses" do
+    with_fake_server(select: "INBOX") do |server, imap|
+      server.on "EXPUNGE" do |resp|
+        resp.untagged("1 EXPUNGE")
+        resp.untagged("1 EXPUNGE")
+        resp.untagged("99 EXPUNGE")
+        resp.done_ok
+      end
+      response = imap.expunge
+      cmd = server.commands.pop
+      assert_equal ["EXPUNGE", nil], [cmd.name, cmd.args]
+      assert_equal [1, 1, 99], response
+      assert_equal [], imap.clear_responses("EXPUNGED")
+    end
+  end
+
+  test "#expunge with a VANISHED response" do
+    with_fake_server(select: "INBOX") do |server, imap|
+      server.on "EXPUNGE" do |resp|
+        resp.untagged("VANISHED 15:456")
+        resp.done_ok
+      end
+      response = imap.expunge
+      cmd = server.commands.pop
+      assert_equal ["EXPUNGE", nil], [cmd.name, cmd.args]
+      assert_equal(
+        Net::IMAP::VanishedData[uids: [15..456], earlier: false],
+        response
+      )
+      assert_equal([], imap.clear_responses("VANISHED"))
+    end
+  end
+
+  test "#expunge with multiple VANISHED responses" do
+    with_fake_server(select: "INBOX") do |server, imap|
+      server.unsolicited("VANISHED 86")
+      server.on "EXPUNGE" do |resp|
+        resp.untagged("VANISHED (EARLIER) 1:5,99,123")
+        resp.untagged("VANISHED 15,456")
+        resp.untagged("VANISHED (EARLIER) 987,1001")
+        resp.done_ok
+      end
+      response = imap.expunge
+      cmd = server.commands.pop
+      assert_equal ["EXPUNGE", nil], [cmd.name, cmd.args]
+      assert_equal(
+        Net::IMAP::VanishedData[uids: [15, 86, 456], earlier: false],
+        response
+      )
+      assert_equal(
+        [
+          Net::IMAP::VanishedData[uids: [1..5, 99, 123], earlier: true],
+          Net::IMAP::VanishedData[uids: [987, 1001],     earlier: true],
+        ],
+        imap.clear_responses("VANISHED")
+      )
+    end
+  end
+
   def test_close
     with_fake_server(select: "inbox") do |server, imap|
       resp = imap.close