fix: Order of messages if Sentbox is synced before Inbox (#7169)

This fixes a Gmail-like scenario when outgoing messages are saved to Sentbox as well and it is
fetched before Inbox by chance, by allowing received outgoing messages to mingle with fresh incoming
ones. The reason is that a received outgoing message may be a reply (explicit or implicit) to an
incoming message received later, so it's better to sort them together purely by timestamp. Another
case is the user sharing their account with someone else (using another device) or having some
auto-reply bot.

This fixes the described scenarios w/o introducing a new message state for outgoing messages because locally sent ones have zero `timestamp_sent`, so we can filter them in `calc_sort_timestamp()`.

As for messages sent locally, there's no need to make them more noticeable even if they are newer,
so received outgoing messages are added after them.

Author: iequidoo

src/chat.rs

@@ -1450,17 +1450,23 @@ impl ChatId {
             // received message purely by timestamp. We could place it just before that seen
             // message, but anyway the user may not notice it.
             //
-            // NB: Received outgoing messages may break sorting of fresh incoming ones, but this
-            // shouldn't happen frequently. Seen incoming messages don't really break sorting of
-            // fresh ones, they rather mean that older incoming messages are actually seen as well.
+            // NB: Seen incoming messages don't really break sorting of fresh ones, they rather mean
+            // that older incoming messages are actually seen as well.
+            // NB: Locally sent messages have zero `timestamp_sent`.
             context
                 .sql
                 .query_row_optional(
                     "SELECT MAX(timestamp), MAX(IIF(state=?,timestamp_sent,0))
                      FROM msgs
                      WHERE chat_id=? AND hidden=0 AND state>?
+                         AND (state!=? OR timestamp_sent=0)
                      HAVING COUNT(*) > 0",
-                    (MessageState::InSeen, self, MessageState::InFresh),
+                    (
+                        MessageState::InSeen,
+                        self,
+                        MessageState::InFresh,
+                        MessageState::OutDelivered,
+                    ),
                     |row| {
                         let ts: i64 = row.get(0)?;
                         let ts_sent_seen: i64 = row.get(1)?;

src/events/payload.rs

@@ -35,7 +35,7 @@ pub enum EventType {
     /// Emitted when an IMAP message has been moved
     ImapMessageMoved(String),
 
-    /// Emitted before going into IDLE on the Inbox folder.
+    /// Emitted before going into IDLE on any folder.
     ImapInboxIdle,
 
     /// Emitted when an new file in the $BLOBDIR was created

src/scheduler.rs

@@ -661,6 +661,7 @@ async fn fetch_idle(
 
     connection.connectivity.set_idle(ctx);
 
+    // Maybe we'll remove watching other folders soon, so use this event for all folders for now.
     ctx.emit_event(EventType::ImapInboxIdle);
 
     if !session.can_idle() {

src/test_utils.rs

@@ -1521,7 +1521,7 @@ async fn write_msg(context: &Context, prefix: &str, msg: &Message, buf: &mut Str
 
     let statestr = match msg.get_state() {
         MessageState::OutPending => " o",
-        MessageState::OutDelivered => " √",
+        MessageState::OutDelivered if msg.timestamp_sent == 0 => " √",
         MessageState::OutMdnRcvd => " √√",
         MessageState::OutFailed => " !!",
         _ => "",

src/tests/verified_chats.rs

@@ -316,9 +316,9 @@ async fn test_old_message_4() -> Result<()> {
     Ok(())
 }
 
-/// Alice is offline for some time.
-/// When they come online, first their sentbox is synced and then their inbox.
-/// This test tests that the messages are still in the right order.
+/// Alice's device#0 is offline for some time.
+/// When it comes online, it sees a message from another device and an incoming message. Messages
+/// may come from different folders.
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn test_old_message_5() -> Result<()> {
     let alice = TestContext::new_alice().await;
@@ -348,7 +348,12 @@ async fn test_old_message_5() -> Result<()> {
     .await?
     .unwrap();
 
-    assert!(msg_sent.sort_timestamp == msg_incoming.sort_timestamp);
+    // If the messages come from the same folder and `msg_sent` is sent by Alice, it's better to
+    // sort `msg_incoming` after it so that it's more visible, but if Alice shares her account with
+    // someone else or has some auto-reply bot, messages should be sorted just by "Date". If the
+    // messages come from different folders, probably `msg_incoming` is already noticed by Alice on
+    // another device (and this is the most often case in practice).
+    assert!(msg_incoming.sort_timestamp < msg_sent.sort_timestamp);
     alice
         .golden_test_chat(msg_sent.chat_id, "test_old_message_5")
         .await;

test-data/golden/receive_imf_older_message_from_2nd_device

@@ -1,5 +1,5 @@
 Single#Chat#10: [email protected] [[email protected]] Icon: 4138c52e5bc1c576cda7dd44d088c07.png 
 --------------------------------------------------------------------------------
 Msg#10: Me (Contact#Contact#Self): We share this account  √
-Msg#11: Me (Contact#Contact#Self): I'm Alice too  √
+Msg#11: Me (Contact#Contact#Self): I'm Alice too 
 --------------------------------------------------------------------------------

test-data/golden/test_old_message_5

@@ -1,5 +1,5 @@
 Single#Chat#10: Bob [[email protected]] Icon: 4138c52e5bc1c576cda7dd44d088c07.png 
 --------------------------------------------------------------------------------
-Msg#10: Me (Contact#Contact#Self): Happy birthday, Bob!  √
 Msg#11:  (Contact#Contact#10): Happy birthday to me, Alice! [FRESH]
+Msg#10: Me (Contact#Contact#Self): Happy birthday, Bob! 
 --------------------------------------------------------------------------------

test-data/golden/test_outgoing_encrypted_msg

@@ -1,5 +1,5 @@
 Single#Chat#10: [email protected] [KEY [email protected]] 🛡️
 --------------------------------------------------------------------------------
 Msg#10: info (Contact#Contact#Info): Messages are end-to-end encrypted. [NOTICED][INFO 🛡️]
-Msg#11🔒: Me (Contact#Contact#Self): Test – This is encrypted, signed, and has an Autocrypt Header without prefer-encrypt=mutual.  √
+Msg#11🔒: Me (Contact#Contact#Self): Test – This is encrypted, signed, and has an Autocrypt Header without prefer-encrypt=mutual. 
 --------------------------------------------------------------------------------

test-data/golden/test_outgoing_mua_msg

@@ -1,4 +1,4 @@
 Single#Chat#11: [email protected] [[email protected]] Icon: 4138c52e5bc1c576cda7dd44d088c07.png 
 --------------------------------------------------------------------------------
-Msg#12: Me (Contact#Contact#Self): One classical MUA message  √
+Msg#12: Me (Contact#Contact#Self): One classical MUA message 
 --------------------------------------------------------------------------------