fix: Save msgs to key-contacts migration state and run migration periodically (#6956)

Save:
- (old contact id) -> (new contact id) mapping.
- The message id starting from which all messages are already migrated.
Run the migration from `housekeeping()` and make the latter run every 2.4 hours untill all messages
are migrated. Migrate >= 1000 messages per run.

Author: iequidoo

src/constants.rs

@@ -257,6 +257,9 @@ pub(crate) const ASM_BODY: &str = "This is the Autocrypt Setup Message \
     If you see this message in a chatmail client (Delta Chat, Arcane Chat, Delta Touch ...), \
     use \"Settings / Add Second Device\" instead.";
 
+/// Period between `sql::housekeeping()` runs.
+pub(crate) const HOUSEKEEPING_PERIOD: i64 = 24 * 60 * 60;
+
 #[cfg(test)]
 mod tests {
     use num_traits::FromPrimitive;

src/context.rs

@@ -1041,6 +1041,13 @@ impl Context {
                 .await?
                 .to_string(),
         );
+        res.insert(
+            "first_key_contacts_msg_id",
+            self.sql
+                .get_raw_config("first_key_contacts_msg_id")
+                .await?
+                .unwrap_or_default(),
+        );
 
         let elapsed = time_elapsed(&self.creation_time);
         res.insert("uptime", duration_to_str(elapsed));

src/scheduler.rs

@@ -15,6 +15,7 @@ use tokio_util::task::TaskTracker;
 
 use self::connectivity::ConnectivityStore;
 use crate::config::{self, Config};
+use crate::constants;
 use crate::contact::{ContactId, RecentlySeenLoop};
 use crate::context::Context;
 use crate::download::{DownloadState, download_msg};
@@ -497,7 +498,8 @@ async fn inbox_fetch_idle(ctx: &Context, imap: &mut Imap, mut session: Session)
 
     match ctx.get_config_i64(Config::LastHousekeeping).await {
         Ok(last_housekeeping_time) => {
-            let next_housekeeping_time = last_housekeeping_time.saturating_add(60 * 60 * 24);
+            let next_housekeeping_time =
+                last_housekeeping_time.saturating_add(constants::HOUSEKEEPING_PERIOD);
             if next_housekeeping_time <= time() {
                 sql::housekeeping(ctx).await.log_err(ctx).ok();
             }

src/sql.rs

@@ -581,6 +581,12 @@ impl Sql {
         Ok(value)
     }
 
+    /// Removes the `key`'s value from the cache.
+    pub(crate) async fn uncache_raw_config(&self, key: &str) {
+        let mut lock = self.config_cache.write().await;
+        lock.remove(key);
+    }
+
     /// Sets configuration for the given key to 32-bit signed integer value.
     pub async fn set_raw_config_int(&self, key: &str, value: i32) -> Result<()> {
         self.set_raw_config(key, Some(&format!("{value}"))).await
@@ -793,6 +799,12 @@ pub async fn housekeeping(context: &Context) -> Result<()> {
         .log_err(context)
         .ok();
 
+    migrations::msgs_to_key_contacts(context)
+        .await
+        .context("migrations::msgs_to_key_contacts")
+        .log_err(context)
+        .ok();
+
     info!(context, "Housekeeping done.");
     Ok(())
 }

src/sql/migrations.rs

@@ -1,5 +1,6 @@
 //! Migrations module.
 
+use std::cmp;
 use std::collections::BTreeMap;
 use std::collections::BTreeSet;
 use std::time::Instant;
@@ -12,7 +13,7 @@ use rusqlite::OptionalExtension;
 
 use crate::config::Config;
 use crate::configure::EnteredLoginParam;
-use crate::constants::ShowEmails;
+use crate::constants::{self, ShowEmails};
 use crate::context::Context;
 use crate::imap;
 use crate::key::DcKey;
@@ -21,7 +22,7 @@ use crate::login_param::ConfiguredLoginParam;
 use crate::message::MsgId;
 use crate::provider::get_provider_by_domain;
 use crate::sql::Sql;
-use crate::tools::inc_and_check;
+use crate::tools::{Time, inc_and_check, time, time_elapsed};
 
 const DBVERSION: i32 = 68;
 const VERSION_CFG: &str = "dbversion";
@@ -1245,6 +1246,10 @@ CREATE INDEX gossip_timestamp_index ON gossip_timestamp (chat_id, fingerprint);
             "key-contacts migration took {:?} in total.",
             start.elapsed()
         );
+        // Schedule `msgs_to_key_contacts()`.
+        context
+            .set_config_internal(Config::LastHousekeeping, None)
+            .await?;
     }
 
     let new_version = sql
@@ -1830,62 +1835,120 @@ fn migrate_key_contacts(
     }
 
     // ======================= Step 5: =======================
-    // Rewrite `from_id` in messages
+    // Prepare for rewriting `from_id`, `to_id` in messages
     {
-        let start = Instant::now();
-
-        let mut encrypted_msgs_stmt = transaction
-            .prepare(
-                "SELECT id, from_id, to_id
-                FROM msgs
-                WHERE chat_id>9
-                AND (param GLOB '*\nc=1*' OR param GLOB 'c=1*')
-                ORDER BY id DESC LIMIT 10000",
+        let mut contacts_map = autocrypt_key_contacts_with_reset_peerstate;
+        for (old, new) in autocrypt_key_contacts {
+            contacts_map.insert(old, new);
+        }
+        transaction
+            .execute(
+                "CREATE TABLE key_contacts_map (
+                    old_id INTEGER PRIMARY KEY,
+                    new_id INTEGER NOT NULL
+                ) WITHOUT ROWID, STRICT",
+                (),
             )
             .context("Step 32")?;
-        let mut rewrite_msg_stmt = transaction
-            .prepare("UPDATE msgs SET from_id=?, to_id=? WHERE id=?")
-            .context("Step 32.1")?;
-
-        struct LoadedMsg {
-            id: u32,
-            from_id: u32,
-            to_id: u32,
+        {
+            let mut stmt = transaction
+                .prepare("INSERT INTO key_contacts_map (old_id, new_id) VALUES (?, ?)")
+                .context("Step 33")?;
+            for ids in contacts_map {
+                stmt.execute(ids).context("Step 34")?;
+            }
         }
+        transaction
+            .execute(
+                "INSERT INTO config (keyname, value) VALUES (
+                    'first_key_contacts_msg_id',
+                    IFNULL((SELECT MAX(id)+1 FROM msgs), 0)
+                )",
+                (),
+            )
+            .context("Step 35")?;
+    }
 
-        let encrypted_msgs = encrypted_msgs_stmt
-            .query_map((), |row| {
-                let id: u32 = row.get(0)?;
-                let from_id: u32 = row.get(1)?;
-                let to_id: u32 = row.get(2)?;
-                Ok(LoadedMsg { id, from_id, to_id })
-            })
-            .context("Step 33")?;
-
-        for msg in encrypted_msgs {
-            let msg = msg.context("Step 34")?;
-
-            let new_from_id = *autocrypt_key_contacts
-                .get(&msg.from_id)
-                .or_else(|| autocrypt_key_contacts_with_reset_peerstate.get(&msg.from_id))
-                .unwrap_or(&msg.from_id);
-
-            let new_to_id = *autocrypt_key_contacts
-                .get(&msg.to_id)
-                .or_else(|| autocrypt_key_contacts_with_reset_peerstate.get(&msg.to_id))
-                .unwrap_or(&msg.to_id);
+    Ok(())
+}
 
-            rewrite_msg_stmt
-                .execute((new_from_id, new_to_id, msg.id))
-                .context("Step 35")?;
+/// Rewrite `from_id`, `to_id` in ~1000 messages starting from the newest ones, to key-contacts.
+pub(crate) async fn msgs_to_key_contacts(context: &Context) -> Result<()> {
+    if context
+        .sql
+        .get_raw_config_int64("first_key_contacts_msg_id")
+        .await?
+        <= Some(0)
+    {
+        return Ok(());
+    }
+    let start = Time::now();
+    let trans_fn = |t: &mut rusqlite::Transaction| {
+        let mut first_key_contacts_msg_id: u64 = t
+            .query_one(
+                "SELECT CAST(value AS INTEGER) FROM config WHERE keyname='first_key_contacts_msg_id'",
+                (),
+                |row| row.get(0),
+            )
+            .context("Get first_key_contacts_msg_id")?;
+        let mut stmt = t
+            .prepare(
+                "UPDATE msgs SET
+                    from_id=IFNULL(
+                        (SELECT new_id FROM key_contacts_map WHERE old_id=msgs.from_id),
+                        from_id
+                    ),
+                    to_id=IFNULL(
+                        (SELECT new_id FROM key_contacts_map WHERE old_id=msgs.to_id),
+                        to_id
+                    )
+                WHERE id>=? AND id<?
+                AND chat_id>9
+                AND (param GLOB '*\nc=1*' OR param GLOB 'c=1*')",
+            )
+            .context("Prepare stmt")?;
+        let msgs_to_migrate = 1000;
+        let mut msgs_migrated: u64 = 0;
+        while first_key_contacts_msg_id > 0 && msgs_migrated < msgs_to_migrate {
+            let start_msg_id = first_key_contacts_msg_id.saturating_sub(msgs_to_migrate);
+            let cnt: u64 = stmt
+                .execute((start_msg_id, first_key_contacts_msg_id))
+                .context("UPDATE msgs")?
+                .try_into()?;
+            msgs_migrated += cnt;
+            first_key_contacts_msg_id = start_msg_id;
         }
-        info!(
-            context,
-            "Rewriting msgs to key-contacts took {:?}.",
-            start.elapsed()
+        t.execute(
+            "UPDATE config SET value=? WHERE keyname='first_key_contacts_msg_id'",
+            (first_key_contacts_msg_id,),
+        )
+        .context("Update first_key_contacts_msg_id")?;
+        Ok((msgs_migrated, first_key_contacts_msg_id))
+    };
+    let (msgs_migrated, first_key_contacts_msg_id) = context.sql.transaction(trans_fn).await?;
+    context
+        .sql
+        .uncache_raw_config("first_key_contacts_msg_id")
+        .await;
+    info!(
+        context,
+        "Rewriting {msgs_migrated} msgs to key-contacts took {:?}.",
+        time_elapsed(&start),
+    );
+
+    if first_key_contacts_msg_id > 0 {
+        let last_housekeeping = context.get_config_i64(Config::LastHousekeeping).await?;
+        let last_housekeeping = cmp::min(
+            last_housekeeping,
+            time().saturating_sub(constants::HOUSEKEEPING_PERIOD / 10 * 9),
         );
+        context
+            .set_config_internal(
+                Config::LastHousekeeping,
+                Some(&last_housekeeping.to_string()),
+            )
+            .await?;
     }
-
     Ok(())
 }