feat: add api to get information about diskspace usage of database. (jsonrpc method: get_storage_usage_report_string) (#7486)

new jsonrpc api: `get_storage_usage_report_string(accountId)`
new rust API: `get_storage_usage(&context)`

Author: Simon-Laux

deltachat-jsonrpc/src/api.rs

@@ -34,6 +34,7 @@ use deltachat::qr_code_generator::{generate_backup_qr, get_securejoin_qr_svg};
 use deltachat::reaction::{get_msg_reactions, send_reaction};
 use deltachat::securejoin;
 use deltachat::stock_str::StockMessage;
+use deltachat::storage_usage::get_storage_usage;
 use deltachat::webxdc::StatusUpdateSerial;
 use deltachat::EventEmitter;
 use sanitize_filename::is_sanitized;
@@ -366,6 +367,13 @@ impl CommandApi {
         ctx.get_info().await
     }
 
+    /// Get storage usage report as formatted string
+    async fn get_storage_usage_report_string(&self, account_id: u32) -> Result<String> {
+        let ctx = self.get_context(account_id).await?;
+        let storage_usage = get_storage_usage(&ctx).await?;
+        Ok(storage_usage.to_string())
+    }
+
     /// Get the blob dir.
     async fn get_blob_dir(&self, account_id: u32) -> Result<Option<String>> {
         let ctx = self.get_context(account_id).await?;

src/lib.rs

@@ -89,6 +89,7 @@ pub mod securejoin;
 mod simplify;
 mod smtp;
 pub mod stock_str;
+pub mod storage_usage;
 mod sync;
 mod timesmearing;
 mod token;

src/storage_usage.rs

@@ -0,0 +1,109 @@
+//! Module to collect and display Disk Space Usage of a Profile.
+use crate::{context::Context, message::MsgId};
+use anyhow::Result;
+use humansize::{BINARY, format_size};
+
+/// Storage Usage Report
+/// Useful for debugging space usage problems in the deltachat database.
+#[derive(Debug)]
+pub struct StorageUsage {
+    /// Total database size, subtract this from the backup size to estimate size of all blobs
+    pub db_size: usize,
+    /// size and row count of the 10 biggest tables
+    pub largest_tables: Vec<(String, usize, Option<usize>)>,
+    /// count and total size of status updates
+    /// for the 10 webxdc apps with the most size usage in status updates
+    pub largest_webxdc_data: Vec<(MsgId, usize, usize)>,
+}
+
+impl std::fmt::Display for StorageUsage {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        writeln!(f, "Storage Usage:")?;
+        let human_db_size = format_size(self.db_size, BINARY);
+        writeln!(f, "[Database Size]: {human_db_size}")?;
+        writeln!(f, "[Largest Tables]:")?;
+        for (name, size, row_count) in &self.largest_tables {
+            let human_table_size = format_size(*size, BINARY);
+            writeln!(
+                f,
+                "   {name:<20} {human_table_size:>10}, {row_count:>6} rows",
+                name = format!("{name}:"),
+                row_count = row_count.map(|c| c.to_string()).unwrap_or("?".to_owned())
+            )?;
+        }
+        writeln!(f, "[Webxdc With Biggest Status Update Space Usage]:")?;
+        for (msg_id, size, update_count) in &self.largest_webxdc_data {
+            let human_size = format_size(*size, BINARY);
+            writeln!(
+                f,
+                "   {msg_id:<8} {human_size:>10} across {update_count:>5} updates",
+                msg_id = format!("{msg_id}:")
+            )?;
+        }
+        Ok(())
+    }
+}
+
+/// Get storage usage information for the Context's database
+pub async fn get_storage_usage(ctx: &Context) -> Result<StorageUsage> {
+    let page_size: usize = ctx
+        .sql
+        .query_get_value("PRAGMA page_size", ())
+        .await?
+        .unwrap_or_default();
+    let page_count: usize = ctx
+        .sql
+        .query_get_value("PRAGMA page_count", ())
+        .await?
+        .unwrap_or_default();
+
+    let mut largest_tables = ctx
+        .sql
+        .query_map_vec(
+            "SELECT name,
+                SUM(pgsize) AS size
+                FROM dbstat
+                WHERE name IN (SELECT name FROM sqlite_master WHERE type='table')
+                GROUP BY name ORDER BY size DESC LIMIT 10",
+            (),
+            |row| {
+                let name: String = row.get(0)?;
+                let size: usize = row.get(1)?;
+                Ok((name, size, None))
+            },
+        )
+        .await?;
+
+    for row in &mut largest_tables {
+        let name = &row.0;
+        let row_count: Result<Option<usize>> = ctx
+            .sql
+            // SECURITY: the table name comes from the db, not from the user
+            .query_get_value(&format!("SELECT COUNT(*) FROM {name}"), ())
+            .await;
+        row.2 = row_count.unwrap_or_default();
+    }
+
+    let largest_webxdc_data = ctx
+        .sql
+        .query_map_vec(
+            "SELECT msg_id, SUM(length(update_item)) as size, COUNT(*) as update_count
+                 FROM msgs_status_updates
+                 GROUP BY msg_id ORDER BY size DESC LIMIT 10",
+            (),
+            |row| {
+                let msg_id: MsgId = row.get(0)?;
+                let size: usize = row.get(1)?;
+                let count: usize = row.get(2)?;
+
+                Ok((msg_id, size, count))
+            },
+        )
+        .await?;
+
+    Ok(StorageUsage {
+        db_size: page_size * page_count,
+        largest_tables,
+        largest_webxdc_data,
+    })
+}