Fixed client mode flow args

Cargo.lock

@@ -1064,7 +1064,7 @@ dependencies = [
 
 [[package]]
 name = "khm"
-version = "0.5.0"
+version = "0.6.1"
 dependencies = [
  "actix-web",
  "base64 0.21.7",

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "khm"
-version = "0.5.0"
+version = "0.6.2"
 edition = "2021"
 authors = ["AB <ab@hexor.cy>"]
 

src/client.rs

@@ -11,6 +11,8 @@ use std::path::Path;
 struct SshKey {
     server: String,
     public_key: String,
+    #[serde(default)]
+    deprecated: bool,
 }
 
 fn read_known_hosts(file_path: &str) -> io::Result<Vec<SshKey>> {
@@ -26,7 +28,11 @@ fn read_known_hosts(file_path: &str) -> io::Result<Vec<SshKey>> {
                 if parts.len() >= 2 {
                     let server = parts[0].to_string();
                     let public_key = parts[1..].join(" ");
-                    keys.push(SshKey { server, public_key });
+                    keys.push(SshKey {
+                        server,
+                        public_key,
+                        deprecated: false, // Keys from known_hosts are not deprecated
+                    });
                 }
             }
             Err(e) => {
@@ -42,10 +48,17 @@ fn write_known_hosts(file_path: &str, keys: &[SshKey]) -> io::Result<()> {
     let path = Path::new(file_path);
     let mut file = File::create(&path)?;
 
-    for key in keys {
+    // Filter out deprecated keys - they should not be written to known_hosts
+    let active_keys: Vec<&SshKey> = keys.iter().filter(|key| !key.deprecated).collect();
+    let active_count = active_keys.len();
+
+    for key in active_keys {
         writeln!(file, "{} {}", key.server, key.public_key)?;
     }
-    info!("Wrote {} keys to known_hosts file", keys.len());
+    info!(
+        "Wrote {} active keys to known_hosts file (filtered out deprecated keys)",
+        active_count
+    );
 
     Ok(())
 }
@@ -167,7 +180,10 @@ pub async fn run_client(args: crate::Args) -> std::io::Result<()> {
         Ok(keys) => keys,
         Err(e) => {
             if e.kind() == io::ErrorKind::NotFound {
-                info!("known_hosts file not found: {}. Starting with empty key list.", args.known_hosts);
+                info!(
+                    "known_hosts file not found: {}. Starting with empty key list.",
+                    args.known_hosts
+                );
                 Vec::new()
             } else {
                 error!("Failed to read known_hosts file: {}", e);
@@ -177,20 +193,29 @@ pub async fn run_client(args: crate::Args) -> std::io::Result<()> {
     };
 
     let host = args.host.expect("host is required in client mode");
-    info!("Client mode: Sending keys to server at {}", host);
+    let flow = args.flow.expect("flow is required in client mode");
+    let url = format!("{}/{}", host, flow);
 
-    if let Err(e) = send_keys_to_server(&host, keys, &args.basic_auth).await {
+    info!("Client mode: Sending keys to server at {}", url);
+
+    if let Err(e) = send_keys_to_server(&url, keys, &args.basic_auth).await {
         error!("Failed to send keys to server: {}", e);
-        return Err(io::Error::new(io::ErrorKind::Other, format!("Network error: {}", e)));
+        return Err(io::Error::new(
+            io::ErrorKind::Other,
+            format!("Network error: {}", e),
+        ));
     }
 
     if args.in_place {
         info!("Client mode: In-place update is enabled. Fetching keys from server.");
-        let server_keys = match get_keys_from_server(&host, &args.basic_auth).await {
+        let server_keys = match get_keys_from_server(&url, &args.basic_auth).await {
             Ok(keys) => keys,
             Err(e) => {
                 error!("Failed to get keys from server: {}", e);
-                return Err(io::Error::new(io::ErrorKind::Other, format!("Network error: {}", e)));
+                return Err(io::Error::new(
+                    io::ErrorKind::Other,
+                    format!("Network error: {}", e),
+                ));
             }
         };
 

src/db.rs

@@ -1,8 +1,10 @@
 use crate::server::SshKey;
-use log::info;
+use log::{error, info};
 use std::collections::HashMap;
 use std::collections::HashSet;
-use tokio_postgres::Client;
+use tokio_postgres::tls::NoTlsStream;
+use tokio_postgres::Socket;
+use tokio_postgres::{Client, Connection, NoTls};
 
 // Structure for storing key processing statistics
 pub struct KeyInsertStats {
@@ -12,11 +14,59 @@ pub struct KeyInsertStats {
     pub key_id_map: Vec<(SshKey, i32)>, // Mapping of keys to their IDs in the database
 }
 
-pub async fn initialize_db_schema(client: &Client) -> Result<(), tokio_postgres::Error> {
+// Simple database client that exits on connection errors
+pub struct DbClient {
+    client: Client,
+}
+
+impl DbClient {
+    pub async fn connect(
+        connection_string: &str,
+    ) -> Result<(Self, Connection<Socket, NoTlsStream>), tokio_postgres::Error> {
+        info!("Connecting to database...");
+        let (client, connection) = tokio_postgres::connect(connection_string, NoTls).await?;
+        info!("Successfully connected to database");
+
+        Ok((DbClient { client }, connection))
+    }
+
+    // Helper function to handle database errors - exits the application on connection errors
+    fn handle_db_error<T>(
+        result: Result<T, tokio_postgres::Error>,
+        operation: &str,
+    ) -> Result<T, tokio_postgres::Error> {
+        match result {
+            Ok(value) => Ok(value),
+            Err(e) => {
+                if Self::is_connection_error(&e) {
+                    error!("Database connection lost during {}: {}", operation, e);
+                    error!("Exiting application due to database connection failure");
+                    std::process::exit(1);
+                } else {
+                    // For non-connection errors, just return the error
+                    Err(e)
+                }
+            }
+        }
+    }
+
+    fn is_connection_error(error: &tokio_postgres::Error) -> bool {
+        // Check if the error is related to connection issues
+        let error_str = error.to_string();
+        error_str.contains("connection closed")
+            || error_str.contains("connection reset")
+            || error_str.contains("broken pipe")
+            || error_str.contains("Connection refused")
+            || error_str.contains("connection terminated")
+            || error.as_db_error().is_none() // Non-database errors are often connection issues
+    }
+
+    pub async fn initialize_schema(&self) -> Result<(), tokio_postgres::Error> {
         info!("Checking and initializing database schema if needed");
 
         // Check if tables exist by querying information_schema
-    let tables_exist = client
+        let result = self
+            .client
             .query(
                 "SELECT EXISTS (
                     SELECT FROM information_schema.tables 
@@ -29,7 +79,9 @@ pub async fn initialize_db_schema(client: &Client) -> Result<(), tokio_postgres:
                 )",
                 &[],
             )
-        .await?
+            .await;
+
+        let tables_exist = Self::handle_db_error(result, "checking table existence")?
             .get(0)
             .map(|row| row.get::<_, bool>(0))
             .unwrap_or(false);
@@ -38,21 +90,25 @@ pub async fn initialize_db_schema(client: &Client) -> Result<(), tokio_postgres:
             info!("Database schema doesn't exist. Creating tables...");
 
             // Create the keys table
-        client
+            let result = self
+                .client
                 .execute(
                     "CREATE TABLE IF NOT EXISTS public.keys (
                         key_id SERIAL PRIMARY KEY,
                         host VARCHAR(255) NOT NULL,
                         key TEXT NOT NULL,
                         updated TIMESTAMP WITH TIME ZONE NOT NULL,
+                        deprecated BOOLEAN NOT NULL DEFAULT FALSE,
                         CONSTRAINT unique_host_key UNIQUE (host, key)
                     )",
                     &[],
                 )
-            .await?;
+                .await;
+            Self::handle_db_error(result, "creating keys table")?;
 
             // Create the flows table
-        client
+            let result = self
+                .client
                 .execute(
                     "CREATE TABLE IF NOT EXISTS public.flows (
                         flow_id SERIAL PRIMARY KEY,
@@ -66,28 +122,62 @@ pub async fn initialize_db_schema(client: &Client) -> Result<(), tokio_postgres:
                     )",
                     &[],
                 )
-            .await?;
+                .await;
+            Self::handle_db_error(result, "creating flows table")?;
 
             // Create an index for faster lookups
-        client
+            let result = self
+                .client
                 .execute(
                     "CREATE INDEX IF NOT EXISTS idx_flows_name ON public.flows(name)",
                     &[],
                 )
-            .await?;
+                .await;
+            Self::handle_db_error(result, "creating index")?;
 
             info!("Database schema created successfully");
         } else {
             info!("Database schema already exists");
+
+            // Check if deprecated column exists, add it if missing (migration)
+            let result = self
+                .client
+                .query(
+                    "SELECT EXISTS (
+                        SELECT FROM information_schema.columns 
+                        WHERE table_schema = 'public' 
+                        AND table_name = 'keys' 
+                        AND column_name = 'deprecated'
+                    )",
+                    &[],
+                )
+                .await;
+
+            let column_exists = Self::handle_db_error(result, "checking deprecated column")?
+                .get(0)
+                .map(|row| row.get::<_, bool>(0))
+                .unwrap_or(false);
+
+            if !column_exists {
+                info!("Adding deprecated column to existing keys table...");
+                let result = self.client
+                    .execute(
+                        "ALTER TABLE public.keys ADD COLUMN deprecated BOOLEAN NOT NULL DEFAULT FALSE",
+                        &[],
+                    )
+                    .await;
+                Self::handle_db_error(result, "adding deprecated column")?;
+                info!("Migration completed: deprecated column added");
+            }
         }
 
         Ok(())
-}
+    }
 
-pub async fn batch_insert_keys(
+    pub async fn batch_insert_keys(
-    client: &Client,
+        &self,
         keys: &[SshKey],
-) -> Result<KeyInsertStats, tokio_postgres::Error> {
+    ) -> Result<KeyInsertStats, tokio_postgres::Error> {
         if keys.is_empty() {
             return Ok(KeyInsertStats {
                 total: 0,
@@ -106,9 +196,10 @@ pub async fn batch_insert_keys(
             key_values.push(&key.public_key);
         }
 
-    // First, check which keys already exist in the database
+        // First, check which keys already exist in the database (including deprecated status)
         let mut existing_keys = HashMap::new();
-    let mut key_query = String::from("SELECT host, key, key_id FROM public.keys WHERE ");
+        let mut key_query =
+            String::from("SELECT host, key, key_id, deprecated FROM public.keys WHERE ");
 
         for i in 0..keys.len() {
             if i > 0 {
@@ -124,24 +215,34 @@ pub async fn batch_insert_keys(
             params.push(&key_values[i]);
         }
 
-    let rows = client.query(&key_query, &params[..]).await?;
+        let result = self.client.query(&key_query, &params[..]).await;
+        let rows = Self::handle_db_error(result, "checking existing keys")?;
 
         for row in rows {
             let host: String = row.get(0);
             let key: String = row.get(1);
             let key_id: i32 = row.get(2);
-        existing_keys.insert((host, key), key_id);
+            let deprecated: bool = row.get(3);
+            existing_keys.insert((host, key), (key_id, deprecated));
         }
 
         // Determine which keys need to be inserted and which already exist
         let mut keys_to_insert = Vec::new();
         let mut unchanged_keys = Vec::new();
+        let mut ignored_deprecated = 0;
 
         for key in keys {
             let key_tuple = (key.server.clone(), key.public_key.clone());
-        if existing_keys.contains_key(&key_tuple) {
+            if let Some((key_id, is_deprecated)) = existing_keys.get(&key_tuple) {
-            unchanged_keys.push((key.clone(), *existing_keys.get(&key_tuple).unwrap()));
+                if *is_deprecated {
+                    // Ignore deprecated keys - don't add them to any flow
+                    ignored_deprecated += 1;
                 } else {
+                    // Key exists and is not deprecated - add to unchanged
+                    unchanged_keys.push((key.clone(), *key_id));
+                }
+            } else {
+                // Key doesn't exist - add to insert list
                 keys_to_insert.push(key.clone());
             }
         }
@@ -150,7 +251,8 @@ pub async fn batch_insert_keys(
 
         // If there are keys to insert, perform the insertion
         if !keys_to_insert.is_empty() {
-        let mut insert_sql = String::from("INSERT INTO public.keys (host, key, updated) VALUES ");
+            let mut insert_sql =
+                String::from("INSERT INTO public.keys (host, key, updated) VALUES ");
 
             let mut insert_params: Vec<&(dyn tokio_postgres::types::ToSql + Sync)> = Vec::new();
             let mut param_count = 1;
@@ -167,7 +269,8 @@ pub async fn batch_insert_keys(
 
             insert_sql.push_str(" RETURNING key_id, host, key");
 
-        let inserted_rows = client.query(&insert_sql, &insert_params[..]).await?;
+            let result = self.client.query(&insert_sql, &insert_params[..]).await;
+            let inserted_rows = Self::handle_db_error(result, "inserting keys")?;
 
             for row in inserted_rows {
                 let host: String = row.get(1);
@@ -200,18 +303,18 @@ pub async fn batch_insert_keys(
         };
 
         info!(
-        "Keys stats: received={}, new={}, unchanged={}",
+            "Keys stats: received={}, new={}, unchanged={}, ignored_deprecated={}",
-        stats.total, stats.inserted, stats.unchanged
+            stats.total, stats.inserted, stats.unchanged, ignored_deprecated
         );
 
         Ok(stats)
-}
+    }
 
-pub async fn batch_insert_flow_keys(
+    pub async fn batch_insert_flow_keys(
-    client: &Client,
+        &self,
         flow_name: &str,
         key_ids: &[i32],
-) -> Result<usize, tokio_postgres::Error> {
+    ) -> Result<usize, tokio_postgres::Error> {
         if key_ids.is_empty() {
             info!("No keys to associate with flow '{}'", flow_name);
             return Ok(0);
@@ -236,7 +339,8 @@ pub async fn batch_insert_flow_keys(
             params.push(key_id);
         }
 
-    let rows = client.query(&existing_query, &params[..]).await?;
+        let result = self.client.query(&existing_query, &params[..]).await;
+        let rows = Self::handle_db_error(result, "checking existing flow associations")?;
 
         let mut existing_associations = HashSet::new();
         for row in rows {
@@ -280,7 +384,8 @@ pub async fn batch_insert_flow_keys(
         }
 
         // Execute query
-    let affected = client.execute(&sql, &insert_params[..]).await?;
+        let result = self.client.execute(&sql, &insert_params[..]).await;
+        let affected = Self::handle_db_error(result, "inserting flow associations")?;
 
         let affected_usize = affected as usize;
 
@@ -292,4 +397,280 @@ pub async fn batch_insert_flow_keys(
         );
 
         Ok(affected_usize)
+    }
+
+    pub async fn get_keys_from_db(
+        &self,
+    ) -> Result<Vec<crate::server::Flow>, tokio_postgres::Error> {
+        let result = self.client.query(
+            "SELECT k.host, k.key, k.deprecated, f.name FROM public.keys k INNER JOIN public.flows f ON k.key_id = f.key_id",
+            &[]
+        ).await;
+        let rows = Self::handle_db_error(result, "getting keys from database")?;
+
+        let mut flows_map: HashMap<String, crate::server::Flow> = HashMap::new();
+
+        for row in rows {
+            let host: String = row.get(0);
+            let key: String = row.get(1);
+            let deprecated: bool = row.get(2);
+            let flow: String = row.get(3);
+
+            let ssh_key = SshKey {
+                server: host,
+                public_key: key,
+                deprecated,
+            };
+
+            if let Some(flow_entry) = flows_map.get_mut(&flow) {
+                flow_entry.servers.push(ssh_key);
+            } else {
+                flows_map.insert(
+                    flow.clone(),
+                    crate::server::Flow {
+                        name: flow,
+                        servers: vec![ssh_key],
+                    },
+                );
+            }
+        }
+
+        info!("Retrieved {} flows from database", flows_map.len());
+        Ok(flows_map.into_values().collect())
+    }
+
+    pub async fn deprecate_key_by_server(
+        &self,
+        server_name: &str,
+        flow_name: &str,
+    ) -> Result<u64, tokio_postgres::Error> {
+        // Update keys to deprecated status for the given server
+        let result = self
+            .client
+            .execute(
+                "UPDATE public.keys 
+                 SET deprecated = TRUE, updated = NOW() 
+                 WHERE host = $1 
+                 AND key_id IN (
+                     SELECT key_id FROM public.flows WHERE name = $2
+                 )",
+                &[&server_name, &flow_name],
+            )
+            .await;
+        let affected = Self::handle_db_error(result, "deprecating key")?;
+
+        info!(
+            "Deprecated {} key(s) for server '{}' in flow '{}'",
+            affected, server_name, flow_name
+        );
+
+        Ok(affected)
+    }
+
+    pub async fn restore_key_by_server(
+        &self,
+        server_name: &str,
+        flow_name: &str,
+    ) -> Result<u64, tokio_postgres::Error> {
+        // Update keys to active status for the given server in the flow
+        let result = self
+            .client
+            .execute(
+                "UPDATE public.keys 
+                 SET deprecated = FALSE, updated = NOW() 
+                 WHERE host = $1 
+                 AND deprecated = TRUE
+                 AND key_id IN (
+                     SELECT key_id FROM public.flows WHERE name = $2
+                 )",
+                &[&server_name, &flow_name],
+            )
+            .await;
+        let affected = Self::handle_db_error(result, "restoring key")?;
+
+        info!(
+            "Restored {} key(s) for server '{}' in flow '{}'",
+            affected, server_name, flow_name
+        );
+
+        Ok(affected)
+    }
+
+    pub async fn permanently_delete_key_by_server(
+        &self,
+        server_name: &str,
+        flow_name: &str,
+    ) -> Result<u64, tokio_postgres::Error> {
+        // First, find the key_ids for the given server in the flow
+        let result = self
+            .client
+            .query(
+                "SELECT k.key_id FROM public.keys k 
+                 INNER JOIN public.flows f ON k.key_id = f.key_id 
+                 WHERE k.host = $1 AND f.name = $2",
+                &[&server_name, &flow_name],
+            )
+            .await;
+        let key_rows = Self::handle_db_error(result, "finding keys to delete")?;
+
+        if key_rows.is_empty() {
+            return Ok(0);
+        }
+
+        let key_ids: Vec<i32> = key_rows.iter().map(|row| row.get::<_, i32>(0)).collect();
+
+        // Delete flow associations first
+        let mut flow_delete_count = 0;
+        for key_id in &key_ids {
+            let result = self
+                .client
+                .execute(
+                    "DELETE FROM public.flows WHERE name = $1 AND key_id = $2",
+                    &[&flow_name, key_id],
+                )
+                .await;
+            let deleted = Self::handle_db_error(result, "deleting flow association")?;
+            flow_delete_count += deleted;
+        }
+
+        // Check if any of these keys are used in other flows
+        let mut keys_to_delete = Vec::new();
+        for key_id in &key_ids {
+            let result = self
+                .client
+                .query_one(
+                    "SELECT COUNT(*) FROM public.flows WHERE key_id = $1",
+                    &[key_id],
+                )
+                .await;
+            let count: i64 = Self::handle_db_error(result, "checking key references")?.get(0);
+
+            if count == 0 {
+                keys_to_delete.push(*key_id);
+            }
+        }
+
+        // Permanently delete keys that are no longer referenced by any flow
+        let mut total_deleted = 0;
+        for key_id in keys_to_delete {
+            let result = self
+                .client
+                .execute("DELETE FROM public.keys WHERE key_id = $1", &[&key_id])
+                .await;
+            let deleted = Self::handle_db_error(result, "deleting key")?;
+            total_deleted += deleted;
+        }
+
+        info!(
+            "Permanently deleted {} flow associations and {} orphaned keys for server '{}' in flow '{}'",
+            flow_delete_count, total_deleted, server_name, flow_name
+        );
+
+        Ok(std::cmp::max(flow_delete_count, total_deleted))
+    }
+}
+
+// Compatibility wrapper for transition
+pub struct ReconnectingDbClient {
+    inner: Option<DbClient>,
+}
+
+impl ReconnectingDbClient {
+    pub fn new(_connection_string: String) -> Self {
+        Self { inner: None }
+    }
+
+    pub async fn connect(&mut self, connection_string: &str) -> Result<(), tokio_postgres::Error> {
+        let (client, connection) = DbClient::connect(connection_string).await?;
+
+        // Spawn connection handler that will exit on error
+        tokio::spawn(async move {
+            if let Err(e) = connection.await {
+                error!("Database connection error: {}", e);
+                error!("Exiting application due to database connection failure");
+                std::process::exit(1);
+            }
+        });
+
+        self.inner = Some(client);
+        Ok(())
+    }
+
+    pub async fn initialize_schema(&self) -> Result<(), tokio_postgres::Error> {
+        match &self.inner {
+            Some(client) => client.initialize_schema().await,
+            None => panic!("Database client not initialized"),
+        }
+    }
+
+    pub async fn batch_insert_keys_reconnecting(
+        &self,
+        keys: Vec<SshKey>,
+    ) -> Result<KeyInsertStats, tokio_postgres::Error> {
+        match &self.inner {
+            Some(client) => client.batch_insert_keys(&keys).await,
+            None => panic!("Database client not initialized"),
+        }
+    }
+
+    pub async fn batch_insert_flow_keys_reconnecting(
+        &self,
+        flow_name: String,
+        key_ids: Vec<i32>,
+    ) -> Result<usize, tokio_postgres::Error> {
+        match &self.inner {
+            Some(client) => client.batch_insert_flow_keys(&flow_name, &key_ids).await,
+            None => panic!("Database client not initialized"),
+        }
+    }
+
+    pub async fn get_keys_from_db_reconnecting(
+        &self,
+    ) -> Result<Vec<crate::server::Flow>, tokio_postgres::Error> {
+        match &self.inner {
+            Some(client) => client.get_keys_from_db().await,
+            None => panic!("Database client not initialized"),
+        }
+    }
+
+    pub async fn deprecate_key_by_server_reconnecting(
+        &self,
+        server_name: String,
+        flow_name: String,
+    ) -> Result<u64, tokio_postgres::Error> {
+        match &self.inner {
+            Some(client) => {
+                client
+                    .deprecate_key_by_server(&server_name, &flow_name)
+                    .await
+            }
+            None => panic!("Database client not initialized"),
+        }
+    }
+
+    pub async fn restore_key_by_server_reconnecting(
+        &self,
+        server_name: String,
+        flow_name: String,
+    ) -> Result<u64, tokio_postgres::Error> {
+        match &self.inner {
+            Some(client) => client.restore_key_by_server(&server_name, &flow_name).await,
+            None => panic!("Database client not initialized"),
+        }
+    }
+
+    pub async fn permanently_delete_key_by_server_reconnecting(
+        &self,
+        server_name: String,
+        flow_name: String,
+    ) -> Result<u64, tokio_postgres::Error> {
+        match &self.inner {
+            Some(client) => {
+                client
+                    .permanently_delete_key_by_server(&server_name, &flow_name)
+                    .await
+            }
+            None => panic!("Database client not initialized"),
+        }
+    }
 }

src/main.rs

@@ -22,7 +22,7 @@ use log::{error, info};
     khm --server --ip 0.0.0.0 --port 1337 --db-host psql.psql.svc --db-name khm --db-user admin --db-password <SECRET> --flows work,home\n\
     \n\
     Running in client mode to send diff and sync ~/.ssh/known_hosts with remote flow `work` in place:\n\
-    khm --host https://khm.example.com/work --known-hosts ~/.ssh/known_hosts --in-place\n\
+    khm --host https://khm.example.com --flow work --known-hosts ~/.ssh/known_hosts --in-place\n\
     \n\
     "
 )]
@@ -96,10 +96,18 @@ struct Args {
     #[arg(
         long,
         required_if_eq("server", "false"),
-        help = "Client mode: Full host address of the server to connect to. Like https://khm.example.com/<FLOW_NAME>"
+        help = "Client mode: Full host address of the server to connect to. Like https://khm.example.com"
     )]
     host: Option<String>,
 
+    /// Flow name to use on the server
+    #[arg(
+        long,
+        required_if_eq("server", "false"),
+        help = "Client mode: Flow name to use on the server"
+    )]
+    flow: Option<String>,
+
     /// Path to the known_hosts file (default: ~/.ssh/known_hosts)
     #[arg(
         long,
@@ -121,13 +129,19 @@ async fn main() -> std::io::Result<()> {
     let args = Args::parse();
 
     // Check if we have the minimum required arguments
-    if !args.server && args.host.is_none() {
+    if !args.server && (args.host.is_none() || args.flow.is_none()) {
         // Neither server mode nor client mode properly configured
-        eprintln!("Error: You must specify either server mode (--server) or client mode (--host)");
+        eprintln!("Error: You must specify either server mode (--server) or client mode (--host and --flow)");
         eprintln!();
         eprintln!("Examples:");
-        eprintln!("  Server mode: {} --server --db-user admin --db-password pass --flows work,home", env!("CARGO_PKG_NAME"));
+        eprintln!(
-        eprintln!("  Client mode: {} --host https://khm.example.com/work", env!("CARGO_PKG_NAME"));
+            "  Server mode: {} --server --db-user admin --db-password pass --flows work,home",
+            env!("CARGO_PKG_NAME")
+        );
+        eprintln!(
+            "  Client mode: {} --host https://khm.example.com --flow work",
+            env!("CARGO_PKG_NAME")
+        );
         eprintln!();
         eprintln!("Use --help for more information.");
         std::process::exit(1);

src/server.rs

@@ -2,16 +2,16 @@ use actix_web::{web, App, HttpRequest, HttpResponse, HttpServer, Responder};
 use log::{error, info};
 use regex::Regex;
 use serde::{Deserialize, Serialize};
-use std::collections::HashMap;
 use std::sync::{Arc, Mutex};
-use tokio_postgres::{Client, NoTls};
 
-use crate::db;
+use crate::db::ReconnectingDbClient;
 
 #[derive(Serialize, Deserialize, Clone, Debug)]
 pub struct SshKey {
     pub server: String,
     pub public_key: String,
+    #[serde(default)]
+    pub deprecated: bool,
 }
 
 #[derive(Serialize, Deserialize, Clone, Debug)]
@@ -35,41 +35,6 @@ pub fn is_valid_ssh_key(key: &str) -> bool {
         || ed25519_re.is_match(key)
 }
 
-pub async fn get_keys_from_db(client: &Client) -> Result<Vec<Flow>, tokio_postgres::Error> {
-    let rows = client.query(
-        "SELECT k.host, k.key, f.name FROM public.keys k INNER JOIN public.flows f ON k.key_id = f.key_id",
-        &[]
-    ).await?;
-
-    let mut flows_map: HashMap<String, Flow> = HashMap::new();
-
-    for row in rows {
-        let host: String = row.get(0);
-        let key: String = row.get(1);
-        let flow: String = row.get(2);
-
-        let ssh_key = SshKey {
-            server: host,
-            public_key: key,
-        };
-
-        if let Some(flow_entry) = flows_map.get_mut(&flow) {
-            flow_entry.servers.push(ssh_key);
-        } else {
-            flows_map.insert(
-                flow.clone(),
-                Flow {
-                    name: flow,
-                    servers: vec![ssh_key],
-                },
-            );
-        }
-    }
-
-    info!("Retrieved {} flows from database", flows_map.len());
-    Ok(flows_map.into_values().collect())
-}
-
 // Extract client hostname from request headers
 fn get_client_hostname(req: &HttpRequest) -> String {
     if let Some(hostname) = req.headers().get("X-Client-Hostname") {
@@ -85,6 +50,7 @@ pub async fn get_keys(
     flow_id: web::Path<String>,
     allowed_flows: web::Data<Vec<String>>,
     req: HttpRequest,
+    query: web::Query<std::collections::HashMap<String, String>>,
 ) -> impl Responder {
     let client_hostname = get_client_hostname(&req);
     let flow_id_str = flow_id.into_inner();
@@ -104,10 +70,25 @@ pub async fn get_keys(
 
     let flows = flows.lock().unwrap();
     if let Some(flow) = flows.iter().find(|flow| flow.name == flow_id_str) {
-        let servers: Vec<&SshKey> = flow.servers.iter().collect();
+        // Check if we should include deprecated keys (default: false for CLI clients)
+        let include_deprecated = query
+            .get("include_deprecated")
+            .map(|v| v == "true")
+            .unwrap_or(false);
+
+        let servers: Vec<&SshKey> = if include_deprecated {
+            // Return all keys (for web interface)
+            flow.servers.iter().collect()
+        } else {
+            // Return only active keys (for CLI clients)
+            flow.servers.iter().filter(|key| !key.deprecated).collect()
+        };
+
         info!(
-            "Returning {} keys for flow '{}' to client '{}'",
+            "Returning {} keys ({} total, deprecated filtered: {}) for flow '{}' to client '{}'",
             servers.len(),
+            flow.servers.len(),
+            !include_deprecated,
             flow_id_str,
             client_hostname
         );
@@ -125,7 +106,7 @@ pub async fn add_keys(
     flows: web::Data<Flows>,
     flow_id: web::Path<String>,
     new_keys: web::Json<Vec<SshKey>>,
-    db_client: web::Data<Arc<Client>>,
+    db_client: web::Data<Arc<ReconnectingDbClient>>,
     allowed_flows: web::Data<Vec<String>>,
     req: HttpRequest,
 ) -> impl Responder {
@@ -171,7 +152,10 @@ pub async fn add_keys(
     );
 
     // Batch insert keys with statistics
-    let key_stats = match crate::db::batch_insert_keys(&db_client, &valid_keys).await {
+    let key_stats = match db_client
+        .batch_insert_keys_reconnecting(valid_keys.clone())
+        .await
+    {
         Ok(stats) => stats,
         Err(e) => {
             error!(
@@ -189,7 +173,9 @@ pub async fn add_keys(
         let key_ids: Vec<i32> = key_stats.key_id_map.iter().map(|(_, id)| *id).collect();
 
         // Batch insert key-flow associations
-        if let Err(e) = crate::db::batch_insert_flow_keys(&db_client, &flow_id_str, &key_ids).await
+        if let Err(e) = db_client
+            .batch_insert_flow_keys_reconnecting(flow_id_str.clone(), key_ids.clone())
+            .await
         {
             error!(
                 "Failed to batch insert flow keys from client '{}' into database: {}",
@@ -213,7 +199,7 @@ pub async fn add_keys(
     }
 
     // Get updated data
-    let updated_flows = match get_keys_from_db(&db_client).await {
+    let updated_flows = match db_client.get_keys_from_db_reconnecting().await {
         Ok(flows) => flows,
         Err(e) => {
             error!(
@@ -268,28 +254,22 @@ pub async fn run_server(args: crate::Args) -> std::io::Result<()> {
         args.db_host, db_user, db_password, args.db_name
     );
 
-    info!("Connecting to database at {}", args.db_host);
+    info!("Creating database client for {}", args.db_host);
-    let (db_client, connection) = match tokio_postgres::connect(&db_conn_str, NoTls).await {
+    let mut db_client_temp = ReconnectingDbClient::new(db_conn_str.clone());
-        Ok((client, conn)) => (client, conn),
+
-        Err(e) => {
+    // Initial connection
+    if let Err(e) = db_client_temp.connect(&db_conn_str).await {
         error!("Failed to connect to the database: {}", e);
         return Err(std::io::Error::new(
             std::io::ErrorKind::ConnectionRefused,
             format!("Database connection error: {}", e),
         ));
     }
-    };
-    let db_client = Arc::new(db_client);
 
-    // Spawn a new thread to run the database connection
+    let db_client = Arc::new(db_client_temp);
-    tokio::spawn(async move {
-        if let Err(e) = connection.await {
-            error!("Connection error: {}", e);
-        }
-    });
 
     // Initialize database schema if needed
-    if let Err(e) = db::initialize_db_schema(&db_client).await {
+    if let Err(e) = db_client.initialize_schema().await {
         error!("Failed to initialize database schema: {}", e);
         return Err(std::io::Error::new(
             std::io::ErrorKind::Other,
@@ -297,7 +277,7 @@ pub async fn run_server(args: crate::Args) -> std::io::Result<()> {
         ));
     }
 
-    let mut initial_flows = match get_keys_from_db(&db_client).await {
+    let mut initial_flows = match db_client.get_keys_from_db_reconnecting().await {
         Ok(flows) => flows,
         Err(e) => {
             error!("Failed to get initial flows from database: {}", e);
@@ -326,13 +306,27 @@ pub async fn run_server(args: crate::Args) -> std::io::Result<()> {
             .app_data(allowed_flows.clone())
             // API routes
             .route("/api/flows", web::get().to(crate::web::get_flows_api))
-            .route("/{flow_id}/keys/{server}", web::delete().to(crate::web::delete_key_by_server))
+            .route(
+                "/{flow_id}/keys/{server}",
+                web::delete().to(crate::web::delete_key_by_server),
+            )
+            .route(
+                "/{flow_id}/keys/{server}/restore",
+                web::post().to(crate::web::restore_key_by_server),
+            )
+            .route(
+                "/{flow_id}/keys/{server}/delete",
+                web::delete().to(crate::web::permanently_delete_key_by_server),
+            )
             // Original API routes
             .route("/{flow_id}/keys", web::get().to(get_keys))
             .route("/{flow_id}/keys", web::post().to(add_keys))
             // Web interface routes
             .route("/", web::get().to(crate::web::serve_web_interface))
-            .route("/static/{filename:.*}", web::get().to(crate::web::serve_static_file))
+            .route(
+                "/static/{filename:.*}",
+                web::get().to(crate::web::serve_static_file),
+            )
     })
     .bind((args.ip.as_str(), args.port))?
     .run()

src/web.rs

@@ -1,39 +1,35 @@
 use actix_web::{web, HttpResponse, Result};
 use log::info;
 use rust_embed::RustEmbed;
-use serde::{Deserialize, Serialize};
 use serde_json::json;
-use std::collections::HashSet;
+use std::sync::Arc;
-use tokio_postgres::Client;
 
-use crate::server::{get_keys_from_db, Flows};
+use crate::db::ReconnectingDbClient;
+use crate::server::Flows;
 
 #[derive(RustEmbed)]
 #[folder = "static/"]
 struct StaticAssets;
 
-#[derive(Deserialize)]
-struct DeleteKeyPath {
-    flow_id: String,
-    server: String,
-}
-
 // API endpoint to get list of available flows
 pub async fn get_flows_api(allowed_flows: web::Data<Vec<String>>) -> Result<HttpResponse> {
     info!("API request for available flows");
     Ok(HttpResponse::Ok().json(&**allowed_flows))
 }
 
-// API endpoint to delete a specific key by server name
+// API endpoint to deprecate a specific key by server name
 pub async fn delete_key_by_server(
     flows: web::Data<Flows>,
     path: web::Path<(String, String)>,
-    db_client: web::Data<std::sync::Arc<Client>>,
+    db_client: web::Data<Arc<ReconnectingDbClient>>,
     allowed_flows: web::Data<Vec<String>>,
 ) -> Result<HttpResponse> {
     let (flow_id_str, server_name) = path.into_inner();
 
-    info!("API request to delete key for server '{}' in flow '{}'", server_name, flow_id_str);
+    info!(
+        "API request to deprecate key for server '{}' in flow '{}'",
+        server_name, flow_id_str
+    );
 
     if !allowed_flows.contains(&flow_id_str) {
         return Ok(HttpResponse::Forbidden().json(json!({
@@ -41,14 +37,142 @@ pub async fn delete_key_by_server(
         })));
     }
 
-    // Delete from database
+    // Deprecate in database
-    match delete_key_from_db(&db_client, &server_name, &flow_id_str).await {
+    match db_client
-        Ok(deleted_count) => {
+        .deprecate_key_by_server_reconnecting(server_name.clone(), flow_id_str.clone())
-            if deleted_count > 0 {
+        .await
-                info!("Deleted {} key(s) for server '{}' in flow '{}'", deleted_count, server_name, flow_id_str);
+    {
+        Ok(deprecated_count) => {
+            if deprecated_count > 0 {
+                info!(
+                    "Deprecated {} key(s) for server '{}' in flow '{}'",
+                    deprecated_count, server_name, flow_id_str
+                );
 
                 // Refresh the in-memory flows
-                let updated_flows = match get_keys_from_db(&db_client).await {
+                let updated_flows = match db_client.get_keys_from_db_reconnecting().await {
+                    Ok(flows) => flows,
+                    Err(e) => {
+                        return Ok(HttpResponse::InternalServerError().json(json!({
+                            "error": format!("Failed to refresh flows: {}", e)
+                        })));
+                    }
+                };
+
+                let mut flows_guard = flows.lock().unwrap();
+                *flows_guard = updated_flows;
+
+                Ok(HttpResponse::Ok().json(json!({
+                    "message": format!("Successfully deprecated {} key(s) for server '{}'", deprecated_count, server_name),
+                    "deprecated_count": deprecated_count
+                })))
+            } else {
+                Ok(HttpResponse::NotFound().json(json!({
+                    "error": format!("No keys found for server '{}'", server_name)
+                })))
+            }
+        }
+        Err(e) => Ok(HttpResponse::InternalServerError().json(json!({
+            "error": format!("Failed to deprecate key: {}", e)
+        }))),
+    }
+}
+
+// API endpoint to restore a deprecated key
+pub async fn restore_key_by_server(
+    flows: web::Data<Flows>,
+    path: web::Path<(String, String)>,
+    db_client: web::Data<Arc<ReconnectingDbClient>>,
+    allowed_flows: web::Data<Vec<String>>,
+) -> Result<HttpResponse> {
+    let (flow_id_str, server_name) = path.into_inner();
+
+    info!(
+        "API request to restore key for server '{}' in flow '{}'",
+        server_name, flow_id_str
+    );
+
+    if !allowed_flows.contains(&flow_id_str) {
+        return Ok(HttpResponse::Forbidden().json(json!({
+            "error": "Flow ID not allowed"
+        })));
+    }
+
+    // Restore in database
+    match db_client
+        .restore_key_by_server_reconnecting(server_name.clone(), flow_id_str.clone())
+        .await
+    {
+        Ok(restored_count) => {
+            if restored_count > 0 {
+                info!(
+                    "Restored {} key(s) for server '{}' in flow '{}'",
+                    restored_count, server_name, flow_id_str
+                );
+
+                // Refresh the in-memory flows
+                let updated_flows = match db_client.get_keys_from_db_reconnecting().await {
+                    Ok(flows) => flows,
+                    Err(e) => {
+                        return Ok(HttpResponse::InternalServerError().json(json!({
+                            "error": format!("Failed to refresh flows: {}", e)
+                        })));
+                    }
+                };
+
+                let mut flows_guard = flows.lock().unwrap();
+                *flows_guard = updated_flows;
+
+                Ok(HttpResponse::Ok().json(json!({
+                    "message": format!("Successfully restored {} key(s) for server '{}'", restored_count, server_name),
+                    "restored_count": restored_count
+                })))
+            } else {
+                Ok(HttpResponse::NotFound().json(json!({
+                    "error": format!("No deprecated keys found for server '{}'", server_name)
+                })))
+            }
+        }
+        Err(e) => Ok(HttpResponse::InternalServerError().json(json!({
+            "error": format!("Failed to restore key: {}", e)
+        }))),
+    }
+}
+
+// API endpoint to permanently delete a key
+pub async fn permanently_delete_key_by_server(
+    flows: web::Data<Flows>,
+    path: web::Path<(String, String)>,
+    db_client: web::Data<Arc<ReconnectingDbClient>>,
+    allowed_flows: web::Data<Vec<String>>,
+) -> Result<HttpResponse> {
+    let (flow_id_str, server_name) = path.into_inner();
+
+    info!(
+        "API request to permanently delete key for server '{}' in flow '{}'",
+        server_name, flow_id_str
+    );
+
+    if !allowed_flows.contains(&flow_id_str) {
+        return Ok(HttpResponse::Forbidden().json(json!({
+            "error": "Flow ID not allowed"
+        })));
+    }
+
+    // Permanently delete from database
+    match db_client
+        .permanently_delete_key_by_server_reconnecting(server_name.clone(), flow_id_str.clone())
+        .await
+    {
+        Ok(deleted_count) => {
+            if deleted_count > 0 {
+                info!(
+                    "Permanently deleted {} key(s) for server '{}' in flow '{}'",
+                    deleted_count, server_name, flow_id_str
+                );
+
+                // Refresh the in-memory flows
+                let updated_flows = match db_client.get_keys_from_db_reconnecting().await {
                     Ok(flows) => flows,
                     Err(e) => {
                         return Ok(HttpResponse::InternalServerError().json(json!({
@@ -70,74 +194,10 @@ pub async fn delete_key_by_server(
                 })))
             }
         }
-        Err(e) => {
+        Err(e) => Ok(HttpResponse::InternalServerError().json(json!({
-            Ok(HttpResponse::InternalServerError().json(json!({
             "error": format!("Failed to delete key: {}", e)
-            })))
+        }))),
     }
-    }
-}
-
-// Helper function to delete a key from database
-async fn delete_key_from_db(
-    client: &Client,
-    server_name: &str,
-    flow_name: &str,
-) -> Result<u64, tokio_postgres::Error> {
-    // First, find the key_ids for the given server
-    let key_rows = client
-        .query("SELECT key_id FROM public.keys WHERE host = $1", &[&server_name])
-        .await?;
-
-    if key_rows.is_empty() {
-        return Ok(0);
-    }
-
-    let key_ids: Vec<i32> = key_rows.iter().map(|row| row.get::<_, i32>(0)).collect();
-
-    // Delete flow associations first
-    let mut flow_delete_count = 0;
-    for key_id in &key_ids {
-        let deleted = client
-            .execute(
-                "DELETE FROM public.flows WHERE name = $1 AND key_id = $2",
-                &[&flow_name, key_id],
-            )
-            .await?;
-        flow_delete_count += deleted;
-    }
-
-    // Check if any of these keys are used in other flows
-    let mut keys_to_delete = Vec::new();
-    for key_id in &key_ids {
-        let count: i64 = client
-            .query_one(
-                "SELECT COUNT(*) FROM public.flows WHERE key_id = $1",
-                &[key_id],
-            )
-            .await?
-            .get(0);
-
-        if count == 0 {
-            keys_to_delete.push(*key_id);
-        }
-    }
-
-    // Delete keys that are no longer referenced by any flow
-    let mut total_deleted = 0;
-    for key_id in keys_to_delete {
-        let deleted = client
-            .execute("DELETE FROM public.keys WHERE key_id = $1", &[&key_id])
-            .await?;
-        total_deleted += deleted;
-    }
-
-    info!(
-        "Deleted {} flow associations and {} orphaned keys for server '{}'",
-        flow_delete_count, total_deleted, server_name
-    );
-
-    Ok(std::cmp::max(flow_delete_count, total_deleted))
 }
 
 // Serve static files from embedded assets
@@ -163,22 +223,16 @@ pub async fn serve_static_file(path: web::Path<String>) -> Result<HttpResponse>
                 .content_type(content_type)
                 .body(content.data.as_ref().to_vec()))
         }
-        None => {
+        None => Ok(HttpResponse::NotFound().body(format!("File not found: {}", file_path))),
-            Ok(HttpResponse::NotFound().body(format!("File not found: {}", file_path)))
-        }
     }
 }
 
 // Serve the main web interface from embedded assets
 pub async fn serve_web_interface() -> Result<HttpResponse> {
     match StaticAssets::get("index.html") {
-        Some(content) => {
+        Some(content) => Ok(HttpResponse::Ok()
-            Ok(HttpResponse::Ok()
             .content_type("text/html; charset=utf-8")
-                .body(content.data.as_ref().to_vec()))
+            .body(content.data.as_ref().to_vec())),
-        }
+        None => Ok(HttpResponse::NotFound().body("Web interface not found")),
-        None => {
-            Ok(HttpResponse::NotFound().body("Web interface not found"))
-        }
     }
 }

static/index.html

@@ -34,7 +34,16 @@
 
             <div class="actions-panel">
                 <button id="addKeyBtn" class="btn btn-primary">Add SSH Key</button>
-                <button id="bulkDeleteBtn" class="btn btn-danger" disabled>Delete Selected</button>
+                <button id="bulkDeleteBtn" class="btn btn-danger" disabled>Deprecate Selected</button>
+                <button id="bulkPermanentDeleteBtn" class="btn btn-danger" disabled style="display: none;">Delete Selected</button>
+                
+                <div class="filter-controls">
+                    <label class="filter-label">
+                        <input type="checkbox" id="showDeprecatedOnly"> 
+                        <span>Show only deprecated keys</span>
+                    </label>
+                </div>
+                
                 <div class="search-box">
                     <input type="text" id="searchInput" placeholder="Search servers or keys...">
                 </div>
@@ -47,9 +56,9 @@
                             <th>
                                 <input type="checkbox" id="selectAll">
                             </th>
-                            <th>Server</th>
+                            <th>Server/Type</th>
-                            <th>Key Type</th>
                             <th>Key Preview</th>
+                            <th></th>
                             <th>Actions</th>
                         </tr>
                     </thead>

static/script.js

@@ -3,9 +3,13 @@ class SSHKeyManager {
         this.currentFlow = null;
         this.keys = [];
         this.filteredKeys = [];
+        this.groupedKeys = {};
+        this.expandedGroups = new Set();
         this.currentPage = 1;
         this.keysPerPage = 20;
+        this.serversPerPage = 10;
         this.selectedKeys = new Set();
+        this.showDeprecatedOnly = false;
         
         this.initializeEventListeners();
         this.loadFlows();
@@ -40,11 +44,31 @@ class SSHKeyManager {
             this.deleteSelectedKeys();
         });
 
+        // Bulk permanent delete button
+        document.getElementById('bulkPermanentDeleteBtn').addEventListener('click', () => {
+            this.permanentlyDeleteSelectedKeys();
+        });
+
         // Search input
         document.getElementById('searchInput').addEventListener('input', (e) => {
             this.filterKeys(e.target.value);
         });
 
+        // Deprecated filter checkbox
+        document.getElementById('showDeprecatedOnly').addEventListener('change', (e) => {
+            this.showDeprecatedOnly = e.target.checked;
+            
+            // Update visual state
+            const filterLabel = e.target.closest('.filter-label');
+            if (e.target.checked) {
+                filterLabel.classList.add('active');
+            } else {
+                filterLabel.classList.remove('active');
+            }
+            
+            this.filterKeys(document.getElementById('searchInput').value);
+        });
+
         // Select all checkbox
         document.getElementById('selectAll').addEventListener('change', (e) => {
             this.toggleSelectAll(e.target.checked);
@@ -127,6 +151,13 @@ class SSHKeyManager {
             option.textContent = flow;
             select.appendChild(option);
         });
+        
+        // Auto-select the first flow if available
+        if (flows.length > 0) {
+            select.value = flows[0];
+            this.currentFlow = flows[0];
+            this.loadKeys();
+        }
     }
 
     async loadKeys() {
@@ -134,11 +165,12 @@ class SSHKeyManager {
 
         try {
             this.showLoading();
-            const response = await fetch(`/${this.currentFlow}/keys`);
+            const response = await fetch(`/${this.currentFlow}/keys?include_deprecated=true`);
             if (!response.ok) throw new Error('Failed to load keys');
             
             this.keys = await response.json();
-            this.filteredKeys = [...this.keys];
+            this.groupKeys();
+            this.filterKeys();
             this.updateStats();
             this.renderTable();
             this.selectedKeys.clear();
@@ -151,16 +183,37 @@ class SSHKeyManager {
         }
     }
 
+    groupKeys() {
+        this.groupedKeys = {};
+        this.keys.forEach(key => {
+            if (!this.groupedKeys[key.server]) {
+                this.groupedKeys[key.server] = [];
+            }
+            this.groupedKeys[key.server].push(key);
+        });
+        
+        // Groups are closed by default - no auto-expand
+    }
+
     filterKeys(searchTerm) {
-        if (!searchTerm.trim()) {
+        let keys = [...this.keys];
-            this.filteredKeys = [...this.keys];
+        
+        // Apply deprecated filter first
+        if (this.showDeprecatedOnly) {
+            keys = keys.filter(key => key.deprecated);
+        }
+        
+        // Then apply search filter
+        if (!searchTerm || !searchTerm.trim()) {
+            this.filteredKeys = keys;
         } else {
             const term = searchTerm.toLowerCase();
-            this.filteredKeys = this.keys.filter(key => 
+            this.filteredKeys = keys.filter(key => 
                 key.server.toLowerCase().includes(term) || 
                 key.public_key.toLowerCase().includes(term)
             );
         }
+        
         this.currentPage = 1;
         this.renderTable();
     }
@@ -172,6 +225,17 @@ class SSHKeyManager {
         document.getElementById('uniqueServers').textContent = uniqueServers.size;
     }
 
+    getGroupedFilteredKeys() {
+        const groupedFilteredKeys = {};
+        this.filteredKeys.forEach(key => {
+            if (!groupedFilteredKeys[key.server]) {
+                groupedFilteredKeys[key.server] = [];
+            }
+            groupedFilteredKeys[key.server].push(key);
+        });
+        return groupedFilteredKeys;
+    }
+
     renderTable() {
         const tbody = document.getElementById('keysTableBody');
         const noKeysMessage = document.getElementById('noKeysMessage');
@@ -185,30 +249,78 @@ class SSHKeyManager {
 
         noKeysMessage.style.display = 'none';
         
-        const startIndex = (this.currentPage - 1) * this.keysPerPage;
+        // Group filtered keys by server
-        const endIndex = startIndex + this.keysPerPage;
+        const groupedFilteredKeys = this.getGroupedFilteredKeys();
-        const pageKeys = this.filteredKeys.slice(startIndex, endIndex);
         
-        tbody.innerHTML = pageKeys.map((key, index) => {
+        // Calculate pagination for grouped view
+        const servers = Object.keys(groupedFilteredKeys).sort();
+        
+        // For pagination, we'll show a reasonable number of server groups per page
+        const startServerIndex = (this.currentPage - 1) * this.serversPerPage;
+        const endServerIndex = startServerIndex + this.serversPerPage;
+        const pageServers = servers.slice(startServerIndex, endServerIndex);
+        
+        let html = '';
+        
+        pageServers.forEach(server => {
+            const serverKeys = groupedFilteredKeys[server];
+            const activeCount = serverKeys.filter(k => !k.deprecated).length;
+            const deprecatedCount = serverKeys.filter(k => k.deprecated).length;
+            const isExpanded = this.expandedGroups.has(server);
+            
+            // Server group header
+            html += `
+                <tr class="host-group-header ${isExpanded ? '' : 'collapsed'}">
+                    <td>
+                        <input type="checkbox" 
+                               data-group="${this.escapeHtml(server)}" 
+                               onchange="sshKeyManager.toggleGroupSelection('${this.escapeHtml(server)}', this.checked)"
+                               onclick="event.stopPropagation()">
+                    </td>
+                    <td colspan="4" onclick="sshKeyManager.toggleGroup('${this.escapeHtml(server)}')" style="cursor: pointer;">
+                        <span class="expand-icon">${isExpanded ? '▼' : '▶'}</span>
+                        <strong>${this.escapeHtml(server)}</strong>
+                        <span class="host-summary">
+                            <span class="key-count">${serverKeys.length} keys</span>
+                            ${deprecatedCount > 0 ? `<span class="deprecated-count">${deprecatedCount} deprecated</span>` : ''}
+                        </span>
+                    </td>
+                </tr>
+            `;
+            
+            // Server keys (if expanded)
+            if (isExpanded) {
+                serverKeys.forEach(key => {
                     const keyType = this.getKeyType(key.public_key);
                     const keyPreview = this.getKeyPreview(key.public_key);
                     const keyId = `${key.server}-${key.public_key}`;
                     
-            return `
+                    html += `
-                <tr>
+                        <tr class="key-row${key.deprecated ? ' deprecated' : ''}">
                             <td>
                                 <input type="checkbox" data-key-id="${keyId}" ${this.selectedKeys.has(keyId) ? 'checked' : ''}>
                             </td>
-                    <td>${this.escapeHtml(key.server)}</td>
+                            <td style="padding-left: 2rem;">
-                    <td><span class="key-type ${keyType.toLowerCase()}">${keyType}</span></td>
+                                <span class="key-type ${keyType.toLowerCase()}">${keyType}</span>
+                                ${key.deprecated ? '<span class="deprecated-badge">DEPRECATED</span>' : ''}
+                            </td>
                             <td><span class="key-preview">${keyPreview}</span></td>
+                            <td></td>
                             <td class="table-actions">
                                 <button class="btn btn-sm btn-secondary" onclick="sshKeyManager.viewKey('${keyId}')">View</button>
-                        <button class="btn btn-sm btn-danger" onclick="sshKeyManager.deleteKey('${keyId}')">Delete</button>
+                                ${key.deprecated ? 
+                                    `<button class="btn btn-sm btn-success" onclick="sshKeyManager.restoreKey('${keyId}')">Restore</button>
+                                     <button class="btn btn-sm btn-danger" onclick="sshKeyManager.permanentlyDeleteKey('${keyId}')">Delete</button>` : 
+                                    `<button class="btn btn-sm btn-danger" onclick="sshKeyManager.deleteKey('${keyId}')">Deprecate</button>`
+                                }
                             </td>
                         </tr>
                     `;
-        }).join('');
+                });
+            }
+        });
+        
+        tbody.innerHTML = html;
 
         // Add event listeners for checkboxes
         tbody.querySelectorAll('input[type="checkbox"]').forEach(checkbox => {
@@ -221,14 +333,78 @@ class SSHKeyManager {
                 }
                 this.updateBulkDeleteButton();
                 this.updateSelectAllCheckbox();
+                this.updateGroupCheckboxes(); // Update group checkboxes when individual keys change
             });
         });
 
+        // Update group checkboxes to show correct indeterminate state
+        this.updateGroupCheckboxes();
         this.updatePagination();
     }
 
+    toggleGroup(server) {
+        if (this.expandedGroups.has(server)) {
+            this.expandedGroups.delete(server);
+        } else {
+            this.expandedGroups.add(server);
+        }
+        this.renderTable();
+    }
+
+    toggleGroupSelection(server, isChecked) {
+        const groupedFilteredKeys = this.getGroupedFilteredKeys();
+        const serverKeys = groupedFilteredKeys[server] || [];
+        
+        serverKeys.forEach(key => {
+            const keyId = `${key.server}-${key.public_key}`;
+            if (isChecked) {
+                this.selectedKeys.add(keyId);
+            } else {
+                this.selectedKeys.delete(keyId);
+            }
+        });
+        
+        this.updateBulkDeleteButton();
+        this.updateSelectAllCheckbox();
+        this.updateGroupCheckboxes();
+        
+        // Update individual checkboxes without full re-render
+        const tbody = document.getElementById('keysTableBody');
+        serverKeys.forEach(key => {
+            const keyId = `${key.server}-${key.public_key}`;
+            const checkbox = tbody.querySelector(`input[data-key-id="${keyId}"]`);
+            if (checkbox) {
+                checkbox.checked = this.selectedKeys.has(keyId);
+            }
+        });
+    }
+
+    updateGroupCheckboxes() {
+        const groupedFilteredKeys = this.getGroupedFilteredKeys();
+        const tbody = document.getElementById('keysTableBody');
+        
+        Object.keys(groupedFilteredKeys).forEach(server => {
+            const serverKeys = groupedFilteredKeys[server];
+            const groupCheckbox = tbody.querySelector(`input[data-group="${server}"]`);
+            
+            if (groupCheckbox) {
+                const allSelected = serverKeys.every(key => 
+                    this.selectedKeys.has(`${key.server}-${key.public_key}`)
+                );
+                const someSelected = serverKeys.some(key => 
+                    this.selectedKeys.has(`${key.server}-${key.public_key}`)
+                );
+                
+                groupCheckbox.checked = allSelected;
+                groupCheckbox.indeterminate = someSelected && !allSelected;
+            }
+        });
+    }
+
     updatePagination() {
-        const totalPages = Math.ceil(this.filteredKeys.length / this.keysPerPage);
+        const groupedFilteredKeys = this.getGroupedFilteredKeys();
+        const totalServers = Object.keys(groupedFilteredKeys).length;
+        const totalPages = Math.ceil(totalServers / this.serversPerPage);
         
         document.getElementById('pageInfo').textContent = `Page ${this.currentPage} of ${totalPages}`;
         document.getElementById('prevPage').disabled = this.currentPage <= 1;
@@ -236,7 +412,10 @@ class SSHKeyManager {
     }
 
     changePage(newPage) {
-        const totalPages = Math.ceil(this.filteredKeys.length / this.keysPerPage);
+        const groupedFilteredKeys = this.getGroupedFilteredKeys();
+        const totalServers = Object.keys(groupedFilteredKeys).length;
+        const totalPages = Math.ceil(totalServers / this.serversPerPage);
+        
         if (newPage >= 1 && newPage <= totalPages) {
             this.currentPage = newPage;
             this.renderTable();
@@ -277,10 +456,50 @@ class SSHKeyManager {
 
     updateBulkDeleteButton() {
         const bulkDeleteBtn = document.getElementById('bulkDeleteBtn');
-        bulkDeleteBtn.disabled = this.selectedKeys.size === 0;
+        const bulkPermanentDeleteBtn = document.getElementById('bulkPermanentDeleteBtn');
-        bulkDeleteBtn.textContent = this.selectedKeys.size > 0 
+        
-            ? `Delete Selected (${this.selectedKeys.size})` 
+        if (this.selectedKeys.size === 0) {
-            : 'Delete Selected';
+            // No keys selected - hide both buttons
+            bulkDeleteBtn.disabled = true;
+            bulkDeleteBtn.textContent = 'Deprecate Selected';
+            bulkPermanentDeleteBtn.style.display = 'none';
+            bulkPermanentDeleteBtn.disabled = true;
+            return;
+        }
+
+        // Count selected active and deprecated keys
+        let activeCount = 0;
+        let deprecatedCount = 0;
+        
+        Array.from(this.selectedKeys).forEach(keyId => {
+            const key = this.findKeyById(keyId);
+            if (key) {
+                if (key.deprecated) {
+                    deprecatedCount++;
+                } else {
+                    activeCount++;
+                }
+            }
+        });
+
+        // Show/hide deprecate button
+        if (activeCount > 0) {
+            bulkDeleteBtn.disabled = false;
+            bulkDeleteBtn.textContent = `Deprecate Selected (${activeCount})`;
+        } else {
+            bulkDeleteBtn.disabled = true;
+            bulkDeleteBtn.textContent = 'Deprecate Selected';
+        }
+
+        // Show/hide permanent delete button
+        if (deprecatedCount > 0) {
+            bulkPermanentDeleteBtn.style.display = 'inline-flex';
+            bulkPermanentDeleteBtn.disabled = false;
+            bulkPermanentDeleteBtn.textContent = `Delete Selected (${deprecatedCount})`;
+        } else {
+            bulkPermanentDeleteBtn.style.display = 'none';
+            bulkPermanentDeleteBtn.disabled = true;
+        }
     }
 
     showAddKeyModal() {
@@ -346,12 +565,17 @@ class SSHKeyManager {
     }
 
     async deleteKey(keyId) {
-        if (!confirm('Are you sure you want to delete this SSH key?')) {
+        const key = this.findKeyById(keyId);
+        if (!key) return;
+        
+        if (key.deprecated) {
+            this.showToast('This key is already deprecated', 'warning');
             return;
         }
 
-        const key = this.findKeyById(keyId);
+        if (!confirm('Are you sure you want to deprecate this SSH key?')) {
-        if (!key) return;
+            return;
+        }
 
         try {
             this.showLoading();
@@ -361,13 +585,79 @@ class SSHKeyManager {
 
             if (!response.ok) {
                 const errorText = await response.text();
-                throw new Error(errorText || 'Failed to delete key');
+                throw new Error(errorText || 'Failed to deprecate key');
             }
 
-            this.showToast('SSH key deleted successfully', 'success');
+            this.showToast('SSH key deprecated successfully', 'success');
             await this.loadKeys();
         } catch (error) {
-            this.showToast('Failed to delete key: ' + error.message, 'error');
+            this.showToast('Failed to deprecate key: ' + error.message, 'error');
+        } finally {
+            this.hideLoading();
+        }
+    }
+
+    async restoreKey(keyId) {
+        const key = this.findKeyById(keyId);
+        if (!key) return;
+        
+        if (!key.deprecated) {
+            this.showToast('This key is not deprecated', 'warning');
+            return;
+        }
+
+        if (!confirm('Are you sure you want to restore this SSH key from deprecated status?')) {
+            return;
+        }
+
+        try {
+            this.showLoading();
+            const response = await fetch(`/${this.currentFlow}/keys/${encodeURIComponent(key.server)}/restore`, {
+                method: 'POST'
+            });
+
+            if (!response.ok) {
+                const errorText = await response.text();
+                throw new Error(errorText || 'Failed to restore key');
+            }
+
+            this.showToast('SSH key restored successfully', 'success');
+            await this.loadKeys();
+        } catch (error) {
+            this.showToast('Failed to restore key: ' + error.message, 'error');
+        } finally {
+            this.hideLoading();
+        }
+    }
+
+    async permanentlyDeleteKey(keyId) {
+        const key = this.findKeyById(keyId);
+        if (!key) return;
+
+        if (!confirm('⚠️ Are you sure you want to PERMANENTLY DELETE this SSH key?\n\nThis action cannot be undone!')) {
+            return;
+        }
+
+        // Double confirmation for permanent deletion
+        if (!confirm('This will permanently remove the key from the database.\n\nConfirm permanent deletion?')) {
+            return;
+        }
+
+        try {
+            this.showLoading();
+            const response = await fetch(`/${this.currentFlow}/keys/${encodeURIComponent(key.server)}/delete`, {
+                method: 'DELETE'
+            });
+
+            if (!response.ok) {
+                const errorText = await response.text();
+                throw new Error(errorText || 'Failed to permanently delete key');
+            }
+
+            this.showToast('SSH key permanently deleted', 'success');
+            await this.loadKeys();
+        } catch (error) {
+            this.showToast('Failed to permanently delete key: ' + error.message, 'error');
         } finally {
             this.hideLoading();
         }
@@ -376,14 +666,25 @@ class SSHKeyManager {
     async deleteSelectedKeys() {
         if (this.selectedKeys.size === 0) return;
 
-        if (!confirm(`Are you sure you want to delete ${this.selectedKeys.size} selected SSH keys?`)) {
+        // Filter out already deprecated keys
+        const activeKeys = Array.from(this.selectedKeys).filter(keyId => {
+            const key = this.findKeyById(keyId);
+            return key && !key.deprecated;
+        });
+
+        if (activeKeys.length === 0) {
+            this.showToast('All selected keys are already deprecated', 'warning');
+            return;
+        }
+
+        if (!confirm(`Are you sure you want to deprecate ${activeKeys.length} selected SSH keys?`)) {
             return;
         }
 
         try {
             this.showLoading();
             
-            const deletePromises = Array.from(this.selectedKeys).map(keyId => {
+            const deprecatePromises = activeKeys.map(keyId => {
                 const key = this.findKeyById(keyId);
                 if (!key) return Promise.resolve();
                 
@@ -392,11 +693,56 @@ class SSHKeyManager {
                 });
             });
 
-            await Promise.all(deletePromises);
+            await Promise.all(deprecatePromises);
-            this.showToast(`${this.selectedKeys.size} SSH keys deleted successfully`, 'success');
+            this.showToast(`${activeKeys.length} SSH keys deprecated successfully`, 'success');
             await this.loadKeys();
         } catch (error) {
-            this.showToast('Failed to delete selected keys: ' + error.message, 'error');
+            this.showToast('Failed to deprecate selected keys: ' + error.message, 'error');
+        } finally {
+            this.hideLoading();
+        }
+    }
+
+    async permanentlyDeleteSelectedKeys() {
+        if (this.selectedKeys.size === 0) return;
+
+        // Filter only deprecated keys
+        const deprecatedKeys = Array.from(this.selectedKeys).filter(keyId => {
+            const key = this.findKeyById(keyId);
+            return key && key.deprecated;
+        });
+
+        if (deprecatedKeys.length === 0) {
+            this.showToast('No deprecated keys selected', 'warning');
+            return;
+        }
+
+        if (!confirm(`⚠️ Are you sure you want to PERMANENTLY DELETE ${deprecatedKeys.length} deprecated SSH keys?\n\nThis action cannot be undone!`)) {
+            return;
+        }
+
+        // Double confirmation for permanent deletion
+        if (!confirm('This will permanently remove the keys from the database.\n\nConfirm permanent deletion?')) {
+            return;
+        }
+
+        try {
+            this.showLoading();
+            
+            const deletePromises = deprecatedKeys.map(keyId => {
+                const key = this.findKeyById(keyId);
+                if (!key) return Promise.resolve();
+                
+                return fetch(`/${this.currentFlow}/keys/${encodeURIComponent(key.server)}/delete`, {
+                    method: 'DELETE'
+                });
+            });
+
+            await Promise.all(deletePromises);
+            this.showToast(`${deprecatedKeys.length} SSH keys permanently deleted`, 'success');
+            await this.loadKeys();
+        } catch (error) {
+            this.showToast('Failed to permanently delete selected keys: ' + error.message, 'error');
         } finally {
             this.hideLoading();
         }

static/style.css

@@ -119,6 +119,15 @@ header h1 {
     background-color: var(--danger-hover);
 }
 
+.btn-success {
+    background-color: var(--success-color);
+    color: white;
+}
+
+.btn-success:hover:not(:disabled) {
+    background-color: #059669;
+}
+
 .btn-sm {
     padding: 0.25rem 0.5rem;
     font-size: 0.75rem;
@@ -161,6 +170,46 @@ header h1 {
     flex-wrap: wrap;
 }
 
+.filter-controls {
+    display: flex;
+    align-items: center;
+    gap: 1rem;
+}
+
+.filter-label {
+    display: flex;
+    align-items: center;
+    gap: 0.5rem;
+    font-size: 0.875rem;
+    color: var(--text-primary);
+    cursor: pointer;
+    user-select: none;
+    padding: 0.5rem 0.75rem;
+    border-radius: var(--border-radius);
+    transition: background-color 0.2s ease;
+}
+
+.filter-label:hover {
+    background-color: var(--background);
+}
+
+.filter-label.active {
+    background-color: var(--primary-color);
+    color: white;
+}
+
+.filter-label.active input[type="checkbox"] {
+    accent-color: white;
+}
+
+.filter-label input[type="checkbox"] {
+    margin: 0;
+}
+
+.filter-label span {
+    white-space: nowrap;
+}
+
 .search-box input {
     padding: 0.5rem 1rem;
     border: 1px solid var(--border);
@@ -201,6 +250,94 @@ header h1 {
     background-color: #f8fafc;
 }
 
+.keys-table tbody tr.deprecated {
+    opacity: 0.6;
+    background-color: #fef2f2;
+}
+
+.keys-table tbody tr.deprecated:hover {
+    background-color: #fee2e2;
+}
+
+.keys-table tbody tr.deprecated .key-preview,
+.keys-table tbody tr.deprecated td:nth-child(2) {
+    text-decoration: line-through;
+    color: var(--text-secondary);
+}
+
+.host-group-header {
+    background-color: #f1f5f9;
+    font-weight: 600;
+    transition: background-color 0.2s ease;
+    border-left: 4px solid var(--primary-color);
+}
+
+.host-group-header:hover {
+    background-color: #e2e8f0;
+}
+
+.host-group-header.collapsed {
+    border-left-color: var(--secondary-color);
+}
+
+.host-group-header .expand-icon {
+    transition: transform 0.2s ease;
+    display: inline-block;
+    margin-right: 0.5rem;
+    user-select: none;
+}
+
+.host-group-header.collapsed .expand-icon {
+    transform: rotate(-90deg);
+}
+
+.host-group-header input[type="checkbox"] {
+    margin: 0;
+}
+
+.host-group-header td:first-child {
+    width: 50px;
+    text-align: center;
+}
+
+.host-group-header td:nth-child(2) {
+    cursor: pointer;
+    user-select: none;
+}
+
+.key-row {
+    border-left: 4px solid transparent;
+}
+
+.key-row.hidden {
+    display: none;
+}
+
+.host-summary {
+    font-size: 0.875rem;
+    color: var(--text-secondary);
+}
+
+.key-count {
+    background-color: var(--primary-color);
+    color: white;
+    padding: 0.125rem 0.375rem;
+    border-radius: 0.25rem;
+    font-size: 0.75rem;
+    font-weight: 500;
+    margin-left: 0.5rem;
+}
+
+.deprecated-count {
+    background-color: var(--danger-color);
+    color: white;
+    padding: 0.125rem 0.375rem;
+    border-radius: 0.25rem;
+    font-size: 0.75rem;
+    font-weight: 500;
+    margin-left: 0.25rem;
+}
+
 .key-preview {
     font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', monospace;
     font-size: 0.875rem;
@@ -226,6 +363,17 @@ header h1 {
 .key-type.ecdsa { background-color: #e0e7ff; color: #3730a3; }
 .key-type.dsa { background-color: #fce7f3; color: #9d174d; }
 
+.deprecated-badge {
+    display: inline-block;
+    padding: 0.25rem 0.5rem;
+    background-color: #fecaca;
+    color: #991b1b;
+    border-radius: 0.25rem;
+    font-size: 0.75rem;
+    font-weight: 500;
+    margin-left: 0.5rem;
+}
+
 .no-keys-message {
     text-align: center;
     padding: 3rem;
@@ -445,6 +593,11 @@ header h1 {
     .actions-panel {
         flex-direction: column;
         align-items: stretch;
+        gap: 1rem;
+    }
+
+    .filter-controls {
+        justify-content: center;
     }
 
     .search-box input {
@@ -480,6 +633,13 @@ input[type="checkbox"] {
     accent-color: var(--primary-color);
 }
 
+/* Indeterminate checkbox styling */
+input[type="checkbox"]:indeterminate {
+    background-color: var(--primary-color);
+    background-image: linear-gradient(90deg, transparent 40%, white 40%, white 60%, transparent 60%);
+    border-color: var(--primary-color);
+}
+
 /* Action buttons in table */
 .table-actions {
     display: flex;