|
| 1 | +//! Snapshot expiry is by reference: anything a retained row points at stays, and content follows. |
| 2 | +
|
| 3 | +use uuid::Uuid; |
| 4 | + |
| 5 | +use crate::snapshots::constants::{CONTENT_HASH_CHUNK, SNAPSHOT_RETENTION_BATCH}; |
| 6 | +use crate::snapshots::models::SnapshotRetention; |
| 7 | +use crate::store::errors::ControlError; |
| 8 | +use crate::store::main::{begin::begin, commit::commit, query::query}; |
| 9 | +use crate::store::models::{ControlStore, ControlTransaction, Parameter, TransactionMode}; |
| 10 | + |
| 11 | +/// Beyond the newest generations, any retained reference keeps a snapshot; superseded progress goes. |
| 12 | +const EXPIRED_SNAPSHOTS: &str = "SELECT s.snapshot_id FROM sqb_snapshots s \ |
| 13 | + WHERE (SELECT COUNT(*) FROM sqb_snapshots newer WHERE newer.location_id=s.location_id \ |
| 14 | + AND newer.target=s.target AND newer.generation>s.generation)>=$1 \ |
| 15 | + AND NOT EXISTS (SELECT 1 FROM sqb_current_snapshots c WHERE c.snapshot_id=s.snapshot_id) \ |
| 16 | + AND NOT EXISTS (SELECT 1 FROM sqb_runs r WHERE r.snapshot_id=s.snapshot_id) \ |
| 17 | + AND NOT EXISTS (SELECT 1 FROM sqb_jobs j WHERE j.snapshot_id=s.snapshot_id) \ |
| 18 | + AND NOT EXISTS (SELECT 1 FROM sqb_triggers t WHERE t.snapshot_id=s.snapshot_id \ |
| 19 | + OR t.removed_in_snapshot_id=s.snapshot_id) \ |
| 20 | + AND NOT EXISTS (SELECT 1 FROM sqb_trigger_state_events e WHERE e.snapshot_id=s.snapshot_id) \ |
| 21 | + AND NOT EXISTS (SELECT 1 FROM sqb_location_work w WHERE w.snapshot_id=s.snapshot_id) \ |
| 22 | + AND NOT EXISTS (SELECT 1 FROM sqb_trigger_progress p WHERE p.snapshot_id=s.snapshot_id \ |
| 23 | + AND NOT EXISTS (SELECT 1 FROM sqb_trigger_progress newer WHERE newer.trigger_id=p.trigger_id \ |
| 24 | + AND newer.storage_id>p.storage_id)) \ |
| 25 | + ORDER BY s.published_at, s.snapshot_id LIMIT $2"; |
| 26 | + |
| 27 | +/// Content-addressed bodies of the expiring snapshots, removed once nothing else references them. |
| 28 | +const CONTENT: [((&str, &str), &str, &str); 3] = [ |
| 29 | + ( |
| 30 | + ("sqb_snapshot_nodes", "node_hash"), |
| 31 | + "sqb_node_definitions", |
| 32 | + "", |
| 33 | + ), |
| 34 | + ( |
| 35 | + ("sqb_snapshot_sources", "text_hash"), |
| 36 | + "sqb_source_texts", |
| 37 | + " AND NOT EXISTS (SELECT 1 FROM sqb_warehouse_fingerprints f \ |
| 38 | + WHERE f.definition_text_hash=sqb_source_texts.text_hash)", |
| 39 | + ), |
| 40 | + ( |
| 41 | + ("sqb_snapshot_documents", "document_hash"), |
| 42 | + "sqb_documents", |
| 43 | + "", |
| 44 | + ), |
| 45 | +]; |
| 46 | + |
| 47 | +/// Batches commit independently; a repeated or interrupted pass finds only what is still expired. |
| 48 | +pub(crate) async fn prune_snapshots( |
| 49 | + store: &ControlStore, |
| 50 | + retention: SnapshotRetention, |
| 51 | +) -> Result<usize, ControlError> { |
| 52 | + let mut deleted = 0; |
| 53 | + loop { |
| 54 | + let transaction = begin(store, TransactionMode::Write).await?; |
| 55 | + let (transaction, rows) = query( |
| 56 | + transaction, |
| 57 | + EXPIRED_SNAPSHOTS, |
| 58 | + vec![ |
| 59 | + Parameter::Integer(retention.kept()), |
| 60 | + Parameter::Integer(SNAPSHOT_RETENTION_BATCH as i64), |
| 61 | + ], |
| 62 | + ) |
| 63 | + .await?; |
| 64 | + let mut expired: Vec<Uuid> = Vec::with_capacity(rows.len()); |
| 65 | + for row in &rows { |
| 66 | + expired.push( |
| 67 | + serde_json::from_value(row["snapshot_id"].clone()) |
| 68 | + .map_err(|error| ControlError::Schema(error.to_string()))?, |
| 69 | + ); |
| 70 | + } |
| 71 | + let transaction = delete_snapshots(transaction, &expired).await?; |
| 72 | + commit(transaction).await?; |
| 73 | + deleted += expired.len(); |
| 74 | + if expired.len() < SNAPSHOT_RETENTION_BATCH { |
| 75 | + return Ok(deleted); |
| 76 | + } |
| 77 | + } |
| 78 | +} |
| 79 | + |
| 80 | +async fn delete_snapshots( |
| 81 | + mut transaction: ControlTransaction, |
| 82 | + expired: &[Uuid], |
| 83 | +) -> Result<ControlTransaction, ControlError> { |
| 84 | + if expired.is_empty() { |
| 85 | + return Ok(transaction); |
| 86 | + } |
| 87 | + let placeholders = (1..=expired.len()) |
| 88 | + .map(|index| format!("${index}")) |
| 89 | + .collect::<Vec<_>>() |
| 90 | + .join(","); |
| 91 | + let ids: Vec<Parameter> = expired.iter().copied().map(Parameter::Id).collect(); |
| 92 | + let mut hashes: Vec<Vec<String>> = Vec::with_capacity(CONTENT.len()); |
| 93 | + for ((table, column), _, _) in CONTENT { |
| 94 | + let rows; |
| 95 | + (transaction, rows) = query( |
| 96 | + transaction, |
| 97 | + &format!("SELECT DISTINCT {column} FROM {table} WHERE snapshot_id IN ({placeholders})"), |
| 98 | + ids.clone(), |
| 99 | + ) |
| 100 | + .await?; |
| 101 | + hashes.push( |
| 102 | + rows.iter() |
| 103 | + .filter_map(|row| row[column].as_str().map(str::to_owned)) |
| 104 | + .collect(), |
| 105 | + ); |
| 106 | + } |
| 107 | + for table in [ |
| 108 | + "sqb_trigger_progress", |
| 109 | + "sqb_snapshot_sources", |
| 110 | + "sqb_snapshot_nodes", |
| 111 | + "sqb_snapshot_documents", |
| 112 | + "sqb_snapshot_changes", |
| 113 | + "sqb_snapshots", |
| 114 | + ] { |
| 115 | + (transaction, _) = query( |
| 116 | + transaction, |
| 117 | + &format!("DELETE FROM {table} WHERE snapshot_id IN ({placeholders})"), |
| 118 | + ids.clone(), |
| 119 | + ) |
| 120 | + .await?; |
| 121 | + } |
| 122 | + for (((table, column), content, elsewhere), hashes) in CONTENT.iter().zip(hashes) { |
| 123 | + for chunk in hashes.chunks(CONTENT_HASH_CHUNK) { |
| 124 | + let placeholders = (1..=chunk.len()) |
| 125 | + .map(|index| format!("${index}")) |
| 126 | + .collect::<Vec<_>>() |
| 127 | + .join(","); |
| 128 | + (transaction, _) = query( |
| 129 | + transaction, |
| 130 | + &format!( |
| 131 | + "DELETE FROM {content} WHERE {column} IN ({placeholders}) AND NOT EXISTS \ |
| 132 | + (SELECT 1 FROM {table} n WHERE n.{column}={content}.{column}){elsewhere}" |
| 133 | + ), |
| 134 | + chunk.iter().cloned().map(Parameter::Text).collect(), |
| 135 | + ) |
| 136 | + .await?; |
| 137 | + } |
| 138 | + } |
| 139 | + Ok(transaction) |
| 140 | +} |
0 commit comments