+105

crates/tranquil-store/src/blockstore/compaction.rs

View file Reviewed

··· 187 187 } 188 188 189 189 #[allow(clippy::too_many_arguments)] 190 190 + pub(super) fn repair_blocks_on_writer_thread<S: StorageIO>( 191 191 + manager: &DataFileManager<S>, 192 192 + index: &BlockIndex, 193 193 + blocks: &[(CidBytes, Vec<u8>)], 194 194 + current_epoch: CommitEpoch, 195 195 + file_ids: &FileIdAllocator, 196 196 + hint_positions: &super::group_commit::ShardHintPositions, 197 197 + epoch: &super::types::EpochCounter, 198 198 + ) -> Result<u64, CompactionError> { 199 199 + if blocks.is_empty() { 200 200 + return Ok(0); 201 201 + } 202 202 + 203 203 + let new_file_id = file_ids.allocate(); 204 204 + let new_handle = manager.open_for_append(new_file_id)?; 205 205 + let mut writer = DataFileWriter::new(manager.io(), new_handle.fd(), new_file_id)?; 206 206 + 207 207 + let hint_path = hint_file_path(manager.data_dir(), new_file_id); 208 208 + let hint_fd = manager.io().open(&hint_path, OpenOptions::read_write())?; 209 209 + let mut hint_writer = HintFileWriter::new(manager.io(), hint_fd); 210 210 + 211 211 + let mut relocations: Vec<(CidBytes, BlockLocation)> = Vec::with_capacity(blocks.len()); 212 212 + let write_result = blocks.iter().try_for_each(|(cid, data)| { 213 213 + let refcount = index.get(cid).map(|e| e.refcount.raw()).unwrap_or(1).max(1); 214 214 + let loc = writer.append_block(cid, data)?; 215 215 + hint_writer.append_relocate(cid, &loc, refcount)?; 216 216 + relocations.push((*cid, loc)); 217 217 + Ok::<_, CompactionError>(()) 218 218 + }); 219 219 + 220 220 + let record_count = u32::try_from(relocations.len()).unwrap_or(u32::MAX); 221 221 + let writer_position = writer.position(); 222 222 + let finalize = write_result 223 223 + .and_then(|()| writer.sync().map_err(CompactionError::from)) 224 224 + .and_then(|()| { 225 225 + hint_writer 226 226 + .append_commit_marker( 227 227 + current_epoch.raw(), 228 228 + record_count, 229 229 + new_file_id, 230 230 + writer_position, 231 231 + ) 232 232 + .map_err(CompactionError::from) 233 233 + }) 234 234 + .and_then(|()| hint_writer.sync().map_err(CompactionError::from)) 235 235 + .and_then(|()| { 236 236 + manager 237 237 + .io() 238 238 + .sync_dir(manager.data_dir()) 239 239 + .map_err(CompactionError::from) 240 240 + }) 241 241 + .and_then(|()| manager.io().barrier().map_err(CompactionError::from)); 242 242 + 243 243 + let final_hint_offset = hint_writer.position(); 244 244 + let _ = manager.io().close(hint_fd); 245 245 + 246 246 + if let Err(e) = 247 247 + finalize.and_then(|()| verify_repaired_blocks(manager, new_file_id, &relocations)) 248 248 + { 249 249 + manager.delete_data_file(new_file_id).ok(); 250 250 + manager 251 251 + .io() 252 252 + .delete(&hint_file_path(manager.data_dir(), new_file_id)) 253 253 + .ok(); 254 254 + return Err(e); 255 255 + } 256 256 + 257 257 + hint_positions.record_extra(new_file_id, final_hint_offset); 258 258 + index.apply_compaction(&relocations, &[]); 259 259 + index 260 260 + .write_checkpoint(epoch.current(), hint_positions) 261 261 + .map_err(CompactionError::Io)?; 262 262 + 263 263 + tracing::info!( 264 264 + dest = %new_file_id, 265 265 + repaired = relocations.len(), 266 266 + "structural repair complete" 267 267 + ); 268 268 + 269 269 + Ok(relocations.len() as u64) 270 270 + } 271 271 + 272 272 + fn verify_repaired_blocks<S: StorageIO>( 273 273 + manager: &DataFileManager<S>, 274 274 + file_id: DataFileId, 275 275 + relocations: &[(CidBytes, BlockLocation)], 276 276 + ) -> Result<(), CompactionError> { 277 277 + let handle = manager.open_for_read(file_id)?; 278 278 + let file_size = manager.io().file_size(handle.fd())?; 279 279 + relocations.iter().try_for_each(|(cid, loc)| { 280 280 + match super::data_file::decode_block_record( 281 281 + manager.io(), 282 282 + handle.fd(), 283 283 + loc.offset, 284 284 + file_size, 285 285 + ) { 286 286 + Ok(Some(ReadBlockRecord::Valid { cid_bytes, .. })) if cid_bytes == *cid => Ok(()), 287 287 + _ => Err(CompactionError::Io(io::Error::other( 288 288 + "repaired block failed read-back verification", 289 289 + ))), 290 290 + } 291 291 + }) 292 292 + } 293 293 + 294 294 + #[allow(clippy::too_many_arguments)] 190 295 fn stream_compact<S: StorageIO>( 191 296 manager: &DataFileManager<S>, 192 297 index: &BlockIndex,

+78

crates/tranquil-store/src/blockstore/group_commit.rs

View file Reviewed

··· 175 175 type ApplyResponse = tokio::sync::oneshot::Sender<Result<(), CommitError>>; 176 176 type CompactResponse = tokio::sync::oneshot::Sender<Result<CompactionResult, CompactionError>>; 177 177 type RepairResponse = tokio::sync::oneshot::Sender<Result<u64, CommitError>>; 178 178 + type RepairBlocksResponse = tokio::sync::oneshot::Sender<Result<u64, CompactionError>>; 179 179 + type RepairBlockSet = Vec<(CidBytes, Vec<u8>)>; 180 180 + type DeferredRepairBlock = (RepairBlockSet, RepairBlocksResponse); 178 181 type QuiesceResponse = tokio::sync::oneshot::Sender<BlockstoreSnapshot>; 179 182 type QuiesceResume = tokio::sync::oneshot::Receiver<()>; 180 183 ··· 196 199 RepairLeaked { 197 200 leaked_cids: Vec<(CidBytes, RefCount)>, 198 201 response: RepairResponse, 202 202 + }, 203 203 + RepairBlocks { 204 204 + blocks: RepairBlockSet, 205 205 + response: RepairBlocksResponse, 199 206 }, 200 207 Quiesce { 201 208 response: QuiesceResponse, ··· 668 675 leaked_cids: Vec<(CidBytes, RefCount)>, 669 676 response: RepairResponse, 670 677 }, 678 678 + RepairBlocks { 679 679 + blocks: RepairBlockSet, 680 680 + response: RepairBlocksResponse, 681 681 + }, 671 682 Quiesce { 672 683 response: QuiesceResponse, 673 684 resume: QuiesceResume, ··· 704 715 leaked_cids, 705 716 response, 706 717 }, 718 718 + CommitRequest::RepairBlocks { blocks, response } => { 719 719 + ClassifyResult::RepairBlocks { blocks, response } 720 720 + } 707 721 CommitRequest::Quiesce { response, resume } => ClassifyResult::Quiesce { response, resume }, 708 722 CommitRequest::Shutdown => ClassifyResult::Shutdown, 709 723 } ··· 720 734 shutdown: bool, 721 735 deferred_compacts: Vec<(DataFileId, u64, CompactResponse)>, 722 736 deferred_repairs: Vec<(Vec<(CidBytes, RefCount)>, RepairResponse)>, 737 737 + deferred_repair_blocks: Vec<DeferredRepairBlock>, 723 738 deferred_quiesces: Vec<(QuiesceResponse, QuiesceResume)>, 724 739 } 725 740 ··· 735 750 shutdown: true, 736 751 deferred_compacts: Vec::new(), 737 752 deferred_repairs: Vec::new(), 753 753 + deferred_repair_blocks: Vec::new(), 738 754 deferred_quiesces: Vec::new(), 739 755 }; 740 756 } ··· 748 764 shutdown: false, 749 765 deferred_compacts: vec![(file_id, grace_period_ms, response)], 750 766 deferred_repairs: Vec::new(), 767 767 + deferred_repair_blocks: Vec::new(), 751 768 deferred_quiesces: Vec::new(), 752 769 }; 753 770 } ··· 760 777 shutdown: false, 761 778 deferred_compacts: Vec::new(), 762 779 deferred_repairs: vec![(leaked_cids, response)], 780 780 + deferred_repair_blocks: Vec::new(), 781 781 + deferred_quiesces: Vec::new(), 782 782 + }; 783 783 + } 784 784 + ClassifyResult::RepairBlocks { blocks, response } => { 785 785 + return DrainResult { 786 786 + entries: Vec::new(), 787 787 + shutdown: false, 788 788 + deferred_compacts: Vec::new(), 789 789 + deferred_repairs: Vec::new(), 790 790 + deferred_repair_blocks: vec![(blocks, response)], 763 791 deferred_quiesces: Vec::new(), 764 792 }; 765 793 } ··· 769 797 shutdown: false, 770 798 deferred_compacts: Vec::new(), 771 799 deferred_repairs: Vec::new(), 800 800 + deferred_repair_blocks: Vec::new(), 772 801 deferred_quiesces: vec![(response, resume)], 773 802 }; 774 803 } ··· 781 810 shutdown: false, 782 811 deferred_compacts: Vec::new(), 783 812 deferred_repairs: Vec::new(), 813 813 + deferred_repair_blocks: Vec::new(), 784 814 deferred_quiesces: Vec::new(), 785 815 }; 786 816 ··· 801 831 response, 802 832 } => { 803 833 r.deferred_repairs.push((leaked_cids, response)); 834 834 + false 835 835 + } 836 836 + ClassifyResult::RepairBlocks { blocks, response } => { 837 837 + r.deferred_repair_blocks.push((blocks, response)); 804 838 false 805 839 } 806 840 ClassifyResult::Quiesce { response, resume } => { ··· 946 980 let _ = response.send(Ok(repaired)); 947 981 continue; 948 982 } 983 983 + Ok(CommitRequest::RepairBlocks { blocks, response }) => { 984 984 + let result = compaction::repair_blocks_on_writer_thread( 985 985 + manager, 986 986 + index, 987 987 + &blocks, 988 988 + epoch.current(), 989 989 + &ctx.file_ids, 990 990 + &ctx.hint_positions, 991 991 + epoch, 992 992 + ); 993 993 + let _ = response.send(result); 994 994 + continue; 995 995 + } 949 996 Ok(CommitRequest::Quiesce { response, resume }) => { 950 997 handle_quiesce(manager, state, epoch, response, resume); 951 998 continue; ··· 1018 1065 let _ = response.send(Ok(repaired)); 1019 1066 }); 1020 1067 1068 1068 + drain 1069 1069 + .deferred_repair_blocks 1070 1070 + .into_iter() 1071 1071 + .for_each(|(blocks, response)| { 1072 1072 + let result = compaction::repair_blocks_on_writer_thread( 1073 1073 + manager, 1074 1074 + index, 1075 1075 + &blocks, 1076 1076 + epoch.current(), 1077 1077 + &ctx.file_ids, 1078 1078 + &ctx.hint_positions, 1079 1079 + epoch, 1080 1080 + ); 1081 1081 + let _ = response.send(result); 1082 1082 + }); 1083 1083 + 1021 1084 maybe_checkpoint( 1022 1085 index, 1023 1086 epoch, ··· 1062 1125 let mut entries: Vec<BatchEntry> = Vec::new(); 1063 1126 let mut compacts: Vec<(DataFileId, u64, CompactResponse)> = Vec::new(); 1064 1127 let mut repairs: Vec<(Vec<(CidBytes, RefCount)>, RepairResponse)> = Vec::new(); 1128 1128 + let mut repair_blocks: Vec<DeferredRepairBlock> = Vec::new(); 1065 1129 1066 1130 std::iter::from_fn(|| receiver.try_recv().ok()).for_each(|req| match classify_request(req) { 1067 1131 ClassifyResult::Batch(entry) => entries.push(entry), ··· 1074 1138 leaked_cids, 1075 1139 response, 1076 1140 } => repairs.push((leaked_cids, response)), 1141 1141 + ClassifyResult::RepairBlocks { blocks, response } => repair_blocks.push((blocks, response)), 1077 1142 ClassifyResult::Shutdown | ClassifyResult::Quiesce { .. } => {} 1078 1143 }); 1079 1144 ··· 1109 1174 let repaired = 1110 1175 index.repair_leaked_refcounts(&leaked_cids, epoch.current(), ctx.clock.wall_millis()); 1111 1176 let _ = response.send(Ok(repaired)); 1177 1177 + }); 1178 1178 + 1179 1179 + repair_blocks.into_iter().for_each(|(blocks, response)| { 1180 1180 + let result = compaction::repair_blocks_on_writer_thread( 1181 1181 + manager, 1182 1182 + index, 1183 1183 + &blocks, 1184 1184 + epoch.current(), 1185 1185 + &ctx.file_ids, 1186 1186 + &ctx.hint_positions, 1187 1187 + epoch, 1188 1188 + ); 1189 1189 + let _ = response.send(result); 1112 1190 }); 1113 1191 1114 1192 shutdown_checkpoint(index, epoch, &ctx.hint_positions);

+113

crates/tranquil-store/src/blockstore/repair.rs

View file Reviewed

··· 1 1 + use std::sync::Arc; 2 2 + 3 3 + use cid::Cid; 4 4 + use jacquard_repo::error::RepoError; 5 5 + use jacquard_repo::mst::Mst; 6 6 + use jacquard_repo::storage::MemoryBlockStore; 7 7 + 8 8 + use crate::clock::Clock; 9 9 + use crate::io::StorageIO; 10 10 + 11 11 + use super::store::{TranquilBlockStore, cid_to_bytes}; 12 12 + use super::types::CidBytes; 13 13 + 14 14 + const REPAIR_FILE_BYTE_BUDGET: usize = 64 * 1024 * 1024; 15 15 + 16 16 + #[derive(Debug, Clone, Copy, PartialEq, Eq)] 17 17 + pub struct RepairOutcome { 18 18 + pub nodes_total: usize, 19 19 + pub nodes_repaired: u64, 20 20 + } 21 21 + 22 22 + fn rebuild_err(context: &str, e: impl std::fmt::Display) -> RepoError { 23 23 + RepoError::storage(std::io::Error::other(format!("{context}: {e}"))) 24 24 + } 25 25 + 26 26 + async fn rebuild_node_blocks( 27 27 + entries: Vec<(String, Cid)>, 28 28 + expected_root: Cid, 29 29 + ) -> Result<Vec<(CidBytes, Vec<u8>)>, RepoError> { 30 30 + let scratch = Arc::new(MemoryBlockStore::new()); 31 31 + let mut mst = Mst::new(scratch); 32 32 + for (key, cid) in &entries { 33 33 + mst.add_mut(key.as_str(), *cid) 34 34 + .await 35 35 + .map_err(|e| rebuild_err("mst rebuild add", e))?; 36 36 + } 37 37 + 38 38 + let (root, blocks) = mst 39 39 + .collect_blocks() 40 40 + .await 41 41 + .map_err(|e| rebuild_err("mst collect_blocks", e))?; 42 42 + 43 43 + if root != expected_root { 44 44 + return Err(RepoError::storage(std::io::Error::new( 45 45 + std::io::ErrorKind::InvalidData, 46 46 + format!( 47 47 + "rebuilt MST root {root} does not match expected root {expected_root}, refusing to repair" 48 48 + ), 49 49 + ))); 50 50 + } 51 51 + 52 52 + blocks 53 53 + .into_iter() 54 54 + .map(|(cid, bytes)| Ok((cid_to_bytes(&cid)?, bytes.to_vec()))) 55 55 + .collect() 56 56 + } 57 57 + 58 58 + fn batch_by_bytes(blocks: Vec<(CidBytes, Vec<u8>)>) -> Vec<Vec<(CidBytes, Vec<u8>)>> { 59 59 + blocks 60 60 + .into_iter() 61 61 + .fold( 62 62 + (Vec::<Vec<(CidBytes, Vec<u8>)>>::new(), 0usize), 63 63 + |(mut batches, current_bytes), item| { 64 64 + let item_len = item.1.len(); 65 65 + match batches.last_mut() { 66 66 + Some(last) if current_bytes + item_len <= REPAIR_FILE_BYTE_BUDGET => { 67 67 + last.push(item); 68 68 + (batches, current_bytes + item_len) 69 69 + } 70 70 + _ => { 71 71 + batches.push(vec![item]); 72 72 + (batches, item_len) 73 73 + } 74 74 + } 75 75 + }, 76 76 + ) 77 77 + .0 78 78 + } 79 79 + 80 80 + pub async fn rebuild_and_repair_mst<S, C>( 81 81 + store: &TranquilBlockStore<S, C>, 82 82 + entries: &[(String, Cid)], 83 83 + expected_root: Cid, 84 84 + ) -> Result<RepairOutcome, RepoError> 85 85 + where 86 86 + S: StorageIO + Send + Sync + 'static, 87 87 + C: Clock, 88 88 + { 89 89 + let store = store.clone(); 90 90 + let entries = entries.to_vec(); 91 91 + tokio::task::spawn_blocking(move || -> Result<RepairOutcome, RepoError> { 92 92 + let handle = tokio::runtime::Handle::current(); 93 93 + let node_blocks = handle.block_on(rebuild_node_blocks(entries, expected_root))?; 94 94 + let nodes_total = node_blocks.len(); 95 95 + 96 96 + let to_repair: Vec<(CidBytes, Vec<u8>)> = node_blocks 97 97 + .into_iter() 98 98 + .filter(|(cid, _)| !matches!(store.get_block_sync(cid), Ok(Some(_)))) 99 99 + .collect(); 100 100 + 101 101 + let nodes_repaired = batch_by_bytes(to_repair) 102 102 + .into_iter() 103 103 + .try_fold(0u64, |acc, batch| store.repair_blocks(batch).map(|n| acc + n)) 104 104 + .map_err(|e| rebuild_err("repair_blocks", e))?; 105 105 + 106 106 + Ok(RepairOutcome { 107 107 + nodes_total, 108 108 + nodes_repaired, 109 109 + }) 110 110 + }) 111 111 + .await 112 112 + .map_err(RepoError::task_failed)? 113 113 + }