From d41c211505d1ef39810dd79b55b6aa8ccc243afd Mon Sep 17 00:00:00 2001 From: Metehan Yurtseven Date: Sat, 24 Oct 2026 04:27:23 +0200 Subject: [PATCH] perf: don't hash all-zero chunks Chunks that contain only zeros have a known length: the Rabin hash of a window of zeros is 0, so runs of zeros are cut at the minimum chunk size (the chunk size for the fixed-size chunker). Checking for zeros is far faster than SHA-256, so such chunks now get a precomputed id. restic does the same since 0.15.0 (restic/restic@c147422). Blob ids and the repository format are unchanged. Co-Authored-By: Claude Opus 5.5 --- crates/core/src/archiver/file_archiver.rs | 47 +++++++++++++++++++++-- crates/core/tests/integration/chunker.rs | 46 +++++++++++++++++++++- 2 files changed, 87 insertions(+), 6 deletions(-) diff --git a/crates/core/src/archiver/file_archiver.rs b/crates/core/src/archiver/file_archiver.rs index fb83389c3..83f8b1dbf 100644 --- a/crates/core/src/archiver/file_archiver.rs +++ b/crates/core/src/archiver/file_archiver.rs @@ -20,7 +20,7 @@ use crate::{ error::{ErrorKind, RusticError, RusticResult}, index::{ReadGlobalIndex, indexer::SharedIndexer}, progress::Progress, - repofile::configfile::ConfigFile, + repofile::configfile::{Chunker, ConfigFile}, }; /// The `FileArchiver` is responsible for archiving files. @@ -35,6 +35,8 @@ pub(crate) struct FileArchiver<'a, BE: DecryptWriteBackend, I: ReadGlobalIndex> index: &'a I, data_packer: Packer, config: ConfigFile, + /// Length and id of the chunks the chunker cuts from a run of zeros. + zero_chunk: (usize, DataId), } impl<'a, BE: DecryptWriteBackend, I: ReadGlobalIndex> FileArchiver<'a, BE, I> { @@ -66,10 +68,18 @@ impl<'a, BE: DecryptWriteBackend, I: ReadGlobalIndex> FileArchiver<'a, BE, I> { PackSizer::from_config(config, BlobType::Data, index.total_size(BlobType::Data)); let data_packer = Packer::new(be, BlobType::Data, indexer, pack_sizer)?; + // The Rabin hash of a window of zeros is 0, so runs of zeros are cut at the minimum size. + let zero_len = match config.chunker() { + Chunker::Rabin => config.chunk_min_size(), + Chunker::FixedSize => config.chunk_size(), + }; + let zero_chunk = (zero_len, DataId::from(hash(&vec![0; zero_len]))); + Ok(Self { index, data_packer, config: config.clone(), + zero_chunk, }) } @@ -148,14 +158,20 @@ impl<'a, BE: DecryptWriteBackend, I: ReadGlobalIndex> FileArchiver<'a, BE, I> { )? .map(|chunk| { let chunk = chunk?; - let id = hash(&chunk); + // Checking for zeros is much faster than hashing, which speeds up sparse files and + // disk images. + let id = if chunk.len() == self.zero_chunk.0 && is_zero(&chunk) { + self.zero_chunk.1 + } else { + DataId::from(hash(&chunk)) + }; let size = chunk.len() as u64; - if !self.index.has_data(&DataId::from(id)) { + if !self.index.has_data(&id) { self.data_packer.add(chunk.into(), BlobId::from(id))?; } p.inc(size); - Ok((DataId::from(id), size)) + Ok((id, size)) }) .collect::>()?; @@ -180,3 +196,26 @@ impl<'a, BE: DecryptWriteBackend, I: ReadGlobalIndex> FileArchiver<'a, BE, I> { self.data_packer.finalize() } } + +/// Checks if `data` contains only zeros. Unlike `iter().all()`, the inner loop is vectorized. +fn is_zero(data: &[u8]) -> bool { + data.chunks(4096) + .all(|block| block.iter().fold(0, |acc, &b| acc | b) == 0) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn is_zero_finds_any_nonzero_byte() { + let mut data = vec![0; 3 * 4096 + 7]; + assert!(is_zero(&data)); + assert!(is_zero(&[])); + for i in [0, 4095, 4096, data.len() - 1] { + data[i] = 1; + assert!(!is_zero(&data), "byte {i}"); + data[i] = 0; + } + } +} diff --git a/crates/core/tests/integration/chunker.rs b/crates/core/tests/integration/chunker.rs index 45fa9a17c..b33b834d1 100644 --- a/crates/core/tests/integration/chunker.rs +++ b/crates/core/tests/integration/chunker.rs @@ -1,12 +1,13 @@ -use std::{path::PathBuf, str::FromStr}; +use std::{collections::HashSet, fs, path::PathBuf, str::FromStr}; use anyhow::Result; use bytesize::ByteSize; use insta::{Settings, assert_ron_snapshot}; use rstest::rstest; +use tempfile::tempdir; use rustic_core::{ - BackupOptions, ConfigOptions, + BackupOptions, CheckOptions, ConfigOptions, PathList, repofile::{Chunker, SnapshotFile}, }; @@ -46,3 +47,44 @@ fn test_chunker_params( Ok(()) } + +#[rstest] +#[case(Chunker::Rabin)] +#[case(Chunker::FixedSize)] +fn test_backup_zeros(set_up_repo: Result, #[case] chunker: Chunker) -> Result<()> { + let mut repo = set_up_repo?.to_indexed_ids()?; + let config_opts = ConfigOptions::default().set_chunker(chunker); + _ = repo.apply_config(&config_opts)?; + + // Runs of zeros between data, so that zero chunks follow and precede other chunks. + let mut data = vec![0; 8 << 20]; + let mut x: u32 = 1; + for range in [0..100_000, 3_000_123..3_100_000, (8 << 20) - 5..8 << 20] { + for b in &mut data[range] { + x = x.wrapping_mul(1_103_515_245).wrapping_add(12345); + *b = (x >> 24) as u8 | 1; + } + } + let dir = tempdir()?; + let file = dir.path().join("zeros"); + fs::write(&file, &data)?; + + let opts = BackupOptions::default().as_path(PathBuf::from_str("test")?); + _ = repo.backup( + &opts, + &PathList::from_iter(Some(dir.path().to_path_buf())), + SnapshotFile::default(), + )?; + + // The zero chunks are stored once, under the hash of their content. + let repo = repo.to_indexed()?; + repo.check(CheckOptions::default().read_data(true))? + .is_ok()?; + let node = repo.node_from_snapshot_path("latest:test/zeros", |_| true)?; + let content = node.content.as_ref().unwrap(); + assert!(content.iter().collect::>().len() < content.len()); + let mut restored = Vec::new(); + repo.dump(&node, &mut restored)?; + assert_eq!(restored, data); + Ok(()) +}