Single file databases + CRC for the root pages (checking the other pages makes everything very slow)
[?]
Feb 18, 2021, 9:53 PM
W2MIZD5BNL7A5HVFWTESF57QU7T6QMEF4RBSLFQXMEEU3XD2NU2QCDependencies
- [2]
OHUZ73MKVersions - [3]
G4JEQLLXDebugging synchronisation - [4]
XEU2QVLCDebugging after plugging this into Pijul - [5]
OFINGD26implementing prev() on cursors (+ some cleanup) - [6]
HN6Z5DU4Cleanup - [7]
6DCQHIFPMinor changes after benchmarking - [8]
YWFYZNLZCleanup + inter-process concurrency - [9]
6DMPXOATMore debugging - [10]
OP6SVMODResetting history - [11]
YXKP4AIWNew file locks, with multiple sets of free pages - [12]
SO25TWFLA few features for integrating it into Pijul - [13]
EAAYH6BQDebugging put - [*]
WS4ZQM4RDebugging, tests, etc. - [*]
UAQX27N4Tests - [*]
OTWDDJE7Trait/type cleanup - [*]
ONES3V46reference counting works for put
Change contents
- edit in sanakirja-core/src/lib.rs at line 225
}}#[cfg(feature = "crc32")]pub fn crc(&self, hasher: &crc32fast::Hasher) -> u32 {let mut hasher = hasher.clone();hasher.reset();// Hash the beginning and the end of the page (i.e. remove// the CRC).unsafe {hasher.update(core::slice::from_raw_parts(self.data, 4));hasher.update(core::slice::from_raw_parts(self.data.offset(8),PAGE_SIZE - 8,)); - edit in sanakirja-core/src/lib.rs at line 241
hasher.finalize()}#[cfg(feature = "crc32")]pub fn crc_check(&self, hasher: &crc32fast::Hasher) -> bool {let crc = unsafe { u32::from_le(*(self.data as *const u32)) };self.crc(hasher) == crc - edit in sanakirja-core/src/lib.rs at line 257
#[cfg(not(feature = "crc32"))] - edit in sanakirja-core/src/lib.rs at line 260
}#[cfg(feature = "crc32")]pub fn clear_dirty(&mut self, hasher: &crc32fast::Hasher) {unsafe {*self.0.data &= 0xfe;let crc = (self.0.data as *mut u32).offset(1);*crc = self.0.crc(hasher)} - replacement in sanakirja-core/Cargo.toml at line 3
version = "0.1.0"authors = ["pe"]version = "1.0.0" - edit in sanakirja-core/Cargo.toml at line 5
description = "Copy-on-write datastructures, storable on disk (or elsewhere) with a stable format."authors = ["Pierre-Étienne Meunier", "Florent Becker"]license = "MIT/Apache-2.0"documentation = "https://docs.rs/sanakirja"repository = "https://nest.pijul.com/pijul/sanakirja"include = ["Cargo.toml","src/lib.rs","src","src/btree","src/btree/page_cursor.rs","src/btree/page_unsized","src/btree/page_unsized/alloc.rs","src/btree/page_unsized/rebalance.rs","src/btree/page_unsized/header.rs","src/btree/page_unsized/put.rs","src/btree/page_unsized/cursor.rs","src/btree/put.rs","src/btree/page","src/btree/page/alloc.rs","src/btree/page/rebalance.rs","src/btree/page/put.rs","src/btree/del.rs","src/btree/mod.rs","src/btree/page_unsized.rs","src/btree/cursor.rs","src/btree/page.rs"] - replacement in sanakirja-core/Cargo.toml at line 34
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html[features]crc32 = [ "crc32fast" ] - replacement in sanakirja-core/Cargo.toml at line 38
log = "*"[3.70978]log = "0.4"crc32fast = { version = "1.2", optional = true, default-features = false } - edit in sanakirja/src/tests.rs at line 22
#[test]pub fn put_growth() {env_logger::try_init().unwrap_or(());let path = tempfile::tempdir().unwrap();let path = path.path().join("db");let l0 = 1 << 13; // 2 pageslet env = Env::new(&path, l0, 1).unwrap();let mut txn = Env::mut_txn_begin(&env).unwrap();let mut db = create_db::<MutTxn<&Env, ()>, u64, u64>(&mut txn).unwrap();let n = 100_000u64;for i in 0..n {put(&mut txn, &mut db, &i, &i).unwrap();}println!("{:?}", env.mmaps);let len = std::fs::metadata(&path).unwrap().len();assert_eq!(len, (l0 << 9) - l0);} - replacement in sanakirja/src/tests.rs at line 412
txn.set_root(0, db.db);txn.set_root(1, db.db); - replacement in sanakirja/src/tests.rs at line 421
let env = Env::new("/tmp/sanakirja0", 4096 * 20, 2).unwrap();let env = Env::new("/tmp/sanakirja0/db", 4096 * 20, 2).unwrap(); - replacement in sanakirja/src/tests.rs at line 535
let env = Env::new("/tmp/sanakirja0", 409_600_000, 2).unwrap();let env = Env::new("/tmp/sanakirja0/db", 409_600_000, 2).unwrap(); - edit in sanakirja/src/lib.rs at line 1
//! Transactional, on-disk datastructures with concurrent readers and//! writers (writers exclude each other).//!//! This crate is based on the no-std crate `sanakirja-core`, whose//! goal is to implement different datastructures. - replacement in sanakirja/src/lib.rs at line 8
pub mod environment;mod environment; - edit in sanakirja/src/lib.rs at line 15[3.4433][18.1749]
#[doc(hidden)] - edit in sanakirja/src/environment/muttxn.rs at line 104
env_.check_crc(v0)?; - edit in sanakirja/src/environment/muttxn.rs at line 137
}#[cfg(feature = "crc32")]fn clear_dirty(p: &mut MutPage) {p.clear_dirty(&HASHER) - edit in sanakirja/src/environment/muttxn.rs at line 144
#[cfg(not(feature = "crc32"))]fn clear_dirty(p: &mut MutPage) {p.clear_dirty()} - replacement in sanakirja/src/environment/muttxn.rs at line 162
p.clear_dirty();clear_dirty(p); - edit in sanakirja/src/environment/muttxn.rs at line 184
set_crc(maps[0].ptr.add(*root * PAGE_SIZE)); - replacement in sanakirja/src/environment/muttxn.rs at line 187
let globptr = maps[0].ptr as *mut GlobalHeader;(&mut *globptr).root = *root as u8;(&mut *(maps[0].ptr as *mut GlobalHeader)).root = *root as u8; - replacement in sanakirja/src/environment/mod.rs at line 15
use std::path::{Path, PathBuf};use std::path::Path; - edit in sanakirja/src/environment/mod.rs at line 31
file: Option<std::fs::File>,#[cfg(feature = "mmap")] - replacement in sanakirja/src/environment/mod.rs at line 53
path: Option<PathBuf>,file: Option<std::fs::File>, - edit in sanakirja/src/environment/mod.rs at line 80
drop(map.file); - edit in sanakirja/src/environment/mod.rs at line 100
#[cfg(feature = "mmap")]#[test]#[should_panic]fn nroots_test() {let path = tempfile::tempdir().unwrap();let path = path.path().join("db");let l0 = 1 << 15; // 8 pagesEnv::new(&path, l0, 19).unwrap();}#[cfg(feature = "mmap")]#[test]fn mmap_growth_test() {let path = tempfile::tempdir().unwrap();let path = path.path().join("db");let l0 = 1 << 15; // 8 pages{let env = Env::new(&path, l0, 2).unwrap();let map1 = env.open_mmap(0, l0).unwrap();println!("{:?}", map1);let map2 = env.open_mmap(1, l0).unwrap();println!("{:?}", map2);map1.flush().unwrap();map2.flush().unwrap();}let len = std::fs::metadata(&path).unwrap().len();assert_eq!(len, (l0 << 2) - l0);}#[cfg(not(feature = "crc32"))]fn set_crc(_ptr: *mut u8) {}#[cfg(feature = "crc32")]fn set_crc(ptr: *mut u8) {unsafe {let root_page = std::slice::from_raw_parts(ptr.add(8), PAGE_SIZE - 8);let mut h = HASHER.clone();h.update(root_page);let globptr = ptr as *mut GlobalHeader;(&mut *globptr).crc = h.finalize().to_le();}} - replacement in sanakirja/src/environment/mod.rs at line 146
let db_path = path.as_ref().join("db");Ok(std::fs::metadata(&db_path)?.len())Ok(std::fs::metadata(&path)?.len()) - replacement in sanakirja/src/environment/mod.rs at line 166
// let length = (1 as u64).shl(log_length);let mut db_path = path.as_ref().join("db0");let db_exists = std::fs::metadata(&db_path).is_ok();let length = if let Ok(meta) = std::fs::metadata(&db_path) {let db_exists = std::fs::metadata(&path).is_ok();let length = if let Ok(meta) = std::fs::metadata(&path) { - replacement in sanakirja/src/environment/mod.rs at line 178
.open(&db_path)?;.open(&path)?; - replacement in sanakirja/src/environment/mod.rs at line 181
let mut env = Self::new_nolock_mmap(Some(file), length, mmap, !db_exists, n_roots)?;db_path.pop();env.path = Some(db_path);Ok(env)Self::new_nolock_mmap(Some(file), length, mmap, !db_exists, n_roots) - edit in sanakirja/src/environment/mod.rs at line 192
assert!(n_roots < ((length >> 12) as usize)); - replacement in sanakirja/src/environment/mod.rs at line 194
let glob = if initialise {if initialise { - replacement in sanakirja/src/environment/mod.rs at line 196
*(map.add(i * PAGE_SIZE) as *mut GlobalHeader) = GlobalHeader {*(map.add(i * PAGE_SIZE) as *mut GlobalHeader) = (GlobalHeader { - replacement in sanakirja/src/environment/mod.rs at line 204
};}).to_le();set_crc(map.add(i * PAGE_SIZE)); - replacement in sanakirja/src/environment/mod.rs at line 209[3.96846]→[3.2459:2525](∅→∅),[3.2525]→[3.96911:96928](∅→∅),[3.96911]→[3.96911:96928](∅→∅),[3.96928]→[3.2526:2592](∅→∅),[3.2592]→[3.96993:97004](∅→∅),[3.96993]→[3.96993:97004](∅→∅)
GlobalHeader::from_le(&*(map as *const GlobalHeader))} else {GlobalHeader::from_le(&*(map as *const GlobalHeader))};}let glob = GlobalHeader::from_le(&*(map as *const GlobalHeader)); - replacement in sanakirja/src/environment/mod.rs at line 220
path: None,file, - edit in sanakirja/src/environment/mod.rs at line 224
file, - edit in sanakirja/src/environment/mod.rs at line 271
} - edit in sanakirja/src/environment/mod.rs at line 272
impl Env { - replacement in sanakirja/src/environment/mod.rs at line 291
.open(path.join("db").with_extension(&format!("lock{}", n)))?,.open(path.with_extension(&format!("lock{}", n)))?, - replacement in sanakirja/src/environment/mod.rs at line 331
if let Some(ref path) = self.path {let mut db_path = path.join(&format!("db{}", i));let file = OpenOptions::new().read(true).write(true).truncate(false).create(true).open(&db_path)?;file.set_len(length)?;db_path.pop();let mut mmap = unsafe { memmap::MmapMut::map_mut(&file)? };let offset = (length0 << i) - length0;if let Some(ref file) = self.file {file.set_len(offset + length)?;let mut mmap = unsafe {memmap::MmapOptions::new().offset(offset).len(length as usize).map_mut(&file)?}; - edit in sanakirja/src/environment/mod.rs at line 343
file: Some(file), - edit in sanakirja/src/environment/mod.rs at line 350
file: None, - edit in sanakirja/src/environment/mod.rs at line 402
}#[cfg(feature = "crc32")]use lazy_static::*;#[cfg(feature = "crc32")]lazy_static! {static ref HASHER: crc32fast::Hasher = crc32fast::Hasher::new(); - edit in sanakirja/src/environment/mod.rs at line 416
let env_ = env.borrow(); - edit in sanakirja/src/environment/mod.rs at line 418
let env_ = env.borrow(); - edit in sanakirja/src/environment/mod.rs at line 433
env_.check_crc(root)?; - edit in sanakirja/src/environment/mod.rs at line 435
}#[cfg(not(feature = "crc32"))]fn check_crc(&self, _root: usize) -> Result<(), crate::CRCError> {Ok(())}#[cfg(feature = "crc32")]fn check_crc(&self, root: usize) -> Result<(), crate::CRCError> {unsafe {let maps = self.mmaps.lock();let page_ptr = maps[0].ptr.add(root * PAGE_SIZE);let crc = (&*(page_ptr as *const GlobalHeader)).crc;let root_page = std::slice::from_raw_parts(page_ptr.add(8), PAGE_SIZE - 8);let mut h = HASHER.clone();h.update(root_page);if h.finalize() != crc {return Err(crate::CRCError {});}}Ok(()) - replacement in sanakirja/src/environment/mod.rs at line 533
Err(CRCError {})Err(crate::CRCError {}) - edit in sanakirja/src/environment/global_header.rs at line 5
#[repr(C)] - edit in sanakirja/src/environment/global_header.rs at line 35
}}pub fn to_le(&self) -> Self {GlobalHeader {version: self.version.to_le(),root: self.root,n_roots: self.n_roots,crc: self.crc.to_le(),free_db: self.free_db.to_le(),length: self.length.to_le(),rc_db: self.rc_db.to_le(), - edit in sanakirja/Cargo.toml at line 9
crc32 = [ "crc32fast", "lazy_static", "sanakirja-core/crc32" ] - edit in sanakirja/Cargo.toml at line 19
crc32fast = { version = "1.2", optional = true, default-features = false }lazy_static = { version = "1.4", optional = true } - edit in sanakirja/Cargo.toml at line 29[2.274]
tempfile = "3.2"