The FileSystem ChangeStore in libpijul loads change content lazily. I'm more concerned about speed than memory usage here, so I made my own implementation that loads the content eagerly and keeps it around.
QWLMNP5FOCONNJDNVKZFHJIM7WHLF7QCJI67YSV5VT2SXEOPZ6TQC R3XMFPCYGXT5CNHYRNVZDGD2TDTCK6XGQ55SPJW56UEQQNV4R64AC ATRA7XTTN62JZQOI7A4JI3Z7L3XVTAEEW2U4AX6UQKEQ6USOXQDQC YNZMKRJDWYOQ3R3XHA2G47YG2SDI7FT7AEXH3OLB3BJGB6ALTRQAC D467LQZ62MTKWYPTMRBYTTR63CZDCE5WGBIGLQMSWAQYPHC3XITAC FIIUZR4LJOB5DPB4CBMPJHMO7C5Q4ZINUVM52UK6SIM5WM7R7ZLAC UFSP7C7B2ERFJ2XVVVNXGZRXUEMFSMRABC5YXXLK6ONMFG6QBNHAC TQBJZLD7Q223IFEBR7SU4FJWPWQWVBCSRPYYYEHXNSDM6DC7UWRAC GKLDTVAKBFKWS3JZVTWJAWYUNPEIUMK6FW77IRD7J26NAVQ52CKQC use std::collections::HashMap;use std::path::{Path, PathBuf};use std::sync::{Arc, RwLock};use libpijul::change::{Change, ChangeError, ChangeHeader};use libpijul::changestore::ChangeStore;use libpijul::pristine::{Hash, Position};use libpijul::{Base32, ChangeId, Merkle, Vertex};use thiserror::Error;#[derive(Clone)]/// A change store that eagerly decompresses the content of changes,/// and keeps them in memory.pub struct EagerChangeStore {changes: Arc<RwLock<HashMap<Hash, Change>>>,tags: Arc<RwLock<HashMap<Merkle, ChangeHeader>>>,changes_dir: PathBuf,}#[derive(Debug, Error)]pub enum Error {#[error(transparent)]Io(#[from] std::io::Error),#[error(transparent)]Utf8(#[from] std::str::Utf8Error),#[error(transparent)]Change(#[from] libpijul::change::ChangeError),#[error(transparent)]Bincode(#[from] bincode::Error),#[error(transparent)]Tag(#[from] libpijul::tag::TagError),#[error(transparent)]Persist(#[from] tempfile::PersistError),}impl EagerChangeStore {pub fn from_root<P: AsRef<Path>>(root: P) -> Self {EagerChangeStore {changes: Arc::default(),tags: Arc::default(),changes_dir: root.as_ref().join(".pijul").join("changes"),}}fn load(&self, h: &Hash) -> Result<(), Error> {{let r = self.changes.read().unwrap();if let Some(_) = r.get(h) {return Ok(());}}let path = self.filename(h);let file_name = path.to_str().unwrap();let p = Change::deserialize(&file_name, Some(h))?;self.changes.write().unwrap().insert(*h, p);Ok(())}fn filename(&self, h: &Hash) -> PathBuf {let mut path = self.changes_dir.clone();let b32 = h.to_base32();let (a, b) = b32.split_at(2);path.push(a);path.push(b);path.set_extension("change");path}}impl ChangeStore for EagerChangeStore {type Error = Error;fn get_change(&self, h: &Hash) -> Result<Change, Self::Error> {self.load(h)?;let r = self.changes.read().unwrap();Ok(r.get(h).unwrap().clone())}fn get_tag_header(&self, h: &Merkle) -> Result<ChangeHeader, Self::Error> {let r = self.tags.read().unwrap();if let Some(t) = r.get(&h) {Ok(t.clone())} else {let mut path = self.changes_dir.clone();let b32 = h.to_base32();let (a, b) = b32.split_at(2);path.push(a);path.push(b);path.set_extension("tag");let mut p = libpijul::tag::OpenTagFile::open(&path, h)?;let header = p.header()?;self.tags.write().unwrap().insert(*h, header.clone());Ok(header)}}fn has_contents(&self, hash: Hash, _: Option<ChangeId>) -> bool {match self.load(&hash) {Ok(_) => {let r = self.changes.read().unwrap();match r.get(&hash) {Some(c) => !c.contents.is_empty(),None => false,}}Err(_) => false,}}fn get_contents<F: Fn(ChangeId) -> Option<Hash>>(&self,hash: F,key: Vertex<ChangeId>,buf: &mut [u8],) -> Result<usize, Self::Error> {if key.end <= key.start {return Ok(0);}assert_eq!(buf.len(), key.end - key.start);let h = hash(key.change).unwrap();self.load(&h)?;let r = self.changes.read().unwrap();let p = r.get(&h).unwrap();let start: usize = key.start.0 .0.try_into().unwrap();let end: usize = key.end.0 .0.try_into().unwrap();buf.clone_from_slice(&p.contents[start..end]);Ok(end - start)}fn get_contents_ext(&self,key: Vertex<Option<Hash>>,buf: &mut [u8],) -> Result<usize, Self::Error> {if let Some(change) = key.change {if key.end <= key.start {return Ok(0);}assert_eq!(key.end - key.start, buf.len());self.load(&change)?;let r = self.changes.read().unwrap();let p = r.get(&change).unwrap();let start: usize = key.start.0 .0.try_into().unwrap();let end: usize = key.end.0 .0.try_into().unwrap();buf.clone_from_slice(&p.contents[start..end]);Ok(end - start)} else {Ok(0)}}fn change_deletes_position<F: Fn(ChangeId) -> Option<Hash>>(&self,hash: F,change: ChangeId,pos: Position<Option<Hash>>,) -> Result<Vec<Hash>, Self::Error> {let r = self.changes.read().unwrap();let change = r.get(&hash(change).unwrap()).unwrap();let mut v = Vec::new();for c in change.changes.iter() {for c in c.iter() {v.extend(c.deletes_pos(pos).into_iter())}}Ok(v)}fn save_change<E: From<Self::Error> + From<ChangeError>,F: FnOnce(&mut Change, &Hash) -> Result<(), E>,>(&self,p: &mut Change,ff: F,) -> Result<Hash, E> {let mut f = match tempfile::NamedTempFile::new_in(&self.changes_dir) {Ok(f) => f,Err(e) => return Err(E::from(Error::from(e))),};let hash = {let w = std::io::BufWriter::new(&mut f);p.serialize(w, ff)?};let file_name = self.filename(&hash);if let Err(e) = std::fs::create_dir_all(file_name.parent().unwrap()) {return Err(E::from(Error::from(e)));}if let Err(e) = f.persist(file_name) {return Err(E::from(Error::from(e)));}let mut w = self.changes.write().unwrap();w.insert(hash, p.clone());Ok(hash)}fn del_change(&self, h: &Hash) -> Result<bool, Self::Error> {let file_name = self.filename(h);std::fs::remove_file(&file_name).unwrap_or(());std::fs::remove_dir(file_name.parent().unwrap()).unwrap_or(()); // fails silently if there are still changes with the same 2-letter prefix.let mut w = self.changes.write().unwrap();Ok(w.remove(h).is_some())}}
tempfile = "3.1"bincode = "1.3"thiserror = "1.0"