new ChangeStore implementation

andybalholm
Apr 14, 2023, 6:18 PM
QWLMNP5FOCONNJDNVKZFHJIM7WHLF7QCJI67YSV5VT2SXEOPZ6TQC

Dependencies

Change contents

  • edit in src/repo.rs at line 27
    [3.180]
    [3.180]
    use crate::changestore::EagerChangeStore;
  • replacement in src/repo.rs at line 33
    [3.229][3.135:200]()
    change_store: libpijul::changestore::filesystem::FileSystem,
    [3.229]
    [3.86]
    change_store: EagerChangeStore,
  • replacement in src/repo.rs at line 94
    [3.818][3.328:442]()
    let change_store =
    libpijul::changestore::filesystem::FileSystem::from_root(&repo_path, 256);
    [3.818]
    [3.818]
    let change_store = EagerChangeStore::from_root(&repo_path);
  • replacement in src/repo.rs at line 209
    [3.725][3.725:851]()
    let change_store =
    libpijul::changestore::filesystem::FileSystem::from_root(&Path::new(&self.path), 256);
    [3.725]
    [3.851]
    let change_store = EagerChangeStore::from_root(&Path::new(&self.path));
  • replacement in src/repo.rs at line 223
    [3.1227][3.1227:1292]()
    change_store: libpijul::changestore::filesystem::FileSystem,
    [3.1227]
    [3.1292]
    change_store: EagerChangeStore,
  • edit in src/main.rs at line 10
    [9.52]
    [9.52]
    mod changestore;
  • file addition: changestore.rs (----------)
    [8.33]
    use std::collections::HashMap;
    use std::path::{Path, PathBuf};
    use std::sync::{Arc, RwLock};
    use libpijul::change::{Change, ChangeError, ChangeHeader};
    use libpijul::changestore::ChangeStore;
    use libpijul::pristine::{Hash, Position};
    use libpijul::{Base32, ChangeId, Merkle, Vertex};
    use thiserror::Error;
    #[derive(Clone)]
    /// A change store that eagerly decompresses the content of changes,
    /// and keeps them in memory.
    pub struct EagerChangeStore {
    changes: Arc<RwLock<HashMap<Hash, Change>>>,
    tags: Arc<RwLock<HashMap<Merkle, ChangeHeader>>>,
    changes_dir: PathBuf,
    }
    #[derive(Debug, Error)]
    pub enum Error {
    #[error(transparent)]
    Io(#[from] std::io::Error),
    #[error(transparent)]
    Utf8(#[from] std::str::Utf8Error),
    #[error(transparent)]
    Change(#[from] libpijul::change::ChangeError),
    #[error(transparent)]
    Bincode(#[from] bincode::Error),
    #[error(transparent)]
    Tag(#[from] libpijul::tag::TagError),
    #[error(transparent)]
    Persist(#[from] tempfile::PersistError),
    }
    impl EagerChangeStore {
    pub fn from_root<P: AsRef<Path>>(root: P) -> Self {
    EagerChangeStore {
    changes: Arc::default(),
    tags: Arc::default(),
    changes_dir: root.as_ref().join(".pijul").join("changes"),
    }
    }
    fn load(&self, h: &Hash) -> Result<(), Error> {
    {
    let r = self.changes.read().unwrap();
    if let Some(_) = r.get(h) {
    return Ok(());
    }
    }
    let path = self.filename(h);
    let file_name = path.to_str().unwrap();
    let p = Change::deserialize(&file_name, Some(h))?;
    self.changes.write().unwrap().insert(*h, p);
    Ok(())
    }
    fn filename(&self, h: &Hash) -> PathBuf {
    let mut path = self.changes_dir.clone();
    let b32 = h.to_base32();
    let (a, b) = b32.split_at(2);
    path.push(a);
    path.push(b);
    path.set_extension("change");
    path
    }
    }
    impl ChangeStore for EagerChangeStore {
    type Error = Error;
    fn get_change(&self, h: &Hash) -> Result<Change, Self::Error> {
    self.load(h)?;
    let r = self.changes.read().unwrap();
    Ok(r.get(h).unwrap().clone())
    }
    fn get_tag_header(&self, h: &Merkle) -> Result<ChangeHeader, Self::Error> {
    let r = self.tags.read().unwrap();
    if let Some(t) = r.get(&h) {
    Ok(t.clone())
    } else {
    let mut path = self.changes_dir.clone();
    let b32 = h.to_base32();
    let (a, b) = b32.split_at(2);
    path.push(a);
    path.push(b);
    path.set_extension("tag");
    let mut p = libpijul::tag::OpenTagFile::open(&path, h)?;
    let header = p.header()?;
    self.tags.write().unwrap().insert(*h, header.clone());
    Ok(header)
    }
    }
    fn has_contents(&self, hash: Hash, _: Option<ChangeId>) -> bool {
    match self.load(&hash) {
    Ok(_) => {
    let r = self.changes.read().unwrap();
    match r.get(&hash) {
    Some(c) => !c.contents.is_empty(),
    None => false,
    }
    }
    Err(_) => false,
    }
    }
    fn get_contents<F: Fn(ChangeId) -> Option<Hash>>(
    &self,
    hash: F,
    key: Vertex<ChangeId>,
    buf: &mut [u8],
    ) -> Result<usize, Self::Error> {
    if key.end <= key.start {
    return Ok(0);
    }
    assert_eq!(buf.len(), key.end - key.start);
    let h = hash(key.change).unwrap();
    self.load(&h)?;
    let r = self.changes.read().unwrap();
    let p = r.get(&h).unwrap();
    let start: usize = key.start.0 .0.try_into().unwrap();
    let end: usize = key.end.0 .0.try_into().unwrap();
    buf.clone_from_slice(&p.contents[start..end]);
    Ok(end - start)
    }
    fn get_contents_ext(
    &self,
    key: Vertex<Option<Hash>>,
    buf: &mut [u8],
    ) -> Result<usize, Self::Error> {
    if let Some(change) = key.change {
    if key.end <= key.start {
    return Ok(0);
    }
    assert_eq!(key.end - key.start, buf.len());
    self.load(&change)?;
    let r = self.changes.read().unwrap();
    let p = r.get(&change).unwrap();
    let start: usize = key.start.0 .0.try_into().unwrap();
    let end: usize = key.end.0 .0.try_into().unwrap();
    buf.clone_from_slice(&p.contents[start..end]);
    Ok(end - start)
    } else {
    Ok(0)
    }
    }
    fn change_deletes_position<F: Fn(ChangeId) -> Option<Hash>>(
    &self,
    hash: F,
    change: ChangeId,
    pos: Position<Option<Hash>>,
    ) -> Result<Vec<Hash>, Self::Error> {
    let r = self.changes.read().unwrap();
    let change = r.get(&hash(change).unwrap()).unwrap();
    let mut v = Vec::new();
    for c in change.changes.iter() {
    for c in c.iter() {
    v.extend(c.deletes_pos(pos).into_iter())
    }
    }
    Ok(v)
    }
    fn save_change<
    E: From<Self::Error> + From<ChangeError>,
    F: FnOnce(&mut Change, &Hash) -> Result<(), E>,
    >(
    &self,
    p: &mut Change,
    ff: F,
    ) -> Result<Hash, E> {
    let mut f = match tempfile::NamedTempFile::new_in(&self.changes_dir) {
    Ok(f) => f,
    Err(e) => return Err(E::from(Error::from(e))),
    };
    let hash = {
    let w = std::io::BufWriter::new(&mut f);
    p.serialize(w, ff)?
    };
    let file_name = self.filename(&hash);
    if let Err(e) = std::fs::create_dir_all(file_name.parent().unwrap()) {
    return Err(E::from(Error::from(e)));
    }
    if let Err(e) = f.persist(file_name) {
    return Err(E::from(Error::from(e)));
    }
    let mut w = self.changes.write().unwrap();
    w.insert(hash, p.clone());
    Ok(hash)
    }
    fn del_change(&self, h: &Hash) -> Result<bool, Self::Error> {
    let file_name = self.filename(h);
    std::fs::remove_file(&file_name).unwrap_or(());
    std::fs::remove_dir(file_name.parent().unwrap()).unwrap_or(()); // fails silently if there are still changes with the same 2-letter prefix.
    let mut w = self.changes.write().unwrap();
    Ok(w.remove(h).is_some())
    }
    }
  • edit in Cargo.toml at line 18
    [2.2602]
    tempfile = "3.1"
    bincode = "1.3"
    thiserror = "1.0"
  • edit in Cargo.lock at line 848
    [10.20468]
    [3.1873]
    "bincode",
  • edit in Cargo.lock at line 856
    [2.2631]
    [3.1827]
    "tempfile",
    "thiserror",