{"header":{"message":"new ChangeStore implementation","description":"The FileSystem ChangeStore in libpijul loads change content lazily.\nI'm more concerned about speed than memory usage here, \nso I made my own implementation that loads the content eagerly and\nkeeps it around.","timestamp":"2023-04-14T18:18:02.406809325Z","authors":[{}]},"hash":"QWLMNP5FOCONNJDNVKZFHJIM7WHLF7QCJI67YSV5VT2SXEOPZ6TQC","authors":[{"login":"andybalholm","key":"BCEXYuKWaQ96btsk8UyBZWHLjn1Brhykv8tuZGPRjzFn"}],"deps":{"hashes":[["QWLMNP5FOCONNJDNVKZFHJIM7WHLF7QCJI67YSV5VT2SXEOPZ6TQC","new ChangeStore implementation"],["AA",null],["R3XMFPCYGXT5CNHYRNVZDGD2TDTCK6XGQ55SPJW56UEQQNV4R64AC","import and export marks"],["ATRA7XTTN62JZQOI7A4JI3Z7L3XVTAEEW2U4AX6UQKEQ6USOXQDQC","RIIR: factor out Repository struct"],["YNZMKRJDWYOQ3R3XHA2G47YG2SDI7FT7AEXH3OLB3BJGB6ALTRQAC","include more information in log"],["D467LQZ62MTKWYPTMRBYTTR63CZDCE5WGBIGLQMSWAQYPHC3XITAC","Load identities"],["FIIUZR4LJOB5DPB4CBMPJHMO7C5Q4ZINUVM52UK6SIM5WM7R7ZLAC","Include file content"],["TQBJZLD7Q223IFEBR7SU4FJWPWQWVBCSRPYYYEHXNSDM6DC7UWRAC","RIIR: hello, world"],["UFSP7C7B2ERFJ2XVVVNXGZRXUEMFSMRABC5YXXLK6ONMFG6QBNHAC","Export metadata"],["GKLDTVAKBFKWS3JZVTWJAWYUNPEIUMK6FW77IRD7J26NAVQ52CKQC","RIIR: add libpijul"]],"deps":["R3XMFPCYGXT5CNHYRNVZDGD2TDTCK6XGQ55SPJW56UEQQNV4R64AC"],"known":["ATRA7XTTN62JZQOI7A4JI3Z7L3XVTAEEW2U4AX6UQKEQ6USOXQDQC","YNZMKRJDWYOQ3R3XHA2G47YG2SDI7FT7AEXH3OLB3BJGB6ALTRQAC","D467LQZ62MTKWYPTMRBYTTR63CZDCE5WGBIGLQMSWAQYPHC3XITAC","FIIUZR4LJOB5DPB4CBMPJHMO7C5Q4ZINUVM52UK6SIM5WM7R7ZLAC"]},"hunks":[{"Edit":{"inode":{"change":3,"pos":32},"path":"src/repo.rs","line":27,"atom":{"NewVertex":{"up_context":[{"change":3,"pos":180}],"down_context":[{"change":3,"pos":180}],"flag":"BLOCK","start":0,"end":43,"inode":{"change":3,"pos":32}}},"contents":[{"Add":""},{"Add":"use crate::changestore::EagerChangeStore;"}]}},{"Replacement":{"inode":{"change":3,"pos":32},"path":"src/repo.rs","line":33,"ins":{"up_context":[{"change":3,"pos":229}],"down_context":[{"change":3,"pos":86}],"flag":"BLOCK","start":44,"end":80,"inode":{"change":3,"pos":32}},"del":{"edges":[{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":3,"pos":229},"to":{"change":3,"start":135,"end":200},"introduced_by":3}],"inode":{"change":3,"pos":32}},"ins_contents":[{"Add":" change_store: EagerChangeStore,"}],"del_contents":[{"Del":" change_store: libpijul::changestore::filesystem::FileSystem,\n"}]}},{"Replacement":{"inode":{"change":3,"pos":32},"path":"src/repo.rs","line":94,"ins":{"up_context":[{"change":3,"pos":818}],"down_context":[{"change":3,"pos":818}],"flag":"BLOCK","start":81,"end":149,"inode":{"change":3,"pos":32}},"del":{"edges":[{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":3,"pos":818},"to":{"change":3,"start":328,"end":442},"introduced_by":3}],"inode":{"change":3,"pos":32}},"ins_contents":[{"Add":" let change_store = EagerChangeStore::from_root(&repo_path);"}],"del_contents":[{"Del":" let change_store =\n libpijul::changestore::filesystem::FileSystem::from_root(&repo_path, 256);\n"}]}},{"Replacement":{"inode":{"change":3,"pos":32},"path":"src/repo.rs","line":209,"ins":{"up_context":[{"change":3,"pos":725}],"down_context":[{"change":3,"pos":851}],"flag":"BLOCK","start":150,"end":230,"inode":{"change":3,"pos":32}},"del":{"edges":[{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":3,"pos":725},"to":{"change":3,"start":725,"end":851},"introduced_by":3}],"inode":{"change":3,"pos":32}},"ins_contents":[{"Add":" let change_store = EagerChangeStore::from_root(&Path::new(&self.path));"}],"del_contents":[{"Del":" let change_store =\n libpijul::changestore::filesystem::FileSystem::from_root(&Path::new(&self.path), 256);\n"}]}},{"Replacement":{"inode":{"change":3,"pos":32},"path":"src/repo.rs","line":223,"ins":{"up_context":[{"change":3,"pos":1227}],"down_context":[{"change":3,"pos":1292}],"flag":"BLOCK","start":231,"end":267,"inode":{"change":3,"pos":32}},"del":{"edges":[{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":3,"pos":1227},"to":{"change":3,"start":1227,"end":1292},"introduced_by":3}],"inode":{"change":3,"pos":32}},"ins_contents":[{"Add":" change_store: EagerChangeStore,"}],"del_contents":[{"Del":" change_store: libpijul::changestore::filesystem::FileSystem,\n"}]}},{"Edit":{"inode":{"change":8,"pos":67},"path":"src/main.rs","line":10,"atom":{"NewVertex":{"up_context":[{"change":9,"pos":52}],"down_context":[{"change":9,"pos":52}],"flag":"BLOCK","start":268,"end":286,"inode":{"change":8,"pos":67}}},"contents":[{"Add":""},{"Add":"mod changestore;"}]}},{"FileAdd":{"meta":{"basename":"changestore.rs","metadata":0},"context":[{"change":8,"pos":33}],"contents":[{"Add":"use std::collections::HashMap;"},{"Add":"use std::path::{Path, PathBuf};"},{"Add":"use std::sync::{Arc, RwLock};"},{"Add":""},{"Add":"use libpijul::change::{Change, ChangeError, ChangeHeader};"},{"Add":"use libpijul::changestore::ChangeStore;"},{"Add":"use libpijul::pristine::{Hash, Position};"},{"Add":"use libpijul::{Base32, ChangeId, Merkle, Vertex};"},{"Add":""},{"Add":"use thiserror::Error;"},{"Add":""},{"Add":"#[derive(Clone)]"},{"Add":"/// A change store that eagerly decompresses the content of changes,"},{"Add":"/// and keeps them in memory."},{"Add":"pub struct EagerChangeStore {"},{"Add":" changes: Arc>>,"},{"Add":" tags: Arc>>,"},{"Add":" changes_dir: PathBuf,"},{"Add":"}"},{"Add":""},{"Add":"#[derive(Debug, Error)]"},{"Add":"pub enum Error {"},{"Add":" #[error(transparent)]"},{"Add":" Io(#[from] std::io::Error),"},{"Add":" #[error(transparent)]"},{"Add":" Utf8(#[from] std::str::Utf8Error),"},{"Add":" #[error(transparent)]"},{"Add":" Change(#[from] libpijul::change::ChangeError),"},{"Add":" #[error(transparent)]"},{"Add":" Bincode(#[from] bincode::Error),"},{"Add":" #[error(transparent)]"},{"Add":" Tag(#[from] libpijul::tag::TagError),"},{"Add":" #[error(transparent)]"},{"Add":" Persist(#[from] tempfile::PersistError),"},{"Add":"}"},{"Add":""},{"Add":"impl EagerChangeStore {"},{"Add":" pub fn from_root>(root: P) -> Self {"},{"Add":" EagerChangeStore {"},{"Add":" changes: Arc::default(),"},{"Add":" tags: Arc::default(),"},{"Add":" changes_dir: root.as_ref().join(\".pijul\").join(\"changes\"),"},{"Add":" }"},{"Add":" }"},{"Add":""},{"Add":" fn load(&self, h: &Hash) -> Result<(), Error> {"},{"Add":" {"},{"Add":" let r = self.changes.read().unwrap();"},{"Add":" if let Some(_) = r.get(h) {"},{"Add":" return Ok(());"},{"Add":" }"},{"Add":" }"},{"Add":""},{"Add":" let path = self.filename(h);"},{"Add":" let file_name = path.to_str().unwrap();"},{"Add":" let p = Change::deserialize(&file_name, Some(h))?;"},{"Add":" self.changes.write().unwrap().insert(*h, p);"},{"Add":" Ok(())"},{"Add":" }"},{"Add":""},{"Add":" fn filename(&self, h: &Hash) -> PathBuf {"},{"Add":" let mut path = self.changes_dir.clone();"},{"Add":" let b32 = h.to_base32();"},{"Add":" let (a, b) = b32.split_at(2);"},{"Add":" path.push(a);"},{"Add":" path.push(b);"},{"Add":" path.set_extension(\"change\");"},{"Add":" path"},{"Add":" }"},{"Add":"}"},{"Add":""},{"Add":"impl ChangeStore for EagerChangeStore {"},{"Add":" type Error = Error;"},{"Add":""},{"Add":" fn get_change(&self, h: &Hash) -> Result {"},{"Add":" self.load(h)?;"},{"Add":" let r = self.changes.read().unwrap();"},{"Add":" Ok(r.get(h).unwrap().clone())"},{"Add":" }"},{"Add":""},{"Add":" fn get_tag_header(&self, h: &Merkle) -> Result {"},{"Add":" let r = self.tags.read().unwrap();"},{"Add":" if let Some(t) = r.get(&h) {"},{"Add":" Ok(t.clone())"},{"Add":" } else {"},{"Add":" let mut path = self.changes_dir.clone();"},{"Add":" let b32 = h.to_base32();"},{"Add":" let (a, b) = b32.split_at(2);"},{"Add":" path.push(a);"},{"Add":" path.push(b);"},{"Add":" path.set_extension(\"tag\");"},{"Add":" let mut p = libpijul::tag::OpenTagFile::open(&path, h)?;"},{"Add":" let header = p.header()?;"},{"Add":" self.tags.write().unwrap().insert(*h, header.clone());"},{"Add":" Ok(header)"},{"Add":" }"},{"Add":" }"},{"Add":""},{"Add":" fn has_contents(&self, hash: Hash, _: Option) -> bool {"},{"Add":" match self.load(&hash) {"},{"Add":" Ok(_) => {"},{"Add":" let r = self.changes.read().unwrap();"},{"Add":" match r.get(&hash) {"},{"Add":" Some(c) => !c.contents.is_empty(),"},{"Add":" None => false,"},{"Add":" }"},{"Add":" }"},{"Add":" Err(_) => false,"},{"Add":" }"},{"Add":" }"},{"Add":""},{"Add":" fn get_contents Option>("},{"Add":" &self,"},{"Add":" hash: F,"},{"Add":" key: Vertex,"},{"Add":" buf: &mut [u8],"},{"Add":" ) -> Result {"},{"Add":" if key.end <= key.start {"},{"Add":" return Ok(0);"},{"Add":" }"},{"Add":" assert_eq!(buf.len(), key.end - key.start);"},{"Add":" let h = hash(key.change).unwrap();"},{"Add":" self.load(&h)?;"},{"Add":" let r = self.changes.read().unwrap();"},{"Add":" let p = r.get(&h).unwrap();"},{"Add":" let start: usize = key.start.0 .0.try_into().unwrap();"},{"Add":" let end: usize = key.end.0 .0.try_into().unwrap();"},{"Add":" buf.clone_from_slice(&p.contents[start..end]);"},{"Add":" Ok(end - start)"},{"Add":" }"},{"Add":""},{"Add":" fn get_contents_ext("},{"Add":" &self,"},{"Add":" key: Vertex>,"},{"Add":" buf: &mut [u8],"},{"Add":" ) -> Result {"},{"Add":" if let Some(change) = key.change {"},{"Add":" if key.end <= key.start {"},{"Add":" return Ok(0);"},{"Add":" }"},{"Add":" assert_eq!(key.end - key.start, buf.len());"},{"Add":" self.load(&change)?;"},{"Add":" let r = self.changes.read().unwrap();"},{"Add":" let p = r.get(&change).unwrap();"},{"Add":" let start: usize = key.start.0 .0.try_into().unwrap();"},{"Add":" let end: usize = key.end.0 .0.try_into().unwrap();"},{"Add":" buf.clone_from_slice(&p.contents[start..end]);"},{"Add":" Ok(end - start)"},{"Add":" } else {"},{"Add":" Ok(0)"},{"Add":" }"},{"Add":" }"},{"Add":""},{"Add":" fn change_deletes_position Option>("},{"Add":" &self,"},{"Add":" hash: F,"},{"Add":" change: ChangeId,"},{"Add":" pos: Position>,"},{"Add":" ) -> Result, Self::Error> {"},{"Add":" let r = self.changes.read().unwrap();"},{"Add":" let change = r.get(&hash(change).unwrap()).unwrap();"},{"Add":" let mut v = Vec::new();"},{"Add":" for c in change.changes.iter() {"},{"Add":" for c in c.iter() {"},{"Add":" v.extend(c.deletes_pos(pos).into_iter())"},{"Add":" }"},{"Add":" }"},{"Add":" Ok(v)"},{"Add":" }"},{"Add":""},{"Add":" fn save_change<"},{"Add":" E: From + From,"},{"Add":" F: FnOnce(&mut Change, &Hash) -> Result<(), E>,"},{"Add":" >("},{"Add":" &self,"},{"Add":" p: &mut Change,"},{"Add":" ff: F,"},{"Add":" ) -> Result {"},{"Add":" let mut f = match tempfile::NamedTempFile::new_in(&self.changes_dir) {"},{"Add":" Ok(f) => f,"},{"Add":" Err(e) => return Err(E::from(Error::from(e))),"},{"Add":" };"},{"Add":" let hash = {"},{"Add":" let w = std::io::BufWriter::new(&mut f);"},{"Add":" p.serialize(w, ff)?"},{"Add":" };"},{"Add":" let file_name = self.filename(&hash);"},{"Add":" if let Err(e) = std::fs::create_dir_all(file_name.parent().unwrap()) {"},{"Add":" return Err(E::from(Error::from(e)));"},{"Add":" }"},{"Add":" if let Err(e) = f.persist(file_name) {"},{"Add":" return Err(E::from(Error::from(e)));"},{"Add":" }"},{"Add":""},{"Add":" let mut w = self.changes.write().unwrap();"},{"Add":" w.insert(hash, p.clone());"},{"Add":" Ok(hash)"},{"Add":" }"},{"Add":""},{"Add":" fn del_change(&self, h: &Hash) -> Result {"},{"Add":" let file_name = self.filename(h);"},{"Add":" std::fs::remove_file(&file_name).unwrap_or(());"},{"Add":" std::fs::remove_dir(file_name.parent().unwrap()).unwrap_or(()); // fails silently if there are still changes with the same 2-letter prefix."},{"Add":" let mut w = self.changes.write().unwrap();"},{"Add":" Ok(w.remove(h).is_some())"},{"Add":" }"},{"Add":"}"}]}},{"Edit":{"inode":{"change":8,"pos":149},"path":"Cargo.toml","line":18,"atom":{"NewVertex":{"up_context":[{"change":2,"pos":2602}],"down_context":[],"flag":"BLOCK","start":6795,"end":6846,"inode":{"change":8,"pos":149}}},"contents":[{"Add":"tempfile = \"3.1\""},{"Add":"bincode = \"1.3\""},{"Add":"thiserror = \"1.0\""}]}},{"Edit":{"inode":{"change":8,"pos":367},"path":"Cargo.lock","line":848,"atom":{"NewVertex":{"up_context":[{"change":10,"pos":20468}],"down_context":[{"change":3,"pos":1873}],"flag":"BLOCK","start":6847,"end":6859,"inode":{"change":8,"pos":367}}},"contents":[{"Add":" \"bincode\","}]}},{"Edit":{"inode":{"change":8,"pos":367},"path":"Cargo.lock","line":856,"atom":{"NewVertex":{"up_context":[{"change":2,"pos":2631}],"down_context":[{"change":3,"pos":1827}],"flag":"BLOCK","start":6860,"end":6887,"inode":{"change":8,"pos":367}}},"contents":[{"Add":" \"tempfile\","},{"Add":" \"thiserror\","}]}}],"owner":"andybalholm","repo":"pijul-export","token":"3Vs58LVlF4OewGov8jYyjjNL0j8QsqpDCXtz1u8gAmU=","login":null}

…

Dependencies

Change contents