Making libpijul deterministic (and getting rid of `rand`)
[?]
May 18, 2021, 8:08 AM
2RXOCWUWOGHEKHT5W73LAHJSOZVRTOGS7BWLSIGEEEBJGMCZBXQACDependencies
- [2]
3MHUNYERChanging --amend flag to respect -m flag - [3]
TPQHDDNCpijul add --force know adds ignored files - [4]
ZAQU2MXMBetter debugging output for `PathId` - [5]
3X4OWIU2Sanakirja 1.2 - [6]
YCEZL7VFMoving to temporary paths when outputting - [7]
K7JPP64SFaster error in unrecord when the change is not on the channel - [8]
G7VOM2IMReturning an error when recording non-existent paths - [9]
P6WE7YKLUnrecord did not check whether a file already existed before adding it back - [10]
V435QOJRUsing path-slash to fix path issues on Windows - [11]
ZBNKSYA6Fixing a bus error when starting a transaction on a full disk - [12]
5BRU2RRWCleanup (debugging a crash related to trees/inodes) - [13]
LCERQSWMCleanup - [14]
MF3WAHBIcommands/record: amend current change if unspecified - [15]
VO5OQW4WRemoving anyhow in libpijul - [16]
OJZWJUF2MUCH faster `pijul add -r` - [17]
PP3E3TREDo not move source if destination is wrong - [18]
B3QWIGDEFixing the Git features with the latest Pijul (+ conflicts in Cargo.toml) - [19]
IM6UFPOZFixing a test related to the tree/revtree tables - [20]
33SQMZYXNew versions of dependencies - [21]
OU243LABSupport for staging - [22]
SXEYMYF7Fixing the bad changes in history (unfortunately, by rebooting). - [23]
YN63NUZOSanakirja 1.0 - [24]
A3DMBJJAUpgrading the `git` subcommand to the latest Sanakirja and Libpijul - [25]
CXM5CBS2Fixing `pijul remove` when files are not deleted - [26]
QL6K2ZM3Tags - [27]
MFTN7GBWPre-tags cleanup + fast Sanakirja - [28]
KQTD46KVUnrecord: restore files *after* having unapplied the *entire* change - [29]
GHO6DWPIRefactoring iterators - [30]
JRENVH5DReqwest 0.11 - [31]
4OCC6D42Recursive add - [32]
ZHABNS3SCanonicalize all paths - [33]
3AMEP2Y5More convenient interface for channels - [34]
IIV3EL2XCleanup, formatting, and fixing the Git feature - [35]
ENKQ3QZGForward the exit status messages from the SSH background loop to the client (solving hangs) - [36]
IXC43DSHFixing a bug in unrecord - [37]
I24UEJQLVarious post-fire fixes - [38]
RXNT67OTSanakirja version, and removing an unwrap - [39]
PJ7T2VFLDo not hang on locked repositories - [40]
5SLOJYHGFixing the Git feature - [41]
UFCZKKLXUpgrading to the latest Sanakirja/Rand - [42]
I52XSRUHMassive cleanup, and simplification - [43]
BD5PC25ADeleting conflict resolution vertices when the sides are deleted - [44]
4VWXL6KQCorrect handling of ignore files - [45]
6DOXSHWGCleanup, and version bump - [46]
3J6IK4W2Explicitly adding .pijul is now forbidden - [47]
CCLLB7OIUpgrading to Sanakirja 0.15 + version bump - [48]
WI2OFAKLhonor hidden files when using the pijul add subcommand - [49]
7UPL3Y2AUnrecord: don't restore the same unrecorded file deletion twice in the inodes and tree tables - [*]
TZVUNELWDocumentation comments - [*]
HMMMKONLFixing alive vertices - [*]
Y6EVFMTADon't output files if they aren't in the current channel
Change contents
- replacement in pijul/src/remote/mod.rs at line 279
txn.archive_with_state(&changes, &mut channel, &state, extra, &mut tarball)?txn.archive_with_state(&changes, &mut channel, &state, extra, &mut tarball, 0)? - edit in pijul/src/remote/local.rs at line 133
0, - replacement in pijul/src/commands/unrecord.rs at line 132
txn.write().unwrap().unrecord(&repo.changes, &channel, &hash)?;txn.write().unwrap().unrecord(&repo.changes, &channel, &hash, 0)?; - edit in pijul/src/commands/unrecord.rs at line 145
0, - replacement in pijul/src/commands/unrecord.rs at line 149
txn.write().unwrap().unrecord(&repo.changes, &channel, &h)?;txn.write().unwrap().unrecord(&repo.changes, &channel, &h, 0)?; - edit in pijul/src/commands/reset.rs at line 130
0, - edit in pijul/src/commands/reset.rs at line 195
0, - edit in pijul/src/commands/reset.rs at line 208
0, - edit in pijul/src/commands/reset.rs at line 234
0, - replacement in pijul/src/commands/record.rs at line 117
txn.unrecord(&repo.changes, &mut channel, &h)?;txn.unrecord(&repo.changes, &mut channel, &h, self.timestamp.unwrap_or(0) as u64)?; - replacement in pijul/src/commands/record.rs at line 309
num_cpus::get(),1, // num_cpus::get(), - replacement in pijul/src/commands/record.rs at line 321
num_cpus::get(),1, // num_cpus::get(), - replacement in pijul/src/commands/record.rs at line 333
num_cpus::get(),1, // num_cpus::get(),self.timestamp.unwrap_or(0) as u64, - edit in pijul/src/commands/pushpull.rs at line 499
0, - edit in pijul/src/commands/pushpull.rs at line 516
0, - replacement in pijul/src/commands/pushpull.rs at line 529
txn.unrecord(&repo.changes, &mut channel, &h)?;txn.unrecord(&repo.changes, &mut channel, &h, 0)?; - edit in pijul/src/commands/protocol.rs at line 220
0, - edit in pijul/src/commands/protocol.rs at line 251
0, - edit in pijul/src/commands/git.rs at line 474
0, - replacement in pijul/src/commands/git.rs at line 585
txn.move_file(&old_path, &tmp_path.to_string_lossy())txn.move_file(&old_path, &tmp_path.to_string_lossy(), 0) - replacement in pijul/src/commands/git.rs at line 614
if let Err(e) = txn.move_file(&a.to_string_lossy(), &b.to_string_lossy()) {if let Err(e) = txn.move_file(&a.to_string_lossy(), &b.to_string_lossy(), 0) { - replacement in pijul/src/commands/git.rs at line 652
txn_.add_dir(&p).unwrap_or(());txn_.add_dir(&p, 0).unwrap_or(()); - replacement in pijul/src/commands/git.rs at line 654
txn_.add_file(&p).unwrap_or(());txn_.add_file(&p, 0).unwrap_or(()); - edit in pijul/src/commands/git.rs at line 760
0, - edit in pijul/src/commands/file_operations.rs at line 22
salt: Option<u64>, - replacement in pijul/src/commands/file_operations.rs at line 65
txn.move_file(&source, &target)?;txn.move_file(&source, &target, self.salt.unwrap_or(0))?; - edit in pijul/src/commands/file_operations.rs at line 122
#[clap(long = "salt")]salt: Option<u64>, - edit in pijul/src/commands/file_operations.rs at line 159
self.salt.unwrap_or(0), - replacement in pijul/src/commands/file_operations.rs at line 171
if let Err(e) = txn.add(&path_str, meta.is_dir()) {if let Err(e) = txn.add(&path_str, meta.is_dir(), self.salt.unwrap_or(0)) { - edit in pijul/src/commands/diff.rs at line 65
0, - edit in pijul/src/commands/debug.rs at line 51
::sanakirja::debug::debug(&txn.txn, &[&txn.tree], "debug.tree", true); - edit in pijul/src/commands/clone.rs at line 32
salt: Option<u64>, - edit in pijul/src/commands/clone.rs at line 103
self.salt.unwrap_or(0), - edit in pijul/src/commands/archive.rs at line 123
0, - edit in pijul/src/commands/apply.rs at line 132[11.24131][53.3672]
0, - edit in pijul/src/commands/apply.rs at line 145[11.24352][53.3988]
0, - replacement in pijul/Cargo.toml at line 68
sanakirja = { version = "1.2.0", features = [ "crc32" ] }sanakirja = { version="1.2", features = [ "crc32" ] } - edit in libpijul/src/working_copy/filesystem.rs at line 109
salt: u64, - edit in libpijul/src/working_copy/filesystem.rs at line 123
salt, - edit in libpijul/src/working_copy/filesystem.rs at line 135
salt, - edit in libpijul/src/working_copy/filesystem.rs at line 147
salt: u64, - replacement in libpijul/src/working_copy/filesystem.rs at line 172
.threads(threads - 1);.threads((threads - 1).max(1)); - replacement in libpijul/src/working_copy/filesystem.rs at line 215
match txn.add(&path_str, is_dir) {match txn.add(&path_str, is_dir, salt) { - edit in libpijul/src/working_copy/filesystem.rs at line 240
salt: u64, - replacement in libpijul/src/working_copy/filesystem.rs at line 246
self.add_prefix_rec(txn.clone(), repo_path, full, threads)?;self.add_prefix_rec(txn.clone(), repo_path, full, threads, salt)?; - edit in libpijul/src/unrecord/working_copy.rs at line 34
salt: u64, - replacement in libpijul/src/unrecord/working_copy.rs at line 42
restore(txn, changes, channel, source, dest)?restore(txn, changes, channel, source, dest, salt)? - edit in libpijul/src/unrecord/working_copy.rs at line 57
salt: u64, - replacement in libpijul/src/unrecord/working_copy.rs at line 64
return_value = restore_inode(txn, changes, source, dest, parent_inode)?;return_value = restore_inode(txn, changes, source, dest, parent_inode, salt)?; - replacement in libpijul/src/unrecord/working_copy.rs at line 87
return_value = restore_inode(txn, changes, source, dest, Inode::ROOT)?;return_value = restore_inode(txn, changes, source, dest, Inode::ROOT, salt)?; - replacement in libpijul/src/unrecord/working_copy.rs at line 89
return_value = restore_inode(txn, changes, source, dest, inode)?;return_value = restore_inode(txn, changes, source, dest, inode, salt)?; - edit in libpijul/src/unrecord/working_copy.rs at line 108
salt: u64, - replacement in libpijul/src/unrecord/working_copy.rs at line 127
let inode = crate::fs::create_new_inode(txn)?;let inode = crate::fs::create_new_inode(txn, &file_id, salt)?; - edit in libpijul/src/unrecord/mod.rs at line 52
salt: u64, - replacement in libpijul/src/unrecord/mod.rs at line 67
unapply(txn, &mut channel, changes, change_id, &change)?;unapply(txn, &mut channel, changes, change_id, &change, salt)?; - edit in libpijul/src/unrecord/mod.rs at line 144
salt: u64, - replacement in libpijul/src/unrecord/mod.rs at line 197
working_copy::undo_file_deletion(txn, changes, channel, change_id, newedges)?working_copy::undo_file_deletion(txn, changes, channel, change_id, newedges, salt)? - edit in libpijul/src/small_string.rs at line 15
}impl std::hash::Hash for SmallStr {fn hash<H: std::hash::Hasher>(&self, hasher: &mut H) {self.as_bytes().hash(hasher)} - replacement in libpijul/src/pristine/sanakirja.rs at line 256
tree: UDb<PathId, Inode>,pub tree: UDb<PathId, Inode>, - replacement in libpijul/src/pristine/sanakirja.rs at line 1409
self.counter += 1;debug!("put_graph {:?} {:?}, counter = {:?}", k, e, self.counter);// self.counter += 1;// debug!("put_graph {:?} {:?}, counter = {:?}", k, e, self.counter); - replacement in libpijul/src/pristine/sanakirja.rs at line 1440
self.counter += 1;debug!("del_graph {:?} {:?}, counter = {:?}", k, e, self.counter);// self.counter += 1;// debug!("del_graph {:?} {:?}, counter = {:?}", k, e, self.counter); - replacement in libpijul/src/pristine/path_id.rs at line 25
#[derive(Eq, PartialEq, Ord, PartialOrd)]#[derive(Hash, Eq, PartialEq, Ord, PartialOrd)] - edit in libpijul/src/pristine/mod.rs at line 855
use rand::Rng; - replacement in libpijul/src/pristine/mod.rs at line 857
Hash::Blake3(ref s) => ChangeId(L64(LittleEndian::read_u64(&s[..]))),Hash::Blake3(ref s) => LittleEndian::read_u64(&s[..]), - replacement in libpijul/src/pristine/mod.rs at line 859
while let Some(ext) = txn.get_external(&p)? {let mut pp = ChangeId(L64(p));while let Some(ext) = txn.get_external(&pp)? { - replacement in libpijul/src/pristine/mod.rs at line 862
p = ChangeId(L64(rand::thread_rng().gen()));p = u64::from_le(p) + 1;pp = ChangeId(L64(p.to_le())); - replacement in libpijul/src/pristine/mod.rs at line 865
Ok(p)Ok(pp) - replacement in libpijul/src/pristine/inode.rs at line 5
pub struct Inode(pub(in crate::pristine) super::L64);pub struct Inode(pub(in crate) super::L64);use byteorder::{ByteOrder, BigEndian}; - replacement in libpijul/src/pristine/inode.rs at line 11
self.0.to_slice_le(&mut b);BigEndian::write_u64(&mut b, (self.0).0); // self.0.to_slice_le(&mut b); - edit in libpijul/src/pristine/inode.rs at line 20
}pub(crate) fn random() -> Self {Inode(L64(rand::random())) - edit in libpijul/src/output/output.rs at line 45
salt: u64, - edit in libpijul/src/output/output.rs at line 60
salt, - edit in libpijul/src/output/output.rs at line 132
salt: u64, - edit in libpijul/src/output/output.rs at line 275
salt, - edit in libpijul/src/output/output.rs at line 375
salt: u64, - replacement in libpijul/src/output/output.rs at line 399[11.680955]→[11.680955:681136](∅→∅),[11.681136]→[11.0:40](∅→∅),[11.40]→[11.681136:681168](∅→∅),[11.681136]→[11.681136:681168](∅→∅)
use rand::Rng;let s: String = rand::thread_rng().sample_iter(&rand::distributions::Alphanumeric).take(30).map(|c| c as char).collect();let s = {let mut c = [0u8; 16];unsafe {*(c.as_mut_ptr() as *mut Position<ChangeId>) = output_item.pos}data_encoding::BASE32_NOPAD.encode(blake3::hash(&c).as_bytes())}; - edit in libpijul/src/output/output.rs at line 408
- edit in libpijul/src/output/output.rs at line 410
- replacement in libpijul/src/output/output.rs at line 445
let inode = create_new_inode(&mut *txn_)?;let inode = create_new_inode(&mut *txn_, &file_id, salt)?; - edit in libpijul/src/lib.rs at line 107
pub type Hasher = std::hash::BuildHasherDefault<twox_hash::XxHash64>; - replacement in libpijul/src/lib.rs at line 109
std::collections::HashMap<K, V, std::hash::BuildHasherDefault<twox_hash::XxHash64>>;std::collections::HashMap<K, V, Hasher>; - replacement in libpijul/src/lib.rs at line 111
std::collections::HashSet<K, std::hash::BuildHasherDefault<twox_hash::XxHash64>>;std::collections::HashSet<K, Hasher>; - edit in libpijul/src/lib.rs at line 288
salt: u64, - replacement in libpijul/src/lib.rs at line 290[11.28778]→[11.725402:725459](∅→∅),[11.94683]→[11.725402:725459](∅→∅),[11.725402]→[11.725402:725459](∅→∅)
unrecord::unrecord(self, channel, changes, hash)unrecord::unrecord(self, channel, changes, hash, salt) - replacement in libpijul/src/lib.rs at line 319[11.726194]→[11.94789:94875](∅→∅),[11.28956]→[11.726264:726311](∅→∅),[11.94875]→[11.726264:726311](∅→∅),[11.726264]→[11.726264:726311](∅→∅)
fn add_file(&mut self, path: &str) -> Result<(), fs::FsError<Self::GraphError>> {fs::add_inode(self, None, path, false)fn add_file(&mut self, path: &str, salt: u64) -> Result<(), fs::FsError<Self::GraphError>> {fs::add_inode(self, None, path, false, salt) - replacement in libpijul/src/lib.rs at line 327[11.726545]→[11.94876:94961](∅→∅),[11.29037]→[11.726614:726660](∅→∅),[11.94961]→[11.726614:726660](∅→∅),[11.726614]→[11.726614:726660](∅→∅)
fn add_dir(&mut self, path: &str) -> Result<(), fs::FsError<Self::GraphError>> {fs::add_inode(self, None, path, true)fn add_dir(&mut self, path: &str, salt: u64) -> Result<(), fs::FsError<Self::GraphError>> {fs::add_inode(self, None, path, true, salt) - replacement in libpijul/src/lib.rs at line 334[11.726871]→[11.94962:95057](∅→∅),[11.29128]→[11.726950:726998](∅→∅),[11.95057]→[11.726950:726998](∅→∅),[11.726950]→[11.726950:726998](∅→∅)
fn add(&mut self, path: &str, is_dir: bool) -> Result<(), fs::FsError<Self::GraphError>> {fs::add_inode(self, None, path, is_dir)fn add(&mut self, path: &str, is_dir: bool, salt: u64) -> Result<(), fs::FsError<Self::GraphError>> {fs::add_inode(self, None, path, is_dir, salt) - replacement in libpijul/src/lib.rs at line 338[11.727005]→[11.95058:95151](∅→∅),[11.29217]→[11.727082:727116](∅→∅),[11.95151]→[11.727082:727116](∅→∅),[11.727082]→[11.727082:727116](∅→∅)
fn move_file(&mut self, a: &str, b: &str) -> Result<(), fs::FsError<Self::GraphError>> {fs::move_file(self, a, b)fn move_file(&mut self, a: &str, b: &str, salt: u64) -> Result<(), fs::FsError<Self::GraphError>> {fs::move_file(self, a, b, salt) - edit in libpijul/src/lib.rs at line 353
salt: u64, - edit in libpijul/src/lib.rs at line 363
salt, - edit in libpijul/src/lib.rs at line 383
salt: u64, - replacement in libpijul/src/lib.rs at line 402[11.95738]→[11.729279:729333](∅→∅),[11.126591]→[11.729279:729333](∅→∅),[11.729279]→[11.729279:729333](∅→∅)
self.unrecord(changes, channel, &h)?;self.unrecord(changes, channel, &h, salt)?; - replacement in libpijul/src/fs.rs at line 61[11.30675]→[11.102288:102389](∅→∅),[11.102389]→[11.738608:738741](∅→∅),[11.738608]→[11.738608:738741](∅→∅),[11.738741]→[11.129467:129533](∅→∅)
pub(crate) fn create_new_inode<T: TreeMutTxnT>(txn: &mut T) -> Result<Inode, TxnErr<T::TreeError>> {let mut already_taken = true;let mut inode: Inode = Inode::ROOT;while already_taken {inode = Inode::random();already_taken = txn.get_revtree(&inode, None)?.is_some();pub(crate) fn create_new_inode<T: TreeMutTxnT>(txn: &mut T, parent_id: &PathId, salt: u64) -> Result<Inode, TxnErr<T::TreeError>> {use std::hash::{BuildHasher, Hash, Hasher};let mut s = crate::Hasher::default().build_hasher();(parent_id, salt).hash(&mut s);let mut i: u64 = s.finish();let mut inode = Inode(L64(i.to_le()));while txn.get_revtree(&inode, None)?.is_some() {i += 1;inode = Inode(L64(i.to_le())); - edit in libpijul/src/fs.rs at line 191
salt: u64, - replacement in libpijul/src/fs.rs at line 217
None => create_new_inode(txn)?,None => create_new_inode(txn, &parent_id, salt)?, - edit in libpijul/src/fs.rs at line 239
salt: u64, - replacement in libpijul/src/fs.rs at line 248
current_inode = make_new_child(txn, current_inode, c, true, None)?;current_inode = make_new_child(txn, current_inode, c, true, None, salt)?; - replacement in libpijul/src/fs.rs at line 252
make_new_child(txn, current_inode, file_name, is_dir, inode)?;make_new_child(txn, current_inode, file_name, is_dir, inode, salt)?; - edit in libpijul/src/fs.rs at line 266
salt: u64, - replacement in libpijul/src/fs.rs at line 269
move_file_by_inode(txn, find_inode(txn, origin)?, destination)?;move_file_by_inode(txn, find_inode(txn, origin)?, destination, salt)?; - edit in libpijul/src/fs.rs at line 277
salt: u64, - replacement in libpijul/src/fs.rs at line 292
add_inode(txn, Some(inode), destination, is_dir)?;add_inode(txn, Some(inode), destination, is_dir, salt)?; - edit in libpijul/src/apply.rs at line 353
- replacement in libpijul/Cargo.toml at line 97
sanakirja = { version = "1.2.0", features = [ "crc32" ] }sanakirja = { version = "1.2", features = [ "crc32" ] } - edit in libpijul/Cargo.toml at line 104
rand = "0.8" - edit in Cargo.lock at line 952
"rand 0.8.3",