MUCH faster `pijul add -r`

[?]

Dec 11, 2020, 5:51 PM

OJZWJUF2TCGZ7RFVY6FPKBS5P3C4BGHZDPVH775OHVNVFMJICKNQC

Dependencies

[2] 2K7JLB4Z No pager on Windows
[3] VO5OQW4W Removing anyhow in libpijul
[4] 4OCC6D42 Recursive add
[5] R3H7D42U Debugging `pijul git`: proper error reporting
[6] SXEYMYF7 Fixing the bad changes in history (unfortunately, by rebooting).
[7] LCERQSWM Cleanup
[*] Q45QHPO4 Feedback on network stuff

Change contents

edit in pijul/src/commands/record.rs at line 189

[4.107478]

[4.107478]

num_cpus::get(),
edit in pijul/src/commands/git.rs at line 697

[4.160881]

[4.188]

let num_cpus = num_cpus::get();
replacement in pijul/src/commands/git.rs at line 699

[4.219]→[4.219:317](∅→∅)

let result = working_copy.record_prefix(txn, channel, changes, &mut state, repo_path, p);

[4.219]

[3.647]

let result =
working_copy.record_prefix(txn, channel, changes, &mut state, repo_path, p, num_cpus);
edit in pijul/src/commands/file_operations.rs at line 101

[4.171507]

[4.171551]

let threads = num_cpus::get();
replacement in pijul/src/commands/file_operations.rs at line 112

[4.459]→[4.459:538](∅→∅)

.add_prefix_rec(&mut txn, &repo.path, &full, &prefix)?

[4.459]

[4.538]

.add_prefix_rec(&mut txn, &repo.path, &full, &prefix, threads)?
edit in pijul/src/commands/diff.rs at line 60

[4.175012]

[4.175012]

num_cpus::get(),
edit in pijul/Cargo.toml at line 78

[9.2859]

[2.309]

num_cpus = "1.13"
edit in libpijul/src/working_copy/filesystem.rs at line 95

[4.210434]

[3.3775]

threads: usize,
replacement in libpijul/src/working_copy/filesystem.rs at line 98

[4.210511]→[4.1391:1481](∅→∅)

self.record_prefix(txn, channel, changes, state, repo_path, prefix.as_ref())?

[4.210511]

[4.210695]

self.record_prefix(
txn,
channel,
changes,
state,
repo_path,
prefix.as_ref(),
threads,
)?
replacement in libpijul/src/working_copy/filesystem.rs at line 109

[4.210738]→[4.1482:1570](∅→∅)

self.record_prefix(txn, channel, changes, state, repo_path, Path::new(""))?

[4.210738]

[4.1589]

self.record_prefix(
txn,
channel,
changes,
state,
repo_path,
Path::new(""),
threads,
)?
edit in libpijul/src/working_copy/filesystem.rs at line 128

[4.1792]

[3.3825]

threads: usize,
replacement in libpijul/src/working_copy/filesystem.rs at line 136

[4.2110]→[4.2110:2677](∅→∅)

if let Ok(meta) = meta {
if meta.is_dir() {
let mut walk = WalkBuilder::new(&full);
walk.standard_filters(true);
let walk = walk.build();
for entry in walk {
let entry = entry?;
let p = entry.path();
if let Some(p) = p.file_name() {
if let Some(p) = p.to_str() {
if p.ends_with("~") || (p.starts_with("#") && p.ends_with("#")) {
continue;

[4.2110]

[4.2677]

let (sender, receiver) = std::sync::mpsc::sync_channel(100);
let full = full.to_path_buf();
let t = std::thread::spawn(move || -> Result<(), AddError<T::Error>> {
if let Ok(meta) = meta {
if meta.is_dir() {
let mut walk = WalkBuilder::new(&full);
walk.standard_filters(true);
walk.threads(threads - 1);
walk.build_parallel().run(|| {
Box::new(|entry| {
let entry: ignore::DirEntry = if let Ok(entry) = entry {
entry
} else {
return ignore::WalkState::Quit;
};
let p = entry.path();
if let Some(p) = p.file_name() {
if let Some(p) = p.to_str() {
if p.ends_with("~") || (p.starts_with("#") && p.ends_with("#"))
{
return ignore::WalkState::Skip;
}
}
}
debug!("entry path = {:?} {:?}", entry.path(), repo_path_);
if let Ok(path) = entry.path().strip_prefix(&repo_path_) {
let is_dir = entry.file_type().unwrap().is_dir();
sender.send((path.to_path_buf(), is_dir)).unwrap();
} else {
debug!("entry = {:?}", entry.path());
replacement in libpijul/src/working_copy/filesystem.rs at line 167

[4.2707]→[4.2707:3413](∅→∅)

}
}
debug!("entry path = {:?} {:?}", entry.path(), repo_path);
if let Ok(path) = entry.path().strip_prefix(&repo_path_) {
let path_str = path.to_str().unwrap();
if !txn.is_tracked(&path_str) {
info!("Adding {:?}", path);
txn.add(path_str, entry.file_type().unwrap().is_dir())?
} else {
debug!("already tracked {:?}", path_str)
}
} else {
debug!("entry = {:?}", entry.path());
}

[4.2707]

[4.3413]

ignore::WalkState::Continue
})
})
} else if let Ok(path) = full.strip_prefix(&repo_path_) {
sender.send((path.to_path_buf(), false)).unwrap();
edit in libpijul/src/working_copy/filesystem.rs at line 173

[4.3431]→[4.3431:3721](∅→∅)

} else if let Ok(path) = full.strip_prefix(&repo_path_) {
let path_str = path.to_str().unwrap();
if !txn.is_tracked(&path_str) {
info!("Adding file {:?}", path);
txn.add(path_str, false)?
}
edit in libpijul/src/working_copy/filesystem.rs at line 174

[4.3735]

[4.210920]

Ok(())
});

while let Ok((path, is_dir)) = receiver.recv() {
info!("Adding {:?}", path);
let path_str = path.to_str().unwrap();
txn.add(path_str, is_dir)?
}

if let Ok(t) = t.join() {
t?
edit in libpijul/src/working_copy/filesystem.rs at line 200

[4.211282]

[3.3868]

threads: usize,
replacement in libpijul/src/working_copy/filesystem.rs at line 203

[4.211500]→[4.3736:3802](∅→∅)

self.add_prefix_rec(txn, repo_path, &full, &prefix)?;

[4.211500]

[4.213590]

self.add_prefix_rec(txn, repo_path, &full, &prefix, threads)?;
edit in libpijul/src/pristine/path_id.rs at line 1

[4.584967]→[4.584968:585023](∅→∅)

// org id iBTHhIzBVluJ4gTIZXlBR1WgC/C6Zw4cKuVCh++X60Y=
edit in libpijul/src/pristine/inode.rs at line 1

[4.643628]→[4.643629:643684](∅→∅)

// org id husqv+oHDFDeBrRtm5ymelT1H84bAvU87dV7ig64UFo=
replacement in libpijul/src/fs.rs at line 74

[4.739847]→[4.739847:739899](∅→∅)

fileid.basename = SmallString::from_str(c);

[4.739847]

[4.739899]

fileid.basename.clone_from_str(c);
replacement in libpijul/src/fs.rs at line 82

[4.740145]→[4.740145:740177](∅→∅)

id.cmp(&fileid)

[4.740145]

[4.740177]

id > fileid
edit in libpijul/src/fs.rs at line 89

[4.740320]→[4.740320:740403](∅→∅)

if !found {
fileid.parent_inode = inode;
}
edit in libpijul/src/fs.rs at line 90

[4.740429]

[4.740429]

fileid.parent_inode = inode;
break;
replacement in libpijul/src/fs.rs at line 217

[4.745165]→[4.745165:745244](∅→∅)

current_inode = make_new_child(txn, current_inode, c, true, None)?

[4.745165]

[4.745244]

current_inode = make_new_child(txn, current_inode, c, true, None)?;