When downloading changes, check whether we have their dependencies and download them too

pmeunier

Feb 28, 2023, 11:57 AM

C3L2TLQWREYOM3YHL37L7PS74YGLHBEDQRSCVMYIU6HKBEPNN2SAC

Dependencies

[2] 6NUSIEGD Fixing concurrency issues in output, when large binary files are involved
[3] 4XLHUME7 Fixing a interlocking when cloning from a particular patch
[4] SXEYMYF7 Fixing the bad changes in history (unfortunately, by rebooting).
[5] Q45QHPO4 Feedback on network stuff
[6] X6YFD4WV Do not download changes if we already have them
[7] MU5GSJAW Partial push and pull (WARNING: breaks the existing protocol)
[8] 5SLOJYHG Fixing the Git feature
[9] TPEH2XNB 1.0.0-alpha.28, with Tokio 1.0
[10] 2RXOCWUW Making libpijul deterministic (and getting rid of `rand`)
[11] ZDK3GNDB Tag transactions (including a massive refactoring of errors)
[12] 2D7P2VKJ Change completions (where the whole progress bar story started)
[13] ZWVYH7WP Pulling local tags
[14] QE64ATLZ Fixing asynchronicity problems in SSH
[15] EUZFFJSO Updating Pijul with the latest changes in Libpijul
[16] I24UEJQL Various post-fire fixes
[17] DO2Y5TY5 Tag synchronisation
[18] 5QTMRUXN Fixing a race condition between progress bars
[19] 3KRGVQFU Do not update the mtime of unmodified files
[20] 3WO4H2MM Fixing async issues in downloads
[21] A3RM526Y Integrating identity malleability
[22] TKEVOH7H Fixing a bug when downloading changes, and making change download more efficient (more async)
[23] LGEJSLTY Fixing output (including its uses in reset and pull)
[24] XA23FMQM Reset only files that have been modified
[25] BNPSVXIC Friendlier progress bars
[26] HXEIH4UQ Pulling more than 100 changes at once

Change contents

replacement in pijul/src/remote/mod.rs at line 1147

[4.1407]→[4.68985:69053](∅→∅),[4.2675]→[4.68985:69053](∅→∅),[4.68985]→[4.68985:69053](∅→∅)

let (mut send, mut recv) = tokio::sync::mpsc::channel(100);

[4.2675]

[4.1206]

let (mut send, recv) = tokio::sync::mpsc::channel(100);
edit in pijul/src/remote/mod.rs at line 1163

[4.3095]

[4.231]

let mut waiting = 0;
edit in pijul/src/remote/mod.rs at line 1175

[4.408]

[4.408]

waiting += 1;
replacement in pijul/src/remote/mod.rs at line 1180

[4.544]→[4.2833:2868](∅→∅)

std::mem::drop(hash_send);

[4.544]

[4.0]

let (u, mut to_apply) = self
.download_changes_rec(repo, hash_send, recv, pro_a, waiting)
.await?;
replacement in pijul/src/remote/mod.rs at line 1187

[4.1454]→[4.593:621](∅→∅),[4.735]→[4.593:621](∅→∅)

for h in to_apply {

[4.1454]

[4.3174]

while let Some(h) = to_apply.recv().await {
edit in pijul/src/remote/mod.rs at line 1189

[4.3215]→[4.3215:3307](∅→∅),[4.3307]→[3.257:315](∅→∅),[3.315]→[4.3360:3473](∅→∅),[4.3360]→[4.3360:3473](∅→∅),[4.920]→[4.70253:70312](∅→∅),[4.3473]→[4.70253:70312](∅→∅),[4.70253]→[4.70253:70312](∅→∅)

while to_download.contains(&h) {
debug!("waiting for {:?}", h);
if let Some((h, _)) = recv.recv().await {
debug!("recv {:?}", h);
to_download.remove(&h);
} else {
break;
}
}
replacement in pijul/src/remote/mod.rs at line 1193

[4.1641]→[4.16553:16600](∅→∅)

if let CS::Change(h) = h {

[4.1641]

[4.16600]

if let CS::Change(ref h) = h {
replacement in pijul/src/remote/mod.rs at line 1213

[4.2330]→[4.17387:17461](∅→∅)

|| { inodes.iter().any(|i| CS::Change(i.change) == *h) };

[4.2330]

[4.2392]

|| { inodes.iter().any(|i| CS::Change(i.change) == h) };
replacement in pijul/src/remote/mod.rs at line 1216

[4.2425]→[4.2425:2467](∅→∅)

to_apply_inodes.push(*h);

[4.2425]

[4.2467]

to_apply_inodes.push(h);
replacement in pijul/src/remote/mod.rs at line 1227

[4.17560]→[4.17560:17643](∅→∅)

txn.apply_change_ws(&repo.changes, &mut channel, h, &mut ws)?;

[4.17560]

[4.17643]

txn.apply_change_rec_ws(&repo.changes, &mut channel, &h, &mut ws)?;
edit in pijul/src/remote/mod.rs at line 1236

[4.1191]→[4.70686:70716](∅→∅),[4.3665]→[4.70686:70716](∅→∅),[4.70686]→[4.70686:70716](∅→∅)

std::mem::drop(recv);
edit in pijul/src/remote/mod.rs at line 1238

[4.321]

[4.1356]

u.await??;
edit in pijul/src/remote/mod.rs at line 1243

[4.70880]

[4.70880]

async fn download_changes_rec(
&mut self,
repo: &mut Repository,
send_hash: tokio::sync::mpsc::UnboundedSender<CS>,
mut recv_signal: tokio::sync::mpsc::Receiver<(CS, bool)>,
pro_n: usize,
mut waiting: usize,
) -> Result<
(
tokio::task::JoinHandle<Result<(), anyhow::Error>>,
tokio::sync::mpsc::Receiver<CS>,
),
anyhow::Error,
> {
let (send_ready, recv_ready) = tokio::sync::mpsc::channel(100);
let mut change_path = repo.changes_dir.clone();
let mut dep_path = repo.changes_dir.clone();
let changes = repo.changes.clone();
let t = tokio::spawn(async move {
let mut ready = Vec::new();
while let Some((hash, follow)) = recv_signal.recv().await {
if let CS::Change(hash) = hash {
waiting -= 1;
if follow {
libpijul::changestore::filesystem::push_filename(&mut change_path, &hash);
std::fs::create_dir_all(change_path.parent().unwrap())?;
use libpijul::changestore::ChangeStore;
let mut needs_dep = false;
for dep in changes.get_dependencies(&hash)? {
let dep: libpijul::pristine::Hash = dep;

libpijul::changestore::filesystem::push_filename(&mut dep_path, &dep);
let has_dep = std::fs::metadata(&dep_path).is_ok();
libpijul::changestore::filesystem::pop_filename(&mut dep_path);
edit in pijul/src/remote/mod.rs at line 1279

[4.70881]

[4.4721]

if !has_dep {
needs_dep = true;
PROGRESS.borrow_mut().unwrap()[pro_n].incr_len();
send_hash.send(CS::Change(dep))?;
waiting += 1
}
}
libpijul::changestore::filesystem::pop_filename(&mut change_path);

if !needs_dep {
send_ready.send(CS::Change(hash)).await?;
} else {
ready.push(CS::Change(hash))
}
} else {
send_ready.send(CS::Change(hash)).await?;
}
}
if waiting == 0 {
break;
}
}
for r in ready {
send_ready.send(r).await?;
}
std::mem::drop(recv_signal);
Ok(())
});
Ok((t, recv_ready))
}
edit in pijul/src/remote/mod.rs at line 1317

[4.71101]→[4.71101:71183](∅→∅)

let (mut send_signal, mut recv_signal) = tokio::sync::mpsc::channel(100);
replacement in pijul/src/remote/mod.rs at line 1318

[4.842]→[4.71261:71319](∅→∅),[4.3773]→[4.71261:71319](∅→∅),[4.71261]→[4.71261:71319](∅→∅)

let mut change_path_ = repo.changes_dir.clone();

[4.3773]

[4.71319]

let (mut send_signal, recv_signal) = tokio::sync::mpsc::channel(100);
replacement in pijul/src/remote/mod.rs at line 1328

[4.1733]→[4.1733:1734](∅→∅)

[4.1733]

[4.71386]

let mut change_path_ = repo.changes_dir.clone();
edit in pijul/src/remote/mod.rs at line 1347

[4.72022]

[4.72022]

let (u, mut recv_ready) = self
.download_changes_rec(repo, send_hash, recv_signal, pro_n, waiting)
.await?;
edit in pijul/src/remote/mod.rs at line 1351

[4.72023]→[4.72023:72079](∅→∅)

let mut change_path = repo.changes_dir.clone();
edit in pijul/src/remote/mod.rs at line 1352

[4.72116]→[3.373:441](∅→∅),[3.441]→[4.17707:17752](∅→∅),[4.72174]→[4.17707:17752](∅→∅),[4.17752]→[3.442:1197](∅→∅),[3.1197]→[4.18201:18219](∅→∅),[4.18201]→[4.18201:18219](∅→∅),[4.18219]→[3.1198:1264](∅→∅),[3.1264]→[4.72577:72591](∅→∅),[4.4140]→[4.72577:72591](∅→∅),[4.18302]→[4.72577:72591](∅→∅),[4.72577]→[4.72577:72591](∅→∅),[4.72670]→[4.72670:72752](∅→∅)

while let Some((hash, follow)) = recv_signal.recv().await {
if let CS::Change(hash) = hash {
waiting -= 1;
if follow {
libpijul::changestore::filesystem::push_filename(&mut change_path, &hash);
std::fs::create_dir_all(change_path.parent().unwrap())?;
use libpijul::changestore::ChangeStore;
hashes.push(CS::Change(hash));
for dep in repo.changes.get_dependencies(&hash)? {
let dep: libpijul::pristine::Hash = dep;
PROGRESS.borrow_mut().unwrap()[pro_n].incr_len();
send_hash.send(CS::Change(dep))?;
waiting += 1
}
libpijul::changestore::filesystem::pop_filename(&mut change_path);
}
}
if waiting == 0 {
break
}
}
std::mem::drop(recv_signal);
std::mem::drop(send_hash);
replacement in pijul/src/remote/mod.rs at line 1355

[4.578]→[3.1265:1311](∅→∅),[3.1311]→[4.18303:18352](∅→∅),[4.247]→[4.18303:18352](∅→∅)

for hash in hashes.iter().rev() {
if let CS::Change(hash) = hash {

[4.578]

[3.1312]

while let Some(hash) = recv_ready.recv().await {
if let CS::Change(ref hash) = hash {
edit in pijul/src/remote/mod.rs at line 1359

[4.18457]

[4.330]

hashes.push(hash);
edit in pijul/src/remote/mod.rs at line 1364

[4.73007]

[4.1873]

u.await??;
replacement in pijul/src/commands/clone.rs at line 108

[4.22604]→[4.22604:22660](∅→∅),[4.22660]→[4.17222:17254](∅→∅),[4.17254]→[4.185679:185706](∅→∅),[4.22699]→[4.185679:185706](∅→∅),[4.185679]→[4.185679:185706](∅→∅),[4.185706]→[4.17255:17295](∅→∅),[4.1926]→[4.185732:185766](∅→∅),[4.17295]→[4.185732:185766](∅→∅),[4.22746]→[4.185732:185766](∅→∅),[4.185732]→[4.185732:185766](∅→∅),[4.185766]→[4.491:509](∅→∅),[4.509]→[2.46:81](∅→∅),[2.81]→[4.1592:1628](∅→∅),[4.22776]→[4.1592:1628](∅→∅),[4.509]→[4.185766:185778](∅→∅),[4.1628]→[4.185766:185778](∅→∅),[4.22776]→[4.185766:185778](∅→∅),[4.185766]→[4.185766:185778](∅→∅)

libpijul::output::output_repository_no_pending(
&repo.working_copy,
&repo.changes,
&txn,
&channel,
"",
true,
None,
1, // num_cpus::get(),
self.salt.unwrap_or(0),
)?;

[4.22558]

[4.502]

if self.partial_paths.is_empty() {
libpijul::output::output_repository_no_pending(
&repo.working_copy,
&repo.changes,
&txn,
&channel,
"",
true,
None,
1, // num_cpus::get(),
self.salt.unwrap_or(0),
)?;
} else {
for p in self.partial_paths.iter() {
libpijul::output::output_repository_no_pending(
&repo.working_copy,
&repo.changes,
&txn,
&channel,
p,
true,
None,
1, // num_cpus::get(),
self.salt.unwrap_or(0),
)?;
}
}