mx00s/pijul - Change TKEVOH7HXON7SOBGXTUDHAHO2U2GPTQRNESP6ERKUQAS526OZIRAC

Fixing a bug when downloading changes, and making change download more efficient (more async)

Created by pmeunier on February 28, 2021

TKEVOH7HXON7SOBGXTUDHAHO2U2GPTQRNESP6ERKUQAS526OZIRAC

Dependencies

In channels

main

Change contents

Replacement in pijul/src/remote/ssh.rs at line 14 [5.25380]
∅:D[3.80] → [4.251:276]
∅:D[6.6054] → [4.251:276]
B:BD[4.251] → [4.251:276]
```
use log::{debug, error};
```
[6.6054]
[4.276]
```
use log::{debug, error, trace};
```

Replacement in pijul/src/remote/ssh.rs at line 400 [5.25380]

B:BD[7.1411] → [7.1411:1466]

        debug!("data {:?} {:?}", channel, data.len());

[7.1411]

[7.1466]

        trace!("data {:?} {:?}", channel, data.len());

Replacement in pijul/src/remote/ssh.rs at line 430 [5.25380]

B:BD[7.2788] → [7.2788:2833]

                    debug!("state changes");

[7.2788]

[7.2833]

                    trace!("state changes");

Insertion in pijul/src/remote/ssh.rs at line 439 [5.25380]

[7.3260]

                            debug!("writing {:?} bytes", *remaining_len);

Insertion in pijul/src/remote/ssh.rs at line 462 [5.25380]

[7.4400]

                                debug!("before pop: {:?}", path);

Insertion in pijul/src/remote/ssh.rs at line 468 [5.25380]

[7.4701]

                                debug!("after pop: {:?}", path);

Replacement in pijul/src/remote/ssh.rs at line 484 [5.25380]

B:BD[7.5504] → [7.5504:5558]

                            debug!("need more data");

[7.5504]

[7.5558]

                            trace!("need more data");

Replacement in pijul/src/remote/ssh.rs at line 488 [5.25380]

B:BD[7.5641] → [7.5641:5707]

                    debug!("finished, {:?} {:?}", p, data.len());

[7.5641]

[7.5707]

                    trace!("finished, {:?} {:?}", p, data.len());

Replacement in pijul/src/remote/ssh.rs at line 797 [5.25380]

B:BD[5.45470] → [7.8706:8746]

        c: &[libpijul::pristine::Hash],

[5.45470]

[7.8746]

        c: &mut tokio::sync::mpsc::Receiver<libpijul::pristine::Hash>,

Replacement in pijul/src/remote/ssh.rs at line 808 [5.25380]

B:BD[8.208] → [8.208:248]

        c: &[libpijul::pristine::Hash],

[8.208]

[8.248]

        c: &mut tokio::sync::mpsc::Receiver<libpijul::pristine::Hash>,

Deletion in pijul/src/remote/ssh.rs at line 813 [5.25380]

B:BD[8.426] → [9.2585:2648]

        if c.is_empty() {
            return Ok(());
        }

Deletion in pijul/src/remote/ssh.rs at line 815 [5.25380]
B:BD[7.8967] → [7.8967:9043]
```
        libpijul::changestore::filesystem::push_filename(&mut path, &c[0]);
```
Insertion in pijul/src/remote/ssh.rs at line 818 [5.25380]
[7.9094]
[7.9094]
```
        debug!("changes_dir = {:?}", changes_dir);
```
Replacement in pijul/src/remote/ssh.rs at line 825 [5.25380]
B:BD[7.9247] → [7.9247:9279]
```
            hashes: c.to_vec(),
```
[7.9247]
[7.9279]
```
            hashes: Vec::new(),
```

Replacement in pijul/src/remote/ssh.rs at line 829 [5.25380]

B:BD[7.9350] → [7.9350:9371]

        for c in c {

[7.9350]

[7.9371]

        let mut len = 0;
        while let Some(c) = c.recv().await {
            if let State::Changes { ref mut hashes, .. } = *self.state.lock().await {
                hashes.push(c);
            }

Insertion in pijul/src/remote/ssh.rs at line 845 [5.25380]
[7.9743]
[7.9743]
```
            len += 1;
```

Replacement in pijul/src/remote/ssh.rs at line 847 [5.25380]

B:BD[7.9753] → [10.29:86]

        let progress = ProgressBar::new(c.len() as u64);

[7.9753]

[11.0]

        let progress = ProgressBar::new(len as u64);

Replacement in pijul/src/remote/ssh.rs at line 876 [5.25380]
B:BD[5.49753] → [7.9928:9964]
```
        debug!("done downloading");
```
[5.49753]
[5.52441]
```
        debug!("done downloading {:?}", changes_dir);
```

Replacement in pijul/src/remote/mod.rs at line 7 [5.52474]

∅:D[12.393] → [3.516:602]

B:BD[4.439] → [3.516:602]

use libpijul::pristine::{Base32, ChannelRef, Hash, Merkle, MutTxnT, RemoteRef, TxnT};

[12.393]

[5.52599]

use libpijul::pristine::{Base32, ChannelRef, GraphIter, Hash, Merkle, MutTxnT, RemoteRef, TxnT};

Replacement in pijul/src/remote/mod.rs at line 350 [5.52474]

B:BD[5.66710] → [7.10655:10700]

        hashes: &[libpijul::pristine::Hash],

[5.66710]

[7.10700]

        hashes: &mut tokio::sync::mpsc::Receiver<libpijul::pristine::Hash>,

Deletion in pijul/src/remote/mod.rs at line 363 [5.52474]
B:BD[5.68102] → [5.68102:68165]
```
        libpijul::changestore::filesystem::pop_filename(path);
```

Insertion in pijul/src/remote/mod.rs at line 376 [5.52474]

[5.69053]


        let mut self_ = std::mem::replace(self, RemoteRepo::None);
        let (hash_send, mut hash_recv) = tokio::sync::mpsc::channel(100);
        let mut change_path_ = repo.path.clone();
        change_path_.push(DOT_DIR);
        change_path_.push("changes");
        let t = tokio::spawn(async move {
            self_
                .download_changes(&mut hash_recv, &mut send, &mut change_path_, false)
                .await?;
            Ok::<_, anyhow::Error>(self_)
        });

Replacement in pijul/src/remote/mod.rs at line 390 [5.52474]
B:BD[5.69111] → [13.188:231]
```
        let mut to_download_ = Vec::new();
```
[5.69111]
[13.231]
```
        let mut len = 0;
```

Replacement in pijul/src/remote/mod.rs at line 394 [5.52474]

B:BD[13.402] → [13.402:440]

                to_download_.push(*h)

[13.402]

[13.440]

                hash_send.send(*h).await?;
                len += 1

Deletion in pijul/src/remote/mod.rs at line 399 [5.52474]
B:BD[13.544] → [13.544:592]
```
        let to_download = to_download_.clone();
```

Deletion in pijul/src/remote/mod.rs at line 400 [5.52474]

B:BD[14.1] → [14.1:293]

∅:D[14.293] → [5.69754:69755]

∅:D[7.11284] → [5.69754:69755]

B:BD[5.69754] → [5.69754:69755]

        let mut self_ = std::mem::replace(self, RemoteRepo::None);
        let t = tokio::spawn(async move {
            self_
                .download_changes(&to_download_, &mut send, &mut change_path_, false)
                .await?;
            Ok::<_, anyhow::Error>(self_)
        });

Replacement in pijul/src/remote/mod.rs at line 403 [5.52474]

B:BD[10.611] → [10.611:686]

            let p = indicatif::ProgressBar::new(to_download.len() as u64);

[10.611]

[11.905]

            let p = indicatif::ProgressBar::new(len);

Replacement in pijul/src/remote/mod.rs at line 497 [5.52474]

B:BD[5.71469] → [5.71469:71529]

B:BD[5.71529] → [7.11285:11427]

∅:D[7.11427] → [5.71732:71746]

B:BD[5.71732] → [5.71732:71746]

            while let Some(hash) = recv_hash.recv().await {
                self_
                    .download_changes(&[hash], &mut send_signal, &mut change_path_, false)
                    .await?;
            }

[5.71428]

[5.71908]

            self_
                .download_changes(&mut recv_hash, &mut send_signal, &mut change_path_, false)
                .await?;

Replacement in pijul/src/remote/mod.rs at line 560 [5.52474]

B:BD[5.74770] → [3.798:855]

    pub async fn complete_changes<T: MutTxnT + TxnTExt>(

[5.74770]

[5.74830]

    pub async fn complete_changes<T: MutTxnT + TxnTExt + GraphIter>(

Insertion in pijul/src/remote/mod.rs at line 568 [5.52474]
[5.75038]
[5.75038]
```
        debug!("complete changes {:?}", changes);
```

Replacement in pijul/src/remote/mod.rs at line 575 [5.52474]

B:BD[5.75446] → [5.75446:75562]

B:BD[5.75562] → [7.11428:11562]

∅:D[7.11562] → [5.75825:75839]

B:BD[5.75825] → [5.75825:75839]

B:BD[5.75983] → [5.75983:76064]

            while let Some(h) = recv_hash.recv().await {
                debug!("downloading full patch: {:?}", h);
                self_
                    .download_changes(&[h], &mut send_sig, &mut changes_dir, true)
                    .await?;
            }
            let result: Result<_, anyhow::Error> = Ok(self_);
            result

[5.75405]

[5.76064]

            self_
                .download_changes(&mut recv_hash, &mut send_sig, &mut changes_dir, true)
                .await?;
            Ok::<_, anyhow::Error>(self_)

Insertion in pijul/src/remote/mod.rs at line 599 [5.52474]

[5.76547]

                debug!("could not find internal for {:?}", sc);

Replacement in pijul/src/remote/mod.rs at line 609 [5.52474]

B:BD[5.76925] → [6.8315:8360]

∅:D[6.8360] → [3.899:942]

∅:D[15.9672] → [3.899:942]

B:BD[3.899] → [3.899:942]

            if txn.is_alive(&channel, &v)? {
                send_hash.send(*c).await?;

[5.76925]

[5.77531]

            let graph = txn.graph(&channel);
            let mut it = txn.iter_graph(graph, Some(&v))?;
            while let Some(x) = txn.next_graph(&graph, &mut it) {
                let (v, e) = x?;
                if v.change > change {
                    break;
                } else if e.flag().is_alive_parent() {
                    send_hash.send(*c).await?;
                    break;
                }

Replacement in pijul/src/remote/local.rs at line 129 [5.79745]

B:BD[5.83512] → [7.11727:11767]

        c: &[libpijul::pristine::Hash],

[5.83512]

[7.11767]

        hashes: &mut tokio::sync::mpsc::Receiver<libpijul::pristine::Hash>,

Replacement in pijul/src/remote/local.rs at line 133 [5.79745]

B:BD[5.83607] → [7.11872:12057]

        for c in c {
            libpijul::changestore::filesystem::push_filename(&mut self.changes_dir, c);
            libpijul::changestore::filesystem::push_filename(&mut path, c);

[5.83607]

[7.12057]

        while let Some(c) = hashes.recv().await {
            libpijul::changestore::filesystem::push_filename(&mut self.changes_dir, &c);
            libpijul::changestore::filesystem::push_filename(&mut path, &c);

Replacement in pijul/src/remote/local.rs at line 149 [5.79745]
B:BD[7.12700] → [16.157:191]
```
            send.send(*c).await?;
```
[7.12700]
[17.3741]
```
            send.send(c).await?;
```

Replacement in pijul/src/remote/http.rs at line 74 [18.10]

B:BD[3.1135] → [3.1135:1180]

        hashes: &[libpijul::pristine::Hash],

[3.1135]

[3.1180]

        hashes: &mut tokio::sync::mpsc::Receiver<libpijul::pristine::Hash>,

Replacement in pijul/src/remote/http.rs at line 79 [18.10]

B:BD[3.1338] → [10.1254:1316]

        let progress = ProgressBar::new(hashes.len() as u64);

[3.1338]

[11.1282]

        let progress = ProgressBar::new(0 as u64);

Replacement in pijul/src/remote/http.rs at line 87 [18.10]
∅:D[12.981] → [18.432:458]
∅:D[3.1338] → [18.432:458]
∅:D[10.1365] → [18.432:458]
∅:D[11.1448] → [18.432:458]
∅:D[19.1538] → [18.432:458]
B:BD[18.432] → [18.432:458]
```
        for c in hashes {
```
[19.1538]
[14.322]
```
        while let Some(c) = hashes.recv().await {
            progress.inc_length(1);
```
Replacement in pijul/src/remote/http.rs at line 97 [18.10]
B:BD[19.1774] → [19.1774:1798]
```
                    *c,
```
[19.1774]
[19.1798]
```
                    c,
```

Replacement in libpijul/src/pristine/sanakirja.rs at line 552 [5.531056]

B:BD[20.38323] → [20.38323:38422]

B:BD[20.38422] → [6.69108:69176]

    fn iter_graph(&self, g: &Self::Graph) -> Result<Self::GraphCursor, TxnErr<Self::GraphError>> {
        Ok(::sanakirja::btree::cursor::Cursor::new(&self.txn, &g)?)

[20.38323]

[20.38539]

    fn iter_graph(
        &self,
        g: &Self::Graph,
        s: Option<&Vertex<ChangeId>>,
    ) -> Result<Self::GraphCursor, TxnErr<Self::GraphError>> {
        let mut c = ::sanakirja::btree::cursor::Cursor::new(&self.txn, &g)?;
        if let Some(s) = s {
            c.set(&self.txn, s, None)?;
        }
        Ok(c)

Replacement in libpijul/src/pristine/sanakirja.rs at line 1643 [5.531056]

B:BD[2.363] → [2.363:445]

                .map_err(|_| SanakirjaError::ChannelRc { c: name0.to_string() })?

[2.363]

[2.445]

                .map_err(|_| SanakirjaError::ChannelRc {
                    c: name0.to_string(),
                })?

Replacement in libpijul/src/pristine/mod.rs at line 350 [5.586748]

B:BD[20.51889] → [20.51889:51987]

    fn iter_graph(&self, g: &Self::Graph) -> Result<Self::GraphCursor, TxnErr<Self::GraphError>>;

[20.51889]

[6.102575]

    fn iter_graph(
        &self,
        g: &Self::Graph,
        s: Option<&Vertex<ChangeId>>,
    ) -> Result<Self::GraphCursor, TxnErr<Self::GraphError>>;

Replacement in libpijul/src/pristine/mod.rs at line 913 [5.586748]

B:BD[21.17878] → [20.54034:54090]

    let mut cursor = txn.iter_graph(&channel).unwrap();

[21.17878]

[3.45536]

    let mut cursor = txn.iter_graph(&channel, None).unwrap();

Replacement in libpijul/src/pristine/mod.rs at line 961 [5.586748]

B:BD[3.46782] → [20.54273:54329]

    let mut cursor = txn.iter_graph(&channel).unwrap();

[3.46782]

[20.54329]

    let mut cursor = txn.iter_graph(&channel, None).unwrap();

Insertion in libpijul/src/pristine/edge.rs at line 53 [5.648958]
[22.9044]
[22.9044]

Insertion in libpijul/src/pristine/edge.rs at line 55 [5.648958]

[22.9058]

    pub fn is_alive_parent(&self) -> bool {
        *self & (EdgeFlags::DELETED | EdgeFlags::PARENT) == EdgeFlags::PARENT
    }
    #[inline]