pijul nest
guest › [sign in] [explore]

Improved push/pull reporting

ammkrn
Jul 29, 2021, 10:15 PM
IVLLXQ5ZWZDKHO4TNQG3TPXN34H6Y2WXPAGSO4PWCYNSKUZWOEJQC

Dependencies

  • [2] OP77HLKN Pushing files in the correct order
  • [3] QZ77NIXC Actually downloading changelists from channels without an id
  • [4] EUZFFJSO Updating Pijul with the latest changes in Libpijul
  • [5] 5XMUEZMZ pijul-clone: avoid panics on parsing remote URLs
  • [6] K6GWUOD5 Styling progress bars
  • [7] 44BN7FWS Do not output files introduced by patches that were not applied during a push
  • [8] A3RM526Y Integrating identity malleability
  • [9] M5FK3ABT Complete dependencies when pushing and pulling
  • [10] BBKV6VMN Fixing push/pull messages, and do not reverse the changes to download/upload
  • [11] SXEYMYF7 Fixing the bad changes in history (unfortunately, by rebooting).
  • [12] HSVGP2G4 Version bump + formatting
  • [13] MU5GSJAW Partial push and pull (WARNING: breaks the existing protocol)
  • [14] I52XSRUH Massive cleanup, and simplification
  • [15] I24UEJQL Various post-fire fixes
  • [16] 5HF7C67M push/pull: fixed "changes" arguments
  • [17] 3AMEP2Y5 More convenient interface for channels
  • [18] 76PCXGML Pushing to, and pulling from the local repository
  • [19] GHO6DWPI Refactoring iterators
  • [20] BE7GUCI2 Completing dependencies only with changes the remote does not have
  • [21] 5OGOE4VW Store the current channel in the pristine
  • [22] VMPAOJS2 Don't output after pushing to a local channel
  • [23] TKEVOH7H Fixing a bug when downloading changes, and making change download more efficient (more async)
  • [24] CVLQ7DBF Committing the remotes even if we do not push anything
  • [25] YS2HLPX6 Don't propose an empty list of changes to push
  • [26] JL4WKA5P Implement the Sanakirja concurrency model in a cross-process way
  • [27] WZVCLZKY address clippy lints
  • [28] QMTANHVN Reset: only output changed files
  • [29] UDHP4ZVB Fixing SSH asynchronicity issues
  • [30] 5QTMRUXN Fixing a race condition between progress bars
  • [31] ZBNKSYA6 Fixing a bus error when starting a transaction on a full disk
  • [32] YN63NUZO Sanakirja 1.0
  • [33] L2VH4BYK Downloading changelists from channels without an id (Nest discussions)
  • [34] Q45QHPO4 Feedback on network stuff
  • [35] 367UBQ6K Forwarding SSH stderr, and progress bar for push
  • [36] L4JXJHWX pijul/*: reorganize imports and remove extern crate
  • [37] CCLLB7OI Upgrading to Sanakirja 0.15 + version bump
  • [*] 2D7P2VKJ Change completions (where the whole progress bar story started)
  • [*] 4H2XTVJ2 Fix some mistakes in the docs

Change contents

  • replacement in pijul/src/remote/mod.rs at line 7
    [5.393]→[5.1031:1128](∅→∅)
    use libpijul::pristine::{Base32, ChannelRef, GraphIter, Hash, Merkle, MutTxnT, RemoteRef, TxnT};
    [5.393]
    [5.52599]
    use libpijul::pristine::{
    sanakirja::MutTxn, Base32, ChangeId, ChannelRef, GraphIter, Hash, Merkle, MutTxnT, RemoteRef,
    TxnT,
    };
  • replacement in pijul/src/remote/mod.rs at line 12
    [5.52622]→[5.52622:52659](∅→∅)
    use libpijul::{MutTxnTExt, TxnTExt};
    [5.52622]
    [5.549]
    use libpijul::{ChannelTxnT, DepsTxnT, GraphTxnT, MutTxnTExt, TxnTExt};
  • edit in pijul/src/remote/mod.rs at line 112
    [5.8343]
    [5.1063]
    }
    }
    // Extracting this saves a little bit of duplication.
    fn get_local_inodes(
    txn: &mut MutTxn<()>,
    channel: &ChannelRef<MutTxn<()>>,
    repo: &Repository,
    path: &[String],
    ) -> Result<HashSet<Position<ChangeId>>, anyhow::Error> {
    let mut paths = HashSet::new();
    for path in path.iter() {
    let (p, ambiguous) = txn.follow_oldest_path(&repo.changes, &channel, path)?;
    if ambiguous {
    bail!("Ambiguous path: {:?}", path)
    }
    paths.insert(p);
    paths.extend(
    libpijul::fs::iter_graph_descendants(txn, &channel.read().graph, p)?
    .map(|x| x.unwrap()),
    );
    }
    Ok(paths)
    }
    /// Embellished [`RemoteDelta`] that has information specific
    /// to a push operation. We want to know what our options are
    /// for changes to upload, whether the remote has unrecorded relevant changes,
    /// and whether the remote has changes we don't know about, since those might
    /// effect whether or not we actually want to go through with the push.
    pub(crate) struct PushDelta<T: MutTxnTExt + TxnTExt> {
    pub to_upload: Vec<Hash>,
    pub remote_ref: Option<RemoteRef<T>>,
    pub remote_unrecs: Vec<(u64, Hash)>,
    pub unknown_changes: Vec<Hash>,
    }
    /// For a [`RemoteRepo`] that's Local, Ssh, or Http
    /// (anything other than a LocalChannel),
    /// [`RemoteDelta`] contains data about the difference between
    /// the "actual" state of the remote ('theirs') and the last version of it
    /// that we cached ('ours'). The dichotomy is the last point at which the two
    /// were the same. `remote_unrecs` is a list of changes which used to be
    /// present in the remote, AND were present in the current channel we're
    /// pulling to or pushing from. The significance of that is that if we knew
    /// about a certain change but did not pull it, the user won't be notified
    /// if it's unrecorded in the remote.
    ///
    /// If the remote we're pulling from or pushing to is a LocalChannel,
    /// (meaning it's just a different channel of the repo we're already in), then
    /// `ours_ge_dichotomy`, `theirs_ge_dichotomy`, and `remote_unrecs` will be empty
    /// since they have no meaning. If we're pulling from a LocalChannel,
    /// there's no cache to have diverged from, and if we're pushing to a LocalChannel,
    /// we're not going to suddenly be surprised by the presence of unknown changes.
    ///
    /// This struct will be created by both a push and pull operation since both
    /// need to update the changelist and will at least try to update the local
    /// remote cache. For a push, this later gets turned into [`PushDelta`].
    pub(crate) struct RemoteDelta<T: MutTxnTExt + TxnTExt> {
    pub inodes: HashSet<Position<Hash>>,
    pub to_download: Vec<Hash>,
    pub remote_ref: Option<RemoteRef<T>>,
    pub ours_ge_dichotomy_set: HashSet<Hash>,
    pub theirs_ge_dichotomy_set: HashSet<Hash>,
    // Keep the Vec representation around as well so that notification
    // for unknown changes during shows the hashes in order.
    pub theirs_ge_dichotomy: Vec<(u64, Hash, Merkle)>,
    pub remote_unrecs: Vec<(u64, Hash)>,
    }
    impl RemoteDelta<MutTxn<()>> {
    /// Make a [`PushDelta`] from a [`RemoteDelta`]
    /// when the remote is a [`RemoteRepo::LocalChannel`].
    pub(crate) fn to_local_channel_push(
    self,
    remote_channel: &str,
    txn: &mut MutTxn<()>,
    path: &[String],
    channel: &ChannelRef<MutTxn<()>>,
    repo: &Repository,
    ) -> Result<PushDelta<MutTxn<()>>, anyhow::Error> {
    let mut to_upload = Vec::<Hash>::new();
    let inodes = get_local_inodes(txn, channel, repo, path)?;
    for x in txn.reverse_log(&*channel.read(), None)? {
    let (_, (h, _)) = x?;
    if let Some(channel) = txn.load_channel(remote_channel)? {
    let channel = channel.read();
    let h_int = txn.get_internal(h)?.unwrap();
    if txn.get_changeset(txn.changes(&channel), h_int)?.is_none() {
    if inodes.is_empty() {
    to_upload.push(h.into())
    } else {
    for p in inodes.iter() {
    if txn.get_touched_files(p, Some(h_int))?.is_some() {
    to_upload.push(h.into());
    break;
    }
    }
    }
    }
    }
    }
    assert!(self.ours_ge_dichotomy_set.is_empty());
    assert!(self.theirs_ge_dichotomy_set.is_empty());
    let d = PushDelta {
    to_upload: to_upload.into_iter().rev().collect(),
    remote_ref: self.remote_ref,
    remote_unrecs: self.remote_unrecs,
    unknown_changes: Vec::new(),
    };
    assert!(d.remote_unrecs.is_empty());
    Ok(d)
    }
    /// Make a [`PushDelta`] from a [`RemoteDelta`] when the remote
    /// is not a LocalChannel.
    pub(crate) fn to_remote_push(
    self,
    txn: &mut MutTxn<()>,
    path: &[String],
    channel: &ChannelRef<MutTxn<()>>,
    repo: &Repository,
    ) -> Result<PushDelta<MutTxn<()>>, anyhow::Error> {
    let mut to_upload = Vec::<Hash>::new();
    let inodes = get_local_inodes(txn, channel, repo, path)?;
    if let Some(ref remote_ref) = self.remote_ref {
    for x in txn.reverse_log(&*channel.read(), None)? {
    let (_, (h, m)) = x?;
    if txn.remote_has_state(remote_ref, &m)? {
    break;
    }
    let h_int = txn.get_internal(h)?.unwrap();
    let h_deser = Hash::from(h);
    // For elements that are in the uncached remote changes (theirs_ge_dichotomy),
    // don't put those in to_upload since the remote we're pushing to
    // already has those changes.
    if !txn.remote_has_change(remote_ref, &h)?
    && !self.theirs_ge_dichotomy_set.contains(&h_deser)
    {
    if inodes.is_empty() {
    to_upload.push(h_deser)
    } else {
    for p in inodes.iter() {
    if txn.get_touched_files(p, Some(h_int))?.is_some() {
    to_upload.push(h_deser);
    break;
    }
    }
    }
    }
    }
    }
    // { h | h \in theirs_ge_dichotomy /\ ~(h \in ours_ge_dichotomy) }
    // The set of their changes >= dichotomy that aren't
    // already known to our set of changes after the dichotomy.
    let unknown_changes = self
    .theirs_ge_dichotomy
    .iter()
    .filter_map(|(_, h, _)| {
    if self.ours_ge_dichotomy_set.contains(h) {
    None
    } else {
    Some(*h)
    }
    })
    .collect::<Vec<Hash>>();
    Ok(PushDelta {
    to_upload: to_upload.into_iter().rev().collect(),
    remote_ref: self.remote_ref,
    remote_unrecs: self.remote_unrecs,
    unknown_changes,
    })
  • edit in pijul/src/remote/mod.rs at line 290
    [5.55184]
    [5.55184]
    /// Create a [`RemoteDelta`] for a [`RemoteRepo::LocalChannel`].
    /// Since this case doesn't have a local remote cache to worry about,
    /// mainly just calculates the `to_download` list of changes.
    pub(crate) fn update_changelist_local_channel(
    remote_channel: &str,
    txn: &mut MutTxn<()>,
    path: &[String],
    current_channel: &ChannelRef<MutTxn<()>>,
    repo: &Repository,
    specific_changes: &[String],
    ) -> Result<RemoteDelta<MutTxn<()>>, anyhow::Error> {
    if !specific_changes.is_empty() {
    let to_download: Result<Vec<libpijul::Hash>, anyhow::Error> = specific_changes
    .iter()
    .map(|h| Ok(txn.hash_from_prefix(h)?.0))
    .collect();
    Ok(RemoteDelta {
    inodes: HashSet::new(),
    to_download: to_download?,
    remote_ref: None,
    ours_ge_dichotomy_set: HashSet::new(),
    theirs_ge_dichotomy: Vec::new(),
    theirs_ge_dichotomy_set: HashSet::new(),
    remote_unrecs: Vec::new(),
    })
    } else {
    let mut inodes = HashSet::new();
    let inodes_ = get_local_inodes(txn, current_channel, repo, path)?;
    let mut to_download = Vec::new();
    inodes.extend(inodes_.iter().map(|x| libpijul::pristine::Position {
    change: txn.get_external(&x.change).unwrap().unwrap().into(),
    pos: x.pos,
    }));
    if let Some(remote_channel) = txn.load_channel(remote_channel)? {
    let remote_channel = remote_channel.read();
    for x in txn.reverse_log(&remote_channel, None)? {
    let (h, m) = x?.1;
    if txn
    .channel_has_state(txn.states(&*current_channel.read()), &m)?
    .is_some()
    {
    break;
    }
    let h_int = txn.get_internal(h)?.unwrap();
    if txn
    .get_changeset(txn.changes(&*current_channel.read()), h_int)?
    .is_none()
    {
    if inodes_.is_empty()
    || inodes_.iter().any(|&inode| {
    txn.get_rev_touched_files(h_int, Some(&inode))
    .unwrap()
    .is_some()
    })
    {
    to_download.push(h.into())
    }
    }
    }
    }
    Ok(RemoteDelta {
    inodes,
    to_download,
    remote_ref: None,
    ours_ge_dichotomy_set: HashSet::new(),
    theirs_ge_dichotomy: Vec::new(),
    theirs_ge_dichotomy_set: HashSet::new(),
    remote_unrecs: Vec::new(),
    })
    }
    }
  • edit in pijul/src/remote/mod.rs at line 449
    [5.57522]
    [5.57522]
    /// Creates a [`RemoteDelta`].
    ///
    /// IF:
    /// the RemoteRepo is a [`RemoteRepo::LocalChannel`], delegate to
    /// the simpler method [`update_changelist_local_channel`], returning the
    /// `to_download` list of changes.
    ///
    /// ELSE:
    /// calculate the `to_download` list of changes. Additionally, if there are
    /// no remote unrecords, update the local remote cache. If there are remote unrecords,
    /// calculate and return information about the difference between our cached version
    /// of the remote, and their version of the remote.
    pub(crate) async fn update_changelist_pushpull(
    &mut self,
    txn: &mut MutTxn<()>,
    path: &[String],
    current_channel: &ChannelRef<MutTxn<()>>,
    force_cache: Option<bool>,
    repo: &Repository,
    specific_changes: &[String],
    ) -> Result<RemoteDelta<MutTxn<()>>, anyhow::Error> {
    debug!("update_changelist");
    if let RemoteRepo::LocalChannel(c) = self {
    return update_changelist_local_channel(
    c,
    txn,
    path,
    current_channel,
    repo,
    specific_changes,
    );
    }
    let id = self.get_id(txn).await?.unwrap();
    let mut remote_ref = if let Some(name) = self.name() {
    txn.open_or_create_remote(id, name).unwrap()
    } else {
    unreachable!()
    };
    let dichotomy_n = self
    .dichotomy_changelist(txn, &remote_ref.lock().remote)
    .await?;
    let ours_ge_dichotomy: Vec<(u64, Hash)> = txn
    .iter_remote(&remote_ref.lock().remote, dichotomy_n)?
    .filter_map(|k| {
    debug!("filter_map {:?}", k);
    match k.unwrap() {
    (k, libpijul::pristine::Pair { a: hash, .. }) => {
    let (k, hash) = (u64::from(*k), Hash::from(*hash));
    if k >= dichotomy_n {
    Some((k, hash))
    } else {
    None
    }
    }
    }
    })
    .collect();
    let (inodes, theirs_ge_dichotomy) =
    self.download_changelist_nocache(dichotomy_n, path).await?;
    let ours_ge_dichotomy_set = ours_ge_dichotomy
    .iter()
    .map(|(_, h)| h)
    .copied()
    .collect::<HashSet<Hash>>();
    let theirs_ge_dichotomy_set = theirs_ge_dichotomy
    .iter()
    .map(|(_, h, _)| h)
    .copied()
    .collect::<HashSet<Hash>>();
    // remote_unrecs = {x: (u64, Hash) | x \in ours_ge_dichot /\ ~(x \in theirs_ge_dichot) /\ x \in current_channel }
    let mut remote_unrecs = Vec::new();
    for (n, hash) in &ours_ge_dichotomy {
    if theirs_ge_dichotomy_set.contains(hash) {
    // If this change is still present in the remote, skip
    continue;
    } else if txn.get_revchanges(&current_channel, &hash)?.is_none() {
    // If this unrecord wasn't in our current channel, skip
    continue;
    } else {
    remote_unrecs.push((*n, *hash))
    }
    }
    let should_cache = force_cache.unwrap_or_else(|| remote_unrecs.is_empty());
    if should_cache {
    for (k, _) in ours_ge_dichotomy.iter().copied() {
    txn.del_remote(&mut remote_ref, k)?;
    }
    for (n, h, m) in theirs_ge_dichotomy.iter().copied() {
    txn.put_remote(&mut remote_ref, n, (h, m))?;
    }
    }
    let state_cond = |txn: &MutTxn<()>, merkle: &libpijul::pristine::SerializedMerkle| {
    txn.channel_has_state(txn.states(&*current_channel.read()), merkle)
    .map(|x| x.is_some())
    };
    let change_cond = |txn: &MutTxn<()>, hash: &Hash| {
    txn.get_revchanges(&current_channel, hash)
    .unwrap()
    .is_none()
    };
  • edit in pijul/src/remote/mod.rs at line 553
    [5.57523]
    [5.1663]
    // IF:
    // The user only wanted to push/pull specific changes
    // ELIF:
    // The user specified no changes and there were no remote unrecords
    // effecting the current channel means we can auto-cache
    // the local remote cache
    // ELSE:
    // The user specified no changes but there were remote unrecords
    // effecting the current channel meaning we can't auto-cache
    // the local remote cache.
    if !specific_changes.is_empty() {
    let to_download = specific_changes
    .iter()
    .map(|h| Ok(txn.hash_from_prefix(h)?.0))
    .collect::<Result<Vec<_>, anyhow::Error>>();
    Ok(RemoteDelta {
    inodes,
    remote_ref: Some(remote_ref),
    to_download: to_download?,
    ours_ge_dichotomy_set,
    theirs_ge_dichotomy,
    theirs_ge_dichotomy_set,
    remote_unrecs,
    })
    } else if should_cache {
    let mut to_download: Vec<Hash> = Vec::new();
    for thing in txn.iter_remote(&remote_ref.lock().remote, 0)? {
    let (_, libpijul::pristine::Pair { a: hash, b: merkle }) = thing?;
    if state_cond(txn, &merkle)? {
    break;
    } else if change_cond(txn, &hash.into()) {
    to_download.push(Hash::from(hash));
    }
    }
    Ok(RemoteDelta {
    inodes,
    remote_ref: Some(remote_ref),
    to_download,
    ours_ge_dichotomy_set,
    theirs_ge_dichotomy,
    theirs_ge_dichotomy_set,
    remote_unrecs,
    })
    } else {
    let mut to_download: Vec<Hash> = Vec::new();
    for thing in txn.iter_remote(&remote_ref.lock().remote, 0)? {
    let (n, libpijul::pristine::Pair { a: hash, b: merkle }) = thing?;
    if u64::from(*n) < dichotomy_n {
    if state_cond(txn, &merkle)? {
    continue;
    } else if change_cond(txn, &hash.into()) {
    to_download.push(Hash::from(hash));
    }
    }
    }
    for (_, hash, merkle) in &theirs_ge_dichotomy {
    if state_cond(txn, &merkle.into())? {
    continue;
    } else if change_cond(txn, &hash) {
    to_download.push(Hash::from(*hash));
    }
    }
    Ok(RemoteDelta {
    inodes,
    remote_ref: Some(remote_ref),
    to_download,
    ours_ge_dichotomy_set,
    theirs_ge_dichotomy,
    theirs_ge_dichotomy_set,
    remote_unrecs,
    })
    }
    }
    /// Get the list of the remote's changes that come after `from: u64`.
    /// Instead of immediately updating the local cache of the remote, return
    /// the change info without changing the cache.
    pub async fn download_changelist_nocache(
    &mut self,
    from: u64,
    paths: &[String],
    ) -> Result<(HashSet<Position<Hash>>, Vec<(u64, Hash, Merkle)>), anyhow::Error> {
    let mut v = Vec::new();
    let f = |v: &mut Vec<(u64, Hash, Merkle)>, n, h, m| {
    debug!("no cache: {:?}", h);
    Ok(v.push((n, h, m)))
    };
    let r = match *self {
    RemoteRepo::Local(ref mut l) => l.download_changelist(f, &mut v, from, paths)?,
    RemoteRepo::Ssh(ref mut s) => s.download_changelist(f, &mut v, from, paths).await?,
    RemoteRepo::Http(ref h) => h.download_changelist(f, &mut v, from, paths).await?,
    RemoteRepo::LocalChannel(_) => HashSet::new(),
    RemoteRepo::None => unreachable!(),
    };
    Ok((r, v))
    }
    /// Uses a binary search to find the integer identifier of the last point
    /// at which our locally cached version of the remote was the same as the 'actual'
    /// state of the remote.
  • edit in pijul/src/remote/mod.rs at line 799
    [3.7]→[3.7:757](∅→∅),[3.757]→[5.65140:65147](∅→∅),[5.65140]→[5.65140:65147](∅→∅)
    pub async fn download_changelist_nocache(
    &mut self,
    from: u64,
    paths: &[String],
    v: &mut Vec<Hash>,
    ) -> Result<HashSet<Position<Hash>>, anyhow::Error> {
    let f = |v: &mut Vec<Hash>, _n, h, _m| {
    debug!("no cache: {:?}", h);
    Ok(v.push(h))
    };
    let r = match *self {
    RemoteRepo::Local(ref mut l) => l.download_changelist(f, v, from, paths)?,
    RemoteRepo::Ssh(ref mut s) => s.download_changelist(f, v, from, paths).await?,
    RemoteRepo::Http(ref h) => h.download_changelist(f, v, from, paths).await?,
    RemoteRepo::LocalChannel(_) => HashSet::new(),
    RemoteRepo::None => unreachable!(),
    };
    Ok(r)
    }
  • edit in pijul/src/commands/pushpull.rs at line 10
    [5.111707]
    [5.12857]
    use libpijul::pristine::sanakirja::MutTxn;
  • edit in pijul/src/commands/pushpull.rs at line 16
    [39.4805]
    [5.1761]
    use crate::remote::{PushDelta, RemoteDelta, RemoteRepo};
  • edit in pijul/src/commands/pushpull.rs at line 78
    [5.113241]
    [40.1194]
    /// Force an update of the local remote cache. May effect some
    /// reporting of unrecords/concurrent changes in the remote.
    #[clap(long = "force-cache", short = 'f')]
    force_cache: bool,
  • edit in pijul/src/commands/pushpull.rs at line 109
    [5.113745]
    [40.1560]
    /// Force an update of the local remote cache. May effect some
    /// reporting of unrecords/concurrent changes in the remote.
    #[clap(long = "force-cache", short = 'f')]
    force_cache: bool,
  • edit in pijul/src/commands/pushpull.rs at line 137
    [5.114453]
    [5.114453]
    /// Gets the `to_upload` vector while trying to auto-update
    /// the local cache if possible. Also calculates whether the remote
    /// has any changes we don't know about.
    async fn to_upload(
    &self,
    txn: &mut MutTxn<()>,
    channel: &mut ChannelRef<MutTxn<()>>,
    repo: &Repository,
    remote: &mut RemoteRepo,
    ) -> Result<PushDelta<MutTxn<()>>, anyhow::Error> {
    let remote_delta = remote
    .update_changelist_pushpull(
    txn,
    &self.path,
    channel,
    Some(self.force_cache),
    repo,
    self.changes.as_slice(),
    )
    .await?;
    if let RemoteRepo::LocalChannel(ref remote_channel) = remote {
    remote_delta.to_local_channel_push(
    remote_channel,
    txn,
    self.path.as_slice(),
    channel,
    repo,
    )
    } else {
    remote_delta.to_remote_push(txn, self.path.as_slice(), channel, repo)
    }
    }
  • replacement in pijul/src/commands/pushpull.rs at line 172
    [5.1981]→[5.15226:15285](∅→∅)
    let repo = Repository::find_root(self.repo_path)?;
    [5.1981]
    [5.114570]
    let repo = Repository::find_root(self.repo_path.clone())?;
  • replacement in pijul/src/commands/pushpull.rs at line 215
    [5.115558]→[4.6921:7019](∅→∅)
    let remote_changes = remote
    .update_changelist(&mut *txn.write(), &self.path)
    [5.115558]
    [4.7019]
    let mut channel = txn.write().open_or_create_channel(&channel_name)?;
    let PushDelta {
    remote_ref,
    to_upload,
    remote_unrecs,
    unknown_changes,
    } = self
    .to_upload(&mut *txn.write(), &mut channel, &repo, &mut remote)
  • edit in pijul/src/commands/pushpull.rs at line 226
    [4.7040]→[4.7040:7114](∅→∅)
    let channel = txn.write().open_or_create_channel(&channel_name)?;
  • edit in pijul/src/commands/pushpull.rs at line 227
    [5.115889]→[5.7039:7118](∅→∅),[5.7118]→[4.7115:7245](∅→∅),[5.7207]→[5.116032:116059](∅→∅),[4.7245]→[5.116032:116059](∅→∅),[5.116032]→[5.116032:116059](∅→∅),[5.116059]→[5.7208:7260](∅→∅),[5.752]→[5.116148:116162](∅→∅),[5.7260]→[5.116148:116162](∅→∅),[5.116148]→[5.116148:116162](∅→∅),[5.116162]→[5.7261:7290](∅→∅),[5.7290]→[5.12957:12983](∅→∅),[5.12983]→[4.7246:7340](∅→∅),[4.7340]→[5.13071:13128](∅→∅),[5.15260]→[5.13071:13128](∅→∅),[5.13071]→[5.13071:13128](∅→∅),[5.13128]→[5.7445:7455](∅→∅),[5.7445]→[5.7445:7455](∅→∅),[5.7455]→[5.116227:116228](∅→∅),[5.116227]→[5.116227:116228](∅→∅),[5.116228]→[5.10462:10513](∅→∅),[5.10513]→[4.7341:7657](∅→∅),[4.7657]→[5.116707:116729](∅→∅),[5.116707]→[5.116707:116729](∅→∅),[5.15374]→[5.13573:13636](∅→∅),[5.5258]→[5.13573:13636](∅→∅),[5.13636]→[4.7658:7728](∅→∅),[5.428]→[5.7889:7935](∅→∅),[5.709]→[5.7889:7935](∅→∅),[4.7728]→[5.7889:7935](∅→∅),[5.13721]→[5.7889:7935](∅→∅),[5.5404]→[5.7889:7935](∅→∅),[5.7935]→[5.10836:10889](∅→∅),[5.10889]→[5.7981:8066](∅→∅),[5.7981]→[5.7981:8066](∅→∅),[5.8066]→[5.10890:11038](∅→∅),[5.11038]→[5.8207:8284](∅→∅),[5.8207]→[5.8207:8284](∅→∅),[5.8284]→[5.5588:5618](∅→∅),[5.5588]→[5.5588:5618](∅→∅),[5.5697]→[5.5697:5745](∅→∅),[5.5745]→[4.7729:8671](∅→∅),[5.5745]→[5.116792:116824](∅→∅),[4.8671]→[5.116792:116824](∅→∅),[5.116792]→[5.116792:116824](∅→∅),[5.116895]→[5.116895:116905](∅→∅)
    let mut paths = HashSet::new();
    for path in self.path.iter() {
    let (p, ambiguous) = txn
    .read()
    .follow_oldest_path(&repo.changes, &channel, path)?;
    if ambiguous {
    bail!("Ambiguous path: {:?}", path)
    }
    paths.insert(p);
    paths.extend(
    libpijul::fs::iter_graph_descendants(&*txn.read(), &channel.read().graph, p)?
    .map(|x| x.unwrap()),
    );
    }
    let mut to_upload: Vec<Hash> = Vec::new();
    {
    let txn = txn.read();
    for x in txn.reverse_log(&*channel.read(), None)? {
    let (_, (h, m)) = x?;
    if let Some((_, ref remote_changes)) = remote_changes {
    if txn.remote_has_state(remote_changes, &m)? {
    break;
    }
    let h_int = txn.get_internal(h)?.unwrap();
    if !txn.remote_has_change(&remote_changes, &h)? {
    if paths.is_empty() {
    to_upload.push(h.into())
    } else {
    for p in paths.iter() {
    if txn.get_touched_files(p, Some(h_int))?.is_some() {
    to_upload.push(h.into());
    break;
    }
    }
    }
    }
    } else if let crate::remote::RemoteRepo::LocalChannel(ref remote_channel) = remote {
    if let Some(channel) = txn.load_channel(remote_channel)? {
    let channel = channel.read();
    let h_int = txn.get_internal(h)?.unwrap();
    if txn.get_changeset(txn.changes(&channel), h_int)?.is_none() {
    if paths.is_empty() {
    to_upload.push(h.into())
    } else {
    for p in paths.iter() {
    if txn.get_touched_files(p, Some(h_int))?.is_some() {
    to_upload.push(h.into());
    break;
    }
    }
    }
    }
    }
    }
    }
    }
  • replacement in pijul/src/commands/pushpull.rs at line 234
    [5.72]→[2.0:29](∅→∅)
    to_upload.reverse();
    [5.72]
    [5.72]
    notify_remote_unrecords(&repo, remote_unrecs.as_slice());
    notify_unknown_changes(unknown_changes.as_slice());
  • edit in pijul/src/commands/pushpull.rs at line 267
    [5.77]→[5.8346:8408](∅→∅),[5.8408]→[4.8707:8741](∅→∅)
    let remote_changes = remote_changes.map(|x| x.1);
    let txn = txn.read();
  • replacement in pijul/src/commands/pushpull.rs at line 269
    [5.158]→[4.8742:8840](∅→∅)
    let comp = complete_deps(&*txn, &remote_changes, &repo.changes, &to_upload, &d)?;
    [5.158]
    [5.232]
    let comp = complete_deps(&*txn.read(), &remote_ref, &repo.changes, &to_upload, &d)?;
  • edit in pijul/src/commands/pushpull.rs at line 300
    [5.117527]
    [5.117527]
    /// Gets the `to_download` vec and calculates any remote unrecords.
    /// If the local remote cache can be auto-updated, it will be.
    async fn to_download(
    &self,
    txn: &mut MutTxn<()>,
    channel: &mut ChannelRef<MutTxn<()>>,
    repo: &mut Repository,
    remote: &mut RemoteRepo,
    ) -> Result<RemoteDelta<MutTxn<()>>, anyhow::Error> {
    let force_cache = if self.force_cache {
    Some(self.force_cache)
    } else {
    None
    };
    let delta = remote
    .update_changelist_pushpull(
    txn,
    &self.path,
    channel,
    force_cache,
    repo,
    self.changes.as_slice(),
    )
    .await?;
    let to_download = remote
    .pull(
    repo,
    txn,
    channel,
    delta.to_download.as_slice(),
    &delta.inodes,
    false,
    )
    .await?;
    Ok(RemoteDelta {
    to_download,
    ..delta
    })
    }
  • replacement in pijul/src/commands/pushpull.rs at line 342
    [5.117585]→[5.15309:15372](∅→∅)
    let mut repo = Repository::find_root(self.repo_path)?;
    [5.117585]
    [4.9020]
    let mut repo = Repository::find_root(self.repo_path.clone())?;
  • replacement in pijul/src/commands/pushpull.rs at line 380
    [5.118434]→[5.8446:8542](∅→∅),[5.8542]→[4.9333:9431](∅→∅)
    let mut inodes: HashSet<libpijul::pristine::Position<libpijul::Hash>> = HashSet::new();
    let remote_changes = remote
    .update_changelist(&mut *txn.write(), &self.path)
    [5.118434]
    [4.9431]
    let RemoteDelta {
    inodes,
    remote_ref,
    mut to_download,
    remote_unrecs,
    ..
    } = self
    .to_download(&mut *txn.write(), &mut channel, &mut repo, &mut remote)
  • replacement in pijul/src/commands/pushpull.rs at line 389
    [4.9452]→[5.8542:8601](∅→∅),[5.15480]→[5.8542:8601](∅→∅),[5.8542]→[5.8542:8601](∅→∅),[5.2129]→[5.118721:118760](∅→∅),[5.8689]→[5.118721:118760](∅→∅),[5.118721]→[5.118721:118760](∅→∅),[5.118760]→[5.11097:11154](∅→∅),[5.11154]→[5.15481:15560](∅→∅),[5.15560]→[5.8760:8812](∅→∅),[5.8760]→[5.8760:8812](∅→∅),[5.8812]→[4.9453:9569](∅→∅),[4.9569]→[5.11155:11199](∅→∅),[5.15454]→[5.11155:11199](∅→∅),[5.13890]→[5.11155:11199](∅→∅),[5.11199]→[5.0:27](∅→∅),[5.13929]→[5.0:27](∅→∅),[5.27]→[4.9570:9650](∅→∅),[4.9650]→[5.105:162](∅→∅),[5.11281]→[5.105:162](∅→∅),[5.15536]→[5.105:162](∅→∅),[5.105]→[5.105:162](∅→∅),[5.162]→[5.6416:6447](∅→∅),[5.2918]→[5.6416:6447](∅→∅),[5.13990]→[5.6416:6447](∅→∅),[5.6416]→[5.6416:6447](∅→∅),[5.6447]→[5.11282:11420](∅→∅),[5.11420]→[5.6565:6702](∅→∅),[5.6565]→[5.6565:6702](∅→∅),[5.6702]→[5.8813:8863](∅→∅),[5.8863]→[4.9651:9689](∅→∅),[4.9689]→[5.8863:9007](∅→∅),[5.8863]→[5.8863:9007](∅→∅),[5.9007]→[5.6861:6896](∅→∅),[5.6861]→[5.6861:6896](∅→∅),[5.6896]→[5.9008:9068](∅→∅),[5.836]→[5.6993:7015](∅→∅),[5.9068]→[5.6993:7015](∅→∅),[5.6993]→[5.6993:7015](∅→∅),[5.7015]→[5.9069:9108](∅→∅),[5.9108]→[5.14067:14103](∅→∅),[5.14103]→[4.9690:9785](∅→∅),[4.9785]→[5.14199:14272](∅→∅),[5.15632]→[5.14199:14272](∅→∅),[5.14199]→[5.14199:14272](∅→∅),[5.14272]→[5.9305:9407](∅→∅),[5.9305]→[5.9305:9407](∅→∅),[5.9407]→[5.11421:11503](∅→∅),[5.11503]→[5.9472:9525](∅→∅),[5.14347]→[5.9472:9525](∅→∅),[5.9472]→[5.9472:9525](∅→∅),[5.9525]→[5.14348:14430](∅→∅),[5.14430]→[4.9786:9850](∅→∅),[4.9850]→[5.14431:14545](∅→∅),[5.15698]→[5.14431:14545](∅→∅),[5.7259]→[5.14431:14545](∅→∅),[5.14545]→[5.163:194](∅→∅),[5.194]→[4.9851:9933](∅→∅),[4.9933]→[5.276:341](∅→∅),[5.11587]→[5.276:341](∅→∅),[5.15782]→[5.276:341](∅→∅),[5.276]→[5.276:341](∅→∅),[5.341]→[5.7403:7464](∅→∅),[5.3003]→[5.7403:7464](∅→∅),[5.14610]→[5.7403:7464](∅→∅),[5.7403]→[5.7403:7464](∅→∅),[5.7464]→[5.14611:14678](∅→∅),[5.14678]→[5.7530:7561](∅→∅),[5.7530]→[5.7530:7561](∅→∅),[5.7561]→[4.9934:10016](∅→∅),[5.512]→[5.7644:7709](∅→∅),[5.794]→[5.7644:7709](∅→∅),[4.10016]→[5.7644:7709](∅→∅),[5.14763]→[5.7644:7709](∅→∅),[5.15866]→[5.7644:7709](∅→∅),[5.7644]→[5.7644:7709](∅→∅),[5.7709]→[5.9526:9641](∅→∅),[5.9641]→[5.11588:11671](∅→∅),[5.11671]→[5.14846:14947](∅→∅),[5.14846]→[5.14846:14947](∅→∅),[5.14947]→[5.9733:9798](∅→∅),[5.9733]→[5.9733:9798](∅→∅),[5.9798]→[5.11672:11731](∅→∅),[5.11731]→[5.8030:8108](∅→∅),[5.8030]→[5.8030:8108](∅→∅),[5.8108]→[5.119084:119102](∅→∅),[5.119084]→[5.119084:119102](∅→∅),[5.119102]→[3.758:921](∅→∅),[3.921]→[5.119102:119116](∅→∅),[5.119102]→[5.119102:119116](∅→∅),[5.119151]→[5.119151:119192](∅→∅),[5.119192]→[4.10017:10051](∅→∅),[4.10051]→[5.2508:2577](∅→∅),[5.119192]→[5.2508:2577](∅→∅),[5.2577]→[5.119271:119431](∅→∅),[5.119271]→[5.119271:119431](∅→∅),[5.119431]→[5.9799:9828](∅→∅)
    let mut to_download = if self.changes.is_empty() {
    debug!("changelist done");
    let mut to_download: Vec<Hash> = Vec::new();
    if let Some((inodes_, remote_changes)) = remote_changes.as_ref() {
    inodes.extend(inodes_.into_iter());
    let txn = txn.read();
    for x in txn.iter_remote(&remote_changes.lock().remote, 0)? {
    let p = x?.1; // (h, m)
    if txn
    .channel_has_state(txn.states(&*channel.read()), &p.b)?
    .is_some()
    {
    break;
    } else if txn.get_revchanges(&channel, &p.a.into())?.is_none() {
    to_download.push(p.a.into())
    }
    }
    } else if let crate::remote::RemoteRepo::LocalChannel(ref remote_channel) = remote {
    let mut inodes_ = HashSet::new();
    let txn = txn.read();
    for path in self.path.iter() {
    let (p, ambiguous) = txn.follow_oldest_path(&repo.changes, &channel, path)?;
    if ambiguous {
    bail!("Ambiguous path: {:?}", path)
    }
    inodes_.insert(p);
    inodes_.extend(
    libpijul::fs::iter_graph_descendants(&*txn, &channel.read().graph, p)?
    .map(|x| x.unwrap()),
    );
    }
    inodes.extend(inodes_.iter().map(|x| libpijul::pristine::Position {
    change: txn.get_external(&x.change).unwrap().unwrap().into(),
    pos: x.pos,
    }));
    if let Some(remote_channel) = txn.load_channel(remote_channel)? {
    let remote_channel = remote_channel.read();
    for x in txn.reverse_log(&remote_channel, None)? {
    let (h, m) = x?.1;
    if txn
    .channel_has_state(txn.states(&*channel.read()), &m)?
    .is_some()
    {
    break;
    }
    let h_int = txn.get_internal(h)?.unwrap();
    if txn
    .get_changeset(txn.changes(&*channel.read()), h_int)?
    .is_none()
    {
    if inodes_.is_empty()
    || inodes_.iter().any(|&inode| {
    txn.get_rev_touched_files(h_int, Some(&inode))
    .unwrap()
    .is_some()
    })
    {
    to_download.push(h.into())
    }
    }
    }
    }
    } else {
    inodes = remote
    .download_changelist_nocache(0, &self.path, &mut to_download)
    .await?
    }
    to_download
    } else {
    let txn = txn.read();
    let r: Result<Vec<libpijul::Hash>, anyhow::Error> = self
    .changes
    .iter()
    .map(|h| Ok(txn.hash_from_prefix(h)?.0))
    .collect();
    r?
    };
    debug!("recording");
    [4.9452]
    [5.15913]
  • replacement in pijul/src/commands/pushpull.rs at line 391
    [5.16142]→[5.0:37](∅→∅),[5.9899]→[5.0:37](∅→∅),[5.37]→[5.9932:9951](∅→∅),[5.9932]→[5.9932:9951](∅→∅),[5.15614]→[5.9951:9978](∅→∅),[5.9951]→[5.9951:9978](∅→∅),[5.9978]→[4.10052:10087](∅→∅),[4.10087]→[5.10004:10154](∅→∅),[5.10004]→[5.10004:10154](∅→∅),[5.10154]→[5.2534:2585](∅→∅)
    let mut to_download = remote
    .pull(
    &mut repo,
    &mut *txn.write(),
    &mut channel,
    &mut to_download,
    &inodes,
    self.all,
    )
    .await?;
    if let Some((_, ref r)) = remote_changes {
    [5.15987]
    [5.2585]
    if let Some(ref r) = remote_ref {
  • edit in pijul/src/commands/pushpull.rs at line 395
    [5.2654]
    [5.10154]
    notify_remote_unrecords(&repo, remote_unrecs.as_slice());
  • edit in pijul/src/commands/pushpull.rs at line 418
    [5.808]
    [5.120336]
    }
    {
    // Now that .pull is always given `false` for `do_apply`...
  • edit in pijul/src/commands/pushpull.rs at line 438
    [5.120589]
    [5.120589]
  • edit in pijul/src/commands/pushpull.rs at line 597
    [5.123266]
    fn notify_remote_unrecords(repo: &Repository, remote_unrecs: &[(u64, Hash)]) {
    use std::fmt::Write;
    if !remote_unrecs.is_empty() {
    let mut s = format!(
    "# The following changes have been unrecorded in the remote.\n\
    # This buffer is only being used to inform you of the remote change;\n\
    # your push will continue when it is closed.\n"
    );
    for (_, hash) in remote_unrecs {
    let header = &repo.changes.get_change(hash).unwrap().header;
    s.push_str("#\n");
    writeln!(&mut s, "# {}", header.message).expect("Infallible write to String");
    writeln!(&mut s, "# {}", header.timestamp).expect("Infallible write to String");
    writeln!(&mut s, "# {}", hash.to_base32()).expect("Infallible write to String");
    }
    if let Err(e) = edit::edit(s.as_str()) {
    log::error!(
    "Notification of remote unrecords experienced an error: {}",
    e
    );
    }
    }
    }
    fn notify_unknown_changes(unknown_changes: &[Hash]) {
    use std::fmt::Write;
    if unknown_changes.is_empty() {
    return;
    } else {
    let mut s = format!(
    "# The following changes are new in the remote\n# (and are not yet known to your local copy):\n#\n"
    );
    let rest_len = unknown_changes.len().saturating_sub(5);
    for hash in unknown_changes.iter().take(5) {
    writeln!(&mut s, "# {}", hash.to_base32()).expect("Infallible write to String");
    }
    if rest_len > 0 {
    let plural = if rest_len == 1 { "" } else { "s" };
    writeln!(&mut s, "# ... plus {} more change{}", rest_len, plural)
    .expect("Infallible write to String");
    }
    if let Err(e) = edit::edit(s.as_str()) {
    log::error!(
    "Notification of unknown changes experienced an error: {}",
    e
    );
    }
    }
    }