Import Git repos without writing anything to disk

pmeunier

Oct 12, 2022, 11:42 AM

RP7YRM5QOINW7FB5BGNZEXVQN7TXB7LLLHXZPPAI6DGFGRP7JPFQC

Dependencies

• [2] 5DUMO2IC Better `pijul git` error message (when recording)
• [3] 3SJ3DJNF Adding a --force option to `pijul add`, to include otherwise ignored files
• [4] UC5C5REV `pijul git`: two more fields in the benchmarks
• [5] XFYALEYN When importing initial Git commits, use walk rather than iter to walk the tree
• [6] 2RXOCWUW Making libpijul deterministic (and getting rid of `rand`)
• [7] A3DMBJJA Upgrading the `git` subcommand to the latest Sanakirja and Libpijul
• [8] RHHNPMZI Fixing `pijul git`
• [9] 5SLOJYHG Fixing the Git feature
• [10] LGEJSLTY Fixing output (including its uses in reset and pull)
• [11] V435QOJR Using path-slash to fix path issues on Windows
• [12] CVAT6LN3 Fixing git import, and adding more useful feedback (with `RUST_LOG="pijul=info"`)
• [13] S7MPXAPH Removing warnings in pijul::commands::git
• [14] VO5OQW4W Removing anyhow in libpijul
• [15] L4JXJHWX pijul/*: reorganize imports and remove extern crate
• [16] SXEYMYF7 Fixing the bad changes in history (unfortunately, by rebooting).
• [17] PIQCNEEB Upgrading to Clap 3.0.0-alpha.5
• [18] YVA72CP2 Default feedback for the `pijul git` command
• [19] LOJL4HMY Recording a prefix we just deleted (mostly useful in `pijul git`, regular records returned NotFound)
• [20] SFY4U6XE remove redundant conditional compilation
• [21] ZDK3GNDB Tag transactions (including a massive refactoring of errors)
• [22] VU4KVXHW Git import was importing parents and hidden files (including .git), in some cases
• [23] SNZ3OAMC use native external subcommand support instead of hand-rolled one
• [24] R3H7D42U Debugging `pijul git`: proper error reporting
• [25] BJOZ25EU Deterministic Git import
• [26] OJZWJUF2 MUCH faster `pijul add -r`
• [27] YGPEHOTE libpijul::fs::add_{file,dir} now return the created Inode
• [28] EUZFFJSO Updating Pijul with the latest changes in Libpijul
• [29] GUL4M5FI Cleanup and formatting
• [30] ZHABNS3S Canonicalize all paths
• [31] I52XSRUH Massive cleanup, and simplification
• [32] 46XERN6R Fixing "addition error" in `pijul git`
• [*] LCERQSWM Cleanup

Change contents

• replacement in pijul/src/main.rs at line 140
  [6.20]→[6.29:52](∅→∅)
  − env_logger_init();
  [6.20]

  [6.100]
  + if cfg!(debug_assertions) {

  + env_logger::init();

  + } else {

  + env_logger_init();

  + }
• edit in pijul/src/commands/git.rs at line 2
  [6.819]→[6.819:857](∅→∅)
  − use canonical_path::CanonicalPathBuf;
• edit in pijul/src/commands/git.rs at line 6
  [6.970]→[6.137281:137305](∅→∅),[6.137281]→[6.137281:137305](∅→∅)
  − use std::cell::RefCell;
• edit in pijul/src/commands/git.rs at line 9
  [6.137419]→[6.137419:137436](∅→∅)
  − use std::rc::Rc;
• replacement in pijul/src/commands/git.rs at line 444
  [6.2635]→[6.14259:14296](∅→∅)
  − &repo.repo.working_copy,
  [6.2635]

  [6.151374]
  + &libpijul::working_copy::sink(),
• replacement in pijul/src/commands/git.rs at line 482
  [6.152146]→[6.152146:152214](∅→∅)
  − ) -> Result<(git2::Object<'a>, BTreeSet<PathBuf>), anyhow::Error> {
  [6.152146]

  [6.152214]
  + ) -> Result<(git2::Object<'a>, BTreeMap<PathBuf, bool>), anyhow::Error> {
• edit in pijul/src/commands/git.rs at line 487
  [6.152395]→[6.152395:153092](∅→∅),[6.153092]→[6.0:84](∅→∅),[6.84]→[6.153092:153321](∅→∅),[6.153092]→[6.153092:153321](∅→∅)
  − let reset_was_useful = Rc::new(RefCell::new(false));

  − let mut builder = git2::build::CheckoutBuilder::new();

  − let repo_path = repo.repo.path.clone();

  − let reset_was_useful_ = reset_was_useful.clone();

  − builder

  − .force()

  − .remove_untracked(true)

  − .remove_ignored(true)

  − .progress(move |file, a, b| {

  − debug!("Git progress: {:?} {:?} {:?}", file, a, b);

  − if let Some(file) = file {

  − let file = repo_path.join(file);

  − if let Ok(meta) = std::fs::metadata(&file) {

  − if !meta.file_type().is_symlink() {

  − *reset_was_useful_.borrow_mut() = true

  − }

  − } else {

  − *reset_was_useful_.borrow_mut() = true

  − }

  − }

  − });

  − builder.notify(|notif, file, _, _, _| {

  − info!("Git reset: {:?} {:?}", notif, file);

  − true

  − });

  − git.reset(&object, git2::ResetType::Hard, Some(&mut builder))?;
• replacement in pijul/src/commands/git.rs at line 491
  [6.153433]→[6.153433:153473](∅→∅)
  − let mut prefixes = BTreeSet::new();
  [6.153433]

  [6.153473]
  + let mut prefixes = BTreeMap::new();
• edit in pijul/src/commands/git.rs at line 511
  [6.154151]

  [6.154151]
  + let is_dir = delta.new_file().mode() == git2::FileMode::Tree;
• replacement in pijul/src/commands/git.rs at line 551
  [6.156162]→[6.156162:156213](∅→∅)
  − prefixes.insert(new_path);
  [6.156162]

  [6.156213]
  + prefixes.insert(new_path, is_dir);
• replacement in pijul/src/commands/git.rs at line 555
  [6.156344]→[6.156344:156395](∅→∅)
  − prefixes.insert(old_path);
  [6.156344]

  [6.156395]
  + prefixes.insert(old_path, is_dir);
• replacement in pijul/src/commands/git.rs at line 562
  [6.156742]→[6.156742:156852](∅→∅)
  − prefixes.insert(old_path);

  − prefixes.insert(new_path);
  [6.156742]

  [6.156852]
  + prefixes.insert(old_path, is_dir);

  + prefixes.insert(new_path, is_dir);
• replacement in pijul/src/commands/git.rs at line 583
  [5.335]→[5.335:379](∅→∅)
  − prefixes.insert(m);
  [5.335]

  [5.379]
  + prefixes.insert(m, t.kind() == Some(git2::ObjectType::Tree));
• edit in pijul/src/commands/git.rs at line 593
  [6.157466]

  [6.157466]
  + }

  +

  + #[derive(Clone)]

  + struct Commit<'a> {

  + r: &'a git2::Repository,

  + c: git2::Commit<'a>,
• edit in pijul/src/commands/git.rs at line 600
  [6.157468]

  [6.157468]
  +

  + impl<'a> libpijul::working_copy::WorkingCopyRead for Commit<'a> {

  + type Error = git2::Error;

  +

  + fn file_metadata(&self, file: &str) -> Result<InodeMetadata, Self::Error> {

  + debug!("metadata {:?}", file);

  + let entry = self.c.tree()?.get_path(Path::new(file))?;

  + let is_dir = entry.kind() == Some(git2::ObjectType::Tree);

  + if is_dir {

  + Ok(InodeMetadata::new(0o100, true))

  + } else {

  + let permissions = entry.filemode();

  + debug!(

  + "permissions = {:o} {:o} {:?}",

  + permissions,

  + permissions & 0o100,

  + is_dir

  + );

  + Ok(InodeMetadata::new(permissions as usize & 0o100, false))

  + }

  + }
• edit in pijul/src/commands/git.rs at line 622
  [6.157469]

  [6.157469]
  + fn read_file(&self, file: &str, buffer: &mut Vec<u8>) -> Result<(), Self::Error> {

  + debug!("read file {:?}", file);

  + let entry = self.c.tree()?.get_path(Path::new(file))?;

  + if let Ok(b) = entry.to_object(self.r)?.peel_to_blob() {

  + buffer.extend(b.content());

  + }

  + debug!("entry {:?}", entry.kind());

  + Ok(())

  + }

  +

  + fn modified_time(&self, _: &str) -> Result<std::time::SystemTime, Self::Error> {

  + Ok(std::time::SystemTime::now())

  + }

  + }

  +
• replacement in pijul/src/commands/git.rs at line 650
  [6.3493]→[6.157898:157976](∅→∅),[6.14809]→[6.157898:157976](∅→∅),[6.157898]→[6.157898:157976](∅→∅),[6.157976]→[6.453:530](∅→∅),[6.530]→[6.157976:158004](∅→∅),[6.157976]→[6.157976:158004](∅→∅),[6.158004]→[6.0:63](∅→∅)
  − for p in prefixes.iter() {

  − if let Ok(m) = std::fs::metadata(&p) {

  − use path_slash::PathExt;

  − let p = p.to_slash_lossy();

  − if m.is_dir() {

  − txn_.add_dir(&p, 0).map(|_| ()).unwrap_or(());
  [6.14809]

  [6.158068]
  + let mut prefixes_ = BTreeMap::new();

  + for (mut p, is_dir) in prefixes {

  + use path_slash::PathExt;

  + loop {

  + debug!("p = {:?}", p);

  + if prefixes_.contains_key(&p) {

  + break;

  + }

  + let p_ = p.to_slash_lossy();

  + debug!("adding prefix {:?}", p_);

  + let (tracked, pos) = libpijul::fs::get_vertex(&*txn_, &p_)?;

  + if !tracked {

  + debug!("not tracked");

  + if is_dir {

  + txn_.add_dir(&p_, 0).map(|_| ()).unwrap_or(());

  + } else {

  + txn_.add_file(&p_, 0).map(|_| ()).unwrap_or(());

  + }

  + }

  + debug!("pos = {:?}", pos);

  + if pos.is_none() || !is_dir {

  + if !p.pop() {

  + prefixes_.insert(PathBuf::new(), true);

  + break;

  + }
• replacement in pijul/src/commands/git.rs at line 676
  [6.158089]→[6.64:128](∅→∅)
  − txn_.add_file(&p, 0).map(|_| ()).unwrap_or(());
  [6.158089]

  [6.158154]
  + prefixes_.insert(p, is_dir);

  + break;
• edit in pijul/src/commands/git.rs at line 686
  [6.158402]→[6.158402:158463](∅→∅)
  − let prefix_vec: Vec<_> = prefixes.into_iter().collect();
• replacement in pijul/src/commands/git.rs at line 711
  [6.14912]→[6.14912:14945](∅→∅)
  − &repo.repo.working_copy,
  [6.14912]

  [6.158558]
  + // &repo.repo.working_copy

  + &Commit {

  + r: git,

  + c: git.find_commit(*child)?,

  + },
• replacement in pijul/src/commands/git.rs at line 717
  [6.158586]→[6.199:258](∅→∅),[6.258]→[6.158611:158632](∅→∅),[6.158611]→[6.158611:158632](∅→∅)
  − &CanonicalPathBuf::canonicalize(&repo.repo.path)?,

  − &prefix_vec,
  [6.158586]

  [6.158632]
  + &prefixes_,
• replacement in pijul/src/commands/git.rs at line 763
  [6.1494]→[6.1494:1622](∅→∅)
  − T: TxnT + TxnTExt + MutTxnTExt + Send + Sync + 'static,

  − C: libpijul::changestore::ChangeStore + Clone + Send + 'static,
  [6.1494]

  [6.1622]
  + T: TxnT + TxnTExt + MutTxnTExt,

  + C: libpijul::changestore::ChangeStore + Clone,

  + W: libpijul::working_copy::WorkingCopyRead + Clone,
• replacement in pijul/src/commands/git.rs at line 769
  [6.15323]→[6.15323:15378](∅→∅)
  − working_copy: &libpijul::working_copy::FileSystem,
  [6.15323]

  [6.160570]
  + working_copy: &W,
• replacement in pijul/src/commands/git.rs at line 771
  [6.160587]→[6.259:293](∅→∅),[6.293]→[6.160609:160635](∅→∅),[6.160609]→[6.160609:160635](∅→∅)
  − repo_path: &CanonicalPathBuf,

  − prefixes: &[PathBuf],
  [6.160587]

  [6.160635]
  + prefixes: &BTreeMap<PathBuf, bool>,
• replacement in pijul/src/commands/git.rs at line 774
  [4.39]→[6.7303:7398](∅→∅),[6.160679]→[6.7303:7398](∅→∅)
  − ) -> Result<(usize, Option<libpijul::Hash>, libpijul::Merkle), libpijul::LocalApplyError<T>> {
  [4.39]

  [6.85]
  + ) -> Result<(usize, Option<libpijul::Hash>, libpijul::Merkle), libpijul::LocalApplyError<T>>

  + where

  + W::Error: 'static,

  + {
• replacement in pijul/src/commands/git.rs at line 781
  [6.160881]→[6.34:70](∅→∅),[6.70]→[6.0:269](∅→∅),[6.269]→[3.24:43](∅→∅),[3.43]→[6.269:606](∅→∅),[6.269]→[6.269:606](∅→∅),[6.606]→[2.0:57](∅→∅)
  − let num_cpus = num_cpus::get();

  − for p in prefixes.iter() {

  − use libpijul::working_copy::filesystem::*;

  − match working_copy.record_prefix(

  − txn.clone(),

  − channel.clone(),

  − changes,

  − &mut state,

  − repo_path.clone(),

  − p,

  − false,

  − num_cpus,

  − 0,

  − ) {

  − Ok(_) => {}

  − Err(Error::Add(AddError::Fs(FsError::NotFound(_)))) => {}

  − Err(Error::Add(AddError::Fs(FsError::AlreadyInRepo(_)))) => {}

  − Err(Error::Add(AddError::Io(e))) if e.kind() == std::io::ErrorKind::NotFound => {}

  − Err(e) => {

  − error!("While adding {:?}: {:?}", p, e);
  [6.160881]

  [6.668]
  + let mut last = None;

  + for (p, _) in prefixes.iter() {

  + if let Some(last) = last {

  + if p.starts_with(&last) {

  + continue;
• edit in pijul/src/commands/git.rs at line 788
  [6.692]

  [6.692]
  + state

  + .record_single_thread(

  + txn.clone(),

  + libpijul::Algorithm::default(),

  + false,

  + &libpijul::DEFAULT_SEPARATOR,

  + channel.clone(),

  + working_copy,

  + changes,

  + p.to_str().unwrap(),

  + )

  + .unwrap();

  + last = Some(p);
• replacement in pijul/src/commands/git.rs at line 803
  [6.727]→[6.727:977](∅→∅),[6.977]→[3.44:63](∅→∅),[3.63]→[6.977:1314](∅→∅),[6.977]→[6.977:1314](∅→∅),[6.1314]→[2.58:122](∅→∅),[2.122]→[6.1376:1390](∅→∅),[6.1376]→[6.1376:1390](∅→∅),[6.1390]→[6.549:559](∅→∅),[6.4996]→[6.549:559](∅→∅),[6.549]→[6.549:559](∅→∅)
  − use libpijul::working_copy::filesystem::*;

  − match working_copy.record_prefix(

  − txn.clone(),

  − channel.clone(),

  − changes,

  − &mut state,

  − repo_path.clone(),

  − Path::new(""),

  − false,

  − num_cpus,

  − 0,

  − ) {

  − Ok(_) => {}

  − Err(Error::Add(AddError::Fs(FsError::NotFound(_)))) => {}

  − Err(Error::Add(AddError::Fs(FsError::AlreadyInRepo(_)))) => {}

  − Err(Error::Add(AddError::Io(e))) if e.kind() == std::io::ErrorKind::NotFound => {}

  − Err(e) => {

  − error!("While adding {:?}: {:?}", prefixes, e);

  − }

  − }
  [6.727]

  [6.559]
  + state

  + .record_single_thread(

  + txn.clone(),

  + libpijul::Algorithm::default(),

  + false,

  + &libpijul::DEFAULT_SEPARATOR,

  + channel.clone(),

  + working_copy,

  + changes,

  + "",

  + )

  + .unwrap();
• edit in libpijul/src/fs.rs at line 166
  [34.9]

  [6.741561]
  + /// Returns whether a path is registered in the working copy.

  + pub fn get_vertex<T: TreeTxnT>(

  + txn: &T,

  + path: &str,

  + ) -> Result<(bool, Option<Position<ChangeId>>), TreeErr<T::TreeError>> {

  + debug!("is_tracked {:?}", path);

  + let (inode, mut remaining_path_components) = closest_in_repo_ancestor(txn, path)?;

  + debug!("/is_tracked {:?}", path);

  + if remaining_path_components.next().is_none() {

  + Ok((true, txn.get_inodes(&inode, None)?.cloned()))

  + } else {

  + Ok((false, None))

  + }

  + }

  +