Better error handling in HTTP

pmeunier

Jul 7, 2021, 12:14 PM

OIOMXESDNMLOTMNYCZZBYSBAQTYPAXXMUHTLA2AYCMNHZMPSLX2AC

Dependencies

• [2] 6FEU6ES7 Fixing network errors
• [3] AI73GKAO Adding a UserAgent header to the http downloader
• [4] L2VH4BYK Downloading changelists from channels without an id (Nest discussions)
• [5] HKA66XOQ Updating Thrussh version for long or massive connections
• [6] WI5BS6BS New published versions
• [7] CCLLB7OI Upgrading to Sanakirja 0.15 + version bump
• [8] GBSL4AZI Version bump
• [9] H565UUPC Use correct pattern for workspace interdependencies
• [10] MU5GSJAW Partial push and pull (WARNING: breaks the existing protocol)
• [11] YX3VCEOM Version bump
• [12] 6DOXSHWG Cleanup, and version bump
• [13] Q45QHPO4 Feedback on network stuff
• [14] 4CEHBW7Y Truncate files when restarting a HTTP transfer
• [15] IQ4FCHPZ HTTP connections: pooling + retry on error
• [16] 3S4DR77Z Version updates
• [17] JRENVH5D Reqwest 0.11
• [18] UN64Q3P2 Version bump
• [19] VYHHOEYH Versions and formatting
• [20] FYUDBQ3C Formatting changes + version bump
• [21] ENKQ3QZG Forward the exit status messages from the SSH background loop to the client (solving hangs)
• [22] SAGSYAPX Various version bumps
• [23] SXEYMYF7 Fixing the bad changes in history (unfortunately, by rebooting).
• [24] OCBM7IFE New release: pijul-1.0.0-alpha.8
• [25] TPEH2XNB 1.0.0-alpha.28, with Tokio 1.0
• [26] 5BRU2RRW Cleanup (debugging a crash related to trees/inodes)
• [27] I6DVZEFU Do not ask for user input if the SSH channel is already closed
• [28] R4RRU65V Recreate the patch file instead of only truncating it, on failed HTTP downloads
• [29] G65S7FAW Version bump and cleanup
• [30] FBXYP7QM Forgot to add remote::http
• [31] WIORLB47 Version bump
• [32] 367UBQ6K Forwarding SSH stderr, and progress bar for push
• [33] HSVGP2G4 Version bump + formatting
• [34] BVVMTOYW Proper renaming of changes downloaded over HTTP
• [35] HDGRZISM Version updates
• [36] VRDOV7DD Versions
• [37] CUHXXBDZ Fixing a bug in replacements, recently introduced during a fix of a graph corruption bug
• [38] FDPGJDXV Compiling Thrussh with feature OpenSSL by default
• [39] ZTVNGFNT Version bump
• [40] OUWD436A Version bump
• [41] XAY4DYRR Version bump
• [42] B3QWIGDE Fixing the Git features with the latest Pijul (+ conflicts in Cargo.toml)
• [43] VBMXB443 Retrying if the HTTP connection drops while reading the body
• [44] IIV3EL2X Cleanup, formatting, and fixing the Git feature
• [45] NX5I5H53 New published versions
• [46] JL4WKA5P Implement the Sanakirja concurrency model in a cross-process way
• [47] JACZWIJ6 Version bump
• [48] G6YZ7U65 Version bump
• [*] V435QOJR Using path-slash to fix path issues on Windows
• [*] 2K7JLB4Z No pager on Windows
• [*] QJXNUQFJ Solving conflicts

Change contents

• replacement in pijul/src/remote/http.rs at line 8
  [4.2274]→[6.0:25](∅→∅),[6.5100]→[6.0:25](∅→∅)
  − use log::{debug, error};
  [4.2274]

  [3.0]
  + use log::{debug, error, trace};
• replacement in pijul/src/remote/http.rs at line 28
  [6.358]→[6.358:406](∅→∅)
  − let mut f = std::fs::File::create(&path_)?;
  [6.358]

  [6.406]
  + let mut f = tokio::fs::File::create(&path_).await?;
• replacement in pijul/src/remote/http.rs at line 33
  [6.578]→[6.0:11](∅→∅)
  − loop {
  [6.578]

  [3.70]
  +

  + let (send, mut recv) = tokio::sync::mpsc::channel::<Option<bytes::Bytes>>(100);

  + let t = tokio::spawn(async move {

  + while let Some(chunk) = recv.recv().await {

  + match chunk {

  + Some(chunk) => {

  + trace!("writing {:?}", chunk.len());

  + use tokio::io::AsyncWriteExt;

  + f.write_all(&chunk).await?;

  + }

  + None => {

  + f.set_len(0).await?;

  + }

  + }

  + }

  + Ok::<_, std::io::Error>(())

  + });

  + let mut done = false;

  + while !done {
• replacement in pijul/src/remote/http.rs at line 64
  [6.1213]→[2.369:396](∅→∅)
  − f.set_len(0)?;
  [6.1213]

  [6.888]
  + send.send(None).await?;
• replacement in pijul/src/remote/http.rs at line 70
  [6.240]→[6.11136:11187](∅→∅)
  − bail!("HTTP error {:?}", res.status())
  [6.240]

  [6.1183]
  + tokio::time::sleep(std::time::Duration::from_secs_f64(delay)).await;

  + send.send(None).await?;

  + delay *= 2.;

  + continue;
• replacement in pijul/src/remote/http.rs at line 75
  [6.1193]→[6.12:38](∅→∅)
  − let done = loop {
  [6.1193]

  [6.99]
  + while !done {
• replacement in pijul/src/remote/http.rs at line 77
  [6.137]→[6.294:448](∅→∅),[6.1261]→[6.294:448](∅→∅),[6.294]→[6.294:448](∅→∅),[6.448]→[6.39:79](∅→∅),[6.79]→[6.490:593](∅→∅),[6.490]→[6.490:593](∅→∅)
  − Ok(Some(chunk)) => {

  − debug!("writing {:?}", chunk.len());

  − f.write_all(&chunk)?;

  − }

  − Ok(None) => break true,

  − Err(_) => {

  − error!("Error while downloading {:?}, retrying", url);
  [6.137]

  [6.1262]
  + Ok(Some(chunk)) => send.send(Some(chunk)).await?,

  + Ok(None) => done = true,

  + Err(e) => {

  + debug!("error {:?}", e);

  + error!("Error while downloading {:?} from {:?}, retrying", c32, url);

  + send.send(None).await?;
• replacement in pijul/src/remote/http.rs at line 85
  [6.719]→[6.80:113](∅→∅)
  − break false;
  [6.719]

  [6.746]
  + break;
• edit in pijul/src/remote/http.rs at line 88
  [6.778]→[6.114:233](∅→∅)
  − };

  − if done {

  − std::fs::rename(&path_, &path_.with_extension("change"))?;

  − break;
• edit in pijul/src/remote/http.rs at line 90
  [6.1330]

  [6.856]
  + std::mem::drop(send);

  + t.await??;

  + if done {

  + std::fs::rename(&path_, &path_.with_extension("change"))?;

  + }
• replacement in pijul/Cargo.toml at line 63
  [6.197665]→[6.1535:1615](∅→∅)
  − tokio = { version = "1.0", features = [ "rt-multi-thread", "macros", "sync" ] }
  [6.197665]

  [5.88]
  + tokio = { version = "1.0", features = [ "rt-multi-thread", "macros", "sync", "fs" ] }
• edit in pijul/Cargo.toml at line 90
  [50.1035]

  [51.309]
  + bytes = "1.0"
• edit in Cargo.lock at line 1511
  [52.33594]

  [52.33594]
  + "bytes",