New config for HTTP remotes

pmeunier

Nov 11, 2022, 1:46 PM

H4AU6QRPRDRFW3V7NN5CJ6DHLEUBYGNLRZ5GYV6ULBGRMOPCJQXQC

Dependencies

[2] DO2Y5TY5 Tag synchronisation
[3] FVQYZQFL Create dialoguer themes based on global config
[4] TI7PCK7J Update `pijul/src/main.rs` to use new identity management
[5] AI73GKAO Adding a UserAgent header to the http downloader
[6] VBMXB443 Retrying if the HTTP connection drops while reading the body
[7] A3RM526Y Integrating identity malleability
[8] SNZ3OAMC use native external subcommand support instead of hand-rolled one
[9] SEWGHUHQ .pijul/config: simplify remotes and hooks
[10] Q45QHPO4 Feedback on network stuff
[11] BVVMTOYW Proper renaming of changes downloaded over HTTP
[12] CCLLB7OI Upgrading to Sanakirja 0.15 + version bump
[13] TPEH2XNB 1.0.0-alpha.28, with Tokio 1.0
[14] R245EVN3 Do not print anything on broken pipe errors
[15] GYXIF25T Proper parsing of URLs
[16] 5BB266P6 Optional colours in the global config file
[17] 7Z3KZV6G Retry HTTP downloads if we don't get a full patch
[18] 76PCXGML Pushing to, and pulling from the local repository
[19] X243Z3Y5 Recording only the required metadata (can even be changed later!)
[20] GFED4ORA Improve error handling for nonexistent channels
[21] JRENVH5D Reqwest 0.11
[22] BNPSVXIC Friendlier progress bars
[23] SXEYMYF7 Fixing the bad changes in history (unfortunately, by rebooting).
[24] 27RZYCM3 Pushing/pulling from/to Nest discussions again
[25] FBXYP7QM Forgot to add remote::http
[26] OIOMXESD Better error handling in HTTP
[27] KI2AFWOS Fixing a panic in pull
[28] C5XGFNKI Simplify return type for remote get_id
[29] I52XSRUH Massive cleanup, and simplification
[30] KWAGWB73 Adding extra dependencies from the config file
[31] JZADJIA3 Handling HTTP errors
[32] 4OJWMSOW Fully replace crate::Identity
[33] IQ4FCHPZ HTTP connections: pooling + retry on error
[34] 367UBQ6K Forwarding SSH stderr, and progress bar for push
[35] ZSFJT4SF Allow remotes to have a different push and pull address
[36] MU5GSJAW Partial push and pull (WARNING: breaks the existing protocol)
[*] VL7ZYKHB Running hooks through shell on Windows and Unix

Change contents

replacement in pijul/src/remote/mod.rs at line 53

[5.53148]→[5.53148:53208](∅→∅),[5.53208]→[5.54:82](∅→∅)

if let Some(name) = self.config.remotes.get(name) {
unknown_remote(

[5.53148]

[5.82]

if let Some(name) = self.config.remotes.iter().find(|e| e.name() == name) {
name.to_remote(channel, no_cert_check, with_path).await
/*unknown_remote(
replacement in pijul/src/remote/mod.rs at line 62

[5.248]→[5.248:267](∅→∅)

.await

[5.248]

[5.53271]

.await*/
edit in pijul/src/remote/mod.rs at line 69

[5.53370]

[5.53370]

impl RemoteConfig {
async fn to_remote(
&self,
channel: &str,
no_cert_check: bool,
with_path: bool,
) -> Result<RemoteRepo, anyhow::Error> {
match self {
RemoteConfig::Ssh { ssh, .. } => {
if let Some(mut sshr) = ssh_remote(ssh, with_path) {
debug!("unknown_remote, ssh = {:?}", ssh);
if let Some(c) = sshr.connect(ssh, channel).await? {
return Ok(RemoteRepo::Ssh(c));
}
}
bail!("Remote not found: {:?}", ssh)
}
RemoteConfig::Http {
http,
headers,
name,
} => {
let mut h = Vec::new();
for (k, v) in headers.iter() {
match v {
RemoteHttpHeader::String(s) => {
h.push((k.clone(), s.clone()));
}
RemoteHttpHeader::Shell(shell) => {
h.push((k.clone(), shell_cmd(&shell.shell)?));
}
}
}
return Ok(RemoteRepo::Http(Http {
url: http.parse().unwrap(),
channel: channel.to_string(),
client: reqwest::ClientBuilder::new()
.danger_accept_invalid_certs(no_cert_check)
.build()?,
headers: h,
name: name.to_string(),
}));
}
}
}
}
edit in pijul/src/remote/mod.rs at line 133

[5.450]

[5.450]

headers: Vec::new(),
edit in pijul/src/remote/http.rs at line 1

[5.10]→[5.4982:5013](∅→∅),[5.1080]→[5.11:55](∅→∅),[5.5013]→[5.11:55](∅→∅),[5.10]→[5.11:55](∅→∅),[5.55]→[5.1192:1193](∅→∅)

use std::collections::HashSet;
use std::io::Write;
use std::path::PathBuf;
edit in pijul/src/remote/http.rs at line 5

[5.32]

[2.23928]

use std::collections::HashSet;
use std::io::Write;
use std::path::PathBuf;
edit in pijul/src/remote/http.rs at line 18

[5.207]

[5.26]

pub headers: Vec<(String, String)>,
edit in pijul/src/remote/http.rs at line 24

[5.1215]

[5.101]

headers: Vec<(String, String)>,
edit in pijul/src/remote/http.rs at line 64

[5.650]

[5.650]
replacement in pijul/src/remote/http.rs at line 67

[5.694]→[5.70:116](∅→∅),[5.11]→[5.70:116](∅→∅)

let mut res = if let Ok(res) = client

[5.694]

[5.116]

let mut req = client
replacement in pijul/src/remote/http.rs at line 70

[2.24506]→[5.179:289](∅→∅),[5.179]→[5.179:289](∅→∅)

.header(reqwest::header::USER_AGENT, USER_AGENT)
.send()
.await
{

[2.24506]

[5.173]

.header(reqwest::header::USER_AGENT, USER_AGENT);
for (k, v) in headers.iter() {
req = req.header(k.as_str(), v.as_str());
}
let mut res = if let Ok(res) = req.send().await {
edit in pijul/src/remote/http.rs at line 156

[5.1740]

[5.1740]

self.headers.clone(),
replacement in pijul/src/remote/http.rs at line 233

[5.1639]→[5.0:28](∅→∅)

let resp = self

[5.1639]

[5.28]

let mut req = self
replacement in pijul/src/remote/http.rs at line 237

[5.1727]→[5.290:355](∅→∅),[5.355]→[2.25772:25800](∅→∅),[2.25800]→[5.1757:1781](∅→∅),[5.1757]→[5.1757:1781](∅→∅),[5.1378]→[5.1781:1806](∅→∅),[5.1781]→[5.1781:1806](∅→∅)

.header(reqwest::header::USER_AGENT, USER_AGENT)
.body(body)
.send()
.await?;

[5.1727]

[5.53]

.header(reqwest::header::USER_AGENT, USER_AGENT);
for (k, v) in self.headers.iter() {
req = req.header(k.as_str(), v.as_str());
}
let resp = req.body(body).send().await?;
replacement in pijul/src/remote/http.rs at line 285

[5.5584]→[5.356:379](∅→∅)

let res = self

[5.5584]

[5.379]

let mut req = self
replacement in pijul/src/remote/http.rs at line 289

[5.448]→[5.448:550](∅→∅)

.header(reqwest::header::USER_AGENT, USER_AGENT)
.send()
.await?;

[5.448]

[5.0]

.header(reqwest::header::USER_AGENT, USER_AGENT);
for (k, v) in self.headers.iter() {
req = req.header(k.as_str(), v.as_str());
}
let res = req.send().await?;
replacement in pijul/src/remote/http.rs at line 341

[5.683]→[5.551:574](∅→∅)

let res = self

[5.683]

[5.574]

let mut req = self
replacement in pijul/src/remote/http.rs at line 345

[5.640]→[5.640:742](∅→∅)

.header(reqwest::header::USER_AGENT, USER_AGENT)
.send()
.await?;

[5.640]

[5.757]

.header(reqwest::header::USER_AGENT, USER_AGENT);
for (k, v) in self.headers.iter() {
req = req.header(k.as_str(), v.as_str());
}
let res = req.send().await?;
replacement in pijul/src/remote/http.rs at line 374

[5.10720]→[5.10720:10743](∅→∅)

let res = self

[5.10720]

[5.10743]

let mut req = self
replacement in pijul/src/remote/http.rs at line 378

[5.10809]→[5.10809:10911](∅→∅)

.header(reqwest::header::USER_AGENT, USER_AGENT)
.send()
.await?;

[5.10809]

[5.10911]

.header(reqwest::header::USER_AGENT, USER_AGENT);
for (k, v) in self.headers.iter() {
req = req.header(k.as_str(), v.as_str());
}
let res = req.send().await?;
replacement in pijul/src/remote/http.rs at line 459

[5.11594]→[5.11594:11617](∅→∅)

let res = self

[5.11594]

[5.11617]

let mut req = self
replacement in pijul/src/remote/http.rs at line 470

[5.11886]→[5.11886:11988](∅→∅)

.header(reqwest::header::USER_AGENT, USER_AGENT)
.send()
.await?;

[5.11886]

[5.11988]

.header(reqwest::header::USER_AGENT, USER_AGENT);
for (k, v) in self.headers.iter() {
req = req.header(k.as_str(), v.as_str());
}
let res = req.send().await?;
replacement in pijul/src/main.rs at line 153

[4.434]→[4.434:496](∅→∅)

log::error!("Error backtrace: {:#?}", e.backtrace());

[4.434]

[5.0]

log::error!("Error: {:#?}", e);
replacement in pijul/src/config.rs at line 134

[5.22]→[5.308:354](∅→∅)

pub remotes: HashMap<String, RemoteName>,

[5.22]

[5.23]

pub remotes: Vec<RemoteConfig>,
replacement in pijul/src/config.rs at line 142

[5.358]→[3.459:487](∅→∅),[3.487]→[5.375:439](∅→∅),[5.375]→[5.375:439](∅→∅)

#[derive(Debug, Serialize)]
pub enum RemoteName {
Name(String),
Split(SplitRemote),

[5.358]

[5.439]

#[derive(Debug, Serialize, Deserialize)]
#[serde(untagged)]
pub enum RemoteConfig {
Ssh {
name: String,
ssh: String,
},
Http {
name: String,
http: String,
#[serde(default)]
headers: HashMap<String, RemoteHttpHeader>,
},
replacement in pijul/src/config.rs at line 157

[5.442]→[5.442:794](∅→∅)

#[derive(Clone, Copy, Debug)]
pub enum Direction {
Push,
Pull,
}

impl RemoteName {
pub fn with_dir(&self, d: Direction) -> &str {
match (self, d) {
(RemoteName::Name(ref s), _) => s,
(RemoteName::Split(ref s), Direction::Pull) => &s.pull,
(RemoteName::Split(ref s), Direction::Push) => &s.push,

[5.442]

[5.794]

impl RemoteConfig {
pub fn name(&self) -> &str {
match self {
RemoteConfig::Ssh { name, .. } => name,
RemoteConfig::Http { name, .. } => name,
edit in pijul/src/config.rs at line 165

[5.812]→[5.812:1608](∅→∅)

use serde::de::{self, MapAccess, Visitor};
use serde::de::{Deserialize, Deserializer};
use std::fmt;
use std::marker::PhantomData;

impl<'de> Deserialize<'de> for RemoteName {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
struct StringOrStruct(PhantomData<fn() -> RemoteName>);
impl<'de> Visitor<'de> for StringOrStruct {
type Value = RemoteName;

fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
formatter.write_str("string or map")
}

fn visit_str<E>(self, value: &str) -> Result<Self::Value, E>
where
E: de::Error,
{
Ok(RemoteName::Name(value.to_string()))
}
replacement in pijul/src/config.rs at line 166

[5.1609]→[5.1609:2296](∅→∅)

fn visit_map<M>(self, map: M) -> Result<Self::Value, M::Error>
where
M: MapAccess<'de>,
{
// `MapAccessDeserializer` is a wrapper that turns a `MapAccess`
// into a `Deserializer`, allowing it to be used as the input to T's
// `Deserialize` implementation. T then deserializes itself using
// the entries from the map visitor.
Ok(RemoteName::Split(Deserialize::deserialize(
de::value::MapAccessDeserializer::new(map),
)?))
}
}
deserializer.deserialize_any(StringOrStruct(PhantomData))
}

[5.1609]

[5.91196]

#[derive(Debug, Serialize, Deserialize)]
#[serde(untagged)]
pub enum RemoteHttpHeader {
String(String),
Shell(Shell),
replacement in pijul/src/config.rs at line 174

[3.529]→[5.2327:2396](∅→∅),[5.2327]→[5.2327:2396](∅→∅)

pub struct SplitRemote {
pub pull: String,
pub push: String,

[3.529]

[5.2396]

pub struct Shell {
pub shell: String,
}

#[derive(Clone, Copy, Debug)]
pub enum Direction {
Push,
Pull,
edit in pijul/src/config.rs at line 199

[38.217]

[38.217]

pub fn shell_cmd(s: &str) -> Result<String, anyhow::Error> {
let out = if cfg!(target_os = "windows") {
std::process::Command::new("cmd")
.args(&["/C", s])
.output()
.expect("failed to execute process")
} else {
std::process::Command::new(std::env::var("SHELL").unwrap_or("sh".to_string()))
.arg("-c")
.arg(s)
.output()
.expect("failed to execute process")
};
Ok(String::from_utf8(out.stdout)?)
}