Fetching URLs (server-side)

pmeunier

Jun 27, 2025, 10:27 PM

F2C3TLOI4KEEWK427YMSHZ47352IEYL24PHCW5ELJID7BCSZCRLAC

Dependencies

• [2] BQ4E3XLA Forwarding stdout/stderr
• [3] ODUDDQRY Adding the OCaml interface
• [4] SI454P2V Documentation and cleanup
• [5] UWQB743K First working shell (with ocaml code)

Change contents

• replacement in src/main.rs at line 120
  [2.718]→[2.718:796](∅→∅)
  − destdir: vec![out.to_str().unwrap().to_string()],
  [2.718]

  [2.796]
  + destdir: out

  + .into_iter()

  + .map(|x| x.to_str().unwrap().to_string())

  + .collect(),
• edit in src/main.rs at line 143
  [3.2205]

  [3.2205]
  + async fn add_url(

  + &self,

  + request: tonic::Request<proto::AddUrlRequest>,

  + ) -> Result<tonic::Response<proto::DerivationReply>, tonic::Status> {

  + let r = request.into_inner();

  + debug!("add_url request {:?}", r);

  + let p = self

  + .deb_client

  + .http_download(

  + &r.url,

  + match r.hash_algorithm {

  + 0 => {

  + let mut h = [0; 32];

  + h.clone_from_slice(&r.hash);

  + Hash::Blake3(h)

  + }

  + 1 => {

  + let mut h = [0; 32];

  + h.clone_from_slice(&r.hash);

  + Hash::Sha256(h)

  + }

  + 2 => {

  + let mut h = [0; 64];

  + h.clone_from_slice(&r.hash);

  + Hash::Sha512(h)

  + }

  + _ => unreachable!(),

  + },

  + )

  + .await

  + .unwrap();

  + Ok(tonic::Response::new(proto::DerivationReply {

  + result: Some(proto::derivation_reply::Result::Ok(

  + proto::DerivationResult {

  + destdir: vec![p.to_str().unwrap().to_string()],

  + paths: Vec::new(),

  + path_patterns: Vec::new(),

  + },

  + )),

  + }))

  + }

  +
• edit in src/lib.rs at line 83
  [3.9770]→[3.9770:9925](∅→∅)
  − /// The contained process failed to produce the file or directory

  − /// called `$DESTDIR`.

  − #[error("Failed to produce $DESTDIR")]

  − NoDestDir,
• edit in src/lib.rs at line 102
  [3.10429]

  [3.10429]
  + }

  +

  + /// Hashes, used for example in URL downloads.

  + #[derive(Debug, Clone, Copy, PartialEq, Eq)]

  + pub enum Hash {

  + #[allow(missing_docs)]

  + Blake3([u8; 32]),

  + #[allow(missing_docs)]

  + Sha256([u8; 32]),

  + #[allow(missing_docs)]

  + Sha512([u8; 64]),

  + }

  +

  + /// Hasher for each different hash algorithm.

  + pub enum Hasher {

  + #[allow(missing_docs)]

  + Blake3(blake3::Hasher),

  + #[allow(missing_docs)]

  + Sha256(sha2::Sha256),

  + #[allow(missing_docs)]

  + Sha512(sha2::Sha512),

  + }

  +

  + impl Hash {

  + /// Create a hasher for this hash algorithm.

  + pub fn hasher(&self) -> Hasher {

  + match self {

  + Hash::Blake3(_) => Hasher::Blake3(blake3::Hasher::new()),

  + Hash::Sha256(_) => Hasher::Sha256(sha2::Sha256::new()),

  + Hash::Sha512(_) => Hasher::Sha512(sha2::Sha512::new()),

  + }

  + }

  + }

  +

  + impl Hasher {

  + /// Update the hasher with new bytes.

  + pub fn update(&mut self, b: &[u8]) {

  + match self {

  + Hasher::Blake3(h) => {

  + h.update(b);

  + }

  + Hasher::Sha256(h) => {

  + h.update(b);

  + }

  + Hasher::Sha512(h) => {

  + h.update(b);

  + }

  + }

  + }

  +

  + /// Create the hash, to be compared with the one supplied by the user.

  + pub fn finalize(self) -> Hash {

  + match self {

  + Hasher::Blake3(h) => Hash::Blake3(h.finalize().into()),

  + Hasher::Sha256(h) => Hash::Sha256(h.finalize().into()),

  + Hasher::Sha512(h) => Hash::Sha512(h.finalize().into()),

  + }

  + }

  + }

  +

  + impl std::ops::Deref for Hash {

  + type Target = [u8];

  + fn deref(&self) -> &Self::Target {

  + match self {

  + Hash::Blake3(h) => &h[..],

  + Hash::Sha256(h) => &h[..],

  + Hash::Sha512(h) => &h[..],

  + }

  + }

  + }

  +

  + impl std::ops::DerefMut for Hash {

  + fn deref_mut(&mut self) -> &mut Self::Target {

  + match self {

  + Hash::Blake3(h) => &mut h[..],

  + Hash::Sha256(h) => &mut h[..],

  + Hash::Sha512(h) => &mut h[..],

  + }

  + }
• edit in src/lib.rs at line 183
  [3.10432]

  [3.10432]
  + impl Client {

  + /// Download a URL.

  + pub async fn http_download(&self, url: &str, h: Hash) -> Result<PathBuf, Error> {

  + let expected_path = self

  + .store_path

  + .join(data_encoding::BASE32_DNSSEC.encode(&h));

  +

  + if let Ok(mut f) = tokio::fs::File::open(&expected_path).await {

  + use tokio::io::AsyncReadExt;

  + let mut buf = [0; 4096];

  + let mut hasher = h.hasher();

  + while let Ok(n) = f.read(&mut buf).await {

  + if n == 0 {

  + break;

  + }

  + hasher.update(&buf[..n]);

  + }

  + let got = hasher.finalize();

  + return if got == h {

  + Ok(expected_path)

  + } else {

  + Err(Error::WrongHash {

  + expected: data_encoding::HEXLOWER.encode(&h),

  + got: data_encoding::HEXLOWER.encode(&got),

  + })

  + };

  + }

  +

  + let mut r = self.client().get(url).send().await?;

  + let mut t: Option<tokio::task::JoinHandle<Result<tokio::fs::File, tokio::io::Error>>> =

  + None;

  +

  + let file = NamedTempFile::new_in(&self.store_path)?;

  + let mut f = Some(tokio::fs::File::create(&file).await?);

  + let mut hasher = h.hasher();

  + while let Some(chunk) = r.chunk().await? {

  + hasher.update(&chunk);

  +

  + if let Some(t) = t.take() {

  + f = Some(t.await.unwrap().unwrap())

  + }

  + let mut f = f.take().unwrap();

  + t = Some(tokio::spawn(async move {

  + f.write_all(&chunk).await?;

  + Ok(f)

  + }))

  + }

  + let got = hasher.finalize();

  + if got == h {

  + file.persist(&expected_path)?;

  + Ok(expected_path)

  + } else {

  + Err(Error::WrongHash {

  + expected: data_encoding::HEXLOWER.encode(&h),

  + got: data_encoding::HEXLOWER.encode(&got),

  + })

  + }

  + }

  + }

  +
• edit in src/lib.rs at line 413
  [3.2478]→[3.13145:13347](∅→∅),[3.13145]→[3.13145:13347](∅→∅)
  − let r = self

  − .client()

  − .get(&format!(

  − "{m}/dists/{release}/{component}/binary-{arch}/Packages.xz"

  − ))

  − .send()

  − .await?;
• edit in src/lib.rs at line 414
  [3.13348]

  [3.13348]
  + let r = {

  + let client = self.client();

  + let permit = self.download_sem.acquire().await.unwrap();

  + let r = client

  + .get(&format!(

  + "{m}/dists/{release}/{component}/binary-{arch}/Packages.xz"

  + ))

  + .send()

  + .await?;

  + drop(permit);

  + r

  + };
• edit in elpe/lib/elpegrpc.proto at line 26
  [3.63522]

  [3.55524]
  +
• edit in elpe/lib/elpegrpc.proto at line 28
  [3.55525]

  [3.55525]
  + enum Hash {

  + blake3 = 0;

  + sha256 = 1;

  + }

  +

  + message AddUrlRequest {

  + string url = 1;

  + Hash hash_algorithm = 2;

  + bytes hash = 3;

  + }

  +
• edit in elpe/lib/elpegrpc.proto at line 87
  [3.63960]

  [3.55985]
  + rpc AddUrl (AddUrlRequest) returns (DerivationReply);