LIMIMBOUN3DCIKKTHOK4TAXAWQXNVQXBTMNZYN22DJX4L4FZQH4AC F2C3TLOI4KEEWK427YMSHZ47352IEYL24PHCW5ELJID7BCSZCRLAC RBGQQMWXRUO4W2P5GFH24VRIP52IW5BNSE2ZOBXKFIYARDOPBFWAC AJLMC7UMTMBYBXLOSMGT5PDNX4D7O7LV4GDV7AN25CPOBPRUFYVAC A7YP2YT6WQ4WDFQTQRHPFC7EFJBCWYHV6HERQD6CUTWUBC4FLBTAC LTB4YNHF75FTVQG6S5PL46CGS4CMDWZRAEXWEIXUHQHIWVYVKFKAC UWQB743KR36X6A6JVNK3VH6KMEFUHOUT6ZS2Y2SWIJR55JSF5WQAC SI454P2VFO6Y6RO2NIOH6ZROURP3OTB2BBXH2F36KJT6YML6YBIAC ODUDDQRY373JMDR6W2BHUWSKEU6TEICQMNZIAIFKJPNUA5UN3C4QC BQ4E3XLAVS2AQAUYRGN6SZ4J35KQ2KIH7BBT2JNZXSPKKCGX7DYAC 6MGFBMON6ASGBRQJXY6VUGGZQHXPQSMQJZOOD5LKS2FZODB6ET7AC BDEVQIAUBDZLH5NNXJ3HD3WK2ART4V6K2FLHMAQVAQYNXGL4TOIQC LIUJQXB752UIHJFF32LKJ7ECUZ2VDIFDDHBLAUVNBRTPDYNCAUYQC YJYXDY6AUDJMBUVKKUKM2GO2XVIKZVQFT4SRD4Z7XSTQBTSJ5ZSAC HX4TXY2DTPEEQPU6CMHSD3X4VX7A4OKV7N3HT3QZKSNHNAZOK7GQC R7J4254ZZREVMWXXDUM772GPLZSCZSIJKOLEAI33WUF2CPS3X6WQC KOWYPLMX4TCQANL33QW5VQTKWBQVCMOVLBQUQCTOK6YVQUHZJR5QC LQBZJUGCEIU56WBYBLE757W2DHPZWJ4JNG3ILB52RDXMXUYAPZCQC 6BIW5YDCZXIH3SNKNTZ3MSHUFGN3VKAYPKI4MWJIFVUY7YQSJJ3AC use elpe::extract::*;use elpe::*;use std::sync::Arc;use tokio::io::AsyncWriteExt;use tokio_stream::wrappers::ReceiverStream;use tokio_util::sync::PollSender;use tonic::codegen::tokio_stream::StreamExt;use tracing::*;pub struct Elpe {deb_client: elpe::Client,sender: tokio::sync::mpsc::UnboundedSender<(crate::container::BuildRequest,tokio::sync::mpsc::Sender<crate::container::Msg>,)>,t: Option<tokio::task::JoinHandle<Result<(), elpe::Error>>>,}pub mod proto {tonic::include_proto!("elpe");}impl Elpe {pub fn new(deb_client: elpe::Client,container_channel: elpe::container::ContainerChannel,) -> Self {let (sender, receiver) = tokio::sync::mpsc::unbounded_channel::<(crate::container::BuildRequest,tokio::sync::mpsc::Sender<crate::container::Msg>,)>();let t = tokio::spawn(crate::container::forward(receiver, container_channel));Elpe {deb_client,sender,t: Some(t),}}pub async fn serve(mut self, addr: std::net::SocketAddr) {let t = self.t.take().unwrap();tokio::select! {_ = tonic::transport::Server::builder().add_service(proto::elpe_server::ElpeServer::new(self)).serve(addr)=> {}_ = t => {}}}}use std::pin::Pin;type ResponseStream =Pin<Box<dyn tokio_stream::Stream<Item = Result<proto::DerivationReply, tonic::Status>> + Send>>;#[tonic::async_trait]impl proto::elpe_server::Elpe for Elpe {async fn add_path(&self,request: tonic::Request<tonic::Streaming<proto::AddPathRequest>>,) -> Result<tonic::Response<proto::DerivationReply>, tonic::Status> {let mut r = request.into_inner();debug!("add_path");let mut current_file = None;let ref store = self.deb_client.store_path();debug!("store");let tmp_dir = tempfile::tempdir_in(store).unwrap();debug!("store {:?}", tmp_dir);let mut hasher = blake3::Hasher::new();debug!("loop");loop {trace!("waiting for next in stream");let Some(r) = r.next().await else { break };let r = r.unwrap();match r.request {Some(proto::add_path_request::Request::File(f)) => {info!("Adding file {:?}", f.name);hasher.update(b"\0f");hasher.update(f.name.as_bytes());hasher.update(b"\0");let p = tmp_dir.path().join(&f.name);tokio::fs::create_dir_all(p.parent().unwrap()).await?;current_file = Some(tokio::fs::File::create(&p).await?)}Some(proto::add_path_request::Request::Directory(d)) => {info!("Adding file {:?}", d.name);hasher.update(b"\0d");hasher.update(d.name.as_bytes());hasher.update(b"\0");let p = tmp_dir.path().join(&d.name);tokio::fs::create_dir_all(&p).await.unwrap();}Some(proto::add_path_request::Request::Contents(c)) => {hasher.update(&c.content);current_file.as_mut().unwrap().write_all(&c.content).await?;}None => break,}}debug!("loop done");let path = store.join(data_encoding::HEXLOWER.encode(hasher.finalize().as_bytes()));use tokio::io::ErrorKind;let new = tmp_dir.into_path();match tokio::fs::rename(&new, &path).await {Ok(()) => (),Err(e) if e.kind() == ErrorKind::DirectoryNotEmpty => (),Err(e) => {tokio::fs::remove_dir_all(&new).await?;return Err(e.into());}}info!("add_path extracted to {:?}", path);Ok(tonic::Response::new(proto::DerivationReply {result: Some(proto::derivation_reply::Result::Ok(proto::DerivationResult {destdir: vec![path.to_str().unwrap().to_string()],paths: Vec::new(),path_patterns: Vec::new(),},)),}))}type DerivationStream = ResponseStream;async fn derivation(&self,request: tonic::Request<proto::DerivationRequest>,) -> Result<tonic::Response<ResponseStream>, tonic::Status> {debug!("derivation request");let now = std::time::Instant::now();let r = request.into_inner();let (tx, rx) = tokio::sync::mpsc::channel(200);debug!("derivation request: {:?} {:?}", r.name, r.paths);self.sender.send((crate::container::BuildRequest {name: r.name.clone(),paths: r.paths,script: r.builder,target: r.target,output_hash: r.output_hash,},tx,)).unwrap();use crate::container::Msg;let output_stream = ReceiverStream::new(rx).map(|x| {Ok(match x {Msg::Ok(out) => proto::DerivationReply {result: Some(proto::derivation_reply::Result::Ok(proto::DerivationResult {destdir: out.into_iter().map(|x| x.to_str().unwrap().to_string()).collect(),paths: Vec::new(),path_patterns: Vec::new(),},)),},Msg::Error(p) => proto::DerivationReply {result: Some(proto::derivation_reply::Result::Error(p)),},Msg::Stdout(p) => proto::DerivationReply {result: Some(proto::derivation_reply::Result::Stdout(p)),},Msg::Stderr(p) => proto::DerivationReply {result: Some(proto::derivation_reply::Result::Stderr(p)),},})});info!("request {:?}: {:?}", r.name, now.elapsed());Ok(tonic::Response::new(Box::pin(output_stream)))}async fn add_url(&self,request: tonic::Request<proto::AddUrlRequest>,) -> Result<tonic::Response<proto::DerivationReply>, tonic::Status> {let r = request.into_inner();debug!("add_url request {:?}", r);let p = self.deb_client.http_download(&r.url,match r.hash_algorithm {0 => {let mut h = [0; 32];h.clone_from_slice(&r.hash);Hash::Blake3(h)}1 => {let mut h = [0; 32];h.clone_from_slice(&r.hash);Hash::Sha256(h)}2 => {let mut h = [0; 64];h.clone_from_slice(&r.hash);Hash::Sha512(h)}_ => unreachable!(),},).await.unwrap();Ok(tonic::Response::new(proto::DerivationReply {result: Some(proto::derivation_reply::Result::Ok(proto::DerivationResult {destdir: vec![p.to_str().unwrap().to_string()],paths: Vec::new(),path_patterns: Vec::new(),},)),}))}async fn ubuntu_release(&self,request: tonic::Request<proto::UbuntuReleaseRequest>,) -> Result<tonic::Response<proto::DerivationReply>, tonic::Status> {debug!("ubuntu release {:?}", request);let now = std::time::Instant::now();let r = request.into_inner();let h = self.deb_client.in_release(r.release.clone()).await.unwrap();let arch = match r.arch {0 => "amd64",1 => "aarch64",_ => unreachable!(),};let p = self.deb_client.packages(&h, &r.repository, arch).await.unwrap();info!("ubuntu release request {:?}: {:?}",r.release,now.elapsed());Ok(tonic::Response::new(proto::DerivationReply {result: Some(proto::derivation_reply::Result::Ok(proto::DerivationResult {destdir: vec![p.to_str().unwrap().to_string()],paths: Vec::new(),path_patterns: Vec::new(),},)),}))}type UbuntuPackageStream = ResponseStream;async fn ubuntu_package(&self,request: tonic::Request<proto::UbuntuPackageRequest>,) -> Result<tonic::Response<Self::UbuntuPackageStream>, tonic::Status> {let now = std::time::Instant::now();let r = request.into_inner();let index: Result<Vec<_>, _> = r.index.into_iter().map(|index| deb::Index::open(&index)).collect();let index = index.unwrap();let link_extra: Vec<_> = r.link_extra.into_iter().map(|l| {(regex::Regex::new(&l.pkg).unwrap(),regex::Regex::new(&l.dep).unwrap(),)}).collect();let (tx, rx) = tokio::sync::mpsc::channel(200);use crate::extract::Msg;let output_stream = ReceiverStream::new(rx).map(|x| {Ok(match x {Msg::Downloading(p) => proto::DerivationReply {result: Some(proto::derivation_reply::Result::Loading(p)),},Msg::Ok(p) => proto::DerivationReply {result: Some(proto::derivation_reply::Result::Ok(proto::DerivationResult {destdir: p.result.iter().rev().map(|x| x.to_str().unwrap().to_string()).collect(),paths: p.paths.into_iter().filter_map(Arc::into_inner).collect(),path_patterns: Vec::new(),},)),},Msg::Error(e) => proto::DerivationReply {result: Some(proto::derivation_reply::Result::Error(e.to_string())),},})});match download_extract_deps(&index,&self.deb_client,&r.name,&link_extra,PollSender::new(tx.clone()),).await{Ok(p) => {info!("path {:?} {:#?}", r.name, p);tx.send(Msg::Ok(p)).await.unwrap();}Err(e) => tx.send(Msg::Error(e)).await.unwrap(),}info!("ubuntu package request {:?}: {:?}", r.name, now.elapsed());Ok(tonic::Response::new(Box::pin(output_stream)))}}
use std::sync::Arc;use tokio::io::AsyncWriteExt;use tokio_stream::wrappers::ReceiverStream;use tokio_util::sync::PollSender;use tonic::codegen::tokio_stream::StreamExt;use tracing::*;
pub struct Elpe {deb_client: elpe::Client,sender: tokio::sync::mpsc::UnboundedSender<(crate::container::BuildRequest,tokio::sync::mpsc::Sender<crate::container::Msg>,)>,}
mod server;
use std::pin::Pin;type ResponseStream =Pin<Box<dyn tokio_stream::Stream<Item = Result<proto::DerivationReply, tonic::Status>> + Send>>;#[tonic::async_trait]impl proto::elpe_server::Elpe for Elpe {async fn add_path(&self,request: tonic::Request<tonic::Streaming<proto::AddPathRequest>>,) -> Result<tonic::Response<proto::DerivationReply>, tonic::Status> {let mut r = request.into_inner();debug!("add_path");let mut current_file = None;let ref store = self.deb_client.store_path();debug!("store");let tmp_dir = tempfile::tempdir_in(store).unwrap();debug!("store {:?}", tmp_dir);let mut hasher = blake3::Hasher::new();debug!("loop");loop {trace!("waiting for next in stream");let Some(r) = r.next().await else { break };let r = r.unwrap();match r.request {Some(proto::add_path_request::Request::File(f)) => {info!("Adding file {:?}", f.name);hasher.update(b"\0f");hasher.update(f.name.as_bytes());hasher.update(b"\0");let p = tmp_dir.path().join(&f.name);tokio::fs::create_dir_all(p.parent().unwrap()).await?;current_file = Some(tokio::fs::File::create(&p).await?)}Some(proto::add_path_request::Request::Directory(d)) => {info!("Adding file {:?}", d.name);hasher.update(b"\0d");hasher.update(d.name.as_bytes());hasher.update(b"\0");let p = tmp_dir.path().join(&d.name);tokio::fs::create_dir_all(&p).await.unwrap();}Some(proto::add_path_request::Request::Contents(c)) => {hasher.update(&c.content);current_file.as_mut().unwrap().write_all(&c.content).await?;}None => break,}}debug!("loop done");let path = store.join(data_encoding::HEXLOWER.encode(hasher.finalize().as_bytes()));use tokio::io::ErrorKind;let new = tmp_dir.into_path();match tokio::fs::rename(&new, &path).await {Ok(()) => (),Err(e) if e.kind() == ErrorKind::DirectoryNotEmpty => (),Err(e) => {tokio::fs::remove_dir_all(&new).await?;return Err(e.into());}}info!("add_path extracted to {:?}", path);Ok(tonic::Response::new(proto::DerivationReply {result: Some(proto::derivation_reply::Result::Ok(proto::DerivationResult {destdir: vec![path.to_str().unwrap().to_string()],paths: Vec::new(),path_patterns: Vec::new(),},)),}))}type DerivationStream = ResponseStream;async fn derivation(&self,request: tonic::Request<proto::DerivationRequest>,) -> Result<tonic::Response<ResponseStream>, tonic::Status> {debug!("derivation request");let r = request.into_inner();let (tx, rx) = tokio::sync::mpsc::channel(200);self.sender.send((crate::container::BuildRequest {name: r.name,paths: r.paths,script: r.builder,target: r.target,output_hash: r.output_hash,},tx,)).unwrap();use crate::container::Msg;let output_stream = ReceiverStream::new(rx).map(|x| {Ok(match x {Msg::Ok(out) => proto::DerivationReply {result: Some(proto::derivation_reply::Result::Ok(proto::DerivationResult {destdir: out.into_iter().map(|x| x.to_str().unwrap().to_string()).collect(),paths: Vec::new(),path_patterns: Vec::new(),},)),},Msg::Error(p) => proto::DerivationReply {result: Some(proto::derivation_reply::Result::Error(p)),},Msg::Stdout(p) => proto::DerivationReply {result: Some(proto::derivation_reply::Result::Stdout(p)),},Msg::Stderr(p) => proto::DerivationReply {result: Some(proto::derivation_reply::Result::Stderr(p)),},})});Ok(tonic::Response::new(Box::pin(output_stream)))}async fn add_url(&self,request: tonic::Request<proto::AddUrlRequest>,) -> Result<tonic::Response<proto::DerivationReply>, tonic::Status> {let r = request.into_inner();debug!("add_url request {:?}", r);let p = self.deb_client.http_download(&r.url,match r.hash_algorithm {0 => {let mut h = [0; 32];h.clone_from_slice(&r.hash);Hash::Blake3(h)}1 => {let mut h = [0; 32];h.clone_from_slice(&r.hash);Hash::Sha256(h)}2 => {let mut h = [0; 64];h.clone_from_slice(&r.hash);Hash::Sha512(h)}_ => unreachable!(),},).await.unwrap();Ok(tonic::Response::new(proto::DerivationReply {result: Some(proto::derivation_reply::Result::Ok(proto::DerivationResult {destdir: vec![p.to_str().unwrap().to_string()],paths: Vec::new(),path_patterns: Vec::new(),},)),}))}async fn ubuntu_release(&self,request: tonic::Request<proto::UbuntuReleaseRequest>,) -> Result<tonic::Response<proto::DerivationReply>, tonic::Status> {debug!("ubuntu release {:?}", request);let r = request.into_inner();let h = self.deb_client.in_release(r.release).await.unwrap();let arch = match r.arch {0 => "amd64",1 => "aarch64",_ => unreachable!(),};let p = self.deb_client.packages(&h, &r.repository, arch).await.unwrap();Ok(tonic::Response::new(proto::DerivationReply {result: Some(proto::derivation_reply::Result::Ok(proto::DerivationResult {destdir: vec![p.to_str().unwrap().to_string()],paths: Vec::new(),path_patterns: Vec::new(),},)),}))}type UbuntuPackageStream = ResponseStream;async fn ubuntu_package(&self,request: tonic::Request<proto::UbuntuPackageRequest>,) -> Result<tonic::Response<Self::UbuntuPackageStream>, tonic::Status> {debug!("request {:?}", request);let r = request.into_inner();let index: Result<Vec<_>, _> = r.index.into_iter().map(|index| deb::Index::open(&index)).collect();let index = index.unwrap();let link_extra: Vec<_> = r.link_extra.into_iter().map(|l| {(regex::Regex::new(&l.pkg).unwrap(),regex::Regex::new(&l.dep).unwrap(),)}).collect();let (tx, rx) = tokio::sync::mpsc::channel(200);use crate::extract::Msg;let output_stream = ReceiverStream::new(rx).map(|x| {Ok(match x {Msg::Downloading(p) => proto::DerivationReply {result: Some(proto::derivation_reply::Result::Loading(p)),},Msg::Ok(p) => proto::DerivationReply {result: Some(proto::derivation_reply::Result::Ok(proto::DerivationResult {destdir: p.result.iter().rev().map(|x| x.to_str().unwrap().to_string()).collect(),paths: p.paths.into_iter().filter_map(Arc::into_inner).collect(),path_patterns: Vec::new(),},)),},Msg::Error(e) => proto::DerivationReply {result: Some(proto::derivation_reply::Result::Error(e.to_string())),},})});match download_extract_deps(&index,&self.deb_client,&r.name,&link_extra,PollSender::new(tx.clone()),).await{Ok(p) => {info!("path {:?} {:#?}", r.name, p);tx.send(Msg::Ok(p)).await.unwrap();}Err(e) => tx.send(Msg::Error(e)).await.unwrap(),}Ok(tonic::Response::new(Box::pin(output_stream)))}}
let (sender, receiver) = tokio::sync::mpsc::unbounded_channel::<(crate::container::BuildRequest,tokio::sync::mpsc::Sender<crate::container::Msg>,)>();
let (reader_out, writer_out) = std::io::pipe().unwrap();let (reader_err, writer_err) = std::io::pipe().unwrap();let reader_out_fd = reader_out.as_raw_fd();let reader_err_fd = reader_err.as_raw_fd();let writer_out_fd = writer_out.as_raw_fd();let writer_err_fd = writer_err.as_raw_fd();
let of_string input =tryScanf.sscanf input "%u.%u.%u-%s" (fun v1 v2 v3 v4 -> (v1, v2, v3, Some v4))with _ -> Scanf.sscanf input "%u.%u.%u" (fun v1 v2 v3 -> (v1, v2, v3, None))let to_string (v1, v2, v3, v4) =match v4 with| None -> Printf.sprintf "%u.%u.%u" v1 v2 v3| Some s -> Printf.sprintf "%u.%u.%u-%s" v1 v2 v3 s
("mkdir -p $DESTDIR; tar -xf " ^ List.hd zipped.destdir^ " -C $DESTDIR; cd $DESTDIR/*; echo -n \\{\\\"files\\\":\\{ \>.cargo-checksum.json\n\n=0\n\for f in $(find . -type f -printf '%P\\n'); do\n\if [[ $n -gt 0 ]]; then\n\echo -n , >>.cargo-checksum.json\n\fi\n\echo -n \\\"$f\\\":\\\"$(sha256sum $f | cut -d\" \" -f 1)\\\" >> \.cargo-checksum.json;\n\n=$((n+1))\n\done\n\echo -n \\},\\\"package\\\":\\\"" ^ hash^ "\\\"\\} >> .cargo-checksum.json")
[%string{|mkdir -p $DESTDIRtar -xf %{List.hd zipped.destdir} -C $DESTDIRcd $DESTDIR/*echo -n \{\"files\":\{ >.cargo-checksum.jsonn=0for f in $(find . -type f -printf '%P\n'); doif [[ $n -gt 0 ]]; thenecho -n , >>.cargo-checksum.jsonfiecho -n \"$f\":\"$(sha256sum $f | cut -d" " -f 1)\" >> .cargo-checksum.jsonn=$((n+1))doneecho -n \},\"package\":\"%{hash}\"\} >> .cargo-checksum.json|}]method! install_phase = Lwt.return ""
Lwt.all(List.map(fun pkg ->trylet source = str_from_toml_table "source" pkg inlet checksum = str_from_toml_table "checksum" pkg inlet name = str_from_toml_table "name" pkg inlet version = str_from_toml_table "version" pkg inlet h = fetch_crate source checksum name version inlet drv = (unzip_checksum h checksum :> derivation) inlet* built = drv#build inn_packages := !n_packages + 1;Lwt.return (Some (name, version, source, drv, built))with Not_found -> Lwt.return None)packages)
Lwt_list.map_s(fun pkg ->trylet source = str_from_toml_table "source" pkg inlet checksum = str_from_toml_table "checksum" pkg inlet name = str_from_toml_table "name" pkg inlet version = str_from_toml_table "version" pkg inlet h = fetch_crate source checksum name version inlet drv = (unzip_checksum h checksum :> derivation) inlet* built = drv#build inn_packages := !n_packages + 1;Lwt.return (Some (name, version, source, drv, built))with Not_found -> Lwt.return None)packages
List.fold_left(fun acc x ->match x with| Some (crate_name, version, _, _, built) ->let target =if Hashtbl.find_opt h crate_name = None then (Hashtbl.add h crate_name ();crate_name)else crate_name ^ "-" ^ versioninList.iter self#add_path built.destdir;acc ^ "ln -s " ^ List.hd built.destdir ^ "/" ^ crate_name ^ "-"^ version ^ " vendor/" ^ target ^ "\n"| None -> acc)"" packages
let package_list =List.fold_left(fun acc x ->match x with| Some (crate_name, version, _, _, built) ->let target =if Hashtbl.find_opt h crate_name = None then (Hashtbl.add h crate_name ();crate_name)else crate_name ^ "-" ^ versioninList.iter self#add_path built.destdir;["ln -s ";List.hd built.destdir;"/";crate_name;"-";version;" vendor/";target;"\n";]@ acc| None -> acc)[] packagesinString.concat "" package_list
"export HOME=/home/me\nmkdir -p $HOME\nexport PATH=" ^ List.hd rustc^ "/usr/lib/rust-" ^ rust_version ^ "/bin:" ^ List.hd cargo^ "/usr/lib/rust-" ^ rust_version^ "/bin:$PATH\nmkdir -p vendor .cargo\n" ^ packages^ "\n\cat <<EOF >> .cargo/config.toml\n\[source.crates-io]\n\replace-with = \"vendored-sources\"\n\n\[source.vendored-sources]\n\directory = \"vendor\"\n\EOF\n\cargo metadata --offline --format-version 1" ^ platform
[%string{|export HOME=/home/memkdir -p $HOMEexport PATH=%{List.hd rustc}/usr/lib/rust-%{rust_version}/bin:%{List.hd cargo}/usr/lib/rust-%{rust_version}/bin:$PATHmkdir -p vendor .cargo%{packages}cat <<EOF >> .cargo/config.toml[source.crates-io]replace-with = "vendored-sources"[source.vendored-sources]directory = "vendor"EOFcargo metadata --offline --format-version 1 %{platform}|}]
type deps = { build_dependencies : string; dependencies : string }
type deps = {build_dependencies : (string * string * string * bool * build_result) list;dependencies : (string * string * string * bool * build_result) list;transitive_build_dependencies : (string, build_result) Hashtbl.t;transitive_dependencies : (string, build_result) Hashtbl.t;}
class compiled_crate rust_version rustc cc platform package path deps features =
class compiled_crate ~rust_version ~rustc ~cc ~host ~target ~package ~path~dependencies ~features ~opt_level ~is_top_level =let package_deps =let d = package |> member "dependencies" |> to_list inlet h = Hashtbl.create (List.length d) inList.iter (fun d -> Hashtbl.add h (d |> member "name" |> to_string) d) d;hin
(s ^ "export HOME=/home/me\nmkdir -p $HOME\nexport PATH="^ List.hd rustc ^ "/usr/lib/rust-" ^ rust_version ^ "/bin:$PATH\n")
[%string{|%{s}for i in %{dependencies} %{build_dependencies}; doif [[ -s $i/lib/env ]]; thensource $i/lib/envfidoneexport HOME=/home/memkdir -p $HOMEexport PATH=%{List.hd rustc}/usr/lib/rust-%{rust_version}/bin:$PATH|}]
method private compile_target setup platform target kind dependencies out =let initial_src_path = target |> member "src_path" |> to_string in
method private compile_target setup target build_target kindtransitive_dependencies dependencies out =let initial_src_path = build_target |> member "src_path" |> to_string in
let platform =match platform with Some p -> " --target " ^ p | None -> ""
let dependencies_str =List.fold_left(fun acc (rename, name, h, is_proc_macro, b) ->acc ^ " --extern " ^ underscorize rename ^ "=" ^ List.hd b.destdir^ "/lib/lib" ^ underscorize name ^ "-" ^ h^if is_proc_macro then ".so"else if kind = "lib" then ".rmeta"else ".rlib")"" dependenciesinlet dependencies_str =Hashtbl.fold(fun _ v acc -> acc ^ " -L dependency=" ^ List.hd v.destdir ^ "/lib")transitive_dependencies dependencies_str
!setup ^ "rustc --crate-name " ^ under ^ edition ^ src_path ^ platform^ " --crate-type bin -C linker=gcc" ^ features_args ^ " -C metadata="^ hash ^ " --out-dir " ^ out ^ dependencies ^ " --cap-lints allow\n"
[%string{|%{!setup}set -xrustc --crate-name %{under} %{edition} %{src_path} %{target} --crate-type bin -C linker=gcc $RUSTC_FLAGS %{features_args} -C metadata=%{hash} --out-dir %{out} %{dependencies_str} -L dependency=../lib --cap-lints allow{ set +x; } 2>/dev/null|}]
!setup ^ "rustc --crate-name " ^ under ^ edition ^ src_path^ " --crate-type " ^ kind ^ platform^ " --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2"^ features_args ^ " -C metadata=" ^ hash ^ " --out-dir " ^ out^ " -C extra-filename=-" ^ hash ^ dependencies^ " --cap-lints allow\n";
[%string{|%{!setup}set -xrustc --crate-name %{under} %{edition} %{src_path} --crate-type %{kind} %{target} --emit=dep-info,metadata,link -C linker=gcc -C embed-bitcode=no -C debuginfo=2 $RUSTC_FLAGS %{if kind = "proc-macro" then " -C prefer-dynamic" else ""} %{features_args} -C metadata=%{hash} --out-dir %{out} -C extra-filename=-%{hash} %{dependencies_str} %{if kind = "proc-macro" then " --extern proc_macro" else ""} --cap-lints allow{ set +x; } 2>/dev/null|}];
val accumulated_deps : (string, compiled_crate) Hashtbl.t option ref =ref Nonemethod accumulated_deps =match !accumulated_deps with| Some acc -> Lwt.return acc| None ->let h = Hashtbl.create 256 inlet* _ =Lwt_list.iter_s(fun d ->let dd = Hashtbl.find package_deps d#name inlet is_build =dd |> member "kind" |> to_option to_string = Some "build"inHashtbl.add h d#name d;let* acc = d#accumulated_deps inLwt.return(if not is_build thenHashtbl.iter (fun k v -> Hashtbl.add h k v) accelse ()))dependenciesinaccumulated_deps := Some h;Lwt.return h
let package_deps =let d = package |> member "dependencies" |> to_list inlet h = Hashtbl.create (List.length d) inList.iter(fun d -> Hashtbl.add h (d |> member "name" |> to_string) d)d;hinlet de = ref "" inlet build_de = ref "" in
let de = ref [] inlet h = Hashtbl.create 256 inlet build_h = Hashtbl.create 256 inlet build_de = ref [] in
Lwt.return (x#name, x#hash, b))
Lwt.return (x, x#name, x#hash, b))dependenciesinlet* _ =Lwt_list.iter_s(fun (x, x_name, x_hash, b) ->let d = Hashtbl.find package_deps x_name inlet is_build =d |> member "kind" |> to_option to_string = Some "build"inList.iter self#add_path b.destdir;let name =match d |> member "rename" |> to_option to_string with| Some rename -> rename| None -> x_nameinlet r, h = if is_build then (build_de, build_h) else (de, h) inlet* acc = x#accumulated_deps inlet* _ =Hashtbl.fold(fun name v acc ->if Hashtbl.find_opt h name = None then (let* _ = acc inlet* b = v#build inList.iter self#add_path b.destdir;Hashtbl.add h name b;Lwt.return ())else Lwt.return ())acc (Lwt.return ())inlet is_proc_macro =List.exists(fun t ->List.exists(fun k -> k = "proc-macro")(t |> member "kind" |> to_list |> filter_string))x#targetsinr := (name, x_name, x_hash, is_proc_macro, b) :: !r;Lwt.return ())
List.iter(fun (x_name, x_hash, b) ->let d = Hashtbl.find package_deps x_name inlet is_build =d |> member "kind" |> to_option to_string = Some "build"inList.iter self#add_path b.destdir;let name =match d |> member "rename" |> to_option to_string with| Some rename -> rename| None -> x_nameinlet r = if is_build then build_de else de inr :=!r ^ " -L dependency=target/deps --extern " ^ name ^ "="^ List.hd b.destdir ^ "/lib/lib" ^ x_name ^ "-" ^ x_hash^ ".rlib")deps;
setup :=!setup^ [%string{|export OUT_DIR=$DESTDIR/libexport CARGO_PKG_VERSION="%{version}"export CARGO_PKG_NAME="%{name}"export CARGO_PKG_VERSION_MAJOR=%{v0#Int}export CARGO_PKG_VERSION_MINOR=%{v1#Int}export CARGO_PKG_VERSION_PATCH=%{v2#Int}export CARGO_PKG_VERSION_PATCH=%{Option.value v4 ~default:""}export TARGET="%{Platform.triple target}"export HOST="%{Platform.triple host}"export OPT_LEVEL=%{opt_level}export CARGO_CFG_UNIX=1export CARGO_CFG_TARGET_ABI=""export CARGO_CFG_TARGET_OS=linuxexport CARGO_CFG_TARGET_ENDIAN="%{Platform.string_of_endianness (Platform.endianness target)}"export CARGO_CFG_TARGET_POINTER_WIDTH=%{(Platform.pointer_width target)#Int}export CARGO_CFG_TARGET_ENV="%{Platform.env target}"export CARGO_CFG_TARGET_ARCH="%{Platform.string_of_arch target.arch}"|}];
setup := !setup ^ "target/build-" ^ hash ^ "/" ^ bs_name ^ "\n"
setup :=!setup^ [%string{|CRATENAME=$(echo %{name} | sed -e "s/\(.*\)-sys$/\U\1/" -e "s/-/_/g")CRATEVERSION=$(echo %{version} | sed -e "s/[\.\+-]/_/g")|}]^ String.trim [%blob "run_build_script.sh"]^ [%string "<<< $(target/build-%{hash}/%{bs_name})\n"]
let rust ?(rust_version = "1.84") ?(platform = None) src =
let rec dfs nodes seen result n =match Hashtbl.find_opt seen n with| Some () -> result| None ->let _ = Hashtbl.add seen n () inlet node = Hashtbl.find nodes n inlet deps = node |> member "dependencies" |> to_list |> filter_string inlet result = List.fold_left (dfs nodes seen) result deps innode :: resultlet topo_sort n_nodes nodes =let h = Hashtbl.create n_nodes inlet nodes_list =List.fold_left(fun list node ->let id = node |> member "id" |> to_string inHashtbl.add h id node;id :: list)[] nodesinlet seen = Hashtbl.create n_nodes inList.fold_left (dfs h seen) [] nodes_listlet rust ?(rust_version = "1.84") ?(target = !Platform.target)?(host = !Platform.host) src =
while read cargo; doeval $(echo $cargo | sed -ne 's/cargo:rustc-link-arg=\(.*\)/RUSTC_FLAGS="-C link_arg=\1 $RUSTC_FLAGS"/p')eval $(echo $cargo | sed -ne 's/cargo:rustc-link-lib=\(.*\)/RUSTC_FLAGS="-l\1 $RUSTC_FLAGS"/p')eval $(echo $cargo | sed -ne 's/cargo:rustc-link-search=\(.*\)/RUSTC_FLAGS="-L\1 $RUSTC_FLAGS"/p')eval $(echo $cargo | sed -ne "s/cargo:rustc-cfg=\(.*\)/RUSTC_FLAGS='--cfg \1 $RUSTC_FLAGS'/p")eval $(echo $cargo | sed -ne 's/cargo:rustc-flags=\(.*\)/RUSTC_FLAGS="\1 $RUSTC_FLAGS"/p')echo $cargo | grep -P "^cargo:(?!:?(rustc-|warning=|rerun-if-changed=|rerun-if-env-changed))" \| gawk -F= "/^cargo::metadata=/ { gsub(/-/, \"_\", \$2); print \"export \" toupper(\"DEP_$(echo $CRATENAME)_\" \$2) \"=\" \"\\\"\"\$3\"\\\"\"; next }/^cargo:/ { sub(/^cargo::?/, \"\", \$1); gsub(/-/, \"_\", \$1); print \"export \" toupper(\"DEP_$(echo $CRATENAME)_\" \$1) \"=\" \"\\\"\"\$2\"\\\"\"; print \"export \" toupper(\"DEP_$(echo $CRATENAME)_$(echo $CRATEVERSION)_\" \$1) \"=\" \"\\\"\"\$2\"\\\"\"; next }" >> target/envdone
type arch = Amd64 | Aarch64type endianness = Little | Bigtype platform = { arch : arch; env : string }let default = { arch = Amd64; env = "gnu" }let endianness p = match p.arch with Amd64 | Aarch64 -> Littlelet env p = p.envlet string_of_endianness = function Little -> "little" | Big -> "big"let string_of_arch = function Amd64 -> "x86_64" | Aarch64 -> "aarch64"let pointer_width _ = 64let triple p =match p.arch with| Amd64 -> "x86_64-unknown-linux-gnu"| Aarch64 -> "aarch64-unknown-linux-gnu"let host = ref defaultlet target = ref default
let rec walk_dir_rec encode f buf path path_name =
let rec is_ignored ignored dir path current =match ignored with| [] -> current| (negated, dir_only, h) :: t ->is_ignored t dir path(if negated && current then not (Str.string_match h path 0)else if (dir_only && dir) || not dir_only thencurrent || Str.string_match h path 0else current)let is_ignored_dir ignored path = is_ignored ignored true path falselet is_ignored_file ignored path = is_ignored ignored false path falselet rec walk_dir_rec encode f buf ignored path path_name =
let req =AddPathRequest.make~request:(`Directory(AddPathRequest.Directory.make ~name:path_name~permissions:0o644 ()))()inlet enc = encode req |> Writer.contents inf (Some enc);walk_dir_rec encode f buf path path_name
if not (is_ignored_dir ignored path_name) then (let req =AddPathRequest.make~request:(`Directory(AddPathRequest.Directory.make ~name:path_name~permissions:0o644 ()))()inlet enc = encode req |> Writer.contents inf (Some enc);walk_dir_rec encode f buf ignored path path_name)else Lwt.return ()
let* file = Lwt_unix.openfile path [ O_RDONLY ] 0 inlet ff =AddPathRequest.File.make ~name:path_name~length:stat.st_size ~permissions:0o644 ()inlet req = AddPathRequest.make ~request:(`File ff) () inlet enc = encode req |> Writer.contents inlet () = f (Some enc) inlet rec read_all n =let* r = Lwt_unix.read file buf 0 4096 inif r != 0 thenlet req =AddPathRequest.make~request:(`Contents(AddPathRequest.FileContents.make ~start:n~content:(Bytes.sub buf 0 r) ()))()inlet enc = encode req |> Writer.contents inlet () = f (Some enc) inread_all (n + r)else Lwt.return ()inread_all 0
if not (is_ignored_file ignored path_name) thenlet* file = Lwt_unix.openfile path [ O_RDONLY ] 0 inlet ff =AddPathRequest.File.make ~name:path_name~length:stat.st_size ~permissions:0o644 ()inlet req = AddPathRequest.make ~request:(`File ff) () inlet enc = encode req |> Writer.contents inlet () = f (Some enc) inlet rec read_all n =let* r = Lwt_unix.read file buf 0 4096 inif r != 0 thenlet req =AddPathRequest.make~request:(`Contents(AddPathRequest.FileContents.make ~start:n~content:(Bytes.sub buf 0 r) ()))()inlet enc = encode req |> Writer.contents inlet () = f (Some enc) inread_all (n + r)else Lwt.return ()inread_all 0else Lwt.return ()
let rec read_ignore ign =trylet line = input_line ign inlet rest = read_ignore ign inlet regexp = Str.regexp {|\(\\#\)\|\(^#\)\|\(^!\)\|\(\*\*\)\|\*|} inlet result = ref "" inlet i = ref 0 inlet negated = ref false inlet _ =trylet ended = ref false inwhile not !ended dolet next = Str.search_forward regexp line !i inlet _ =if next - !i > 0 thenresult := !result ^ Str.quote (String.sub line !i (next - !i))else ()inlet m = Str.matched_string line inlet _ =match m with| "!" -> negated := true| "#" -> ended := true| "**" -> result := !result ^ ".*"| "*" -> result := !result ^ "[^/]*"| "\\#" -> result := !result ^ "#"| m -> result := !result ^ Str.quote mini := next + String.length mdonewith Not_found ->result :=!result ^ Str.quote (String.sub line !i (String.length line - !i))inif String.length !result > 0 thenlet dir_only = !result.[String.length !result - 1] = '/' inlet result =if dir_only then String.sub !result 0 (String.length !result - 1)else !resultinif String.contains result '/' then(!negated, dir_only, Str.regexp result) :: restelse(!negated, dir_only, Str.regexp ({|\(\(.+/\)\|^\)|} ^ result)) :: restelse restwith End_of_file -> []
let c =match !backend_conn with None -> failwith "no conn" | Some c -> cinlet* res = add_path c p inlet res, _ = Result.get_ok res inmatch res with| `Ok r -> Lwt.return { destdir = r.destdir; paths = r.paths }| `Error e -> raise (DerivationError e)| _ -> assert false
match !cached with| None ->let c =match !backend_conn with None -> failwith "no conn" | Some c -> cinlet ignored =try read_ignore (open_in ".ignore") with Sys_error _ -> []inlet* res = add_path c ignored p inlet res, _ = Result.get_ok res inlet result =match res with| `Ok r -> { destdir = r.destdir; paths = r.paths }| `Error e -> raise (DerivationError e)| _ -> assert falseincached := Some result;Lwt.return result| Some cached -> Lwt.return cached
(name elpe)(public_name elpe)(modes byte)(libraries grpc-lwt lwt lwt.unix h2 h2-lwt-unix ocaml-protoc-plugin core yojson toml hex))
(name elpe)(public_name elpe)(modes byte)(preprocess (pps ppx_blob ppx_string))(preprocessor_deps (file run_build_script.sh))(libraries grpc-lwt lwt lwt.unix h2 h2-lwt-unix ocaml-protoc-plugin core yojson toml hex))
(targets elpegrpc.ml)(deps(:proto elpegrpc.proto))(action(runprotoc-I."--ocaml_out=annot=[@@deriving show { with_path = false }, eq]:."%{proto})))
(targets elpegrpc.ml)(deps(:proto elpegrpc.proto))(action(runprotoc-I."--ocaml_out=annot=[@@deriving show { with_path = false }, eq]:."%{proto})))
let req =Elpegrpc.Elpe.UbuntuReleaseRequest.make ~release ~arch ~repository ()inlet open Ocaml_protoc_plugin inlet encode, decode =Service.make_client_functions Elpegrpc.Elpe.Elpe.ubuntuReleaseinlet enc = encode req |> Writer.contents inClient.call ~service:"elpe.Elpe" ~rpc:"UbuntuRelease"~do_request:(H2_lwt_unix.Client.request connection ~error_handler:(fun _ ->failwith "Error"))~handler:(Client.Rpc.unary enc ~f:(fun decoder ->let+ decoder = decoder inmatch decoder with| Some decoder -> (Reader.create decoder |> decode |> function| Ok v -> v| Error e ->failwith(Printf.sprintf "Could not decode request: %s"(Result.show_error e)))| None -> Elpegrpc.Elpe.Elpe.Derivation.Response.make ()))()
Lwt_mutex.with_lock ubuntu_release_lock (fun () ->match Hashtbl.find_opt ubuntu_releases (release, arch, repository) with| None ->let req =Elpegrpc.Elpe.UbuntuReleaseRequest.make ~release ~arch ~repository()inlet open Ocaml_protoc_plugin inlet encode, decode =Service.make_client_functions Elpegrpc.Elpe.Elpe.ubuntuReleaseinlet enc = encode req |> Writer.contents inlet* r =Client.call ~service:"elpe.Elpe" ~rpc:"UbuntuRelease"~do_request:(H2_lwt_unix.Client.request connection ~error_handler:(fun _ ->failwith "Error"))~handler:(Client.Rpc.unary enc ~f:(fun decoder ->let+ decoder = decoder inmatch decoder with| Some decoder -> (Reader.create decoder |> decode |> function| Ok v -> v| Error e ->failwith(Printf.sprintf "Could not decode request: %s"(Result.show_error e)))| None -> Elpegrpc.Elpe.Elpe.Derivation.Response.make ()))()inHashtbl.add ubuntu_releases (release, arch, repository) r;Lwt.return r| Some x -> Lwt.return x)
let link_extra =List.map(fun (pkg, dep) -> Elpegrpc.Elpe.LinkExtra.make ~pkg ~dep ())link_extra
let lock =try Hashtbl.find ubuntu_packages_locks (index, name)with Not_found ->let lock = Lwt_mutex.create () inHashtbl.add ubuntu_packages_locks (index, name) lock;lock
let req =Elpegrpc.Elpe.UbuntuPackageRequest.make ~index ~name ~link_extra ()inlet open Ocaml_protoc_plugin inlet encode, decode =Service.make_client_functions Elpegrpc.Elpe.Elpe.ubuntuPackageinlet enc = encode req |> Writer.contents inlet result = ref None inlet* x =Client.call ~service:"elpe.Elpe" ~rpc:"UbuntuPackage"~do_request:(H2_lwt_unix.Client.request connection ~error_handler:(fun _ ->failwith "Error"))~handler:(Client.Rpc.server_streaming enc ~f:(fun responses ->Lwt_stream.iter_s(fun str ->Reader.create str |> decode |> function| Ok (`Ok v) ->result := Some v;Lwt.return ()| Ok (`Loading v) ->Printf.eprintf "Downloading %s\n" v;Stdlib.flush Stdlib.stderr;Lwt.return ()| Error e ->failwith(Printf.sprintf "Could not decode request: %s"(Result.show_error e))| _ -> Lwt.return ())responses))()inmatch x with| Ok _ -> Lwt.return (Ok (Option.get !result))| Error e -> Lwt.return (Error e)type arch = Amd64 | Aarch64
Lwt_mutex.with_lock lock (fun () ->match Hashtbl.find_opt ubuntu_packages (index, name) with| Some x -> Lwt.return x| None ->let link_extra =List.map(fun (pkg, dep) -> Elpegrpc.Elpe.LinkExtra.make ~pkg ~dep ())link_extrainlet req =Elpegrpc.Elpe.UbuntuPackageRequest.make ~index ~name ~link_extra ()inlet open Ocaml_protoc_plugin inlet encode, decode =Service.make_client_functions Elpegrpc.Elpe.Elpe.ubuntuPackageinlet enc = encode req |> Writer.contents inlet result = ref None inlet* x =Client.call ~service:"elpe.Elpe" ~rpc:"UbuntuPackage"~do_request:(H2_lwt_unix.Client.request connection ~error_handler:(fun _ ->failwith "Error"))~handler:(Client.Rpc.server_streaming enc ~f:(fun responses ->Lwt_stream.iter_s(fun str ->Reader.create str |> decode |> function| Ok (`Ok v) ->result := Some v;Lwt.return ()| Ok (`Loading v) ->Printf.eprintf "Downloading %s\n%!" v;Lwt.return ()| Error e ->failwith(Printf.sprintf "Could not decode request: %s"(Result.show_error e))| _ -> Lwt.return ())responses))()inlet result =match x with Ok _ -> Ok (Option.get !result) | Error e -> Error einHashtbl.add ubuntu_packages (index, name) result;Lwt.return result)
let cc =object (self)inherit std_derivationmethod name = "cc"method! build_inputs = Lwt.return [ ubuntu "coreutils"; ubuntu "gcc" ]method! unpack_phase = Lwt.return ""method! build_phase =let* gcc = (ubuntu "gcc")#build inList.iter self#add_path gcc.destdir;Lwt.return("mkdir -p $DESTDIR/usr/bin; ln -s " ^ List.hd gcc.destdir^ "/usr/bin/gcc $DESTDIR/usr/bin/cc")end