LIMIMBOUN3DCIKKTHOK4TAXAWQXNVQXBTMNZYN22DJX4L4FZQH4AC
F2C3TLOI4KEEWK427YMSHZ47352IEYL24PHCW5ELJID7BCSZCRLAC
RBGQQMWXRUO4W2P5GFH24VRIP52IW5BNSE2ZOBXKFIYARDOPBFWAC
AJLMC7UMTMBYBXLOSMGT5PDNX4D7O7LV4GDV7AN25CPOBPRUFYVAC
A7YP2YT6WQ4WDFQTQRHPFC7EFJBCWYHV6HERQD6CUTWUBC4FLBTAC
LTB4YNHF75FTVQG6S5PL46CGS4CMDWZRAEXWEIXUHQHIWVYVKFKAC
UWQB743KR36X6A6JVNK3VH6KMEFUHOUT6ZS2Y2SWIJR55JSF5WQAC
SI454P2VFO6Y6RO2NIOH6ZROURP3OTB2BBXH2F36KJT6YML6YBIAC
ODUDDQRY373JMDR6W2BHUWSKEU6TEICQMNZIAIFKJPNUA5UN3C4QC
BQ4E3XLAVS2AQAUYRGN6SZ4J35KQ2KIH7BBT2JNZXSPKKCGX7DYAC
6MGFBMON6ASGBRQJXY6VUGGZQHXPQSMQJZOOD5LKS2FZODB6ET7AC
BDEVQIAUBDZLH5NNXJ3HD3WK2ART4V6K2FLHMAQVAQYNXGL4TOIQC
LIUJQXB752UIHJFF32LKJ7ECUZ2VDIFDDHBLAUVNBRTPDYNCAUYQC
YJYXDY6AUDJMBUVKKUKM2GO2XVIKZVQFT4SRD4Z7XSTQBTSJ5ZSAC
HX4TXY2DTPEEQPU6CMHSD3X4VX7A4OKV7N3HT3QZKSNHNAZOK7GQC
R7J4254ZZREVMWXXDUM772GPLZSCZSIJKOLEAI33WUF2CPS3X6WQC
KOWYPLMX4TCQANL33QW5VQTKWBQVCMOVLBQUQCTOK6YVQUHZJR5QC
LQBZJUGCEIU56WBYBLE757W2DHPZWJ4JNG3ILB52RDXMXUYAPZCQC
6BIW5YDCZXIH3SNKNTZ3MSHUFGN3VKAYPKI4MWJIFVUY7YQSJJ3AC
use elpe::extract::*;
use elpe::*;
use std::sync::Arc;
use tokio::io::AsyncWriteExt;
use tokio_stream::wrappers::ReceiverStream;
use tokio_util::sync::PollSender;
use tonic::codegen::tokio_stream::StreamExt;
use tracing::*;
pub struct Elpe {
deb_client: elpe::Client,
sender: tokio::sync::mpsc::UnboundedSender<(
crate::container::BuildRequest,
tokio::sync::mpsc::Sender<crate::container::Msg>,
)>,
t: Option<tokio::task::JoinHandle<Result<(), elpe::Error>>>,
}
pub mod proto {
tonic::include_proto!("elpe");
}
impl Elpe {
pub fn new(
deb_client: elpe::Client,
container_channel: elpe::container::ContainerChannel,
) -> Self {
let (sender, receiver) = tokio::sync::mpsc::unbounded_channel::<(
crate::container::BuildRequest,
tokio::sync::mpsc::Sender<crate::container::Msg>,
)>();
let t = tokio::spawn(crate::container::forward(receiver, container_channel));
Elpe {
deb_client,
sender,
t: Some(t),
}
}
pub async fn serve(mut self, addr: std::net::SocketAddr) {
let t = self.t.take().unwrap();
tokio::select! {
_ = tonic::transport::Server::builder()
.add_service(proto::elpe_server::ElpeServer::new(self))
.serve(addr)
=> {}
_ = t => {}
}
}
}
use std::pin::Pin;
type ResponseStream =
Pin<Box<dyn tokio_stream::Stream<Item = Result<proto::DerivationReply, tonic::Status>> + Send>>;
#[tonic::async_trait]
impl proto::elpe_server::Elpe for Elpe {
async fn add_path(
&self,
request: tonic::Request<tonic::Streaming<proto::AddPathRequest>>,
) -> Result<tonic::Response<proto::DerivationReply>, tonic::Status> {
let mut r = request.into_inner();
debug!("add_path");
let mut current_file = None;
let ref store = self.deb_client.store_path();
debug!("store");
let tmp_dir = tempfile::tempdir_in(store).unwrap();
debug!("store {:?}", tmp_dir);
let mut hasher = blake3::Hasher::new();
debug!("loop");
loop {
trace!("waiting for next in stream");
let Some(r) = r.next().await else { break };
let r = r.unwrap();
match r.request {
Some(proto::add_path_request::Request::File(f)) => {
info!("Adding file {:?}", f.name);
hasher.update(b"\0f");
hasher.update(f.name.as_bytes());
hasher.update(b"\0");
let p = tmp_dir.path().join(&f.name);
tokio::fs::create_dir_all(p.parent().unwrap()).await?;
current_file = Some(tokio::fs::File::create(&p).await?)
}
Some(proto::add_path_request::Request::Directory(d)) => {
info!("Adding file {:?}", d.name);
hasher.update(b"\0d");
hasher.update(d.name.as_bytes());
hasher.update(b"\0");
let p = tmp_dir.path().join(&d.name);
tokio::fs::create_dir_all(&p).await.unwrap();
}
Some(proto::add_path_request::Request::Contents(c)) => {
hasher.update(&c.content);
current_file.as_mut().unwrap().write_all(&c.content).await?;
}
None => break,
}
}
debug!("loop done");
let path = store.join(data_encoding::HEXLOWER.encode(hasher.finalize().as_bytes()));
use tokio::io::ErrorKind;
let new = tmp_dir.into_path();
match tokio::fs::rename(&new, &path).await {
Ok(()) => (),
Err(e) if e.kind() == ErrorKind::DirectoryNotEmpty => (),
Err(e) => {
tokio::fs::remove_dir_all(&new).await?;
return Err(e.into());
}
}
info!("add_path extracted to {:?}", path);
Ok(tonic::Response::new(proto::DerivationReply {
result: Some(proto::derivation_reply::Result::Ok(
proto::DerivationResult {
destdir: vec![path.to_str().unwrap().to_string()],
paths: Vec::new(),
path_patterns: Vec::new(),
},
)),
}))
}
type DerivationStream = ResponseStream;
async fn derivation(
&self,
request: tonic::Request<proto::DerivationRequest>,
) -> Result<tonic::Response<ResponseStream>, tonic::Status> {
debug!("derivation request");
let now = std::time::Instant::now();
let r = request.into_inner();
let (tx, rx) = tokio::sync::mpsc::channel(200);
debug!("derivation request: {:?} {:?}", r.name, r.paths);
self.sender
.send((
crate::container::BuildRequest {
name: r.name.clone(),
paths: r.paths,
script: r.builder,
target: r.target,
output_hash: r.output_hash,
},
tx,
))
.unwrap();
use crate::container::Msg;
let output_stream = ReceiverStream::new(rx).map(|x| {
Ok(match x {
Msg::Ok(out) => proto::DerivationReply {
result: Some(proto::derivation_reply::Result::Ok(
proto::DerivationResult {
destdir: out
.into_iter()
.map(|x| x.to_str().unwrap().to_string())
.collect(),
paths: Vec::new(),
path_patterns: Vec::new(),
},
)),
},
Msg::Error(p) => proto::DerivationReply {
result: Some(proto::derivation_reply::Result::Error(p)),
},
Msg::Stdout(p) => proto::DerivationReply {
result: Some(proto::derivation_reply::Result::Stdout(p)),
},
Msg::Stderr(p) => proto::DerivationReply {
result: Some(proto::derivation_reply::Result::Stderr(p)),
},
})
});
info!("request {:?}: {:?}", r.name, now.elapsed());
Ok(tonic::Response::new(Box::pin(output_stream)))
}
async fn add_url(
&self,
request: tonic::Request<proto::AddUrlRequest>,
) -> Result<tonic::Response<proto::DerivationReply>, tonic::Status> {
let r = request.into_inner();
debug!("add_url request {:?}", r);
let p = self
.deb_client
.http_download(
&r.url,
match r.hash_algorithm {
0 => {
let mut h = [0; 32];
h.clone_from_slice(&r.hash);
Hash::Blake3(h)
}
1 => {
let mut h = [0; 32];
h.clone_from_slice(&r.hash);
Hash::Sha256(h)
}
2 => {
let mut h = [0; 64];
h.clone_from_slice(&r.hash);
Hash::Sha512(h)
}
_ => unreachable!(),
},
)
.await
.unwrap();
Ok(tonic::Response::new(proto::DerivationReply {
result: Some(proto::derivation_reply::Result::Ok(
proto::DerivationResult {
destdir: vec![p.to_str().unwrap().to_string()],
paths: Vec::new(),
path_patterns: Vec::new(),
},
)),
}))
}
async fn ubuntu_release(
&self,
request: tonic::Request<proto::UbuntuReleaseRequest>,
) -> Result<tonic::Response<proto::DerivationReply>, tonic::Status> {
debug!("ubuntu release {:?}", request);
let now = std::time::Instant::now();
let r = request.into_inner();
let h = self.deb_client.in_release(r.release.clone()).await.unwrap();
let arch = match r.arch {
0 => "amd64",
1 => "aarch64",
_ => unreachable!(),
};
let p = self
.deb_client
.packages(&h, &r.repository, arch)
.await
.unwrap();
info!(
"ubuntu release request {:?}: {:?}",
r.release,
now.elapsed()
);
Ok(tonic::Response::new(proto::DerivationReply {
result: Some(proto::derivation_reply::Result::Ok(
proto::DerivationResult {
destdir: vec![p.to_str().unwrap().to_string()],
paths: Vec::new(),
path_patterns: Vec::new(),
},
)),
}))
}
type UbuntuPackageStream = ResponseStream;
async fn ubuntu_package(
&self,
request: tonic::Request<proto::UbuntuPackageRequest>,
) -> Result<tonic::Response<Self::UbuntuPackageStream>, tonic::Status> {
let now = std::time::Instant::now();
let r = request.into_inner();
let index: Result<Vec<_>, _> = r
.index
.into_iter()
.map(|index| deb::Index::open(&index))
.collect();
let index = index.unwrap();
let link_extra: Vec<_> = r
.link_extra
.into_iter()
.map(|l| {
(
regex::Regex::new(&l.pkg).unwrap(),
regex::Regex::new(&l.dep).unwrap(),
)
})
.collect();
let (tx, rx) = tokio::sync::mpsc::channel(200);
use crate::extract::Msg;
let output_stream = ReceiverStream::new(rx).map(|x| {
Ok(match x {
Msg::Downloading(p) => proto::DerivationReply {
result: Some(proto::derivation_reply::Result::Loading(p)),
},
Msg::Ok(p) => proto::DerivationReply {
result: Some(proto::derivation_reply::Result::Ok(
proto::DerivationResult {
destdir: p
.result
.iter()
.rev()
.map(|x| x.to_str().unwrap().to_string())
.collect(),
paths: p.paths.into_iter().filter_map(Arc::into_inner).collect(),
path_patterns: Vec::new(),
},
)),
},
Msg::Error(e) => proto::DerivationReply {
result: Some(proto::derivation_reply::Result::Error(e.to_string())),
},
})
});
match download_extract_deps(
&index,
&self.deb_client,
&r.name,
&link_extra,
PollSender::new(tx.clone()),
)
.await
{
Ok(p) => {
info!("path {:?} {:#?}", r.name, p);
tx.send(Msg::Ok(p)).await.unwrap();
}
Err(e) => tx.send(Msg::Error(e)).await.unwrap(),
}
info!("ubuntu package request {:?}: {:?}", r.name, now.elapsed());
Ok(tonic::Response::new(Box::pin(output_stream)))
}
}
use std::sync::Arc;
use tokio::io::AsyncWriteExt;
use tokio_stream::wrappers::ReceiverStream;
use tokio_util::sync::PollSender;
use tonic::codegen::tokio_stream::StreamExt;
use tracing::*;
pub struct Elpe {
deb_client: elpe::Client,
sender: tokio::sync::mpsc::UnboundedSender<(
crate::container::BuildRequest,
tokio::sync::mpsc::Sender<crate::container::Msg>,
)>,
}
mod server;
use std::pin::Pin;
type ResponseStream =
Pin<Box<dyn tokio_stream::Stream<Item = Result<proto::DerivationReply, tonic::Status>> + Send>>;
#[tonic::async_trait]
impl proto::elpe_server::Elpe for Elpe {
async fn add_path(
&self,
request: tonic::Request<tonic::Streaming<proto::AddPathRequest>>,
) -> Result<tonic::Response<proto::DerivationReply>, tonic::Status> {
let mut r = request.into_inner();
debug!("add_path");
let mut current_file = None;
let ref store = self.deb_client.store_path();
debug!("store");
let tmp_dir = tempfile::tempdir_in(store).unwrap();
debug!("store {:?}", tmp_dir);
let mut hasher = blake3::Hasher::new();
debug!("loop");
loop {
trace!("waiting for next in stream");
let Some(r) = r.next().await else { break };
let r = r.unwrap();
match r.request {
Some(proto::add_path_request::Request::File(f)) => {
info!("Adding file {:?}", f.name);
hasher.update(b"\0f");
hasher.update(f.name.as_bytes());
hasher.update(b"\0");
let p = tmp_dir.path().join(&f.name);
tokio::fs::create_dir_all(p.parent().unwrap()).await?;
current_file = Some(tokio::fs::File::create(&p).await?)
}
Some(proto::add_path_request::Request::Directory(d)) => {
info!("Adding file {:?}", d.name);
hasher.update(b"\0d");
hasher.update(d.name.as_bytes());
hasher.update(b"\0");
let p = tmp_dir.path().join(&d.name);
tokio::fs::create_dir_all(&p).await.unwrap();
}
Some(proto::add_path_request::Request::Contents(c)) => {
hasher.update(&c.content);
current_file.as_mut().unwrap().write_all(&c.content).await?;
}
None => break,
}
}
debug!("loop done");
let path = store.join(data_encoding::HEXLOWER.encode(hasher.finalize().as_bytes()));
use tokio::io::ErrorKind;
let new = tmp_dir.into_path();
match tokio::fs::rename(&new, &path).await {
Ok(()) => (),
Err(e) if e.kind() == ErrorKind::DirectoryNotEmpty => (),
Err(e) => {
tokio::fs::remove_dir_all(&new).await?;
return Err(e.into());
}
}
info!("add_path extracted to {:?}", path);
Ok(tonic::Response::new(proto::DerivationReply {
result: Some(proto::derivation_reply::Result::Ok(
proto::DerivationResult {
destdir: vec![path.to_str().unwrap().to_string()],
paths: Vec::new(),
path_patterns: Vec::new(),
},
)),
}))
}
type DerivationStream = ResponseStream;
async fn derivation(
&self,
request: tonic::Request<proto::DerivationRequest>,
) -> Result<tonic::Response<ResponseStream>, tonic::Status> {
debug!("derivation request");
let r = request.into_inner();
let (tx, rx) = tokio::sync::mpsc::channel(200);
self.sender
.send((
crate::container::BuildRequest {
name: r.name,
paths: r.paths,
script: r.builder,
target: r.target,
output_hash: r.output_hash,
},
tx,
))
.unwrap();
use crate::container::Msg;
let output_stream = ReceiverStream::new(rx).map(|x| {
Ok(match x {
Msg::Ok(out) => proto::DerivationReply {
result: Some(proto::derivation_reply::Result::Ok(
proto::DerivationResult {
destdir: out
.into_iter()
.map(|x| x.to_str().unwrap().to_string())
.collect(),
paths: Vec::new(),
path_patterns: Vec::new(),
},
)),
},
Msg::Error(p) => proto::DerivationReply {
result: Some(proto::derivation_reply::Result::Error(p)),
},
Msg::Stdout(p) => proto::DerivationReply {
result: Some(proto::derivation_reply::Result::Stdout(p)),
},
Msg::Stderr(p) => proto::DerivationReply {
result: Some(proto::derivation_reply::Result::Stderr(p)),
},
})
});
Ok(tonic::Response::new(Box::pin(output_stream)))
}
async fn add_url(
&self,
request: tonic::Request<proto::AddUrlRequest>,
) -> Result<tonic::Response<proto::DerivationReply>, tonic::Status> {
let r = request.into_inner();
debug!("add_url request {:?}", r);
let p = self
.deb_client
.http_download(
&r.url,
match r.hash_algorithm {
0 => {
let mut h = [0; 32];
h.clone_from_slice(&r.hash);
Hash::Blake3(h)
}
1 => {
let mut h = [0; 32];
h.clone_from_slice(&r.hash);
Hash::Sha256(h)
}
2 => {
let mut h = [0; 64];
h.clone_from_slice(&r.hash);
Hash::Sha512(h)
}
_ => unreachable!(),
},
)
.await
.unwrap();
Ok(tonic::Response::new(proto::DerivationReply {
result: Some(proto::derivation_reply::Result::Ok(
proto::DerivationResult {
destdir: vec![p.to_str().unwrap().to_string()],
paths: Vec::new(),
path_patterns: Vec::new(),
},
)),
}))
}
async fn ubuntu_release(
&self,
request: tonic::Request<proto::UbuntuReleaseRequest>,
) -> Result<tonic::Response<proto::DerivationReply>, tonic::Status> {
debug!("ubuntu release {:?}", request);
let r = request.into_inner();
let h = self.deb_client.in_release(r.release).await.unwrap();
let arch = match r.arch {
0 => "amd64",
1 => "aarch64",
_ => unreachable!(),
};
let p = self
.deb_client
.packages(&h, &r.repository, arch)
.await
.unwrap();
Ok(tonic::Response::new(proto::DerivationReply {
result: Some(proto::derivation_reply::Result::Ok(
proto::DerivationResult {
destdir: vec![p.to_str().unwrap().to_string()],
paths: Vec::new(),
path_patterns: Vec::new(),
},
)),
}))
}
type UbuntuPackageStream = ResponseStream;
async fn ubuntu_package(
&self,
request: tonic::Request<proto::UbuntuPackageRequest>,
) -> Result<tonic::Response<Self::UbuntuPackageStream>, tonic::Status> {
debug!("request {:?}", request);
let r = request.into_inner();
let index: Result<Vec<_>, _> = r
.index
.into_iter()
.map(|index| deb::Index::open(&index))
.collect();
let index = index.unwrap();
let link_extra: Vec<_> = r
.link_extra
.into_iter()
.map(|l| {
(
regex::Regex::new(&l.pkg).unwrap(),
regex::Regex::new(&l.dep).unwrap(),
)
})
.collect();
let (tx, rx) = tokio::sync::mpsc::channel(200);
use crate::extract::Msg;
let output_stream = ReceiverStream::new(rx).map(|x| {
Ok(match x {
Msg::Downloading(p) => proto::DerivationReply {
result: Some(proto::derivation_reply::Result::Loading(p)),
},
Msg::Ok(p) => proto::DerivationReply {
result: Some(proto::derivation_reply::Result::Ok(
proto::DerivationResult {
destdir: p
.result
.iter()
.rev()
.map(|x| x.to_str().unwrap().to_string())
.collect(),
paths: p.paths.into_iter().filter_map(Arc::into_inner).collect(),
path_patterns: Vec::new(),
},
)),
},
Msg::Error(e) => proto::DerivationReply {
result: Some(proto::derivation_reply::Result::Error(e.to_string())),
},
})
});
match download_extract_deps(
&index,
&self.deb_client,
&r.name,
&link_extra,
PollSender::new(tx.clone()),
)
.await
{
Ok(p) => {
info!("path {:?} {:#?}", r.name, p);
tx.send(Msg::Ok(p)).await.unwrap();
}
Err(e) => tx.send(Msg::Error(e)).await.unwrap(),
}
Ok(tonic::Response::new(Box::pin(output_stream)))
}
}
let (sender, receiver) = tokio::sync::mpsc::unbounded_channel::<(
crate::container::BuildRequest,
tokio::sync::mpsc::Sender<crate::container::Msg>,
)>();
let (reader_out, writer_out) = std::io::pipe().unwrap();
let (reader_err, writer_err) = std::io::pipe().unwrap();
let reader_out_fd = reader_out.as_raw_fd();
let reader_err_fd = reader_err.as_raw_fd();
let writer_out_fd = writer_out.as_raw_fd();
let writer_err_fd = writer_err.as_raw_fd();
let of_string input =
try
Scanf.sscanf input "%u.%u.%u-%s" (fun v1 v2 v3 v4 -> (v1, v2, v3, Some v4))
with _ -> Scanf.sscanf input "%u.%u.%u" (fun v1 v2 v3 -> (v1, v2, v3, None))
let to_string (v1, v2, v3, v4) =
match v4 with
| None -> Printf.sprintf "%u.%u.%u" v1 v2 v3
| Some s -> Printf.sprintf "%u.%u.%u-%s" v1 v2 v3 s
("mkdir -p $DESTDIR; tar -xf " ^ List.hd zipped.destdir
^ " -C $DESTDIR; cd $DESTDIR/*; echo -n \\{\\\"files\\\":\\{ \
>.cargo-checksum.json\n\
n=0\n\
for f in $(find . -type f -printf '%P\\n'); do\n\
if [[ $n -gt 0 ]]; then\n\
echo -n , >>.cargo-checksum.json\n\
fi\n\
echo -n \\\"$f\\\":\\\"$(sha256sum $f | cut -d\" \" -f 1)\\\" >> \
.cargo-checksum.json;\n\
n=$((n+1))\n\
done\n\
echo -n \\},\\\"package\\\":\\\"" ^ hash
^ "\\\"\\} >> .cargo-checksum.json")
[%string
{|mkdir -p $DESTDIR
tar -xf %{List.hd zipped.destdir} -C $DESTDIR
cd $DESTDIR/*
echo -n \{\"files\":\{ >.cargo-checksum.json
n=0
for f in $(find . -type f -printf '%P\n'); do
if [[ $n -gt 0 ]]; then
echo -n , >>.cargo-checksum.json
fi
echo -n \"$f\":\"$(sha256sum $f | cut -d" " -f 1)\" >> .cargo-checksum.json
n=$((n+1))
done
echo -n \},\"package\":\"%{hash}\"\} >> .cargo-checksum.json
|}]
method! install_phase = Lwt.return ""
Lwt.all
(List.map
(fun pkg ->
try
let source = str_from_toml_table "source" pkg in
let checksum = str_from_toml_table "checksum" pkg in
let name = str_from_toml_table "name" pkg in
let version = str_from_toml_table "version" pkg in
let h = fetch_crate source checksum name version in
let drv = (unzip_checksum h checksum :> derivation) in
let* built = drv#build in
n_packages := !n_packages + 1;
Lwt.return (Some (name, version, source, drv, built))
with Not_found -> Lwt.return None)
packages)
Lwt_list.map_s
(fun pkg ->
try
let source = str_from_toml_table "source" pkg in
let checksum = str_from_toml_table "checksum" pkg in
let name = str_from_toml_table "name" pkg in
let version = str_from_toml_table "version" pkg in
let h = fetch_crate source checksum name version in
let drv = (unzip_checksum h checksum :> derivation) in
let* built = drv#build in
n_packages := !n_packages + 1;
Lwt.return (Some (name, version, source, drv, built))
with Not_found -> Lwt.return None)
packages
List.fold_left
(fun acc x ->
match x with
| Some (crate_name, version, _, _, built) ->
let target =
if Hashtbl.find_opt h crate_name = None then (
Hashtbl.add h crate_name ();
crate_name)
else crate_name ^ "-" ^ version
in
List.iter self#add_path built.destdir;
acc ^ "ln -s " ^ List.hd built.destdir ^ "/" ^ crate_name ^ "-"
^ version ^ " vendor/" ^ target ^ "\n"
| None -> acc)
"" packages
let package_list =
List.fold_left
(fun acc x ->
match x with
| Some (crate_name, version, _, _, built) ->
let target =
if Hashtbl.find_opt h crate_name = None then (
Hashtbl.add h crate_name ();
crate_name)
else crate_name ^ "-" ^ version
in
List.iter self#add_path built.destdir;
[
"ln -s ";
List.hd built.destdir;
"/";
crate_name;
"-";
version;
" vendor/";
target;
"\n";
]
@ acc
| None -> acc)
[] packages
in
String.concat "" package_list
"export HOME=/home/me\nmkdir -p $HOME\nexport PATH=" ^ List.hd rustc
^ "/usr/lib/rust-" ^ rust_version ^ "/bin:" ^ List.hd cargo
^ "/usr/lib/rust-" ^ rust_version
^ "/bin:$PATH\nmkdir -p vendor .cargo\n" ^ packages
^ "\n\
cat <<EOF >> .cargo/config.toml\n\
[source.crates-io]\n\
replace-with = \"vendored-sources\"\n\n\
[source.vendored-sources]\n\
directory = \"vendor\"\n\
EOF\n\
cargo metadata --offline --format-version 1" ^ platform
[%string
{|export HOME=/home/me
mkdir -p $HOME
export PATH=%{List.hd rustc}/usr/lib/rust-%{rust_version}/bin:%{List.hd cargo}/usr/lib/rust-%{rust_version}/bin:$PATH
mkdir -p vendor .cargo
%{packages}
cat <<EOF >> .cargo/config.toml
[source.crates-io]
replace-with = "vendored-sources"
[source.vendored-sources]
directory = "vendor"
EOF
cargo metadata --offline --format-version 1 %{platform}
|}]
type deps = { build_dependencies : string; dependencies : string }
type deps = {
build_dependencies : (string * string * string * bool * build_result) list;
dependencies : (string * string * string * bool * build_result) list;
transitive_build_dependencies : (string, build_result) Hashtbl.t;
transitive_dependencies : (string, build_result) Hashtbl.t;
}
class compiled_crate rust_version rustc cc platform package path deps features =
class compiled_crate ~rust_version ~rustc ~cc ~host ~target ~package ~path
~dependencies ~features ~opt_level ~is_top_level =
let package_deps =
let d = package |> member "dependencies" |> to_list in
let h = Hashtbl.create (List.length d) in
List.iter (fun d -> Hashtbl.add h (d |> member "name" |> to_string) d) d;
h
in
(s ^ "export HOME=/home/me\nmkdir -p $HOME\nexport PATH="
^ List.hd rustc ^ "/usr/lib/rust-" ^ rust_version ^ "/bin:$PATH\n")
[%string
{|%{s}
for i in %{dependencies} %{build_dependencies}; do
if [[ -s $i/lib/env ]]; then
source $i/lib/env
fi
done
export HOME=/home/me
mkdir -p $HOME
export PATH=%{List.hd rustc}/usr/lib/rust-%{rust_version}/bin:$PATH
|}]
method private compile_target setup platform target kind dependencies out =
let initial_src_path = target |> member "src_path" |> to_string in
method private compile_target setup target build_target kind
transitive_dependencies dependencies out =
let initial_src_path = build_target |> member "src_path" |> to_string in
let platform =
match platform with Some p -> " --target " ^ p | None -> ""
let dependencies_str =
List.fold_left
(fun acc (rename, name, h, is_proc_macro, b) ->
acc ^ " --extern " ^ underscorize rename ^ "=" ^ List.hd b.destdir
^ "/lib/lib" ^ underscorize name ^ "-" ^ h
^
if is_proc_macro then ".so"
else if kind = "lib" then ".rmeta"
else ".rlib")
"" dependencies
in
let dependencies_str =
Hashtbl.fold
(fun _ v acc -> acc ^ " -L dependency=" ^ List.hd v.destdir ^ "/lib")
transitive_dependencies dependencies_str
!setup ^ "rustc --crate-name " ^ under ^ edition ^ src_path ^ platform
^ " --crate-type bin -C linker=gcc" ^ features_args ^ " -C metadata="
^ hash ^ " --out-dir " ^ out ^ dependencies ^ " --cap-lints allow\n"
[%string
{|
%{!setup}
set -x
rustc --crate-name %{under} %{edition} %{src_path} %{target} --crate-type bin -C linker=gcc $RUSTC_FLAGS %{features_args} -C metadata=%{hash} --out-dir %{out} %{dependencies_str} -L dependency=../lib --cap-lints allow
{ set +x; } 2>/dev/null
|}]
!setup ^ "rustc --crate-name " ^ under ^ edition ^ src_path
^ " --crate-type " ^ kind ^ platform
^ " --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2"
^ features_args ^ " -C metadata=" ^ hash ^ " --out-dir " ^ out
^ " -C extra-filename=-" ^ hash ^ dependencies
^ " --cap-lints allow\n";
[%string
{|
%{!setup}
set -x
rustc --crate-name %{under} %{edition} %{src_path} --crate-type %{kind} %{target} --emit=dep-info,metadata,link -C linker=gcc -C embed-bitcode=no -C debuginfo=2 $RUSTC_FLAGS %{if kind = "proc-macro" then " -C prefer-dynamic" else ""} %{features_args} -C metadata=%{hash} --out-dir %{out} -C extra-filename=-%{hash} %{dependencies_str} %{if kind = "proc-macro" then " --extern proc_macro" else ""} --cap-lints allow
{ set +x; } 2>/dev/null
|}];
val accumulated_deps : (string, compiled_crate) Hashtbl.t option ref =
ref None
method accumulated_deps =
match !accumulated_deps with
| Some acc -> Lwt.return acc
| None ->
let h = Hashtbl.create 256 in
let* _ =
Lwt_list.iter_s
(fun d ->
let dd = Hashtbl.find package_deps d#name in
let is_build =
dd |> member "kind" |> to_option to_string = Some "build"
in
Hashtbl.add h d#name d;
let* acc = d#accumulated_deps in
Lwt.return
(if not is_build then
Hashtbl.iter (fun k v -> Hashtbl.add h k v) acc
else ()))
dependencies
in
accumulated_deps := Some h;
Lwt.return h
let package_deps =
let d = package |> member "dependencies" |> to_list in
let h = Hashtbl.create (List.length d) in
List.iter
(fun d -> Hashtbl.add h (d |> member "name" |> to_string) d)
d;
h
in
let de = ref "" in
let build_de = ref "" in
let de = ref [] in
let h = Hashtbl.create 256 in
let build_h = Hashtbl.create 256 in
let build_de = ref [] in
Lwt.return (x#name, x#hash, b))
Lwt.return (x, x#name, x#hash, b))
dependencies
in
let* _ =
Lwt_list.iter_s
(fun (x, x_name, x_hash, b) ->
let d = Hashtbl.find package_deps x_name in
let is_build =
d |> member "kind" |> to_option to_string = Some "build"
in
List.iter self#add_path b.destdir;
let name =
match d |> member "rename" |> to_option to_string with
| Some rename -> rename
| None -> x_name
in
let r, h = if is_build then (build_de, build_h) else (de, h) in
let* acc = x#accumulated_deps in
let* _ =
Hashtbl.fold
(fun name v acc ->
if Hashtbl.find_opt h name = None then (
let* _ = acc in
let* b = v#build in
List.iter self#add_path b.destdir;
Hashtbl.add h name b;
Lwt.return ())
else Lwt.return ())
acc (Lwt.return ())
in
let is_proc_macro =
List.exists
(fun t ->
List.exists
(fun k -> k = "proc-macro")
(t |> member "kind" |> to_list |> filter_string))
x#targets
in
r := (name, x_name, x_hash, is_proc_macro, b) :: !r;
Lwt.return ())
List.iter
(fun (x_name, x_hash, b) ->
let d = Hashtbl.find package_deps x_name in
let is_build =
d |> member "kind" |> to_option to_string = Some "build"
in
List.iter self#add_path b.destdir;
let name =
match d |> member "rename" |> to_option to_string with
| Some rename -> rename
| None -> x_name
in
let r = if is_build then build_de else de in
r :=
!r ^ " -L dependency=target/deps --extern " ^ name ^ "="
^ List.hd b.destdir ^ "/lib/lib" ^ x_name ^ "-" ^ x_hash
^ ".rlib")
deps;
setup :=
!setup
^ [%string
{|export OUT_DIR=$DESTDIR/lib
export CARGO_PKG_VERSION="%{version}"
export CARGO_PKG_NAME="%{name}"
export CARGO_PKG_VERSION_MAJOR=%{v0#Int}
export CARGO_PKG_VERSION_MINOR=%{v1#Int}
export CARGO_PKG_VERSION_PATCH=%{v2#Int}
export CARGO_PKG_VERSION_PATCH=%{Option.value v4 ~default:""}
export TARGET="%{Platform.triple target}"
export HOST="%{Platform.triple host}"
export OPT_LEVEL=%{opt_level}
export CARGO_CFG_UNIX=1
export CARGO_CFG_TARGET_ABI=""
export CARGO_CFG_TARGET_OS=linux
export CARGO_CFG_TARGET_ENDIAN="%{Platform.string_of_endianness (Platform.endianness target)}"
export CARGO_CFG_TARGET_POINTER_WIDTH=%{(Platform.pointer_width target)#Int}
export CARGO_CFG_TARGET_ENV="%{Platform.env target}"
export CARGO_CFG_TARGET_ARCH="%{Platform.string_of_arch target.arch}"
|}];
setup := !setup ^ "target/build-" ^ hash ^ "/" ^ bs_name ^ "\n"
setup :=
!setup
^ [%string
{|CRATENAME=$(echo %{name} | sed -e "s/\(.*\)-sys$/\U\1/" -e "s/-/_/g")
CRATEVERSION=$(echo %{version} | sed -e "s/[\.\+-]/_/g")
|}]
^ String.trim [%blob "run_build_script.sh"]
^ [%string "<<< $(target/build-%{hash}/%{bs_name})\n"]
let rust ?(rust_version = "1.84") ?(platform = None) src =
let rec dfs nodes seen result n =
match Hashtbl.find_opt seen n with
| Some () -> result
| None ->
let _ = Hashtbl.add seen n () in
let node = Hashtbl.find nodes n in
let deps = node |> member "dependencies" |> to_list |> filter_string in
let result = List.fold_left (dfs nodes seen) result deps in
node :: result
let topo_sort n_nodes nodes =
let h = Hashtbl.create n_nodes in
let nodes_list =
List.fold_left
(fun list node ->
let id = node |> member "id" |> to_string in
Hashtbl.add h id node;
id :: list)
[] nodes
in
let seen = Hashtbl.create n_nodes in
List.fold_left (dfs h seen) [] nodes_list
let rust ?(rust_version = "1.84") ?(target = !Platform.target)
?(host = !Platform.host) src =
while read cargo; do
eval $(echo $cargo | sed -ne 's/cargo:rustc-link-arg=\(.*\)/RUSTC_FLAGS="-C link_arg=\1 $RUSTC_FLAGS"/p')
eval $(echo $cargo | sed -ne 's/cargo:rustc-link-lib=\(.*\)/RUSTC_FLAGS="-l\1 $RUSTC_FLAGS"/p')
eval $(echo $cargo | sed -ne 's/cargo:rustc-link-search=\(.*\)/RUSTC_FLAGS="-L\1 $RUSTC_FLAGS"/p')
eval $(echo $cargo | sed -ne "s/cargo:rustc-cfg=\(.*\)/RUSTC_FLAGS='--cfg \1 $RUSTC_FLAGS'/p")
eval $(echo $cargo | sed -ne 's/cargo:rustc-flags=\(.*\)/RUSTC_FLAGS="\1 $RUSTC_FLAGS"/p')
echo $cargo | grep -P "^cargo:(?!:?(rustc-|warning=|rerun-if-changed=|rerun-if-env-changed))" \
| gawk -F= "/^cargo::metadata=/ { gsub(/-/, \"_\", \$2); print \"export \" toupper(\"DEP_$(echo $CRATENAME)_\" \$2) \"=\" \"\\\"\"\$3\"\\\"\"; next }
/^cargo:/ { sub(/^cargo::?/, \"\", \$1); gsub(/-/, \"_\", \$1); print \"export \" toupper(\"DEP_$(echo $CRATENAME)_\" \$1) \"=\" \"\\\"\"\$2\"\\\"\"; print \"export \" toupper(\"DEP_$(echo $CRATENAME)_$(echo $CRATEVERSION)_\" \$1) \"=\" \"\\\"\"\$2\"\\\"\"; next }" >> target/env
done
type arch = Amd64 | Aarch64
type endianness = Little | Big
type platform = { arch : arch; env : string }
let default = { arch = Amd64; env = "gnu" }
let endianness p = match p.arch with Amd64 | Aarch64 -> Little
let env p = p.env
let string_of_endianness = function Little -> "little" | Big -> "big"
let string_of_arch = function Amd64 -> "x86_64" | Aarch64 -> "aarch64"
let pointer_width _ = 64
let triple p =
match p.arch with
| Amd64 -> "x86_64-unknown-linux-gnu"
| Aarch64 -> "aarch64-unknown-linux-gnu"
let host = ref default
let target = ref default
let rec walk_dir_rec encode f buf path path_name =
let rec is_ignored ignored dir path current =
match ignored with
| [] -> current
| (negated, dir_only, h) :: t ->
is_ignored t dir path
(if negated && current then not (Str.string_match h path 0)
else if (dir_only && dir) || not dir_only then
current || Str.string_match h path 0
else current)
let is_ignored_dir ignored path = is_ignored ignored true path false
let is_ignored_file ignored path = is_ignored ignored false path false
let rec walk_dir_rec encode f buf ignored path path_name =
let req =
AddPathRequest.make
~request:
(`Directory
(AddPathRequest.Directory.make ~name:path_name
~permissions:0o644 ()))
()
in
let enc = encode req |> Writer.contents in
f (Some enc);
walk_dir_rec encode f buf path path_name
if not (is_ignored_dir ignored path_name) then (
let req =
AddPathRequest.make
~request:
(`Directory
(AddPathRequest.Directory.make ~name:path_name
~permissions:0o644 ()))
()
in
let enc = encode req |> Writer.contents in
f (Some enc);
walk_dir_rec encode f buf ignored path path_name)
else Lwt.return ()
let* file = Lwt_unix.openfile path [ O_RDONLY ] 0 in
let ff =
AddPathRequest.File.make ~name:path_name
~length:stat.st_size ~permissions:0o644 ()
in
let req = AddPathRequest.make ~request:(`File ff) () in
let enc = encode req |> Writer.contents in
let () = f (Some enc) in
let rec read_all n =
let* r = Lwt_unix.read file buf 0 4096 in
if r != 0 then
let req =
AddPathRequest.make
~request:
(`Contents
(AddPathRequest.FileContents.make ~start:n
~content:(Bytes.sub buf 0 r) ()))
()
in
let enc = encode req |> Writer.contents in
let () = f (Some enc) in
read_all (n + r)
else Lwt.return ()
in
read_all 0
if not (is_ignored_file ignored path_name) then
let* file = Lwt_unix.openfile path [ O_RDONLY ] 0 in
let ff =
AddPathRequest.File.make ~name:path_name
~length:stat.st_size ~permissions:0o644 ()
in
let req = AddPathRequest.make ~request:(`File ff) () in
let enc = encode req |> Writer.contents in
let () = f (Some enc) in
let rec read_all n =
let* r = Lwt_unix.read file buf 0 4096 in
if r != 0 then
let req =
AddPathRequest.make
~request:
(`Contents
(AddPathRequest.FileContents.make ~start:n
~content:(Bytes.sub buf 0 r) ()))
()
in
let enc = encode req |> Writer.contents in
let () = f (Some enc) in
read_all (n + r)
else Lwt.return ()
in
read_all 0
else Lwt.return ()
let rec read_ignore ign =
try
let line = input_line ign in
let rest = read_ignore ign in
let regexp = Str.regexp {|\(\\#\)\|\(^#\)\|\(^!\)\|\(\*\*\)\|\*|} in
let result = ref "" in
let i = ref 0 in
let negated = ref false in
let _ =
try
let ended = ref false in
while not !ended do
let next = Str.search_forward regexp line !i in
let _ =
if next - !i > 0 then
result := !result ^ Str.quote (String.sub line !i (next - !i))
else ()
in
let m = Str.matched_string line in
let _ =
match m with
| "!" -> negated := true
| "#" -> ended := true
| "**" -> result := !result ^ ".*"
| "*" -> result := !result ^ "[^/]*"
| "\\#" -> result := !result ^ "#"
| m -> result := !result ^ Str.quote m
in
i := next + String.length m
done
with Not_found ->
result :=
!result ^ Str.quote (String.sub line !i (String.length line - !i))
in
if String.length !result > 0 then
let dir_only = !result.[String.length !result - 1] = '/' in
let result =
if dir_only then String.sub !result 0 (String.length !result - 1)
else !result
in
if String.contains result '/' then
(!negated, dir_only, Str.regexp result) :: rest
else
(!negated, dir_only, Str.regexp ({|\(\(.+/\)\|^\)|} ^ result)) :: rest
else rest
with End_of_file -> []
let c =
match !backend_conn with None -> failwith "no conn" | Some c -> c
in
let* res = add_path c p in
let res, _ = Result.get_ok res in
match res with
| `Ok r -> Lwt.return { destdir = r.destdir; paths = r.paths }
| `Error e -> raise (DerivationError e)
| _ -> assert false
match !cached with
| None ->
let c =
match !backend_conn with None -> failwith "no conn" | Some c -> c
in
let ignored =
try read_ignore (open_in ".ignore") with Sys_error _ -> []
in
let* res = add_path c ignored p in
let res, _ = Result.get_ok res in
let result =
match res with
| `Ok r -> { destdir = r.destdir; paths = r.paths }
| `Error e -> raise (DerivationError e)
| _ -> assert false
in
cached := Some result;
Lwt.return result
| Some cached -> Lwt.return cached
(name elpe)
(public_name elpe)
(modes byte)
(libraries grpc-lwt lwt lwt.unix h2 h2-lwt-unix ocaml-protoc-plugin core yojson toml hex))
(name elpe)
(public_name elpe)
(modes byte)
(preprocess (pps ppx_blob ppx_string))
(preprocessor_deps (file run_build_script.sh))
(libraries grpc-lwt lwt lwt.unix h2 h2-lwt-unix ocaml-protoc-plugin core yojson toml hex))
(targets elpegrpc.ml)
(deps
(:proto elpegrpc.proto))
(action
(run
protoc
-I
.
"--ocaml_out=annot=[@@deriving show { with_path = false }, eq]:."
%{proto})))
(targets elpegrpc.ml)
(deps
(:proto elpegrpc.proto))
(action
(run
protoc
-I
.
"--ocaml_out=annot=[@@deriving show { with_path = false }, eq]:."
%{proto})))
let req =
Elpegrpc.Elpe.UbuntuReleaseRequest.make ~release ~arch ~repository ()
in
let open Ocaml_protoc_plugin in
let encode, decode =
Service.make_client_functions Elpegrpc.Elpe.Elpe.ubuntuRelease
in
let enc = encode req |> Writer.contents in
Client.call ~service:"elpe.Elpe" ~rpc:"UbuntuRelease"
~do_request:
(H2_lwt_unix.Client.request connection ~error_handler:(fun _ ->
failwith "Error"))
~handler:
(Client.Rpc.unary enc ~f:(fun decoder ->
let+ decoder = decoder in
match decoder with
| Some decoder -> (
Reader.create decoder |> decode |> function
| Ok v -> v
| Error e ->
failwith
(Printf.sprintf "Could not decode request: %s"
(Result.show_error e)))
| None -> Elpegrpc.Elpe.Elpe.Derivation.Response.make ()))
()
Lwt_mutex.with_lock ubuntu_release_lock (fun () ->
match Hashtbl.find_opt ubuntu_releases (release, arch, repository) with
| None ->
let req =
Elpegrpc.Elpe.UbuntuReleaseRequest.make ~release ~arch ~repository
()
in
let open Ocaml_protoc_plugin in
let encode, decode =
Service.make_client_functions Elpegrpc.Elpe.Elpe.ubuntuRelease
in
let enc = encode req |> Writer.contents in
let* r =
Client.call ~service:"elpe.Elpe" ~rpc:"UbuntuRelease"
~do_request:
(H2_lwt_unix.Client.request connection ~error_handler:(fun _ ->
failwith "Error"))
~handler:
(Client.Rpc.unary enc ~f:(fun decoder ->
let+ decoder = decoder in
match decoder with
| Some decoder -> (
Reader.create decoder |> decode |> function
| Ok v -> v
| Error e ->
failwith
(Printf.sprintf "Could not decode request: %s"
(Result.show_error e)))
| None -> Elpegrpc.Elpe.Elpe.Derivation.Response.make ()))
()
in
Hashtbl.add ubuntu_releases (release, arch, repository) r;
Lwt.return r
| Some x -> Lwt.return x)
let link_extra =
List.map
(fun (pkg, dep) -> Elpegrpc.Elpe.LinkExtra.make ~pkg ~dep ())
link_extra
let lock =
try Hashtbl.find ubuntu_packages_locks (index, name)
with Not_found ->
let lock = Lwt_mutex.create () in
Hashtbl.add ubuntu_packages_locks (index, name) lock;
lock
let req =
Elpegrpc.Elpe.UbuntuPackageRequest.make ~index ~name ~link_extra ()
in
let open Ocaml_protoc_plugin in
let encode, decode =
Service.make_client_functions Elpegrpc.Elpe.Elpe.ubuntuPackage
in
let enc = encode req |> Writer.contents in
let result = ref None in
let* x =
Client.call ~service:"elpe.Elpe" ~rpc:"UbuntuPackage"
~do_request:
(H2_lwt_unix.Client.request connection ~error_handler:(fun _ ->
failwith "Error"))
~handler:
(Client.Rpc.server_streaming enc ~f:(fun responses ->
Lwt_stream.iter_s
(fun str ->
Reader.create str |> decode |> function
| Ok (`Ok v) ->
result := Some v;
Lwt.return ()
| Ok (`Loading v) ->
Printf.eprintf "Downloading %s\n" v;
Stdlib.flush Stdlib.stderr;
Lwt.return ()
| Error e ->
failwith
(Printf.sprintf "Could not decode request: %s"
(Result.show_error e))
| _ -> Lwt.return ())
responses))
()
in
match x with
| Ok _ -> Lwt.return (Ok (Option.get !result))
| Error e -> Lwt.return (Error e)
type arch = Amd64 | Aarch64
Lwt_mutex.with_lock lock (fun () ->
match Hashtbl.find_opt ubuntu_packages (index, name) with
| Some x -> Lwt.return x
| None ->
let link_extra =
List.map
(fun (pkg, dep) -> Elpegrpc.Elpe.LinkExtra.make ~pkg ~dep ())
link_extra
in
let req =
Elpegrpc.Elpe.UbuntuPackageRequest.make ~index ~name ~link_extra ()
in
let open Ocaml_protoc_plugin in
let encode, decode =
Service.make_client_functions Elpegrpc.Elpe.Elpe.ubuntuPackage
in
let enc = encode req |> Writer.contents in
let result = ref None in
let* x =
Client.call ~service:"elpe.Elpe" ~rpc:"UbuntuPackage"
~do_request:
(H2_lwt_unix.Client.request connection ~error_handler:(fun _ ->
failwith "Error"))
~handler:
(Client.Rpc.server_streaming enc ~f:(fun responses ->
Lwt_stream.iter_s
(fun str ->
Reader.create str |> decode |> function
| Ok (`Ok v) ->
result := Some v;
Lwt.return ()
| Ok (`Loading v) ->
Printf.eprintf "Downloading %s\n%!" v;
Lwt.return ()
| Error e ->
failwith
(Printf.sprintf "Could not decode request: %s"
(Result.show_error e))
| _ -> Lwt.return ())
responses))
()
in
let result =
match x with Ok _ -> Ok (Option.get !result) | Error e -> Error e
in
Hashtbl.add ubuntu_packages (index, name) result;
Lwt.return result)
let cc =
object (self)
inherit std_derivation
method name = "cc"
method! build_inputs = Lwt.return [ ubuntu "coreutils"; ubuntu "gcc" ]
method! unpack_phase = Lwt.return ""
method! build_phase =
let* gcc = (ubuntu "gcc")#build in
List.iter self#add_path gcc.destdir;
Lwt.return
("mkdir -p $DESTDIR/usr/bin; ln -s " ^ List.hd gcc.destdir
^ "/usr/bin/gcc $DESTDIR/usr/bin/cc")
end