Rust builder: compiling the crates

pmeunier

Jun 29, 2025, 9:08 PM

6BIW5YDCZXIH3SNKNTZ3MSHUFGN3VKAYPKI4MWJIFVUY7YQSJJ3AC

Dependencies

[2] D7MVXDG2 Adding an example
[3] R7J4254Z Reorganisation of the frontend
[4] LQBZJUGC Adding a basic Rust builder
[5] ODUDDQRY Adding the OCaml interface
[6] UWQB743K First working shell (with ocaml code)
[7] LIUJQXB7 Allow merging two packages based on regular expressions of their name
[8] 7Q4257EP Formatting, cleanup and Rust 2021 compatibility
[9] VAKO4QFF Formatting

Change contents

replacement in examples/hello/build.ml at line 14

[2.397]→[2.397:638](∅→∅)

(ubuntu "gcc" :> derivation);
(ubuntu "libc6-dev" :> derivation);
(ubuntu "make" :> derivation);
(ubuntu "coreutils" :> derivation);
(ubuntu "libstdc++-13-dev" :> derivation);

[2.397]

[2.638]

ubuntu "gcc";
ubuntu "libc6-dev";
ubuntu "make";
ubuntu "coreutils";
ubuntu "libstdc++-13-dev";
edit in elpe/lib/rust.ml at line 3

[4.65]

[4.65]

open Yojson.Basic.Util
edit in elpe/lib/rust.ml at line 41

[4.1156]

[4.1156]

let packages toml =
let packages =
match
Toml.Types.Table.find (Toml.Types.Table.Key.of_string "package") toml
with
| Toml.Types.TArray (Toml.Types.NodeTable arr) -> arr
| _ -> failwith "Unexpected type"
in
let n_packages = ref 0 in
let* packages =
Lwt.all
(List.map
(fun pkg ->
try
let source = str_from_toml_table "source" pkg in
let checksum = str_from_toml_table "checksum" pkg in
let name = str_from_toml_table "name" pkg in
let version = str_from_toml_table "version" pkg in
let h = fetch_crate source checksum name version in
let drv = (unzip h :> derivation) in
let* built = drv#build in
n_packages := !n_packages + 1;
Lwt.return (Some (name, version, source, drv, built))
with Not_found -> Lwt.return None)
packages)
in
replacement in elpe/lib/rust.ml at line 69

[4.1157]→[4.1157:1192](∅→∅)

let rust ?(version = "1.84") src =

[4.1157]

[4.1192]

(* Create a hash table of package id -> derivation of the unzipped
source. *)
let h = Hashtbl.create !n_packages in
List.iter
(fun x ->
match x with
| Some (crate_name, version, source, drv, built) ->
Hashtbl.add h (crate_name, version, source) (drv, built)
| None -> ())
packages;
Lwt.return (h, packages)

let metadata rust_version cargo rustc platform src n_packages packages =
replacement in elpe/lib/rust.ml at line 84

[4.1235]→[4.1235:1260](∅→∅)

method name = "rust"

[4.1235]

[4.1260]

method name = "test"
edit in elpe/lib/rust.ml at line 86

[4.1293]→[4.1293:1370](∅→∅)

method! build_inputs = Lwt.return [ (ubuntu "coreutils" :> derivation) ]
replacement in elpe/lib/rust.ml at line 87

[4.1371]→[4.1371:1398](∅→∅)

method packages toml =

[4.1371]

[4.1398]

method! build_phase =
List.iter self#add_path cargo;
List.iter self#add_path rustc;
let platform =
match platform with Some s -> " --filter-platform " ^ s | None -> ""
in
replacement in elpe/lib/rust.ml at line 94

[4.1419]→[4.1419:1630](∅→∅)

match
Toml.Types.Table.find (Toml.Types.Table.Key.of_string "package") toml
with
| Toml.Types.TArray (Toml.Types.NodeTable arr) -> arr
| _ -> failwith "Unexpected type"

[4.1419]

[4.1630]

let h = Hashtbl.create n_packages in
List.fold_left
(fun acc x ->
match x with
| Some (crate_name, version, _, _, built) ->
let target =
if Hashtbl.find_opt h crate_name = None then
crate_name ^ "-" ^ version
else crate_name
in
acc ^ "ln -s " ^ List.hd built.destdir ^ "/" ^ crate_name ^ "-"
^ version ^ " vendor/" ^ target ^ "\n"
| None -> acc)
"" packages
in
let setup =
"export HOME=/home/me\nmkdir -p $HOME\nexport PATH=" ^ List.hd rustc
^ "/usr/lib/rust-" ^ rust_version ^ "/bin:" ^ List.hd cargo
^ "/usr/lib/rust-" ^ rust_version
^ "/bin:$PATH\nmkdir -p vendor .cargo\n" ^ packages
^ "\n\
cat <<EOF >> .cargo/config.toml\n\
[source.crates-io]\n\
replace-with = \"vendored-sources\"\n\n\
[source.vendored-sources]\n\
directory = \"vendor\"\n\
EOF\n\
cargo metadata --offline --format-version 1" ^ platform
edit in elpe/lib/rust.ml at line 123

[4.1639]

[4.1639]

Lwt.return setup

method! build_inputs = Lwt.return [ (ubuntu "coreutils" :> derivation) ]
end
replacement in elpe/lib/rust.ml at line 128

[4.1640]→[4.1640:2324](∅→∅)

let* packages =
Lwt.all
(List.map
(fun pkg ->
try
let source = str_from_toml_table "source" pkg in
let checksum = str_from_toml_table "checksum" pkg in
let name = str_from_toml_table "name" pkg in
let version = str_from_toml_table "version" pkg in
let h = fetch_crate source checksum name version in
let* drv = self#derivation (unzip h :> derivation) in
Lwt.return (Some (name, version, h, drv))
with Not_found -> Lwt.return None)
packages)
in
let h = Hashtbl.create 16 in

[4.1640]

[4.2324]

let compile_crate rust_version rustc platform package path deps features =
let target_platform = "x86_64-unknown-linux-gnu" in
object (self)
inherit std_derivation
method name = package |> member "name" |> to_string
method! src = Lwt.return path

method! post_setup =
List.iter self#add_path rustc;
replacement in elpe/lib/rust.ml at line 138

[4.2341]→[4.2341:2913](∅→∅)

(List.fold_left
(fun acc x ->
match x with
| Some (crate_name, version, _, drv) ->
let target =
if Hashtbl.find_opt h crate_name = None then
let _ = Hashtbl.add h crate_name () in
crate_name ^ "-" ^ version
else crate_name
in
acc ^ "ln -s " ^ List.hd drv.destdir ^ "/" ^ crate_name ^ "-"
^ version ^ " vendor/" ^ target ^ "\n"
| None -> acc)
"" packages)

[4.2341]

[4.2913]

("export HOME=/home/me\nmkdir -p $HOME\nexport PATH=" ^ List.hd rustc
^ "/usr/lib/rust-" ^ rust_version ^ "/bin:$PATH\n")
replacement in elpe/lib/rust.ml at line 142

[4.2940]→[4.2940:3114](∅→∅)

let* src = self#src in
let* src_built = src#build in
let lock =
Stdlib.open_in
(Filename.concat (List.hd src_built.destdir) "Cargo.lock")

[4.2940]

[4.3114]

let platform =
match platform with Some s -> " --filter-platform " ^ s | None -> ""
in
let features =
String.concat ", " (List.map (fun x -> "\"" ^ x ^ "\"") features)
replacement in elpe/lib/rust.ml at line 148

[4.3123]→[4.3123:3274](∅→∅)

let parsed =
match Toml.Parser.from_channel lock with
| `Ok toml -> toml
| `Error (err, _) -> failwith ("Error: " ^ err)

[4.3123]

[4.3274]

let setup = ref "set -x\nrustc --print sysroot\n" in
let dependencies = "" in
let compile_target target kind =
let src_path = target |> member "src_path" |> to_string in
let name = target |> member "name" |> to_string in
let edition =
try " --edition=" ^ (target |> member "edition" |> to_string) ^ " "
with Not_found -> (
try " --edition=" ^ (package |> member "edition" |> to_string) ^ " "
with Not_found -> " ")
in

setup :=
!setup ^ "rustc --crate-name " ^ name ^ edition ^ src_path
^ " --crate-type " ^ kind
^ " --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 \
--cfg 'feature=\"cargo-build \"' --cfg 'feature=\"default\"' \
--check-cfg 'cfg(docsrs,test)' --check-cfg 'cfg(feature, values("
^ features
^ "))' -C metadata=db5f6b051938318f -C \
extra-filename=-99d0a40c2aab0586 --out-dir $DESTDIR/lib "
^ dependencies ^ " --cap-lints allow\n"
replacement in elpe/lib/rust.ml at line 172

[4.3283]→[4.3283:3475](∅→∅)

let* packages = self#packages parsed in
let metadata =
object (self)
inherit std_derivation
method name = "test"
method! src = Lwt.return src

[4.3283]

[4.3475]

List.iter
(fun target ->
List.iter
(fun kind ->
match kind with
| "lib" | "rlib" | "dylib" | "cdylib" | "staticlib" | "proc-macro"
| "bin" ->
compile_target target kind
| _ -> ())
(target |> member "kind" |> to_list |> filter_string))
(package |> member "targets" |> to_list);
Lwt.return !setup
replacement in elpe/lib/rust.ml at line 185

[4.3476]→[4.3476:4398](∅→∅)

method! build_phase =
let* rustc = self#derivation (ubuntu ("rustc-" ^ version)) in
let rustc = List.hd rustc.destdir in
let* cargo = self#derivation (ubuntu ("cargo-" ^ version)) in
let cargo = List.hd cargo.destdir in
let setup =
"export HOME=/home/me\nmkdir -p $HOME\nexport PATH=" ^ rustc
^ "/usr/lib/rust-" ^ version ^ "/bin:" ^ cargo ^ "/usr/lib/rust-"
^ version ^ "/bin:$PATH\nmkdir -p vendor .cargo\n" ^ packages
^ "\n\
cat <<EOF >> .cargo/config.toml\n\
[source.crates-io]\n\
replace-with = \"vendored-sources\"\n\n\
[source.vendored-sources]\n\
directory = \"vendor\"\n\
EOF\n\
cargo metadata --offline --format-version 1"
in
Lwt.return setup

[4.3476]

[4.4398]

method! build_inputs = Lwt.return [ (ubuntu "coreutils" :> derivation) ]
end
replacement in elpe/lib/rust.ml at line 188

[4.4399]→[4.4399:4506](∅→∅)

method! build_inputs =
Lwt.return [ (ubuntu "coreutils" :> derivation) ]
end

[4.4399]

[4.4506]

let rust ?(rust_version = "1.84") ?(platform = None) ?(crate = None) src =
let* src_built = src#build in
let lock =
Stdlib.open_in (Filename.concat (List.hd src_built.destdir) "Cargo.lock")
in
let parsed =
match Toml.Parser.from_channel lock with
| `Ok toml -> toml
| `Error (err, _) -> failwith ("Error: " ^ err)
in
let* packages_drv, packages = packages parsed in
let n_packages = Hashtbl.length packages_drv in

let* rustc = (ubuntu ("rustc-" ^ rust_version))#build in
let rustc = rustc.destdir in
let* cargo = (ubuntu ("cargo-" ^ rust_version))#build in
let cargo = cargo.destdir in

let metadata =
metadata rust_version cargo rustc platform src n_packages packages
in
let buf = metadata#set_stdout in
let* _ = metadata#build in
let meta_json = Yojson.Basic.from_string (Buffer.contents buf) in
let resolved = Hashtbl.create n_packages in

let nodes = meta_json |> member "resolve" |> member "nodes" |> to_list in

let packages_meta = Hashtbl.create n_packages in
List.iter
(fun pkg -> Hashtbl.add packages_meta (pkg |> member "id" |> to_string) pkg)
(meta_json |> member "packages" |> to_list);

List.iter
(fun node ->
let id = node |> member "id" |> to_string in

let meta = Hashtbl.find packages_meta id in
let name = meta |> member "name" |> to_string in
let version = meta |> member "version" |> to_string in
let drv =
try
let drv, _ =
Hashtbl.find packages_drv
(name, version, meta |> member "source" |> to_string)
in
drv
with Not_found | Yojson.Basic.Util.Type_error _ -> src
replacement in elpe/lib/rust.ml at line 238

[4.4516]→[4.4516:4792](∅→∅)

let buf = metadata#set_stdout in
let* _ = metadata#build in
let meta_json = Yojson.Basic.from_string (Buffer.contents buf) in
let open Yojson.Basic.Util in
let members =
meta_json |> member "workspace_members" |> to_list |> filter_string

[4.4516]

[4.4792]

let dependencies =
List.map
(fun dep -> Hashtbl.find resolved dep)
(node |> member "dependencies" |> to_list |> filter_string)
in
let deps = node |> member "deps" |> to_list in
let features = node |> member "features" |> to_list |> filter_string in

let compiled =
compile_crate rust_version rustc platform meta drv deps features
replacement in elpe/lib/rust.ml at line 249

[4.4801]→[4.4801:4862](∅→∅)

let _ = List.map (fun x -> print_endline x) members in

[4.4801]

[4.4862]

Hashtbl.add resolved id compiled)
nodes;
replacement in elpe/lib/rust.ml at line 252

[4.4863]→[4.4863:4889](∅→∅)

Lwt.return ""
end

[4.4863]

Lwt.return
(List.map (Hashtbl.find resolved)
(meta_json |> member "workspace_members" |> to_list |> filter_string))
replacement in elpe/lib/elpe.ml at line 119

[5.62151]→[5.72695:72823](∅→∅)

let last_built_module : std_derivation option ref = ref None
let build (spec : std_derivation) = last_built_module := Some spec

[5.62151]

let last_built_module : derivation Lwt.t option ref = ref None
let build_lwt (spec : derivation Lwt.t) = last_built_module := Some spec
let build (spec : derivation) = last_built_module := Some (Lwt.return spec)
replacement in elpe/lib/dune at line 6

[5.71]→[5.62510:62511](∅→∅),[5.231]→[5.62510:62511](∅→∅),[3.406]→[5.62510:62511](∅→∅),[5.72940]→[5.62510:62511](∅→∅),[5.62510]→[5.62510:62511](∅→∅)

[3.406]

[5.62511]

(env (dev (flags (:standard -warn-error -A))))
edit in elpe/lib/derivation.ml at line 195

[3.6814]

[3.6814]

method add_path r =
if Hashtbl.find_opt extra_paths_h r = None then
let _ = Hashtbl.add extra_paths_h r () in
extra_paths := r :: !extra_paths
else ()
replacement in elpe/lib/derivation.ml at line 203

[3.6888]→[3.6888:7132](∅→∅)

List.fold_right
(fun r () ->
if Hashtbl.find_opt extra_paths_h r = None then
let _ = Hashtbl.add extra_paths_h r () in
extra_paths := r :: !extra_paths
else ())
path.destdir ();

[3.6888]

[3.7132]

List.fold_right (fun r () -> self#add_path r) path.destdir ();
edit in elpe/lib/derivation.ml at line 281

[3.9639]

[3.9639]

val cached_build = ref None
replacement in elpe/lib/derivation.ml at line 284

[3.9659]→[3.9659:10404](∅→∅)

let* phases =
Lwt.all
[
self#setup;
self#unpack_phase;
self#configure_phase;
self#build_phase;
self#install_phase;
]
in
let builder = String.concat "\n" phases in
let c =
match !backend_conn with None -> failwith "no conn" | Some c -> c
in
let* output_hash = self#output_hash in
let* r =
derivation c ~name:self#name ~builder ~paths:!extra_paths
~stdout:!(self#stdout) ~stderr:!(self#stderr) ~target:self#target
~output_hash
in
match r with
| `Ok r -> Lwt.return { destdir = r.destdir; paths = r.paths }
| `Error e -> failwith e
| _ -> assert false

[3.9659]

[3.10404]

match !cached_build with
| Some cached -> Lwt.return cached
| None -> (
let* phases =
Lwt.all
[
self#setup;
self#unpack_phase;
self#configure_phase;
self#build_phase;
self#install_phase;
]
in
let builder = String.concat "\n" phases in
let c =
match !backend_conn with None -> failwith "no conn" | Some c -> c
in
let* output_hash = self#output_hash in
let* r =
derivation c ~name:self#name ~builder ~paths:!extra_paths
~stdout:!(self#stdout) ~stderr:!(self#stderr) ~target:self#target
~output_hash
in
match r with
| `Ok r ->
let c = { destdir = r.destdir; paths = r.paths } in
cached_build := Some c;
Lwt.return c
| `Error e -> failwith e
| _ -> assert false)
edit in default.nix at line 119

[5.69926]

[3.11908]

ocamlPackages.toml
ocamlPackages.yojson