Parse rustc self-profile data using `analyzeme`

finchie
May 8, 2024, 6:55 AM
L6QJNN62E4LA65OOTWGFK25QQNIJGLK3SJEEKAHOB5B7RXDUEZUQC

Dependencies

  • [2] BRXHJFU7 Refactor measurements into `annotations` module
  • [3] JVYWRCPT Add basic chart visualisation
  • [4] ZPFD3275 Switch from `cargo_metadata`+`petgraph` to `guppy`
  • [5] Q3Z6XMP5 Migrate dependency tree to `petgraph::Graph`
  • [6] UQJO24KB Use `forceatlas2` to construct graph layout
  • [*] 7CVIL7UJ Create simple metadata parser
  • [*] UXJFRBBL Move graph functionality into `graph` module
  • [*] LOR3KOXG Parse JSON output from `cargo build --timings`

Change contents

  • edit in src/main.rs at line 1
    [8.49]
    [9.0]
    #![feature(let_chains)]
  • file addition: self_profile.rs (----------)
    [2.3345]
    use std::cell::OnceCell;
    use std::collections::HashMap;
    use std::convert::Infallible;
    use std::str::FromStr;
    use analyzeme::ProfilingData;
    use camino::Utf8PathBuf;
    use cargo_metadata::Edition;
    use serde::de::value::StrDeserializer;
    use serde::Deserialize;
    /// When a crate name is set to 3 underscores (`___`), we assume this is cargo
    /// probing rustc without actually compiling anything. Clarification:
    /// https://rust-lang.zulipchat.com/#narrow/stream/246057-t-cargo/topic/New.20visualizations.20for.20cargo-timing.2Ehtml/near/436821099
    const CARGO_PROBE_CRATE_NAME: &str = "___";
    const PROFDATA_EXTENSION: &str = "mm_profdata";
    const PROFILES_DIR: &str = "target/profiles";
    #[derive(Clone, Copy, Debug, Deserialize)]
    #[serde(rename_all = "kebab-case")]
    pub enum Emit {
    #[serde(rename = "asm")]
    Assembly,
    DepInfo,
    Link,
    #[serde(rename = "llvm-bc")]
    LlvmBitcode,
    LlvmIr,
    Metadata,
    Mir,
    #[serde(rename = "obj")]
    Object,
    }
    #[derive(Clone, Debug)]
    pub enum RustcFilename {
    StandardInput,
    File(Utf8PathBuf),
    }
    impl FromStr for RustcFilename {
    type Err = Infallible;
    fn from_str(s: &str) -> Result<Self, Self::Err> {
    Ok(match s {
    // Cargo sets the filename to be a single hyphen (`-`: U+002D) when probing rustc
    // This instructs rustc to read the file from standard input
    "-" => Self::StandardInput,
    _ => {
    let path = Utf8PathBuf::from_str(s).unwrap();
    // TODO: some build-scripts seem to generate relative paths, so the following code would fail:
    // assert!(path_buf.is_file(), "Path does not exist: {path_buf:#?}");
    Self::File(path)
    }
    })
    }
    }
    #[derive(Clone, Debug)]
    // TODO: investigate using `rustc --print` to validate this information
    // TODO: reference data directly
    pub struct RustcArgs {
    pub filename: RustcFilename,
    pub crate_name: String,
    pub edition: Edition,
    pub crate_types: Vec<super::timings::TargetKind>,
    pub emit: Vec<Emit>,
    pub externs: Vec<String>,
    pub codegen_options: Vec<String>,
    pub unstable_options: Vec<String>,
    pub extra_args: HashMap<String, Vec<String>>,
    }
    impl RustcArgs {
    fn single_value(map: &mut HashMap<String, Vec<String>>, key: &str) -> Option<String> {
    let mut values = map.remove(key)?;
    assert_eq!(values.len(), 1);
    Some(values.remove(0))
    }
    pub fn parse_from(source: &str) -> Self {
    let mut args_map: HashMap<String, Vec<String>> = HashMap::new();
    let mut arg_chunks = source.split_ascii_whitespace();
    let command_name = arg_chunks.next().expect("Arguments cannot be empty");
    assert_eq!(command_name, "rustc");
    let filename = OnceCell::new();
    while let Some(chunk) = arg_chunks.next() {
    let (prefix, arg) = if let Some(long_arg) = chunk.strip_prefix("--") {
    ("--", long_arg)
    } else if let Some(short_arg) = chunk.strip_prefix('-')
    && chunk != "-"
    {
    ("-", short_arg)
    } else {
    let insertion_result = filename.set(chunk.to_string());
    if let Err(previous_filename) = insertion_result {
    panic!("Found multiple unexpected arguments, unable to parse filename. First: `{previous_filename}` Second: `{chunk}`")
    }
    continue;
    };
    let (arg_name, value) = if let Some((name, value)) = chunk.split_once('=') {
    (name.strip_prefix(prefix).unwrap(), value)
    } else {
    let value = arg_chunks.next().expect(&format!(
    "No arguments left, but expected value for argument: `{chunk}`"
    ));
    (arg, value)
    };
    args_map
    .entry(arg_name.to_string())
    .or_default()
    .push(value.to_string());
    }
    Self {
    filename: RustcFilename::from_str(
    filename.get().expect("Missing filename in arguments"),
    )
    .unwrap(),
    crate_name: Self::single_value(&mut args_map, "crate-name").unwrap(),
    edition: Self::single_value(&mut args_map, "edition")
    .map(|edition| deserialize_string(&edition))
    .unwrap_or(Edition::E2015),
    crate_types: args_map
    .remove("crate-type")
    .map(deserialize_vec)
    .unwrap_or_default(),
    emit: args_map
    .remove("emit")
    .map(deserialize_vec)
    .unwrap_or_default(),
    externs: args_map.remove("extern").unwrap_or_default(),
    codegen_options: args_map.remove("C").unwrap_or_default(),
    unstable_options: args_map.remove("Z").unwrap_or_default(),
    extra_args: args_map,
    }
    }
    }
    #[derive(Debug)]
    pub struct SelfProfile {
    pub args: RustcArgs,
    pub data: ProfilingData,
    }
    #[derive(Debug)]
    pub struct ProfileCollection {
    pub crates: Vec<SelfProfile>,
    pub probes: Vec<SelfProfile>,
    }
    impl ProfileCollection {
    pub fn new() -> Self {
    let mut crate_profiles = Vec::new();
    let mut probe_profiles = Vec::new();
    let valid_profiles = std::fs::read_dir(PROFILES_DIR)
    .unwrap()
    .into_iter()
    .filter_map(|potential_entry| potential_entry.ok())
    .filter_map(|entry| Utf8PathBuf::from_path_buf(entry.path()).ok())
    .filter(|path| path.is_file())
    .filter(|path| path.extension() == Some(&PROFDATA_EXTENSION));
    for profile_path in valid_profiles {
    let relative_filename = profile_path.file_name().unwrap();
    let (file_name, _extension) = relative_filename.rsplit_once('.').unwrap();
    // Self-profiles have paths like:
    // CRATE_NAME-RUSTC_PID.mm_profdata
    // We need to special-case the crate name cargo uses when probing rustc;
    // these profiles are not associated to any specific crate in the graph
    let (crate_name, process_id) = file_name.rsplit_once('-').unwrap();
    // Make sure PID is only numbers
    assert!(
    process_id.chars().all(|character| character.is_numeric()),
    "{}",
    process_id
    );
    let data = ProfilingData::new(profile_path.as_std_path()).unwrap();
    let cmd = &data.metadata().cmd;
    let args = RustcArgs::parse_from(cmd);
    match args.filename {
    RustcFilename::StandardInput => {
    if crate_name != CARGO_PROBE_CRATE_NAME {
    if let Some(suffix) = crate_name.strip_prefix("probe") {
    assert!(
    suffix.chars().all(|c| c.is_numeric()),
    "Unexpected suffix for probe crate name: {suffix}"
    );
    } else {
    panic!("Unexpected probe with name: {crate_name}")
    }
    }
    probe_profiles.push(SelfProfile { args, data });
    }
    RustcFilename::File(_) => {
    crate_profiles.push(SelfProfile { args, data });
    }
    }
    }
    Self {
    crates: crate_profiles,
    probes: probe_profiles,
    }
    }
    }
    fn deserialize_string<'a, T: Deserialize<'a>>(source: &str) -> T {
    let deserializer = StrDeserializer::<serde::de::value::Error>::new(source);
    T::deserialize(deserializer).unwrap()
    }
    fn deserialize_vec<'a, T: Deserialize<'a>>(source: Vec<String>) -> Vec<T> {
    source
    .iter()
    .map(|arg| arg.split_terminator(','))
    .flatten()
    .map(deserialize_string::<T>)
    .collect()
    }
  • edit in src/annotations/mod.rs at line 11
    [2.3746]
    [2.3746]
    mod self_profile;
  • edit in src/annotations/mod.rs at line 50
    [2.4555]
    [2.4555]
    let profile_data = self_profile::ProfileCollection::new();
    dbg!(profile_data.probes.len(), profile_data.crates.len());
  • edit in Cargo.toml at line 7
    [8.1040]
    [10.2890]
    analyzeme = { git = "https://github.com/rust-lang/measureme" }
    camino = "1.1.6"
  • edit in Cargo.lock at line 16
    [3.1619]
    [3.1619]
    ]
    [[package]]
    name = "analyzeme"
    version = "9.2.0"
    source = "git+https://github.com/rust-lang/measureme?tag=9.2.0#9f51cde2e5dd3ef0392f0f6a7201f4946502ef41"
    dependencies = [
    "byteorder",
    "measureme 9.2.0",
    "memchr",
    "rustc-hash",
    "serde",
    "serde_json",
  • edit in Cargo.lock at line 32
    [3.1783]
    [3.1783]
    name = "analyzeme"
    version = "11.0.1"
    source = "git+https://github.com/rust-lang/measureme#7bbe7a985a1ed5cc1043e1c8582b4c7e17efc5b7"
    dependencies = [
    "analyzeme 9.2.0",
    "decodeme 10.1.2",
    "decodeme 11.0.1",
    "measureme 10.1.2",
    "measureme 11.0.1",
    "memchr",
    "rustc-hash",
    "serde",
    ]
    [[package]]
  • edit in Cargo.lock at line 74
    [3.2347]
    [8.1219]
    name = "byteorder"
    version = "1.5.0"
    source = "registry+https://github.com/rust-lang/crates.io-index"
    checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
    [[package]]
  • edit in Cargo.lock at line 185
    [3.2076]
    [3.2790]
    [[package]]
    name = "decodeme"
    version = "10.1.2"
    source = "git+https://github.com/rust-lang/measureme?tag=10.1.2#f9f84d1a79c46e9927926c177c33eb3ea3c72979"
    dependencies = [
    "measureme 10.1.2",
    "memchr",
    "rustc-hash",
    "serde",
    "serde_json",
    ]
    [[package]]
    name = "decodeme"
    version = "11.0.1"
    source = "git+https://github.com/rust-lang/measureme#7bbe7a985a1ed5cc1043e1c8582b4c7e17efc5b7"
    dependencies = [
    "measureme 11.0.1",
    "memchr",
    "rustc-hash",
    "serde",
    "serde_json",
    ]
  • edit in Cargo.lock at line 214
    [8.2014]
    [10.3002]
    "analyzeme 11.0.1",
    "camino",
  • replacement in Cargo.lock at line 262
    [3.3303][3.3303:3319]()
    "parking_lot",
    [3.3303]
    [3.3319]
    "parking_lot 0.12.1",
  • edit in Cargo.lock at line 349
    [3.2262]
    [3.2262]
    ]
    [[package]]
    name = "instant"
    version = "0.1.12"
    source = "registry+https://github.com/rust-lang/crates.io-index"
    checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c"
    dependencies = [
    "cfg-if",
  • edit in Cargo.lock at line 398
    [3.3986]
    [3.3986]
    name = "measureme"
    version = "9.2.0"
    source = "git+https://github.com/rust-lang/measureme?tag=9.2.0#9f51cde2e5dd3ef0392f0f6a7201f4946502ef41"
    dependencies = [
    "log",
    "memmap2",
    "parking_lot 0.11.2",
    "perf-event-open-sys 1.0.1",
    "rustc-hash",
    "smallvec",
    ]
    [[package]]
    name = "measureme"
    version = "10.1.2"
    source = "git+https://github.com/rust-lang/measureme?tag=10.1.2#f9f84d1a79c46e9927926c177c33eb3ea3c72979"
    dependencies = [
    "log",
    "memmap2",
    "parking_lot 0.12.1",
    "perf-event-open-sys 3.0.0",
    "rustc-hash",
    "smallvec",
    ]
    [[package]]
    name = "measureme"
    version = "11.0.1"
    source = "git+https://github.com/rust-lang/measureme#7bbe7a985a1ed5cc1043e1c8582b4c7e17efc5b7"
    dependencies = [
    "log",
    "memmap2",
    "parking_lot 0.12.1",
    "perf-event-open-sys 3.0.0",
    "rustc-hash",
    "smallvec",
    ]
    [[package]]
  • edit in Cargo.lock at line 441
    [3.4163]
    [8.2224]
    [[package]]
    name = "memmap2"
    version = "0.2.3"
    source = "registry+https://github.com/rust-lang/crates.io-index"
    checksum = "723e3ebdcdc5c023db1df315364573789f8857c11b631a2fdfad7c00f5c046b4"
    dependencies = [
    "libc",
    ]
  • edit in Cargo.lock at line 474
    [3.3842]
    [3.3842]
    version = "0.11.2"
    source = "registry+https://github.com/rust-lang/crates.io-index"
    checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99"
    dependencies = [
    "instant",
    "lock_api",
    "parking_lot_core 0.8.6",
    ]
    [[package]]
    name = "parking_lot"
  • replacement in Cargo.lock at line 490
    [3.4034][3.4034:4055]()
    "parking_lot_core",
    [3.4034]
    [3.4055]
    "parking_lot_core 0.9.9",
    ]
    [[package]]
    name = "parking_lot_core"
    version = "0.8.6"
    source = "registry+https://github.com/rust-lang/crates.io-index"
    checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc"
    dependencies = [
    "cfg-if",
    "instant",
    "libc",
    "redox_syscall 0.2.16",
    "smallvec",
    "winapi",
  • replacement in Cargo.lock at line 515
    [3.4282][3.4282:4300]()
    "redox_syscall",
    [3.4282]
    [3.4300]
    "redox_syscall 0.4.1",
  • edit in Cargo.lock at line 530
    [3.3848]
    [3.4358]
    name = "perf-event-open-sys"
    version = "1.0.1"
    source = "registry+https://github.com/rust-lang/crates.io-index"
    checksum = "ce9bedf5da2c234fdf2391ede2b90fabf585355f33100689bc364a3ea558561a"
    dependencies = [
    "libc",
    ]
    [[package]]
    name = "perf-event-open-sys"
    version = "3.0.0"
    source = "registry+https://github.com/rust-lang/crates.io-index"
    checksum = "b29be2ba35c12c6939f6bc73187f728bba82c3c062ecdc5fa90ea739282a1f58"
    dependencies = [
    "libc",
    ]
    [[package]]
  • edit in Cargo.lock at line 678
    [3.5772]
    [3.5772]
    version = "0.2.16"
    source = "registry+https://github.com/rust-lang/crates.io-index"
    checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
    dependencies = [
    "bitflags",
    ]
    [[package]]
    name = "redox_syscall"
  • edit in Cargo.lock at line 693
    [3.5965]
    [3.5965]
    [[package]]
    name = "rustc-hash"
    version = "1.1.0"
    source = "registry+https://github.com/rust-lang/crates.io-index"
    checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
  • edit in Cargo.lock at line 855
    [3.4959]
    [3.6174]
    name = "winapi"
    version = "0.3.9"
    source = "registry+https://github.com/rust-lang/crates.io-index"
    checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
    dependencies = [
    "winapi-i686-pc-windows-gnu",
    "winapi-x86_64-pc-windows-gnu",
    ]
    [[package]]
    name = "winapi-i686-pc-windows-gnu"
    version = "0.4.0"
    source = "registry+https://github.com/rust-lang/crates.io-index"
    checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
    [[package]]
    name = "winapi-x86_64-pc-windows-gnu"
    version = "0.4.0"
    source = "registry+https://github.com/rust-lang/crates.io-index"
    checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
    [[package]]