Fixing tests

darleybarreto
Jul 31, 2021, 2:07 PM
NOSZA4ZMMICEZS4NPOE66REH2V5AWQLIUIQC7GGIKDWPNBSQFIXQC

Dependencies

  • [2] 6XB4RDRQ Adding changes from original impl
  • [3] HEFGMMUM Start oxidizing 1/N
  • [4] IH334Q5A Reboot
  • [5] IB4L4N5M Fixing the lifetimes for `Seekable`
  • [6] SZHL3TSD Fixing tests
  • [7] UPRY2FEL Remove bindgen dependency
  • [8] GKNZUJV6 Removing unecessary unsafe blocks
  • [9] L7S3YSSM Clippy + rustfmt
  • [10] HELRYMZK Manual bindings (Windows and Clang aren't very good friends), and getting rid of anyhow
  • [11] K6L5F26W Fixing sequential test.

Change contents

  • replacement in src/lib.rs at line 16
    [3.175][3.175:216]()
    use twox_hash::{xxh3::hash64, XxHash64};
    [3.175]
    [3.216]
    use twox_hash::XxHash64;
  • replacement in src/lib.rs at line 18
    [3.217][3.217:399]()
    const BLOCKSIZELOG_MAX: usize = 17;
    const BLOCKSIZE_MAX: usize = 1 << BLOCKSIZELOG_MAX;
    const SEEKABLE_BUFF_SIZE: usize = BLOCKSIZE_MAX;
    const SEEKABLE_MAXFRAMES: usize = 0x8000000;
    [3.217]
    [3.399]
    const BLOCK_SIZE_LOG_MAX: usize = 17;
    const BLOCK_SIZE_MAX: usize = 1 << BLOCK_SIZE_LOG_MAX;
    const SEEKABLE_BUFF_SIZE: usize = BLOCK_SIZE_MAX;
    const SEEKABLE_MAX_FRAMES: usize = 0x8000000;
  • edit in src/lib.rs at line 32
    [2.94][3.241471:241472](),[3.665][3.241471:241472](),[3.241471][3.241471:241472](),[3.241472][3.0:30](),[3.30][3.241502:241541](),[3.72][3.241502:241541](),[3.241502][3.241502:241541]()
    pub fn out_size() -> size_t {
    unsafe { ZSTD_DStreamOutSize() }
    }
  • replacement in src/lib.rs at line 37
    [3.744][3.744:763]()
    checksum: u64,
    [3.744]
    [3.763]
    checksum: u32,
  • replacement in src/lib.rs at line 43
    [3.850][3.850:874]()
    checksum_flag: u64,
    [3.850]
    [3.874]
    checksum_flag: u32,
  • replacement in src/lib.rs at line 67
    [2.133][3.1234:1253](),[3.1234][3.1234:1253]()
    checksum: u64,
    [2.133]
    [3.241806]
    checksum: u32,
  • replacement in src/lib.rs at line 72
    [3.1306][3.1306:1330]()
    checksum_flag: u64,
    [3.1306]
    [3.1330]
    checksum_flag: u32,
  • replacement in src/lib.rs at line 307
    [3.5624][3.5624:5676]()
    self.xxh_state.finish() & 0xFFFFFFFFu64
    [3.5624]
    [3.5676]
    self.xxh_state.finish() as u32 & 0xFFFFFFFFu32
  • replacement in src/lib.rs at line 380
    [3.247876][3.2969:3009]()
    unsafe { ZSTD_CStreamInSize() }
    [3.247876]
    [3.247925]
    BLOCK_SIZE_MAX
  • replacement in src/lib.rs at line 519
    [3.207][3.3462:3514](),[3.3514][2.975:1054]()
    pub fn init(source: R) -> Result<Self, Error> {
    let mut seekable = Self::make_seekable(source, ptr::null_mut(), None);
    [3.207]
    [3.3587]
    pub fn init(source: R, size: Option<usize>) -> Result<Self, Error> {
    let mut seekable = Self::make_seekable(source, ptr::null_mut(), size);
  • edit in src/lib.rs at line 525
    [3.3650]
    [2.1055]
    // The parameter size is the size of a buffer. So if the source is not one, the size is None.
  • replacement in src/lib.rs at line 615
    [3.151][2.1306:1377](),[2.1377][3.4852:4879](),[3.4852][3.4852:4879]()
    let a = slice_to_num(&self.in_buff[pos..pos + 4])? as u64;
    c_offset += a;
    [3.151]
    [3.12131]
    c_offset += slice_to_num(&self.in_buff[pos..pos + 4])? as u64;
  • replacement in src/lib.rs at line 621
    [3.12289][3.12289:12380]()
    entries[idx].checksum = slice_to_num(&self.in_buff[pos..pos + 4])? as u64;
    [3.12289]
    [3.12380]
    entries[idx].checksum = slice_to_num(&self.in_buff[pos..pos + 4])?;
  • replacement in src/lib.rs at line 634
    [3.12572][3.12572:12634]()
    self.seek_table.checksum_flag = checksum_flag as u64;
    [3.12572]
    [3.12634]
    self.seek_table.checksum_flag = checksum_flag as u32;
  • replacement in src/lib.rs at line 772
    [3.16267][3.16267:16336]()
    && (self.xxh_state.finish() & 0xFFFFFFFFu64)
    [3.16267]
    [3.16336]
    && (self.xxh_state.finish() as u32 & 0xFFFFFFFFu32)
  • replacement in src/lib.rs at line 911
    [3.20775][3.20775:20794]()
    checksum: u64,
    [3.20775]
    [3.259155]
    checksum: u32,
  • replacement in src/lib.rs at line 940
    [3.21179][3.21179:21211]()
    let checksum = hash64(src);
    [3.21179]
    [3.1239]
    let mut hasher = XxHash64::with_seed(0);
    hasher.write(src);
    let checksum = hasher.finish() as u32;
  • replacement in src/lib.rs at line 962
    [3.21505][3.21505:21601]()
    pub fn log_frame(&mut self, c_size: u32, d_size: u32, checksum: u64) -> Result<(), Error> {
    [3.21505]
    [3.7005]
    pub fn log_frame(&mut self, c_size: u32, d_size: u32, checksum: u32) -> Result<(), Error> {
  • replacement in src/lib.rs at line 964
    [3.7044][3.7044:7166]()
    if size == SEEKABLE_MAXFRAMES {
    return Err(Error::FIndexTooLarge(size, SEEKABLE_MAXFRAMES as usize));
    [3.7044]
    [3.21715]
    if size == SEEKABLE_MAX_FRAMES {
    return Err(Error::FIndexTooLarge(size, SEEKABLE_MAX_FRAMES as usize));
  • replacement in src/lib.rs at line 1064
    [3.24437][3.1474:1569]()
    ret = self.stwrite32(output, self.entries[i].checksum as u32, start + 8, pos);
    [3.24437]
    [3.24529]
    ret = self.stwrite32(output, self.entries[i].checksum, start + 8, pos);
  • edit in src/lib.rs at line 1115
    [3.261353]
    [3.1178]
    chunk_size: usize,
  • replacement in src/lib.rs at line 1120
    [3.261453][3.261453:261558]()
    let chunk_size = 256;
    let n = src.len() / chunk_size + if src.len() % 256 == 0 { 0 } else { 1 };
    [3.261453]
    [3.261558]
    let n = src.len() / chunk_size + if src.len() % chunk_size == 0 { 0 } else { 1 };
  • edit in src/lib.rs at line 1124
    [3.261627]
    [3.261627]
    let mut frames: Vec<CompressedFrame<D>> = Vec::with_capacity(n);
  • replacement in src/lib.rs at line 1133
    [3.262053][3.262053:262054]()
    [3.262053]
    [3.25956]
  • replacement in src/lib.rs at line 1135
    [3.26025][3.26025:26059]()
    unsafe { frames.set_len(n) };
    [3.26025]
    [3.26059]
    unsafe {frames.set_len(n)};
  • replacement in src/lib.rs at line 1142
    [3.262356][3.8398:8462]()
    let t = frame.as_slice();
    output.write_all(t)?;
    [3.262356]
    [3.8462]
    output.write_all(frame.as_slice())?;
  • replacement in src/lib.rs at line 1147
    [3.262481][3.262481:263412](),[3.263412][3.8535:8593](),[3.62][3.263470:263511](),[3.8593][3.263470:263511](),[3.263470][3.263470:263511](),[3.263511][2.4759:4834](),[3.129][3.263568:263654](),[2.4834][3.263568:263654](),[3.263568][3.263568:263654](),[3.263654][2.4835:4897](),[3.192][3.263707:263713](),[2.4897][3.263707:263713](),[3.8724][3.263707:263713](),[3.263707][3.263707:263713](),[3.263713][3.193:254](),[3.254][3.263774:263818](),[3.26266][3.263774:263818](),[3.263774][3.263774:263818](),[3.263818][3.1772:3316](),[3.3316][3.265:266](),[3.10350][3.265:266](),[3.264963][3.265:266](),[3.266][3.3317:3453](),[3.3453][2.4898:4973](),[2.4973][3.1494:1626](),[3.3519][3.1494:1626](),[3.1626][3.3605:3862](),[3.3605][3.3605:3862]()
    }
    #[test]
    fn test_par() {
    let input = b"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Ut diam ante, sollicitudin a dolor et, volutpat elementum nulla. Etiam nec efficitur nibh, quis rutrum risus. Maecenas quis lorem malesuada, aliquet mi vel, viverra nunc. Donec et nulla sed velit sagittis varius. Suspendisse vestibulum, neque lobortis ornare vestibulum, orci turpis vulputate nisi, ut sodales neque purus eget magna. Nunc condimentum, diam eu consequat venenatis, est nisl semper lorem, et lobortis velit justo sed nulla. Nunc sit amet tempor nunc, vel posuere ipsum. Cras erat tortor, pulvinar ac pretium eu, auctor ac nibh. Duis iaculis porta magna, eu lobortis elit. Duis vitae massa eros. Nulla non magna accumsan, egestas quam sit amet, laoreet lectus.";
    let mut output = Vec::new();
    parallel_compress::<&mut Vec<u8>, [u8; 256]>(input, &mut output, 10, 4).unwrap();
    let mut decomp = Vec::new();
    let mut s = Seekable::init_buf(&mut output).unwrap();
    for frame in 0..s.get_num_frames() {
    let size = s.get_frame_decompressed_size(frame).unwrap() as usize;
    let n = decomp.len();
    decomp.extend(std::iter::repeat(0).take(size));
    s.decompress_frame(&mut decomp[n..], frame).unwrap();
    }
    println!("{:?}", std::str::from_utf8(&decomp).unwrap());
    assert_eq!(&input[..], &decomp[..])
    }
    #[test]
    fn test() {
    let mut cstream = SeekableCStream::new(10, 256).unwrap();
    let input = b"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Ut diam ante, sollicitudin a dolor et, volutpat elementum nulla. Etiam nec efficitur nibh, quis rutrum risus. Maecenas quis lorem malesuada, aliquet mi vel, viverra nunc. Donec et nulla sed velit sagittis varius. Suspendisse vestibulum, neque lobortis ornare vestibulum, orci turpis vulputate nisi, ut sodales neque purus eget magna. Nunc condimentum, diam eu consequat venenatis, est nisl semper lorem, et lobortis velit justo sed nulla. Nunc sit amet tempor nunc, vel posuere ipsum. Cras erat tortor, pulvinar ac pretium eu, auctor ac nibh. Duis iaculis porta magna, eu lobortis elit. Duis vitae massa eros. Nulla non magna accumsan, egestas quam sit amet, laoreet lectus.";
    let mut input_pos = 0;
    let mut output = vec![0; input.len()];
    let mut output_pos = 0;
    while input_pos < input.len() {
    let (a, b) = cstream
    .compress(&mut output[output_pos..], &input[input_pos..])
    .unwrap();
    output_pos += a;
    input_pos += b;
    }
    while let Ok(n) = cstream.end_stream(&mut output[output_pos..]) {
    if n == 0 {
    break;
    }
    output_pos += n;
    }
    output.truncate(output_pos);
    {
    use std::io::Write;
    let mut file = std::fs::File::create("test").unwrap();
    file.write_all(&output).unwrap();
    }
    println!("input len = {:?}, pos = {:?}", input.len(), output_pos);
    let mut decomp = Vec::new();
    let mut s = { Seekable::init_buf(&mut output).unwrap() };
    for frame in 0..s.get_num_frames() {
    let size = s.get_frame_decompressed_size(frame).unwrap() as usize;
    println!(
    "{:?} {:?}",
    size,
    s.get_frame_decompressed_offset(frame).unwrap()
    );
    let n = decomp.len();
    decomp.extend(std::iter::repeat(0).take(size));
    s.decompress_frame(&mut decomp[n..], frame).unwrap();
    }
    println!("{:?}", std::str::from_utf8(&decomp).unwrap());
    assert_eq!(&input[..], &decomp[..])
    }
    [3.262481]
    }
  • file addition: tests (d--r------)
    [1.0]
  • file addition: tests.rs (----------)
    [0.1672]
    use zstd_seekable::{parallel_compress, Seekable, SeekableCStream};
    const data: &[u8; 731] = b"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Ut diam ante, sollicitudin a dolor et, volutpat elementum nulla. Etiam nec efficitur nibh, quis rutrum risus. Maecenas quis lorem malesuada, aliquet mi vel, viverra nunc. Donec et nulla sed velit sagittis varius. Suspendisse vestibulum, neque lobortis ornare vestibulum, orci turpis vulputate nisi, ut sodales neque purus eget magna. Nunc condimentum, diam eu consequat venenatis, est nisl semper lorem, et lobortis velit justo sed nulla. Nunc sit amet tempor nunc, vel posuere ipsum. Cras erat tortor, pulvinar ac pretium eu, auctor ac nibh. Duis iaculis porta magna, eu lobortis elit. Duis vitae massa eros. Nulla non magna accumsan, egestas quam sit amet, laoreet lectus.";
    #[test]
    fn test_par_1() {
    let mut output = Vec::new();
    const chunk_size: usize = 256;
    parallel_compress::<&mut Vec<u8>, [u8; chunk_size]>(data, &mut output, 10, 4, chunk_size).unwrap();
    let mut decomp = Vec::new();
    let mut s = { Seekable::init_buf(&mut output[..]).unwrap() };
    let mut start_offset = 0;
    let end_offset = data.len();
    let mut i = 0;
    for frame in 0..s.get_num_frames() {
    let size = s.get_frame_decompressed_size(frame).unwrap() as usize;
    let n = decomp.len();
    decomp.extend(std::iter::repeat(0).take(size));
    s.decompress_frame(&mut decomp[n..], frame).unwrap();
    }
    println!("{:?}", std::str::from_utf8(&decomp).unwrap());
    assert_eq!(&data[..], &decomp[..])
    }
    #[test]
    fn test_par_2() {
    let mut output = Vec::new();
    const chunk_size: usize = 256;
    parallel_compress::<&mut Vec<u8>, [u8; chunk_size]>(data, &mut output, 10, 4, chunk_size).unwrap();
    let b_size = 1024;
    let mut decomp = vec![0;b_size];
    let mut s = { Seekable::init_buf(output.as_mut_slice()).unwrap() };
    let mut start_offset = 0;
    let end_offset = data.len();
    let mut i = 0;
    while start_offset < end_offset {
    let dst_size = std::cmp::min(end_offset - start_offset, b_size);
    let r = s.decompress(&mut decomp[i*dst_size..(i+1)*dst_size], dst_size as u64, start_offset as u64).unwrap();
    if r == 0 {
    break;
    }
    i += 1;
    start_offset += r;
    }
    decomp.truncate(end_offset);
    println!("{:?}", std::str::from_utf8(&decomp).unwrap());
    assert_eq!(&data[..], &decomp[..])
    }
    #[test]
    fn test() {
    let mut cstream = SeekableCStream::new(10, 256).unwrap();
    let mut input_pos = 0;
    let mut output = vec![0; data.len()];
    let mut output_pos = 0;
    while input_pos < data.len() {
    let (a, b) = cstream
    .compress(&mut output[output_pos..], &data[input_pos..])
    .unwrap();
    output_pos += a;
    input_pos += b;
    }
    while let Ok(n) = cstream.end_stream(&mut output[output_pos..]) {
    if n == 0 {
    break;
    }
    output_pos += n;
    }
    output.truncate(output_pos);
    {
    use std::io::Write;
    let mut file = std::fs::File::create("test").unwrap();
    file.write_all(&output).unwrap();
    }
    println!("data len = {:?}, pos = {:?}", data.len(), output_pos);
    let mut decomp = Vec::new();
    let mut s = { Seekable::init_buf(&mut output).unwrap() };
    for frame in 0..s.get_num_frames() {
    let size = s.get_frame_decompressed_size(frame).unwrap() as usize;
    let n = decomp.len();
    decomp.extend(std::iter::repeat(0).take(size));
    s.decompress_frame(&mut decomp[n..], frame).unwrap();
    }
    println!("{:?}", std::str::from_utf8(&decomp).unwrap());
    assert_eq!(&data[..], &decomp[..])
    }
  • file addition: Cargo.lock (----------)
    [1.0]
    # This file is automatically @generated by Cargo.
    # It is not intended for manual editing.
    [[package]]
    name = "cfg-if"
    version = "0.1.10"
    source = "registry+https://github.com/rust-lang/crates.io-index"
    checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822"
    [[package]]
    name = "cfg-if"
    version = "1.0.0"
    source = "registry+https://github.com/rust-lang/crates.io-index"
    checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
    [[package]]
    name = "getrandom"
    version = "0.1.16"
    source = "registry+https://github.com/rust-lang/crates.io-index"
    checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce"
    dependencies = [
    "cfg-if 1.0.0",
    "libc",
    "wasi",
    ]
    [[package]]
    name = "hermit-abi"
    version = "0.1.19"
    source = "registry+https://github.com/rust-lang/crates.io-index"
    checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
    dependencies = [
    "libc",
    ]
    [[package]]
    name = "libc"
    version = "0.2.98"
    source = "registry+https://github.com/rust-lang/crates.io-index"
    checksum = "320cfe77175da3a483efed4bc0adc1968ca050b098ce4f2f1c13a56626128790"
    [[package]]
    name = "num_cpus"
    version = "1.13.0"
    source = "registry+https://github.com/rust-lang/crates.io-index"
    checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3"
    dependencies = [
    "hermit-abi",
    "libc",
    ]
    [[package]]
    name = "pkg-config"
    version = "0.3.19"
    source = "registry+https://github.com/rust-lang/crates.io-index"
    checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c"
    [[package]]
    name = "ppv-lite86"
    version = "0.2.10"
    source = "registry+https://github.com/rust-lang/crates.io-index"
    checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857"
    [[package]]
    name = "proc-macro2"
    version = "1.0.27"
    source = "registry+https://github.com/rust-lang/crates.io-index"
    checksum = "f0d8caf72986c1a598726adc988bb5984792ef84f5ee5aa50209145ee8077038"
    dependencies = [
    "unicode-xid",
    ]
    [[package]]
    name = "quote"
    version = "1.0.9"
    source = "registry+https://github.com/rust-lang/crates.io-index"
    checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7"
    dependencies = [
    "proc-macro2",
    ]
    [[package]]
    name = "rand"
    version = "0.7.3"
    source = "registry+https://github.com/rust-lang/crates.io-index"
    checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03"
    dependencies = [
    "getrandom",
    "libc",
    "rand_chacha",
    "rand_core",
    "rand_hc",
    ]
    [[package]]
    name = "rand_chacha"
    version = "0.2.2"
    source = "registry+https://github.com/rust-lang/crates.io-index"
    checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402"
    dependencies = [
    "ppv-lite86",
    "rand_core",
    ]
    [[package]]
    name = "rand_core"
    version = "0.5.1"
    source = "registry+https://github.com/rust-lang/crates.io-index"
    checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19"
    dependencies = [
    "getrandom",
    ]
    [[package]]
    name = "rand_hc"
    version = "0.2.0"
    source = "registry+https://github.com/rust-lang/crates.io-index"
    checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c"
    dependencies = [
    "rand_core",
    ]
    [[package]]
    name = "static_assertions"
    version = "1.1.0"
    source = "registry+https://github.com/rust-lang/crates.io-index"
    checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
    [[package]]
    name = "syn"
    version = "1.0.73"
    source = "registry+https://github.com/rust-lang/crates.io-index"
    checksum = "f71489ff30030d2ae598524f61326b902466f72a0fb1a8564c001cc63425bcc7"
    dependencies = [
    "proc-macro2",
    "quote",
    "unicode-xid",
    ]
    [[package]]
    name = "thiserror"
    version = "1.0.26"
    source = "registry+https://github.com/rust-lang/crates.io-index"
    checksum = "93119e4feac1cbe6c798c34d3a53ea0026b0b1de6a120deef895137c0529bfe2"
    dependencies = [
    "thiserror-impl",
    ]
    [[package]]
    name = "thiserror-impl"
    version = "1.0.26"
    source = "registry+https://github.com/rust-lang/crates.io-index"
    checksum = "060d69a0afe7796bf42e9e2ff91f5ee691fb15c53d38b4b62a9a53eb23164745"
    dependencies = [
    "proc-macro2",
    "quote",
    "syn",
    ]
    [[package]]
    name = "threadpool"
    version = "1.8.1"
    source = "registry+https://github.com/rust-lang/crates.io-index"
    checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa"
    dependencies = [
    "num_cpus",
    ]
    [[package]]
    name = "twox-hash"
    version = "1.6.0"
    source = "registry+https://github.com/rust-lang/crates.io-index"
    checksum = "04f8ab788026715fa63b31960869617cba39117e520eb415b0139543e325ab59"
    dependencies = [
    "cfg-if 0.1.10",
    "rand",
    "static_assertions",
    ]
    [[package]]
    name = "unicode-xid"
    version = "0.2.2"
    source = "registry+https://github.com/rust-lang/crates.io-index"
    checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3"
    [[package]]
    name = "wasi"
    version = "0.9.0+wasi-snapshot-preview1"
    source = "registry+https://github.com/rust-lang/crates.io-index"
    checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519"
    [[package]]
    name = "zstd-seekable"
    version = "0.1.7"
    dependencies = [
    "libc",
    "pkg-config",
    "thiserror",
    "threadpool",
    "twox-hash",
    ]