Debugging (including moving to xxhash-rust instead of twox-hash)

pmeunier

Sep 6, 2021, 9:10 AM

GXWDF2M2F5VA4JR6TC5RI5TFVWY7CA6C4ZW2UXGLD66Z4VSY6PMAC

Dependencies

• [2] MHZGMWJE Reorganising the files
• [3] NOSZA4ZM Fixing tests
• [4] 57ARVVZE Adding one more test
• [5] HEFGMMUM Start oxidizing 1/N
• [6] 7C44WWMY Cleanup + compatibility with previous versions
• [7] HELRYMZK Manual bindings (Windows and Clang aren't very good friends), and getting rid of anyhow
• [8] TKSHSYFM Using pkg-config to find the libraries (+ some cleanup)
• [9] IB4L4N5M Fixing the lifetimes for `Seekable`
• [10] IH334Q5A Reboot
• [11] K6L5F26W Fixing sequential test.
• [12] UPRY2FEL Remove bindgen dependency
• [13] 6XB4RDRQ Adding changes from original impl

Change contents

• replacement in tests/tests.rs at line 1
  [3.1706]→[3.1707:1774](∅→∅)
  − use zstd_seekable::{parallel_compress, Seekable, SeekableCStream};
  [3.1706]

  [3.1774]
  + use zstd_seekable::{Seekable, SeekableCStream};

  + #[cfg(feature = "threadpool")]

  + use zstd_seekable::parallel_compress;
• replacement in tests/tests.rs at line 5
  [3.1775]→[3.1775:2536](∅→∅)
  − const data: &[u8; 731] = b"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Ut diam ante, sollicitudin a dolor et, volutpat elementum nulla. Etiam nec efficitur nibh, quis rutrum risus. Maecenas quis lorem malesuada, aliquet mi vel, viverra nunc. Donec et nulla sed velit sagittis varius. Suspendisse vestibulum, neque lobortis ornare vestibulum, orci turpis vulputate nisi, ut sodales neque purus eget magna. Nunc condimentum, diam eu consequat venenatis, est nisl semper lorem, et lobortis velit justo sed nulla. Nunc sit amet tempor nunc, vel posuere ipsum. Cras erat tortor, pulvinar ac pretium eu, auctor ac nibh. Duis iaculis porta magna, eu lobortis elit. Duis vitae massa eros. Nulla non magna accumsan, egestas quam sit amet, laoreet lectus.";
  [3.1775]

  [3.2536]
  + const DATA: &[u8; 731] = b"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Ut diam ante, sollicitudin a dolor et, volutpat elementum nulla. Etiam nec efficitur nibh, quis rutrum risus. Maecenas quis lorem malesuada, aliquet mi vel, viverra nunc. Donec et nulla sed velit sagittis varius. Suspendisse vestibulum, neque lobortis ornare vestibulum, orci turpis vulputate nisi, ut sodales neque purus eget magna. Nunc condimentum, diam eu consequat venenatis, est nisl semper lorem, et lobortis velit justo sed nulla. Nunc sit amet tempor nunc, vel posuere ipsum. Cras erat tortor, pulvinar ac pretium eu, auctor ac nibh. Duis iaculis porta magna, eu lobortis elit. Duis vitae massa eros. Nulla non magna accumsan, egestas quam sit amet, laoreet lectus.";
• edit in tests/tests.rs at line 8
  [3.2545]

  [3.2545]
  + #[cfg(feature = "threadpool")]
• replacement in tests/tests.rs at line 11
  [3.2596]→[3.176:305](∅→∅)
  − const chunk_size: usize = 200;

  − parallel_compress::<&mut Vec<u8>, [u8; chunk_size]>(data, &mut output, 10, 4, chunk_size)
  [3.2596]

  [3.305]
  + const CHUNK_SIZE: usize = 200;

  + parallel_compress::<&mut Vec<u8>, [u8; CHUNK_SIZE]>(DATA, &mut output, 10, 4, CHUNK_SIZE)
• edit in tests/tests.rs at line 16
  [3.2842]→[3.2842:2924](∅→∅)
  − let mut start_offset = 0;

  − let end_offset = data.len();

  − let mut i = 0;
• replacement in tests/tests.rs at line 25
  [3.3257]→[3.3257:3296](∅→∅)
  − assert_eq!(&data[..], &decomp[..])
  [3.3257]

  [3.3296]
  + assert_eq!(&DATA[..], &decomp[..])
• edit in tests/tests.rs at line 29
  [3.3307]

  [3.3307]
  + #[cfg(feature = "threadpool")]
• replacement in tests/tests.rs at line 32
  [3.3358]→[3.325:454](∅→∅)
  − const chunk_size: usize = 128;

  − parallel_compress::<&mut Vec<u8>, [u8; chunk_size]>(data, &mut output, 10, 4, chunk_size)
  [3.3358]

  [3.454]
  + const CHUNK_SIZE: usize = 128;

  + parallel_compress::<&mut Vec<u8>, [u8; CHUNK_SIZE]>(DATA, &mut output, 10, 4, CHUNK_SIZE)
• replacement in tests/tests.rs at line 39
  [3.3659]→[3.3659:3692](∅→∅)
  − let end_offset = data.len();
  [3.3659]

  [3.3692]
  + let end_offset = DATA.len();
• replacement in tests/tests.rs at line 46
  [3.556]→[3.556:652](∅→∅)
  − &mut decomp[i * dst_size..(i + 1) * dst_size],

  − dst_size as u64,
  [3.556]

  [3.652]
  + &mut decomp[i * dst_size..i * dst_size + dst_size],
• replacement in tests/tests.rs at line 60
  [3.4135]→[3.4135:4174](∅→∅)
  − assert_eq!(&data[..], &decomp[..])
  [3.4135]

  [3.4174]
  + assert_eq!(&DATA[..], &decomp[..])
• edit in tests/tests.rs at line 64
  [3.4185]

  [3.727]
  + #[cfg(feature = "threadpool")]
• replacement in tests/tests.rs at line 67
  [3.778]→[3.778:907](∅→∅)
  − const chunk_size: usize = 100;

  − parallel_compress::<&mut Vec<u8>, [u8; chunk_size]>(data, &mut output, 10, 4, chunk_size)
  [3.778]

  [3.907]
  + const CHUNK_SIZE: usize = 100;

  + parallel_compress::<&mut Vec<u8>, [u8; CHUNK_SIZE]>(DATA, &mut output, 10, 4, CHUNK_SIZE)
• replacement in tests/tests.rs at line 74
  [3.1088]→[3.1088:1121](∅→∅)
  − let end_offset = data.len();
  [3.1088]

  [3.1121]
  + let end_offset = DATA.len();
• replacement in tests/tests.rs at line 81
  [3.1346]→[3.1346:1425](∅→∅)
  − .decompress(&mut decomp[..], dst_size as u64, start_offset as u64)
  [3.1346]

  [3.1425]
  + .decompress(&mut decomp[..dst_size], start_offset as u64)
• replacement in tests/tests.rs at line 85
  [3.1525]→[3.1525:1601](∅→∅)
  − assert_eq!(&data[i * b_size..i * b_size + dst_size], &decomp[..r]);
  [3.1525]

  [3.1601]
  + assert_eq!(&DATA[i * b_size..i * b_size + dst_size], &decomp[..r]);
• replacement in tests/tests.rs at line 101
  [3.4286]→[3.4286:4328](∅→∅)
  − let mut output = vec![0; data.len()];
  [3.4286]

  [3.4328]
  + let mut output = vec![0; DATA.len()];
• replacement in tests/tests.rs at line 103
  [3.4356]→[3.4356:4391](∅→∅)
  − while input_pos < data.len() {
  [3.4356]

  [3.4391]
  + while input_pos < DATA.len() {
• replacement in tests/tests.rs at line 105
  [3.4420]→[3.4420:4489](∅→∅)
  − .compress(&mut output[output_pos..], &data[input_pos..])
  [3.4420]

  [3.4489]
  + .compress(&mut output[output_pos..], &DATA[input_pos..])
• replacement in tests/tests.rs at line 122
  [3.4895]→[3.4895:4964](∅→∅)
  − println!("data len = {:?}, pos = {:?}", data.len(), output_pos);
  [3.4895]

  [3.4964]
  + println!("data len = {:?}, pos = {:?}", DATA.len(), output_pos);
• replacement in tests/tests.rs at line 125
  [3.4998]→[3.4998:5060](∅→∅)
  − let mut s = { Seekable::init_buf(&mut output).unwrap() };
  [3.4998]

  [3.5060]
  + let mut s = { Seekable::init_buf(&output).unwrap() };
• replacement in tests/tests.rs at line 133
  [3.5391]→[3.5391:5430](∅→∅)
  − assert_eq!(&data[..], &decomp[..])
  [3.5391]

  [3.1748]
  + assert_eq!(&DATA[..], &decomp[..])
• edit in src/lib.rs at line 4
  [3.241338]→[3.241338:241375](∅→∅),[3.241375]→[3.0:24](∅→∅)
  − #[macro_use]

  − extern crate thiserror;

  − extern crate twox_hash;
• edit in src/lib.rs at line 9
  [2.25]

  [3.174]
  + use thiserror::*;
• replacement in src/lib.rs at line 33
  [3.242156]→[3.242156:242173](∅→∅)
  − #[derive(Error)]
  [3.1333]

  [3.242173]
  + #[derive(Debug, Error)]
• edit in src/lib.rs at line 35
  [3.242190]

  [3.242200]
  + #[error("Could not open file {}", 0)]
• replacement in src/lib.rs at line 37
  [3.242230]→[3.67:85](∅→∅)
  − ZSTD(size_t),
  [3.242230]

  [3.1667]
  + #[error(transparent)]

  + ZSTD(#[from] ZSTDError),

  + #[error(transparent)]
• edit in src/lib.rs at line 41
  [3.1694]

  [3.174]
  + #[error("Null pointer")]
• edit in src/lib.rs at line 43
  [3.184]

  [3.184]
  + #[error("Frame index too large, expected at most {}, found {}", 0, 1)]
• edit in src/lib.rs at line 45
  [3.218]

  [3.218]
  + #[error("Unknown error")]
• edit in src/lib.rs at line 47
  [3.231]

  [3.170]
  + #[error("An I/O error occurred when reading/seeking")]
• replacement in src/lib.rs at line 49
  [3.186]→[3.231:254](∅→∅),[3.231]→[3.231:254](∅→∅)
  − InvalidConversion,
  [3.186]

  [3.254]
  + #[error("Unsupported frame parameter. Expected at most {}, found {}", 0, 1)]
• edit in src/lib.rs at line 51
  [3.291]

  [3.291]
  + #[error("Unknown frame descriptor {}", 0)]
• edit in src/lib.rs at line 53
  [3.315]

  [3.315]
  + #[error("Corrupted block detected {}", 0)]
• edit in src/lib.rs at line 55
  [3.345]

  [3.187]
  + #[error("Destination buffer too small. Expected at most {}, found {}", 0, 1)]
• replacement in src/lib.rs at line 59
  [3.242251]→[3.1720:1748](∅→∅)
  − impl fmt::Debug for Error {
  [3.242251]

  [3.1748]
  + #[derive(Error)]

  + pub struct ZSTDError(size_t);

  +

  + impl fmt::Display for ZSTDError {
• replacement in src/lib.rs at line 64
  [3.1809]→[3.242355:242377](∅→∅),[3.242355]→[3.242355:242377](∅→∅),[3.242377]→[3.379:468](∅→∅),[3.468]→[3.242508:242613](∅→∅),[3.242508]→[3.242508:242613](∅→∅),[3.242613]→[3.469:519](∅→∅),[3.519]→[3.242662:242677](∅→∅),[3.242662]→[3.242662:242677](∅→∅),[3.242677]→[3.520:564](∅→∅),[3.564]→[3.217:274](∅→∅),[3.274]→[3.613:794](∅→∅),[3.613]→[3.613:794](∅→∅),[3.794]→[3.275:431](∅→∅),[3.431]→[3.865:1040](∅→∅),[3.865]→[3.865:1040](∅→∅),[3.1040]→[3.432:511](∅→∅),[3.511]→[3.1121:1157](∅→∅),[3.1121]→[3.1121:1157](∅→∅),[3.1157]→[3.512:686](∅→∅),[3.686]→[3.1316:1387](∅→∅),[3.1316]→[3.1316:1387](∅→∅),[3.1387]→[3.687:771](∅→∅),[3.771]→[3.1466:1502](∅→∅),[3.1466]→[3.1466:1502](∅→∅)
  − match *self {

  − Error::CouldNotOpenFile(ref f) => write!(fmt, "Could not open file {}.", f),

  − Error::ZSTD(r) => unsafe {

  − let error = CStr::from_ptr(ZSTD_getErrorName(r));

  − write!(fmt, "ZSTD({:?}).", error)

  − },

  − Error::Io(ref e) => e.fmt(fmt),

  − Error::Null => write!(fmt, "Null pointer."),

  − Error::FIndexTooLarge(a, b) => write!(

  − fmt,

  − "Frame index too large. Expected at most {}, found {}.",

  − a, b

  − ),

  − Error::Generic => write!(fmt, "Error (generic)."),

  − Error::SeekableIo => write!(fmt, "An I/O error occurred when reading/seeking."),

  − Error::InvalidConversion => write!(fmt, "Invalid slice to convert from bytes to u32."),

  − Error::FParamUnsupported(a, b) => write!(

  − fmt,

  − "Unsupported frame parameter. Expected at most {}, found {}.",

  − a, b

  − ),

  − Error::PrefixUnknown(p) => write!(fmt, "Unknown frame descriptor {}", p),

  − Error::Corruption(ref s) => write!(fmt, "Corrupted block detected {}.", s),

  − Error::DSizeTooSmall(a, b) => write!(

  − fmt,

  − "Destination buffer is too small. Expected at least {}, found {}.",

  − b, a

  − ),
  [3.1809]

  [3.242677]
  + unsafe {

  + let error = CStr::from_ptr(ZSTD_getErrorName(self.0));

  + write!(fmt, "{}", error.to_str().unwrap())
• replacement in src/lib.rs at line 71
  [3.242696]→[3.1868:1898](∅→∅)
  − impl fmt::Display for Error {
  [3.242696]

  [3.1898]
  + impl fmt::Debug for ZSTDError {
• replacement in src/lib.rs at line 73
  [3.1959]→[3.242802:242836](∅→∅),[3.242802]→[3.242802:242836](∅→∅)
  − write!(fmt, "{:?}", self)
  [3.1959]

  [3.243076]
  + write!(fmt, "{}", self)
• replacement in src/lib.rs at line 98
  [3.247734]→[3.247734:247783](∅→∅)
  − return Err(Error::ZSTD(result));
  [3.247734]

  [3.247783]
  + return Err(Error::ZSTD(ZSTDError(result)));
• replacement in src/lib.rs at line 183
  [3.3175]→[3.3175:3224](∅→∅)
  − return Err(Error::ZSTD(result));
  [3.3175]

  [3.3224]
  + return Err(Error::ZSTD(ZSTDError(result)));
• replacement in src/lib.rs at line 219
  [3.919]→[3.919:968](∅→∅)
  − return Err(Error::ZSTD(result));
  [3.919]

  [3.251306]
  + return Err(Error::ZSTD(ZSTDError(result)));
• replacement in src/lib.rs at line 241
  [3.3386]→[3.3386:3436](∅→∅)
  − return Err(Error::ZSTD(_result));
  [3.3386]

  [3.3436]
  + return Err(Error::ZSTD(ZSTDError(_result)));
• edit in src/lib.rs at line 248
  [3.8818]
  +
• replacement in src/decompress/mod.rs at line 3
  [2.710]→[2.710:799](∅→∅)
  − Error, ZSTD_DStream, ZSTD_reset_session_only, BLOCK_SIZE_MAX, MAGIC_SKIPPABLE_START,
  [2.710]

  [2.799]
  + Error, ZSTDError, ZSTD_DStream, ZSTD_reset_session_only, BLOCK_SIZE_MAX, MAGIC_SKIPPABLE_START,
• replacement in src/decompress/mod.rs at line 8
  [2.912]→[2.912:960](∅→∅)
  − use std::io::Read;

  − use twox_hash::xxh3::Hash64;
  [2.912]

  [2.960]
  + use xxhash_rust::xxh64::Xxh64;
• replacement in src/decompress/mod.rs at line 11
  [2.1011]→[2.1011:1060](∅→∅)
  − const SEEKABLE_NO_OUTPUT_PROGRESS_MAX: u32 = 16;
  [2.1011]

  [2.1060]
  + // const SEEKABLE_NO_OUTPUT_PROGRESS_MAX: u32 = 16;
• replacement in src/decompress/mod.rs at line 17
  [2.1186]→[2.1186:1216](∅→∅)
  − Ok(u32::from_ne_bytes(b))
  [2.1186]

  [2.1216]
  + Ok(u32::from_le_bytes(b))
• replacement in src/decompress/mod.rs at line 28
  [2.1431]→[2.1431:1533](∅→∅)
  − in_buff: [u8; SEEKABLE_BUFF_SIZE],

  − out_buff: [u8; SEEKABLE_BUFF_SIZE],

  − xxh_state: Hash64,
  [2.1431]

  [2.1533]
  + in_buff: Vec<u8>,

  + out_buff: Vec<u8>,

  + xxh_state: Xxh64,
• replacement in src/decompress/mod.rs at line 75
  [2.2548]→[2.2548:2671](∅→∅)
  − let dstream = unsafe { ZSTD_createDStream() };

  − if dstream.is_null() {

  − return Err(Error::Null);
  [2.2548]

  [2.2671]
  + unsafe {

  + let dstream = ZSTD_createDStream();

  + if dstream.is_null() {

  + Err(Error::Null)

  + } else {

  + Ok(Seekable {

  + dstream,

  + seek_table: SeekTable::new(),

  + src: source,

  + inner_buf_size: size,

  + decompressed_offset: 0,

  + cur_frame: 0,

  + in_buff: vec![0; SEEKABLE_BUFF_SIZE],

  + out_buff: vec![0; SEEKABLE_BUFF_SIZE],

  + xxh_state: Xxh64::new(0),

  + })

  + }
• edit in src/decompress/mod.rs at line 93
  [2.2681]→[2.2681:3036](∅→∅)
  − Ok(Seekable {

  − dstream,

  − seek_table: SeekTable::new(),

  − src: source,

  − inner_buf_size: size,

  − decompressed_offset: 0,

  − cur_frame: 0,

  − in_buff: [0; SEEKABLE_BUFF_SIZE],

  − out_buff: [0; SEEKABLE_BUFF_SIZE],

  − xxh_state: Hash64::with_seed(0),

  − })
• edit in src/decompress/mod.rs at line 97
  [2.3150]→[2.3150:3210](∅→∅)
  − assert!(self.in_buff.len() == SEEKABLE_BUFF_SIZE);

  −
• replacement in src/decompress/mod.rs at line 100
  [2.3306]→[2.3306:3425](∅→∅)
  − let mut handle = (&mut self.src).take(SEEK_TABLE_FOOTER_SIZE as u64);

  − handle.read(&mut self.in_buff)?;
  [2.3306]

  [2.3425]
  + self.src.read_exact(&mut self.in_buff[..SEEK_TABLE_FOOTER_SIZE])?;
• replacement in src/decompress/mod.rs at line 125
  [2.4317]→[2.4317:4414](∅→∅)
  − handle = (&mut self.src).take(to_read as u64);

  − handle.read(&mut self.in_buff)?;

  −
  [2.4317]

  [2.4414]
  + self.src.read_exact(&mut self.in_buff[..to_read])?;
• replacement in src/decompress/mod.rs at line 139
  [2.4897]→[2.4897:4927](∅→∅)
  − let mut pos = 0usize;
  [2.4897]

  [2.4927]
  + let mut pos = 8;
• edit in src/decompress/mod.rs at line 146
  [2.5211]

  [2.5211]
  + self.in_buff.copy_within(pos..pos + offset, 0); // offset..offset + to_read, pos);
• replacement in src/decompress/mod.rs at line 148
  [2.5212]→[2.5212:5424](∅→∅)
  − self.in_buff.copy_within(offset..offset + to_read, pos);

  −

  − handle = (&mut self.src).take(to_read as u64);

  − handle.read(&mut self.in_buff[offset..offset + to_read])?;
  [2.5212]

  [2.5424]
  + self.src.read_exact(&mut self.in_buff[offset..offset + to_read])?;
• edit in src/decompress/mod.rs at line 158
  [2.5636]→[2.5636:5637](∅→∅)
  −
• replacement in src/decompress/mod.rs at line 189
  [2.6544]→[2.6544:6591](∅→∅)
  − Err(Error::ZSTD(dstream_init))
  [2.6544]

  [2.6591]
  + Err(Error::ZSTD(ZSTDError(dstream_init)))
• edit in src/decompress/mod.rs at line 224
  [2.7843]→[2.7843:7888](∅→∅)
  − let mut no_output_progress_c = 0u32;
• edit in src/decompress/mod.rs at line 242
  [2.8550]

  [2.8550]
  + self.xxh_state.reset(0);
• edit in src/decompress/mod.rs at line 244
  [2.8551]→[2.8551:8607](∅→∅)
  − self.xxh_state = Hash64::with_seed(0);

  −
• replacement in src/decompress/mod.rs at line 247
  [2.8762]→[2.8762:8814](∅→∅)
  − return Err(Error::ZSTD(r));
  [2.8762]

  [2.8814]
  + return Err(Error::ZSTD(ZSTDError(r)));
• replacement in src/decompress/mod.rs at line 290
  [2.10474]→[2.10474:10532](∅→∅)
  − return Err(Error::ZSTD(to_read));
  [2.10474]

  [2.10532]
  + return Err(Error::ZSTD(ZSTDError(to_read)));
• replacement in src/decompress/mod.rs at line 292
  [2.10554]→[2.10554:10573](∅→∅)
  − };
  [2.10554]

  [2.10573]
  + }
• replacement in src/decompress/mod.rs at line 296
  [2.10672]→[2.10672:10767](∅→∅)
  − &slice_tmp[prev_out_pos..prev_out_pos + (out_tmp.pos - prev_out_pos)],
  [2.10672]

  [2.10767]
  + &slice_tmp[prev_out_pos..out_tmp.pos],
• edit in src/decompress/mod.rs at line 301
  [2.10885]→[2.10885:11222](∅→∅)
  −

  − if forward_progress == 0 {

  − no_output_progress_c += 1;

  − if no_output_progress_c > SEEKABLE_NO_OUTPUT_PROGRESS_MAX {

  − return Err(Error::SeekableIo);

  − }

  − } else {

  − no_output_progress_c = 0;

  − }
• edit in src/decompress/mod.rs at line 305
  [2.11376]

  [2.11376]
  + // frame complete

  +

  + // verify checksum

  + let f = self.xxh_state.finish();

  + let f = f as u32;
• replacement in src/decompress/mod.rs at line 311
  [2.11434]→[2.11434:11492](∅→∅)
  − && self.xxh_state.finish() as u32
  [2.11434]

  [2.11492]
  + && f
• replacement in src/decompress/mod.rs at line 327
  [2.12126]→[2.12126:12254](∅→∅)
  − let mut handle = (&mut self.src).take(to_read as u64);

  − handle.read(&mut self.in_buff)?;
  [2.12126]

  [2.12254]
  + self.in_buff.resize(to_read, 0u8);

  + self.src.read_exact(&mut self.in_buff)?;
• replacement in src/decompress/mod.rs at line 351
  [2.12707]→[2.12707:12788](∅→∅)
  − fn get_frame(&'_ self, frame_index: usize) -> Result<&'_ SeekEntry, Error> {
  [2.12707]

  [2.12788]
  + fn get_frame(&self, frame_index: usize) -> Result<&'_ SeekEntry, Error> {
• replacement in src/decompress/mod.rs at line 403
  [2.14639]→[2.14639:14690](∅→∅)
  − impl<'a> Seekable<std::io::Cursor<&'a mut [u8]>> {
  [2.14639]

  [2.14690]
  + impl<'a> Seekable<std::io::Cursor<&'a [u8]>> {
• replacement in src/decompress/mod.rs at line 405
  [2.14746]→[2.14746:14812](∅→∅)
  − pub fn init_buf(input: &'a mut [u8]) -> Result<Self, Error> {
  [2.14746]

  [2.14812]
  + pub fn init_buf(input: &'a [u8]) -> Result<Self, Error> {
• replacement in src/decompress/mod.rs at line 408
  [2.14894]→[2.14894:14967](∅→∅)
  − let mut seekable = Seekable::make_seekable(source, Some(size))?;
  [2.14894]

  [2.14967]
  + let mut seekable = if let Ok(s) = Seekable::make_seekable(source, Some(size)) {

  + s

  + } else {

  + panic!("")

  + };
• edit in src/decompress/mod.rs at line 427
  [2.15433]
  +

  +

  + #[test]

  + fn pijul_change() {

  +

  + let mut h = xxhash_rust::xxh64::Xxh64::new(0);

  + h.update(&[4]);

  + let d = 0x64b9da3ed69d6732;

  + let f = h.finish();

  + assert_eq!(d, f);

  +

  + let change = include_bytes!("../../.pijul/changes/IH/334Q5ACWE4TNQYYOOF6GWV6CRXOEM6542NVNPA6HRIZ3CBFKEAC.change");

  + use serde_derive::*;

  + #[derive(Deserialize)]

  + pub struct Offsets {

  + pub version: u64,

  + pub hashed_len: u64,

  + pub unhashed_off: u64,

  + pub unhashed_len: u64,

  + pub contents_off: u64,

  + pub contents_len: u64,

  + pub total: u64,

  + }

  +

  + let off0 = std::mem::size_of::<Offsets>();

  + let offsets: Offsets = bincode::deserialize(&change[..off0]).unwrap();

  +

  + let mut s = Seekable::init_buf(

  + &change[off0..offsets.unhashed_off as usize],

  + ).unwrap();

  +

  + let mut buf_ = Vec::new();

  + buf_.resize(offsets.hashed_len as usize, 0);

  + s.decompress(&mut buf_[..], 0).unwrap();

  +

  + }
• replacement in src/compress/mod.rs at line 1
  [2.15486]→[2.15487:15541](∅→∅)
  − use super::{CStream, Error, ZSTD_reset_session_only};
  [2.15486]

  [2.15541]
  + use super::{CStream, Error, ZSTD_reset_session_only, ZSTDError};
• replacement in src/compress/mod.rs at line 6
  [2.15624]→[2.15624:15663](∅→∅)
  − use twox_hash::xxh3::{hash64, Hash64};
  [2.15624]

  [2.15663]
  + use xxhash_rust::xxh64::Xxh64;

  + // use twox_hash::xxh3::Hash64;
• replacement in src/compress/mod.rs at line 21
  [2.15952]→[2.15952:15975](∅→∅)
  − xxh_state: Hash64,
  [2.15952]

  [2.15975]
  + xxh_state: Xxh64,
• replacement in src/compress/mod.rs at line 52
  [2.16982]→[2.16982:17035](∅→∅)
  − return Err(Error::ZSTD(result));
  [2.16982]

  [2.17035]
  + return Err(Error::ZSTD(ZSTDError(result)));
• replacement in src/compress/mod.rs at line 61
  [2.17296]→[2.17296:17345](∅→∅)
  − xxh_state: Hash64::with_seed(0),
  [2.17296]

  [2.17345]
  + xxh_state: Xxh64::new(0),
• replacement in src/compress/mod.rs at line 93
  [2.18210]→[2.18210:18254](∅→∅)
  − return Err(Error::ZSTD(r));
  [2.18210]

  [2.18254]
  + return Err(Error::ZSTD(ZSTDError(r)));
• replacement in src/compress/mod.rs at line 98
  [2.18326]→[2.18326:18377](∅→∅)
  − self.xxh_state = Hash64::with_seed(0);
  [2.18326]

  [2.18377]
  + self.xxh_state = Xxh64::new(0);
• replacement in src/compress/mod.rs at line 139
  [2.19760]→[2.19760:19806](∅→∅)
  − return Err(Error::ZSTD(ret));
  [2.19760]

  [2.19806]
  + return Err(Error::ZSTD(ZSTDError(ret)));
• replacement in src/compress/mod.rs at line 147
  [2.19998]→[2.19998:20044](∅→∅)
  − return Err(Error::ZSTD(ret));
  [2.19998]

  [2.20044]
  + return Err(Error::ZSTD(ZSTDError(ret)));
• replacement in src/compress/mod.rs at line 166
  [2.20636]→[2.20636:20688](∅→∅)
  − return Err(Error::ZSTD(end_frame));
  [2.20636]

  [2.20688]
  + return Err(Error::ZSTD(ZSTDError(end_frame)));
• replacement in src/compress/mod.rs at line 178
  [2.21003]→[2.21003:21048](∅→∅)
  − return Err(Error::ZSTD(result));
  [2.21003]

  [2.21048]
  + return Err(Error::ZSTD(ZSTDError(result)));
• edit in src/compress/mod.rs at line 183
  [2.21098]→[2.21098:21247](∅→∅)
  − }

  −

  − pub trait Dst: Send {

  − fn as_mut_ptr(&mut self) -> *mut u8;

  − fn as_slice(&self) -> &[u8];

  − fn len(&self) -> usize;

  − fn new() -> Self;
• replacement in src/compress/mod.rs at line 185
  [2.21250]→[2.21250:21473](∅→∅)
  − impl<const N: usize> Dst for [u8; N] {

  − fn as_mut_ptr(&mut self) -> *mut u8 {

  − self.as_mut().as_mut_ptr()

  − }

  − fn as_slice(&self) -> &[u8] {

  − self.as_ref()

  − }

  − fn len(&self) -> usize {

  − N
  [2.21250]

  [2.21473]
  + #[cfg(feature = "threadpool")]

  + mod parallel_compress {

  + use super::*;

  + use xxhash_rust::xxh64::xxh64;

  + pub trait Dst: Send {

  + fn as_mut_ptr(&mut self) -> *mut u8;

  + fn as_slice(&self) -> &[u8];

  + fn len(&self) -> usize;

  + fn new() -> Self;
• replacement in src/compress/mod.rs at line 195
  [2.21479]→[2.21479:21567](∅→∅)
  − fn new() -> Self {

  − unsafe { std::mem::MaybeUninit::uninit().assume_init() }
  [2.21479]

  [2.21567]
  +

  + impl<const N: usize> Dst for [u8; N] {

  + fn as_mut_ptr(&mut self) -> *mut u8 {

  + self.as_mut().as_mut_ptr()

  + }

  + fn as_slice(&self) -> &[u8] {

  + self.as_ref()

  + }

  + fn len(&self) -> usize {

  + N

  + }

  + fn new() -> Self {

  + unsafe { std::mem::MaybeUninit::uninit().assume_init() }

  + }
• edit in src/compress/mod.rs at line 210
  [2.21573]→[2.21573:21575](∅→∅)
  − }
• replacement in src/compress/mod.rs at line 211
  [2.21576]→[2.21576:21680](∅→∅)
  − struct CompressedFrame<D: Dst> {

  − src_size: u32,

  − dst_size: u32,

  − checksum: u32,

  − dst: D,

  − }
  [2.21576]

  [2.21680]
  + struct CompressedFrame<D: Dst> {

  + src_size: u32,

  + dst_size: u32,

  + checksum: u32,

  + dst: D,

  + }
• replacement in src/compress/mod.rs at line 218
  [2.21681]→[2.21681:21804](∅→∅)
  − impl<D: Dst> CompressedFrame<D> {

  − fn as_slice(&self) -> &[u8] {

  − &self.dst.as_slice()[..self.dst_size as usize]
  [2.21681]

  [2.21804]
  + impl<D: Dst> CompressedFrame<D> {

  + fn as_slice(&self) -> &[u8] {

  + &self.dst.as_slice()[..self.dst_size as usize]

  + }
• edit in src/compress/mod.rs at line 223
  [2.21810]→[2.21810:21812](∅→∅)
  − }
• replacement in src/compress/mod.rs at line 224
  [2.21813]→[2.21813:21864](∅→∅)
  − unsafe impl<D: Dst> Send for CompressedFrame<D> {}
  [2.21813]

  [2.21864]
  + unsafe impl<D: Dst> Send for CompressedFrame<D> {}
• replacement in src/compress/mod.rs at line 226
  [2.21865]→[2.21865:21984](∅→∅)
  − fn compress_frame<D: Dst>(src: &[u8], level: usize) -> Result<CompressedFrame<D>, Error> {

  − let mut dst = D::new();
  [2.21865]

  [2.21984]
  + fn compress_frame<D: Dst>(src: &[u8], level: usize) -> Result<CompressedFrame<D>, Error> {

  + let mut dst = D::new();
• replacement in src/compress/mod.rs at line 229
  [2.21985]→[2.21985:22234](∅→∅)
  − let ret = unsafe {

  − let ret = ZSTD_compress(

  − dst.as_mut_ptr() as *mut c_void,

  − dst.len() as size_t,

  − src.as_ptr() as *const c_void,

  − src.len() as size_t,

  − level as c_int,

  − );
  [2.21985]

  [2.22234]
  + let ret = unsafe {

  + let ret = ZSTD_compress(

  + dst.as_mut_ptr() as *mut c_void,

  + dst.len() as size_t,

  + src.as_ptr() as *const c_void,

  + src.len() as size_t,

  + level as c_int,

  + );
• replacement in src/compress/mod.rs at line 238
  [2.22235]→[2.22235:22323](∅→∅)
  − if ZSTD_isError(ret) != 0 {

  − return Err(Error::ZSTD(ret));

  − }
  [2.22235]

  [2.22323]
  + if ZSTD_isError(ret) != 0 {

  + return Err(Error::ZSTD(ZSTDError(ret)));

  + }
• replacement in src/compress/mod.rs at line 242
  [2.22324]→[2.22324:22343](∅→∅)
  − ret

  − };
  [2.22324]

  [2.22343]
  + ret

  + };
• replacement in src/compress/mod.rs at line 245
  [2.22344]→[2.22344:22383](∅→∅)
  − let checksum = hash64(src) as u32;
  [2.22344]

  [2.22383]
  + let checksum = xxh64(src, 0) as u32;
• replacement in src/compress/mod.rs at line 247
  [2.22384]→[2.22384:22515](∅→∅)
  − Ok(CompressedFrame {

  − src_size: src.len() as u32,

  − dst_size: ret as u32,

  − checksum,

  − dst,

  − })

  − }
  [2.22384]

  [2.22515]
  + Ok(CompressedFrame {

  + src_size: src.len() as u32,

  + dst_size: ret as u32,

  + checksum,

  + dst,

  + })

  + }
• replacement in src/compress/mod.rs at line 255
  [2.22516]→[2.22516:22771](∅→∅)
  − pub fn parallel_compress<W: std::io::Write, D: Dst + 'static>(

  − src: &'static [u8],

  − mut output: W,

  − level: usize,

  − jobs: usize,

  − chunk_size: usize,

  − ) -> Result<(), Error> {

  − use std::sync::mpsc::channel;

  − use threadpool::ThreadPool;
  [2.22516]

  [2.22771]
  + pub fn parallel_compress<W: std::io::Write, D: Dst + 'static>(

  + src: &'static [u8],

  + mut output: W,

  + level: usize,

  + jobs: usize,

  + chunk_size: usize,

  + ) -> Result<(), Error> {

  + use std::sync::mpsc::channel;

  + use threadpool::ThreadPool;
• replacement in src/compress/mod.rs at line 265
  [2.22772]→[2.22772:22896](∅→∅)
  − let n = src.len() / chunk_size + if src.len() % chunk_size == 0 { 0 } else { 1 };

  − let pool = ThreadPool::new(jobs);
  [2.22772]

  [2.22896]
  + let n = src.len() / chunk_size + if src.len() % chunk_size == 0 { 0 } else { 1 };

  + let pool = ThreadPool::new(jobs);
• replacement in src/compress/mod.rs at line 268
  [2.22897]→[2.22897:23221](∅→∅)
  − let (tx, rx) = channel();

  − for (i, chunk) in src.chunks(chunk_size).enumerate() {

  − let tx = tx.clone();

  − pool.execute(move || {

  − let frame = compress_frame(chunk, level);

  − tx.send((i, frame))

  − .expect("channel will be there waiting for the pool");

  − });

  − }
  [2.22897]

  [2.23221]
  + let (tx, rx) = channel();

  + for (i, chunk) in src.chunks(chunk_size).enumerate() {

  + let tx = tx.clone();

  + pool.execute(move || {

  + let frame = compress_frame(chunk, level);

  + tx.send((i, frame))

  + .expect("channel will be there waiting for the pool");

  + });

  + }
• replacement in src/compress/mod.rs at line 278
  [2.23222]→[2.23222:23401](∅→∅)
  − let mut frames: Vec<CompressedFrame<D>> = Vec::with_capacity(n);

  − unsafe { frames.set_len(n) };

  − for (i, frame) in rx.iter().take(n) {

  − frames[i] = frame?;

  − }
  [2.23222]

  [2.23401]
  + let mut frames: Vec<CompressedFrame<D>> = Vec::with_capacity(n);

  + unsafe { frames.set_len(n) };

  + for (i, frame) in rx.iter().take(n) {

  + frames[i] = frame?;

  + }
• replacement in src/compress/mod.rs at line 284
  [2.23402]→[2.23402:23587](∅→∅)
  − let mut log = FrameLog::new();

  − for frame in frames.iter() {

  − output.write_all(frame.as_slice())?;

  − log.log_frame(frame.dst_size, frame.src_size, frame.checksum)?;
  [2.23402]

  [2.23587]
  + let mut log = FrameLog::new();

  + for frame in frames.iter() {

  + output.write_all(frame.as_slice())?;

  + log.log_frame(frame.dst_size, frame.src_size, frame.checksum)?;

  + }

  + log.write_all(&mut output)?;

  + Ok(())
• edit in src/compress/mod.rs at line 292
  [2.23593]→[2.23593:23637](∅→∅)
  − log.write_all(&mut output)?;

  − Ok(())
• edit in src/compress/mod.rs at line 293
  [2.23639]
  +

  + #[cfg(feature = "threadpool")]

  + pub use parallel_compress::*;
• replacement in src/compress/framelog.rs at line 2
  [2.23690]→[2.23690:23717](∅→∅)
  − Error, ZSTD_outBuffer,
  [2.23690]

  [2.23717]
  + Error,
• replacement in src/compress/framelog.rs at line 8
  [2.23829]→[2.23829:23842](∅→∅)
  − use libc::*;
  [2.23829]

  [2.23842]
  + // use libc::*;
• edit in src/compress/framelog.rs at line 31
  [2.24204]

  [2.24204]
  + #[cfg(feature = "threadpool")]
• edit in src/compress/framelog.rs at line 176
  [2.28299]

  [2.28299]
  + #[cfg(feature = "threadpool")]
• edit in src/compress/framelog.rs at line 178
  [2.28392]

  [2.28392]
  + use crate::ZSTD_outBuffer;

  + use libc::c_void;
• edit in src/bindings.rs at line 34
  [3.9081]

  [3.2238]
  + #[cfg(feature = "threadpool")]
• replacement in Cargo.toml at line 21
  [3.270438]→[3.270438:270457](∅→∅),[3.270457]→[3.26728:26748](∅→∅)
  − threadpool = "1.8"

  − twox-hash = "1.6.0"
  [3.270438]

  [3.270457]
  + threadpool = { version = "1.8", optional = true }

  + xxhash-rust = { version = "0.8.2", features = [ "xxh64" ] }
• replacement in Cargo.toml at line 25
  [3.270490]→[3.213:231](∅→∅)
  − pkg-config = "0.3"
  [3.270479]
  + pkg-config = "0.3"

  +

  + [dev-dependencies]

  + bincode = "1.0"

  + serde = "1.0"

  + serde_derive = "1.0"
• replacement in Cargo.lock at line 3
  [3.5559]→[3.5559:5749](∅→∅)
  − [[package]]

  − name = "cfg-if"

  − version = "0.1.10"

  − source = "registry+https://github.com/rust-lang/crates.io-index"

  − checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822"
  [3.5559]

  [3.5749]
  + version = 3
• replacement in Cargo.lock at line 6
  [3.5762]→[3.5762:5796](∅→∅)
  − name = "cfg-if"

  − version = "1.0.0"
  [3.5762]

  [3.5796]
  + name = "bincode"

  + version = "1.3.3"
• replacement in Cargo.lock at line 9
  [3.5861]→[3.5861:6133](∅→∅)
  − checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"

  −

  − [[package]]

  − name = "getrandom"

  − version = "0.1.16"

  − source = "registry+https://github.com/rust-lang/crates.io-index"

  − checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce"
  [3.5861]

  [3.6133]
  + checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad"
• replacement in Cargo.lock at line 11
  [3.6150]→[3.6150:6185](∅→∅)
  − "cfg-if 1.0.0",

  − "libc",

  − "wasi",
  [3.6150]

  [3.6185]
  + "serde",
• edit in Cargo.lock at line 46
  [3.7043]→[3.7043:7238](∅→∅)
  − name = "ppv-lite86"

  − version = "0.2.10"

  − source = "registry+https://github.com/rust-lang/crates.io-index"

  − checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857"

  −

  − [[package]]
• replacement in Cargo.lock at line 47
  [3.7259]→[3.7259:7278](∅→∅)
  − version = "1.0.27"
  [3.7259]

  [3.7278]
  + version = "1.0.29"
• replacement in Cargo.lock at line 49
  [3.7343]→[3.7343:7421](∅→∅)
  − checksum = "f0d8caf72986c1a598726adc988bb5984792ef84f5ee5aa50209145ee8077038"
  [3.7343]

  [3.7421]
  + checksum = "b9f5105d4fdaab20335ca9565e106a5d9b82b6219b5ba735731124ac6711d23d"
• edit in Cargo.lock at line 61
  [3.7678]→[3.7678:8193](∅→∅)
  − ]

  −

  − [[package]]

  − name = "rand"

  − version = "0.7.3"

  − source = "registry+https://github.com/rust-lang/crates.io-index"

  − checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03"

  − dependencies = [

  − "getrandom",

  − "libc",

  − "rand_chacha",

  − "rand_core",

  − "rand_hc",

  − ]

  −

  − [[package]]

  − name = "rand_chacha"

  − version = "0.2.2"

  − source = "registry+https://github.com/rust-lang/crates.io-index"

  − checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402"

  − dependencies = [

  − "ppv-lite86",

  − "rand_core",
• replacement in Cargo.lock at line 64
  [3.8208]→[3.8208:8245](∅→∅)
  − name = "rand_core"

  − version = "0.5.1"
  [3.8208]

  [3.8245]
  + name = "serde"

  + version = "1.0.130"
• replacement in Cargo.lock at line 67
  [3.8310]→[3.8310:8421](∅→∅)
  − checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19"

  − dependencies = [

  − "getrandom",

  − ]
  [3.8310]

  [3.8421]
  + checksum = "f12d06de37cf59146fbdecab66aa99f9fe4f78722e3607577a5375d66bd0c913"
• replacement in Cargo.lock at line 70
  [3.8434]→[3.8434:8469](∅→∅)
  − name = "rand_hc"

  − version = "0.2.0"
  [3.8434]

  [3.8469]
  + name = "serde_derive"

  + version = "1.0.130"
• replacement in Cargo.lock at line 73
  [3.8534]→[3.8534:8612](∅→∅)
  − checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c"
  [3.8534]

  [3.8612]
  + checksum = "d7bc1a1ab1961464eae040d96713baa5a724a8152c1222492465b54322ec508b"
• replacement in Cargo.lock at line 75
  [3.8629]→[3.8629:8643](∅→∅)
  − "rand_core",
  [3.8629]

  [3.8643]
  + "proc-macro2",

  + "quote",

  + "syn",
• edit in Cargo.lock at line 79
  [3.8645]→[3.8645:8846](∅→∅)
  −

  − [[package]]

  − name = "static_assertions"

  − version = "1.1.0"

  − source = "registry+https://github.com/rust-lang/crates.io-index"

  − checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
• replacement in Cargo.lock at line 82
  [3.8872]→[3.8872:8891](∅→∅)
  − version = "1.0.73"
  [3.8872]

  [3.8891]
  + version = "1.0.76"
• replacement in Cargo.lock at line 84
  [3.8956]→[3.8956:9034](∅→∅)
  − checksum = "f71489ff30030d2ae598524f61326b902466f72a0fb1a8564c001cc63425bcc7"
  [3.8956]

  [3.9034]
  + checksum = "c6f107db402c2c2055242dbf4d2af0e69197202e9faacbef9571bbe47f5a1b84"
• replacement in Cargo.lock at line 93
  [3.9127]→[3.9127:9146](∅→∅)
  − version = "1.0.26"
  [3.9127]

  [3.9146]
  + version = "1.0.29"
• replacement in Cargo.lock at line 95
  [3.9211]→[3.9211:9289](∅→∅)
  − checksum = "93119e4feac1cbe6c798c34d3a53ea0026b0b1de6a120deef895137c0529bfe2"
  [3.9211]

  [3.9289]
  + checksum = "602eca064b2d83369e2b2f34b09c70b605402801927c65c11071ac911d299b88"
• replacement in Cargo.lock at line 102
  [3.9364]→[3.9364:9383](∅→∅)
  − version = "1.0.26"
  [3.9364]

  [3.9383]
  + version = "1.0.29"
• replacement in Cargo.lock at line 104
  [3.9448]→[3.9448:9526](∅→∅)
  − checksum = "060d69a0afe7796bf42e9e2ff91f5ee691fb15c53d38b4b62a9a53eb23164745"
  [3.9448]

  [3.9526]
  + checksum = "bad553cc2c78e8de258400763a647e80e6d1b31ee237275d756f6836d204494c"
• edit in Cargo.lock at line 118
  [3.9803]→[3.9803:10064](∅→∅)
  − ]

  −

  − [[package]]

  − name = "twox-hash"

  − version = "1.6.0"

  − source = "registry+https://github.com/rust-lang/crates.io-index"

  − checksum = "04f8ab788026715fa63b31960869617cba39117e520eb415b0139543e325ab59"

  − dependencies = [

  − "cfg-if 0.1.10",

  − "rand",

  − "static_assertions",
• replacement in Cargo.lock at line 127
  [3.10274]→[3.10274:10329](∅→∅)
  − name = "wasi"

  − version = "0.9.0+wasi-snapshot-preview1"
  [3.10274]

  [3.10329]
  + name = "xxhash-rust"

  + version = "0.8.2"
• replacement in Cargo.lock at line 130
  [3.10394]→[3.10394:10472](∅→∅)
  − checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519"
  [3.10394]

  [3.10472]
  + checksum = "e575e15bedf6e57b5c2d763ffc6c3c760143466cbd09d762d539680ab5992ded"
• edit in Cargo.lock at line 136
  [3.10543]

  [3.10543]
  + "bincode",
• edit in Cargo.lock at line 139
  [3.10567]

  [3.10567]
  + "serde",

  + "serde_derive",
• replacement in Cargo.lock at line 143
  [3.10596]→[3.10596:10610](∅→∅)
  − "twox-hash",
  [3.10596]

  [3.10610]
  + "xxhash-rust",