NOSZA4ZMMICEZS4NPOE66REH2V5AWQLIUIQC7GGIKDWPNBSQFIXQC
6XB4RDRQIINYARVQUO3U3ZG7652UIUXBUF33U7JEJ7ASRI47EKHAC
HEFGMMUMJ2GGJGF2BEKF7EUL24AO53WJTXKBJSEGT64AV3Y4TBZQC
IH334Q5ACWE4TNQYYOOF6GWV6CRXOEM6542NVNPA6HRIZ3CBFKEAC
HELRYMZKHZW4S2IEYXA4IE7VOYYL7FH6TVHGFHJ4J3KRBBLHPQPAC
UPRY2FELRWZCL5FGUS3EKS2RXEY7D2FGXYGLJWDNJT23PHITRWMAC
K6L5F26WJMIYPP3Y3CBBUYMAK7C5SVR7NYFNU45NGU5NMDCSJJIQC
IB4L4N5M765IYALNZENJRKOGUPBRK2AMAB6IRNWPDCKNKKRMOB5AC
L7S3YSSMT6JCK4A7MCXZDMDEFBP7XJITHFRDMDKJ5L2RDVHI4AHAC
GKNZUJV6XUPOHKZEOOPOWMCYRBTMR36WTLOBEIPWZ55DX6SRUAVAC
SZHL3TSDHAQ3C4JKQRNYFDWOSXURPIN72HKTXQRY7YCABQDJE5FQC
const BLOCKSIZELOG_MAX: usize = 17;
const BLOCKSIZE_MAX: usize = 1 << BLOCKSIZELOG_MAX;
const SEEKABLE_BUFF_SIZE: usize = BLOCKSIZE_MAX;
const SEEKABLE_MAXFRAMES: usize = 0x8000000;
const BLOCK_SIZE_LOG_MAX: usize = 17;
const BLOCK_SIZE_MAX: usize = 1 << BLOCK_SIZE_LOG_MAX;
const SEEKABLE_BUFF_SIZE: usize = BLOCK_SIZE_MAX;
const SEEKABLE_MAX_FRAMES: usize = 0x8000000;
pub fn init(source: R) -> Result<Self, Error> {
let mut seekable = Self::make_seekable(source, ptr::null_mut(), None);
pub fn init(source: R, size: Option<usize>) -> Result<Self, Error> {
let mut seekable = Self::make_seekable(source, ptr::null_mut(), size);
}
#[test]
fn test_par() {
let input = b"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Ut diam ante, sollicitudin a dolor et, volutpat elementum nulla. Etiam nec efficitur nibh, quis rutrum risus. Maecenas quis lorem malesuada, aliquet mi vel, viverra nunc. Donec et nulla sed velit sagittis varius. Suspendisse vestibulum, neque lobortis ornare vestibulum, orci turpis vulputate nisi, ut sodales neque purus eget magna. Nunc condimentum, diam eu consequat venenatis, est nisl semper lorem, et lobortis velit justo sed nulla. Nunc sit amet tempor nunc, vel posuere ipsum. Cras erat tortor, pulvinar ac pretium eu, auctor ac nibh. Duis iaculis porta magna, eu lobortis elit. Duis vitae massa eros. Nulla non magna accumsan, egestas quam sit amet, laoreet lectus.";
let mut output = Vec::new();
parallel_compress::<&mut Vec<u8>, [u8; 256]>(input, &mut output, 10, 4).unwrap();
let mut decomp = Vec::new();
let mut s = Seekable::init_buf(&mut output).unwrap();
for frame in 0..s.get_num_frames() {
let size = s.get_frame_decompressed_size(frame).unwrap() as usize;
let n = decomp.len();
decomp.extend(std::iter::repeat(0).take(size));
s.decompress_frame(&mut decomp[n..], frame).unwrap();
}
println!("{:?}", std::str::from_utf8(&decomp).unwrap());
assert_eq!(&input[..], &decomp[..])
}
#[test]
fn test() {
let mut cstream = SeekableCStream::new(10, 256).unwrap();
let input = b"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Ut diam ante, sollicitudin a dolor et, volutpat elementum nulla. Etiam nec efficitur nibh, quis rutrum risus. Maecenas quis lorem malesuada, aliquet mi vel, viverra nunc. Donec et nulla sed velit sagittis varius. Suspendisse vestibulum, neque lobortis ornare vestibulum, orci turpis vulputate nisi, ut sodales neque purus eget magna. Nunc condimentum, diam eu consequat venenatis, est nisl semper lorem, et lobortis velit justo sed nulla. Nunc sit amet tempor nunc, vel posuere ipsum. Cras erat tortor, pulvinar ac pretium eu, auctor ac nibh. Duis iaculis porta magna, eu lobortis elit. Duis vitae massa eros. Nulla non magna accumsan, egestas quam sit amet, laoreet lectus.";
let mut input_pos = 0;
let mut output = vec![0; input.len()];
let mut output_pos = 0;
while input_pos < input.len() {
let (a, b) = cstream
.compress(&mut output[output_pos..], &input[input_pos..])
.unwrap();
output_pos += a;
input_pos += b;
}
while let Ok(n) = cstream.end_stream(&mut output[output_pos..]) {
if n == 0 {
break;
}
output_pos += n;
}
output.truncate(output_pos);
{
use std::io::Write;
let mut file = std::fs::File::create("test").unwrap();
file.write_all(&output).unwrap();
}
println!("input len = {:?}, pos = {:?}", input.len(), output_pos);
let mut decomp = Vec::new();
let mut s = { Seekable::init_buf(&mut output).unwrap() };
for frame in 0..s.get_num_frames() {
let size = s.get_frame_decompressed_size(frame).unwrap() as usize;
println!(
"{:?} {:?}",
size,
s.get_frame_decompressed_offset(frame).unwrap()
);
let n = decomp.len();
decomp.extend(std::iter::repeat(0).take(size));
s.decompress_frame(&mut decomp[n..], frame).unwrap();
}
println!("{:?}", std::str::from_utf8(&decomp).unwrap());
assert_eq!(&input[..], &decomp[..])
}
}
use zstd_seekable::{parallel_compress, Seekable, SeekableCStream};
const data: &[u8; 731] = b"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Ut diam ante, sollicitudin a dolor et, volutpat elementum nulla. Etiam nec efficitur nibh, quis rutrum risus. Maecenas quis lorem malesuada, aliquet mi vel, viverra nunc. Donec et nulla sed velit sagittis varius. Suspendisse vestibulum, neque lobortis ornare vestibulum, orci turpis vulputate nisi, ut sodales neque purus eget magna. Nunc condimentum, diam eu consequat venenatis, est nisl semper lorem, et lobortis velit justo sed nulla. Nunc sit amet tempor nunc, vel posuere ipsum. Cras erat tortor, pulvinar ac pretium eu, auctor ac nibh. Duis iaculis porta magna, eu lobortis elit. Duis vitae massa eros. Nulla non magna accumsan, egestas quam sit amet, laoreet lectus.";
#[test]
fn test_par_1() {
let mut output = Vec::new();
const chunk_size: usize = 256;
parallel_compress::<&mut Vec<u8>, [u8; chunk_size]>(data, &mut output, 10, 4, chunk_size).unwrap();
let mut decomp = Vec::new();
let mut s = { Seekable::init_buf(&mut output[..]).unwrap() };
let mut start_offset = 0;
let end_offset = data.len();
let mut i = 0;
for frame in 0..s.get_num_frames() {
let size = s.get_frame_decompressed_size(frame).unwrap() as usize;
let n = decomp.len();
decomp.extend(std::iter::repeat(0).take(size));
s.decompress_frame(&mut decomp[n..], frame).unwrap();
}
println!("{:?}", std::str::from_utf8(&decomp).unwrap());
assert_eq!(&data[..], &decomp[..])
}
#[test]
fn test_par_2() {
let mut output = Vec::new();
const chunk_size: usize = 256;
parallel_compress::<&mut Vec<u8>, [u8; chunk_size]>(data, &mut output, 10, 4, chunk_size).unwrap();
let b_size = 1024;
let mut decomp = vec![0;b_size];
let mut s = { Seekable::init_buf(output.as_mut_slice()).unwrap() };
let mut start_offset = 0;
let end_offset = data.len();
let mut i = 0;
while start_offset < end_offset {
let dst_size = std::cmp::min(end_offset - start_offset, b_size);
let r = s.decompress(&mut decomp[i*dst_size..(i+1)*dst_size], dst_size as u64, start_offset as u64).unwrap();
if r == 0 {
break;
}
i += 1;
start_offset += r;
}
decomp.truncate(end_offset);
println!("{:?}", std::str::from_utf8(&decomp).unwrap());
assert_eq!(&data[..], &decomp[..])
}
#[test]
fn test() {
let mut cstream = SeekableCStream::new(10, 256).unwrap();
let mut input_pos = 0;
let mut output = vec![0; data.len()];
let mut output_pos = 0;
while input_pos < data.len() {
let (a, b) = cstream
.compress(&mut output[output_pos..], &data[input_pos..])
.unwrap();
output_pos += a;
input_pos += b;
}
while let Ok(n) = cstream.end_stream(&mut output[output_pos..]) {
if n == 0 {
break;
}
output_pos += n;
}
output.truncate(output_pos);
{
use std::io::Write;
let mut file = std::fs::File::create("test").unwrap();
file.write_all(&output).unwrap();
}
println!("data len = {:?}, pos = {:?}", data.len(), output_pos);
let mut decomp = Vec::new();
let mut s = { Seekable::init_buf(&mut output).unwrap() };
for frame in 0..s.get_num_frames() {
let size = s.get_frame_decompressed_size(frame).unwrap() as usize;
let n = decomp.len();
decomp.extend(std::iter::repeat(0).take(size));
s.decompress_frame(&mut decomp[n..], frame).unwrap();
}
println!("{:?}", std::str::from_utf8(&decomp).unwrap());
assert_eq!(&data[..], &decomp[..])
}
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
[[package]]
name = "cfg-if"
version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822"
[[package]]
name = "cfg-if"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "getrandom"
version = "0.1.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce"
dependencies = [
"cfg-if 1.0.0",
"libc",
"wasi",
]
[[package]]
name = "hermit-abi"
version = "0.1.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
dependencies = [
"libc",
]
[[package]]
name = "libc"
version = "0.2.98"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "320cfe77175da3a483efed4bc0adc1968ca050b098ce4f2f1c13a56626128790"
[[package]]
name = "num_cpus"
version = "1.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3"
dependencies = [
"hermit-abi",
"libc",
]
[[package]]
name = "pkg-config"
version = "0.3.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c"
[[package]]
name = "ppv-lite86"
version = "0.2.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857"
[[package]]
name = "proc-macro2"
version = "1.0.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f0d8caf72986c1a598726adc988bb5984792ef84f5ee5aa50209145ee8077038"
dependencies = [
"unicode-xid",
]
[[package]]
name = "quote"
version = "1.0.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7"
dependencies = [
"proc-macro2",
]
[[package]]
name = "rand"
version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03"
dependencies = [
"getrandom",
"libc",
"rand_chacha",
"rand_core",
"rand_hc",
]
[[package]]
name = "rand_chacha"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402"
dependencies = [
"ppv-lite86",
"rand_core",
]
[[package]]
name = "rand_core"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19"
dependencies = [
"getrandom",
]
[[package]]
name = "rand_hc"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c"
dependencies = [
"rand_core",
]
[[package]]
name = "static_assertions"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
[[package]]
name = "syn"
version = "1.0.73"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f71489ff30030d2ae598524f61326b902466f72a0fb1a8564c001cc63425bcc7"
dependencies = [
"proc-macro2",
"quote",
"unicode-xid",
]
[[package]]
name = "thiserror"
version = "1.0.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93119e4feac1cbe6c798c34d3a53ea0026b0b1de6a120deef895137c0529bfe2"
dependencies = [
"thiserror-impl",
]
[[package]]
name = "thiserror-impl"
version = "1.0.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "060d69a0afe7796bf42e9e2ff91f5ee691fb15c53d38b4b62a9a53eb23164745"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "threadpool"
version = "1.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa"
dependencies = [
"num_cpus",
]
[[package]]
name = "twox-hash"
version = "1.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04f8ab788026715fa63b31960869617cba39117e520eb415b0139543e325ab59"
dependencies = [
"cfg-if 0.1.10",
"rand",
"static_assertions",
]
[[package]]
name = "unicode-xid"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3"
[[package]]
name = "wasi"
version = "0.9.0+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519"
[[package]]
name = "zstd-seekable"
version = "0.1.7"
dependencies = [
"libc",
"pkg-config",
"thiserror",
"threadpool",
"twox-hash",
]