Arbitrary byte strings, and version 1.3.1

pmeunier

Dec 13, 2022, 12:57 PM

C36737FJVMZSBWSQV7ZEYRYUPJVNLK3MGY3EJVAPYHYAM36J7TJQC

Dependencies

• [2] YDHYZA77 Adding the RootPage trait to access the raw bytes of a root page (needed in libpijul)
• [3] XOXTGNPZ Adding a contiguous memory allocator (for large blocks)
• [4] OHUZ73MK Versions
• [5] SYURNHHL Adding the `put_mut` and `set_left_child` methods
• [6] PUOGOIJ3 Debugging, impls and version bump
• [7] BPR2HVMR Version bump
• [8] VRAQTH26 Fixing two crashes on corrupted files
• [9] OHG5NX6K Refactoring, drop and the Check trait
• [10] GPP7KJSF Version bump
• [11] 4Z4GEJTF Version bump
• [12] FQ567GAX Version bumps
• [13] 5LSYTRQ6 More docs, example, and fixing the free page diagnostic function for mutable transactions
• [14] TJ2R4HAZ Accessing the root pages (unsafely, of course)
• [15] P5NWMJ2H Version bump
• [16] M6PHQUGL fallocate only when necessary
• [17] YXKP4AIW New file locks, with multiple sets of free pages
• [18] IIIOKJTE Version bump
• [19] GGEFV4YY Some page splits were not properly handled in deletions
• [20] DASFQGOR Debugging
• [21] 2ZRCQBXP Version bump
• [22] 77TAHKV4 Fixing a logical error (again) in del
• [23] 7T2CCH3P Fixing a segfault (wrong offset in page_unsized::del)
• [24] J7LJZBME Setting reverse cursor to last by default
• [25] E4MD6T3L Proofreading and commenting of this crate (massive bug fixes included)
• [26] DEKK3RUI Fixing a bug when splitting unsized pages
• [27] SO25TWFL A few features for integrating it into Pijul
• [28] OP6SVMOD Resetting history
• [29] RLVQDUPY Fixing a double-free error introduced in 1.2.13
• [30] W2MIZD5B Single file databases + CRC for the root pages (checking the other pages makes everything very slow)
• [31] WTXLZDYI Fixing bus errors on a full disk
• [32] Z33OHFPA Version bump
• [33] PRDUE4YA Cleanup + published on crates.io
• [*] QYDGYIZR Split trait Representable into its mandatory part and an optional part
• [*] CCNPHVQC Cleanup, comments, renaming

Change contents

• edit in sanakirja-core/src/lib.rs at line 164
  [35.1687]

  [35.1687]
  + }

  +

  + #[derive(Debug, Clone, Copy)]

  + #[repr(C)]

  + struct Ref {

  + p: u64,

  + len: u64,

  + }

  +

  + pub union Slice<'b> {

  + len: u16,

  + page: Ref,

  + mem: Mem<'b>,

  + }

  +

  + #[derive(Clone, Copy)]

  + #[repr(C)]

  + struct Mem<'b> {

  + _len: u16,

  + m: &'b [u8],

  + }

  +

  + impl<'a> core::fmt::Debug for Slice<'a> {

  + fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {

  + write!(fmt, "Slice({:?})", unsafe { self.len })

  + }

  + }

  +

  + impl<'a> core::convert::From<&'a [u8]> for Slice<'a> {

  + fn from(m: &'a [u8]) -> Slice<'a> {

  + let s = Slice {

  + mem: Mem { _len: 513, m },

  + };

  + s

  + }

  + }

  +

  + impl<'a> Slice<'a> {

  + pub fn as_bytes<T: LoadPage>(&self, txn: &T) -> Result<&[u8], T::Error> {

  + Ok(unsafe {

  + let len = u16::from_le(self.len) & 0xfff;

  + if len == 512 {

  + // Stored externally

  + let p = txn.load_page(u64::from_le(self.page.p) & !0xfff)?;

  + core::slice::from_raw_parts(p.data, u64::from_le(self.page.len) as usize)

  + } else if len == 513 {

  + // Stored in memory, not on any page

  + self.mem.m

  + } else {

  + core::slice::from_raw_parts(

  + (&self.len as *const u16 as *const u8).add(2),

  + len as usize,

  + )

  + }

  + })

  + }

  + }

  +

  + impl<'a> Storable for Slice<'a> {

  + type PageReferences = Pages;

  + fn page_references(&self) -> Self::PageReferences {

  + unsafe {

  + let len = u16::from_le(self.len);

  + if len == 512 {

  + let plen = u64::from_le(self.page.len);

  + let len_up = ((plen + PAGE_SIZE as u64 - 1) / PAGE_SIZE as u64)

  + * PAGE_SIZE as u64;

  + let offset = u64::from_le(self.page.p) & !0xfff;

  + Pages {

  + offset,

  + limit: offset + len_up,

  + }

  + } else {

  + Pages {

  + offset: 0,

  + limit: 0,

  + }

  + }

  + }

  + }

  + fn compare<T: LoadPage>(&self, t: &T, b: &Self) -> core::cmp::Ordering {

  + self.as_bytes(t).unwrap().cmp(b.as_bytes(t).unwrap())

  + }

  + }

  +

  + pub struct Pages {

  + offset: u64,

  + limit: u64,

  + }

  +

  + impl Iterator for Pages {

  + type Item = u64;

  + fn next(&mut self) -> Option<Self::Item> {

  + if self.offset >= self.limit {

  + None

  + } else {

  + let o = self.offset;

  + self.offset += PAGE_SIZE as u64;

  + Some(o)

  + }

  + }
• edit in sanakirja-core/src/lib.rs at line 267
  [35.1690]

  [35.1690]
  + impl<'b> UnsizedStorable for Slice<'b> {

  + const ALIGN: usize = 8;

  + fn size(&self) -> usize {

  + let s = unsafe {

  + if u16::from_le(self.len) == 512 {

  + // Stored externally

  + 16

  + } else if u16::from_le(self.len) == 513 {

  + // Stored in memory, not on any page

  + if self.mem.m.len() > 510 {

  + 16

  + } else {

  + 2 + self.mem.m.len()

  + }

  + } else {

  + u16::from_le(self.len) as usize

  + }

  + };

  + s

  + }

  + unsafe fn from_raw_ptr<'a, T>(_: &T, p: *const u8) -> &'a Self {

  + &*(p as *const Self)

  + }

  + unsafe fn onpage_size(p: *const u8) -> usize {

  + let p = &*(p as *const Self);

  + if u16::from_le(p.len) == 512 {

  + // Stored externally

  + 16

  + } else if u16::from_le(p.len) == 513 {

  + // Stored in memory, not on any page

  + 2 + p.mem.m.len()

  + } else {

  + u16::from_le(p.len) as usize

  + }

  + }

  +

  + unsafe fn write_to_page<T: AllocPage>(&self, t: &mut T, p: *mut u8) {

  + if self.len == 512 {

  + // Stored externally

  + core::ptr::copy(&self.page as *const Ref as *const u8, p, 16)

  + } else if self.len == 513 {

  + // Alloc ?

  + if self.mem.m.len() > 510 {

  + let len = self.mem.m.len();

  + let page = if len > 4096 {

  + t.alloc_contiguous(

  + (((len + PAGE_SIZE - 1) / PAGE_SIZE) * PAGE_SIZE)

  + as u64

  + ).unwrap()

  + } else {

  + t.alloc_page().unwrap()

  + };

  + core::ptr::copy_nonoverlapping(self.mem.m.as_ptr(), page.0.data, len);

  + let p = &mut *(p as *mut Ref);

  + p.len = (self.mem.m.len() as u64).to_le();

  + p.p = (page.0.offset | 512).to_le();

  + } else {

  + let len = self.mem.m.len();

  + *(p as *mut u16) = (len as u16).to_le();

  + core::ptr::copy_nonoverlapping(self.mem.m.as_ptr(), p.add(2), len)

  + }

  + } else {

  + core::ptr::copy(&self.len as *const u16 as *const u8, p, 2 + u16::from_le(self.len) as usize)

  + }

  + }

  + }

  +
• edit in sanakirja-core/src/lib.rs at line 503
  [4.3229]

  [36.2799]
  + /// Allocate many contiguous pages, return the first one

  + fn alloc_contiguous(&mut self, length: u64) -> Result<MutPage, Self::Error>;
• replacement in sanakirja-core/Cargo.toml at line 3
  [4.70812]→[3.0:18](∅→∅)
  − version = "1.3.0"
  [4.70812]

  [4.70847]
  + version = "1.3.2"
• replacement in sanakirja/src/lib.rs at line 108
  [2.67]→[4.2260:2378](∅→∅),[4.6634]→[4.2260:2378](∅→∅)
  − pub use sanakirja_core::{btree, direct_repr, LoadPage, AllocPage, Storable, UnsizedStorable, MutPage, CowPage, Page};
  [2.67]

  [4.312]
  + pub use sanakirja_core::{btree, direct_repr, LoadPage, AllocPage, Storable, UnsizedStorable, MutPage, CowPage, Page, Slice};
• replacement in sanakirja/Cargo.toml at line 3
  [4.108441]→[3.4436:4454](∅→∅)
  − version = "1.3.0"
  [4.108441]

  [4.108459]
  + version = "1.3.1"
• replacement in sanakirja/Cargo.toml at line 32
  [4.84]→[3.4455:4522](∅→∅)
  − sanakirja-core = { path = "../sanakirja-core", version = "1.3.0" }
  [4.84]

  [4.5528]
  + sanakirja-core = { path = "../sanakirja-core", version = "1.3.1" }