New file locks, with multiple sets of free pages

Dependencies

• [2] WS4ZQM4R Debugging, tests, etc.
• [3] EAAYH6BQ Debugging put
• [4] FMN7X4J2 Micro-improvements, now noticeably faster than std::collections::BTreeMap
• [5] OP6SVMOD Resetting history
• [6] PXF3R6SV Improving test coverage for btree::cursor
• [7] ONES3V46 reference counting works for put

Change contents

• replacement in sanakirja/src/environment/muttxn.rs at line 6
  [3.80263]→[2.37103:37172](∅→∅)
  − impl<E:Borrow<Env<Exclusive>>, T> std::fmt::Debug for MutTxn<E, T> {
  [3.80263]

  [2.37172]
  + impl<E:Borrow<Env>, T> std::fmt::Debug for MutTxn<E, T> {
• replacement in sanakirja/src/environment/muttxn.rs at line 13
  [3.80290]→[3.80290:80340](∅→∅)
  − pub struct MutTxn<E: Borrow<Env<Exclusive>>, T> {
  [3.80290]

  [3.80340]
  + pub struct MutTxn<E: Borrow<Env>, T> {
• replacement in sanakirja/src/environment/muttxn.rs at line 33
  [3.80903]→[3.80903:80962](∅→∅)
  − impl<E: Borrow<Env<Exclusive>>, T> Drop for MutTxn<E, T> {
  [3.80903]

  [3.80962]
  + impl<E: Borrow<Env>, T> Drop for MutTxn<E, T> {
• replacement in sanakirja/src/environment/muttxn.rs at line 38
  [3.81081]→[3.81081:81129](∅→∅)
  − env.roots[*root].unlock_exclusive()
  [3.81081]

  [3.81129]
  + env.roots[*root].rw.unlock_exclusive();

  + if let Some(ref f) = env.roots[*root].lock_file {

  + f.unlock().unwrap_or(())

  + }
• replacement in sanakirja/src/environment/muttxn.rs at line 52
  [3.81284]→[3.81284:81368](∅→∅)
  − impl<'a, E: Borrow<Env<Exclusive>>, T> Commit for MutTxn<E, &'a mut MutTxn<E, T>> {
  [3.81284]

  [2.37309]
  + impl<'a, E: Borrow<Env>, T> Commit for MutTxn<E, &'a mut MutTxn<E, T>> {
• replacement in sanakirja/src/environment/muttxn.rs at line 72
  [3.81942]→[3.81942:81964](∅→∅)
  − impl Env<Exclusive> {
  [3.81942]

  [3.81964]
  + impl Env {
• replacement in sanakirja/src/environment/muttxn.rs at line 82
  [3.82396]→[3.82396:82455](∅→∅)
  − env_.roots[(*v + 1) % n].lock_exclusive();
  [3.82396]

  [3.82455]
  + env_.roots[*v].rw.lock_exclusive();

  + if let Some(ref f) = env_.roots[*v].lock_file {

  + f.lock_exclusive()?

  + }
• edit in sanakirja/src/environment/muttxn.rs at line 87
  [3.82513]

  [3.82513]
  + // Root of the last MutTxn.

  + let v0 = *v + n - 1;
• replacement in sanakirja/src/environment/muttxn.rs at line 90
  [3.82586]→[3.82586:82745](∅→∅)
  − let page_ptr = maps.offset((*v * PAGE_SIZE) as isize);

  − let next_page_ptr = maps.offset((((*v + 1) % n) * PAGE_SIZE) as isize);
  [3.82586]

  [3.82745]
  + let page_ptr = maps.offset((v0 * PAGE_SIZE) as isize);

  + let next_page_ptr = maps.offset((*v * PAGE_SIZE) as isize);
• replacement in sanakirja/src/environment/muttxn.rs at line 131
  [3.84060]→[3.84060:84119](∅→∅)
  − impl<E: Borrow<Env<Exclusive>>> Commit for MutTxn<E, ()> {
  [3.84060]

  [3.84119]
  + impl<E: Borrow<Env>> Commit for MutTxn<E, ()> {
• edit in sanakirja/src/environment/muttxn.rs at line 164
  [3.85644]→[3.85644:85733](∅→∅)
  − (&mut *globptr).root = ((&mut *globptr).root + 1) % (&mut *globptr).n_roots;
• edit in sanakirja/src/environment/muttxn.rs at line 165
  [3.85828]

  [3.85828]
  + (&mut *globptr).root = *root as u8;
• edit in sanakirja/src/environment/muttxn.rs at line 170
  [3.85909]

  [3.85909]
  + env.roots[*root].rw.unlock_exclusive();

  + if let Some(ref f) = env.roots[*root].lock_file {

  + f.unlock().unwrap_or(())

  + }
• replacement in sanakirja/src/environment/muttxn.rs at line 180
  [3.86004]→[3.86004:86054](∅→∅)
  − impl<E: Borrow<Env<Exclusive>>, T> MutTxn<E, T> {
  [3.86004]

  [3.86054]
  + impl<E: Borrow<Env>, T> MutTxn<E, T> {
• replacement in sanakirja/src/environment/muttxn.rs at line 214
  [3.87011]→[3.87011:87091](∅→∅)
  − impl<E: Borrow<Env<Exclusive>>, T> sanakirja_core::AllocPage for MutTxn<E, T> {
  [3.87011]

  [3.87091]
  + impl<E: Borrow<Env>, T> sanakirja_core::AllocPage for MutTxn<E, T> {
• replacement in sanakirja/src/environment/muttxn.rs at line 336
  [3.90733]→[3.90733:90812](∅→∅)
  − impl<E: Borrow<Env<Exclusive>>, A> sanakirja_core::LoadPage for MutTxn<E, A> {
  [3.90733]

  [3.90812]
  + impl<E: Borrow<Env>, A> sanakirja_core::LoadPage for MutTxn<E, A> {
• replacement in sanakirja/src/environment/mod.rs at line 4
  [3.91687]→[3.91687:91733](∅→∅)
  − use std::borrow::Borrow;

  − use std::path::Path;
  [3.91687]

  [3.91733]
  + #[cfg(feature = "mmap")]

  + use fs2::FileExt;
• edit in sanakirja/src/environment/mod.rs at line 8
  [2.39944]

  [3.91760]
  +

  + use std::borrow::Borrow;

  + #[cfg(not(feature = "mmap"))]

  + use std::path::Path;

  + #[cfg(feature = "mmap")]

  + use std::path::{Path, PathBuf};

  + #[cfg(feature = "mmap")]

  + use std::fs::OpenOptions;

  + use std::sync::atomic::{AtomicUsize, Ordering};
• edit in sanakirja/src/environment/mod.rs at line 46
  [3.92395]→[3.92395:92827](∅→∅)
  − }

  − }

  −

  − /// Represents an exclusive lock taken on the environment.

  − pub struct Exclusive(std::fs::File);

  −

  − #[cfg(feature = "mmap")]

  − impl Drop for Exclusive {

  − fn drop(&mut self) {

  − self.0.unlock().unwrap_or(());

  − }

  − }

  −

  − /// Represents a shared lock taken on the environment.

  − pub struct Shared(std::fs::File);

  −

  − #[cfg(feature = "mmap")]

  − impl Drop for Shared {

  − fn drop(&mut self) {

  − self.0.unlock().unwrap_or(());
• replacement in sanakirja/src/environment/mod.rs at line 60
  [3.93379]→[3.93379:93399](∅→∅)
  − pub struct Env<T> {
  [3.93379]

  [3.93399]
  + pub struct Env {
• edit in sanakirja/src/environment/mod.rs at line 63
  [3.93455]→[3.93455:93659](∅→∅)
  − /// It is undefined behavior to have a file mmapped for than once.

  − #[cfg(feature = "mmap")]

  − lock_file: Option<T>,

  − #[cfg(not(feature = "mmap"))]

  − lock_file: std::marker::PhantomData<T>,
• replacement in sanakirja/src/environment/mod.rs at line 67
  [3.93735]→[3.93735:93775](∅→∅)
  − roots: Vec<parking_lot::RawRwLock>,
  [3.93735]

  [3.93775]
  + roots: Vec<RootLock>,

  + // Root number of the next MutTxn.
• replacement in sanakirja/src/environment/mod.rs at line 72
  [3.93802]→[3.93802:93870](∅→∅)
  − unsafe impl<T> Send for Env<T> {}

  − unsafe impl<T> Sync for Env<T> {}
  [3.93802]

  [3.93870]
  + struct RootLock {

  + /// It is undefined behavior to have a file mmapped for than once.

  + #[cfg(feature = "mmap")]

  + lock_file: Option<std::fs::File>,

  +

  + rw: parking_lot::RawRwLock,

  + n_txn: AtomicUsize,

  + }

  +

  + unsafe impl Send for Env {}

  + unsafe impl Sync for Env {}
• replacement in sanakirja/src/environment/mod.rs at line 85
  [3.93896]→[3.93896:93922](∅→∅)
  − impl<T> Drop for Env<T> {
  [3.93896]

  [3.93922]
  + impl Drop for Env {
• replacement in sanakirja/src/environment/mod.rs at line 87
  [3.93947]→[3.93947:94005](∅→∅)
  − for map in self.mmaps.lock().unwrap().drain(..) {
  [3.93947]

  [3.94005]
  + for map in self.mmaps.lock().drain(..) {
• replacement in sanakirja/src/environment/mod.rs at line 95
  [3.94110]→[3.94110:94136](∅→∅)
  − impl<T> Drop for Env<T> {
  [3.94110]

  [3.94136]
  + impl Drop for Env {
• replacement in sanakirja/src/environment/mod.rs at line 105
  [3.94354]→[3.94354:94393](∅→∅)
  − pub struct Txn<L, E: Borrow<Env<L>>> {
  [3.94354]

  [3.94393]
  + pub struct Txn<E: Borrow<Env>> {
• edit in sanakirja/src/environment/mod.rs at line 107
  [3.94405]→[3.94405:94444](∅→∅)
  − lock: std::marker::PhantomData<L>,
• replacement in sanakirja/src/environment/mod.rs at line 110
  [3.94464]→[3.94464:94481](∅→∅)
  − impl<T> Env<T> {
  [3.94464]

  [3.94481]
  + impl Env {
• replacement in sanakirja/src/environment/mod.rs at line 129
  [3.95236]→[3.95236:95328](∅→∅)
  − pub unsafe fn new_nolock<P: AsRef<Path>>(path: P, length: u64) -> Result<Self, Error> {
  [3.95236]

  [3.95328]
  + pub unsafe fn new_nolock<P: AsRef<Path>>(path: P, length: u64, n_roots: usize) -> Result<Self, Error> {
• replacement in sanakirja/src/environment/mod.rs at line 138
  [3.95677]→[3.95677:95726](∅→∅)
  − let length = length.next_power_of_two();
  [3.95677]

  [3.95726]
  + let length = (length + 4095) & !4096;
• replacement in sanakirja/src/environment/mod.rs at line 147
  [3.95982]→[3.95982:96066](∅→∅)
  − let mut env = Self::new_nolock_mmap(Some(file), length, mmap, !db_exists)?;
  [3.95982]

  [3.96066]
  + let mut env = Self::new_nolock_mmap(Some(file), length, mmap, !db_exists, n_roots)?;
• edit in sanakirja/src/environment/mod.rs at line 159
  [3.96325]

  [3.96325]
  + n_roots: usize,
• edit in sanakirja/src/environment/mod.rs at line 162
  [3.96393]→[3.96393:96418](∅→∅)
  − let n_roots = 2;
• replacement in sanakirja/src/environment/mod.rs at line 164
  [3.96487]→[3.96487:96570](∅→∅)
  − *(map.offset(i * PAGE_SIZE) as *mut GlobalHeader) = GlobalHeader {
  [3.96487]

  [3.96570]
  + *(map.add(i * PAGE_SIZE) as *mut GlobalHeader) = GlobalHeader {
• replacement in sanakirja/src/environment/mod.rs at line 167
  [3.96645]→[3.96645:96674](∅→∅)
  − n_roots,
  [3.96645]

  [3.96674]
  + n_roots: n_roots as u8,
• replacement in sanakirja/src/environment/mod.rs at line 169
  [3.96702]→[3.96702:96751](∅→∅)
  − length: n_roots * PAGE_SIZE,
  [3.96702]

  [3.96751]
  + length,
• replacement in sanakirja/src/environment/mod.rs at line 174
  [3.96846]→[3.96846:96911](∅→∅)
  − GlobalHeader::from_le(*(map as *const GlobalHeader))
  [3.96846]

  [3.96911]
  + GlobalHeader::from_le(&*(map as *const GlobalHeader))
• replacement in sanakirja/src/environment/mod.rs at line 176
  [3.96928]→[3.96928:96993](∅→∅)
  − GlobalHeader::from_le(*(map as *const GlobalHeader))
  [3.96928]

  [3.96993]
  + GlobalHeader::from_le(&*(map as *const GlobalHeader))
• replacement in sanakirja/src/environment/mod.rs at line 178
  [3.97004]→[3.97004:97157](∅→∅)
  − let mut versions = Vec::with_capacity(glob.n_versions as usize);

  − for _ in 0..glob.n_versions {

  − versions.push(Mutex::new(()))
  [3.97004]

  [3.97157]
  + let mut roots = Vec::with_capacity(glob.n_roots as usize);

  + for _ in 0..glob.n_roots {

  + roots.push(RootLock {

  + rw: <parking_lot::RawRwLock as parking_lot::lock_api::RawRwLock>::INIT,

  + n_txn: AtomicUsize::new(0),

  + lock_file: None,

  + })
• edit in sanakirja/src/environment/mod.rs at line 194
  [3.97367]→[3.97367:97396](∅→∅)
  − lock_file: None,
• replacement in sanakirja/src/environment/mod.rs at line 195
  [3.97442]→[3.97442:97514](∅→∅)
  − mutable: parking_lot::RawMutex::INIT,

  − versions,
  [3.97442]

  [3.97514]
  + roots,

  + root: Mutex::new(glob.root as usize),
• replacement in sanakirja/src/environment/mod.rs at line 202
  [3.97582]→[3.97582:97678](∅→∅)
  − unsafe fn new_nolock_mmap(length: u64, initialise: bool) -> Result<Env<Exclusive>, Error> {
  [3.97582]

  [3.97678]
  + unsafe fn new_nolock_mmap(length: u64, initialise: bool) -> Result<Env, Error> {
• replacement in sanakirja/src/environment/mod.rs at line 222
  [3.98453]→[3.98453:98544](∅→∅)
  − roots.push(<parking_lot::RawRwLock as parking_lot::lock_api::RawRwLock>::INIT)
  [3.98453]

  [3.98544]
  + roots.push(RootLock {

  + rw: <parking_lot::RawRwLock as parking_lot::lock_api::RawRwLock>::INIT,

  + n_txn: AtomicUsize::new(0),

  + })
• edit in sanakirja/src/environment/mod.rs at line 233
  [3.98710]→[3.98710:98759](∅→∅)
  − lock_file: std::marker::PhantomData,
• replacement in sanakirja/src/environment/mod.rs at line 241
  [3.98910]→[3.98910:99664](∅→∅)
  − impl Env<Shared> {

  − /// Initialize an environment. `length` must be a strictly

  − /// positive multiple of 4096. The same file can only be open in

  − /// one process or thread at the same time, and this is enforced

  − /// by a locked file.

  − #[cfg(feature = "mmap")]

  − pub fn new_shared<P: AsRef<Path>>(path: P, length: u64) -> Result<Env<Shared>, Error> {

  − let lock_file = OpenOptions::new()

  − .read(true)

  − .write(true)

  − .truncate(false)

  − .create(true)

  − .open(path.as_ref().join("db").with_extension("lock"))?;

  − lock_file.lock_shared()?;

  − let mut env = unsafe { Self::new_nolock(path, length)? };

  − env.lock_file = Some(Shared(lock_file));

  − Ok(env)

  − }
  [3.98910]

  [3.99664]
  + impl Env {
• replacement in sanakirja/src/environment/mod.rs at line 244
  [3.99728]→[3.99728:99844](∅→∅)
  − /// positive multiple of 4096. Returns an error if the database is

  − /// locked by another process or thread.
  [3.99728]

  [3.99844]
  + /// positive multiple of 4096.

  + ///

  + /// The `n_roots` parameter is the maximum number of mutable

  + /// transactions that can commit during a single immutable

  + /// transaction, and must be at most 255. If it is 1, mutable

  + /// transactions exclude all immutable transactions.
• replacement in sanakirja/src/environment/mod.rs at line 251
  [3.99873]→[3.99873:101798](∅→∅)
  − pub fn try_new_shared<P: AsRef<Path>>(path: P, length: u64) -> Result<Env<Shared>, Error> {

  − let lock_file = OpenOptions::new()

  − .read(true)

  − .write(true)

  − .truncate(false)

  − .create(true)

  − .open(path.as_ref().join("db").with_extension("lock"))?;

  − lock_file.try_lock_shared()?;

  − let mut env = unsafe { Self::new_nolock(path, length)? };

  − env.lock_file = Some(Shared(lock_file));

  − Ok(env)

  − }

  − }

  −

  − impl Env<Exclusive> {

  − /// Initialize an environment. `length` must be a strictly

  − /// positive multiple of 4096. The same file can only be open in

  − /// one process or thread at the same time, and this is enforced

  − /// by a locked file.

  − #[cfg(feature = "mmap")]

  − pub fn new<P: AsRef<Path>>(path: P, length: u64) -> Result<Env<Exclusive>, Error> {

  − let lock_file = OpenOptions::new()

  − .read(true)

  − .write(true)

  − .truncate(false)

  − .create(true)

  − .open(path.as_ref().join("db").with_extension("lock"))?;

  − lock_file.lock_exclusive()?;

  − let mut env = unsafe { Self::new_nolock(path, length)? };

  − env.lock_file = Some(Exclusive(lock_file));

  − Ok(env)

  − }

  −

  − /// Initialize an environment. `length` must be a strictly

  − /// positive multiple of 4096. Returns an error if the database is

  − /// locked by another process or thread.

  − #[cfg(feature = "mmap")]

  − pub fn try_new<P: AsRef<Path>>(path: P, length: u64) -> Result<Env<Exclusive>, Error> {

  − let lock_file = OpenOptions::new()

  − .read(true)

  − .write(true)

  − .truncate(false)

  − .create(true)

  − .open(path.as_ref().join("db").with_extension("lock"))?;

  − lock_file.try_lock_exclusive()?;

  − let mut env = unsafe { Self::new_nolock(path, length)? };

  − env.lock_file = Some(Exclusive(lock_file));
  [3.99873]

  [3.101798]
  + pub fn new<P: AsRef<Path>>(path: P, length: u64, n_roots: usize) -> Result<Env, Error> {

  + assert!(n_roots < 256);

  + let path = path.as_ref();

  + let mut env = unsafe { Self::new_nolock(path, length, n_roots)? };

  + for (n, l) in env.roots.iter_mut().enumerate() {

  + l.lock_file = Some(

  + OpenOptions::new()

  + .read(true)

  + .write(true)

  + .truncate(false)

  + .create(true)

  + .open(path.join("db").with_extension(&format!("lock{}", n)))?

  + );

  + }
• replacement in sanakirja/src/environment/mod.rs at line 271
  [3.101972]→[3.101972:102110](∅→∅)
  − pub fn new_anon(length: u64) -> Result<Env<Exclusive>, Error> {

  − let length = std::cmp::max(length, 4096).next_power_of_two();
  [3.101972]

  [3.102110]
  + pub fn new_anon(length: u64, n_roots: usize) -> Result<Env, Error> {

  + let length = (std::cmp::max(length, 4096) + 4095) & !4096;
• replacement in sanakirja/src/environment/mod.rs at line 274
  [3.102174]→[3.102174:102241](∅→∅)
  − unsafe { Self::new_nolock_mmap(None, length, mmap, true) }
  [3.102174]

  [3.102241]
  + unsafe { Self::new_nolock_mmap(None, length, mmap, true, n_roots) }
• replacement in sanakirja/src/environment/mod.rs at line 280
  [3.102404]→[3.102404:102472](∅→∅)
  − pub fn new_anon(length: u64) -> Result<Env<Exclusive>, Error> {
  [3.102404]

  [3.102472]
  + pub fn new_anon(length: u64) -> Result<Env, Error> {
• edit in sanakirja/src/environment/mod.rs at line 284
  [3.102603]→[3.102603:102605](∅→∅)
  − }
• edit in sanakirja/src/environment/mod.rs at line 285
  [3.102606]→[3.102606:102623](∅→∅)
  − impl<L> Env<L> {
• replacement in sanakirja/src/environment/mod.rs at line 354
  [3.105034]→[3.105034:105090](∅→∅)
  − for m in self.mmaps.lock().unwrap().drain(..) {
  [3.105034]

  [3.105090]
  + for m in self.mmaps.lock().drain(..) {
• replacement in sanakirja/src/environment/mod.rs at line 357
  [3.105126]→[3.105126:105211](∅→∅)
  − if let Some(lock_file) = self.lock_file.take() {

  − drop(lock_file)
  [3.105126]

  [3.105211]
  + for l in self.roots.drain(..) {

  + if let Some(ref f) = l.lock_file {

  + f.unlock().unwrap_or(())

  + }
• replacement in sanakirja/src/environment/mod.rs at line 377
  [3.105584]→[3.105584:105601](∅→∅)
  − impl<L> Env<L> {
  [3.105584]

  [3.105601]
  + impl Env {
• replacement in sanakirja/src/environment/mod.rs at line 379
  [3.105640]→[3.105640:105716](∅→∅)
  − pub fn txn_begin<E: Borrow<Self>>(env: E) -> Result<Txn<L, E>, Error> {
  [3.105640]

  [3.105716]
  + pub fn txn_begin<E: Borrow<Self>>(env: E) -> Result<Txn<E>, Error> {
• replacement in sanakirja/src/environment/mod.rs at line 384
  [3.105889]→[3.105889:105933](∅→∅)
  − env_.roots[root].lock_shared();
  [3.105889]

  [3.105933]
  +

  + let old_n_txn = env_.roots[root].n_txn.fetch_add(1, Ordering::SeqCst);

  + if old_n_txn == 0 {

  + env_.roots[root].rw.lock_shared();

  + if let Some(ref f) = env_.roots[root].lock_file {

  + f.lock_shared()?;

  + }

  + }
• replacement in sanakirja/src/environment/mod.rs at line 394
  [3.105961]→[3.105961:106068](∅→∅)
  − Ok(Txn {

  − env,

  − lock: std::marker::PhantomData,

  − root,

  − })
  [3.105961]

  [3.106068]
  + Ok(Txn { env, root })
• replacement in sanakirja/src/environment/mod.rs at line 398
  [3.106077]→[3.106077:106125](∅→∅)
  − impl<L, E: Borrow<Env<L>>> Drop for Txn<L, E> {
  [3.106077]

  [3.106125]
  + impl<E: Borrow<Env>> Drop for Txn<E> {
• replacement in sanakirja/src/environment/mod.rs at line 401
  [3.106187]→[3.106187:106243](∅→∅)
  − unsafe { env.roots[self.root].unlock_shared() }
  [3.106187]

  [3.106243]
  + let old_n_txn = env.roots[self.root].n_txn.fetch_sub(1, Ordering::SeqCst);

  + if old_n_txn == 1 {

  + unsafe { env.roots[self.root].rw.unlock_shared() }

  + if let Some(ref f) = env.roots[self.root].lock_file {

  + f.unlock().unwrap_or(())

  + }

  + }
• replacement in sanakirja/src/environment/mod.rs at line 411
  [3.106252]→[3.106252:106320](∅→∅)
  − impl<L, E: Borrow<Env<L>>> sanakirja_core::LoadPage for Txn<L, E> {
  [3.106252]

  [3.106320]
  + impl<E: Borrow<Env>> sanakirja_core::LoadPage for Txn<E> {
• edit in sanakirja/Cargo.toml at line 8
  [3.108614]

  [3.108614]
  +

  + [features]

  + default = [ "mmap" ]

  + mmap = [ "memmap", "fs2" ]
• edit in sanakirja/Cargo.toml at line 17
  [3.2944]

  [3.108728]
  + # lmdb-rs = "*"

  + memmap = { version = "0.7", optional = true }

  + fs2 = { version = "0.4", optional = true }

  + sanakirja-core = { path = "../sanakirja-core", version = "*" }

  +

  + [dev-dependencies]
• edit in sanakirja/Cargo.toml at line 24
  [3.108745]→[3.108745:108821](∅→∅)
  − lmdb-rs = "*"

  − sanakirja-core = { path = "../sanakirja-core", version = "*" }