gigapixel/pgvector_rs - Change UV2RTMNSNI3AJFVVD66OT3CEF7WU7TLAPJKACGX7BBSSWAEBAAAAC

apparently had to reset the whole pijul thing

Created by BCbPPm1DzTKCMDwRhA2WJN63Uswk1sQh9wdbcbpfnWPZ on September 12, 2021

UV2RTMNSNI3AJFVVD66OT3CEF7WU7TLAPJKACGX7BBSSWAEBAAAAC

Dependencies

In channels

setup-mem-cache-as-index

Change contents

Insertion in src/vec_block_alloc.rs at line 4 [3.28]
[2.27]
[3.55]
```
use std::iter::repeat;
```
Replacement in src/vec_block_alloc.rs at line 11 [3.28]
B:BD[3.127] → [3.127:146]
```
#[derive(Default)]
```
[3.127]
[3.146]
```
#[derive(Default, Debug)]
```
Insertion in src/vec_block_alloc.rs at line 14 [3.28]
[3.281]
[3.281]
```
pub type BlockResult<T> = Result<T, F32BlockError>;
```

Replacement in src/vec_block_alloc.rs at line 19 [3.28]

B:BD[3.346] → [3.346:465]

    #[error("Trying to write over end of block: Start {write_start}, Length {write_length}, block size {block_size}")]

[3.346]

[3.465]

    #[error("Trying to write over end of block: Start {write_start}, Length {write_length}, block ends at {block_end}")]

Replacement in src/vec_block_alloc.rs at line 23 [3.28]

B:BD[3.537] → [3.537:564]

        block_size: usize,

[3.537]

[3.564]

        block_end: usize,
    },
    #[error("Trying to push past block limit")]
    LimitExceded {
        push_start: usize,
        push_size: usize,
        limit: usize,

Insertion in src/vec_block_alloc.rs at line 39 [3.28]

[3.846]

    #[error(
        "Trying to start at index {start} that is beyond the length of written data {data_len}"
    )]
    StartEmpty { data_len: usize, start: usize },
    #[error("Only support pushing bounded iterators")]
    PushUnboundedIter,

Replacement in src/vec_block_alloc.rs at line 52 [3.28]

B:BD[3.1048] → [3.1048:1079]

            block_size: block,

[3.1048]

[3.1079]

            block_end: block,
        }
    }
    fn limit_exceded(start: usize, length: usize, limit: usize) -> F32BlockError {
        F32BlockError::LimitExceded {
            push_start: start,
            push_size: length,
            limit,

Replacement in src/vec_block_alloc.rs at line 65 [3.28]

B:BD[3.1098] → [3.1098:1372]

impl<const Size: usize> F32Block<Size> {
    fn write_vec(&mut self, vec: &Vec<f32>, start: usize) -> Result<(), F32BlockError> {
        let end = start + vec.len();
        if end < start || (end) >= Size {
            Err(F32BlockError::overflow(start, vec.len(), Size))

[3.1098]

[3.1372]

struct Padded<'a> {
    dimension: u16,
    data: &'a [f32],
}
impl<'a> Padded<'a> {
    fn iterator_with_dim(self) -> impl Iterator<Item = f32> + 'a {
        self.data
            .iter()
            .map(|x| *x)
            .chain(repeat(0.0))
            .take(self.dimension as usize)
    }
}
/// This will write over a piece of the block
impl<const SIZE: usize> F32Block<SIZE> {
    fn write_slice<'a, 'b>(&'a mut self, vec: &'b [f32], start: usize) -> BlockResult<()> {
        let vec_len = vec.len();
        let data_len = self.0.len();
        let end = vec_len + start;
        if end <= data_len {
            for (new_val, old_val) in vec.iter().zip(self.0[start..].iter_mut()) {
                *old_val = *new_val;
            }
            Ok(())
        } else {
            Err(F32BlockError::overflow(start, vec_len, data_len))
        }
    }
    fn write_iter<IterType: Iterator<Item = f32>>(
        &mut self,
        start: usize,
        iter: IterType,
    ) -> BlockResult<()> {
        if let (_, Some(upper_bound)) = iter.size_hint() {
            let data_len = self.0.len();
            if start + upper_bound > data_len {
                for (old_val, new_val) in self.0[start..].iter_mut().zip(iter) {
                    *old_val = new_val;
                }
                Ok(())
            } else {
                Err(F32BlockError::overflow(start, upper_bound, data_len))
            }

Replacement in src/vec_block_alloc.rs at line 111 [3.28]

B:BD[3.1389] → [3.1389:1717]

            let vec_data = &mut self.0;
            let data_len = vec_data.len();
            if start > data_len {
                (data_len..start)
                    .map(|_| &0.0)
                    .chain(vec.iter())
                    .map(|x| vec_data.push(*x))
                    .collect::<Result<Vec<()>, f32>>()

[3.1389]

[3.1717]

            Err(F32BlockError::PushUnboundedIter)
        }
    }
    fn push_slice<'a, 'b>(&'a mut self, vec: &'b [f32]) -> BlockResult<()> {
        let data_len = self.0.len();
        let vec_len = vec.len();
        self.0
            .extend_from_slice(vec)
            .map_err(|_| F32BlockError::limit_exceded(data_len, vec_len, SIZE))
    }
    /// We allow pushing iterators, but we require an upper bound for the size hint of the
    /// iterator.
    fn push_iter<IterType: Iterator<Item = f32>>(&mut self, iter: IterType) -> BlockResult<()> {
        if let (_, Some(upper_bound)) = iter.size_hint() {
            let data_len = self.0.len();
            if data_len + upper_bound <= SIZE {
                iter.map(|el| self.0.push(el))
                    .collect::<Result<Vec<_>, _>>()
                    .map_err(|_| F32BlockError::PushUnboundedIter)

Deletion in src/vec_block_alloc.rs at line 133 [3.28]

B:BD[3.1750] → [3.1750:1832]

                    .map_err(|_| F32BlockError::overflow(start, vec.len(), Size))

Replacement in src/vec_block_alloc.rs at line 134 [3.28]

B:BD[3.1853] → [3.1853:2413]

                let intersect = data_len.min(end);
                let intersect_shift = intersect - start;
                vec_data[start..intersect]
                    .iter_mut()
                    .zip(vec.iter())
                    .for_each(|(buffer, new)| *buffer = *new);
                vec[intersect_shift..]
                    .iter()
                    .map(|x| vec_data.push(*x))
                    .collect::<Result<Vec<()>, f32>>()
                    .map_err(|_| F32BlockError::overflow(start, vec.len(), Size))?;
                Ok(())

[3.1853]

[3.2413]

                Err(F32BlockError::limit_exceded(data_len, upper_bound, SIZE))

Insertion in src/vec_block_alloc.rs at line 136 [3.28]

[3.2427]

        } else {
            Err(F32BlockError::PushUnboundedIter)

Replacement in src/vec_block_alloc.rs at line 142 [3.28]

B:BD[3.2524] → [3.2524:2597]

        const SEGMENT_SIZE: usize = 4096;
        let end = start + len;

[3.2524]

[3.2597]

        let mut end = start + len;

Replacement in src/vec_block_alloc.rs at line 144 [3.28]

B:BD[3.2634] → [3.2634:2807]

        if end < start || end > data_len {
            Err(F32BlockError::DeleteEmpty {
                data_len,
                start,
                len,
            })

[3.2634]

[3.2807]

        if end > data_len {
            Err(F32BlockError::overflow(start, len, data_len))

Replacement in src/vec_block_alloc.rs at line 147 [3.28]

B:BD[3.2824] → [3.2824:3814]

            let mut tmp = [0.0; SEGMENT_SIZE];
            let to_move = data_len - end;
            (end..data_len).step_by(SEGMENT_SIZE).for_each(|i| {
                let vec_data = &self.0;
                tmp.iter_mut()
                    .zip(vec_data[i..(i + SEGMENT_SIZE)].iter())
                    .for_each(|(tmp_val, vec_data_val)| *tmp_val = *vec_data_val);
                let vec_data = &mut self.0;
                vec_data[(i - len)..(i - len + SEGMENT_SIZE)]
                    .iter_mut()
                    .zip(tmp.iter())
                    .for_each(|(vec_data_val, tmp_val)| *vec_data_val = *tmp_val);
            });
            (0..len)
                .map(|_| {
                    self.0.pop().ok_or(F32BlockError::DeleteEmpty {
                        data_len,
                        start,
                        len,
                    })
                })
                .collect::<Result<Vec<f32>, F32BlockError>>()
                .map(|_| ())

[3.2824]

[3.3814]

            let mut loop_start = start;
            loop {
                let (initial, tail) = self.0.split_at_mut(end);
                for (to_delete, to_assign) in initial[loop_start..].iter_mut().zip(tail.iter()) {
                    *to_delete = *to_assign;
                }
                end += len;
                loop_start += len;
                if end >= data_len {
                    break;
                }
            }
            self.0.truncate(data_len - len);
            Ok(())

Replacement in src/vec_block_alloc.rs at line 164 [3.28]

B:BD[3.3831] → [3.3831:3967]

    fn read_data(
        &self,
        start: usize,
        len: usize,
    ) -> Result<impl Iterator<Item = &f32>, F32BlockError> {

[3.3831]

[3.3967]

    fn read_data(&self, start: usize, len: usize) -> BlockResult<&[f32]> {

Replacement in src/vec_block_alloc.rs at line 167 [3.28]

B:BD[3.4035] → [3.4035:4142]

        if (end < start) || (end >= data_len) {
            Err(F32BlockError::overflow(start, len, Size))

[3.4035]

[3.4142]

        if end > data_len {
            Err(F32BlockError::overflow(start, len, SIZE))

Replacement in src/vec_block_alloc.rs at line 170 [3.28]
B:BD[3.4159] → [3.4159:4201]
```
            Ok(self.0[start..end].iter())
```
[3.4159]
[3.4201]
```
            Ok(&self.0[start..end])
```

Insertion in src/vec_block_alloc.rs at line 177 [3.28]

[3.4255]

    const TOL: f32 = 1.0e-7;
    const BLOCK_SIZE: usize = 8192;

Deletion in src/vec_block_alloc.rs at line 182 [3.28]
B:BD[3.4345] → [3.4345:4374]
```
    use core::cell::RefCell;
```

Replacement in src/vec_block_alloc.rs at line 188 [3.28]

B:BD[3.4542] → [3.4542:4614]

        const BLOCK_SIZE: usize = 10000;
        let good_start = 1000;

[3.4542]

[3.4614]

        let good_start = 0;

Replacement in src/vec_block_alloc.rs at line 190 [3.28]

B:BD[3.4642] → [3.4642:4690]

        let bad_start = 10000 - (vec_size / 2);

[3.4642]

[3.4690]

        let end = vec_size * 2;
        let bad_start = BLOCK_SIZE - (vec_size / 2);

Replacement in src/vec_block_alloc.rs at line 195 [3.28]

B:BD[3.4771] → [3.4771:4986]

        let mem = RefCell::new(Box::new(F32Block::<BLOCK_SIZE>::default()));
        assert!(mem
            .borrow_mut()
            .write_vec(&vec, good_start)
            .map(|_| true)
            .unwrap());

[3.4771]

[3.4986]

        let mut mem = F32Block::<BLOCK_SIZE>::default();
        mem.push_iter((0..end).map(|x| (x as f32))).unwrap();
        mem.write_slice(&vec, good_start).unwrap();
        let read = mem.read_data(good_start, vec_size).unwrap();
        let diff: f32 = read
            .iter()
            .zip(vec.iter())
            .map(|(x, y)| (x - y).abs())
            .sum();
        assert!(diff <= TOL);

Replacement in src/vec_block_alloc.rs at line 207 [3.28]

B:BD[3.5006] → [3.5006:5145]

            mem.borrow_mut().write_vec(&vec, bad_start).unwrap_err(),
            F32BlockError::overflow(bad_start, vec_size, BLOCK_SIZE)

[3.5006]

[3.5145]

            mem.write_slice(&vec, bad_start).unwrap_err(),
            F32BlockError::overflow(bad_start, vec_size, end)
        );
        assert_eq!(
            mem.write_slice(&vec, BLOCK_SIZE + vec_size).unwrap_err(),
            F32BlockError::overflow(BLOCK_SIZE + vec_size, vec_size, end)

Replacement in src/vec_block_alloc.rs at line 217 [3.28]

B:BD[3.5175] → [3.5175:5486]

    fn test_delete() -> Result<(), F32BlockError> {
        const BLOCK_SIZE: usize = 100000;
        let good_start = 10000; // something less than BLOCK_SIZE;
        let len = 10000; // something less than BLOCK_SIZE - good_start;
        let mem = RefCell::new(Box::new(F32Block::<BLOCK_SIZE>::default()));

[3.5175]

[3.5486]

    fn test_push() {
        let mut mem = F32Block::<BLOCK_SIZE>::default();
        let vec_size = 512;
        let vec = (0..vec_size).map(|x| (x as f32).sin()).collect::<Vec<_>>();
        let pushes = (0..20)
            .map(|_| mem.push_slice(&vec))
            .collect::<Result<Vec<()>, F32BlockError>>();
        assert!(pushes.is_err());
    }
    #[test]
    fn test_delete() {
        let good_start = 1024; // something less than BLOCK_SIZE;
        let len = good_start * 1024; // something less than BLOCK_SIZE - good_start;
        let mut mem = F32Block::<BLOCK_SIZE>::default();

Replacement in src/vec_block_alloc.rs at line 234 [3.28]

B:BD[3.5506] → [3.5506:5710]

            mem.borrow_mut().free(good_start, len).unwrap_err(),
            F32BlockError::DeleteEmpty {
                data_len: 0,
                start: good_start,
                len
            }

[3.5506]

[3.5710]

            mem.free(good_start, len).unwrap_err(),
            F32BlockError::overflow(good_start, len, BLOCK_SIZE)

Replacement in src/vec_block_alloc.rs at line 237 [3.28]

B:BD[3.5721] → [3.5721:6049]

        mem.borrow_mut().write_vec(
            &(0..(good_start + len))
                .map(|x| (x as f32).cos())
                .collect::<Vec<_>>(),
            0,
        )?;
        assert!(mem
            .borrow_mut()
            .free(good_start, len)
            .map(|_| true)
            .unwrap());
        Ok(())

[3.5721]

[3.6049]


        mem.push_iter((0..len).map(|x| (x as f32).cos())).unwrap();
        assert!(mem.free(good_start, len).is_ok());
        assert!(mem.free(good_start, len).is_err());

Replacement in src/vec_block_alloc.rs at line 245 [3.28]
B:BD[3.6126] → [2.28:47]
```
#[derive(Default)]
```
[3.6126]
[3.6145]
```
#[derive(Default, Debug)]
```
Deletion in src/vec_block_alloc.rs at line 248 [3.28]
B:BD[3.6221] → [3.6221:6239]
```
    start: usize,
```
Deletion in src/vec_block_alloc.rs at line 257 [3.28]
B:BD[3.6547] → [3.6547:6569]
```
            start: 0,
```

Insertion in src/vec_block_alloc.rs at line 261 [3.28]

[3.6661]

    pub fn is_set(&self) -> bool {
        self.dimension > 0
    }

Replacement in src/vec_block_alloc.rs at line 273 [3.28]

B:BD[3.6848] → [3.6848:6906]

        self.start + self.len * (self.dimension as usize)

[3.6848]

[3.6906]

        self.len * (self.dimension as usize)

Replacement in src/vec_block_alloc.rs at line 276 [3.28]

B:BD[3.6913] → [3.6913:6990]

    pub fn add_vec(&mut self, vec: &Vec<f32>) -> Result<(), F32BlockError> {

[3.6913]

[3.6990]

    pub fn add_vec(&mut self, vec: &[f32]) -> Result<(), F32BlockError> {
        //

Replacement in src/vec_block_alloc.rs at line 279 [3.28]

B:BD[3.7032] → [3.7032:7249]

        let end = self.end();
        let write_vec = vec[..self.dim()].iter().map(|x| *x).collect::<Vec<_>>(); // Unecessary allocation ! need to implement as iters
        self.f32_data.write_vec(&write_vec, end)?;

[3.7032]

[3.7249]

        let padded = Padded {
            dimension: self.dimension,
            data: vec,
        };
        self.f32_data.push_iter(padded.iterator_with_dim())?;

Replacement in src/vec_block_alloc.rs at line 293 [3.28]

B:BD[3.7500] → [3.7500:7562]

            let slice_start = self.start + index * dimension;

[3.7500]

[3.7562]

            let slice_start = index * dimension;

Replacement in src/vec_block_alloc.rs at line 301 [3.28]

B:BD[3.7735] → [3.7735:7831]

    pub fn get_iter(&self, index: usize) -> Result<impl Iterator<Item = &f32>, F32BlockError> {

[3.7735]

[3.7831]

    pub fn get_slice(&self, index: usize) -> Result<&[f32], F32BlockError> {

Replacement in src/vec_block_alloc.rs at line 303 [3.28]

B:BD[3.7867] → [3.7867:8034]

        let slice_start = self.start + (dimension) * index;
        let slice_end = slice_start + (dimension);
        self.f32_data.read_data(slice_start, slice_end)

[3.7867]

[3.8034]

        let slice_start = dimension * index;
        let slice_end = slice_start + dimension;
        self.f32_data.read_data(slice_start, dimension)
    }
    fn pad<'b>(&'_ self, data: &'b [f32]) -> Padded<'b> {
        Padded {
            dimension: self.dimension,
            data,
        }
    }
    pub fn update_vec(&mut self, index: usize, data: &[f32]) -> Result<(), F32BlockError> {
        let padded = self.pad(data);
        self.f32_data
            .write_iter(index * self.dim(), padded.iterator_with_dim())?;
        Ok(())

Insertion in src/vec_block_alloc.rs at line 321 [3.28]
[3.8041]
[3.8041]
```
    // todo SIMD????
```

Replacement in src/vec_block_alloc.rs at line 326 [3.28]

B:BD[3.8206] → [3.8206:8335]

                let data_iter = self.get_iter(index)?;
                Ok(data_iter.zip(vec.iter()).map(|(&a, &b)| a * b).sum())

[3.8206]

[3.8335]

                let data = self.get_slice(index)?;
                Ok(data.iter().zip(vec.iter()).map(|(&a, &b)| a * b).sum())

Replacement in src/mem_cache_index/mod.rs at line 1 [2.206]
B:BD[2.206] → [2.207:219]
```
mod build;
```
[2.206]
[2.219]
```
pub mod build;
mod insert;
pub mod options;
```

Replacement in src/mem_cache_index/mod.rs at line 5 [2.206]

B:BD[2.231] → [2.231:232]

[2.231]

[2.232]

#[pg_guard]
pub extern "C" fn amvalidate(_opclassoid: pg_sys::Oid) -> bool {
    true
}
/// ```sql
/// CREATE OR REPLACE FUNCTION amhandler(internal) RETURNS index_am_handler PARALLEL SAFE IMMUTABLE STRICT COST 0.0001 LANGUAGE c AS 'MODULE_PATHNAME', '@FUNCTION_NAME@';
/// CREATE ACCESS METHOD vector TYPE INDEX HANDLER amhandler;
/// ```

Replacement in src/mem_cache_index/mod.rs at line 14 [2.206]

B:BD[2.245] → [2.245:328]

fn amhandler(_fcinfi: pg_sys::FunctionCallInfo) -> PgBox<pg_sys::IndexAmRoutine> {

[2.245]

[2.328]

pub fn amhandler(_fcinfi: pg_sys::FunctionCallInfo) -> PgBox<pg_sys::IndexAmRoutine> {

Replacement in src/mem_cache_index/mod.rs at line 21 [2.206]

B:BD[2.573] → [2.573:693]

    amroutine.amkeytype = pg_sys::InvalidOid; // InvalidOid = 0? https://www.postgresql.org/docs/9.6/lo-interfaces.html

[2.573]

[2.693]

    amroutine.amkeytype = pg_sys::InvalidOid;

Replacement in src/mem_cache_index/mod.rs at line 23 [2.206]

B:BD[2.739] → [2.739:1134]

    /*
    amroutine.ambuildempty = todo!
    amroutine.aminsert = todo!;
    amroutine.ambulkdelete = todo!;
    amroutine.amvacuumcleanup = todo!;
    amroutine.amcostestimate = todo!;
    amroutine.amoptions = todo!;
    amroutine.amvalidate = todo!;
    amroutine.ambeginscan = todo!;
    amroutine.amrescan = todo!;
    amroutine.amgettuple = todo!;
    amroutine.amendscan = todo!;
    */

[2.739]

[2.1134]

    amroutine.ambuildempty = Some(build::vec_mem_cache_buildempty);
    amroutine.aminsert = None; // todo!();
    amroutine.ambulkdelete = None; // todo!();
    amroutine.amvacuumcleanup = None; // todo!();
    amroutine.amcostestimate = None; //todo!();
    amroutine.amoptions = Some(options::vec_mem_cache_options);
    amroutine.amvalidate = Some(amvalidate);
    amroutine.ambeginscan = None; //todo!();
    amroutine.amrescan = None; //todo!();
    amroutine.amgettuple = None; //todo!();
    amroutine.amendscan = None; //todo!();

Insertion in src/mem_cache_index/mod.rs at line 37 [2.206]

[2.1151]


#[cfg(test)]
mod test {
    // We want to try out something like
    // ```sql
    // CREATE TABLE public.vecs_2
    //(
    //    idx bigserial,
    //    vec real[] CONSTRAINT vecs_2_dim512 CHECK (cardinality(vec) = 512),
    //    PRIMARY KEY (idx)
    //);
    //
    // ```
    fn test_constraints() {
        todo!() // YOLO
    }
}

Insertion in src/mem_cache_index/build.rs at line 1 [2.1163]
[2.1163]
[2.1164]
```
use super::options;
```
Insertion in src/mem_cache_index/build.rs at line 3 [2.1163]
[2.1192]
[2.1192]
```
use crate::VECTOR_ALLOCATOR00;
```
Insertion in src/mem_cache_index/build.rs at line 10 [2.1163]
[2.1371]
[2.1371]
```
    let mut vector_allocator = VECTOR_ALLOCATOR00.exclusive();
```

Insertion in src/mem_cache_index/build.rs at line 12 [2.1163]

[2.1414]

    if vector_allocator.is_set() {
        panic!("Index already set. We've only got room for one in here")
    }

Insertion in src/mem_cache_index/build.rs at line 16 [2.1163]

[2.1450]

    let index_relation = unsafe { PgRelation::from_pg(indexrel) };
    let opts = options::VecMemCacheOptions::from_relation(&index_relation);
    vector_allocator.set_dim(opts.get_dim());
    // check that the indexrel has a realarray column and a bigint column

Insertion in src/mem_cache_index/build.rs at line 24 [2.1163]

[2.1598]

}
struct BuildState<'a> {
    tupdesc: &'a PgTupleDesc<'a>,
    dimension: u16,
    // attributes: Vec<CategorizedAttribute<'a>>,
    memcxt: PgMemoryContexts,
    pub rows_added: u32,
}
impl<'a> BuildState<'a> {
    fn new(
        tupdesc: &'a PgTupleDesc,
        dimension: u16,
        // attributes: Vec<CategorizedAttribute<'a>>,
    ) -> Self {
        BuildState {
            tupdesc,
            dimension,
            memcxt: PgMemoryContexts::new("pgvec_rs context"),
            rows_added: 0,
        }
    }
}
#[cfg(feature = "pg13")]
#[pg_guard]
unsafe extern "C" fn build_callback(
    _index: pg_sys::Relation,
    ctid: pg_sys::ItemPointer,
    values: *mut pg_sys::Datum,
    _isnull: *mut bool,
    _tuple_is_alive: bool,
    state: *mut std::os::raw::c_void,
) {
    build_callback_internal(*ctid, values, state);
}
#[inline(always)]
unsafe extern "C" fn build_callback_internal(
    ctid: pg_sys::ItemPointerData,
    values: *mut pg_sys::Datum,
    state: *mut std::os::raw::c_void,
) {
    check_for_interrupts!();
    let state = (state as *mut BuildState).as_mut().unwrap();
    let old_context = state.memcxt.set_as_current();
    // TODO: Add check for structure
    let values = std::slice::from_raw_parts(values, 1);
    let VecRow { id: _, data: vec } = row_to_vec_iter(&state.tupdesc, values[0]).unwrap();
    let mut vec_allocator = VECTOR_ALLOCATOR00.exclusive();
    vec_allocator.add_vec(&vec);
    state.rows_added += 1;
}
fn do_heap_scan<'a>(
    index_info: *mut pg_sys::IndexInfo,
    heap_relation: &'a PgRelation,
    index_relation: &'a PgRelation,
    tupdesc: &'a PgTupleDesc,
) -> u32 {
    // Should be able a to get a dimension from the description, but we'll hard code it now YOLO
    let dimension = 512;
    let mut state = BuildState::new(&tupdesc, dimension);
    unsafe {
        pg_sys::IndexBuildHeapScan(
            heap_relation.as_ptr(),
            index_relation.as_ptr(),
            index_info,
            Some(build_callback),
            &mut state,
        );
    }
    state.rows_added
}
#[derive(Default, Clone)]
struct VecRowPartial {
    id: Option<i64>,
    data: Option<Vec<f32>>,
}
impl VecRowPartial {
    fn check(self) -> Result<VecRow, &'static str> {
        match self {
            VecRowPartial {
                id: Some(id),
                data: Some(data),
            } => Ok(VecRow { id, data }),
            VecRowPartial {
                id: None,
                data: Some(_),
            } => Err("Missing Id for vec row"),
            VecRowPartial {
                id: Some(_),
                data: None,
            } => Err("Missing vector data for vec row"),
            VecRowPartial {
                id: None,
                data: None,
            } => Err("Missing all data for vector"),
        }
    }
}
struct VecRow {
    id: i64,
    data: Vec<f32>,

Insertion in src/mem_cache_index/build.rs at line 138 [2.1163]

[2.1600]


#[inline]
unsafe fn row_to_vec_iter<'a>(
    tupdesc: &'a PgTupleDesc,
    row: pg_sys::Datum,
) -> Result<VecRow, &'static str> {
    let td = pg_sys::pg_detoast_datum(row as *mut pg_sys::varlena) as pg_sys::HeapTupleHeader;
    let mut tmptup = pg_sys::HeapTupleData {
        t_len: varsize(td as *mut pg_sys::varlena) as u32,
        t_self: Default::default(),
        t_tableOid: 0,
        t_data: td,
    };
    let mut datums = vec![0 as pg_sys::Datum; tupdesc.natts as usize];
    let mut nulls = vec![false; tupdesc.natts as usize];
    pg_sys::heap_deform_tuple(
        &mut tmptup,
        tupdesc.as_ptr(),
        datums.as_mut_ptr(),
        nulls.as_mut_ptr(),
    );
    let mut drop_cnt = 0;
    let mut vec_row = VecRowPartial::default();
    tupdesc
        .iter()
        .map(|attribute| {
            let is_dropped = attribute.is_dropped();
            let array_type = unsafe { pg_sys::get_element_type(attribute.type_oid().value()) };
            let (base_oid, is_array) = if array_type != pg_sys::InvalidOid {
                (PgOid::from(array_type), true)
            } else {
                (attribute.type_oid(), false)
            };
            let typoid = base_oid;
            let fill_process: Result<(), &'static str> = match &typoid {
                PgOid::BuiltIn(builtin) => match (builtin, is_array) {
                    (PgBuiltInOids::FLOAT4OID, true) => {
                        if let None = vec_row.data {
                            let data =
                                unsafe { Vec::<f32>::from_datum(row, false, builtin.value()) }
                                    .unwrap();
                            vec_row.data = Some(data);
                            Ok(())
                        } else {
                            Err("Received more than one vector data entries in row")
                        }
                    }
                    (PgBuiltInOids::INT8OID, false) => {
                        if let None = vec_row.id {
                            let id =
                                unsafe { i64::from_datum(row, false, builtin.value()) }.unwrap();
                            vec_row.id = Some(id);
                            Ok(())
                        } else {
                            Err("Received more than one id in the row")
                        }
                    }
                    (_, _) => {
                        Err("This row is not the right shape.  It should be (bigint, real[])")
                    }
                },
                _ => Err("This element is not an accepted type"), // todo: communicate the right and wrong types
            };
            fill_process
        })
        .collect::<Result<Vec<()>, &'static str>>()?;
    vec_row.check()
}
#[pg_guard]
pub extern "C" fn vec_mem_cache_buildempty(_index_relation: pg_sys::Relation) {}

Replacement in src/lib.rs at line 1 [3.9696]

B:BD[3.9696] → [2.1601:1622]

mod mem_cache_index;

[3.9696]

[3.9697]

#![feature(array_map)]
#![feature(once_cell)]
#![feature(const_extern_fn)]
pub mod mem_cache_index;
mod shmem_blocks;

Replacement in src/lib.rs at line 8 [3.9696]

B:BD[3.9728] → [3.9728:9771]

use crate::vec_block_alloc::F32BlockError;

[3.9728]

[3.9771]

pub mod vector_query;
use crate::shmem_blocks::ShMemBlocksInner;

Insertion in src/lib.rs at line 11 [3.9696]
[3.9783]
[3.9783]
```
use rand::{distributions::Uniform, thread_rng, Rng};
```

Insertion in src/lib.rs at line 13 [3.9696]

[3.9797]

use std::fmt::format;
use std::sync::Arc;
use std::sync::Mutex;

Insertion in src/lib.rs at line 17 [3.9696]

[3.9838]

#[macro_use]
extern crate lazy_static;
const MAX_SPACE: usize = 131_072;
lazy_static! {
    static ref SHMEM_STEP: Arc<Mutex<usize>> = Arc::new(Mutex::new(0));
    static ref LOCKS: Arc<Mutex<Vec<PgLwLock<FiniteDimAllocator<MAX_SPACE>>>>> =
        Arc::new(Mutex::new(Vec::new()));
    static ref HOOK_LOCK: Mutex<bool> = Mutex::new(false);
}

Deletion in src/lib.rs at line 33 [3.9696]

B:BD[3.9960] → [3.9960:10158]

}
#[pg_extern]
fn l2_dist(a: Vector, b: Vector) -> f32 {
    let Vector(a) = a;
    let Vector(b) = b;
    let sq: f32 = a.iter().zip(b.iter()).map(|(a, b)| (a - b) * (a - b)).sum();
    sq.sqrt()

Deletion in src/lib.rs at line 34 [3.9696]
B:BD[3.10160] → [3.10160:10161]

Deletion in src/lib.rs at line 41 [3.9696]

B:BD[3.10375] → [3.10375:11148]

#[pg_extern]
fn ip(a: Array<f32>, b: Array<f32>) -> f32 {
    a.iter()
        .zip(b.iter())
        .map(|pair| unwrap_pair(pair))
        .map(|(a, b)| a * b)
        .sum()
}
const TOL: f32 = 1.0e-8;
#[pg_extern]
fn cosine_similarity(a: Array<f32>, b: Array<f32>) -> f32 {
    let mut a_norm_sq = 0.0;
    let mut b_norm_sq = 0.0;
    let a_norm_sq_ref = &mut a_norm_sq;
    let b_norm_sq_ref = &mut b_norm_sq;
    let mut sq_sum: f32 = a
        .iter()
        .zip(b.iter())
        .map(|pair| unwrap_pair(pair))
        .map(|(a, b)| {
            *a_norm_sq_ref += a * a;
            *b_norm_sq_ref += b * b;
            a * b
        })
        .sum();
    sq_sum /= TOL + a_norm_sq.sqrt();
    sq_sum /= TOL + b_norm_sq.sqrt();
    sq_sum.min(1.).max(-1.0)
}

Deletion in src/lib.rs at line 43 [3.9696]
B:BD[2.1751] → [2.1751:1784]
```
const MAX_SPACE: usize = 500000;
```

Replacement in src/lib.rs at line 44 [3.9696]

B:BD[3.11219] → [3.11219:11304]

B:BD[3.11304] → [4.73:1844]

static VECTOR_ALLOCATOR0: PgLwLock<FiniteDimAllocator<MAX_SPACE>> = PgLwLock::new();
// static VECTOR_ALLOCATOR1: PgLwLock<FiniteDimAllocator<MAX_SPACE>> = PgLwLock::new();
// static VECTOR_ALLOCATOR2: PgLwLock<FiniteDimAllocator<MAX_SPACE>> = PgLwLock::new();
// static VECTOR_ALLOCATOR3: PgLwLock<FiniteDimAllocator<MAX_SPACE>> = PgLwLock::new();
// static VECTOR_ALLOCATOR4: PgLwLock<FiniteDimAllocator<MAX_SPACE>> = PgLwLock::new();
// static VECTOR_ALLOCATOR5: PgLwLock<FiniteDimAllocator<MAX_SPACE>> = PgLwLock::new();
// static VECTOR_ALLOCATOR6: PgLwLock<FiniteDimAllocator<MAX_SPACE>> = PgLwLock::new();
// static VECTOR_ALLOCATOR7: PgLwLock<FiniteDimAllocator<MAX_SPACE>> = PgLwLock::new();
// static VECTOR_ALLOCATOR8: PgLwLock<FiniteDimAllocator<MAX_SPACE>> = PgLwLock::new();
// static VECTOR_ALLOCATOR9: PgLwLock<FiniteDimAllocator<MAX_SPACE>> = PgLwLock::new();
// static VECTOR_ALLOCATOR10: PgLwLock<FiniteDimAllocator<MAX_SPACE>> = PgLwLock::new();
// static VECTOR_ALLOCATOR11: PgLwLock<FiniteDimAllocator<MAX_SPACE>> = PgLwLock::new();
// static VECTOR_ALLOCATOR12: PgLwLock<FiniteDimAllocator<MAX_SPACE>> = PgLwLock::new();
// static VECTOR_ALLOCATOR13: PgLwLock<FiniteDimAllocator<MAX_SPACE>> = PgLwLock::new();
// static VECTOR_ALLOCATOR14: PgLwLock<FiniteDimAllocator<MAX_SPACE>> = PgLwLock::new();
// static VECTOR_ALLOCATOR15: PgLwLock<FiniteDimAllocator<MAX_SPACE>> = PgLwLock::new();
// static VECTOR_ALLOCATOR16: PgLwLock<FiniteDimAllocator<MAX_SPACE>> = PgLwLock::new();
// static VECTOR_ALLOCATOR17: PgLwLock<FiniteDimAllocator<MAX_SPACE>> = PgLwLock::new();
// static VECTOR_ALLOCATOR18: PgLwLock<FiniteDimAllocator<MAX_SPACE>> = PgLwLock::new();
// static VECTOR_ALLOCATOR19: PgLwLock<FiniteDimAllocator<MAX_SPACE>> = PgLwLock::new();
// static VECTOR_ALLOCATOR20: PgLwLock<FiniteDimAllocator<MAX_SPACE>> = PgLwLock::new();

[3.11219]

[3.13015]

static VECTOR_ALLOCATOR00: PgLwLock<FiniteDimAllocator<MAX_SPACE>> = PgLwLock::new();
static SHMEM_BLOCKS: shmem_blocks::ShMemBlocks<MAX_SPACE> =
    shmem_blocks::ShMemBlocks::<MAX_SPACE>::new();

Insertion in src/lib.rs at line 49 [3.9696]

[3.13016]

//static VECVECVEC: [PgLwLock<FiniteDimAllocator<MAX_SPACE>>; 400] =
//    [(); 400].map(|_| PgLwLock::new());

Replacement in src/lib.rs at line 54 [3.9696]

B:BD[3.13059] → [3.13059:13098]

B:BD[3.13098] → [4.1845:2776]

    pg_shmem_init!(VECTOR_ALLOCATOR0);
    //     pg_shmem_init!(VECTOR_ALLOCATOR1);
    //     pg_shmem_init!(VECTOR_ALLOCATOR2);
    //     pg_shmem_init!(VECTOR_ALLOCATOR3);
    //     pg_shmem_init!(VECTOR_ALLOCATOR4);
    //     pg_shmem_init!(VECTOR_ALLOCATOR5);
    //     pg_shmem_init!(VECTOR_ALLOCATOR6);
    //     pg_shmem_init!(VECTOR_ALLOCATOR7);
    //     pg_shmem_init!(VECTOR_ALLOCATOR8);
    //     pg_shmem_init!(VECTOR_ALLOCATOR9);
    //     pg_shmem_init!(VECTOR_ALLOCATOR10);
    //     pg_shmem_init!(VECTOR_ALLOCATOR11);
    //     pg_shmem_init!(VECTOR_ALLOCATOR12);
    //     pg_shmem_init!(VECTOR_ALLOCATOR13);
    //     pg_shmem_init!(VECTOR_ALLOCATOR14);
    //     pg_shmem_init!(VECTOR_ALLOCATOR15);
    //     pg_shmem_init!(VECTOR_ALLOCATOR16);
    //     pg_shmem_init!(VECTOR_ALLOCATOR17);
    //     pg_shmem_init!(VECTOR_ALLOCATOR18);
    //     pg_shmem_init!(VECTOR_ALLOCATOR19);
    //     pg_shmem_init!(VECTOR_ALLOCATOR20);

[3.13059]

[3.13889]

    // pg_shmem_init!(VECTOR_ALLOCATOR00);
    unsafe {
        let lock = std::ffi::CString::new(SHMEM_BLOCKS.get_name()).expect("CString::new failed");
        let size = SHMEM_BLOCKS.get_size();
        for _ in 0..size {
            let pg_lw_lock = PgLwLock::<FiniteDimAllocator<MAX_SPACE>>::new();
            let name = pg_lw_lock.get_name();
            let lock_name = std::ffi::CString::new(name).expect("CString::new failed");
            pg_sys::RequestAddinShmemSpace(
                size * std::mem::size_of::<FiniteDimAllocator<MAX_SPACE>>(),
            );
            pg_sys::RequestNamedLWLockTranche(lock_name.as_ptr(), 1);
            let mut locks = LOCKS.lock().unwrap();
            locks.push(pg_lw_lock)
        }
        pg_sys::RequestAddinShmemSpace(
            size * std::mem::size_of::<PgLwLock<FiniteDimAllocator<MAX_SPACE>>>(),
        );
        pg_sys::RequestNamedLWLockTranche(lock.as_ptr(), 1);
    }
    unsafe {
        let size = SHMEM_BLOCKS.get_size();
        for _ in 0..size {
            let hook_lock = HOOK_LOCK.lock().expect("LOCKING SHMEM HOOKS");
            static mut PREV_SHMEM_STARTUP_HOOK: Option<(unsafe extern "C" fn(), usize)> = None;
            PREV_SHMEM_STARTUP_HOOK = pg_sys::shmem_startup_hook.map(|i| (i, 0));
            pg_sys::shmem_startup_hook = Some(shmem_hook);
            println!("startup hook: {:?}", PREV_SHMEM_STARTUP_HOOK);
            println!("DONE SETTING SHMEM_HOOKS: {:?}", hook_lock);
            #[pg_guard]
            extern "C" fn shmem_hook() {
                unsafe {
                    PREV_SHMEM_STARTUP_HOOK =
                        PREV_SHMEM_STARTUP_HOOK.map(|(i, count)| (i, count + 1));
                    if let Some((i, count)) = PREV_SHMEM_STARTUP_HOOK {
                        if count < SHMEM_BLOCKS.get_size() {
                            i()
                        };
                    }
                    let addin_shmem_init_lock: *mut pg_sys::LWLock =
                        &mut (*pg_sys::MainLWLockArray.add(21)).lock;
                    pg_sys::LWLockAcquire(addin_shmem_init_lock, pg_sys::LWLockMode_LW_EXCLUSIVE);
                    let mut found = false;
                    let mut step_num = SHMEM_STEP.lock().unwrap();
                    let locks_vec = LOCKS.lock().expect("Unable to lock locks vector");
                    let lock = locks_vec.get(*step_num).unwrap();
                    let lock_name = lock.get_name();
                    let shm_name = std::ffi::CString::new(lock_name).expect("CString::new failed");
                    println!("About to attach pointer");
                    let fv_shmem = pg_sys::ShmemInitStruct(
                        shm_name.into_raw(),
                        std::mem::size_of::<FiniteDimAllocator<MAX_SPACE>>(),
                        &mut found,
                    ) as *mut FiniteDimAllocator<MAX_SPACE>;
                    lock.attach(fv_shmem);
                    println!("About to write pointer");
                    *step_num += 1;
                    pg_sys::LWLockRelease(addin_shmem_init_lock);
                }
            }
        }
        static mut PREV_SHMEM_STARTUP_HOOK: Option<unsafe extern "C" fn()> = None;
        let hook_lock = HOOK_LOCK.lock().unwrap();
        PREV_SHMEM_STARTUP_HOOK = pg_sys::shmem_startup_hook;
        pg_sys::shmem_startup_hook = Some(shmem_hook);
        #[pg_guard]
        extern "C" fn shmem_hook() {
            unsafe {
                if let Some(i) = PREV_SHMEM_STARTUP_HOOK {
                    i();
                }
                let mut found = false;
                let addin_shmem_init_lock: *mut pg_sys::LWLock =
                    &mut (*pg_sys::MainLWLockArray.add(21)).lock;
                pg_sys::LWLockAcquire(addin_shmem_init_lock, pg_sys::LWLockMode_LW_EXCLUSIVE);
                let shm_name =
                    std::ffi::CString::new(SHMEM_BLOCKS.get_name()).expect("CString::new failed");
                let fv_shmem = pg_sys::ShmemInitStruct(
                    shm_name.into_raw(),
                    SHMEM_BLOCKS.get_size()
                        * std::mem::size_of::<PgLwLock<FiniteDimAllocator<MAX_SPACE>>>(),
                    &mut found,
                ) as *mut PgLwLock<FiniteDimAllocator<MAX_SPACE>>;
                SHMEM_BLOCKS
                    .inner
                    .set(ShMemBlocksInner { start: fv_shmem })
                    .expect("Failed to set inner shmem");
                for offset in 0..SHMEM_BLOCKS.get_size() {
                    let lock = LOCKS
                        .lock()
                        .expect("Locks should be set")
                        .pop()
                        .expect("Locks should be initialised at this point");
                    *lock.exclusive() = FiniteDimAllocator::<MAX_SPACE>::default();
                    println!("written!");
                    std::ptr::write(fv_shmem.add(offset), lock);
                    let lock_ref = fv_shmem.add(offset).as_ref().unwrap();
                    println!("{:?}", *lock_ref.share())
                }
                pg_sys::LWLockRelease(addin_shmem_init_lock);
            }
        }
    }

Insertion in src/lib.rs at line 174 [3.9696]

[3.13891]


#[pg_extern]
fn set_allocator_dim(block: i32, dimemsion: i16) {
    let error = format!("Block not found, only {:?}", SHMEM_BLOCKS.get_size());
    let block = SHMEM_BLOCKS.get_block(block as usize).expect(&error);
    block.exclusive().set_dim(dimemsion as u16)
}

Replacement in src/lib.rs at line 183 [3.9696]

B:BD[3.13904] → [3.13904:14003]

fn set_allocator_dim(dimemsion: i16) {
    VECTOR_ALLOCATOR0.exclusive().set_dim(dimemsion as u16)

[3.13904]

[3.14003]

fn fill_vec_allocator(block: i32, vecs: i32) {
    let mut rng = thread_rng();
    let unif = Uniform::new(0.0, 1.0);
    set_allocator_dim(block, 512);
    (0..vecs).for_each(|_| {
        let vec = (0..512).map(|_| rng.sample(unif)).collect::<Vec<_>>();
        SHMEM_BLOCKS
            .get_block(block as usize)
            .unwrap()
            .exclusive()
            .add_vec(&vec);
    });

Replacement in src/lib.rs at line 198 [3.9696]
B:BD[3.14019] → [3.14019:14057]
```
fn push_vec_to_mem(vec: Array<f32>) {
```
[3.14019]
[3.14057]
```
fn push_vec_to_mem(block: i32, vec: Array<f32>) {
```

Replacement in src/lib.rs at line 203 [3.9696]

B:BD[3.14161] → [3.14161:14210]

    VECTOR_ALLOCATOR0.exclusive().add_vec(&vec);

[3.14161]

[3.14210]

    SHMEM_BLOCKS
        .get_block(block as usize)
        .unwrap()
        .exclusive()
        .add_vec(&vec);

Insertion in src/lib.rs at line 212 [3.9696]
[3.14240]
[3.14240]
```
    block: i32,
```

Replacement in src/lib.rs at line 222 [3.9696]

B:BD[3.14568] → [3.14568:14610]

    let return_scores = VECTOR_ALLOCATOR0

[3.14568]

[3.14610]

    let return_scores = SHMEM_BLOCKS
        .get_block(block as usize)
        .unwrap()

Deletion in src/lib.rs at line 236 [3.9696]

B:BD[3.14860] → [3.14860:15260]

#[cfg(any(test, feature = "pg_test"))]
mod tests {
    use crate::Vector;
    use pgx::*;
    #[pg_test]
    fn test_hello_pgvector_rs() {
        let vec1 = (0..512)
            .map(|x| x as f32)
            .map(|x| x.sin())
            .collect::<Vec<_>>();
        let vec1 = Vector(vec1);
        let vec2 = vec1.clone();
        assert!(crate::l2_dist(vec1, vec2).abs() <= 1.0e-10);
    }
}

Deletion in Cargo.toml at line 11 [3.15618]

B:BD[3.15747] → [3.15747:15864]

pg10 = ["pgx/pg10", "pgx-tests/pg10" ]
pg11 = ["pgx/pg11", "pgx-tests/pg11" ]
pg12 = ["pgx/pg12", "pgx-tests/pg12" ]

Replacement in Cargo.toml at line 15 [3.15618]
B:BD[3.15932] → [3.15932:15947]
```
pgx = "0.1.21"
```
[3.15932]
[3.15947]
```
pgx = "0.1.22"
```
Insertion in Cargo.toml at line 21 [3.15618]
[3.16033]
[3.16033]
```
memoffset = "0.6.1"
```
Insertion in Cargo.toml at line 24 [3.15618]
[3.16073]
[3.16073]
```
uuid = "0.8.2"
lazy_static = "1.4.0"
```