aeveris/ml-preproc - Change PGIJJK2MGH2SHXUAJQ6LA3HZVC7KJC563WGD6XIEP3J436YW3NUAC

Fix HashingVectorizer to work the same as scikit-learn. Extend test

Created by Gz5AN8tiLGjsQNTG24sqRr5jNodcDoAzyZSgZGoaYTaD on July 16, 2021

PGIJJK2MGH2SHXUAJQ6LA3HZVC7KJC563WGD6XIEP3J436YW3NUAC

Dependencies

In channels

main

Change contents

File addition: murmur.rs (----------)
[3.15]
Replacement in src/lib.rs at line 10 [3.47]
B:BD[2.57] → [2.57:111]
```
use ndarray_linalg::{normalize, Norm, NormalizeAxis};
```
[2.57]
[3.220]
```
use ndarray_linalg::Norm;
```
Insertion in src/lib.rs at line 18 [3.47]
[2.133]
[3.383]
```
    normalise: bool,
```

Replacement in src/lib.rs at line 22 [3.47]

B:BD[3.411] → [2.134:204]

    pub fn new(n_features: u32, n_min: usize, n_max: usize) -> Self {

[3.411]

[3.482]

    pub fn new(n_features: u32, n_min: usize, n_max: usize, normalise: bool) -> Self {

Insertion in src/lib.rs at line 27 [3.47]
[3.558]
[3.558]
```
            normalise,
```

Insertion in src/lib.rs at line 37 [3.47]

[3.811]

[2.397]

            let lower = s.to_lowercase();

Replacement in src/lib.rs at line 39 [3.47]

B:BD[2.442] → [2.442:545]

            assert_eq!(v.shape(), &[16]);
            for ng in NGramIter::new(&normalize_str(s), 5) {

[2.442]

[3.872]

            for ng in NGramIter::new(&lower, 5) {

Replacement in src/lib.rs at line 41 [3.47]

B:BD[3.942] → [2.546:595]

                let f_idx = h % self.n_features;

[3.942]

[2.595]

                // NOTE: yes, we need all that fuckery because scikit-learn uses a i32 for whatever stupid reason. Fuck them
                let f_idx = (h as i32).abs() as u32 % self.n_features;

Insertion in src/lib.rs at line 44 [3.47]

[2.664]

                // This should be analogous to scikit-learn's comparison >= 0

Insertion in src/lib.rs at line 48 [3.47]

[3.956]


            if self.normalise {
                let v_norm = v.norm();
                v /= v_norm;
            }

Deletion in src/lib.rs at line 54 [3.47]

B:BD[3.966] → [2.786:882]

        println!("{:?}", target);
        let target = normalize(target, NormalizeAxis::Row).0;

Replacement in src/lib.rs at line 134 [3.47]
B:BD[3.2869] → [2.975:1013]
```
    use ndarray::{array, Array, Ix2};
```
[3.2869]
[2.1013]
```
    use ndarray::{array, Array};
```

Replacement in src/lib.rs at line 139 [3.47]

B:BD[3.2907] → [3.2907:3149]

    fn five_gram_tests() {
        let input = "this is a test!";
        let expected: HashSet<_> = [" this", "this ", " is ", " a ", " test", "test!", "est! "]
            .iter()
            .map(|&s| s.to_owned())
            .collect();

[3.2907]

[3.3149]

    fn five_gram_test() {
        let input = "This is the first document.";
        let expected: HashSet<_> = [
            " docu", " firs", " is ", " the ", " This", "cumen", "docum", "ent. ", "first",
            "irst ", "ment.", "ocume", "This ", "ument",
        ]
        .iter()
        .map(|&s| s.to_owned())
        .collect();

Replacement in src/lib.rs at line 153 [3.47]

B:BD[2.1082] → [2.1082:1153]

    fn vect_test() {
        let v = HashingVectorizer::new(16, 5, 5);

[2.1082]

[2.1153]

    fn vectorization_test() {
        let v = HashingVectorizer::new(16, 5, 5, false);

Replacement in src/lib.rs at line 162 [3.47]

B:BD[2.1408] → [2.1408:1511]

        //assert_eq!(val, Vec::new());
        let t: Array<f64, _> = Array::zeros((input.len(), 16));

[2.1408]

[2.1511]

        let t: Array<f64, _> = array![
            [
                0.0, 0.0, -1.0, -1.0, 1.0, 0.0, 0.0, -1.0, -1.0, 0.0, 0.0, -1.0, -1.0, 0.0, -1.0,
                0.0
            ],
            [
                0.0, 1.0, -2.0, -2.0, 1.0, 1.0, 0.0, 1.0, 1.0, 0.0, -1.0, 0.0, -1.0, 0.0, -1.0,
                -1.0,
            ],
            [-2.0, 0.0, 0.0, 0.0, 1.0, 1.0, 0.0, -1.0, 0.0, 0.0, 0.0, -1.0, 0.0, 1.0, -1.0, 0.0],
            [0.0, 0.0, -2.0, -1.0, 1.0, 0.0, 0.0, -1.0, -1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
        ];