mx00s/pijul - Change 4DNDMC7IUZNYLDEQQYYF5K3G2QWWXGQENTEWPNTM6XKQEFPW7L3QC

Fixing a number of bugs related to encodings (extra newlines + misdetection in linux2x)

Created by pmeunier on July 3, 2021

4DNDMC7IUZNYLDEQQYYF5K3G2QWWXGQENTEWPNTM6XKQEFPW7L3QC

Dependencies

In channels

main

Change contents

Replacement in libpijul/src/text_encoding.rs at line 2 [5.19]
B:BD[5.34] → [5.34:35]
[5.34]
[5.35]
```
use std::borrow::Cow;
```

Replacement in libpijul/src/text_encoding.rs at line 17 [5.19]

B:BD[5.424] → [5.424:595]

    pub(crate) fn decode(&self, text: &[u8]) -> String {
        let (decoded, ..) = self.0.decode(&text);
        String::from_utf8(decoded.as_bytes().to_vec()).unwrap()

[5.424]

[5.595]

    pub(crate) fn decode<'a>(&self, text: &'a [u8]) -> Cow<'a, str> {
        self.0.decode(&text).0

Replacement in libpijul/src/text_encoding.rs at line 21 [5.19]

B:BD[2.1] → [2.1:145]

    pub(crate) fn encode(&self, text: &str) -> Vec<u8> {
        let (encoded, ..) = self.0.encode(text);
        encoded.into_owned().to_vec()

[2.1]

[2.145]

    pub(crate) fn encode<'a>(&self, text: &'a str) -> Cow<'a, [u8]> {
        self.0.encode(text).0

Replacement in libpijul/src/diff/mod.rs at line 16 [6.793247]
B:BD[6.793459] → [6.793459:793495]
```
#[derive(Debug, Hash, Clone, Copy)]
```
[6.793459]
[6.793495]
```
#[derive(Hash, Clone, Copy)]
```
Insertion in libpijul/src/diff/mod.rs at line 22 [6.793247]
[6.793593]
[4.11]
```
    ptr: *const u8,
```

Insertion in libpijul/src/diff/mod.rs at line 25 [6.793247]

[4.14]

impl<'a> std::fmt::Debug for Line<'a> {
    fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
        write!(fmt, "Line {{ l: {:?} }}", std::str::from_utf8(self.l))
    }
}

Insertion in libpijul/src/diff/mod.rs at line 38 [6.793247]
[4.198]
[4.198]
```
            ptr: std::ptr::null(),
```

Replacement in libpijul/src/diff/mod.rs at line 51 [6.793247]

B:BD[6.794015] → [6.794015:794064]

        self.l == b.l && self.cyclic == b.cyclic

[6.794015]

[6.794064]

        (self.ptr == b.ptr || self.l == b.l) && self.cyclic == b.cyclic

Insertion in libpijul/src/diff/mod.rs at line 100 [6.793247]
[4.1344]
[4.1344]
```
                ptr: l.as_ptr(),
```
Insertion in libpijul/src/diff/mod.rs at line 116 [6.793247]
[4.1757]
[4.1757]
```
                ptr: l.as_ptr(),
```
Insertion in libpijul/src/diff/mod.rs at line 139 [6.793247]
[7.783]
[4.1803]
```
        debug!("encoding = {:?}", encoding);
```

Deletion in libpijul/src/diff/mod.rs at line 141 [6.793247]

B:BD[4.1860] → [4.1860:1978]

            // self.diff_binary(changes, txn, txn.graph(channel), path, inode, a, &b)?;
            // return Ok(());

Replacement in libpijul/src/diff/mod.rs at line 151 [6.793247]

B:BD[8.68] → [8.68:115]

        trace!("{:?} {:?}", lines_a, lines_b);

[8.68]

[6.796653]

        if log::log_enabled!(log::Level::Trace) {
            for l in lines_a.iter() {
                trace!("a: {:?}", l)
            }
            for l in lines_b.iter() {
                trace!("b: {:?}", l)
            }
        }

Deletion in libpijul/src/diff/mod.rs at line 190 [6.793247]

B:BD[6.797337] → [6.797337:797338]

B:BD[6.797338] → [9.116096:116146]

∅:D[9.116146] → [6.797383:797440]

B:BD[6.797383] → [6.797383:797440]

B:BD[6.797440] → [9.116147:116175]

∅:D[9.116175] → [6.797470:797588]

B:BD[6.797470] → [6.797470:797588]

B:BD[6.797588] → [9.116176:116221]

B:BD[9.116221] → [10.86503:86541]

∅:D[10.86541] → [11.52:129]

B:BD[6.797638] → [11.52:129]

B:BD[11.129] → [10.86542:86778]

∅:D[10.86778] → [6.797895:798199]

B:BD[6.797895] → [6.797895:798199]

B:BD[6.798199] → [9.116222:116349]

B:BD[9.116349] → [12.136081:136139]

∅:D[12.136139] → [6.798352:798400]

B:BD[6.798352] → [6.798352:798400]

B:BD[6.798400] → [12.136140:136198]

∅:D[12.136198] → [6.798456:798787]

B:BD[6.798456] → [6.798456:798787]

B:BD[6.798787] → [12.136199:136320]

∅:D[12.136320] → [6.798904:798951]

B:BD[6.798904] → [6.798904:798951]

B:BD[6.798951] → [12.136321:136509]

∅:D[12.136509] → [6.799133:799187]

B:BD[6.799133] → [6.799133:799187]

B:BD[6.799187] → [12.136510:136578]

∅:D[12.136578] → [6.799253:799309]

B:BD[6.799253] → [6.799253:799309]

B:BD[6.799309] → [12.136579:136648]

∅:D[12.136648] → [6.799376:799524]

B:BD[6.799376] → [6.799376:799524]

B:BD[6.799524] → [12.136649:136705]

∅:D[12.136705] → [6.799578:799632]

B:BD[6.799578] → [6.799578:799632]

B:BD[6.799632] → [12.136706:136774]

∅:D[12.136774] → [6.799698:799869]

B:BD[6.799698] → [6.799698:799869]

B:BD[6.799869] → [10.86779:86822]

∅:D[13.2400] → [6.799918:800168]

∅:D[10.86822] → [6.799918:800168]

B:BD[6.799918] → [6.799918:800168]

B:BD[6.800168] → [7.1103:1135]

∅:D[7.1135] → [6.800168:800193]

B:BD[6.800168] → [6.800168:800193]

B:BD[6.800193] → [10.86823:86869]

∅:D[13.2451] → [14.548:592]

∅:D[10.86869] → [14.548:592]

B:BD[6.800245] → [14.548:592]

∅:D[14.592] → [6.800348:800587]

B:BD[6.800348] → [6.800348:800587]

B:BD[6.800587] → [12.136775:136879]

∅:D[12.136879] → [6.800691:800730]

B:BD[6.800691] → [6.800691:800730]

B:BD[6.800730] → [7.1136:1164]

∅:D[7.1164] → [9.116350:116377]

B:BD[6.800730] → [9.116350:116377]

∅:D[9.116377] → [6.800741:800747]

B:BD[6.800741] → [6.800741:800747]


    fn diff_binary<T: GraphTxnT, C: ChangeStore>(
        &mut self,
        changes: &C,
        txn: &T,
        channel: &T::Graph,
        path: String,
        inode: Position<Option<ChangeId>>,
        ret: &crate::alive::Graph,
        b: &[u8],
    ) -> Result<(), TxnErr<T::GraphError>> {
        self.has_binary_files = true;
        use crate::change::{Atom, EdgeMap, Hunk, Local, NewEdge, NewVertex};
        let mut contents = self.contents.lock().unwrap();
        let pos = contents.len();
        contents.extend_from_slice(&b[..]);
        let pos_end = contents.len();
        contents.push(0);
        std::mem::drop(contents);
        let mut edges = Vec::new();
        let mut deleted = Vec::new();
        for v in ret.lines.iter() {
            debug!("v.vertex = {:?}, inode = {:?}", v.vertex, inode);
            if Some(v.vertex.change) == inode.change && v.vertex.end == inode.pos {
                continue;
            }
            for e in iter_adjacent(txn, channel, v.vertex, EdgeFlags::PARENT, EdgeFlags::all())? {
                let e = e?;
                if e.flag().contains(EdgeFlags::PSEUDO) {
                    continue;
                }
                if e.flag().contains(EdgeFlags::FOLDER) {
                    if log_enabled!(log::Level::Debug) {
                        let f = std::fs::File::create("debug_diff_binary").unwrap();
                        ret.debug(changes, txn, channel, false, true, f).unwrap();
                    }
                    panic!("e.flag.contains(EdgeFlags::FOLDER)");
                }
                if e.flag().contains(EdgeFlags::PARENT) {
                    if e.flag().contains(EdgeFlags::DELETED) {
                        deleted.push(NewEdge {
                            previous: e.flag() - EdgeFlags::PARENT,
                            flag: e.flag() - EdgeFlags::PARENT,
                            from: e.dest().to_option(),
                            to: v.vertex.to_option(),
                            introduced_by: Some(e.introduced_by()),
                        })
                    } else {
                        let previous = e.flag() - EdgeFlags::PARENT;
                        edges.push(NewEdge {
                            previous,
                            flag: previous | EdgeFlags::DELETED,
                            from: e.dest().to_option(),
                            to: v.vertex.to_option(),
                            introduced_by: Some(e.introduced_by()),
                        })
                    }
                }
            }
        }
        // Kill all of `ret`, add `b` instead.
        if !deleted.is_empty() {
            self.actions.push(Hunk::Edit {
                local: Local {
                    line: 0,
                    path: path.clone(),
                },
                change: Atom::EdgeMap(EdgeMap {
                    edges: deleted,
                    inode,
                }),
                encoding: None,
            })
        }
        self.actions.push(Hunk::Replacement {
            local: Local { line: 0, path },
            change: Atom::EdgeMap(EdgeMap { edges, inode }),
            replacement: Atom::NewVertex(NewVertex {
                up_context: vec![inode],
                down_context: Vec::new(),
                flag: EdgeFlags::empty(),
                start: ChangePosition(pos.into()),
                end: ChangePosition(pos_end.into()),
                inode,
            }),
            encoding: None,
        });
        Ok(())
    }

Replacement in libpijul/src/diff/bin.rs at line 49 [4.2530]
B:BD[4.3860] → [4.3860:3884]
```
    let mut i = window;
```
[4.3860]
[4.3884]
```
    let mut i = window.min(b.len());
```
Insertion in libpijul/src/diff/bin.rs at line 62 [4.2530]
[4.4302]
[4.4302]
```
                        ptr: old.as_ptr(),
```

Insertion in libpijul/src/diff/bin.rs at line 68 [4.2530]

[4.4496]

                        } else {
                            break

Replacement in libpijul/src/diff/bin.rs at line 95 [4.2530]

B:BD[4.5184] → [4.5184:5254]

            Chunk::Old { start, end, .. } => lines.push(super::Line {

[4.5184]

[4.5254]

            Chunk::Old { start, end, ptr, .. } => lines.push(super::Line {

Insertion in libpijul/src/diff/bin.rs at line 97 [4.2530]
[4.5289]
[4.5289]
```
                ptr,
```
Insertion in libpijul/src/diff/bin.rs at line 118 [4.2530]
[4.5732]
[4.5732]
```
        ptr: *const u8,
```

Replacement in libpijul/src/change/text_changes.rs at line 1281 [15.37976]

B:BD[2.987] → [2.987:1033]

        None => String::from(h).into_bytes(),

[2.987]

[2.1033]

        None => std::borrow::Cow::Borrowed(h.as_bytes()),

Replacement in libpijul/src/change/text_changes.rs at line 1320 [15.37976]

B:BD[16.522] → [17.6111:6169]

        for a in encoding.decode(&contents).split('\n') {

[16.522]

[18.1236]

        let dec = encoding.decode(&contents);
        let dec = if dec.ends_with("\n") {
            &dec[..dec.len() - 1]
        } else {
            &dec
        };
        for a in dec.split('\n') {

Insertion in chardetng/src/lib.rs at line 2950 [3.12027]
[3.128274]
[3.128274]

Replacement in chardetng/src/lib.rs at line 2962 [3.12027]

B:BD[3.128794] → [3.128794:128849]

        if self.non_ascii_seen == 0 && self.esc_seen {

[3.128794]

[3.128849]

        if self.non_ascii_seen == 0
            && self.esc_seen
        {

Replacement in chardetng/src/lib.rs at line 2966 [3.12027]

B:BD[3.128923] → [3.128923:128967]

                return (ISO_2022_JP, true);

[3.128923]

[3.128967]

                return (ISO_2022_JP, true)

Replacement in chardetng/src/lib.rs at line 3041 [3.12027]
B:BD[3.132034] → [3.132034:132063]
```
        (encoding, max > 0)
```
[3.132034]
[3.132063]
```
        (encoding, max >= 0)
```

Insertion in chardetng/src/lib.rs at line 3386 [3.12027]

[3.143335]

    }
    #[test]
    fn test_en_windows1252() {
        // "Don't "
        check_bytes(&[68,111,110,180,116,32], WINDOWS_1252);

Insertion in Cargo.lock at line 21 [6.1030385]

[6.1030901]

[19.16895]

name = "adler32"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234"
[[package]]

Deletion in Cargo.lock at line 212 [6.1030385]

B:BD[20.2369] → [20.2369:2512]

source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "81a81b0d8f8ee23417182818b4f06312c5f535c2b04eef1773f7c24bbdf8c500"

Insertion in Cargo.lock at line 213 [6.1030385]
[20.2529]
[20.2529]
```
 "arrayvec",
```
Insertion in Cargo.lock at line 215 [6.1030385]
[20.2547]
[20.2547]
```
 "detone",
```
Insertion in Cargo.lock at line 218 [6.1030385]
[21.466]
[20.2580]
```
 "rayon",
```

Insertion in Cargo.lock at line 326 [6.1030385]

[20.5083]

name = "crossbeam-channel"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "06ed27e177f16d65f0f0c22a213e17c696ace5dd64b14258b52f9417ccb52db4"
dependencies = [
 "cfg-if 1.0.0",
 "crossbeam-utils",
]
[[package]]

Insertion in Cargo.lock at line 439 [6.1030385]

[20.7564]

name = "detone"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f7104c193859c8141dcbb2008bd3d93e581d0fa7bb47b0d9f5e15c89d1b55514"
[[package]]

Insertion in Cargo.lock at line 1074 [6.1030385]
[20.22974]
[20.22974]
```
 "adler32",
```

Insertion in Cargo.lock at line 1705 [6.1030385]

[20.39097]

]
[[package]]
name = "rayon"
version = "1.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c06aca804d41dbc8ba42dfd964f0d01334eceb64314b9ecf7c5fad5188a06d90"
dependencies = [
 "autocfg",
 "crossbeam-deque",
 "either",
 "rayon-core",

Insertion in Cargo.lock at line 1720 [6.1030385]

[20.39310]

name = "rayon-core"
version = "1.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d78120e2c850279833f1dd3582f730c4ab53ed95aeaaaa862a2a5c71b1656d8e"
dependencies = [
 "crossbeam-channel",
 "crossbeam-deque",
 "crossbeam-utils",
 "lazy_static",
 "num_cpus",
]
[[package]]

Fixing a number of bugs related to encodings (extra newlines + misdetection in linux2x)

Dependencies

In channels

Change contents

Replacement in libpijul/src/text_encoding.rs at line 2 [5.19]

Replacement in libpijul/src/text_encoding.rs at line 17 [5.19]

Replacement in libpijul/src/text_encoding.rs at line 21 [5.19]

Replacement in libpijul/src/diff/mod.rs at line 16 [6.793247]

Insertion in libpijul/src/diff/mod.rs at line 22 [6.793247]

Insertion in libpijul/src/diff/mod.rs at line 25 [6.793247]

Insertion in libpijul/src/diff/mod.rs at line 38 [6.793247]

Replacement in libpijul/src/diff/mod.rs at line 51 [6.793247]

Insertion in libpijul/src/diff/mod.rs at line 100 [6.793247]

Insertion in libpijul/src/diff/mod.rs at line 116 [6.793247]

Insertion in libpijul/src/diff/mod.rs at line 139 [6.793247]

Deletion in libpijul/src/diff/mod.rs at line 141 [6.793247]

Replacement in libpijul/src/diff/mod.rs at line 151 [6.793247]

Deletion in libpijul/src/diff/mod.rs at line 190 [6.793247]

Replacement in libpijul/src/diff/bin.rs at line 49 [4.2530]

Insertion in libpijul/src/diff/bin.rs at line 62 [4.2530]

Insertion in libpijul/src/diff/bin.rs at line 68 [4.2530]

Replacement in libpijul/src/diff/bin.rs at line 95 [4.2530]

Insertion in libpijul/src/diff/bin.rs at line 97 [4.2530]

Insertion in libpijul/src/diff/bin.rs at line 118 [4.2530]

Replacement in libpijul/src/change/text_changes.rs at line 1281 [15.37976]

Replacement in libpijul/src/change/text_changes.rs at line 1320 [15.37976]

Insertion in chardetng/src/lib.rs at line 2950 [3.12027]

Replacement in chardetng/src/lib.rs at line 2962 [3.12027]

Replacement in chardetng/src/lib.rs at line 2966 [3.12027]

Replacement in chardetng/src/lib.rs at line 3041 [3.12027]

Insertion in chardetng/src/lib.rs at line 3386 [3.12027]

Insertion in Cargo.lock at line 21 [6.1030385]

Deletion in Cargo.lock at line 212 [6.1030385]

Insertion in Cargo.lock at line 213 [6.1030385]

Insertion in Cargo.lock at line 215 [6.1030385]

Insertion in Cargo.lock at line 218 [6.1030385]

Insertion in Cargo.lock at line 326 [6.1030385]

Insertion in Cargo.lock at line 439 [6.1030385]

Insertion in Cargo.lock at line 1074 [6.1030385]

Insertion in Cargo.lock at line 1705 [6.1030385]

Insertion in Cargo.lock at line 1720 [6.1030385]