jcdickinson/pijul - Change VMOYG7MKEWTUEEY2EOL256RWCVPGRD63IFOSKXHBGJ6VSRITLMOAC

text file decoding for new files

Created by rohan on December 11, 2020

VMOYG7MKEWTUEEY2EOL256RWCVPGRD63IFOSKXHBGJ6VSRITLMOAC

Dependencies

In channels

main

Change contents

Replacement in libpijul/src/tests/text.rs at line 34 [2.10]

B:BD[2.969] → [2.969:1116]

    for l in std::str::from_utf8(&v).unwrap().lines() {
        error!("{:?}", l);
    }
    let lines = std::str::from_utf8(&v).unwrap().lines();

[2.969]

[2.1116]


    let lines: Vec<&str> = std::str::from_utf8(&v)
        .unwrap()
        .lines()
        .filter(|l| l.starts_with("+"))
        .collect();

Replacement in libpijul/src/tests/text.rs at line 41 [2.10]

B:BD[2.1132] → [2.1132:1331]

        1,
        lines
            .clone()
            .filter(|l| l.starts_with("+") && l.contains("French / Français (Windows CP 1252)"))
            .count()
    );
    assert_eq!(
        1,

[2.1132]

[2.1331]

        vec!["+ French / Français (Windows CP 1252)", "+ €‚ƒ„…†‡, Salut"],

Deletion in libpijul/src/tests/text.rs at line 43 [2.10]

B:BD[2.1345] → [2.1345:1455]

            .filter(|l| l.starts_with("+") && l.contains("€‚ƒ„…†‡, Salut"))
            .count()

Replacement in libpijul/src/tests/text.rs at line 91 [2.10]

B:BD[2.2793] → [2.2793:2882]

    for l in std::str::from_utf8(&v).unwrap().lines() {
        error!("{:?}", l);
    }

[2.2793]

[2.2882]

    let lines: Vec<&str> = std::str::from_utf8(&v)
        .unwrap()
        .lines()
        .filter(|l| l.starts_with(|c| c == '-' || c == '+'))
        .collect();

Replacement in libpijul/src/tests/text.rs at line 97 [2.10]

B:BD[2.2898] → [2.2898:3128]

        1,
        std::str::from_utf8(&v)
            .unwrap()
            .lines()
            .filter(|l| l.starts_with("-")
                && l.contains("French / Français (ISO Latin-1 / ISO 8859-1)"))
            .count()

[2.2898]

[2.3128]

        vec![
            "- French / Français (ISO Latin-1 / ISO 8859-1)",
            "+ Français / French (ISO Latin-1 / ISO 8859-1)"
        ],
        lines

Insertion in libpijul/src/record.rs at line 2 [3.488880]

[3.488937]

use std::collections::{HashMap, HashSet};
use chardetng::EncodingDetector;
// use encoding_rs::Encoding;

Deletion in libpijul/src/record.rs at line 16 [3.488880]
B:BD[3.489225] → [3.489225:489267]
```
use std::collections::{HashMap, HashSet};
```

Replacement in libpijul/src/record.rs at line 243 [3.488880]

B:BD[3.496552] → [3.496552:496630]

            working_copy.read_file(&item.full_path, &mut self.rec.contents)?;

[3.496552]

[3.496630]


            let mut uncoded = Vec::new();
            working_copy.read_file(&item.full_path, &mut uncoded)?;
            let encoding = if tree_magic_mini::from_u8(&uncoded).starts_with("text/") {
                let mut detector = EncodingDetector::new();
                detector.feed(&uncoded, true);
                let encoding = detector.guess(None, true);
                debug!("guessed encoding = {:?}", encoding.name());
                let (decoded, encoding, malformed) = encoding.decode(&uncoded);
                debug!("final encoding = {:?}", encoding.name());
                if !malformed {
                    self.rec.contents.append(&mut decoded.as_bytes().to_vec());
                    Some(encoding)
                } else {
                    warn!("text file was malformed");
                    self.rec.contents.append(&mut uncoded);
                    None
                }
            } else {
                self.rec.contents.append(&mut uncoded);
                None
            };

Deletion in libpijul/src/change/text_changes.rs at line 1 [4.37976]
B:BD[4.37976] → [4.37977:38019]
B:BD[4.38019] → [2.5213:5246]
```
use super::*;
use crate::changestore::*;
use chardetng::EncodingDetector;
```
Insertion in libpijul/src/change/text_changes.rs at line 5 [4.37976]
[4.38112]
[4.38112]
```
use super::*;
use crate::changestore::*;
```

Replacement in libpijul/src/change/text_changes.rs at line 1161 [4.37976]

B:BD[2.5313] → [2.5313:5752]

∅:D[2.5752] → [4.84993:85003]

B:BD[4.84993] → [4.84993:85003]

B:BD[4.85003] → [2.5753:5804]

        let mut detector = EncodingDetector::new();
        detector.feed(&contents, true);
        let encoding = detector.guess(None, true);
        debug!("guessed encoding = {:?}", encoding.name());
        let (contents, encoding, malformed) = encoding.decode(&contents);
        debug!("final encoding = {:?}", encoding.name());
        if malformed {
            warn!("text file was malformed, should probably try binary instead")
        }
        for a in contents.split_terminator('\n') {

[2.5313]

[2.5804]

        for a in std::str::from_utf8(&contents)
            .unwrap()
            .split_terminator('\n')
        {