pijul/pijul - Change 246V5TYIUL7CFN7G5Y7A35EEM6IJPN532ROEYVSM7Q4HCQSWPDBQC

decode existing files

Created by rohan on December 11, 2020

246V5TYIUL7CFN7G5Y7A35EEM6IJPN532ROEYVSM7Q4HCQSWPDBQC

Dependencies

In channels

main

In discussions

#179 supporting non-UTF8 text file encodings

Change contents

Insertion in libpijul/src/working_copy/mod.rs at line 2 [3.198172]
[3.198228]
[3.198228]
```
use chardetng::EncodingDetector;
// use encoding_rs::Encoding;
```

Insertion in libpijul/src/working_copy/mod.rs at line 37 [3.198172]

[4.2100]

[3.199207]


    fn decode_file(&mut self, file: &str, buffer: &mut Vec<u8>) -> Result<(), Self::Error> {
        let mut uncoded = Vec::new();
        self.read_file(&file, &mut uncoded)?;
        let (mut decoded, encoding) = if tree_magic_mini::from_u8(&uncoded).starts_with("text/") {
            let mut detector = EncodingDetector::new();
            detector.feed(&uncoded, true);
            let encoding = detector.guess(None, true);
            debug!("guessed encoding = {:?}", encoding.name());
            let (decoded, encoding, malformed) = encoding.decode(&uncoded);
            debug!("final encoding = {:?}", encoding.name());
            if !malformed {
                (decoded.as_bytes().to_vec(), Some(encoding))
            } else {
                warn!("text file was malformed");
                (uncoded, None)
            }
        } else {
            (uncoded, None)
        };
        buffer.append(&mut decoded);
        Ok(())
    }

Deletion in libpijul/src/record.rs at line 3 [3.488880]
B:BD[2.613] → [2.613:677]
```
use chardetng::EncodingDetector;
// use encoding_rs::Encoding;
```

Replacement in libpijul/src/record.rs at line 240 [3.488880]

B:BD[3.496552] → [2.679:1701]


            let mut uncoded = Vec::new();
            working_copy.read_file(&item.full_path, &mut uncoded)?;
            let encoding = if tree_magic_mini::from_u8(&uncoded).starts_with("text/") {
                let mut detector = EncodingDetector::new();
                detector.feed(&uncoded, true);
                let encoding = detector.guess(None, true);
                debug!("guessed encoding = {:?}", encoding.name());
                let (decoded, encoding, malformed) = encoding.decode(&uncoded);
                debug!("final encoding = {:?}", encoding.name());
                if !malformed {
                    self.rec.contents.append(&mut decoded.as_bytes().to_vec());
                    Some(encoding)
                } else {
                    warn!("text file was malformed");
                    self.rec.contents.append(&mut uncoded);
                    None
                }
            } else {
                self.rec.contents.append(&mut uncoded);
                None
            };

[3.496552]

[3.496630]

            working_copy.decode_file(&item.full_path, &mut self.rec.contents)?;

Replacement in libpijul/src/record.rs at line 392 [3.488880]

B:BD[4.12123] → [4.12123:12179]

                    .read_file(&item.full_path, &mut b)

[4.12123]

[4.12179]

                    .decode_file(&item.full_path, &mut b)