Adding a patched chardetng (temporarily)
Created by pmeunier on July 1, 2021
SGXOEWHUET7RVS5HANSYE7QRQAG56NIKTC56MLPZ5XBCXODEWUXQC Dependencies
- [2]
SYP2J7VC6DWMSZDNVROYYE33GY5TJJEJUU3XWUEPI7LSX2FXJXFAC - [3]
6CZYYOG7C7ULY5Q4WSRVVC3QVSYACV3KFEYXDCH63OXXX2QFFFBAC - [4]
SXEYMYF7P4RZMZ46WPL4IZUTSQ2ATBWYZX7QNVMS3SGOYXYOHAGQC - [5]
246V5TYIUL7CFN7G5Y7A35EEM6IJPN532ROEYVSM7Q4HCQSWPDBQC - [6]
UM5DLRPBCQZQBEDJDLDPKODOKLACUHZD6YL6S4JRNKW6JLPNUVSAC - [7]
TVVW53HZGYPODAXEQ4BFZNSPBOFG6JEDVOKIYIDZMWFAMOBKOR2QC - [8]
NYOF5766GLBTWQV2KTVRAJMGVJNJ37Z5BLJMFPZA3HG7X2Q2RXPAC - [9]
52SOYOCNCODMHEV6ASPY4WXWYNDXYID46VMDF4W7QYALNA7LMGLAC - [10]
6HNRL5RT76NH5YNSUN7B4FHNRZXKNLX4DROFGMO4R5P2U7JWOL2QC - [11]
Q3GU26WDEYE2HXMM4WWNEGVMSXLQAKRJBSB2EJZA6JTHPKQHXCXQC - [12]
JRENVH5DF2F4SOV7UNJENFA7VDI3H63XK76R3LFZK6QCW7JIBLSQC - [13]
A3RM526Y7LUXNYW4TL56YKQ5GVOK2R5D7JJVTSQ6TT5MEXIR6YAAC - [14]
I24UEJQLCH2SOXA4UHIYWTRDCHSOPU7AFTRUOTX7HZIAV4AZKYEQC
In channels
main
Change contents
Replacement in libpijul/src/working_copy/mod.rs at line 38 [4.198172]
Replacement in libpijul/src/working_copy/mod.rs at line 42 [4.198172]
Replacement in libpijul/Cargo.toml at line 119 [4.1020742]
Insertion in libpijul/Cargo.toml at line 140 [4.1020742]
File addition: chardetng
[1.0]File addition: src
[0.241]File addition: tld.rs
[0.257]/* Any copyright is dedicated to the Public Domain.* https://creativecommons.org/publicdomain/zero/1.0/ */#[derive(Clone, Copy, Debug, PartialEq)]pub enum Tld {CentralWindows,Cyrillic,Western,Greek,TurkishAzeri,Hebrew,Arabic,Baltic,Vietnamese,Thai,Simplified,Traditional,Japanese,Korean,SimplifiedTraditional,TraditionalSimplified,CentralIso,IcelandicFaroese,WesternCyrillic,CentralCyrillic,WesternArabic,Generic,Eu,}pub fn classify_tld(tld: &[u8]) -> Tld {if tld.len() == 2 {let key = [tld[0], tld[1]];if let Ok(i) = TWO_LETTER_KEYS.binary_search(&key) {TWO_LETTER_VALUES[i]} else {Tld::Western}} else if tld.len() == 3 {match tld {b"edu" | b"gov" | b"mil" => Tld::Western,_ => Tld::Generic,}} else if tld.starts_with(b"xn--") && tld.len() >= 8 {// It's unclear is including the IDNs here is a good idea.// Clearly, they are an anachronism relative to the era// of legacy encodings. The idea, consistent with previous// approach in Firefox is to address the case where one// of these TLDs is configured as an alternative name for// a server that also serves the same content from a// two-ASCII-letter TLD. This makes the detection result// the same either way even though otherwise this thing// does not make much sense.if let Ok(i) = PUNYCODE_KEYS.binary_search(&&tld[4..]) {PUNYCODE_VALUES[i]} else {Tld::Generic}} else {Tld::Generic}}static TWO_LETTER_VALUES: [Tld; 87] = [Tld::Generic, // acTld::Arabic, // aeTld::Arabic, // afTld::Generic, // aiTld::WesternCyrillic, // amTld::TurkishAzeri, // azTld::CentralCyrillic, // baTld::Cyrillic, // bgTld::Arabic, // bhTld::Cyrillic, // byTld::Generic, // bzTld::Generic, // cbTld::Generic, // ccTld::Generic, // cdTld::Simplified, // cnTld::Generic, // cxTld::Greek, // cyTld::CentralWindows, // czTld::Generic, // djTld::Arabic, // dzTld::Arabic, // egTld::Eu, // euTld::Generic, // fmTld::IcelandicFaroese, // foTld::WesternCyrillic, // geTld::Greek, // grTld::TraditionalSimplified, // hkTld::CentralWindows, // hrTld::CentralIso, // huTld::Hebrew, // ilTld::Generic, // inTld::Arabic, // iqTld::Arabic, // irTld::IcelandicFaroese, // isTld::Arabic, // joTld::Japanese, // jpTld::Cyrillic, // kgTld::Korean, // kpTld::Korean, // krTld::Arabic, // kwTld::Cyrillic, // kzTld::Generic, // laTld::Arabic, // lbTld::Baltic, // ltTld::Baltic, // lvTld::Arabic, // lyTld::Arabic, // maTld::Cyrillic, // mdTld::Generic, // meTld::Cyrillic, // mkTld::Cyrillic, // mnTld::TraditionalSimplified, // moTld::Arabic, // mrTld::Generic, // msTld::WesternArabic, // myTld::Generic, // nuTld::Arabic, // omTld::Arabic, // pkTld::CentralIso, // plTld::Arabic, // psTld::Arabic, // qaTld::CentralWindows, // roTld::Cyrillic, // rsTld::Cyrillic, // ruTld::Arabic, // saTld::Arabic, // sdTld::SimplifiedTraditional, // sgTld::CentralIso, // siTld::CentralWindows, // skTld::Generic, // stTld::Cyrillic, // suTld::Arabic, // syTld::Thai, // thTld::Cyrillic, // tjTld::Generic, // tkTld::Cyrillic, // tmTld::Arabic, // tnTld::Generic, // toTld::TurkishAzeri, // trTld::Generic, // tvTld::Traditional, // twTld::Cyrillic, // uaTld::Cyrillic, // uzTld::Generic, // vcTld::Vietnamese, // vnTld::Generic, // vuTld::Arabic, // ye];static TWO_LETTER_KEYS: [[u8; 2]; 87] = [[b'a', b'c'], // Generic[b'a', b'e'], // Arabic[b'a', b'f'], // Arabic[b'a', b'i'], // Generic[b'a', b'm'], // WesternCyrillic[b'a', b'z'], // TurkishAzeri[b'b', b'a'], // CentralCyrillic[b'b', b'g'], // Cyrillic[b'b', b'h'], // Arabic[b'b', b'y'], // Cyrillic[b'b', b'z'], // Generic[b'c', b'b'], // Generic[b'c', b'c'], // Generic[b'c', b'd'], // Generic[b'c', b'n'], // Simplified[b'c', b'x'], // Generic[b'c', b'y'], // Greek[b'c', b'z'], // CentralWindows[b'd', b'j'], // Generic[b'd', b'z'], // Arabic[b'e', b'g'], // Arabic[b'e', b'u'], // Eu[b'f', b'm'], // Generic[b'f', b'o'], // IcelandicFaroese[b'g', b'e'], // WesternCyrillic[b'g', b'r'], // Greek[b'h', b'k'], // TraditionalSimplified[b'h', b'r'], // CentralWindows[b'h', b'u'], // CentralIso[b'i', b'l'], // Hebrew[b'i', b'n'], // Generic[b'i', b'q'], // Arabic[b'i', b'r'], // Arabic[b'i', b's'], // IcelandicFaroese[b'j', b'o'], // Arabic[b'j', b'p'], // Japanese[b'k', b'g'], // Cyrillic[b'k', b'p'], // Korean[b'k', b'r'], // Korean[b'k', b'w'], // Arabic[b'k', b'z'], // Cyrillic[b'l', b'a'], // Generic[b'l', b'b'], // Arabic[b'l', b't'], // Baltic[b'l', b'v'], // Baltic[b'l', b'y'], // Arabic[b'm', b'a'], // Arabic[b'm', b'd'], // Cyrillic[b'm', b'e'], // Generic[b'm', b'k'], // Cyrillic[b'm', b'n'], // Cyrillic[b'm', b'o'], // TraditionalSimplified[b'm', b'r'], // Arabic[b'm', b's'], // Generic[b'm', b'y'], // WesternArabic[b'n', b'u'], // Generic[b'o', b'm'], // Arabic[b'p', b'k'], // Arabic[b'p', b'l'], // CentralIso[b'p', b's'], // Arabic[b'q', b'a'], // Arabic[b'r', b'o'], // CentralWindows[b'r', b's'], // Cyrillic[b'r', b'u'], // Cyrillic[b's', b'a'], // Arabic[b's', b'd'], // Arabic[b's', b'g'], // SimplifiedTraditional[b's', b'i'], // CentralIso[b's', b'k'], // CentralWindows[b's', b't'], // Generic[b's', b'u'], // Cyrillic[b's', b'y'], // Arabic[b't', b'h'], // Thai[b't', b'j'], // Cyrillic[b't', b'k'], // Generic[b't', b'm'], // Cyrillic[b't', b'n'], // Arabic[b't', b'o'], // Generic[b't', b'r'], // TurkishAzeri[b't', b'v'], // Generic[b't', b'w'], // Traditional[b'u', b'a'], // Cyrillic[b'u', b'z'], // Cyrillic[b'v', b'c'], // Generic[b'v', b'n'], // Vietnamese[b'v', b'u'], // Generic[b'y', b'e'], // Arabic];static PUNYCODE_KEYS: [&'static [u8]; 46] = [b"3e0b707e", // Koreanb"54b7fta0cc", // Westernb"80ao21a", // Cyrillicb"90a3ac", // Cyrillicb"90ae", // Cyrillicb"90ais", // Cyrillicb"clchc0ea0b2g2a9gcd", // SimplifiedTraditionalb"d1alf", // Cyrillicb"e1a4c", // Eub"fiqs8S", // Simplifiedb"fiqz9S", // Simplifiedb"fzc2c9e2c", // Westernb"j1amh", // Cyrillicb"j6w193g", // TraditionalSimplifiedb"kprw13d", // Traditionalb"kpry57d", // Traditionalb"l1acc", // Cyrillicb"lgbbat1ad8j", // Arabicb"mgb2ddes", // Arabicb"mgb9awbf", // Arabicb"mgba3a4f16a", // Arabicb"mgbaam7a8h", // Arabicb"mgbah1a3hjkrd", // Arabicb"mgbai9azgqp6j", // Arabicb"mgbayh7gpa", // Arabicb"mgbc0a9azcg", // Arabicb"mgbcpq6gpa1a", // Arabicb"mgberp4a5d4ar", // Arabicb"mgbpl2fh", // Arabicb"mgbtx2b", // Arabicb"mgbx4cd0ab", // WesternArabicb"mix891f", // TraditionalSimplifiedb"node", // WesternCyrillicb"o3cw4h", // Thaib"ogbpf8fl", // Arabicb"p1ai", // Cyrillicb"pgbs0dh", // Arabicb"q7ce6a", // Arabicb"qxa6a", // Eub"qxam", // Greekb"wgbh1c", // Arabicb"wgbl6a", // Arabicb"xkc2al3hye2a", // Westernb"y9a3aq", // WesternCyrillicb"yfro4i67o", // SimplifiedTraditionalb"ygbi2ammx", // Arabic];static PUNYCODE_VALUES: [Tld; 46] = [Tld::Korean, // 3e0b707eTld::Western, // 54b7fta0ccTld::Cyrillic, // 80ao21aTld::Cyrillic, // 90a3acTld::Cyrillic, // 90aeTld::Cyrillic, // 90aisTld::SimplifiedTraditional, // clchc0ea0b2g2a9gcdTld::Cyrillic, // d1alfTld::Eu, // e1a4cTld::Simplified, // fiqs8STld::Simplified, // fiqz9STld::Western, // fzc2c9e2cTld::Cyrillic, // j1amhTld::TraditionalSimplified, // j6w193gTld::Traditional, // kprw13dTld::Traditional, // kpry57dTld::Cyrillic, // l1accTld::Arabic, // lgbbat1ad8jTld::Arabic, // mgb2ddesTld::Arabic, // mgb9awbfTld::Arabic, // mgba3a4f16aTld::Arabic, // mgbaam7a8hTld::Arabic, // mgbah1a3hjkrdTld::Arabic, // mgbai9azgqp6jTld::Arabic, // mgbayh7gpaTld::Arabic, // mgbc0a9azcgTld::Arabic, // mgbcpq6gpa1aTld::Arabic, // mgberp4a5d4arTld::Arabic, // mgbpl2fhTld::Arabic, // mgbtx2bTld::WesternArabic, // mgbx4cd0abTld::TraditionalSimplified, // mix891fTld::WesternCyrillic, // nodeTld::Thai, // o3cw4hTld::Arabic, // ogbpf8flTld::Cyrillic, // p1aiTld::Arabic, // pgbs0dhTld::Arabic, // q7ce6aTld::Eu, // qxa6aTld::Greek, // qxamTld::Arabic, // wgbh1cTld::Arabic, // wgbl6aTld::Western, // xkc2al3hye2aTld::WesternCyrillic, // y9a3aqTld::SimplifiedTraditional, // yfro4i67oTld::Arabic, // ygbi2ammx];File addition: lib.rs
[0.257]// Copyright Mozilla Foundation. See the COPYRIGHT// file at the top-level directory of this distribution.//// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or// https://www.apache.org/licenses/LICENSE-2.0> or the MIT license// <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your// option. This file may not be copied, modified, or distributed// except according to those terms.//! `chardetng` is a character encoding detector for legacy Web content.//!//! It is optimized for binary size in applications that already depend//! on `encoding_rs` for other reasons.#[cfg(feature = "multithreading")]use rayon::prelude::*;#[cfg(feature = "multithreading")]use arrayvec::ArrayVec;#[cfg(all(target_arch = "x86", target_feature = "sse2"))]use core::arch::x86::__m128i;#[cfg(all(target_arch = "x86", target_feature = "sse2"))]use core::arch::x86::_mm_movemask_epi8;#[cfg(all(target_arch = "x86_64", target_feature = "sse2"))]use core::arch::x86_64::__m128i;#[cfg(all(target_arch = "x86_64", target_feature = "sse2"))]use core::arch::x86_64::_mm_movemask_epi8;use encoding_rs::Decoder;use encoding_rs::DecoderResult;use encoding_rs::Encoding;use encoding_rs::BIG5;use encoding_rs::EUC_JP;use encoding_rs::EUC_KR;use encoding_rs::GBK;use encoding_rs::ISO_2022_JP;use encoding_rs::ISO_8859_8;use encoding_rs::SHIFT_JIS;use encoding_rs::UTF_8;use encoding_rs::WINDOWS_1255;mod data;mod tld;use data::*;use tld::classify_tld;use tld::Tld;const LATIN_ADJACENCY_PENALTY: i64 = -50;const IMPLAUSIBILITY_PENALTY: i64 = -220;const ORDINAL_BONUS: i64 = 300;const IMPLAUSIBLE_LATIN_CASE_TRANSITION_PENALTY: i64 = -180;const NON_LATIN_CAPITALIZATION_BONUS: i64 = 40;const NON_LATIN_ALL_CAPS_PENALTY: i64 = -40;const NON_LATIN_MIXED_CASE_PENALTY: i64 = -20;// Manually calibrated relative to windows-1256 Arabicconst CJK_BASE_SCORE: i64 = 41;const CJK_SECONDARY_BASE_SCORE: i64 = 20; // Was 20const SHIFT_JIS_SCORE_PER_KANA: i64 = 20;const SHIFT_JIS_SCORE_PER_LEVEL_1_KANJI: i64 = CJK_BASE_SCORE;const SHIFT_JIS_SCORE_PER_LEVEL_2_KANJI: i64 = CJK_SECONDARY_BASE_SCORE;// Manually calibrated relative to windows-1256 Persian and Urduconst SHIFT_JIS_INITIAL_HALF_WIDTH_KATAKANA_PENALTY: i64 = -75;const HALF_WIDTH_KATAKANA_SCORE: i64 = 1;// Unclear if this is a good idea; seems not harmful, but can't be sure.const HALF_WIDTH_KATAKANA_VOICING_SCORE: i64 = 10;const SHIFT_JIS_PUA_PENALTY: i64 = -(CJK_BASE_SCORE * 10); // Should this be larger?const SHIFT_JIS_EXTENSION_PENALTY: i64 = SHIFT_JIS_PUA_PENALTY * 2;const SHIFT_JIS_SINGLE_BYTE_EXTENSION_PENALTY: i64 = SHIFT_JIS_EXTENSION_PENALTY;const EUC_JP_SCORE_PER_KANA: i64 = CJK_BASE_SCORE + (CJK_BASE_SCORE / 3); // Relative to Big5const EUC_JP_SCORE_PER_NEAR_OBSOLETE_KANA: i64 = CJK_BASE_SCORE - 1;const EUC_JP_SCORE_PER_LEVEL_1_KANJI: i64 = CJK_BASE_SCORE;const EUC_JP_SCORE_PER_LEVEL_2_KANJI: i64 = CJK_SECONDARY_BASE_SCORE;const EUC_JP_SCORE_PER_OTHER_KANJI: i64 = CJK_SECONDARY_BASE_SCORE / 4;const EUC_JP_INITIAL_KANA_PENALTY: i64 = -((CJK_BASE_SCORE / 3) + 1);const EUC_JP_EXTENSION_PENALTY: i64 = -(CJK_BASE_SCORE * 50); // Needs to be more severe than for Shift_JIS to avoid misdetecting EUC-KR!const BIG5_SCORE_PER_LEVEL_1_HANZI: i64 = CJK_BASE_SCORE;const BIG5_SCORE_PER_OTHER_HANZI: i64 = CJK_SECONDARY_BASE_SCORE;const BIG5_PUA_PENALTY: i64 = -(CJK_BASE_SCORE * 30); // More severe than other PUA penalties to avoid misdetecting EUC-KR! (25 as the multiplier is too little)const BIG5_SINGLE_BYTE_EXTENSION_PENALTY: i64 = -(CJK_BASE_SCORE * 40);const EUC_KR_SCORE_PER_EUC_HANGUL: i64 = CJK_BASE_SCORE + 1;const EUC_KR_SCORE_PER_NON_EUC_HANGUL: i64 = CJK_SECONDARY_BASE_SCORE / 5;const EUC_KR_SCORE_PER_HANJA: i64 = CJK_SECONDARY_BASE_SCORE / 2;const EUC_KR_HANJA_AFTER_HANGUL_PENALTY: i64 = -(CJK_BASE_SCORE * 10);const EUC_KR_LONG_WORD_PENALTY: i64 = -6;const EUC_KR_PUA_PENALTY: i64 = GBK_PUA_PENALTY - 1; // Break tie in favor of GBKconst EUC_KR_MAC_KOREAN_PENALTY: i64 = EUC_KR_PUA_PENALTY * 2;const EUC_KR_SINGLE_BYTE_EXTENSION_PENALTY: i64 = EUC_KR_MAC_KOREAN_PENALTY;const GBK_SCORE_PER_LEVEL_1: i64 = CJK_BASE_SCORE;const GBK_SCORE_PER_LEVEL_2: i64 = CJK_SECONDARY_BASE_SCORE;const GBK_SCORE_PER_NON_EUC: i64 = CJK_SECONDARY_BASE_SCORE / 4;const GBK_PUA_PENALTY: i64 = -(CJK_BASE_SCORE * 10); // Factor should be at least 2, but should it be larger?const GBK_SINGLE_BYTE_EXTENSION_PENALTY: i64 = GBK_PUA_PENALTY * 4;const CJK_LATIN_ADJACENCY_PENALTY: i64 = -CJK_BASE_SCORE; // smaller penalty than LATIN_ADJACENCY_PENALTYconst CJ_PUNCTUATION: i64 = CJK_BASE_SCORE / 2;const CJK_OTHER: i64 = CJK_SECONDARY_BASE_SCORE / 4;/// Latin letter caseless classconst LATIN_LETTER: u8 = 1;fn contains_upper_case_period_or_non_ascii(label: &[u8]) -> bool {for &b in label.into_iter() {if b >= 0x80 {return true;}if b == b'.' {return true;}if b >= b'A' && b <= b'Z' {return true;}}false}// For Latin, we only penalize pairwise bad transitions// if one participant is non-ASCII. This avoids violating// the principle that ASCII pairs never contribute to the// score. (Maybe that's a bad principle, though!)#[derive(PartialEq)]enum LatinCaseState {Space,Upper,Lower,AllCaps,}// Fon non-Latin, we calculate case-related penalty// or bonus on a per-non-Latin-word basis.#[derive(PartialEq)]enum NonLatinCaseState {Space,Upper,Lower,UpperLower,AllCaps,Mix,}struct NonLatinCasedCandidate {data: &'static SingleByteData,prev: u8,case_state: NonLatinCaseState,prev_ascii: bool,current_word_len: u64,longest_word: u64,ibm866: bool,prev_was_a0: bool, // Only used with IBM866}impl NonLatinCasedCandidate {fn new(data: &'static SingleByteData) -> Self {NonLatinCasedCandidate {data: data,prev: 0,case_state: NonLatinCaseState::Space,prev_ascii: true,current_word_len: 0,longest_word: 0,ibm866: data == &SINGLE_BYTE_DATA[IBM866_INDEX],prev_was_a0: false,}}fn feed(&mut self, buffer: &[u8]) -> Option<i64> {let mut score = 0i64;for &b in buffer {let class = self.data.classify(b);if class == 255 {return None;}let caseless_class = class & 0x7F;let ascii = b < 0x80;let ascii_pair = self.prev_ascii && ascii;let non_ascii_alphabetic = self.data.is_non_latin_alphabetic(caseless_class, false);// The purpose of this state machine is to avoid misdetecting Greek as// Cyrillic by://// * Giving a small bonus to words that start with an upper-case letter// and are lower-case for the rest.// * Giving a large penalty to start with one lower-case letter followed// by all upper-case (obviously upper and lower case inverted, which// unfortunately is possible due to KOI8-U).// * Giving a small per-word penalty to all-uppercase KOI8-U (to favor// all-lowercase Greek over all-caps KOI8-U).// * Giving large penalties for mixed-case other than initial upper-case.// This also helps relative to non-cased encodings.// ASCII doesn't participate in non-Latin casing.if caseless_class == LATIN_LETTER {// Latin// Mark this word as a mess. If there end up being non-Latin// letters in this word, the ASCII-adjacency penalty gets// applied to Latin/non-Latin pairs and the mix penalty// to non-Latin/non-Latin pairs.// XXX Apply penalty hereself.case_state = NonLatinCaseState::Mix;} else if !non_ascii_alphabetic {// Spacematch self.case_state {NonLatinCaseState::Space| NonLatinCaseState::Upper| NonLatinCaseState::Lower => {}NonLatinCaseState::UpperLower => {// Intentionally applied only once per word.score += NON_LATIN_CAPITALIZATION_BONUS;}NonLatinCaseState::AllCaps => {// Intentionally applied only once per word.if self.data == &SINGLE_BYTE_DATA[KOI8_U_INDEX] {// Apply only to KOI8-U.score += NON_LATIN_ALL_CAPS_PENALTY;}}NonLatinCaseState::Mix => {// Per letterscore += NON_LATIN_MIXED_CASE_PENALTY * (self.current_word_len as i64);}}self.case_state = NonLatinCaseState::Space;} else if (class >> 7) == 0 {// Lower casematch self.case_state {NonLatinCaseState::Space => {self.case_state = NonLatinCaseState::Lower;}NonLatinCaseState::Upper => {self.case_state = NonLatinCaseState::UpperLower;}NonLatinCaseState::Lower| NonLatinCaseState::UpperLower| NonLatinCaseState::Mix => {}NonLatinCaseState::AllCaps => {self.case_state = NonLatinCaseState::Mix;}}} else {// Upper casematch self.case_state {NonLatinCaseState::Space => {self.case_state = NonLatinCaseState::Upper;}NonLatinCaseState::Upper => {self.case_state = NonLatinCaseState::AllCaps;}NonLatinCaseState::Lower | NonLatinCaseState::UpperLower => {self.case_state = NonLatinCaseState::Mix;}NonLatinCaseState::AllCaps | NonLatinCaseState::Mix => {}}}// XXX Apply penalty if > 16if non_ascii_alphabetic {self.current_word_len += 1;} else {if self.current_word_len > self.longest_word {self.longest_word = self.current_word_len;}self.current_word_len = 0;}let is_a0 = b == 0xA0;if !ascii_pair {// 0xA0 is no-break space in many other encodings, so avoid// assigning score to IBM866 when 0xA0 occurs next to itself// or a space-like byte.if !(self.ibm866&& ((is_a0 && (self.prev_was_a0 || self.prev == 0))|| caseless_class == 0 && self.prev_was_a0)){score += self.data.score(caseless_class, self.prev, false);}if self.prev == LATIN_LETTER && non_ascii_alphabetic {score += LATIN_ADJACENCY_PENALTY;} else if caseless_class == LATIN_LETTER&& self.data.is_non_latin_alphabetic(self.prev, false){score += LATIN_ADJACENCY_PENALTY;}}self.prev_ascii = ascii;self.prev = caseless_class;self.prev_was_a0 = is_a0;}Some(score)}}enum OrdinalState {Other,Space,PeriodAfterN,OrdinalExpectingSpace,OrdinalExpectingSpaceUndoImplausibility,OrdinalExpectingSpaceOrDigit,OrdinalExpectingSpaceOrDigitUndoImplausibily,UpperN,LowerN,FeminineAbbreviationStartLetter,Digit,Roman,}struct LatinCandidate {data: &'static SingleByteData,prev: u8,case_state: LatinCaseState,prev_non_ascii: u32,ordinal_state: OrdinalState, // Used only when `windows1252 == true`windows1252: bool,}impl LatinCandidate {fn new(data: &'static SingleByteData) -> Self {LatinCandidate {data: data,prev: 0,case_state: LatinCaseState::Space,prev_non_ascii: 0,ordinal_state: OrdinalState::Space,windows1252: data == &SINGLE_BYTE_DATA[WINDOWS_1252_INDEX],}}fn feed(&mut self, buffer: &[u8]) -> Option<i64> {let mut score = 0i64;for &b in buffer {let class = self.data.classify(b);if class == 255 {return None;}let caseless_class = class & 0x7F;let ascii = b < 0x80;let ascii_pair = self.prev_non_ascii == 0 && ascii;let non_ascii_penalty = match self.prev_non_ascii {0 | 1 | 2 => 0,3 => -5,4 => -20,_ => -200,};score += non_ascii_penalty;// XXX if has Vietnamese-only characters and word length > 7,// apply penaltyif !self.data.is_latin_alphabetic(caseless_class) {self.case_state = LatinCaseState::Space;} else if (class >> 7) == 0 {// Penalizing lower case after two upper case// is important for avoiding misdetecting// windows-1250 as windows-1252 (byte 0x9F).if self.case_state == LatinCaseState::AllCaps && !ascii_pair {score += IMPLAUSIBLE_LATIN_CASE_TRANSITION_PENALTY;}self.case_state = LatinCaseState::Lower;} else {match self.case_state {LatinCaseState::Space => {self.case_state = LatinCaseState::Upper;}LatinCaseState::Upper | LatinCaseState::AllCaps => {self.case_state = LatinCaseState::AllCaps;}LatinCaseState::Lower => {if !ascii_pair {// XXX How bad is this for Irish Gaelic?score += IMPLAUSIBLE_LATIN_CASE_TRANSITION_PENALTY;}self.case_state = LatinCaseState::Upper;}}}// Treat pairing space-like, which can be non-ASCII, with ASCII as// ASCIIish enough not to get a score in order to avoid giving// ASCII i and I in windows-1254 next to windows-125x apostrophe/quote// a score. This avoids detecting English I’ as Turkish.let ascii_ish_pair = ascii_pair|| (ascii && self.prev == 0)|| (caseless_class == 0 && self.prev_non_ascii == 0);if !ascii_ish_pair {score += self.data.score(caseless_class, self.prev, false);}if self.windows1252 {// This state machine assigns score to the sequences// * " º " (Spanish)// * " ª " (Spanish)// * ".ª " (Spanish)// * ".º " (Spanish)// * "n.º1" (Spanish)// * " Mª " (Spanish)// * " Dª " (Spanish)// * " Nª " (Spanish)// * " Sª " (Spanish)// * " 3º " (Italian, where 3 is an ASCII digit)// * " 3ª " (Italian, where 3 is an ASCII digit)// * " Xº " (Italian, where X is a small Roman numeral)// * " Xª " (Italian, where X is a small Roman numeral)// * " Nº1" (Italian, where 1 is an ASCII digit)// * " Nº " (Italian)// which are problematic to deal with by pairwise scoring// without messing up Romanian detection.// Initial scmatch self.ordinal_state {OrdinalState::Other => {if caseless_class == 0 {self.ordinal_state = OrdinalState::Space;}}OrdinalState::Space => {if caseless_class == 0 {// pass} else if b == 0xAA || b == 0xBA {self.ordinal_state = OrdinalState::OrdinalExpectingSpace;} else if b == b'M' || b == b'D' || b == b'S' {self.ordinal_state = OrdinalState::FeminineAbbreviationStartLetter;} else if b == b'N' {// numero or Nuestraself.ordinal_state = OrdinalState::UpperN;} else if b == b'n' {// numeroself.ordinal_state = OrdinalState::LowerN;} else if caseless_class == (ASCII_DIGIT as u8) {self.ordinal_state = OrdinalState::Digit;} else if caseless_class == 9 /* I */ || caseless_class == 22 /* V */ || caseless_class == 24/* X */{self.ordinal_state = OrdinalState::Roman;} else {self.ordinal_state = OrdinalState::Other;}}OrdinalState::OrdinalExpectingSpace => {if caseless_class == 0 {score += ORDINAL_BONUS;self.ordinal_state = OrdinalState::Space;} else {self.ordinal_state = OrdinalState::Other;}}OrdinalState::OrdinalExpectingSpaceUndoImplausibility => {if caseless_class == 0 {score += ORDINAL_BONUS - IMPLAUSIBILITY_PENALTY;self.ordinal_state = OrdinalState::Space;} else {self.ordinal_state = OrdinalState::Other;}}OrdinalState::OrdinalExpectingSpaceOrDigit => {if caseless_class == 0 {score += ORDINAL_BONUS;self.ordinal_state = OrdinalState::Space;} else if caseless_class == (ASCII_DIGIT as u8) {score += ORDINAL_BONUS;// Deliberately set to `Other`self.ordinal_state = OrdinalState::Other;} else {self.ordinal_state = OrdinalState::Other;}}OrdinalState::OrdinalExpectingSpaceOrDigitUndoImplausibily => {if caseless_class == 0 {score += ORDINAL_BONUS - IMPLAUSIBILITY_PENALTY;self.ordinal_state = OrdinalState::Space;} else if caseless_class == (ASCII_DIGIT as u8) {score += ORDINAL_BONUS - IMPLAUSIBILITY_PENALTY;// Deliberately set to `Other`self.ordinal_state = OrdinalState::Other;} else {self.ordinal_state = OrdinalState::Other;}}OrdinalState::UpperN => {if b == 0xAA {self.ordinal_state =OrdinalState::OrdinalExpectingSpaceUndoImplausibility;} else if b == 0xBA {self.ordinal_state =OrdinalState::OrdinalExpectingSpaceOrDigitUndoImplausibily;} else if b == b'.' {self.ordinal_state = OrdinalState::PeriodAfterN;} else if caseless_class == 0 {self.ordinal_state = OrdinalState::Space;} else {self.ordinal_state = OrdinalState::Other;}}OrdinalState::LowerN => {if b == 0xBA {self.ordinal_state =OrdinalState::OrdinalExpectingSpaceOrDigitUndoImplausibily;} else if b == b'.' {self.ordinal_state = OrdinalState::PeriodAfterN;} else if caseless_class == 0 {self.ordinal_state = OrdinalState::Space;} else {self.ordinal_state = OrdinalState::Other;}}OrdinalState::FeminineAbbreviationStartLetter => {if b == 0xAA {self.ordinal_state =OrdinalState::OrdinalExpectingSpaceUndoImplausibility;} else if caseless_class == 0 {self.ordinal_state = OrdinalState::Space;} else {self.ordinal_state = OrdinalState::Other;}}OrdinalState::Digit => {if b == 0xAA || b == 0xBA {self.ordinal_state = OrdinalState::OrdinalExpectingSpace;} else if caseless_class == 0 {self.ordinal_state = OrdinalState::Space;} else if caseless_class == (ASCII_DIGIT as u8) {// pass} else {self.ordinal_state = OrdinalState::Other;}}OrdinalState::Roman => {if b == 0xAA || b == 0xBA {self.ordinal_state =OrdinalState::OrdinalExpectingSpaceUndoImplausibility;} else if caseless_class == 0 {self.ordinal_state = OrdinalState::Space;} else if caseless_class == 9 /* I */ || caseless_class == 22 /* V */ || caseless_class == 24/* X */{// pass} else {self.ordinal_state = OrdinalState::Other;}}OrdinalState::PeriodAfterN => {if b == 0xBA {self.ordinal_state = OrdinalState::OrdinalExpectingSpaceOrDigit;} else if caseless_class == 0 {self.ordinal_state = OrdinalState::Space;} else {self.ordinal_state = OrdinalState::Other;}}}}if ascii {self.prev_non_ascii = 0;} else {self.prev_non_ascii += 1;}self.prev = caseless_class;}Some(score)}}struct ArabicFrenchCandidate {data: &'static SingleByteData,prev: u8,case_state: LatinCaseState,prev_ascii: bool,current_word_len: u64,longest_word: u64,}impl ArabicFrenchCandidate {fn new(data: &'static SingleByteData) -> Self {ArabicFrenchCandidate {data: data,prev: 0,case_state: LatinCaseState::Space,prev_ascii: true,current_word_len: 0,longest_word: 0,}}fn feed(&mut self, buffer: &[u8]) -> Option<i64> {let mut score = 0i64;for &b in buffer {let class = self.data.classify(b);if class == 255 {return None;}let caseless_class = class & 0x7F;let ascii = b < 0x80;let ascii_pair = self.prev_ascii && ascii;if caseless_class != LATIN_LETTER {// We compute case penalties for French onlyself.case_state = LatinCaseState::Space;} else if (class >> 7) == 0 {if self.case_state == LatinCaseState::AllCaps && !ascii_pair {score += IMPLAUSIBLE_LATIN_CASE_TRANSITION_PENALTY;}self.case_state = LatinCaseState::Lower;} else {match self.case_state {LatinCaseState::Space => {self.case_state = LatinCaseState::Upper;}LatinCaseState::Upper | LatinCaseState::AllCaps => {self.case_state = LatinCaseState::AllCaps;}LatinCaseState::Lower => {if !ascii_pair {score += IMPLAUSIBLE_LATIN_CASE_TRANSITION_PENALTY;}self.case_state = LatinCaseState::Upper;}}}// Count only Arabic word length and ignore Frenchlet non_ascii_alphabetic = self.data.is_non_latin_alphabetic(caseless_class, true);// XXX apply penalty if > 23if non_ascii_alphabetic {self.current_word_len += 1;} else {if self.current_word_len > self.longest_word {self.longest_word = self.current_word_len;}self.current_word_len = 0;}if !ascii_pair {score += self.data.score(caseless_class, self.prev, true);if self.prev == LATIN_LETTER && non_ascii_alphabetic {score += LATIN_ADJACENCY_PENALTY;} else if caseless_class == LATIN_LETTER&& self.data.is_non_latin_alphabetic(self.prev, true){score += LATIN_ADJACENCY_PENALTY;}}self.prev_ascii = ascii;self.prev = caseless_class;}Some(score)}}struct CaselessCandidate {data: &'static SingleByteData,prev: u8,prev_ascii: bool,current_word_len: u64,longest_word: u64,}impl CaselessCandidate {fn new(data: &'static SingleByteData) -> Self {CaselessCandidate {data: data,prev: 0,prev_ascii: true,current_word_len: 0,longest_word: 0,}}fn feed(&mut self, buffer: &[u8]) -> Option<i64> {let mut score = 0i64;for &b in buffer {let class = self.data.classify(b);if class == 255 {return None;}let caseless_class = class & 0x7F;let ascii = b < 0x80;let ascii_pair = self.prev_ascii && ascii;let non_ascii_alphabetic = self.data.is_non_latin_alphabetic(caseless_class, false);// Apply penalty if > 23 and not Thaiif non_ascii_alphabetic {self.current_word_len += 1;} else {if self.current_word_len > self.longest_word {self.longest_word = self.current_word_len;}self.current_word_len = 0;}if !ascii_pair {score += self.data.score(caseless_class, self.prev, false);if self.prev == LATIN_LETTER && non_ascii_alphabetic {score += LATIN_ADJACENCY_PENALTY;} else if caseless_class == LATIN_LETTER&& self.data.is_non_latin_alphabetic(self.prev, false){score += LATIN_ADJACENCY_PENALTY;}}self.prev_ascii = ascii;self.prev = caseless_class;}Some(score)}}fn is_ascii_punctuation(byte: u8) -> bool {match byte {b'.' | b',' | b':' | b';' | b'?' | b'!' => true,_ => false,}}struct LogicalCandidate {data: &'static SingleByteData,prev: u8,prev_ascii: bool,plausible_punctuation: u64,current_word_len: u64,longest_word: u64,}impl LogicalCandidate {fn new(data: &'static SingleByteData) -> Self {LogicalCandidate {data: data,prev: 0,prev_ascii: true,plausible_punctuation: 0,current_word_len: 0,longest_word: 0,}}fn feed(&mut self, buffer: &[u8]) -> Option<i64> {let mut score = 0i64;for &b in buffer {let class = self.data.classify(b);if class == 255 {return None;}let caseless_class = class & 0x7F;let ascii = b < 0x80;let ascii_pair = self.prev_ascii && ascii;let non_ascii_alphabetic = self.data.is_non_latin_alphabetic(caseless_class, false);// XXX apply penalty if > 22if non_ascii_alphabetic {self.current_word_len += 1;} else {if self.current_word_len > self.longest_word {self.longest_word = self.current_word_len;}self.current_word_len = 0;}if !ascii_pair {score += self.data.score(caseless_class, self.prev, false);let prev_non_ascii_alphabetic = self.data.is_non_latin_alphabetic(self.prev, false);if caseless_class == 0 && prev_non_ascii_alphabetic && is_ascii_punctuation(b) {self.plausible_punctuation += 1;}if self.prev == LATIN_LETTER && non_ascii_alphabetic {score += LATIN_ADJACENCY_PENALTY;} else if caseless_class == LATIN_LETTER && prev_non_ascii_alphabetic {score += LATIN_ADJACENCY_PENALTY;}}self.prev_ascii = ascii;self.prev = caseless_class;}Some(score)}}struct VisualCandidate {data: &'static SingleByteData,prev: u8,prev_ascii: bool,prev_punctuation: bool,plausible_punctuation: u64,current_word_len: u64,longest_word: u64,}impl VisualCandidate {fn new(data: &'static SingleByteData) -> Self {VisualCandidate {data: data,prev: 0,prev_ascii: true,prev_punctuation: false,plausible_punctuation: 0,current_word_len: 0,longest_word: 0,}}fn feed(&mut self, buffer: &[u8]) -> Option<i64> {let mut score = 0i64;for &b in buffer {let class = self.data.classify(b);if class == 255 {return None;}let caseless_class = class & 0x7F;let ascii = b < 0x80;let ascii_pair = self.prev_ascii && ascii;let non_ascii_alphabetic = self.data.is_non_latin_alphabetic(caseless_class, false);// XXX apply penalty if > 22if non_ascii_alphabetic {self.current_word_len += 1;} else {if self.current_word_len > self.longest_word {self.longest_word = self.current_word_len;}self.current_word_len = 0;}if !ascii_pair {score += self.data.score(caseless_class, self.prev, false);if non_ascii_alphabetic && self.prev_punctuation {self.plausible_punctuation += 1;}if self.prev == LATIN_LETTER && non_ascii_alphabetic {score += LATIN_ADJACENCY_PENALTY;} else if caseless_class == LATIN_LETTER&& self.data.is_non_latin_alphabetic(self.prev, false){score += LATIN_ADJACENCY_PENALTY;}}self.prev_ascii = ascii;self.prev = caseless_class;self.prev_punctuation = caseless_class == 0 && is_ascii_punctuation(b);}Some(score)}}struct Utf8Candidate {decoder: Decoder,}impl Utf8Candidate {fn feed(&mut self, buffer: &[u8], last: bool) -> Option<i64> {let mut dst = [0u8; 1024];let mut total_read = 0;loop {let (result, read, _) = self.decoder.decode_to_utf8_without_replacement(&buffer[total_read..],&mut dst,last,);total_read += read;match result {DecoderResult::InputEmpty => {return Some(0);}DecoderResult::Malformed(_, _) => {return None;}DecoderResult::OutputFull => {continue;}}}}}struct Iso2022Candidate {decoder: Decoder,}impl Iso2022Candidate {fn feed(&mut self, buffer: &[u8], last: bool) -> Option<i64> {let mut dst = [0u16; 1024];let mut total_read = 0;loop {let (result, read, _) = self.decoder.decode_to_utf16_without_replacement(&buffer[total_read..],&mut dst,last,);total_read += read;match result {DecoderResult::InputEmpty => {return Some(0);}DecoderResult::Malformed(_, _) => {return None;}DecoderResult::OutputFull => {continue;}}}}}#[derive(PartialEq)]enum LatinCj {AsciiLetter,Cj,Other,}#[derive(PartialEq, Copy, Clone)]enum HalfWidthKatakana {DakutenForbidden,DakutenAllowed,DakutenOrHandakutenAllowed,}#[derive(PartialEq)]enum LatinKorean {AsciiLetter,Hangul,Hanja,Other,}fn cjk_extra_score(u: u16, table: &'static [u16; 128]) -> i64 {if let Some(pos) = table.iter().position(|&x| x == u) {((128 - pos) / 16) as i64} else {0}}struct GbkCandidate {decoder: Decoder,prev_byte: u8,prev: LatinCj,pending_score: Option<i64>,}impl GbkCandidate {fn maybe_set_as_pending(&mut self, s: i64) -> i64 {assert!(self.pending_score.is_none());if self.prev == LatinCj::Cj || !more_problematic_lead(self.prev_byte) {s} else {self.pending_score = Some(s);0}}fn feed(&mut self, buffer: &[u8], last: bool) -> Option<i64> {let mut score = 0i64;let mut src = [0u8];let mut dst = [0u16; 2];for &b in buffer {src[0] = b;let (result, read, written) = self.decoder.decode_to_utf16_without_replacement(&src, &mut dst, false);if written == 1 {let u = dst[0];if (u >= u16::from(b'a') && u <= u16::from(b'z'))|| (u >= u16::from(b'A') && u <= u16::from(b'Z')){self.pending_score = None; // Discard pending scoreif self.prev == LatinCj::Cj {score += CJK_LATIN_ADJACENCY_PENALTY;}self.prev = LatinCj::AsciiLetter;} else if u == 0x20AC {// euro signself.pending_score = None; // Discard pending score// Should there even be a penalty?self.prev = LatinCj::Other;} else if u >= 0x4E00 && u <= 0x9FA5 {if let Some(pending) = self.pending_score {score += pending;self.pending_score = None;}if b >= 0xA1 && b <= 0xFE {match self.prev_byte {0xA1..=0xD7 => {score += GBK_SCORE_PER_LEVEL_1;score +=cjk_extra_score(u, &data::DETECTOR_DATA.frequent_simplified);}0xD8..=0xFE => score += GBK_SCORE_PER_LEVEL_2,_ => {score += GBK_SCORE_PER_NON_EUC;}}} else {score += self.maybe_set_as_pending(GBK_SCORE_PER_NON_EUC);}if self.prev == LatinCj::AsciiLetter {score += CJK_LATIN_ADJACENCY_PENALTY;}self.prev = LatinCj::Cj;} else if (u >= 0x3400 && u < 0xA000) || (u >= 0xF900 && u < 0xFB00) {if let Some(pending) = self.pending_score {score += pending;self.pending_score = None;}// XXX score?if self.prev == LatinCj::AsciiLetter {score += CJK_LATIN_ADJACENCY_PENALTY;}self.prev = LatinCj::Cj;} else if u >= 0xE000 && u < 0xF900 {if let Some(pending) = self.pending_score {score += pending;self.pending_score = None;}// Treat the GB18030-required PUA mappings as non-EUC ideographs.match u {0xE78D..=0xE796| 0xE816..=0xE818| 0xE81E| 0xE826| 0xE82B| 0xE82C| 0xE831| 0xE832| 0xE83B| 0xE843| 0xE854| 0xE855| 0xE864 => {score += GBK_SCORE_PER_NON_EUC;if self.prev == LatinCj::AsciiLetter {score += CJK_LATIN_ADJACENCY_PENALTY;}self.prev = LatinCj::Cj;}_ => {score += GBK_PUA_PENALTY;self.prev = LatinCj::Other;}}} else {match u {0x3000 // Distinct from Korean, space| 0x3001 // Distinct from Korean, enumeration comma| 0x3002 // Distinct from Korean, full stop| 0xFF08 // Distinct from Korean, parenthesis| 0xFF09 // Distinct from Korean, parenthesis| 0xFF01 // Distinct from Japanese, exclamation| 0xFF0C // Distinct from Japanese, comma| 0xFF1B // Distinct from Japanese, semicolon| 0xFF1F // Distinct from Japanese, question=> {if let Some(pending) = self.pending_score {score += pending;self.pending_score = None;}score += CJ_PUNCTUATION;}0..=0x7F => {self.pending_score = None; // Discard pending score}_ => {if let Some(pending) = self.pending_score {score += pending;self.pending_score = None;}score += CJK_OTHER;}}self.prev = LatinCj::Other;}} else if written == 2 {if let Some(pending) = self.pending_score {score += pending;self.pending_score = None;}let u = dst[0];if u >= 0xDB80 && u <= 0xDBFF {score += GBK_PUA_PENALTY;self.prev = LatinCj::Other;} else if u >= 0xD480 && u < 0xD880 {score += GBK_SCORE_PER_NON_EUC;if self.prev == LatinCj::AsciiLetter {score += CJK_LATIN_ADJACENCY_PENALTY;}self.prev = LatinCj::Cj;} else {score += CJK_OTHER;self.prev = LatinCj::Other;}}match result {DecoderResult::InputEmpty => {assert_eq!(read, 1);}DecoderResult::Malformed(malformed_len, _) => {if (self.prev_byte == 0xA0 || self.prev_byte == 0xFE || self.prev_byte == 0xFD)&& (b < 0x80 || b == 0xFF){// Mac OS Chinese Simplified single-byte that conflicts with code page GBK lead byte// followed by ASCII or a non-conflicting single-byte extension.self.pending_score = None; // Just in casescore += GBK_SINGLE_BYTE_EXTENSION_PENALTY;if (b >= b'a' && b <= b'z') || (b >= b'A' && b <= b'Z') {self.prev = LatinCj::AsciiLetter;} else if b == 0xFF {score += GBK_SINGLE_BYTE_EXTENSION_PENALTY;self.prev = LatinCj::Other;} else {self.prev = LatinCj::Other;}// The GBK decoder has the pending ASCII concept, which is// a problem with this trickery, so let's reset the state.self.decoder = GBK.new_decoder_without_bom_handling();} else if malformed_len == 1 && b == 0xFF {// Mac OS Chinese Simplified single-byte extension that doesn't conflict with lead bytesself.pending_score = None; // Just in casescore += GBK_SINGLE_BYTE_EXTENSION_PENALTY;self.prev = LatinCj::Other;// The GBK decoder has the pending ASCII concept, which is// a problem with this trickery, so let's reset the state.self.decoder = GBK.new_decoder_without_bom_handling();} else {return None;}}DecoderResult::OutputFull => {unreachable!();}}self.prev_byte = b;}if last {let (result, _, _) = self.decoder.decode_to_utf16_without_replacement(b"", &mut dst, true);match result {DecoderResult::InputEmpty => {}DecoderResult::Malformed(_, _) => {return None;}DecoderResult::OutputFull => {unreachable!();}}}Some(score)}}// Shift_JIS and Big5fn problematic_lead(b: u8) -> bool {match b {0x91..=0x97 | 0x9A | 0x8A | 0x9B | 0x8B | 0x9E | 0x8E | 0xB0 => true,_ => false,}}// GBK and EUC-KRfn more_problematic_lead(b: u8) -> bool {problematic_lead(b) || b == 0x82 || b == 0x84 || b == 0x85 || b == 0xA0}struct ShiftJisCandidate {decoder: Decoder,half_width_katakana_seen: bool,half_width_katakana_state: HalfWidthKatakana,prev: LatinCj,prev_byte: u8,pending_score: Option<i64>,}impl ShiftJisCandidate {fn maybe_set_as_pending(&mut self, s: i64) -> i64 {assert!(self.pending_score.is_none());if self.prev == LatinCj::Cj || !problematic_lead(self.prev_byte) {s} else {self.pending_score = Some(s);0}}fn feed(&mut self, buffer: &[u8], last: bool) -> Option<i64> {let mut score = 0i64;let mut src = [0u8];let mut dst = [0u16; 2];for &b in buffer {src[0] = b;let (result, read, written) = self.decoder.decode_to_utf16_without_replacement(&src, &mut dst, false);if written > 0 {let half_width_katakana_state = self.half_width_katakana_state;self.half_width_katakana_state = HalfWidthKatakana::DakutenForbidden;let u = dst[0];if (u >= u16::from(b'a') && u <= u16::from(b'z'))|| (u >= u16::from(b'A') && u <= u16::from(b'Z')){self.pending_score = None; // Discard pending scoreif self.prev == LatinCj::Cj {score += CJK_LATIN_ADJACENCY_PENALTY;}self.prev = LatinCj::AsciiLetter;} else if u >= 0xFF61 && u <= 0xFF9F {if !self.half_width_katakana_seen {self.half_width_katakana_seen = true;// To avoid misdetecting title-length inputsscore += SHIFT_JIS_INITIAL_HALF_WIDTH_KATAKANA_PENALTY;}self.pending_score = None; // Discard pending scorescore += HALF_WIDTH_KATAKANA_SCORE;if (u >= 0xFF76 && u <= 0xFF84) || u == 0xFF73 {self.half_width_katakana_state = HalfWidthKatakana::DakutenAllowed;} else if u >= 0xFF8A && u <= 0xFF8E {self.half_width_katakana_state =HalfWidthKatakana::DakutenOrHandakutenAllowed;} else if u == 0xFF9E {if half_width_katakana_state == HalfWidthKatakana::DakutenForbidden {score += IMPLAUSIBILITY_PENALTY;} else {score += HALF_WIDTH_KATAKANA_VOICING_SCORE;}} else if u == 0xFF9F {if half_width_katakana_state!= HalfWidthKatakana::DakutenOrHandakutenAllowed{score += IMPLAUSIBILITY_PENALTY;} else {score += HALF_WIDTH_KATAKANA_VOICING_SCORE;}}if self.prev == LatinCj::AsciiLetter {score += CJK_LATIN_ADJACENCY_PENALTY;}self.prev = LatinCj::Cj;} else if u >= 0x3040 && u < 0x3100 {if let Some(pending) = self.pending_score {score += pending;self.pending_score = None;}score += SHIFT_JIS_SCORE_PER_KANA;if self.prev == LatinCj::AsciiLetter {score += CJK_LATIN_ADJACENCY_PENALTY;}self.prev = LatinCj::Cj;} else if (u >= 0x3400 && u < 0xA000) || (u >= 0xF900 && u < 0xFB00) {if let Some(pending) = self.pending_score {score += pending;self.pending_score = None;}if self.prev_byte < 0x98 || (self.prev_byte == 0x98 && b < 0x73) {score += self.maybe_set_as_pending(SHIFT_JIS_SCORE_PER_LEVEL_1_KANJI+ cjk_extra_score(u, &data::DETECTOR_DATA.frequent_kanji),);} else {score += self.maybe_set_as_pending(SHIFT_JIS_SCORE_PER_LEVEL_2_KANJI);}if self.prev == LatinCj::AsciiLetter {score += CJK_LATIN_ADJACENCY_PENALTY;}self.prev = LatinCj::Cj;} else if u >= 0xE000 && u < 0xF900 {if let Some(pending) = self.pending_score {score += pending;self.pending_score = None;}score += SHIFT_JIS_PUA_PENALTY;self.prev = LatinCj::Other;} else {match u {0x3000 // Distinct from Korean, space| 0x3001 // Distinct from Korean, enumeration comma| 0x3002 // Distinct from Korean, full stop| 0xFF08 // Distinct from Korean, parenthesis| 0xFF09 // Distinct from Korean, parenthesis=> {if let Some(pending) = self.pending_score {score += pending;self.pending_score = None;}// Not really needed for CJK distinction// but let's give non-zero score for these// common byte pairs anyway.score += CJ_PUNCTUATION;}0..=0x7F => {self.pending_score = None; // Discard pending score}0x80 => {// This is a control character that overlaps euro// in windows-1252 and happens to be a non-error// is Shift_JIS.self.pending_score = None; // Discard pending scorescore += IMPLAUSIBILITY_PENALTY;}_ => {if let Some(pending) = self.pending_score {score += pending;self.pending_score = None;}score += CJK_OTHER;}}self.prev = LatinCj::Other;}}match result {DecoderResult::InputEmpty => {assert_eq!(read, 1);}DecoderResult::Malformed(malformed_len, _) => {if (((self.prev_byte >= 0x81 && self.prev_byte <= 0x9F)|| (self.prev_byte >= 0xE0 && self.prev_byte <= 0xFC))&& ((b >= 0x40 && b <= 0x7E) || (b >= 0x80 && b <= 0xFC)))&& !((self.prev_byte == 0x82 && b >= 0xFA)|| (self.prev_byte == 0x84 && ((b >= 0xDD && b <= 0xE4) || b >= 0xFB))|| (self.prev_byte == 0x86 && b >= 0xF2 && b <= 0xFA)|| (self.prev_byte == 0x87 && b >= 0x77 && b <= 0x7D)|| (self.prev_byte == 0xFC && b >= 0xF5)){// Shift_JIS2004 or MacJapaneseif let Some(pending) = self.pending_score {score += pending;self.pending_score = None;}score += SHIFT_JIS_EXTENSION_PENALTY;// Approximate boundaryif self.prev_byte < 0x87 {self.prev = LatinCj::Other;} else {if self.prev == LatinCj::AsciiLetter {score += CJK_LATIN_ADJACENCY_PENALTY;}self.prev = LatinCj::Cj;}} else if malformed_len == 1 && (b == 0xA0 || b >= 0xFD) {self.pending_score = None; // Just in casescore += SHIFT_JIS_SINGLE_BYTE_EXTENSION_PENALTY;self.prev = LatinCj::Other;} else {return None;}}DecoderResult::OutputFull => {unreachable!();}}self.prev_byte = b;}if last {let (result, _, _) = self.decoder.decode_to_utf16_without_replacement(b"", &mut dst, true);match result {DecoderResult::InputEmpty => {}DecoderResult::Malformed(_, _) => {return None;}DecoderResult::OutputFull => {unreachable!();}}}Some(score)}}struct EucJpCandidate {decoder: Decoder,non_ascii_seen: bool,half_width_katakana_state: HalfWidthKatakana,prev: LatinCj,prev_byte: u8,prev_prev_byte: u8,}impl EucJpCandidate {fn feed(&mut self, buffer: &[u8], last: bool) -> Option<i64> {let mut score = 0i64;let mut src = [0u8];let mut dst = [0u16; 2];for &b in buffer {src[0] = b;let (result, read, written) = self.decoder.decode_to_utf16_without_replacement(&src, &mut dst, false);if written > 0 {let half_width_katakana_state = self.half_width_katakana_state;self.half_width_katakana_state = HalfWidthKatakana::DakutenForbidden;let u = dst[0];if !self.non_ascii_seen && u >= 0x80 {self.non_ascii_seen = true;if u >= 0xFF61 && u <= 0xFF9F {// return None;}if u >= 0x3040 && u < 0x3100 {// Remove the kana advantage over initial Big5// hanzi.score += EUC_JP_INITIAL_KANA_PENALTY;}}if (u >= u16::from(b'a') && u <= u16::from(b'z'))|| (u >= u16::from(b'A') && u <= u16::from(b'Z')){if self.prev == LatinCj::Cj {score += CJK_LATIN_ADJACENCY_PENALTY;}self.prev = LatinCj::AsciiLetter;} else if u >= 0xFF61 && u <= 0xFF9F {score += HALF_WIDTH_KATAKANA_SCORE;if (u >= 0xFF76 && u <= 0xFF84) || u == 0xFF73 {self.half_width_katakana_state = HalfWidthKatakana::DakutenAllowed;} else if u >= 0xFF8A && u <= 0xFF8E {self.half_width_katakana_state =HalfWidthKatakana::DakutenOrHandakutenAllowed;} else if u == 0xFF9E {if half_width_katakana_state == HalfWidthKatakana::DakutenForbidden {score += IMPLAUSIBILITY_PENALTY;} else {score += HALF_WIDTH_KATAKANA_VOICING_SCORE;}} else if u == 0xFF9F {if half_width_katakana_state!= HalfWidthKatakana::DakutenOrHandakutenAllowed{score += IMPLAUSIBILITY_PENALTY;} else {score += HALF_WIDTH_KATAKANA_VOICING_SCORE;}}if self.prev == LatinCj::AsciiLetter {score += CJK_LATIN_ADJACENCY_PENALTY;}self.prev = LatinCj::Other;} else if (u >= 0x3041 && u <= 0x3093) || (u >= 0x30A1 && u <= 0x30F6) {match u {0x3090 // hiragana wi| 0x3091 // hiragana we| 0x30F0 // katakana wi| 0x30F1 // katakana we=> {// Remove advantage over Big5 Hanziscore += EUC_JP_SCORE_PER_NEAR_OBSOLETE_KANA;}_ => {score += EUC_JP_SCORE_PER_KANA;}}if self.prev == LatinCj::AsciiLetter {score += CJK_LATIN_ADJACENCY_PENALTY;}self.prev = LatinCj::Cj;} else if (u >= 0x3400 && u < 0xA000) || (u >= 0xF900 && u < 0xFB00) {if self.prev_prev_byte == 0x8F {score += EUC_JP_SCORE_PER_OTHER_KANJI;} else if self.prev_byte < 0xD0 {score += EUC_JP_SCORE_PER_LEVEL_1_KANJI;score += cjk_extra_score(u, &data::DETECTOR_DATA.frequent_kanji);} else {score += EUC_JP_SCORE_PER_LEVEL_2_KANJI;}if self.prev == LatinCj::AsciiLetter {score += CJK_LATIN_ADJACENCY_PENALTY;}self.prev = LatinCj::Cj;} else {match u {0x3000 // Distinct from Korean, space| 0x3001 // Distinct from Korean, enumeration comma| 0x3002 // Distinct from Korean, full stop| 0xFF08 // Distinct from Korean, parenthesis| 0xFF09 // Distinct from Korean, parenthesis=> {score += CJ_PUNCTUATION;}0..=0x7F => {}_ => {score += CJK_OTHER;}}self.prev = LatinCj::Other;}}match result {DecoderResult::InputEmpty => {assert_eq!(read, 1);}DecoderResult::Malformed(_, _) => {if b >= 0xA1&& b <= 0xFE&& self.prev_byte >= 0xA1&& self.prev_byte <= 0xFE&& ((self.prev_prev_byte != 0x8F&& !(self.prev_byte == 0xA8 && b >= 0xDF && b <= 0xE6)&& !(self.prev_byte == 0xAC && b >= 0xF4 && b <= 0xFC)&& !(self.prev_byte == 0xAD && b >= 0xD8 && b <= 0xDE))|| (self.prev_prev_byte == 0x8F&& self.prev_byte != 0xA2&& self.prev_byte != 0xA6&& self.prev_byte != 0xA7&& self.prev_byte != 0xA9&& self.prev_byte != 0xAA&& self.prev_byte != 0xAB&& self.prev_byte != 0xED&& !(self.prev_byte == 0xFE && b >= 0xF7))){score += EUC_JP_EXTENSION_PENALTY;if self.prev == LatinCj::AsciiLetter {score += CJK_LATIN_ADJACENCY_PENALTY;}self.prev = LatinCj::Cj;} else {return None;}}DecoderResult::OutputFull => {unreachable!();}}self.prev_prev_byte = self.prev_byte;self.prev_byte = b;}if last {let (result, _, _) = self.decoder.decode_to_utf16_without_replacement(b"", &mut dst, true);match result {DecoderResult::InputEmpty => {}DecoderResult::Malformed(_, _) => {return None;}DecoderResult::OutputFull => {unreachable!();}}}Some(score)}}struct Big5Candidate {decoder: Decoder,prev: LatinCj,prev_byte: u8,pending_score: Option<i64>,}impl Big5Candidate {fn maybe_set_as_pending(&mut self, s: i64) -> i64 {assert!(self.pending_score.is_none());if self.prev == LatinCj::Cj || !problematic_lead(self.prev_byte) {s} else {self.pending_score = Some(s);0}}fn feed(&mut self, buffer: &[u8], last: bool) -> Option<i64> {let mut score = 0i64;let mut src = [0u8];let mut dst = [0u16; 2];for &b in buffer {src[0] = b;let (result, read, written) = self.decoder.decode_to_utf16_without_replacement(&src, &mut dst, false);if written == 1 {let u = dst[0];if (u >= u16::from(b'a') && u <= u16::from(b'z'))|| (u >= u16::from(b'A') && u <= u16::from(b'Z')){self.pending_score = None; // Discard pending scoreif self.prev == LatinCj::Cj {score += CJK_LATIN_ADJACENCY_PENALTY;}self.prev = LatinCj::AsciiLetter;} else if (u >= 0x3400 && u < 0xA000) || (u >= 0xF900 && u < 0xFB00) {if let Some(pending) = self.pending_score {score += pending;self.pending_score = None;}match self.prev_byte {0xA4..=0xC6 => {score += self.maybe_set_as_pending(BIG5_SCORE_PER_LEVEL_1_HANZI);// score += cjk_extra_score(u, &data::DETECTOR_DATA.frequent_traditional);}_ => {score += self.maybe_set_as_pending(BIG5_SCORE_PER_OTHER_HANZI);}}if self.prev == LatinCj::AsciiLetter {score += CJK_LATIN_ADJACENCY_PENALTY;}self.prev = LatinCj::Cj;} else {match u {0x3000 // Distinct from Korean, space| 0x3001 // Distinct from Korean, enumeration comma| 0x3002 // Distinct from Korean, full stop| 0xFF08 // Distinct from Korean, parenthesis| 0xFF09 // Distinct from Korean, parenthesis| 0xFF01 // Distinct from Japanese, exclamation| 0xFF0C // Distinct from Japanese, comma| 0xFF1B // Distinct from Japanese, semicolon| 0xFF1F // Distinct from Japanese, question=> {if let Some(pending) = self.pending_score {score += pending;self.pending_score = None;}// Not really needed for CJK distinction// but let's give non-zero score for these// common byte pairs anyway.score += CJ_PUNCTUATION;}0..=0x7F => {self.pending_score = None; // Discard pending score}_ => {if let Some(pending) = self.pending_score {score += pending;self.pending_score = None;}score += CJK_OTHER;}}self.prev = LatinCj::Other;}} else if written == 2 {if let Some(pending) = self.pending_score {score += pending;self.pending_score = None;}if dst[0] == 0xCA || dst[0] == 0xEA {score += CJK_OTHER;self.prev = LatinCj::Other;} else {debug_assert!(dst[0] >= 0xD480 && dst[0] < 0xD880);score += self.maybe_set_as_pending(BIG5_SCORE_PER_OTHER_HANZI);if self.prev == LatinCj::AsciiLetter {score += CJK_LATIN_ADJACENCY_PENALTY;}self.prev = LatinCj::Cj;}}match result {DecoderResult::InputEmpty => {assert_eq!(read, 1);}DecoderResult::Malformed(malformed_len, _) => {if self.prev_byte >= 0x81&& self.prev_byte <= 0xFE&& ((b >= 0x40 && b <= 0x7E) || (b >= 0xA1 && b <= 0xFE)){// The byte pair is in the Big5 range but unmapped.// Treat as PUA to avoid rejecting Big5-UAO, etc.// We don't reprocess `b` even if ASCII, since it's// logically part of the pair.if let Some(pending) = self.pending_score {score += pending;self.pending_score = None;}score += BIG5_PUA_PENALTY;// Assume Hanzi semanticsif self.prev == LatinCj::AsciiLetter {score += CJK_LATIN_ADJACENCY_PENALTY;}self.prev = LatinCj::Cj;} else if (self.prev_byte == 0xA0|| self.prev_byte == 0xFD|| self.prev_byte == 0xFE)&& (b < 0x80 || b == 0xFF){// Mac OS Chinese Traditional single-byte that conflicts with code page Big5 lead byte// followed by ASCII or a non-conflicting single-byte extension.self.pending_score = None; // Just in casescore += BIG5_SINGLE_BYTE_EXTENSION_PENALTY;if (b >= b'a' && b <= b'z') || (b >= b'A' && b <= b'Z') {self.prev = LatinCj::AsciiLetter;} else if b == 0xFF {score += BIG5_SINGLE_BYTE_EXTENSION_PENALTY;self.prev = LatinCj::Other;} else {self.prev = LatinCj::Other;}} else if malformed_len == 1 && b == 0xFF {// Mac OS Chinese Traditional single-byte extension that doesn't conflict with lead bytesself.pending_score = None; // Just in casescore += BIG5_SINGLE_BYTE_EXTENSION_PENALTY;self.prev = LatinCj::Other;} else {return None;}}DecoderResult::OutputFull => {unreachable!();}}self.prev_byte = b;}if last {let (result, _, _) = self.decoder.decode_to_utf16_without_replacement(b"", &mut dst, true);match result {DecoderResult::InputEmpty => {}DecoderResult::Malformed(_, _) => {return None;}DecoderResult::OutputFull => {unreachable!();}}}Some(score)}}struct EucKrCandidate {decoder: Decoder,prev_byte: u8,prev_was_euc_range: bool,prev: LatinKorean,current_word_len: u64,pending_score: Option<i64>,}impl EucKrCandidate {fn maybe_set_as_pending(&mut self, s: i64) -> i64 {assert!(self.pending_score.is_none());if self.prev == LatinKorean::Hangul || !more_problematic_lead(self.prev_byte) {s} else {self.pending_score = Some(s);0}}fn feed(&mut self, buffer: &[u8], last: bool) -> Option<i64> {let mut score = 0i64;let mut src = [0u8];let mut dst = [0u16; 2];for &b in buffer {let in_euc_range = b >= 0xA1 && b <= 0xFE;src[0] = b;let (result, read, written) = self.decoder.decode_to_utf16_without_replacement(&src, &mut dst, false);if written > 0 {let u = dst[0];if (u >= u16::from(b'a') && u <= u16::from(b'z'))|| (u >= u16::from(b'A') && u <= u16::from(b'Z')){self.pending_score = None; // Discard pending scorematch self.prev {LatinKorean::Hangul | LatinKorean::Hanja => {score += CJK_LATIN_ADJACENCY_PENALTY;}_ => {}}self.prev = LatinKorean::AsciiLetter;self.current_word_len = 0;} else if u >= 0xAC00 && u <= 0xD7A3 {if let Some(pending) = self.pending_score {score += pending;self.pending_score = None;}if self.prev_was_euc_range && in_euc_range {score += EUC_KR_SCORE_PER_EUC_HANGUL;score += cjk_extra_score(u, &data::DETECTOR_DATA.frequent_hangul);} else {score += self.maybe_set_as_pending(EUC_KR_SCORE_PER_NON_EUC_HANGUL);}if self.prev == LatinKorean::AsciiLetter {score += CJK_LATIN_ADJACENCY_PENALTY;}self.prev = LatinKorean::Hangul;self.current_word_len += 1;if self.current_word_len > 5 {score += EUC_KR_LONG_WORD_PENALTY;}} else if (u >= 0x4E00 && u < 0xAC00) || (u >= 0xF900 && u <= 0xFA0B) {if let Some(pending) = self.pending_score {score += pending;self.pending_score = None;}score += EUC_KR_SCORE_PER_HANJA;match self.prev {LatinKorean::AsciiLetter => {score += CJK_LATIN_ADJACENCY_PENALTY;}LatinKorean::Hangul => {score += EUC_KR_HANJA_AFTER_HANGUL_PENALTY;}_ => {}}self.prev = LatinKorean::Hanja;self.current_word_len += 1;if self.current_word_len > 5 {score += EUC_KR_LONG_WORD_PENALTY;}} else {if u >= 0x80 {if let Some(pending) = self.pending_score {score += pending;self.pending_score = None;}score += CJK_OTHER;} else {self.pending_score = None; // Discard pending score}self.prev = LatinKorean::Other;self.current_word_len = 0;}}match result {DecoderResult::InputEmpty => {assert_eq!(read, 1);}DecoderResult::Malformed(malformed_len, _) => {if (self.prev_byte == 0xC9 || self.prev_byte == 0xFE) && b >= 0xA1 && b <= 0xFE{if let Some(pending) = self.pending_score {score += pending;self.pending_score = None;}// The byte pair is in code page 949 EUDC rangescore += EUC_KR_PUA_PENALTY;// Assume Hanja semanticsmatch self.prev {LatinKorean::AsciiLetter => {score += CJK_LATIN_ADJACENCY_PENALTY;}LatinKorean::Hangul => {score += EUC_KR_HANJA_AFTER_HANGUL_PENALTY;}_ => {}}self.prev = LatinKorean::Hanja;self.current_word_len += 1;if self.current_word_len > 5 {score += EUC_KR_LONG_WORD_PENALTY;}} else if (self.prev_byte == 0xA1|| (self.prev_byte >= 0xA3 && self.prev_byte <= 0xA8)|| (self.prev_byte >= 0xAA && self.prev_byte <= 0xAD))&& (b >= 0x7B && b <= 0x7D){if let Some(pending) = self.pending_score {score += pending;self.pending_score = None;}// MacKorean symbols in range not part of code page 949score += EUC_KR_MAC_KOREAN_PENALTY;self.prev = LatinKorean::Other;self.current_word_len = 0;} else if (self.prev_byte >= 0x81 && self.prev_byte <= 0x84)&& (b <= 0x80 || b == 0xFF){// MacKorean single-byte that conflicts with code page 949 lead byte// followed by ASCII or a non-conflicting single-byte extension.self.pending_score = None; // Just in casescore += EUC_KR_SINGLE_BYTE_EXTENSION_PENALTY;if (b >= b'a' && b <= b'z') || (b >= b'A' && b <= b'Z') {self.prev = LatinKorean::AsciiLetter;} else if b == 0x80 || b == 0xFF {score += EUC_KR_SINGLE_BYTE_EXTENSION_PENALTY;self.prev = LatinKorean::Other;} else {self.prev = LatinKorean::Other;}self.current_word_len = 0;} else if malformed_len == 1 && (b == 0x80 || b == 0xFF) {// MacKorean single-byte extensions that don't conflict with lead bytesself.pending_score = None; // Just in casescore += EUC_KR_SINGLE_BYTE_EXTENSION_PENALTY;self.prev = LatinKorean::Other;self.current_word_len = 0;} else {return None;}}DecoderResult::OutputFull => {unreachable!();}}self.prev_was_euc_range = in_euc_range;self.prev_byte = b;}if last {let (result, _, _) = self.decoder.decode_to_utf16_without_replacement(b"", &mut dst, true);match result {DecoderResult::InputEmpty => {}DecoderResult::Malformed(_, _) => {return None;}DecoderResult::OutputFull => {unreachable!();}}}Some(score)}}enum InnerCandidate {Latin(LatinCandidate),NonLatinCased(NonLatinCasedCandidate),Caseless(CaselessCandidate),ArabicFrench(ArabicFrenchCandidate),Logical(LogicalCandidate),Visual(VisualCandidate),Utf8(Utf8Candidate),Iso2022(Iso2022Candidate),Shift(ShiftJisCandidate),EucJp(EucJpCandidate),EucKr(EucKrCandidate),Big5(Big5Candidate),Gbk(GbkCandidate),}impl InnerCandidate {fn feed(&mut self, buffer: &[u8], last: bool) -> Option<i64> {match self {InnerCandidate::Latin(c) => {if let Some(new_score) = c.feed(buffer) {if last {// Treat EOF as space-likeif let Some(additional_score) = c.feed(b" ") {Some(new_score + additional_score)} else {None}} else {Some(new_score)}} else {None}}InnerCandidate::NonLatinCased(c) => {if let Some(new_score) = c.feed(buffer) {if last {// Treat EOF as space-likeif let Some(additional_score) = c.feed(b" ") {Some(new_score + additional_score)} else {None}} else {Some(new_score)}} else {None}}InnerCandidate::Caseless(c) => {if let Some(new_score) = c.feed(buffer) {if last {// Treat EOF as space-likeif let Some(additional_score) = c.feed(b" ") {Some(new_score + additional_score)} else {None}} else {Some(new_score)}} else {None}}InnerCandidate::ArabicFrench(c) => {if let Some(new_score) = c.feed(buffer) {if last {// Treat EOF as space-likeif let Some(additional_score) = c.feed(b" ") {Some(new_score + additional_score)} else {None}} else {Some(new_score)}} else {None}}InnerCandidate::Logical(c) => {if let Some(new_score) = c.feed(buffer) {if last {// Treat EOF as space-likeif let Some(additional_score) = c.feed(b" ") {Some(new_score + additional_score)} else {None}} else {Some(new_score)}} else {None}}InnerCandidate::Visual(c) => {if let Some(new_score) = c.feed(buffer) {if last {// Treat EOF as space-likeif let Some(additional_score) = c.feed(b" ") {Some(new_score + additional_score)} else {None}} else {Some(new_score)}} else {None}}InnerCandidate::Utf8(c) => c.feed(buffer, last),InnerCandidate::Iso2022(c) => c.feed(buffer, last),InnerCandidate::Shift(c) => c.feed(buffer, last),InnerCandidate::EucJp(c) => c.feed(buffer, last),InnerCandidate::EucKr(c) => c.feed(buffer, last),InnerCandidate::Big5(c) => c.feed(buffer, last),InnerCandidate::Gbk(c) => c.feed(buffer, last),}}}fn encoding_for_tld(tld: Tld) -> usize {match tld {Tld::CentralWindows | Tld::CentralCyrillic => EncodingDetector::CENTRAL_WINDOWS_INDEX,Tld::Cyrillic => EncodingDetector::CYRILLIC_WINDOWS_INDEX,Tld::Generic | Tld::Western | Tld::WesternCyrillic | Tld::WesternArabic | Tld::Eu => {EncodingDetector::WESTERN_INDEX}Tld::IcelandicFaroese => EncodingDetector::ICELANDIC_INDEX,Tld::Greek => EncodingDetector::GREEK_ISO_INDEX,Tld::TurkishAzeri => EncodingDetector::TURKISH_INDEX,Tld::Hebrew => EncodingDetector::LOGICAL_INDEX,Tld::Arabic => EncodingDetector::ARABIC_WINDOWS_INDEX,Tld::Baltic => EncodingDetector::BALTIC_WINDOWS_INDEX,Tld::Vietnamese => EncodingDetector::VIETNAMESE_INDEX,Tld::Thai => EncodingDetector::THAI_INDEX,Tld::Simplified | Tld::SimplifiedTraditional => EncodingDetector::GBK_INDEX,Tld::Traditional | Tld::TraditionalSimplified => EncodingDetector::BIG5_INDEX,Tld::Japanese => EncodingDetector::SHIFT_JIS_INDEX,Tld::Korean => EncodingDetector::EUC_KR_INDEX,Tld::CentralIso => EncodingDetector::CENTRAL_ISO_INDEX,}}fn encoding_is_native_to_tld(tld: Tld, encoding: usize) -> bool {match tld {Tld::CentralWindows => encoding == EncodingDetector::CENTRAL_WINDOWS_INDEX,Tld::Cyrillic => {encoding == EncodingDetector::CYRILLIC_WINDOWS_INDEX|| encoding == EncodingDetector::CYRILLIC_KOI_INDEX|| encoding == EncodingDetector::CYRILLIC_IBM_INDEX|| encoding == EncodingDetector::CYRILLIC_ISO_INDEX}Tld::Western => encoding == EncodingDetector::WESTERN_INDEX,Tld::Greek => {encoding == EncodingDetector::GREEK_WINDOWS_INDEX|| encoding == EncodingDetector::GREEK_ISO_INDEX}Tld::TurkishAzeri => encoding == EncodingDetector::TURKISH_INDEX,Tld::Hebrew => encoding == EncodingDetector::LOGICAL_INDEX,Tld::Arabic => {encoding == EncodingDetector::ARABIC_WINDOWS_INDEX|| encoding == EncodingDetector::ARABIC_ISO_INDEX}Tld::Baltic => {encoding == EncodingDetector::BALTIC_WINDOWS_INDEX|| encoding == EncodingDetector::BALTIC_ISO13_INDEX|| encoding == EncodingDetector::BALTIC_ISO4_INDEX}Tld::Vietnamese => encoding == EncodingDetector::VIETNAMESE_INDEX,Tld::Thai => encoding == EncodingDetector::THAI_INDEX,Tld::Simplified => encoding == EncodingDetector::GBK_INDEX,Tld::Traditional => encoding == EncodingDetector::BIG5_INDEX,Tld::Japanese => {encoding == EncodingDetector::SHIFT_JIS_INDEX|| encoding == EncodingDetector::EUC_JP_INDEX}Tld::Korean => encoding == EncodingDetector::EUC_KR_INDEX,Tld::SimplifiedTraditional | Tld::TraditionalSimplified => {encoding == EncodingDetector::GBK_INDEX || encoding == EncodingDetector::BIG5_INDEX}Tld::CentralIso => encoding == EncodingDetector::CENTRAL_ISO_INDEX,Tld::IcelandicFaroese => encoding == EncodingDetector::ICELANDIC_INDEX,Tld::WesternCyrillic => {encoding == EncodingDetector::WESTERN_INDEX|| encoding == EncodingDetector::CYRILLIC_WINDOWS_INDEX|| encoding == EncodingDetector::CYRILLIC_KOI_INDEX|| encoding == EncodingDetector::CYRILLIC_IBM_INDEX|| encoding == EncodingDetector::CYRILLIC_ISO_INDEX}Tld::CentralCyrillic => {encoding == EncodingDetector::CENTRAL_WINDOWS_INDEX|| encoding == EncodingDetector::CENTRAL_ISO_INDEX|| encoding == EncodingDetector::CYRILLIC_WINDOWS_INDEX|| encoding == EncodingDetector::CYRILLIC_KOI_INDEX|| encoding == EncodingDetector::CYRILLIC_IBM_INDEX|| encoding == EncodingDetector::CYRILLIC_ISO_INDEX}Tld::WesternArabic => {encoding == EncodingDetector::WESTERN_INDEX|| encoding == EncodingDetector::ARABIC_WINDOWS_INDEX|| encoding == EncodingDetector::ARABIC_ISO_INDEX}Tld::Eu => {encoding == EncodingDetector::WESTERN_INDEX|| encoding == EncodingDetector::ICELANDIC_INDEX|| encoding == EncodingDetector::CENTRAL_WINDOWS_INDEX|| encoding == EncodingDetector::CENTRAL_ISO_INDEX|| encoding == EncodingDetector::CYRILLIC_WINDOWS_INDEX|| encoding == EncodingDetector::CYRILLIC_KOI_INDEX|| encoding == EncodingDetector::CYRILLIC_IBM_INDEX|| encoding == EncodingDetector::CYRILLIC_ISO_INDEX|| encoding == EncodingDetector::GREEK_WINDOWS_INDEX|| encoding == EncodingDetector::GREEK_ISO_INDEX|| encoding == EncodingDetector::BALTIC_WINDOWS_INDEX|| encoding == EncodingDetector::BALTIC_ISO13_INDEX|| encoding == EncodingDetector::BALTIC_ISO4_INDEX}Tld::Generic => false,}}fn score_adjustment(score: i64, encoding: usize, tld: Tld) -> i64 {if score < 1 {return 0;}// This is the most ad hoc part of this library.let (divisor, constant) = match tld {Tld::Generic => {unreachable!();}Tld::CentralWindows | Tld::CentralIso => {match encoding {EncodingDetector::WESTERN_INDEX| EncodingDetector::ICELANDIC_INDEX| EncodingDetector::BALTIC_WINDOWS_INDEX| EncodingDetector::BALTIC_ISO4_INDEX| EncodingDetector::BALTIC_ISO13_INDEX| EncodingDetector::VIETNAMESE_INDEX| EncodingDetector::TURKISH_INDEX => {// XXX Tune this better instead of this kind of absolute.return score;}_ => (50, 60),}}Tld::Cyrillic => {match encoding {EncodingDetector::BIG5_INDEX| EncodingDetector::GBK_INDEX| EncodingDetector::EUC_JP_INDEX| EncodingDetector::CENTRAL_WINDOWS_INDEX| EncodingDetector::CENTRAL_ISO_INDEX| EncodingDetector::GREEK_WINDOWS_INDEX| EncodingDetector::GREEK_ISO_INDEX| EncodingDetector::VISUAL_INDEX| EncodingDetector::LOGICAL_INDEX| EncodingDetector::BALTIC_WINDOWS_INDEX| EncodingDetector::BALTIC_ISO4_INDEX| EncodingDetector::BALTIC_ISO13_INDEX| EncodingDetector::TURKISH_INDEX => {// XXX Tune this better instead of this kind of absolute.return score;}_ => (50, 60),}}Tld::Western | Tld::WesternCyrillic | Tld::WesternArabic => {match encoding {EncodingDetector::CENTRAL_WINDOWS_INDEX| EncodingDetector::CENTRAL_ISO_INDEX| EncodingDetector::BALTIC_WINDOWS_INDEX| EncodingDetector::BALTIC_ISO4_INDEX| EncodingDetector::BALTIC_ISO13_INDEX| EncodingDetector::TURKISH_INDEX| EncodingDetector::VIETNAMESE_INDEX => {// XXX Tune this better instead of this kind of absolute.return score;}_ => (50, 60),}}Tld::Greek => {match encoding {EncodingDetector::BIG5_INDEX| EncodingDetector::GBK_INDEX| EncodingDetector::EUC_JP_INDEX| EncodingDetector::CENTRAL_WINDOWS_INDEX| EncodingDetector::CENTRAL_ISO_INDEX| EncodingDetector::CYRILLIC_WINDOWS_INDEX| EncodingDetector::CYRILLIC_ISO_INDEX| EncodingDetector::CYRILLIC_KOI_INDEX| EncodingDetector::CYRILLIC_IBM_INDEX| EncodingDetector::VISUAL_INDEX| EncodingDetector::LOGICAL_INDEX| EncodingDetector::BALTIC_WINDOWS_INDEX| EncodingDetector::BALTIC_ISO4_INDEX| EncodingDetector::BALTIC_ISO13_INDEX| EncodingDetector::TURKISH_INDEX => {// XXX Tune this better instead of this kind of absolute.return score;}_ => (50, 60),}}Tld::TurkishAzeri => {match encoding {EncodingDetector::CENTRAL_WINDOWS_INDEX| EncodingDetector::CENTRAL_ISO_INDEX| EncodingDetector::BALTIC_WINDOWS_INDEX| EncodingDetector::BALTIC_ISO4_INDEX| EncodingDetector::BALTIC_ISO13_INDEX| EncodingDetector::VIETNAMESE_INDEX| EncodingDetector::ICELANDIC_INDEX => {// XXX Tune this better instead of this kind of absolute.return score;}_ => (50, 60),}}Tld::Hebrew => {match encoding {EncodingDetector::CENTRAL_WINDOWS_INDEX| EncodingDetector::CENTRAL_ISO_INDEX| EncodingDetector::CYRILLIC_WINDOWS_INDEX| EncodingDetector::CYRILLIC_ISO_INDEX| EncodingDetector::CYRILLIC_KOI_INDEX| EncodingDetector::CYRILLIC_IBM_INDEX| EncodingDetector::GREEK_WINDOWS_INDEX| EncodingDetector::GREEK_ISO_INDEX| EncodingDetector::BALTIC_WINDOWS_INDEX| EncodingDetector::BALTIC_ISO4_INDEX| EncodingDetector::BALTIC_ISO13_INDEX| EncodingDetector::VIETNAMESE_INDEX| EncodingDetector::TURKISH_INDEX => {// XXX Tune this better instead of this kind of absolute.return score;}_ => (50, 60),}}Tld::Arabic => {match encoding {EncodingDetector::BIG5_INDEX| EncodingDetector::GBK_INDEX| EncodingDetector::EUC_JP_INDEX| EncodingDetector::EUC_KR_INDEX| EncodingDetector::CENTRAL_WINDOWS_INDEX| EncodingDetector::CENTRAL_ISO_INDEX| EncodingDetector::CYRILLIC_WINDOWS_INDEX| EncodingDetector::CYRILLIC_ISO_INDEX| EncodingDetector::CYRILLIC_KOI_INDEX| EncodingDetector::CYRILLIC_IBM_INDEX| EncodingDetector::GREEK_WINDOWS_INDEX| EncodingDetector::GREEK_ISO_INDEX| EncodingDetector::VISUAL_INDEX| EncodingDetector::LOGICAL_INDEX| EncodingDetector::BALTIC_WINDOWS_INDEX| EncodingDetector::BALTIC_ISO4_INDEX| EncodingDetector::BALTIC_ISO13_INDEX| EncodingDetector::VIETNAMESE_INDEX| EncodingDetector::TURKISH_INDEX => {// XXX Tune this better instead of this kind of absolute.return score;}_ => (50, 60),}}Tld::Baltic => {match encoding {EncodingDetector::CENTRAL_WINDOWS_INDEX| EncodingDetector::CENTRAL_ISO_INDEX| EncodingDetector::ICELANDIC_INDEX| EncodingDetector::TURKISH_INDEX| EncodingDetector::VIETNAMESE_INDEX => {// XXX Tune this better instead of this kind of absolute.return score;}_ => (50, 60),}}Tld::Vietnamese => {match encoding {EncodingDetector::CENTRAL_WINDOWS_INDEX| EncodingDetector::CENTRAL_ISO_INDEX| EncodingDetector::BALTIC_WINDOWS_INDEX| EncodingDetector::BALTIC_ISO4_INDEX| EncodingDetector::BALTIC_ISO13_INDEX| EncodingDetector::TURKISH_INDEX| EncodingDetector::ICELANDIC_INDEX => {// XXX Tune this better instead of this kind of absolute.return score;}_ => (50, 60),}}Tld::Thai => {match encoding {EncodingDetector::BIG5_INDEX| EncodingDetector::GBK_INDEX| EncodingDetector::EUC_JP_INDEX| EncodingDetector::EUC_KR_INDEX| EncodingDetector::SHIFT_JIS_INDEX| EncodingDetector::CENTRAL_WINDOWS_INDEX| EncodingDetector::CENTRAL_ISO_INDEX| EncodingDetector::CYRILLIC_WINDOWS_INDEX| EncodingDetector::CYRILLIC_ISO_INDEX| EncodingDetector::CYRILLIC_KOI_INDEX| EncodingDetector::CYRILLIC_IBM_INDEX| EncodingDetector::GREEK_WINDOWS_INDEX| EncodingDetector::GREEK_ISO_INDEX| EncodingDetector::ARABIC_WINDOWS_INDEX| EncodingDetector::ARABIC_ISO_INDEX| EncodingDetector::VISUAL_INDEX| EncodingDetector::LOGICAL_INDEX| EncodingDetector::BALTIC_WINDOWS_INDEX| EncodingDetector::BALTIC_ISO4_INDEX| EncodingDetector::BALTIC_ISO13_INDEX| EncodingDetector::TURKISH_INDEX => {// XXX Tune this better instead of this kind of absolute.return score;}_ => (50, 60),}}Tld::Simplified| Tld::Traditional| Tld::TraditionalSimplified| Tld::SimplifiedTraditional| Tld::Japanese| Tld::Korean => {// If TLD default is valid, everything else scores zeroreturn score;}Tld::IcelandicFaroese => {match encoding {EncodingDetector::CENTRAL_WINDOWS_INDEX| EncodingDetector::CENTRAL_ISO_INDEX| EncodingDetector::BALTIC_WINDOWS_INDEX| EncodingDetector::BALTIC_ISO4_INDEX| EncodingDetector::BALTIC_ISO13_INDEX| EncodingDetector::TURKISH_INDEX| EncodingDetector::VIETNAMESE_INDEX => {// XXX Tune this better instead of this kind of absolute.return score;}_ => (50, 60),}}Tld::CentralCyrillic => {match encoding {EncodingDetector::BIG5_INDEX| EncodingDetector::GBK_INDEX| EncodingDetector::EUC_JP_INDEX| EncodingDetector::GREEK_WINDOWS_INDEX| EncodingDetector::GREEK_ISO_INDEX| EncodingDetector::VISUAL_INDEX| EncodingDetector::LOGICAL_INDEX| EncodingDetector::BALTIC_WINDOWS_INDEX| EncodingDetector::BALTIC_ISO4_INDEX| EncodingDetector::BALTIC_ISO13_INDEX| EncodingDetector::TURKISH_INDEX => {// XXX Tune this better instead of this kind of absolute.return score;}_ => (50, 60),}}Tld::Eu => {match encoding {EncodingDetector::BIG5_INDEX| EncodingDetector::GBK_INDEX| EncodingDetector::EUC_JP_INDEX| EncodingDetector::TURKISH_INDEX| EncodingDetector::VIETNAMESE_INDEX => {// XXX Tune this better instead of this kind of absolute.return score;}_ => (50, 60),}}};(score / divisor) + constant}cfg_if::cfg_if! {if #[cfg(feature = "multithreading")] {#[repr(align(64))] // Align to cache lines to avoid false sharing in the Rayon casestruct Candidate {inner: InnerCandidate,score: Option<i64>,}} else {struct Candidate {inner: InnerCandidate,score: Option<i64>,}}}impl Candidate {fn feed(&mut self, buffer: &[u8], last: bool) {if let Some(old_score) = self.score {if let Some(new_score) = self.inner.feed(buffer, last) {self.score = Some(old_score + new_score);} else {self.score = None;}}}#[cfg(feature = "multithreading")]fn qualified(&self) -> bool {!self.score.is_none()}fn new_latin(data: &'static SingleByteData) -> Self {Candidate {inner: InnerCandidate::Latin(LatinCandidate::new(data)),score: Some(0),}}fn new_non_latin_cased(data: &'static SingleByteData) -> Self {Candidate {inner: InnerCandidate::NonLatinCased(NonLatinCasedCandidate::new(data)),score: Some(0),}}fn new_caseless(data: &'static SingleByteData) -> Self {Candidate {inner: InnerCandidate::Caseless(CaselessCandidate::new(data)),score: Some(0),}}fn new_arabic_french(data: &'static SingleByteData) -> Self {Candidate {inner: InnerCandidate::ArabicFrench(ArabicFrenchCandidate::new(data)),score: Some(0),}}fn new_logical(data: &'static SingleByteData) -> Self {Candidate {inner: InnerCandidate::Logical(LogicalCandidate::new(data)),score: Some(0),}}fn new_visual(data: &'static SingleByteData) -> Self {Candidate {inner: InnerCandidate::Visual(VisualCandidate::new(data)),score: Some(0),}}fn new_utf_8() -> Self {Candidate {inner: InnerCandidate::Utf8(Utf8Candidate {decoder: UTF_8.new_decoder_without_bom_handling(),}),score: Some(0),}}fn new_iso_2022_jp() -> Self {Candidate {inner: InnerCandidate::Iso2022(Iso2022Candidate {decoder: ISO_2022_JP.new_decoder_without_bom_handling(),}),score: Some(0),}}fn new_shift_jis() -> Self {Candidate {inner: InnerCandidate::Shift(ShiftJisCandidate {decoder: SHIFT_JIS.new_decoder_without_bom_handling(),half_width_katakana_seen: false,half_width_katakana_state: HalfWidthKatakana::DakutenForbidden,prev: LatinCj::Other,prev_byte: 0,pending_score: None,}),score: Some(0),}}fn new_euc_jp() -> Self {Candidate {inner: InnerCandidate::EucJp(EucJpCandidate {decoder: EUC_JP.new_decoder_without_bom_handling(),non_ascii_seen: false,half_width_katakana_state: HalfWidthKatakana::DakutenForbidden,prev: LatinCj::Other,prev_byte: 0,prev_prev_byte: 0,}),score: Some(0),}}fn new_euc_kr() -> Self {Candidate {inner: InnerCandidate::EucKr(EucKrCandidate {decoder: EUC_KR.new_decoder_without_bom_handling(),prev_byte: 0,prev_was_euc_range: false,prev: LatinKorean::Other,current_word_len: 0,pending_score: None,}),score: Some(0),}}fn new_big5() -> Self {Candidate {inner: InnerCandidate::Big5(Big5Candidate {decoder: BIG5.new_decoder_without_bom_handling(),prev: LatinCj::Other,prev_byte: 0,pending_score: None,}),score: Some(0),}}fn new_gbk() -> Self {Candidate {inner: InnerCandidate::Gbk(GbkCandidate {decoder: GBK.new_decoder_without_bom_handling(),prev: LatinCj::Other,prev_byte: 0,pending_score: None,}),score: Some(0),}}fn score(&self, encoding: usize, tld: Tld, expectation_is_valid: bool) -> Option<i64> {match &self.inner {InnerCandidate::NonLatinCased(c) => {if c.longest_word < 2 {return None;}}InnerCandidate::Caseless(c) => {if c.longest_word < 2 && !encoding_is_native_to_tld(tld, encoding) {return None;}}InnerCandidate::ArabicFrench(c) => {if c.longest_word < 2 && !encoding_is_native_to_tld(tld, encoding) {return None;}}InnerCandidate::Logical(c) => {if c.longest_word < 2 && !encoding_is_native_to_tld(tld, encoding) {return None;}}InnerCandidate::Visual(c) => {if c.longest_word < 2 && !encoding_is_native_to_tld(tld, encoding) {return None;}}_ => {}}if tld == Tld::Generic {return self.score;}if let Some(score) = self.score {if encoding == encoding_for_tld(tld) {return Some(score + 1);}if encoding_is_native_to_tld(tld, encoding) {return Some(score);}if expectation_is_valid {return Some(score - score_adjustment(score, encoding, tld));}// If expectation is no longer valid, fall back to// generic behavior.// XXX Flipped Chinese and Centralreturn Some(score);}None}fn plausible_punctuation(&self) -> u64 {match &self.inner {InnerCandidate::Logical(c) => {return c.plausible_punctuation;}InnerCandidate::Visual(c) => {return c.plausible_punctuation;}_ => {unreachable!();}}}fn encoding(&self) -> &'static Encoding {match &self.inner {InnerCandidate::Latin(c) => {return c.data.encoding;}InnerCandidate::NonLatinCased(c) => {return c.data.encoding;}InnerCandidate::Caseless(c) => {return c.data.encoding;}InnerCandidate::ArabicFrench(c) => {return c.data.encoding;}InnerCandidate::Logical(c) => {return c.data.encoding;}InnerCandidate::Visual(c) => {return c.data.encoding;}InnerCandidate::Shift(_) => {return SHIFT_JIS;}InnerCandidate::EucJp(_) => {return EUC_JP;}InnerCandidate::Big5(_) => {return BIG5;}InnerCandidate::EucKr(_) => {return EUC_KR;}InnerCandidate::Gbk(_) => {return GBK;}InnerCandidate::Utf8(_) => {return UTF_8;}InnerCandidate::Iso2022(_) => {return ISO_2022_JP;}}}}// LLVM doesn't autovectorize this properly for SSE2, so let's help manually.cfg_if::cfg_if! {if #[cfg(target_feature = "sse2")] {fn count_non_ascii(buffer: &[u8]) -> u64 {let mut count = 0;let (prefix, simd, suffix) = unsafe { buffer.align_to::<__m128i>() };for &b in prefix {if b >= 0x80 {count += 1;}}for &s in simd {count += unsafe {_mm_movemask_epi8(s)}.count_ones() as u64;}for &b in suffix {if b >= 0x80 {count += 1;}}count}} else {fn count_non_ascii(buffer: &[u8]) -> u64 {let mut count = 0;for &b in buffer {if b >= 0x80 {count += 1;}}count}}}#[derive(Clone, Copy)]enum BeforeNonAscii {None,One([u8; 1]),Two([u8; 2]),}impl BeforeNonAscii {fn as_slice(&self) -> &[u8] {match self {BeforeNonAscii::None => b"",BeforeNonAscii::One(arr) => &arr[..],BeforeNonAscii::Two(arr) => &arr[..],}}fn push(&mut self, buffer: &[u8]) {let len = buffer.len();if len >= 2 {let arr = [buffer[len - 2], buffer[len - 1]];*self = BeforeNonAscii::Two(arr);} else if len == 1 {match self {BeforeNonAscii::None => {let arr = [buffer[0]];*self = BeforeNonAscii::One(arr);}BeforeNonAscii::One(first) => {let arr = [first[0], buffer[0]];*self = BeforeNonAscii::Two(arr);}BeforeNonAscii::Two(first) => {let arr = [first[1], buffer[0]];*self = BeforeNonAscii::Two(arr);}}}}}/// A Web browser-oriented detector for guessing what character/// encoding a stream of bytes is encoded in.////// The bytes are fed to the detector incrementally using the `feed`/// method. The current guess of the detector can be queried using/// the `guess` method. The guessing parameters are arguments to the/// `guess` method rather than arguments to the constructor in order/// to enable the application to check if the arguments affect the/// guessing outcome. (The specific use case is to disable UI for/// re-running the detector with UTF-8 allowed and the top-level/// domain name ignored if those arguments don't change the guess.)pub struct EncodingDetector {candidates: [Candidate; 27],non_ascii_seen: u64,// We need to feed up to two bytes of context before non-ASCII// thanks to Spanish n.º.last_before_non_ascii: BeforeNonAscii,esc_seen: bool,closed: bool,}impl EncodingDetector {cfg_if::cfg_if! {if #[cfg(feature = "multithreading")] {fn feed_impl(&mut self, buffer: &[u8], last: bool) {if buffer.len() < 10 {self.candidates.iter_mut().for_each(|candidate| candidate.feed(buffer, last));self.non_ascii_seen += count_non_ascii(buffer);return;}// Collect only qualified candidates to avoid Rayon// performing thread synchronization only to bail// out immediately when trying a disqualified// candidate.let mut qualified = ArrayVec::<[_; 27]>::new();for candidate in self.candidates.iter_mut() {if candidate.qualified() {qualified.push(candidate);}}let (_, non_ascii) = rayon::join(|| qualified.par_iter_mut().for_each(|candidate| candidate.feed(buffer, last)),|| count_non_ascii(buffer));self.non_ascii_seen += non_ascii;}} else {fn feed_impl(&mut self, buffer: &[u8], last: bool) {self.candidates.iter_mut().for_each(|candidate| candidate.feed(buffer, last));self.non_ascii_seen += count_non_ascii(buffer);}}}/// Inform the detector of a chunk of input.////// The byte stream is represented as a sequence of calls to this/// method such that the concatenation of the arguments to this/// method form the byte stream. It does not matter how the application/// chooses to chunk the stream. It is OK to call this method with/// a zero-length byte slice.////// The end of the stream is indicated by calling this method with/// `last` set to `true`. In that case, the end of the stream is/// considered to occur after the last byte of the `buffer` (which/// may be zero-length) passed in the same call. Once this method/// has been called with `last` set to `true` this method must not/// be called again.////// If you want to perform detection on just the prefix of a longer/// stream, do not pass `last=true` after the prefix if the stream/// actually still continues.////// Returns `true` if after processing `buffer` the stream has/// contained at least one non-ASCII byte and `false` if only/// ASCII has been seen so far.////// # Panics////// If this method has previously been called with `last` set to `true`.pub fn feed(&mut self, buffer: &[u8], last: bool) -> bool {assert!(!self.closed,"Must not feed again after feeding with last equaling true.");if last {self.closed = true;}let start = if self.non_ascii_seen == 0 && !self.esc_seen {let up_to = Encoding::ascii_valid_up_to(buffer);let start = if let Some(escape) = memchr::memchr(0x1B, &buffer[..up_to]) {self.esc_seen = true;escape} else {up_to};if start == buffer.len() {self.last_before_non_ascii.push(buffer);return self.non_ascii_seen != 0;}if start == 0 || start == 1 {let last_before = self.last_before_non_ascii;self.last_before_non_ascii = BeforeNonAscii::None;self.feed_impl(last_before.as_slice(), false);0} else {start - 2}} else {0};self.feed_impl(&buffer[start..], last);self.non_ascii_seen != 0}/// Guess the encoding given the bytes pushed to the detector so far/// (via `feed()`), the top-level domain name from which the bytes were/// loaded, and an indication of whether to consider UTF-8 as a permissible/// guess.////// The `tld` argument takes the rightmost DNS label of the hostname of the/// host the stream was loaded from in lower-case ASCII form. That is, if/// the label is an internationalized top-level domain name, it must be/// provided in its Punycode form. If the TLD that the stream was loaded/// from is unavalable, `None` may be passed instead, which is equivalent/// to passing `Some(b"com")`.////// If the `allow_utf8` argument is set to `false`, the return value of/// this method won't be `encoding_rs::UTF_8`. When performing detection/// on `text/html` on non-`file:` URLs, Web browsers must pass `false`,/// unless the user has taken a specific contextual action to request an/// override. This way, Web developers cannot start depending on UTF-8/// detection. Such reliance would make the Web Platform more brittle.////// Returns the guessed encoding.////// # Panics////// If `tld` contains non-ASCII, period, or upper-case letters. (The panic/// condition is intentionally limited to signs of failing to extract the/// label correctly, failing to provide it in its Punycode form, and failure/// to lower-case it. Full DNS label validation is intentionally not performed/// to avoid panics when the reality doesn't match the specs.)pub fn guess(&self, tld: Option<&[u8]>, allow_utf8: bool) -> &'static Encoding {self.guess_assess(tld, allow_utf8).0}/// Same as `guess()`, but also returns a Boolean indicating/// whether the guessed encoding had a higher score than at least/// one other candidate. If this method returns `false`, the/// guessed encoding is likely to be wrong.pub fn guess_assess(&self, tld: Option<&[u8]>, allow_utf8: bool) -> (&'static Encoding, bool) {let mut tld_type = tld.map_or(Tld::Generic, |tld| {assert!(!contains_upper_case_period_or_non_ascii(tld));classify_tld(tld)});if self.non_ascii_seen == 0 && self.esc_seen {if self.candidates[Self::ISO_2022_JP_INDEX].score.is_some() {return (ISO_2022_JP, true);}}if self.candidates[Self::UTF_8_INDEX].score.is_some() {if allow_utf8 {return (UTF_8, true);}// Various test cases that prohibit UTF-8 detection want to// see windows-1252 specifically. These tests run on generic// domains. However, if we returned windows-1252 on// some non-generic domains, we'd cause reloads.return (self.candidates[encoding_for_tld(tld_type)].encoding(), true);}let mut encoding = self.candidates[encoding_for_tld(tld_type)].encoding();let mut max = 0i64;let mut expectation_is_valid = false;if tld_type != Tld::Generic {for (i, candidate) in self.candidates.iter().enumerate().skip(Self::FIRST_NORMAL) {if encoding_is_native_to_tld(tld_type, i) && candidate.score.is_some() {expectation_is_valid = true;break;}}}if !expectation_is_valid {// Flip Chinese and Central aroundmatch tld_type {Tld::Simplified => {if self.candidates[Self::BIG5_INDEX].score.is_some() {tld_type = Tld::Traditional;expectation_is_valid = true;}}Tld::Traditional => {if self.candidates[Self::GBK_INDEX].score.is_some() {tld_type = Tld::Simplified;expectation_is_valid = true;}}Tld::CentralWindows => {if self.candidates[Self::CENTRAL_ISO_INDEX].score.is_some() {tld_type = Tld::CentralIso;expectation_is_valid = true;}}Tld::CentralIso => {if self.candidates[Self::CENTRAL_WINDOWS_INDEX].score.is_some() {tld_type = Tld::CentralWindows;expectation_is_valid = true;}}_ => {}}}for (i, candidate) in self.candidates.iter().enumerate().skip(Self::FIRST_NORMAL) {if let Some(score) = candidate.score(i, tld_type, expectation_is_valid) {if score > max {max = score;encoding = candidate.encoding();}}}let visual = &self.candidates[Self::VISUAL_INDEX];if let Some(visual_score) = visual.score(Self::VISUAL_INDEX, tld_type, expectation_is_valid){if (visual_score > max || encoding == WINDOWS_1255)&& visual.plausible_punctuation()> self.candidates[Self::LOGICAL_INDEX].plausible_punctuation(){// max = visual_score;encoding = ISO_8859_8;}}(encoding, max > 0)}// XXX Test-only API#[cfg(feature = "testing-only-no-semver-guarantees-do-not-use")]pub fn find_score(&self, encoding: &'static Encoding) -> Option<i64> {let mut tld_type = Tld::Generic;let mut expectation_is_valid = false;if tld_type != Tld::Generic {for (i, candidate) in self.candidates.iter().enumerate().skip(Self::FIRST_NORMAL) {if encoding_is_native_to_tld(tld_type, i) && candidate.score.is_some() {expectation_is_valid = true;break;}}}if !expectation_is_valid {// Flip Chinese and Central aroundmatch tld_type {Tld::Simplified => {if self.candidates[Self::BIG5_INDEX].score.is_some() {tld_type = Tld::Traditional;expectation_is_valid = true;}}Tld::Traditional => {if self.candidates[Self::GBK_INDEX].score.is_some() {tld_type = Tld::Simplified;expectation_is_valid = true;}}Tld::CentralWindows => {if self.candidates[Self::CENTRAL_ISO_INDEX].score.is_some() {tld_type = Tld::CentralIso;expectation_is_valid = true;}}Tld::CentralIso => {if self.candidates[Self::CENTRAL_WINDOWS_INDEX].score.is_some() {tld_type = Tld::CentralWindows;expectation_is_valid = true;}}_ => {}}}for (i, candidate) in self.candidates.iter().enumerate() {if encoding == candidate.encoding() {return candidate.score(i, tld_type, expectation_is_valid);}}Some(0)}const FIRST_NORMAL: usize = 3;const UTF_8_INDEX: usize = 0;const ISO_2022_JP_INDEX: usize = 1;const VISUAL_INDEX: usize = 2;const GBK_INDEX: usize = 3;const EUC_JP_INDEX: usize = 4;const EUC_KR_INDEX: usize = 5;const SHIFT_JIS_INDEX: usize = 6;const BIG5_INDEX: usize = 7;const WESTERN_INDEX: usize = 8;const CYRILLIC_WINDOWS_INDEX: usize = 9;const CENTRAL_WINDOWS_INDEX: usize = 10;const CENTRAL_ISO_INDEX: usize = 11;const ARABIC_WINDOWS_INDEX: usize = 12;const ICELANDIC_INDEX: usize = 13;const TURKISH_INDEX: usize = 14;const THAI_INDEX: usize = 15;const LOGICAL_INDEX: usize = 16;const GREEK_WINDOWS_INDEX: usize = 17;const GREEK_ISO_INDEX: usize = 18;const BALTIC_WINDOWS_INDEX: usize = 19;const BALTIC_ISO13_INDEX: usize = 20;const CYRILLIC_KOI_INDEX: usize = 21;const CYRILLIC_IBM_INDEX: usize = 22;const ARABIC_ISO_INDEX: usize = 23;const VIETNAMESE_INDEX: usize = 24;const BALTIC_ISO4_INDEX: usize = 25;const CYRILLIC_ISO_INDEX: usize = 26;/// Creates a new instance of the detector.pub fn new() -> Self {EncodingDetector {candidates: [Candidate::new_utf_8(), // 0Candidate::new_iso_2022_jp(), // 1Candidate::new_visual(&SINGLE_BYTE_DATA[ISO_8859_8_INDEX]), // 2Candidate::new_gbk(), // 3Candidate::new_euc_jp(), // 4Candidate::new_euc_kr(), // 5Candidate::new_shift_jis(), // 6Candidate::new_big5(), // 7Candidate::new_latin(&SINGLE_BYTE_DATA[WINDOWS_1252_INDEX]), // 8Candidate::new_non_latin_cased(&SINGLE_BYTE_DATA[WINDOWS_1251_INDEX]), // 9Candidate::new_latin(&SINGLE_BYTE_DATA[WINDOWS_1250_INDEX]), // 10Candidate::new_latin(&SINGLE_BYTE_DATA[ISO_8859_2_INDEX]), // 11Candidate::new_arabic_french(&SINGLE_BYTE_DATA[WINDOWS_1256_INDEX]), // 12Candidate::new_latin(&SINGLE_BYTE_DATA[WINDOWS_1252_ICELANDIC_INDEX]), // 13Candidate::new_latin(&SINGLE_BYTE_DATA[WINDOWS_1254_INDEX]), // 14Candidate::new_caseless(&SINGLE_BYTE_DATA[WINDOWS_874_INDEX]), // 15Candidate::new_logical(&SINGLE_BYTE_DATA[WINDOWS_1255_INDEX]), // 16Candidate::new_non_latin_cased(&SINGLE_BYTE_DATA[WINDOWS_1253_INDEX]), // 17Candidate::new_non_latin_cased(&SINGLE_BYTE_DATA[ISO_8859_7_INDEX]), // 18Candidate::new_latin(&SINGLE_BYTE_DATA[WINDOWS_1257_INDEX]), // 19Candidate::new_latin(&SINGLE_BYTE_DATA[ISO_8859_13_INDEX]), // 20Candidate::new_non_latin_cased(&SINGLE_BYTE_DATA[KOI8_U_INDEX]), // 21Candidate::new_non_latin_cased(&SINGLE_BYTE_DATA[IBM866_INDEX]), // 22Candidate::new_caseless(&SINGLE_BYTE_DATA[ISO_8859_6_INDEX]), // 23Candidate::new_latin(&SINGLE_BYTE_DATA[WINDOWS_1258_INDEX]), // 24Candidate::new_latin(&SINGLE_BYTE_DATA[ISO_8859_4_INDEX]), // 25Candidate::new_non_latin_cased(&SINGLE_BYTE_DATA[ISO_8859_5_INDEX]), // 26],non_ascii_seen: 0,last_before_non_ascii: BeforeNonAscii::None,esc_seen: false,closed: false,}}}#[cfg(test)]mod tests {use super::*;use detone::IterDecomposeVietnamese;use encoding_rs::IBM866;use encoding_rs::ISO_8859_2;use encoding_rs::ISO_8859_4;use encoding_rs::ISO_8859_5;use encoding_rs::ISO_8859_6;use encoding_rs::ISO_8859_7;use encoding_rs::KOI8_U;use encoding_rs::WINDOWS_1250;use encoding_rs::WINDOWS_1251;use encoding_rs::WINDOWS_1252;use encoding_rs::WINDOWS_1253;use encoding_rs::WINDOWS_1254;use encoding_rs::WINDOWS_1256;use encoding_rs::WINDOWS_1257;use encoding_rs::WINDOWS_1258;use encoding_rs::WINDOWS_874;fn check_bytes(bytes: &[u8], encoding: &'static Encoding) {let mut det = EncodingDetector::new();det.feed(bytes, true);let enc = det.guess(None, false);let (decoded, _) = enc.decode_without_bom_handling(bytes);println!("{:?}", decoded);assert_eq!(enc, encoding);}fn check(input: &str, encoding: &'static Encoding) {let orthographic;let (bytes, _, _) = if encoding == WINDOWS_1258 {orthographic = input.chars().decompose_vietnamese_tones(true).collect::<String>();encoding.encode(&orthographic)} else {encoding.encode(input)};check_bytes(&bytes, encoding);}#[test]fn test_i_apostrophe() {let mut det = EncodingDetector::new();det.feed(b"I\x92", true);let enc = det.guess(None, false);assert_eq!(enc, WINDOWS_1252);}#[test]fn test_streaming_numero_one_by_one() {let mut det = EncodingDetector::new();det.feed(b"n", false);det.feed(b".", false);det.feed(b"\xBA", false);det.feed(b"1", true);let enc = det.guess(None, false);assert_eq!(enc, WINDOWS_1252);}#[test]fn test_streaming_numero_two_together() {let mut det = EncodingDetector::new();det.feed(b"n.", false);det.feed(b"\xBA", false);det.feed(b"1", true);let enc = det.guess(None, false);assert_eq!(enc, WINDOWS_1252);}#[test]fn test_streaming_numero_one_by_one_extra_before() {let mut det = EncodingDetector::new();det.feed(b" n", false);det.feed(b".", false);det.feed(b"\xBA", false);det.feed(b"1", true);let enc = det.guess(None, false);assert_eq!(enc, WINDOWS_1252);}#[test]fn test_streaming_numero_one_before() {let mut det = EncodingDetector::new();det.feed(b"n", false);det.feed(b".\xBA", false);det.feed(b"1", true);let enc = det.guess(None, false);assert_eq!(enc, WINDOWS_1252);}#[test]fn test_streaming_numero_longer_first_buffer() {let mut det = EncodingDetector::new();det.feed(b"rrn.", false);det.feed(b"\xBA", false);det.feed(b"1", true);let enc = det.guess(None, false);assert_eq!(enc, WINDOWS_1252);}#[test]fn test_empty() {let mut det = EncodingDetector::new();let seen_non_ascii = det.feed(b"", true);let enc = det.guess(None, false);assert_eq!(enc, WINDOWS_1252);assert!(!seen_non_ascii);}#[test]fn test_fi() {check("Ääni", WINDOWS_1252);}#[test]fn test_fi_bis() {check("Tämä", WINDOWS_1252);}#[test]fn test_pt() {check("Este é um teste de codificação de caracteres.",WINDOWS_1252,);}#[test]fn test_is() {check("Þetta er kóðunarpróf á staf. Fyrir sum tungumál sem nota latneska stafi þurfum við meira inntak til að taka ákvörðunina.", WINDOWS_1252);}#[test]fn test_ru_short() {check("Русский", WINDOWS_1251);}#[test]fn test_ru() {check("Это тест кодировки символов.", WINDOWS_1251);}#[test]fn test_ru_iso() {check("Это тест кодировки символов.", ISO_8859_5);}#[test]fn test_ru_ibm() {check("Это тест кодировки символов.", IBM866);}#[test]fn test_ru_koi() {check("Это тест кодировки символов.", KOI8_U);}#[test]fn test_uk() {check("Це тест на кодування символів.", WINDOWS_1251);}#[test]fn test_uk_koi() {check("Це тест на кодування символів.", KOI8_U);}#[test]fn test_el_short() {check("Ελληνικά", WINDOWS_1253);}#[test]fn test_el() {check("Πρόκειται για δοκιμή κωδικοποίησης χαρακτήρων: Άρης",WINDOWS_1253,);}#[test]fn test_el_iso() {check("Πρόκειται για δοκιμή κωδικοποίησης χαρακτήρων: Άρης",ISO_8859_7,);}#[test]fn test_de() {check("Straße", WINDOWS_1252);}#[test]fn test_he() {check("\u{5E2}\u{5D1}\u{5E8}\u{5D9}\u{5EA}", WINDOWS_1255);}#[test]fn test_2022() {check("日本語", ISO_2022_JP);}#[test]fn test_th() {check("นี่คือการทดสอบการเข้ารหัสอักขระ", WINDOWS_874);}#[test]fn test_vi() {check("Đây là một thử nghiệm mã hóa ký tự.", WINDOWS_1258);}#[test]fn test_tr() {check("Bu bir karakter kodlama testidir. Latince karakterleri kullanan bazı dillerde karar vermek için daha fazla girdiye ihtiyacımız var.", WINDOWS_1254);}#[test]fn test_simplified() {check("这是一个字符编码测试。", GBK);}#[test]fn test_traditional() {check("這是一個字符編碼測試。", BIG5);}#[test]fn test_ko() {check("이것은 문자 인코딩 테스트입니다.", EUC_KR);}#[test]fn test_shift() {check("これは文字実験です。", SHIFT_JIS);}#[test]fn test_euc() {check("これは文字実験です。", EUC_JP);}#[test]fn test_ar() {check("هذا هو اختبار ترميز الأحرف.", WINDOWS_1256);}#[test]fn test_ar_iso() {check("هذا هو اختبار ترميز الأحرف.", ISO_8859_6);}#[test]fn test_fa() {check("این یک تست رمزگذاری کاراکتر است.", WINDOWS_1256);}#[test]fn test_visual() {check(".םיוות דודיק ןחבמ והז", ISO_8859_8);}#[test]fn test_yi() {check("דאָס איז אַ טעסט פֿאַר קאָדירונג פון כאַראַקטער.", WINDOWS_1255);}#[test]fn test_it() {check("è", WINDOWS_1252);}#[test]fn test_en() {check("isn’t", WINDOWS_1252);}#[test]fn test_en_bis() {check("Rock ’n Roll", WINDOWS_1252);}#[test]fn test_ca() {check("Codificació de caràcters", WINDOWS_1252);}#[test]fn test_et() {check("või", WINDOWS_1252);}#[test]fn test_pl_iso() {check("To jest test kodowania znaków. W przypadku niektórych języków, które używają znaków łacińskich, potrzebujemy więcej danych, aby podjąć decyzję.", ISO_8859_2);}#[test]fn test_pl() {check("To jest test kodowania znaków. W przypadku niektórych języków, które używają znaków łacińskich, potrzebujemy więcej danych, aby podjąć decyzję.", WINDOWS_1250);}#[test]fn test_lt() {check("Tai simbolių kodavimo testas. Kai kurioms kalboms, naudojančioms lotyniškus rašmenis, mums reikia daugiau informacijos, kad galėtume priimti sprendimą.", WINDOWS_1257);}// TODO: Detected as ISO-8859-2.// #[test]// fn test_lt_windows_iso_8859_4() {// check("Tai simbolių kodavimo testas. Kai kurioms kalboms, naudojančioms lotyniškus rašmenis, mums reikia daugiau informacijos, kad galėtume priimti sprendimą.", ISO_8859_4);// }#[test]fn test_lv() {check("Šis ir rakstzīmju kodēšanas tests. Dažās valodās, kurās tiek izmantotas latīņu valodas burti, lēmuma pieņemšanai mums ir nepieciešams vairāk ieguldījuma.", WINDOWS_1257);}#[test]fn test_lv_iso_8859_4() {check("Šis ir rakstzīmju kodēšanas tests. Dažās valodās, kurās tiek izmantotas latīņu valodas burti, lēmuma pieņemšanai mums ir nepieciešams vairāk ieguldījuma.", ISO_8859_4);}#[test]fn test_a0() {// Test that this isn't IBM866. TODO: What about GBK with fully paired 0xA0?check("\u{A0}\u{A0} \u{A0}", WINDOWS_1252);}#[test]fn test_a0a0() {// Test that this isn't GBK or EUC-KR.check("\u{A0}\u{A0}", WINDOWS_1252);}#[test]fn test_space_masculine_space() {check(" º ", WINDOWS_1252);}#[test]fn test_space_feminine_space() {check(" ª ", WINDOWS_1252);}#[test]fn test_period_masculine_space() {check(".º ", WINDOWS_1252);}#[test]fn test_period_feminine_space() {check(".ª ", WINDOWS_1252);}#[test]fn test_maria() {check(" Mª ", WINDOWS_1252);}#[test]fn test_dona() {check(" Dª ", WINDOWS_1252);}#[test]fn test_nuestra() {check(" Nª ", WINDOWS_1252);}#[test]fn test_senora() {check(" Sª ", WINDOWS_1252);}#[test]fn test_digit_feminine() {check(" 42ª ", WINDOWS_1252);}#[test]fn test_digit_masculine() {check(" 42º ", WINDOWS_1252);}#[test]fn test_roman_feminine() {check(" XIVª ", WINDOWS_1252);}#[test]fn test_roman_masculine() {check(" XIVº ", WINDOWS_1252);}#[test]fn test_numero_uno() {check("Nº1", WINDOWS_1252);}#[test]fn test_numero() {check("Nº", WINDOWS_1252);}#[test]fn test_euro() {check(" €9", WINDOWS_1252);}#[test]fn test_shift_jis_half_width_katakana() {check("ﾊｰﾄﾞｳｪｱﾊｰﾄﾞｳｪｱﾊｰﾄﾞｳｪｱﾊｰﾄﾞｳｪｱﾊｰﾄﾞｳｪｱ", SHIFT_JIS);}#[test]fn test_big5_pua() {let mut v = Vec::new();for _ in 0..40 {v.extend_from_slice(b"\xA4\x40");}v.extend_from_slice(b"\x81\x40\xA4\x40");check_bytes(&v, BIG5);}#[test]fn test_big5_single_byte_a0() {let mut v = Vec::new();for _ in 0..80 {v.extend_from_slice(b"\xA4\x40");}v.extend_from_slice(b"\x81\x40\xA0 ");check_bytes(&v, BIG5);}#[test]fn test_big5_single_byte_ff() {let mut v = Vec::new();for _ in 0..80 {v.extend_from_slice(b"\xA4\x40");}v.extend_from_slice(b"\x81\x40\xFF ");check_bytes(&v, BIG5);}#[test]fn test_not_big5() {let mut v = Vec::new();for _ in 0..40 {v.extend_from_slice(b"\xA4\x40");}v.extend_from_slice(b"\x81\x40\xA0\xA0");check_bytes(&v, IBM866);}#[test]fn test_euc_kr_pua() {let mut v = Vec::new();v.extend_from_slice(b"\xC9\xA1\xB0\xA1 ");for _ in 0..40 {v.extend_from_slice(b"\xC5\xD7\xBD\xBA\xC6\xAE. ");}check_bytes(&v, EUC_KR);}#[test]fn test_euc_kr_pua_bis() {let mut v = Vec::new();v.extend_from_slice(b"\xFE\xA1\xB0\xA1 ");for _ in 0..40 {v.extend_from_slice(b"\xC5\xD7\xBD\xBA\xC6\xAE. ");}check_bytes(&v, EUC_KR);}#[test]fn test_euc_kr_single_byte_ff() {let mut v = Vec::new();v.extend_from_slice(b"\xFF ");for _ in 0..40 {v.extend_from_slice(b"\xC5\xD7\xBD\xBA\xC6\xAE. ");}check_bytes(&v, EUC_KR);}#[test]fn test_euc_kr_single_byte_81() {let mut v = Vec::new();v.extend_from_slice(b"\x81 ");for _ in 0..40 {v.extend_from_slice(b"\xC5\xD7\xBD\xBA\xC6\xAE. ");}check_bytes(&v, EUC_KR);}#[test]fn test_euc_kr_single_byte_84() {let mut v = Vec::new();v.extend_from_slice(b"\x84 ");for _ in 0..40 {v.extend_from_slice(b"\xC5\xD7\xBD\xBA\xC6\xAE. ");}check_bytes(&v, EUC_KR);}#[test]fn test_not_euc_kr() {let mut v = Vec::new();v.extend_from_slice(b"\xC9\xA0\xB0\xA1 ");for _ in 0..40 {v.extend_from_slice(b"\xC5\xD7\xBD\xBA\xC6\xAE. ");}check_bytes(&v, GBK);}#[test]fn test_shift_jis_x0213() {let mut v = Vec::new();v.extend_from_slice(b"\x87\xE5");for _ in 0..40 {v.extend_from_slice(b"\x82\xC9\x82\xD9\x82\xF1\x82\xB2");}check_bytes(&v, SHIFT_JIS);}#[test]fn test_shift_jis_single_byte_fd() {let mut v = Vec::new();v.extend_from_slice(b"\xFD");for _ in 0..40 {v.extend_from_slice(b"\x82\xC9\x82\xD9\x82\xF1\x82\xB2");}check_bytes(&v, SHIFT_JIS);}#[test]fn test_not_shift_jis() {let mut v = Vec::new();v.extend_from_slice(b"\x84\xE0");for _ in 0..40 {v.extend_from_slice(b"\x82\xC9\x82\xD9\x82\xF1\x82\xB2");}check_bytes(&v, GBK);}#[test]fn test_not_shift_jis_bis() {let mut v = Vec::new();v.extend_from_slice(b"\x87\x7D");for _ in 0..40 {v.extend_from_slice(b"\x82\xC9\x82\xD9\x82\xF1\x82\xB2");}check_bytes(&v, GBK);}#[test]fn test_euc_jp_x0213() {let mut v = Vec::new();v.extend_from_slice(b"\xAD\xBF");for _ in 0..80 {v.extend_from_slice(b"\xA4\xCB\xA4\xDB\xA4\xF3\xA4\xB4");}check_bytes(&v, EUC_JP);}#[test]fn test_euc_jp_x0213_other_plane() {let mut v = Vec::new();v.extend_from_slice(b"\x8F\xFE\xF6");for _ in 0..80 {v.extend_from_slice(b"\xA4\xCB\xA4\xDB\xA4\xF3\xA4\xB4");}check_bytes(&v, EUC_JP);}#[test]fn test_not_euc_jp() {let mut v = Vec::new();v.extend_from_slice(b"\x8F\xFE\xF7");for _ in 0..80 {v.extend_from_slice(b"\xA4\xCB\xA4\xDB\xA4\xF3\xA4\xB4");}check_bytes(&v, WINDOWS_1252);}#[test]fn test_not_euc_jp_bis() {let mut v = Vec::new();v.extend_from_slice(b"\xA8\xDF");for _ in 0..80 {v.extend_from_slice(b"\xA4\xCB\xA4\xDB\xA4\xF3\xA4\xB4");}check_bytes(&v, BIG5);}#[test]fn test_gbk_single_byte_ff() {let mut v = Vec::new();v.extend_from_slice(b"\xFF");for _ in 0..80 {v.extend_from_slice(b"\xB5\xC4");}check_bytes(&v, GBK);}#[test]fn test_gbk_single_byte_a0() {let mut v = Vec::new();v.extend_from_slice(b"\xA0 ");for _ in 0..80 {v.extend_from_slice(b"\xB5\xC4");}check_bytes(&v, GBK);}#[test]fn test_gbk_single_byte_fe() {let mut v = Vec::new();v.extend_from_slice(b"\xFE ");for _ in 0..80 {v.extend_from_slice(b"\xB5\xC4");}check_bytes(&v, GBK);}#[test]fn test_not_gbk_single_byte_fc() {let mut v = Vec::new();v.extend_from_slice(b"\xFC ");for _ in 0..80 {v.extend_from_slice(b"\xB5\xC4");}check_bytes(&v, ISO_8859_5);}}File addition: data.rs
[0.257]/* Any copyright is dedicated to the Public Domain.* https://creativecommons.org/publicdomain/zero/1.0/ */use super::IMPLAUSIBILITY_PENALTY;use encoding_rs::Encoding;use encoding_rs::IBM866_INIT;use encoding_rs::ISO_8859_13_INIT;use encoding_rs::ISO_8859_2_INIT;use encoding_rs::ISO_8859_4_INIT;use encoding_rs::ISO_8859_5_INIT;use encoding_rs::ISO_8859_6_INIT;use encoding_rs::ISO_8859_7_INIT;use encoding_rs::ISO_8859_8_INIT;use encoding_rs::KOI8_U_INIT;use encoding_rs::WINDOWS_1250_INIT;use encoding_rs::WINDOWS_1251_INIT;use encoding_rs::WINDOWS_1252_INIT;use encoding_rs::WINDOWS_1253_INIT;use encoding_rs::WINDOWS_1254_INIT;use encoding_rs::WINDOWS_1255_INIT;use encoding_rs::WINDOWS_1256_INIT;use encoding_rs::WINDOWS_1257_INIT;use encoding_rs::WINDOWS_1258_INIT;use encoding_rs::WINDOWS_874_INIT;const PLAUSIBLE_NEXT_TO_ALPHABETIC_ON_EITHER_SIDE: usize = 0;const IMPLAUSIBLE_NEXT_TO_ALPHABETIC_ON_EITHER_SIDE: usize = 1;const IMPLAUSIBLE_BEFORE_ALPHABETIC: usize = 2;const IMPLAUSIBLE_AFTER_ALPHABETIC: usize = 3;const PLAUSIBLE_NEXT_TO_NON_ASCII_ALPHABETIC_ON_EITHER_SIDE: usize = 4;const PLAUSIBLE_NEXT_TO_ASCII_ALPHABETIC_ON_EITHER_SIDE: usize = 5;const WINDOWS_1256_ZWNJ: usize = 2;pub const ASCII_DIGIT: usize = 100;#[repr(align(64))] // Align to cache linespub struct DetectorData {pub frequent_simplified: [u16; 128],pub frequent_kanji: [u16; 128],pub frequent_hangul: [u16; 128],latin_ascii: [u8; 128],non_latin_ascii: [u8; 128],turkish_ascii: [u8; 128],windows_1258: [u8; 128],windows_1250: [u8; 128],iso_8859_2: [u8; 128],windows_1251: [u8; 128],koi8_u: [u8; 128],iso_8859_5: [u8; 128],ibm866: [u8; 128],windows_1252: [u8; 128],windows_1252_icelandic: [u8; 128],windows_1253: [u8; 128],iso_8859_7: [u8; 128],windows_1254: [u8; 128],windows_1255: [u8; 128],iso_8859_8: [u8; 128],windows_1256: [u8; 128],iso_8859_6: [u8; 128],windows_1257: [u8; 128],iso_8859_13: [u8; 128],iso_8859_4: [u8; 128],windows_874: [u8; 128],vietnamese: [u8; 1975],central: [u8; 3895],cyrillic: [u8; 2112],western: [u8; 2752],icelandic: [u8; 871],greek: [u8; 1365],turkish: [u8; 845],hebrew: [u8; 1292],arabic: [u8; 2805],baltic: [u8; 1387],thai: [u8; 5180],}#[rustfmt::skip]pub static DETECTOR_DATA: DetectorData = DetectorData {frequent_simplified: [0x7684, 0x5E74, 0x56FD, 0x65E5, 0x6708, 0x4E2D, 0x4EBA, 0x4E00, 0x5927, 0x4E3A, 0x5728, 0x662F, 0x5B66, 0x6709, 0x884C, 0x4F1A,0x65AF, 0x4E8E, 0x5730, 0x533A, 0x6587, 0x548C, 0x5C14, 0x540D, 0x7B2C, 0x516C, 0x65F6, 0x5C0F, 0x90E8, 0x4E0D, 0x5E02, 0x53F0,0x4EE5, 0x4E0A, 0x540E, 0x52A8, 0x51FA, 0x4E2A, 0x672C, 0x4F5C, 0x5BB6, 0x65B0, 0x6210, 0x897F, 0x5B9A, 0x91CD, 0x751F, 0x4E4B,0x7535, 0x4E3B, 0x5B50, 0x7528, 0x7279, 0x5206, 0x6C11, 0x4E86, 0x4E9A, 0x5458, 0x514B, 0x5357, 0x653F, 0x7AD9, 0x5FB7, 0x4E0E,0x7403, 0x4E1C, 0x79D1, 0x91CC, 0x9053, 0x5C71, 0x6CD5, 0x65B9, 0x5317, 0x5411, 0x5929, 0x53D1, 0x7269, 0x6765, 0x5230, 0x673A,0x661F, 0x8DEF, 0x76EE, 0x7F8E, 0x6751, 0x9AD8, 0x957F, 0x519B, 0x5229, 0x4E09, 0x62C9, 0x8F66, 0x5DDE, 0x57FA, 0x6D77, 0x81EA,0x4E0B, 0x8D5B, 0x9762, 0x52A0, 0x4ED6, 0x9A6C, 0x5176, 0x53C2, 0x53BF, 0x4EE3, 0x5185, 0x7406, 0x4E16, 0x4E8C, 0x7EBF, 0x53CA,0x5EFA, 0x8868, 0x4F4D, 0x7F57, 0x7531, 0x7ACB, 0x591A, 0x53EF, 0x534E, 0x6797, 0x7EF4, 0x5EA6, 0x4E8B, 0x5E73, 0x5916, 0x4F53,],frequent_kanji: [0x5E74, 0x65E5, 0x6708, 0x5927, 0x672C, 0x5B66, 0x4EBA, 0x56FD, 0x4F1A, 0x4E2D, 0x51FA, 0x4E00, 0x8005, 0x5E02, 0x4F5C, 0x540D,0x90E8, 0x7528, 0x5730, 0x884C, 0x5834, 0x7530, 0x7B2C, 0x751F, 0x5408, 0x5B50, 0x9053, 0x4E0A, 0x6771, 0x6642, 0x770C, 0x4EE3,0x5C71, 0x793E, 0x4E8B, 0x753B, 0x65B0, 0x624B, 0x9AD8, 0x6210, 0x6226, 0x7269, 0x5F8C, 0x767A, 0x9577, 0x7ACB, 0x5206, 0x5DDD,0x8A18, 0x6821, 0x9593, 0x696D, 0x95A2, 0x6240, 0x5B9A, 0x9078, 0x5C0F, 0x76EE, 0x52D5, 0x548C, 0x6587, 0x91CE, 0x540C, 0x524D,0x5185, 0x958B, 0x7DDA, 0x81EA, 0x53F7, 0x516C, 0x99C5, 0x9001, 0x56DE, 0x753A, 0x9664, 0x4E3B, 0x5BB6, 0x5229, 0x8ECA, 0x901A,0x4EAC, 0x8868, 0x5CF6, 0x4E0B, 0x4E16, 0x65B9, 0x6751, 0x66F8, 0x5168, 0x660E, 0x9023, 0x5E73, 0x653E, 0x4F53, 0x7684, 0x5F0F,0x756A, 0x5EA6, 0x5317, 0x5165, 0x5916, 0x983C, 0x8A9E, 0x5973, 0x8A71, 0x6A5F, 0x8A2D, 0x539F, 0x4E09, 0x524A, 0x533A, 0x6D77,0x4F9D, 0x5F53, 0x73FE, 0x5BFE, 0x4F4D, 0x6570, 0x5316, 0x795E, 0x66F2, 0x7406, 0x6559, 0x7279, 0x7248, 0x5728, 0x6CD5, 0x898B,],frequent_hangul: [0xC774, 0xC758, 0xB2E4, 0xAE30, 0xC5D0, 0xB85C, 0xC0AC, 0xB144, 0xC2A4, 0xB9AC, 0xB294, 0xC77C, 0xD558, 0xAC00, 0xC2DC, 0xC9C0,0xB300, 0xC11C, 0xBD84, 0xAD6D, 0xD55C, 0xB3C4, 0xC778, 0xACE0, 0xB958, 0xC790, 0xC8FC, 0xC544, 0xC744, 0xB77C, 0xC218, 0xC81C,0xC815, 0xC6D4, 0xB098, 0xAD6C, 0xC804, 0xC5B4, 0xC740, 0xADF8, 0xBD80, 0xB97C, 0xB3D9, 0xC120, 0xC73C, 0xBB38, 0xD2B8, 0xC6A9,0xBCF4, 0xC704, 0xB4DC, 0xACFC, 0xAD50, 0xC0C1, 0xB9C8, 0xC7A5, 0xD559, 0xC6D0, 0xC131, 0xD654, 0xC5ED, 0xB2C8, 0xBBF8, 0xACF5,0xACBD, 0xD574, 0xC624, 0xC6B0, 0xBA85, 0xC788, 0xD06C, 0xC601, 0xC18C, 0xC870, 0xD68C, 0xC5EC, 0xBBFC, 0xD1A0, 0xBE44, 0xC138,0xB974, 0xC720, 0xC2E0, 0xD0A4, 0xC911, 0xACC4, 0xD0C0, 0xC5F0, 0xD504, 0xAD00, 0xB418, 0xC801, 0xCE58, 0xB808, 0xCE74, 0xC9C4,0xC640, 0xD130, 0xB4E4, 0xBAA9, 0xACA8, 0xAC8C, 0xAC1C, 0xBC29, 0xD30C, 0xC0B0, 0xD638, 0xCD9C, 0xC74C, 0xB9BC, 0xBA74, 0xC791,0xB9CC, 0xB2E8, 0xB118, 0xBAA8, 0xC694, 0xC5C8, 0xC0DD, 0xB0A8, 0xC7AC, 0xBB34, 0xD6C4, 0xD45C, 0xAD70, 0xD3EC, 0xB2F9, 0xB178,],latin_ascii: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,100,100,100,100,100,100,100,100,100,100, 0, 0, 0, 0, 0, 0,0,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154, 0, 0, 0, 0, 0,0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 0, 0, 0, 0, 0,],non_latin_ascii: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,100,100,100,100,100,100,100,100,100,100, 0, 0, 0, 0, 0, 0,0,129,129,129,129,129,129,129,129,129,129,129,129,129,129,129,129,129,129,129,129,129,129,129,129,129,129, 0, 0, 0, 0, 0,0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0,],turkish_ascii: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,100,100,100,100,100,100,100,100,100,100, 0, 0, 0, 0, 0, 0,0,129,130,131,132,133,134,135,136,154,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153, 0, 0, 0, 0, 0,0, 1, 2, 3, 4, 5, 6, 7, 8, 27, 9, 10, 11, 12, 13, 14,15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 0, 0, 0, 0, 0,],windows_1258: [0,255, 0, 53, 0, 0, 0, 0, 0, 0,255, 0,155,255,255,255,255, 0, 0, 0, 0, 0, 0, 0, 0, 0,255, 0, 27,255,255,179,0, 55, 53, 53, 53, 53, 52, 53, 53, 55, 53, 53, 56, 52, 54, 53,55, 56, 54, 54, 53, 55, 54, 52, 53, 54, 53, 53, 55, 55, 55, 55,159,160,161,162,155,155,155,155,163,164,165,155, 28,167,168,169,170,155, 29,172,173,174,155, 56,155,175,176,177,155,178, 30, 27,31, 32, 33, 34, 27, 27, 27, 27, 35, 36, 37, 27, 38, 39, 40, 41,42, 27, 43, 44, 45, 46, 27, 56, 27, 47, 48, 49, 27, 50, 53, 51,],windows_1250: [0,255, 0,255, 0, 0, 0, 0,255, 0,156, 0,157,158,159,160,255, 0, 0, 0, 0, 0, 0, 0,255, 0, 28, 0, 29, 30, 31, 32,0, 69, 69,161, 69,162, 68, 69, 69, 71,163, 68, 69, 68, 70,165,71, 69, 69, 33, 69, 71, 70, 68, 69, 34, 35, 68,164, 0, 36, 37,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188, 72,189,190,191,192,193,194,195, 27,38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53,54, 55, 56, 57, 58, 59, 60, 72, 61, 62, 63, 64, 65, 66, 67, 69,],iso_8859_2: [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,162, 69,161, 69,164,157, 69, 69,156,163,158,160, 68,159,165,71, 34, 69, 33, 69, 36, 29, 69, 69, 28, 35, 30, 32, 0, 31, 37,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188, 72,189,190,191,192,193,194,195, 27,38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53,54, 55, 56, 57, 58, 59, 60, 72, 61, 62, 63, 64, 65, 66, 67, 69,],windows_1251: [131,130, 0, 2, 0, 0, 0, 0, 0, 0,132, 0,133,130,134,135,3, 0, 0, 0, 0, 0, 0, 0,255, 0, 4, 0, 5, 2, 6, 7,0,136, 8,140, 47,130, 46, 47,138, 49,139, 49, 50, 46, 48,141,49, 50,137, 9, 2, 49, 48, 46, 10, 47, 11, 48, 12,130, 2, 13,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29,30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45,],koi8_u: [47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47,47, 47, 47, 47, 47, 46, 0, 50, 50, 50, 0, 47, 49, 48, 46, 50,47, 47, 47, 10, 11, 47, 9, 13, 47, 47, 47, 47, 47, 2, 8, 47,47, 47, 47,138,139, 47,137,141, 47, 47, 47, 47, 47,130,136, 49,44, 14, 15, 36, 18, 19, 34, 17, 35, 22, 23, 24, 25, 26, 27, 28,29, 45, 30, 31, 32, 33, 20, 16, 42, 41, 21, 38, 43, 39, 37, 40,172,142,143,164,146,147,162,145,163,150,151,152,153,154,155,156,157,173,158,159,160,161,148,144,170,169,149,166,171,167,165,168,],iso_8859_5: [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,138,131,130,139,130,137,141,140,132,133,134,130, 46,136,135,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29,30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45,47, 10, 3, 2, 11, 2, 9, 13, 12, 4, 5, 6, 2, 47, 8, 7,],ibm866: [142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29,47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47,47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47,47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47,30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45,138, 10,139, 11,141, 13,136, 8, 49, 46, 46, 0, 47, 47, 47, 3,],windows_1252: [0,255, 0, 60, 0, 0, 0, 0, 0, 0,156, 0,157,255,185,255,255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 29,255, 57,186,0, 62, 60, 60, 60, 60, 59, 60, 60, 62, 60, 59, 63, 59, 61, 60,62, 63, 61, 61, 60, 62, 61, 59, 60, 61, 60, 59, 62, 62, 62, 62,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,188,174,175,176,177,178,179, 63,180,181,182,183,184,188,188, 27,30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45,60, 46, 47, 48, 49, 50, 51, 63, 52, 53, 54, 55, 56, 60, 60, 58,],windows_1252_icelandic: [0,255, 0, 41, 0, 0, 0, 0, 0, 0,155, 0,155,255,155,255,255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 27,255, 27,155,0, 43, 41, 41, 41, 41, 40, 41, 41, 43, 41, 41, 44, 40, 42, 41,43, 44, 42, 42, 41, 43, 42, 40, 41, 42, 41, 41, 43, 43, 43, 43,155,156,155,155,157,155,158,155,155,159,155,155,155,160,155,155,161,155,155,162,155,155,163, 44,164,155,165,155,155,166,167, 27,27, 28, 27, 27, 29, 27, 30, 27, 27, 31, 27, 27, 27, 32, 27, 27,33, 27, 27, 34, 27, 27, 35, 44, 36, 27, 37, 27, 27, 38, 39, 27,],windows_1253: [38,255, 0, 38, 0, 0, 0, 0,255, 0,255, 0,255,255,255,255,255, 0, 0, 0, 0, 0, 0, 0,255, 0,255, 0,255,255,255,255,0, 38,131, 38, 38, 38, 37, 38, 38, 40,255, 40, 37, 37, 39, 37,40, 37, 39, 39, 0, 40, 39, 37,132,133,134, 39,162, 40,163,164,2,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,255,153,154,155,156,157,158,159,160,161, 3, 4, 5, 6,2, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21,22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36,255,],iso_8859_7: [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0, 40, 39, 38, 38, 38, 37, 38, 38, 40, 38, 40, 37, 37,255, 37,40, 37, 39, 39, 0, 38,131, 37,132,133,134, 39,162, 40,163,164,2,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,255,153,154,155,156,157,158,159,160,161, 3, 4, 5, 6,2, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21,22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36,255,],windows_1254: [0,255, 0, 40, 0, 0, 0, 0, 0, 0,156, 0,156,255,255,255,255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 28,255,255,156,0, 42, 40, 40, 40, 40, 39, 40, 40, 42, 40, 42, 43, 39, 41, 40,42, 43, 41, 41, 40, 42, 41, 39, 40, 41, 40, 41, 42, 42, 42, 42,156,156,158,156,157,156,156,159,156,156,160,156,156,156,161,156,162,156,156,156,156,156,163, 43,156,156,156,164,165,155,166, 28,28, 28, 30, 28, 29, 28, 28, 31, 28, 28, 32, 28, 28, 28, 33, 28,34, 28, 28, 28, 28, 28, 35, 43, 28, 28, 28, 36, 37, 26, 38, 28,],windows_1255: [0,255, 0, 37, 0, 0, 0, 0, 0, 0,255, 0,255,255,255,255,255, 0, 0, 0, 0, 0, 0, 0, 0, 0,255, 0,255,255,255,255,0, 39, 37, 37, 37, 37, 36, 37, 37, 39, 40, 40, 40, 36, 38, 37,39, 40, 38, 38, 37, 39, 38, 36, 37, 38, 40, 40, 39, 39, 39, 39,2, 2, 2, 2, 2, 2, 2, 3, 4, 2, 2, 2, 5, 2, 36, 37,36, 2, 2, 0, 6, 7, 8, 41, 41,255,255,255,255,255,255,255,9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24,25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35,255,255, 37, 37,255,],iso_8859_8: [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255, 37, 37, 37, 37, 36, 37, 37, 39, 40, 40, 40, 36, 38, 37,39, 40, 38, 38, 37, 39, 38, 36, 37, 38, 40, 40, 39, 39, 39,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, 36,9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24,25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35,255,255, 37, 37,255,],windows_1256: [0, 3, 0, 54, 0, 0, 0, 0, 0, 0, 4, 0,129, 5, 6, 7,8, 0, 0, 0, 0, 0, 0, 0, 9, 0, 10, 0, 1, 2, 53, 11,0, 58, 54, 54, 54, 54, 53, 54, 54, 56, 12, 56, 57, 53, 55, 54,56, 57, 55, 55, 54, 56, 55, 53, 54, 55, 58, 55, 56, 56, 56, 58,13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28,29, 30, 31, 32, 33, 34, 35, 57, 36, 37, 38, 39, 40, 41, 42, 43,1, 44, 1, 45, 46, 47, 48, 1, 1, 1, 1, 1, 49, 50, 1, 1,51, 51, 51, 51, 1, 51, 51, 57, 51, 1, 51, 1, 1, 54, 54, 52,],iso_8859_6: [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255, 54,255,255,255,255,255,255,255, 58, 53,255,255,255,255,255,255,255,255,255,255,255,255,255, 58,255,255,255, 58,255, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28,29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39,255,255,255,255,255,40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 51, 51, 51, 51,51, 51, 51,255,255,255,255,255,255,255,255,255,255,255,255,255,],windows_1257: [0,255, 0,255, 0, 0, 0, 0,255, 0,255, 0,255, 47, 47, 47,255, 0, 0, 0, 0, 0, 0, 0,255, 0,255, 0,255, 47, 47,255,0,255, 47, 47, 47,255, 46, 47,155, 49,156, 49, 50, 46, 48,155,49, 50, 48, 48, 47, 49, 48, 46, 27, 48, 28, 48, 49, 49, 49, 27,157,158,159,155,155,155,160,161,162,155,155,163,164,165,166,167,168,155,169,155,170,155,155, 50,171,155,155,172,155,155,173, 27,29, 30, 31, 27, 27, 27, 32, 33, 34, 27, 27, 35, 36, 37, 38, 39,40, 27, 41, 27, 42, 27, 27, 50, 43, 27, 27, 44, 27, 27, 45, 47,],iso_8859_13: [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0, 48, 47, 47, 47, 49, 46, 47,155, 49,156, 49, 50, 46, 48,155,49, 50, 48, 48, 46, 49, 48, 46, 27, 48, 28, 48, 49, 49, 49, 27,157,158,159,155,155,155,160,161,162,155,155,163,164,165,166,167,168,155,169,155,170,155,155, 50,171,155,155,172,155,155,173, 27,29, 30, 31, 27, 27, 27, 32, 33, 34, 27, 27, 35, 36, 37, 38, 39,40, 27, 41, 27, 42, 27, 27, 50, 43, 27, 27, 44, 27, 27, 45, 46,],iso_8859_4: [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,157, 27,156, 47,155,167, 47, 47,168,161,164,155, 46,173, 47,49, 29, 47, 28, 47, 27, 39, 47, 47, 40, 33, 36, 27,155, 45, 27,159,155,155,155,155,155,155,158,162,155,160,155,163,155,155,166,155,169,170,165,155,155,155, 50,155,171,155,155,155,155,172, 27,31, 27, 27, 27, 27, 27, 27, 30, 34, 27, 32, 27, 35, 27, 27, 38,27, 41, 42, 37, 27, 27, 27, 50, 27, 43, 27, 27, 27, 27, 44, 47,],windows_874: [77,255,255,255,255, 0,255,255,255,255,255,255,255,255,255,255,255, 0, 0, 0, 0, 0, 0, 0,255,255,255,255,255,255,255,255,0, 2, 3, 71, 4, 71, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14,15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30,31, 32, 33, 34, 35, 36, 71, 37, 38, 39, 40, 41, 42, 43, 44, 45,46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56,255,255,255,255, 77,57, 58, 59, 60, 61, 71, 62, 63, 64, 65, 66, 67, 68, 69, 70, 77,77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77,255,255,255,255,],vietnamese: [0, 58, 58, 22, 33, 7, 0, 0, 0, 0, 1, 57, 3, 0, 0, 0, 39, 14, 8, 3, 0, 11, 0, 5, 0, // ,0, 3, 16, 2, 0, 0, 0, 0, 0, 0, 0, 3, 1,255, 0, 17, 2, 1, 0, 0, 1, 0, 0, 1,255, // a,0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,255, // b,0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 51, 12, 0, 0, 0, 48, 0, 0, 0, 0, 2, 0, 0, 0, // c,0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,255, // d,0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,255, 0, 0, 0, 0, 0, 0, 0, 0,255, 0, 0, // e,0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,255,255,255, // f,0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,255, // g,0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,255,255, 0, 0, 0, 0, 0, 0, 0, 0, 0,255, // h,0, 27, 11, 1, 16, 3, 0, 0, 0, 0, 0, 16, 0, 0,255, 10, 34, 0, 2, 1, 0, 0, 0, 0,255, // i,0, 0,255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,255,255, 0, 0, 0, 0,255, 0, 0, 0, 0,255, // j,0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,255, // k,0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,255, // l,0, 6, 3, 0, 2, 0, 2, 22, 0, 0, 2, 4, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0,255, // m,0, 59, 23, 10, 19, 22, 18, 5, 0, 0, 28, 61, 6, 0, 0, 0, 39, 4, 20, 9, 6, 1, 0, 1,255, // n,0, 5, 13, 0, 5, 3, 0, 0, 0, 0, 0, 3, 0,255, 0, 4, 2, 0, 0,255,255, 0,255, 0,255, // o,0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 13, 0, 0,255, 0, 10, 0, 0, 0, 0, 0, 0, 0,255, // p,0, 0, 0,255, 0, 0, 0,255, 0, 0, 0, 0, 0,255, 0, 0,255, 0, 0,255, 0, 0,255, 0,255, // q,0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,255, // r,0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,255, // s,0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0,255, // t,0, 12, 3, 1, 1, 0, 4, 0, 0, 0, 6, 4, 0,255,255, 0, 4, 0, 0, 0, 0, 0,255, 1, 0, // u,0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,255, 0,255, 0,255, // v,0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,255, 0, 0, 0, 0, 0, 0,255, 0,255, 0,255, // w,0, 0,255, 0, 0, 0,255,255, 0, 0, 0, 0, 0,255, 0, 0, 0, 0, 0, 0, 0, 0,255, 0,255, // x,0, 18, 2, 0, 18, 1, 4, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 0, 0,255, 0, 0,255, 0,255, // y,0, 0,255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,255, 0,255, // z,0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,255,255, 0, 0,255,255, 0, 0, 0, 0, 0,255,255,255,255, 0,255,255,255, 0, 0,255,255, // ß,0, 95,255,255,255, 1,255,255,255, 11,255,255,255,255,255, 6,255,255,255,255,255, 8,255,255,255, 4,255,255,255,255,255,255,255, 14, 2,255,255, 17,255,255,255,255,255,255,255, 8, 16,255,255,255, 5,255, // ̀,0, 39,255,255,255, 0,255,255,255, 5,255,255,255,255,255, 3,255,255,255,255,255, 20,255,255,255, 1,255,255,255,255,255,255,255, 3, 0,255,255, 31,255,255,255,255,255,255,255, 10, 10,255,255,255, 5,255, // ̉,0, 12,255,255,255, 1,255,255,255, 5,255,255,255,255,255, 0,255,255,255,255,255, 3,255,255,255, 1,255,255,255,255,255,255,255, 2, 0,255,255, 3,255,255,255,255,255,255,255, 1, 0,255,255,255, 7,255, // ̃,0, 0, 3, 0, 0, 0, 0, 12, 15, 0, 0, 0, 17, 1, 6, 15, 0,255, 0, 0, 1, 0, 16, 0, 0, 0, 0, 0, 0,255,255, 0, 0,255,255,255,255,255,255, 0,255,255, 1, 0,255,255, 0,255,255,255, 0,255, // à,2, 0, 2, 13, 0, 0, 0, 0, 26, 3, 0, 0, 1, 1, 0, 1, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0,255,255,255, 0, 0,255,255,255,255,255,255, 0,255,255, 3,255,255,255,255,255,255,255,255,255, // á,3, 0, 1, 7, 5, 0, 0, 1, 23, 0, 0, 0, 3, 3, 0, 0, 0, 0, 3, 1, 7, 13, 16, 0, 0, 0, 0, 0, 0, 0, 0,255,255, 0, 0,255, 0, 0, 0,255,255,255, 8, 0,255,255, 0,255, 0,255, 0,255, // â,0, 0, 3, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 23, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0,255, 0, 0, 0,255,255, 0, 0,255,255,255, 0,255,255,255, 1, 0,255,255, 0,255,255,255,255,255, // ă,0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,255,255,255,255,255,255,255, 0, 0,255,255, 0,255,255, 0,255,255,255,255,255,255,255,255,255, // è,0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,255,255,255, 0,255,255,255,255, 0,255,255,255,255,255, 0,255,255,255,255,255,255,255,255,255, // é,0, 0, 1, 0, 0, 0, 0, 0, 28, 66, 0, 6, 2, 0, 1, 0, 0, 0, 2, 0, 4, 0, 3, 0, 0, 12, 0,255, 0, 0, 0,255,255,255,255, 0, 0, 0, 0, 0,255,255, 13, 0,255, 0,255,255,255,255, 0,255, // ê,0, 63,255,255,255, 5,255,255,255, 24,255,255,255,255,255, 22,255,255,255,255,255, 18,255,255,255, 3,255,255,255,255,255,255,255, 17, 6,255,255, 30,255,255,255,255,255,255,255, 23, 20,255,255,255, 10,255, // ́,0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0,255, 0, 0,255,255,255,255, 0, 0, 0,255,255, 0,255,255, 0, 0,255,255,255,255,255, // í,0, 0, 0, 0, 0, 0, 0, 0, 0,255, 0, 0, 0, 0, 0, 0, 0,255, 0, 0, 0, 0, 0, 0,255, 0, 0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, // î,0, 0,255,255, 0, 0, 0, 0, 0, 0,255, 0, 0, 0, 0, 0,255,255, 0,255, 0, 0,255,255, 0, 0, 0,255,255,255,255,255,255,255,255,255, 0,255, 0,255,255,255,255,255,255,255,255,255,255,255,255,255, // ï,97, 0, 0, 0, 0, 0, 0, 0, 0, 0,255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,255, 0, 0, 0, 0,255,255,255,255,255, 0, 0,255,255,255, 0, 0, 0, 0, 0,255,255,255,255,255, // đ,0, 42,255,255,255, 0,255,255,255, 15,255,255,255,255,255, 18,255,255,255,255,255, 8,255,255,255, 0,255,255,255,255,255,255,255, 34, 4,255,255, 24,255,255,255,255,255,255,255, 41, 17,255,255,255, 17,255, // ̣,0, 0, 2, 10, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,255,255,255,255,255,255,255,255,255,255,255, 0,255,255, 2,255, 0,255,255,255,255,255, 0,255, // ó,2, 0, 7, 8, 0, 0, 0, 2, 18, 0, 0, 0, 1, 17, 2, 0, 0, 0, 1, 6, 5, 15, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0,255,255,255,255,255, 0, 0,255,255,255, 22, 0,255, 0, 0,255,255,255, 0,255, // ô,5, 0, 1, 1, 0, 0, 0, 0, 8, 2, 0, 0, 2, 1, 0, 0, 0,255, 1, 2, 0, 0, 4, 0, 0, 0, 0,255, 0, 0, 0,255,255,255, 0,255,255,255, 0,255,255,255, 2, 0,255, 0, 0,255,255,255, 42,255, // ơ,0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0,255, 0,255,255,255,255, 0,255,255,255,255,255,255,255,255, 0,255,255,255,255, 0,255,255,255,255, // ù,0, 0, 0, 1, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,255,255,255,255,255,255,255,255,255, 0,255,255,255,255, 0,255, 0,255,255,255, 0,255,255,255, // ú,0,255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,255,255, 0, 0, 0,255,255, 0,255, 0, 0, 0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, // û,1, 0, 1, 4, 4,255, 0, 7, 28, 1, 0, 0, 4, 0, 3, 0, 0, 0, 4, 7, 9, 0, 2, 0, 0, 0, 0,255, 0, 0, 0,255,255, 0, 0,255,255,255, 0,255,255,255, 15, 0,255,255, 0,255,255,255, 0,255, // ư,0, 0,255,255,255,255,255,255,255,255,255,255, 0,255,255, 0,255,255, 0,255,255, 0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, // ÿ,// , a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t, u, v, w, x, y, z, ß, ̀, ̉, ̃, à, á, â, ă, è, é, ê, ́, í, î, ï, đ, ̣, ó, ô, ơ, ù, ú, û, ư, ÿ,],central: [0, 42, 11, 51, 30, 3,154, 77, 18, 20, 23, 0,139, 0,254, 1, 0, 79, 0, 70,132,121, 0, 52,241, 14, 5, 7, 17, 8, 74, 0, 58, 0, 8, 33, 36, 9, 1,105, 8, // ,0, 70, 0, 11,146, 0,115, 0, 11, 23, 12, 0, 0, 0, 2, 0, 0, 66, 0,107, 1, 0, 0, 0, 10, 0, 10, 25, 0, 8, 1, 0, 1, 0, 9, 0, 0, 0, 0, 0, 48, // a,0, 0, 0, 0, 18, 0, 1, 1, 0, 4, 2, 0, 32, 0, 1, 0, 0, 0, 0, 2, 11, 5, 0, 0, 2, 0, 0, 0, 0, 0, 8, 0, 4, 6, 1, 0, 10, 0, 0, 1, 0, // b,0, 2, 62, 0, 0, 0, 9, 45, 9, 0, 5, 0, 47, 0, 8, 0, 0, 0, 0, 0, 5, 31, 0, 3, 33, 3, 0,255, 14, 0, 8, 0, 0, 0, 0, 0, 13, 0, 1, 48, 0, // c,0, 0, 0, 0, 10, 6, 1, 19, 0, 0, 2, 0, 23, 0, 6, 0,255, 0, 0, 0, 11, 28, 0, 5, 8, 0, 0, 0, 0, 0, 33, 0, 4, 2, 0, 1, 10, 0, 1, 0, 0, // d,0, 70, 0, 0,111, 0, 16, 0, 27, 0, 36, 0, 0, 0, 0, 0,255, 61, 0,145, 5, 0, 0, 0, 0, 0, 0,133, 0, 0, 0, 0, 2, 0, 64, 0, 0, 1, 0, 0, 34, // e,0, 0, 0, 0, 0,255, 0, 0, 0, 0, 0,255, 1, 0, 0, 0,255, 0, 0, 0, 0, 0, 0, 0, 0, 0,255,255, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, // f,0, 0, 0, 0, 2, 0, 2, 5, 0, 0, 0,255, 64, 0, 2, 0, 0, 0, 0, 0, 61, 9, 0, 0, 4, 0, 0,255, 0, 0, 7, 0, 1, 2, 0, 0, 13, 0, 4, 0, 0, // g,0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 4, 11, 2, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 4, 0, 0, 5, 0, // h,0,110, 0, 0,103, 0, 0, 0,157, 0, 0, 0, 2, 1, 7, 0, 0,225, 0,177, 0, 0, 0, 0, 0, 7, 0, 6, 0, 0, 6, 0, 5, 0, 24, 0, 0, 0, 0, 0,242, // i,0, 7, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0,255, 0,255, 27, 1, 0, 0, 10, 9, 0, 0, 0, 0, 0, 6, 0, 3, 0, 0, 0, 8, 0, 0, 0, 0, // j,0,193, 0, 0,210, 0, 21, 0, 1, 12, 8, 0, 21, 0, 0, 0, 0, 1, 0,186, 42, 8, 0, 8, 19, 5, 0, 0, 0, 0, 13, 0, 9, 10, 2, 0, 1, 2, 5, 2, 2, // k,0, 10, 15, 0, 1, 0, 0, 0, 0, 0, 2, 0, 92, 3, 12, 0, 0, 0, 0, 31, 48, 0, 0, 15, 9, 4, 0, 0, 0, 0, 24, 1, 13, 16, 0, 1, 7, 1, 56, 2, 0, // l,0, 2, 9, 0, 1, 0, 1, 0, 0, 2, 0, 0, 22, 4, 16, 0, 0, 0, 0, 0, 15, 0, 0, 7, 27, 10, 0, 0, 0, 0, 4, 0, 1, 2, 0, 4, 4, 0, 0, 22, 0, // m,0, 37, 15, 0, 39, 6, 14, 0, 1, 5, 20, 0, 76, 90, 10, 0, 0, 6, 0,121, 54, 0, 0, 16, 20,238, 0, 0, 0, 0, 37, 2, 2, 17, 1, 0, 4, 1, 4, 1, 0, // n,0, 16, 0, 3, 7, 0, 93, 0, 7, 21, 12, 0, 0, 0, 0, 0, 0, 2, 1, 11, 4, 0, 0, 0, 5, 0, 0, 1, 0, 4, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, // o,0, 23, 1, 0, 0, 0, 1, 3, 0, 0, 0, 0, 9, 0, 3, 0, 1, 0, 0, 0, 44, 38, 0, 3, 2, 0,255, 0, 0, 0, 6, 0, 2, 0, 0, 0, 6, 0, 0, 0, 0, // p,255, 0,255,255, 0, 0, 0,255, 0,255,255,255, 0, 0, 0, 0,255,255,255, 0, 0, 0, 0,255, 0, 0,255,255,255,255, 0, 0,255, 0,255,255, 0,255, 0,255,255, // q,0, 1, 11, 0, 1, 5, 4, 0, 0, 0, 0, 0, 96, 14, 70, 1, 0, 0, 0, 15, 68, 0, 0, 8, 20, 4, 0, 0, 0, 0, 58, 3, 5, 27, 0, 0, 14, 4, 3, 3, 0, // r,0, 1, 0, 0, 10, 0, 0, 6, 0, 9, 4, 0,133, 0, 15, 0, 0, 0, 0, 1,147, 9, 0, 11, 27, 2, 0, 0, 66, 0, 11, 5, 13, 12, 3, 4, 9, 6, 2, 5, 0, // s,0,254, 0, 0, 0, 0, 3, 10, 74, 0, 0, 0, 63, 7, 46, 5, 0, 0, 0, 6, 30, 23, 0, 22, 45, 3, 0, 0, 0, 0, 6, 1, 5, 13, 0, 0, 7, 0, 3, 2, 0, // t,0, 7, 0, 0, 35, 0, 35, 0, 10, 6, 4,255, 1, 9, 17, 0, 0, 16, 0, 22, 0, 0, 0, 0, 0, 0, 0, 42, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, // u,0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 1, 0, 0, 0, 0, 8, 38, 0, 0, 2, 12, 1, 0, 0,255, 0, 2, 9, 6, 14, 0, 6, 2, 6, 0, 8, 0, // v,0, 0, 33, 0, 0, 1, 0, 0, 0,255, 0,255, 0, 0, 0, 0,255, 0,255, 0, 0, 0, 0, 0, 0, 0,255,255, 0,255,113, 0, 0, 0, 0, 0, 0,255, 0, 0, 0, // w,255, 0,255,255, 0,255, 0,255,255,255,255,255, 0,255, 0, 0,255, 0,255, 0, 3,255, 0, 0, 0,255,255,255,255,255, 0, 0, 0, 0, 0,255, 0, 0, 0, 0,255, // x,255, 0, 0, 0, 0, 0, 37, 0, 0, 0, 37,255, 0, 0, 0, 0,255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,255, 0, 0, 0, 0, 0, 0, 0, 0, 0,255, 0, 0, 0, // y,0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 46, 0, 9, 2, 0, 0,255, 0, 8, 29, 0, 4, 5, 0, 0,255, 0,255, 4, 1, 8, 31, 0, 1, 42, 1, 3, 3, 0, // z,0, 0,255,255,255, 0,255,255, 0, 0,255, 0, 0, 0, 0, 1,255,255, 0, 0, 0, 0,255, 0,255, 0,255, 0,255,255,255,255,255,255,255,255,255,255,255,255,255,255, 0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, 0,255,255,255,255, 0,255,255, // ß,180, 89, 2, 0, 5, 92, 0, 23, 0,166, 40, 4, 4, 15, 12,133,125, 0, 60, 0, 3, 37, 10, 0, 0, 13, 5,255, 9,255, 0, 8,255,255, 0,255, 0,255, 0, 4, 0, 0, 0, 0, 0,255, 3, 0, 0, 0, 1, 3, 0,255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1,255, 0, 3,255, // š,53, 6, 0, 0, 0, 25, 0, 0, 0, 6, 4, 0, 0, 1, 0, 89, 0, 0, 0, 0, 0, 1, 0, 1,255, 5, 0,255,255, 0,255,255, 0, 0, 0,255,255, 0,255, 0,255,255,255,255, 0,255,255,255, 6,255, 0,255,255,255,255, 0,255, 0, 0,255,255,255,255, 0,255,255,255,255, // ś,1, 13, 0, 0, 0, 1, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0,255, 1, 28, 0, 0, 0,255,255, 3, 0,255, 2,255, 0, 0,255,255,255,255,255,255,255, 0,255,255, 1,255,255,255, 0, 0, 0,255, 0, 3,255, 0,255,255, 0, 0, 0,255,255,255, 0, 2,255,255, 0,255, // ť,96, 34, 0, 0, 21, 89, 0, 0, 0, 28, 0, 0, 5, 1, 3,221, 0, 0,127, 0, 0, 77, 1, 0, 0, 2, 0,255, 0,255, 0, 0,255,255, 0,255, 0,255, 0, 3,255, 0, 0, 5, 0,255, 0, 1, 0,255, 5, 5,255, 0, 0,255, 0, 0, 4,255,255, 0, 2, 1,255, 0, 1,255, // ž,4, 7, 0,255, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,255, 0, 0,255, 0,255, 0,255, 0, 0,255,255, 0,255,255, 0, 0, 0,255,255, 0,255,255,255, 0,255,255,255,255,255,255, 0,255, 0,255,255,255,255,255,255, 3, 0,255,255,255,255,255,255,255,255,255, // ź,18,129, 4, 3, 21, 21, 0, 23, 1, 42, 0, 15, 0, 3, 0, 40, 11, 0, 2, 40, 2, 56, 0, 19,255, 32, 15,255, 0, 0,255, 0, 0, 0, 4,255,255, 0,255, 0,255, 0, 0,255, 0,255, 0, 0, 2, 0, 0, 0,255,255,255,255,255, 30, 0,255, 0,255, 0,255,255,255,255,255, // ł,0, 0, 2, 5, 3, 0, 0, 2, 0, 38, 44, 9, 8, 1, 16, 0, 0, 0, 7, 9, 5, 0, 0, 7, 0, 0, 28,255, 0, 0,255, 0, 0, 7, 0, 0,255, 2,255,255,255,255,255,255, 0, 0, 0, 0, 0,255,255, 0,255,255,255,255, 0, 0,255,255,255,255,255,255,255,255, 0, 0, // ą,140, 48, 0, 1, 0, 50, 0, 0, 0, 33, 0, 0, 0, 0, 1, 19, 0, 0, 4, 0, 0, 12, 0, 0, 0, 0, 0,255,255,255,255,255,255,255,255, 0,255,255,255, 0, 2, 3, 0,255,255,255,255, 0, 0,255,255, 0, 4,255,255,255,255,255,255,255, 0,255,255, 0,255, 0,255, 0, // ş,8, 3, 0, 0, 7, 56, 0, 0, 4, 1, 0, 0, 0, 0, 0, 13, 0,255, 0, 0, 0, 2, 0,255,255, 0, 0,255, 0,255,255, 0,255,255,255,255, 0,255,255, 5,255, 0,255, 0,255,255, 0,255,255,255, 0, 0,255,255,255,255,255,255, 0,255,255,255,255, 0,255,255, 0,255, // ľ,30, 16, 0, 0, 3, 26,255, 0, 0, 5, 0, 5, 0, 0, 1, 27, 0,255, 1, 0, 0, 24, 0, 0, 0, 10, 0,255,255, 0,255,255, 0, 1, 11,255,255, 0,255,255,255,255, 0,255, 0,255,255, 0, 9, 0,255,255,255, 0,255, 0,255, 5, 0,255,255,255,255,255,255,255,255,255, // ż,0, 0, 0,255, 0, 0, 0, 0, 0, 0,255, 0,255, 0,255, 0, 0,255, 0, 0, 0,255, 0,255,255,255, 0,255,255,255,255, 0,255,255,255,255,255,255, 0, 0,255,255,255,255,255,255, 0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, // ŕ,37, 0, 10, 9, 29, 2, 7, 20, 20, 13, 34, 45, 62, 52,115, 0, 15, 0, 97, 50, 87, 8, 57, 0, 0, 14, 82,255, 0, 0, 0, 3, 0, 0,255, 0, 0, 0, 0, 0,255,255,255,255, 0, 0, 7, 0,255, 0, 0, 0, 0, 0, 0, 0, 0, 0,255, 0, 0, 3,255, 0, 0, 0, 0, 0, // á,0, 0, 1, 19, 2, 0, 5, 2, 5, 0, 0, 0, 1, 48, 1, 0, 8, 0, 17, 2, 12, 0, 9, 0, 0, 0, 1,255, 0,255,255, 0,255,255,255, 0,255,255,255,255, 0, 0,255,255, 0, 0, 0, 0,255,255,255,255, 0,255, 0,255, 0,255,255,255,255,255,255,255,255, 0,255, 0, // â,0, 0, 14,106, 14, 0, 9, 21, 1, 0, 0, 0, 51, 27, 62, 0, 36, 0, 72, 66,133, 7, 12, 0, 1, 0, 28,255, 0,255,255, 0,255,255,255, 0,255,255,255,255, 0, 0,255,255,255, 0, 0,255,255,255,255,255, 0,255, 0, 0,255,255,255,255,255,255,255,255,255,255,255, 35, // ă,0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0,255, 0,255,255, 0, 0, 0,255, 0,255, 0,255,255,255,255, 0,255,255, 0, 0,255,255,255,255,255,255, 0,255,255, 0,255,255,255, 0,255,255,255,255, 0, 0,255, // ä,0, 0, 0,255, 5, 0,255, 0, 0, 0,255, 0,255, 0, 0, 0, 0,255, 0, 0, 1, 0, 0,255,255,255,255,255,255,255,255, 0,255,255,255,255,255,255,255,255,255,255,255,255,255,255, 0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, // ĺ,15, 18, 0, 0, 0, 91, 0, 0, 0, 96, 0, 1, 0, 0, 0, 29,156,255, 1, 0, 0, 37, 0, 0, 0, 4, 0,255, 14, 40,255, 0, 0, 0, 1,255,255, 0,255, 0,255,255,255,255, 0,255, 0, 0, 1,255, 0, 0,255,255,255, 0,255, 0,255,255,255,255,255,255,255,255,255,255, // ć,0, 0,255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0,255,255, 0, 0, 0,255,255,255,255,255,255,255,255, 0,255,255,255, 0, 0,255, 0,255,255, 0,255, 0,255,255,255, 0,255,255,255,255,255,255, 0,255, 0,255,255,255,255, 0, 0,255, // ç,118,219, 32, 0, 1, 87, 0, 0, 2,176, 3, 1, 3, 12, 26,106, 2, 0, 12, 4, 0, 54, 5, 0, 0, 1, 0,255, 76, 0, 0, 0,255, 0,255,255, 0,255, 0, 6, 0, 0, 3, 0, 0,255, 0, 0,255,255, 0, 3,255,255, 0,255, 0, 0, 0,255, 0, 0, 0, 10,255, 0, 0,255, // č,106, 1, 6, 3, 16, 0, 11, 14, 4, 2, 5, 60, 49, 41, 86, 0, 11, 0, 28, 50, 74, 1, 32, 0, 0, 10, 33, 0, 0,255, 0, 0, 0, 0,255, 0,255, 0,255, 0,255,255,255,255, 0, 0, 0, 0,255,255,255, 0,255, 0, 0, 0, 0, 0,255, 0, 0, 0,255, 0, 0, 0, 0, 0, // é,0, 0, 4, 1, 2, 0, 0, 4, 0,101, 53, 10, 5, 7, 32, 0, 2, 0, 14, 2, 41, 0, 0, 7, 0, 0, 26,255, 0, 0,255, 0, 0, 5, 0, 0, 0, 1,255,255,255,255,255,255, 0, 0, 0, 0, 0, 0,255, 0,255,255,255, 0,255,255,255,255,255,255,255, 0,255,255, 0, 0, // ę,0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,255, 0,255,255,255,255, 0,255,255,255, 0,255,255,255,255,255,255,255, 0, 0,255,255, 0,255,255, 0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, // ë,0, 0, 10, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 36, 54, 0, 6, 0, 0, 0, 23, 0, 37, 0,255, 0, 0,255, 0,255, 0, 0, 0,255,255,255,255,255,255, 0,255,255,255,255, 0,255, 0, 0,255,255, 0, 0,255, 0,255, 0, 0,255,255,255,255, 0,255,255,255,255,255, 0, // ě,11, 1, 6, 25, 14, 2, 1, 2, 5, 3, 21, 5, 12, 21,177, 0, 7, 0, 38, 11, 28, 1, 26, 0, 0, 7, 14,255, 29, 0, 0, 7, 0,255,255, 0, 0,255, 0, 4,255, 0,255,255, 0, 0, 14, 0,255,255, 0, 0, 0, 0, 0,255, 0, 0,255, 0,255, 51,255, 0, 0, 0,255, 0, // í,239, 0, 1, 0, 2, 1, 0, 0, 1, 3, 2, 9, 8, 7, 7, 0, 1, 0, 7, 3, 5, 0, 4, 0, 0, 0, 0,255,255,255,255,255,255,255,255, 1,255,255,255, 0, 0, 0,255,255,255, 0, 0,255, 0, 0,255, 0, 0,255,255,255,255,255,255,255,255,255,255,255,255,255,255, 0, // î,4, 4, 0,255, 0, 4,255,255,255, 0, 0,255, 0, 0, 0, 0,255,255, 0,255, 0, 0, 1,255,255, 0, 0,255,255,255,255, 0,255,255,255,255,255,255,255, 0,255,255, 0,255,255,255, 0, 0,255,255, 0, 0,255, 0,255,255, 0, 0,255,255,255,255,255, 0,255,255,255,255, // ď,4, 31, 0,255, 0, 41,255, 0, 0, 1, 0, 0, 0, 0, 1, 69, 0,255, 4, 0,255, 3, 0, 0,255, 0, 0,255, 0,255,255, 63,255,255,255,255,255,255,255, 0,255,255,255,255,255,255, 0,255,255,255,255,255,255,255, 0,255,255, 0,255,255,255,255,255,255,255,255,255,255, // đ,0, 48,255, 0, 0, 16,255,255, 0, 12, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 1,255, 0,255, 6, 0,255,255, 0,255,255, 0, 0, 0, 0,255, 0,255, 0,255,255,255,255,255,255,255, 0, 0, 0,255, 0,255,255, 0, 0,255, 0, 0,255,255,255, 0, 0,255,255,255,255, // ń,0, 1, 0, 0, 3, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0,255, 0, 0, 0, 0, 0,255,255, 0, 0,255, 0,255, 0, 0,255,255,255,255, 0,255, 0, 0,255,255,255, 0,255,255, 0, 0,255,255, 0, 0,255, 0,255, 0, 0, 0, 0,255,255, 0, 0, 0,255, 0, 0,255, // ň,3, 0, 8, 9, 10, 2, 3, 53, 3, 17, 6, 22, 42, 5, 7, 0, 26, 0, 52, 7, 32, 0, 2, 31, 0, 2, 20,255, 0, 0,255, 0, 0, 16,255,255,255, 1,255, 0,255,255, 0,255, 0, 0, 0, 0, 0,255,255, 0,255,255,255, 0, 0, 0,255, 0,255, 0,255, 0, 0, 0, 0, 0, // ó,0, 0, 0, 1, 3, 0, 0, 0, 1, 0, 0, 3, 0, 4, 0, 0, 13,255, 1, 0, 0, 0, 1, 0, 0, 0, 0,255,255,255,255, 0,255, 0,255,255, 0, 0,255,255,255,255,255,255,255, 0,255, 0,255,255,255, 0,255,255, 0,255,255, 0, 0,255,255,255,255,255,255,255,255,255, // ô,4, 0, 5, 0, 12, 0, 7, 1, 0, 0, 0, 5, 22, 0, 9, 0, 0,255, 9, 13, 33, 0, 2, 0, 0, 3, 23,255,255,255,255, 0,255,255,255,255,255,255,255, 0,255,255,255,255,255, 0,255, 0,255,255,255, 0,255,255,255,255,255, 0,255, 0, 0,255,255, 0, 0, 0,255,255, // ő,13, 0, 2, 0, 6, 0, 5, 3, 2, 0, 1, 57, 7, 1, 5, 0, 0,255, 8, 2, 22, 0, 1, 0, 0, 2, 15,255, 0,255,255, 0,255, 0,255, 0,255,255,255, 0,255,255, 0,255,255, 0, 0, 0,255,255,255, 0,255,255,255,255,255, 0,255, 0, 0,255,255, 0, 0, 0,255,255, // ö,16, 6, 5, 0, 3, 4, 0, 0, 2, 2, 0, 4, 0, 1, 0, 12, 66,255, 0, 0, 19, 2, 1, 0, 0, 2, 1,255, 0,255,255, 0,255,255,255,255,255,255, 0, 9,255,255,255,255,255,255, 0, 0,255, 0, 3, 5,255,255,255,255, 0, 0,255,255,255, 0, 0, 2,255,255, 0,255, // ř,0, 0, 0, 2, 5, 0, 0, 0, 2, 0, 1, 5, 4, 3, 3, 0, 7, 0, 10, 1, 4, 0, 3, 0, 0, 0, 1, 0, 0,255, 0, 0,255, 0,255, 0,255,255,255, 0,255,255,255,255, 0,255, 0, 0,255,255,255, 0,255, 0,255,255, 0, 0,255,255, 0, 0, 0, 0,255, 0, 0,255, // ů,27, 0, 0, 36, 2, 0, 0, 1, 1, 4, 26, 8, 4, 2, 8, 0, 1, 0, 14, 40, 14, 0, 2, 0, 0, 1, 4,255, 0,255, 0, 0,255,255,255,255, 0,255,255, 0,255,255,255,255, 0, 0, 0, 0,255,255, 0, 0,255, 0, 0,255, 0, 0,255, 0,255, 0, 0, 0, 0,255, 0, 0, // ú,0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0,255, 4, 3, 3, 0, 1, 0, 0, 2, 1,255,255,255,255,255,255,255,255,255,255,255,255, 0,255,255,255,255,255,255,255, 0,255,255,255, 0,255,255,255,255,255, 0,255, 0,255,255,255, 0, 0, 0,255,255, // ű,5, 0, 2, 0, 0, 0, 3, 1, 1, 0, 0, 8, 2, 2, 1, 0, 15, 0, 14, 4, 4, 0, 1, 1, 0, 3, 13,255, 0,255, 0, 0,255,255,255, 0,255, 0,255, 0,255,255, 0,255,255, 0, 0, 0,255, 0,255, 0,255,255,255,255, 0, 0,255, 0, 0,255,255, 0, 0, 0,255,255, // ü,0, 0, 4, 0, 2, 0, 0, 0, 3, 0, 0, 52, 4, 1, 69, 0, 0,255, 13, 0, 11, 0, 56, 0,255, 0, 1,255, 0,255,255, 0,255,255,255,255,255,255,255, 0,255,255, 0,255,255,255, 0, 0,255,255, 0, 0,255,255,255,255, 0,255,255,255, 0, 0, 0, 0,255, 0, 0,255, // ý,20,111, 1, 33, 0, 31, 0, 0, 0, 31, 0, 0, 5, 1, 78, 5, 2, 0, 10, 1, 0, 25, 2, 0, 0, 0, 0,255,255,255,255,255,255,255,255, 0,255,255,255, 0, 0, 25,255,255,255,255,255, 0,255,255,255, 0, 4,255,255,255,255, 0,255,255,255,255,255, 0,255,255,255, 0, // ţ,// , a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t, u, v, w, x, y, z, ß, š, ś, ť, ž, ź, ł, ą, ş, ľ, ż, ŕ, á, â, ă, ä, ĺ, ć, ç, č, é, ę, ë, ě, í, î, ď, đ, ń, ň, ó, ô, ő, ö, ř, ů, ú, ű, ü, ý, ţ,],cyrillic: [0, 0, 0, 0, 1, 0, 16, 38, 0, 2, 5, 10,121, 4, 20, 25, 26, 53, 9, 5, 61, 23, 20, 26, 15, 95, 60, 2, 26, 15, 25, 29, 0, 14, 6, 6, 25, 1, 0, 27, 25, 8, 5, 39, // ,0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // a,0, 0, 0,255, 0, 0,255, 0,255, 0,255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,255,255, 0, 0,255, 0, 0, // ѓ,0, 0,255, 0, 0, 0, 0, 0,255,255,255,255, 0,255, 2, 0, 0, 0, 0, 0, 0, 0, 0,255, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,255,255,255,255,255,255,255, // ђ,0, 0, 0, 0, 0, 0, 0, 0,255,255,255,255, 0,255, 0, 0, 0, 0, 0, 4, 0, 0, 0,255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,255,255,255,255,255,255,255, // љ,0, 0, 0, 0, 0, 0, 0, 0,255,255,255, 0, 0,255, 5, 0, 0, 0, 0, 2, 0, 0, 0,255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,255,255,255,255,255,255,255, // њ,0, 0,255, 0, 0, 0, 0, 0,255, 0,255,255, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1,255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,255,255,255,255,255,255,255, // ћ,0, 0, 0, 0, 0, 0, 0, 0,255,255,255,255, 0,255, 0, 0, 0, 0, 0, 0, 0, 0, 0,255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,255,255,255,255,255,255,255, // џ,7, 0, 0,255,255,255,255,255, 0, 1, 0,255,255,255, 15, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,255,255, 1, 0, 0, 0, 1, // ў,12, 0, 0,255,255, 0,255,255, 0, 2, 0, 0, 0, 0, 2, 3, 15, 5, 5, 0, 0, 4, 0, 0, 21, 15, 10, 17, 0, 6, 14, 4, 6, 0, 3, 1, 8, 1, 0, 0, 0, 2, 0, 0, 0, 0, // і,0, 0,255,255,255,255,255,255, 0, 0, 0,255,255, 0, 4, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // ё,6, 0, 0,255,255,255,255,255, 0, 0,255, 5,255, 0, 1, 7, 0, 3, 2, 0, 0, 0, 0, 0, 0, 2, 2, 2, 0, 0, 2, 2, 5, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // є,12, 0, 0, 0, 0, 0, 0, 0,255, 0,255, 0, 0, 0, 5, 1, 0, 0, 0, 2, 0, 0, 20,255, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,255,255, 0,255,255,255,255, // ј,9, 0, 0,255,255,255,255,255,255, 5,255, 0, 0, 13, 3, 3, 0, 4, 1, 0, 1, 2, 0, 0, 0, 1, 0, 0, 4, 0, 0, 1, 3, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // ї,32, 0, 0, 2, 2, 2, 0, 0, 0, 1, 0, 0, 28, 0, 23, 22, 26, 22, 19, 0, 3, 12, 5, 0, 44, 38, 18, 58, 1, 21, 44, 17, 54, 1, 2, 28, 5, 8, 3, 1, 9, 0, 12, 0, 0, 0, // а,40, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7, 0, 0, 0, 1, 7, 0, 1, 1, 0, 0, 7, 4, 1, 9, 0, 1, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, // б,31, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 3, 0, 0, 19, 0, 0, 1, 1, 6, 0, 2, 6, 0, 1, 0, 1, 0, 32, 0, 2, 2, 23, 9, 0, 0, 0, 1, 0, 0, 1, 1, 0, 3, 0, 2, // в,23, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7, 0, 1, 20, 0, 0, 1, 0, 9, 0, 0, 9, 7, 0, 5, 2, 18, 11, 0, 8, 3, 2, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 13, 0, 3, // г,26, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 2, 0, 2, 19, 0, 1, 5, 0, 13, 2, 2, 3, 2, 0, 6, 1, 12, 30, 0, 4, 0, 0, 7, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5, 0, 1, // д,12, 0, 0, 1, 4, 5, 0, 0, 0, 0, 0, 0, 24, 1, 5, 7, 11, 3, 12, 1, 6, 6, 11, 0, 3, 15, 14, 14, 4, 8, 25, 14, 29, 0, 1, 1, 4, 8, 8, 2, 0, 3, 1, 0, 0, 0, // е,6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 2, 2, 0, 0, 0, 0, 0, 3, 2, 1, 2, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, // ж,19, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 6, 0, 0, 0, 11, 8, 0, 0, 8, 0, 0, 0, 0, 0, 4, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, // з,24, 0, 0, 0, 0, 1, 5, 0, 0, 0, 0, 0, 1, 0, 1, 10, 16, 21, 22, 0, 6, 5, 6, 1, 15, 15, 8, 38, 2, 4, 27, 9, 15, 0, 3, 8, 12, 7, 6, 1, 0, 0, 0, 0, 0, 0, // и,6, 0, 0, 0,255,255,255,255, 0, 7, 0, 0,255, 4, 21, 0, 0, 0, 0, 5, 0, 0, 39, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 5, 0, 3, 0, 0, // й,54, 0, 0, 0, 0, 0, 0, 0, 1, 8, 0, 0, 0, 0, 10, 0, 1, 0, 1, 11, 0, 0, 12, 0, 1, 2, 0, 4, 8, 0, 2, 23, 2, 4, 0, 2, 3, 3, 8, 0, 0, 3, 16, 1, 4, 3, // к,12, 0, 0, 0, 0, 0, 0, 0, 2, 6, 0, 6, 0, 4, 29, 12, 4, 5, 2, 18, 0, 0, 17, 4, 5, 11, 0, 0, 21, 2, 3, 4, 1, 15, 1, 0, 0, 0, 0, 0, 4, 3, 2, 12, 0, 2, // л,23, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4, 0, 0, 17, 1, 0, 0, 0, 7, 0, 1, 13, 2, 0, 0, 0, 0, 13, 0, 2, 4, 0, 2, 0, 0, 0, 0, 0, 0, 1, 4, 2, 4, 1, 1, // м,42, 0, 0, 0, 0, 0, 0, 0, 4, 12, 6, 7, 1, 7, 76, 0, 22, 1, 4, 27, 1, 3, 34, 30, 0, 7, 1, 13, 24, 1, 3, 5, 3, 4, 0, 1, 0, 4, 1, 0, 2, 18, 7, 16, 0, 4, // н,37, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 1, 10, 27, 22, 15, 1, 2, 3, 7, 5, 32, 11, 7, 38, 8, 21, 24, 11, 23, 0, 2, 10, 2, 2, 3, 2, 0, 0, 1, 0, 0, 0, // о,47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 0, 2, 0, 1, 2, 4, 0, 0, 2, 0, 6, 0, 0, 5, 0, 2, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, // п,19, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8, 0, 5, 47, 4, 6, 6, 5, 23, 0, 0, 5, 2, 6, 0, 0, 0, 23, 22, 0, 1, 14, 9, 1, 0, 1, 0, 0, 0, 7, 2, 8, 16, 0, 3, // р,53, 0, 0, 0, 0, 0, 0, 0, 4, 9, 2, 0, 1, 2, 21, 1, 4, 1, 2, 11, 0, 0, 12, 2, 4, 7, 1, 13, 15, 1, 4, 6, 3, 6, 0, 0, 0, 0, 0, 0, 1, 2, 3, 5, 0, 1, // с,28, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 1, 0, 1, 32, 0, 1, 3, 0, 12, 0, 1, 22, 1, 4, 7, 1, 6, 23, 0, 14, 41, 14, 3, 0, 1, 1, 1, 21, 0, 2, 2, 6, 2, 1, 4, // т,15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2, 4, 2, 4, 6, 3, 0, 2, 0, 0, 6, 5, 6, 3, 0, 3, 7, 4, 7, 18, 1, 6, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, // у,8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // ф,41, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 2, 30, 0, 2, 0, 0, 11, 0, 0, 5, 1, 14, 3, 0, 3, 6, 0, 7, 0, 0, 1, 0, 1, 0, 2, 0, 0, 0, 4, 3, 5, 0, 0, // х,8, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 7, 0, 0, 0, 0, 4, 0, 0, 7, 1, 0, 1, 0, 2, 1, 0, 0, 9, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 0, 0, 1, 1, // ц,6, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 5, 0, 1, 5, 0, 2, 0, 0, 6, 0, 0, 1, 0, 0, 3, 0, 2, 0, 0, 2, 0, 1, 0, 0, 3, 0, 0, 2, 0, 0, 0, 0, // ч,12, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 17, 0, 0, 1, 0, 2, 0, 0, 26, 0, 0, 0, 0, 0, 22, 2, 6, 0, 0, 5, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, // ш,2, 0,255, 0,255,255,255,255,255, 0, 0, 0,255, 0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, // щ,0, 0,255,255,255,255, 0,255, 0, 0, 0,255,255,255, 0, 3, 4, 0, 2, 0, 0, 0, 0, 0, 11, 0, 1, 0, 0, 2, 2, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // ъ,1, 0, 0,255,255,255,255,255, 0, 0, 0, 0, 0,255, 0, 3, 11, 0, 4, 0, 2, 1, 0, 0, 0, 3, 1, 16, 0, 0, 22, 2, 10, 0, 0, 0, 8, 6, 3, 0, 0, 0, 0, 0, 0, 0, // ы,0, 0, 0,255,255, 0, 0, 0,255, 0, 0, 0, 0, 0, 5, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 7, 3, 0, 1, 13, 7, 7, 0, 35, 6, 0, 0, 0, 0, 0, 0, 0, 6, 0, // ь,10, 0, 0,255,255,255,255,255, 0, 0, 0, 0,255, 0, 0, 1, 1, 10, 11, 0, 2, 2, 0, 0, 0, 9, 3, 9, 0, 0, 7, 6, 9, 0, 0, 8, 3, 2, 1, 0, 0, 0, 0, 17, 0, 0, // э,14, 0, 0, 0,255,255,255,255, 0, 0, 0, 0,255, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // ю,5, 0, 0,255,255,255,255,255, 0, 9, 0, 0,255, 0, 11, 0, 3, 0, 0, 0, 0, 2, 24, 0, 0, 5, 2, 14, 1, 0, 2, 3, 1, 0, 0, 1, 3, 0, 0, 0, 0, 16, 1, 0, 0, 0, // я,// , a, ѓ, ђ, љ, њ, ћ, џ, ў, і, ё, є, ј, ї, а, б, в, г, д, е, ж, з, и, й, к, л, м, н, о, п, р, с, т, у, ф, х, ц, ч, ш, щ, ъ, ы, ь, э, ю, я,],western: [18, 3, 0,254, 74, 0, 5,254,254, 2, 25,254,149, 4,254, 66,148,254, 0,254,122,238, 8, 1, 20, 13,254, 35, 20, 3, 1, 0, // ,0, 3, 0, 0, 0, 0, 0, 5, 2, 0, 86, 9, 76, 0, 0, 0,241, 0, 0, 49, 0, 0, 0, 0, 11, 2, 0, 34, 0, 1, 2, 0, // a,19, 0, 0, 5, 5, 0, 0, 8, 13, 5, 0, 34, 22, 0, 0, 0, 4, 0, 0, 0, 6, 1, 3, 3, 42, 37, 8, 8, 0, 67, 0, 0, // b,0, 0, 0, 9, 6, 1, 0, 22, 10, 1, 0, 19, 54, 1, 0, 1, 18, 3, 1, 2, 40, 7, 0, 0, 6, 0, 3, 5, 1, 34, 0, 0, // c,0, 0, 0, 5, 5, 0, 0, 12, 45, 16, 1, 6, 42, 0, 13, 3, 10, 0, 2, 0, 66, 11, 5, 8, 33,104, 3, 4, 0, 19, 0, 0, // d,63, 5, 0, 0, 0, 0, 2, 33, 15, 1, 3, 0, 87, 0, 0, 0, 0, 0, 1, 21, 0, 0, 0, 49, 1, 11, 0, 3, 0, 9, 1, 0, // e,0, 0, 0, 8, 8, 0, 0, 10, 2, 7, 0,162, 23, 0, 13, 0, 4, 0, 0, 0, 1, 3, 0, 0, 15, 4, 0, 0, 0, 4, 0, 0, // f,1, 0, 0, 14, 16, 24, 0, 29, 11, 41, 0, 13, 86, 0, 14, 9, 3, 0, 0, 0, 20, 8, 7, 7, 13, 37, 14, 0, 0, 12, 0, 0, // g,1, 0, 0, 0, 0, 0, 0, 47, 2, 0, 0, 0, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 29, 20, 0, 0, 0, 0, 45, 0, 0, // h,5, 4, 0,166,120, 0, 0,144, 0, 2, 3, 88,254, 0, 0, 0, 0, 0, 0, 3, 28,107, 0,112, 8, 2, 44, 32, 0, 3, 3, 0, // i,0, 0, 0, 0, 0, 0, 0, 39, 9, 0, 0, 2, 1, 0, 2, 0, 0, 0, 0, 4, 0, 0, 0, 16, 18, 44, 0, 0, 0, 0, 0,255, // j,0, 2, 0, 0, 1, 0, 0, 48, 31, 32, 1, 60, 1, 0, 4, 0, 1, 0, 0, 0, 1, 3, 0, 2, 20, 47, 0, 0, 0, 20, 0, 0, // k,4, 0, 0, 12, 16, 0, 0, 54, 40, 48, 0, 64, 36, 0, 39, 6, 12, 3, 0, 0, 27, 9, 3, 24, 42, 33, 2, 9, 7, 77, 0, 0, // l,0, 0, 0, 14, 5, 4, 0, 60, 11, 4, 3, 48, 30, 7, 28, 1, 10, 1, 0, 0, 24, 41, 3, 3, 19, 24, 1, 8, 2, 36, 0, 0, // m,1, 1, 0, 24, 91, 16, 0,132, 62, 73, 1, 56, 71, 33, 78, 7, 35, 2, 3, 0, 94,254, 10, 21, 33, 38, 24, 21, 1, 61, 0, 0, // n,0, 1, 0, 0, 0, 0,254, 6, 0, 1, 27, 0, 13, 0, 0, 84,127, 0, 0, 62, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, // o,0, 0, 0, 5, 2, 0, 0, 9, 15, 0, 0, 4, 34, 0, 6, 0, 6, 0, 0, 0, 20, 12, 9, 28, 10, 22, 0, 3, 0, 7, 0, 0, // p,0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 33, 1, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0,255,255, // q,0, 0, 0, 83, 62, 1, 0,198,139,125, 0,229, 94, 54,190, 38, 18, 1, 0, 0,176, 24, 16, 29,193,181, 13, 13, 2,131, 0, 0, // r,1, 0, 0, 41, 34, 0, 0, 41, 24, 42, 0, 68,113, 15,159, 6, 43, 19, 4, 58, 14, 18, 1, 4, 48, 42, 4, 12, 9, 20, 0, 0, // s,7, 1, 0, 14, 20, 8, 0, 56, 37, 31, 0,104, 67, 14,113, 3, 50, 9, 5, 0, 89, 7, 19, 22, 13, 14, 40, 12, 15, 18, 0, 0, // t,0, 1, 5, 1, 2, 0, 0, 30, 0, 0, 1, 15, 2, 0, 1, 0, 1, 0, 0, 2, 4, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, // u,0, 2, 0, 1, 6, 0, 0, 29, 33, 13, 0, 19, 46, 0, 15, 0, 7, 0, 1, 31, 2, 2, 3, 1, 32, 27, 0, 0, 1, 1, 0, 0, // v,0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3, 0, 0, 4, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0,255, // w,0, 0, 0, 1, 16, 0, 0, 23, 0, 0, 0, 3, 14, 0, 0, 0, 2, 3, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0,255, 0, // x,0, 0, 0, 0, 0, 0, 0, 58, 8, 0, 0, 1, 1, 62, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 6, 82, 0, 0, 0, 0, 0,255, // y,0, 0, 0, 0, 2, 0, 0, 0, 14, 0, 0, 7, 3, 0, 6, 0, 3, 5, 0, 0, 0, 0, 4, 0, 1, 0, 0, 0, 0, 0, 0, 0, // z,0, 29, 0, 0, 0, 15, 0, 0, 0, 11, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0,255,255, 0, 0,255,255, 4, 0, 0,255,255, 0,255, 0,255, 0, 0,255,255,255, 0, 0, 0, 8, 0,255, 0, 0, 2, 0, 0, // ß,6, 2, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 10, 1, 0, 0, 0, 0, 0, 0, 0,255, 0, 1, 0, 0, 0, 0, 0,255, 0, 0, 0, 0, 0, 0, 0,255,255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // š,3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,255, 0, 0, 0, 0, 0, 0, 0, 0, 0,255, 0, 0,255,255,255, 0, 0, 0,255,255,255, 0,255,255,255,255, 0, 0,255,255,255,255,255,255, 0,255,255,255, 0,255,255, // œ,107, 0, 22, 16, 18, 14, 6, 24, 46, 15, 2, 0, 42, 18, 17, 0, 36, 0, 34, 4,254, 1, 2, 0, 0, 1, 0, 0, 0,255, 0, 0, 0, 0, 0,255,255, 0, 0, 0, 0, 0, 0, 0,255, 0, 0, 0,255,255,255,255,255, 0, 0,255, 0, 0, 0, // à,41, 0, 10, 8, 21, 34, 5, 5, 60, 18, 5, 1, 29, 42, 26, 2, 16, 0, 27, 9, 43, 28, 7, 0, 0, 1, 4, 0, 0,255, 0, 0,255,255,255, 0,255, 0, 0, 0, 0,255, 0, 0, 0, 0, 0, 0, 0, 0,255,255,255, 0, 0, 0, 0, 0,255, // á,24, 0, 1, 2, 0, 0, 0, 0, 7, 0, 0, 0, 3, 1, 0, 0, 0, 0, 2, 0, 5, 0, 1, 0, 0, 0, 0,255, 0,255, 0, 0, 0,255, 0,255, 0, 0, 0, 2, 0,255, 0,255, 0, 0, 0, 0,255, 0,255,255,255,255,255, 0,255, 0,255, // â,0, 0, 0, 1, 2, 3, 0, 1, 2, 12, 0, 0, 1, 7, 29, 4, 1,255, 11, 66, 11, 0, 1, 0, 0, 0, 0,255, 0,255,255,255, 0, 0, 0,255,255,127,255,255,255,255,255, 0, 0,255, 0, 0,255,255, 0,255,255,255,255,255,255,255,255, // ã,134, 1, 11, 0, 25, 6, 15, 11, 61, 24,123, 95,114, 68, 53, 1, 49, 0, 60, 98,198, 0, 88, 29, 0, 6, 12, 0, 0,255, 0,255, 0, 0,118, 0,255, 0,255, 0,255, 0,255, 0,255,255, 0,255,255, 0,255, 2,255,255,255, 0, 0, 0,255, // ä,156, 0, 12, 14, 19, 3, 12, 47, 17, 3, 12, 5, 30, 47, 22, 0,205, 0,184, 70, 19, 0, 22, 8, 0, 6, 1,255, 0,255,255, 0,255, 0, 0, 0, 0, 0,255, 0,255, 0,255, 0, 0,255,255,255,255,255,255, 0, 0,255,255,255,255,255,255, // å,26, 0, 7, 0, 4, 0, 23, 8, 15, 0, 18, 19, 56, 23, 24, 0, 9, 0, 82, 37, 24, 0, 71, 0, 0, 0, 0,255, 0,255,255, 0,255,255, 0, 0, 0, 0,255, 0,255,255,255, 0,255,255, 0,255,255,255,255, 0, 0,255,255,255,255, 0,255, // æ,17,112, 0, 2, 0, 15, 0, 0, 0, 35, 0, 0, 2, 0, 59, 9, 1, 0, 36, 0, 0, 8, 0, 0, 0, 0, 0,255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,255,255, // ç,254, 0, 9, 14, 20, 0, 15, 6, 70,144, 14, 45, 47, 92, 16, 3,123, 0, 38, 23,115, 52, 22, 42, 2, 80, 19,255, 0,255, 0, 0,255,255, 0,255,255, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0,255, 0,255,255,255, 0, 0, 0, 1,255,255, // è,152, 2, 19, 24, 85, 0, 29, 23, 26, 25, 2, 9, 43, 60, 62, 1, 32, 0,122, 45,169, 15, 13, 30, 7, 4, 8, 0, 0,255, 0, 0, 0, 0, 0,255, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1,255, 0, 0, 0, 0,255, 0, 0, 0, 0, 0, 0, // é,5, 0, 0, 3, 7, 0, 0, 10, 2, 3, 0, 26, 6, 6, 20, 1, 2, 0, 20, 1, 11, 5, 5, 2, 0, 0, 1,255, 0,255,255,255, 0,255,255,255,255, 0, 0, 0, 0, 0,255, 0, 0, 0, 0,255, 0, 0,255,255,255, 0,255, 0, 0, 0,255, // ê,36, 2, 23, 15, 36,143, 5, 23, 52, 52, 66, 48, 92, 57,216, 10,125, 35, 89, 58,254, 9, 24, 14, 0, 0, 8,255, 0,255, 0,255,255,255, 0, 0,255, 1, 0, 0, 0, 0, 0,255, 0, 0, 0,255,255,255, 0, 0, 0, 0,255, 0, 0, 0,255, // ë,12, 0, 1, 4, 6, 0, 3, 21, 10, 0, 0, 0, 18, 8, 4, 0, 1, 0, 65, 35, 8, 3, 0, 0, 0, 0, 0,255, 0,255, 0, 0,255,255,255,255,255,255, 0, 0, 0,255, 0, 0, 0,255, 0, 0,255, 0,255,255,255, 0,255,255, 0, 0,255, // ì,40, 72, 7, 10, 16, 2, 23, 10, 34, 0, 0, 1, 34, 15, 21, 1, 3, 0,203, 28, 58, 23, 11, 0, 10, 0, 2, 0, 0, 0, 0, 0, 0,255, 0,255,255, 0, 0, 0, 0,255, 0, 0,255,255, 1,255, 0,255,255, 0,255,255, 0,255, 2, 0,255, // í,6, 5, 1, 9, 5, 0, 0, 0, 22, 0, 9, 8, 8, 6, 9, 1, 10, 0, 20, 6,182, 0, 13, 0, 0, 24, 1,255, 0,255,255,255, 0, 0,255, 0,255, 0,255, 0, 0, 0, 0, 0, 0, 0, 0, 0,255, 0,255,255,255,255,255, 0,255,255,255, // î,0, 6, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0,255,255, 0, 0, 0, 0,255, 0,255, 0, 0, 0, 0, 0, 0,255, 0, 0, 0, 0,255, 0, 0, 0, 0,255,255, 0, 0, 0,255,255, // ï,0,254, 0, 0, 0, 26, 0, 0, 0, 61, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0,255,255,255, 0, 0, 0, 0, 0, 0,255,255, 0, 0, 0,255, 0, 1, 0, 0, 0,255, 0, 0, 0, 0, 0, 0, 0,255, 0,255,255, // ñ,20, 0, 56, 43, 8,162, 14, 3, 23, 19, 2,118, 31, 26, 46, 0, 20, 0, 23, 6, 24, 19, 6, 21, 5, 27, 63,255, 0,255, 0, 0,255,255,255,255,255, 3, 0,255,255,255, 0, 0,255, 0, 0, 0, 0,255, 0,255,255, 0,255,255, 0,255,255, // ò,67, 0, 12, 15, 9, 7, 8, 66, 13,254, 3, 23, 14, 16, 16, 0, 8, 0, 29, 11, 26, 0, 5, 5, 1, 10, 13,255, 0,255,255, 0,255, 0, 0,255,255, 1,255, 0,255,255, 0, 0,255, 0, 1, 0, 0, 0, 0,255,255,255, 0,255,255, 0,255, // ó,18, 3, 3, 12, 1, 0, 2, 0, 7, 0, 1, 0, 2, 2, 8, 0, 6, 0, 6, 7, 4, 0, 2, 0, 0, 0, 1,255, 0, 0,255, 0, 0,255,255,255, 0, 0, 0, 0, 0,255,255, 0, 0, 0, 0, 0, 0, 0,255,255,255,255, 0, 0,255,255,255, // ô,29, 2, 0, 0, 0, 0, 0, 0, 5, 2, 22, 30, 25, 38, 19, 0, 33,255, 4, 39, 24, 0, 88, 0, 0, 0, 0,255, 0,255,255, 0,255, 0,255,255,255, 36,255,255,255,255,255, 0,255,255, 0,255, 0, 0, 6, 0,255,255,255, 0, 0, 0,255, // õ,44, 0, 33, 0, 25, 0,142, 5, 46, 10, 25, 32, 26, 13, 6, 0, 3, 0, 30, 8, 35, 0, 25, 5, 0, 44, 7, 0, 0,255,255, 0,255,255, 73, 0,255, 0, 0, 0,255,255,255,255,255, 0, 0,255, 0, 0, 0, 39, 0,255,255,255, 0, 0, 0, // ö,52, 0, 21, 0, 57, 0,119, 12, 47, 3, 59, 33, 45, 15, 12, 0, 3, 0, 52, 82, 49, 1, 11, 0, 0, 0, 0, 0,255, 0,255,255,255,255,255, 0, 0, 0,255, 0,255,255,255, 0,255,255, 0,255,255,255,255, 0, 0,255,255,255,255,255, 0, // ø,25, 0, 4, 3, 53, 0, 0, 2, 12, 72, 0, 0, 30, 0, 0,254, 0, 0, 6, 3, 3, 0, 0, 0, 0, 0, 0,255, 0,255, 0,255, 0,255,255,255,255, 0, 0, 0, 0,255, 0,255,255,255,255, 0,255, 0, 0,255,255, 0, 0, 0, 0, 0, 0, // ù,19, 2, 1, 7, 9, 1, 12, 5, 9, 41, 1, 0, 10, 7, 9, 0, 8, 0, 12, 28, 8, 0, 0, 0, 0, 1, 0,255, 0,255,255, 0,255,255,255,255, 0, 0,255, 0,255,255,255, 0,255,255, 0, 0, 0,255, 0,255,255, 0, 0,255,255, 0,255, // ú,0, 0, 0, 0, 1, 5, 0, 0, 1, 0, 0, 0, 0, 0, 0, 45, 0, 0, 3, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0,255,255,255, 0,255,255,255,255, 0,255, 0,255,255,255, 0, 0,255,255,255,255, 0,255,255,255, 0,255, 0, 0,255, 0, // û,95, 2, 19, 0, 6, 2,121, 9, 15, 1, 5, 44, 18, 26, 7, 0, 11, 2, 68, 49, 20, 0, 2, 17, 0, 0, 6, 0, 0,255, 0,255,255,255, 0,255,255, 0,255, 0,255, 0,255,255,255, 0, 0,255,255,255, 0, 0,255, 0, 0, 0, 31, 0, 0, // ü,1, 1, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,255, 0, 0, 0, 0, 0, 0, 0, 0,255, 0, 0,255,255, 0, 0,255, 0,255, 0,255,255,255,255, 0, 0, 0, 0,255, 0, 0, 0, 0, 0,255, // ž,0, 0, 0, 0, 0, 0,255, 0, 0,255, 0, 0, 0, 0, 0, 0, 0,255, 0, 0, 0, 0, 0, 0,255,255, 0,255,255,255,255,255,255, 0,255, 0,255,255,255,255,255,255,255,255,255,255,255,255,255, 0, 0,255, 0,255,255,255, 0, 0, 0, // ÿ,// , a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t, u, v, w, x, y, z, ß, š, œ, à, á, â, ã, ä, å, æ, ç, è, é, ê, ë, ì, í, î, ï, ñ, ò, ó, ô, õ, ö, ø, ù, ú, û, ü, ž, ÿ,],icelandic: [0, 68, 0, 0, 2,122,156, 5, 1, 1, 5, 1, 0, // ,0, 2,255, 0, 0, 6, 51, 2, 0, 0, 5, 0, 19, // a,0, 1, 0, 0, 0, 4, 2, 2, 0, 0, 0, 0, 0, // b,0, 0, 0, 0, 0, 0,255, 0, 0, 0, 0, 0,255, // c,0, 0, 0, 2, 0, 2, 0, 0, 0, 5, 0, 1,255, // d,0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 22, // e,0, 4, 0, 1, 0, 5, 3, 3, 13, 0, 0, 0, 0, // f,0, 4, 0, 3, 1, 7, 2, 10, 12, 19, 7, 3, 0, // g,0, 2, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, // h,0, 1, 0, 0, 0, 0, 69, 0, 0, 0, 1, 2, 4, // i,0, 0, 0, 1, 0, 0, 2, 0, 0, 0, 0, 1, 8, // j,0, 3, 0, 5, 1, 21, 1, 10, 4, 10, 11, 0,255, // k,0, 30, 0, 8, 9, 4, 6, 78, 20, 18, 4, 1, 0, // l,0, 2, 0, 5, 0, 8, 2, 9, 1, 3, 1, 1, 0, // m,0, 9, 4, 4, 0, 11, 2, 18, 11, 6, 13, 3, 0, // n,0, 0,255, 0, 0, 0, 0, 0,255,255, 0, 0, 3, // o,0, 1, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0,255, // p,0,255,255,255, 0, 0,255,255,255,255,255,255,255, // q,0, 45, 1, 13, 7, 2, 7, 25, 17, 59, 9, 8, 7, // r,0, 8, 1, 2, 1, 37, 13, 5, 0, 1, 9, 9, 0, // s,0, 17, 0, 14, 7, 6, 1, 17, 3, 3, 14, 5, 0, // t,0, 0, 0, 0, 0, 7, 61, 0, 0, 0, 0, 3, 1, // u,0, 5, 0, 2, 0, 3, 4, 3, 0, 9, 0, 0, 6, // v,0, 0,255,255,255,255,255, 0, 0, 0,255,255,255, // w,0, 0, 0, 0, 0, 0,255, 0, 0,255, 0,255,255, // x,0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, // y,0, 0,255, 0, 0, 0,255, 0, 0,255, 0, 0,255, // z,0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,255, 0, 0,255,255, 0,255, 0,255, 0, // ß,83, 0, 2, 0, 4, 0, 2, 3, 6, 0, 16, 3, 9, 23, 5, 0, 2,255, 24, 3, 7, 0, 5, 0, 0, 0, 0, 0, 0,255, 0,255, 0, 0, 0,255,255, 0, 0, 8, // á,0, 0, 0,255, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0,255, 0,255, 0, 0, 0,255, 1, 0,255,255, 0,255,255, 0,255,255,255,255,255,255,255,255,255,255, // ä,4, 0, 7, 0, 3, 0, 8, 1, 4, 0, 0, 0, 5, 2, 3, 0, 4,255, 18, 5, 8, 0, 8, 0, 0, 0,255,255, 0,255, 0,255, 0, 0, 0,255,255, 0, 0, 2, // æ,1, 0, 0, 0, 0, 0, 8, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0,255, 6, 6, 0, 0, 1, 0, 0, 0, 0, 0,255,255,255, 0,255,255,255,255,255,255,255, 0, // é,126, 2, 2, 0, 1, 0, 1, 2, 0, 0, 0, 3, 16, 2, 5, 0, 1, 0, 25, 28, 19, 0, 29, 0, 0, 0, 0, 0, 0,255, 0,255, 0, 0, 0,255,255, 0,255, 0, // í,0, 95, 0,255, 0, 24, 5, 5, 0,122, 0, 0, 0, 0, 0, 7, 0,255, 46, 0, 0, 20, 2,255,255, 6,255, 0, 8,255, 23, 0, 36, 0, 13, 7, 21, 1, 2,255, // ð,8, 0, 63, 0, 7, 0, 22, 2, 3, 4, 39, 9, 5, 4, 5, 0, 1,255, 13, 8, 21, 0, 5, 0, 0, 0, 0, 0, 0,255, 0,255, 0, 0, 0, 0,255, 0, 0, 5, // ó,9, 0, 1, 0, 0, 0, 1, 2, 8, 0, 18, 2, 10, 4, 3, 0, 0,255, 4, 6, 11, 0, 5, 0, 0, 0, 0,255, 0,255, 0,255, 0, 0, 0, 0,255, 0, 0, 0, // ö,17, 0, 3,255, 3, 0, 40, 1, 11, 0, 14, 1, 13, 2, 2, 0, 0,255, 14, 9, 12, 0, 12,255,255, 0,255,255,255,255,255,255, 0, 0, 0,255,255, 0,255,255, // ø,20, 0, 9, 0, 0, 0, 0, 2, 7, 0, 6, 3, 1, 1, 10, 0, 0,255, 12, 3, 3, 0, 0, 0, 0, 0, 0,255, 0,255, 0, 0, 0, 0, 0,255,255, 0, 0, 1, // ú,1, 0, 9, 0, 2,255, 0, 0, 0,255, 0, 1, 6, 0, 7,255, 0,255, 1, 4, 9,255, 0,255, 0, 0, 0,255,255,255,255,255,255, 0, 0,255,255, 0,255, 4, // ý,87, 1,255,255, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0,255, 0, 0, 0, 0, 0,255,255, 0,255,255, 0,255, 0, 0, 2, 0, 0, 0,255, 0, 0, 0, // þ,// , a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t, u, v, w, x, y, z, ß, á, ä, æ, é, í, ð, ó, ö, ø, ú, ý, þ,],greek: [0, 12, 0, 16, 6, 69, 1, 0, 0, 24, 0, 33, 0, 41, 2, 2, 1, 50, 0, 44, 1, 2,105, 1, 2, 33, 0, 0, 0, 2, 0, 0, 15, 5, 1, // ,255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // a,0, 0,255, 0, 0,255,255, 0,255,255, 0, 0,255, 0,255, 0,255,255,255,255,255, 0,255,255,255, 0,255, 0,255,255,255, 0,255,255,255,255,255, // ΐ,4, 0, 0, 0, 0,255, 0, 0, 1, 1, 0, 0, 0, 0, 0, 4, 7, 4, 3, 3, 0, 0, 3, 6,255, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0,255, // ά,9, 0,255,255, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 1, 3, 2, 6, 2, 0, 0, 2, 2,255, 0, 5, 0, 1, 1, 0, 0, 0, 0, 0,255,255, // έ,3, 0,255,255,255,255, 0, 0, 0, 1, 1, 0, 0, 0, 1, 0, 10, 3, 1, 2, 0, 0, 0, 2,255, 0, 4, 0, 1, 1, 0, 0, 0,255,255,255,255, // ή,0, 0,255, 0, 0,255, 0, 5, 1, 2, 2, 15, 0, 0, 0, 0, 1, 4, 1, 5, 0, 5, 2, 14,255, 3, 4, 0, 1, 1, 0, 0, 0, 0, 0,255,255, // ί,46, 0, 0, 0, 1, 0, 22, 0, 4, 3, 4, 0, 1, 0, 2, 20, 34, 8, 15, 23, 1, 0, 13, 18,255, 5, 30, 1, 4, 3, 0, 0, 0, 0, 0, 0, 0, // α,10, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 2, 0, 0, 1, 0, 0,255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // β,15, 0, 0, 1, 1, 0, 0, 7, 0, 1, 0, 2, 0, 6, 0, 1, 0, 0, 0, 0, 0, 5, 0, 5,255, 0, 0, 2, 0, 0, 0, 1, 0, 0, 1, 0, 0, // γ,19, 0, 0, 3, 0, 0, 4, 2, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 3, 0, 4, 0, 0,255, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, // δ,33, 0, 0, 0, 0, 0, 2, 0, 2, 4, 7, 0, 2, 0, 5, 2, 9, 9, 16, 8, 1, 0, 10, 7,255, 15, 15, 0, 1, 5, 0, 0, 0, 0, 0, 0, 0, // ε,1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,255, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, // ζ,10, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 4, 0, 2, 8, 3, 6, 1, 0, 1, 3,255, 14, 41, 0, 0, 1, 0, 0, 0,255, 0, 0, 0, // η,5, 0, 0, 1, 0, 1, 0, 5, 0, 0, 0, 2, 0, 1, 0, 1, 0, 0, 0, 1, 0, 1, 0, 1,255, 0, 0, 1, 0, 0, 0, 0, 0,255, 0, 0, 0, // θ,6, 0, 0, 0, 0, 0, 0, 28, 2, 8, 12, 19, 0, 0, 0, 0, 3, 9, 5, 10, 0, 12, 6, 19,255, 6, 19, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, // ι,44, 0, 0, 1, 1, 0, 2, 8, 0, 5, 0, 5, 0, 4, 0, 33, 0, 0, 0, 0, 0, 3, 0, 2,255, 4, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, // κ,7, 0, 0, 5, 3, 0, 1, 10, 1, 1, 0, 9, 0, 2, 1, 3, 3, 9, 0, 0, 0, 10, 4, 0,255, 0, 0, 3, 0, 0, 0, 0, 0, 0, 3, 1, 0, // λ,20, 0, 0, 1, 1, 3, 0, 5, 0, 1, 0, 3, 0, 8, 1, 3, 0, 0, 2, 0, 0, 8, 0, 2,255, 5, 0, 4, 0, 0, 0, 2, 0, 0, 3, 1, 0, // μ,12, 0, 0, 6, 8, 1, 7, 30, 0, 1, 0, 10, 0, 14, 1, 8, 0, 0, 0, 1, 0, 16, 0, 1,255, 0, 0, 8, 0, 1, 0, 13, 0, 0, 5, 3, 9, // ν,1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,255, 0, 0, 0, 0, 0, 0, 0, 0,255, 0, 0, 0, // ξ,20, 0, 0, 0, 0, 0, 9, 0, 3, 9, 6, 0, 1, 0, 2, 14, 12, 13, 10, 14, 0, 0, 26, 21,255, 4, 54, 0, 4, 3, 0, 0, 0, 0, 0, 0, 0, // ο,45, 0, 0, 1, 0, 0, 0, 13, 0, 0, 0, 8, 0, 0, 0, 1, 0, 0, 5, 0, 0, 6, 0, 0,255, 2, 0, 5, 0, 0, 0, 1, 0, 0, 2, 1, 0, // π,3, 0, 0, 5, 5, 1, 2, 17, 3, 4, 4, 19, 0, 3, 1, 4, 5, 0, 0, 0, 0, 14, 12, 0,255, 0, 9, 5, 1, 4, 0, 2, 0, 0, 3, 2, 1, // ρ,0, 0, 0, 1, 6, 6, 0, 14, 0, 0, 0, 9, 0, 21, 0, 6, 0, 0, 0, 0, 0, 17, 0, 0,255, 0, 0, 5, 0, 0, 0, 4, 0, 0, 7, 1, 0, // ς,43, 0, 0, 3, 2, 3, 4, 10, 0, 0, 0, 4, 0, 6, 0, 13, 0, 0, 0, 1, 0, 7, 0, 0,255, 2, 2, 5, 0, 0, 0, 3, 0, 0, 3, 2, 1, // σ,73, 0, 0, 4, 2, 3, 3, 21, 0, 0, 0, 10, 0, 5, 0, 4, 4, 0, 0, 16, 0, 3, 2, 2,255, 35, 0, 5, 0, 0, 0, 3, 0, 0, 7, 1, 1, // τ,4, 0,255, 0, 0, 0, 0, 4, 0, 0, 1, 5, 0, 0, 1, 0, 2, 1, 0, 0, 0, 53, 0, 1,255, 7, 1, 0, 1, 0, 0, 0,255,255, 0, 0, 0, // υ,6, 0, 0, 1, 0, 0, 0, 5, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0,255, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // φ,9, 0, 0, 0, 3, 0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 4,255, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // χ,0, 0,255, 0, 0, 0, 0, 0, 0, 0, 0, 0,255, 0, 0, 0, 0, 0, 0, 0,255, 0, 0, 0,255, 0, 0, 0, 0, 0, 0, 0,255, 0, 0, 0, 0, // ψ,2, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 1, 1, 1, 0, 3, 0, 0, 1, 4,255, 1, 6, 0, 1, 1, 0, 0, 0,255, 0, 0, 0, // ω,0, 0,255, 0, 0, 0, 0, 1, 0,255,255, 0,255, 0,255, 0, 0, 0, 0, 0,255, 0, 0, 0,255, 0, 0, 0, 0, 0,255, 0, 0,255, 0, 0, 0, // ϊ,0, 0,255, 0, 0,255, 0, 0,255,255, 0, 0,255,255,255, 0,255, 0,255,255,255, 0,255, 0,255, 0, 0, 0,255,255,255, 0,255,255, 0,255,255, // ϋ,6, 0, 0,255, 0,255, 0, 0, 0, 1, 1, 0, 0, 0, 0, 2, 11, 1, 4, 3, 0, 0, 9, 5,255, 1, 4, 0, 0, 0, 0, 0, 0,255, 0,255,255, // ό,0, 0,255,255, 0,255,255, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 1, 0, 0, 0, 13, 0, 0,255, 2, 0, 0, 0, 0, 0, 0,255,255, 0,255,255, // ύ,0, 0,255,255,255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 1, 0, 2, 0, 0, 0, 3,255, 0, 1, 0, 0, 1, 0, 0, 0,255,255,255,255, // ώ,// , a, ΐ, ά, έ, ή, ί, α, β, γ, δ, ε, ζ, η, θ, ι, κ, λ, μ, ν, ξ, ο, π, ρ, ς, σ, τ, υ, φ, χ, ψ, ω, ϊ, ϋ, ό, ύ, ώ,],turkish: [195,254, 0,140, 0, 12,220,165, 2, 1, 58, 25, 27, // ,1, 23, 0, 2, 0, 19, 0, 0, 4, 0, 0, 1, 26, // a,2, 53, 0, 12, 0, 0, 3, 5, 0, 0, 1, 1, 0, // b,13, 31, 0, 4, 0, 0, 0, 0, 0, 0, 0, 2, 0, // c,7,161, 0, 22, 0, 0, 11, 4, 1, 3, 1, 2, 16, // d,0, 18, 0, 0, 0, 22, 0, 0, 6, 0, 1, 0, 14, // e,3, 19, 0, 12, 0, 0, 0, 1, 0, 0, 0, 8, 1, // f,0, 45, 0, 0, 0, 0, 1, 2, 0, 0, 2, 0, 0, // g,0, 27, 0, 21, 0, 0, 1, 2, 0, 0, 0, 2, 0, // h,0, 7, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, // j,37,114, 0, 30, 0, 1, 8, 25, 0, 2, 4, 14, 6, // k,60,157, 0, 45, 0, 4, 3, 9, 13, 19, 1, 21, 18, // l,39,105, 0, 19, 0, 2, 6, 5, 0, 1, 2, 22, 10, // m,105,198, 0, 63, 0, 0, 89, 46, 0, 28, 13, 36, 0, // n,1, 54, 0, 0, 0, 7, 0, 0, 0, 0,255, 0, 1, // o,2, 22, 0, 0, 0, 0, 0, 3, 0, 0, 3, 6, 0, // p,8, 12, 0, 6, 0, 0, 0, 0, 0, 0, 0, 1, 1, // q,44,125, 0,124, 0, 0, 21, 23, 6, 22, 10, 42, 1, // r,18,123, 0, 48, 0, 0, 0, 17, 0, 2, 3, 7, 0, // s,5,117, 0, 35, 0, 1, 2, 7, 0, 1, 1, 4, 25, // t,0, 3, 0, 0, 0, 1, 0, 0, 23, 0, 0, 0, 7, // u,1, 30, 0, 4, 0, 0, 2, 2, 0, 11, 0, 1, 0, // v,0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, // w,3, 9, 0, 3, 0, 0, 3, 0, 0, 0, 0, 1, 0, // x,8, 73, 0, 14, 0, 0, 1, 5, 0, 8, 5, 7, 0, // y,12, 34, 0, 10, 0, 0, 10, 5, 0, 10, 0, 26, 0, // z,25, 0, 2, 22, 54, 0, 8, 2, 4, 0, 38, 68, 26, 55, 0, 10, 7, 67, 64, 38, 0, 0, 0, 4, 44, 9, 3, 0, 0, 0, 0, 15,255,255, 28, 0,255, 0, 17, // ı,228, 15, 82, 26,125, 8, 28, 37, 54, 38,120,164, 71, 79, 6, 17, 7,254,151,193, 4, 68, 61, 9, 15, 20, 0, 2, 0, 0, 0, 26, 0, 0, 22, 0, 0, 0, 36, // i,0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,255, 0, 0, // ß,36, 0, 20, 12, 81, 0, 14, 9, 24, 0, 17,101, 55, 24, 0, 1, 12, 35, 31, 42, 0, 36, 0, 7, 21, 21, 0, 1, 0, 0,255, 4,255,255, 0, 0,255, 3, 24, // ä,0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,255, 0, 0,255, 0, 0,255, 0,255, 0, // â,31, 17, 0, 0, 0, 11, 0, 0, 2, 0, 10, 8, 1, 2, 1, 0, 0, 4, 0, 4, 3, 7, 0, 1, 0, 0, 1, 19, 0, 3, 0, 0, 0, 0, 0, 0, 0, 5, 1, // ç,23, 0, 9, 1, 20, 0, 1, 6, 13, 7, 17, 18, 15, 48, 0, 12, 0, 57, 7, 23, 0, 20, 23, 2, 57, 5,255, 0, 0,255,255, 3, 0, 0, 0,255, 0,255, 6, // ê,20, 0, 6, 5, 24, 0, 4, 2, 6, 16, 25, 21, 15, 33, 0, 4, 1, 52, 12, 20, 0, 13, 17, 0, 16, 10,255, 0, 0,255, 0, 2, 0, 0, 0,255, 0,255, 7, // î,0, 35, 0, 0, 0, 12, 0, 0, 0,255, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 13, 16, 0, 0, 0, 0,255, 0, 0, 2, 0, 2, 0, // ğ,30, 0, 11, 0, 6, 0, 0, 22, 1, 0, 10, 0, 2, 4, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 14, 1, 0, 0, 0, 0,255, 1,255,255, 0, 0,255, 0, 0, // ö,45, 0, 20, 0, 4, 0, 0, 0, 1, 0, 2, 1, 4, 5, 0, 1, 0, 10, 2, 5, 0, 0, 0, 0, 1, 0,255, 0, 0,255,255, 3, 0, 0, 0,255, 0, 0, 3, // û,20, 0, 11, 5, 26, 0, 0, 15, 3, 0, 11, 26, 21, 17, 0, 0, 1, 15, 8, 34, 0, 2, 0, 1, 17, 2, 0, 0, 0, 0, 0, 6,255, 0, 2, 0,255, 0, 3, // ü,33, 45, 1, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 14, 0, 0, 12, 0, 1, 5, 0, 0, 28, 33, 0, 11, 0, 0, 6, 5, 0, 0, 2, 6, 0, // ş,// , a, b, c, d, e, f, g, h, j, k, l, m, n, o, p, q, r, s, t, u, v, w, x, y, z, ı, i, ß, ä, â, ç, ê, î, ğ, ö, û, ü, ş,],hebrew: [0, 1, 0, 0, 0, 0, 0, 28, 29, 17, 24,144, 46, 26, 7, 88, 68, 22, 2, 71,106, 3,200, 0, 35, 69, 10, 2, 9, 2, 13,104, 19,138, // ,0,255,255, 0,255,255,255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,255, 0, 0, 0, 0, 0, 0, 0, // a,0, 0, 0, 0, 0, 0, 0,255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,255, 0, 0, 0, 0, 0, // ְ,0, 0, 0, 0,255,255, 0,255, 1, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0,255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,255, 0, 0, 0, 0, 0, // ַ,0,255, 0, 0, 0,255, 0,255,255, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,255, 0,255, 0, 0, 0, 0, 0, // ָ,0,255, 0, 0, 0, 0,255,255,255, 0, 0, 0, 0, 0, 0, 0,255, 0, 0, 0, 0, 0,255, 0,255, 0, 0,255, 0, 3,255, 0, 0, 0, 0, 0, // ּ,1,255, 0, 0, 0, 0,255,255, 0, 0, 0, 0, 0,255,255,255,255, 0, 0,255, 0, 0,255,255,255, 0, 0, 0,255,255,255, 0, 0, 0, 0, 0, // װ,0,255,255,255,255, 0, 0,255,255, 0, 0,255, 0, 0, 0, 0,255, 0,255,255,255, 0,255, 0,255, 0,255,255,255, 0,255, 0, 0, 0, 0,255, // ױ,0,255, 0, 0,255, 0, 0,255,255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,255, 0, 0,255, 0,255, 0, 0, 0,255, 0,255, 0, 0, 0, 0, 0, // ײ,171, 0, 0, 0, 0, 0, 0,255, 0, 0, 20, 14, 15, 21, 22, 5, 0, 22, 15, 0, 2, 17, 0, 16, 0, 16, 6, 2, 0, 27, 0, 3, 25, 30, 5, 3, // א,101, 0, 0, 0, 0, 0,255, 0, 0, 11, 2, 2, 2, 7, 22, 0, 4, 1, 15, 0, 3, 6, 0, 5, 0, 2, 1, 7, 0, 0, 0, 3, 5, 16, 6, 4, // ב,44, 0, 0, 0, 0, 0,255, 0, 0, 7, 1, 0, 2, 5, 9, 0, 0, 6, 12, 0, 0, 3, 0, 2, 0, 17, 1, 15, 0, 1, 0, 0, 0, 7, 1, 1, // ג,70, 0, 0, 0, 0, 0,255, 0, 0, 9, 4, 4, 0, 4, 16, 0, 5, 0, 15, 0, 3, 5, 0, 8, 0, 20, 3, 7, 0, 2,255, 1, 2, 4, 0, 0, // ד,158, 0, 0, 0, 0, 0, 0,255,255, 5, 8, 2, 6, 3, 8, 5, 4, 2, 36, 0, 4, 17, 0, 11, 0, 16, 1, 4, 0, 4, 0, 2, 5, 16, 9, 5, // ה,42, 0, 0, 0, 0, 0, 0,255,255, 48, 21, 16, 18, 19, 73, 5, 13, 12, 38, 0, 11, 22, 0, 30, 0, 25, 10, 13, 0, 32, 0, 18, 22, 34, 19, 17, // ו,32, 0, 0, 0, 0, 0,255, 0, 0, 8, 0, 0, 2, 1, 6, 0, 2, 0, 25,255, 1, 0, 0, 2, 0, 1, 0, 4, 0, 0,255, 0, 0, 0, 0, 0, // ז,18, 0, 0, 0, 0, 0,255,255,255, 8, 4, 0, 0, 6, 7, 0, 0, 1, 10, 0, 1, 5, 0, 6, 0, 2, 0, 0, 0, 1, 0, 1, 0, 4, 4, 5, // ח,22, 0, 0, 7, 1, 0,255, 0, 0, 28, 1, 4, 0, 2, 6, 0, 0, 0, 30, 0, 5, 9, 0, 4, 0, 20, 21, 12, 0, 7, 0, 1, 10, 16, 20, 0, // ט,31, 0, 0, 0, 0, 0, 0,255,255, 65, 31, 9, 45, 19, 60, 22, 12, 19, 78, 0, 9, 37, 0, 25, 0, 45, 15, 12, 0, 15, 0, 15, 17, 50, 25, 13, // י,0,255, 0, 0, 0, 0,255, 0, 0, 5, 0, 0, 0, 0, 2, 0, 0, 0, 10, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0,255, 0, 0, 4, 0, 0, // ך,29, 0, 0, 0, 0, 0,255, 0, 0, 5, 2, 0, 0, 4, 7, 2, 0, 0, 5, 0, 0, 4, 0, 4, 0, 1, 1, 3, 0, 0,255, 0, 0, 6, 2, 1, // כ,64, 0, 0, 1, 0, 0, 0, 0, 0, 35, 7, 8, 2, 5, 29, 1, 6, 4, 20, 0, 8, 4, 0, 7, 0, 1, 2, 18, 0, 4, 0, 2, 6, 2, 29, 2, // ל,0, 0, 0, 0, 0, 0,255, 0, 0, 2, 0, 3, 1, 3, 9, 0, 0, 0, 65, 0, 0, 2, 0, 0, 0, 0, 0, 11,255, 0,255, 0, 0, 0, 5, 1, // ם,89, 0, 0, 0, 0, 0,255, 0, 0, 12, 11, 1, 2, 21, 15, 2, 3, 1, 10, 0, 2, 8, 0, 5, 0, 1, 2, 7, 0, 0, 0, 2, 1, 5, 7, 10, // מ,1,255, 0, 0, 0, 0, 0, 0, 0, 13, 8, 3, 3, 1, 48, 1, 0, 12, 37, 0, 3, 3, 0, 3, 0, 0, 2, 31, 0, 3,255, 1, 1, 15, 5, 1, // ן,24, 0, 0, 2, 0, 0, 0, 0, 0, 35, 8, 2, 1, 5, 26, 0, 1, 1, 35, 0, 4, 4, 0, 8, 0, 0, 1, 30, 0, 5, 0, 0, 2, 3, 13, 2, // נ,22, 0, 0, 0, 0, 0, 0, 0, 0, 16, 4, 0, 0, 6, 14, 0, 2, 1, 17, 0, 1, 2, 0, 6, 0, 5, 0, 12, 0, 2,255, 0, 3, 5, 0, 0, // ס,33, 0, 0, 0, 0, 1, 0, 0, 0, 0, 16, 44, 38, 14, 25, 8, 0, 46, 36, 0, 5, 20, 0, 25, 0, 25, 5, 1, 0, 11, 0, 5, 15, 19, 18, 1, // ע,0,255, 0, 0, 0, 0,255, 0,255, 0, 0, 0, 1, 0, 2, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0,255, 0, 0, 0, 0, 0, // ף,76, 0, 0, 0, 0, 0,255, 0, 0, 13, 2, 0, 0, 8, 12, 0, 0, 0, 9, 0, 1, 4, 0, 5, 0, 2, 10, 8, 0, 0,255, 1, 1, 4, 5, 3, // פ,0, 0, 0, 0, 0, 0,255,255,255, 0, 2, 0, 0, 0, 1, 0, 0, 0, 1,255, 0, 0,