boogerlad/mj - Change I4MTA4E4GTMWJ4NC42QRWV55JRZ2K2W7SDSH5GAW2N4MIAQWS3DQC

strain family, brand, producer normalization

Created by boogerlad on March 3, 2020

I4MTA4E4GTMWJ4NC42QRWV55JRZ2K2W7SDSH5GAW2N4MIAQWS3DQC

Dependencies

In channels

main

Change contents

Replacement in scrape/common.js at line 106 [3.4897]

B:BD[3.8259] → [3.8259:8402]

	normalize(brand, title, type) {//todo: deal with new types(edibles, vapes, ...)
		if(title.includes('(')) {
			console.log('fuck', title);

[3.8259]

[3.8402]

	roman2number(s) {//https://leetcode.com/problems/roman-to-integer/
		s = s.replace(/\s/g, '');
		let sum = 0;
		const m = {
			I: 1,
			V: 5,
			X: 10,
			L: 50,
			C: 100,
			D: 500,
			M: 1000

Replacement in scrape/common.js at line 118 [3.4897]

B:BD[3.8407] → [3.8407:8903]

		let ot = title.toLowerCase();
		title = title.replace(brand, '');
		brand = brand.toLowerCase().replace('.', '').replace(/&|(\b(and|health|therapeutics|co|company|inc|cannabis)\b)/g, '').trim();
		if(brand.includes('sundial')) {
			brand = brand.replace(/\b(calm|ease|flow|lift|spark)\b/g, '').trim();
			title = title.replace(/\b(calm|ease|flow|lift|spark)\b/g, '').trim();
		} else if(brand === 'alta vie') {
			brand = 'altavie';
		} else if(brand === 'twd') {
			brand = 'tweed';

[3.8407]

[3.8903]

		for(let i = s.length - 1; i >= 0; --i) {
			sum += m[s[i]];
			if(//lol thank god that s[-1] is undefined lmao instead of crash
				s[i - 1] === 'I' && (s[i] === 'V' || s[i] === 'X') ||
				s[i - 1] === 'X' && (s[i] === 'L' || s[i] === 'C') ||
				s[i - 1] === 'C' && (s[i] === 'D' || s[i] === 'M')
			) {
				sum -= m[s[--i]];
			}

Replacement in scrape/common.js at line 128 [3.4897]

B:BD[3.8908] → [3.8908:9074]

B:BD[3.9074] → [2.0:44]

∅:D[4.235] → [3.9118:9312]

∅:D[2.44] → [3.9118:9312]

B:BD[3.9118] → [3.9118:9312]

∅:D[4.654] → [3.9312:9893]

B:BD[3.9312] → [3.9312:9893]

		//let ratio = title.match(/\d+:\d+/);
		let q = title.match(specialqty);
		title = title
			.toLowerCase()
			.replace(inBrackets, '')
			.replace(brand, '')
			.replace(/twd(\.?)|"|-|−|%|&|:/g, '')
			.replace(/\b(liquid|and|oral|drop(s?)|((hard|soft)(-|−| )?)?(capsule|cap)(s?)|pre(-|−| )?roll(s?)|pack(s?)|joint(s?)|oil|flower|spray)\b/g, '')
			.replace(specialqty, '')
			.trim();
		if(ot.includes('fiveup') || /5\s*up/.test(ot)) {
			title = '5up';
			//early return?
		} else if(ot.includes('tenup') || /10\s*up/.test(ot)) {
			title = '10up';
			//early return?
		} else if(title.includes('reserve') && brand.includes('edison')) {
			title = title.replace('reserve', '').trim();
			brand = 'edison reserve';
		} else if(title.includes('select') && brand.includes('canaca')) {
			title = title.replace('select', '').trim();
			brand = 'canaca select';
		} else if(brand === 'tantalus labs') {
			title = title.replace(/bc|sungrown/g, '').trim();

[3.8908]

[3.9893]

		return ' ' + sum + ' ';
	},
	strain_family_clean(s) {
		s = s
		.normalize("NFD")//https://stackoverflow.com/a/37511463
		.replace(/[\u0300-\u036f]/g, "")
		//.replace(/((^|\s)(?=[MDCLXVI])M*(C[MD]|D?C{0,3})(X[CL]|L?X{0,3})(I[XV]|V?I{0,3})($|\s))/g, roman2number) fuck you "Josh D OG"
		//https://stackoverflow.com/a/6713327
		//https://stackoverflow.com/a/36576402
		//https://stackoverflow.com/a/11622109
		//perhaps we should delete words like og, kush?
		.replace(/(^|\s)(I|II|III|IV|V|VI|VII|VIII|IX|X|XI|XII|XIII|XIV|XV|XVI|XVII|XVIII|XIX|XX|XXI|XXII|XXIII|XXIV|XXV|XXVI|XXVII|XXVIII|XXIX|XXX)($|\s)/g, roman2number)
		.toLowerCase()
		.replace(/\b(bc|cbd)\b/, "")
		.replace(/\bpurps\b/, 'purple')
		.replace(/[^a-z0-9 ]/g, "")
		.split(/\s+/)
		.sort()
		.join('');
		switch(s) {
			case 'cland':
				s = 'candyland';
				break;
			case 'cheese':
				s = 'cheeseuk';
				break;
			case 'chemdawg':
				s = 'chemdog';
				break;
			case 'cookiesgirlscout':
				s = 'gsc';
				break;

Replacement in scrape/common.js at line 162 [3.4897]

B:BD[3.9898] → [3.9898:10012]

		if(title.includes('casa') && title.includes('blanca') && brand.includes('edison')) {
			title = 'casablanca';

[3.9898]

[3.10012]

		if(s === 'cland') {
			s = 'candyland';
		} else if(s === 'cheese') {
			s = 'cheeseuk';
		} else if(s === 'chemdawg') {
			s = 'chemdog';

Replacement in scrape/common.js at line 169 [3.4897]

B:BD[3.10017] → [3.10017:10107]

		if(type === 'oil' || type === 'spray') {
			title = title.replace('mist', '').trim();

[3.10017]

[3.10107]

		return s;
	},
	normalize(brand, title, type/*, producer_in, description*/) {//todo: deal with new types(edibles, vapes, ...)
		let producer;
		if(title.includes('(')) {
			console.log('fuck', title);

Replacement in scrape/common.js at line 176 [3.4897]

B:BD[3.10112] → [3.10112:10323]

		if(title.length === 0) {
			//if(ratio === null) {
				title = ot;
			//} else {
			//	title = ratio[0];
			//}
		}
		if(type === 'capsule' && q !== null) {
			title += ' ' + q[0].replace(/\s+/, '');

[3.10112]

[3.10323]

		title = title.toLowerCase();
		if(title.includes('fiveup') || /5\s*up/.test(title)) {
			return ['TILRAY', 'Canaca', '5up', '5up'];
		} else if(title.includes('tenup') || /10\s*up/.test(title)) {
			return ['TILRAY', 'Canaca', '10up', '10up'];
		} else {
			brand = brand.toLowerCase();
			title = title
				.replace(inBrackets, '')
				.replace(brand, '')
				.replace(/twd(\.?)|"|-|−|%|&|:/g, '')
				.replace(/\b(liquid|and|oral|drop(s?)|((hard|soft)(-|−| )?)?(capsule|cap)(s?)|pre(-|−| )?roll(s?)|pack(s?)|joint(s?)|oil|flower|spray)\b/g, '')
			;
			if(type !== 'capsule') {
				title = title.replace(specialqty, '');
				if(type === 'spray') {
					title = title.replace('mist', '');
				}
			}
			//logo is canonical capitalization and spelling
			if(brand.includes('edison')) {
				producer = 'ORGANIGRAM';
				if(title.includes('reserve')) {
					title = title.replace('reserve', '');
					brand = 'EDISON RESERVE';
				} else if(title.includes('bytes')) {
					title = title.replace('bytes', '');
					brand = 'EDISON BYTES';
				} else {
					brand = 'EDISON';
				}
				if(/casa\s*blanca/.test(title)) {
					title = 'casablanca';
				}
			} else if(brand.includes('sundial')) {
				producer = 'SUNDIAL';
				//interesting behavior with .match and /g but we only use [0] anyways
				//https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/match#Return_value
				brand = brand.match(/\b(calm|ease|flow|lift|spark)\b/) || title.match(/\b(calm|ease|flow|lift|spark)\b/);
				if(brand !== null) {
					brand = brand[0].toUpperCase();
				}
				title = title.replace(/\b(calm|ease|flow|lift|spark)\b/, '');
			} else if(/alta\s*vie/.test(brand)) {
				producer = 'Aurora';
				brand = 'AltaVie';
			} else if(brand.startsWith('tw')) {//may be insufficient
				producer = 'CANOPY GROWTH';
				brand = 'Tweed';
			} else if(brand.includes('canaca')) {
				producer = 'TILRAY';
				if(title.includes('select') || brand.includes('select')) {
					brand = 'Canaca Select';
					title = title.replace('select', '');
				} else {
					brand = 'Canaca';
				}
			} else if(brand.includes('tantalus')) {
				producer = 'Tantalus Labs';
				brand = null;
				title = title.replace(/bc|sungrown/g, '');
			} else if(/48\s*north/.test(brand)) {
				producer = '48North';
				brand = null;
			} else if(/7\sacres*/.test(brand)) {
				producer = 'Supreme';
				brand = '7ACRES';
			} else if(/ace\s*valley/.test(brand)) {
				producer = 'flowr';
				brand = 'Ace Valley';
			} else if(/acreage pharms\s*/.test(brand)) {
				producer = 'Acreage Pharms';
				brand = null;
			} else if(brand.includes('aurora')) {
				producer = 'Aurora';
				if(brand.includes('drift')) {
					brand = 'Drift';
					title = title.replace('drift', '');
				} else {
					brand = null;
				}
			} else if(brand.includes('beleave')) {
				producer = 'beleave';
				brand = null;
			} else if(brand.includes('bhang')) {
				producer = 'Bhang';
				brand = null;
			} else if(brand.includes('blazery')) {
				producer = 'Zenabis';
				brand = 'Blazery';
			} else if(brand.includes('blissed')) {
				//producer = ;rely on quicky.licensed_producer or description. fuck trec brands. fuck rebranders!!!!
				//producer_in or search description
				brand = 'BLISSED';
			} else if(/broken\s*coast/.test(brand)) {
				producer = 'aphria';
				brand = 'BROKEN COAST';
			} else if(/canna\s*farms/.test(brand)) {
				producer = 'VIVO';
				brand = 'CANNA FARMS';
			} else if(/chowie\s*wowie/.test(brand)) {
				producer = 'TILRAY';
				brand = 'ChOWie wOWie';
			} else if(/color\s*cannabis/.test(brand)) {
				producer = 'weedmd';
				brand = 'Color';
			} else if(brand.includes('cove')) {
				producer = 'CRONOS';
				brand = 'COVE';
			} else if(/dna\s*genetics/.test(brand)) {
				producer = 'CANOPY GROWTH';
				brand = 'DNA GENETICS';
			} else if(brand.includes('dosecann')) {
				producer = 'Auxly';
				brand = 'Dosecann';
			} else if(brand.includes('dosist')) {
				producer = 'dosist';
				brand = null;
			} else if(brand.includes('emerald')) {
				producer = 'emerald';
				brand = 'null';
			} else if(brand.includes('everie')) {
				producer = 'TILRAY';
				brand = 'Everie';
			} else if(brand.startsWith('eve')) {
				producer = 'Natural MedCo';
				brand = 'eVe';
			} else if(brand.includes('FIGR')) {
				producer = "Canada's Island Garden";
				brand = 'FIGR';
			} else if(brand.includes('fireside')) {
				producer = 'VIVO';
				brand = 'FIRESIDE';
			} else if(brand.includes('flowr')) {
				producer = 'flowr';
				brand = null;
			} else if(brand.includes('foray')) {
				producer = 'Auxly';
				brand = 'Foray';
			} else if(brand.includes('gage')) {
				producer = 'RADICAL';
				brand = 'GAGE';
			} else if(/good\s*supply/.test(brand)) {
				producer = 'aphria';
				brand = 'GOOD SUPPLY';
			} else if(brand.includes('grail')) {
				producer = 'TILRAY';
				brand = 'GRAIL';
			} else if(brand.includes('haven')) {
				producer = 'TERRASCEND';
				brand = 'Haven Street';
			} else if(brand.includes('hexo')) {
				producer = 'HEXO';
				brand = null;
			} else if(/high\s*tide/.test(brand)) {
				producer = 'WAYLAND';
				brand = 'HIGH TIDE';
			} else if(brand.includes('houseplant')) {
				producer = 'CANOPY GROWTH';
				brand = 'Houseplant';
			} else if(brand.includes('indiva')) {
				producer = 'INDIVA';
				brand = 'INDIVA';
			} else if(brand.includes('irisa')) {
				producer = 'TILRAY';
				brand = 'IRISA';
			} else if(brand.includes('kiwi')) {
				producer = 'WAYLAND';
				brand = 'KIWI';
			} else if(brand.includes('kke')) {
				producer = 'Supreme';
				brand = "KKE";
			} else if(/kolab\s*project/.test(brand)) {
				producer = 'Auxly';
				brand = 'KOLAB PROJECT';
			} else if(brand.includes('lbs')) {
				producer = 'CANOPY GROWTH';
				brand = 'LBS';
			} else if(brand.includes('liiv')) {
				producer = 'CannTrust';
				brand = 'liiv';
			} else if(brand.includes('lumina')) {
				producer = 'VIVO';
				brand = 'LUMINA';
			} else if(/marley\s*natural/.test(brand)) {
				producer = 'TILRAY';
				brand = 'MARLEY NATURAL';
			} else if(brand.includes('namaste')) {
				producer = 'Zenabis';
				brand = 'NAMASTE';
			} else if(/northern\s*green\s*canada/.test(brand)) {
				producer = 'NORTHERN GREEN CANADA';
				brand = null;
			} else if(/northern\s*harvest/.test(brand)) {
				producer = 'WAYLAND';
				brand = 'NORTHERN HARVEST';
			} else if(/original\s*stash/.test(brand)) {
				producer = 'HEXO';
				brand = 'ORIGINAL STASH';
			} else if(/plain\s*packaging/.test(brand)) {
				producer = 'CANOPY GROWTH';
				brand = 'PLAIN PACKAGING';
			} else if(/pure\s*sunfarms/.test(brand)) {
				producer = 'PURE SUNFARMS';
				brand = null;
			} else if(brand.includes('qwest')) {//technically wrong, but I don't want to duplicate the brand in the title. fucktards
				producer = 'dECIBEL';
				brand = 'QWEST';
				if(title.includes('ice')) {
					brand += ' RESERVE';
					title = 'ice';
				} else if(title.includes('spirit')) {
					brand += ' RESERVE';
					title = 'spirit';
				} else if(title.includes('wonder')) {
					title = 'wonder';
				} else if(title.includes('zest')) {
					title = 'zest';
				} else if(title.includes('vision')) {
					title = 'vision';
				} else {
					console.log('wow qwest made a new product!!11')
				}
			} else if(brand.includes('redecan')) {
				producer = 'REDECAN';
				brand = null;
			} else if(brand.includes('aqualitas')) {
				producer = 'aqualitas';
				brand = 'reef';
			} else if(brand.includes('riff')) {
				producer = 'aphria';
				brand = 'RIFF';
			} else if(/royal\s*high/.test(brand)) {
				producer = 'HARVEST ONE';
				brand = 'ROYAL HIGH';
			} else if(/san\s*rafael/.test(brand)) {
				producer = 'Aurora';
				brand = "San Rafael '71";
			} else if(brand.includes('saturday')) {
				producer = 'weedmd';
				brand = 'SATURDAY';
			} else if(/(seven|7)\s*oak/.test(brand)) {
				producer = 'beleave';
				brand = 'SEVEN OAKS';
			} else if(brand.includes('solei')) {
				producer = 'aphria';
				brand = 'Solei';
			} else if(brand.includes('spinach')) {
				producer = 'CRONOS';
				brand = 'Spinach';
			} else if(/sugar\s*leaf/.test(brand)) {
				producer = 'Supreme';
				brand = 'sugar leaf';
			} else if(brand.includes('symbl')) {
				producer = 'Aleafia';
				brand = 'symbl';
			} else if(brand.includes('synr')) {
				producer = 'CannTrust';
				brand = 'SYNR.G';
			} else if(/thc\s*biomed/.test(brand)) {
				producer = 'THC BioMed';
				brand = null;
			} else if(/the\s*batch/.test(brand)) {
				producer = 'TILRAY';
				brand = 'THE BATCH';
			} else if(/the\s*green\s*organic\s*dutchman/.test(brand)) {
				producer = 'THE GREEN ORGANIC DUTCHMAN';
				brand = null;
			} else if(/tokyo\s*smoke/.test(brand)) {
				producer = 'CANOPY';
				brand = 'TOKYO SMOKE';
			} else if(/top\s*leaf/.test(brand)) {
				producer = 'SUNDIAL';
				brand = 'TOPLEAF';
			} else if(brand.includes('trailblazer')) {
				producer = 'ORGANIGRAM';
				brand = 'TRAILBLAZER';
			} else if(brand.includes('up')) {
				producer = 'HEXO';
				brand = 'UP';
			} else if(/van\s*der\s*pop/.test(brand)) {
				producer = 'CANOPY';
				brand = 'Van der Pop';
			} else if(brand.includes('vertical')) {
				producer = 'AgMEDICA';
				brand = 'VERTICAL';
			} else if(brand.includes('wayfarer')) {
				producer = 'MediPharm Labs';
				brand = 'Wayfarer';
			} else if(brand.includes('wdbx')) {
				producer = 	'CANADA HOUSE';
				brand = 'WDBX';
			} else if(brand.includes('whistler')) {
				producer = 'Aurora';
				brand = 'Whistler';
			} else if(brand.includes('wink')) {
				//producer = ;rely on quicky.licensed_producer or description. fuck trec brands. fuck rebranders!!!!
				//producer_in or search description
				brand = 'WINK';
			} else if(brand.includes('woodstock')) {
				producer = 'Aurora';
				brand = 'WOODSTOCK';
			} else if(brand.includes('xscape')) {
				producer = 'CannTrust';
				brand = 'XSCAPE';
			}
			join_title = title.split(/\s+/).sort().join('');
			return [producer, brand, title, join_title];

Deletion in scrape/common.js at line 485 [3.4897]

B:BD[3.10328] → [3.10328:10407]

		title = title.split(/\s+/).sort().join('');
		return [brand, title, type];