boogerlad/mj - Change PK7ZUXD7ELS4V6SNYYFWP45JKWKJOVWBGFDRSFY3LP47IU2M55BAC

capsule is a measurement unit too, oops. refetch image when older than 30 days and reupload if different, skip image scrape if no cloudinary url env, fake image scrape if cloudinary url is fuck(obviously, refetch is skipped if fuck). new portion / quantity parser. determine spray earlier, but need to rethink concentrations...

Created by kevin on February 28, 2020

PK7ZUXD7ELS4V6SNYYFWP45JKWKJOVWBGFDRSFY3LP47IU2M55BAC

Dependencies

In channels

main

Change contents

Replacement in initialize.sql at line 77 [5.17]

B:BD[4.1602] → [3.0:50]

create type measurement_unit as enum('ml', 'g');

[4.1602]

[5.1670]

create type measurement_unit as enum('ml', 'g', 'capsules');

Insertion in scrape/image.sh at line 2 [2.10640]

[2.10654]

#$1 is url to source image
#$2 is optional url to destination image to be diffed with $1

Replacement in scrape/image.sh at line 14 [2.10640]

B:BD[2.11242] → [2.11242:11274]

if [[ $status = "404" ]]; then

[2.11242]

[2.11274]

if [[ $status = "200" ]]; then
	type="$(identify -format %[magick] ${filename})";
	if [[ $type = PNG ]]; then
		convert -strip "$filename" -fuzz 10% -trim +repage "$filename";
	elif [[ $type = JPEG ]]; then
		jpegtran -perfect -copy none -trim -optimize -crop $(convert "$filename" -fuzz 10% -trim -format "%wx%h%X%Y" info:-) -outfile "$filename" "$filename";
	fi
	if [ -z "$2" ]; then
		echo "/tmp/$filename";
	else
		string2="$(curl -O -J -s -w "%{filename_effective}, %{http_code}" $2)";
		readarray -c1 -C 'mfcb val_trim a' -td, <<<"$string2,"; unset 'a[-1]'; declare -p a > /dev/null;
		filename2="${a[0]}"
		status2="${a[1]}"
		if [[ $status2 = "200" ]]; then
			#nice! cloudinary filename duplicate file extension, so no chance of overwrite due to collision
			if cmp -s "$filename" "filename2"; then
				echo "same";
			else
				echo "/tmp/$filename";
			fi
		else
			echo "$status2";
		fi
	fi
else

Replacement in scrape/image.sh at line 41 [2.10640]

B:BD[2.11292] → [2.11292:11661]

	exit 0;
fi
type="$(identify -format %[magick] ${filename})";
if [[ $type = PNG ]]; then
	convert -strip "$filename" -fuzz 10% -trim +repage "$filename";
elif [[ $type = JPEG ]]; then
	jpegtran -perfect -copy none -trim -optimize -crop $(convert "$filename" -fuzz 10% -trim -format "%wx%h%X%Y" info:-) -outfile "$filename" "$filename";
fi
echo "/tmp/$filename";

[2.11292]

fi

Replacement in scrape/ocs.ca.js at line 104 [2.11674]

B:BD[2.14612] → [2.14612:14663]

			let minTHC, maxTHC, minCBD, maxCBD, unit, UOM;

[2.14612]

[2.14663]

			let minTHC, maxTHC, minCBD, maxCBD, unit;
			let UOM = 'g';
			for(let j = 0; j < e[i][0].tags.length; ++j) {
				if(e[i][0].tags[j] === 'subsubcategory--Oral Sprays') {
					e[i][0].type = 'spray';
				}
			}

Deletion in scrape/ocs.ca.js at line 117 [2.11674]
B:BD[2.15108] → [2.15108:15124]
```
				UOM = 'g';
```

Replacement in scrape/ocs.ca.js at line 123 [2.11674]

B:BD[2.15346] → [2.15346:15380]

				if(e[i][0].type === 'oil') {

[2.15346]

[2.15380]

				if(e[i][0].type === 'spray' || e[i][0].type === 'oil' || e[i][0].type === 'tea') {

Replacement in scrape/ocs.ca.js at line 125 [2.11674]

B:BD[2.15398] → [2.15398:15599]

				} else if(e[i][0].type === 'tea') {
					UOM = 'sachet';
				} else if(e[i][0].type === 'chocolate') {
					UOM = 'piece';
				} else {//capsule, gummy, mint, cookie
					UOM = e[i][0].type;

[2.15398]

[2.15599]

				} else if(e[i][0].type === 'capsule') {
					UOM = 'capsules';

Replacement in scrape/ocs.ca.js at line 128 [2.11674]

B:BD[2.15606] → [2.15606:15774]

				if(e[i][0].type === 'chocolate' || e[i][0].type === 'gummy' || e[i][0].type === 'mint') {
					let n = new Big(e[i][0].meta.variant.retail_pack_number_of_items);

[2.15606]

[2.15774]

				if(e[i][0].type === 'chocolate' || e[i][0].type === 'gummy' || e[i][0].type === 'mint' || e[i][0].type === 'spray' || e[i][0].type === 'tea') {
					let n = e[i][0].meta.variant;
					if(e[i][0].type === 'spray') {
						n = new Big(n.retail_pack_net_content);
					} else {
						n = new Big(n.retail_pack_number_of_items);
					}

Deletion in scrape/ocs.ca.js at line 139 [2.11674]

B:BD[2.15982] → [2.15982:16137]

				}
			}
			for(let j = 0; j < e[i][0].tags.length; ++j) {
				if(e[i][0].tags[j] === 'subsubcategory--Oral Sprays') {
					e[i][0].type = 'spray';

Replacement in scrape/ocs.ca.js at line 142 [2.11674]

B:BD[2.16231] → [2.16231:16282]

			let fuckoff = e[i][0].meta.product.plant_type;

[2.16231]

[2.16282]

			let strain = e[i][0].meta.product.plant_type;

Replacement in scrape/ocs.ca.js at line 149 [2.11674]
B:BD[2.16473] → [2.16473:16511]
```
					f2 = e[i][0].tags[f].slice(12);
```
[2.16473]
[2.16511]
```
					f2 = e[i][0].tags[f].substring(12);
```

Replacement in scrape/ocs.ca.js at line 152 [2.11674]

B:BD[2.16524] → [2.16524:16574]

			if(fuckoff === undefined) {
				fuckoff = f2

[2.16524]

[2.16574]

			if(strain === undefined) {
				strain = f2

Replacement in scrape/ocs.ca.js at line 155 [2.11674]
B:BD[2.16580] → [2.16580:16605]
```
			if(f2 !== fuckoff) {
```
[2.16580]
[2.16605]
```
			if(f2 !== strain) {
```

Replacement in scrape/ocs.ca.js at line 158 [2.11674]

B:BD[2.16661] → [2.16661:16688]

				console.log(fuckoff);

[2.16661]

[2.16688]

				console.log(strain);
			}
			let spaceIndex = strain.search(/\s+/);
			if(spaceIndex > 0) {//usually should be !== -1, but if the space is at 0, we're fucked anyways
				strain = strain.substring(0, spaceIndex);

Insertion in scrape/ocs.ca.js at line 164 [2.11674]
[2.16694]
[2.16694]
```
			strain = strain.toLowerCase();
```

Replacement in scrape/ocs.ca.js at line 171 [2.11674]

B:BD[2.16915] → [2.16915:17023]

				let terpene = e[i][0].tags[j].split('terpenes--')[1];
				if(terpene) {
					terpenes.push(terpene);

[2.16915]

[2.17023]

				if(e[i][0].tags[j].startsWith('terpenes--')) {
					let terpene = e[i][0].tags[j].substring(10).toLowerCase();
					if(terpene) {
						if(terpene.includes('remove')) {
							if(terpenes.length) {
								console.log(terpenes);
								console.log(e[i][0].tags);
								console.log(`${url}/products/${e[i][0].handle}`);
							}
							terpenes = null;
							break;
						} else {
							terpenes.push(terpene);
						}
					}

Replacement in scrape/ocs.ca.js at line 211 [2.11674]
B:BD[2.17518] → [2.17518:17563]
```
					fuckoff.toLowerCase().split(/\s+/)[0],
```
[2.17518]
[2.17563]
```
					strain,
```

Replacement in scrape/ocs.ca.js at line 237 [2.11674]

B:BD[2.18231] → [2.18231:18954]

			let pics = new Set((await request({
				url: `${url}/products/${e[i][0].handle}.json`,
				headers: {'User-Agent': "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.87 Safari/537.36"}
			}).then(JSON.parse)).product.images.map(a => a.src.split('?')[0]));
			for(let pic of pics) {
				let product_images = await return_key('select ID from product_image where product = $1 and source = $2', [productID, pic]);
				if(product_images.length === 0) {
					let response = (await exec('./image.sh ' + pic)).stdout.trim();
					if(response !== '404') {
						let ar = await cloudinary.uploader.upload(response, {public_id: response.slice(5), unique_filename: false});

[2.18231]

[2.18954]

			if(process.env.CLOUDINARY_URL) {
				let pics = new Set((await request({
					url: `${url}/products/${e[i][0].handle}.json`,
					headers: {'User-Agent': "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.87 Safari/537.36"}
				}).then(JSON.parse)).product.images.map(({src}) => {
					let questionIndex = src.indexOf('?');
					if(questionIndex === -1) {
						return src;
					} else {
						return src.substring(0, questionIndex);
					}
				}));
				for(let pic of pics) {
					let product_images = (await postgres.query(`select id, case when now() - system_time_start > '30 days' then destination else null end as destination from product_image where product = $1 and source = $2`, [productID, pic])).rows;
					if(product_images.length === 0) {
						let destination;
						if(process.env.CLOUDINARY_URL === 'fake') {
							//https://stackoverflow.com/a/13108449/
							let name = pic.split('/');
							name = name.pop() || name.pop();
							name += name.substring(name.lastIndexOf('.'));
							destination = "https://res.cloudinary.com/the-eternal-tao/image/upload/" + name;
						} else {
							let response = (await exec('./image.sh ' + pic)).stdout.trim();
							if(response.startsWith('/tmp/')) {
								destination = (await cloudinary.uploader.upload(response, {public_id: response.substring(5), unique_filename: false})).secure_url;
							} else {
								continue;
							}
						}

Replacement in scrape/ocs.ca.js at line 278 [2.11674]
B:BD[2.19202] → [2.19202:19242]
```
							[productID, pic, ar.secure_url]
```
[2.19202]
[2.19242]
```
							[productID, pic, destination]
```

Replacement in scrape/ocs.ca.js at line 281 [2.11674]

B:BD[2.19289] → [2.19289:19359]

							console.error("how the FUCK", productID, pic, ar.secure_url);

[2.19289]

[2.19359]

							console.error("how the FUCK", productID, pic, destination);

Insertion in scrape/ocs.ca.js at line 285 [2.11674]

[2.19440]

					} else if(product_images.length === 1) {
						if(product_images[0].destination !== null && process.env.CLOUDINARY_URL !== 'fake') {
							let response = (await exec(`./image.sh ${pic} ${product_images[0].destination}`)).stdout.trim();
							if(response.startsWith('/tmp/')) {
								await postgres.query('update product_image set destination = $1 where id = $2', [
									(await cloudinary.uploader.upload(response, {public_id: response.substring(5), unique_filename: false})).secure_url,
									product_images[0].id
								]);
							}
						}
					} else {
						console.error('data integrity issue', product_images);
						//product match, source match, but destinations are different, which should be unpossible

Deletion in scrape/ocs.ca.js at line 299 [2.11674]

B:BD[2.19448] → [2.19448:19825]

				} else if(product_images.length === 1) {
					//console.log('already persisted :)');
					//have a flag that once a month, we fetch images anyways just to see if there's a diff? thanks cloudinary :)
				} else {
					console.error('data integrity issue', product_images);
					//product match, source match, but destinations are different, which should be unpossible

Replacement in scrape/ocs.ca.js at line 317 [2.11674]
B:BD[2.20678] → [2.20678:20704]
```
				let portions = null;
```
[2.20678]
[2.20704]
```
				let portions;
```

Replacement in scrape/ocs.ca.js at line 319 [2.11674]

B:BD[2.20723] → [2.20723:21329]

				switch(e[i][j].type) {
					case 'capsule':
						quantity = e[i][j].meta.variant.retail_pack_number_of_items;
						break;
					case 'disposable':
					case 'cartridge':
					case 'flower':
					case 'spray':
					case 'oil':
						if(e[i][j].meta.variant && e[i][j].meta.variant.retail_pack_net_content) {
							quantity = e[i][j].meta.variant.retail_pack_net_content;
						} else {
							quantity = e[i][j].variant_title.match(qty)[3];
						}
						break;
					case 'tea':
					case 'gummy':
					case 'mint':
					case 'preroll':
					case 'cookie':
					case 'chocolate':

[2.20723]

[2.21329]

				if(e[i][j].variant_title === 'Default Title') {
					portions = e[i][j].meta.variant.retail_pack_number_of_items
					quantity = e[i][j].meta.variant.retail_pack_net_content;
				} else {
					[, , portions, quantity] = e[i][j].variant_title.match(qty);
					if(portions === undefined && e[i][0/*since we only modify the first index*/].type === 'preroll' && e[i][j].meta.variant.retail_pack_number_of_items > 1) {

Replacement in scrape/ocs.ca.js at line 326 [2.11674]

B:BD[2.21397] → [2.21397:21643]

						if(portions === 0) {
							[, , portions, quantity] = e[i][j].variant_title.match(qty);
						} else {
							quantity = (new Big(e[i][j].meta.variant.retail_pack_net_content)).div(new Big(portions)).toString();
						}
						break;

[2.21397]

[2.21643]

						quantity = (new Big(quantity)).div(portions);
					}

Replacement in scrape/ocs.ca.js at line 329 [2.11674]

B:BD[2.21650] → [2.21650:21761]

				if(portions !== null && portions < 2) {
					//console.log(productID, e[i][j].type, portions, quantity);

[2.21650]

[2.21761]

				if(!portions || portions < 2) {