{"header":{"message":"The web scrape functions","description":null,"timestamp":"2022-01-11T00:29:45.277171Z","authors":[{"email":"quick.dudley@gmail.com","full_name":"Jeremy List","name":"quickdudley"}]},"hash":"PQ4BG3ZJU5SY6XQDJ3SURLAOWGXGXLONIRZNUJXZVMHLYXWOORXQC","authors":[],"deps":{"hashes":[["PQ4BG3ZJU5SY6XQDJ3SURLAOWGXGXLONIRZNUJXZVMHLYXWOORXQC","The web scrape functions"],["AA",null],["KUANIPWFDXDFKJ2LH4FQ6APYOPLNYJ5LVGLSBSC75WUUZPFHILCAC","Add function for adding name to database"],["AV73DYWQ2UVZJYTN3UIW3TDMWNAETLXZRD7VHKTQLFQGHKJ5KI6QC","Initial functions for using sqlite in async environment"],["RNW6D77774CYWWM7JIFXI5TGKBOU6ADJIEZB5N2FXGDCTLUXEQZQC","Minor tidy"],["HMOBTVJ4FEPZWMUV2JDPZYH3EFCW6ED5M6KKNKQJQZVKTDAKTVFQC","Initialize crate and add expected dependencies"]],"deps":["KUANIPWFDXDFKJ2LH4FQ6APYOPLNYJ5LVGLSBSC75WUUZPFHILCAC"],"known":["AV73DYWQ2UVZJYTN3UIW3TDMWNAETLXZRD7VHKTQLFQGHKJ5KI6QC","RNW6D77774CYWWM7JIFXI5TGKBOU6ADJIEZB5N2FXGDCTLUXEQZQC"]},"hunks":[{"Replacement":{"inode":{"change":6,"pos":49},"path":"src/main.rs","line":7,"ins":{"up_context":[{"change":3,"pos":301}],"down_context":[{"change":2,"pos":1325}],"flag":"BLOCK","start":0,"end":52,"inode":{"change":6,"pos":49}},"del":{"edges":[{"previous":"BLOCK","flag":"BLOCK | DELETED","from":{"change":3,"pos":301},"to":{"change":2,"start":1272,"end":1325},"introduced_by":2}],"inode":{"change":6,"pos":49}},"ins_contents":[{"Add":" let db = names_database::AsyncConnection::open("}],"del_contents":[{"Del":" let _db = names_database::AsyncConnection::open(\n"}]}},{"FileAdd":{"meta":{"basename":"gather.rs","metadata":0},"context":[{"change":6,"pos":15}],"contents":[{"Add":"use std::borrow::Borrow;"},{"Add":""},{"Add":"#[derive(PartialEq, Eq, Copy, Clone)]"},{"Add":"pub enum Gender {"},{"Add":" Masculine,"},{"Add":" Feminine,"},{"Add":" Neutral,"},{"Add":"}"},{"Add":""},{"Add":"impl Gender {"},{"Add":" pub fn allow(&self, other: &Self) -> bool {"},{"Add":" match self {"},{"Add":" Self::Masculine => other != &Self::Feminine,"},{"Add":" Self::Feminine => other != &Self::Masculine,"},{"Add":" Self::Neutral => other == &Self::Neutral,"},{"Add":" }"},{"Add":" }"},{"Add":"}"},{"Add":""},{"Add":"pub fn all_names(filter: Gender) -> tokio::sync::mpsc::Receiver {"},{"Add":" let (tx, rx) = tokio::sync::mpsc::channel(256);"},{"Add":" let client = reqwest::Client::new();"},{"Add":" for initial in 'A'..='Z' {"},{"Add":" let tx = tx.clone();"},{"Add":" let client = client.clone();"},{"Add":" tokio::spawn(async move {"},{"Add":" let mut page = 0;"},{"Add":" loop {"},{"Add":" let results ="},{"Add":" fetch_name_page(&client, initial, page).await.unwrap();"},{"Add":" for (name, gender) in &results.names {"},{"Add":" if filter.allow(gender) {"},{"Add":" tx.send(name.clone()).await.unwrap();"},{"Add":" }"},{"Add":" }"},{"Add":" if results.has_next {"},{"Add":" page += 1;"},{"Add":" } else {"},{"Add":" break;"},{"Add":" }"},{"Add":" }"},{"Add":" });"},{"Add":" }"},{"Add":" rx"},{"Add":"}"},{"Add":""},{"Add":"struct ResultPage {"},{"Add":" names: Vec<(String, Gender)>,"},{"Add":" has_next: bool,"},{"Add":"}"},{"Add":""},{"Add":"async fn fetch_name_page("},{"Add":" client: &reqwest::Client,"},{"Add":" letter: char,"},{"Add":" page: usize,"},{"Add":") -> Result> {"},{"Add":" let page_header = format!(\"+Baby+Names+starting+with+{}\", letter);"},{"Add":" let starts = String::from(letter);"},{"Add":" let mut form = vec!["},{"Add":" (\"advanced\", \"1\"),"},{"Add":" (\"starts\", starts.borrow()),"},{"Add":" (\"end\", \"\"),"},{"Add":" (\"meaning\", \"\"),"},{"Add":" (\"origin\", \"\"),"},{"Add":" (\"nat\", \"\"),"},{"Add":" (\"startswith\", \"\"),"},{"Add":" (\"endswith\", \"\"),"},{"Add":" (\"gender\", \"\"),"},{"Add":" (\"cat\", \"\"),"},{"Add":" (\"syl\", \"\"),"},{"Add":" (\"page_header\", page_header.borrow()),"},{"Add":" ];"},{"Add":" let mut offset = String::new();"},{"Add":" match page {"},{"Add":" 0 => {"},{"Add":" form.extend("},{"Add":" [(\"offset\", \"66\"), (\"offset\", \"66\"), (\"Next\", \"Previous\")]"},{"Add":" .iter(),"},{"Add":" );"},{"Add":" }"},{"Add":" 1 => {"},{"Add":" form.extend([(\"offset\", \"0\"), (\"Next\", \"Next\")].iter());"},{"Add":" }"},{"Add":" _ => {"},{"Add":" offset = format!(\"{}\", (page - 1) * 66);"},{"Add":" form.extend("},{"Add":" ["},{"Add":" (\"offset\", offset.borrow()),"},{"Add":" (\"offset\", offset.borrow()),"},{"Add":" (\"Next\", \"Next\"),"},{"Add":" ]"},{"Add":" .iter(),"},{"Add":" );"},{"Add":" }"},{"Add":" }"},{"Add":" let response = client"},{"Add":" .post(\"https://babynames.com/names/search.php\")"},{"Add":" .form(&form[..])"},{"Add":" .send()"},{"Add":" .await?;"},{"Add":" StatusError::ensure_success(response.status())?;"},{"Add":" let text = response.text().await?;"},{"Add":" Ok(tokio::task::spawn_blocking(move || {"},{"Add":" let doc = scraper::Html::parse_document(text.borrow());"},{"Add":" let next_button = scraper::Selector::parse(\"input.next-btn\").unwrap();"},{"Add":" let mut next_button = doc.select(&next_button);"},{"Add":" let has_next = match next_button.next() {"},{"Add":" None => false,"},{"Add":" Some(button) => match button.value().attr(\"type\").map(str::trim) {"},{"Add":" Some(\"hidden\") => true,"},{"Add":" _ => false,"},{"Add":" },"},{"Add":" };"},{"Add":" let name_selector ="},{"Add":" scraper::Selector::parse(\"ul.searchresults a\").unwrap();"},{"Add":" let names = doc"},{"Add":" .select(&name_selector)"},{"Add":" .map(|item| {"},{"Add":" let gender = match item.value().attr(\"class\") {"},{"Add":" Some(\"M\") => Gender::Masculine,"},{"Add":" Some(\"F\") => Gender::Feminine,"},{"Add":" _ => Gender::Neutral,"},{"Add":" };"},{"Add":" let name: String = item.text().map(|s| s.trim()).collect();"},{"Add":" (name, gender)"},{"Add":" })"},{"Add":" .collect();"},{"Add":" ResultPage { names, has_next }"},{"Add":" })"},{"Add":" .await?)"},{"Add":"}"},{"Add":""},{"Add":"pub struct StatusError(reqwest::StatusCode);"},{"Add":""},{"Add":"impl StatusError {"},{"Add":" fn ensure_success(status: reqwest::StatusCode) -> Result<(), Self> {"},{"Add":" if status.is_success() {"},{"Add":" Ok(())"},{"Add":" } else {"},{"Add":" Err(Self(status))"},{"Add":" }"},{"Add":" }"},{"Add":"}"},{"Add":""},{"Add":"impl std::error::Error for StatusError {}"},{"Add":"impl std::fmt::Debug for StatusError {"},{"Add":" fn fmt("},{"Add":" &self,"},{"Add":" formatter: &mut std::fmt::Formatter<'_>,"},{"Add":" ) -> Result<(), std::fmt::Error> {"},{"Add":" ::fmt(&self.0, formatter)"},{"Add":" }"},{"Add":"}"},{"Add":""},{"Add":"impl std::fmt::Display for StatusError {"},{"Add":" fn fmt("},{"Add":" &self,"},{"Add":" formatter: &mut std::fmt::Formatter<'_>,"},{"Add":" ) -> Result<(), std::fmt::Error> {"},{"Add":" ::fmt(&self.0, formatter)"},{"Add":" }"},{"Add":"}"}]}}],"owner":"quickdudley","repo":"namepick","token":"Le/SgM4wdwLkPC8uhHEexqpt8ypAKsBQFKyYtozgycQ=","login":null}

…

Dependencies

Change contents