Finished framework for checking series specifications with data.

[?]

Jun 18, 2021, 9:11 AM

LVMGQJGH34VSGZCB74IQEJIVFNJE6OPQQCF7IPS72NPVMIUFBV4AC

Dependencies

[2] 77SIQZ3E Separating out spec generation.
[3] 5POF332L Working on fn cpi_included().
[4] 4MG5JFXT First record.
[5] GUXZCEWW Added Country enum.
[6] UCSU3QE4 Created keytree generator for u and cpi.
[7] CUADTSHQ Save csv data as multiple files.
[8] AIFRDCG2 Split off countries mod into a separate crate.
[*] AT753JPO Selected US unemployment series.

Change contents

file addition: ui.rs (-xw-xw-x--)

[3.6]

//! Deserialize data to be served as JSON to build Unemployment/Inflation scatter-plots.

use std::convert::TryInto;
use std::collections::HashMap;

use serde::{Serialize};

use countries::Country;
use keytree::{
KeyTreeRef,
Error,
ErrorKind,
};
use time_series::{
DateRange,
MonthlyDate,
RegularTimeSeries,
};

use crate::{
DataType,
};

use crate::ts::{
data_from_csv,
SeriesMetaData,
};

/// ```
/// ui:
/// ui_graphic:
/// time_series:
/// data_type: u
/// series: AUSURAMS_a
/// transform: ident
/// time_series:
/// data_type: i
/// series: _
/// transform: ident
/// x: 0
/// y: 1
/// ```
pub struct UISpec(pub HashMap<Country, Vec<UIGraphicSpec>>);

impl<'a> TryInto<UISpec> for KeyTreeRef<'a> {
type Error = Error;

fn try_into(self) -> Result<UISpec, Error> {

let countries: Vec<CountrySpec> = self.vec("ui::country")?;

let mut h = HashMap::new();

for country in countries {
h.insert(country.key, country.value.0);
}
Ok(UISpec(h))
}
}

pub struct CountrySpec {
key: Country,
value: ValueSpec,
}

impl<'a> TryInto<CountrySpec> for KeyTreeRef<'a> {
type Error = Error;

fn try_into(self) -> Result<CountrySpec, Error> {

let country_str: String = self.at("country::key")?;

let country = Country::from_str(&country_str).unwrap();

Ok(
CountrySpec {
key: country,
value: self.at("country::value")?,
}
)
}
}

pub struct ValueSpec(Vec<UIGraphicSpec>);

impl<'a> TryInto<ValueSpec> for KeyTreeRef<'a> {
type Error = Error;

fn try_into(self) -> Result<ValueSpec, Error> {
Ok(ValueSpec(self.vec("value::graphic")?))
}
}

/// An integer reference to a series, for x and y coordinates.
/// ```
/// ui:
/// country:
/// key: Australia
/// value:
/// graphic:
/// title: Unemployment and Inflation for Australia
/// time_series:
/// data_type: u
/// series: AUSURAMS_a
/// transform: ident
/// time_series:
/// data_type: i
/// series: _
/// transform: ident
/// line:
/// x: 0
/// y: 1
/// ```
/// For instance in this example, x refers to the first time-series AUSURAMS_a, and y refers to the
/// second time-series.
pub struct UILineSpec {
x: String,
y: String,
start_date: Option<MonthlyDate>,
end_date: Option<MonthlyDate>,
}

impl<'a> TryInto<UILineSpec> for KeyTreeRef<'a> {
type Error = Error;

fn try_into(self) -> Result<UILineSpec, Error> {
let start_date_str: Option<String> = self.op("line::start_date")?;

let start_date_opt = match start_date_str {
Some(date) => Some(parse_date(&date)?),
None => None,
};

let end_date_str: Option<String> = self.op("line::end_date")?;

let end_date_opt = match end_date_str {
Some(date) => Some(parse_date(&date)?),
None => None,
};

Ok(
UILineSpec {
x: self.at("line::x")?,
y: self.at("line::y")?,
start_date: start_date_opt,
end_date: end_date_opt,
}
)
}
}

fn parse_date(s: &str) -> Result<MonthlyDate, Error> {
let year = s[..=4].parse::<usize>().map_err(|err| keytree::Error(Box::new(ErrorKind::User(err.to_string()))))?;
let month = s[5..].parse::<usize>().map_err(|err| keytree::Error(Box::new(ErrorKind::User(err.to_string()))))?;
Ok(MonthlyDate::ym(year, month))
}

#[test]
fn test_parse_date() {
let s = "1999-03";
assert!(
parse_date(s),
MonthlyDate(1999, 3),
)
}

// Structs UIJson, UIGraphJson and UILineJson encapsulate data to be converted to JSON.

/// `UIJson` are a collection of `LineJson` convertible to JSON for a ui graphic.
#[derive(Serialize)]
pub struct UIJson {
country: Country,
graphics: Vec<UIGraphicJson>,
}

impl UIJson {

/// Build a new `UIJson` from a `UISpec` specification.
pub fn new(ui_spec: &UISpec, country: Country) -> Self {

let ui_country_spec = ui_spec.0.get(&country).unwrap();

let mut builder = UIJson {
country: country,
graphics: Vec::new(),
};

for graphic_spec in ui_country_spec {
builder.graphics.push(UIGraphicJson::new(graphic_spec, country));
}

builder
}
}

/// `UIGraphicJson` is a JSON representation of a UI graph.
#[derive(Serialize)]
pub struct UIGraphicJson {
title: String,
lines: Vec<UILineJson>,
}

impl UIGraphicJson {

pub fn new(graphic_spec: &UIGraphicSpec, country: Country) -> Self {

let mut builder = UIGraphicJson {
title: graphic_spec.title.clone(),
lines: Vec::new(),
};

for line_spec in &graphic_spec.line {
builder.lines.push(UILineJson::new(&line_spec, country));
};

builder
}
}

/// `UILineJson` are two time-series
#[derive(Serialize)]
pub struct UILineJson {
x: RegularTimeSeries<1>,
y: RegularTimeSeries<1>,

x_meta: SeriesMetaData,
y_meta: SeriesMetaData,
}

impl UILineJson {
/// Build a new `UILineJson` from a `UILineSpec` specification.
pub fn new(line_spec: &UILineSpec, country: Country) -> Self {

let (mut x_data, x_meta) = data_from_csv(&line_spec.x, country, DataType::U);

let range = DateRange::new(
line_spec.start_date,
line_spec.end_date,
);

x_data.with_range(&range);

let (mut y_data, y_meta) = data_from_csv(&line_spec.y, country, DataType::Inf);

y_data.with_range(&range);

UILineJson {
x: x_data,
y: y_data,
x_meta: x_meta,
y_meta: y_meta,
}
}
}

/// ```
/// ui:
/// country:
/// key: Australia
/// value:
/// graphic:
/// title: Unemployment and Inflation for Australia
/// line:
/// x: AUSURAMS_a
/// y: _
/// start_date: _
/// end_date: _
/// line:
/// ```
pub struct UIGraphicSpec {
pub title: String,
pub line: Vec<UILineSpec>,
}

impl<'a> TryInto<UIGraphicSpec> for KeyTreeRef<'a> {
type Error = Error;

fn try_into(self) -> Result<UIGraphicSpec, Error> {
Ok(
UIGraphicSpec {
title: self.at("ui_graphic::title")?,
line: self.vec("ui_graphic::line")?,
}
)
}
}
edit in src/ts.rs at line 6

[2.163]→[2.163:177](∅→∅)

use std::fmt;
edit in src/ts.rs at line 9

[2.215]→[2.215:231](∅→∅)

use serde_json;
file addition: spec_to_data.rs (-xw-xw-x--)

[3.6]

//! Use a specification to fetch and save data to disk.

use std::convert::TryInto;
use std::fs;
use std::path::PathBuf;

use countries::Country;
use fred_api::Fred;
use keytree::{KeyTree, KeyTreeRef};

use crate::ts::SourceSpec;

/// Read the specification, pull the data down from FRED and save to disk.
/// ```
/// save_data(spec: "source_spec.keytree", "/path/test_data");
/// ```
pub fn save_data(spec: &str, full_path: &str) {

let spec = fs::read_to_string(&spec).unwrap();

let kt = KeyTree::parse(&spec).unwrap();
let source_spec: SourceSpec = kt.to_ref().try_into().unwrap();

for series_id in &source_spec.all_series() {
save_series_meta(series_id, full_path);
save_data_csv(series_id, full_path);
}
}

/// Save FRED series metadata including title to disk as json.
/// ```
/// save_series_meta("LRUNTTTTSIQ156S");
/// ```
pub fn save_series_meta(series_id: &str, full_path: &str) {
dbg!(&series_id);
let data = match Fred::series(series_id) {
Ok(series) => series.to_string(),
Err(err) => {
println!("{}", err);
panic!();
},
};

let file_path = &format!(
"{}/{}.meta",
full_path,
series_id,
);

if PathBuf::from(file_path).exists() {
println!("Cannot write data as filepath {} already exists.", file_path);
panic!();
} else {
fs::write(file_path, &data).unwrap();
};
}

/// Save FRED data to disk as csv, using full path. Will fail if an existing filepath is
/// encountered.
/// ```
/// save_data_csv("LRUNTTTTSIQ156S");
/// ```
/// To handle breaks the data is saved in a collection of files "LRUNTTTTSIQ156S_a.csv",
/// "LRUNTTTTSIQ156S_b.csv" etc.
///
pub fn save_data_csv(series_id: &str, path: &str) {

let alphabet = "abcdefghijklmnopqrstuvwxyz";
let mut chars = alphabet.chars();
let mut contains_data = false;
let mut data = String::new();

// Select the observation data from the FRED data.

let observations = match Fred::series_observations(series_id) {
Ok(series_obs) => series_obs.observations,
Err(err) => {
println!("{}", err);
panic! ();
},
};

for obs in observations.iter() {

let value: Result<f32, _> = obs.value.parse();
dbg!(&value);

match value {
Ok(_) => {
contains_data = true;
data.push_str(&obs.to_string());
data.push('\n');
},

Err(_) => {
if contains_data {

// Remove last linefeed.
data.pop();

fs::create_dir_all(&path).unwrap();
let file_name = &format!(
"{}_{}",
series_id,
chars.next().expect("Too many breaks"),
);
let file_path = &format!(
"{}/{}.csv",
path,
file_name,
);
println!(" graphic:");
println!(
" data: {}",
&file_name,
);

if PathBuf::from(file_path).exists() {
println!("Cannot write data as filepath {} already exists.", file_path);
panic!();
} else {
fs::write(file_path, &data).unwrap();
};

data = String::new();
contains_data = false;
}
},
}
}

if contains_data {
fs::create_dir_all(&path).unwrap();

let file_name = &format!(
"{}_{}",
series_id,
chars.next().expect("Too many breaks"),
);
let file_path = &format!(
"{}/{}.csv",
path,
file_name,
);
println!(" graphic:");
println!(
" data: {}",
&file_name,
);

if PathBuf::from(file_path).exists() {
println!("Cannot write data as filepath {} already exists.", file_path);
panic!();
} else {
fs::write(file_path, &data).unwrap();
};
}
}
edit in src/main.rs at line 1

[3.10]

[10.1]

use std::convert::TryInto;
use std::fs;
edit in src/main.rs at line 5

[10.21]

[3.0]

use keytree::serialize::IntoKeyTree;
replacement in src/main.rs at line 9

[3.16]→[2.9542:9570](∅→∅)

use ui_data::build_spec::*;

[3.16]

[3.110]

use ui_data::build_source_spec::*;
replacement in src/main.rs at line 32

[2.10145]→[2.10145:10186](∅→∅)

println!("{}", build_data_spec());

[2.10145]

[3.292]

// println!("{}", build_data_spec().keytree());

let spec = DataSpec::from_file("source_data.keytree");
println!("{}", spec.keytree());

let checked = spec.check();
println!("{}", checked.keytree());
replacement in src/lib.rs at line 3

[2.10274]→[2.10274:10391](∅→∅)

//! Probably the most
//! effective way to find stuff, is to search on tags and then filter on series title, such as

[2.10274]

[2.10391]

//! ### Step 1: Find the data you want.
//!
//! Probably the most effective way to find stuff, is to search on tags and then filter on series
//! title, such as
edit in src/lib.rs at line 16

[2.10572]

[2.10572]

//! ```
//! The filters such as `has_phrase` are found in the `fred_api` crate.
//!
//! ### Step 2 Build a specification of data sources.
//!
//! The results of Step 1 are integrated into algorithms in the `build_spec` module. These can then
//! be used to generate collections of series.
//!
//! These specifications are then edited by hand, and cannot be programatically regenerated after
//! editting. To modify the spec. programmatically, a newly generated `Keytree` string can be
//! manually appended.
//!
//! Example:
//! ```
//! println!("{}", build_data_spec().keytree());
//! ```
//! Save the output as file `source_data.keytree`. Then
replacement in src/lib.rs at line 34

[2.10580]→[2.10580:10650](∅→∅)

//! The filters such as `has_phrase` are found in the fred_api crate.

[2.10580]

[3.798]

//! let spec = DataSpec::from_file("source_data.keytree");
//! println!("{}", spec.keytree());
//! ```
//!
//! ### Step 3. Check that data is well-formed and record results in specification.
//!
//! Check each data series, to see if the data is well-formed. Save the output specification.
//!
//! ### Step 4 Save the data on disk.
//!
//! Use the specification from step 3 to build or update data files.
//!
//! ### Step 5 Present data to serve.
//!
replacement in src/lib.rs at line 49

[3.799]→[2.10651:10693](∅→∅)

pub mod build_spec;
pub mod spec_to_data;

[3.799]

[2.10693]

pub mod build_source_spec;
pub mod check_data;
// pub mod spec_to_data;
edit in src/lib.rs at line 56

[3.366]→[3.799:812](∅→∅),[2.10732]→[3.799:812](∅→∅),[3.799]→[3.799:812](∅→∅),[3.812]→[2.10733:10757](∅→∅)

use std::fs;
use std::path::PathBuf;
edit in src/lib.rs at line 59

[3.478]→[3.151:175](∅→∅)

use countries::Country;
edit in src/lib.rs at line 60

[3.498]→[2.10781:10808](∅→∅)

use fred_api::SeriesItems;
file addition: check_data.rs (-xw-xw-x--)

[3.6]

use std::convert::TryInto;
use std::fs;

use countries::Country;
use fred_api::{
Fred,
};
use keytree::{KeyTree, KeyTreeRef};
use keytree::Error;
use keytree::serialize::{
IntoKeyTree,
KeyTreeString,
};

use crate::build_source_spec::DataSpec;
use crate::DataType;

impl DataSpec {
pub fn check(&self) -> CheckedDataSpec {
let mut builder = CheckedDataSpec(Vec::new());

for series_spec in self.0 {
println!("{}", series_spec.id);
match Fred::series(&series_spec.id) {
Ok(series) => {
let checked_series = CheckedSourceSeries {
data_type: series_spec.data_type,
country: series_spec.country,
id: series_spec.id.clone(),
error: "ok".to_string(),
};
builder.0.push(checked_series);
},
Err(err) => {
let checked_series = CheckedSourceSeries {
data_type: series_spec.data_type,
country: series_spec.country,
id: series_spec.id.clone(),
error: err.to_string(),
};
builder.0.push(checked_series);
}
}
}
builder
}
}

/// ```
/// series:
/// data_type: u
/// country: United States
/// id: LRUNTTTTUSQ156S
/// ```
pub struct CheckedDataSpec(Vec<CheckedSourceSeries>);

impl CheckedDataSpec {

/// Read in keytree data from file.
pub fn from_file(path: &str) -> Self {
let source_spec = fs::read_to_string(path).unwrap();
let kt = KeyTree::parse(&source_spec).unwrap();
kt.to_ref().try_into().unwrap()
}

/// Read in source_data.keytree, download series, and return
/// `CheckedSourceSeries`.
pub fn update(&self) {
for series in &self.0 {
let data = match Fred::series(&series.id) {
Ok(data) => println!("{}", data.keytree()),
Err(err) => println!("{:?}", err),
};
}
}
}

impl<'a> TryInto<CheckedDataSpec> for KeyTreeRef<'a> {
type Error = Error;

fn try_into(self) -> Result<CheckedDataSpec, Error> {
Ok(
CheckedDataSpec(self.vec("seriess:series")?)
)
}
}

impl IntoKeyTree for CheckedDataSpec {
fn keytree(&self) -> KeyTreeString {
let mut kt = KeyTreeString::new();
kt.push_key(0, "seriess");
for series in &self.0 {
kt.push_keytree(1, series.keytree());
}
kt
}
}

pub struct CheckedSourceSeries {
data_type: DataType,
country: Country,
id: String,
error: String,
}

impl IntoKeyTree for CheckedSourceSeries {
fn keytree(&self) -> KeyTreeString {
let mut kt = KeyTreeString::new();
kt.push_key(0, "series");
kt.push_keyvalue(1, "data_type", &self.data_type.to_string());
kt.push_keyvalue(1, "country", &self.country.to_string());
kt.push_keyvalue(1, "id", &self.id);
kt.push_keyvalue(1, "error", &self.error);
kt
}
}

impl<'a> TryInto<CheckedSourceSeries> for KeyTreeRef<'a> {
type Error = keytree::Error;

fn try_into(self) -> Result<CheckedSourceSeries, keytree::Error> {
let data_type_str: String = self.at("series::data_type")?;
let data_type = DataType::from_str(&data_type_str).unwrap();

let country_str: String = self.at("series::country")?;
let country = Country::from_str(&country_str).unwrap();

Ok(
CheckedSourceSeries{
data_type,
country,
id: self.at("series::id")?,
error: self.at("series::error")?,
}
)
}
}
file addition: build_source_spec.rs (-xw-xw-x--)

[3.6]

//! Builds generic specifications.

use std::convert::TryInto;
use std::fs;

use countries::Country;
use fred_api::{
Fred,
SeriesItems,
};
use keytree::{
KeyTree,
KeyTreeRef,
};
use keytree::Error;
use keytree::serialize::{
IntoKeyTree,
KeyTreeString,
};

use crate::DataType;

/// Return all the countries with good data as a `Vec`.
pub fn countries_with_data() -> Vec<Country> {
vec!(
Country::Australia,
Country::Austria,
Country::Belgium,
Country::Canada,
Country::Chile,
Country::CzechRepublic,
Country::Denmark,
Country::Estonia,
Country::Finland,
Country::France,
Country::Germany,
Country::Greece,
Country::Ireland,
Country::Israel,
Country::Italy,
Country::Japan,
Country::Latvia,
Country::Netherlands,
Country::NewZealand,
Country::Norway,
Country::Poland,
Country::Serbia,
Country::SouthKorea,
Country::Spain,
Country::Sweden,
Country::Switzerland,
Country::UnitedKingdom,
Country::UnitedStates,
)
}

/// Returns relevant CPI series for a country, by pre-selecting series titles for each country.
///
/// ```
/// println!("{}", generic_cpi_series_spec(&Country::NewZealand, "cpi_series.keytree"));
/// ```
pub fn generic_cpi_series_spec(country: &Country, path: &str) -> SeriesItems {
let titles = match country {
Country::Australia => {
vec!(
"Consumer Price Index: All items: Total: Total for Australia",
"Consumer Price Index: All Items for Australia",
"Consumer Price Index in Australia (DISCONTINUED)",
"Consumer Price Index: Total All Items for Australia",
"Consumer Price Index for Australia",
"Inflation, consumer prices for Australia",
)
}
Country::Austria => {
vec!(
"Consumer Price Index for Austria",
"Consumer Price Index: All items: Total: Total for Austria",
"Consumer Price Index: All Items for Austria",
"Consumer Price Index in Austria (DISCONTINUED)",
"Consumer Price Index: Harmonized Prices: Total All Items for Austria",
"Consumer Price Index: Harmonized Prices: Total All Items for Austria // (DISCONTINUED)",
"Consumer Price Index: Total All Items for Austria",
"Harmonized Index of Consumer Prices: All Items for Austria",
"Harmonized Index of Consumer Prices in Austria (DISCONTINUED)",
"Inflation, consumer prices for Austria",
"Consumer Price Index: All Items (Harmonized Index of Consumer Prices) for // Austria",
"Consumer Price Index: All Items (Harmonized Index of Consumer Prices) for Austria (DISCONTINUED)",
)
}
Country::Belgium => {
vec!(
"Consumer Price Index for Belgium",
"Consumer Price Index: Harmonized Prices: Total All Items for Belgium",
"Consumer Price Index: Harmonized Prices: Total All Items for Belgium (DISCONTINUED)",
"Consumer Price Index: Total All Items for Belgium",
"Harmonized Index of Consumer Prices: All Items for Belgium",
"Harmonized Index of Consumer Prices in Belgium (DISCONTINUED)",
"Consumer Price Index: All Items for Belgium",
"Consumer Price Index in Belgium (DISCONTINUED)",
"Consumer Price Index: All items: Total: Total for Belgium",
)
}
Country::Canada => {
vec!(
"Inflation, consumer prices for Canada",
"Consumer Price Index for Canada",
"Consumer Price Index: Total All Items for Canada",
"Consumer Price Index in Canada (DISCONTINUED)",
"Consumer Price Index of All Items in Canada",
"Consumer Price Index: All items: Total: Total for Canada",
)
}
Country::Chile => {
vec!(
"Consumer Price Index for Chile",
"Inflation, consumer prices for Chile",
"Consumer Price Index: Total All Items for Chile",
"Consumer Price Index: All Items for Chile",
"Consumer Price Index: All items: Total: Total for Chile",
)
}
Country::CzechRepublic => {
vec!(
"Consumer Price Index for Czech Republic",
"Consumer Price Index: All Items (Harmonized Index of Consumer Prices) for Czech Republic",
"Consumer Price Index: All Items (Harmonized Index of Consumer Prices) for Czech Republic (DISCONTINUED)",
"Consumer Price Index: All Items for Czech Republic",
"Consumer Price Index: All items: Total: Total for the Czech Republic",
"Consumer Price Index: Harmonized Prices: Total All Items for the Czech Republic",
"Consumer Price Index: Harmonized Prices: Total All Items for the Czech Republic (DISCONTINUED)",
"Consumer Price Index: Total All Items for the Czech Republic",
"Harmonized Index of Consumer Prices: All Items for Czech Republic",
)
}
Country::Denmark => {
vec!(
"Inflation, consumer prices for Denmark",
"Consumer Price Index: All Items (Harmonized Index of Consumer Prices) for Denmark",
"Consumer Price Index: All Items (Harmonized Index of Consumer Prices) for Denmark (DISCONTINUED)",
"Harmonized Index of Consumer Prices in Denmark (DISCONTINUED)",
"Consumer Price Index: All Items for Denmark",
"Consumer Price Index in Denmark (DISCONTINUED)",
"Consumer Price Index: All items: Total: Total for Denmark",
"Consumer Price Index for Denmark",
"Consumer Price Index: Total All Items for Denmark",
"Harmonized Index of Consumer Prices: All Items for Denmark",
)
}
Country::Estonia => {
vec!(
"Inflation, consumer prices for Estonia",
"Consumer Price Index: All items: Total: Total for Estonia",
"Consumer Price Index for Estonia",
"Consumer Price Index: Harmonized Prices: Total All Items for Estonia",
"Consumer Price Index: Harmonized Prices: Total All Items for Estonia (DISCONTINUED)",
"Consumer Price Index: Total All Items for Estonia",
"Harmonized Index of Consumer Prices: All Items for Estonia",
)
}
Country::Finland => {
vec!(
"Inflation, consumer prices for Finland",
"Consumer Price Index: All Items (Harmonized Index of Consumer Prices) for Finland",
"Consumer Price Index: All Items (Harmonized Index of Consumer Prices) for Finland (DISCONTINUED)",
"Consumer Price Index: All Items for Finland",
"Consumer Price Index: OECD Groups: Services: Total for Finland",
"Consumer Price Index: All items: Total: Total for Finland",
"Consumer Price Index: Harmonized Prices: Total All Items for Finland (DISCONTINUED)",
"Consumer Price Index for Finland",
"Consumer Price Index: Harmonized Prices: Total All Items for Finland",
"Consumer Price Index: Total All Items for Finland",
"Harmonized Index of Consumer Prices: All Items for Finland",
)
}
Country::France => {
vec!(
"Harmonized Index of Consumer Prices in France (DISCONTINUED)",
"Consumer Price Index: All Items (Harmonized Index of Consumer Prices) for France (DISCONTINUED)",
"Consumer Price Index: All Items (Harmonized Index of Consumer Prices) for France",
"Consumer Price Index in France (DISCONTINUED)",
"Consumer Price Index of All Items in France",
"Consumer Price Index: All items: Total: Total for France",
"Inflation, consumer prices for France",
"Consumer Price Index for France",
"Consumer Price Index: Harmonized Prices: Total All Items for France (DISCONTINUED)",
"Consumer Price Index: Harmonized Prices: Total All Items for France",
"Consumer Price Index: Total All Items for France",
)
}
Country::Germany => {
vec!(
"Inflation, consumer prices for Germany",
"Harmonized Index of Consumer Prices in Germany (DISCONTINUED)",
"Consumer Price Index: All Items (Harmonized Index of Consumer Prices) for Germany (DISCONTINUED)",
"Consumer Price Index: All Items (Harmonized Index of Consumer Prices) for Germany",
"Consumer Price Index in Germany (DISCONTINUED)",
"Consumer Price Index of All Items in Germany",
"Consumer Price Index: All items: Total: Total for Germany",
"Consumer Price Index for Germany",
"Consumer Price Index: Harmonized Prices: Total All Items for Germany (DISCONTINUED)",
"Consumer Price Index: Harmonized Prices: Total All Items for Germany",
"Consumer Price Index: Total All Items for Germany",
"Harmonized Index of Consumer Prices: All Items for Germany (including former GDR from 1991)",
)
}
Country::Greece => {
vec!(
"Consumer Price Index: All Items (Harmonized Index of Consumer Prices) for Greece",
"Consumer Price Index: All Items (Harmonized Index of Consumer Prices) for Greece (DISCONTINUED)",
"Consumer Price Index: All Items for Greece",
"Consumer Price Index: All items: Total: Total for Greece",
"Inflation, consumer prices for Greece",
"Consumer Price Index for Greece",
"Consumer Price Index: Harmonized Prices: Total All Items for Greece",
"Consumer Price Index: Harmonized Prices: Total All Items for Greece (DISCONTINUED)",
"Consumer Price Index: Total All Items for Greece",
"Harmonized Index of Consumer Prices: All Items for Greece",
)
}
Country::Ireland => {
vec!(
"Consumer Price Index: All Items (Harmonized Index of Consumer Prices) for Ireland",
"Consumer Price Index: All Items (Harmonized Index of Consumer Prices) for Ireland (DISCONTINUED)",
"Consumer Price Index: OECD Groups: Services: Total for Ireland",
// "Consumer Price Index: All Items for Ireland", // Does not parse
// "Consumer Price Index: All items: Total: Total for Ireland", // Does not parse

"Consumer Price Index: Harmonized Prices: Total All Items for Ireland (DISCONTINUED)",
"Consumer Price Index for Ireland",
"Consumer Price Index: Harmonized Prices: Total All Items for Ireland",
"Harmonized Index of Consumer Prices: All Items for Ireland",
)
}
Country::Israel => {
vec!(
"Consumer Price Index: All Items for Israel",
"Consumer Price Index: All items: Total: Total for Israel",
"Consumer Price Index for Israel",
"Inflation, consumer prices for Israel",
"Consumer Price Index: Total All Items for Israel",
)
}
Country::Italy => {
vec!(
"Harmonized Index of Consumer Prices in Italy (DISCONTINUED)",
"Consumer Price Index: All Items (Harmonized Index of Consumer Prices) for Italy (DISCONTINUED)",
"Consumer Price Index: All Items (Harmonized Index of Consumer Prices) for Italy",
"Consumer Price Index: Food for Italy",
"Consumer Price Index in Italy (DISCONTINUED)",
"Consumer Price Index of All Items in Italy",
"Consumer Price Index: All items: Total: Total for Italy",
"Inflation, consumer prices for Italy",
"Consumer Price Index for Italy",
"Consumer Price Index: Harmonized Prices: Total All Items for Italy (DISCONTINUED)",
"Consumer Price Index: Harmonized Prices: Total All Items for Italy",
"Consumer Price Index: Total All Items for Italy",
"Harmonized Index of Consumer Prices: All Items for Italy",
)
}
Country::Japan => {
vec!(
"Harmonized Index of Consumer Prices in Japan (DISCONTINUED)",
"Consumer Price Index in Japan (DISCONTINUED)",
"Consumer Price Index of All Items in Japan",
"Consumer Price Index: All items: Total: Total for Japan",
"Inflation, consumer prices for Japan",
"Not Seasonally Adjusted",
)
}
Country::Latvia => {
vec!(
"Consumer Price Index: All items: Total: Total for Latvia",
"Inflation, consumer prices for Latvia",
"Harmonized Index of Consumer Prices: Unprocessed Food for Latvia",
"Consumer Price Index for Latvia",
)
}
Country::Netherlands => {
vec!(
"Harmonized Index of Consumer Prices in Netherlands (DISCONTINUED)",
"Consumer Price Index: All Items (Harmonized Index of Consumer Prices) for Netherlands (DISCONTINUED)",
"Consumer Price Index: All Items (Harmonized Index of Consumer Prices) for Netherlands",
"Consumer Price Index in Netherlands (DISCONTINUED)",
"Consumer Price Index: All Items for Netherlands",
"Consumer Price Index: All items: Total: Total for the Netherlands",
"Consumer Price Index for Netherlands",
"Consumer Price Index: Harmonized Prices: Total All Items for the Netherlands",
"Consumer Price Index: Harmonized Prices: Total All Items for the Netherlands (DISCONTINUED)",
"Consumer Price Index: Total All Items for the Netherlands",
"Harmonized Index of Consumer Prices: All Items for Netherlands",
)
}
Country::NewZealand => {
vec!(
"Consumer Price Index: All Items for New Zealand",
"Consumer Price Index: All Items Excluding Food and Energy for New Zealand",
"Consumer Price Index: All items: Total: Total for New Zealand",
"Consumer Price Index for New Zealand",
"Inflation, consumer prices for New Zealand",
"Consumer Price Index: Total All Items for New Zealand",
)
}
Country::Norway => {
vec!(
"Consumer Price Index: All Items (Harmonized Index of Consumer Prices) for Norway",
"Consumer Price Index: All Items (Harmonized Index of Consumer Prices) for Norway (DISCONTINUED)",
"Consumer Price Index: All Items for Norway",
"Consumer Price Index in Norway (DISCONTINUED)",
"Consumer Price Index: All items: Total: Total for Norway",
"Consumer Price Index: Harmonized Prices: Total All Items for Norway (DISCONTINUED)",
"Consumer Price Index for Norway",
"Consumer Price Index: Harmonized Prices: Total All Items for Norway",
"Consumer Price Index: Total All Items for Norway",
"Harmonized Index of Consumer Prices: All Items for Norway",
)
}
Country::Poland => {
vec!(
"Consumer Price Index: All Items (Harmonized Index of Consumer Prices) for Poland",
"Consumer Price Index: All Items (Harmonized Index of Consumer Prices) for Poland (DISCONTINUED)",
"Consumer Price Index: Food for Poland",
"Consumer Price Index: All Items for Poland",
"Consumer Price Index: All items: Total: Total for Poland",
"Inflation, consumer prices for Poland",
"Consumer Price Index for Poland",
"Consumer Price Index: Total All Items for Poland",
"Harmonized Index of Consumer Prices: All Items for Poland",
)
}
Country::Serbia => {
vec!(
"Consumer Price Index for Serbia",
"Inflation, consumer prices for Serbia",
)
}
Country::SouthKorea => {
vec!(
"Consumer Price Index: All Items for Korea",
"Consumer Price Index: All items: Total: Total for the Republic of Korea",
"Inflation, consumer prices for the Republic of Korea",
"Consumer Price Index for Republic of Korea",
"Consumer Price Index: Total All Items for the Republic of Korea",
)
}
Country::Spain => {
vec!(
"Inflation, consumer prices for Spain",
"Consumer Price Index: All Items (Harmonized Index of Consumer Prices) for Spain (DISCONTINUED)",
"Consumer Price Index: All Items (Harmonized Index of Consumer Prices) for Spain",
"Consumer Price Index in Spain (DISCONTINUED)",
"Consumer Price Index: All Items for Spain",
"Consumer Price Index: All items: Total: Total for Spain",
"Consumer Price Index for Spain",
"Consumer Price Index: Harmonized Prices: Total All Items for Spain (DISCONTINUED)",
"Consumer Price Index: Harmonized Prices: Total All Items for Spain",
"Consumer Price Index: Total All Items for Spain",
"Harmonized Index of Consumer Prices: All Items for Spain",
)
}
Country::Sweden => {
vec!(
"Harmonized Index of Consumer Prices in Sweden (DISCONTINUED)",
"Consumer Price Index: All Items (Harmonized Index of Consumer Prices) for Sweden",
"Consumer Price Index: All Items (Harmonized Index of Consumer Prices) for Sweden (DISCONTINUED)",
"Consumer Price Index: All Items for Sweden",
"Consumer Price Index in Sweden (DISCONTINUED)",
"Consumer Price Index: All items: Total: Total for Sweden",
"Consumer Price Index: Harmonized Prices: Total All Items for Sweden",
"Consumer Price Index: Harmonized Prices: Total All Items for Sweden (DISCONTINUED)",
"Consumer Price Index for Sweden",
"Consumer Price Index: Total, Net All Items for Sweden (DISCONTINUED)",
"Consumer Price Index: Total All Items for Sweden",
"Harmonized Index of Consumer Prices: All Items for Sweden",
)
}
Country::Switzerland => {
vec!(
"Consumer Price Index: Harmonized Prices: Total All Items for Switzerland",
"Consumer Price Index: Harmonized Prices: Total All Items for Switzerland (DISCONTINUED)",
"Consumer Price Index for Switzerland",
"Consumer Price Index: Total All Items for Switzerland",
"Harmonized Index of Consumer Prices: All Items for Switzerland",
"Consumer Price Index: All Items (Harmonized Index of Consumer Prices) for Switzerland",
"Consumer Price Index: All Items (Harmonized Index of Consumer Prices) for Switzerland (DISCONTINUED)",
)
}
Country::UnitedKingdom => {
vec!(
"Harmonized Index of Consumer Prices in the United Kingdom (DISCONTINUED)",
"Consumer Price Index: All Items (Harmonized Index of Consumer Prices) for United Kingdom (DISCONTINUED)",
"Consumer Price Index: All Items (Harmonized Index of Consumer Prices) for United Kingdom",
"Consumer Price Index in the United Kingdom (DISCONTINUED)",
"Consumer Price Index of All Items in the United Kingdom",
"Consumer Price Index: All Items for United Kingdom",
"Consumer Price Index: All items: Total: Total for the United Kingdom",
"Inflation, consumer prices for the United Kingdom",
"Consumer Price Index for United Kingdom",
"Consumer Price Index in the United Kingdom",
"Consumer Price Inflation in the United Kingdom",
"Consumer Price Index: Harmonized Prices: Total All Items for the United Kingdom (DISCONTINUED)",
"Consumer Price Index: Harmonized Prices: Total All Items for the United Kingdom",
"Consumer Price Index: Total All Items for the United Kingdom",
"Harmonized Index of Consumer Prices: All Items for United Kingdom",
)
}
Country::UnitedStates => {
vec!(
"Harmonized Index of Consumer Prices: All Items for United States",
"Consumer Price Index: Total All Items for the United States",
"Consumer Price Index: Harmonized Prices: Total All Items for the United States",
"Consumer Price Index: Harmonized Prices: Total All Items for the United States (DISCONTINUED)",
"Research Consumer Price Index: All Items",
"Consumer Price Index for United States",
"Flexible Price Consumer Price Index",
"Inflation, consumer prices for the United States",
"Consumer Price Index, All Items for United States",
"Rate of Change (6 Month Span at Annual Rate), Consumer Price Index, All Items (Centered) for United States",
"Median Consumer Price Index",
"Sticky Price Consumer Price Index",
"16% Trimmed-Mean Consumer Price Index",
"Consumer Price Index: All Items for the United States",
"Consumer Price Index of All Items in United States",
"Consumer Price Index in the United States (DISCONTINUED)",
"Consumer Price Index: All Items (Harmonized Index of Consumer Prices) for the United States",
"Consumer Price Index: All Items (Harmonized Index of Consumer Prices) for the United States (DISCONTINUED)",
"Harmonized Index of Consumer Prices in the United States (DISCONTINUED)",
)
}
_ => panic!(),
};
match country {
Country::UnitedStates => {
match Fred::tags_series("cpi;usa") {
Ok(tags_series) => {
tags_series.seriess
.equals_one_of(titles)
},
Err(json_error) => {
println!("{}", json_error);
panic!();
},
}
},
_ => {
match Fred::tags_series(&to_tag("cpi", &country)) {
Ok(tags_series) => {
tags_series.seriess
.equals_one_of(titles)
},
Err(json_error) => {
println!("{}", json_error);
panic!();
},
}
},
}
}

fn to_tag(tag: &str, country: &Country) -> String {
format!(
"{};{}",
tag,
country.to_string().to_lowercase()
)
}

/// Return relevant unemployment rate series for a country.
/// ```
/// println!("{}", unemployment_series(&Country::Canada, "u_series.keytree"));
/// ```
pub fn generic_u_series_spec(country: &Country) -> SeriesItems {
if let Country::UnitedStates = country {
// Need to use a different search technique for US data.

let exclude_phrase = vec!(
"Male",
"Female",
"Men",
"Women",
"Youth",
);

let one_of = vec!(
"Unemployment Rate for United States",
"Unemployment Rate: Aged 15 and Over: All Persons for the United States",
"Unemployment Rate: Aged 15-74: All Persons for the United States",
"Harmonized Unemployment Rate: Total: All Persons for the United States",
"Unemployment Rate - 18 Years and Over",
);

let tag_series = Fred::tags_series("unemployment;rate;usa;nation").unwrap().seriess;
tag_series
.exclude_phrases(exclude_phrase)
.equals_one_of(one_of)
} else {
let (exclude_phrase, include_phrase) = match country {
Country::Australia => {
(
vec!(
"Male",
"Female",
"55-64",
"25-54",
"15-24",
"20 to 24",
"Youth",
"Women",
"Teenagers",
),
vec!(
"Rate"
),
)
}
Country::Austria => {
(
vec!(
"Male",
"Female",
"55-64",
"25-54",
"15-24",
"15-64", // series includes 15-74
"20 to 24",
"Youth",
"Women",
"Teenagers",
),
vec!(
"Rate"
),
)
}
Country::Belgium => {
(
vec!(
"Male",
"Female",
"55-64",
"25-54",
"15-24",
"15-64", // series includes 15-74
"20 to 24",
"Youth",
"Women",
"Teenagers",
),
vec!(
"Rate"
),
)
}
Country::Canada => {
(
vec!(
"Male",
"Female",
"15-64",
"55-64",
"25-54",
"15-24",
"20 to 24",
"Youth",
"Women",
"Teenagers",
),
vec!(
"Rate"
),
)
}
Country::Chile => {
(
vec!(
"Male",
"Female",
"Youth",
"15-24",
"15-64",
"25-54",
"55-64",
),
vec!(
"Rate"
),
)
}
Country::CzechRepublic => {
(
vec!(
"Male",
"Female",
"Youth",
"15-24",
"15-64",
"25-54",
"55-64",
),
vec!(
"Rate"
),
)
}
Country::Denmark => {
(
vec!(
"Male",
"Female",
"Youth",
"15-24",
"15-64",
"25-54",
"55-64",
),
vec!(
"Rate"
),
)
}
Country::Estonia => {
(
vec!(
"Male",
"Female",
"Youth",
"15-24",
"15-64",
"25-54",
"55-64",
),
vec!(
"Rate",
),
)
}
Country::Finland => {
(
vec!(
"Male",
"Female",
"Youth",
"15-24",
"15-64",
"25-54",
"55-64",
),
vec!(
"Rate",
),
)
}
Country::France => {
(
vec!(
"Male",
"Men",
"Female",
"Women",
"Youth",
"Teenagers",
"15-24",
"15-64",
"25-54",
"55-64",
),
vec!(
"Rate",
),
)
}
Country::Germany => {
(
vec!(
"Male",
"Men",
"Female",
"Youth",
"Women",
"Teenagers",
"20 to 24",
"15-24",
"15-64",
"25-54",
"55-64",
),
vec!(
"Rate",
),
)
}
Country::Greece => {
(
vec!(
"Male",
"Female",
"Youth",
"15-24",
"15-64",
"25-54",
"55-64",
),
vec!(
"Rate",
),
)
}
Country::Ireland => {
(
vec!(
"Male",
"Female",
"Youth",
"15-24",
"15-64",
"25-54",
"55-64",
),
vec!(
"Rate",
),
)
}
Country::Israel => {
(
vec!(
"Male",
"Female",
"Youth",
"15-24",
"15-64",
"25-54",
"55-64",
),
vec!(
"Rate",
),
)
}
Country::Italy => {
(
vec!(
"Male",
"Female",
"Youth",
"Men",
"Women",
"Teenagers",
"15-24",
"15-64",
"25-54",
"55-64",
),
vec!(
"Rate",
),
)
}
Country::Japan => {
(
vec!(
"Male",
"Female",
"Youth",
"Men",
"Women",
"Teenagers",
"20 to 24",
"15-24",
"15-64",
"25-54",
"55-64",
),
vec!(
"Rate",
),
)
}
Country::Latvia => {
(
vec!(
"Youth",
"Male",
"Female",
"25 and over",
"15-24",
"15-64",
"25-54",
"55-64",
),
vec!(
"nemployment",
),
)
}
Country::Netherlands => {
(
vec!(
"Male",
"Female",
"Youth",
"Women",
"Teenagers",
"Men",
"20 to 24",
"15-24",
"15-64",
"25-54",
"55-64",
),
vec!(
"Rate",
),
)
}
Country::NewZealand => {
(
vec!(
"Male",
"Female",
"55-64",
"25-54",
"15-24",
"Youth",
),
vec!(
"Rate",
),
)
}
Country::Norway => {
(
vec!(
"Male",
"Female",
"15-24",
"15-64",
"25-54",
"55-64",
),
vec!(
"Rate",
),
)
}
Country::Poland => {
(
vec!(
"Male",
"Female",
"Youth",
"15-24",
"15-64",
"25-54",
"55-64",
),
vec!(
"Rate",
),
)
}
Country::Serbia => {
(
vec!(
),
vec!(
"",
),
)
}
Country::SouthKorea => {
(
vec!(
"Male",
"Female",
"Youth",
"15-24",
"15-64",
"25-54",
"55-64",
),
vec!(
"Rate",
),
)
}
Country::Spain => {
(
vec!(
"Male",
"Female",
"Youth",
"15-24",
"15-64",
"25-54",
"55-64",
),
vec!(
"Rate",
),
)
}
Country::Sweden => {
(
vec!(
"Male",
"Female",
"Youth",
"Men",
"Women",
"Teenagers",
"15-24",
"15-64",
"25-54",
"55-64",
"20 to 24",
),
vec!(
"Rate",
),
)
}
Country::Switzerland => {
(
vec!(
"Male",
"Female",
"Youth",
"15-24",
"15-64",
"25-54",
"55-64",
),
vec!(
"Rate",
),
)
}
Country::UnitedKingdom => {
(
vec!(
"Male",
"Female",
"Youth",
"Men",
"Women",
"Teenagers",
"20 to 24",
"15-24",
"15-64",
"25-54",
"55-64",
),
vec!(
"Rate",
),
)
}
_ => panic!(),

};
match Fred::tags_series(&to_tag("unemployment", country)) {
Ok(tags_series) => {
tags_series.seriess
.exclude_phrases(exclude_phrase)
.only_include(include_phrase)
},
Err(err) => {
println!("{}", err);
panic!();
},
}
}
}

/// ```
/// seriess:
/// series:
/// data_type: u
/// country: Australia
/// id: AUSURAMS
/// ```
pub fn build_data_spec() -> DataSpec {
let mut seriess = DataSpec(Vec::new());

let data_type = DataType::U;
for country in countries_with_data() {
println!("{}", country);
for series_item in generic_u_series_spec(&country).iter() {
seriess.0.push(
SourceSeries {
data_type,
country,
id: series_item.id.clone(),
}
)
}
}
seriess
}

/// ```
/// series:
/// data_type: u
/// country: United States
/// id: LRUNTTTTUSQ156S
/// ```
pub struct DataSpec(pub Vec<SourceSeries>);

impl DataSpec {

/// Read in keytree data from file.
pub fn from_file(path: &str) -> Self {
let source_spec = fs::read_to_string(path).unwrap();
let kt = KeyTree::parse(&source_spec).unwrap();
kt.to_ref().try_into().unwrap()
}

/// Read in source_data.keytree, download series, and return
/// `CheckedSourceSeries`.
pub fn update(&self) {
for series in &self.0 {
let data = match Fred::series(&series.id) {
Ok(data) => println!("{}", data.keytree()),
Err(err) => println!("{:?}", err),
};
}
}
}

impl<'a> TryInto<DataSpec> for KeyTreeRef<'a> {
type Error = Error;

fn try_into(self) -> Result<DataSpec, Error> {
Ok(
DataSpec(self.vec("seriess:series")?)
)
}
}

impl IntoKeyTree for DataSpec {
fn keytree(&self) -> KeyTreeString {
let mut kt = KeyTreeString::new();
kt.push_key(0, "seriess");
for series in &self.0 {
kt.push_keytree(1, series.keytree());
}
kt
}
}

pub struct SourceSeries {
pub data_type: DataType,
pub country: Country,
pub id: String,
}

impl IntoKeyTree for SourceSeries {
fn keytree(&self) -> KeyTreeString {
let mut kt = KeyTreeString::new();
kt.push_key(0, "series");
kt.push_keyvalue(1, "data_type", &self.data_type.to_string());
kt.push_keyvalue(1, "country", &self.country.to_string());
kt.push_keyvalue(1, "id", &self.id);
kt
}
}

impl<'a> TryInto<SourceSeries> for KeyTreeRef<'a> {
type Error = keytree::Error;

fn try_into(self) -> Result<SourceSeries, keytree::Error> {
let data_type_str: String = self.at("series::data_type")?;
let data_type = DataType::from_str(&data_type_str).unwrap();

let country_str: String = self.at("series::country")?;
let country = Country::from_str(&country_str).unwrap();

Ok(
SourceSeries{
data_type,
country,
id: self.at("series::id")?,
}
)
}
}