lib.rs
//! Collect unemployment rate, inflation rate and interest rate data server side, then serve this
//! data as JSON in a format suitable for building UI graphics using D3. This is somewhat annoying
//! complex because the software automation must interact with manually edited specifications.
//!
//! ## Step 1. Build a generic data specification.
//!
//! The data series selector specification that determines what data series to download from Fred
//! looks like
//! ```text
//! selectors:
//! series:
//! country: Australia
//! data_type: u
//! tag: unemployment
//! exclude: Male
//! exclude: Female
//! exclude: 55-64
//! exclude: 25-54
//! exclude: 15-24
//! exclude: 20 to 24
//! exclude: Youth
//! exclude: Women
//! exclude: Teenagers
//! require: Rate
//!
//! series:
//! country: Austria
//! data_type: u
//! tag: unemployment
//! exclude: Male
//! exclude: Female
//! exclude: 55-64
//! exclude: 25-54
//! exclude: 15-24
//! ```
//! For example, the selector specification for unemployment in Australia, looks up the tag
//! `unemployment` in conjuction with `australia` then filters out any title that includes the
//! phrases `Male`, `Female`, `55-64` etc. It also filters out any title that does not include the
//! phrase `Rate`.
//!
//! ```
//! println!("{}", DataSelector::from_file("series_selector.keytree")
//! .into_data_spec()
//! .unwrap()
//! .keytree());
//! ```
//!
//! Copy the output of the above command from the terminal and paste the results in
//! `source_data.keytree`. The reason we don't automatically save these files, is that we have to be
//! careful not to overwrite files that may have been manually edited. So overwriting a
//! specification file must be done manually through copy/paste. The resulting The data series
//! specification `source_data.keytree` will look something like
//!
//! ```
//! seriess:
//! series:
//! data_type: u
//! country: Australia
//! id: AUSURAMS
//! series:
//! data_type: u
//! country: Australia
//! id: AUSURANAA
//! series:
//! data_type: u
//! country: Australia
//! id: AUSURAQS
//! series:
//! data_type: u
//! country: Australia
//! id: AUSURHARMADSMEI
//! series:
//! data_type: u
//! country: Australia
//! id: AUSURHARMMDSMEI
//! ```
//!
//! ## Step 2. Use the data specification to write data to file.
//! ```
//! DataSpec::from_file("source_data.keytree")
//! .unwrap()
//! .write("/full/path/to/data");
//! ```
//!
//! If there is a break in the connection we can use
//! ```
//! DataSpec::from_file("source_data.keytree")
//! .unwrap()
//! .resume_write("NORURTOTADSMEI", "/full/path/to/data");
//! ```
//! to resume. To update the data files due to a change of the DataSpec,
//! ```
//! data_spec
//! .update_write("/full/path/to/data")
//! .unwrap();
//! ```
//! `update_write()` will also remove any series that have been removed from
//! the data specification. Deleting the data files directly is useful to re-download data files.
//!
//! ## Step 3. Generate a generic time-series graphics specification.
//!
//! The generated specification specifies the time-series graphics.
//!
//! ```
//! let data_spec = DataSpec::from_file("source_data.keytree")
//! .unwrap();
//! println!("{}", data_spec.generic_ts_spec().keytree());
//! ```
//! Again, to overwrite an existing specification file, the terminal output needs to be copied and
//! pasted over an existing specification file.
//!
//! ## Step 4. Serve Time-series
//!
//! A time-series specification looks like
//! ```
//! ts_spec:
//! page:
//! country: Australia
//! data_type: u
//! index: 0
//! graphic:
//! series:
//! data_type: u
//! series_id: AUSURAMS
//! series:
//! data_type: u
//! series_id: AUSURANAA
//! series:
//! data_type: u
//! series_id: AUSURAQS
//! series:
//! data_type: u
//! series_id: AUSURHARMADSMEI
//! series:
//! data_type: u
//! series_id: AUSURHARMMDSMEI
//! series:
//! data_type: u
//! series_id: AUSURHARMQDSMEI
//! ```
//! It specifies the way time-series graphics are presented in the client browser. A generic
//! time-series specification can be built using
//! ```
//! // include command here
//! ```
//!
//! This step converts the specification in JSON data and meta-data that is served to the client.
//! The client Javascript then builds the graphics dynamically. Take both specifications and load
//! data from "/full/path/to/data".
//!
//! ```
//! let data_spec = DataSpec::from_file("source_data.keytree").unwrap();
//! let ts_spec = TSSpec::from_file("ts_spec.keytree");
//! ```
//!
//! Then `TSSpec` uses `IndexedDataSpec` as an argument to be converted into `TSJson`.
//!
//! ```
//! let ts_json = ts_spec.into_json(&data_spec, "/full/path/to/data").unwrap();
//! ```
//! All JSON is generated at server startup into an immutable `HashMap` in memory.
//!
//! ## Step 5. Serve UI Scatterplots
//!
pub mod error;
pub mod fred;
pub mod parser;
pub mod ts;
pub mod ui;
use std::collections::BTreeMap;
use std::convert::TryInto;
use std::{ file, fmt, fs, line };
use std::path::Path;
use std::str::FromStr;
use serde::Serialize;
use walkdir::WalkDir;
use countries::Country;
use fred_api::{ Fred };
use keytree::{ KeyTree, KeyTreeRef };
use keytree::serialize::{ IntoKeyTree, KeyTreeString };
use time_series::{ RegularTimeSeries, TimeSeries };
use crate::error::*;
use crate::parser::*;
use crate::ts::{ GraphicCategory, Transform };
/// A date that can be parsed from a string like "07-08-2010".
#[derive(Clone, Debug, Serialize)]
pub struct MonthlyDate(time_series::MonthlyDate);
impl FromStr for MonthlyDate {
type Err = Error;
fn from_str(s: &str) -> Result<Self, Self::Err> {
let year = s[..4].parse().map_err(|_| {
err(file!(), line!(), &format!("Parse datapoint {} failed", s))
})?;
let month = s[5..7].parse().map_err(|_| {
err(file!(), line!(), &format!("Parse datapoint {} failed", s))
})?;
Ok(MonthlyDate(time_series::MonthlyDate::ym(year, month)))
}
}
impl fmt::Display for MonthlyDate {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{}-{}-01", self.0.year(), self.0.month())
}
}
/// `DateRange` newtype.
#[derive(Clone, Copy, Debug, Serialize)]
pub struct DateRange(time_series::DateRange);
impl DateRange {
/// Create a time_series::DateRange.
pub fn new(
first_date: &Option<MonthlyDate>,
last_date: &Option<MonthlyDate>) -> DateRange
{
DateRange(
time_series::DateRange::new(
&first_date.clone().map(|d| d.0),
&last_date.clone().map(|d| d.0),
)
)
}
pub fn first_date(&self) -> Option<MonthlyDate> {
self.0.first_date().map(|date| MonthlyDate(date))
}
pub fn last_date(&self) -> Option<MonthlyDate> {
self.0.last_date().map(|date| MonthlyDate(date))
}
}
impl fmt::Display for DateRange {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let s = match (self.0.first_date(), self.0.last_date()) {
(None, None) => {
format!(
"open"
)
},
(Some(d1), None) => {
format!(
"from {}-{}-01",
d1.year(),
d1.month(),
)
},
(None, Some(d2)) => {
format!(
"to {}-{}-01",
d2.year(),
d2.month()
)
},
(Some(d1), Some(d2)) => {
format!(
"between {}-{}-01 and {}-{}-01",
d1.year(),
d1.month(),
d2.year(),
d2.month(),
)
},
};
write!(f, "{}", s)
}
}
/// Represents a FRED series id like `LRHUTTTTAUA156N` or a transformation on a FRED series_id
/// like `LRHUTTTTAUA156N_a`.
#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)]
pub struct SeriesId(String);
impl SeriesId {
/// Create a new SeriesId from a string.
pub fn new(s: &str) -> Self {
SeriesId(s.to_string())
}
/// Return the component without transformation modifications.
pub fn stem(&self) -> Self {
let inner = self.0.split('_').next().unwrap().clone();
SeriesId(String::from(inner))
}
}
impl FromStr for SeriesId {
type Err = ();
fn from_str(s: &str) -> Result<Self, ()> {
Ok(SeriesId(String::from(s)))
}
}
impl fmt::Display for SeriesId {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{}", self.0)
}
}
/// Unemployment rate, interest rate, inflation rate etc.
#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)]
pub enum DataType {
/// Unemployment rate
U,
/// CPI
Cpi,
/// Inflation rate
Inf,
/// Interest rate
Int,
}
impl FromStr for DataType {
type Err = Error;
/// Parse a string into a `DataType` or return `None` on failure.
fn from_str(s: &str) -> Result<Self, Self::Err> {
match s {
"u" => Ok(DataType::U),
"cpi" => Ok(DataType::Cpi),
"inf" => Ok(DataType::Inf),
"int" => Ok(DataType::Int),
_ => Err(err(file!(), line!(), &format!("Failed to parse datatype {}.", s))),
}
}
}
impl fmt::Display for DataType {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let s = match self {
DataType::U => "u",
DataType::Cpi => "cpi",
DataType::Inf => "inf",
DataType::Int => "int",
};
write!(f, "{}", s)
}
}
/// ```
/// println!("{}", title("LFACTTTTKRA657N"));
/// ```
pub fn title(series_id: &str) -> Result<String, Error> {
let sid = SeriesId::from_str(series_id).unwrap();
let mut s = String::new();
let seriess = match Fred::series(&sid.to_string()) {
Ok(series) => series.seriess,
Err(e) => { return Err(err(file!(), line!(), &e.to_string())) },
};
for series in seriess.iter() {
s.push_str(&series.title)
}
Ok(s)
}
/// Return series from tags. Tags look like "loans;australia".
pub fn interest_rate_series(tags: &str) {
let tags_series = Fred::tags_series(tags).unwrap();
let series_items = tags_series.seriess;
let iter = series_items.iter();
for item in iter {
println!();
println!("{}", item.title);
println!("{}", item.id);
let tags = Fred::series_tags(&item.id).unwrap();
println!("{}", tags.one_line());
}
}
// `DataSpec` is set up in what seems like an overly complex way in order to maintain
// the order is which data is `DataSelector` orders the series. We want to maintain an
// ordering by (DataType, Country). But sometimes we also need to search by SeriesId so also need a
// reverse lookup.
/// A generic specification of all data series. Output looks like
/// ```
/// series:
/// data_type: u
/// country: United States
/// id: LRUNTTTTUSQ156S
/// ```
#[derive(Debug)]
pub struct DataSpec {
map: BTreeMap<(DataType, Country), Vec<SeriesSpec>>,
reverse: BTreeMap<SeriesId, (DataType, Country)>,
}
impl DataSpec {
/// Used to build a generic specification.
pub fn empty() -> Self {
DataSpec {
map: BTreeMap::new(),
reverse: BTreeMap::new(),
}
}
/// Get a `SeriesSpec` from a `SeriesId`.
pub fn get_series_spec(&self, series_id: &SeriesId) -> Option<SeriesSpec> {
let key = match self.reverse.get(&series_id) {
Some(key) => key,
None => { return None },
};
let seriess = self.map.get(&key).unwrap();
match seriess.iter().find(|&series| {
&series.series_id == series_id
}) {
Some(series_spec) => Some(series_spec.clone()),
None => None,
}
}
pub (crate) fn new() -> Self {
DataSpec {
map: BTreeMap::new(),
reverse: BTreeMap::new(),
}
}
/// Return a `DataSpec` built from a collection of `SeriesSpec`.
pub (crate) fn from_vec(v: Vec<SeriesSpec>) -> Self {
let mut data_spec = DataSpec::new();
for series_spec in v {
data_spec.insert(&series_spec);
}
data_spec
}
/// Insert a `SeriesSpec`.
pub fn insert(&mut self, series_spec: &SeriesSpec) {
// Expect country to be Some while building source.
match self.map.get_mut(&(series_spec.data_type, series_spec.country.unwrap())) {
None => {
self.map.insert(
// Expect country to be Some while building source.
(series_spec.data_type, series_spec.country.unwrap()),
vec!(series_spec.clone())
);
self.reverse.insert(
series_spec.series_id.clone(),
// Expect country to be Some while building source.
(series_spec.data_type, series_spec.country.unwrap()),
);
},
Some(value) => {
value.push(series_spec.clone());
self.reverse.insert(
series_spec.series_id.clone(),
// Expect country to be Some while building source.
(series_spec.data_type, series_spec.country.unwrap()),
);
},
}
}
/// Build a generic `Spec` from `Self`.
pub fn generic_ts_spec(&self) -> ts::Spec {
let mut spec = ts::Spec::empty();
for ((data_type, country), series_specs) in self.map.iter() {
let mut page_spec = ts::PageSpec::new(
*country, // country
*data_type, // data_type
0, // index
None, // height
);
let mut collated_graphic = ts::GraphicSpec {
category_opt: Some(GraphicCategory::Collation),
series_ids: Vec::new(),
graphic_range: None,
note: None,
};
let mut single_graphics = Vec::new();
for series_spec in series_specs {
let ts_series_spec = SeriesSpec {
country: None,
data_type: series_spec.data_type,
series_id: series_spec.series_id.clone(),
transforms: Vec::new(),
parser: ParserSpec::FredSpec(FredSpec::empty()),
};
// Series
page_spec.seriess.insert(series_spec.series_id.clone(), ts_series_spec);
// Single-series graphics
let single_series_graphic = ts::GraphicSpec {
category_opt: Some(GraphicCategory::Source),
series_ids: vec!(series_spec.series_id.clone()),
graphic_range: None,
note: None,
};
single_graphics.push(single_series_graphic);
// Collated graphic
collated_graphic.series_ids.push(series_spec.series_id.clone());
};
let mut graphics = vec!(collated_graphic);
graphics.append(&mut single_graphics);
page_spec.graphics = graphics;
spec.0.push(page_spec);
}
spec
}
// /// Append `other` to `Self`.
// pub fn append(&mut self, other: &mut DataSpec) {
// self.0.append(&mut other.0)
// }
// fn series_from_index(&self, i: usize) -> &Series {
// &self.0[i]
// }
/// Read in keytree data from file.
pub fn from_file(path: &str) -> Result<Self, Error> {
dbg!(path);
let source_spec = match fs::read_to_string(path) {
Ok(ss) => ss,
Err(e) => { return Err(
err(file!(), line!(), &format!("Failed to read file {}.", e))
)},
};
let kt = KeyTree::parse(&source_spec).unwrap();
kt.to_ref().try_into().map_err(|e: keytree::error::Error| {
err(file!(), line!(), &e.to_string())
})
}
/// Save FRED data to disk as csv, using full path. Will fail if an existing filepath is
/// encountered.
/// ```
/// let mut source = DataSpec::from_file("checked_data.keytree");
/// source.write(&root_dir);
/// ```
/// To path from root is "/{data_type}/{country}/LRUNTTTTSIQ156S.csv"
///
pub fn write(&self, root_path: &str) -> Result<(), Error> {
for (_, series_specs) in &self.map {
for series_spec in series_specs {
series_spec.write_data_to_file(root_path)?;
series_spec.write_meta_to_file(root_path)?;
}
}
Ok(())
}
// Need to keep a record of all files and remove files that shouldn't exist.
//
// i.e. its not sufficient to check if a file exists. We need to check if a file remains. So we
// need to create a file list.
/// Only make requests and updata data to Fred for files that are in `DataSpec` but do not exist
/// as data files.
pub fn update_write(&self, root_path: &str) -> Result<(), Error> {
for (_, series_specs) in &self.map {
for series_spec in series_specs {
if !series_spec.exists(root_path)? {
dbg!(&series_spec);
series_spec.write_data_to_file(root_path)?;
series_spec.write_meta_to_file(root_path)?;
}
}
}
self.remove_old(root_path)?;
Ok(())
}
/// Run through data files, query and remove any files that are not in directory.
pub fn remove_old(&self, root_path: &str) -> Result<(), Error> {
for entry in WalkDir::new(root_path) {
let entry = entry.unwrap();
if !entry.file_type().is_dir() {
let pathbuf = entry.path();
let mut path_iter = pathbuf
.iter()
.rev()
.map(|os_str| os_str.to_str()
.unwrap());
let mut file_parts = path_iter
.next()
.unwrap()
.split('.');
let file_stem = file_parts.next().unwrap();
let file_ext = file_parts.next().unwrap();
if file_ext == "csv" || file_ext == "meta" {
let series_id = SeriesId::new(file_stem);
match self.get_series_spec(&series_id) {
Some(_) => {},
None => {
println!("remove file: {}", entry.path().display());
// match fs::remove_file(entry.path()) {
// Ok(_) => {},
// Err(_) => {
// return Err(file_error(file!(), line!()))
// },
// }
},
}
}
};
}
Ok(())
}
/// Same as `write()` except that it starts in the specification at `series_id`. Useful if there
/// is a break in the connection when writing.
pub fn resume_write(&self, series_id: &str, root_path: &str) -> Result<(), Error> {
let sid = SeriesId::new(series_id);
for (_, series_specs) in self
.map
.iter()
.skip_while(|_| {
match self.get_series_spec(&sid) {
Some(series_spec) => sid != series_spec.series_id,
None => true,
}
})
{
for series_spec in series_specs {
series_spec.write_data_to_file(root_path)?;
series_spec.write_meta_to_file(root_path)?;
}
}
Ok(())
}
}
impl<'a> TryInto<DataSpec> for KeyTreeRef<'a> {
type Error = keytree::Error;
fn try_into(self) -> Result<DataSpec, Self::Error> {
let v: Vec<SeriesSpec> = self.vec_at("seriess::series")?;
Ok(DataSpec::from_vec(v))
}
}
impl IntoKeyTree for DataSpec {
fn keytree(&self) -> KeyTreeString {
let mut kt = KeyTreeString::new();
kt.push_key(0, "seriess");
for (_, series_specs) in &self.map {
for series_spec in series_specs {
kt.push_keytree(1, series_spec.keytree());
}
}
kt
}
}
// The country field is always `Some`. The `Option` is used in initialization from a keytree file.
// The transforms field is set is the time-series and ui specifications, and so is empty when a
// SeriesSpec is initialized from a source specification.
#[derive(Clone, Debug)]
pub struct SeriesSpec {
country: Option<Country>,
pub data_type: DataType,
pub series_id: SeriesId,
pub transforms: Vec<Transform>,
pub parser: ParserSpec,
}
impl SeriesSpec {
pub fn country(&self) -> Country {
self.country.unwrap()
}
pub fn data_path(&self, root_path: &str, extension: &str) -> String {
format!(
"{}/{}/{}/{}.{}",
root_path,
self.data_type,
self.country().as_path(),
self.series_id.stem(),
extension,
)
}
pub fn dir_path(&self, root_path: &str) -> String {
format!(
"{}/{}/{}",
root_path,
self.data_type,
self.country().as_path(),
)
}
/// Read metadata from file.
pub fn read_meta_from_file(&self, root_path: &str) -> SeriesMetaData
{
let meta_str = fs::read_to_string(
&self.data_path(root_path, "meta")
).unwrap();
let kt = KeyTree::parse(&meta_str).unwrap();
kt.to_ref().try_into().unwrap()
}
/// Check if data and meta-data exist in a file.
pub fn exists(&self, root_path: &str) -> Result<bool, Error> {
Ok(
Path::new(&self.data_path(root_path, "csv")).exists() &&
Path::new(&self.data_path(root_path, "meta")).exists()
)
}
pub fn read_data_with_transforms(
&self,
root_path: &str) -> Result<RegularTimeSeries<1>, Error>
{
let mut rts = self.read_data_without_transform(root_path)?;
// Do transforms before constraining dates.
for transform in &self.transforms {
rts = match transform {
Transform::ToMonthly => rts.to_monthly(0),
Transform::ToQuarterly => rts.to_quarterly(0),
Transform::YearOnYear => {
match rts.to_year_on_year(0) {
Ok(rts) => rts,
Err(e) => { return Err(err(file!(), line!(), &e.to_string())) },
}
},
Transform::DateRange(range) => rts.range(&range.0),
};
}
Ok(rts)
}
pub fn read_data_without_transform(
&self,
root_path: &str) -> Result<RegularTimeSeries<1>, Error>
{
let ts = TimeSeries::<1>::from_csv(&self.data_path(root_path, "csv"))
.map_err(|e| {
err(file!(), line!(), &e.to_string())
})?;
let rts = ts.try_into()
.map_err(|e: time_series::error::Error| {
err(file!(), line!(), &e.to_string())
})?;
Ok(rts)
}
pub fn to_data(
&self,
parser: ParserSpec) -> Result<RegularTimeSeries<1>, Error>
{
// We want to handle data fetching and parsing in the Parser implementation, so make the
// Parser the receiver.
match self.parser {
ParserSpec::FredSpec(fred_spec) => FredParser(fred_spec).to_data(self),
ParserSpec::FredDailySpec(fred_daily_spec) => FredDailyParser(fred_daily_spec).to_data(self),
}
}
/// Fetches data as specified in `source_data.keytree` and saves to disk.
pub fn write_data_to_file(&self, root_path: &str) -> Result<(), Error> {
let rts = self.to_data(self.parser)?;
// Build csv manually
let mut s = String::new();
for dp in rts.iter(time_series::DateRange::new(&None, &None)) {
s.push_str(&format!(
"{}-{}-01, {}",
dp.date().year(),
dp.date().month(),
dp.value(0).to_string()
))
}
fs::create_dir_all(&self.dir_path(root_path))
.map_err(|e| err(file!(), line!(), &format!("Failed to create dir {}.", e)))?;
fs::write(self.data_path(root_path, "csv"), s)
.map_err(|e| err(file!(), line!(), &e.to_string()))?;
Ok(())
}
/// Fetches data as specified in `source_data.keytree` and saves meta_data to disk.
pub fn write_meta_to_file(&self, root_path: &str) -> Result<(), Error> {
let series = Fred::series(&self.series_id.to_string())
.map_err(|e| err(file!(), line!(), &e.to_string()))?;
let series_item = series.seriess.iter().next().ok_or_else(|| {
err(file!(), line!(), &format!("Expected series data for {}", self.series_id))
})?;
let meta = SeriesMetaData {
realtime: series.realtime_start.clone(),
series_id: self.series_id.clone(),
title: series_item.title.clone(),
observation_start: series_item.observation_start.clone(),
observation_end: series_item.observation_end.clone(),
frequency: series_item.frequency.clone(),
seasonal_adjustment: series_item.seasonal_adjustment.clone(),
};
let json = serde_json::to_string(&meta)
.map_err(|e| err(file!(), line!(), &e.to_string()))?;
fs::create_dir_all(self.dir_path(root_path))
.map_err(|e| err(file!(), line!(), &format!("Failed to create dir {}", e)))?;
let path = self.data_path(root_path, "meta");
println!("Writing {}", path);
fs::write(path, json)
.map_err(|e| err(file!(), line!(), &e.to_string()))?;
Ok(())
}
}
impl IntoKeyTree for SeriesSpec {
fn keytree(&self) -> KeyTreeString {
let mut kt = KeyTreeString::new();
kt.push_key(0, "series");
if let Some(country) = self.country {
kt.push_value(1, "country", country);
};
kt.push_value(1, "data_type", &self.data_type);
kt.push_value(1, "series_id", &self.series_id);
for f in &self.transforms {
kt.push_value(1, "f", f);
}
if let ParserSpec::FredDailySpec(_) = self.parser {
kt.push_value(1, "parse_cat", "fred_daily")
};
if let ParserSpec::FredSpec(parser_spec) = self.parser {
if parser_spec.drop_first != 0 {
kt.push_key(1, "parser");
kt.push_value(2, "drop_first", parser_spec.drop_first.to_string());
}
};
kt
}
}
// series:
// data_type: int
// country: United States
// series_id: DPRIME
// parse_cat: fred_daily
//
// seriess:
// series:
// country: United States
// data_type: int
// series_id: DPRIME
impl<'a> TryInto<SeriesSpec> for KeyTreeRef<'a> {
type Error = keytree::Error;
fn try_into(self) -> Result<SeriesSpec, keytree::Error> {
let first_date = self.opt_value("series::first_date")?;
let last_date = self.opt_value("series::last_date")?;
let date_range = DateRange::new(&first_date, &last_date);
let mut transforms: Vec<Transform> = self.opt_vec_value("series::transform")?;
if first_date.is_some() || last_date.is_some() {
transforms.push(Transform::DateRange(date_range));
}
// let parser_str_opt: Option<String> = self.opt_value("series::parser")?;
// let parser = match parser_str_opt {
// None => Parser::Fred,
// Some(parser_str) => {
// Parser::from_str(&parser_str)
// .map_err(|err| keytree::error::external(file!(), line!(), &err.to_string()))?
// },
// };
let a_parser: Option<String> = self.opt_value("series::parse_cat::fred_daily")?;
let b_parser: Option<String> = self.opt_value("series::parse_cat::fred")?;
let parser = match (a_parser.is_some(), b_parser.is_some()) {
(false, false) => ParserSpec::FredDailySpec(FredDailySpec::empty()),
(true, false) => ParserSpec::FredDailySpec(FredDailySpec::empty()),
(false, true) => ParserSpec::FredSpec(self.at("series::parser")?),
(true, true) => Err(keytree::error::err(file!(), line!(), "Only one parser allowed."))?,
};
Ok(
SeriesSpec{
country: self.opt_value("series::country")?,
data_type: self.value("series::data_type")?,
series_id: self.value("series::series_id")?,
transforms,
parser,
}
)
}
}
/// Source meta-data is stored in a file as a String:
///
/// ```text
/// series:
/// realtime_start: 2021-06-03
/// realtime_end: 2021-06-03
/// series_items:
/// series_item:
/// realtime: 2021-06-03
/// series_id: AUSCPALTT01IXNBQ
/// title: Consumer Price Index: All items: Total: Total for Australia
/// observation_start: 1960-01-01
/// observation_end: 2021-01-01
/// frequency: Quarterly
/// seasonal_adjustment: Not Seasonally Adjusted
/// notes: (see JSON data for notes)
/// ```
#[derive(Debug, Serialize)]
pub struct SeriesMetaData {
realtime: String,
series_id: SeriesId,
title: String,
observation_start: String,
observation_end: String,
frequency: String,
seasonal_adjustment: String,
}
impl SeriesMetaData {
///
pub fn from_file(
data_type: DataType,
country: Country,
series_id: SeriesId,
root_path: &str) -> Result<Self, Error>
{
let path = &format!(
"{}/{}/{}/{}.meta",
root_path,
data_type,
country,
series_id,
);
let meta_str = fs::read_to_string(&path).unwrap();
let kt = KeyTree::parse(&meta_str).unwrap();
Ok(kt.to_ref().try_into().unwrap())
}
}
impl<'a> TryInto<SeriesMetaData> for KeyTreeRef<'a> {
type Error = keytree::Error;
fn try_into(self) -> Result<SeriesMetaData, Self::Error> {
Ok(
SeriesMetaData {
realtime: self.value("series_meta::realtime")?,
series_id: self.value("series_meta::series_id")?,
title: self.value("series_meta::title")?,
observation_start: self.value("series_meta::observation_start")?,
observation_end: self.value("series_meta::observation_end")?,
frequency: self.value("series_meta::frequency")?,
seasonal_adjustment: self.value("series_meta::seasonal_adjustment")?,
}
)
}
}
impl IntoKeyTree for SeriesMetaData {
fn keytree(&self) -> KeyTreeString {
let mut kt = KeyTreeString::new();
kt.push_key(0, "series_meta");
kt.push_value(1, "realtime", &self.realtime);
kt.push_value(1, "series_id", &self.series_id.to_string());
kt.push_value(1, "title", &self.title);
kt.push_value(1, "observation_start", &self.observation_start);
kt.push_value(1, "observation_end", &self.observation_end);
kt.push_value(1, "frequency", &self.frequency);
kt.push_value(1, "seasonal_adjustment", &self.seasonal_adjustment);
kt
}
}
/// A component of `Json` that is a single data series. Series are referenced by graphics through an
/// index, so that each series can be used multiple times, without being downloaded multiple times.
#[derive(Debug, Serialize)]
pub struct SeriesJson {
series_id: SeriesId,
rts: RegularTimeSeries<1>,
meta: Option<SeriesMetaData>,
transforms: Vec<Transform>,
}
#[derive(Clone, Copy, Debug, Serialize)]
/// Specifies the range of a graphic
pub struct GraphicRange {
min: f32,
max: f32,
}
impl FromStr for GraphicRange {
type Err = Error;
fn from_str(s: &str) -> Result<Self, Self::Err> {
let segment: Vec<&str> = s.split(" to ").collect();
if segment[0].is_empty() || segment[1].is_empty() {
return Err(
err(file!(), line!(), &format!("Parse graphic range {} failed", s)))
};
let min = segment[0].parse()
.map_err(|_| err(file!(), line!(), &format!("Parse graphic range {} failed", s)))?;
let max = segment[1].parse()
.map_err(|_| err(file!(), line!(), &format!("Parse graphic range {} failed", s)))?;
Ok(GraphicRange { min, max })
}
}
impl fmt::Display for GraphicRange {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{} to {}", self.min, self.max)
}
}
// /// The United States prime interest rate data is daily. To buid a monthly time-series, we read
// /// through raw csv data, calculate a monthly value and add to to the time-series. The data include
// /// missing days, so we need the mechanism to ignore datepoints with value ".".
//
// ts.try_into().map_err(|err: time_series::error::Error| {
// external(
// file!(),
// line!(),
// &err.to_string(),
// )
// })
// }