Completed first try at ts Json data-structure.
[?]
CrEcTsRjb1hHQjHuumqRfqdbVV4X58iLEubi4noaDPFa
Jul 21, 2021, 7:43 AM
5B2HBV3JTNBNJYEJ4BXFQDEP2D552SBJFPY6STKSK7SPFZYSTCFACDependencies
- [2]
TSY4YBBZChanged ts Spec datastructures to new format. - [3]
2CCG6KUPRedo time-series spec. - [4]
XI5ALEH6Take advantage of keytree FromStr functionality. - [5]
UKQAGL5FChecked ts_json toolchain. - [6]
GUXZCEWWAdded Country enum. - [7]
TTR5IFSGWorking on building generic TSSpec. - [8]
K4CH53V4Added GPL2 license, included missing source files. - [9]
4MG5JFXTFirst record. - [10]
77SIQZ3ESeparating out spec generation. - [11]
GQVS55HIFinished generate_ts_spec() function. - [12]
YJXKWWM6Added max() min() n GraphicJson and cleaned up serialization - [13]
SAHJYVNBRemoved checking functionality. - [14]
SPSFTMLRCompleted loading ts_data from specification. - [15]
UUD3CJZLMaking error handling more comprehensive. - [16]
2SABVMY3Finished into_json() functionality.
Change contents
- edit in src/ts.rs at line 3
use std::collections::BTreeMap; - replacement in src/ts.rs at line 11
// use serde::{// Serialize,// Serializer,// };// use serde::ser::SerializeStruct;use serde::{Serialize,Serializer,};use serde::ser::SerializeStruct; - replacement in src/ts.rs at line 26
// use time_series::{// RegularTimeSeries,// };use time_series::{RegularTimeSeries,}; - edit in src/ts.rs at line 32
data_from_file, - replacement in src/ts.rs at line 36
meta_from_file,SeriesSpec, - edit in src/ts.rs at line 42
#[derive(Debug)]pub struct Json(BTreeMap<(Country, DataType, usize), Vec<GraphicJson>>);#[derive(Debug, Serialize)]pub struct GraphicJson {title_opt: Option<String>,graphics: Vec<SeriesJson>,text_spec: TextSpec,}#[derive(Debug, Serialize)]pub struct SeriesJson {rts: RegularTimeSeries<1>,meta: Option<SeriesMetaData>,transforms: Vec<Transform>,} - replacement in src/ts.rs at line 61
/// Time-series specification./// ```/// ts_spec:/// page:/// country: Australia/// data_type: u/// index: 0/// graphic:/// series:/// data_type: u/// series_id: AUSURAMS/// series:/// data_type: u/// series_id: AUSURANAA/// ```#[derive(Debug)] - edit in src/ts.rs at line 85
}pub (crate) fn into_json(&self, root_path: &str) -> Result<Json, Error> {let mut json = BTreeMap::new();for page_spec in &self.0 {let PageSpec {country,data_type,index,seriess,graphics,} = page_spec;// Iterate over graphics.let mut v: Vec<GraphicJson> = Vec::new();for graphic_spec in graphics {let GraphicSpec {title_opt,height_opt,series_ids,text_spec,..} = graphic_spec;// Construct GraphicJsonlet t_opt = match title_opt {Some(title_opt) => Some(title_opt.clone()),None => None,};let mut graphic_json = GraphicJson {title_opt: t_opt,graphics: Vec::new(),text_spec: graphic_spec.text_spec,};for series_id in series_ids {graphic_json.graphics.push(page_spec.into_series_json(series_id, *country, root_path)?);}// Push GraphicJson to temporary Vec.v.push(graphic_json);}// Push key and value to jsonjson.insert((country, data_type, index), v);}Ok(Json(BTreeMap::new())) - edit in src/ts.rs at line 183
#[derive(Debug)] - edit in src/ts.rs at line 185
/// - edit in src/ts.rs at line 186
/// - edit in src/ts.rs at line 187
/// - replacement in src/ts.rs at line 188
///pub series: Vec<SeriesSpec>,///pub seriess: BTreeMap<SeriesId, SeriesSpec>, - edit in src/ts.rs at line 192
impl PageSpec {pub (crate) fn into_series_json(&self,series_id: &SeriesId,country: Country,root_path: &str) -> Result<SeriesJson, Error>{let series_spec = match self.seriess.get(&series_id) {Some(series) => series,None => {return Err(failed_to_reference_series(file!(),line!(),&series_id.to_string() ,))}};let rts = series_spec.data_with_transforms()?;let meta = series_spec.meta(country, root_path);Ok(SeriesJson {rts: rts,meta: Some(meta),transforms: series_spec.transforms.clone(),})}} - edit in src/ts.rs at line 228
let seriess_vec: Vec<SeriesSpec> = self.vec_at("page::series")?;let mut map = BTreeMap::new();for series_spec in seriess_vec {map.insert(series_spec.series_id.clone(), series_spec);} - replacement in src/ts.rs at line 240
series: self.vec_at("page::series")?,seriess: map, - replacement in src/ts.rs at line 254
for series in &self.series {for (_, series) in self.seriess.iter() { - edit in src/ts.rs at line 258
- edit in src/ts.rs at line 267
#[derive(Copy, Clone, Debug, Serialize)] - edit in src/ts.rs at line 269
/// Specifies that graphic displays Links to series. - edit in src/ts.rs at line 270
/// Specifies that graphics displays metadata. - edit in src/ts.rs at line 271
/// Specifies that graphics do not display text. - edit in src/ts.rs at line 318
#[derive(Debug)] - replacement in src/ts.rs at line 320
///pub title: Option<String>,///pub height: Option<f32>,///pub series_id: Vec<SeriesId>,///pub fid: Vec<String>,///pub title_opt: Option<String>,pub height_opt: Option<f32>,pub series_ids: Vec<SeriesId>, - replacement in src/ts.rs at line 332
title: self.opt_value("graphic::title")?,height: self.opt_value("graphic::height")?,series_id: self.vec_value("graphic::series_id")?,fid: self.vec_value("graphic::fid")?,title_opt: self.opt_value("graphic::title")?,height_opt: self.opt_value("graphic::height")?,series_ids: self.vec_value("graphic::series_id")?, - replacement in src/ts.rs at line 346
if let Some(title) = &self.title {if let Some(title) = &self.title_opt { - replacement in src/ts.rs at line 350
if let Some(h) = &self.height {if let Some(h) = &self.height_opt { - replacement in src/ts.rs at line 354
for series_id in &self.series_id {for series_id in &self.series_ids { - edit in src/ts.rs at line 356
}for fid in &self.fid {kt.push_value(1, "fid", fid) - edit in src/ts.rs at line 363[2.2745]→[3.10062:10206](∅→∅),[3.7063]→[3.10062:10206](∅→∅),[3.10206]→[3.7109:7121](∅→∅),[3.7109]→[3.7109:7121](∅→∅),[3.7121]→[3.10207:10258](∅→∅),[3.10258]→[3.7177:7185](∅→∅),[3.7177]→[3.7177:7185](∅→∅),[3.7185]→[3.2179:2196](∅→∅),[3.2196]→[3.10259:10283](∅→∅),[3.7185]→[3.10259:10283](∅→∅),[3.10283]→[2.2746:2946](∅→∅),[2.2946]→[3.7479:7482](∅→∅),[3.7479]→[3.7479:7482](∅→∅),[3.7482]→[3.10334:10384](∅→∅),[3.10384]→[3.7534:7568](∅→∅),[3.7534]→[3.7534:7568](∅→∅),[3.7568]→[3.10385:10447](∅→∅),[3.10447]→[3.2548:2744](∅→∅),[3.2744]→[3.7629:7641](∅→∅),[3.10447]→[3.7629:7641](∅→∅),[3.7629]→[3.7629:7641](∅→∅),[3.7641]→[3.2745:3048](∅→∅),[3.2243]→[3.11011:11035](∅→∅),[3.3048]→[3.11011:11035](∅→∅),[3.7790]→[3.11011:11035](∅→∅),[3.11011]→[3.11011:11035](∅→∅),[3.8099]→[3.8099:8105](∅→∅),[3.8105]→[3.3049:3051](∅→∅),[3.3051]→[2.2947:3213](∅→∅),[2.3213]→[3.3051:3052](∅→∅),[3.3051]→[3.3051:3052](∅→∅),[3.3052]→[2.3214:3670](∅→∅)
// This is differentiated from lib::SeriesSpec in that is is just for interacting with///// Component of time-series specification./// ```/// series:/// data_type: u/// series_id: AUSURAMS/// ```#[derive(Debug)]pub struct SeriesSpec {///pub data_type: DataType,///pub series_id: SeriesId,///pub date_range: DateRange,///pub transforms: Vec<Transform>,///pub fid: Option<FID>,}impl<'a> TryInto<SeriesSpec> for KeyTreeRef<'a> {type Error = keytree::Error;fn try_into(self) -> Result<SeriesSpec, keytree::Error> {let first_date = self.opt_value("series::first_date")?;let last_date = self.opt_value("series::last_date")?;let date_range = DateRange::new(&first_date, &last_date);Ok(SeriesSpec {data_type: self.value("series::data_type")?,series_id: self.value("series::series_id")?,date_range,transforms: self.vec_value("series::transform")?,fid: self.opt_value("series::fid")?,})}}impl IntoKeyTree for SeriesSpec {fn keytree(&self) -> KeyTreeString {let mut kt = KeyTreeString::new();kt.push_key(0, "series");kt.push_value(1, "data_type", &self.data_type);kt.push_value(1, "series_id", &self.series_id);if let Some(first_date) = self.date_range.first_date() {kt.push_value(1, "first_date", first_date);}if let Some(last_date) = self.date_range.last_date() {kt.push_value(1, "last_date", last_date);};for f in &self.transforms {kt.push_value(1, "f", f);}if let Some(fid) = &self.fid {kt.push_value(1, "fid", fid);}kt}} - replacement in src/ts.rs at line 366
#[derive(Debug)]#[derive(Copy, Clone, Debug, Serialize)] - replacement in src/ts.rs at line 392[2.3790]→[2.3790:3989](∅→∅),[2.3989]→[3.3750:3831](∅→∅),[3.3750]→[3.3750:3831](∅→∅),[3.3831]→[3.8109:8110](∅→∅),[3.8109]→[3.8109:8110](∅→∅),[3.8110]→[3.3832:4252](∅→∅)
Transform::ToMonthly => write!(f, "to_monthly"),Transform::ToQuarterly => write!(f, "to_quarterly"),Transform::YearOnYear => write!(f, "yoy"),}}}/// SeriesIdentifierAfterTransform#[derive(Debug)]pub struct FID(String);impl FID {pub (crate) fn new(series_id: &SeriesId, f: Vec<Transform>) -> Self {let mut s = format!("{}_",series_id,);for transform in f {match transform {Transform::ToMonthly => { s.push('m') },Transform::ToQuarterly => { s.push('q') },Transform::YearOnYear => { s.push('y') },}Transform::ToMonthly => write!(f, "to_monthly"),Transform::ToQuarterly => write!(f, "to_quarterly"),Transform::YearOnYear => write!(f, "yoy"), - edit in src/ts.rs at line 396
FID(s) - edit in src/ts.rs at line 398[3.4285]→[3.11144:11145](∅→∅),[3.11144]→[3.11144:11145](∅→∅),[3.11145]→[3.4286:4331](∅→∅),[3.4331]→[3.10473:10478](∅→∅),[3.11145]→[3.10473:10478](∅→∅),[3.10478]→[3.4332:4427](∅→∅),[3.4427]→[2.3990:4122](∅→∅)
impl FromStr for FID {type Err = Error;fn from_str(s: &str) -> Result<Self, Self::Err> {Ok(FID(String::from(s)))}}impl fmt::Display for FID {fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {write!(f, "{}", self.0)}} - edit in src/lib.rs at line 1
#![deny(missing_docs)] - edit in src/lib.rs at line 205
PageSpec, - edit in src/lib.rs at line 207
Transform, - replacement in src/lib.rs at line 249
#[derive(Debug)]#[derive(Clone, Debug)] - replacement in src/lib.rs at line 434
match self.map.get_mut(&(series_spec.data_type, series_spec.country)) {// Expect country to be Some while building source.match self.map.get_mut(&(series_spec.data_type, series_spec.country.unwrap())) { - replacement in src/lib.rs at line 439
(series_spec.data_type, series_spec.country),// Expect country to be Some while building source.(series_spec.data_type, series_spec.country.unwrap()), - replacement in src/lib.rs at line 446
(series_spec.data_type, series_spec.country),// Expect country to be Some while building source.(series_spec.data_type, series_spec.country.unwrap()), - replacement in src/lib.rs at line 455
(series_spec.data_type, series_spec.country),// Expect country to be Some while building source.(series_spec.data_type, series_spec.country.unwrap()), - replacement in src/lib.rs at line 469
let mut page_spec = ts::PageSpec {let mut page_spec = PageSpec { - replacement in src/lib.rs at line 473
series: Vec::new(),seriess: BTreeMap::new(), - replacement in src/lib.rs at line 478
title: None,height: None,series_id: Vec::new(),fid: Vec::new(),title_opt: None,height_opt: None,series_ids: Vec::new(), - replacement in src/lib.rs at line 488
let ts_series_spec = ts::SeriesSpec {let ts_series_spec = SeriesSpec {country: None, - replacement in src/lib.rs at line 494
fid: None,drop_first: series_spec.drop_first, - replacement in src/lib.rs at line 499
page_spec.series.push(ts_series_spec);page_spec.seriess.insert(series_spec.series_id.clone(), ts_series_spec); - replacement in src/lib.rs at line 504
title: None,height: None,series_id: vec!(series_spec.series_id.clone()),fid: Vec::new(),title_opt: None,height_opt: None,series_ids: vec!(series_spec.series_id.clone()), - replacement in src/lib.rs at line 514
collated_graphic.series_id.push(series_spec.series_id.clone());collated_graphic.series_ids.push(series_spec.series_id.clone()); - replacement in src/lib.rs at line 673
///// We conflate both FRED-facing series and client-facing series. The reason for this is so that// the time_series() functionality is in one place. The down-side is that we need to make// the transforms optional, as they are not used in FRED-facing functionality, and// the country field optional as it is specified at a higher level in JsonSpec. - replacement in src/lib.rs at line 679
///pub country: Option<Country>, - edit in src/lib.rs at line 681
///pub country: Country,/// - replacement in src/lib.rs at line 682
///pub date_range: DateRange,pub transforms: Vec<Transform>, - edit in src/lib.rs at line 688
// We pass country because `series_spec.country` is None./// Return the meta data for a `SeriesSpec`.pub fn meta(&self, country: Country, root_path: &str) -> SeriesMetaData{let path = data_path(root_path,self.data_type,country,self.series_id.clone(),"meta");let meta_str = fs::read_to_string(path).unwrap();let kt = KeyTree::parse(&meta_str).unwrap();kt.to_ref().try_into().unwrap()} - edit in src/lib.rs at line 706
- replacement in src/lib.rs at line 713
self.country,// Expect country to be Some while building source.self.country.unwrap(), - replacement in src/lib.rs at line 722
self.country,// Expect country to be Some while building source.self.country.unwrap(), - edit in src/lib.rs at line 760
}}pub fn data_with_transforms(&self) -> Result<RegularTimeSeries<1>, Error> {let mut rts = self.data_without_transform()?;// Do transforms before constraining dates.for transform in &self.transforms {match transform {Transform::ToMonthly => rts.to_monthly(0),Transform::ToQuarterly => rts.to_quarterly(0),Transform::YearOnYear => {match rts.to_year_on_year(0) {Ok(rts) => rts,Err(err) => {return Err(keytree_error(file!(),line!(),&err.to_string(),))},}}}; - edit in src/lib.rs at line 790
rts.with_range(&self.date_range.0);Ok(rts) - edit in src/lib.rs at line 793
- replacement in src/lib.rs at line 796
pub fn into_time_series(&self) -> Result<RegularTimeSeries<1>, Error> {pub fn data_without_transform(&self) -> Result<RegularTimeSeries<1>, Error> { - replacement in src/lib.rs at line 829
&self.country.to_string(),// Expect country to be Some while building source.&self.country.unwrap().to_string(), - replacement in src/lib.rs at line 857
&self.country.to_string(),// Expect country to be Some while building source.&self.country.unwrap().to_string(), - replacement in src/lib.rs at line 885
let rts = self.into_time_series()?;let rts = self.data_without_transform()?; - replacement in src/lib.rs at line 908
self.country.as_path(),// Expect country to be Some while building source.self.country.unwrap().as_path(), - replacement in src/lib.rs at line 972
self.country.as_path(),// Expect country to be Some while building source.self.country.unwrap().as_path(), - edit in src/lib.rs at line 992
- edit in src/lib.rs at line 994
if let Some(country) = self.country {kt.push_value(1, "country", country);}; - replacement in src/lib.rs at line 1000
kt.push_value(1, "country", &self.country); - replacement in src/lib.rs at line 1002
kt.push_opt_value(1, "drop_first", self.drop_first);if let Some(first_date) = self.date_range.first_date() {kt.push_value(1, "first_date", first_date);}if let Some(last_date) = self.date_range.last_date() {kt.push_value(1, "last_date", last_date);};for f in &self.transforms {kt.push_value(1, "f", f);}if let Some(drop) = &self.drop_first {kt.push_value(1, "drop_first", drop);} - edit in src/lib.rs at line 1026
let first_date = self.opt_value("series::first_date")?;let last_date = self.opt_value("series::last_date")?; - edit in src/lib.rs at line 1032
country: self.opt_value("series::country")?, - edit in src/lib.rs at line 1034
country: self.value("series::country")?, - edit in src/lib.rs at line 1035
date_range: DateRange::new(&first_date, &last_date),transforms: self.vec_value("series::transforms")?, - edit in src/lib.rs at line 1130
/// Read csv data from file and return a time-series.pub fn data_from_file(country: Country,data_type: DataType,series_id: SeriesId,root_path: &str) -> Result<RegularTimeSeries<1>, Error>{let path = data_path(root_path,data_type,country,series_id.clone(),"csv");match TimeSeries::<1>::from_csv(&path) {Ok(ts) => {match ts.try_into() {Ok(rts) => Ok(rts),Err(err) => {Err(time_series_from_csv_failed(file!(),line!(),&data_type.to_string(),&country.to_string(),&series_id.to_string(),&err.to_string(),))},}},Err(err) => Err(time_series_from_csv_failed(file!(),line!(),&data_type.to_string(),&country.to_string(),&series_id.to_string(),&err.to_string(),))}}/// Return the meta data for a `SeriesSpec`.pub fn meta_from_file(country: Country,data_type: DataType,series_id: SeriesId,root_path: &str) -> SeriesMetaData{let path = data_path(root_path,data_type,country,series_id.clone(),"meta");let meta_str = fs::read_to_string(path).unwrap();let kt = KeyTree::parse(&meta_str).unwrap();kt.to_ref().try_into().unwrap()} - edit in src/fred.rs at line 17
DateRange, - replacement in src/fred.rs at line 115
country: tag_selector.country,country: Some(tag_selector.country), - edit in src/fred.rs at line 117
date_range: DateRange::new(&None, &None),transforms: Vec::new(), - edit in src/fred.rs at line 168
country: Some(tag_selector.country), - edit in src/fred.rs at line 170
country: tag_selector.country, - edit in src/fred.rs at line 171
date_range: DateRange::new(&None, &None),transforms: Vec::new(), - replacement in src/error.rs at line 89
///pub fn failed_to_reference_series(code_file: &str,code_line: u32,series_id: &str) -> Error{Error(format!("[ui_date:06:{}:{}] Could not find graphic reference [{}] in series.",code_file,code_line,series_id,))} - edit in src/error.rs at line 127
/// - edit in src/error.rs at line 140
///