ts.rs
// #![allow(dead_code)]
use std::collections::BTreeMap;
use std::convert::{ TryInto };
use std::{fmt, fs};
use std::str::FromStr;
use serde::Serialize;
use countries::Country;
use keytree::{ KeyTree, KeyTreeRef };
use keytree::serialize::{ KeyTreeString, IntoKeyTree };
use crate::{ DataType, DateRange, GraphicRange, MonthlyDate, SeriesId, SeriesJson, SeriesSpec };
use crate::error::*;
// Default height in pixels of a graphic.
static GRAPHIC_HEIGHT: f32 = 300.0;
/// Top level Javascript-facing data-structure, that is loaded into memory by the server, and acts
/// as an immutable datastore for time-series web-pages.
#[derive(Debug)]
pub struct Json(BTreeMap<(Country, DataType, usize), PageValue>);
impl Json {
pub fn into_ts_data(&self) -> Result<TSData, Error> {
let mut map: BTreeMap<(Country, DataType, usize), String> = BTreeMap::new();
for (key, value) in self.0.iter() {
let json = serde_json::to_string(&value)
.map_err(|_| err(file!(), line!(), "Failed to serialize JSON."))?;
map.insert(*key, json);
}
Ok(TSData(map))
}
}
pub struct TSData(pub BTreeMap<(Country, DataType, usize), String>);
// The JSON that is served for a given page.
#[derive(Debug, Serialize)]
pub struct PageValue {
country: Country,
data_type: DataType,
index: usize,
seriess: Vec<SeriesJson>,
graphics: Vec<GraphicJson>,
first_date: MonthlyDate,
last_date: MonthlyDate,
max: f32,
min: f32,
height: f32,
}
/// The values in `Json` that are served one per web-page.
#[derive(Debug, Serialize)]
pub struct GraphicJson {
category: GraphicCategory,
// Reference to series data.
series_ref: Vec<usize>,
// Specifies what kind of text appears under the graphic.
caption_spec: CaptionSpec,
graphic_range: Option<GraphicRange>,
note: Option<String>,
}
/// The top-level Fred-facing time-series specification. By convention, the file 'ts_spec.keytree'
/// can be deserialized into this type using the `KeyTree` library.
#[derive(Debug)]
pub struct Spec(pub Vec<PageSpec>);
impl Spec {
pub (crate) fn empty() -> Self {
Spec(Vec::new())
}
// This has to be the sole initialization method for Spec, because we need to specifically
// downcast counties to series specifications.
/// Read in ts specification from file.
/// ```
/// let ts_spec = ts::Spec::from_file("ts_spec.keytree");
/// ```
pub fn from_file(path: &str) -> Result<Self, Error> {
let source_spec = match fs::read_to_string(path) {
Ok(ss) => ss,
Err(e) => { return Err(
err(file!(), line!(), &format!("Failed to read {}.", e))
)},
};
let kt = KeyTree::parse(&source_spec).unwrap();
let mut spec: Spec = kt.to_ref().try_into().map_err(|e: keytree::error::Error| {
err(file!(), line!(), &e.to_string())
})?;
// We need to do some post-processing to
// Iterate over PageSpecs and
spec.0.iter_mut().for_each(|page_spec| page_spec.downcast_countries());
Ok(spec)
}
// This function is responsible for converting `Spec` into `Json`. The iteration is simple, but
// the building of `Json` is a little complex. This is the point where Fred-facing data-structures
// meet Javascript-facing client data-structures, and hence the incongruence.
pub fn into_json(&self, root_path: &str) -> Result<Json, Error> {
let mut json = BTreeMap::new();
// Iterate over pages.
for page_spec in &self.0 {
let PageSpec {
country,
data_type,
index,
height_opt,
seriess,
graphics,
} = page_spec;
// Iterate over graphics.
let mut v_graphics: Vec<GraphicJson> = Vec::new();
// Build series data from series specification.
// We want to build a BTreeMap<SeriesId, SeriesJson> from a BTreeMap<SeriesId, SeriesSpec>
let mut m_series: BTreeMap<SeriesId, (usize, SeriesJson)> = BTreeMap::new();
for (i, (series_id, _)) in seriess.iter().enumerate() {
m_series.insert(
series_id.clone(),
(
i,
page_spec.into_series_json(series_id, root_path)?,
)
);
}
// Build graphics from graphics specification.
for graphic_spec in graphics {
let GraphicSpec {
category_opt,
series_ids,
graphic_range,
note,
} = graphic_spec;
// Construct GraphicJson for a given Graphic
let category = match category_opt {
None => GraphicCategory::Source,
Some(GraphicCategory::Cleaned) => GraphicCategory::Cleaned,
Some(GraphicCategory::Collation) => GraphicCategory::Collation,
Some(GraphicCategory::Source) => GraphicCategory::Source,
};
// Source graphics are expected to have one series (otherwise meta will overflow
// box).
if let GraphicCategory::Source = category {
if !graphic_spec.assert_has_one_series() {
return Err(err(
file!(),
line!(),
&format!("Expected graphic ({} {} {}) to have one series.",
country,
data_type,
index,
),
))
}
};
// Sets the kind of text display under each graphic.
let caption_spec = match graphic_spec.category_opt {
Some(GraphicCategory::Cleaned) => CaptionSpec::Link,
Some(GraphicCategory::Collation) => CaptionSpec::Link,
Some(GraphicCategory::Source) => CaptionSpec::Meta,
// Default is assumed implicitly to be GraphicClass::Source.
None => CaptionSpec::Meta,
};
let mut graphic_json = GraphicJson {
category: category,
series_ref: Vec::new(),
caption_spec: caption_spec,
graphic_range: *graphic_range,
note: note.clone(),
};
for series_id in series_ids {
let ix = match m_series.get(series_id) {
Some((ix, _)) => ix,
None => {
return Err(err(
file!(),
line!(),
&format!("Failed to lookup {}.", &series_id.to_string()),
))
},
};
graphic_json.series_ref.push(*ix);
}
// Push GraphicJson to Vec<GraphicJson>
v_graphics.push(graphic_json);
}
// Now that we have iterated over all the graphics on one page we can insert the
// collection of SeriesJsons and the collection of GraphicJsons into Json.
// Convert v_series to a Vec.
let mut v_series: Vec<SeriesJson> = Vec::new();
for (_, (_, series_json)) in m_series.into_iter() {
v_series.push(series_json);
};
let first_date = v_series.iter()
.map(|series_json| series_json.rts.first_date())
.min()
.unwrap();
let last_date = v_series.iter()
.map(|series_json| series_json.rts.last_date())
.max()
.unwrap();
let max = v_series.iter()
.map(|series_json| series_json.rts.max(0))
.fold(f32::NEG_INFINITY, |a, b| a.max(b));
let min = v_series.iter()
.map(|series_json| series_json.rts.min(0))
.fold(f32::INFINITY, |a, b| a.min(b));
let height = match height_opt {
Some(h) => *h,
None => GRAPHIC_HEIGHT,
};
let page_value = PageValue {
country: *country,
data_type: *data_type,
index: *index,
seriess: v_series,
graphics: v_graphics,
first_date: MonthlyDate(first_date),
last_date: MonthlyDate(last_date),
max: max,
min: min,
height: height,
};
json.insert((*country, *data_type, *index), page_value);
}
Ok(Json(json))
}
}
impl<'a> TryInto<Spec> for KeyTreeRef<'a> {
type Error = keytree::Error;
fn try_into(self) -> Result<Spec, keytree::Error> {
Ok(Spec(self.vec_at("ts_spec::page")?))
}
}
impl IntoKeyTree for Spec {
fn keytree(&self) -> KeyTreeString {
let mut kt = KeyTreeString::new();
kt.push_key(0, "ts_spec");
for page_spec in &self.0 {
kt.push_keytree(1, page_spec.keytree());
}
kt
}
}
// There are two important functions that need to be built.
// 1. SeriesSpec.into_json(). This is done in lib.rs.
// 2. PageJson.into_data()
/// Component of time-series specification.
/// ```
/// page:
/// country: Australia
/// data_type: u
/// index: 0
/// graphic:
/// series:
/// data_type: u
/// series_id: AUSURAMS
/// series:
/// data_type: u
/// series_id: AUSURANAA
/// ```
#[derive(Debug)]
pub struct PageSpec {
country: Country,
data_type: DataType,
index: usize,
height_opt: Option<f32>,
pub seriess: BTreeMap<SeriesId, SeriesSpec>,
pub graphics: Vec<GraphicSpec>,
}
impl PageSpec {
pub fn new(
country: Country,
data_type: DataType,
index: usize,
height_opt: Option<f32>) -> Self
{
PageSpec {
country,
data_type,
index,
height_opt,
seriess: BTreeMap::new(),
graphics: Vec::new(),
}
}
// Because country is specified in the PageSpec level of a specification, on initialization of a
// PageSpec, we need to set the country in series. The only way to initialize is through the
// Spec::from_file() function.
pub (crate) fn downcast_countries(&mut self) {
for mut country in self.seriess.iter().map(|(_, series_spec)| series_spec.country) {
country = Some(self.country);
}
}
pub (crate) fn into_series_json(
&self,
series_id: &SeriesId,
root_path: &str) -> Result<SeriesJson, Error>
{
let series_spec = match self.seriess.get(&series_id) {
Some(series) => series,
None => {
return Err(
err(file!(), line!(), &format!("Series {} lookup failed.", series_id.to_string()))
)
}
};
let rts = series_spec.read_data_with_transforms(root_path)?;
let meta = series_spec.read_meta_from_file(root_path);
Ok(
SeriesJson {
series_id: series_spec.series_id.clone(),
rts: rts,
meta: Some(meta),
transforms: series_spec.transforms.clone(),
}
)
}
}
impl<'a> TryInto<PageSpec> for KeyTreeRef<'a> {
type Error = keytree::Error;
fn try_into(self) -> Result<PageSpec, keytree::Error> {
let seriess_vec: Vec<SeriesSpec> = self.vec_at("page::series")?;
let mut map = BTreeMap::new();
for series_spec in seriess_vec {
map.insert(series_spec.series_id.clone(), series_spec);
}
Ok(
PageSpec {
country: self.value("page::country")?,
data_type: self.value("page::data_type")?,
index: self.value("page::index")?,
height_opt: self.opt_value("page::height")?,
seriess: map,
graphics: self.vec_at("page::graphic")?,
}
)
}
}
impl IntoKeyTree for PageSpec {
fn keytree(&self) -> KeyTreeString {
let mut kt = KeyTreeString::new();
kt.push_key(0, "page");
kt.push_value(1, "country", self.country);
kt.push_value(1, "data_type", self.data_type);
kt.push_value(1, "index", self.index);
for (_, series) in self.seriess.iter() {
kt.push_keytree(1, series.keytree());
}
for graphic in &self.graphics {
kt.push_keytree(1, graphic.keytree());
}
kt
}
}
/// Specifies what kind of text is displayed under a graphic.
#[derive(Copy, Clone, Debug, Serialize)]
pub enum CaptionSpec {
Link,
Meta,
None,
}
impl FromStr for CaptionSpec {
type Err = Error;
fn from_str(s: &str) -> Result<Self, Self::Err> {
match s {
"link" => Ok(CaptionSpec::Link),
"meta" => Ok(CaptionSpec::Meta),
_ => Err(err(file!(), line!(), &format!("Failed to parse {}.", s))),
}
}
}
impl fmt::Display for CaptionSpec {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
CaptionSpec::Link => write!(f, "link"),
CaptionSpec::Meta => write!(f, "meta"),
CaptionSpec::None => write!(f, "none"),
}
}
}
impl IntoKeyTree for CaptionSpec {
fn keytree(&self) -> KeyTreeString {
let mut kt = KeyTreeString::new();
match self {
CaptionSpec::Link => kt.push_value(0, "caption", "link"),
CaptionSpec::Meta => kt.push_value(0, "caption", "meta"),
CaptionSpec::None => {},
}
kt
}
}
#[derive(Debug, Serialize)]
pub enum GraphicCategory {
/// Generally the top graphic which displays all time-series.
Collation,
/// A single time-series which displays un-transformed data directly from source.
Source,
/// Data that has been selected and transformed.
Cleaned,
}
impl FromStr for GraphicCategory {
type Err = Error;
fn from_str(s: &str) -> Result<Self, Error> {
match s {
"collation" => Ok(GraphicCategory::Collation),
"source" => Ok(GraphicCategory::Source),
"cleaned" => Ok(GraphicCategory::Cleaned),
_ => Err(err(file!(), line!(), "Failed to parse GraphicClass")),
}
}
}
impl fmt::Display for GraphicCategory {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let s = match self {
GraphicCategory::Collation => "collation",
GraphicCategory::Source => "source",
GraphicCategory::Cleaned => "cleaned",
};
write!(f, "{}", s)
}
}
/// Component of time-series specification.
/// ```
/// graphic:
/// series:
/// data_type: u
/// series_id: AUSURAMS
/// series:
/// data_type: u
/// series_id: AUSURANAA
/// ```
#[derive(Debug)]
pub struct GraphicSpec {
pub category_opt: Option<GraphicCategory>,
pub series_ids: Vec<SeriesId>,
pub graphic_range: Option<GraphicRange>,
pub note: Option<String>,
}
impl GraphicSpec {
pub (crate) fn assert_has_one_series(&self) -> bool {
self.series_ids.len() == 1
}
}
impl<'a> TryInto<GraphicSpec> for KeyTreeRef<'a> {
type Error = keytree::Error;
fn try_into(self) -> Result<GraphicSpec, keytree::Error> {
Ok(
GraphicSpec {
category_opt: self.opt_value("graphic::category")?,
series_ids: self.vec_value("graphic::series_id")?,
graphic_range: self.opt_value("graphic::range")?,
note: self.opt_value("graphic::note")?,
}
)
}
}
impl IntoKeyTree for GraphicSpec {
fn keytree(&self) -> KeyTreeString {
let mut kt = KeyTreeString::new();
kt.push_key(0, "graphic" );
if let Some(class) = &self.category_opt {
kt.push_value(1, "class", class);
}
if let Some(range) = &self.graphic_range {
kt.push_value(1, "graphic", range);
}
if let Some(note) = &self.note {
kt.push_value(1, "note", note);
}
for series_id in &self.series_ids {
kt.push_value(1, "series_id", series_id);
}
kt
}
}
/// Transforms from a TimeSeries<1> to another TimeSeries<1>.
#[derive(Copy, Clone, Debug, Serialize)]
pub enum Transform {
ToMonthly,
ToQuarterly,
/// Year on year percentage change.
YearOnYear,
DateRange(DateRange),
}
impl FromStr for Transform {
type Err = Error;
fn from_str(s: &str) -> Result<Self, Self::Err> {
match s {
"to_monthly" => Ok(Transform::ToMonthly),
"to_quarterly" => Ok(Transform::ToQuarterly),
"yoy" => Ok(Transform::YearOnYear),
_ => Err(err(file!(), line!(), &format!("Could not parse {}", s))),
}
}
}
impl fmt::Display for Transform {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Transform::ToMonthly => write!(f, "to_monthly"),
Transform::ToQuarterly => write!(f, "to_quarterly"),
Transform::YearOnYear => write!(f, "yoy"),
Transform::DateRange(date_range)
=> write!(f, "{}", date_range.to_string()),
}
}
}