//! Collect unemployment rate, inflation rate and interest rate data server side, then serve this
//! data as JSON in a format suitable for building UI graphics using D3. This is somewhat annoying
//! complex because the software automation must interact with manually edited specifications.
//! 
//! ## Step 1. Build a generic data specification.
//!
//! The data series selector specification that determines what data series to download from Fred
//! looks like
//! ```text
//! selectors:
//!     series:
//!         country:    Australia
//!         data_type:  u
//!         tag:        unemployment
//!         exclude:    Male
//!         exclude:    Female
//!         exclude:    55-64
//!         exclude:    25-54
//!         exclude:    15-24
//!         exclude:    20 to 24
//!         exclude:    Youth
//!         exclude:    Women
//!         exclude:    Teenagers
//!         require:    Rate
//! 
//!     series:
//!         country:    Austria
//!         data_type:  u
//!         tag:        unemployment
//!         exclude:    Male
//!         exclude:    Female
//!         exclude:    55-64
//!         exclude:    25-54
//!         exclude:    15-24
//! ```
//! For example, the selector specification for unemployment in Australia, looks up the tag
//! `unemployment` in conjuction with `australia` then filters out any title that includes the
//! phrases `Male`, `Female`, `55-64` etc. It also filters out any title that does not include the
//! phrase `Rate`.
//!
//! ```
//! println!("{}", DataSelector::from_file("series_selector.keytree")
//!     .into_data_spec()
//!     .unwrap()
//!     .keytree());
//! ```
//!
//! Copy the output of the above command from the terminal and paste the results in
//! `source_data.keytree`. The reason we don't automatically save these files, is that we have to be
//! careful not to overwrite files that may have been manually edited. So overwriting a
//! specification file must be done manually through copy/paste. The resulting The data series
//! specification `source_data.keytree` will look something like
//!
//! ```
//! seriess:
//!     series:
//!         data_type:  u
//!         country:    Australia
//!         id:         AUSURAMS
//!     series:
//!         data_type:  u
//!         country:    Australia
//!         id:         AUSURANAA
//!     series:
//!         data_type:  u
//!         country:    Australia
//!         id:         AUSURAQS
//!     series:
//!         data_type:  u
//!         country:    Australia
//!         id:         AUSURHARMADSMEI
//!     series:
//!         data_type:  u
//!         country:    Australia
//!         id:         AUSURHARMMDSMEI
//! ```
//!
//! ## Step 2. Use the data specification to write data to file.
//! ```
//! DataSpec::from_file("source_data.keytree")
//!     .unwrap()
//!     .write("/full/path/to/data");
//! ```
//!
//! If there is a break in the connection we can use
//! ```
//! DataSpec::from_file("source_data.keytree")
//!     .unwrap()
//!     .resume_write("NORURTOTADSMEI", "/full/path/to/data");
//! ```
//! to resume. To update the data files due to a change of the DataSpec,
//! ```
//! data_spec
//!     .update_write("/full/path/to/data")
//!     .unwrap();
//! ```
//! `update_write()` will also remove any series that have been removed from
//! the data specification. Deleting the data files directly is useful to re-download data files.
//!
//! ## Step 3. Generate a generic time-series graphics specification.
//!
//! The generated specification specifies the time-series graphics.
//!
//! ```
//! let data_spec = DataSpec::from_file("source_data.keytree")
//!     .unwrap();
//! println!("{}", data_spec.generic_ts_spec().keytree());
//! ```
//! Again, to overwrite an existing specification file, the terminal output needs to be copied and
//! pasted over an existing specification file.
//!
//! ## Step 4. Serve Time-series
//!
//! A time-series specification looks like
//! ```
//! ts_spec:
//!     page:
//!         country:            Australia
//!         data_type:          u
//!         index:              0
//!         graphic:
//!             series:
//!                 data_type:  u
//!                 series_id:  AUSURAMS
//!             series:
//!                 data_type:  u
//!                 series_id:  AUSURANAA
//!             series:
//!                 data_type:  u
//!                 series_id:  AUSURAQS
//!             series:
//!                 data_type:  u
//!                 series_id:  AUSURHARMADSMEI
//!             series:
//!                 data_type:  u
//!                 series_id:  AUSURHARMMDSMEI
//!             series:
//!                 data_type:  u
//!                 series_id:  AUSURHARMQDSMEI
//! ```
//! It specifies the way time-series graphics are presented in the client browser. A generic
//! time-series specification can be built using
//! ```
//! // include command here
//! ```
//!
//! This step converts the specification in JSON data and meta-data that is served to the client.
//! The client Javascript then builds the graphics dynamically. Take both specifications and load
//! data from "/full/path/to/data".
//!
//! ```
//! let data_spec = DataSpec::from_file("source_data.keytree").unwrap();
//! let ts_spec = TSSpec::from_file("ts_spec.keytree");
//! ```
//!
//! Then `TSSpec` uses `IndexedDataSpec` as an argument to be converted into `TSJson`.
//!
//! ```
//! let ts_json = ts_spec.into_json(&data_spec, "/full/path/to/data").unwrap();
//! ```
//! All JSON is generated at server startup into an immutable `HashMap` in memory.
//!
//! ## Step 5. Serve UI Scatterplots
//!

pub mod error;
pub mod fred;
pub mod parser;
pub mod ts;
pub mod ui;

use std::collections::BTreeMap;
use std::convert::TryInto;
use std::{ file, fmt, fs, line };
use std::path::Path;
use std::str::FromStr;

use serde::Serialize;
use walkdir::WalkDir;

use countries::Country;
use fred_api::{ Fred };
use keytree::{ KeyTree, KeyTreeRef };
use keytree::serialize::{ IntoKeyTree, KeyTreeString };
use time_series::{ RegularTimeSeries, TimeSeries };

use crate::error::*;
use crate::parser::*; 
use crate::ts::{ GraphicCategory, Transform };

/// A date that can be parsed from a string like "07-08-2010".
#[derive(Clone, Debug, Serialize)]
pub struct MonthlyDate(time_series::MonthlyDate);

impl FromStr for MonthlyDate {
    type Err = Error;
    
    fn from_str(s: &str) -> Result<Self, Self::Err> {
        let year = s[..4].parse().map_err(|_| {
            err(file!(), line!(), &format!("Parse datapoint {} failed", s))
        })?;

        let month = s[5..7].parse().map_err(|_| {
            err(file!(), line!(), &format!("Parse datapoint {} failed", s))
        })?;

        Ok(MonthlyDate(time_series::MonthlyDate::ym(year, month)))
    }
}

impl fmt::Display for MonthlyDate {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(f, "{}-{}-01", self.0.year(), self.0.month())
    }
}

/// `DateRange` newtype.
#[derive(Clone, Copy, Debug, Serialize)]
pub struct DateRange(time_series::DateRange);

impl DateRange {

    /// Create a time_series::DateRange.
    pub fn new(
        first_date: &Option<MonthlyDate>,
        last_date: &Option<MonthlyDate>) -> DateRange
    {
        DateRange(
            time_series::DateRange::new(
                &first_date.clone().map(|d| d.0),
                &last_date.clone().map(|d| d.0),
            )
        )
    }

    pub fn first_date(&self) -> Option<MonthlyDate> {
        self.0.first_date().map(|date| MonthlyDate(date))
    }

    pub fn last_date(&self) -> Option<MonthlyDate> {
        self.0.last_date().map(|date| MonthlyDate(date))
    }
}

impl fmt::Display for DateRange {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        let s = match (self.0.first_date(), self.0.last_date()) {
            (None, None) => {
                format!(
                    "open"
                )
            },
            (Some(d1), None) => {
                format!(
                    "from {}-{}-01",
                    d1.year(),
                    d1.month(),
                )
            },
            (None, Some(d2)) => {
                format!(
                    "to {}-{}-01",
                     d2.year(),
                    d2.month()
                )
            },
            (Some(d1), Some(d2)) => {
                format!(
                    "between {}-{}-01 and {}-{}-01",
                    d1.year(),
                    d1.month(),
                    d2.year(),
                    d2.month(),
                )
            },
        };
        write!(f, "{}", s)
    }
}

/// Represents a FRED series id like `LRHUTTTTAUA156N` or a transformation on a FRED series_id
/// like `LRHUTTTTAUA156N_a`.
#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)]
pub struct SeriesId(String);

impl SeriesId {

    /// Create a new SeriesId from a string.
    pub fn new(s: &str) -> Self {
        SeriesId(s.to_string())
    }

    /// Return the component without transformation modifications.
    pub fn stem(&self) -> Self {
        let inner = self.0.split('_').next().unwrap().clone();
        SeriesId(String::from(inner)) 
    }
}

impl FromStr for SeriesId {
    type Err = ();

    fn from_str(s: &str) -> Result<Self, ()> {
        Ok(SeriesId(String::from(s)))
    }
}

impl fmt::Display for SeriesId {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(f, "{}", self.0)
    }
}

/// Unemployment rate, interest rate, inflation rate etc.
#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)]
pub enum DataType {
    /// Unemployment rate
    U,
    /// CPI
    Cpi,
    /// Inflation rate
    Inf,
    /// Interest rate
    Int,
}

impl FromStr for DataType {
    type Err = Error;

    /// Parse a string into a `DataType` or return `None` on failure.
    fn from_str(s: &str) -> Result<Self, Self::Err> {
        match s {
            "u"     => Ok(DataType::U),
            "cpi"   => Ok(DataType::Cpi),
            "inf"   => Ok(DataType::Inf),
            "int"   => Ok(DataType::Int),
            _       => Err(err(file!(), line!(), &format!("Failed to parse datatype {}.", s))),
        }
    }
}

impl fmt::Display for DataType {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        let s = match self {
            DataType::U => "u",
            DataType::Cpi => "cpi",
            DataType::Inf => "inf",
            DataType::Int => "int",
        };
        write!(f, "{}", s)
    }
}

/// ```
/// println!("{}", title("LFACTTTTKRA657N"));
/// ```
pub fn title(series_id: &str) -> Result<String, Error> {
    let sid = SeriesId::from_str(series_id).unwrap();
    let mut s = String::new();
    let seriess = match Fred::series(&sid.to_string()) {
        Ok(series) => series.seriess,
        Err(e) => { return Err(err(file!(), line!(), &e.to_string())) },
    }; 
    for series in seriess.iter() {
        s.push_str(&series.title)
    }
    Ok(s)
}

/// Return series from tags. Tags look like "loans;australia".
pub fn interest_rate_series(tags: &str) {
    let tags_series = Fred::tags_series(tags).unwrap();
    let series_items = tags_series.seriess;
    let iter = series_items.iter();
    for item in iter {
        println!();
        println!("{}", item.title);  
        println!("{}", item.id);
        let tags = Fred::series_tags(&item.id).unwrap();
        println!("{}", tags.one_line());
    }  
}

// `DataSpec` is set up in what seems like an overly complex way in order to maintain
// the order is which data is `DataSelector` orders the series. We want to maintain an 
// ordering by (DataType, Country). But sometimes we also need to search by SeriesId so also need a
// reverse lookup.
/// A generic specification of all data series. Output looks like
/// ```
/// series:
///      data_type:  u
///      country:    United States
///      id:         LRUNTTTTUSQ156S
/// ```
#[derive(Debug)]
pub struct DataSpec {
    map: BTreeMap<(DataType, Country), Vec<SeriesSpec>>,
    reverse: BTreeMap<SeriesId, (DataType, Country)>,
}

impl DataSpec {

    /// Used to build a generic specification.
    pub fn empty() -> Self {
        DataSpec {
            map: BTreeMap::new(),
            reverse: BTreeMap::new(),
        }
    }

    /// Get a `SeriesSpec` from a `SeriesId`.
    pub fn get_series_spec(&self, series_id: &SeriesId) -> Option<SeriesSpec> {

        let key = match self.reverse.get(&series_id) {
            Some(key) => key,
            None => { return None },
        };
        let seriess = self.map.get(&key).unwrap();      

        match seriess.iter().find(|&series| {
            &series.series_id == series_id
        }) {
            Some(series_spec) => Some(series_spec.clone()),
            None => None,
        } 
    }

    pub (crate) fn new() -> Self {
        DataSpec {
            map: BTreeMap::new(),
            reverse: BTreeMap::new(),
        }
    }

    /// Return a `DataSpec` built from a collection of `SeriesSpec`.
    pub (crate) fn from_vec(v: Vec<SeriesSpec>) -> Self {
        let mut data_spec = DataSpec::new();
        for series_spec in v {
            data_spec.insert(&series_spec);
        }
        data_spec
    }

    /// Insert a `SeriesSpec`.
    pub fn insert(&mut self, series_spec: &SeriesSpec) {
        // Expect country to be Some while building source.
        match self.map.get_mut(&(series_spec.data_type, series_spec.country.unwrap())) {
            None => {

                self.map.insert(
                    // Expect country to be Some while building source.
                    (series_spec.data_type, series_spec.country.unwrap()),
                    vec!(series_spec.clone())
                );

                self.reverse.insert(
                    series_spec.series_id.clone(),
                    // Expect country to be Some while building source.
                    (series_spec.data_type, series_spec.country.unwrap()),
                );
            },
            Some(value) => {
                value.push(series_spec.clone());

                self.reverse.insert(
                    series_spec.series_id.clone(),
                    // Expect country to be Some while building source.
                    (series_spec.data_type, series_spec.country.unwrap()),
                );
            },
        }
    }

    /// Build a generic `Spec` from `Self`.
    pub fn generic_ts_spec(&self) -> ts::Spec {

        let mut spec = ts::Spec::empty();

        for ((data_type, country), series_specs) in self.map.iter() {

            let mut page_spec = ts::PageSpec::new(
                *country,           // country
                *data_type,         // data_type
                0,                  // index
                None,               // height
            );

            let mut collated_graphic = ts::GraphicSpec {
                category_opt:   Some(GraphicCategory::Collation),
                series_ids:     Vec::new(),
                graphic_range:  None,
                note:           None,
            };

            let mut single_graphics = Vec::new();

            for series_spec in series_specs {

                let ts_series_spec = SeriesSpec {
                    country:        None,
                    data_type:      series_spec.data_type,
                    series_id:      series_spec.series_id.clone(),
                    transforms:     Vec::new(),
                    parser:         ParserSpec::FredSpec(FredSpec::empty()),
                };

                // Series

                page_spec.seriess.insert(series_spec.series_id.clone(), ts_series_spec);

                // Single-series graphics

                let single_series_graphic = ts::GraphicSpec {
                    category_opt:   Some(GraphicCategory::Source),
                    series_ids:     vec!(series_spec.series_id.clone()), 
                    graphic_range:  None,
                    note:           None,
                };

                single_graphics.push(single_series_graphic);

                // Collated graphic

                collated_graphic.series_ids.push(series_spec.series_id.clone());
            };

            let mut graphics = vec!(collated_graphic);
            graphics.append(&mut single_graphics);
            page_spec.graphics = graphics;

            spec.0.push(page_spec); 
        }
        spec
    }

    // /// Append `other` to `Self`.
    // pub fn append(&mut self, other: &mut DataSpec) {
    //     self.0.append(&mut other.0)
    // }

    // fn series_from_index(&self, i: usize) -> &Series {
    //     &self.0[i]
    // }

    /// Read in keytree data from file.
    pub fn from_file(path: &str) -> Result<Self, Error> {
        dbg!(path);
        let source_spec = match fs::read_to_string(path) {
            Ok(ss) => ss,
            Err(e) => { return Err(
                err(file!(), line!(), &format!("Failed to read file {}.", e))
            )},
        };
        let kt = KeyTree::parse(&source_spec).unwrap();
        kt.to_ref().try_into().map_err(|e: keytree::error::Error| {
            err(file!(), line!(), &e.to_string())
        }) 
    }

    /// Save FRED data to disk as csv, using full path. Will fail if an existing filepath is
    /// encountered.
    /// ```
    /// let mut source = DataSpec::from_file("checked_data.keytree");
    /// source.write(&root_dir);
    /// ```
    /// To path from root is "/{data_type}/{country}/LRUNTTTTSIQ156S.csv"
    /// 
    pub fn write(&self, root_path: &str) -> Result<(), Error> {
        for (_, series_specs) in &self.map {
            for series_spec in series_specs {
                series_spec.write_data_to_file(root_path)?;
                series_spec.write_meta_to_file(root_path)?; 
            }
        }
        Ok(())
    }

    // Need to keep a record of all files and remove files that shouldn't exist.
    // 
    // i.e. its not sufficient to check if a file exists. We need to check if a file remains. So we
    // need to create a file list.

    /// Only make requests and updata data to Fred for files that are in `DataSpec` but do not exist
    /// as data files.
    pub fn update_write(&self, root_path: &str) -> Result<(), Error> {
        for (_, series_specs) in &self.map {
            for series_spec in series_specs {
                if !series_spec.exists(root_path)? {
                    dbg!(&series_spec);
                    series_spec.write_data_to_file(root_path)?;
                    series_spec.write_meta_to_file(root_path)?;
                }
            }
        }
        self.remove_old(root_path)?;
        Ok(())
    }

    /// Run through data files, query and remove any files that are not in directory.
    pub fn remove_old(&self, root_path: &str) -> Result<(), Error> {

        for entry in WalkDir::new(root_path) {

            let entry = entry.unwrap();

            if !entry.file_type().is_dir() {

                let pathbuf = entry.path(); 

                let mut path_iter = pathbuf
                    .iter()
                    .rev()
                    .map(|os_str| os_str.to_str()
                    .unwrap());

                let mut file_parts = path_iter
                    .next()
                    .unwrap()
                    .split('.');

                let file_stem = file_parts.next().unwrap();
                let file_ext = file_parts.next().unwrap();

                if file_ext == "csv" || file_ext == "meta" {

                    let series_id = SeriesId::new(file_stem);

                    match self.get_series_spec(&series_id) {
                        Some(_) => {},
                        None => {
                            println!("remove file: {}", entry.path().display());
                            // match fs::remove_file(entry.path()) {
                            //     Ok(_) => {},
                            //     Err(_) => {
                            //         return Err(file_error(file!(), line!()))
                            //      },
                            // }
                        },
                    }
                }
            };
        }
        Ok(())
    }

    /// Same as `write()` except that it starts in the specification at `series_id`. Useful if there
    /// is a break in the connection when writing.
    pub fn resume_write(&self, series_id: &str, root_path: &str) -> Result<(), Error> {
        let sid = SeriesId::new(series_id);
        for (_, series_specs) in self
            .map
            .iter()
            .skip_while(|_| {
                match self.get_series_spec(&sid) {
                    Some(series_spec) => sid != series_spec.series_id,
                    None => true,
                }
            })
        {
            for series_spec in series_specs {
                series_spec.write_data_to_file(root_path)?;
                series_spec.write_meta_to_file(root_path)?;
            }
        }
        Ok(())
    }
}

impl<'a> TryInto<DataSpec> for KeyTreeRef<'a> {
    type Error = keytree::Error;

    fn try_into(self) -> Result<DataSpec, Self::Error> {
        let v: Vec<SeriesSpec> = self.vec_at("seriess::series")?;
        Ok(DataSpec::from_vec(v))
    }
}

impl IntoKeyTree for DataSpec {
    fn keytree(&self) -> KeyTreeString {
        let mut kt = KeyTreeString::new();
        kt.push_key(0, "seriess");
        for (_, series_specs) in &self.map {
            for series_spec in series_specs {
                kt.push_keytree(1, series_spec.keytree());
            }
        }
        kt
    }
}

// The country field is always `Some`. The `Option` is used in initialization from a keytree file.
// The transforms field is set is the time-series and ui specifications, and so is empty when a
// SeriesSpec is initialized from a source specification.
#[derive(Clone, Debug)]
pub struct SeriesSpec {
    country:            Option<Country>,
    pub data_type:      DataType,
    pub series_id:      SeriesId,
    pub transforms:     Vec<Transform>,
    pub parser:         ParserSpec,
}

impl SeriesSpec {

    pub fn country(&self) -> Country {
        self.country.unwrap()
    }

    pub fn data_path(&self, root_path: &str, extension: &str) -> String {
        format!(
            "{}/{}/{}/{}.{}",
            root_path,
            self.data_type,
            self.country().as_path(),
            self.series_id.stem(),
            extension,
        )
    }

    pub fn dir_path(&self, root_path: &str) -> String {
        format!(
            "{}/{}/{}",
            root_path,
            self.data_type,
            self.country().as_path(),
        )
    }

    /// Read metadata from file.
    pub fn read_meta_from_file(&self, root_path: &str) -> SeriesMetaData
    {
        let meta_str = fs::read_to_string(
            &self.data_path(root_path, "meta")
        ).unwrap();

        let kt = KeyTree::parse(&meta_str).unwrap();
        kt.to_ref().try_into().unwrap()
    }

    /// Check if data and meta-data exist in a file.
    pub fn exists(&self, root_path: &str) -> Result<bool, Error> {
        
        Ok(
            Path::new(&self.data_path(root_path, "csv")).exists() &&
            Path::new(&self.data_path(root_path, "meta")).exists()
        )
    }

    pub fn read_data_with_transforms(
        &self,
        root_path: &str) -> Result<RegularTimeSeries<1>, Error>
    {

        let mut rts = self.read_data_without_transform(root_path)?;

        // Do transforms before constraining dates.
       
        for transform in &self.transforms {

            rts = match transform {
                Transform::ToMonthly => rts.to_monthly(0),
                Transform::ToQuarterly => rts.to_quarterly(0),
                Transform::YearOnYear => {
                    match rts.to_year_on_year(0) {
                        Ok(rts) => rts,
                        Err(e) => { return Err(err(file!(), line!(), &e.to_string())) },
                    }
                },
                Transform::DateRange(range) => rts.range(&range.0),
            };
        }
        Ok(rts)
    }

    pub fn read_data_without_transform(
        &self,
        root_path: &str) -> Result<RegularTimeSeries<1>, Error>
    {
        let ts = TimeSeries::<1>::from_csv(&self.data_path(root_path, "csv"))
            .map_err(|e| {
                err(file!(), line!(), &e.to_string())
            })?;

        let rts = ts.try_into()
            .map_err(|e: time_series::error::Error| {
                err(file!(), line!(), &e.to_string())
            })?;

        Ok(rts)
    }

    pub fn to_data(
        &self,
        parser: ParserSpec) -> Result<RegularTimeSeries<1>, Error>
    {
        // We want to handle data fetching and parsing in the Parser implementation, so make the
        // Parser the receiver.
        match self.parser {
            ParserSpec::FredSpec(fred_spec) => FredParser(fred_spec).to_data(self),
            ParserSpec::FredDailySpec(fred_daily_spec) => FredDailyParser(fred_daily_spec).to_data(self),
        }
    }

    /// Fetches data as specified in `source_data.keytree` and saves to disk.
    pub fn write_data_to_file(&self, root_path: &str) -> Result<(), Error> {

        let rts = self.to_data(self.parser)?;

        // Build csv manually

        let mut s = String::new();
        for dp in rts.iter(time_series::DateRange::new(&None, &None)) {
            s.push_str(&format!(
                "{}-{}-01, {}",
                dp.date().year(),
                dp.date().month(),
                dp.value(0).to_string()
            ))
        }

        fs::create_dir_all(&self.dir_path(root_path))
            .map_err(|e| err(file!(), line!(), &format!("Failed to create dir {}.", e)))?;

        fs::write(self.data_path(root_path, "csv"), s)
            .map_err(|e| err(file!(), line!(), &e.to_string()))?;

        Ok(())
    }

    /// Fetches data as specified in `source_data.keytree` and saves meta_data to disk.
    pub fn write_meta_to_file(&self, root_path: &str) -> Result<(), Error> {

        let series =  Fred::series(&self.series_id.to_string())
            .map_err(|e| err(file!(), line!(), &e.to_string()))?;

        let series_item = series.seriess.iter().next().ok_or_else(|| {
            err(file!(), line!(), &format!("Expected series data for {}", self.series_id))
        })?;

        let meta = SeriesMetaData {
            realtime:               series.realtime_start.clone(),
            series_id:              self.series_id.clone(),
            title:                  series_item.title.clone(),
            observation_start:      series_item.observation_start.clone(),
            observation_end:        series_item.observation_end.clone(),
            frequency:              series_item.frequency.clone(),
            seasonal_adjustment:    series_item.seasonal_adjustment.clone(),
        };
        let json = serde_json::to_string(&meta)
            .map_err(|e| err(file!(), line!(), &e.to_string()))?;

        fs::create_dir_all(self.dir_path(root_path))
            .map_err(|e| err(file!(), line!(), &format!("Failed to create dir {}", e)))?;

        let path = self.data_path(root_path, "meta"); 

        println!("Writing {}", path);
        fs::write(path, json)
            .map_err(|e| err(file!(), line!(), &e.to_string()))?;

        Ok(())
    }
}

impl IntoKeyTree for SeriesSpec {
    fn keytree(&self) -> KeyTreeString {
        let mut kt = KeyTreeString::new();

        kt.push_key(0, "series");

        if let Some(country) = self.country {
            kt.push_value(1, "country", country);
        };

        kt.push_value(1, "data_type", &self.data_type);

        kt.push_value(1, "series_id", &self.series_id);

        for f in &self.transforms {
            kt.push_value(1, "f", f); 
        }

        if let ParserSpec::FredDailySpec(_) = self.parser {
            kt.push_value(1, "parse_cat", "fred_daily")
        };

        if let ParserSpec::FredSpec(parser_spec) = self.parser {
            if parser_spec.drop_first != 0 {
                kt.push_key(1, "parser");
                kt.push_value(2, "drop_first", parser_spec.drop_first.to_string());
            }
        };

        kt
    }
}

//     series:
//         data_type:          int
//         country:            United States
//         series_id:          DPRIME
//         parse_cat:          fred_daily
// 
// seriess:
//     series:
//         country:    United States
//         data_type:  int
//         series_id:  DPRIME

impl<'a> TryInto<SeriesSpec> for KeyTreeRef<'a> {
    type Error = keytree::Error;

    fn try_into(self) -> Result<SeriesSpec, keytree::Error> {

        let first_date  = self.opt_value("series::first_date")?;
        let last_date   = self.opt_value("series::last_date")?;

        let date_range = DateRange::new(&first_date, &last_date);

        let mut transforms: Vec<Transform> = self.opt_vec_value("series::transform")?;
   
        if first_date.is_some() || last_date.is_some() {
            transforms.push(Transform::DateRange(date_range));
        }

        // let parser_str_opt: Option<String> = self.opt_value("series::parser")?;
        // let parser = match parser_str_opt {
        //     None => Parser::Fred,
        //     Some(parser_str) => {
        //         Parser::from_str(&parser_str)
        //             .map_err(|err| keytree::error::external(file!(), line!(), &err.to_string()))?
        //     },
        // };

        let a_parser: Option<String> = self.opt_value("series::parse_cat::fred_daily")?;
        let b_parser: Option<String> = self.opt_value("series::parse_cat::fred")?;

        let parser = match (a_parser.is_some(), b_parser.is_some()) {

            (false, false)  => ParserSpec::FredDailySpec(FredDailySpec::empty()),
            (true, false)   => ParserSpec::FredDailySpec(FredDailySpec::empty()),
            (false, true)   => ParserSpec::FredSpec(self.at("series::parser")?),
            (true, true)    => Err(keytree::error::err(file!(), line!(), "Only one parser allowed."))?,
        };

        Ok(
            SeriesSpec{
                country:    self.opt_value("series::country")?,
                data_type:  self.value("series::data_type")?, 
                series_id:  self.value("series::series_id")?,
                transforms,
                parser,
            }
        )
    }
}

/// Source meta-data is stored in a file as a String:
///
/// ```text
/// series:
///     realtime_start:                 2021-06-03
///     realtime_end:                   2021-06-03
///     series_items:
///         series_item:
///             realtime:               2021-06-03
///             series_id:              AUSCPALTT01IXNBQ
///             title:                  Consumer Price Index: All items: Total: Total for Australia
///             observation_start:      1960-01-01
///             observation_end:        2021-01-01
///             frequency:              Quarterly
///             seasonal_adjustment:    Not Seasonally Adjusted
///             notes:                  (see JSON data for notes)
/// ```
#[derive(Debug, Serialize)]
pub struct SeriesMetaData {
    realtime: String,
    series_id: SeriesId,
    title: String,
    observation_start: String,
    observation_end: String,
    frequency: String,
    seasonal_adjustment: String,
}  

impl SeriesMetaData {

    ///
    pub fn from_file(
        data_type: DataType,
        country: Country,
        series_id: SeriesId,
        root_path: &str) -> Result<Self, Error>
    {
        let path = &format!(
            "{}/{}/{}/{}.meta",
            root_path,
            data_type,
            country,
            series_id,
        );

        let meta_str = fs::read_to_string(&path).unwrap();
        let kt = KeyTree::parse(&meta_str).unwrap();

        Ok(kt.to_ref().try_into().unwrap())
    }
} 

impl<'a> TryInto<SeriesMetaData> for KeyTreeRef<'a> {
    type Error = keytree::Error;

    fn try_into(self) -> Result<SeriesMetaData, Self::Error> {
        Ok(
            SeriesMetaData {
                realtime:               self.value("series_meta::realtime")?,
                series_id:              self.value("series_meta::series_id")?,
                title:                  self.value("series_meta::title")?,
                observation_start:      self.value("series_meta::observation_start")?,
                observation_end:        self.value("series_meta::observation_end")?,
                frequency:              self.value("series_meta::frequency")?,
                seasonal_adjustment:    self.value("series_meta::seasonal_adjustment")?,
            }
        )
    }
}

impl IntoKeyTree for SeriesMetaData {
    fn keytree(&self) -> KeyTreeString {
        let mut kt = KeyTreeString::new();

        kt.push_key(0, "series_meta");
        kt.push_value(1, "realtime", &self.realtime);
        kt.push_value(1, "series_id", &self.series_id.to_string());
        kt.push_value(1, "title", &self.title);
        kt.push_value(1, "observation_start", &self.observation_start);
        kt.push_value(1, "observation_end", &self.observation_end);
        kt.push_value(1, "frequency", &self.frequency);
        kt.push_value(1, "seasonal_adjustment", &self.seasonal_adjustment);

        kt
    }
}

/// A component of `Json` that is a single data series. Series are referenced by graphics through an
/// index, so that each series can be used multiple times, without being downloaded multiple times.
#[derive(Debug, Serialize)]
pub struct SeriesJson {
    series_id:  SeriesId,
    rts:        RegularTimeSeries<1>,
    meta:       Option<SeriesMetaData>,
    transforms: Vec<Transform>,
}

#[derive(Clone, Copy, Debug, Serialize)]
/// Specifies the range of a graphic
pub struct GraphicRange {
    min:    f32,
    max:    f32,
}

impl FromStr for GraphicRange {
    type Err = Error;

    fn from_str(s: &str) -> Result<Self, Self::Err> {

        let segment: Vec<&str> = s.split(" to ").collect();
        if segment[0].is_empty() || segment[1].is_empty() {
            return Err(
                err(file!(), line!(), &format!("Parse graphic range {} failed", s)))
        };
        let min = segment[0].parse()
            .map_err(|_| err(file!(), line!(), &format!("Parse graphic range {} failed", s)))?;

        let max = segment[1].parse()
            .map_err(|_| err(file!(), line!(), &format!("Parse graphic range {} failed", s)))?;
        
        Ok(GraphicRange { min, max })
    }
}

impl fmt::Display for GraphicRange {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(f, "{} to {}", self.min, self.max)
    }
}

// /// The United States prime interest rate data is daily. To buid a monthly time-series, we read
// /// through raw csv data, calculate a monthly value and add to to the time-series. The data include
// /// missing days, so we need the mechanism to ignore datepoints with value ".".
// 
//     ts.try_into().map_err(|err: time_series::error::Error| {
//         external(
//             file!(),
//             line!(),
//             &err.to_string(),
//         )
//     })
// }