korrat/beancount-importers - Change XQHYMSDYDG3MFDUEGXADTNXA4B7AQRSYA5JCP3S3KQYL527DORHQC

Add importer for Union Investment transactions

Created by korrat on June 22, 2023

XQHYMSDYDG3MFDUEGXADTNXA4B7AQRSYA5JCP3S3KQYL527DORHQC

Dependencies

In channels

main

Change contents

File addition: uniondepot (d--x------)
[4.1]
File addition: src (d--x------)
[0.1]

File addition: lib.rs (---x------)

[0.25]

use core::fmt;
use core::fmt::Display;
use core::fmt::Formatter;
use core::hash::Hash;
use core::ops::Index;
use core::str::FromStr;
use std::collections::HashMap;
use beancount_importers_framework::error::ImporterBuilderError;
use beancount_importers_framework::error::UninitializedFieldSnafu;
use beancount_types::Account;
use beancount_types::AccountTemplate;
use beancount_types::Amount;
use beancount_types::Commodity;
use beancount_types::CostBasis;
use beancount_types::Directive;
use beancount_types::Link;
use beancount_types::Seg;
use beancount_types::Transaction;
use derive_builder::Builder;
use miette::Diagnostic;
use miette::IntoDiagnostic as _;
use serde::Deserialize;
use snafu::Backtrace;
use snafu::OptionExt as _;
use snafu::Snafu;
use time::format_description::well_known::Iso8601;
use time::Date;
use xxhash_rust::xxh3::Xxh3;
// TODO documentation
// TODO balance assertions
// TODO metadata + linking
// TODO adjust accounts
#[derive(Builder, Clone, Debug, Deserialize)]
#[builder(
    build_fn(error = "ImporterBuilderError", skip),
    name = "ImporterBuilder"
)]
pub struct Config {
    #[builder(setter(into), try_setter)]
    pub base_account: AccountTemplate<TemplateSelector>,
    #[builder(setter(into), try_setter)]
    pub capital_gains_account: AccountTemplate<TemplateSelector>,
    #[builder(field(type = "HashMap<String, Commodity>"), setter(into))]
    #[serde(default)]
    pub currencies: HashMap<String, Commodity>,
    #[builder(setter(into), try_setter)]
    pub distributions_account: AccountTemplate<TemplateSelector>,
    #[builder(setter(into), try_setter)]
    pub fee_account: AccountTemplate<TemplateSelector>,
    #[builder(setter(into))]
    pub payee: String,
    #[builder(field(type = "HashMap<String, Commodity>"), setter(into))]
    #[serde(default)]
    pub positions: HashMap<String, Commodity>,
    #[builder(setter(into), try_setter)]
    pub reference_account: AccountTemplate<TemplateSelector>,
}
#[derive(Debug, Diagnostic, Snafu)]
pub enum Error {}
#[derive(Debug, Deserialize)]
pub struct Importer {
    #[serde(flatten)]
    config: Config,
    #[serde(default = "csv::Importer::semicolon_delimited", skip_deserializing)]
    importer: csv::Importer,
}
impl Importer {
    const NAME: &str = "uniondepot/transactions";
    pub fn builder() -> ImporterBuilder {
        ImporterBuilder::default()
    }
    pub fn new(config: Config) -> Self {
        let importer = csv::Importer::semicolon_delimited();
        Self { config, importer }
    }
}
impl Importer {
    fn lookup_currency(&self, name: &str) -> Result<Commodity, Error> {
        self.config
            .currencies
            .get(name)
            .copied()
            .ok_or_else(|| todo!())
    }
    fn lookup_commodity(&self, position: &str) -> Result<Commodity, Error> {
        self.config
            .positions
            .get(position)
            .copied()
            .ok_or_else(|| todo!())
    }
}
impl beancount_importers_framework::ImporterProtocol for Importer {
    type Error = miette::Report;
    fn account(&self, _file: &camino::Utf8Path) -> Result<Account, Self::Error> {
        Ok(self.config.base_account.base().to_owned())
    }
    fn date(&self, file: &camino::Utf8Path) -> Option<Result<Date, Self::Error>> {
        self.importer
            .date(file, self)
            .map(|result| result.map_err(Self::Error::from))
    }
    fn extract(
        &self,
        file: &camino::Utf8Path,
        existing: &[Directive],
    ) -> Result<Vec<Directive>, Self::Error> {
        self.importer
            .extract(file, existing, self)
            .map_err(Self::Error::from)
    }
    fn filename(&self, _file: &camino::Utf8Path) -> Option<Result<String, Self::Error>> {
        Some(Ok(String::from("transactions.csv")))
    }
    fn identify(&self, file: &camino::Utf8Path) -> Result<bool, Self::Error> {
        const EXPECTED_HEADERS: &[&str] = &[
            "DATUM",
            "UNIONDEPOTNUMMER",
            "FONDS/PRODUKT",
            "TRANSAKTIONSART",
            "STATUS",
            "FONDSPREIS",
            "ANTEILE",
            "VOLUMEN",
            "EINHEIT",
        ];
        self.importer
            .identify(file, EXPECTED_HEADERS)
            .into_diagnostic()
    }
    fn name(&self) -> &'static str {
        Self::NAME
    }
    fn typetag_deserialize(&self) {}
}
impl csv::RecordImporter for Importer {
    type Error = Error;
    fn date(&self, record: &csv::Record) -> Option<Result<Date, Self::Error>> {
        Some(parse_order_date(record))
    }
    fn extract(
        &self,
        _existing: &[Directive],
        record: &csv::Record,
    ) -> Result<Vec<Directive>, Self::Error> {
        const DEPOT_ID_COLUMN_INDEX: usize = 1;
        const PRICE_COLUMN_INDEX: usize = 5;
        const TOTAL_COLUMN_INDEX: usize = 7;
        const TRANSACTION_KIND_COLUMN_INDEX: usize = 3;
        const UNIT_COLUMN_INDEX: usize = 8;
        const UNITS_COLUMN_INDEX: usize = 6;
        let context = TemplateContext::try_from(&record[DEPOT_ID_COLUMN_INDEX])?;
        let account = self.config.base_account.render(&context);
        let stock = self.lookup_commodity(context.position).unwrap();
        let date = parse_order_date(record)?;
        let currency: Commodity = self.lookup_currency(&record[UNIT_COLUMN_INDEX])?;
        let units = {
            let value: &str = &record[UNITS_COLUMN_INDEX];
            german_decimal::parse(value).map_err(|_| todo!())
        }?;
        let unit_cost = {
            let price = &record[PRICE_COLUMN_INDEX];
            german_decimal::parse(price).map_err(|_| todo!())
        }?;
        let total = {
            let value: &str = &record[TOTAL_COLUMN_INDEX];
            german_decimal::parse(value).map_err(|_| todo!())
        }?;
        let total = Amount {
            amount: total,
            commodity: currency,
        };
        let transaction_kind = TransactionKind::from_str(&record[TRANSACTION_KIND_COLUMN_INDEX])?;
        let opposite_account = match transaction_kind {
            TransactionKind::DepositFee | TransactionKind::ForeignFees => {
                self.config.fee_account.render(&context)
            }
            TransactionKind::Distribution => self.config.distributions_account.render(&context),
            TransactionKind::Purchase | TransactionKind::Reversal | TransactionKind::Sale => {
                self.config.reference_account.render(&context)
            }
        };
        let (cost_basis, price) = match transaction_kind {
            TransactionKind::Distribution
            | TransactionKind::Purchase
            | TransactionKind::Reversal => (
                CostBasis::PerUnit(Amount {
                    amount: unit_cost,
                    commodity: currency,
                }),
                None,
            ),
            TransactionKind::DepositFee | TransactionKind::ForeignFees | TransactionKind::Sale => (
                CostBasis::Empty,
                Some(Amount {
                    amount: unit_cost,
                    commodity: currency,
                }),
            ),
        };
        let mut transaction = Transaction::on(date);
        transaction
            .set_payee(&self.config.payee)
            .set_narration(transaction_kind.to_string())
            .build_posting(account, |posting| {
                posting
                    .set_amount(Amount::new(units, stock))
                    .set_cost(cost_basis);
                posting.price = price;
            })
            .build_posting(opposite_account, |posting| {
                posting.set_amount(-total);
            });
        if matches!(
            transaction_kind,
            TransactionKind::Purchase | TransactionKind::Sale
        ) {
            transaction.add_link(
                Link::try_from(format!("^union-investment.{:X}", hash(date, &context))).unwrap(),
            );
        }
        if matches!(
            transaction_kind,
            TransactionKind::DepositFee | TransactionKind::ForeignFees | TransactionKind::Sale
        ) {
            // Third leg for capital gains
            transaction.build_posting(
                self.config.capital_gains_account.render(&context),
                |_posting| {},
            );
        }
        Ok(vec![Directive::from(transaction)])
    }
}
impl ImporterBuilder {
    pub fn clear_currencies(&mut self) -> &mut Self {
        self.currencies.clear();
        self
    }
    pub fn clear_positions(&mut self) -> &mut Self {
        self.positions.clear();
        self
    }
    pub fn try_add_currency<C>(
        &mut self,
        name: impl Into<String>,
        commodity: C,
    ) -> Result<&mut Self, C::Error>
    where
        C: TryInto<Commodity>,
    {
        self.currencies.insert(name.into(), commodity.try_into()?);
        Ok(self)
    }
    pub fn try_add_position<C>(
        &mut self,
        position: impl Into<String>,
        commodity: C,
    ) -> Result<&mut Self, C::Error>
    where
        C: TryInto<Commodity>,
    {
        self.positions
            .insert(position.into(), commodity.try_into()?);
        Ok(self)
    }
}
impl ImporterBuilder {
    pub fn build(&self) -> Result<Importer, ImporterBuilderError> {
        let config = Config {
            base_account: self.base_account.clone().context(UninitializedFieldSnafu {
                field: "base_account",
                importer: Importer::NAME,
            })?,
            capital_gains_account: self.capital_gains_account.clone().context(
                UninitializedFieldSnafu {
                    field: "capital_gains_account",
                    importer: Importer::NAME,
                },
            )?,
            distributions_account: self.distributions_account.clone().context(
                UninitializedFieldSnafu {
                    field: "distributions_account",
                    importer: Importer::NAME,
                },
            )?,
            fee_account: self.fee_account.clone().context(UninitializedFieldSnafu {
                field: "fee_account",
                importer: Importer::NAME,
            })?,
            payee: self.payee.clone().context(UninitializedFieldSnafu {
                field: "payee",
                importer: Importer::NAME,
            })?,
            positions: self.positions.clone(),
            reference_account: self
                .reference_account
                .clone()
                .context(UninitializedFieldSnafu {
                    field: "reference_account",
                    importer: Importer::NAME,
                })?,
            currencies: self.currencies.clone(),
        };
        Ok(Importer::new(config))
    }
}
#[derive(Debug)]
struct TemplateContext<'i> {
    depot: &'i Seg,
    position: &'i Seg,
}
impl Index<&TemplateSelector> for TemplateContext<'_> {
    type Output = Seg;
    fn index(&self, selector: &TemplateSelector) -> &Self::Output {
        match selector {
            TemplateSelector::Depot => self.depot,
            TemplateSelector::Position => self.position,
        }
    }
}
impl<'i> TryFrom<&'i str> for TemplateContext<'i> {
    type Error = Error;
    fn try_from(value: &'i str) -> Result<Self, Self::Error> {
        let (depot, position) = value.split_once('/').ok_or_else(|| todo!())?;
        let depot = <&Seg>::try_from(depot).map_err(|_| todo!())?;
        let position = <&Seg>::try_from(position).map_err(|_| todo!())?;
        Ok(Self { depot, position })
    }
}
#[derive(Clone, Copy, Debug)]
pub enum TemplateSelector {
    Depot,
    Position,
}
impl FromStr for TemplateSelector {
    type Err = TemplateSelectorError;
    fn from_str(selector: &str) -> Result<Self, Self::Err> {
        match selector {
            "depot" => Ok(Self::Depot),
            "position" => Ok(Self::Position),
            selector => TemplateSelectorSnafu { selector }.fail(),
        }
    }
}
#[derive(Debug, Diagnostic, Snafu)]
pub struct TemplateSelectorError {
    selector: String,
    backtrace: Backtrace,
}
#[derive(Clone, Copy, Debug)]
enum TransactionKind {
    DepositFee,
    Distribution,
    ForeignFees,
    Purchase,
    Reversal,
    Sale,
}
impl Display for TransactionKind {
    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
        f.write_str(match self {
            TransactionKind::DepositFee => "Sale To Cover Deposit Fees",
            TransactionKind::Distribution => "Reinvestment of Distribution",
            TransactionKind::ForeignFees => "Sale To Cover Foreign Fees",
            TransactionKind::Purchase => "Purchase",
            TransactionKind::Reversal => "Reversal of Purchase",
            TransactionKind::Sale => "Sale",
        })
    }
}
impl FromStr for TransactionKind {
    type Err = Error;
    fn from_str(s: &str) -> Result<Self, Self::Err> {
        let value = match s {
            "Ertragsausschüttung" => Self::Distribution,
            "Fremde Gebühren" => Self::ForeignFees,
            "Kauf" => Self::Purchase,
            "Storno wegen Rücklastschrift" => Self::Reversal,
            "Verkauf" => Self::Sale,
            "Verkauf wg. Depotgebühr UID mit Postbox" => Self::DepositFee,
            other => todo!("unsupported transaction kind {other:?}"),
        };
        Ok(value)
    }
}
fn parse_order_date(record: &csv::Record) -> Result<Date, Error> {
    const ORDER_DATE_COLUMN_INDEX: usize = 0;
    Date::parse(&record[ORDER_DATE_COLUMN_INDEX], &Iso8601::PARSING).map_err(|_| todo!())
}
fn hash(date: Date, context: &TemplateContext) -> u128 {
    let mut hasher = Xxh3::new();
    (context.depot, context.position, date.year(), date.month()).hash(&mut hasher);
    hasher.digest128()
}

File addition: Cargo.toml (---x------)

[0.1]

[package]
name = "uniondepot"
edition.workspace      = true
publish.workspace      = true
rust-version.workspace = true
version.workspace      = true
[dependencies]
# Workspace dependencies
beancount-importers-framework.path = "../../framework"
beancount-types.path               = "../../common/beancount-types"
csv.path                           = "../csv"
german-decimal.path                = "../../common/german-decimal"
# Inherited dependencies
camino.workspace         = true
derive_builder.workspace = true
miette.workspace         = true
serde.workspace          = true
snafu.workspace          = true
time.workspace           = true
xxhash-rust.workspace    = true

Insertion in common/beancount-types/src/commodity.rs at line 10 [5.8128]
[5.8156]
[2.0]
```
use miette::Diagnostic;
```
Replacement in common/beancount-types/src/commodity.rs at line 112 [5.8128]
∅:D[6.6200] → [5.9091:9115]
B:BD[5.9091] → [5.9091:9115]
```
#[derive(Debug, Snafu)]
```
[6.6200]
[5.9115]
```
#[derive(Debug, Diagnostic, Snafu)]
```
Replacement in common/beancount-types/Cargo.toml at line 4 [5.25786]
B:BD[5.25864] → [7.145:170]
B:BD[7.170] → [6.16351:16376]
```
edition.workspace = true
publish.workspace = true
```
[5.25864]
[7.170]
```
edition.workspace      = true
publish.workspace      = true
```
Replacement in common/beancount-types/Cargo.toml at line 7 [5.25786]
B:BD[7.200] → [7.200:225]
```
version.workspace = true
```
[7.200]
[5.25961]
```
version.workspace      = true
```

Replacement in common/beancount-types/Cargo.toml at line 11 [5.25786]

B:BD[7.251] → [7.251:331]

B:BD[7.331] → [8.11794:11818]

∅:D[8.11818] → [6.16377:16399]

B:BD[7.331] → [6.16377:16399]

∅:D[6.16399] → [7.331:381]

B:BD[7.331] → [7.331:381]

arrayvec.workspace = true
delegate.workspace = true
lazy-regex.workspace = true
miette.workspace = true
momo.workspace = true
once_cell.workspace = true
regex.workspace = true

[7.251]

[7.381]

arrayvec.workspace     = true
delegate.workspace     = true
lazy-regex.workspace   = true
miette.workspace       = true
momo.workspace         = true
once_cell.workspace    = true
regex.workspace        = true

Replacement in common/beancount-types/Cargo.toml at line 19 [5.25786]
B:BD[7.411] → [9.752:775]
∅:D[9.775] → [7.411:434]
B:BD[7.411] → [7.411:434]
B:BD[7.434] → [10.7807:7828]
∅:D[10.7828] → [7.434:456]
B:BD[7.434] → [7.434:456]
B:BD[7.456] → [11.1573:1598]
```
serde.workspace = true
snafu.workspace = true
tap.workspace = true
time.workspace = true
time-tz.workspace = true
```
[7.411]
[5.26127]
```
serde.workspace        = true
snafu.workspace        = true
tap.workspace          = true
time-tz.workspace      = true
time.workspace         = true
```
Replacement in common/beancount-types/Cargo.toml at line 30 [5.25786]
B:BD[7.482] → [7.482:505]
```
insta.workspace = true
```
[7.482]
[7.505]
```
insta.workspace             = true
```
Replacement in common/beancount-types/Cargo.toml at line 32 [5.25786]
B:BD[7.540] → [7.540:567]
```
test-case.workspace = true
```
[7.540]
```
test-case.workspace         = true
```
Insertion in Cargo.toml at line 15 [13.9831]
[3.15727]
[12.91793541]
```
    "importers/uniondepot",
```

Insertion in Cargo.lock at line 4259 [13.9916]

[8.25985]

[14.54299]

name = "uniondepot"
version = "0.0.0-dev.0"
dependencies = [
 "beancount-importers-framework",
 "beancount-types",
 "camino",
 "csv 0.0.0-dev.0",
 "derive_builder",
 "german-decimal",
 "miette",
 "serde",
 "snafu",
 "time 0.3.22",
 "xxhash-rust",
]
[[package]]

Add importer for Union Investment transactions

Dependencies

In channels

Change contents

File addition: uniondepot (d--x------)

File addition: src (d--x------)

File addition: lib.rs (---x------)

File addition: Cargo.toml (---x------)

Insertion in common/beancount-types/src/commodity.rs at line 10 [5.8128]

Replacement in common/beancount-types/src/commodity.rs at line 112 [5.8128]

Replacement in common/beancount-types/Cargo.toml at line 4 [5.25786]

Replacement in common/beancount-types/Cargo.toml at line 7 [5.25786]

Replacement in common/beancount-types/Cargo.toml at line 11 [5.25786]

Replacement in common/beancount-types/Cargo.toml at line 19 [5.25786]

Replacement in common/beancount-types/Cargo.toml at line 30 [5.25786]

Replacement in common/beancount-types/Cargo.toml at line 32 [5.25786]

Insertion in Cargo.toml at line 15 [13.9831]

Insertion in Cargo.lock at line 4259 [13.9916]