Add an importer for the VR-Bank CSV format

korrat
Jun 26, 2023, 1:33 PM
6A5YLGWVK7F6DQWDTSVFRIPFQ5VLIUN3P6DW64QHDDE6RUARDEBQC

Dependencies

  • [2] XQHYMSDY Add importer for Union Investment transactions
  • [3] NQ455YWR Extract balance deduplication logic into shared utility
  • [4] R524JUUE Implement metadata & price directives
  • [*] KB7Y4PJI Implement importers for Amazon accounts
  • [*] QNGOXZL4 Add a basic framework
  • [*] R5K55SCB Move tagging of directives with source to framework runner
  • [*] I2P2FTLE add basic parser for german decimals
  • [*] RI7HQBYA Add generator and parser for ISO20022 messages
  • [*] YDK6X6PP add a library of important types for beancount
  • [*] 5S4MZHL5 pretty print decimals using icu

Change contents

  • file addition: vr-bank (d--r------)
    [6.1]
  • file addition: src (d--r------)
    [0.19]
  • file addition: lib.rs (---r------)
    [0.36]
    extern crate alloc;
    use core::fmt;
    use core::fmt::Display;
    use core::fmt::Formatter;
    use core::ops::Index;
    use core::str::FromStr;
    use beancount_importers_framework::error::ImporterBuilderError;
    use beancount_importers_framework::error::UninitializedFieldSnafu;
    use beancount_types::Account;
    use beancount_types::AccountTemplate;
    use beancount_types::Amount;
    use beancount_types::Balance;
    use beancount_types::Commodity;
    use beancount_types::Directive;
    use beancount_types::MetadataKey;
    use beancount_types::Seg;
    use beancount_types::Transaction;
    use derive_builder::Builder;
    use hashbrown::HashMap;
    use miette::Diagnostic;
    use miette::IntoDiagnostic as _;
    use serde::Deserialize;
    use snafu::Backtrace;
    use snafu::OptionExt as _;
    use snafu::Snafu;
    use time::macros::format_description;
    use time::Date;
    // TODO documentation
    // TODO balance assertions
    // TODO metadata + linking
    // TODO adjust accounts
    #[derive(Builder, Clone, Debug, Deserialize)]
    #[builder(
    build_fn(error = "ImporterBuilderError", skip),
    name = "ImporterBuilder"
    )]
    pub struct Config {
    #[builder(setter(into), try_setter)]
    pub account: AccountTemplate<TemplateSelector>,
    #[builder(setter(into), try_setter)]
    pub fallback_account: Account,
    #[builder(field(type = "HashMap<String, Account>"))]
    pub known_ibans: HashMap<String, Account>,
    }
    #[derive(Debug, Diagnostic, Snafu)]
    pub enum Error {}
    #[derive(Deserialize)]
    pub struct Importer {
    #[serde(flatten)]
    config: Config,
    #[serde(default = "csv::Importer::semicolon_delimited", skip_deserializing)]
    importer: csv::Importer,
    }
    impl fmt::Debug for Importer {
    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
    f.debug_struct("Importer")
    .field("config", &self.config)
    .field("importer", &self.importer)
    .finish_non_exhaustive()
    }
    }
    impl Importer {
    const NAME: &str = "vr-bank/transactions";
    pub fn builder() -> ImporterBuilder {
    ImporterBuilder::default()
    }
    pub fn new(config: Config) -> Self {
    let importer = csv::Importer::semicolon_delimited();
    Self { config, importer }
    }
    }
    impl beancount_importers_framework::ImporterProtocol for Importer {
    type Error = miette::Report;
    fn account(&self, _file: &camino::Utf8Path) -> Result<Account, Self::Error> {
    Ok(self.config.account.base().to_owned())
    }
    fn date(&self, file: &camino::Utf8Path) -> Option<Result<Date, Self::Error>> {
    self.importer
    .date(file, self)
    .map(|result| result.map_err(Self::Error::from))
    }
    fn extract(
    &self,
    file: &camino::Utf8Path,
    existing: &[Directive],
    ) -> Result<Vec<Directive>, Self::Error> {
    self.importer
    .extract(file, existing, self)
    .map_err(Self::Error::from)
    }
    fn filename(&self, _file: &camino::Utf8Path) -> Option<Result<String, Self::Error>> {
    Some(Ok(String::from("transactions.csv")))
    }
    fn identify(&self, file: &camino::Utf8Path) -> Result<bool, Self::Error> {
    const EXPECTED_HEADERS: &[&str] = &[
    "Bezeichnung Auftragskonto",
    "IBAN Auftragskonto",
    "BIC Auftragskonto",
    "Bankname Auftragskonto",
    "Buchungstag",
    "Valutadatum",
    "Name Zahlungsbeteiligter",
    "IBAN Zahlungsbeteiligter",
    "BIC (SWIFT-Code) Zahlungsbeteiligter",
    "Buchungstext",
    "Verwendungszweck",
    "Betrag",
    "Waehrung",
    "Saldo nach Buchung",
    "Bemerkung",
    "Kategorie",
    "Steuerrelevant",
    "Glaeubiger ID",
    "Mandatsreferenz",
    ];
    self.importer
    .identify(file, EXPECTED_HEADERS)
    .into_diagnostic()
    }
    fn name(&self) -> &'static str {
    Self::NAME
    }
    fn typetag_deserialize(&self) {}
    }
    impl csv::RecordImporter for Importer {
    type Error = Error;
    fn date(&self, record: &csv::Record) -> Option<Result<Date, Self::Error>> {
    Some(parse_transaction_date(record))
    }
    fn extract(
    &self,
    _existing: &[Directive],
    record: &csv::Record,
    ) -> Result<Vec<Directive>, Self::Error> {
    let date = parse_transaction_date(record)?;
    let commodity = Commodity::try_from(&record[12]).map_err(|_err| -> Error { todo!() })?;
    // TODO handle transaction
    let transaction = {
    let mut transaction = Transaction::on(date);
    transaction.set_payee(&record[6]).set_narration(&record[10]);
    let context = TemplateContext {};
    let account = self.config.account.render(&context);
    let amount = {
    let amount = &record[11];
    let amount = german_decimal::parse(amount).map_err(|_| -> Error { todo!() })?;
    Amount::new(amount, commodity)
    };
    let opposite_iban = &record[7];
    let opposite_account = self
    .config
    .known_ibans
    .get(opposite_iban)
    .unwrap_or(&self.config.fallback_account);
    transaction
    .add_meta(
    MetadataKey::try_from("other-iban").expect("valid key"),
    opposite_iban,
    )
    .build_posting(account, |posting| {
    posting.set_amount(amount);
    })
    .build_posting(opposite_account, |_posting| {});
    transaction
    };
    // TODO create an interface for matchers that can modify transactions
    let balance = {
    let amount = {
    let amount = &record[13];
    let amount = german_decimal::parse(amount).map_err(|_| -> Error { todo!() })?;
    Amount::new(amount, commodity)
    };
    Balance::new(date.next_day().unwrap(), self.config.account.base(), amount)
    };
    Ok(vec![Directive::from(transaction), Directive::from(balance)])
    }
    }
    impl ImporterBuilder {
    pub fn build(&mut self) -> Result<Importer, ImporterBuilderError> {
    let config = Config {
    account: self.account.clone().context(UninitializedFieldSnafu {
    field: "account",
    importer: Importer::NAME,
    })?,
    fallback_account: self
    .fallback_account
    .clone()
    .context(UninitializedFieldSnafu {
    field: "fallback_account",
    importer: Importer::NAME,
    })?,
    known_ibans: self.known_ibans.clone(),
    };
    Ok(Importer::new(config))
    }
    pub fn clear_known_ibans(&mut self) -> &mut Self {
    self.known_ibans.clear();
    self
    }
    pub fn try_add_known_iban<A>(
    &mut self,
    iban: impl Into<String>,
    account: A,
    ) -> Result<&mut Self, A::Error>
    where
    A: TryInto<Account>,
    {
    self.known_ibans.insert(iban.into(), account.try_into()?);
    Ok(self)
    }
    }
    #[derive(Debug)]
    struct TemplateContext {}
    impl Index<&TemplateSelector> for TemplateContext {
    type Output = Seg;
    fn index(&self, selector: &TemplateSelector) -> &Self::Output {
    match *selector {}
    }
    }
    #[derive(Clone, Copy, Debug)]
    pub enum TemplateSelector {}
    impl FromStr for TemplateSelector {
    type Err = TemplateSelectorError;
    fn from_str(selector: &str) -> Result<Self, Self::Err> {
    TemplateSelectorSnafu { selector }.fail()
    }
    }
    #[derive(Debug, Diagnostic, Snafu)]
    pub struct TemplateSelectorError {
    selector: String,
    backtrace: Backtrace,
    }
    #[derive(Clone, Copy, Debug)]
    enum TransactionKind {
    DepositFee,
    Distribution,
    ForeignFees,
    Purchase,
    Reversal,
    Sale,
    }
    impl Display for TransactionKind {
    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
    f.write_str(match self {
    TransactionKind::DepositFee => "Sale To Cover Deposit Fees",
    TransactionKind::Distribution => "Reinvestment of Distribution",
    TransactionKind::ForeignFees => "Sale To Cover Foreign Fees",
    TransactionKind::Purchase => "Purchase",
    TransactionKind::Reversal => "Reversal of Purchase",
    TransactionKind::Sale => "Sale",
    })
    }
    }
    impl FromStr for TransactionKind {
    type Err = Error;
    fn from_str(s: &str) -> Result<Self, Self::Err> {
    let value = match s {
    "Ertragsausschüttung" => Self::Distribution,
    "Fremde Gebühren" => Self::ForeignFees,
    "Kauf" => Self::Purchase,
    "Storno wegen Rücklastschrift" => Self::Reversal,
    "Verkauf" => Self::Sale,
    "Verkauf wg. Depotgebühr UID mit Postbox" => Self::DepositFee,
    other => todo!("unsupported transaction kind {other:?}"),
    };
    Ok(value)
    }
    }
    fn parse_transaction_date(record: &csv::Record) -> Result<Date, Error> {
    use time::format_description::FormatItem;
    const DATE_COLUMN_INDEX: usize = 5;
    const DATE_FORMAT: &[FormatItem] = format_description!("[day].[month].[year]");
    Date::parse(&record[DATE_COLUMN_INDEX], DATE_FORMAT).map_err(|_| todo!())
    }
  • file addition: Cargo.toml (---r------)
    [0.19]
    [package]
    name = "vr-bank"
    edition.workspace = true
    publish.workspace = true
    rust-version.workspace = true
    version.workspace = true
    [dependencies]
    # Workspace dependencies
    beancount-importers-framework.path = "../../framework"
    beancount-types.path = "../../common/beancount-types"
    csv.path = "../csv"
    german-decimal.path = "../../common/german-decimal"
    # Inherited dependencies
    camino.workspace = true
    derive_builder.workspace = true
    hashbrown.workspace = true
    miette.workspace = true
    serde.workspace = true
    snafu.workspace = true
    time.workspace = true
  • edit in framework/src/utilities.rs at line 1
    [3.1]
    [3.2]
    use core::cell::RefCell;
  • replacement in framework/src/utilities.rs at line 3
    [3.36][3.36:63]()
    use core::hash::Hash as _;
    [3.36]
    [3.63]
    use core::hash::Hash;
  • edit in framework/src/utilities.rs at line 12
    [3.272]
    [3.272]
    use beancount_types::Transaction;
  • edit in framework/src/utilities.rs at line 18
    [3.396]
    [3.396]
    use xxhash_rust::xxh3::Xxh3;
  • edit in framework/src/utilities.rs at line 21
    [3.426]
    [3.426]
    pub struct ApplyTransactionHook<I, F>
    where
    I: ImporterProtocol,
    F: FnMut(&mut Transaction),
    {
    inner: I,
    hook: RefCell<F>,
    }
    impl<I, F> ApplyTransactionHook<I, F>
    where
    I: ImporterProtocol,
    F: FnMut(&mut Transaction),
    {
    pub fn new(inner: I, hook: F) -> Self {
    let hook = RefCell::new(hook);
    Self { inner, hook }
    }
    }
    impl<I, F> ImporterProtocol for ApplyTransactionHook<I, F>
    where
    I: ImporterProtocol,
    F: FnMut(&mut Transaction),
    {
    type Error = I::Error;
    delegate! {
    to (self.inner) {
    fn account(&self, file: &Utf8Path) -> Result<Account, Self::Error>;
    fn date(&self, file: &Utf8Path) -> Option<Result<Date, Self::Error>>;
    fn filename(&self, file: &Utf8Path) -> Option<Result<String, Self::Error>>;
    fn identify(&self, file: &Utf8Path) -> Result<bool, Self::Error>;
    fn name(&self) -> &'static str;
    fn typetag_deserialize(&self);
    }
    }
    fn extract(
    &self,
    file: &camino::Utf8Path,
    existing: &[Directive],
    ) -> Result<Vec<Directive>, Self::Error> {
    let mut directives = self.inner.extract(file, existing)?;
    let mut hook = self.hook.borrow_mut();
    for directive in &mut directives {
    let Directive::Transaction(transaction) = directive else {
    continue;
    };
  • edit in framework/src/utilities.rs at line 74
    [3.427]
    [3.427]
    hook(transaction)
    }
    Ok(directives)
    }
    }
  • replacement in framework/src/utilities.rs at line 85
    [3.510][3.510:537]()
    F: Fn(&Balance) -> &T,
    [3.510]
    [3.537]
    F: Fn(&Balance) -> T,
  • replacement in framework/src/utilities.rs at line 96
    [3.655][3.655:682]()
    F: Fn(&Balance) -> &T,
    [3.655]
    [3.682]
    F: Fn(&Balance) -> T,
  • replacement in framework/src/utilities.rs at line 107
    [3.850][3.850:877]()
    F: Fn(&Balance) -> &T,
    [3.850]
    [3.877]
    F: Fn(&Balance) -> T,
  • replacement in framework/src/utilities.rs at line 109
    [3.879][3.879:934]()
    fn key<'b>(&self, balance: &'b Balance) -> &'b T {
    [3.879]
    [3.934]
    fn key(&self, balance: &Balance) -> T {
  • replacement in framework/src/utilities.rs at line 151
    [3.1985][3.1985:2012]()
    F: Fn(&Balance) -> &T,
    [3.1985]
    [3.2012]
    F: Fn(&Balance) -> T,
  • replacement in framework/src/utilities.rs at line 199
    [3.3468][3.3468:3499]()
    F: Fn(&Balance) -> &T,
    [3.3468]
    [3.3499]
    F: Fn(&Balance) -> T,
  • edit in framework/src/utilities.rs at line 203
    [3.3555]
    [3.3555]
    fn apply_transaction_hook<F>(self, hook: F) -> ApplyTransactionHook<Self, F>
    where
    Self: Sized + ImporterProtocol,
    F: FnMut(&mut Transaction),
    {
    ApplyTransactionHook::new(self, hook)
    }
  • edit in framework/src/utilities.rs at line 215
    [3.3630]
    [3.3630]
    pub fn hash<T: Hash>(data: T) -> u128 {
    let mut hasher = Xxh3::new();
    data.hash(&mut hasher);
    hasher.digest128()
    }
  • edit in framework/Cargo.toml at line 27
    [8.2395]
    xxhash-rust.workspace = true
  • edit in common/beancount-types/src/metadata/kv.rs at line 159
    [4.7506]
    [4.7506]
  • replacement in common/beancount-types/src/metadata/kv.rs at line 162
    [4.7535][4.7535:7584]()
    pub fn as_number(&self) -> Option<Decimal> {
    [4.7535]
    [4.7584]
    pub const fn as_number(&self) -> Option<Decimal> {
  • replacement in common/beancount-types/src/metadata/kv.rs at line 183
    [4.7995][4.7995:8105]()
    Value::Number(inner) => inner.fmt(f),
    Value::String(inner) => write!(f, "{inner:?}"),
    [4.7995]
    [4.8105]
    Self::Number(inner) => inner.fmt(f),
    Self::String(inner) => write!(f, "{inner:?}"),
  • edit in common/beancount-types/src/metadata/kv.rs at line 231
    [4.8893]
    [4.8893]
    impl PartialEq<&str> for Value {
    fn eq(&self, other: &&str) -> bool {
    self.as_str().map_or(false, |value| value == *other)
    }
    }
    impl PartialEq<Value> for &str {
    fn eq(&self, other: &Value) -> bool {
    other == self
    }
    }
  • edit in Cargo.toml at line 16
    [2.15340]
    [10.91793541]
    "importers/vr-bank",
  • edit in Cargo.lock at line 270
    [7.15471]
    [7.15471]
    "xxhash-rust",
  • edit in Cargo.lock at line 4382
    [11.38045]
    [12.55499]
    [[package]]
    name = "vr-bank"
    version = "0.0.0-dev.0"
    dependencies = [
    "beancount-importers-framework",
    "beancount-types",
    "camino",
    "csv 0.0.0-dev.0",
    "derive_builder",
    "german-decimal",
    "hashbrown 0.14.0",
    "miette",
    "serde",
    "snafu",
    "time 0.3.22",
    ]