First commit
This commit is contained in:
@@ -0,0 +1,161 @@
|
||||
"""Detect recurring payments (subscriptions, rent, insurance...) and flag missed ones."""
|
||||
from collections import Counter, defaultdict
|
||||
from datetime import date, timedelta
|
||||
from decimal import Decimal
|
||||
from statistics import median
|
||||
|
||||
from dateutil.relativedelta import relativedelta
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from categorize import normalize
|
||||
from models import Category, Recurring, Transaction
|
||||
|
||||
# name, (min gap, max gap) in days, min occurrences, grace days before "missed"
|
||||
FREQUENCIES = {
|
||||
"weekly": ((5, 9), 4, 3),
|
||||
"monthly": ((25, 36), 3, 6),
|
||||
"quarterly": ((80, 100), 3, 14),
|
||||
"yearly": ((350, 380), 2, 21),
|
||||
}
|
||||
|
||||
|
||||
def step(freq: str) -> relativedelta:
|
||||
return {"weekly": relativedelta(weeks=1), "monthly": relativedelta(months=1),
|
||||
"quarterly": relativedelta(months=3), "yearly": relativedelta(years=1)}[freq]
|
||||
|
||||
|
||||
def grace(freq: str) -> int:
|
||||
return FREQUENCIES[freq][2]
|
||||
|
||||
|
||||
def tx_key(tx: Transaction) -> str:
|
||||
return normalize(tx.counterparty) or normalize(tx.description)
|
||||
|
||||
|
||||
def _clusters(txs: list[Transaction], tol: float = 0.25) -> list[list[Transaction]]:
|
||||
"""Split one counterparty's payments into groups of similar amounts."""
|
||||
out: list[list[Transaction]] = []
|
||||
for tx in sorted(txs, key=lambda t: abs(t.amount)):
|
||||
if out:
|
||||
ref = median([abs(t.amount) for t in out[-1]])
|
||||
if ref and abs(abs(tx.amount) - ref) / ref <= Decimal(str(tol)):
|
||||
out[-1].append(tx)
|
||||
continue
|
||||
out.append([tx])
|
||||
return out
|
||||
|
||||
|
||||
def _classify(dates: list[date]) -> str | None:
|
||||
gaps = [(b - a).days for a, b in zip(dates, dates[1:])]
|
||||
if not gaps:
|
||||
return None
|
||||
med = median(gaps)
|
||||
for name, ((lo, hi), min_n, _) in FREQUENCIES.items():
|
||||
if lo <= med <= hi and len(dates) >= min_n:
|
||||
regular = sum(lo <= g <= hi for g in gaps) / len(gaps)
|
||||
if regular >= 0.7:
|
||||
return name
|
||||
return None
|
||||
|
||||
|
||||
def _excluded_category_ids(db: Session) -> set[int]:
|
||||
return set(db.scalars(select(Category.id).where(Category.kind == "transfer")))
|
||||
|
||||
|
||||
def detect(db: Session, today: date | None = None) -> int:
|
||||
"""Scan the last ~13 months of outgoing payments and create/update Recurring rows."""
|
||||
today = today or date.today()
|
||||
excluded = _excluded_category_ids(db)
|
||||
txs = db.scalars(select(Transaction).where(Transaction.amount < 0,
|
||||
Transaction.date >= today - timedelta(days=400)))
|
||||
groups: dict[str, list[Transaction]] = defaultdict(list)
|
||||
for tx in txs:
|
||||
if tx.category_id in excluded:
|
||||
continue
|
||||
k = tx_key(tx)
|
||||
if k:
|
||||
groups[k].append(tx)
|
||||
|
||||
existing = list(db.scalars(select(Recurring)))
|
||||
created = 0
|
||||
for key, items in groups.items():
|
||||
for cl in _clusters(items):
|
||||
# one payment per day is enough to establish the rhythm
|
||||
by_day = {t.date: t for t in cl}
|
||||
dates = sorted(by_day)
|
||||
freq = _classify(dates)
|
||||
if not freq:
|
||||
continue
|
||||
last = dates[-1]
|
||||
if last + step(freq) * 3 + timedelta(days=grace(freq)) < today:
|
||||
continue # stopped more than ~3 cycles ago: probably cancelled
|
||||
amount = Decimal(median([t.amount for t in sorted(cl, key=lambda t: t.date)[-3:]])).quantize(Decimal("0.01"))
|
||||
rec = next((r for r in existing if r.match == key and r.amount
|
||||
and abs(abs(r.amount) - abs(amount)) / abs(r.amount) <= Decimal("0.3")), None)
|
||||
if rec is None:
|
||||
rec = Recurring(name=cl[-1].counterparty[:120] or key, match=key, source="detected",
|
||||
status="detected", amount=amount)
|
||||
db.add(rec)
|
||||
existing.append(rec)
|
||||
created += 1
|
||||
if rec.source == "detected":
|
||||
rec.frequency = freq
|
||||
rec.amount = amount
|
||||
rec.occurrences = len(dates)
|
||||
rec.last_date = max(last, rec.last_date or last)
|
||||
rec.next_date = rec.last_date + step(rec.frequency)
|
||||
rec.account_id = Counter(t.account_id for t in cl).most_common(1)[0][0]
|
||||
cats = [t.category_id for t in cl if t.category_id]
|
||||
rec.category_id = Counter(cats).most_common(1)[0][0] if cats else rec.category_id
|
||||
db.flush()
|
||||
for t in cl:
|
||||
t.recurring_id = rec.id
|
||||
db.flush()
|
||||
refresh(db, today)
|
||||
db.commit()
|
||||
return created
|
||||
|
||||
|
||||
def refresh(db: Session, today: date | None = None) -> None:
|
||||
"""Advance last/next dates for every recurring item using the newest matching payment."""
|
||||
today = today or date.today()
|
||||
for rec in db.scalars(select(Recurring)):
|
||||
since = rec.last_date or (rec.next_date - timedelta(days=grace(rec.frequency) * 2) if rec.next_date else None)
|
||||
q = select(Transaction).where(Transaction.amount < 0 if rec.amount < 0 else Transaction.amount > 0)
|
||||
if since:
|
||||
q = q.where(Transaction.date > since - timedelta(days=0 if rec.last_date else 1))
|
||||
tol = Decimal(rec.tolerance_pct) / 100
|
||||
hits = [t for t in db.scalars(q.order_by(Transaction.date))
|
||||
if rec.match in tx_key(t) or rec.match in normalize(t.description)]
|
||||
hits = [t for t in hits if abs(abs(t.amount) - abs(rec.amount)) <= abs(rec.amount) * max(tol, Decimal("0.5"))]
|
||||
for t in hits:
|
||||
t.recurring_id = rec.id
|
||||
if rec.last_date is None or t.date > rec.last_date:
|
||||
rec.last_date = t.date
|
||||
rec.occurrences = (rec.occurrences or 0) + 1
|
||||
if rec.last_date:
|
||||
nxt = rec.last_date + step(rec.frequency)
|
||||
if rec.next_date is None or nxt > rec.next_date or rec.source == "detected":
|
||||
rec.next_date = nxt
|
||||
db.flush()
|
||||
|
||||
|
||||
def state(rec: Recurring, today: date | None = None) -> str:
|
||||
"""ok | due-soon | missed | ignored"""
|
||||
today = today or date.today()
|
||||
if rec.status == "ignored":
|
||||
return "ignored"
|
||||
if not rec.next_date:
|
||||
return "ok"
|
||||
if today > rec.next_date + timedelta(days=grace(rec.frequency)):
|
||||
return "missed"
|
||||
if rec.next_date - today <= timedelta(days=7):
|
||||
return "due-soon"
|
||||
return "ok"
|
||||
|
||||
|
||||
def monthly_equivalent(rec: Recurring) -> Decimal:
|
||||
f = {"weekly": Decimal("52") / 12, "monthly": Decimal(1), "quarterly": Decimal(1) / 3,
|
||||
"yearly": Decimal(1) / 12}[rec.frequency]
|
||||
return (abs(rec.amount) * f).quantize(Decimal("0.01"))
|
||||
Reference in New Issue
Block a user