"""News sources for the filter. Two kinds of headlines are gathered for each asset: - asset news: about the company or coin itself (Google News, and Finnhub if you add a key); - macro news: central-bank announcements that move whole markets (ECB for euro markets, the Federal Reserve for US markets and crypto). Every source is optional and isolated: if one fails, the others still count, and the failure is shown on the dashboard instead of stopping the bot. Why not Reuters, Bloomberg or Dow Jones directly? Their real-time feeds are licensed products that cost thousands per month and forbid redistribution, which is far out of proportion for a bot that acts on hourly candles. The sources below are free. """ import time import urllib.parse from calendar import timegm from dataclasses import dataclass from datetime import datetime, timedelta, timezone import feedparser import httpx from .config import Instrument, settings MAX_AGE = 3 * 86400 # ignore anything older than 3 days HEADERS = {"User-Agent": "Yote-In-Vest/1.0 (+personal research tool)"} LABELS = {"google": "Google News", "ecb": "European Central Bank", "fed": "Federal Reserve", "finnhub": "Finnhub"} # Which central bank matters for which exchange. London has none for now (no Bank of England feed). MACRO_FOR = {"AEX": ["ecb"], "EPA": ["ecb"], "XETRA": ["ecb"], "LSE": [], "US": ["fed"], "CRYPTO": ["fed", "ecb"]} INFO = { "google": ("asset", "Headlines about each company or coin, from news sites worldwide. Free, no key."), "ecb": ("macro", "ECB press releases and speeches. Used for Euronext, Xetra and crypto."), "fed": ("macro", "Federal Reserve press releases. Used for US stocks and crypto."), "finnhub": ("asset", "Company news for US stocks and a crypto news feed. Needs a free API key."), } @dataclass class Item: source: str title: str published: str # "YYYY-MM-DD HH:MM" (UTC) or "" kind: str # "asset" or "macro" def line(i: Item) -> str: tag = i.source if i.kind == "asset" else f"macro ยท {i.source}" title = i.title.replace("<", "").replace(">", "")[:220] # no markup inside the prompt return f"{i.published} | [{tag}] {title}" def _fresh(ts: float | None, now: float) -> bool: return ts is None or now - ts <= MAX_AGE def _fmt(ts: float | None) -> str: return datetime.fromtimestamp(ts, timezone.utc).strftime("%Y-%m-%d %H:%M") if ts else "" def _rss(url: str, source: str, kind: str, limit: int, now: float) -> list[Item]: resp = httpx.get(url, headers=HEADERS, timeout=10, follow_redirects=True) resp.raise_for_status() out = [] for e in feedparser.parse(resp.text).entries: t = e.get("published_parsed") or e.get("updated_parsed") ts = timegm(t) if t else None if e.get("title") and _fresh(ts, now): out.append(Item(source, e.title.strip(), _fmt(ts), kind)) if len(out) >= limit: break return out def google(query: str, now: float) -> list[Item]: url = ("https://news.google.com/rss/search?q=" + urllib.parse.quote(f"{query} when:3d") + "&hl=en&gl=NL&ceid=NL:en") return _rss(url, LABELS["google"], "asset", 10, now) def macro(key: str, now: float) -> list[Item]: url = {"ecb": settings.ecb_feed_url, "fed": settings.fed_feed_url}[key] return _rss(url, LABELS[key], "macro", 3, now) def finnhub(inst: Instrument, now: float) -> list[Item]: key = settings.finnhub_key base = "https://finnhub.io/api/v1" if inst.asset_class == "crypto": r = httpx.get(f"{base}/news", params={"category": "crypto", "token": key}, headers=HEADERS, timeout=10) r.raise_for_status() words = {inst.tv_symbol[:3].lower(), inst.news_query.split()[0].lower()} # "btc", "bitcoin" rows = [x for x in r.json() if any(w in (x.get("headline", "") + x.get("summary", "")).lower() for w in words)] elif inst.exchange == "US": today = datetime.now(timezone.utc).date() r = httpx.get(f"{base}/company-news", headers=HEADERS, timeout=10, params={ "symbol": inst.tv_symbol, "from": (today - timedelta(days=3)).isoformat(), "to": today.isoformat(), "token": key}) r.raise_for_status() rows = r.json() else: return [] # the free plan covers US companies only out = [] for x in rows: ts = x.get("datetime") if x.get("headline") and _fresh(ts, now): out.append(Item(LABELS["finnhub"], x["headline"].strip(), _fmt(ts), "asset")) out.sort(key=lambda i: i.published, reverse=True) return out[:6] def gather(query: str, inst: Instrument | None = None, now: float | None = None): """Collect headlines. Returns (items newest-first, {source: count}, [error messages]).""" now = time.time() if now is None else now enabled = settings.news_sources items: list[Item] = [] counts: dict[str, int] = {} errors: list[str] = [] def run(key, fn): if key not in enabled: return try: got = fn() counts[LABELS[key]] = len(got) items.extend(got) except Exception as e: counts[LABELS[key]] = 0 errors.append(f"{LABELS[key]}: {e}"[:140]) run("google", lambda: google(query, now)) if inst is not None: for m in MACRO_FOR.get(inst.exchange, []): run(m, lambda m=m: macro(m, now)) # Finnhub's free plan covers US companies and crypto only; don't ask (or list it) elsewhere. if settings.finnhub_key and (inst.exchange == "US" or inst.asset_class == "crypto"): run("finnhub", lambda: finnhub(inst, now)) items.sort(key=lambda i: i.published, reverse=True) return items, counts, errors def status() -> list[dict]: """Which sources are on, for the dashboard.""" out = [] for key, (kind, text) in INFO.items(): configured = key != "finnhub" or bool(settings.finnhub_key) out.append({"key": key, "name": LABELS[key], "kind": kind, "about": text, "enabled": key in settings.news_sources and configured, "needs_key": key == "finnhub", "configured": configured}) return out