158 lines
6.2 KiB
Python
158 lines
6.2 KiB
Python
"""News sources for the filter.
|
|
|
|
Two kinds of headlines are gathered for each asset:
|
|
- asset news: about the company or coin itself (Google News, and Finnhub if you add a key);
|
|
- macro news: central-bank announcements that move whole markets (ECB for euro markets,
|
|
the Federal Reserve for US markets and crypto).
|
|
|
|
Every source is optional and isolated: if one fails, the others still count, and the
|
|
failure is shown on the dashboard instead of stopping the bot.
|
|
|
|
Why not Reuters, Bloomberg or Dow Jones directly? Their real-time feeds are licensed
|
|
products that cost thousands per month and forbid redistribution, which is far out of
|
|
proportion for a bot that acts on hourly candles. The sources below are free.
|
|
"""
|
|
import time
|
|
import urllib.parse
|
|
from calendar import timegm
|
|
from dataclasses import dataclass
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
import feedparser
|
|
import httpx
|
|
|
|
from .config import Instrument, settings
|
|
|
|
MAX_AGE = 3 * 86400 # ignore anything older than 3 days
|
|
HEADERS = {"User-Agent": "Yote-In-Vest/1.0 (+personal research tool)"}
|
|
|
|
LABELS = {"google": "Google News", "ecb": "European Central Bank", "fed": "Federal Reserve",
|
|
"finnhub": "Finnhub"}
|
|
|
|
# Which central bank matters for which exchange. London has none for now (no Bank of England feed).
|
|
MACRO_FOR = {"AEX": ["ecb"], "EPA": ["ecb"], "XETRA": ["ecb"], "LSE": [],
|
|
"US": ["fed"], "CRYPTO": ["fed", "ecb"]}
|
|
|
|
INFO = {
|
|
"google": ("asset", "Headlines about each company or coin, from news sites worldwide. Free, no key."),
|
|
"ecb": ("macro", "ECB press releases and speeches. Used for Euronext, Xetra and crypto."),
|
|
"fed": ("macro", "Federal Reserve press releases. Used for US stocks and crypto."),
|
|
"finnhub": ("asset", "Company news for US stocks and a crypto news feed. Needs a free API key."),
|
|
}
|
|
|
|
|
|
@dataclass
|
|
class Item:
|
|
source: str
|
|
title: str
|
|
published: str # "YYYY-MM-DD HH:MM" (UTC) or ""
|
|
kind: str # "asset" or "macro"
|
|
|
|
|
|
def line(i: Item) -> str:
|
|
tag = i.source if i.kind == "asset" else f"macro · {i.source}"
|
|
title = i.title.replace("<", "").replace(">", "")[:220] # no markup inside the prompt
|
|
return f"{i.published} | [{tag}] {title}"
|
|
|
|
|
|
def _fresh(ts: float | None, now: float) -> bool:
|
|
return ts is None or now - ts <= MAX_AGE
|
|
|
|
|
|
def _fmt(ts: float | None) -> str:
|
|
return datetime.fromtimestamp(ts, timezone.utc).strftime("%Y-%m-%d %H:%M") if ts else ""
|
|
|
|
|
|
def _rss(url: str, source: str, kind: str, limit: int, now: float) -> list[Item]:
|
|
resp = httpx.get(url, headers=HEADERS, timeout=10, follow_redirects=True)
|
|
resp.raise_for_status()
|
|
out = []
|
|
for e in feedparser.parse(resp.text).entries:
|
|
t = e.get("published_parsed") or e.get("updated_parsed")
|
|
ts = timegm(t) if t else None
|
|
if e.get("title") and _fresh(ts, now):
|
|
out.append(Item(source, e.title.strip(), _fmt(ts), kind))
|
|
if len(out) >= limit:
|
|
break
|
|
return out
|
|
|
|
|
|
def google(query: str, now: float) -> list[Item]:
|
|
url = ("https://news.google.com/rss/search?q=" + urllib.parse.quote(f"{query} when:3d")
|
|
+ "&hl=en&gl=NL&ceid=NL:en")
|
|
return _rss(url, LABELS["google"], "asset", 10, now)
|
|
|
|
|
|
def macro(key: str, now: float) -> list[Item]:
|
|
url = {"ecb": settings.ecb_feed_url, "fed": settings.fed_feed_url}[key]
|
|
return _rss(url, LABELS[key], "macro", 3, now)
|
|
|
|
|
|
def finnhub(inst: Instrument, now: float) -> list[Item]:
|
|
key = settings.finnhub_key
|
|
base = "https://finnhub.io/api/v1"
|
|
if inst.asset_class == "crypto":
|
|
r = httpx.get(f"{base}/news", params={"category": "crypto", "token": key},
|
|
headers=HEADERS, timeout=10)
|
|
r.raise_for_status()
|
|
words = {inst.tv_symbol[:3].lower(), inst.news_query.split()[0].lower()} # "btc", "bitcoin"
|
|
rows = [x for x in r.json() if any(w in (x.get("headline", "") + x.get("summary", "")).lower()
|
|
for w in words)]
|
|
elif inst.exchange == "US":
|
|
today = datetime.now(timezone.utc).date()
|
|
r = httpx.get(f"{base}/company-news", headers=HEADERS, timeout=10, params={
|
|
"symbol": inst.tv_symbol, "from": (today - timedelta(days=3)).isoformat(),
|
|
"to": today.isoformat(), "token": key})
|
|
r.raise_for_status()
|
|
rows = r.json()
|
|
else:
|
|
return [] # the free plan covers US companies only
|
|
out = []
|
|
for x in rows:
|
|
ts = x.get("datetime")
|
|
if x.get("headline") and _fresh(ts, now):
|
|
out.append(Item(LABELS["finnhub"], x["headline"].strip(), _fmt(ts), "asset"))
|
|
out.sort(key=lambda i: i.published, reverse=True)
|
|
return out[:6]
|
|
|
|
|
|
def gather(query: str, inst: Instrument | None = None, now: float | None = None):
|
|
"""Collect headlines. Returns (items newest-first, {source: count}, [error messages])."""
|
|
now = time.time() if now is None else now
|
|
enabled = settings.news_sources
|
|
items: list[Item] = []
|
|
counts: dict[str, int] = {}
|
|
errors: list[str] = []
|
|
|
|
def run(key, fn):
|
|
if key not in enabled:
|
|
return
|
|
try:
|
|
got = fn()
|
|
counts[LABELS[key]] = len(got)
|
|
items.extend(got)
|
|
except Exception as e:
|
|
counts[LABELS[key]] = 0
|
|
errors.append(f"{LABELS[key]}: {e}"[:140])
|
|
|
|
run("google", lambda: google(query, now))
|
|
if inst is not None:
|
|
for m in MACRO_FOR.get(inst.exchange, []):
|
|
run(m, lambda m=m: macro(m, now))
|
|
# Finnhub's free plan covers US companies and crypto only; don't ask (or list it) elsewhere.
|
|
if settings.finnhub_key and (inst.exchange == "US" or inst.asset_class == "crypto"):
|
|
run("finnhub", lambda: finnhub(inst, now))
|
|
items.sort(key=lambda i: i.published, reverse=True)
|
|
return items, counts, errors
|
|
|
|
|
|
def status() -> list[dict]:
|
|
"""Which sources are on, for the dashboard."""
|
|
out = []
|
|
for key, (kind, text) in INFO.items():
|
|
configured = key != "finnhub" or bool(settings.finnhub_key)
|
|
out.append({"key": key, "name": LABELS[key], "kind": kind, "about": text,
|
|
"enabled": key in settings.news_sources and configured,
|
|
"needs_key": key == "finnhub", "configured": configured})
|
|
return out
|