From 9d596f63731e0cce39358015ab534bd01f5ce3d4 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Thu, 23 Jul 2026 19:41:22 -0700 Subject: [PATCH] add drugs.com and brutalist as news rss feeds (#7602) Co-authored-by: deeleeramone <> --- openbb_platform/extensions/news/README.md | 12 +- .../news/integration/test_news_api.py | 50 ++++ .../news/integration/test_news_python.py | 19 ++ .../extensions/news/openbb_news/parser.py | 141 +++++++++-- .../extensions/news/openbb_news/registry.py | 220 ++++++++++++++++++ .../extensions/news/openbb_news/rss.py | 13 +- .../extensions/news/tests/conftest.py | 44 +++- .../extensions/news/tests/test_parser.py | 219 +++++++++++++++++ .../extensions/news/tests/test_registry.py | 119 ++++++++++ .../extensions/news/tests/test_rss.py | 43 +++- 10 files changed, 857 insertions(+), 23 deletions(-) diff --git a/openbb_platform/extensions/news/README.md b/openbb_platform/extensions/news/README.md index 418f3f248b9..3c5ccb61c3d 100644 --- a/openbb_platform/extensions/news/README.md +++ b/openbb_platform/extensions/news/README.md @@ -1,12 +1,14 @@ # openbb-news RSS news feeds for the OpenBB Platform — one Workspace Newsfeed widget backed by -a registry of 400+ vetted feeds (Benzinga, GlobeNewswire by full ICB industry +a registry of 500+ vetted feeds (Benzinga, GlobeNewswire by full ICB industry breakdown + subject/event-type breakdown — earnings, M&A, dividends, IPOs, etc., PR Newswire regional + category, Axios, Yahoo Finance, Fortune, BBC, Fox News, -Drudge Report, Google News regional + topic). The widget defaults to -**Benzinga → Markets** so it loads finance-relevant articles immediately on -first open. +Drudge Report, Drugs.com, Google News regional + topic, plus the full set of +publishers aggregated by brutalist.report grouped under its eight topics — +Tech, News, Business, Science, Gaming, Culture, Politics, Sports). The widget +defaults to **Benzinga → Markets** so it loads finance-relevant articles +immediately on first open. ## Install @@ -75,7 +77,7 @@ internal = "Internal Sources" ### 3. Keep the bundled defaults — `[news] merge_defaults` Without this flag, the presence of any `[news.rss_feeds]` entries replaces -the 438 bundled feeds entirely. Set `merge_defaults = true` to keep +the 562 bundled feeds entirely. Set `merge_defaults = true` to keep everything and add your feeds on top. User keys that collide with bundled keys override the bundled URL. diff --git a/openbb_platform/extensions/news/integration/test_news_api.py b/openbb_platform/extensions/news/integration/test_news_api.py index bb20323db68..4ef4e692583 100644 --- a/openbb_platform/extensions/news/integration/test_news_api.py +++ b/openbb_platform/extensions/news/integration/test_news_api.py @@ -35,6 +35,9 @@ def test_rss_providers_lists_outlets(): assert "bbc" in values assert "globenewswire" in values assert "pr_newswire" in values + assert "drugs_com" in values + assert "brutalist_tech" in values + assert "brutalist_sports" in values @pytest.mark.integration @@ -67,6 +70,53 @@ def test_rss_fetches_articles_without_body(): assert "body" in first +@pytest.mark.integration +def test_rss_feeds_filtered_by_drugs_com_outlet(): + result = _get("rss_feeds", params={"outlet": "drugs_com"}) + assert result.status_code == 200 + values = {entry["value"] for entry in result.json()} + assert values == { + "drugs_com_medical_news", + "drugs_com_headline_news", + "drugs_com_fda_alerts", + "drugs_com_new_drug_approvals", + "drugs_com_new_drug_applications", + "drugs_com_clinical_trials", + } + + +@pytest.mark.integration +def test_rss_drugs_com_fetches_full_body(): + result = _get( + "rss", + params={"source": "drugs_com_medical_news", "limit": 3, "fetch_body": "true"}, + ) + assert result.status_code == 200 + results = result.json()["results"] + assert 1 <= len(results) <= 3 + for article in results: + assert len(article["body"]) > len(article["excerpt"]) + assert "Whatever your topic of interest" not in article["body"] + assert "Disclaimer: Statistical data" not in article["body"] + assert "ddc-opengraph-logomark" not in article["body"] + + +@pytest.mark.integration +def test_rss_brutalist_gated_host_fetches_full_body(): + # ESPN, PBS, BleepingComputer, Heavy and Techmeme need pinned request + # headers; a plain session gets 403/202/406. Exercise one of them live. + result = _get( + "rss", + params={"source": "brutalist_tech_bleepingcomputer", "limit": 2}, + ) + assert result.status_code == 200 + results = result.json()["results"] + assert 1 <= len(results) <= 2 + for article in results: + assert article["title"] + assert len(article["body"]) > len(article["excerpt"]) + + @pytest.mark.integration def test_rss_returns_empty_when_no_source_and_no_outlet(): result = _get("rss") diff --git a/openbb_platform/extensions/news/integration/test_news_python.py b/openbb_platform/extensions/news/integration/test_news_python.py index 7a855796224..01e7f6f6933 100644 --- a/openbb_platform/extensions/news/integration/test_news_python.py +++ b/openbb_platform/extensions/news/integration/test_news_python.py @@ -25,3 +25,22 @@ def test_rss_python_fetches_articles(obb): assert hasattr(first, "url") assert hasattr(first, "excerpt") assert hasattr(first, "body") + + +@pytest.mark.integration +def test_rss_python_drugs_com_fetches_full_body(obb): + out = obb.news.rss(source="drugs_com_medical_news", limit=3, fetch_body=True) + assert isinstance(out, OBBject) + assert 1 <= len(out.results) <= 3 + for article in out.results: + assert article.url.startswith("https://www.drugs.com/") + assert len(article.body) > len(article.excerpt) + assert "Whatever your topic of interest" not in article.body + + +@pytest.mark.integration +def test_rss_python_brutalist_topic_default(obb): + out = obb.news.rss(outlet="brutalist_business", limit=2, fetch_body=False) + assert isinstance(out, OBBject) + assert 1 <= len(out.results) <= 2 + assert out.results[0].title diff --git a/openbb_platform/extensions/news/openbb_news/parser.py b/openbb_platform/extensions/news/openbb_news/parser.py index ea6cde443e0..18f0376fd72 100644 --- a/openbb_platform/extensions/news/openbb_news/parser.py +++ b/openbb_platform/extensions/news/openbb_news/parser.py @@ -4,7 +4,7 @@ import re from datetime import datetime, timezone from html.parser import HTMLParser from time import struct_time -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, NamedTuple if TYPE_CHECKING: from aiohttp import ClientSession @@ -29,6 +29,85 @@ _ARTICLE_TIMEOUT_SEC = 6.0 _TIMESTAMP_RE = re.compile(r"^\d{1,2}:\d{2}(?::\d{2})?$") +_WEBKIT_UA = ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36" + " (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36" +) +_SAFARI_UA = ( + "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_7) AppleWebKit/605.1.15" + " (KHTML, like Gecko) Version/18.2 Safari/605.1.15" +) +_FEED_ACCEPT = ( + "text/html,application/xhtml+xml,application/xml;q=0.9," + "application/rss+xml,application/atom+xml,*/*;q=0.8" +) +# The shared session decodes only gzip/deflate. Overriding request headers +# drops its default Accept-Encoding, so aiohttp advertises br, which then +# arrives undecoded and unparseable; pin an encoding the session can decode. +_SAFE_ENCODING = "gzip, deflate" + + +def _pinned(user_agent: str | None = None) -> dict[str, str]: + """Build a browser-like header set the shared session can decode.""" + headers = {"Accept": _FEED_ACCEPT, "Accept-Encoding": _SAFE_ENCODING} + if user_agent: + headers["User-Agent"] = user_agent + return headers + + +# Hosts whose rotating-UA / JSON-Accept defaults get 403/406/202 or br-encoded. +_HOST_HEADERS: dict[str, dict[str, str]] = { + # Drugs.com article pages 403 the rotating Gecko user agents. + "drugs.com": _pinned(_WEBKIT_UA), + # Cloudflare/Akamai edges gate these feeds on the user agent; Safari passes. + "espn.com": _pinned(_SAFARI_UA), + "heavy.com": _pinned(_SAFARI_UA), + "bleepingcomputer.com": _pinned(_SAFARI_UA), + "pbs.org": _pinned(_SAFARI_UA), + # Techmeme's feed rejects the session default Accept: application/json. + "techmeme.com": _pinned(), +} + + +class _SiteRule(NamedTuple): + """Article extraction rule for a single host.""" + + container: str + """XPath for the element holding the article.""" + + cut: tuple[str, ...] + """XPath tests; the first matching child of ``container`` ends the article.""" + + hero_skip: tuple[str, ...] = () + """Substrings marking a hero image as a site placeholder rather than art.""" + + +_SITE_RULES: dict[str, _SiteRule] = { + # Drugs.com has no
and its
holds the sidebar, so //main + # swallows related teasers, newsletter pitches and vendor copyright. + "drugs.com": _SiteRule( + container="//div[contains(@class, 'ddc-main-content')]", + cut=( + "self::*[contains(@class, 'ddc-disclaimer')]", + "self::*[contains(@class, 'more-resources')]", + "self::h2[normalize-space()='More news resources']", + "self::h2[normalize-space()='Subscribe to our newsletter']", + ), + hero_skip=("ddc-opengraph-logomark",), + ), +} + + +def _host(url: str | None) -> str: + """Return the hostname of ``url`` without a ``www.`` prefix.""" + if not url: + return "" + from urllib.parse import urlparse + + hostname = (urlparse(url).hostname or "").lower() + return hostname[4:] if hostname.startswith("www.") else hostname + + class _TagStripper(HTMLParser): """Collect text data while discarding tags.""" @@ -123,10 +202,13 @@ async def fetch_feed(session: "ClientSession", url: str): import aiohttp import feedparser + kwargs: dict = {"timeout": aiohttp.ClientTimeout(total=_FEED_TIMEOUT_SEC)} + headers = _HOST_HEADERS.get(_host(url)) + if headers: + kwargs["headers"] = headers + try: - response = await session.get( - url, timeout=aiohttp.ClientTimeout(total=_FEED_TIMEOUT_SEC) - ) + response = await session.get(url, **kwargs) async with response: response.raise_for_status() payload = await response.read() @@ -219,6 +301,29 @@ def _article_markdown(node) -> str: return "\n\n".join(parts) +def _cut_tail(container, cut: tuple[str, ...]) -> None: + """Drop the first child matching ``cut`` and every element after it.""" + cutting = False + for child in list(container): + if not isinstance(child.tag, str): + continue + if not cutting and any(child.xpath(f"boolean({test})") for test in cut): + cutting = True + if cutting: + container.remove(child) + + +def _site_body(doc, rule: _SiteRule) -> str: + """Render the article container defined by ``rule`` as markdown.""" + best = "" + for container in doc.xpath(rule.container): + _cut_tail(container, rule.cut) + text = _article_markdown(container) + if _looks_clean(text) and len(text) > len(best): + best = text + return best + + def _extract_image_from_doc(doc) -> str | None: """Find a hero image URL from meta tags, JSON-LD, or article body.""" for path in ( @@ -242,7 +347,7 @@ def _extract_image_from_doc(doc) -> str | None: return None -def _extract_body(payload: bytes) -> str | None: +def _extract_body(payload: bytes, url: str | None = None) -> str | None: """Return the article body as markdown with inline images.""" from lxml import html @@ -251,16 +356,21 @@ def _extract_body(payload: bytes) -> str | None: except Exception: # noqa: BLE001 return None + rule = _SITE_RULES.get(_host(url)) + hero = _extract_image_from_doc(doc) + if hero and rule and any(marker in hero for marker in rule.hero_skip): + hero = None body = _jsonld_field(doc, "articleBody") if not (body and _looks_clean(body)): _strip_chrome(doc) - best: str = "" - for article in doc.xpath("//article"): - text = _article_markdown(article) - if _looks_clean(text) and len(text) > len(best): - best = text + best: str = _site_body(doc, rule) if rule else "" + if not best: + for article in doc.xpath("//article"): + text = _article_markdown(article) + if _looks_clean(text) and len(text) > len(best): + best = text if not best: for main in doc.xpath("//main"): text = _article_markdown(main) @@ -278,13 +388,16 @@ async def fetch_article_body(session: "ClientSession", url: str) -> str | None: """Fetch ``url`` and return its article body.""" import aiohttp + kwargs: dict = {"timeout": aiohttp.ClientTimeout(total=_ARTICLE_TIMEOUT_SEC)} + headers = _HOST_HEADERS.get(_host(url)) + if headers: + kwargs["headers"] = headers + try: - response = await session.get( - url, timeout=aiohttp.ClientTimeout(total=_ARTICLE_TIMEOUT_SEC) - ) + response = await session.get(url, **kwargs) async with response: response.raise_for_status() payload = await response.read() except (aiohttp.ClientError, TimeoutError): return None - return _extract_body(payload) + return _extract_body(payload, url) diff --git a/openbb_platform/extensions/news/openbb_news/registry.py b/openbb_platform/extensions/news/openbb_news/registry.py index f64631c7c80..0d826410ba5 100644 --- a/openbb_platform/extensions/news/openbb_news/registry.py +++ b/openbb_platform/extensions/news/openbb_news/registry.py @@ -10,6 +10,15 @@ _PROVIDER_LABELS: dict[str, str] = { "benzinga": "Benzinga", "cbc": "CBC", "drudge_report": "Drudge Report", + "drugs_com": "Drugs.com", + "brutalist_tech": "Brutalist — Tech", + "brutalist_news": "Brutalist — News", + "brutalist_business": "Brutalist — Business", + "brutalist_science": "Brutalist — Science", + "brutalist_gaming": "Brutalist — Gaming", + "brutalist_culture": "Brutalist — Culture", + "brutalist_politics": "Brutalist — Politics", + "brutalist_sports": "Brutalist — Sports", "fox_news": "Fox News", "fortune": "Fortune", "globenewswire": "GlobeNewswire", @@ -597,6 +606,36 @@ _CURATED_FEEDS: dict[str, tuple[str, str, str]] = { "Sports — Soccer", "https://www.cbc.ca/webfeed/rss/rss-sports-soccer", ), + "drugs_com_medical_news": ( + "drugs_com", + "Daily MedNews", + "https://www.drugs.com/feeds/medical_news.xml", + ), + "drugs_com_headline_news": ( + "drugs_com", + "News for Health Professionals", + "https://www.drugs.com/feeds/headline_news.xml", + ), + "drugs_com_fda_alerts": ( + "drugs_com", + "FDA MedWatch Drug Alerts", + "https://www.drugs.com/feeds/fda_alerts.xml", + ), + "drugs_com_new_drug_approvals": ( + "drugs_com", + "New Drug Approvals", + "https://www.drugs.com/feeds/new_drug_approvals.xml", + ), + "drugs_com_new_drug_applications": ( + "drugs_com", + "New Drug Applications", + "https://www.drugs.com/feeds/new_drug_applications.xml", + ), + "drugs_com_clinical_trials": ( + "drugs_com", + "Clinical Trial Results", + "https://www.drugs.com/feeds/clinical_trials.xml", + ), "fox_news_latest": ( "fox_news", "Latest Headlines", @@ -720,6 +759,171 @@ _CURATED_FEEDS: dict[str, tuple[str, str, str]] = { ), } +# The publisher sources aggregated by brutalist.report, grouped under its eight +# topics. Each entry is (feed_key_suffix, publisher_label, native_feed_url); the +# provider id is ``brutalist_``. Feeds validated through the shared async +# session. Hosts needing pinned request headers are handled in parser._HOST_HEADERS. +_BRUTALIST_TOPIC_LABELS: dict[str, str] = { + "tech": "Tech", + "news": "News", + "business": "Business", + "science": "Science", + "gaming": "Gaming", + "culture": "Culture", + "politics": "Politics", + "sports": "Sports", +} +_BRUTALIST_FEEDS: dict[str, tuple[tuple[str, str, str], ...]] = { + "tech": ( + ( + "androidauthority", + "Android Authority", + "https://www.androidauthority.com/feed", + ), + ("appleinsider", "AppleInsider", "https://appleinsider.com/rss/news/"), + ( + "arstechnica", + "ArsTechnica", + "https://feeds.arstechnica.com/arstechnica/index", + ), + ( + "bleepingcomputer", + "Bleeping Computer", + "https://www.bleepingcomputer.com/feed/", + ), + ("cloudflare", "Cloudflare Blog", "https://www.blog.cloudflare.com/rss/"), + ("cnet", "CNET", "https://www.cnet.com/rss/news/"), + ("coindesk", "Coindesk", "https://www.coindesk.com/arc/outboundfeeds/rss/"), + ("daringfireball", "Daring Fireball", "https://daringfireball.net/feeds/main"), + ("engadget", "Engadget", "https://www.engadget.com/category/news/feed/"), + ("hackaday", "Hackaday", "https://hackaday.com/feed/"), + ("hn", "Hacker News", "https://news.ycombinator.com/rss"), + ("lwn", "Linux Weekly News", "https://lwn.net/headlines/rss"), + ("phoronix", "Phoronix", "https://www.phoronix.com/rss.php"), + ("servethehome", "Serve The Home", "https://www.servethehome.com/feed/"), + ("slashdot", "Slashdot", "https://rss.slashdot.org/Slashdot/slashdotMain"), + ("techmeme", "Techmeme", "https://www.techmeme.com/feed.xml"), + ("techradar", "TechRadar", "https://www.techradar.com/feeds.xml"), + ("register", "The Register", "https://www.theregister.com/headlines.atom"), + ("shortcut", "The Shortcut", "https://theshortcut.com/feed"), + ("verge", "The Verge", "https://www.theverge.com/rss/index.xml"), + ("venturebeat", "Venturebeat", "https://venturebeat.com/feed/"), + ("wired", "Wired", "https://www.wired.com/feed/rss"), + ), + "news": ( + ("abcnews", "ABC News", "https://feeds.abcnews.com/abcnews/topstories"), + ("aljazeera", "Al Jazeera", "https://www.aljazeera.com/xml/rss/all.xml"), + ("bbc", "BBC", "https://feeds.bbci.co.uk/news/rss.xml"), + ( + "foxnews", + "Fox News", + "https://moxie.foxnews.com/google-publisher/latest.xml", + ), + ("nbcnews", "NBC News", "https://feeds.nbcnews.com/nbcnews/public/news"), + ("npr", "NPR", "https://feeds.npr.org/1002/rss.xml"), + ( + "pbsnewshour", + "PBS NewsHour", + "https://www.pbs.org/newshour/feeds/rss/headlines", + ), + ("quartz", "Quartz", "https://qz.com/rss"), + ( + "conversation", + "The Conversation", + "https://theconversation.com/us/articles.atom", + ), + ("guardian", "The Guardian", "https://www.theguardian.com/international/rss"), + ("intercept", "The Intercept", "https://theintercept.com/feed/?rss"), + ( + "nytimes", + "The New York Times", + "https://rss.nytimes.com/services/xml/rss/nyt/HomePage.xml", + ), + ("onion", "The Onion", "https://theonion.com/feed/"), + ( + "wsj", + "The Wall Street Journal", + "https://feeds.a.dj.com/rss/RSSWorldNews.xml", + ), + ), + "business": ( + ( + "businessinsider", + "Business Insider", + "https://feeds.businessinsider.com/custom/all", + ), + ("cnbc", "CNBC", "https://www.cnbc.com/id/100003114/device/rss/rss.html"), + ("fastcompany", "FastCompany", "https://www.fastcompany.com/latest/rss"), + ("fortune", "Fortune", "https://fortune.com/feed/fortune-feeds/?id=3230629"), + ( + "hbr", + "Harvard Business Review", + "http://feeds.harvardbusiness.org/harvardbusiness/", + ), + ( + "marketwatch", + "MarketWatch", + "https://feeds.content.dowjones.io/public/rss/mw_topstories", + ), + ("yahoofinance", "Yahoo Finance", "https://finance.yahoo.com/news/rssindex"), + ("zerohedge", "Zero Hedge", "https://feeds.feedburner.com/zerohedge/feed"), + ), + "science": ( + ("phys", "Phys", "https://phys.org/rss-feed/"), + ("quantamag", "Quanta Magazine", "https://www.quantamagazine.org/quanta/feed/"), + ("sciencedaily", "ScienceDaily", "https://www.sciencedaily.com/rss/all.xml"), + ( + "scientificamerican", + "Scientific American", + "https://www.scientificamerican.com/platform/syndication/rss/", + ), + ), + "gaming": ( + ("gamespot", "GameSpot", "https://www.gamespot.com/feeds/mashup/"), + ("kotaku", "Kotaku", "https://kotaku.com/feed"), + ("pcgamer", "PC Gamer", "https://www.pcgamer.com/feeds.xml"), + ("polygon", "Polygon", "https://www.polygon.com/rss/index.xml"), + ), + "culture": ( + ("billboard", "Billboard", "https://www.billboard.com/feed/rss/"), + ("esquire", "Esquire", "https://www.esquire.com/rss/all.xml/"), + ("heavy", "Heavy", "https://heavy.com/feed/"), + ("mashable", "Mashable", "https://mashable.com/feeds/rss/all"), + ("reddit", "Reddit Front Page", "https://www.reddit.com/.rss"), + ("avclub", "The A.V. Club", "https://www.avclub.com/rss"), + ("newyorker", "The New Yorker", "https://www.newyorker.com/feed/rss"), + ("thepointsguy", "The Points Guy", "https://thepointsguy.com/feed/"), + ("tmz", "TMZ", "https://www.tmz.com/rss.xml"), + ("vanityfair", "Vanity Fair", "https://www.vanityfair.com/feed/rss"), + ("variety", "Variety", "https://variety.com/feed/rss/"), + ("vox", "Vox", "https://www.vox.com/rss/index.xml"), + ), + "politics": ( + ("axios", "Axios", "https://api.axios.com/feed/"), + ("salon", "Salon", "https://www.salon.com/feed/"), + ( + "talkingpointsmemo", + "Talking Points Memo", + "https://talkingpointsmemo.com/feed", + ), + ( + "drudge", + "The Drudge Report", + "https://feeds.feedburner.com/DrudgeReportFeed", + ), + ("hill", "The Hill", "https://thehill.com/news/feed/"), + ("thenewrepublic", "The New Republic", "https://newrepublic.com/rss.xml"), + ), + "sports": ( + ("espn_mlb", "ESPN MLB", "https://www.espn.com/espn/rss/mlb/news"), + ("espn_nba", "ESPN NBA", "https://www.espn.com/espn/rss/nba/news"), + ("espn_nfl", "ESPN NFL", "https://www.espn.com/espn/rss/nfl/news"), + ("espn_nhl", "ESPN NHL", "https://www.espn.com/espn/rss/nhl/news"), + ("espn_soccer", "ESPN SOCCER", "https://www.espn.com/espn/rss/soccer/news"), + ("yahoosports", "Yahoo Sports", "https://sports.yahoo.com/rss/"), + ), +} + def _slug(value: str) -> str: return value.replace("/", "_").replace("-", "_") @@ -791,6 +995,13 @@ def _build_feed_registry() -> tuple[dict[str, str], dict[str, dict[str, str]]]: feeds[key] = url meta[key] = {"provider": provider, "label": label} + for topic, entries in _BRUTALIST_FEEDS.items(): + provider = f"brutalist_{topic}" + for suffix, label, url in entries: + key = f"{provider}_{suffix}" + feeds[key] = url + meta[key] = {"provider": provider, "label": label} + return feeds, meta @@ -924,11 +1135,20 @@ _DEFAULT_FEED_BY_PROVIDER: dict[str, str] = { "bbc": "bbc_world", "benzinga": "benzinga_markets", "cbc": "cbc_business", + "drugs_com": "drugs_com_medical_news", "fox_news": "fox_news_latest", "globenewswire": "globenewswire_all", "google_news": "google_news_us", "pr_newswire": "pr_newswire_global", "wired": "wired_business", + "brutalist_tech": "brutalist_tech_verge", + "brutalist_news": "brutalist_news_guardian", + "brutalist_business": "brutalist_business_fortune", + "brutalist_science": "brutalist_science_scientificamerican", + "brutalist_gaming": "brutalist_gaming_polygon", + "brutalist_culture": "brutalist_culture_vox", + "brutalist_politics": "brutalist_politics_axios", + "brutalist_sports": "brutalist_sports_yahoosports", } diff --git a/openbb_platform/extensions/news/openbb_news/rss.py b/openbb_platform/extensions/news/openbb_news/rss.py index 996c69af560..10b3c4b4e46 100644 --- a/openbb_platform/extensions/news/openbb_news/rss.py +++ b/openbb_platform/extensions/news/openbb_news/rss.py @@ -166,7 +166,16 @@ async def rss( for entry, fetched_body in zip(entries, bodies): summary_html = entry.get("summary") or entry.get("description") or "" excerpt = truncate(strip_html(summary_html)) - body = fetched_body or html_to_markdown(summary_html) or excerpt + url = (entry.get("link") or "").strip() + if fetched_body: + body = fetched_body + else: + # No full body — some sources paywall or block article pages (NYT, + # WSJ). Fall back to the summary and link out so the reader can + # still reach the full story. + body = html_to_markdown(summary_html) or excerpt + if url: + body = f"{body}\n\n[Read the full story at the source ↗]({url})" out.append( NewsItemData( title=(entry.get("title") or "(untitled)").strip(), @@ -174,7 +183,7 @@ async def rss( entry.get("published_parsed") or entry.get("updated_parsed") ), author=entry.get("author") or feed_title or "Unknown", - url=(entry.get("link") or "").strip(), + url=url, excerpt=excerpt, body=body, ) diff --git a/openbb_platform/extensions/news/tests/conftest.py b/openbb_platform/extensions/news/tests/conftest.py index ea99fe3b37c..60db9759658 100644 --- a/openbb_platform/extensions/news/tests/conftest.py +++ b/openbb_platform/extensions/news/tests/conftest.py @@ -38,6 +38,46 @@ _SAMPLE_JSONLD_HTML = b""" """ +# Mirrors the real drugs.com article DOM: no
, sidebar inside
, +# and trailing furniture after the disclaimer. +_SAMPLE_DRUGS_COM_HTML = b""" + + +
+
+
Home News Consumer News Print page
+ +

FDA Clears First Cholesterol Pill

+ + +

THURSDAY, July 16, 2026 — A new daily pill will lower cholesterol.

+

The FDA acted today to approve the drug for adults.

+
  • Sources
+

Disclaimer: Statistical data in medical articles provide general trends.

+
+

© 2026 HealthDay. All rights reserved.

+

Read this next

+

Related teaser one.

Related teaser two.

+

More news resources

+
  • FDA Medwatch Drug Alerts
+

Subscribe to our newsletter

+

Whatever your topic of interest, subscribe to our newsletters.

+
+
+
+

Podcast pitch paragraph.

+

Sidebar drug teaser one.

+

Sidebar drug teaser two.

+
+
+ +""" + + +@pytest.fixture +def sample_drugs_com_html() -> bytes: + return _SAMPLE_DRUGS_COM_HTML + @pytest.fixture def sample_rss() -> bytes: @@ -83,10 +123,12 @@ class _FakeSession: def __init__(self, response_map: dict) -> None: self.response_map = response_map self.calls: list[str] = [] + self.headers_sent: list[dict | None] = [] self.closed = False - async def get(self, url: str, **_kwargs: Any) -> _FakeResponse: + async def get(self, url: str, **kwargs: Any) -> _FakeResponse: self.calls.append(url) + self.headers_sent.append(kwargs.get("headers")) result = self.response_map.get(url) if result is None: raise KeyError(f"unmocked URL: {url}") diff --git a/openbb_platform/extensions/news/tests/test_parser.py b/openbb_platform/extensions/news/tests/test_parser.py index 8df3a104894..8d1195634f8 100644 --- a/openbb_platform/extensions/news/tests/test_parser.py +++ b/openbb_platform/extensions/news/tests/test_parser.py @@ -232,6 +232,20 @@ async def test_fetch_feed_http_error(stub_session_factory): assert len(parsed.entries) == 0 +async def test_fetch_feed_pins_headers_for_gated_host(stub_session_factory, sample_rss): + url = "https://www.espn.com/espn/rss/nfl/news" + session = stub_session_factory({url: sample_rss}) + await parser.fetch_feed(session, url) + assert session.headers_sent == [parser._HOST_HEADERS["espn.com"]] + + +async def test_fetch_feed_no_headers_for_normal_host(stub_session_factory, sample_rss): + url = "https://feeds.bbci.co.uk/news/rss.xml" + session = stub_session_factory({url: sample_rss}) + await parser.fetch_feed(session, url) + assert session.headers_sent == [None] + + def test_extract_body_jsonld_top_level(sample_jsonld_html): assert parser._extract_body(sample_jsonld_html) == "JSON-LD top-level body." @@ -592,6 +606,211 @@ def test_jsonld_field_image_object_with_url(): assert body.startswith("![](https://x.test/obj.jpg)") +DRUGS_COM_URL = "https://www.drugs.com/news/fda-clears-cholesterol-pill-130672.html" + + +def test_host_strips_www(): + assert parser._host("https://www.drugs.com/news/x.html") == "drugs.com" + + +def test_host_without_www(): + assert parser._host("https://drugs.com/news/x.html") == "drugs.com" + + +def test_host_is_lowercased(): + assert parser._host("https://WWW.Drugs.COM/news/x.html") == "drugs.com" + + +def test_host_empty_url(): + assert parser._host(None) == "" + assert parser._host("") == "" + + +def test_host_url_without_hostname(): + assert parser._host("not-a-url") == "" + + +def test_cut_tail_removes_marker_and_everything_after(): + doc = html.fromstring( + b"
" + b"

Keep one.

" + b"" + b"

Keep two.

" + b"

More news resources

" + b"
  • link
" + b"

Newsletter pitch.

" + b"
" + ) + container = doc.xpath("//div")[0] + parser._cut_tail(container, parser._SITE_RULES["drugs.com"].cut) + text = container.text_content() + assert "Keep one." in text + assert "Keep two." in text + assert "More news resources" not in text + assert "Newsletter pitch." not in text + + +def test_cut_tail_without_marker_keeps_all(): + doc = html.fromstring(b"

One.

Two.

") + container = doc.xpath("//div")[0] + parser._cut_tail(container, parser._SITE_RULES["drugs.com"].cut) + assert len(container.xpath(".//p")) == 2 + + +def test_site_body_no_container_returns_empty(): + doc = html.fromstring(b"

One.

Two.

") + assert parser._site_body(doc, parser._SITE_RULES["drugs.com"]) == "" + + +def test_site_body_picks_largest_container(): + doc = html.fromstring( + b"" + b"
" + b"

Short one.

Short two.

" + b"
" + b"

Longer container paragraph one with much more substance.

" + b"

Longer container paragraph two with additional detail.

" + b"" + ) + body = parser._site_body(doc, parser._SITE_RULES["drugs.com"]) + assert "Longer container paragraph one" in body + assert "Short one." not in body + + +def test_site_body_skips_unclean_container(): + doc = html.fromstring( + b"
" + b"

[[ template ]] junk

{{ binding }} more junk

" + b"
" + ) + assert parser._site_body(doc, parser._SITE_RULES["drugs.com"]) == "" + + +def test_extract_body_drugs_com_keeps_article_text(sample_drugs_com_html): + body = parser._extract_body(sample_drugs_com_html, DRUGS_COM_URL) + assert body is not None + assert "By Ellyn Vohnoutka HealthDay Reporter" in body + assert "A new daily pill will lower cholesterol." in body + assert "The FDA acted today to approve the drug for adults." in body + + +def test_extract_body_drugs_com_drops_trailing_furniture(sample_drugs_com_html): + body = parser._extract_body(sample_drugs_com_html, DRUGS_COM_URL) + assert body is not None + assert "Disclaimer:" not in body + assert "HealthDay. All rights reserved." not in body + assert "Related teaser one." not in body + assert "Whatever your topic of interest" not in body + + +def test_extract_body_drugs_com_drops_sidebar(sample_drugs_com_html): + body = parser._extract_body(sample_drugs_com_html, DRUGS_COM_URL) + assert body is not None + assert "Sidebar drug teaser one." not in body + assert "Podcast pitch paragraph." not in body + assert "ddc-podcast-cover" not in body + + +def test_extract_body_drugs_com_skips_placeholder_hero(sample_drugs_com_html): + body = parser._extract_body(sample_drugs_com_html, DRUGS_COM_URL) + assert body is not None + assert "ddc-opengraph-logomark" not in body + assert not body.startswith("![") + + +def test_extract_body_drugs_com_keeps_real_hero(): + payload = ( + b"" + b'' + b"
" + b"

Lede paragraph of the story.

Second paragraph.

" + b"
" + ) + body = parser._extract_body(payload, DRUGS_COM_URL) + assert body is not None + assert body.split("\n\n")[0] == "![](https://media.test/featured.jpg)" + + +def test_extract_body_drugs_com_falls_back_when_container_missing(): + payload = ( + b"
" + b"

Generic paragraph one.

Generic paragraph two.

" + b"
" + ) + body = parser._extract_body(payload, DRUGS_COM_URL) + assert body == "Generic paragraph one.\n\nGeneric paragraph two." + + +def test_extract_body_unknown_host_uses_generic_path(sample_drugs_com_html): + body = parser._extract_body(sample_drugs_com_html, "https://other.test/x.html") + assert body is not None + assert "Whatever your topic of interest" in body + + +def test_extract_body_without_url_uses_generic_path(sample_drugs_com_html): + body = parser._extract_body(sample_drugs_com_html) + assert body is not None + assert "Whatever your topic of interest" in body + + +async def test_fetch_article_body_applies_site_rule( + stub_session_factory, sample_drugs_com_html +): + session = stub_session_factory({DRUGS_COM_URL: sample_drugs_com_html}) + body = await parser.fetch_article_body(session, DRUGS_COM_URL) + assert body is not None + assert "A new daily pill will lower cholesterol." in body + assert "Whatever your topic of interest" not in body + + +async def test_fetch_article_body_pins_user_agent_for_drugs_com( + stub_session_factory, sample_drugs_com_html +): + session = stub_session_factory({DRUGS_COM_URL: sample_drugs_com_html}) + await parser.fetch_article_body(session, DRUGS_COM_URL) + assert session.headers_sent == [parser._HOST_HEADERS["drugs.com"]] + assert session.headers_sent[0]["User-Agent"] == parser._WEBKIT_UA + + +def test_drugs_com_pinned_user_agent_is_not_gecko(): + # drugs.com articles 403 any Gecko user agent. + ua = parser._HOST_HEADERS["drugs.com"]["User-Agent"] + assert "Gecko/" not in ua + assert "AppleWebKit" in ua + + +def test_host_headers_pin_safe_encoding(): + # Overriding headers drops the session default Accept-Encoding; every pinned + # host must exclude br, which the shared session cannot decode. + for host, headers in parser._HOST_HEADERS.items(): + assert headers["Accept-Encoding"] == "gzip, deflate", host + assert "br" not in headers["Accept-Encoding"], host + + +def test_host_headers_cover_expected_hosts(): + assert set(parser._HOST_HEADERS) == { + "drugs.com", + "espn.com", + "heavy.com", + "bleepingcomputer.com", + "pbs.org", + "techmeme.com", + } + + +def test_pinned_helper_omits_user_agent_when_none(): + assert "User-Agent" not in parser._pinned() + assert parser._pinned("UA/1.0")["User-Agent"] == "UA/1.0" + + +async def test_fetch_article_body_keeps_session_user_agent_for_other_hosts( + stub_session_factory, sample_article_html +): + session = stub_session_factory({"https://x.test/a": sample_article_html}) + await parser.fetch_article_body(session, "https://x.test/a") + assert session.headers_sent == [None] + + def test_extract_image_jsonld_fallback_path(): payload = ( b"" diff --git a/openbb_platform/extensions/news/tests/test_registry.py b/openbb_platform/extensions/news/tests/test_registry.py index c3b680939d1..1a0fcd38877 100644 --- a/openbb_platform/extensions/news/tests/test_registry.py +++ b/openbb_platform/extensions/news/tests/test_registry.py @@ -55,6 +55,7 @@ def test_list_providers_default(monkeypatch): assert providers["benzinga"] == "Benzinga" assert providers["cbc"] == "CBC" assert providers["drudge_report"] == "Drudge Report" + assert providers["drugs_com"] == "Drugs.com" assert providers["fox_news"] == "Fox News" assert providers["fortune"] == "Fortune" assert providers["globenewswire"] == "GlobeNewswire" @@ -62,12 +63,129 @@ def test_list_providers_default(monkeypatch): assert providers["pr_newswire"] == "PR Newswire" assert providers["yahoo_finance"] == "Yahoo Finance" assert providers["wired"] == "Wired" + assert providers["brutalist_tech"] == "Brutalist — Tech" + assert providers["brutalist_business"] == "Brutalist — Business" + assert providers["brutalist_sports"] == "Brutalist — Sports" assert "custom" not in providers assert "barchart" not in providers assert "cnn" not in providers assert "seeking_alpha" not in providers +def test_list_feed_choices_drugs_com(monkeypatch): + monkeypatch.setattr(registry, "load_config", lambda: {}) + choices = registry.list_feed_choices("drugs_com") + labels = [c["label"] for c in choices] + assert "Daily MedNews" in labels + assert "News for Health Professionals" in labels + assert "FDA MedWatch Drug Alerts" in labels + assert "New Drug Approvals" in labels + assert "New Drug Applications" in labels + assert "Clinical Trial Results" in labels + assert len(choices) == 6 + assert labels == sorted(labels) + + +def test_drugs_com_feed_urls(monkeypatch): + monkeypatch.setattr(registry, "load_config", lambda: {}) + assert ( + registry.get_feed_url("drugs_com_medical_news") + == "https://www.drugs.com/feeds/medical_news.xml" + ) + assert ( + registry.get_feed_url("drugs_com_headline_news") + == "https://www.drugs.com/feeds/headline_news.xml" + ) + assert ( + registry.get_feed_url("drugs_com_fda_alerts") + == "https://www.drugs.com/feeds/fda_alerts.xml" + ) + assert ( + registry.get_feed_url("drugs_com_new_drug_approvals") + == "https://www.drugs.com/feeds/new_drug_approvals.xml" + ) + assert ( + registry.get_feed_url("drugs_com_new_drug_applications") + == "https://www.drugs.com/feeds/new_drug_applications.xml" + ) + assert ( + registry.get_feed_url("drugs_com_clinical_trials") + == "https://www.drugs.com/feeds/clinical_trials.xml" + ) + + +_BRUTALIST_PROVIDER_COUNTS = { + "brutalist_tech": 22, + "brutalist_news": 14, + "brutalist_business": 8, + "brutalist_science": 4, + "brutalist_gaming": 4, + "brutalist_culture": 12, + "brutalist_politics": 6, + "brutalist_sports": 6, +} + + +def test_brutalist_providers_present(monkeypatch): + monkeypatch.setattr(registry, "load_config", lambda: {}) + providers = registry.list_providers() + for pid in _BRUTALIST_PROVIDER_COUNTS: + assert pid in providers + assert providers[pid].startswith("Brutalist — ") + + +def test_brutalist_feed_choices_per_topic(monkeypatch): + monkeypatch.setattr(registry, "load_config", lambda: {}) + for pid, count in _BRUTALIST_PROVIDER_COUNTS.items(): + choices = registry.list_feed_choices(pid) + labels = [c["label"] for c in choices] + assert len(choices) == count, pid + assert labels == sorted(labels), pid + + +def test_brutalist_total_feed_count(): + total = sum(len(v) for v in registry._BRUTALIST_FEEDS.values()) + assert total == 76 + assert set(registry._BRUTALIST_FEEDS) == set(registry._BRUTALIST_TOPIC_LABELS) + + +def test_brutalist_feed_keys_unique_and_registered(monkeypatch): + monkeypatch.setattr(registry, "load_config", lambda: {}) + feeds = registry.list_feeds() + keys = [] + for topic, entries in registry._BRUTALIST_FEEDS.items(): + for suffix, _label, url in entries: + key = f"brutalist_{topic}_{suffix}" + keys.append(key) + assert feeds.get(key) == url + assert len(keys) == len(set(keys)) + + +def test_brutalist_sample_feed_urls(): + assert ( + registry.get_feed_url("brutalist_tech_verge") + == "https://www.theverge.com/rss/index.xml" + ) + assert ( + registry.get_feed_url("brutalist_sports_espn_nfl") + == "https://www.espn.com/espn/rss/nfl/news" + ) + assert ( + registry.get_feed_url("brutalist_politics_axios") + == "https://api.axios.com/feed/" + ) + + +def test_brutalist_defaults_resolve(monkeypatch): + monkeypatch.setattr(registry, "load_config", lambda: {}) + feeds = registry.list_feeds() + for pid in _BRUTALIST_PROVIDER_COUNTS: + default = registry.default_feed_for(pid) + assert default is not None + assert default in feeds + assert default.startswith(pid + "_") + + def test_get_feed_url_drudge_report(): assert ( registry.get_feed_url("drudge_report") @@ -223,6 +341,7 @@ def test_default_feed_for_curated_picks(monkeypatch): "bbc": "bbc_world", "benzinga": "benzinga_markets", "cbc": "cbc_business", + "drugs_com": "drugs_com_medical_news", "fox_news": "fox_news_latest", "globenewswire": "globenewswire_all", "google_news": "google_news_us", diff --git a/openbb_platform/extensions/news/tests/test_rss.py b/openbb_platform/extensions/news/tests/test_rss.py index 4af6f693430..5706a33d561 100644 --- a/openbb_platform/extensions/news/tests/test_rss.py +++ b/openbb_platform/extensions/news/tests/test_rss.py @@ -29,7 +29,10 @@ async def test_rss_with_body(mocked_pipeline): assert items[0].excerpt == "A short summary." assert items[0].url == "https://example.test/article/1" assert "First paragraph" in items[0].body + # The extracted article body is used verbatim — no source link appended. + assert "Read the full story" not in items[0].body assert items[1].excerpt == "Plain text summary without HTML." + # Item 1 has no link, so the summary stands alone. assert items[1].body == items[1].excerpt assert items[1].url == "" assert items[1].author == "Sample Feed" @@ -37,7 +40,45 @@ async def test_rss_with_body(mocked_pipeline): async def test_rss_without_body(mocked_pipeline): out = await rss.rss(source="example", limit=5, fetch_body=False) - assert all(item.body == item.excerpt for item in out.results) + results = out.results + # Item 0 has a link: the summary is followed by a source link. + assert results[0].body.startswith(results[0].excerpt) + assert ( + "[Read the full story at the source ↗](https://example.test/article/1)" + in results[0].body + ) + # Item 1 has no link, so the summary stands alone. + assert results[1].body == results[1].excerpt + + +async def test_rss_appends_source_link_when_body_fetch_fails( + monkeypatch, stub_session_factory +): + feed = ( + b"" + b'Paywalled' + b"Locked Story" + b"https://paywall.test/story" + b"Just the teaser." + b"Mon, 19 May 2026 12:00:00 GMT" + b"" + ) + monkeypatch.setattr( + "openbb_news.registry.get_feed_url", lambda key: "http://feed.test/pw" + ) + stub_session_factory( + { + "http://feed.test/pw": feed, + # Article page 403s / yields no extractable body. + "https://paywall.test/story": (b"Access Denied", 403), + } + ) + item = (await rss.rss(source="x", limit=1, fetch_body=True)).results[0] + assert item.excerpt == "Just the teaser." + assert item.body == ( + "Just the teaser.\n\n" + "[Read the full story at the source ↗](https://paywall.test/story)" + ) async def test_rss_limit_caps_entries(mocked_pipeline):