add drugs.com and brutalist as news rss feeds (#7602)

Co-authored-by: deeleeramone <>

openbb_platform/extensions/news/README.md

@@ -1,12 +1,14 @@
 # openbb-news
 
 RSS news feeds for the OpenBB Platform — one Workspace Newsfeed widget backed by
-a registry of 400+ vetted feeds (Benzinga, GlobeNewswire by full ICB industry
+a registry of 500+ vetted feeds (Benzinga, GlobeNewswire by full ICB industry
 breakdown + subject/event-type breakdown — earnings, M&A, dividends, IPOs, etc.,
 PR Newswire regional + category, Axios, Yahoo Finance, Fortune, BBC, Fox News,
-Drudge Report, Google News regional + topic). The widget defaults to
-**Benzinga → Markets** so it loads finance-relevant articles immediately on
-first open.
+Drudge Report, Drugs.com, Google News regional + topic, plus the full set of
+publishers aggregated by brutalist.report grouped under its eight topics —
+Tech, News, Business, Science, Gaming, Culture, Politics, Sports). The widget
+defaults to **Benzinga → Markets** so it loads finance-relevant articles
+immediately on first open.
 
 ## Install
 
@@ -75,7 +77,7 @@ internal      = "Internal Sources"
 ### 3. Keep the bundled defaults — `[news] merge_defaults`
 
 Without this flag, the presence of any `[news.rss_feeds]` entries replaces
-the 438 bundled feeds entirely. Set `merge_defaults = true` to keep
+the 562 bundled feeds entirely. Set `merge_defaults = true` to keep
 everything and add your feeds on top. User keys that collide with bundled
 keys override the bundled URL.
 

openbb_platform/extensions/news/integration/test_news_api.py

@@ -35,6 +35,9 @@ def test_rss_providers_lists_outlets():
     assert "bbc" in values
     assert "globenewswire" in values
     assert "pr_newswire" in values
+    assert "drugs_com" in values
+    assert "brutalist_tech" in values
+    assert "brutalist_sports" in values
 
 
 @pytest.mark.integration
@@ -67,6 +70,53 @@ def test_rss_fetches_articles_without_body():
     assert "body" in first
 
 
+@pytest.mark.integration
+def test_rss_feeds_filtered_by_drugs_com_outlet():
+    result = _get("rss_feeds", params={"outlet": "drugs_com"})
+    assert result.status_code == 200
+    values = {entry["value"] for entry in result.json()}
+    assert values == {
+        "drugs_com_medical_news",
+        "drugs_com_headline_news",
+        "drugs_com_fda_alerts",
+        "drugs_com_new_drug_approvals",
+        "drugs_com_new_drug_applications",
+        "drugs_com_clinical_trials",
+    }
+
+
+@pytest.mark.integration
+def test_rss_drugs_com_fetches_full_body():
+    result = _get(
+        "rss",
+        params={"source": "drugs_com_medical_news", "limit": 3, "fetch_body": "true"},
+    )
+    assert result.status_code == 200
+    results = result.json()["results"]
+    assert 1 <= len(results) <= 3
+    for article in results:
+        assert len(article["body"]) > len(article["excerpt"])
+        assert "Whatever your topic of interest" not in article["body"]
+        assert "Disclaimer: Statistical data" not in article["body"]
+        assert "ddc-opengraph-logomark" not in article["body"]
+
+
+@pytest.mark.integration
+def test_rss_brutalist_gated_host_fetches_full_body():
+    # ESPN, PBS, BleepingComputer, Heavy and Techmeme need pinned request
+    # headers; a plain session gets 403/202/406. Exercise one of them live.
+    result = _get(
+        "rss",
+        params={"source": "brutalist_tech_bleepingcomputer", "limit": 2},
+    )
+    assert result.status_code == 200
+    results = result.json()["results"]
+    assert 1 <= len(results) <= 2
+    for article in results:
+        assert article["title"]
+        assert len(article["body"]) > len(article["excerpt"])
+
+
 @pytest.mark.integration
 def test_rss_returns_empty_when_no_source_and_no_outlet():
     result = _get("rss")

openbb_platform/extensions/news/integration/test_news_python.py

@@ -25,3 +25,22 @@ def test_rss_python_fetches_articles(obb):
     assert hasattr(first, "url")
     assert hasattr(first, "excerpt")
     assert hasattr(first, "body")
+
+
+@pytest.mark.integration
+def test_rss_python_drugs_com_fetches_full_body(obb):
+    out = obb.news.rss(source="drugs_com_medical_news", limit=3, fetch_body=True)
+    assert isinstance(out, OBBject)
+    assert 1 <= len(out.results) <= 3
+    for article in out.results:
+        assert article.url.startswith("https://www.drugs.com/")
+        assert len(article.body) > len(article.excerpt)
+        assert "Whatever your topic of interest" not in article.body
+
+
+@pytest.mark.integration
+def test_rss_python_brutalist_topic_default(obb):
+    out = obb.news.rss(outlet="brutalist_business", limit=2, fetch_body=False)
+    assert isinstance(out, OBBject)
+    assert 1 <= len(out.results) <= 2
+    assert out.results[0].title

openbb_platform/extensions/news/openbb_news/parser.py

@@ -4,7 +4,7 @@ import re
 from datetime import datetime, timezone
 from html.parser import HTMLParser
 from time import struct_time
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, NamedTuple
 
 if TYPE_CHECKING:
     from aiohttp import ClientSession
@@ -29,6 +29,85 @@ _ARTICLE_TIMEOUT_SEC = 6.0
 _TIMESTAMP_RE = re.compile(r"^\d{1,2}:\d{2}(?::\d{2})?$")
 
 
+_WEBKIT_UA = (
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
+    " (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
+)
+_SAFARI_UA = (
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_7) AppleWebKit/605.1.15"
+    " (KHTML, like Gecko) Version/18.2 Safari/605.1.15"
+)
+_FEED_ACCEPT = (
+    "text/html,application/xhtml+xml,application/xml;q=0.9,"
+    "application/rss+xml,application/atom+xml,*/*;q=0.8"
+)
+# The shared session decodes only gzip/deflate. Overriding request headers
+# drops its default Accept-Encoding, so aiohttp advertises br, which then
+# arrives undecoded and unparseable; pin an encoding the session can decode.
+_SAFE_ENCODING = "gzip, deflate"
+
+
+def _pinned(user_agent: str | None = None) -> dict[str, str]:
+    """Build a browser-like header set the shared session can decode."""
+    headers = {"Accept": _FEED_ACCEPT, "Accept-Encoding": _SAFE_ENCODING}
+    if user_agent:
+        headers["User-Agent"] = user_agent
+    return headers
+
+
+# Hosts whose rotating-UA / JSON-Accept defaults get 403/406/202 or br-encoded.
+_HOST_HEADERS: dict[str, dict[str, str]] = {
+    # Drugs.com article pages 403 the rotating Gecko user agents.
+    "drugs.com": _pinned(_WEBKIT_UA),
+    # Cloudflare/Akamai edges gate these feeds on the user agent; Safari passes.
+    "espn.com": _pinned(_SAFARI_UA),
+    "heavy.com": _pinned(_SAFARI_UA),
+    "bleepingcomputer.com": _pinned(_SAFARI_UA),
+    "pbs.org": _pinned(_SAFARI_UA),
+    # Techmeme's feed rejects the session default Accept: application/json.
+    "techmeme.com": _pinned(),
+}
+
+
+class _SiteRule(NamedTuple):
+    """Article extraction rule for a single host."""
+
+    container: str
+    """XPath for the element holding the article."""
+
+    cut: tuple[str, ...]
+    """XPath tests; the first matching child of ``container`` ends the article."""
+
+    hero_skip: tuple[str, ...] = ()
+    """Substrings marking a hero image as a site placeholder rather than art."""
+
+
+_SITE_RULES: dict[str, _SiteRule] = {
+    # Drugs.com has no <article> and its <main> holds the sidebar, so //main
+    # swallows related teasers, newsletter pitches and vendor copyright.
+    "drugs.com": _SiteRule(
+        container="//div[contains(@class, 'ddc-main-content')]",
+        cut=(
+            "self::*[contains(@class, 'ddc-disclaimer')]",
+            "self::*[contains(@class, 'more-resources')]",
+            "self::h2[normalize-space()='More news resources']",
+            "self::h2[normalize-space()='Subscribe to our newsletter']",
+        ),
+        hero_skip=("ddc-opengraph-logomark",),
+    ),
+}
+
+
+def _host(url: str | None) -> str:
+    """Return the hostname of ``url`` without a ``www.`` prefix."""
+    if not url:
+        return ""
+    from urllib.parse import urlparse
+
+    hostname = (urlparse(url).hostname or "").lower()
+    return hostname[4:] if hostname.startswith("www.") else hostname
+
+
 class _TagStripper(HTMLParser):
     """Collect text data while discarding tags."""
 
@@ -123,10 +202,13 @@ async def fetch_feed(session: "ClientSession", url: str):
     import aiohttp
     import feedparser
 
+    kwargs: dict = {"timeout": aiohttp.ClientTimeout(total=_FEED_TIMEOUT_SEC)}
+    headers = _HOST_HEADERS.get(_host(url))
+    if headers:
+        kwargs["headers"] = headers
+
     try:
-        response = await session.get(
-            url, timeout=aiohttp.ClientTimeout(total=_FEED_TIMEOUT_SEC)
-        )
+        response = await session.get(url, **kwargs)
         async with response:
             response.raise_for_status()
             payload = await response.read()
@@ -219,6 +301,29 @@ def _article_markdown(node) -> str:
     return "\n\n".join(parts)
 
 
+def _cut_tail(container, cut: tuple[str, ...]) -> None:
+    """Drop the first child matching ``cut`` and every element after it."""
+    cutting = False
+    for child in list(container):
+        if not isinstance(child.tag, str):
+            continue
+        if not cutting and any(child.xpath(f"boolean({test})") for test in cut):
+            cutting = True
+        if cutting:
+            container.remove(child)
+
+
+def _site_body(doc, rule: _SiteRule) -> str:
+    """Render the article container defined by ``rule`` as markdown."""
+    best = ""
+    for container in doc.xpath(rule.container):
+        _cut_tail(container, rule.cut)
+        text = _article_markdown(container)
+        if _looks_clean(text) and len(text) > len(best):
+            best = text
+    return best
+
+
 def _extract_image_from_doc(doc) -> str | None:
     """Find a hero image URL from meta tags, JSON-LD, or article body."""
     for path in (
@@ -242,7 +347,7 @@ def _extract_image_from_doc(doc) -> str | None:
     return None
 
 
-def _extract_body(payload: bytes) -> str | None:
+def _extract_body(payload: bytes, url: str | None = None) -> str | None:
     """Return the article body as markdown with inline images."""
     from lxml import html
 
@@ -251,16 +356,21 @@ def _extract_body(payload: bytes) -> str | None:
     except Exception:  # noqa: BLE001
         return None
 
+    rule = _SITE_RULES.get(_host(url))
+
     hero = _extract_image_from_doc(doc)
+    if hero and rule and any(marker in hero for marker in rule.hero_skip):
+        hero = None
 
     body = _jsonld_field(doc, "articleBody")
     if not (body and _looks_clean(body)):
         _strip_chrome(doc)
-        best: str = ""
-        for article in doc.xpath("//article"):
-            text = _article_markdown(article)
-            if _looks_clean(text) and len(text) > len(best):
-                best = text
+        best: str = _site_body(doc, rule) if rule else ""
+        if not best:
+            for article in doc.xpath("//article"):
+                text = _article_markdown(article)
+                if _looks_clean(text) and len(text) > len(best):
+                    best = text
         if not best:
             for main in doc.xpath("//main"):
                 text = _article_markdown(main)
@@ -278,13 +388,16 @@ async def fetch_article_body(session: "ClientSession", url: str) -> str | None:
     """Fetch ``url`` and return its article body."""
     import aiohttp
 
+    kwargs: dict = {"timeout": aiohttp.ClientTimeout(total=_ARTICLE_TIMEOUT_SEC)}
+    headers = _HOST_HEADERS.get(_host(url))
+    if headers:
+        kwargs["headers"] = headers
+
     try:
-        response = await session.get(
-            url, timeout=aiohttp.ClientTimeout(total=_ARTICLE_TIMEOUT_SEC)
-        )
+        response = await session.get(url, **kwargs)
         async with response:
             response.raise_for_status()
             payload = await response.read()
     except (aiohttp.ClientError, TimeoutError):
         return None
-    return _extract_body(payload)
+    return _extract_body(payload, url)

openbb_platform/extensions/news/openbb_news/registry.py

@@ -10,6 +10,15 @@ _PROVIDER_LABELS: dict[str, str] = {
     "benzinga": "Benzinga",
     "cbc": "CBC",
     "drudge_report": "Drudge Report",
+    "drugs_com": "Drugs.com",
+    "brutalist_tech": "Brutalist — Tech",
+    "brutalist_news": "Brutalist — News",
+    "brutalist_business": "Brutalist — Business",
+    "brutalist_science": "Brutalist — Science",
+    "brutalist_gaming": "Brutalist — Gaming",
+    "brutalist_culture": "Brutalist — Culture",
+    "brutalist_politics": "Brutalist — Politics",
+    "brutalist_sports": "Brutalist — Sports",
     "fox_news": "Fox News",
     "fortune": "Fortune",
     "globenewswire": "GlobeNewswire",
@@ -597,6 +606,36 @@ _CURATED_FEEDS: dict[str, tuple[str, str, str]] = {
         "Sports — Soccer",
         "https://www.cbc.ca/webfeed/rss/rss-sports-soccer",
     ),
+    "drugs_com_medical_news": (
+        "drugs_com",
+        "Daily MedNews",
+        "https://www.drugs.com/feeds/medical_news.xml",
+    ),
+    "drugs_com_headline_news": (
+        "drugs_com",
+        "News for Health Professionals",
+        "https://www.drugs.com/feeds/headline_news.xml",
+    ),
+    "drugs_com_fda_alerts": (
+        "drugs_com",
+        "FDA MedWatch Drug Alerts",
+        "https://www.drugs.com/feeds/fda_alerts.xml",
+    ),
+    "drugs_com_new_drug_approvals": (
+        "drugs_com",
+        "New Drug Approvals",
+        "https://www.drugs.com/feeds/new_drug_approvals.xml",
+    ),
+    "drugs_com_new_drug_applications": (
+        "drugs_com",
+        "New Drug Applications",
+        "https://www.drugs.com/feeds/new_drug_applications.xml",
+    ),
+    "drugs_com_clinical_trials": (
+        "drugs_com",
+        "Clinical Trial Results",
+        "https://www.drugs.com/feeds/clinical_trials.xml",
+    ),
     "fox_news_latest": (
         "fox_news",
         "Latest Headlines",
@@ -720,6 +759,171 @@ _CURATED_FEEDS: dict[str, tuple[str, str, str]] = {
     ),
 }
 
+# The publisher sources aggregated by brutalist.report, grouped under its eight
+# topics. Each entry is (feed_key_suffix, publisher_label, native_feed_url); the
+# provider id is ``brutalist_<topic>``. Feeds validated through the shared async
+# session. Hosts needing pinned request headers are handled in parser._HOST_HEADERS.
+_BRUTALIST_TOPIC_LABELS: dict[str, str] = {
+    "tech": "Tech",
+    "news": "News",
+    "business": "Business",
+    "science": "Science",
+    "gaming": "Gaming",
+    "culture": "Culture",
+    "politics": "Politics",
+    "sports": "Sports",
+}
+_BRUTALIST_FEEDS: dict[str, tuple[tuple[str, str, str], ...]] = {
+    "tech": (
+        (
+            "androidauthority",
+            "Android Authority",
+            "https://www.androidauthority.com/feed",
+        ),
+        ("appleinsider", "AppleInsider", "https://appleinsider.com/rss/news/"),
+        (
+            "arstechnica",
+            "ArsTechnica",
+            "https://feeds.arstechnica.com/arstechnica/index",
+        ),
+        (
+            "bleepingcomputer",
+            "Bleeping Computer",
+            "https://www.bleepingcomputer.com/feed/",
+        ),
+        ("cloudflare", "Cloudflare Blog", "https://www.blog.cloudflare.com/rss/"),
+        ("cnet", "CNET", "https://www.cnet.com/rss/news/"),
+        ("coindesk", "Coindesk", "https://www.coindesk.com/arc/outboundfeeds/rss/"),
+        ("daringfireball", "Daring Fireball", "https://daringfireball.net/feeds/main"),
+        ("engadget", "Engadget", "https://www.engadget.com/category/news/feed/"),
+        ("hackaday", "Hackaday", "https://hackaday.com/feed/"),
+        ("hn", "Hacker News", "https://news.ycombinator.com/rss"),
+        ("lwn", "Linux Weekly News", "https://lwn.net/headlines/rss"),
+        ("phoronix", "Phoronix", "https://www.phoronix.com/rss.php"),
+        ("servethehome", "Serve The Home", "https://www.servethehome.com/feed/"),
+        ("slashdot", "Slashdot", "https://rss.slashdot.org/Slashdot/slashdotMain"),
+        ("techmeme", "Techmeme", "https://www.techmeme.com/feed.xml"),
+        ("techradar", "TechRadar", "https://www.techradar.com/feeds.xml"),
+        ("register", "The Register", "https://www.theregister.com/headlines.atom"),
+        ("shortcut", "The Shortcut", "https://theshortcut.com/feed"),
+        ("verge", "The Verge", "https://www.theverge.com/rss/index.xml"),
+        ("venturebeat", "Venturebeat", "https://venturebeat.com/feed/"),
+        ("wired", "Wired", "https://www.wired.com/feed/rss"),
+    ),
+    "news": (
+        ("abcnews", "ABC News", "https://feeds.abcnews.com/abcnews/topstories"),
+        ("aljazeera", "Al Jazeera", "https://www.aljazeera.com/xml/rss/all.xml"),
+        ("bbc", "BBC", "https://feeds.bbci.co.uk/news/rss.xml"),
+        (
+            "foxnews",
+            "Fox News",
+            "https://moxie.foxnews.com/google-publisher/latest.xml",
+        ),
+        ("nbcnews", "NBC News", "https://feeds.nbcnews.com/nbcnews/public/news"),
+        ("npr", "NPR", "https://feeds.npr.org/1002/rss.xml"),
+        (
+            "pbsnewshour",
+            "PBS NewsHour",
+            "https://www.pbs.org/newshour/feeds/rss/headlines",
+        ),
+        ("quartz", "Quartz", "https://qz.com/rss"),
+        (
+            "conversation",
+            "The Conversation",
+            "https://theconversation.com/us/articles.atom",
+        ),
+        ("guardian", "The Guardian", "https://www.theguardian.com/international/rss"),
+        ("intercept", "The Intercept", "https://theintercept.com/feed/?rss"),
+        (
+            "nytimes",
+            "The New York Times",
+            "https://rss.nytimes.com/services/xml/rss/nyt/HomePage.xml",
+        ),
+        ("onion", "The Onion", "https://theonion.com/feed/"),
+        (
+            "wsj",
+            "The Wall Street Journal",
+            "https://feeds.a.dj.com/rss/RSSWorldNews.xml",
+        ),
+    ),
+    "business": (
+        (
+            "businessinsider",
+            "Business Insider",
+            "https://feeds.businessinsider.com/custom/all",
+        ),
+        ("cnbc", "CNBC", "https://www.cnbc.com/id/100003114/device/rss/rss.html"),
+        ("fastcompany", "FastCompany", "https://www.fastcompany.com/latest/rss"),
+        ("fortune", "Fortune", "https://fortune.com/feed/fortune-feeds/?id=3230629"),
+        (
+            "hbr",
+            "Harvard Business Review",
+            "http://feeds.harvardbusiness.org/harvardbusiness/",
+        ),
+        (
+            "marketwatch",
+            "MarketWatch",
+            "https://feeds.content.dowjones.io/public/rss/mw_topstories",
+        ),
+        ("yahoofinance", "Yahoo Finance", "https://finance.yahoo.com/news/rssindex"),
+        ("zerohedge", "Zero Hedge", "https://feeds.feedburner.com/zerohedge/feed"),
+    ),
+    "science": (
+        ("phys", "Phys", "https://phys.org/rss-feed/"),
+        ("quantamag", "Quanta Magazine", "https://www.quantamagazine.org/quanta/feed/"),
+        ("sciencedaily", "ScienceDaily", "https://www.sciencedaily.com/rss/all.xml"),
+        (
+            "scientificamerican",
+            "Scientific American",
+            "https://www.scientificamerican.com/platform/syndication/rss/",
+        ),
+    ),
+    "gaming": (
+        ("gamespot", "GameSpot", "https://www.gamespot.com/feeds/mashup/"),
+        ("kotaku", "Kotaku", "https://kotaku.com/feed"),
+        ("pcgamer", "PC Gamer", "https://www.pcgamer.com/feeds.xml"),
+        ("polygon", "Polygon", "https://www.polygon.com/rss/index.xml"),
+    ),
+    "culture": (
+        ("billboard", "Billboard", "https://www.billboard.com/feed/rss/"),
+        ("esquire", "Esquire", "https://www.esquire.com/rss/all.xml/"),
+        ("heavy", "Heavy", "https://heavy.com/feed/"),
+        ("mashable", "Mashable", "https://mashable.com/feeds/rss/all"),
+        ("reddit", "Reddit Front Page", "https://www.reddit.com/.rss"),
+        ("avclub", "The A.V. Club", "https://www.avclub.com/rss"),
+        ("newyorker", "The New Yorker", "https://www.newyorker.com/feed/rss"),
+        ("thepointsguy", "The Points Guy", "https://thepointsguy.com/feed/"),
+        ("tmz", "TMZ", "https://www.tmz.com/rss.xml"),
+        ("vanityfair", "Vanity Fair", "https://www.vanityfair.com/feed/rss"),
+        ("variety", "Variety", "https://variety.com/feed/rss/"),
+        ("vox", "Vox", "https://www.vox.com/rss/index.xml"),
+    ),
+    "politics": (
+        ("axios", "Axios", "https://api.axios.com/feed/"),
+        ("salon", "Salon", "https://www.salon.com/feed/"),
+        (
+            "talkingpointsmemo",
+            "Talking Points Memo",
+            "https://talkingpointsmemo.com/feed",
+        ),
+        (
+            "drudge",
+            "The Drudge Report",
+            "https://feeds.feedburner.com/DrudgeReportFeed",
+        ),
+        ("hill", "The Hill", "https://thehill.com/news/feed/"),
+        ("thenewrepublic", "The New Republic", "https://newrepublic.com/rss.xml"),
+    ),
+    "sports": (
+        ("espn_mlb", "ESPN MLB", "https://www.espn.com/espn/rss/mlb/news"),
+        ("espn_nba", "ESPN NBA", "https://www.espn.com/espn/rss/nba/news"),
+        ("espn_nfl", "ESPN NFL", "https://www.espn.com/espn/rss/nfl/news"),
+        ("espn_nhl", "ESPN NHL", "https://www.espn.com/espn/rss/nhl/news"),
+        ("espn_soccer", "ESPN SOCCER", "https://www.espn.com/espn/rss/soccer/news"),
+        ("yahoosports", "Yahoo Sports", "https://sports.yahoo.com/rss/"),
+    ),
+}
+
 
 def _slug(value: str) -> str:
     return value.replace("/", "_").replace("-", "_")
@@ -791,6 +995,13 @@ def _build_feed_registry() -> tuple[dict[str, str], dict[str, dict[str, str]]]:
         feeds[key] = url
         meta[key] = {"provider": provider, "label": label}
 
+    for topic, entries in _BRUTALIST_FEEDS.items():
+        provider = f"brutalist_{topic}"
+        for suffix, label, url in entries:
+            key = f"{provider}_{suffix}"
+            feeds[key] = url
+            meta[key] = {"provider": provider, "label": label}
+
     return feeds, meta
 
 
@@ -924,11 +1135,20 @@ _DEFAULT_FEED_BY_PROVIDER: dict[str, str] = {
     "bbc": "bbc_world",
     "benzinga": "benzinga_markets",
     "cbc": "cbc_business",
+    "drugs_com": "drugs_com_medical_news",
     "fox_news": "fox_news_latest",
     "globenewswire": "globenewswire_all",
     "google_news": "google_news_us",
     "pr_newswire": "pr_newswire_global",
     "wired": "wired_business",
+    "brutalist_tech": "brutalist_tech_verge",
+    "brutalist_news": "brutalist_news_guardian",
+    "brutalist_business": "brutalist_business_fortune",
+    "brutalist_science": "brutalist_science_scientificamerican",
+    "brutalist_gaming": "brutalist_gaming_polygon",
+    "brutalist_culture": "brutalist_culture_vox",
+    "brutalist_politics": "brutalist_politics_axios",
+    "brutalist_sports": "brutalist_sports_yahoosports",
 }
 
 

openbb_platform/extensions/news/openbb_news/rss.py

@@ -166,7 +166,16 @@ async def rss(
     for entry, fetched_body in zip(entries, bodies):
         summary_html = entry.get("summary") or entry.get("description") or ""
         excerpt = truncate(strip_html(summary_html))
-        body = fetched_body or html_to_markdown(summary_html) or excerpt
+        url = (entry.get("link") or "").strip()
+        if fetched_body:
+            body = fetched_body
+        else:
+            # No full body — some sources paywall or block article pages (NYT,
+            # WSJ). Fall back to the summary and link out so the reader can
+            # still reach the full story.
+            body = html_to_markdown(summary_html) or excerpt
+            if url:
+                body = f"{body}\n\n[Read the full story at the source ↗]({url})"
         out.append(
             NewsItemData(
                 title=(entry.get("title") or "(untitled)").strip(),
@@ -174,7 +183,7 @@ async def rss(
                     entry.get("published_parsed") or entry.get("updated_parsed")
                 ),
                 author=entry.get("author") or feed_title or "Unknown",
-                url=(entry.get("link") or "").strip(),
+                url=url,
                 excerpt=excerpt,
                 body=body,
             )

openbb_platform/extensions/news/tests/conftest.py

@@ -38,6 +38,46 @@ _SAMPLE_JSONLD_HTML = b"""<!DOCTYPE html><html><body>
 </body></html>
 """
 
+# Mirrors the real drugs.com article DOM: no <article>, sidebar inside <main>,
+# and trailing furniture after the disclaimer.
+_SAMPLE_DRUGS_COM_HTML = b"""<!DOCTYPE html><html><head>
+<meta property="og:image" content="https://www.drugs.com/img/social/ddc-opengraph-logomark.png">
+</head><body>
+<main id="container" class="ddc-main-container ddc-width-container">
+<div class="ddc-main-content">
+<div class="ddc-main-content-head">Home News Consumer News Print page</div>
+<nav>All News Consumer Pro New Drugs</nav>
+<h1 class="ddc-title-length-xl">FDA Clears First Cholesterol Pill</h1>
+<p class="ddc-mgt-2 ddc-mgb-0 ddc-media-metadata">By Ellyn Vohnoutka HealthDay Reporter</p>
+<!-- comment node between children -->
+<p>THURSDAY, July 16, 2026 &mdash; A new daily pill will lower cholesterol.</p>
+<p>The FDA acted today to approve the drug for adults.</p>
+<div class="ddc-reference-list"><ul><li>Sources</li></ul></div>
+<p class="ddc-disclaimer">Disclaimer: Statistical data in medical articles provide general trends.</p>
+<div class="ddc-mgt-3 ddc-mgb-3"><img src="/img/logo/vendor/healthday-logo.png"/>
+<p class="ddc-mgt-0">&copy; 2026 HealthDay. All rights reserved.</p></div>
+<div class="more-resources"><h2>Read this next</h2>
+<div class="ddc-media-list"><p>Related teaser one.</p><p>Related teaser two.</p></div></div>
+<h2>More news resources</h2>
+<ul><li>FDA Medwatch Drug Alerts</li></ul>
+<h2>Subscribe to our newsletter</h2>
+<p>Whatever your topic of interest, subscribe to our newsletters.</p>
+</div>
+<div class="ddc-main-sidebar">
+<div class="ddc-sidebox ddc-sidebox-podcast"><img src="/img/banners/ddc-podcast-cover.png"/>
+<p>Podcast pitch paragraph.</p></div>
+<div class="ddc-sidebox ddc-sidebox-news"><p>Sidebar drug teaser one.</p>
+<p>Sidebar drug teaser two.</p></div>
+</div>
+</main>
+</body></html>
+"""
+
+
+@pytest.fixture
+def sample_drugs_com_html() -> bytes:
+    return _SAMPLE_DRUGS_COM_HTML
+
 
 @pytest.fixture
 def sample_rss() -> bytes:
@@ -83,10 +123,12 @@ class _FakeSession:
     def __init__(self, response_map: dict) -> None:
         self.response_map = response_map
         self.calls: list[str] = []
+        self.headers_sent: list[dict | None] = []
         self.closed = False
 
-    async def get(self, url: str, **_kwargs: Any) -> _FakeResponse:
+    async def get(self, url: str, **kwargs: Any) -> _FakeResponse:
         self.calls.append(url)
+        self.headers_sent.append(kwargs.get("headers"))
         result = self.response_map.get(url)
         if result is None:
             raise KeyError(f"unmocked URL: {url}")

openbb_platform/extensions/news/tests/test_parser.py

@@ -232,6 +232,20 @@ async def test_fetch_feed_http_error(stub_session_factory):
     assert len(parsed.entries) == 0
 
 
+async def test_fetch_feed_pins_headers_for_gated_host(stub_session_factory, sample_rss):
+    url = "https://www.espn.com/espn/rss/nfl/news"
+    session = stub_session_factory({url: sample_rss})
+    await parser.fetch_feed(session, url)
+    assert session.headers_sent == [parser._HOST_HEADERS["espn.com"]]
+
+
+async def test_fetch_feed_no_headers_for_normal_host(stub_session_factory, sample_rss):
+    url = "https://feeds.bbci.co.uk/news/rss.xml"
+    session = stub_session_factory({url: sample_rss})
+    await parser.fetch_feed(session, url)
+    assert session.headers_sent == [None]
+
+
 def test_extract_body_jsonld_top_level(sample_jsonld_html):
     assert parser._extract_body(sample_jsonld_html) == "JSON-LD top-level body."
 
@@ -592,6 +606,211 @@ def test_jsonld_field_image_object_with_url():
     assert body.startswith("![](https://x.test/obj.jpg)")
 
 
+DRUGS_COM_URL = "https://www.drugs.com/news/fda-clears-cholesterol-pill-130672.html"
+
+
+def test_host_strips_www():
+    assert parser._host("https://www.drugs.com/news/x.html") == "drugs.com"
+
+
+def test_host_without_www():
+    assert parser._host("https://drugs.com/news/x.html") == "drugs.com"
+
+
+def test_host_is_lowercased():
+    assert parser._host("https://WWW.Drugs.COM/news/x.html") == "drugs.com"
+
+
+def test_host_empty_url():
+    assert parser._host(None) == ""
+    assert parser._host("") == ""
+
+
+def test_host_url_without_hostname():
+    assert parser._host("not-a-url") == ""
+
+
+def test_cut_tail_removes_marker_and_everything_after():
+    doc = html.fromstring(
+        b"<div id='c'>"
+        b"<p>Keep one.</p>"
+        b"<!-- comment -->"
+        b"<p>Keep two.</p>"
+        b"<h2>More news resources</h2>"
+        b"<ul><li>link</li></ul>"
+        b"<p>Newsletter pitch.</p>"
+        b"</div>"
+    )
+    container = doc.xpath("//div")[0]
+    parser._cut_tail(container, parser._SITE_RULES["drugs.com"].cut)
+    text = container.text_content()
+    assert "Keep one." in text
+    assert "Keep two." in text
+    assert "More news resources" not in text
+    assert "Newsletter pitch." not in text
+
+
+def test_cut_tail_without_marker_keeps_all():
+    doc = html.fromstring(b"<div><p>One.</p><p>Two.</p></div>")
+    container = doc.xpath("//div")[0]
+    parser._cut_tail(container, parser._SITE_RULES["drugs.com"].cut)
+    assert len(container.xpath(".//p")) == 2
+
+
+def test_site_body_no_container_returns_empty():
+    doc = html.fromstring(b"<html><body><p>One.</p><p>Two.</p></body></html>")
+    assert parser._site_body(doc, parser._SITE_RULES["drugs.com"]) == ""
+
+
+def test_site_body_picks_largest_container():
+    doc = html.fromstring(
+        b"<html><body>"
+        b"<div class='ddc-main-content'>"
+        b"<p>Short one.</p><p>Short two.</p></div>"
+        b"<div class='ddc-main-content'>"
+        b"<p>Longer container paragraph one with much more substance.</p>"
+        b"<p>Longer container paragraph two with additional detail.</p></div>"
+        b"</body></html>"
+    )
+    body = parser._site_body(doc, parser._SITE_RULES["drugs.com"])
+    assert "Longer container paragraph one" in body
+    assert "Short one." not in body
+
+
+def test_site_body_skips_unclean_container():
+    doc = html.fromstring(
+        b"<html><body><div class='ddc-main-content'>"
+        b"<p>[[ template ]] junk</p><p>{{ binding }} more junk</p>"
+        b"</div></body></html>"
+    )
+    assert parser._site_body(doc, parser._SITE_RULES["drugs.com"]) == ""
+
+
+def test_extract_body_drugs_com_keeps_article_text(sample_drugs_com_html):
+    body = parser._extract_body(sample_drugs_com_html, DRUGS_COM_URL)
+    assert body is not None
+    assert "By Ellyn Vohnoutka HealthDay Reporter" in body
+    assert "A new daily pill will lower cholesterol." in body
+    assert "The FDA acted today to approve the drug for adults." in body
+
+
+def test_extract_body_drugs_com_drops_trailing_furniture(sample_drugs_com_html):
+    body = parser._extract_body(sample_drugs_com_html, DRUGS_COM_URL)
+    assert body is not None
+    assert "Disclaimer:" not in body
+    assert "HealthDay. All rights reserved." not in body
+    assert "Related teaser one." not in body
+    assert "Whatever your topic of interest" not in body
+
+
+def test_extract_body_drugs_com_drops_sidebar(sample_drugs_com_html):
+    body = parser._extract_body(sample_drugs_com_html, DRUGS_COM_URL)
+    assert body is not None
+    assert "Sidebar drug teaser one." not in body
+    assert "Podcast pitch paragraph." not in body
+    assert "ddc-podcast-cover" not in body
+
+
+def test_extract_body_drugs_com_skips_placeholder_hero(sample_drugs_com_html):
+    body = parser._extract_body(sample_drugs_com_html, DRUGS_COM_URL)
+    assert body is not None
+    assert "ddc-opengraph-logomark" not in body
+    assert not body.startswith("![")
+
+
+def test_extract_body_drugs_com_keeps_real_hero():
+    payload = (
+        b"<html><head>"
+        b'<meta property="og:image" content="https://media.test/featured.jpg">'
+        b"</head><body><div class='ddc-main-content'>"
+        b"<p>Lede paragraph of the story.</p><p>Second paragraph.</p>"
+        b"</div></body></html>"
+    )
+    body = parser._extract_body(payload, DRUGS_COM_URL)
+    assert body is not None
+    assert body.split("\n\n")[0] == "![](https://media.test/featured.jpg)"
+
+
+def test_extract_body_drugs_com_falls_back_when_container_missing():
+    payload = (
+        b"<html><body><article>"
+        b"<p>Generic paragraph one.</p><p>Generic paragraph two.</p>"
+        b"</article></body></html>"
+    )
+    body = parser._extract_body(payload, DRUGS_COM_URL)
+    assert body == "Generic paragraph one.\n\nGeneric paragraph two."
+
+
+def test_extract_body_unknown_host_uses_generic_path(sample_drugs_com_html):
+    body = parser._extract_body(sample_drugs_com_html, "https://other.test/x.html")
+    assert body is not None
+    assert "Whatever your topic of interest" in body
+
+
+def test_extract_body_without_url_uses_generic_path(sample_drugs_com_html):
+    body = parser._extract_body(sample_drugs_com_html)
+    assert body is not None
+    assert "Whatever your topic of interest" in body
+
+
+async def test_fetch_article_body_applies_site_rule(
+    stub_session_factory, sample_drugs_com_html
+):
+    session = stub_session_factory({DRUGS_COM_URL: sample_drugs_com_html})
+    body = await parser.fetch_article_body(session, DRUGS_COM_URL)
+    assert body is not None
+    assert "A new daily pill will lower cholesterol." in body
+    assert "Whatever your topic of interest" not in body
+
+
+async def test_fetch_article_body_pins_user_agent_for_drugs_com(
+    stub_session_factory, sample_drugs_com_html
+):
+    session = stub_session_factory({DRUGS_COM_URL: sample_drugs_com_html})
+    await parser.fetch_article_body(session, DRUGS_COM_URL)
+    assert session.headers_sent == [parser._HOST_HEADERS["drugs.com"]]
+    assert session.headers_sent[0]["User-Agent"] == parser._WEBKIT_UA
+
+
+def test_drugs_com_pinned_user_agent_is_not_gecko():
+    # drugs.com articles 403 any Gecko user agent.
+    ua = parser._HOST_HEADERS["drugs.com"]["User-Agent"]
+    assert "Gecko/" not in ua
+    assert "AppleWebKit" in ua
+
+
+def test_host_headers_pin_safe_encoding():
+    # Overriding headers drops the session default Accept-Encoding; every pinned
+    # host must exclude br, which the shared session cannot decode.
+    for host, headers in parser._HOST_HEADERS.items():
+        assert headers["Accept-Encoding"] == "gzip, deflate", host
+        assert "br" not in headers["Accept-Encoding"], host
+
+
+def test_host_headers_cover_expected_hosts():
+    assert set(parser._HOST_HEADERS) == {
+        "drugs.com",
+        "espn.com",
+        "heavy.com",
+        "bleepingcomputer.com",
+        "pbs.org",
+        "techmeme.com",
+    }
+
+
+def test_pinned_helper_omits_user_agent_when_none():
+    assert "User-Agent" not in parser._pinned()
+    assert parser._pinned("UA/1.0")["User-Agent"] == "UA/1.0"
+
+
+async def test_fetch_article_body_keeps_session_user_agent_for_other_hosts(
+    stub_session_factory, sample_article_html
+):
+    session = stub_session_factory({"https://x.test/a": sample_article_html})
+    await parser.fetch_article_body(session, "https://x.test/a")
+    assert session.headers_sent == [None]
+
+
 def test_extract_image_jsonld_fallback_path():
     payload = (
         b"<html><body>"

openbb_platform/extensions/news/tests/test_registry.py

@@ -55,6 +55,7 @@ def test_list_providers_default(monkeypatch):
     assert providers["benzinga"] == "Benzinga"
     assert providers["cbc"] == "CBC"
     assert providers["drudge_report"] == "Drudge Report"
+    assert providers["drugs_com"] == "Drugs.com"
     assert providers["fox_news"] == "Fox News"
     assert providers["fortune"] == "Fortune"
     assert providers["globenewswire"] == "GlobeNewswire"
@@ -62,12 +63,129 @@ def test_list_providers_default(monkeypatch):
     assert providers["pr_newswire"] == "PR Newswire"
     assert providers["yahoo_finance"] == "Yahoo Finance"
     assert providers["wired"] == "Wired"
+    assert providers["brutalist_tech"] == "Brutalist — Tech"
+    assert providers["brutalist_business"] == "Brutalist — Business"
+    assert providers["brutalist_sports"] == "Brutalist — Sports"
     assert "custom" not in providers
     assert "barchart" not in providers
     assert "cnn" not in providers
     assert "seeking_alpha" not in providers
 
 
+def test_list_feed_choices_drugs_com(monkeypatch):
+    monkeypatch.setattr(registry, "load_config", lambda: {})
+    choices = registry.list_feed_choices("drugs_com")
+    labels = [c["label"] for c in choices]
+    assert "Daily MedNews" in labels
+    assert "News for Health Professionals" in labels
+    assert "FDA MedWatch Drug Alerts" in labels
+    assert "New Drug Approvals" in labels
+    assert "New Drug Applications" in labels
+    assert "Clinical Trial Results" in labels
+    assert len(choices) == 6
+    assert labels == sorted(labels)
+
+
+def test_drugs_com_feed_urls(monkeypatch):
+    monkeypatch.setattr(registry, "load_config", lambda: {})
+    assert (
+        registry.get_feed_url("drugs_com_medical_news")
+        == "https://www.drugs.com/feeds/medical_news.xml"
+    )
+    assert (
+        registry.get_feed_url("drugs_com_headline_news")
+        == "https://www.drugs.com/feeds/headline_news.xml"
+    )
+    assert (
+        registry.get_feed_url("drugs_com_fda_alerts")
+        == "https://www.drugs.com/feeds/fda_alerts.xml"
+    )
+    assert (
+        registry.get_feed_url("drugs_com_new_drug_approvals")
+        == "https://www.drugs.com/feeds/new_drug_approvals.xml"
+    )
+    assert (
+        registry.get_feed_url("drugs_com_new_drug_applications")
+        == "https://www.drugs.com/feeds/new_drug_applications.xml"
+    )
+    assert (
+        registry.get_feed_url("drugs_com_clinical_trials")
+        == "https://www.drugs.com/feeds/clinical_trials.xml"
+    )
+
+
+_BRUTALIST_PROVIDER_COUNTS = {
+    "brutalist_tech": 22,
+    "brutalist_news": 14,
+    "brutalist_business": 8,
+    "brutalist_science": 4,
+    "brutalist_gaming": 4,
+    "brutalist_culture": 12,
+    "brutalist_politics": 6,
+    "brutalist_sports": 6,
+}
+
+
+def test_brutalist_providers_present(monkeypatch):
+    monkeypatch.setattr(registry, "load_config", lambda: {})
+    providers = registry.list_providers()
+    for pid in _BRUTALIST_PROVIDER_COUNTS:
+        assert pid in providers
+        assert providers[pid].startswith("Brutalist — ")
+
+
+def test_brutalist_feed_choices_per_topic(monkeypatch):
+    monkeypatch.setattr(registry, "load_config", lambda: {})
+    for pid, count in _BRUTALIST_PROVIDER_COUNTS.items():
+        choices = registry.list_feed_choices(pid)
+        labels = [c["label"] for c in choices]
+        assert len(choices) == count, pid
+        assert labels == sorted(labels), pid
+
+
+def test_brutalist_total_feed_count():
+    total = sum(len(v) for v in registry._BRUTALIST_FEEDS.values())
+    assert total == 76
+    assert set(registry._BRUTALIST_FEEDS) == set(registry._BRUTALIST_TOPIC_LABELS)
+
+
+def test_brutalist_feed_keys_unique_and_registered(monkeypatch):
+    monkeypatch.setattr(registry, "load_config", lambda: {})
+    feeds = registry.list_feeds()
+    keys = []
+    for topic, entries in registry._BRUTALIST_FEEDS.items():
+        for suffix, _label, url in entries:
+            key = f"brutalist_{topic}_{suffix}"
+            keys.append(key)
+            assert feeds.get(key) == url
+    assert len(keys) == len(set(keys))
+
+
+def test_brutalist_sample_feed_urls():
+    assert (
+        registry.get_feed_url("brutalist_tech_verge")
+        == "https://www.theverge.com/rss/index.xml"
+    )
+    assert (
+        registry.get_feed_url("brutalist_sports_espn_nfl")
+        == "https://www.espn.com/espn/rss/nfl/news"
+    )
+    assert (
+        registry.get_feed_url("brutalist_politics_axios")
+        == "https://api.axios.com/feed/"
+    )
+
+
+def test_brutalist_defaults_resolve(monkeypatch):
+    monkeypatch.setattr(registry, "load_config", lambda: {})
+    feeds = registry.list_feeds()
+    for pid in _BRUTALIST_PROVIDER_COUNTS:
+        default = registry.default_feed_for(pid)
+        assert default is not None
+        assert default in feeds
+        assert default.startswith(pid + "_")
+
+
 def test_get_feed_url_drudge_report():
     assert (
         registry.get_feed_url("drudge_report")
@@ -223,6 +341,7 @@ def test_default_feed_for_curated_picks(monkeypatch):
         "bbc": "bbc_world",
         "benzinga": "benzinga_markets",
         "cbc": "cbc_business",
+        "drugs_com": "drugs_com_medical_news",
         "fox_news": "fox_news_latest",
         "globenewswire": "globenewswire_all",
         "google_news": "google_news_us",

openbb_platform/extensions/news/tests/test_rss.py

@@ -29,7 +29,10 @@ async def test_rss_with_body(mocked_pipeline):
     assert items[0].excerpt == "A short summary."
     assert items[0].url == "https://example.test/article/1"
     assert "First paragraph" in items[0].body
+    # The extracted article body is used verbatim — no source link appended.
+    assert "Read the full story" not in items[0].body
     assert items[1].excerpt == "Plain text summary without HTML."
+    # Item 1 has no link, so the summary stands alone.
     assert items[1].body == items[1].excerpt
     assert items[1].url == ""
     assert items[1].author == "Sample Feed"
@@ -37,7 +40,45 @@ async def test_rss_with_body(mocked_pipeline):
 
 async def test_rss_without_body(mocked_pipeline):
     out = await rss.rss(source="example", limit=5, fetch_body=False)
-    assert all(item.body == item.excerpt for item in out.results)
+    results = out.results
+    # Item 0 has a link: the summary is followed by a source link.
+    assert results[0].body.startswith(results[0].excerpt)
+    assert (
+        "[Read the full story at the source ↗](https://example.test/article/1)"
+        in results[0].body
+    )
+    # Item 1 has no link, so the summary stands alone.
+    assert results[1].body == results[1].excerpt
+
+
+async def test_rss_appends_source_link_when_body_fetch_fails(
+    monkeypatch, stub_session_factory
+):
+    feed = (
+        b"<?xml version='1.0' encoding='UTF-8'?>"
+        b'<rss version="2.0"><channel><title>Paywalled</title>'
+        b"<item><title>Locked Story</title>"
+        b"<link>https://paywall.test/story</link>"
+        b"<description>Just the teaser.</description>"
+        b"<pubDate>Mon, 19 May 2026 12:00:00 GMT</pubDate>"
+        b"</item></channel></rss>"
+    )
+    monkeypatch.setattr(
+        "openbb_news.registry.get_feed_url", lambda key: "http://feed.test/pw"
+    )
+    stub_session_factory(
+        {
+            "http://feed.test/pw": feed,
+            # Article page 403s / yields no extractable body.
+            "https://paywall.test/story": (b"Access Denied", 403),
+        }
+    )
+    item = (await rss.rss(source="x", limit=1, fetch_body=True)).results[0]
+    assert item.excerpt == "Just the teaser."
+    assert item.body == (
+        "Just the teaser.\n\n"
+        "[Read the full story at the source ↗](https://paywall.test/story)"
+    )
 
 
 async def test_rss_limit_caps_entries(mocked_pipeline):