[V5] openbb-news: Refactor News for V5 and Add Configurable RSS Endpoint (#7507)

* refactor news for v5 and add rss endpoint

* remove poetry.lock

---------

Co-authored-by: deeleeramone <>
This commit is contained in:
Danglewood
2026-05-29 00:45:36 -07:00
committed by GitHub
parent 8ea35b0bd0
commit 43b14e1746
18 changed files with 7271 additions and 2375 deletions

View File

@@ -0,0 +1,163 @@
name: 📦 publish openbb-news
# Triggers exclusively on the merge of a ``release/openbb-news-v*``
# branch into ``develop`` (or ``v5``). Four sequential jobs — verify the
# version bump, build once, test the built source across the full
# cross-platform matrix, and only publish to PyPI if the matrix is green.
on:
pull_request:
types: [closed]
branches: [develop, v5]
workflow_dispatch:
permissions:
contents: read
jobs:
version-check:
name: Verify version bump
if: ${{ (github.event_name == 'pull_request' && github.event.pull_request.merged == true && startsWith(github.event.pull_request.head.ref, 'release/openbb-news-v')) || github.event_name == 'workflow_dispatch' }}
runs-on: ubuntu-latest
outputs:
version: ${{ steps.parse.outputs.version }}
defaults:
run:
shell: bash
working-directory: openbb_platform/extensions/news
steps:
- uses: actions/checkout@v6
with:
ref: develop
- name: Parse [project].version from pyproject.toml
id: parse
run: |
VERSION=$(grep -m1 -E '^version[[:space:]]*=' pyproject.toml | sed -E 's/^version[[:space:]]*=[[:space:]]*"([^"]+)".*/\1/')
if [ -z "$VERSION" ]; then
echo "::error::Could not parse [project].version from pyproject.toml."
exit 1
fi
echo "Local version: $VERSION"
echo "version=$VERSION" >> "$GITHUB_OUTPUT"
- name: Fail if version already exists on PyPI
env:
VERSION: ${{ steps.parse.outputs.version }}
run: |
STATUS=$(curl -sS -o /dev/null -w "%{http_code}" "https://pypi.org/pypi/openbb-news/${VERSION}/json")
case "$STATUS" in
200)
echo "::error::openbb-news ${VERSION} is already published on PyPI."
echo "Bump openbb_platform/extensions/news/pyproject.toml [project].version and re-merge a fresh release branch."
exit 1
;;
404)
echo "openbb-news ${VERSION} is not yet on PyPI — proceeding."
;;
*)
echo "::error::Unexpected status $STATUS from PyPI JSON API. Aborting to avoid a silent skip."
exit 1
;;
esac
- name: Verify release branch name matches pyproject version
if: github.event_name == 'pull_request'
env:
VERSION: ${{ steps.parse.outputs.version }}
BRANCH: ${{ github.event.pull_request.head.ref }}
run: |
EXPECTED="release/openbb-news-v${VERSION}"
if [ "$BRANCH" != "$EXPECTED" ]; then
echo "::error::Release branch '${BRANCH}' does not match pyproject version '${VERSION}'."
echo "Expected branch name: '${EXPECTED}'."
echo "Either rename the release branch or update [project].version in pyproject.toml so they agree."
exit 1
fi
echo "Branch '${BRANCH}' matches pyproject version '${VERSION}'."
build:
name: Build sdist + wheel
needs: version-check
runs-on: ubuntu-latest
defaults:
run:
shell: bash
working-directory: openbb_platform/extensions/news
steps:
- uses: actions/checkout@v6
with:
ref: develop
- uses: actions/setup-python@v6
with:
python-version: "3.11"
- run: python -m pip install --upgrade pip uv
- name: Build sdist + wheel
run: uv build --no-sources
- uses: actions/upload-artifact@v4
with:
name: openbb-news-dist
path: openbb_platform/extensions/news/dist/
if-no-files-found: error
retention-days: 7
test:
name: Test (${{ matrix.os }} / Python ${{ matrix.python_version }})
needs: build
strategy:
fail-fast: false
matrix:
include:
- { os: ubuntu-latest, python_version: "3.10" }
- { os: ubuntu-latest, python_version: "3.11" }
- { os: ubuntu-latest, python_version: "3.12" }
- { os: ubuntu-latest, python_version: "3.13" }
- { os: ubuntu-latest, python_version: "3.14" }
- { os: macos-latest, python_version: "3.10" }
- { os: macos-latest, python_version: "3.14" }
- { os: windows-latest, python_version: "3.10" }
- { os: windows-latest, python_version: "3.14" }
runs-on: ${{ matrix.os }}
defaults:
run:
shell: bash
working-directory: openbb_platform/extensions/news
steps:
- uses: actions/checkout@v6
with:
ref: develop
- uses: actions/setup-python@v6
with:
python-version: ${{ matrix.python_version }}
allow-prereleases: true
- run: python -m pip install --upgrade pip uv
- name: Install openbb-news from source (+ dev group)
run: |
uv pip install --system -e .
uv pip install --system --group dev
- name: Run unit tests
run: pytest tests --cov=openbb_news --cov-report=term-missing
publish:
name: Publish to PyPI
needs: test
runs-on: ubuntu-latest
permissions:
id-token: write # required for PyPI OIDC trusted publishing
steps:
- uses: actions/download-artifact@v4
with:
name: openbb-news-dist
path: dist
- uses: pypa/gh-action-pypi-publish@release/v1
with:
packages-dir: dist

View File

@@ -0,0 +1,122 @@
name: 🧪 test openbb-news
# PR-test for the News extension. Triggers when a PR touches any file
# under ``openbb_platform/extensions/news/`` or this workflow itself.
#
# Build-once, test-many: a single ``build`` job produces the wheel + sdist;
# the ``lint`` and ``test`` jobs install the extension editable from source
# (``[tool.uv.sources]`` pins openbb-core to the sibling tree, since
# openbb-core>=2.0.0 is not yet on PyPI) and run across every supported
# (OS, Python) combination. Unit tests are pure compute — no network.
on:
pull_request:
branches: [develop, main, v5, feature/v5-platform-api]
paths:
- "openbb_platform/extensions/news/**"
- ".github/workflows/test-unit-openbb-news.yml"
workflow_dispatch:
permissions:
contents: read
concurrency:
group: test-openbb-news-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
build:
name: Build sdist + wheel
runs-on: ubuntu-latest
defaults:
run:
shell: bash
working-directory: openbb_platform/extensions/news
steps:
- uses: actions/checkout@v6
- uses: actions/setup-python@v6
with:
python-version: "3.11"
- run: python -m pip install --upgrade pip uv
- name: Build sdist + wheel
run: uv build --no-sources
- uses: actions/upload-artifact@v4
with:
name: openbb-news-dist
path: openbb_platform/extensions/news/dist/
if-no-files-found: error
retention-days: 7
lint:
name: Lint (ruff + ty)
needs: build
runs-on: ubuntu-latest
defaults:
run:
shell: bash
working-directory: openbb_platform/extensions/news
steps:
- uses: actions/checkout@v6
- uses: actions/setup-python@v6
with:
python-version: "3.11"
- run: python -m pip install --upgrade pip uv
- name: Install openbb-news in editable mode (+ dev group)
run: |
uv pip install --system -e .
uv pip install --system --group dev
- name: ruff check
run: ruff check openbb_news tests integration
- name: ruff format --check
run: ruff format --check openbb_news tests integration
- name: ty check
run: ty check openbb_news
test:
name: Test (${{ matrix.os }} / Python ${{ matrix.python_version }})
needs: build
strategy:
fail-fast: false
matrix:
include:
- { os: ubuntu-latest, python_version: "3.10" }
- { os: ubuntu-latest, python_version: "3.11" }
- { os: ubuntu-latest, python_version: "3.12" }
- { os: ubuntu-latest, python_version: "3.13" }
- { os: ubuntu-latest, python_version: "3.14" }
- { os: macos-latest, python_version: "3.10" }
- { os: macos-latest, python_version: "3.14" }
- { os: windows-latest, python_version: "3.10" }
- { os: windows-latest, python_version: "3.14" }
runs-on: ${{ matrix.os }}
defaults:
run:
shell: bash
working-directory: openbb_platform/extensions/news
steps:
- uses: actions/checkout@v6
- uses: actions/setup-python@v6
with:
python-version: ${{ matrix.python_version }}
allow-prereleases: true
- run: python -m pip install --upgrade pip uv
- name: Install openbb-news from source (+ dev group)
run: |
uv pip install --system -e .
uv pip install --system --group dev
- name: Run unit tests
run: pytest tests --cov=openbb_news --cov-report=term-missing --cov-fail-under=100

View File

@@ -1,13 +1,119 @@
# OpenBB News Extension
# openbb-news
This extension provides news for the OpenBB Platform.
RSS news feeds for the OpenBB Platform — one Workspace Newsfeed widget backed by
a registry of 400+ vetted feeds (Benzinga, GlobeNewswire by full ICB industry
breakdown + subject/event-type breakdown — earnings, M&A, dividends, IPOs, etc.,
PR Newswire regional + category, Axios, Yahoo Finance, Fortune, BBC, Fox News,
Drudge Report, Google News regional + topic). The widget defaults to
**Benzinga → Markets** so it loads finance-relevant articles immediately on
first open.
## Installation
To install the extension, run the following command in this folder:
## Install
```bash
pip install openbb-news
uv pip install -e .
```
Documentation available [here](https://docs.openbb.co/platform/developer_guide/contributing).
## Endpoints
| GET path | Returns |
| --- | --- |
| `/api/v1/news/rss?outlet=<id>&source=<key>&limit=<n>&fetch_body=<bool>` | `OBBject[list[NewsItemData]]` — `{title, date, author, url, excerpt, body}` per article. Empty when `source` is unset and no default can be resolved. |
| `/api/v1/news/rss_providers` | `list[{label, value}]` — drives the widget's **Provider** dropdown. |
| `/api/v1/news/rss_feeds?outlet=<id>` | `list[{label, value}]` — drives the widget's **Feed** dropdown, filtered by the selected provider. |
`fetch_body=true` extracts the full article body from each entry's URL via
JSON-LD `articleBody` first, then the largest `<article>` element after
stripping site chrome (`nav`/`header`/`footer`/`aside`/`script`/`style`/`form`/
`iframe`/`button`/`svg`). Requires at least two real `<p>` paragraphs and
rejects pages whose body contains unrendered template markers (`[[`, `{{`);
falls back to the RSS summary on any miss. Bounded `lru_cache(maxsize=256)`
keeps repeat fetches in-process; no disk persistence.
## Pluggable feeds — `openbb.toml`
Three levers in `~/.openbb_platform/openbb.toml`, all under `[news]`:
### 1. Add or replace feeds — `[news.rss_feeds]`
**Simple form** — `key = "url-string"`. The feed appears under a single
"Custom" provider; the label is a humanized form of the key.
```toml
[news.rss_feeds]
my_internal = "https://internal.example.com/news.rss"
acme_blog = "https://acme.example.com/feed"
```
**Rich form** — `key = { url, provider, label }` (or the equivalent table
syntax). Lets you bucket feeds into your own provider and give them a
display label, so the widget's Provider → Feed cascade groups them.
```toml
[news.rss_feeds.alpha_research]
url = "https://alpha.example.com/rss"
provider = "acme_research"
label = "Alpha Research"
[news.rss_feeds.beta_research]
url = "https://acme.example.com/beta/rss"
provider = "acme_research"
label = "Beta Research"
```
### 2. Pretty provider names — `[news.rss_providers]`
By default a provider id is humanized (`acme_research` → "Acme Research"). To
control the display label exactly, map provider id → label:
```toml
[news.rss_providers]
acme_research = "Acme Research"
internal = "Internal Sources"
```
### 3. Keep the bundled defaults — `[news] merge_defaults`
Without this flag, the presence of any `[news.rss_feeds]` entries replaces
the 438 bundled feeds entirely. Set `merge_defaults = true` to keep
everything and add your feeds on top. User keys that collide with bundled
keys override the bundled URL.
```toml
[news]
merge_defaults = true
[news.rss_feeds]
my_internal = "https://internal.example.com/news.rss"
```
### Full example
```toml
[news]
merge_defaults = true
[news.rss_providers]
acme_research = "Acme Research"
[news.rss_feeds.alpha]
url = "https://alpha.example.com/rss"
provider = "acme_research"
label = "Alpha Research"
[news.rss_feeds.beta]
url = "https://beta.example.com/rss"
provider = "acme_research"
label = "Beta Research"
```
The Workspace Newsfeed widget rebuilds its **Provider** + **Feed** dropdowns
from this config on each request — restart the server only if the file was
empty at boot.
## Tests
```bash
pytest tests --cov=openbb_news --cov-fail-under=100
pytest integration -m integration # hits the live feeds
```

View File

@@ -0,0 +1,34 @@
"""Integration fixtures."""
import threading
import time
import pytest
@pytest.fixture(scope="session", autouse=True)
def api_server(pytestconfig):
if pytestconfig.getoption("markexpr") == "not integration":
yield None
return
import uvicorn
from openbb_core.api.rest_api import app
config = uvicorn.Config(app, host="0.0.0.0", port=8000, log_level="warning") # noqa: S104
server = uvicorn.Server(config)
thread = threading.Thread(target=server.run, daemon=True)
thread.start()
deadline = time.time() + 60
while time.time() < deadline:
if server.started:
break
time.sleep(0.25)
else: # pragma: no cover
raise RuntimeError("The OpenBB REST API did not start within 60s.")
yield server
server.should_exit = True
thread.join(timeout=15)

View File

@@ -1,209 +1,88 @@
"""Test News API."""
"""HTTP API integration tests."""
import base64
import pytest
import requests
from openbb_core.env import Env
from openbb_core.provider.utils.helpers import get_querystring
_BASE_URL = "http://127.0.0.1:8000/api/v1/news"
_headers: dict = {}
@pytest.fixture(scope="session")
def headers():
"""Generate headers for API requests with basic authentication."""
def get_headers() -> dict:
if _headers:
return _headers
userpass = f"{Env().API_USERNAME}:{Env().API_PASSWORD}"
userpass_bytes = userpass.encode("ascii")
base64_bytes = base64.b64encode(userpass_bytes)
return {"Authorization": f"Basic {base64_bytes.decode('ascii')}"}
base64_bytes = base64.b64encode(userpass.encode("ascii"))
_headers["Authorization"] = f"Basic {base64_bytes.decode('ascii')}"
return _headers
# pylint: disable=redefined-outer-name
def _get(path: str, params: dict | None = None) -> requests.Response:
return requests.get(
f"{_BASE_URL}/{path}", headers=get_headers(), params=params, timeout=60
)
@pytest.mark.parametrize(
"params",
[
(
{
"display": "full",
"date": None,
"start_date": None,
"end_date": None,
"updated_since": None,
"published_since": None,
"sort": "created",
"order": "desc",
"isin": None,
"cusip": None,
"channels": "General",
"topics": "earnings",
"authors": None,
"content_types": "headline",
"provider": "benzinga",
"limit": 20,
}
),
(
{
"provider": "fmp",
"limit": 30,
"start_date": None,
"end_date": None,
"topic": "general",
"page": 1,
}
),
(
{
"provider": "intrinio",
"limit": 20,
"start_date": None,
"end_date": None,
"source": "yahoo",
"topic": None,
"is_spam": False,
"sentiment": None,
"language": None,
"word_count_greater_than": None,
"word_count_less_than": None,
"business_relevance_greater_than": None,
"business_relevance_less_than": None,
}
),
(
{
"provider": "biztoc",
"source": None,
"term": "microsoft",
"start_date": None,
"end_date": None,
}
),
(
{
"provider": "tiingo",
"limit": 30,
"source": "bloomberg.com",
"start_date": None,
"end_date": None,
"offset": 0,
}
),
],
)
@pytest.mark.integration
def test_news_world(params, headers):
"""Test retrieval of world news with various parameters."""
params = {p: v for p, v in params.items() if v}
query_str = get_querystring(params, [])
url = f"http://0.0.0.0:8000/api/v1/news/world?{query_str}"
result = requests.get(url, headers=headers, timeout=10)
assert isinstance(result, requests.Response)
def test_rss_providers_lists_outlets():
result = _get("rss_providers")
assert result.status_code == 200
payload = result.json()
assert isinstance(payload, list)
values = {entry["value"] for entry in payload}
assert "bbc" in values
assert "globenewswire" in values
assert "pr_newswire" in values
@pytest.mark.parametrize(
"params",
[
(
{
"symbol": "AAPL",
"limit": 20,
"provider": "benzinga",
"date": "2023-01-01",
"start_date": "2023-01-01",
"end_date": "2023-06-06",
}
),
(
{
"display": "full",
"date": None,
"start_date": "2023-06-01",
"end_date": "2023-06-06",
"updated_since": 1,
"published_since": 1,
"sort": "created",
"order": "asc",
"isin": None,
"cusip": None,
"channels": None,
"topics": "AAPL",
"authors": None,
"content_types": "headline",
"provider": "benzinga",
"symbol": "AAPL,MSFT",
"limit": 20,
}
),
(
{
"provider": "fmp",
"symbol": "AAPL",
"limit": 20,
"page": 1,
"start_date": None,
"end_date": None,
"press_release": False,
}
),
(
{
"provider": "yfinance",
"symbol": "AAPL",
"limit": 20,
"start_date": None,
"end_date": None,
}
),
(
{
"provider": "intrinio",
"symbol": "AAPL",
"limit": 20,
"start_date": "2024-01-02",
"end_date": "2024-01-03",
"source": "yahoo",
"topic": None,
"is_spam": False,
"sentiment": None,
"language": None,
"word_count_greater_than": None,
"word_count_less_than": None,
"business_relevance_greater_than": None,
"business_relevance_less_than": None,
}
),
(
{
"provider": "tiingo",
"symbol": "AAPL,MSFT",
"limit": 20,
"source": "bloomberg.com",
"start_date": None,
"end_date": None,
"offset": None,
}
),
(
{
"provider": "tmx",
"symbol": "RBC",
"limit": 20,
"page": 1,
}
),
],
)
@pytest.mark.integration
def test_news_company(params, headers):
"""Test retrieval of company-specific news with various parameters."""
params = {p: v for p, v in params.items() if v}
query_str = get_querystring(params, [])
url = f"http://0.0.0.0:8000/api/v1/news/company?{query_str}"
result = requests.get(url, headers=headers, timeout=10)
assert isinstance(result, requests.Response)
def test_rss_feeds_filtered_by_outlet():
result = _get("rss_feeds", params={"outlet": "bbc"})
assert result.status_code == 200
payload = result.json()
assert isinstance(payload, list)
values = {entry["value"] for entry in payload}
assert "bbc_world" in values
assert "bbc_business" in values
@pytest.mark.integration
def test_rss_fetches_articles_without_body():
result = _get(
"rss",
params={"source": "bbc_world", "limit": 3, "fetch_body": "false"},
)
assert result.status_code == 200
body = result.json()
results = body["results"]
assert 1 <= len(results) <= 3
first = results[0]
assert first["title"]
assert first["date"]
assert first["author"]
assert "url" in first
assert "excerpt" in first
assert "body" in first
@pytest.mark.integration
def test_rss_returns_empty_when_no_source_and_no_outlet():
result = _get("rss")
assert result.status_code == 200
assert result.json()["results"] == []
@pytest.mark.integration
def test_rss_auto_resolves_default_when_only_outlet_given():
result = _get("rss", params={"outlet": "bbc", "limit": 1, "fetch_body": "false"})
assert result.status_code == 200
body = result.json()
assert len(body["results"]) == 1
@pytest.mark.integration
def test_rss_unknown_source_returns_error():
result = _get("rss", params={"source": "not_a_feed", "limit": 1})
assert result.status_code >= 400

View File

@@ -1,190 +1,27 @@
"""Test news extension."""
"""Python interface integration tests."""
import pytest
from openbb_core.app.model.obbject import OBBject
@pytest.fixture(scope="session")
def obb(pytestconfig): # pylint: disable=inconsistent-return-statements
"""Fixture to setup obb."""
def obb(pytestconfig):
if pytestconfig.getoption("markexpr") != "not integration":
import openbb # pylint: disable=import-outside-toplevel
import openbb
return openbb.obb
return None
# pylint: disable=redefined-outer-name
@pytest.mark.parametrize(
"params",
[
(
{
"display": "full",
"date": None,
"start_date": "2023-05-01",
"end_date": "2023-05-31",
"updated_since": None,
"published_since": None,
"sort": "created",
"order": "asc",
"isin": None,
"cusip": None,
"channels": "General",
"topics": "car",
"authors": None,
"content_types": "Car",
"provider": "benzinga",
"limit": 20,
}
),
(
{
"provider": "fmp",
"limit": 20,
"start_date": None,
"end_date": None,
"page": 0,
"topic": "general",
}
),
(
{
"provider": "intrinio",
"limit": 20,
"start_date": "2024-01-02",
"end_date": "2024-01-03",
"source": "yahoo",
"topic": None,
"is_spam": False,
"sentiment": None,
"language": None,
"word_count_greater_than": None,
"word_count_less_than": None,
"business_relevance_greater_than": None,
"business_relevance_less_than": None,
}
),
(
{
"provider": "biztoc",
"source": None,
"term": "microsoft",
"start_date": None,
"end_date": None,
}
),
(
{
"provider": "tiingo",
"limit": 30,
"source": "bloomberg.com",
"start_date": None,
"end_date": None,
"offset": 0,
}
),
],
)
@pytest.mark.integration
def test_news_world(params, obb):
"""Test the news world endpoint."""
result = obb.news.world(**params)
assert result
assert isinstance(result, OBBject)
@pytest.mark.parametrize(
"params",
[
(
{
"display": "full",
"date": None,
"start_date": None,
"end_date": None,
"updated_since": None,
"published_since": None,
"sort": "created",
"order": "desc",
"isin": None,
"cusip": None,
"channels": "General",
"topics": "earnings",
"authors": None,
"content_types": "headline",
"provider": "benzinga",
"symbol": "AAPL,MSFT",
"limit": 20,
}
),
(
{
"provider": "fmp",
"symbol": "AAPL",
"limit": 20,
"page": 1,
"start_date": None,
"end_date": None,
"press_release": False,
}
),
(
{
"provider": "yfinance",
"symbol": "AAPL",
"limit": 20,
"start_date": None,
"end_date": None,
}
),
(
{
"provider": "intrinio",
"symbol": "AAPL",
"limit": 20,
"start_date": "2024-01-02",
"end_date": "2024-01-03",
"source": "yahoo",
"topic": None,
"is_spam": False,
"sentiment": None,
"language": None,
"word_count_greater_than": None,
"word_count_less_than": None,
"business_relevance_greater_than": None,
"business_relevance_less_than": None,
}
),
(
{
"provider": "tiingo",
"symbol": "AAPL",
"limit": 20,
"source": "bloomberg.com",
"start_date": None,
"end_date": None,
"offset": None,
}
),
(
{
"provider": "tmx",
"symbol": "RBC",
"limit": 20,
"page": 1,
}
),
],
)
@pytest.mark.integration
def test_news_company(params, obb):
"""Test the news company endpoint."""
params = {p: v for p, v in params.items() if v}
result = obb.news.company(**params)
assert result
assert isinstance(result, OBBject)
assert len(result.results) > 0
def test_rss_python_fetches_articles(obb):
out = obb.news.rss(source="bbc_world", limit=3, fetch_body=False)
assert isinstance(out, OBBject)
assert 1 <= len(out.results) <= 3
first = out.results[0]
assert first.title
assert first.date
assert first.author
assert hasattr(first, "url")
assert hasattr(first, "excerpt")
assert hasattr(first, "body")

View File

@@ -0,0 +1,290 @@
"""RSS/Atom feed and article body helpers."""
import re
from datetime import datetime, timezone
from html.parser import HTMLParser
from time import struct_time
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from aiohttp import ClientSession
_CHROME_TAGS: tuple[str, ...] = (
"script",
"style",
"noscript",
"iframe",
"nav",
"header",
"footer",
"aside",
"form",
"button",
"svg",
)
_TEMPLATE_MARKERS: tuple[str, ...] = ("[[", "{{")
_MIN_PARAGRAPHS = 2
_FEED_TIMEOUT_SEC = 8.0
_ARTICLE_TIMEOUT_SEC = 6.0
_TIMESTAMP_RE = re.compile(r"^\d{1,2}:\d{2}(?::\d{2})?$")
class _TagStripper(HTMLParser):
"""Collect text data while discarding tags."""
def __init__(self) -> None:
super().__init__()
self.parts: list[str] = []
def handle_data(self, data: str) -> None:
self.parts.append(data)
def strip_html(text: str) -> str:
"""Strip HTML tags and collapse whitespace."""
if not text:
return ""
stripper = _TagStripper()
stripper.feed(text)
return " ".join("".join(stripper.parts).split())
def html_to_markdown(fragment: str) -> str:
"""Render an HTML fragment as markdown, preserving anchors and images."""
if not fragment:
return ""
from lxml import html as _html
try:
root = _html.fragment_fromstring(fragment, create_parent="div")
except Exception: # noqa: BLE001
return strip_html(fragment)
parts: list[str] = []
def emit(el) -> None:
tag = el.tag.lower() if isinstance(el.tag, str) else ""
if tag == "a":
href = (el.get("href") or "").strip()
inner = " ".join(el.text_content().split())
if inner and href:
parts.append(f"[{inner}]({href})")
elif inner:
parts.append(inner)
parts.append(" ")
if el.tail:
parts.append(el.tail)
return
if tag == "img":
src = (el.get("src") or "").strip()
alt = (el.get("alt") or "").strip()
if src:
parts.append(f"![{alt}]({src})")
if el.tail:
parts.append(el.tail)
return
if tag == "br":
parts.append("\n")
elif tag == "p" and parts and not parts[-1].endswith("\n"):
parts.append("\n\n")
if el.text:
parts.append(el.text)
for child in el:
emit(child)
if tag == "p":
parts.append("\n\n")
if el.tail:
parts.append(el.tail)
emit(root)
rendered = "".join(parts)
lines = [" ".join(line.split()) for line in rendered.split("\n")]
return "\n".join(lines).strip()
def truncate(text: str, limit: int = 280) -> str:
"""Truncate at the nearest word boundary within ``limit``."""
if len(text) <= limit:
return text
cut = text[:limit].rsplit(" ", 1)[0]
return cut.rstrip(",.;:") + "…"
def struct_time_to_iso(t: "struct_time | None") -> str:
"""Convert a feedparser ``struct_time`` to ISO 8601."""
if t is None:
return datetime.now(timezone.utc).isoformat()
return datetime(*t[:6], tzinfo=timezone.utc).isoformat()
async def fetch_feed(session: "ClientSession", url: str):
"""Fetch and parse an RSS/Atom feed."""
import aiohttp
import feedparser
try:
response = await session.get(
url, timeout=aiohttp.ClientTimeout(total=_FEED_TIMEOUT_SEC)
)
async with response:
response.raise_for_status()
payload = await response.read()
except (aiohttp.ClientError, TimeoutError):
return feedparser.parse(b"")
return feedparser.parse(payload)
def _strip_chrome(doc) -> None:
"""Remove chrome tags in place."""
selector = " | ".join(f"//{tag}" for tag in _CHROME_TAGS)
for el in doc.xpath(selector):
parent = el.getparent()
if parent is not None:
parent.remove(el)
def _looks_clean(text: str) -> bool:
"""Reject empty results and unrendered template fragments."""
return bool(text) and not any(marker in text for marker in _TEMPLATE_MARKERS)
def _jsonld_field(doc, field: str) -> str | None:
"""Pull a top-level string field from any JSON-LD script."""
import json
for script in doc.xpath('//script[@type="application/ld+json"]/text()'):
try:
data = json.loads(script)
except (json.JSONDecodeError, TypeError):
continue
for entry in data if isinstance(data, list) else [data]:
if not isinstance(entry, dict):
continue
for candidate in (entry, *(entry.get("@graph") or [])):
if not isinstance(candidate, dict):
continue
value = candidate.get(field)
if isinstance(value, str) and value.strip():
return value.strip()
if isinstance(value, list) and value:
first = value[0]
if isinstance(first, str) and first.strip():
return first.strip()
if isinstance(first, dict):
nested = first.get("url")
if isinstance(nested, str) and nested.strip():
return nested.strip()
if isinstance(value, dict):
nested = value.get("url")
if isinstance(nested, str) and nested.strip():
return nested.strip()
return None
def _article_markdown(node) -> str:
"""Render an article container as markdown with text + inline images."""
parts: list[str] = []
text_count = [0]
def walk(el) -> None:
tag = el.tag.lower() if isinstance(el.tag, str) else ""
if tag == "p":
text = " ".join(el.text_content().split())
if text and not _TIMESTAMP_RE.match(text):
parts.append(text)
text_count[0] += 1
return
if tag == "figure":
srcs = el.xpath(".//img/@src")
if srcs:
src = str(srcs[0]).strip()
cap_nodes = el.xpath(".//figcaption//text()")
cap = " ".join("".join(cap_nodes).split()) if cap_nodes else ""
if src:
parts.append(f"![{cap}]({src})")
return
if tag == "img":
src = (el.get("src") or "").strip()
alt = (el.get("alt") or "").strip()
if src:
parts.append(f"![{alt}]({src})")
return
for child in el:
walk(child)
walk(node)
if text_count[0] < _MIN_PARAGRAPHS:
return ""
return "\n\n".join(parts)
def _extract_image_from_doc(doc) -> str | None:
"""Find a hero image URL from meta tags, JSON-LD, or article body."""
for path in (
'//meta[@property="og:image"]/@content',
'//meta[@name="og:image"]/@content',
'//meta[@name="twitter:image"]/@content',
'//meta[@property="twitter:image"]/@content',
):
for value in doc.xpath(path):
url = str(value).strip()
if url:
return url
jsonld = _jsonld_field(doc, "image")
if jsonld:
return jsonld
for path in ("//article//img/@src", "//main//img/@src"):
for value in doc.xpath(path):
url = str(value).strip()
if url:
return url
return None
def _extract_body(payload: bytes) -> str | None:
"""Return the article body as markdown with inline images."""
from lxml import html
try:
doc = html.fromstring(payload, parser=html.HTMLParser(encoding="utf-8"))
except Exception: # noqa: BLE001
return None
hero = _extract_image_from_doc(doc)
body = _jsonld_field(doc, "articleBody")
if not (body and _looks_clean(body)):
_strip_chrome(doc)
best: str = ""
for article in doc.xpath("//article"):
text = _article_markdown(article)
if _looks_clean(text) and len(text) > len(best):
best = text
if not best:
for main in doc.xpath("//main"):
text = _article_markdown(main)
if _looks_clean(text) and len(text) > len(best):
best = text
body = best or None
if body and hero and "![" not in body:
body = f"![]({hero})\n\n{body}"
return body
async def fetch_article_body(session: "ClientSession", url: str) -> str | None:
"""Fetch ``url`` and return its article body."""
import aiohttp
try:
response = await session.get(
url, timeout=aiohttp.ClientTimeout(total=_ARTICLE_TIMEOUT_SEC)
)
async with response:
response.raise_for_status()
payload = await response.read()
except (aiohttp.ClientError, TimeoutError):
return None
return _extract_body(payload)

View File

@@ -0,0 +1,943 @@
"""RSS feed registry."""
from urllib.parse import quote
from openbb_core.app.config.loader import load_config
_PROVIDER_LABELS: dict[str, str] = {
"axios": "Axios",
"bbc": "BBC",
"benzinga": "Benzinga",
"cbc": "CBC",
"drudge_report": "Drudge Report",
"fox_news": "Fox News",
"fortune": "Fortune",
"globenewswire": "GlobeNewswire",
"google_news": "Google News",
"yahoo_finance": "Yahoo Finance",
"pr_newswire": "PR Newswire",
"wired": "Wired",
"custom": "Custom",
}
_PRNEWSWIRE_REGIONS: tuple[str, ...] = (
"apac",
"apac/zh",
"br",
"jp",
"kr",
"mx",
"id",
"vn",
)
_PRNEWSWIRE_REGION_LABELS: dict[str, str] = {
"global": "Global",
"apac": "APAC",
"apac/zh": "APAC (Chinese)",
"br": "Brazil",
"jp": "Japan",
"kr": "Korea",
"mx": "Mexico",
"id": "Indonesia",
"vn": "Vietnam",
}
_PRNEWSWIRE_CATEGORIES: tuple[str, ...] = (
"business-technology",
"consumer-products-retail",
"consumer-technology",
"energy",
"entertainment-media",
"environment",
"general-business",
"heavy-industry-manufacturing",
"health",
"people-culture",
"policy-public-interest",
"sports",
"telecommunications",
"travel",
"financial-services",
"automotive-transportation",
)
_PRNEWSWIRE_CATEGORY_LABELS: dict[str, str] = {
"business-technology": "Business Technology",
"consumer-products-retail": "Consumer Products & Retail",
"consumer-technology": "Consumer Technology",
"energy": "Energy",
"entertainment-media": "Entertainment & Media",
"environment": "Environment",
"general-business": "General Business",
"heavy-industry-manufacturing": "Heavy Industry & Manufacturing",
"health": "Health",
"people-culture": "People & Culture",
"policy-public-interest": "Policy & Public Interest",
"sports": "Sports",
"telecommunications": "Telecommunications",
"travel": "Travel",
"financial-services": "Financial Services",
"automotive-transportation": "Automotive & Transportation",
}
_PRNEWSWIRE_GLOBAL_SKIP: frozenset[str] = frozenset({"business-technology"})
_GLOBENEWSWIRE_INDUSTRIES: tuple[tuple[str, str], ...] = (
("2713", "Aerospace"),
("5751", "Airlines"),
("7537", "Alternative Electricity"),
("587", "Alternative Fuels"),
("1753", "Aluminum"),
("5371", "Apparel Retailers"),
("8771", "Asset Managers and Custodians"),
("3355", "Auto Parts"),
("40101010", "Auto Services"),
("3353", "Automobiles"),
("8355", "Banks"),
("1000", "Basic Materials"),
("4573", "Biotechnology"),
("3533", "Brewers"),
("5553", "Broadcasting and Entertainment"),
("5373", "Broadline Retailers"),
("50101025", "Building Climate Control"),
("2353", "Building Materials and Fixtures"),
("50101035", "Building Materials Other"),
("50101020", "Building, Roofing, Wallboard and Plumbing"),
("2791", "Business Support Services"),
("2793", "Business Training and Employment Agencies"),
("20103020", "Cannabis Producers"),
("50101030", "Cement"),
("55201010", "Chemicals and Synthetic Fibers"),
("55201000", "Chemicals Diversified"),
("30204000", "Closed End Investments"),
("3763", "Clothing and Accessories"),
("1771", "Coal"),
("50206050", "Commercial Vehicle Equipment Leasing"),
("50206015", "Commercial Vehicles and Parts"),
("2753", "Commercial Vehicles and Trucks"),
("1353", "Commodity Chemicals"),
("9572", "Computer Hardware"),
("9533", "Computer Services"),
("50101010", "Construction"),
("3743", "Consumer Electronics"),
("8773", "Consumer Finance"),
("3000", "Consumer Goods"),
("5000", "Consumer Services"),
("2723", "Containers and Packaging"),
("7535", "Conventional Electricity"),
("55102040", "Copper"),
("2717", "Defense"),
("2771", "Delivery Services"),
("1773", "Diamonds and Gemstones"),
("3535", "Distillers and Vintners"),
("30202000", "Diversified Financial Services"),
("2727", "Diversified Industrials"),
("55101000", "Diversified Materials"),
("8674", "Diversified REITs"),
("5333", "Drug Retailers"),
("3722", "Durable Household Products"),
("50202010", "Electrical Components"),
("2733", "Electrical Components and Equipment"),
("2737", "Electronic Equipment"),
("50202020", "Electronic Equipment Control and Filter"),
("50202025", "Electronic Equipment Gauges and Meters"),
("50202040", "Electronic Equipment Other"),
("50202030", "Electronic Equipment Pollution Control"),
("9574", "Electronic Office Equipment"),
("1", "Energy"),
("50101015", "Engineering and Contracting Services"),
("8985", "Equity Investment Instruments"),
("533", "Exploration and Production"),
("3573", "Farming and Fishing"),
("55201015", "Fertilizers"),
("2795", "Financial Administration"),
("30201030", "Financial Data Providers"),
("8000", "Financials"),
("6535", "Fixed Line Telecommunications"),
("3577", "Food Products"),
("5337", "Food Retailers and Wholesalers"),
("3765", "Footwear"),
("1733", "Forestry"),
("50205030", "Forms and Bulk Printing Services"),
("8532", "Full Line Insurance"),
("3726", "Furnishings"),
("5752", "Gambling"),
("7573", "Gas Distribution"),
("1775", "General Mining"),
("50203020", "Glass"),
("1777", "Gold Mining"),
("4000", "Health Care"),
("4533", "Health Care Providers"),
("35102010", "Health Care REITs"),
("2357", "Heavy Construction"),
("3728", "Home Construction"),
("5375", "Home Improvement Retailers"),
("8677", "Hotel and Lodging REITs"),
("5753", "Hotels"),
("8671", "Industrial and Office REITs"),
("2757", "Industrial Machinery"),
("35102020", "Industrial REITs"),
("2797", "Industrial Suppliers"),
("2000", "Industrials"),
("35102025", "Infrastructure REITs"),
("8534", "Insurance Brokers"),
("537", "Integrated Oil and Gas"),
("9535", "Internet"),
("8777", "Investment Services"),
("1757", "Iron and Steel"),
("8575", "Life Insurance"),
("50204010", "Machinery Agricultural"),
("50204020", "Machinery Construction and Handling"),
("50204030", "Machinery Engines"),
("50204000", "Machinery Industrial"),
("50204050", "Machinery Specialty"),
("50204040", "Machinery Tools"),
("2773", "Marine Transportation"),
("5555", "Media Agencies"),
("4535", "Medical Equipment"),
("20102020", "Medical Services"),
("4537", "Medical Supplies"),
("55102015", "Metal Fabricating"),
("6575", "Mobile Telecommunications"),
("8779", "Mortgage Finance"),
("8676", "Mortgage REITs"),
("7575", "Multiutilities"),
("3724", "Nondurable Household Products"),
("8995", "Nonequity Investment Instruments"),
("1755", "Nonferrous Metals"),
("35102030", "Office REITs"),
("573", "Oil Equipment and Services"),
("30205000", "Open End and Miscellaneous Investment Vehicles"),
("35102070", "Other Specialty REITs"),
("50203010", "Paints and Coatings"),
("1737", "Paper"),
("3767", "Personal Products"),
("4577", "Pharmaceuticals"),
("577", "Pipelines"),
("50203015", "Plastics"),
("1779", "Platinum and Precious Metals"),
("10102020", "Production Technology Equipment"),
("50205020", "Professional Business Support Services"),
("8536", "Property and Casualty Insurance"),
("5557", "Publishing"),
("50206025", "Railroad Equipment"),
("2775", "Railroads"),
("35", "Real Estate"),
("8633", "Real Estate Holding and Development"),
("8637", "Real Estate Services"),
("3745", "Recreational Products"),
("5755", "Recreational Services"),
("8538", "Reinsurance"),
("583", "Renewable Energy Equipment"),
("8673", "Residential REITs"),
("5757", "Restaurants and Bars"),
("8672", "Retail REITs"),
("50205040", "Security Services"),
("9576", "Semiconductors"),
("3537", "Soft Drinks"),
("9537", "Software"),
("5377", "Specialized Consumer Services"),
("1357", "Specialty Chemicals"),
("8775", "Specialty Finance"),
("8675", "Specialty REITs"),
("5379", "Specialty Retailers"),
("35102050", "Storage REITs"),
("9000", "Technology"),
("6000", "Telecommunications"),
("9578", "Telecommunications Equipment"),
("55101020", "Textile Products"),
("35102060", "Timber REITs"),
("3357", "Tires"),
("3785", "Tobacco"),
("3747", "Toys"),
("50205015", "Transaction Processing Services"),
("2777", "Transportation Services"),
("5759", "Travel and Tourism"),
("2779", "Trucking"),
("7000", "Utilities"),
("2799", "Waste and Disposal Services"),
("7577", "Water"),
)
def _globenewswire_industry_url(code: str, label: str) -> str:
"""Build a GlobeNewswire industry feed URL."""
title = quote(f"GlobeNewswire - Industry News on {label}", safe="-")
return f"https://www.globenewswire.com/AtomFeed/industry/{code}/feedTitle/{title}"
def _globenewswire_industry_key(label: str) -> str:
"""Slug an industry label into a feed key."""
return _globenewswire_slug(label, prefix="globenewswire_")
_GLOBENEWSWIRE_SUBJECTS: tuple[tuple[str, str], ...] = (
("1", "Advisory"),
("3", "Analyst Recommendations"),
("2", "Annual Meetings and Shareholder Rights"),
("65", "Annual Report"),
("4", "Arts and Entertainment"),
("5", "Bankruptcy"),
("6", "Bond Market News"),
("74", "Bonds Market Information"),
("7", "Business Contracts"),
("8", "Calendar Of Events"),
("58", "Changes In Company's Own Shares"),
("57", "Changes In Share Capital And Votes"),
("71", "Changes To Observation Segment"),
("84", "Class Action"),
("90", "Clinical Study"),
("9", "Company Announcement"),
("10", "Company Regulatory Filings"),
("89", "Conference Calls, Webcasts"),
("88", "Contests, Awards"),
("61", "Corporate Action"),
("75", "Derivative Market Information"),
("11", "Directors And Officers"),
("12", "Dividend Reports and Estimates"),
("13", "Earnings Releases and Operating Results"),
("14", "Economic Research And Reports"),
("91", "Environmental, Social, And Governance Criteria"),
("76", "Equity Market Information"),
("85", "European Regulatory News"),
("70", "Exchange Announcement"),
("77", "Exchange Members"),
("78", "Exchange News"),
("15", "Fashion"),
("16", "Feature Article"),
("17", "Financing Agreements"),
("69", "First North Announcement"),
("79", "First North Information"),
("18", "Food"),
("19", "Government News"),
("20", "Health"),
("21", "Initial Public Offerings"),
("22", "Insider's Buy, Sell"),
("66", "Interim Information"),
("80", "Investment Fund Information"),
("83", "Investment Opinion"),
("23", "Joint Venture"),
("24", "Law and Legal Issues"),
("25", "Licensing Agreements"),
("26", "Lifestyle"),
("59", "Major Shareholder Announcements"),
("86", "Management Changes"),
("67", "Management Statements"),
("81", "Market Research Reports"),
("27", "Mergers and Acquisitions"),
("64", "Mutual Fund Information"),
("62", "Net Asset Value"),
("28", "Other News"),
("29", "Partnerships"),
("30", "Patents"),
("87", "Philanthropy"),
("34", "Politics"),
("31", "Pre Release Comments"),
("72", "Press Releases"),
("32", "Product, Services Announcement"),
("63", "Prospectus, Announcement Of Prospectus"),
("33", "Proxy Statements And Analysis"),
("73", "Regulatory Information"),
("35", "Religion"),
("36", "Research Analysis And Reports"),
("37", "Restructuring, Recapitalization"),
("38", "Sports"),
("39", "Stock Market News"),
("40", "Tax Issues, Accounting"),
("43", "Technical Analysis"),
("41", "Trade Show"),
("68", "Trading Information"),
("42", "Travel"),
("82", "Warrants And Certificates"),
)
def _globenewswire_subject_url(code: str, label: str) -> str:
"""Build a GlobeNewswire subject feed URL."""
title = quote(f"GlobeNewswire - {label}", safe="-")
return (
f"https://www.globenewswire.com/AtomFeed/subjectcode/{code}/feedTitle/{title}"
)
def _globenewswire_subject_key(label: str) -> str:
"""Slug a subject label into a feed key."""
return _globenewswire_slug(label, prefix="globenewswire_subject_")
def _globenewswire_slug(label: str, prefix: str) -> str:
"""Slug a label."""
import re
slug = label.lower().replace("&", "and").replace("/", "_").replace("'", "")
slug = slug.replace(":", "").replace(",", "").replace(".", "")
slug = re.sub(r"[^a-z0-9]+", "_", slug).strip("_")
return f"{prefix}{slug}"
_CURATED_FEEDS: dict[str, tuple[str, str, str]] = {
"axios_main": ("axios", "Latest", "https://api.axios.com/feed/"),
"benzinga_main": ("benzinga", "Latest", "https://www.benzinga.com/feed"),
"benzinga_markets": (
"benzinga",
"Markets",
"https://www.benzinga.com/markets/feed",
),
"benzinga_general": (
"benzinga",
"General News",
"https://www.benzinga.com/general/feed",
),
"benzinga_cryptocurrency": (
"benzinga",
"Cryptocurrency",
"https://www.benzinga.com/topic/cryptocurrency/feed",
),
"benzinga_government": (
"benzinga",
"Government",
"https://www.benzinga.com/topic/government/feed",
),
"globenewswire_all": (
"globenewswire",
"All News",
"https://www.globenewswire.com/RssFeed/orgclass/1/feedTitle/GlobeNewswire-AllNews",
),
"globenewswire_us": (
"globenewswire",
"United States",
"https://www.globenewswire.com/AtomFeed/country/United%20States/feedTitle/GlobeNewswire%20-%20News%20from%20United%20States",
),
"bbc_world": ("bbc", "World", "http://feeds.bbci.co.uk/news/world/rss.xml"),
"bbc_uk": ("bbc", "UK", "http://feeds.bbci.co.uk/news/uk/rss.xml"),
"bbc_business": (
"bbc",
"Business",
"http://feeds.bbci.co.uk/news/business/rss.xml",
),
"bbc_politics": (
"bbc",
"Politics",
"http://feeds.bbci.co.uk/news/politics/rss.xml",
),
"bbc_health": ("bbc", "Health", "http://feeds.bbci.co.uk/news/health/rss.xml"),
"bbc_education": (
"bbc",
"Education",
"http://feeds.bbci.co.uk/news/education/rss.xml",
),
"bbc_science": (
"bbc",
"Science & Environment",
"http://feeds.bbci.co.uk/news/science_and_environment/rss.xml",
),
"bbc_technology": (
"bbc",
"Technology",
"http://feeds.bbci.co.uk/news/technology/rss.xml",
),
"bbc_entertainment": (
"bbc",
"Entertainment & Arts",
"http://feeds.bbci.co.uk/news/entertainment_and_arts/rss.xml",
),
"cbc_world": ("cbc", "World", "https://www.cbc.ca/webfeed/rss/rss-world"),
"cbc_canada": ("cbc", "Canada", "https://www.cbc.ca/webfeed/rss/rss-canada"),
"cbc_business": (
"cbc",
"Business",
"https://www.cbc.ca/webfeed/rss/rss-business",
),
"cbc_politics": (
"cbc",
"Politics",
"https://www.cbc.ca/webfeed/rss/rss-politics",
),
"cbc_technology": (
"cbc",
"Technology",
"https://www.cbc.ca/webfeed/rss/rss-technology",
),
"cbc_health": ("cbc", "Health", "https://www.cbc.ca/webfeed/rss/rss-health"),
"cbc_arts": ("cbc", "Arts", "https://www.cbc.ca/webfeed/rss/rss-arts"),
"cbc_indigenous": (
"cbc",
"Indigenous",
"https://www.cbc.ca/webfeed/rss/rss-Indigenous",
),
"cbc_thenational": (
"cbc",
"The National",
"https://www.cbc.ca/webfeed/rss/rss-thenational",
),
"cbc_canada_britishcolumbia": (
"cbc",
"Region — British Columbia",
"https://www.cbc.ca/webfeed/rss/rss-canada-britishcolumbia",
),
"cbc_canada_calgary": (
"cbc",
"Region — Calgary",
"https://www.cbc.ca/webfeed/rss/rss-canada-calgary",
),
"cbc_canada_edmonton": (
"cbc",
"Region — Edmonton",
"https://www.cbc.ca/webfeed/rss/rss-canada-edmonton",
),
"cbc_canada_hamilton": (
"cbc",
"Region — Hamilton",
"https://www.cbc.ca/webfeed/rss/rss-canada-hamiltonnews",
),
"cbc_canada_kamloops": (
"cbc",
"Region — Kamloops",
"https://www.cbc.ca/webfeed/rss/rss-canada-kamloops",
),
"cbc_canada_kitchenerwaterloo": (
"cbc",
"Region — Kitchener-Waterloo",
"https://www.cbc.ca/webfeed/rss/rss-canada-kitchenerwaterloo",
),
"cbc_canada_london": (
"cbc",
"Region — London",
"https://www.cbc.ca/webfeed/rss/rss-canada-london",
),
"cbc_canada_manitoba": (
"cbc",
"Region — Manitoba",
"https://www.cbc.ca/webfeed/rss/rss-canada-manitoba",
),
"cbc_canada_montreal": (
"cbc",
"Region — Montreal",
"https://www.cbc.ca/webfeed/rss/rss-canada-montreal",
),
"cbc_canada_newbrunswick": (
"cbc",
"Region — New Brunswick",
"https://www.cbc.ca/webfeed/rss/rss-canada-newbrunswick",
),
"cbc_canada_newfoundland": (
"cbc",
"Region — Newfoundland",
"https://www.cbc.ca/webfeed/rss/rss-canada-newfoundland",
),
"cbc_canada_north": (
"cbc",
"Region — North",
"https://www.cbc.ca/webfeed/rss/rss-canada-north",
),
"cbc_canada_novascotia": (
"cbc",
"Region — Nova Scotia",
"https://www.cbc.ca/webfeed/rss/rss-canada-novascotia",
),
"cbc_canada_ottawa": (
"cbc",
"Region — Ottawa",
"https://www.cbc.ca/webfeed/rss/rss-canada-ottawa",
),
"cbc_canada_pei": (
"cbc",
"Region — Prince Edward Island",
"https://www.cbc.ca/webfeed/rss/rss-canada-pei",
),
"cbc_canada_saskatchewan": (
"cbc",
"Region — Saskatchewan",
"https://www.cbc.ca/webfeed/rss/rss-canada-saskatchewan",
),
"cbc_canada_saskatoon": (
"cbc",
"Region — Saskatoon",
"https://www.cbc.ca/webfeed/rss/rss-canada-saskatoon",
),
"cbc_canada_sudbury": (
"cbc",
"Region — Sudbury",
"https://www.cbc.ca/webfeed/rss/rss-canada-sudbury",
),
"cbc_canada_thunderbay": (
"cbc",
"Region — Thunder Bay",
"https://www.cbc.ca/webfeed/rss/rss-canada-thunderbay",
),
"cbc_canada_toronto": (
"cbc",
"Region — Toronto",
"https://www.cbc.ca/webfeed/rss/rss-canada-toronto",
),
"cbc_canada_windsor": (
"cbc",
"Region — Windsor",
"https://www.cbc.ca/webfeed/rss/rss-canada-windsor",
),
"cbc_sports_nhl": (
"cbc",
"Sports — NHL",
"https://www.cbc.ca/webfeed/rss/rss-sports-nhl",
),
"cbc_sports_nba": (
"cbc",
"Sports — NBA",
"https://www.cbc.ca/webfeed/rss/rss-sports-nba",
),
"cbc_sports_nfl": (
"cbc",
"Sports — NFL",
"https://www.cbc.ca/webfeed/rss/rss-sports-nfl",
),
"cbc_sports_mlb": (
"cbc",
"Sports — MLB",
"https://www.cbc.ca/webfeed/rss/rss-sports-mlb",
),
"cbc_sports_soccer": (
"cbc",
"Sports — Soccer",
"https://www.cbc.ca/webfeed/rss/rss-sports-soccer",
),
"fox_news_latest": (
"fox_news",
"Latest Headlines",
"https://moxie.foxnews.com/google-publisher/latest.xml",
),
"fox_news_world": (
"fox_news",
"World",
"https://moxie.foxnews.com/google-publisher/world.xml",
),
"fox_news_us": (
"fox_news",
"US",
"https://moxie.foxnews.com/google-publisher/us.xml",
),
"fox_news_politics": (
"fox_news",
"Politics",
"https://moxie.foxnews.com/google-publisher/politics.xml",
),
"google_news_us": (
"google_news",
"Top — US",
"https://news.google.com/rss?hl=en-US&gl=US&ceid=US:en",
),
"google_news_ca": (
"google_news",
"Top — Canada",
"https://news.google.com/rss?hl=en-CA&gl=CA&ceid=CA:en",
),
"google_news_uk": (
"google_news",
"Top — UK",
"https://news.google.com/rss?hl=en-GB&gl=GB&ceid=GB:en",
),
"google_news_au": (
"google_news",
"Top — Australia",
"https://news.google.com/rss?hl=en-AU&gl=AU&ceid=AU:en",
),
"google_news_business": (
"google_news",
"Business",
"https://news.google.com/rss/headlines/section/topic/BUSINESS?hl=en-US&gl=US&ceid=US:en",
),
"google_news_technology": (
"google_news",
"Technology",
"https://news.google.com/rss/headlines/section/topic/TECHNOLOGY?hl=en-US&gl=US&ceid=US:en",
),
"google_news_world": (
"google_news",
"World",
"https://news.google.com/rss/headlines/section/topic/WORLD?hl=en-US&gl=US&ceid=US:en",
),
"google_news_nation": (
"google_news",
"Nation",
"https://news.google.com/rss/headlines/section/topic/NATION?hl=en-US&gl=US&ceid=US:en",
),
"drudge_report": (
"drudge_report",
"Latest",
"http://feeds.feedburner.com/DrudgeReportFeed",
),
"fortune": (
"fortune",
"Latest",
"https://fortune.com/feed/fortune-feeds/?id=3230629",
),
"yahoo_finance": (
"yahoo_finance",
"Latest",
"https://finance.yahoo.com/news/rssindex",
),
"wired_main": ("wired", "Latest", "https://www.wired.com/feed/rss"),
"wired_backchannel": (
"wired",
"Backchannel",
"https://www.wired.com/feed/category/backchannel/latest/rss",
),
"wired_business": (
"wired",
"Business",
"https://www.wired.com/feed/category/business/latest/rss",
),
"wired_culture": (
"wired",
"Culture",
"https://www.wired.com/feed/category/culture/latest/rss",
),
"wired_gear": (
"wired",
"Gear",
"https://www.wired.com/feed/category/gear/latest/rss",
),
"wired_ideas": (
"wired",
"Ideas",
"https://www.wired.com/feed/category/ideas/latest/rss",
),
"wired_science": (
"wired",
"Science",
"https://www.wired.com/feed/category/science/latest/rss",
),
"wired_security": (
"wired",
"Security",
"https://www.wired.com/feed/category/security/latest/rss",
),
"wired_ai": (
"wired",
"Tag — AI",
"https://www.wired.com/feed/tag/ai/latest/rss",
),
"wired_guide": (
"wired",
"Tag — Wired Guide",
"https://www.wired.com/feed/tag/wired-guide/latest/rss",
),
}
def _slug(value: str) -> str:
return value.replace("/", "_").replace("-", "_")
def _humanize(key: str) -> str:
return key.replace("_", " ").title()
def _build_feed_registry() -> tuple[dict[str, str], dict[str, dict[str, str]]]:
feeds: dict[str, str] = {}
meta: dict[str, dict[str, str]] = {}
feeds["pr_newswire_global"] = (
"https://www.prnewswire.com/rss/news-releases-list.rss"
)
meta["pr_newswire_global"] = {"provider": "pr_newswire", "label": "Global"}
for region in _PRNEWSWIRE_REGIONS:
key = f"pr_newswire_{_slug(region)}"
feeds[key] = f"https://www.prnewswire.com/{region}/rss/news-releases-list.rss"
meta[key] = {
"provider": "pr_newswire",
"label": _PRNEWSWIRE_REGION_LABELS[region],
}
for cat in _PRNEWSWIRE_CATEGORIES:
if cat in _PRNEWSWIRE_GLOBAL_SKIP:
continue
key = f"pr_newswire_global_{_slug(cat)}"
feeds[key] = (
f"https://www.prnewswire.com/rss/{cat}-latest-news/{cat}-latest-news-list.rss"
)
meta[key] = {
"provider": "pr_newswire",
"label": f"Global — {_PRNEWSWIRE_CATEGORY_LABELS[cat]}",
}
for region in _PRNEWSWIRE_REGIONS:
region_label = _PRNEWSWIRE_REGION_LABELS[region]
for cat in _PRNEWSWIRE_CATEGORIES:
key = f"pr_newswire_{_slug(region)}_{_slug(cat)}"
feeds[key] = (
f"https://www.prnewswire.com/{region}/rss/"
f"{cat}-latest-news/{cat}-latest-news-list.rss"
)
meta[key] = {
"provider": "pr_newswire",
"label": f"{region_label} — {_PRNEWSWIRE_CATEGORY_LABELS[cat]}",
}
for code, label in _GLOBENEWSWIRE_INDUSTRIES:
key = _globenewswire_industry_key(label)
feeds[key] = _globenewswire_industry_url(code, label)
meta[key] = {
"provider": "globenewswire",
"label": f"Industry — {label}",
}
for code, label in _GLOBENEWSWIRE_SUBJECTS:
key = _globenewswire_subject_key(label)
feeds[key] = _globenewswire_subject_url(code, label)
meta[key] = {
"provider": "globenewswire",
"label": f"Subject — {label}",
}
for key, (provider, label, url) in _CURATED_FEEDS.items():
feeds[key] = url
meta[key] = {"provider": provider, "label": label}
return feeds, meta
_DEFAULT_FEEDS, _DEFAULT_META = _build_feed_registry()
def _user_news_config() -> dict | None:
"""Return the ``[news]`` table from ``openbb.toml``."""
config = load_config()
if not isinstance(config, dict):
return None
news = config.get("news")
return news if isinstance(news, dict) else None
def _user_feed_entries() -> dict[str, dict[str, str]] | None:
"""Return normalized user feeds keyed by feed id."""
news = _user_news_config()
if not news:
return None
raw = news.get("rss_feeds")
if not isinstance(raw, dict) or not raw:
return None
out: dict[str, dict[str, str]] = {}
for key, value in raw.items():
skey = str(key)
if isinstance(value, str) and value:
out[skey] = {
"url": value,
"provider": "custom",
"label": _humanize(skey),
}
elif isinstance(value, dict):
url = value.get("url")
if not isinstance(url, str) or not url:
continue
out[skey] = {
"url": url,
"provider": str(value.get("provider") or "custom"),
"label": str(value.get("label") or _humanize(skey)),
}
return out or None
def _user_provider_labels() -> dict[str, str]:
"""Return user-defined provider labels from ``[news.rss_providers]``."""
news = _user_news_config()
if not news:
return {}
providers = news.get("rss_providers")
if isinstance(providers, dict):
return {str(k): str(v) for k, v in providers.items() if isinstance(v, str)}
return {}
def _should_merge_defaults() -> bool:
"""Return ``True`` when ``[news] merge_defaults = true``."""
news = _user_news_config()
return bool(news and news.get("merge_defaults"))
def list_feeds() -> dict[str, str]:
"""Return the active ``{key: url}`` map."""
user = _user_feed_entries()
if user is None:
return dict(_DEFAULT_FEEDS)
user_urls = {k: v["url"] for k, v in user.items()}
if _should_merge_defaults():
merged = dict(_DEFAULT_FEEDS)
merged.update(user_urls)
return merged
return user_urls
def _feed_meta() -> dict[str, dict[str, str]]:
"""Return ``{key: {provider, label}}`` for the active feed set."""
user = _user_feed_entries()
if user is None:
return dict(_DEFAULT_META)
user_meta = {
k: {"provider": v["provider"], "label": v["label"]} for k, v in user.items()
}
if _should_merge_defaults():
merged = dict(_DEFAULT_META)
merged.update(user_meta)
return merged
return user_meta
def list_providers() -> dict[str, str]:
"""Return ``{provider_id: label}`` for active providers."""
user_labels = _user_provider_labels()
providers_in_use = {entry["provider"] for entry in _feed_meta().values()}
result: dict[str, str] = {}
for pid in sorted(providers_in_use):
if pid in user_labels:
result[pid] = user_labels[pid]
elif pid in _PROVIDER_LABELS:
result[pid] = _PROVIDER_LABELS[pid]
else:
result[pid] = _humanize(pid)
return result
def list_feed_choices(provider: str | None = None) -> list[dict[str, str]]:
"""Return ``[{label, value}]`` for feeds under ``provider``."""
if not provider:
return []
meta = _feed_meta()
choices = [
{"label": entry["label"], "value": key}
for key, entry in meta.items()
if entry["provider"] == provider
]
return sorted(choices, key=lambda c: c["label"])
def get_feed_url(key: str) -> str:
"""Look up a feed URL by key."""
feeds = list_feeds()
url = feeds.get(key)
if url is None:
known = ", ".join(sorted(feeds))
raise ValueError(f"Unknown RSS feed source '{key}'. Known sources: {known}.")
return url
_DEFAULT_FEED_BY_PROVIDER: dict[str, str] = {
"axios": "axios_main",
"bbc": "bbc_world",
"benzinga": "benzinga_markets",
"cbc": "cbc_business",
"fox_news": "fox_news_latest",
"globenewswire": "globenewswire_all",
"google_news": "google_news_us",
"pr_newswire": "pr_newswire_global",
"wired": "wired_business",
}
def default_feed_for(provider: str | None) -> str | None:
"""Return the default feed key for ``provider``."""
if not provider:
return None
preferred = _DEFAULT_FEED_BY_PROVIDER.get(provider)
if preferred and preferred in list_feeds():
return preferred
choices = list_feed_choices(provider)
return choices[0]["value"] if choices else None

View File

@@ -0,0 +1,12 @@
"""News router."""
from openbb_core.app.router import Router
from openbb_news.news_router import router as _news_router
from openbb_news.rss import router as _rss_router
router = Router(prefix="", description="Financial market news data.")
router.include_router(_news_router)
router.include_router(_rss_router)
__all__ = ["router"]

View File

@@ -0,0 +1,206 @@
"""RSS news feed endpoint."""
import asyncio
from datetime import (
date as dateType,
datetime,
)
from typing import TYPE_CHECKING, Annotated
from fastapi import Query
from openbb_core.app.model.example import APIEx
from openbb_core.app.model.obbject import OBBject
from openbb_core.app.router import Router
from openbb_core.app.service.system_service import SystemService
from openbb_core.provider.abstract.data import Data
from pydantic import Field
if TYPE_CHECKING:
from aiohttp import ClientSession
api_prefix = SystemService().system_settings.api_settings.prefix
router = Router(prefix="", description="RSS news feeds.")
async def _gather_bodies(
session: "ClientSession", links: list[str | None]
) -> list[str | None]:
"""Fetch article bodies concurrently."""
from openbb_news.parser import fetch_article_body
async def one(link: str | None) -> str | None:
if not link:
return None
try:
return await fetch_article_body(session, link)
except asyncio.CancelledError:
raise
except Exception: # noqa: BLE001
return None
return await asyncio.gather(*(one(link) for link in links))
class NewsItemData(Data):
"""One article in Newsfeed-widget shape."""
title: str = Field(description="Article headline.")
date: datetime | dateType | str = Field(description="ISO 8601 publication date.")
author: str = Field(description="Author (falls back to the feed name).")
url: str = Field(description="Canonical article URL from the RSS feed.")
excerpt: str = Field(description="Short plain-text summary.")
body: str = Field(description="Article body as markdown with inline images.")
@router.command(
methods=["GET"],
widget_config={
"name": "RSS News Feed",
"description": (
"Articles from any RSS/Atom feed registered under"
" [news.rss_feeds] in openbb.toml."
),
"type": "newsfeed",
"category": "News",
"subCategory": "RSS",
"source": ["RSS"],
"gridData": {"w": 20, "h": 15},
"refetchInterval": 300000,
"params": [
{
"paramName": "outlet",
"label": "Provider",
"value": "benzinga",
"description": "News outlet grouping the available feeds.",
"type": "endpoint",
"optionsEndpoint": f"{api_prefix}/news/rss_providers",
"style": {"popupWidth": 280},
},
{
"paramName": "source",
"label": "Feed",
"value": None,
"description": "Specific feed within the selected provider.",
"type": "endpoint",
"optionsEndpoint": f"{api_prefix}/news/rss_feeds",
"optionsParams": {"outlet": "$outlet"},
"style": {"popupWidth": 360},
"optional": True,
},
{
"paramName": "limit",
"label": "Limit",
"value": 20,
"description": "Maximum number of articles to return.",
"type": "number",
"optional": True,
},
{
"paramName": "fetch_body",
"label": "Fetch full body",
"value": True,
"description": "Fetch each article URL for its full body.",
"type": "boolean",
"optional": True,
},
],
},
examples=[APIEx(parameters={"source": "yahoo_finance"})],
)
async def rss(
outlet: Annotated[
str | None,
Query(title="Provider", description="Outlet id."),
] = None,
source: Annotated[
str | None,
Query(title="Source", description="Configured feed key."),
] = None,
limit: Annotated[
int,
Query(title="Limit", description="Maximum articles to return.", ge=1),
] = 20,
fetch_body: Annotated[
bool,
Query(title="Fetch body", description="Fetch each article URL for its body."),
] = True,
) -> OBBject[list[NewsItemData]]:
"""Fetch a registered RSS feed."""
from openbb_core.provider.utils.helpers import get_async_requests_session
from openbb_news.parser import (
fetch_feed,
html_to_markdown,
strip_html,
struct_time_to_iso,
truncate,
)
from openbb_news.registry import default_feed_for, get_feed_url
if not source:
source = default_feed_for(outlet)
if not source:
return OBBject(results=[])
url = get_feed_url(source)
async with await get_async_requests_session() as session:
parsed = await fetch_feed(session, url)
feed_title = (parsed.feed.get("title") if parsed.feed else "") or ""
entries = sorted(
parsed.entries,
key=lambda e: (
e.get("published_parsed") or e.get("updated_parsed") or (0,) * 9
),
reverse=True,
)[:limit]
if fetch_body and entries:
links = [entry.get("link") for entry in entries]
bodies = await _gather_bodies(session, links)
else:
bodies = [None] * len(entries)
out: list[NewsItemData] = []
for entry, fetched_body in zip(entries, bodies):
summary_html = entry.get("summary") or entry.get("description") or ""
excerpt = truncate(strip_html(summary_html))
body = fetched_body or html_to_markdown(summary_html) or excerpt
out.append(
NewsItemData(
title=(entry.get("title") or "(untitled)").strip(),
date=struct_time_to_iso(
entry.get("published_parsed") or entry.get("updated_parsed")
),
author=entry.get("author") or feed_title or "Unknown",
url=(entry.get("link") or "").strip(),
excerpt=excerpt,
body=body,
)
)
return OBBject(results=out)
@router.command(methods=["GET"], widget_config={"exclude": True})
async def rss_providers() -> list[dict[str, str]]:
"""Return [{label, value}] for active news providers."""
from openbb_news.registry import list_providers
return [
{"label": label, "value": value} for value, label in list_providers().items()
]
@router.command(methods=["GET"], widget_config={"exclude": True})
async def rss_feeds(
outlet: Annotated[
str | None,
Query(title="Outlet", description="Outlet id."),
] = None,
) -> list[dict[str, str]]:
"""Return [{label, value}] for feeds under outlet."""
from openbb_news.registry import list_feed_choices
return list_feed_choices(outlet)

File diff suppressed because it is too large Load Diff

View File

@@ -1,19 +1,94 @@
[tool.poetry]
[project]
name = "openbb-news"
version = "1.6.1"
description = "News extension for OpenBB"
authors = ["OpenBB Team <hello@openbb.co>"]
license = "AGPL-3.0-only"
version = "2.0.0"
description = "RSS news feeds extension for OpenBB"
authors = [{ name = "OpenBB Team", email = "hello@openbb.co" }]
license = { text = "AGPL-3.0-only" }
readme = "README.md"
packages = [{ include = "openbb_news" }]
requires-python = ">=3.10,<4"
dependencies = [
"openbb-core[pandas]>=2.0.0",
"feedparser>=6.0",
"lxml>=5.0",
]
[tool.poetry.dependencies]
python = ">=3.10,<4"
openbb-core = "^1.6.3"
[project.entry-points."openbb_core_extension"]
news = "openbb_news.router:router"
[project.urls]
Homepage = "https://openbb.co"
Repository = "https://github.com/OpenBB-finance/OpenBB"
[dependency-groups]
dev = ["openbb-devtools>=2.0.0"]
[build-system]
requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
requires = ["hatchling"]
build-backend = "hatchling.build"
[tool.poetry.plugins."openbb_core_extension"]
news = "openbb_news.news_router:router"
[tool.hatch.build.targets.wheel]
packages = ["openbb_news"]
[tool.hatch.build.targets.sdist]
include = ["openbb_news", "README.md", "pyproject.toml"]
[tool.uv.sources]
openbb-core = { path = "../../core", editable = true }
openbb-devtools = { path = "../devtools", editable = true }
[tool.ruff]
line-length = 88
target-version = "py310"
fix = true
[tool.ruff.lint]
select = ["D", "E", "W", "F", "S", "UP", "I", "PLC", "PLE", "PLR", "PLW", "SIM", "T20"]
ignore = [
"PLR2004", "PLR0913", "PLR0915", "PLC0415",
"E402", "E501",
"D101", "D102", "D103", "D105", "D107",
"D202", "D203", "D205", "D206", "D210", "D212", "D213", "D214",
"D215", "D300", "D400", "D401", "D404", "D405", "D406", "D407",
"D408", "D409", "D410", "D411", "D413", "D415", "D416", "D417",
]
[tool.ruff.lint.per-file-ignores]
"openbb_news/*.py" = ["PLR0911", "PLR0912", "PLR0913", "PLR0914", "PLR0915", "PLR0917", "ARG001", "ARG002", "SLF001"]
"tests/**" = ["S101", "S104", "S105", "S106", "S108", "D", "F841", "PLR0911", "PLW0108", "SIM105", "SIM117", "E741", "SLF001"]
"integration/**" = ["S101", "S104", "S105", "S106", "S108", "D", "PLC0415", "PLR0911", "PLR0913", "PLR0914", "PLR0915", "PLR0917"]
"**/__init__.py" = ["F401"]
[tool.ruff.lint.isort]
combine-as-imports = true
force-wrap-aliases = true
[tool.ruff.lint.pydocstyle]
convention = "numpy"
[tool.ty]
[tool.ty.environment]
python-version = "3.10"
[tool.ty.src]
include = ["openbb_news"]
exclude = ["tests", "integration"]
[tool.ty.rules]
unresolved-import = "warn"
[tool.coverage.run]
omit = ["openbb_news/news_router.py"]
[tool.coverage.report]
exclude_lines = ["pragma: no cover", "if TYPE_CHECKING:", "raise NotImplementedError", "@overload"]
[tool.pytest.ini_options]
asyncio_mode = "auto"
addopts = "-q --strict-markers"
testpaths = ["tests"]
markers = [
"integration: end-to-end tests that hit live services (excluded from the coverage gate)",
]
filterwarnings = [
"ignore:Inheritance class ClientSession from ClientSession is discouraged:DeprecationWarning",
]

View File

@@ -0,0 +1,134 @@
"""Shared test fixtures."""
from typing import Any
import pytest
_SAMPLE_RSS = b"""<?xml version="1.0" encoding="UTF-8"?>
<rss version="2.0">
<channel>
<title>Sample Feed</title>
<item>
<title>First Article</title>
<link>https://example.test/article/1</link>
<description>&lt;p&gt;A short summary.&lt;/p&gt;</description>
<pubDate>Mon, 19 May 2026 13:00:00 GMT</pubDate>
<author>Jane Doe</author>
</item>
<item>
<title>Second Article</title>
<description>Plain text summary without HTML.</description>
<pubDate>Mon, 19 May 2026 12:00:00 GMT</pubDate>
</item>
</channel>
</rss>
"""
_SAMPLE_ARTICLE_HTML = b"""<!DOCTYPE html><html><body>
<article>
<p>First paragraph of the article.</p>
<p>Second paragraph with more detail.</p>
<p>Third paragraph closing the story.</p>
</article>
</body></html>
"""
_SAMPLE_JSONLD_HTML = b"""<!DOCTYPE html><html><body>
<script type="application/ld+json">{"@type":"NewsArticle","articleBody":"JSON-LD top-level body."}</script>
</body></html>
"""
@pytest.fixture
def sample_rss() -> bytes:
return _SAMPLE_RSS
@pytest.fixture
def sample_article_html() -> bytes:
return _SAMPLE_ARTICLE_HTML
@pytest.fixture
def sample_jsonld_html() -> bytes:
return _SAMPLE_JSONLD_HTML
class _FakeResponse:
def __init__(self, content: bytes = b"", status: int = 200) -> None:
self._content = content
self.status = status
async def read(self) -> bytes:
return self._content
def raise_for_status(self) -> None:
if self.status >= 400:
import aiohttp
raise aiohttp.ClientResponseError(
request_info=None, # type: ignore[arg-type]
history=(),
status=self.status,
)
async def __aenter__(self) -> "_FakeResponse":
return self
async def __aexit__(self, *_args: Any) -> None:
return None
class _FakeSession:
def __init__(self, response_map: dict) -> None:
self.response_map = response_map
self.calls: list[str] = []
self.closed = False
async def get(self, url: str, **_kwargs: Any) -> _FakeResponse:
self.calls.append(url)
result = self.response_map.get(url)
if result is None:
raise KeyError(f"unmocked URL: {url}")
if isinstance(result, BaseException):
raise result
return result
async def __aenter__(self) -> "_FakeSession":
return self
async def __aexit__(self, *_args: Any) -> None:
self.closed = True
async def close(self) -> None:
self.closed = True
@pytest.fixture
def stub_session_factory(monkeypatch):
def factory(response_map: dict) -> _FakeSession:
normalized: dict = {}
for url, value in response_map.items():
if isinstance(value, (bytes, bytearray)):
normalized[url] = _FakeResponse(bytes(value))
elif (
isinstance(value, tuple)
and len(value) == 2
and isinstance(value[0], (bytes, bytearray))
):
content, status = value
normalized[url] = _FakeResponse(bytes(content), status=int(status))
else:
normalized[url] = value
session = _FakeSession(normalized)
async def _factory(*_a: Any, **_kw: Any) -> _FakeSession:
return session
monkeypatch.setattr(
"openbb_core.provider.utils.helpers.get_async_requests_session",
_factory,
)
return session
return factory

View File

@@ -0,0 +1,606 @@
"""Parser tests."""
import time
from lxml import html
from openbb_news import parser
def test_strip_html_empty():
assert parser.strip_html("") == ""
def test_strip_html_plain():
assert parser.strip_html("hello world") == "hello world"
def test_strip_html_tags_and_whitespace():
assert parser.strip_html("<p>Hello <b>world</b>\n!</p>") == "Hello world !"
def test_html_to_markdown_empty():
assert parser.html_to_markdown("") == ""
def test_html_to_markdown_preserves_anchor_links():
out = parser.html_to_markdown(
'<a href="https://nyt.test/x" target="_blank">NYT headline</a>&nbsp;'
'<a href="https://bbc.test/x">BBC headline</a>'
)
assert "[NYT headline](https://nyt.test/x)" in out
assert "[BBC headline](https://bbc.test/x)" in out
def test_html_to_markdown_strips_non_anchor_tags():
out = parser.html_to_markdown("<p>Hello <b>bold</b> world</p>")
assert "bold" in out
assert "<" not in out
def test_html_to_markdown_anchor_without_href():
out = parser.html_to_markdown('<a target="_blank">just text</a>')
assert "just text" in out
assert "[" not in out
def test_html_to_markdown_anchor_without_text():
out = parser.html_to_markdown('<a href="https://x.test/y"></a>')
assert "[" not in out
def test_html_to_markdown_handles_paragraphs_and_breaks():
out = parser.html_to_markdown("<p>Line one</p><p>Line two</p>")
assert "Line one" in out
assert "Line two" in out
assert "Line one" in out.split("\n")[0]
def test_html_to_markdown_handles_br():
out = parser.html_to_markdown("First<br/>Second")
assert "First" in out
assert "Second" in out
def test_html_to_markdown_paragraph_after_text():
out = parser.html_to_markdown("intro text<p>paragraph</p>")
assert "intro text" in out
assert "paragraph" in out
lines = [line for line in out.split("\n") if line.strip()]
assert lines.index("intro text") < lines.index("paragraph")
def test_html_to_markdown_lxml_parse_failure_falls_back(monkeypatch):
from lxml import html as _html
def boom(*_a, **_kw):
raise ValueError("synthetic parse failure")
monkeypatch.setattr(_html, "fragment_fromstring", boom)
out = parser.html_to_markdown('<a href="x">link</a>')
assert out == "link"
def test_truncate_short():
assert parser.truncate("abc", limit=10) == "abc"
def test_truncate_long():
text = "word " * 100
truncated = parser.truncate(text, limit=50)
assert len(truncated) <= 60
assert truncated.endswith("…")
def test_struct_time_to_iso_none():
iso = parser.struct_time_to_iso(None)
assert "T" in iso and iso.endswith("+00:00")
def test_struct_time_to_iso_value():
st = time.struct_time((2026, 5, 19, 12, 0, 0, 0, 139, 0))
assert parser.struct_time_to_iso(st) == "2026-05-19T12:00:00+00:00"
def test_article_markdown_joins_with_blank_lines(sample_article_html):
doc = html.fromstring(sample_article_html)
article = doc.xpath("//article")[0]
text = parser._article_markdown(article)
assert "First paragraph of the article." in text
assert text.count("\n\n") == 2
def test_article_markdown_requires_two_paragraphs():
doc = html.fromstring(b"<div><p>only one paragraph</p></div>")
node = doc.xpath("//div")[0]
assert parser._article_markdown(node) == ""
def test_article_markdown_filters_video_chapter_timestamps():
doc = html.fromstring(
b"<div>"
b"<p>Real opening paragraph with content.</p>"
b"<p>2:26</p>"
b"<p>12:44</p>"
b"<p>0:58</p>"
b"<p>1:30:00</p>"
b"<p>Real closing paragraph.</p>"
b"</div>"
)
text = parser._article_markdown(doc.xpath("//div")[0])
assert "2:26" not in text
assert "12:44" not in text
assert "0:58" not in text
assert "1:30:00" not in text
assert "Real opening paragraph" in text
assert "Real closing paragraph" in text
def test_article_markdown_keeps_timestamp_inside_sentence():
doc = html.fromstring(
b"<div>"
b"<p>The video begins at 2:26 with the host introducing.</p>"
b"<p>By 12:44 the segment closes.</p>"
b"</div>"
)
text = parser._article_markdown(doc.xpath("//div")[0])
assert "2:26" in text
assert "12:44" in text
def test_article_markdown_ignores_non_paragraph_text():
doc = html.fromstring(b"<div>plain wrapper text <span>here</span></div>")
node = doc.xpath("//div")[0]
assert parser._article_markdown(node) == ""
def test_article_markdown_skips_figure_without_img():
doc = html.fromstring(
b"<article>"
b"<p>One.</p>"
b"<figure><figcaption>No image here</figcaption></figure>"
b"<p>Two.</p>"
b"</article>"
)
text = parser._article_markdown(doc.xpath("//article")[0])
assert "![" not in text
assert "One." in text
assert "Two." in text
def test_article_markdown_skips_img_without_src():
doc = html.fromstring(b"<article><p>One.</p><img/><p>Two.</p></article>")
text = parser._article_markdown(doc.xpath("//article")[0])
assert "![" not in text
def test_article_markdown_figure_without_caption():
doc = html.fromstring(
b"<article>"
b"<p>One.</p>"
b'<figure><img src="https://x.test/p.jpg"/></figure>'
b"<p>Two.</p>"
b"</article>"
)
text = parser._article_markdown(doc.xpath("//article")[0])
assert "![](https://x.test/p.jpg)" in text
def test_strip_chrome_removes_chrome_tags():
doc = html.fromstring(
b"<html><body>"
b"<nav>nav junk</nav><header>h</header><footer>f</footer>"
b"<aside>a</aside><form>fm</form><script>s</script><style>.x{}</style>"
b"<noscript>n</noscript><iframe>i</iframe><button>b</button><svg>sv</svg>"
b"<article><p>kept one</p><p>kept two</p></article>"
b"</body></html>"
)
parser._strip_chrome(doc)
leftover_tags = {el.tag for el in doc.iter() if isinstance(el.tag, str)}
for tag in parser._CHROME_TAGS:
assert tag not in leftover_tags
assert "article" in leftover_tags
def test_looks_clean_rejects_templates_and_empty():
assert parser._looks_clean("Some real text.")
assert not parser._looks_clean("")
assert not parser._looks_clean("[[ type === 'moc' ]] body text")
assert not parser._looks_clean("body with {{ binding }} marker")
async def test_fetch_feed_parses(stub_session_factory, sample_rss):
session = stub_session_factory({"http://feed.test/rss": sample_rss})
parsed = await parser.fetch_feed(session, "http://feed.test/rss")
assert len(parsed.entries) == 2
assert parsed.entries[0].title == "First Article"
async def test_fetch_feed_client_error(stub_session_factory):
import aiohttp
session = stub_session_factory(
{"http://feed.test/rss": aiohttp.ClientError("boom")}
)
parsed = await parser.fetch_feed(session, "http://feed.test/rss")
assert len(parsed.entries) == 0
async def test_fetch_feed_http_error(stub_session_factory):
session = stub_session_factory({"http://feed.test/rss": (b"", 500)})
parsed = await parser.fetch_feed(session, "http://feed.test/rss")
assert len(parsed.entries) == 0
def test_extract_body_jsonld_top_level(sample_jsonld_html):
assert parser._extract_body(sample_jsonld_html) == "JSON-LD top-level body."
def test_extract_body_jsonld_graph():
payload = (
b'<html><body><script type="application/ld+json">'
b'{"@graph":[{"@type":"Other"},'
b'{"@type":"NewsArticle","articleBody":"Body from graph."}]}'
b"</script></body></html>"
)
assert parser._extract_body(payload) == "Body from graph."
def test_extract_body_jsonld_list():
payload = (
b'<html><body><script type="application/ld+json">'
b'[{"@type":"Other"},'
b'{"@type":"NewsArticle","articleBody":"Found in list."}]'
b"</script></body></html>"
)
assert parser._extract_body(payload) == "Found in list."
def test_extract_body_jsonld_invalid_then_article():
payload = (
b"<html><body>"
b'<script type="application/ld+json">not json</script>'
b"<article><p>First sentence.</p><p>Second sentence.</p></article>"
b"</body></html>"
)
assert parser._extract_body(payload) == "First sentence.\n\nSecond sentence."
def test_extract_body_article_selector(sample_article_html):
body = parser._extract_body(sample_article_html)
assert body is not None
assert "First paragraph" in body
assert body.count("\n\n") == 2
def test_extract_body_picks_largest_article():
payload = (
b"<html><body>"
b"<article><p>Related card one.</p><p>Related card two.</p></article>"
b"<article>"
b"<p>Main story paragraph one with more substance.</p>"
b"<p>Main story paragraph two with detail.</p>"
b"<p>Main story paragraph three closing the piece.</p>"
b"</article>"
b"</body></html>"
)
body = parser._extract_body(payload)
assert body is not None
assert "Main story paragraph one" in body
assert "Related card" not in body
def test_extract_body_main_fallback_when_no_article():
payload = (
b"<html><body>"
b"<nav><p>nav link</p><p>nav link two</p></nav>"
b"<main>"
b"<div><p>Story paragraph one with content.</p>"
b"<p>Story paragraph two with more detail.</p></div>"
b"</main>"
b"</body></html>"
)
assert (
parser._extract_body(payload)
== "Story paragraph one with content.\n\nStory paragraph two with more detail."
)
def test_extract_body_main_requires_two_paragraphs():
payload = b"<html><body><main><p>Only one paragraph here.</p></main></body></html>"
assert parser._extract_body(payload) is None
def test_extract_body_article_wins_over_main():
payload = (
b"<html><body>"
b"<main>"
b"<p>Main paragraph one.</p><p>Main paragraph two.</p>"
b"<p>Main paragraph three.</p><p>Main paragraph four.</p>"
b"</main>"
b"<article><p>Article paragraph one.</p><p>Article paragraph two.</p></article>"
b"</body></html>"
)
assert (
parser._extract_body(payload)
== "Article paragraph one.\n\nArticle paragraph two."
)
def test_extract_body_rejects_unrendered_templates():
payload = (
b"<html><body>"
b"<article>"
b"<p>[[ type === 'moc' ? 'MARKET ON CLOSE' : 'FREE WEBINAR' ]]</p>"
b"<p>[[ timeLabel ]] WATCH LIVE</p>"
b"</article>"
b"</body></html>"
)
assert parser._extract_body(payload) is None
def test_extract_body_strips_chrome_before_extracting():
payload = (
b"<html><body>"
b"<nav><p>nav link one</p><p>nav link two</p></nav>"
b"<header><p>banner text</p><p>banner two</p></header>"
b"<article><p>Real story paragraph.</p><p>Story continues here.</p></article>"
b"<footer><p>footer note</p><p>footer two</p></footer>"
b"</body></html>"
)
assert (
parser._extract_body(payload)
== "Real story paragraph.\n\nStory continues here."
)
def test_extract_body_no_container():
assert parser._extract_body(b"<html><body>nothing</body></html>") is None
def test_extract_body_html_parse_error():
assert parser._extract_body(b"") is None
def test_extract_body_jsonld_non_dict_entries():
payload = (
b"<html><body>"
b'<script type="application/ld+json">["string", 42, null]</script>'
b"<article><p>Fallback one.</p><p>Fallback two.</p></article>"
b"</body></html>"
)
assert parser._extract_body(payload) == "Fallback one.\n\nFallback two."
def test_extract_body_jsonld_graph_no_body():
payload = (
b"<html><body>"
b'<script type="application/ld+json">'
b'{"@type":"NewsArticle","@graph":[{"@type":"X"}]}'
b"</script>"
b"<article><p>Graph fallback one.</p><p>Graph fallback two.</p></article>"
b"</body></html>"
)
assert parser._extract_body(payload) == "Graph fallback one.\n\nGraph fallback two."
def test_extract_body_jsonld_graph_non_dict_entry():
payload = (
b"<html><body>"
b'<script type="application/ld+json">'
b'{"@type":"NewsArticle","@graph":["scalar",null,42,'
b'{"@type":"NewsArticle","articleBody":"Graph article body."}]}'
b"</script></body></html>"
)
assert parser._extract_body(payload) == "Graph article body."
def test_extract_body_jsonld_template_falls_through_to_article():
payload = (
b"<html><body>"
b'<script type="application/ld+json">'
b'{"@type":"NewsArticle","articleBody":"[[ binding ]] template junk"}'
b"</script>"
b"<article><p>Clean paragraph one.</p><p>Clean paragraph two.</p></article>"
b"</body></html>"
)
assert (
parser._extract_body(payload) == "Clean paragraph one.\n\nClean paragraph two."
)
def test_extract_body_preserves_utf8_smart_punctuation():
payload = (
"<html><body><article>"
"<p>The Court’s ruling — issued Tuesday — was clear.</p>"
"<p>Dominion’s lawyers said “we won”.</p>"
"</article></body></html>"
).encode()
body = parser._extract_body(payload)
assert body is not None
assert "Court’s" in body
assert "Dominion’s" in body
assert "“we won”" in body
assert "—" in body
for artifact in ("’", "–", " "):
assert artifact not in body
async def test_fetch_article_body_end_to_end(stub_session_factory, sample_article_html):
session = stub_session_factory({"https://x.test/a": sample_article_html})
body = await parser.fetch_article_body(session, "https://x.test/a")
assert body is not None
assert "First paragraph" in body
async def test_fetch_article_body_client_error(stub_session_factory):
import aiohttp
session = stub_session_factory({"https://x.test/a": aiohttp.ClientError("boom")})
assert await parser.fetch_article_body(session, "https://x.test/a") is None
async def test_fetch_article_body_timeout(stub_session_factory):
session = stub_session_factory(
{"https://x.test/a": TimeoutError("simulated timeout")}
)
assert await parser.fetch_article_body(session, "https://x.test/a") is None
async def test_fetch_article_body_http_error(stub_session_factory):
session = stub_session_factory({"https://x.test/a": (b"", 403)})
assert await parser.fetch_article_body(session, "https://x.test/a") is None
async def test_fetch_article_body_no_article_returns_none(stub_session_factory):
session = stub_session_factory(
{"https://x.test/a": b"<html><body>nothing</body></html>"}
)
assert await parser.fetch_article_body(session, "https://x.test/a") is None
def test_extract_body_preserves_inline_img():
payload = (
b"<html><body><article>"
b"<p>Opening paragraph.</p>"
b'<img src="https://x.test/inline.jpg" alt="caption text"/>'
b"<p>Closing paragraph.</p>"
b"</article></body></html>"
)
body = parser._extract_body(payload)
assert body is not None
lines = body.split("\n\n")
assert lines == [
"Opening paragraph.",
"![caption text](https://x.test/inline.jpg)",
"Closing paragraph.",
]
def test_extract_body_preserves_figure_with_caption():
payload = (
b"<html><body><article>"
b"<p>Lead in.</p>"
b'<figure><img src="https://x.test/photo.jpg"/>'
b"<figcaption>A long caption describing the photo.</figcaption>"
b"</figure>"
b"<p>After the photo.</p>"
b"</article></body></html>"
)
body = parser._extract_body(payload)
assert body is not None
assert "![A long caption describing the photo.](https://x.test/photo.jpg)" in body
assert "Lead in." in body
assert "After the photo." in body
def test_extract_body_prepends_og_image_when_no_inline_image():
payload = (
b"<html><head>"
b'<meta property="og:image" content="https://x.test/hero.jpg">'
b"</head><body><article>"
b"<p>Paragraph one.</p><p>Paragraph two.</p>"
b"</article></body></html>"
)
body = parser._extract_body(payload)
assert body is not None
assert body.split("\n\n")[0] == "![](https://x.test/hero.jpg)"
assert "Paragraph one." in body
def test_extract_body_skips_hero_when_inline_image_present():
payload = (
b"<html><head>"
b'<meta property="og:image" content="https://x.test/hero.jpg">'
b"</head><body><article>"
b"<p>Lead.</p>"
b'<img src="https://x.test/inline.jpg"/>'
b"<p>Body.</p>"
b"</article></body></html>"
)
body = parser._extract_body(payload)
assert body is not None
assert "https://x.test/hero.jpg" not in body
assert "![](https://x.test/inline.jpg)" in body
def test_extract_body_jsonld_articlebody_with_og_hero():
payload = (
b"<html><head>"
b'<meta property="og:image" content="https://x.test/hero.jpg">'
b"</head><body>"
b'<script type="application/ld+json">'
b'{"@type":"NewsArticle","articleBody":"Pure text body."}'
b"</script></body></html>"
)
body = parser._extract_body(payload)
assert body == "![](https://x.test/hero.jpg)\n\nPure text body."
def test_html_to_markdown_preserves_inline_img():
out = parser.html_to_markdown(
'<p>Lead</p><img src="https://x.test/inline.jpg" alt="alt text"/>'
)
assert "![alt text](https://x.test/inline.jpg)" in out
assert "Lead" in out
def test_html_to_markdown_img_tail_text():
out = parser.html_to_markdown(
'<img src="https://x.test/inline.jpg"/> trailing text after img'
)
assert "![](https://x.test/inline.jpg)" in out
assert "trailing text after img" in out
def test_jsonld_field_list_of_strings():
payload = (
b"<html><body>"
b'<script type="application/ld+json">'
b'{"@type":"NewsArticle","image":["https://x.test/first.jpg","https://x.test/second.jpg"]}'
b"</script></body></html>"
)
body = parser._extract_body(payload)
assert body is None or body.startswith("![](https://x.test/first.jpg)")
def test_jsonld_field_list_of_image_objects():
payload = (
b"<html><body>"
b'<script type="application/ld+json">'
b'{"@type":"NewsArticle","image":[{"url":"https://x.test/o.jpg"}]}'
b"</script>"
b"<article><p>One.</p><p>Two.</p></article>"
b"</body></html>"
)
body = parser._extract_body(payload)
assert body is not None
assert body.startswith("![](https://x.test/o.jpg)")
def test_jsonld_field_image_object_with_url():
payload = (
b"<html><body>"
b'<script type="application/ld+json">'
b'{"@type":"NewsArticle","image":{"@type":"ImageObject","url":"https://x.test/obj.jpg"}}'
b"</script>"
b"<article><p>One.</p><p>Two.</p></article>"
b"</body></html>"
)
body = parser._extract_body(payload)
assert body is not None
assert body.startswith("![](https://x.test/obj.jpg)")
def test_extract_image_jsonld_fallback_path():
payload = (
b"<html><body>"
b'<script type="application/ld+json">'
b'{"@type":"NewsArticle","image":"https://x.test/ld.jpg"}'
b"</script>"
b"<article><p>One.</p><p>Two.</p></article>"
b"</body></html>"
)
body = parser._extract_body(payload)
assert body is not None
assert body.startswith("![](https://x.test/ld.jpg)")

View File

@@ -0,0 +1,546 @@
"""Registry tests."""
import pytest
from openbb_news import registry
def test_list_feeds_returns_defaults(monkeypatch):
monkeypatch.setattr(registry, "load_config", lambda: {})
feeds = registry.list_feeds()
assert "pr_newswire_global" in feeds
assert "bbc_world" in feeds
assert feeds["bbc_world"].endswith("/world/rss.xml")
def test_list_feeds_user_override(monkeypatch):
monkeypatch.setattr(
registry,
"load_config",
lambda: {"news": {"rss_feeds": {"my_feed": "https://my.test/rss"}}},
)
feeds = registry.list_feeds()
assert feeds == {"my_feed": "https://my.test/rss"}
def test_list_feeds_ignores_non_dict_override(monkeypatch):
monkeypatch.setattr(
registry, "load_config", lambda: {"news": {"rss_feeds": ["not", "a", "dict"]}}
)
feeds = registry.list_feeds()
assert "bbc_world" in feeds
def test_list_feeds_ignores_empty_override(monkeypatch):
monkeypatch.setattr(registry, "load_config", lambda: {"news": {"rss_feeds": {}}})
feeds = registry.list_feeds()
assert "bbc_world" in feeds
def test_get_feed_url_known():
url = registry.get_feed_url("bbc_world")
assert url.endswith("/world/rss.xml")
def test_get_feed_url_unknown_raises():
with pytest.raises(ValueError, match="Unknown RSS feed source"):
registry.get_feed_url("not_a_real_source")
def test_list_providers_default(monkeypatch):
monkeypatch.setattr(registry, "load_config", lambda: {})
providers = registry.list_providers()
assert providers["axios"] == "Axios"
assert providers["bbc"] == "BBC"
assert providers["benzinga"] == "Benzinga"
assert providers["cbc"] == "CBC"
assert providers["drudge_report"] == "Drudge Report"
assert providers["fox_news"] == "Fox News"
assert providers["fortune"] == "Fortune"
assert providers["globenewswire"] == "GlobeNewswire"
assert providers["google_news"] == "Google News"
assert providers["pr_newswire"] == "PR Newswire"
assert providers["yahoo_finance"] == "Yahoo Finance"
assert providers["wired"] == "Wired"
assert "custom" not in providers
assert "barchart" not in providers
assert "cnn" not in providers
assert "seeking_alpha" not in providers
def test_get_feed_url_drudge_report():
assert (
registry.get_feed_url("drudge_report")
== "http://feeds.feedburner.com/DrudgeReportFeed"
)
def test_list_feed_choices_cbc(monkeypatch):
monkeypatch.setattr(registry, "load_config", lambda: {})
choices = registry.list_feed_choices("cbc")
labels = [c["label"] for c in choices]
assert "Business" in labels
assert "World" in labels
assert "Region — Toronto" in labels
assert "Sports — NHL" in labels
assert labels == sorted(labels)
def test_get_feed_url_cbc_business():
assert (
registry.get_feed_url("cbc_business")
== "https://www.cbc.ca/webfeed/rss/rss-business"
)
def test_list_feed_choices_benzinga(monkeypatch):
monkeypatch.setattr(registry, "load_config", lambda: {})
choices = registry.list_feed_choices("benzinga")
labels = [c["label"] for c in choices]
assert "Markets" in labels
assert "Cryptocurrency" in labels
assert "Latest" in labels
assert labels == sorted(labels)
def test_list_feed_choices_globenewswire(monkeypatch):
monkeypatch.setattr(registry, "load_config", lambda: {})
choices = registry.list_feed_choices("globenewswire")
labels = [c["label"] for c in choices]
assert "All News" in labels
assert "United States" in labels
assert "Industry — Energy" in labels
assert "Industry — Financials" in labels
assert "Industry — Banks" in labels
assert "Industry — Pharmaceuticals" in labels
assert "Industry — Gold Mining" in labels
assert "Industry — Semiconductors" in labels
assert "Industry — Diversified REITs" in labels
assert "Subject — Earnings Releases and Operating Results" in labels
assert "Subject — Dividend Reports and Estimates" in labels
assert "Subject — Mergers and Acquisitions" in labels
assert len(choices) == 253
assert labels == sorted(labels)
def test_get_feed_url_globenewswire_all():
url = registry.get_feed_url("globenewswire_all")
assert url.startswith("https://www.globenewswire.com/RssFeed/")
assert "AllNews" in url
def test_get_feed_url_globenewswire_industry():
url = registry.get_feed_url("globenewswire_energy")
assert url.startswith(
"https://www.globenewswire.com/AtomFeed/industry/1/feedTitle/"
)
assert "Energy" in url
def test_globenewswire_industry_key_helper():
assert (
registry._globenewswire_industry_key("Iron and Steel")
== "globenewswire_iron_and_steel"
)
assert (
registry._globenewswire_industry_key(
"Building, Roofing, Wallboard and Plumbing"
)
== "globenewswire_building_roofing_wallboard_and_plumbing"
)
assert (
registry._globenewswire_industry_key("REITs & Real Estate")
== "globenewswire_reits_and_real_estate"
)
def test_globenewswire_industries_unique_keys():
keys = [
registry._globenewswire_industry_key(label)
for _code, label in registry._GLOBENEWSWIRE_INDUSTRIES
]
assert len(keys) == len(set(keys))
def test_get_feed_url_globenewswire_subject():
url = registry.get_feed_url(
"globenewswire_subject_earnings_releases_and_operating_results"
)
assert url.startswith(
"https://www.globenewswire.com/AtomFeed/subjectcode/13/feedTitle/"
)
assert "Earnings" in url
def test_globenewswire_subject_key_helper():
assert (
registry._globenewswire_subject_key("Insider's Buy, Sell")
== "globenewswire_subject_insiders_buy_sell"
)
assert (
registry._globenewswire_subject_key("Mergers and Acquisitions")
== "globenewswire_subject_mergers_and_acquisitions"
)
def test_globenewswire_subjects_unique_keys():
keys = [
registry._globenewswire_subject_key(label)
for _code, label in registry._GLOBENEWSWIRE_SUBJECTS
]
assert len(keys) == len(set(keys))
def test_get_feed_url_benzinga_markets():
assert (
registry.get_feed_url("benzinga_markets")
== "https://www.benzinga.com/markets/feed"
)
def test_list_feed_choices_google_news(monkeypatch):
monkeypatch.setattr(registry, "load_config", lambda: {})
choices = registry.list_feed_choices("google_news")
labels = [c["label"] for c in choices]
assert "Business" in labels
assert "Technology" in labels
assert "Top — US" in labels
assert "Top — Canada" in labels
assert labels == sorted(labels)
def test_get_feed_url_google_news_us():
assert (
registry.get_feed_url("google_news_us")
== "https://news.google.com/rss?hl=en-US&gl=US&ceid=US:en"
)
def test_default_feed_for_curated_picks(monkeypatch):
monkeypatch.setattr(registry, "load_config", lambda: {})
expected = {
"axios": "axios_main",
"bbc": "bbc_world",
"benzinga": "benzinga_markets",
"cbc": "cbc_business",
"fox_news": "fox_news_latest",
"globenewswire": "globenewswire_all",
"google_news": "google_news_us",
"pr_newswire": "pr_newswire_global",
"wired": "wired_business",
}
for provider, feed_key in expected.items():
assert registry.default_feed_for(provider) == feed_key
def test_default_feed_for_falls_back_to_first_alphabetical(monkeypatch):
monkeypatch.setattr(registry, "load_config", lambda: {})
assert registry.default_feed_for("fortune") == "fortune"
def test_curated_defaults_reference_real_feeds(monkeypatch):
monkeypatch.setattr(registry, "load_config", lambda: {})
feeds = registry.list_feeds()
for provider, feed_key in registry._DEFAULT_FEED_BY_PROVIDER.items():
assert feed_key in feeds, f"{provider} default {feed_key!r} missing"
def test_every_multi_feed_provider_has_a_curated_default(monkeypatch):
monkeypatch.setattr(registry, "load_config", lambda: {})
for provider in registry.list_providers():
if provider == "custom":
continue
choices = registry.list_feed_choices(provider)
if len(choices) > 1:
assert provider in registry._DEFAULT_FEED_BY_PROVIDER, (
f"multi-feed provider {provider!r} needs a curated default"
)
def test_default_feed_for_unknown_provider(monkeypatch):
monkeypatch.setattr(registry, "load_config", lambda: {})
assert registry.default_feed_for("totally_made_up") is None
def test_default_feed_for_empty_provider():
assert registry.default_feed_for(None) is None
assert registry.default_feed_for("") is None
def test_default_feed_for_drops_stale_curated_default(monkeypatch):
monkeypatch.setattr(
registry,
"load_config",
lambda: {"news": {"rss_feeds": {"only_one": "https://x.test/rss"}}},
)
assert registry.default_feed_for("axios") is None
assert registry.default_feed_for("custom") == "only_one"
def test_list_feed_choices_axios(monkeypatch):
monkeypatch.setattr(registry, "load_config", lambda: {})
choices = registry.list_feed_choices("axios")
assert choices == [{"label": "Latest", "value": "axios_main"}]
def test_get_feed_url_axios_main():
assert registry.get_feed_url("axios_main") == "https://api.axios.com/feed/"
def test_list_feed_choices_wired(monkeypatch):
monkeypatch.setattr(registry, "load_config", lambda: {})
choices = registry.list_feed_choices("wired")
labels = [c["label"] for c in choices]
assert "Latest" in labels
assert "Backchannel" in labels
assert "Business" in labels
assert "Culture" in labels
assert "Gear" in labels
assert "Ideas" in labels
assert "Science" in labels
assert "Security" in labels
assert "Tag — AI" in labels
assert "Tag — Wired Guide" in labels
assert len(choices) == 10
assert labels == sorted(labels)
def test_get_feed_url_wired_main():
assert registry.get_feed_url("wired_main") == "https://www.wired.com/feed/rss"
def test_get_feed_url_wired_business():
assert (
registry.get_feed_url("wired_business")
== "https://www.wired.com/feed/category/business/latest/rss"
)
def test_list_providers_user_feeds_are_custom(monkeypatch):
monkeypatch.setattr(
registry,
"load_config",
lambda: {"news": {"rss_feeds": {"my_feed": "https://my.test/rss"}}},
)
providers = registry.list_providers()
assert providers == {"custom": "Custom"}
def test_list_feed_choices_filters_by_provider(monkeypatch):
monkeypatch.setattr(registry, "load_config", lambda: {})
bbc = registry.list_feed_choices("bbc")
labels = [c["label"] for c in bbc]
assert "World" in labels
assert "Politics" in labels
assert "Technology" in labels
assert labels == sorted(labels)
fox = registry.list_feed_choices("fox_news")
fox_labels = [c["label"] for c in fox]
assert "Latest Headlines" in fox_labels
assert "Politics" in fox_labels
def test_list_feed_choices_pr_newswire(monkeypatch):
monkeypatch.setattr(registry, "load_config", lambda: {})
choices = registry.list_feed_choices("pr_newswire")
labels = [c["label"] for c in choices]
assert "Global" in labels
assert "APAC — General Business" in labels
assert "APAC (Chinese)" in labels
assert labels == sorted(labels)
def test_list_feed_choices_no_provider(monkeypatch):
monkeypatch.setattr(registry, "load_config", lambda: {})
assert registry.list_feed_choices(None) == []
assert registry.list_feed_choices("") == []
def test_list_feed_choices_unknown_provider(monkeypatch):
monkeypatch.setattr(registry, "load_config", lambda: {})
assert registry.list_feed_choices("totally_made_up") == []
def test_list_feed_choices_custom_feeds(monkeypatch):
monkeypatch.setattr(
registry,
"load_config",
lambda: {"news": {"rss_feeds": {"my_feed": "https://my.test/rss"}}},
)
choices = registry.list_feed_choices("custom")
assert choices == [{"label": "My Feed", "value": "my_feed"}]
def test_user_feeds_rich_form_with_provider_and_label(monkeypatch):
monkeypatch.setattr(
registry,
"load_config",
lambda: {
"news": {
"rss_feeds": {
"alpha": {
"url": "https://alpha.test/rss",
"provider": "internal",
"label": "Alpha Research",
},
"beta": {
"url": "https://beta.test/rss",
"provider": "internal",
"label": "Beta Research",
},
}
}
},
)
feeds = registry.list_feeds()
assert feeds == {
"alpha": "https://alpha.test/rss",
"beta": "https://beta.test/rss",
}
choices = registry.list_feed_choices("internal")
assert {c["value"] for c in choices} == {"alpha", "beta"}
labels = {c["label"] for c in choices}
assert labels == {"Alpha Research", "Beta Research"}
def test_user_provider_labels_apply(monkeypatch):
monkeypatch.setattr(
registry,
"load_config",
lambda: {
"news": {
"rss_providers": {"internal": "Internal Sources"},
"rss_feeds": {
"x": {"url": "https://x.test/rss", "provider": "internal"},
},
}
},
)
providers = registry.list_providers()
assert providers == {"internal": "Internal Sources"}
def test_user_provider_unknown_id_falls_back_to_humanized(monkeypatch):
monkeypatch.setattr(
registry,
"load_config",
lambda: {
"news": {
"rss_feeds": {
"x": {"url": "https://x.test/rss", "provider": "acme_research"},
},
}
},
)
providers = registry.list_providers()
assert providers == {"acme_research": "Acme Research"}
def test_user_feeds_merge_defaults_preserves_bundled(monkeypatch):
monkeypatch.setattr(
registry,
"load_config",
lambda: {
"news": {
"merge_defaults": True,
"rss_feeds": {"x": "https://x.test/rss"},
}
},
)
feeds = registry.list_feeds()
assert "bbc_world" in feeds
assert "pr_newswire_global" in feeds
assert feeds["x"] == "https://x.test/rss"
providers = registry.list_providers()
assert "bbc" in providers
assert "custom" in providers
def test_user_feeds_merge_override_replaces_bundled_key(monkeypatch):
monkeypatch.setattr(
registry,
"load_config",
lambda: {
"news": {
"merge_defaults": True,
"rss_feeds": {"bbc_world": "https://my.test/bbc"},
}
},
)
feeds = registry.list_feeds()
assert feeds["bbc_world"] == "https://my.test/bbc"
def test_user_feeds_rich_form_ignores_missing_url(monkeypatch):
monkeypatch.setattr(
registry,
"load_config",
lambda: {
"news": {
"rss_feeds": {
"good": {"url": "https://good.test/rss"},
"bad": {"provider": "internal"},
"also_bad": 42,
}
}
},
)
feeds = registry.list_feeds()
assert feeds == {"good": "https://good.test/rss"}
def test_user_feeds_rich_form_default_provider_is_custom(monkeypatch):
monkeypatch.setattr(
registry,
"load_config",
lambda: {
"news": {
"rss_feeds": {
"x": {"url": "https://x.test/rss", "label": "Pretty Label"},
}
}
},
)
providers = registry.list_providers()
assert providers == {"custom": "Custom"}
choices = registry.list_feed_choices("custom")
assert choices == [{"label": "Pretty Label", "value": "x"}]
def test_load_config_returning_non_dict_falls_back_to_defaults(monkeypatch):
monkeypatch.setattr(registry, "load_config", lambda: None)
feeds = registry.list_feeds()
assert "bbc_world" in feeds
def test_user_provider_labels_ignores_non_string_values(monkeypatch):
monkeypatch.setattr(
registry,
"load_config",
lambda: {
"news": {
"rss_providers": {"internal": 42},
"rss_feeds": {
"x": {"url": "https://x.test/rss", "provider": "internal"},
},
}
},
)
providers = registry.list_providers()
assert providers == {"internal": "Internal"}
def test_slug_helper():
assert registry._slug("apac/zh") == "apac_zh"
assert registry._slug("business-technology") == "business_technology"
def test_humanize_helper():
assert registry._humanize("my_internal_feed") == "My Internal Feed"
def test_default_meta_covers_every_feed():
assert set(registry._DEFAULT_FEEDS) == set(registry._DEFAULT_META)
for entry in registry._DEFAULT_META.values():
assert entry["provider"]
assert entry["label"]

View File

@@ -0,0 +1,24 @@
"""Router tests."""
from openbb_core.app.router import Router
from openbb_news.router import router
def test_router_aggregates_rss_commands():
assert isinstance(router, Router)
paths = {getattr(route, "path", "") for route in router.api_router.routes}
assert "/rss" in paths
assert "/rss_providers" in paths
assert "/rss_feeds" in paths
def test_router_aggregates_provider_news_commands():
from openbb_core.app.provider_interface import ProviderInterface
paths = {getattr(route, "path", "") for route in router.api_router.routes}
model_map = ProviderInterface().map
if "CompanyNews" in model_map:
assert "/company" in paths
if "WorldNews" in model_map:
assert "/world" in paths

View File

@@ -0,0 +1,350 @@
"""RSS endpoint tests."""
import pytest
from openbb_news import registry, rss
@pytest.fixture
def mocked_pipeline(monkeypatch, stub_session_factory, sample_rss, sample_article_html):
monkeypatch.setattr(
"openbb_news.registry.get_feed_url",
lambda key: f"http://feed.test/{key}",
)
stub_session_factory(
{
"http://feed.test/example": sample_rss,
"https://example.test/article/1": sample_article_html,
}
)
async def test_rss_with_body(mocked_pipeline):
out = await rss.rss(source="example", limit=5, fetch_body=True)
items = out.results
assert len(items) == 2
assert items[0].title == "First Article"
assert items[0].date.endswith("+00:00")
assert items[0].author == "Jane Doe"
assert items[0].excerpt == "A short summary."
assert items[0].url == "https://example.test/article/1"
assert "First paragraph" in items[0].body
assert items[1].excerpt == "Plain text summary without HTML."
assert items[1].body == items[1].excerpt
assert items[1].url == ""
assert items[1].author == "Sample Feed"
async def test_rss_without_body(mocked_pipeline):
out = await rss.rss(source="example", limit=5, fetch_body=False)
assert all(item.body == item.excerpt for item in out.results)
async def test_rss_limit_caps_entries(mocked_pipeline):
out = await rss.rss(source="example", limit=1, fetch_body=False)
assert len(out.results) == 1
async def test_rss_empty_feed(monkeypatch, stub_session_factory):
monkeypatch.setattr(
"openbb_news.registry.get_feed_url", lambda key: "http://feed.test/empty"
)
stub_session_factory({"http://feed.test/empty": b"<rss><channel></channel></rss>"})
out = await rss.rss(source="empty", limit=5, fetch_body=True)
assert out.results == []
async def test_rss_unknown_source():
with pytest.raises(ValueError, match="Unknown RSS feed source"):
await rss.rss(source="totally_made_up", limit=1, fetch_body=False)
async def test_rss_returns_empty_when_no_source():
out = await rss.rss(source=None, limit=5, fetch_body=False)
assert out.results == []
out2 = await rss.rss(source="", limit=5, fetch_body=False)
assert out2.results == []
async def test_rss_auto_selects_curated_default_for_outlet(
monkeypatch, stub_session_factory, sample_rss
):
monkeypatch.setattr(registry, "default_feed_for", lambda outlet: "curated_default")
monkeypatch.setattr(
"openbb_news.registry.get_feed_url",
lambda key: f"http://feed.test/{key}",
)
stub_session_factory({"http://feed.test/curated_default": sample_rss})
out = await rss.rss(outlet="any", source=None, limit=5, fetch_body=False)
assert len(out.results) == 2
assert out.results[0].title == "First Article"
async def test_rss_returns_empty_for_unknown_outlet(monkeypatch):
monkeypatch.setattr(registry, "default_feed_for", lambda outlet: None)
out = await rss.rss(outlet="ghost", source=None, limit=5, fetch_body=False)
assert out.results == []
async def test_rss_sorts_newest_first(monkeypatch, stub_session_factory):
feed = (
b"<?xml version='1.0'?><rss version='2.0'><channel>"
b"<title>Reversed Feed</title>"
b"<item><title>Oldest</title>"
b"<pubDate>Mon, 17 May 2026 09:00:00 GMT</pubDate></item>"
b"<item><title>Middle</title>"
b"<pubDate>Mon, 18 May 2026 09:00:00 GMT</pubDate></item>"
b"<item><title>Newest</title>"
b"<pubDate>Mon, 19 May 2026 09:00:00 GMT</pubDate></item>"
b"</channel></rss>"
)
monkeypatch.setattr(
"openbb_news.registry.get_feed_url", lambda key: "http://feed.test/rev"
)
stub_session_factory({"http://feed.test/rev": feed})
out = await rss.rss(source="x", limit=5, fetch_body=False)
assert [item.title for item in out.results] == ["Newest", "Middle", "Oldest"]
async def test_rss_sorts_undated_entries_to_bottom(monkeypatch, stub_session_factory):
feed = (
b"<?xml version='1.0'?><rss version='2.0'><channel>"
b"<title>Mixed Feed</title>"
b"<item><title>No Date</title>"
b"<description>no pubdate</description></item>"
b"<item><title>Dated</title>"
b"<pubDate>Mon, 19 May 2026 09:00:00 GMT</pubDate></item>"
b"</channel></rss>"
)
monkeypatch.setattr(
"openbb_news.registry.get_feed_url", lambda key: "http://feed.test/mixed"
)
stub_session_factory({"http://feed.test/mixed": feed})
out = await rss.rss(source="x", limit=5, fetch_body=False)
assert [item.title for item in out.results] == ["Dated", "No Date"]
async def test_rss_handles_missing_title(monkeypatch, stub_session_factory):
monkeypatch.setattr(
"openbb_news.registry.get_feed_url", lambda key: "http://feed.test/notitle"
)
feed = (
b"<?xml version='1.0'?><rss version='2.0'><channel>"
b"<item><description>Body only</description></item>"
b"</channel></rss>"
)
stub_session_factory({"http://feed.test/notitle": feed})
items = (await rss.rss(source="x", limit=1, fetch_body=False)).results
assert items[0].title == "(untitled)"
assert items[0].author == "Unknown"
async def test_rss_providers_lists_known_providers(monkeypatch):
monkeypatch.setattr(registry, "load_config", lambda: {})
out = await rss.rss_providers()
keys = [entry["value"] for entry in out]
assert "bbc" in keys
assert "pr_newswire" in keys
labels = {entry["value"]: entry["label"] for entry in out}
assert labels["bbc"] == "BBC"
assert labels["axios"] == "Axios"
async def test_rss_feeds_filtered_by_provider(monkeypatch):
monkeypatch.setattr(registry, "load_config", lambda: {})
out = await rss.rss_feeds(outlet="bbc")
labels = [entry["label"] for entry in out]
assert "World" in labels
assert "Politics" in labels
assert labels == sorted(labels)
fox_out = await rss.rss_feeds(outlet="fox_news")
fox_labels = [entry["label"] for entry in fox_out]
assert "Latest Headlines" in fox_labels
assert "Politics" in fox_labels
async def test_rss_feeds_no_provider_returns_empty(monkeypatch):
monkeypatch.setattr(registry, "load_config", lambda: {})
assert await rss.rss_feeds(outlet=None) == []
def test_news_item_data_shape():
item = rss.NewsItemData(
title="t",
date="2026-01-01T00:00:00+00:00",
author="a",
url="https://example.test/a",
excerpt="e",
body="b",
)
assert item.title == "t"
assert item.body == "b"
assert item.url == "https://example.test/a"
async def test_rss_body_uses_markdown_links_when_extraction_fails(
monkeypatch, stub_session_factory
):
feed = (
b"<?xml version='1.0' encoding='UTF-8'?>"
b'<rss version="2.0"><channel><title>Aggregator</title>'
b"<item>"
b"<title>Cluster</title>"
b"<link>https://aggregator.test/x</link>"
b"<description><![CDATA["
b'<a href="https://nyt.test/story" target="_blank">NYT headline</a>&nbsp;'
b'<a href="https://bbc.test/story" target="_blank">BBC headline</a>&nbsp;'
b"]]></description>"
b"<pubDate>Mon, 19 May 2026 12:00:00 GMT</pubDate>"
b"</item></channel></rss>"
)
monkeypatch.setattr(
"openbb_news.registry.get_feed_url", lambda key: "http://feed.test/agg"
)
stub_session_factory(
{
"http://feed.test/agg": feed,
"https://aggregator.test/x": b"<html><body><div>not an article</div></body></html>",
}
)
items = (await rss.rss(source="x", limit=1, fetch_body=True)).results
assert len(items) == 1
body = items[0].body
assert "[NYT headline](https://nyt.test/story)" in body
assert "[BBC headline](https://bbc.test/story)" in body
assert items[0].url == "https://aggregator.test/x"
async def test_rss_body_includes_inline_image_from_article(
monkeypatch, stub_session_factory
):
feed = (
b"<?xml version='1.0' encoding='UTF-8'?>"
b'<rss version="2.0"><channel><title>X</title>'
b"<item><title>Story</title>"
b"<link>https://aggregator.test/story</link>"
b"<description>x</description>"
b"<pubDate>Wed, 20 May 2026 13:35:34 EDT</pubDate>"
b"</item></channel></rss>"
)
article = (
b"<html><body><article>"
b"<p>Opening sentence.</p>"
b'<img src="https://x.test/inline.jpg" alt="photo"/>'
b"<p>Closing sentence.</p>"
b"</article></body></html>"
)
monkeypatch.setattr(
"openbb_news.registry.get_feed_url", lambda key: "http://feed.test/x"
)
stub_session_factory(
{"http://feed.test/x": feed, "https://aggregator.test/story": article}
)
items = (await rss.rss(source="x", limit=1, fetch_body=True)).results
body = items[0].body
assert "![photo](https://x.test/inline.jpg)" in body
assert body.index("Opening sentence.") < body.index("![photo]")
assert body.index("![photo]") < body.index("Closing sentence.")
async def test_rss_body_prepends_og_image_when_no_inline(
monkeypatch, stub_session_factory
):
feed = (
b"<?xml version='1.0' encoding='UTF-8'?>"
b'<rss version="2.0"><channel><title>X</title>'
b"<item><title>Story</title>"
b"<link>https://aggregator.test/story</link>"
b"<description>x</description>"
b"<pubDate>Wed, 20 May 2026 13:35:34 EDT</pubDate>"
b"</item></channel></rss>"
)
article = (
b"<html><head>"
b'<meta property="og:image" content="https://x.test/hero.jpg">'
b"</head><body><article>"
b"<p>One.</p><p>Two.</p>"
b"</article></body></html>"
)
monkeypatch.setattr(
"openbb_news.registry.get_feed_url", lambda key: "http://feed.test/x"
)
stub_session_factory(
{"http://feed.test/x": feed, "https://aggregator.test/story": article}
)
items = (await rss.rss(source="x", limit=1, fetch_body=True)).results
assert items[0].body.startswith("![](https://x.test/hero.jpg)")
async def test_rss_body_preserves_cbc_style_description_image(
monkeypatch, stub_session_factory
):
feed = (
b"<?xml version='1.0' encoding='UTF-8'?>"
b'<rss version="2.0"><channel><title>CBC</title>'
b"<item><title>Story</title>"
b"<link>https://aggregator.test/story</link>"
b"<description><![CDATA["
b"<img src='https://i.cbc.ca/hero.jpg' alt='photo'/>"
b"<p>Lead text.</p>"
b"]]></description>"
b"<pubDate>Wed, 20 May 2026 13:35:34 EDT</pubDate>"
b"</item></channel></rss>"
)
monkeypatch.setattr(
"openbb_news.registry.get_feed_url", lambda key: "http://feed.test/x"
)
stub_session_factory(
{
"http://feed.test/x": feed,
"https://aggregator.test/story": b"<html><body><div>not an article</div></body></html>",
}
)
items = (await rss.rss(source="x", limit=1, fetch_body=True)).results
assert "![photo](https://i.cbc.ca/hero.jpg)" in items[0].body
async def test_gather_bodies_returns_none_for_empty_link(stub_session_factory):
session = stub_session_factory({})
out = await rss._gather_bodies(session, [None, ""])
assert out == [None, None]
assert session.calls == []
async def test_gather_bodies_swallows_per_item_exceptions(
monkeypatch, stub_session_factory, sample_article_html
):
import aiohttp
session = stub_session_factory(
{
"https://good.test/a": sample_article_html,
"https://bad.test/a": aiohttp.ClientError("synthetic failure"),
}
)
out = await rss._gather_bodies(
session, ["https://good.test/a", "https://bad.test/a"]
)
assert out[0] is not None
assert "First paragraph" in out[0]
assert out[1] is None
async def test_gather_bodies_propagates_cancellation(monkeypatch):
import asyncio
async def cancelling_fetch(session, url):
raise asyncio.CancelledError
monkeypatch.setattr("openbb_news.parser.fetch_article_body", cancelling_fetch)
with pytest.raises(asyncio.CancelledError):
await rss._gather_bodies(object(), ["https://x.test/a"])
async def test_gather_bodies_swallows_unexpected_exceptions(monkeypatch):
async def broken_fetch(session, url):
raise RuntimeError("internal bug")
monkeypatch.setattr("openbb_news.parser.fetch_article_body", broken_fetch)
out = await rss._gather_bodies(object(), ["https://x.test/a", "https://x.test/b"])
assert out == [None, None]

3556
openbb_platform/extensions/news/uv.lock generated vendored Normal file

File diff suppressed because it is too large Load Diff