mirror of
https://github.com/OpenBB-finance/OpenBB.git
synced 2026-10-27 04:31:13 +00:00
[V5] openbb-news: Refactor News for V5 and Add Configurable RSS Endpoint (#7507)
* refactor news for v5 and add rss endpoint * remove poetry.lock --------- Co-authored-by: deeleeramone <>
This commit is contained in:
163
.github/workflows/publish-openbb-news.yml
vendored
Normal file
163
.github/workflows/publish-openbb-news.yml
vendored
Normal file
@@ -0,0 +1,163 @@
|
||||
name: 📦 publish openbb-news
|
||||
|
||||
# Triggers exclusively on the merge of a ``release/openbb-news-v*``
|
||||
# branch into ``develop`` (or ``v5``). Four sequential jobs — verify the
|
||||
# version bump, build once, test the built source across the full
|
||||
# cross-platform matrix, and only publish to PyPI if the matrix is green.
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [closed]
|
||||
branches: [develop, v5]
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
version-check:
|
||||
name: Verify version bump
|
||||
if: ${{ (github.event_name == 'pull_request' && github.event.pull_request.merged == true && startsWith(github.event.pull_request.head.ref, 'release/openbb-news-v')) || github.event_name == 'workflow_dispatch' }}
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
version: ${{ steps.parse.outputs.version }}
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
working-directory: openbb_platform/extensions/news
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
ref: develop
|
||||
|
||||
- name: Parse [project].version from pyproject.toml
|
||||
id: parse
|
||||
run: |
|
||||
VERSION=$(grep -m1 -E '^version[[:space:]]*=' pyproject.toml | sed -E 's/^version[[:space:]]*=[[:space:]]*"([^"]+)".*/\1/')
|
||||
if [ -z "$VERSION" ]; then
|
||||
echo "::error::Could not parse [project].version from pyproject.toml."
|
||||
exit 1
|
||||
fi
|
||||
echo "Local version: $VERSION"
|
||||
echo "version=$VERSION" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Fail if version already exists on PyPI
|
||||
env:
|
||||
VERSION: ${{ steps.parse.outputs.version }}
|
||||
run: |
|
||||
STATUS=$(curl -sS -o /dev/null -w "%{http_code}" "https://pypi.org/pypi/openbb-news/${VERSION}/json")
|
||||
case "$STATUS" in
|
||||
200)
|
||||
echo "::error::openbb-news ${VERSION} is already published on PyPI."
|
||||
echo "Bump openbb_platform/extensions/news/pyproject.toml [project].version and re-merge a fresh release branch."
|
||||
exit 1
|
||||
;;
|
||||
404)
|
||||
echo "openbb-news ${VERSION} is not yet on PyPI — proceeding."
|
||||
;;
|
||||
*)
|
||||
echo "::error::Unexpected status $STATUS from PyPI JSON API. Aborting to avoid a silent skip."
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
- name: Verify release branch name matches pyproject version
|
||||
if: github.event_name == 'pull_request'
|
||||
env:
|
||||
VERSION: ${{ steps.parse.outputs.version }}
|
||||
BRANCH: ${{ github.event.pull_request.head.ref }}
|
||||
run: |
|
||||
EXPECTED="release/openbb-news-v${VERSION}"
|
||||
if [ "$BRANCH" != "$EXPECTED" ]; then
|
||||
echo "::error::Release branch '${BRANCH}' does not match pyproject version '${VERSION}'."
|
||||
echo "Expected branch name: '${EXPECTED}'."
|
||||
echo "Either rename the release branch or update [project].version in pyproject.toml so they agree."
|
||||
exit 1
|
||||
fi
|
||||
echo "Branch '${BRANCH}' matches pyproject version '${VERSION}'."
|
||||
|
||||
build:
|
||||
name: Build sdist + wheel
|
||||
needs: version-check
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
working-directory: openbb_platform/extensions/news
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
ref: develop
|
||||
|
||||
- uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: "3.11"
|
||||
|
||||
- run: python -m pip install --upgrade pip uv
|
||||
|
||||
- name: Build sdist + wheel
|
||||
run: uv build --no-sources
|
||||
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: openbb-news-dist
|
||||
path: openbb_platform/extensions/news/dist/
|
||||
if-no-files-found: error
|
||||
retention-days: 7
|
||||
|
||||
test:
|
||||
name: Test (${{ matrix.os }} / Python ${{ matrix.python_version }})
|
||||
needs: build
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- { os: ubuntu-latest, python_version: "3.10" }
|
||||
- { os: ubuntu-latest, python_version: "3.11" }
|
||||
- { os: ubuntu-latest, python_version: "3.12" }
|
||||
- { os: ubuntu-latest, python_version: "3.13" }
|
||||
- { os: ubuntu-latest, python_version: "3.14" }
|
||||
- { os: macos-latest, python_version: "3.10" }
|
||||
- { os: macos-latest, python_version: "3.14" }
|
||||
- { os: windows-latest, python_version: "3.10" }
|
||||
- { os: windows-latest, python_version: "3.14" }
|
||||
runs-on: ${{ matrix.os }}
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
working-directory: openbb_platform/extensions/news
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
ref: develop
|
||||
|
||||
- uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: ${{ matrix.python_version }}
|
||||
allow-prereleases: true
|
||||
|
||||
- run: python -m pip install --upgrade pip uv
|
||||
|
||||
- name: Install openbb-news from source (+ dev group)
|
||||
run: |
|
||||
uv pip install --system -e .
|
||||
uv pip install --system --group dev
|
||||
|
||||
- name: Run unit tests
|
||||
run: pytest tests --cov=openbb_news --cov-report=term-missing
|
||||
|
||||
publish:
|
||||
name: Publish to PyPI
|
||||
needs: test
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
id-token: write # required for PyPI OIDC trusted publishing
|
||||
steps:
|
||||
- uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: openbb-news-dist
|
||||
path: dist
|
||||
|
||||
- uses: pypa/gh-action-pypi-publish@release/v1
|
||||
with:
|
||||
packages-dir: dist
|
||||
122
.github/workflows/test-unit-openbb-news.yml
vendored
Normal file
122
.github/workflows/test-unit-openbb-news.yml
vendored
Normal file
@@ -0,0 +1,122 @@
|
||||
name: 🧪 test openbb-news
|
||||
|
||||
# PR-test for the News extension. Triggers when a PR touches any file
|
||||
# under ``openbb_platform/extensions/news/`` or this workflow itself.
|
||||
#
|
||||
# Build-once, test-many: a single ``build`` job produces the wheel + sdist;
|
||||
# the ``lint`` and ``test`` jobs install the extension editable from source
|
||||
# (``[tool.uv.sources]`` pins openbb-core to the sibling tree, since
|
||||
# openbb-core>=2.0.0 is not yet on PyPI) and run across every supported
|
||||
# (OS, Python) combination. Unit tests are pure compute — no network.
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [develop, main, v5, feature/v5-platform-api]
|
||||
paths:
|
||||
- "openbb_platform/extensions/news/**"
|
||||
- ".github/workflows/test-unit-openbb-news.yml"
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: test-openbb-news-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: Build sdist + wheel
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
working-directory: openbb_platform/extensions/news
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: "3.11"
|
||||
|
||||
- run: python -m pip install --upgrade pip uv
|
||||
|
||||
- name: Build sdist + wheel
|
||||
run: uv build --no-sources
|
||||
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: openbb-news-dist
|
||||
path: openbb_platform/extensions/news/dist/
|
||||
if-no-files-found: error
|
||||
retention-days: 7
|
||||
|
||||
lint:
|
||||
name: Lint (ruff + ty)
|
||||
needs: build
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
working-directory: openbb_platform/extensions/news
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: "3.11"
|
||||
|
||||
- run: python -m pip install --upgrade pip uv
|
||||
|
||||
- name: Install openbb-news in editable mode (+ dev group)
|
||||
run: |
|
||||
uv pip install --system -e .
|
||||
uv pip install --system --group dev
|
||||
|
||||
- name: ruff check
|
||||
run: ruff check openbb_news tests integration
|
||||
|
||||
- name: ruff format --check
|
||||
run: ruff format --check openbb_news tests integration
|
||||
|
||||
- name: ty check
|
||||
run: ty check openbb_news
|
||||
|
||||
test:
|
||||
name: Test (${{ matrix.os }} / Python ${{ matrix.python_version }})
|
||||
needs: build
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- { os: ubuntu-latest, python_version: "3.10" }
|
||||
- { os: ubuntu-latest, python_version: "3.11" }
|
||||
- { os: ubuntu-latest, python_version: "3.12" }
|
||||
- { os: ubuntu-latest, python_version: "3.13" }
|
||||
- { os: ubuntu-latest, python_version: "3.14" }
|
||||
- { os: macos-latest, python_version: "3.10" }
|
||||
- { os: macos-latest, python_version: "3.14" }
|
||||
- { os: windows-latest, python_version: "3.10" }
|
||||
- { os: windows-latest, python_version: "3.14" }
|
||||
runs-on: ${{ matrix.os }}
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
working-directory: openbb_platform/extensions/news
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: ${{ matrix.python_version }}
|
||||
allow-prereleases: true
|
||||
|
||||
- run: python -m pip install --upgrade pip uv
|
||||
|
||||
- name: Install openbb-news from source (+ dev group)
|
||||
run: |
|
||||
uv pip install --system -e .
|
||||
uv pip install --system --group dev
|
||||
|
||||
- name: Run unit tests
|
||||
run: pytest tests --cov=openbb_news --cov-report=term-missing --cov-fail-under=100
|
||||
120
openbb_platform/extensions/news/README.md
vendored
120
openbb_platform/extensions/news/README.md
vendored
@@ -1,13 +1,119 @@
|
||||
# OpenBB News Extension
|
||||
# openbb-news
|
||||
|
||||
This extension provides news for the OpenBB Platform.
|
||||
RSS news feeds for the OpenBB Platform — one Workspace Newsfeed widget backed by
|
||||
a registry of 400+ vetted feeds (Benzinga, GlobeNewswire by full ICB industry
|
||||
breakdown + subject/event-type breakdown — earnings, M&A, dividends, IPOs, etc.,
|
||||
PR Newswire regional + category, Axios, Yahoo Finance, Fortune, BBC, Fox News,
|
||||
Drudge Report, Google News regional + topic). The widget defaults to
|
||||
**Benzinga → Markets** so it loads finance-relevant articles immediately on
|
||||
first open.
|
||||
|
||||
## Installation
|
||||
|
||||
To install the extension, run the following command in this folder:
|
||||
## Install
|
||||
|
||||
```bash
|
||||
pip install openbb-news
|
||||
uv pip install -e .
|
||||
```
|
||||
|
||||
Documentation available [here](https://docs.openbb.co/platform/developer_guide/contributing).
|
||||
## Endpoints
|
||||
|
||||
| GET path | Returns |
|
||||
| --- | --- |
|
||||
| `/api/v1/news/rss?outlet=<id>&source=<key>&limit=<n>&fetch_body=<bool>` | `OBBject[list[NewsItemData]]` — `{title, date, author, url, excerpt, body}` per article. Empty when `source` is unset and no default can be resolved. |
|
||||
| `/api/v1/news/rss_providers` | `list[{label, value}]` — drives the widget's **Provider** dropdown. |
|
||||
| `/api/v1/news/rss_feeds?outlet=<id>` | `list[{label, value}]` — drives the widget's **Feed** dropdown, filtered by the selected provider. |
|
||||
|
||||
`fetch_body=true` extracts the full article body from each entry's URL via
|
||||
JSON-LD `articleBody` first, then the largest `<article>` element after
|
||||
stripping site chrome (`nav`/`header`/`footer`/`aside`/`script`/`style`/`form`/
|
||||
`iframe`/`button`/`svg`). Requires at least two real `<p>` paragraphs and
|
||||
rejects pages whose body contains unrendered template markers (`[[`, `{{`);
|
||||
falls back to the RSS summary on any miss. Bounded `lru_cache(maxsize=256)`
|
||||
keeps repeat fetches in-process; no disk persistence.
|
||||
|
||||
## Pluggable feeds — `openbb.toml`
|
||||
|
||||
Three levers in `~/.openbb_platform/openbb.toml`, all under `[news]`:
|
||||
|
||||
### 1. Add or replace feeds — `[news.rss_feeds]`
|
||||
|
||||
**Simple form** — `key = "url-string"`. The feed appears under a single
|
||||
"Custom" provider; the label is a humanized form of the key.
|
||||
|
||||
```toml
|
||||
[news.rss_feeds]
|
||||
my_internal = "https://internal.example.com/news.rss"
|
||||
acme_blog = "https://acme.example.com/feed"
|
||||
```
|
||||
|
||||
**Rich form** — `key = { url, provider, label }` (or the equivalent table
|
||||
syntax). Lets you bucket feeds into your own provider and give them a
|
||||
display label, so the widget's Provider → Feed cascade groups them.
|
||||
|
||||
```toml
|
||||
[news.rss_feeds.alpha_research]
|
||||
url = "https://alpha.example.com/rss"
|
||||
provider = "acme_research"
|
||||
label = "Alpha Research"
|
||||
|
||||
[news.rss_feeds.beta_research]
|
||||
url = "https://acme.example.com/beta/rss"
|
||||
provider = "acme_research"
|
||||
label = "Beta Research"
|
||||
```
|
||||
|
||||
### 2. Pretty provider names — `[news.rss_providers]`
|
||||
|
||||
By default a provider id is humanized (`acme_research` → "Acme Research"). To
|
||||
control the display label exactly, map provider id → label:
|
||||
|
||||
```toml
|
||||
[news.rss_providers]
|
||||
acme_research = "Acme Research"
|
||||
internal = "Internal Sources"
|
||||
```
|
||||
|
||||
### 3. Keep the bundled defaults — `[news] merge_defaults`
|
||||
|
||||
Without this flag, the presence of any `[news.rss_feeds]` entries replaces
|
||||
the 438 bundled feeds entirely. Set `merge_defaults = true` to keep
|
||||
everything and add your feeds on top. User keys that collide with bundled
|
||||
keys override the bundled URL.
|
||||
|
||||
```toml
|
||||
[news]
|
||||
merge_defaults = true
|
||||
|
||||
[news.rss_feeds]
|
||||
my_internal = "https://internal.example.com/news.rss"
|
||||
```
|
||||
|
||||
### Full example
|
||||
|
||||
```toml
|
||||
[news]
|
||||
merge_defaults = true
|
||||
|
||||
[news.rss_providers]
|
||||
acme_research = "Acme Research"
|
||||
|
||||
[news.rss_feeds.alpha]
|
||||
url = "https://alpha.example.com/rss"
|
||||
provider = "acme_research"
|
||||
label = "Alpha Research"
|
||||
|
||||
[news.rss_feeds.beta]
|
||||
url = "https://beta.example.com/rss"
|
||||
provider = "acme_research"
|
||||
label = "Beta Research"
|
||||
```
|
||||
|
||||
The Workspace Newsfeed widget rebuilds its **Provider** + **Feed** dropdowns
|
||||
from this config on each request — restart the server only if the file was
|
||||
empty at boot.
|
||||
|
||||
## Tests
|
||||
|
||||
```bash
|
||||
pytest tests --cov=openbb_news --cov-fail-under=100
|
||||
pytest integration -m integration # hits the live feeds
|
||||
```
|
||||
|
||||
34
openbb_platform/extensions/news/integration/conftest.py
Normal file
34
openbb_platform/extensions/news/integration/conftest.py
Normal file
@@ -0,0 +1,34 @@
|
||||
"""Integration fixtures."""
|
||||
|
||||
import threading
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture(scope="session", autouse=True)
|
||||
def api_server(pytestconfig):
|
||||
if pytestconfig.getoption("markexpr") == "not integration":
|
||||
yield None
|
||||
return
|
||||
|
||||
import uvicorn
|
||||
from openbb_core.api.rest_api import app
|
||||
|
||||
config = uvicorn.Config(app, host="0.0.0.0", port=8000, log_level="warning") # noqa: S104
|
||||
server = uvicorn.Server(config)
|
||||
thread = threading.Thread(target=server.run, daemon=True)
|
||||
thread.start()
|
||||
|
||||
deadline = time.time() + 60
|
||||
while time.time() < deadline:
|
||||
if server.started:
|
||||
break
|
||||
time.sleep(0.25)
|
||||
else: # pragma: no cover
|
||||
raise RuntimeError("The OpenBB REST API did not start within 60s.")
|
||||
|
||||
yield server
|
||||
|
||||
server.should_exit = True
|
||||
thread.join(timeout=15)
|
||||
@@ -1,209 +1,88 @@
|
||||
"""Test News API."""
|
||||
"""HTTP API integration tests."""
|
||||
|
||||
import base64
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
from openbb_core.env import Env
|
||||
from openbb_core.provider.utils.helpers import get_querystring
|
||||
|
||||
_BASE_URL = "http://127.0.0.1:8000/api/v1/news"
|
||||
_headers: dict = {}
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def headers():
|
||||
"""Generate headers for API requests with basic authentication."""
|
||||
def get_headers() -> dict:
|
||||
if _headers:
|
||||
return _headers
|
||||
userpass = f"{Env().API_USERNAME}:{Env().API_PASSWORD}"
|
||||
userpass_bytes = userpass.encode("ascii")
|
||||
base64_bytes = base64.b64encode(userpass_bytes)
|
||||
|
||||
return {"Authorization": f"Basic {base64_bytes.decode('ascii')}"}
|
||||
base64_bytes = base64.b64encode(userpass.encode("ascii"))
|
||||
_headers["Authorization"] = f"Basic {base64_bytes.decode('ascii')}"
|
||||
return _headers
|
||||
|
||||
|
||||
# pylint: disable=redefined-outer-name
|
||||
def _get(path: str, params: dict | None = None) -> requests.Response:
|
||||
return requests.get(
|
||||
f"{_BASE_URL}/{path}", headers=get_headers(), params=params, timeout=60
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"params",
|
||||
[
|
||||
(
|
||||
{
|
||||
"display": "full",
|
||||
"date": None,
|
||||
"start_date": None,
|
||||
"end_date": None,
|
||||
"updated_since": None,
|
||||
"published_since": None,
|
||||
"sort": "created",
|
||||
"order": "desc",
|
||||
"isin": None,
|
||||
"cusip": None,
|
||||
"channels": "General",
|
||||
"topics": "earnings",
|
||||
"authors": None,
|
||||
"content_types": "headline",
|
||||
"provider": "benzinga",
|
||||
"limit": 20,
|
||||
}
|
||||
),
|
||||
(
|
||||
{
|
||||
"provider": "fmp",
|
||||
"limit": 30,
|
||||
"start_date": None,
|
||||
"end_date": None,
|
||||
"topic": "general",
|
||||
"page": 1,
|
||||
}
|
||||
),
|
||||
(
|
||||
{
|
||||
"provider": "intrinio",
|
||||
"limit": 20,
|
||||
"start_date": None,
|
||||
"end_date": None,
|
||||
"source": "yahoo",
|
||||
"topic": None,
|
||||
"is_spam": False,
|
||||
"sentiment": None,
|
||||
"language": None,
|
||||
"word_count_greater_than": None,
|
||||
"word_count_less_than": None,
|
||||
"business_relevance_greater_than": None,
|
||||
"business_relevance_less_than": None,
|
||||
}
|
||||
),
|
||||
(
|
||||
{
|
||||
"provider": "biztoc",
|
||||
"source": None,
|
||||
"term": "microsoft",
|
||||
"start_date": None,
|
||||
"end_date": None,
|
||||
}
|
||||
),
|
||||
(
|
||||
{
|
||||
"provider": "tiingo",
|
||||
"limit": 30,
|
||||
"source": "bloomberg.com",
|
||||
"start_date": None,
|
||||
"end_date": None,
|
||||
"offset": 0,
|
||||
}
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.integration
|
||||
def test_news_world(params, headers):
|
||||
"""Test retrieval of world news with various parameters."""
|
||||
params = {p: v for p, v in params.items() if v}
|
||||
|
||||
query_str = get_querystring(params, [])
|
||||
url = f"http://0.0.0.0:8000/api/v1/news/world?{query_str}"
|
||||
result = requests.get(url, headers=headers, timeout=10)
|
||||
assert isinstance(result, requests.Response)
|
||||
def test_rss_providers_lists_outlets():
|
||||
result = _get("rss_providers")
|
||||
assert result.status_code == 200
|
||||
payload = result.json()
|
||||
assert isinstance(payload, list)
|
||||
values = {entry["value"] for entry in payload}
|
||||
assert "bbc" in values
|
||||
assert "globenewswire" in values
|
||||
assert "pr_newswire" in values
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"params",
|
||||
[
|
||||
(
|
||||
{
|
||||
"symbol": "AAPL",
|
||||
"limit": 20,
|
||||
"provider": "benzinga",
|
||||
"date": "2023-01-01",
|
||||
"start_date": "2023-01-01",
|
||||
"end_date": "2023-06-06",
|
||||
}
|
||||
),
|
||||
(
|
||||
{
|
||||
"display": "full",
|
||||
"date": None,
|
||||
"start_date": "2023-06-01",
|
||||
"end_date": "2023-06-06",
|
||||
"updated_since": 1,
|
||||
"published_since": 1,
|
||||
"sort": "created",
|
||||
"order": "asc",
|
||||
"isin": None,
|
||||
"cusip": None,
|
||||
"channels": None,
|
||||
"topics": "AAPL",
|
||||
"authors": None,
|
||||
"content_types": "headline",
|
||||
"provider": "benzinga",
|
||||
"symbol": "AAPL,MSFT",
|
||||
"limit": 20,
|
||||
}
|
||||
),
|
||||
(
|
||||
{
|
||||
"provider": "fmp",
|
||||
"symbol": "AAPL",
|
||||
"limit": 20,
|
||||
"page": 1,
|
||||
"start_date": None,
|
||||
"end_date": None,
|
||||
"press_release": False,
|
||||
}
|
||||
),
|
||||
(
|
||||
{
|
||||
"provider": "yfinance",
|
||||
"symbol": "AAPL",
|
||||
"limit": 20,
|
||||
"start_date": None,
|
||||
"end_date": None,
|
||||
}
|
||||
),
|
||||
(
|
||||
{
|
||||
"provider": "intrinio",
|
||||
"symbol": "AAPL",
|
||||
"limit": 20,
|
||||
"start_date": "2024-01-02",
|
||||
"end_date": "2024-01-03",
|
||||
"source": "yahoo",
|
||||
"topic": None,
|
||||
"is_spam": False,
|
||||
"sentiment": None,
|
||||
"language": None,
|
||||
"word_count_greater_than": None,
|
||||
"word_count_less_than": None,
|
||||
"business_relevance_greater_than": None,
|
||||
"business_relevance_less_than": None,
|
||||
}
|
||||
),
|
||||
(
|
||||
{
|
||||
"provider": "tiingo",
|
||||
"symbol": "AAPL,MSFT",
|
||||
"limit": 20,
|
||||
"source": "bloomberg.com",
|
||||
"start_date": None,
|
||||
"end_date": None,
|
||||
"offset": None,
|
||||
}
|
||||
),
|
||||
(
|
||||
{
|
||||
"provider": "tmx",
|
||||
"symbol": "RBC",
|
||||
"limit": 20,
|
||||
"page": 1,
|
||||
}
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.integration
|
||||
def test_news_company(params, headers):
|
||||
"""Test retrieval of company-specific news with various parameters."""
|
||||
params = {p: v for p, v in params.items() if v}
|
||||
|
||||
query_str = get_querystring(params, [])
|
||||
url = f"http://0.0.0.0:8000/api/v1/news/company?{query_str}"
|
||||
result = requests.get(url, headers=headers, timeout=10)
|
||||
assert isinstance(result, requests.Response)
|
||||
def test_rss_feeds_filtered_by_outlet():
|
||||
result = _get("rss_feeds", params={"outlet": "bbc"})
|
||||
assert result.status_code == 200
|
||||
payload = result.json()
|
||||
assert isinstance(payload, list)
|
||||
values = {entry["value"] for entry in payload}
|
||||
assert "bbc_world" in values
|
||||
assert "bbc_business" in values
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
def test_rss_fetches_articles_without_body():
|
||||
result = _get(
|
||||
"rss",
|
||||
params={"source": "bbc_world", "limit": 3, "fetch_body": "false"},
|
||||
)
|
||||
assert result.status_code == 200
|
||||
body = result.json()
|
||||
results = body["results"]
|
||||
assert 1 <= len(results) <= 3
|
||||
first = results[0]
|
||||
assert first["title"]
|
||||
assert first["date"]
|
||||
assert first["author"]
|
||||
assert "url" in first
|
||||
assert "excerpt" in first
|
||||
assert "body" in first
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
def test_rss_returns_empty_when_no_source_and_no_outlet():
|
||||
result = _get("rss")
|
||||
assert result.status_code == 200
|
||||
assert result.json()["results"] == []
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
def test_rss_auto_resolves_default_when_only_outlet_given():
|
||||
result = _get("rss", params={"outlet": "bbc", "limit": 1, "fetch_body": "false"})
|
||||
assert result.status_code == 200
|
||||
body = result.json()
|
||||
assert len(body["results"]) == 1
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
def test_rss_unknown_source_returns_error():
|
||||
result = _get("rss", params={"source": "not_a_feed", "limit": 1})
|
||||
assert result.status_code >= 400
|
||||
|
||||
@@ -1,190 +1,27 @@
|
||||
"""Test news extension."""
|
||||
"""Python interface integration tests."""
|
||||
|
||||
import pytest
|
||||
from openbb_core.app.model.obbject import OBBject
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def obb(pytestconfig): # pylint: disable=inconsistent-return-statements
|
||||
"""Fixture to setup obb."""
|
||||
|
||||
def obb(pytestconfig):
|
||||
if pytestconfig.getoption("markexpr") != "not integration":
|
||||
import openbb # pylint: disable=import-outside-toplevel
|
||||
import openbb
|
||||
|
||||
return openbb.obb
|
||||
return None
|
||||
|
||||
|
||||
# pylint: disable=redefined-outer-name
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"params",
|
||||
[
|
||||
(
|
||||
{
|
||||
"display": "full",
|
||||
"date": None,
|
||||
"start_date": "2023-05-01",
|
||||
"end_date": "2023-05-31",
|
||||
"updated_since": None,
|
||||
"published_since": None,
|
||||
"sort": "created",
|
||||
"order": "asc",
|
||||
"isin": None,
|
||||
"cusip": None,
|
||||
"channels": "General",
|
||||
"topics": "car",
|
||||
"authors": None,
|
||||
"content_types": "Car",
|
||||
"provider": "benzinga",
|
||||
"limit": 20,
|
||||
}
|
||||
),
|
||||
(
|
||||
{
|
||||
"provider": "fmp",
|
||||
"limit": 20,
|
||||
"start_date": None,
|
||||
"end_date": None,
|
||||
"page": 0,
|
||||
"topic": "general",
|
||||
}
|
||||
),
|
||||
(
|
||||
{
|
||||
"provider": "intrinio",
|
||||
"limit": 20,
|
||||
"start_date": "2024-01-02",
|
||||
"end_date": "2024-01-03",
|
||||
"source": "yahoo",
|
||||
"topic": None,
|
||||
"is_spam": False,
|
||||
"sentiment": None,
|
||||
"language": None,
|
||||
"word_count_greater_than": None,
|
||||
"word_count_less_than": None,
|
||||
"business_relevance_greater_than": None,
|
||||
"business_relevance_less_than": None,
|
||||
}
|
||||
),
|
||||
(
|
||||
{
|
||||
"provider": "biztoc",
|
||||
"source": None,
|
||||
"term": "microsoft",
|
||||
"start_date": None,
|
||||
"end_date": None,
|
||||
}
|
||||
),
|
||||
(
|
||||
{
|
||||
"provider": "tiingo",
|
||||
"limit": 30,
|
||||
"source": "bloomberg.com",
|
||||
"start_date": None,
|
||||
"end_date": None,
|
||||
"offset": 0,
|
||||
}
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.integration
|
||||
def test_news_world(params, obb):
|
||||
"""Test the news world endpoint."""
|
||||
result = obb.news.world(**params)
|
||||
assert result
|
||||
assert isinstance(result, OBBject)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"params",
|
||||
[
|
||||
(
|
||||
{
|
||||
"display": "full",
|
||||
"date": None,
|
||||
"start_date": None,
|
||||
"end_date": None,
|
||||
"updated_since": None,
|
||||
"published_since": None,
|
||||
"sort": "created",
|
||||
"order": "desc",
|
||||
"isin": None,
|
||||
"cusip": None,
|
||||
"channels": "General",
|
||||
"topics": "earnings",
|
||||
"authors": None,
|
||||
"content_types": "headline",
|
||||
"provider": "benzinga",
|
||||
"symbol": "AAPL,MSFT",
|
||||
"limit": 20,
|
||||
}
|
||||
),
|
||||
(
|
||||
{
|
||||
"provider": "fmp",
|
||||
"symbol": "AAPL",
|
||||
"limit": 20,
|
||||
"page": 1,
|
||||
"start_date": None,
|
||||
"end_date": None,
|
||||
"press_release": False,
|
||||
}
|
||||
),
|
||||
(
|
||||
{
|
||||
"provider": "yfinance",
|
||||
"symbol": "AAPL",
|
||||
"limit": 20,
|
||||
"start_date": None,
|
||||
"end_date": None,
|
||||
}
|
||||
),
|
||||
(
|
||||
{
|
||||
"provider": "intrinio",
|
||||
"symbol": "AAPL",
|
||||
"limit": 20,
|
||||
"start_date": "2024-01-02",
|
||||
"end_date": "2024-01-03",
|
||||
"source": "yahoo",
|
||||
"topic": None,
|
||||
"is_spam": False,
|
||||
"sentiment": None,
|
||||
"language": None,
|
||||
"word_count_greater_than": None,
|
||||
"word_count_less_than": None,
|
||||
"business_relevance_greater_than": None,
|
||||
"business_relevance_less_than": None,
|
||||
}
|
||||
),
|
||||
(
|
||||
{
|
||||
"provider": "tiingo",
|
||||
"symbol": "AAPL",
|
||||
"limit": 20,
|
||||
"source": "bloomberg.com",
|
||||
"start_date": None,
|
||||
"end_date": None,
|
||||
"offset": None,
|
||||
}
|
||||
),
|
||||
(
|
||||
{
|
||||
"provider": "tmx",
|
||||
"symbol": "RBC",
|
||||
"limit": 20,
|
||||
"page": 1,
|
||||
}
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.integration
|
||||
def test_news_company(params, obb):
|
||||
"""Test the news company endpoint."""
|
||||
params = {p: v for p, v in params.items() if v}
|
||||
|
||||
result = obb.news.company(**params)
|
||||
assert result
|
||||
assert isinstance(result, OBBject)
|
||||
assert len(result.results) > 0
|
||||
def test_rss_python_fetches_articles(obb):
|
||||
out = obb.news.rss(source="bbc_world", limit=3, fetch_body=False)
|
||||
assert isinstance(out, OBBject)
|
||||
assert 1 <= len(out.results) <= 3
|
||||
first = out.results[0]
|
||||
assert first.title
|
||||
assert first.date
|
||||
assert first.author
|
||||
assert hasattr(first, "url")
|
||||
assert hasattr(first, "excerpt")
|
||||
assert hasattr(first, "body")
|
||||
|
||||
290
openbb_platform/extensions/news/openbb_news/parser.py
Normal file
290
openbb_platform/extensions/news/openbb_news/parser.py
Normal file
@@ -0,0 +1,290 @@
|
||||
"""RSS/Atom feed and article body helpers."""
|
||||
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from html.parser import HTMLParser
|
||||
from time import struct_time
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from aiohttp import ClientSession
|
||||
|
||||
_CHROME_TAGS: tuple[str, ...] = (
|
||||
"script",
|
||||
"style",
|
||||
"noscript",
|
||||
"iframe",
|
||||
"nav",
|
||||
"header",
|
||||
"footer",
|
||||
"aside",
|
||||
"form",
|
||||
"button",
|
||||
"svg",
|
||||
)
|
||||
_TEMPLATE_MARKERS: tuple[str, ...] = ("[[", "{{")
|
||||
_MIN_PARAGRAPHS = 2
|
||||
_FEED_TIMEOUT_SEC = 8.0
|
||||
_ARTICLE_TIMEOUT_SEC = 6.0
|
||||
_TIMESTAMP_RE = re.compile(r"^\d{1,2}:\d{2}(?::\d{2})?$")
|
||||
|
||||
|
||||
class _TagStripper(HTMLParser):
|
||||
"""Collect text data while discarding tags."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.parts: list[str] = []
|
||||
|
||||
def handle_data(self, data: str) -> None:
|
||||
self.parts.append(data)
|
||||
|
||||
|
||||
def strip_html(text: str) -> str:
|
||||
"""Strip HTML tags and collapse whitespace."""
|
||||
if not text:
|
||||
return ""
|
||||
stripper = _TagStripper()
|
||||
stripper.feed(text)
|
||||
return " ".join("".join(stripper.parts).split())
|
||||
|
||||
|
||||
def html_to_markdown(fragment: str) -> str:
|
||||
"""Render an HTML fragment as markdown, preserving anchors and images."""
|
||||
if not fragment:
|
||||
return ""
|
||||
|
||||
from lxml import html as _html
|
||||
|
||||
try:
|
||||
root = _html.fragment_fromstring(fragment, create_parent="div")
|
||||
except Exception: # noqa: BLE001
|
||||
return strip_html(fragment)
|
||||
|
||||
parts: list[str] = []
|
||||
|
||||
def emit(el) -> None:
|
||||
tag = el.tag.lower() if isinstance(el.tag, str) else ""
|
||||
if tag == "a":
|
||||
href = (el.get("href") or "").strip()
|
||||
inner = " ".join(el.text_content().split())
|
||||
if inner and href:
|
||||
parts.append(f"[{inner}]({href})")
|
||||
elif inner:
|
||||
parts.append(inner)
|
||||
parts.append(" ")
|
||||
if el.tail:
|
||||
parts.append(el.tail)
|
||||
return
|
||||
if tag == "img":
|
||||
src = (el.get("src") or "").strip()
|
||||
alt = (el.get("alt") or "").strip()
|
||||
if src:
|
||||
parts.append(f"")
|
||||
if el.tail:
|
||||
parts.append(el.tail)
|
||||
return
|
||||
if tag == "br":
|
||||
parts.append("\n")
|
||||
elif tag == "p" and parts and not parts[-1].endswith("\n"):
|
||||
parts.append("\n\n")
|
||||
if el.text:
|
||||
parts.append(el.text)
|
||||
for child in el:
|
||||
emit(child)
|
||||
if tag == "p":
|
||||
parts.append("\n\n")
|
||||
if el.tail:
|
||||
parts.append(el.tail)
|
||||
|
||||
emit(root)
|
||||
rendered = "".join(parts)
|
||||
lines = [" ".join(line.split()) for line in rendered.split("\n")]
|
||||
return "\n".join(lines).strip()
|
||||
|
||||
|
||||
def truncate(text: str, limit: int = 280) -> str:
|
||||
"""Truncate at the nearest word boundary within ``limit``."""
|
||||
if len(text) <= limit:
|
||||
return text
|
||||
cut = text[:limit].rsplit(" ", 1)[0]
|
||||
return cut.rstrip(",.;:") + "…"
|
||||
|
||||
|
||||
def struct_time_to_iso(t: "struct_time | None") -> str:
|
||||
"""Convert a feedparser ``struct_time`` to ISO 8601."""
|
||||
if t is None:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
return datetime(*t[:6], tzinfo=timezone.utc).isoformat()
|
||||
|
||||
|
||||
async def fetch_feed(session: "ClientSession", url: str):
|
||||
"""Fetch and parse an RSS/Atom feed."""
|
||||
import aiohttp
|
||||
import feedparser
|
||||
|
||||
try:
|
||||
response = await session.get(
|
||||
url, timeout=aiohttp.ClientTimeout(total=_FEED_TIMEOUT_SEC)
|
||||
)
|
||||
async with response:
|
||||
response.raise_for_status()
|
||||
payload = await response.read()
|
||||
except (aiohttp.ClientError, TimeoutError):
|
||||
return feedparser.parse(b"")
|
||||
return feedparser.parse(payload)
|
||||
|
||||
|
||||
def _strip_chrome(doc) -> None:
|
||||
"""Remove chrome tags in place."""
|
||||
selector = " | ".join(f"//{tag}" for tag in _CHROME_TAGS)
|
||||
for el in doc.xpath(selector):
|
||||
parent = el.getparent()
|
||||
if parent is not None:
|
||||
parent.remove(el)
|
||||
|
||||
|
||||
def _looks_clean(text: str) -> bool:
|
||||
"""Reject empty results and unrendered template fragments."""
|
||||
return bool(text) and not any(marker in text for marker in _TEMPLATE_MARKERS)
|
||||
|
||||
|
||||
def _jsonld_field(doc, field: str) -> str | None:
|
||||
"""Pull a top-level string field from any JSON-LD script."""
|
||||
import json
|
||||
|
||||
for script in doc.xpath('//script[@type="application/ld+json"]/text()'):
|
||||
try:
|
||||
data = json.loads(script)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
continue
|
||||
for entry in data if isinstance(data, list) else [data]:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
for candidate in (entry, *(entry.get("@graph") or [])):
|
||||
if not isinstance(candidate, dict):
|
||||
continue
|
||||
value = candidate.get(field)
|
||||
if isinstance(value, str) and value.strip():
|
||||
return value.strip()
|
||||
if isinstance(value, list) and value:
|
||||
first = value[0]
|
||||
if isinstance(first, str) and first.strip():
|
||||
return first.strip()
|
||||
if isinstance(first, dict):
|
||||
nested = first.get("url")
|
||||
if isinstance(nested, str) and nested.strip():
|
||||
return nested.strip()
|
||||
if isinstance(value, dict):
|
||||
nested = value.get("url")
|
||||
if isinstance(nested, str) and nested.strip():
|
||||
return nested.strip()
|
||||
return None
|
||||
|
||||
|
||||
def _article_markdown(node) -> str:
|
||||
"""Render an article container as markdown with text + inline images."""
|
||||
parts: list[str] = []
|
||||
text_count = [0]
|
||||
|
||||
def walk(el) -> None:
|
||||
tag = el.tag.lower() if isinstance(el.tag, str) else ""
|
||||
if tag == "p":
|
||||
text = " ".join(el.text_content().split())
|
||||
if text and not _TIMESTAMP_RE.match(text):
|
||||
parts.append(text)
|
||||
text_count[0] += 1
|
||||
return
|
||||
if tag == "figure":
|
||||
srcs = el.xpath(".//img/@src")
|
||||
if srcs:
|
||||
src = str(srcs[0]).strip()
|
||||
cap_nodes = el.xpath(".//figcaption//text()")
|
||||
cap = " ".join("".join(cap_nodes).split()) if cap_nodes else ""
|
||||
if src:
|
||||
parts.append(f"")
|
||||
return
|
||||
if tag == "img":
|
||||
src = (el.get("src") or "").strip()
|
||||
alt = (el.get("alt") or "").strip()
|
||||
if src:
|
||||
parts.append(f"")
|
||||
return
|
||||
for child in el:
|
||||
walk(child)
|
||||
|
||||
walk(node)
|
||||
if text_count[0] < _MIN_PARAGRAPHS:
|
||||
return ""
|
||||
return "\n\n".join(parts)
|
||||
|
||||
|
||||
def _extract_image_from_doc(doc) -> str | None:
|
||||
"""Find a hero image URL from meta tags, JSON-LD, or article body."""
|
||||
for path in (
|
||||
'//meta[@property="og:image"]/@content',
|
||||
'//meta[@name="og:image"]/@content',
|
||||
'//meta[@name="twitter:image"]/@content',
|
||||
'//meta[@property="twitter:image"]/@content',
|
||||
):
|
||||
for value in doc.xpath(path):
|
||||
url = str(value).strip()
|
||||
if url:
|
||||
return url
|
||||
jsonld = _jsonld_field(doc, "image")
|
||||
if jsonld:
|
||||
return jsonld
|
||||
for path in ("//article//img/@src", "//main//img/@src"):
|
||||
for value in doc.xpath(path):
|
||||
url = str(value).strip()
|
||||
if url:
|
||||
return url
|
||||
return None
|
||||
|
||||
|
||||
def _extract_body(payload: bytes) -> str | None:
|
||||
"""Return the article body as markdown with inline images."""
|
||||
from lxml import html
|
||||
|
||||
try:
|
||||
doc = html.fromstring(payload, parser=html.HTMLParser(encoding="utf-8"))
|
||||
except Exception: # noqa: BLE001
|
||||
return None
|
||||
|
||||
hero = _extract_image_from_doc(doc)
|
||||
|
||||
body = _jsonld_field(doc, "articleBody")
|
||||
if not (body and _looks_clean(body)):
|
||||
_strip_chrome(doc)
|
||||
best: str = ""
|
||||
for article in doc.xpath("//article"):
|
||||
text = _article_markdown(article)
|
||||
if _looks_clean(text) and len(text) > len(best):
|
||||
best = text
|
||||
if not best:
|
||||
for main in doc.xpath("//main"):
|
||||
text = _article_markdown(main)
|
||||
if _looks_clean(text) and len(text) > len(best):
|
||||
best = text
|
||||
body = best or None
|
||||
|
||||
if body and hero and "\n\n{body}"
|
||||
|
||||
return body
|
||||
|
||||
|
||||
async def fetch_article_body(session: "ClientSession", url: str) -> str | None:
|
||||
"""Fetch ``url`` and return its article body."""
|
||||
import aiohttp
|
||||
|
||||
try:
|
||||
response = await session.get(
|
||||
url, timeout=aiohttp.ClientTimeout(total=_ARTICLE_TIMEOUT_SEC)
|
||||
)
|
||||
async with response:
|
||||
response.raise_for_status()
|
||||
payload = await response.read()
|
||||
except (aiohttp.ClientError, TimeoutError):
|
||||
return None
|
||||
return _extract_body(payload)
|
||||
943
openbb_platform/extensions/news/openbb_news/registry.py
Normal file
943
openbb_platform/extensions/news/openbb_news/registry.py
Normal file
@@ -0,0 +1,943 @@
|
||||
"""RSS feed registry."""
|
||||
|
||||
from urllib.parse import quote
|
||||
|
||||
from openbb_core.app.config.loader import load_config
|
||||
|
||||
_PROVIDER_LABELS: dict[str, str] = {
|
||||
"axios": "Axios",
|
||||
"bbc": "BBC",
|
||||
"benzinga": "Benzinga",
|
||||
"cbc": "CBC",
|
||||
"drudge_report": "Drudge Report",
|
||||
"fox_news": "Fox News",
|
||||
"fortune": "Fortune",
|
||||
"globenewswire": "GlobeNewswire",
|
||||
"google_news": "Google News",
|
||||
"yahoo_finance": "Yahoo Finance",
|
||||
"pr_newswire": "PR Newswire",
|
||||
"wired": "Wired",
|
||||
"custom": "Custom",
|
||||
}
|
||||
|
||||
_PRNEWSWIRE_REGIONS: tuple[str, ...] = (
|
||||
"apac",
|
||||
"apac/zh",
|
||||
"br",
|
||||
"jp",
|
||||
"kr",
|
||||
"mx",
|
||||
"id",
|
||||
"vn",
|
||||
)
|
||||
_PRNEWSWIRE_REGION_LABELS: dict[str, str] = {
|
||||
"global": "Global",
|
||||
"apac": "APAC",
|
||||
"apac/zh": "APAC (Chinese)",
|
||||
"br": "Brazil",
|
||||
"jp": "Japan",
|
||||
"kr": "Korea",
|
||||
"mx": "Mexico",
|
||||
"id": "Indonesia",
|
||||
"vn": "Vietnam",
|
||||
}
|
||||
_PRNEWSWIRE_CATEGORIES: tuple[str, ...] = (
|
||||
"business-technology",
|
||||
"consumer-products-retail",
|
||||
"consumer-technology",
|
||||
"energy",
|
||||
"entertainment-media",
|
||||
"environment",
|
||||
"general-business",
|
||||
"heavy-industry-manufacturing",
|
||||
"health",
|
||||
"people-culture",
|
||||
"policy-public-interest",
|
||||
"sports",
|
||||
"telecommunications",
|
||||
"travel",
|
||||
"financial-services",
|
||||
"automotive-transportation",
|
||||
)
|
||||
_PRNEWSWIRE_CATEGORY_LABELS: dict[str, str] = {
|
||||
"business-technology": "Business Technology",
|
||||
"consumer-products-retail": "Consumer Products & Retail",
|
||||
"consumer-technology": "Consumer Technology",
|
||||
"energy": "Energy",
|
||||
"entertainment-media": "Entertainment & Media",
|
||||
"environment": "Environment",
|
||||
"general-business": "General Business",
|
||||
"heavy-industry-manufacturing": "Heavy Industry & Manufacturing",
|
||||
"health": "Health",
|
||||
"people-culture": "People & Culture",
|
||||
"policy-public-interest": "Policy & Public Interest",
|
||||
"sports": "Sports",
|
||||
"telecommunications": "Telecommunications",
|
||||
"travel": "Travel",
|
||||
"financial-services": "Financial Services",
|
||||
"automotive-transportation": "Automotive & Transportation",
|
||||
}
|
||||
_PRNEWSWIRE_GLOBAL_SKIP: frozenset[str] = frozenset({"business-technology"})
|
||||
|
||||
_GLOBENEWSWIRE_INDUSTRIES: tuple[tuple[str, str], ...] = (
|
||||
("2713", "Aerospace"),
|
||||
("5751", "Airlines"),
|
||||
("7537", "Alternative Electricity"),
|
||||
("587", "Alternative Fuels"),
|
||||
("1753", "Aluminum"),
|
||||
("5371", "Apparel Retailers"),
|
||||
("8771", "Asset Managers and Custodians"),
|
||||
("3355", "Auto Parts"),
|
||||
("40101010", "Auto Services"),
|
||||
("3353", "Automobiles"),
|
||||
("8355", "Banks"),
|
||||
("1000", "Basic Materials"),
|
||||
("4573", "Biotechnology"),
|
||||
("3533", "Brewers"),
|
||||
("5553", "Broadcasting and Entertainment"),
|
||||
("5373", "Broadline Retailers"),
|
||||
("50101025", "Building Climate Control"),
|
||||
("2353", "Building Materials and Fixtures"),
|
||||
("50101035", "Building Materials Other"),
|
||||
("50101020", "Building, Roofing, Wallboard and Plumbing"),
|
||||
("2791", "Business Support Services"),
|
||||
("2793", "Business Training and Employment Agencies"),
|
||||
("20103020", "Cannabis Producers"),
|
||||
("50101030", "Cement"),
|
||||
("55201010", "Chemicals and Synthetic Fibers"),
|
||||
("55201000", "Chemicals Diversified"),
|
||||
("30204000", "Closed End Investments"),
|
||||
("3763", "Clothing and Accessories"),
|
||||
("1771", "Coal"),
|
||||
("50206050", "Commercial Vehicle Equipment Leasing"),
|
||||
("50206015", "Commercial Vehicles and Parts"),
|
||||
("2753", "Commercial Vehicles and Trucks"),
|
||||
("1353", "Commodity Chemicals"),
|
||||
("9572", "Computer Hardware"),
|
||||
("9533", "Computer Services"),
|
||||
("50101010", "Construction"),
|
||||
("3743", "Consumer Electronics"),
|
||||
("8773", "Consumer Finance"),
|
||||
("3000", "Consumer Goods"),
|
||||
("5000", "Consumer Services"),
|
||||
("2723", "Containers and Packaging"),
|
||||
("7535", "Conventional Electricity"),
|
||||
("55102040", "Copper"),
|
||||
("2717", "Defense"),
|
||||
("2771", "Delivery Services"),
|
||||
("1773", "Diamonds and Gemstones"),
|
||||
("3535", "Distillers and Vintners"),
|
||||
("30202000", "Diversified Financial Services"),
|
||||
("2727", "Diversified Industrials"),
|
||||
("55101000", "Diversified Materials"),
|
||||
("8674", "Diversified REITs"),
|
||||
("5333", "Drug Retailers"),
|
||||
("3722", "Durable Household Products"),
|
||||
("50202010", "Electrical Components"),
|
||||
("2733", "Electrical Components and Equipment"),
|
||||
("2737", "Electronic Equipment"),
|
||||
("50202020", "Electronic Equipment Control and Filter"),
|
||||
("50202025", "Electronic Equipment Gauges and Meters"),
|
||||
("50202040", "Electronic Equipment Other"),
|
||||
("50202030", "Electronic Equipment Pollution Control"),
|
||||
("9574", "Electronic Office Equipment"),
|
||||
("1", "Energy"),
|
||||
("50101015", "Engineering and Contracting Services"),
|
||||
("8985", "Equity Investment Instruments"),
|
||||
("533", "Exploration and Production"),
|
||||
("3573", "Farming and Fishing"),
|
||||
("55201015", "Fertilizers"),
|
||||
("2795", "Financial Administration"),
|
||||
("30201030", "Financial Data Providers"),
|
||||
("8000", "Financials"),
|
||||
("6535", "Fixed Line Telecommunications"),
|
||||
("3577", "Food Products"),
|
||||
("5337", "Food Retailers and Wholesalers"),
|
||||
("3765", "Footwear"),
|
||||
("1733", "Forestry"),
|
||||
("50205030", "Forms and Bulk Printing Services"),
|
||||
("8532", "Full Line Insurance"),
|
||||
("3726", "Furnishings"),
|
||||
("5752", "Gambling"),
|
||||
("7573", "Gas Distribution"),
|
||||
("1775", "General Mining"),
|
||||
("50203020", "Glass"),
|
||||
("1777", "Gold Mining"),
|
||||
("4000", "Health Care"),
|
||||
("4533", "Health Care Providers"),
|
||||
("35102010", "Health Care REITs"),
|
||||
("2357", "Heavy Construction"),
|
||||
("3728", "Home Construction"),
|
||||
("5375", "Home Improvement Retailers"),
|
||||
("8677", "Hotel and Lodging REITs"),
|
||||
("5753", "Hotels"),
|
||||
("8671", "Industrial and Office REITs"),
|
||||
("2757", "Industrial Machinery"),
|
||||
("35102020", "Industrial REITs"),
|
||||
("2797", "Industrial Suppliers"),
|
||||
("2000", "Industrials"),
|
||||
("35102025", "Infrastructure REITs"),
|
||||
("8534", "Insurance Brokers"),
|
||||
("537", "Integrated Oil and Gas"),
|
||||
("9535", "Internet"),
|
||||
("8777", "Investment Services"),
|
||||
("1757", "Iron and Steel"),
|
||||
("8575", "Life Insurance"),
|
||||
("50204010", "Machinery Agricultural"),
|
||||
("50204020", "Machinery Construction and Handling"),
|
||||
("50204030", "Machinery Engines"),
|
||||
("50204000", "Machinery Industrial"),
|
||||
("50204050", "Machinery Specialty"),
|
||||
("50204040", "Machinery Tools"),
|
||||
("2773", "Marine Transportation"),
|
||||
("5555", "Media Agencies"),
|
||||
("4535", "Medical Equipment"),
|
||||
("20102020", "Medical Services"),
|
||||
("4537", "Medical Supplies"),
|
||||
("55102015", "Metal Fabricating"),
|
||||
("6575", "Mobile Telecommunications"),
|
||||
("8779", "Mortgage Finance"),
|
||||
("8676", "Mortgage REITs"),
|
||||
("7575", "Multiutilities"),
|
||||
("3724", "Nondurable Household Products"),
|
||||
("8995", "Nonequity Investment Instruments"),
|
||||
("1755", "Nonferrous Metals"),
|
||||
("35102030", "Office REITs"),
|
||||
("573", "Oil Equipment and Services"),
|
||||
("30205000", "Open End and Miscellaneous Investment Vehicles"),
|
||||
("35102070", "Other Specialty REITs"),
|
||||
("50203010", "Paints and Coatings"),
|
||||
("1737", "Paper"),
|
||||
("3767", "Personal Products"),
|
||||
("4577", "Pharmaceuticals"),
|
||||
("577", "Pipelines"),
|
||||
("50203015", "Plastics"),
|
||||
("1779", "Platinum and Precious Metals"),
|
||||
("10102020", "Production Technology Equipment"),
|
||||
("50205020", "Professional Business Support Services"),
|
||||
("8536", "Property and Casualty Insurance"),
|
||||
("5557", "Publishing"),
|
||||
("50206025", "Railroad Equipment"),
|
||||
("2775", "Railroads"),
|
||||
("35", "Real Estate"),
|
||||
("8633", "Real Estate Holding and Development"),
|
||||
("8637", "Real Estate Services"),
|
||||
("3745", "Recreational Products"),
|
||||
("5755", "Recreational Services"),
|
||||
("8538", "Reinsurance"),
|
||||
("583", "Renewable Energy Equipment"),
|
||||
("8673", "Residential REITs"),
|
||||
("5757", "Restaurants and Bars"),
|
||||
("8672", "Retail REITs"),
|
||||
("50205040", "Security Services"),
|
||||
("9576", "Semiconductors"),
|
||||
("3537", "Soft Drinks"),
|
||||
("9537", "Software"),
|
||||
("5377", "Specialized Consumer Services"),
|
||||
("1357", "Specialty Chemicals"),
|
||||
("8775", "Specialty Finance"),
|
||||
("8675", "Specialty REITs"),
|
||||
("5379", "Specialty Retailers"),
|
||||
("35102050", "Storage REITs"),
|
||||
("9000", "Technology"),
|
||||
("6000", "Telecommunications"),
|
||||
("9578", "Telecommunications Equipment"),
|
||||
("55101020", "Textile Products"),
|
||||
("35102060", "Timber REITs"),
|
||||
("3357", "Tires"),
|
||||
("3785", "Tobacco"),
|
||||
("3747", "Toys"),
|
||||
("50205015", "Transaction Processing Services"),
|
||||
("2777", "Transportation Services"),
|
||||
("5759", "Travel and Tourism"),
|
||||
("2779", "Trucking"),
|
||||
("7000", "Utilities"),
|
||||
("2799", "Waste and Disposal Services"),
|
||||
("7577", "Water"),
|
||||
)
|
||||
|
||||
|
||||
def _globenewswire_industry_url(code: str, label: str) -> str:
|
||||
"""Build a GlobeNewswire industry feed URL."""
|
||||
title = quote(f"GlobeNewswire - Industry News on {label}", safe="-")
|
||||
return f"https://www.globenewswire.com/AtomFeed/industry/{code}/feedTitle/{title}"
|
||||
|
||||
|
||||
def _globenewswire_industry_key(label: str) -> str:
|
||||
"""Slug an industry label into a feed key."""
|
||||
return _globenewswire_slug(label, prefix="globenewswire_")
|
||||
|
||||
|
||||
_GLOBENEWSWIRE_SUBJECTS: tuple[tuple[str, str], ...] = (
|
||||
("1", "Advisory"),
|
||||
("3", "Analyst Recommendations"),
|
||||
("2", "Annual Meetings and Shareholder Rights"),
|
||||
("65", "Annual Report"),
|
||||
("4", "Arts and Entertainment"),
|
||||
("5", "Bankruptcy"),
|
||||
("6", "Bond Market News"),
|
||||
("74", "Bonds Market Information"),
|
||||
("7", "Business Contracts"),
|
||||
("8", "Calendar Of Events"),
|
||||
("58", "Changes In Company's Own Shares"),
|
||||
("57", "Changes In Share Capital And Votes"),
|
||||
("71", "Changes To Observation Segment"),
|
||||
("84", "Class Action"),
|
||||
("90", "Clinical Study"),
|
||||
("9", "Company Announcement"),
|
||||
("10", "Company Regulatory Filings"),
|
||||
("89", "Conference Calls, Webcasts"),
|
||||
("88", "Contests, Awards"),
|
||||
("61", "Corporate Action"),
|
||||
("75", "Derivative Market Information"),
|
||||
("11", "Directors And Officers"),
|
||||
("12", "Dividend Reports and Estimates"),
|
||||
("13", "Earnings Releases and Operating Results"),
|
||||
("14", "Economic Research And Reports"),
|
||||
("91", "Environmental, Social, And Governance Criteria"),
|
||||
("76", "Equity Market Information"),
|
||||
("85", "European Regulatory News"),
|
||||
("70", "Exchange Announcement"),
|
||||
("77", "Exchange Members"),
|
||||
("78", "Exchange News"),
|
||||
("15", "Fashion"),
|
||||
("16", "Feature Article"),
|
||||
("17", "Financing Agreements"),
|
||||
("69", "First North Announcement"),
|
||||
("79", "First North Information"),
|
||||
("18", "Food"),
|
||||
("19", "Government News"),
|
||||
("20", "Health"),
|
||||
("21", "Initial Public Offerings"),
|
||||
("22", "Insider's Buy, Sell"),
|
||||
("66", "Interim Information"),
|
||||
("80", "Investment Fund Information"),
|
||||
("83", "Investment Opinion"),
|
||||
("23", "Joint Venture"),
|
||||
("24", "Law and Legal Issues"),
|
||||
("25", "Licensing Agreements"),
|
||||
("26", "Lifestyle"),
|
||||
("59", "Major Shareholder Announcements"),
|
||||
("86", "Management Changes"),
|
||||
("67", "Management Statements"),
|
||||
("81", "Market Research Reports"),
|
||||
("27", "Mergers and Acquisitions"),
|
||||
("64", "Mutual Fund Information"),
|
||||
("62", "Net Asset Value"),
|
||||
("28", "Other News"),
|
||||
("29", "Partnerships"),
|
||||
("30", "Patents"),
|
||||
("87", "Philanthropy"),
|
||||
("34", "Politics"),
|
||||
("31", "Pre Release Comments"),
|
||||
("72", "Press Releases"),
|
||||
("32", "Product, Services Announcement"),
|
||||
("63", "Prospectus, Announcement Of Prospectus"),
|
||||
("33", "Proxy Statements And Analysis"),
|
||||
("73", "Regulatory Information"),
|
||||
("35", "Religion"),
|
||||
("36", "Research Analysis And Reports"),
|
||||
("37", "Restructuring, Recapitalization"),
|
||||
("38", "Sports"),
|
||||
("39", "Stock Market News"),
|
||||
("40", "Tax Issues, Accounting"),
|
||||
("43", "Technical Analysis"),
|
||||
("41", "Trade Show"),
|
||||
("68", "Trading Information"),
|
||||
("42", "Travel"),
|
||||
("82", "Warrants And Certificates"),
|
||||
)
|
||||
|
||||
|
||||
def _globenewswire_subject_url(code: str, label: str) -> str:
|
||||
"""Build a GlobeNewswire subject feed URL."""
|
||||
title = quote(f"GlobeNewswire - {label}", safe="-")
|
||||
return (
|
||||
f"https://www.globenewswire.com/AtomFeed/subjectcode/{code}/feedTitle/{title}"
|
||||
)
|
||||
|
||||
|
||||
def _globenewswire_subject_key(label: str) -> str:
|
||||
"""Slug a subject label into a feed key."""
|
||||
return _globenewswire_slug(label, prefix="globenewswire_subject_")
|
||||
|
||||
|
||||
def _globenewswire_slug(label: str, prefix: str) -> str:
|
||||
"""Slug a label."""
|
||||
import re
|
||||
|
||||
slug = label.lower().replace("&", "and").replace("/", "_").replace("'", "")
|
||||
slug = slug.replace(":", "").replace(",", "").replace(".", "")
|
||||
slug = re.sub(r"[^a-z0-9]+", "_", slug).strip("_")
|
||||
return f"{prefix}{slug}"
|
||||
|
||||
|
||||
_CURATED_FEEDS: dict[str, tuple[str, str, str]] = {
|
||||
"axios_main": ("axios", "Latest", "https://api.axios.com/feed/"),
|
||||
"benzinga_main": ("benzinga", "Latest", "https://www.benzinga.com/feed"),
|
||||
"benzinga_markets": (
|
||||
"benzinga",
|
||||
"Markets",
|
||||
"https://www.benzinga.com/markets/feed",
|
||||
),
|
||||
"benzinga_general": (
|
||||
"benzinga",
|
||||
"General News",
|
||||
"https://www.benzinga.com/general/feed",
|
||||
),
|
||||
"benzinga_cryptocurrency": (
|
||||
"benzinga",
|
||||
"Cryptocurrency",
|
||||
"https://www.benzinga.com/topic/cryptocurrency/feed",
|
||||
),
|
||||
"benzinga_government": (
|
||||
"benzinga",
|
||||
"Government",
|
||||
"https://www.benzinga.com/topic/government/feed",
|
||||
),
|
||||
"globenewswire_all": (
|
||||
"globenewswire",
|
||||
"All News",
|
||||
"https://www.globenewswire.com/RssFeed/orgclass/1/feedTitle/GlobeNewswire-AllNews",
|
||||
),
|
||||
"globenewswire_us": (
|
||||
"globenewswire",
|
||||
"United States",
|
||||
"https://www.globenewswire.com/AtomFeed/country/United%20States/feedTitle/GlobeNewswire%20-%20News%20from%20United%20States",
|
||||
),
|
||||
"bbc_world": ("bbc", "World", "http://feeds.bbci.co.uk/news/world/rss.xml"),
|
||||
"bbc_uk": ("bbc", "UK", "http://feeds.bbci.co.uk/news/uk/rss.xml"),
|
||||
"bbc_business": (
|
||||
"bbc",
|
||||
"Business",
|
||||
"http://feeds.bbci.co.uk/news/business/rss.xml",
|
||||
),
|
||||
"bbc_politics": (
|
||||
"bbc",
|
||||
"Politics",
|
||||
"http://feeds.bbci.co.uk/news/politics/rss.xml",
|
||||
),
|
||||
"bbc_health": ("bbc", "Health", "http://feeds.bbci.co.uk/news/health/rss.xml"),
|
||||
"bbc_education": (
|
||||
"bbc",
|
||||
"Education",
|
||||
"http://feeds.bbci.co.uk/news/education/rss.xml",
|
||||
),
|
||||
"bbc_science": (
|
||||
"bbc",
|
||||
"Science & Environment",
|
||||
"http://feeds.bbci.co.uk/news/science_and_environment/rss.xml",
|
||||
),
|
||||
"bbc_technology": (
|
||||
"bbc",
|
||||
"Technology",
|
||||
"http://feeds.bbci.co.uk/news/technology/rss.xml",
|
||||
),
|
||||
"bbc_entertainment": (
|
||||
"bbc",
|
||||
"Entertainment & Arts",
|
||||
"http://feeds.bbci.co.uk/news/entertainment_and_arts/rss.xml",
|
||||
),
|
||||
"cbc_world": ("cbc", "World", "https://www.cbc.ca/webfeed/rss/rss-world"),
|
||||
"cbc_canada": ("cbc", "Canada", "https://www.cbc.ca/webfeed/rss/rss-canada"),
|
||||
"cbc_business": (
|
||||
"cbc",
|
||||
"Business",
|
||||
"https://www.cbc.ca/webfeed/rss/rss-business",
|
||||
),
|
||||
"cbc_politics": (
|
||||
"cbc",
|
||||
"Politics",
|
||||
"https://www.cbc.ca/webfeed/rss/rss-politics",
|
||||
),
|
||||
"cbc_technology": (
|
||||
"cbc",
|
||||
"Technology",
|
||||
"https://www.cbc.ca/webfeed/rss/rss-technology",
|
||||
),
|
||||
"cbc_health": ("cbc", "Health", "https://www.cbc.ca/webfeed/rss/rss-health"),
|
||||
"cbc_arts": ("cbc", "Arts", "https://www.cbc.ca/webfeed/rss/rss-arts"),
|
||||
"cbc_indigenous": (
|
||||
"cbc",
|
||||
"Indigenous",
|
||||
"https://www.cbc.ca/webfeed/rss/rss-Indigenous",
|
||||
),
|
||||
"cbc_thenational": (
|
||||
"cbc",
|
||||
"The National",
|
||||
"https://www.cbc.ca/webfeed/rss/rss-thenational",
|
||||
),
|
||||
"cbc_canada_britishcolumbia": (
|
||||
"cbc",
|
||||
"Region — British Columbia",
|
||||
"https://www.cbc.ca/webfeed/rss/rss-canada-britishcolumbia",
|
||||
),
|
||||
"cbc_canada_calgary": (
|
||||
"cbc",
|
||||
"Region — Calgary",
|
||||
"https://www.cbc.ca/webfeed/rss/rss-canada-calgary",
|
||||
),
|
||||
"cbc_canada_edmonton": (
|
||||
"cbc",
|
||||
"Region — Edmonton",
|
||||
"https://www.cbc.ca/webfeed/rss/rss-canada-edmonton",
|
||||
),
|
||||
"cbc_canada_hamilton": (
|
||||
"cbc",
|
||||
"Region — Hamilton",
|
||||
"https://www.cbc.ca/webfeed/rss/rss-canada-hamiltonnews",
|
||||
),
|
||||
"cbc_canada_kamloops": (
|
||||
"cbc",
|
||||
"Region — Kamloops",
|
||||
"https://www.cbc.ca/webfeed/rss/rss-canada-kamloops",
|
||||
),
|
||||
"cbc_canada_kitchenerwaterloo": (
|
||||
"cbc",
|
||||
"Region — Kitchener-Waterloo",
|
||||
"https://www.cbc.ca/webfeed/rss/rss-canada-kitchenerwaterloo",
|
||||
),
|
||||
"cbc_canada_london": (
|
||||
"cbc",
|
||||
"Region — London",
|
||||
"https://www.cbc.ca/webfeed/rss/rss-canada-london",
|
||||
),
|
||||
"cbc_canada_manitoba": (
|
||||
"cbc",
|
||||
"Region — Manitoba",
|
||||
"https://www.cbc.ca/webfeed/rss/rss-canada-manitoba",
|
||||
),
|
||||
"cbc_canada_montreal": (
|
||||
"cbc",
|
||||
"Region — Montreal",
|
||||
"https://www.cbc.ca/webfeed/rss/rss-canada-montreal",
|
||||
),
|
||||
"cbc_canada_newbrunswick": (
|
||||
"cbc",
|
||||
"Region — New Brunswick",
|
||||
"https://www.cbc.ca/webfeed/rss/rss-canada-newbrunswick",
|
||||
),
|
||||
"cbc_canada_newfoundland": (
|
||||
"cbc",
|
||||
"Region — Newfoundland",
|
||||
"https://www.cbc.ca/webfeed/rss/rss-canada-newfoundland",
|
||||
),
|
||||
"cbc_canada_north": (
|
||||
"cbc",
|
||||
"Region — North",
|
||||
"https://www.cbc.ca/webfeed/rss/rss-canada-north",
|
||||
),
|
||||
"cbc_canada_novascotia": (
|
||||
"cbc",
|
||||
"Region — Nova Scotia",
|
||||
"https://www.cbc.ca/webfeed/rss/rss-canada-novascotia",
|
||||
),
|
||||
"cbc_canada_ottawa": (
|
||||
"cbc",
|
||||
"Region — Ottawa",
|
||||
"https://www.cbc.ca/webfeed/rss/rss-canada-ottawa",
|
||||
),
|
||||
"cbc_canada_pei": (
|
||||
"cbc",
|
||||
"Region — Prince Edward Island",
|
||||
"https://www.cbc.ca/webfeed/rss/rss-canada-pei",
|
||||
),
|
||||
"cbc_canada_saskatchewan": (
|
||||
"cbc",
|
||||
"Region — Saskatchewan",
|
||||
"https://www.cbc.ca/webfeed/rss/rss-canada-saskatchewan",
|
||||
),
|
||||
"cbc_canada_saskatoon": (
|
||||
"cbc",
|
||||
"Region — Saskatoon",
|
||||
"https://www.cbc.ca/webfeed/rss/rss-canada-saskatoon",
|
||||
),
|
||||
"cbc_canada_sudbury": (
|
||||
"cbc",
|
||||
"Region — Sudbury",
|
||||
"https://www.cbc.ca/webfeed/rss/rss-canada-sudbury",
|
||||
),
|
||||
"cbc_canada_thunderbay": (
|
||||
"cbc",
|
||||
"Region — Thunder Bay",
|
||||
"https://www.cbc.ca/webfeed/rss/rss-canada-thunderbay",
|
||||
),
|
||||
"cbc_canada_toronto": (
|
||||
"cbc",
|
||||
"Region — Toronto",
|
||||
"https://www.cbc.ca/webfeed/rss/rss-canada-toronto",
|
||||
),
|
||||
"cbc_canada_windsor": (
|
||||
"cbc",
|
||||
"Region — Windsor",
|
||||
"https://www.cbc.ca/webfeed/rss/rss-canada-windsor",
|
||||
),
|
||||
"cbc_sports_nhl": (
|
||||
"cbc",
|
||||
"Sports — NHL",
|
||||
"https://www.cbc.ca/webfeed/rss/rss-sports-nhl",
|
||||
),
|
||||
"cbc_sports_nba": (
|
||||
"cbc",
|
||||
"Sports — NBA",
|
||||
"https://www.cbc.ca/webfeed/rss/rss-sports-nba",
|
||||
),
|
||||
"cbc_sports_nfl": (
|
||||
"cbc",
|
||||
"Sports — NFL",
|
||||
"https://www.cbc.ca/webfeed/rss/rss-sports-nfl",
|
||||
),
|
||||
"cbc_sports_mlb": (
|
||||
"cbc",
|
||||
"Sports — MLB",
|
||||
"https://www.cbc.ca/webfeed/rss/rss-sports-mlb",
|
||||
),
|
||||
"cbc_sports_soccer": (
|
||||
"cbc",
|
||||
"Sports — Soccer",
|
||||
"https://www.cbc.ca/webfeed/rss/rss-sports-soccer",
|
||||
),
|
||||
"fox_news_latest": (
|
||||
"fox_news",
|
||||
"Latest Headlines",
|
||||
"https://moxie.foxnews.com/google-publisher/latest.xml",
|
||||
),
|
||||
"fox_news_world": (
|
||||
"fox_news",
|
||||
"World",
|
||||
"https://moxie.foxnews.com/google-publisher/world.xml",
|
||||
),
|
||||
"fox_news_us": (
|
||||
"fox_news",
|
||||
"US",
|
||||
"https://moxie.foxnews.com/google-publisher/us.xml",
|
||||
),
|
||||
"fox_news_politics": (
|
||||
"fox_news",
|
||||
"Politics",
|
||||
"https://moxie.foxnews.com/google-publisher/politics.xml",
|
||||
),
|
||||
"google_news_us": (
|
||||
"google_news",
|
||||
"Top — US",
|
||||
"https://news.google.com/rss?hl=en-US&gl=US&ceid=US:en",
|
||||
),
|
||||
"google_news_ca": (
|
||||
"google_news",
|
||||
"Top — Canada",
|
||||
"https://news.google.com/rss?hl=en-CA&gl=CA&ceid=CA:en",
|
||||
),
|
||||
"google_news_uk": (
|
||||
"google_news",
|
||||
"Top — UK",
|
||||
"https://news.google.com/rss?hl=en-GB&gl=GB&ceid=GB:en",
|
||||
),
|
||||
"google_news_au": (
|
||||
"google_news",
|
||||
"Top — Australia",
|
||||
"https://news.google.com/rss?hl=en-AU&gl=AU&ceid=AU:en",
|
||||
),
|
||||
"google_news_business": (
|
||||
"google_news",
|
||||
"Business",
|
||||
"https://news.google.com/rss/headlines/section/topic/BUSINESS?hl=en-US&gl=US&ceid=US:en",
|
||||
),
|
||||
"google_news_technology": (
|
||||
"google_news",
|
||||
"Technology",
|
||||
"https://news.google.com/rss/headlines/section/topic/TECHNOLOGY?hl=en-US&gl=US&ceid=US:en",
|
||||
),
|
||||
"google_news_world": (
|
||||
"google_news",
|
||||
"World",
|
||||
"https://news.google.com/rss/headlines/section/topic/WORLD?hl=en-US&gl=US&ceid=US:en",
|
||||
),
|
||||
"google_news_nation": (
|
||||
"google_news",
|
||||
"Nation",
|
||||
"https://news.google.com/rss/headlines/section/topic/NATION?hl=en-US&gl=US&ceid=US:en",
|
||||
),
|
||||
"drudge_report": (
|
||||
"drudge_report",
|
||||
"Latest",
|
||||
"http://feeds.feedburner.com/DrudgeReportFeed",
|
||||
),
|
||||
"fortune": (
|
||||
"fortune",
|
||||
"Latest",
|
||||
"https://fortune.com/feed/fortune-feeds/?id=3230629",
|
||||
),
|
||||
"yahoo_finance": (
|
||||
"yahoo_finance",
|
||||
"Latest",
|
||||
"https://finance.yahoo.com/news/rssindex",
|
||||
),
|
||||
"wired_main": ("wired", "Latest", "https://www.wired.com/feed/rss"),
|
||||
"wired_backchannel": (
|
||||
"wired",
|
||||
"Backchannel",
|
||||
"https://www.wired.com/feed/category/backchannel/latest/rss",
|
||||
),
|
||||
"wired_business": (
|
||||
"wired",
|
||||
"Business",
|
||||
"https://www.wired.com/feed/category/business/latest/rss",
|
||||
),
|
||||
"wired_culture": (
|
||||
"wired",
|
||||
"Culture",
|
||||
"https://www.wired.com/feed/category/culture/latest/rss",
|
||||
),
|
||||
"wired_gear": (
|
||||
"wired",
|
||||
"Gear",
|
||||
"https://www.wired.com/feed/category/gear/latest/rss",
|
||||
),
|
||||
"wired_ideas": (
|
||||
"wired",
|
||||
"Ideas",
|
||||
"https://www.wired.com/feed/category/ideas/latest/rss",
|
||||
),
|
||||
"wired_science": (
|
||||
"wired",
|
||||
"Science",
|
||||
"https://www.wired.com/feed/category/science/latest/rss",
|
||||
),
|
||||
"wired_security": (
|
||||
"wired",
|
||||
"Security",
|
||||
"https://www.wired.com/feed/category/security/latest/rss",
|
||||
),
|
||||
"wired_ai": (
|
||||
"wired",
|
||||
"Tag — AI",
|
||||
"https://www.wired.com/feed/tag/ai/latest/rss",
|
||||
),
|
||||
"wired_guide": (
|
||||
"wired",
|
||||
"Tag — Wired Guide",
|
||||
"https://www.wired.com/feed/tag/wired-guide/latest/rss",
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _slug(value: str) -> str:
|
||||
return value.replace("/", "_").replace("-", "_")
|
||||
|
||||
|
||||
def _humanize(key: str) -> str:
|
||||
return key.replace("_", " ").title()
|
||||
|
||||
|
||||
def _build_feed_registry() -> tuple[dict[str, str], dict[str, dict[str, str]]]:
|
||||
feeds: dict[str, str] = {}
|
||||
meta: dict[str, dict[str, str]] = {}
|
||||
|
||||
feeds["pr_newswire_global"] = (
|
||||
"https://www.prnewswire.com/rss/news-releases-list.rss"
|
||||
)
|
||||
meta["pr_newswire_global"] = {"provider": "pr_newswire", "label": "Global"}
|
||||
|
||||
for region in _PRNEWSWIRE_REGIONS:
|
||||
key = f"pr_newswire_{_slug(region)}"
|
||||
feeds[key] = f"https://www.prnewswire.com/{region}/rss/news-releases-list.rss"
|
||||
meta[key] = {
|
||||
"provider": "pr_newswire",
|
||||
"label": _PRNEWSWIRE_REGION_LABELS[region],
|
||||
}
|
||||
|
||||
for cat in _PRNEWSWIRE_CATEGORIES:
|
||||
if cat in _PRNEWSWIRE_GLOBAL_SKIP:
|
||||
continue
|
||||
key = f"pr_newswire_global_{_slug(cat)}"
|
||||
feeds[key] = (
|
||||
f"https://www.prnewswire.com/rss/{cat}-latest-news/{cat}-latest-news-list.rss"
|
||||
)
|
||||
meta[key] = {
|
||||
"provider": "pr_newswire",
|
||||
"label": f"Global — {_PRNEWSWIRE_CATEGORY_LABELS[cat]}",
|
||||
}
|
||||
|
||||
for region in _PRNEWSWIRE_REGIONS:
|
||||
region_label = _PRNEWSWIRE_REGION_LABELS[region]
|
||||
for cat in _PRNEWSWIRE_CATEGORIES:
|
||||
key = f"pr_newswire_{_slug(region)}_{_slug(cat)}"
|
||||
feeds[key] = (
|
||||
f"https://www.prnewswire.com/{region}/rss/"
|
||||
f"{cat}-latest-news/{cat}-latest-news-list.rss"
|
||||
)
|
||||
meta[key] = {
|
||||
"provider": "pr_newswire",
|
||||
"label": f"{region_label} — {_PRNEWSWIRE_CATEGORY_LABELS[cat]}",
|
||||
}
|
||||
|
||||
for code, label in _GLOBENEWSWIRE_INDUSTRIES:
|
||||
key = _globenewswire_industry_key(label)
|
||||
feeds[key] = _globenewswire_industry_url(code, label)
|
||||
meta[key] = {
|
||||
"provider": "globenewswire",
|
||||
"label": f"Industry — {label}",
|
||||
}
|
||||
|
||||
for code, label in _GLOBENEWSWIRE_SUBJECTS:
|
||||
key = _globenewswire_subject_key(label)
|
||||
feeds[key] = _globenewswire_subject_url(code, label)
|
||||
meta[key] = {
|
||||
"provider": "globenewswire",
|
||||
"label": f"Subject — {label}",
|
||||
}
|
||||
|
||||
for key, (provider, label, url) in _CURATED_FEEDS.items():
|
||||
feeds[key] = url
|
||||
meta[key] = {"provider": provider, "label": label}
|
||||
|
||||
return feeds, meta
|
||||
|
||||
|
||||
_DEFAULT_FEEDS, _DEFAULT_META = _build_feed_registry()
|
||||
|
||||
|
||||
def _user_news_config() -> dict | None:
|
||||
"""Return the ``[news]`` table from ``openbb.toml``."""
|
||||
config = load_config()
|
||||
if not isinstance(config, dict):
|
||||
return None
|
||||
news = config.get("news")
|
||||
return news if isinstance(news, dict) else None
|
||||
|
||||
|
||||
def _user_feed_entries() -> dict[str, dict[str, str]] | None:
|
||||
"""Return normalized user feeds keyed by feed id."""
|
||||
news = _user_news_config()
|
||||
if not news:
|
||||
return None
|
||||
raw = news.get("rss_feeds")
|
||||
if not isinstance(raw, dict) or not raw:
|
||||
return None
|
||||
|
||||
out: dict[str, dict[str, str]] = {}
|
||||
for key, value in raw.items():
|
||||
skey = str(key)
|
||||
if isinstance(value, str) and value:
|
||||
out[skey] = {
|
||||
"url": value,
|
||||
"provider": "custom",
|
||||
"label": _humanize(skey),
|
||||
}
|
||||
elif isinstance(value, dict):
|
||||
url = value.get("url")
|
||||
if not isinstance(url, str) or not url:
|
||||
continue
|
||||
out[skey] = {
|
||||
"url": url,
|
||||
"provider": str(value.get("provider") or "custom"),
|
||||
"label": str(value.get("label") or _humanize(skey)),
|
||||
}
|
||||
return out or None
|
||||
|
||||
|
||||
def _user_provider_labels() -> dict[str, str]:
|
||||
"""Return user-defined provider labels from ``[news.rss_providers]``."""
|
||||
news = _user_news_config()
|
||||
if not news:
|
||||
return {}
|
||||
providers = news.get("rss_providers")
|
||||
if isinstance(providers, dict):
|
||||
return {str(k): str(v) for k, v in providers.items() if isinstance(v, str)}
|
||||
return {}
|
||||
|
||||
|
||||
def _should_merge_defaults() -> bool:
|
||||
"""Return ``True`` when ``[news] merge_defaults = true``."""
|
||||
news = _user_news_config()
|
||||
return bool(news and news.get("merge_defaults"))
|
||||
|
||||
|
||||
def list_feeds() -> dict[str, str]:
|
||||
"""Return the active ``{key: url}`` map."""
|
||||
user = _user_feed_entries()
|
||||
if user is None:
|
||||
return dict(_DEFAULT_FEEDS)
|
||||
user_urls = {k: v["url"] for k, v in user.items()}
|
||||
if _should_merge_defaults():
|
||||
merged = dict(_DEFAULT_FEEDS)
|
||||
merged.update(user_urls)
|
||||
return merged
|
||||
return user_urls
|
||||
|
||||
|
||||
def _feed_meta() -> dict[str, dict[str, str]]:
|
||||
"""Return ``{key: {provider, label}}`` for the active feed set."""
|
||||
user = _user_feed_entries()
|
||||
if user is None:
|
||||
return dict(_DEFAULT_META)
|
||||
user_meta = {
|
||||
k: {"provider": v["provider"], "label": v["label"]} for k, v in user.items()
|
||||
}
|
||||
if _should_merge_defaults():
|
||||
merged = dict(_DEFAULT_META)
|
||||
merged.update(user_meta)
|
||||
return merged
|
||||
return user_meta
|
||||
|
||||
|
||||
def list_providers() -> dict[str, str]:
|
||||
"""Return ``{provider_id: label}`` for active providers."""
|
||||
user_labels = _user_provider_labels()
|
||||
providers_in_use = {entry["provider"] for entry in _feed_meta().values()}
|
||||
result: dict[str, str] = {}
|
||||
for pid in sorted(providers_in_use):
|
||||
if pid in user_labels:
|
||||
result[pid] = user_labels[pid]
|
||||
elif pid in _PROVIDER_LABELS:
|
||||
result[pid] = _PROVIDER_LABELS[pid]
|
||||
else:
|
||||
result[pid] = _humanize(pid)
|
||||
return result
|
||||
|
||||
|
||||
def list_feed_choices(provider: str | None = None) -> list[dict[str, str]]:
|
||||
"""Return ``[{label, value}]`` for feeds under ``provider``."""
|
||||
if not provider:
|
||||
return []
|
||||
meta = _feed_meta()
|
||||
choices = [
|
||||
{"label": entry["label"], "value": key}
|
||||
for key, entry in meta.items()
|
||||
if entry["provider"] == provider
|
||||
]
|
||||
return sorted(choices, key=lambda c: c["label"])
|
||||
|
||||
|
||||
def get_feed_url(key: str) -> str:
|
||||
"""Look up a feed URL by key."""
|
||||
feeds = list_feeds()
|
||||
url = feeds.get(key)
|
||||
if url is None:
|
||||
known = ", ".join(sorted(feeds))
|
||||
raise ValueError(f"Unknown RSS feed source '{key}'. Known sources: {known}.")
|
||||
return url
|
||||
|
||||
|
||||
_DEFAULT_FEED_BY_PROVIDER: dict[str, str] = {
|
||||
"axios": "axios_main",
|
||||
"bbc": "bbc_world",
|
||||
"benzinga": "benzinga_markets",
|
||||
"cbc": "cbc_business",
|
||||
"fox_news": "fox_news_latest",
|
||||
"globenewswire": "globenewswire_all",
|
||||
"google_news": "google_news_us",
|
||||
"pr_newswire": "pr_newswire_global",
|
||||
"wired": "wired_business",
|
||||
}
|
||||
|
||||
|
||||
def default_feed_for(provider: str | None) -> str | None:
|
||||
"""Return the default feed key for ``provider``."""
|
||||
if not provider:
|
||||
return None
|
||||
preferred = _DEFAULT_FEED_BY_PROVIDER.get(provider)
|
||||
if preferred and preferred in list_feeds():
|
||||
return preferred
|
||||
choices = list_feed_choices(provider)
|
||||
return choices[0]["value"] if choices else None
|
||||
12
openbb_platform/extensions/news/openbb_news/router.py
Normal file
12
openbb_platform/extensions/news/openbb_news/router.py
Normal file
@@ -0,0 +1,12 @@
|
||||
"""News router."""
|
||||
|
||||
from openbb_core.app.router import Router
|
||||
|
||||
from openbb_news.news_router import router as _news_router
|
||||
from openbb_news.rss import router as _rss_router
|
||||
|
||||
router = Router(prefix="", description="Financial market news data.")
|
||||
router.include_router(_news_router)
|
||||
router.include_router(_rss_router)
|
||||
|
||||
__all__ = ["router"]
|
||||
206
openbb_platform/extensions/news/openbb_news/rss.py
Normal file
206
openbb_platform/extensions/news/openbb_news/rss.py
Normal file
@@ -0,0 +1,206 @@
|
||||
"""RSS news feed endpoint."""
|
||||
|
||||
import asyncio
|
||||
from datetime import (
|
||||
date as dateType,
|
||||
datetime,
|
||||
)
|
||||
from typing import TYPE_CHECKING, Annotated
|
||||
|
||||
from fastapi import Query
|
||||
from openbb_core.app.model.example import APIEx
|
||||
from openbb_core.app.model.obbject import OBBject
|
||||
from openbb_core.app.router import Router
|
||||
from openbb_core.app.service.system_service import SystemService
|
||||
from openbb_core.provider.abstract.data import Data
|
||||
from pydantic import Field
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from aiohttp import ClientSession
|
||||
|
||||
api_prefix = SystemService().system_settings.api_settings.prefix
|
||||
|
||||
router = Router(prefix="", description="RSS news feeds.")
|
||||
|
||||
|
||||
async def _gather_bodies(
|
||||
session: "ClientSession", links: list[str | None]
|
||||
) -> list[str | None]:
|
||||
"""Fetch article bodies concurrently."""
|
||||
from openbb_news.parser import fetch_article_body
|
||||
|
||||
async def one(link: str | None) -> str | None:
|
||||
if not link:
|
||||
return None
|
||||
try:
|
||||
return await fetch_article_body(session, link)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception: # noqa: BLE001
|
||||
return None
|
||||
|
||||
return await asyncio.gather(*(one(link) for link in links))
|
||||
|
||||
|
||||
class NewsItemData(Data):
|
||||
"""One article in Newsfeed-widget shape."""
|
||||
|
||||
title: str = Field(description="Article headline.")
|
||||
date: datetime | dateType | str = Field(description="ISO 8601 publication date.")
|
||||
author: str = Field(description="Author (falls back to the feed name).")
|
||||
url: str = Field(description="Canonical article URL from the RSS feed.")
|
||||
excerpt: str = Field(description="Short plain-text summary.")
|
||||
body: str = Field(description="Article body as markdown with inline images.")
|
||||
|
||||
|
||||
@router.command(
|
||||
methods=["GET"],
|
||||
widget_config={
|
||||
"name": "RSS News Feed",
|
||||
"description": (
|
||||
"Articles from any RSS/Atom feed registered under"
|
||||
" [news.rss_feeds] in openbb.toml."
|
||||
),
|
||||
"type": "newsfeed",
|
||||
"category": "News",
|
||||
"subCategory": "RSS",
|
||||
"source": ["RSS"],
|
||||
"gridData": {"w": 20, "h": 15},
|
||||
"refetchInterval": 300000,
|
||||
"params": [
|
||||
{
|
||||
"paramName": "outlet",
|
||||
"label": "Provider",
|
||||
"value": "benzinga",
|
||||
"description": "News outlet grouping the available feeds.",
|
||||
"type": "endpoint",
|
||||
"optionsEndpoint": f"{api_prefix}/news/rss_providers",
|
||||
"style": {"popupWidth": 280},
|
||||
},
|
||||
{
|
||||
"paramName": "source",
|
||||
"label": "Feed",
|
||||
"value": None,
|
||||
"description": "Specific feed within the selected provider.",
|
||||
"type": "endpoint",
|
||||
"optionsEndpoint": f"{api_prefix}/news/rss_feeds",
|
||||
"optionsParams": {"outlet": "$outlet"},
|
||||
"style": {"popupWidth": 360},
|
||||
"optional": True,
|
||||
},
|
||||
{
|
||||
"paramName": "limit",
|
||||
"label": "Limit",
|
||||
"value": 20,
|
||||
"description": "Maximum number of articles to return.",
|
||||
"type": "number",
|
||||
"optional": True,
|
||||
},
|
||||
{
|
||||
"paramName": "fetch_body",
|
||||
"label": "Fetch full body",
|
||||
"value": True,
|
||||
"description": "Fetch each article URL for its full body.",
|
||||
"type": "boolean",
|
||||
"optional": True,
|
||||
},
|
||||
],
|
||||
},
|
||||
examples=[APIEx(parameters={"source": "yahoo_finance"})],
|
||||
)
|
||||
async def rss(
|
||||
outlet: Annotated[
|
||||
str | None,
|
||||
Query(title="Provider", description="Outlet id."),
|
||||
] = None,
|
||||
source: Annotated[
|
||||
str | None,
|
||||
Query(title="Source", description="Configured feed key."),
|
||||
] = None,
|
||||
limit: Annotated[
|
||||
int,
|
||||
Query(title="Limit", description="Maximum articles to return.", ge=1),
|
||||
] = 20,
|
||||
fetch_body: Annotated[
|
||||
bool,
|
||||
Query(title="Fetch body", description="Fetch each article URL for its body."),
|
||||
] = True,
|
||||
) -> OBBject[list[NewsItemData]]:
|
||||
"""Fetch a registered RSS feed."""
|
||||
from openbb_core.provider.utils.helpers import get_async_requests_session
|
||||
|
||||
from openbb_news.parser import (
|
||||
fetch_feed,
|
||||
html_to_markdown,
|
||||
strip_html,
|
||||
struct_time_to_iso,
|
||||
truncate,
|
||||
)
|
||||
from openbb_news.registry import default_feed_for, get_feed_url
|
||||
|
||||
if not source:
|
||||
source = default_feed_for(outlet)
|
||||
if not source:
|
||||
return OBBject(results=[])
|
||||
|
||||
url = get_feed_url(source)
|
||||
|
||||
async with await get_async_requests_session() as session:
|
||||
parsed = await fetch_feed(session, url)
|
||||
feed_title = (parsed.feed.get("title") if parsed.feed else "") or ""
|
||||
entries = sorted(
|
||||
parsed.entries,
|
||||
key=lambda e: (
|
||||
e.get("published_parsed") or e.get("updated_parsed") or (0,) * 9
|
||||
),
|
||||
reverse=True,
|
||||
)[:limit]
|
||||
|
||||
if fetch_body and entries:
|
||||
links = [entry.get("link") for entry in entries]
|
||||
bodies = await _gather_bodies(session, links)
|
||||
else:
|
||||
bodies = [None] * len(entries)
|
||||
|
||||
out: list[NewsItemData] = []
|
||||
for entry, fetched_body in zip(entries, bodies):
|
||||
summary_html = entry.get("summary") or entry.get("description") or ""
|
||||
excerpt = truncate(strip_html(summary_html))
|
||||
body = fetched_body or html_to_markdown(summary_html) or excerpt
|
||||
out.append(
|
||||
NewsItemData(
|
||||
title=(entry.get("title") or "(untitled)").strip(),
|
||||
date=struct_time_to_iso(
|
||||
entry.get("published_parsed") or entry.get("updated_parsed")
|
||||
),
|
||||
author=entry.get("author") or feed_title or "Unknown",
|
||||
url=(entry.get("link") or "").strip(),
|
||||
excerpt=excerpt,
|
||||
body=body,
|
||||
)
|
||||
)
|
||||
|
||||
return OBBject(results=out)
|
||||
|
||||
|
||||
@router.command(methods=["GET"], widget_config={"exclude": True})
|
||||
async def rss_providers() -> list[dict[str, str]]:
|
||||
"""Return [{label, value}] for active news providers."""
|
||||
from openbb_news.registry import list_providers
|
||||
|
||||
return [
|
||||
{"label": label, "value": value} for value, label in list_providers().items()
|
||||
]
|
||||
|
||||
|
||||
@router.command(methods=["GET"], widget_config={"exclude": True})
|
||||
async def rss_feeds(
|
||||
outlet: Annotated[
|
||||
str | None,
|
||||
Query(title="Outlet", description="Outlet id."),
|
||||
] = None,
|
||||
) -> list[dict[str, str]]:
|
||||
"""Return [{label, value}] for feeds under outlet."""
|
||||
from openbb_news.registry import list_feed_choices
|
||||
|
||||
return list_feed_choices(outlet)
|
||||
1987
openbb_platform/extensions/news/poetry.lock
generated
vendored
1987
openbb_platform/extensions/news/poetry.lock
generated
vendored
File diff suppressed because it is too large
Load Diff
101
openbb_platform/extensions/news/pyproject.toml
vendored
101
openbb_platform/extensions/news/pyproject.toml
vendored
@@ -1,19 +1,94 @@
|
||||
[tool.poetry]
|
||||
[project]
|
||||
name = "openbb-news"
|
||||
version = "1.6.1"
|
||||
description = "News extension for OpenBB"
|
||||
authors = ["OpenBB Team <hello@openbb.co>"]
|
||||
license = "AGPL-3.0-only"
|
||||
version = "2.0.0"
|
||||
description = "RSS news feeds extension for OpenBB"
|
||||
authors = [{ name = "OpenBB Team", email = "hello@openbb.co" }]
|
||||
license = { text = "AGPL-3.0-only" }
|
||||
readme = "README.md"
|
||||
packages = [{ include = "openbb_news" }]
|
||||
requires-python = ">=3.10,<4"
|
||||
dependencies = [
|
||||
"openbb-core[pandas]>=2.0.0",
|
||||
"feedparser>=6.0",
|
||||
"lxml>=5.0",
|
||||
]
|
||||
|
||||
[tool.poetry.dependencies]
|
||||
python = ">=3.10,<4"
|
||||
openbb-core = "^1.6.3"
|
||||
[project.entry-points."openbb_core_extension"]
|
||||
news = "openbb_news.router:router"
|
||||
|
||||
[project.urls]
|
||||
Homepage = "https://openbb.co"
|
||||
Repository = "https://github.com/OpenBB-finance/OpenBB"
|
||||
|
||||
[dependency-groups]
|
||||
dev = ["openbb-devtools>=2.0.0"]
|
||||
|
||||
[build-system]
|
||||
requires = ["poetry-core"]
|
||||
build-backend = "poetry.core.masonry.api"
|
||||
requires = ["hatchling"]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[tool.poetry.plugins."openbb_core_extension"]
|
||||
news = "openbb_news.news_router:router"
|
||||
[tool.hatch.build.targets.wheel]
|
||||
packages = ["openbb_news"]
|
||||
|
||||
[tool.hatch.build.targets.sdist]
|
||||
include = ["openbb_news", "README.md", "pyproject.toml"]
|
||||
|
||||
[tool.uv.sources]
|
||||
openbb-core = { path = "../../core", editable = true }
|
||||
openbb-devtools = { path = "../devtools", editable = true }
|
||||
|
||||
[tool.ruff]
|
||||
line-length = 88
|
||||
target-version = "py310"
|
||||
fix = true
|
||||
|
||||
[tool.ruff.lint]
|
||||
select = ["D", "E", "W", "F", "S", "UP", "I", "PLC", "PLE", "PLR", "PLW", "SIM", "T20"]
|
||||
ignore = [
|
||||
"PLR2004", "PLR0913", "PLR0915", "PLC0415",
|
||||
"E402", "E501",
|
||||
"D101", "D102", "D103", "D105", "D107",
|
||||
"D202", "D203", "D205", "D206", "D210", "D212", "D213", "D214",
|
||||
"D215", "D300", "D400", "D401", "D404", "D405", "D406", "D407",
|
||||
"D408", "D409", "D410", "D411", "D413", "D415", "D416", "D417",
|
||||
]
|
||||
|
||||
[tool.ruff.lint.per-file-ignores]
|
||||
"openbb_news/*.py" = ["PLR0911", "PLR0912", "PLR0913", "PLR0914", "PLR0915", "PLR0917", "ARG001", "ARG002", "SLF001"]
|
||||
"tests/**" = ["S101", "S104", "S105", "S106", "S108", "D", "F841", "PLR0911", "PLW0108", "SIM105", "SIM117", "E741", "SLF001"]
|
||||
"integration/**" = ["S101", "S104", "S105", "S106", "S108", "D", "PLC0415", "PLR0911", "PLR0913", "PLR0914", "PLR0915", "PLR0917"]
|
||||
"**/__init__.py" = ["F401"]
|
||||
|
||||
[tool.ruff.lint.isort]
|
||||
combine-as-imports = true
|
||||
force-wrap-aliases = true
|
||||
|
||||
[tool.ruff.lint.pydocstyle]
|
||||
convention = "numpy"
|
||||
|
||||
[tool.ty]
|
||||
[tool.ty.environment]
|
||||
python-version = "3.10"
|
||||
|
||||
[tool.ty.src]
|
||||
include = ["openbb_news"]
|
||||
exclude = ["tests", "integration"]
|
||||
|
||||
[tool.ty.rules]
|
||||
unresolved-import = "warn"
|
||||
|
||||
[tool.coverage.run]
|
||||
omit = ["openbb_news/news_router.py"]
|
||||
|
||||
[tool.coverage.report]
|
||||
exclude_lines = ["pragma: no cover", "if TYPE_CHECKING:", "raise NotImplementedError", "@overload"]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
asyncio_mode = "auto"
|
||||
addopts = "-q --strict-markers"
|
||||
testpaths = ["tests"]
|
||||
markers = [
|
||||
"integration: end-to-end tests that hit live services (excluded from the coverage gate)",
|
||||
]
|
||||
filterwarnings = [
|
||||
"ignore:Inheritance class ClientSession from ClientSession is discouraged:DeprecationWarning",
|
||||
]
|
||||
|
||||
134
openbb_platform/extensions/news/tests/conftest.py
Normal file
134
openbb_platform/extensions/news/tests/conftest.py
Normal file
@@ -0,0 +1,134 @@
|
||||
"""Shared test fixtures."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
_SAMPLE_RSS = b"""<?xml version="1.0" encoding="UTF-8"?>
|
||||
<rss version="2.0">
|
||||
<channel>
|
||||
<title>Sample Feed</title>
|
||||
<item>
|
||||
<title>First Article</title>
|
||||
<link>https://example.test/article/1</link>
|
||||
<description><p>A short summary.</p></description>
|
||||
<pubDate>Mon, 19 May 2026 13:00:00 GMT</pubDate>
|
||||
<author>Jane Doe</author>
|
||||
</item>
|
||||
<item>
|
||||
<title>Second Article</title>
|
||||
<description>Plain text summary without HTML.</description>
|
||||
<pubDate>Mon, 19 May 2026 12:00:00 GMT</pubDate>
|
||||
</item>
|
||||
</channel>
|
||||
</rss>
|
||||
"""
|
||||
|
||||
_SAMPLE_ARTICLE_HTML = b"""<!DOCTYPE html><html><body>
|
||||
<article>
|
||||
<p>First paragraph of the article.</p>
|
||||
<p>Second paragraph with more detail.</p>
|
||||
<p>Third paragraph closing the story.</p>
|
||||
</article>
|
||||
</body></html>
|
||||
"""
|
||||
|
||||
_SAMPLE_JSONLD_HTML = b"""<!DOCTYPE html><html><body>
|
||||
<script type="application/ld+json">{"@type":"NewsArticle","articleBody":"JSON-LD top-level body."}</script>
|
||||
</body></html>
|
||||
"""
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_rss() -> bytes:
|
||||
return _SAMPLE_RSS
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_article_html() -> bytes:
|
||||
return _SAMPLE_ARTICLE_HTML
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_jsonld_html() -> bytes:
|
||||
return _SAMPLE_JSONLD_HTML
|
||||
|
||||
|
||||
class _FakeResponse:
|
||||
def __init__(self, content: bytes = b"", status: int = 200) -> None:
|
||||
self._content = content
|
||||
self.status = status
|
||||
|
||||
async def read(self) -> bytes:
|
||||
return self._content
|
||||
|
||||
def raise_for_status(self) -> None:
|
||||
if self.status >= 400:
|
||||
import aiohttp
|
||||
|
||||
raise aiohttp.ClientResponseError(
|
||||
request_info=None, # type: ignore[arg-type]
|
||||
history=(),
|
||||
status=self.status,
|
||||
)
|
||||
|
||||
async def __aenter__(self) -> "_FakeResponse":
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *_args: Any) -> None:
|
||||
return None
|
||||
|
||||
|
||||
class _FakeSession:
|
||||
def __init__(self, response_map: dict) -> None:
|
||||
self.response_map = response_map
|
||||
self.calls: list[str] = []
|
||||
self.closed = False
|
||||
|
||||
async def get(self, url: str, **_kwargs: Any) -> _FakeResponse:
|
||||
self.calls.append(url)
|
||||
result = self.response_map.get(url)
|
||||
if result is None:
|
||||
raise KeyError(f"unmocked URL: {url}")
|
||||
if isinstance(result, BaseException):
|
||||
raise result
|
||||
return result
|
||||
|
||||
async def __aenter__(self) -> "_FakeSession":
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *_args: Any) -> None:
|
||||
self.closed = True
|
||||
|
||||
async def close(self) -> None:
|
||||
self.closed = True
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def stub_session_factory(monkeypatch):
|
||||
def factory(response_map: dict) -> _FakeSession:
|
||||
normalized: dict = {}
|
||||
for url, value in response_map.items():
|
||||
if isinstance(value, (bytes, bytearray)):
|
||||
normalized[url] = _FakeResponse(bytes(value))
|
||||
elif (
|
||||
isinstance(value, tuple)
|
||||
and len(value) == 2
|
||||
and isinstance(value[0], (bytes, bytearray))
|
||||
):
|
||||
content, status = value
|
||||
normalized[url] = _FakeResponse(bytes(content), status=int(status))
|
||||
else:
|
||||
normalized[url] = value
|
||||
session = _FakeSession(normalized)
|
||||
|
||||
async def _factory(*_a: Any, **_kw: Any) -> _FakeSession:
|
||||
return session
|
||||
|
||||
monkeypatch.setattr(
|
||||
"openbb_core.provider.utils.helpers.get_async_requests_session",
|
||||
_factory,
|
||||
)
|
||||
return session
|
||||
|
||||
return factory
|
||||
606
openbb_platform/extensions/news/tests/test_parser.py
Normal file
606
openbb_platform/extensions/news/tests/test_parser.py
Normal file
@@ -0,0 +1,606 @@
|
||||
"""Parser tests."""
|
||||
|
||||
import time
|
||||
|
||||
from lxml import html
|
||||
|
||||
from openbb_news import parser
|
||||
|
||||
|
||||
def test_strip_html_empty():
|
||||
assert parser.strip_html("") == ""
|
||||
|
||||
|
||||
def test_strip_html_plain():
|
||||
assert parser.strip_html("hello world") == "hello world"
|
||||
|
||||
|
||||
def test_strip_html_tags_and_whitespace():
|
||||
assert parser.strip_html("<p>Hello <b>world</b>\n!</p>") == "Hello world !"
|
||||
|
||||
|
||||
def test_html_to_markdown_empty():
|
||||
assert parser.html_to_markdown("") == ""
|
||||
|
||||
|
||||
def test_html_to_markdown_preserves_anchor_links():
|
||||
out = parser.html_to_markdown(
|
||||
'<a href="https://nyt.test/x" target="_blank">NYT headline</a> '
|
||||
'<a href="https://bbc.test/x">BBC headline</a>'
|
||||
)
|
||||
assert "[NYT headline](https://nyt.test/x)" in out
|
||||
assert "[BBC headline](https://bbc.test/x)" in out
|
||||
|
||||
|
||||
def test_html_to_markdown_strips_non_anchor_tags():
|
||||
out = parser.html_to_markdown("<p>Hello <b>bold</b> world</p>")
|
||||
assert "bold" in out
|
||||
assert "<" not in out
|
||||
|
||||
|
||||
def test_html_to_markdown_anchor_without_href():
|
||||
out = parser.html_to_markdown('<a target="_blank">just text</a>')
|
||||
assert "just text" in out
|
||||
assert "[" not in out
|
||||
|
||||
|
||||
def test_html_to_markdown_anchor_without_text():
|
||||
out = parser.html_to_markdown('<a href="https://x.test/y"></a>')
|
||||
assert "[" not in out
|
||||
|
||||
|
||||
def test_html_to_markdown_handles_paragraphs_and_breaks():
|
||||
out = parser.html_to_markdown("<p>Line one</p><p>Line two</p>")
|
||||
assert "Line one" in out
|
||||
assert "Line two" in out
|
||||
assert "Line one" in out.split("\n")[0]
|
||||
|
||||
|
||||
def test_html_to_markdown_handles_br():
|
||||
out = parser.html_to_markdown("First<br/>Second")
|
||||
assert "First" in out
|
||||
assert "Second" in out
|
||||
|
||||
|
||||
def test_html_to_markdown_paragraph_after_text():
|
||||
out = parser.html_to_markdown("intro text<p>paragraph</p>")
|
||||
assert "intro text" in out
|
||||
assert "paragraph" in out
|
||||
lines = [line for line in out.split("\n") if line.strip()]
|
||||
assert lines.index("intro text") < lines.index("paragraph")
|
||||
|
||||
|
||||
def test_html_to_markdown_lxml_parse_failure_falls_back(monkeypatch):
|
||||
from lxml import html as _html
|
||||
|
||||
def boom(*_a, **_kw):
|
||||
raise ValueError("synthetic parse failure")
|
||||
|
||||
monkeypatch.setattr(_html, "fragment_fromstring", boom)
|
||||
out = parser.html_to_markdown('<a href="x">link</a>')
|
||||
assert out == "link"
|
||||
|
||||
|
||||
def test_truncate_short():
|
||||
assert parser.truncate("abc", limit=10) == "abc"
|
||||
|
||||
|
||||
def test_truncate_long():
|
||||
text = "word " * 100
|
||||
truncated = parser.truncate(text, limit=50)
|
||||
assert len(truncated) <= 60
|
||||
assert truncated.endswith("…")
|
||||
|
||||
|
||||
def test_struct_time_to_iso_none():
|
||||
iso = parser.struct_time_to_iso(None)
|
||||
assert "T" in iso and iso.endswith("+00:00")
|
||||
|
||||
|
||||
def test_struct_time_to_iso_value():
|
||||
st = time.struct_time((2026, 5, 19, 12, 0, 0, 0, 139, 0))
|
||||
assert parser.struct_time_to_iso(st) == "2026-05-19T12:00:00+00:00"
|
||||
|
||||
|
||||
def test_article_markdown_joins_with_blank_lines(sample_article_html):
|
||||
doc = html.fromstring(sample_article_html)
|
||||
article = doc.xpath("//article")[0]
|
||||
text = parser._article_markdown(article)
|
||||
assert "First paragraph of the article." in text
|
||||
assert text.count("\n\n") == 2
|
||||
|
||||
|
||||
def test_article_markdown_requires_two_paragraphs():
|
||||
doc = html.fromstring(b"<div><p>only one paragraph</p></div>")
|
||||
node = doc.xpath("//div")[0]
|
||||
assert parser._article_markdown(node) == ""
|
||||
|
||||
|
||||
def test_article_markdown_filters_video_chapter_timestamps():
|
||||
doc = html.fromstring(
|
||||
b"<div>"
|
||||
b"<p>Real opening paragraph with content.</p>"
|
||||
b"<p>2:26</p>"
|
||||
b"<p>12:44</p>"
|
||||
b"<p>0:58</p>"
|
||||
b"<p>1:30:00</p>"
|
||||
b"<p>Real closing paragraph.</p>"
|
||||
b"</div>"
|
||||
)
|
||||
text = parser._article_markdown(doc.xpath("//div")[0])
|
||||
assert "2:26" not in text
|
||||
assert "12:44" not in text
|
||||
assert "0:58" not in text
|
||||
assert "1:30:00" not in text
|
||||
assert "Real opening paragraph" in text
|
||||
assert "Real closing paragraph" in text
|
||||
|
||||
|
||||
def test_article_markdown_keeps_timestamp_inside_sentence():
|
||||
doc = html.fromstring(
|
||||
b"<div>"
|
||||
b"<p>The video begins at 2:26 with the host introducing.</p>"
|
||||
b"<p>By 12:44 the segment closes.</p>"
|
||||
b"</div>"
|
||||
)
|
||||
text = parser._article_markdown(doc.xpath("//div")[0])
|
||||
assert "2:26" in text
|
||||
assert "12:44" in text
|
||||
|
||||
|
||||
def test_article_markdown_ignores_non_paragraph_text():
|
||||
doc = html.fromstring(b"<div>plain wrapper text <span>here</span></div>")
|
||||
node = doc.xpath("//div")[0]
|
||||
assert parser._article_markdown(node) == ""
|
||||
|
||||
|
||||
def test_article_markdown_skips_figure_without_img():
|
||||
doc = html.fromstring(
|
||||
b"<article>"
|
||||
b"<p>One.</p>"
|
||||
b"<figure><figcaption>No image here</figcaption></figure>"
|
||||
b"<p>Two.</p>"
|
||||
b"</article>"
|
||||
)
|
||||
text = parser._article_markdown(doc.xpath("//article")[0])
|
||||
assert "![" not in text
|
||||
assert "One." in text
|
||||
assert "Two." in text
|
||||
|
||||
|
||||
def test_article_markdown_skips_img_without_src():
|
||||
doc = html.fromstring(b"<article><p>One.</p><img/><p>Two.</p></article>")
|
||||
text = parser._article_markdown(doc.xpath("//article")[0])
|
||||
assert "![" not in text
|
||||
|
||||
|
||||
def test_article_markdown_figure_without_caption():
|
||||
doc = html.fromstring(
|
||||
b"<article>"
|
||||
b"<p>One.</p>"
|
||||
b'<figure><img src="https://x.test/p.jpg"/></figure>'
|
||||
b"<p>Two.</p>"
|
||||
b"</article>"
|
||||
)
|
||||
text = parser._article_markdown(doc.xpath("//article")[0])
|
||||
assert "" in text
|
||||
|
||||
|
||||
def test_strip_chrome_removes_chrome_tags():
|
||||
doc = html.fromstring(
|
||||
b"<html><body>"
|
||||
b"<nav>nav junk</nav><header>h</header><footer>f</footer>"
|
||||
b"<aside>a</aside><form>fm</form><script>s</script><style>.x{}</style>"
|
||||
b"<noscript>n</noscript><iframe>i</iframe><button>b</button><svg>sv</svg>"
|
||||
b"<article><p>kept one</p><p>kept two</p></article>"
|
||||
b"</body></html>"
|
||||
)
|
||||
parser._strip_chrome(doc)
|
||||
leftover_tags = {el.tag for el in doc.iter() if isinstance(el.tag, str)}
|
||||
for tag in parser._CHROME_TAGS:
|
||||
assert tag not in leftover_tags
|
||||
assert "article" in leftover_tags
|
||||
|
||||
|
||||
def test_looks_clean_rejects_templates_and_empty():
|
||||
assert parser._looks_clean("Some real text.")
|
||||
assert not parser._looks_clean("")
|
||||
assert not parser._looks_clean("[[ type === 'moc' ]] body text")
|
||||
assert not parser._looks_clean("body with {{ binding }} marker")
|
||||
|
||||
|
||||
async def test_fetch_feed_parses(stub_session_factory, sample_rss):
|
||||
session = stub_session_factory({"http://feed.test/rss": sample_rss})
|
||||
parsed = await parser.fetch_feed(session, "http://feed.test/rss")
|
||||
assert len(parsed.entries) == 2
|
||||
assert parsed.entries[0].title == "First Article"
|
||||
|
||||
|
||||
async def test_fetch_feed_client_error(stub_session_factory):
|
||||
import aiohttp
|
||||
|
||||
session = stub_session_factory(
|
||||
{"http://feed.test/rss": aiohttp.ClientError("boom")}
|
||||
)
|
||||
parsed = await parser.fetch_feed(session, "http://feed.test/rss")
|
||||
assert len(parsed.entries) == 0
|
||||
|
||||
|
||||
async def test_fetch_feed_http_error(stub_session_factory):
|
||||
session = stub_session_factory({"http://feed.test/rss": (b"", 500)})
|
||||
parsed = await parser.fetch_feed(session, "http://feed.test/rss")
|
||||
assert len(parsed.entries) == 0
|
||||
|
||||
|
||||
def test_extract_body_jsonld_top_level(sample_jsonld_html):
|
||||
assert parser._extract_body(sample_jsonld_html) == "JSON-LD top-level body."
|
||||
|
||||
|
||||
def test_extract_body_jsonld_graph():
|
||||
payload = (
|
||||
b'<html><body><script type="application/ld+json">'
|
||||
b'{"@graph":[{"@type":"Other"},'
|
||||
b'{"@type":"NewsArticle","articleBody":"Body from graph."}]}'
|
||||
b"</script></body></html>"
|
||||
)
|
||||
assert parser._extract_body(payload) == "Body from graph."
|
||||
|
||||
|
||||
def test_extract_body_jsonld_list():
|
||||
payload = (
|
||||
b'<html><body><script type="application/ld+json">'
|
||||
b'[{"@type":"Other"},'
|
||||
b'{"@type":"NewsArticle","articleBody":"Found in list."}]'
|
||||
b"</script></body></html>"
|
||||
)
|
||||
assert parser._extract_body(payload) == "Found in list."
|
||||
|
||||
|
||||
def test_extract_body_jsonld_invalid_then_article():
|
||||
payload = (
|
||||
b"<html><body>"
|
||||
b'<script type="application/ld+json">not json</script>'
|
||||
b"<article><p>First sentence.</p><p>Second sentence.</p></article>"
|
||||
b"</body></html>"
|
||||
)
|
||||
assert parser._extract_body(payload) == "First sentence.\n\nSecond sentence."
|
||||
|
||||
|
||||
def test_extract_body_article_selector(sample_article_html):
|
||||
body = parser._extract_body(sample_article_html)
|
||||
assert body is not None
|
||||
assert "First paragraph" in body
|
||||
assert body.count("\n\n") == 2
|
||||
|
||||
|
||||
def test_extract_body_picks_largest_article():
|
||||
payload = (
|
||||
b"<html><body>"
|
||||
b"<article><p>Related card one.</p><p>Related card two.</p></article>"
|
||||
b"<article>"
|
||||
b"<p>Main story paragraph one with more substance.</p>"
|
||||
b"<p>Main story paragraph two with detail.</p>"
|
||||
b"<p>Main story paragraph three closing the piece.</p>"
|
||||
b"</article>"
|
||||
b"</body></html>"
|
||||
)
|
||||
body = parser._extract_body(payload)
|
||||
assert body is not None
|
||||
assert "Main story paragraph one" in body
|
||||
assert "Related card" not in body
|
||||
|
||||
|
||||
def test_extract_body_main_fallback_when_no_article():
|
||||
payload = (
|
||||
b"<html><body>"
|
||||
b"<nav><p>nav link</p><p>nav link two</p></nav>"
|
||||
b"<main>"
|
||||
b"<div><p>Story paragraph one with content.</p>"
|
||||
b"<p>Story paragraph two with more detail.</p></div>"
|
||||
b"</main>"
|
||||
b"</body></html>"
|
||||
)
|
||||
assert (
|
||||
parser._extract_body(payload)
|
||||
== "Story paragraph one with content.\n\nStory paragraph two with more detail."
|
||||
)
|
||||
|
||||
|
||||
def test_extract_body_main_requires_two_paragraphs():
|
||||
payload = b"<html><body><main><p>Only one paragraph here.</p></main></body></html>"
|
||||
assert parser._extract_body(payload) is None
|
||||
|
||||
|
||||
def test_extract_body_article_wins_over_main():
|
||||
payload = (
|
||||
b"<html><body>"
|
||||
b"<main>"
|
||||
b"<p>Main paragraph one.</p><p>Main paragraph two.</p>"
|
||||
b"<p>Main paragraph three.</p><p>Main paragraph four.</p>"
|
||||
b"</main>"
|
||||
b"<article><p>Article paragraph one.</p><p>Article paragraph two.</p></article>"
|
||||
b"</body></html>"
|
||||
)
|
||||
assert (
|
||||
parser._extract_body(payload)
|
||||
== "Article paragraph one.\n\nArticle paragraph two."
|
||||
)
|
||||
|
||||
|
||||
def test_extract_body_rejects_unrendered_templates():
|
||||
payload = (
|
||||
b"<html><body>"
|
||||
b"<article>"
|
||||
b"<p>[[ type === 'moc' ? 'MARKET ON CLOSE' : 'FREE WEBINAR' ]]</p>"
|
||||
b"<p>[[ timeLabel ]] WATCH LIVE</p>"
|
||||
b"</article>"
|
||||
b"</body></html>"
|
||||
)
|
||||
assert parser._extract_body(payload) is None
|
||||
|
||||
|
||||
def test_extract_body_strips_chrome_before_extracting():
|
||||
payload = (
|
||||
b"<html><body>"
|
||||
b"<nav><p>nav link one</p><p>nav link two</p></nav>"
|
||||
b"<header><p>banner text</p><p>banner two</p></header>"
|
||||
b"<article><p>Real story paragraph.</p><p>Story continues here.</p></article>"
|
||||
b"<footer><p>footer note</p><p>footer two</p></footer>"
|
||||
b"</body></html>"
|
||||
)
|
||||
assert (
|
||||
parser._extract_body(payload)
|
||||
== "Real story paragraph.\n\nStory continues here."
|
||||
)
|
||||
|
||||
|
||||
def test_extract_body_no_container():
|
||||
assert parser._extract_body(b"<html><body>nothing</body></html>") is None
|
||||
|
||||
|
||||
def test_extract_body_html_parse_error():
|
||||
assert parser._extract_body(b"") is None
|
||||
|
||||
|
||||
def test_extract_body_jsonld_non_dict_entries():
|
||||
payload = (
|
||||
b"<html><body>"
|
||||
b'<script type="application/ld+json">["string", 42, null]</script>'
|
||||
b"<article><p>Fallback one.</p><p>Fallback two.</p></article>"
|
||||
b"</body></html>"
|
||||
)
|
||||
assert parser._extract_body(payload) == "Fallback one.\n\nFallback two."
|
||||
|
||||
|
||||
def test_extract_body_jsonld_graph_no_body():
|
||||
payload = (
|
||||
b"<html><body>"
|
||||
b'<script type="application/ld+json">'
|
||||
b'{"@type":"NewsArticle","@graph":[{"@type":"X"}]}'
|
||||
b"</script>"
|
||||
b"<article><p>Graph fallback one.</p><p>Graph fallback two.</p></article>"
|
||||
b"</body></html>"
|
||||
)
|
||||
assert parser._extract_body(payload) == "Graph fallback one.\n\nGraph fallback two."
|
||||
|
||||
|
||||
def test_extract_body_jsonld_graph_non_dict_entry():
|
||||
payload = (
|
||||
b"<html><body>"
|
||||
b'<script type="application/ld+json">'
|
||||
b'{"@type":"NewsArticle","@graph":["scalar",null,42,'
|
||||
b'{"@type":"NewsArticle","articleBody":"Graph article body."}]}'
|
||||
b"</script></body></html>"
|
||||
)
|
||||
assert parser._extract_body(payload) == "Graph article body."
|
||||
|
||||
|
||||
def test_extract_body_jsonld_template_falls_through_to_article():
|
||||
payload = (
|
||||
b"<html><body>"
|
||||
b'<script type="application/ld+json">'
|
||||
b'{"@type":"NewsArticle","articleBody":"[[ binding ]] template junk"}'
|
||||
b"</script>"
|
||||
b"<article><p>Clean paragraph one.</p><p>Clean paragraph two.</p></article>"
|
||||
b"</body></html>"
|
||||
)
|
||||
assert (
|
||||
parser._extract_body(payload) == "Clean paragraph one.\n\nClean paragraph two."
|
||||
)
|
||||
|
||||
|
||||
def test_extract_body_preserves_utf8_smart_punctuation():
|
||||
payload = (
|
||||
"<html><body><article>"
|
||||
"<p>The Court’s ruling — issued Tuesday — was clear.</p>"
|
||||
"<p>Dominion’s lawyers said “we won”.</p>"
|
||||
"</article></body></html>"
|
||||
).encode()
|
||||
body = parser._extract_body(payload)
|
||||
assert body is not None
|
||||
assert "Court’s" in body
|
||||
assert "Dominion’s" in body
|
||||
assert "“we won”" in body
|
||||
assert "—" in body
|
||||
for artifact in ("’", "–", " "):
|
||||
assert artifact not in body
|
||||
|
||||
|
||||
async def test_fetch_article_body_end_to_end(stub_session_factory, sample_article_html):
|
||||
session = stub_session_factory({"https://x.test/a": sample_article_html})
|
||||
body = await parser.fetch_article_body(session, "https://x.test/a")
|
||||
assert body is not None
|
||||
assert "First paragraph" in body
|
||||
|
||||
|
||||
async def test_fetch_article_body_client_error(stub_session_factory):
|
||||
import aiohttp
|
||||
|
||||
session = stub_session_factory({"https://x.test/a": aiohttp.ClientError("boom")})
|
||||
assert await parser.fetch_article_body(session, "https://x.test/a") is None
|
||||
|
||||
|
||||
async def test_fetch_article_body_timeout(stub_session_factory):
|
||||
session = stub_session_factory(
|
||||
{"https://x.test/a": TimeoutError("simulated timeout")}
|
||||
)
|
||||
assert await parser.fetch_article_body(session, "https://x.test/a") is None
|
||||
|
||||
|
||||
async def test_fetch_article_body_http_error(stub_session_factory):
|
||||
session = stub_session_factory({"https://x.test/a": (b"", 403)})
|
||||
assert await parser.fetch_article_body(session, "https://x.test/a") is None
|
||||
|
||||
|
||||
async def test_fetch_article_body_no_article_returns_none(stub_session_factory):
|
||||
session = stub_session_factory(
|
||||
{"https://x.test/a": b"<html><body>nothing</body></html>"}
|
||||
)
|
||||
assert await parser.fetch_article_body(session, "https://x.test/a") is None
|
||||
|
||||
|
||||
def test_extract_body_preserves_inline_img():
|
||||
payload = (
|
||||
b"<html><body><article>"
|
||||
b"<p>Opening paragraph.</p>"
|
||||
b'<img src="https://x.test/inline.jpg" alt="caption text"/>'
|
||||
b"<p>Closing paragraph.</p>"
|
||||
b"</article></body></html>"
|
||||
)
|
||||
body = parser._extract_body(payload)
|
||||
assert body is not None
|
||||
lines = body.split("\n\n")
|
||||
assert lines == [
|
||||
"Opening paragraph.",
|
||||
"",
|
||||
"Closing paragraph.",
|
||||
]
|
||||
|
||||
|
||||
def test_extract_body_preserves_figure_with_caption():
|
||||
payload = (
|
||||
b"<html><body><article>"
|
||||
b"<p>Lead in.</p>"
|
||||
b'<figure><img src="https://x.test/photo.jpg"/>'
|
||||
b"<figcaption>A long caption describing the photo.</figcaption>"
|
||||
b"</figure>"
|
||||
b"<p>After the photo.</p>"
|
||||
b"</article></body></html>"
|
||||
)
|
||||
body = parser._extract_body(payload)
|
||||
assert body is not None
|
||||
assert "" in body
|
||||
assert "Lead in." in body
|
||||
assert "After the photo." in body
|
||||
|
||||
|
||||
def test_extract_body_prepends_og_image_when_no_inline_image():
|
||||
payload = (
|
||||
b"<html><head>"
|
||||
b'<meta property="og:image" content="https://x.test/hero.jpg">'
|
||||
b"</head><body><article>"
|
||||
b"<p>Paragraph one.</p><p>Paragraph two.</p>"
|
||||
b"</article></body></html>"
|
||||
)
|
||||
body = parser._extract_body(payload)
|
||||
assert body is not None
|
||||
assert body.split("\n\n")[0] == ""
|
||||
assert "Paragraph one." in body
|
||||
|
||||
|
||||
def test_extract_body_skips_hero_when_inline_image_present():
|
||||
payload = (
|
||||
b"<html><head>"
|
||||
b'<meta property="og:image" content="https://x.test/hero.jpg">'
|
||||
b"</head><body><article>"
|
||||
b"<p>Lead.</p>"
|
||||
b'<img src="https://x.test/inline.jpg"/>'
|
||||
b"<p>Body.</p>"
|
||||
b"</article></body></html>"
|
||||
)
|
||||
body = parser._extract_body(payload)
|
||||
assert body is not None
|
||||
assert "https://x.test/hero.jpg" not in body
|
||||
assert "" in body
|
||||
|
||||
|
||||
def test_extract_body_jsonld_articlebody_with_og_hero():
|
||||
payload = (
|
||||
b"<html><head>"
|
||||
b'<meta property="og:image" content="https://x.test/hero.jpg">'
|
||||
b"</head><body>"
|
||||
b'<script type="application/ld+json">'
|
||||
b'{"@type":"NewsArticle","articleBody":"Pure text body."}'
|
||||
b"</script></body></html>"
|
||||
)
|
||||
body = parser._extract_body(payload)
|
||||
assert body == "\n\nPure text body."
|
||||
|
||||
|
||||
def test_html_to_markdown_preserves_inline_img():
|
||||
out = parser.html_to_markdown(
|
||||
'<p>Lead</p><img src="https://x.test/inline.jpg" alt="alt text"/>'
|
||||
)
|
||||
assert "" in out
|
||||
assert "Lead" in out
|
||||
|
||||
|
||||
def test_html_to_markdown_img_tail_text():
|
||||
out = parser.html_to_markdown(
|
||||
'<img src="https://x.test/inline.jpg"/> trailing text after img'
|
||||
)
|
||||
assert "" in out
|
||||
assert "trailing text after img" in out
|
||||
|
||||
|
||||
def test_jsonld_field_list_of_strings():
|
||||
payload = (
|
||||
b"<html><body>"
|
||||
b'<script type="application/ld+json">'
|
||||
b'{"@type":"NewsArticle","image":["https://x.test/first.jpg","https://x.test/second.jpg"]}'
|
||||
b"</script></body></html>"
|
||||
)
|
||||
body = parser._extract_body(payload)
|
||||
assert body is None or body.startswith("")
|
||||
|
||||
|
||||
def test_jsonld_field_list_of_image_objects():
|
||||
payload = (
|
||||
b"<html><body>"
|
||||
b'<script type="application/ld+json">'
|
||||
b'{"@type":"NewsArticle","image":[{"url":"https://x.test/o.jpg"}]}'
|
||||
b"</script>"
|
||||
b"<article><p>One.</p><p>Two.</p></article>"
|
||||
b"</body></html>"
|
||||
)
|
||||
body = parser._extract_body(payload)
|
||||
assert body is not None
|
||||
assert body.startswith("")
|
||||
|
||||
|
||||
def test_jsonld_field_image_object_with_url():
|
||||
payload = (
|
||||
b"<html><body>"
|
||||
b'<script type="application/ld+json">'
|
||||
b'{"@type":"NewsArticle","image":{"@type":"ImageObject","url":"https://x.test/obj.jpg"}}'
|
||||
b"</script>"
|
||||
b"<article><p>One.</p><p>Two.</p></article>"
|
||||
b"</body></html>"
|
||||
)
|
||||
body = parser._extract_body(payload)
|
||||
assert body is not None
|
||||
assert body.startswith("")
|
||||
|
||||
|
||||
def test_extract_image_jsonld_fallback_path():
|
||||
payload = (
|
||||
b"<html><body>"
|
||||
b'<script type="application/ld+json">'
|
||||
b'{"@type":"NewsArticle","image":"https://x.test/ld.jpg"}'
|
||||
b"</script>"
|
||||
b"<article><p>One.</p><p>Two.</p></article>"
|
||||
b"</body></html>"
|
||||
)
|
||||
body = parser._extract_body(payload)
|
||||
assert body is not None
|
||||
assert body.startswith("")
|
||||
546
openbb_platform/extensions/news/tests/test_registry.py
Normal file
546
openbb_platform/extensions/news/tests/test_registry.py
Normal file
@@ -0,0 +1,546 @@
|
||||
"""Registry tests."""
|
||||
|
||||
import pytest
|
||||
|
||||
from openbb_news import registry
|
||||
|
||||
|
||||
def test_list_feeds_returns_defaults(monkeypatch):
|
||||
monkeypatch.setattr(registry, "load_config", lambda: {})
|
||||
feeds = registry.list_feeds()
|
||||
assert "pr_newswire_global" in feeds
|
||||
assert "bbc_world" in feeds
|
||||
assert feeds["bbc_world"].endswith("/world/rss.xml")
|
||||
|
||||
|
||||
def test_list_feeds_user_override(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
registry,
|
||||
"load_config",
|
||||
lambda: {"news": {"rss_feeds": {"my_feed": "https://my.test/rss"}}},
|
||||
)
|
||||
feeds = registry.list_feeds()
|
||||
assert feeds == {"my_feed": "https://my.test/rss"}
|
||||
|
||||
|
||||
def test_list_feeds_ignores_non_dict_override(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
registry, "load_config", lambda: {"news": {"rss_feeds": ["not", "a", "dict"]}}
|
||||
)
|
||||
feeds = registry.list_feeds()
|
||||
assert "bbc_world" in feeds
|
||||
|
||||
|
||||
def test_list_feeds_ignores_empty_override(monkeypatch):
|
||||
monkeypatch.setattr(registry, "load_config", lambda: {"news": {"rss_feeds": {}}})
|
||||
feeds = registry.list_feeds()
|
||||
assert "bbc_world" in feeds
|
||||
|
||||
|
||||
def test_get_feed_url_known():
|
||||
url = registry.get_feed_url("bbc_world")
|
||||
assert url.endswith("/world/rss.xml")
|
||||
|
||||
|
||||
def test_get_feed_url_unknown_raises():
|
||||
with pytest.raises(ValueError, match="Unknown RSS feed source"):
|
||||
registry.get_feed_url("not_a_real_source")
|
||||
|
||||
|
||||
def test_list_providers_default(monkeypatch):
|
||||
monkeypatch.setattr(registry, "load_config", lambda: {})
|
||||
providers = registry.list_providers()
|
||||
assert providers["axios"] == "Axios"
|
||||
assert providers["bbc"] == "BBC"
|
||||
assert providers["benzinga"] == "Benzinga"
|
||||
assert providers["cbc"] == "CBC"
|
||||
assert providers["drudge_report"] == "Drudge Report"
|
||||
assert providers["fox_news"] == "Fox News"
|
||||
assert providers["fortune"] == "Fortune"
|
||||
assert providers["globenewswire"] == "GlobeNewswire"
|
||||
assert providers["google_news"] == "Google News"
|
||||
assert providers["pr_newswire"] == "PR Newswire"
|
||||
assert providers["yahoo_finance"] == "Yahoo Finance"
|
||||
assert providers["wired"] == "Wired"
|
||||
assert "custom" not in providers
|
||||
assert "barchart" not in providers
|
||||
assert "cnn" not in providers
|
||||
assert "seeking_alpha" not in providers
|
||||
|
||||
|
||||
def test_get_feed_url_drudge_report():
|
||||
assert (
|
||||
registry.get_feed_url("drudge_report")
|
||||
== "http://feeds.feedburner.com/DrudgeReportFeed"
|
||||
)
|
||||
|
||||
|
||||
def test_list_feed_choices_cbc(monkeypatch):
|
||||
monkeypatch.setattr(registry, "load_config", lambda: {})
|
||||
choices = registry.list_feed_choices("cbc")
|
||||
labels = [c["label"] for c in choices]
|
||||
assert "Business" in labels
|
||||
assert "World" in labels
|
||||
assert "Region — Toronto" in labels
|
||||
assert "Sports — NHL" in labels
|
||||
assert labels == sorted(labels)
|
||||
|
||||
|
||||
def test_get_feed_url_cbc_business():
|
||||
assert (
|
||||
registry.get_feed_url("cbc_business")
|
||||
== "https://www.cbc.ca/webfeed/rss/rss-business"
|
||||
)
|
||||
|
||||
|
||||
def test_list_feed_choices_benzinga(monkeypatch):
|
||||
monkeypatch.setattr(registry, "load_config", lambda: {})
|
||||
choices = registry.list_feed_choices("benzinga")
|
||||
labels = [c["label"] for c in choices]
|
||||
assert "Markets" in labels
|
||||
assert "Cryptocurrency" in labels
|
||||
assert "Latest" in labels
|
||||
assert labels == sorted(labels)
|
||||
|
||||
|
||||
def test_list_feed_choices_globenewswire(monkeypatch):
|
||||
monkeypatch.setattr(registry, "load_config", lambda: {})
|
||||
choices = registry.list_feed_choices("globenewswire")
|
||||
labels = [c["label"] for c in choices]
|
||||
assert "All News" in labels
|
||||
assert "United States" in labels
|
||||
assert "Industry — Energy" in labels
|
||||
assert "Industry — Financials" in labels
|
||||
assert "Industry — Banks" in labels
|
||||
assert "Industry — Pharmaceuticals" in labels
|
||||
assert "Industry — Gold Mining" in labels
|
||||
assert "Industry — Semiconductors" in labels
|
||||
assert "Industry — Diversified REITs" in labels
|
||||
assert "Subject — Earnings Releases and Operating Results" in labels
|
||||
assert "Subject — Dividend Reports and Estimates" in labels
|
||||
assert "Subject — Mergers and Acquisitions" in labels
|
||||
assert len(choices) == 253
|
||||
assert labels == sorted(labels)
|
||||
|
||||
|
||||
def test_get_feed_url_globenewswire_all():
|
||||
url = registry.get_feed_url("globenewswire_all")
|
||||
assert url.startswith("https://www.globenewswire.com/RssFeed/")
|
||||
assert "AllNews" in url
|
||||
|
||||
|
||||
def test_get_feed_url_globenewswire_industry():
|
||||
url = registry.get_feed_url("globenewswire_energy")
|
||||
assert url.startswith(
|
||||
"https://www.globenewswire.com/AtomFeed/industry/1/feedTitle/"
|
||||
)
|
||||
assert "Energy" in url
|
||||
|
||||
|
||||
def test_globenewswire_industry_key_helper():
|
||||
assert (
|
||||
registry._globenewswire_industry_key("Iron and Steel")
|
||||
== "globenewswire_iron_and_steel"
|
||||
)
|
||||
assert (
|
||||
registry._globenewswire_industry_key(
|
||||
"Building, Roofing, Wallboard and Plumbing"
|
||||
)
|
||||
== "globenewswire_building_roofing_wallboard_and_plumbing"
|
||||
)
|
||||
assert (
|
||||
registry._globenewswire_industry_key("REITs & Real Estate")
|
||||
== "globenewswire_reits_and_real_estate"
|
||||
)
|
||||
|
||||
|
||||
def test_globenewswire_industries_unique_keys():
|
||||
keys = [
|
||||
registry._globenewswire_industry_key(label)
|
||||
for _code, label in registry._GLOBENEWSWIRE_INDUSTRIES
|
||||
]
|
||||
assert len(keys) == len(set(keys))
|
||||
|
||||
|
||||
def test_get_feed_url_globenewswire_subject():
|
||||
url = registry.get_feed_url(
|
||||
"globenewswire_subject_earnings_releases_and_operating_results"
|
||||
)
|
||||
assert url.startswith(
|
||||
"https://www.globenewswire.com/AtomFeed/subjectcode/13/feedTitle/"
|
||||
)
|
||||
assert "Earnings" in url
|
||||
|
||||
|
||||
def test_globenewswire_subject_key_helper():
|
||||
assert (
|
||||
registry._globenewswire_subject_key("Insider's Buy, Sell")
|
||||
== "globenewswire_subject_insiders_buy_sell"
|
||||
)
|
||||
assert (
|
||||
registry._globenewswire_subject_key("Mergers and Acquisitions")
|
||||
== "globenewswire_subject_mergers_and_acquisitions"
|
||||
)
|
||||
|
||||
|
||||
def test_globenewswire_subjects_unique_keys():
|
||||
keys = [
|
||||
registry._globenewswire_subject_key(label)
|
||||
for _code, label in registry._GLOBENEWSWIRE_SUBJECTS
|
||||
]
|
||||
assert len(keys) == len(set(keys))
|
||||
|
||||
|
||||
def test_get_feed_url_benzinga_markets():
|
||||
assert (
|
||||
registry.get_feed_url("benzinga_markets")
|
||||
== "https://www.benzinga.com/markets/feed"
|
||||
)
|
||||
|
||||
|
||||
def test_list_feed_choices_google_news(monkeypatch):
|
||||
monkeypatch.setattr(registry, "load_config", lambda: {})
|
||||
choices = registry.list_feed_choices("google_news")
|
||||
labels = [c["label"] for c in choices]
|
||||
assert "Business" in labels
|
||||
assert "Technology" in labels
|
||||
assert "Top — US" in labels
|
||||
assert "Top — Canada" in labels
|
||||
assert labels == sorted(labels)
|
||||
|
||||
|
||||
def test_get_feed_url_google_news_us():
|
||||
assert (
|
||||
registry.get_feed_url("google_news_us")
|
||||
== "https://news.google.com/rss?hl=en-US&gl=US&ceid=US:en"
|
||||
)
|
||||
|
||||
|
||||
def test_default_feed_for_curated_picks(monkeypatch):
|
||||
monkeypatch.setattr(registry, "load_config", lambda: {})
|
||||
expected = {
|
||||
"axios": "axios_main",
|
||||
"bbc": "bbc_world",
|
||||
"benzinga": "benzinga_markets",
|
||||
"cbc": "cbc_business",
|
||||
"fox_news": "fox_news_latest",
|
||||
"globenewswire": "globenewswire_all",
|
||||
"google_news": "google_news_us",
|
||||
"pr_newswire": "pr_newswire_global",
|
||||
"wired": "wired_business",
|
||||
}
|
||||
for provider, feed_key in expected.items():
|
||||
assert registry.default_feed_for(provider) == feed_key
|
||||
|
||||
|
||||
def test_default_feed_for_falls_back_to_first_alphabetical(monkeypatch):
|
||||
monkeypatch.setattr(registry, "load_config", lambda: {})
|
||||
assert registry.default_feed_for("fortune") == "fortune"
|
||||
|
||||
|
||||
def test_curated_defaults_reference_real_feeds(monkeypatch):
|
||||
monkeypatch.setattr(registry, "load_config", lambda: {})
|
||||
feeds = registry.list_feeds()
|
||||
for provider, feed_key in registry._DEFAULT_FEED_BY_PROVIDER.items():
|
||||
assert feed_key in feeds, f"{provider} default {feed_key!r} missing"
|
||||
|
||||
|
||||
def test_every_multi_feed_provider_has_a_curated_default(monkeypatch):
|
||||
monkeypatch.setattr(registry, "load_config", lambda: {})
|
||||
for provider in registry.list_providers():
|
||||
if provider == "custom":
|
||||
continue
|
||||
choices = registry.list_feed_choices(provider)
|
||||
if len(choices) > 1:
|
||||
assert provider in registry._DEFAULT_FEED_BY_PROVIDER, (
|
||||
f"multi-feed provider {provider!r} needs a curated default"
|
||||
)
|
||||
|
||||
|
||||
def test_default_feed_for_unknown_provider(monkeypatch):
|
||||
monkeypatch.setattr(registry, "load_config", lambda: {})
|
||||
assert registry.default_feed_for("totally_made_up") is None
|
||||
|
||||
|
||||
def test_default_feed_for_empty_provider():
|
||||
assert registry.default_feed_for(None) is None
|
||||
assert registry.default_feed_for("") is None
|
||||
|
||||
|
||||
def test_default_feed_for_drops_stale_curated_default(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
registry,
|
||||
"load_config",
|
||||
lambda: {"news": {"rss_feeds": {"only_one": "https://x.test/rss"}}},
|
||||
)
|
||||
assert registry.default_feed_for("axios") is None
|
||||
assert registry.default_feed_for("custom") == "only_one"
|
||||
|
||||
|
||||
def test_list_feed_choices_axios(monkeypatch):
|
||||
monkeypatch.setattr(registry, "load_config", lambda: {})
|
||||
choices = registry.list_feed_choices("axios")
|
||||
assert choices == [{"label": "Latest", "value": "axios_main"}]
|
||||
|
||||
|
||||
def test_get_feed_url_axios_main():
|
||||
assert registry.get_feed_url("axios_main") == "https://api.axios.com/feed/"
|
||||
|
||||
|
||||
def test_list_feed_choices_wired(monkeypatch):
|
||||
monkeypatch.setattr(registry, "load_config", lambda: {})
|
||||
choices = registry.list_feed_choices("wired")
|
||||
labels = [c["label"] for c in choices]
|
||||
assert "Latest" in labels
|
||||
assert "Backchannel" in labels
|
||||
assert "Business" in labels
|
||||
assert "Culture" in labels
|
||||
assert "Gear" in labels
|
||||
assert "Ideas" in labels
|
||||
assert "Science" in labels
|
||||
assert "Security" in labels
|
||||
assert "Tag — AI" in labels
|
||||
assert "Tag — Wired Guide" in labels
|
||||
assert len(choices) == 10
|
||||
assert labels == sorted(labels)
|
||||
|
||||
|
||||
def test_get_feed_url_wired_main():
|
||||
assert registry.get_feed_url("wired_main") == "https://www.wired.com/feed/rss"
|
||||
|
||||
|
||||
def test_get_feed_url_wired_business():
|
||||
assert (
|
||||
registry.get_feed_url("wired_business")
|
||||
== "https://www.wired.com/feed/category/business/latest/rss"
|
||||
)
|
||||
|
||||
|
||||
def test_list_providers_user_feeds_are_custom(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
registry,
|
||||
"load_config",
|
||||
lambda: {"news": {"rss_feeds": {"my_feed": "https://my.test/rss"}}},
|
||||
)
|
||||
providers = registry.list_providers()
|
||||
assert providers == {"custom": "Custom"}
|
||||
|
||||
|
||||
def test_list_feed_choices_filters_by_provider(monkeypatch):
|
||||
monkeypatch.setattr(registry, "load_config", lambda: {})
|
||||
bbc = registry.list_feed_choices("bbc")
|
||||
labels = [c["label"] for c in bbc]
|
||||
assert "World" in labels
|
||||
assert "Politics" in labels
|
||||
assert "Technology" in labels
|
||||
assert labels == sorted(labels)
|
||||
fox = registry.list_feed_choices("fox_news")
|
||||
fox_labels = [c["label"] for c in fox]
|
||||
assert "Latest Headlines" in fox_labels
|
||||
assert "Politics" in fox_labels
|
||||
|
||||
|
||||
def test_list_feed_choices_pr_newswire(monkeypatch):
|
||||
monkeypatch.setattr(registry, "load_config", lambda: {})
|
||||
choices = registry.list_feed_choices("pr_newswire")
|
||||
labels = [c["label"] for c in choices]
|
||||
assert "Global" in labels
|
||||
assert "APAC — General Business" in labels
|
||||
assert "APAC (Chinese)" in labels
|
||||
assert labels == sorted(labels)
|
||||
|
||||
|
||||
def test_list_feed_choices_no_provider(monkeypatch):
|
||||
monkeypatch.setattr(registry, "load_config", lambda: {})
|
||||
assert registry.list_feed_choices(None) == []
|
||||
assert registry.list_feed_choices("") == []
|
||||
|
||||
|
||||
def test_list_feed_choices_unknown_provider(monkeypatch):
|
||||
monkeypatch.setattr(registry, "load_config", lambda: {})
|
||||
assert registry.list_feed_choices("totally_made_up") == []
|
||||
|
||||
|
||||
def test_list_feed_choices_custom_feeds(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
registry,
|
||||
"load_config",
|
||||
lambda: {"news": {"rss_feeds": {"my_feed": "https://my.test/rss"}}},
|
||||
)
|
||||
choices = registry.list_feed_choices("custom")
|
||||
assert choices == [{"label": "My Feed", "value": "my_feed"}]
|
||||
|
||||
|
||||
def test_user_feeds_rich_form_with_provider_and_label(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
registry,
|
||||
"load_config",
|
||||
lambda: {
|
||||
"news": {
|
||||
"rss_feeds": {
|
||||
"alpha": {
|
||||
"url": "https://alpha.test/rss",
|
||||
"provider": "internal",
|
||||
"label": "Alpha Research",
|
||||
},
|
||||
"beta": {
|
||||
"url": "https://beta.test/rss",
|
||||
"provider": "internal",
|
||||
"label": "Beta Research",
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
)
|
||||
feeds = registry.list_feeds()
|
||||
assert feeds == {
|
||||
"alpha": "https://alpha.test/rss",
|
||||
"beta": "https://beta.test/rss",
|
||||
}
|
||||
choices = registry.list_feed_choices("internal")
|
||||
assert {c["value"] for c in choices} == {"alpha", "beta"}
|
||||
labels = {c["label"] for c in choices}
|
||||
assert labels == {"Alpha Research", "Beta Research"}
|
||||
|
||||
|
||||
def test_user_provider_labels_apply(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
registry,
|
||||
"load_config",
|
||||
lambda: {
|
||||
"news": {
|
||||
"rss_providers": {"internal": "Internal Sources"},
|
||||
"rss_feeds": {
|
||||
"x": {"url": "https://x.test/rss", "provider": "internal"},
|
||||
},
|
||||
}
|
||||
},
|
||||
)
|
||||
providers = registry.list_providers()
|
||||
assert providers == {"internal": "Internal Sources"}
|
||||
|
||||
|
||||
def test_user_provider_unknown_id_falls_back_to_humanized(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
registry,
|
||||
"load_config",
|
||||
lambda: {
|
||||
"news": {
|
||||
"rss_feeds": {
|
||||
"x": {"url": "https://x.test/rss", "provider": "acme_research"},
|
||||
},
|
||||
}
|
||||
},
|
||||
)
|
||||
providers = registry.list_providers()
|
||||
assert providers == {"acme_research": "Acme Research"}
|
||||
|
||||
|
||||
def test_user_feeds_merge_defaults_preserves_bundled(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
registry,
|
||||
"load_config",
|
||||
lambda: {
|
||||
"news": {
|
||||
"merge_defaults": True,
|
||||
"rss_feeds": {"x": "https://x.test/rss"},
|
||||
}
|
||||
},
|
||||
)
|
||||
feeds = registry.list_feeds()
|
||||
assert "bbc_world" in feeds
|
||||
assert "pr_newswire_global" in feeds
|
||||
assert feeds["x"] == "https://x.test/rss"
|
||||
providers = registry.list_providers()
|
||||
assert "bbc" in providers
|
||||
assert "custom" in providers
|
||||
|
||||
|
||||
def test_user_feeds_merge_override_replaces_bundled_key(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
registry,
|
||||
"load_config",
|
||||
lambda: {
|
||||
"news": {
|
||||
"merge_defaults": True,
|
||||
"rss_feeds": {"bbc_world": "https://my.test/bbc"},
|
||||
}
|
||||
},
|
||||
)
|
||||
feeds = registry.list_feeds()
|
||||
assert feeds["bbc_world"] == "https://my.test/bbc"
|
||||
|
||||
|
||||
def test_user_feeds_rich_form_ignores_missing_url(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
registry,
|
||||
"load_config",
|
||||
lambda: {
|
||||
"news": {
|
||||
"rss_feeds": {
|
||||
"good": {"url": "https://good.test/rss"},
|
||||
"bad": {"provider": "internal"},
|
||||
"also_bad": 42,
|
||||
}
|
||||
}
|
||||
},
|
||||
)
|
||||
feeds = registry.list_feeds()
|
||||
assert feeds == {"good": "https://good.test/rss"}
|
||||
|
||||
|
||||
def test_user_feeds_rich_form_default_provider_is_custom(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
registry,
|
||||
"load_config",
|
||||
lambda: {
|
||||
"news": {
|
||||
"rss_feeds": {
|
||||
"x": {"url": "https://x.test/rss", "label": "Pretty Label"},
|
||||
}
|
||||
}
|
||||
},
|
||||
)
|
||||
providers = registry.list_providers()
|
||||
assert providers == {"custom": "Custom"}
|
||||
choices = registry.list_feed_choices("custom")
|
||||
assert choices == [{"label": "Pretty Label", "value": "x"}]
|
||||
|
||||
|
||||
def test_load_config_returning_non_dict_falls_back_to_defaults(monkeypatch):
|
||||
monkeypatch.setattr(registry, "load_config", lambda: None)
|
||||
feeds = registry.list_feeds()
|
||||
assert "bbc_world" in feeds
|
||||
|
||||
|
||||
def test_user_provider_labels_ignores_non_string_values(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
registry,
|
||||
"load_config",
|
||||
lambda: {
|
||||
"news": {
|
||||
"rss_providers": {"internal": 42},
|
||||
"rss_feeds": {
|
||||
"x": {"url": "https://x.test/rss", "provider": "internal"},
|
||||
},
|
||||
}
|
||||
},
|
||||
)
|
||||
providers = registry.list_providers()
|
||||
assert providers == {"internal": "Internal"}
|
||||
|
||||
|
||||
def test_slug_helper():
|
||||
assert registry._slug("apac/zh") == "apac_zh"
|
||||
assert registry._slug("business-technology") == "business_technology"
|
||||
|
||||
|
||||
def test_humanize_helper():
|
||||
assert registry._humanize("my_internal_feed") == "My Internal Feed"
|
||||
|
||||
|
||||
def test_default_meta_covers_every_feed():
|
||||
assert set(registry._DEFAULT_FEEDS) == set(registry._DEFAULT_META)
|
||||
for entry in registry._DEFAULT_META.values():
|
||||
assert entry["provider"]
|
||||
assert entry["label"]
|
||||
24
openbb_platform/extensions/news/tests/test_router.py
Normal file
24
openbb_platform/extensions/news/tests/test_router.py
Normal file
@@ -0,0 +1,24 @@
|
||||
"""Router tests."""
|
||||
|
||||
from openbb_core.app.router import Router
|
||||
|
||||
from openbb_news.router import router
|
||||
|
||||
|
||||
def test_router_aggregates_rss_commands():
|
||||
assert isinstance(router, Router)
|
||||
paths = {getattr(route, "path", "") for route in router.api_router.routes}
|
||||
assert "/rss" in paths
|
||||
assert "/rss_providers" in paths
|
||||
assert "/rss_feeds" in paths
|
||||
|
||||
|
||||
def test_router_aggregates_provider_news_commands():
|
||||
from openbb_core.app.provider_interface import ProviderInterface
|
||||
|
||||
paths = {getattr(route, "path", "") for route in router.api_router.routes}
|
||||
model_map = ProviderInterface().map
|
||||
if "CompanyNews" in model_map:
|
||||
assert "/company" in paths
|
||||
if "WorldNews" in model_map:
|
||||
assert "/world" in paths
|
||||
350
openbb_platform/extensions/news/tests/test_rss.py
Normal file
350
openbb_platform/extensions/news/tests/test_rss.py
Normal file
@@ -0,0 +1,350 @@
|
||||
"""RSS endpoint tests."""
|
||||
|
||||
import pytest
|
||||
|
||||
from openbb_news import registry, rss
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mocked_pipeline(monkeypatch, stub_session_factory, sample_rss, sample_article_html):
|
||||
monkeypatch.setattr(
|
||||
"openbb_news.registry.get_feed_url",
|
||||
lambda key: f"http://feed.test/{key}",
|
||||
)
|
||||
stub_session_factory(
|
||||
{
|
||||
"http://feed.test/example": sample_rss,
|
||||
"https://example.test/article/1": sample_article_html,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
async def test_rss_with_body(mocked_pipeline):
|
||||
out = await rss.rss(source="example", limit=5, fetch_body=True)
|
||||
items = out.results
|
||||
assert len(items) == 2
|
||||
assert items[0].title == "First Article"
|
||||
assert items[0].date.endswith("+00:00")
|
||||
assert items[0].author == "Jane Doe"
|
||||
assert items[0].excerpt == "A short summary."
|
||||
assert items[0].url == "https://example.test/article/1"
|
||||
assert "First paragraph" in items[0].body
|
||||
assert items[1].excerpt == "Plain text summary without HTML."
|
||||
assert items[1].body == items[1].excerpt
|
||||
assert items[1].url == ""
|
||||
assert items[1].author == "Sample Feed"
|
||||
|
||||
|
||||
async def test_rss_without_body(mocked_pipeline):
|
||||
out = await rss.rss(source="example", limit=5, fetch_body=False)
|
||||
assert all(item.body == item.excerpt for item in out.results)
|
||||
|
||||
|
||||
async def test_rss_limit_caps_entries(mocked_pipeline):
|
||||
out = await rss.rss(source="example", limit=1, fetch_body=False)
|
||||
assert len(out.results) == 1
|
||||
|
||||
|
||||
async def test_rss_empty_feed(monkeypatch, stub_session_factory):
|
||||
monkeypatch.setattr(
|
||||
"openbb_news.registry.get_feed_url", lambda key: "http://feed.test/empty"
|
||||
)
|
||||
stub_session_factory({"http://feed.test/empty": b"<rss><channel></channel></rss>"})
|
||||
out = await rss.rss(source="empty", limit=5, fetch_body=True)
|
||||
assert out.results == []
|
||||
|
||||
|
||||
async def test_rss_unknown_source():
|
||||
with pytest.raises(ValueError, match="Unknown RSS feed source"):
|
||||
await rss.rss(source="totally_made_up", limit=1, fetch_body=False)
|
||||
|
||||
|
||||
async def test_rss_returns_empty_when_no_source():
|
||||
out = await rss.rss(source=None, limit=5, fetch_body=False)
|
||||
assert out.results == []
|
||||
out2 = await rss.rss(source="", limit=5, fetch_body=False)
|
||||
assert out2.results == []
|
||||
|
||||
|
||||
async def test_rss_auto_selects_curated_default_for_outlet(
|
||||
monkeypatch, stub_session_factory, sample_rss
|
||||
):
|
||||
monkeypatch.setattr(registry, "default_feed_for", lambda outlet: "curated_default")
|
||||
monkeypatch.setattr(
|
||||
"openbb_news.registry.get_feed_url",
|
||||
lambda key: f"http://feed.test/{key}",
|
||||
)
|
||||
stub_session_factory({"http://feed.test/curated_default": sample_rss})
|
||||
out = await rss.rss(outlet="any", source=None, limit=5, fetch_body=False)
|
||||
assert len(out.results) == 2
|
||||
assert out.results[0].title == "First Article"
|
||||
|
||||
|
||||
async def test_rss_returns_empty_for_unknown_outlet(monkeypatch):
|
||||
monkeypatch.setattr(registry, "default_feed_for", lambda outlet: None)
|
||||
out = await rss.rss(outlet="ghost", source=None, limit=5, fetch_body=False)
|
||||
assert out.results == []
|
||||
|
||||
|
||||
async def test_rss_sorts_newest_first(monkeypatch, stub_session_factory):
|
||||
feed = (
|
||||
b"<?xml version='1.0'?><rss version='2.0'><channel>"
|
||||
b"<title>Reversed Feed</title>"
|
||||
b"<item><title>Oldest</title>"
|
||||
b"<pubDate>Mon, 17 May 2026 09:00:00 GMT</pubDate></item>"
|
||||
b"<item><title>Middle</title>"
|
||||
b"<pubDate>Mon, 18 May 2026 09:00:00 GMT</pubDate></item>"
|
||||
b"<item><title>Newest</title>"
|
||||
b"<pubDate>Mon, 19 May 2026 09:00:00 GMT</pubDate></item>"
|
||||
b"</channel></rss>"
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"openbb_news.registry.get_feed_url", lambda key: "http://feed.test/rev"
|
||||
)
|
||||
stub_session_factory({"http://feed.test/rev": feed})
|
||||
out = await rss.rss(source="x", limit=5, fetch_body=False)
|
||||
assert [item.title for item in out.results] == ["Newest", "Middle", "Oldest"]
|
||||
|
||||
|
||||
async def test_rss_sorts_undated_entries_to_bottom(monkeypatch, stub_session_factory):
|
||||
feed = (
|
||||
b"<?xml version='1.0'?><rss version='2.0'><channel>"
|
||||
b"<title>Mixed Feed</title>"
|
||||
b"<item><title>No Date</title>"
|
||||
b"<description>no pubdate</description></item>"
|
||||
b"<item><title>Dated</title>"
|
||||
b"<pubDate>Mon, 19 May 2026 09:00:00 GMT</pubDate></item>"
|
||||
b"</channel></rss>"
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"openbb_news.registry.get_feed_url", lambda key: "http://feed.test/mixed"
|
||||
)
|
||||
stub_session_factory({"http://feed.test/mixed": feed})
|
||||
out = await rss.rss(source="x", limit=5, fetch_body=False)
|
||||
assert [item.title for item in out.results] == ["Dated", "No Date"]
|
||||
|
||||
|
||||
async def test_rss_handles_missing_title(monkeypatch, stub_session_factory):
|
||||
monkeypatch.setattr(
|
||||
"openbb_news.registry.get_feed_url", lambda key: "http://feed.test/notitle"
|
||||
)
|
||||
feed = (
|
||||
b"<?xml version='1.0'?><rss version='2.0'><channel>"
|
||||
b"<item><description>Body only</description></item>"
|
||||
b"</channel></rss>"
|
||||
)
|
||||
stub_session_factory({"http://feed.test/notitle": feed})
|
||||
items = (await rss.rss(source="x", limit=1, fetch_body=False)).results
|
||||
assert items[0].title == "(untitled)"
|
||||
assert items[0].author == "Unknown"
|
||||
|
||||
|
||||
async def test_rss_providers_lists_known_providers(monkeypatch):
|
||||
monkeypatch.setattr(registry, "load_config", lambda: {})
|
||||
out = await rss.rss_providers()
|
||||
keys = [entry["value"] for entry in out]
|
||||
assert "bbc" in keys
|
||||
assert "pr_newswire" in keys
|
||||
labels = {entry["value"]: entry["label"] for entry in out}
|
||||
assert labels["bbc"] == "BBC"
|
||||
assert labels["axios"] == "Axios"
|
||||
|
||||
|
||||
async def test_rss_feeds_filtered_by_provider(monkeypatch):
|
||||
monkeypatch.setattr(registry, "load_config", lambda: {})
|
||||
out = await rss.rss_feeds(outlet="bbc")
|
||||
labels = [entry["label"] for entry in out]
|
||||
assert "World" in labels
|
||||
assert "Politics" in labels
|
||||
assert labels == sorted(labels)
|
||||
fox_out = await rss.rss_feeds(outlet="fox_news")
|
||||
fox_labels = [entry["label"] for entry in fox_out]
|
||||
assert "Latest Headlines" in fox_labels
|
||||
assert "Politics" in fox_labels
|
||||
|
||||
|
||||
async def test_rss_feeds_no_provider_returns_empty(monkeypatch):
|
||||
monkeypatch.setattr(registry, "load_config", lambda: {})
|
||||
assert await rss.rss_feeds(outlet=None) == []
|
||||
|
||||
|
||||
def test_news_item_data_shape():
|
||||
item = rss.NewsItemData(
|
||||
title="t",
|
||||
date="2026-01-01T00:00:00+00:00",
|
||||
author="a",
|
||||
url="https://example.test/a",
|
||||
excerpt="e",
|
||||
body="b",
|
||||
)
|
||||
assert item.title == "t"
|
||||
assert item.body == "b"
|
||||
assert item.url == "https://example.test/a"
|
||||
|
||||
|
||||
async def test_rss_body_uses_markdown_links_when_extraction_fails(
|
||||
monkeypatch, stub_session_factory
|
||||
):
|
||||
feed = (
|
||||
b"<?xml version='1.0' encoding='UTF-8'?>"
|
||||
b'<rss version="2.0"><channel><title>Aggregator</title>'
|
||||
b"<item>"
|
||||
b"<title>Cluster</title>"
|
||||
b"<link>https://aggregator.test/x</link>"
|
||||
b"<description><![CDATA["
|
||||
b'<a href="https://nyt.test/story" target="_blank">NYT headline</a> '
|
||||
b'<a href="https://bbc.test/story" target="_blank">BBC headline</a> '
|
||||
b"]]></description>"
|
||||
b"<pubDate>Mon, 19 May 2026 12:00:00 GMT</pubDate>"
|
||||
b"</item></channel></rss>"
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"openbb_news.registry.get_feed_url", lambda key: "http://feed.test/agg"
|
||||
)
|
||||
stub_session_factory(
|
||||
{
|
||||
"http://feed.test/agg": feed,
|
||||
"https://aggregator.test/x": b"<html><body><div>not an article</div></body></html>",
|
||||
}
|
||||
)
|
||||
items = (await rss.rss(source="x", limit=1, fetch_body=True)).results
|
||||
assert len(items) == 1
|
||||
body = items[0].body
|
||||
assert "[NYT headline](https://nyt.test/story)" in body
|
||||
assert "[BBC headline](https://bbc.test/story)" in body
|
||||
assert items[0].url == "https://aggregator.test/x"
|
||||
|
||||
|
||||
async def test_rss_body_includes_inline_image_from_article(
|
||||
monkeypatch, stub_session_factory
|
||||
):
|
||||
feed = (
|
||||
b"<?xml version='1.0' encoding='UTF-8'?>"
|
||||
b'<rss version="2.0"><channel><title>X</title>'
|
||||
b"<item><title>Story</title>"
|
||||
b"<link>https://aggregator.test/story</link>"
|
||||
b"<description>x</description>"
|
||||
b"<pubDate>Wed, 20 May 2026 13:35:34 EDT</pubDate>"
|
||||
b"</item></channel></rss>"
|
||||
)
|
||||
article = (
|
||||
b"<html><body><article>"
|
||||
b"<p>Opening sentence.</p>"
|
||||
b'<img src="https://x.test/inline.jpg" alt="photo"/>'
|
||||
b"<p>Closing sentence.</p>"
|
||||
b"</article></body></html>"
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"openbb_news.registry.get_feed_url", lambda key: "http://feed.test/x"
|
||||
)
|
||||
stub_session_factory(
|
||||
{"http://feed.test/x": feed, "https://aggregator.test/story": article}
|
||||
)
|
||||
items = (await rss.rss(source="x", limit=1, fetch_body=True)).results
|
||||
body = items[0].body
|
||||
assert "" in body
|
||||
assert body.index("Opening sentence.") < body.index("![photo]")
|
||||
assert body.index("![photo]") < body.index("Closing sentence.")
|
||||
|
||||
|
||||
async def test_rss_body_prepends_og_image_when_no_inline(
|
||||
monkeypatch, stub_session_factory
|
||||
):
|
||||
feed = (
|
||||
b"<?xml version='1.0' encoding='UTF-8'?>"
|
||||
b'<rss version="2.0"><channel><title>X</title>'
|
||||
b"<item><title>Story</title>"
|
||||
b"<link>https://aggregator.test/story</link>"
|
||||
b"<description>x</description>"
|
||||
b"<pubDate>Wed, 20 May 2026 13:35:34 EDT</pubDate>"
|
||||
b"</item></channel></rss>"
|
||||
)
|
||||
article = (
|
||||
b"<html><head>"
|
||||
b'<meta property="og:image" content="https://x.test/hero.jpg">'
|
||||
b"</head><body><article>"
|
||||
b"<p>One.</p><p>Two.</p>"
|
||||
b"</article></body></html>"
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"openbb_news.registry.get_feed_url", lambda key: "http://feed.test/x"
|
||||
)
|
||||
stub_session_factory(
|
||||
{"http://feed.test/x": feed, "https://aggregator.test/story": article}
|
||||
)
|
||||
items = (await rss.rss(source="x", limit=1, fetch_body=True)).results
|
||||
assert items[0].body.startswith("")
|
||||
|
||||
|
||||
async def test_rss_body_preserves_cbc_style_description_image(
|
||||
monkeypatch, stub_session_factory
|
||||
):
|
||||
feed = (
|
||||
b"<?xml version='1.0' encoding='UTF-8'?>"
|
||||
b'<rss version="2.0"><channel><title>CBC</title>'
|
||||
b"<item><title>Story</title>"
|
||||
b"<link>https://aggregator.test/story</link>"
|
||||
b"<description><![CDATA["
|
||||
b"<img src='https://i.cbc.ca/hero.jpg' alt='photo'/>"
|
||||
b"<p>Lead text.</p>"
|
||||
b"]]></description>"
|
||||
b"<pubDate>Wed, 20 May 2026 13:35:34 EDT</pubDate>"
|
||||
b"</item></channel></rss>"
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"openbb_news.registry.get_feed_url", lambda key: "http://feed.test/x"
|
||||
)
|
||||
stub_session_factory(
|
||||
{
|
||||
"http://feed.test/x": feed,
|
||||
"https://aggregator.test/story": b"<html><body><div>not an article</div></body></html>",
|
||||
}
|
||||
)
|
||||
items = (await rss.rss(source="x", limit=1, fetch_body=True)).results
|
||||
assert "" in items[0].body
|
||||
|
||||
|
||||
async def test_gather_bodies_returns_none_for_empty_link(stub_session_factory):
|
||||
session = stub_session_factory({})
|
||||
out = await rss._gather_bodies(session, [None, ""])
|
||||
assert out == [None, None]
|
||||
assert session.calls == []
|
||||
|
||||
|
||||
async def test_gather_bodies_swallows_per_item_exceptions(
|
||||
monkeypatch, stub_session_factory, sample_article_html
|
||||
):
|
||||
import aiohttp
|
||||
|
||||
session = stub_session_factory(
|
||||
{
|
||||
"https://good.test/a": sample_article_html,
|
||||
"https://bad.test/a": aiohttp.ClientError("synthetic failure"),
|
||||
}
|
||||
)
|
||||
out = await rss._gather_bodies(
|
||||
session, ["https://good.test/a", "https://bad.test/a"]
|
||||
)
|
||||
assert out[0] is not None
|
||||
assert "First paragraph" in out[0]
|
||||
assert out[1] is None
|
||||
|
||||
|
||||
async def test_gather_bodies_propagates_cancellation(monkeypatch):
|
||||
import asyncio
|
||||
|
||||
async def cancelling_fetch(session, url):
|
||||
raise asyncio.CancelledError
|
||||
|
||||
monkeypatch.setattr("openbb_news.parser.fetch_article_body", cancelling_fetch)
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await rss._gather_bodies(object(), ["https://x.test/a"])
|
||||
|
||||
|
||||
async def test_gather_bodies_swallows_unexpected_exceptions(monkeypatch):
|
||||
async def broken_fetch(session, url):
|
||||
raise RuntimeError("internal bug")
|
||||
|
||||
monkeypatch.setattr("openbb_news.parser.fetch_article_body", broken_fetch)
|
||||
out = await rss._gather_bodies(object(), ["https://x.test/a", "https://x.test/b"])
|
||||
assert out == [None, None]
|
||||
3556
openbb_platform/extensions/news/uv.lock
generated
vendored
Normal file
3556
openbb_platform/extensions/news/uv.lock
generated
vendored
Normal file
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user