mirror of
https://github.com/OpenBB-finance/OpenBB.git
synced 2026-10-27 12:40:35 +00:00
feat(platform): standardize exchange inputs with ISO 10383 MIC codes (#7351)
* feat(platform): standardize exchange inputs with ISO 10383 MIC codes Add Exchange type in provider utils that standardizes exchange inputs across providers using ISO 10383 Market Identifier Codes (MIC). Similar to the Country type from PR #7333, this allows providers to accept: - ISO 10383 MIC codes (e.g., 'XNAS', 'XNYS') - Exchange acronyms (e.g., 'NASDAQ', 'NYSE') - Full exchange names (e.g., 'New York Stock Exchange') - lower_snake_case names (e.g., 'new_york_stock_exchange') The Exchange class inherits from str (storing the MIC code) while providing access to mic, acronym, name, country, and city properties. Example usage in FMP equity screener demonstrates provider integration pattern: - exchange: Exchange | None with validator for provider-specific validation - Maps Exchange.acronym to FMP's expected lowercase exchange codes Addresses part of #6969 * refactor(platform): use ExchangeData TypedDict for proper typing - Add ExchangeData TypedDict with explicit field types (consistent with CountryData) - Update function signatures and class attributes to use ExchangeData - Add _last_updated metadata to exchange_data.json - Add Wikipedia reference URL to docstring Aligns Exchange implementation with Country pattern from PR #7352. * fix: add missing major exchanges including Cboe (XCBO, XCBT) * refactor: expand to full ISO 10383 MIC list (2,272 active MICs) Address review feedback to include all active MICs, not just curated majors. - Expand exchange_data.json from 66 to 2,272 active MICs (operating + segments) - Remove country and city fields (out of scope for this PR) - Add scripts/update_exchange_data.py to regenerate from official ISO 20022 CSV source using stdlib only (no external dependencies) - Use first-write-wins lookup with operating MICs ordered before segments to resolve acronym collisions (e.g. LSE -> XLON, not XPVT) - Update tests to reflect simplified data model * feat: add city, country, and website fields to exchange data * refactor: move update_exchange_data.py to provider utils * style: fix black formatting in update_exchange_data.py --------- Co-authored-by: Danglewood <85772166+deeleeramone@users.noreply.github.com>
This commit is contained in:
18161
openbb_platform/core/openbb_core/provider/utils/exchange_data.json
vendored
Normal file
18161
openbb_platform/core/openbb_core/provider/utils/exchange_data.json
vendored
Normal file
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,205 @@
|
||||
"""Utilities for standardizing exchange inputs across providers using ISO 10383 MIC codes.
|
||||
|
||||
This module provides an Exchange type that inherits from str for type checker compatibility
|
||||
while providing full access to ISO 10383 Market Identifier Code (MIC) data via a static dataset.
|
||||
|
||||
Providers can access mic, acronym, and name properties as needed.
|
||||
|
||||
References:
|
||||
- ISO 10383: https://www.iso20022.org/market-identifier-codes
|
||||
- MIC Registry (CSV): https://www.iso20022.org/sites/default/files/ISO10383_MIC/ISO10383_MIC.csv
|
||||
- Wikipedia: https://en.wikipedia.org/wiki/Market_Identifier_Code
|
||||
"""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any, TypedDict
|
||||
|
||||
from pydantic_core import core_schema
|
||||
|
||||
|
||||
class ExchangeData(TypedDict, total=False):
|
||||
"""Type definition for exchange data dictionary."""
|
||||
|
||||
mic: str
|
||||
acronym: str
|
||||
name: str
|
||||
|
||||
|
||||
def _load_exchange_data() -> dict[str, ExchangeData]:
|
||||
"""Load exchange data from JSON and build lookup indices.
|
||||
|
||||
Returns a dict with lookup keys (mic, acronym, name variants) mapping to exchange data.
|
||||
"""
|
||||
data_path = Path(__file__).parent / "exchange_data.json"
|
||||
with open(data_path, encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
|
||||
lookup: dict[str, ExchangeData] = {}
|
||||
for exchange in data["exchanges"]:
|
||||
# Index by MIC (case-insensitive)
|
||||
lookup[exchange["mic"].upper()] = exchange
|
||||
lookup[exchange["mic"].lower()] = exchange
|
||||
|
||||
# Index by acronym (case-insensitive, first-write-wins to preserve
|
||||
# operating MIC priority over segment MICs sharing the same acronym)
|
||||
acronym = exchange["acronym"]
|
||||
acronym_upper = acronym.upper()
|
||||
acronym_lower = acronym.lower()
|
||||
if acronym_upper not in lookup:
|
||||
lookup[acronym_upper] = exchange
|
||||
if acronym_lower not in lookup:
|
||||
lookup[acronym_lower] = exchange
|
||||
|
||||
# Index by name (case-insensitive)
|
||||
name_lower = exchange["name"].lower()
|
||||
if name_lower not in lookup:
|
||||
lookup[name_lower] = exchange
|
||||
if exchange["name"] not in lookup:
|
||||
lookup[exchange["name"]] = exchange
|
||||
|
||||
# Index by snake_case name
|
||||
snake_name = (
|
||||
name_lower.replace(" ", "_")
|
||||
.replace("-", "_")
|
||||
.replace(",", "")
|
||||
.replace("'", "")
|
||||
)
|
||||
if snake_name not in lookup:
|
||||
lookup[snake_name] = exchange
|
||||
|
||||
return lookup
|
||||
|
||||
|
||||
# Load once at module import
|
||||
_EXCHANGE_LOOKUP = _load_exchange_data()
|
||||
|
||||
|
||||
class Exchange(str):
|
||||
"""Exchange string type with full ISO 10383 MIC data access.
|
||||
|
||||
Inherits from str (storing MIC code) for type checker compatibility
|
||||
while providing access to mic, acronym, and name properties.
|
||||
|
||||
Accepts:
|
||||
- ISO 10383 MIC codes (e.g., "XNAS", "XNYS")
|
||||
- Exchange acronyms (e.g., "NASDAQ", "NYSE")
|
||||
- Full exchange names (e.g., "New York Stock Exchange")
|
||||
- lower_snake_case names (e.g., "new_york_stock_exchange")
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> e = Exchange("nasdaq")
|
||||
>>> str(e)
|
||||
'XNAS'
|
||||
>>> e.mic
|
||||
'XNAS'
|
||||
>>> e.acronym
|
||||
'NASDAQ'
|
||||
>>> e.name
|
||||
'NASDAQ - ALL MARKETS'
|
||||
"""
|
||||
|
||||
_exchange_data: ExchangeData
|
||||
|
||||
def __new__(cls, value: Any) -> "Exchange":
|
||||
"""Create a new Exchange instance.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
value : Any
|
||||
Exchange input (MIC, acronym, name, or lower_snake_case).
|
||||
|
||||
Returns
|
||||
-------
|
||||
Exchange
|
||||
An Exchange instance storing the MIC code as its string value.
|
||||
|
||||
Raises
|
||||
------
|
||||
ValueError
|
||||
If the exchange cannot be resolved.
|
||||
"""
|
||||
exchange_data = (
|
||||
value._exchange_data
|
||||
if isinstance(value, Exchange)
|
||||
else cls._lookup_exchange(value)
|
||||
)
|
||||
|
||||
# Create str instance with the MIC code
|
||||
instance = super().__new__(cls, exchange_data["mic"])
|
||||
# Store the exchange data dict
|
||||
instance._exchange_data = exchange_data
|
||||
return instance
|
||||
|
||||
@staticmethod
|
||||
def _lookup_exchange(value: Any) -> ExchangeData:
|
||||
"""Look up an exchange from various input formats.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
value : Any
|
||||
Exchange input to look up.
|
||||
|
||||
Returns
|
||||
-------
|
||||
ExchangeData
|
||||
The exchange data dictionary with mic, acronym, name.
|
||||
|
||||
Raises
|
||||
------
|
||||
ValueError
|
||||
If the exchange cannot be found.
|
||||
"""
|
||||
val = str(value).strip()
|
||||
|
||||
# Convert lower_snake_case to lookup key
|
||||
if "_" in val:
|
||||
val = val.replace("_", " ")
|
||||
|
||||
# Try direct lookup
|
||||
lookup_key = val.lower()
|
||||
if lookup_key in _EXCHANGE_LOOKUP:
|
||||
return _EXCHANGE_LOOKUP[lookup_key]
|
||||
|
||||
# Try uppercase (common for MICs)
|
||||
if val.upper() in _EXCHANGE_LOOKUP:
|
||||
return _EXCHANGE_LOOKUP[val.upper()]
|
||||
|
||||
# Try original case
|
||||
if val in _EXCHANGE_LOOKUP:
|
||||
return _EXCHANGE_LOOKUP[val]
|
||||
|
||||
raise ValueError(
|
||||
f"Invalid exchange: '{value}'. "
|
||||
"Accepts ISO 10383 MIC codes (e.g., 'XNAS', 'XNYS'), "
|
||||
"acronyms (e.g., 'NASDAQ', 'NYSE'), "
|
||||
"or exchange names (e.g., 'New York Stock Exchange')."
|
||||
)
|
||||
|
||||
@property
|
||||
def mic(self) -> str:
|
||||
"""ISO 10383 Market Identifier Code (e.g., 'XNAS')."""
|
||||
return self._exchange_data["mic"]
|
||||
|
||||
@property
|
||||
def acronym(self) -> str:
|
||||
"""Exchange acronym/short name (e.g., 'NASDAQ')."""
|
||||
return self._exchange_data["acronym"]
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
"""Full exchange name (e.g., 'NASDAQ - ALL MARKETS')."""
|
||||
return self._exchange_data["name"]
|
||||
|
||||
@classmethod
|
||||
def __get_pydantic_core_schema__(cls, _source_type: Any, _handler: Any) -> Any:
|
||||
"""Return the Pydantic core schema for validation."""
|
||||
return core_schema.no_info_after_validator_function(
|
||||
cls,
|
||||
core_schema.str_schema(),
|
||||
)
|
||||
|
||||
|
||||
# Backwards-compatible alias
|
||||
ExchangeParam = Exchange
|
||||
@@ -0,0 +1,158 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Update exchange data from ISO 10383 (Market Identifier Codes).
|
||||
|
||||
Downloads the official ISO 10383 MIC registry and regenerates exchange_data.json.
|
||||
Uses only Python standard library (no external dependencies).
|
||||
|
||||
Source:
|
||||
https://www.iso20022.org/market-identifier-codes
|
||||
|
||||
Usage:
|
||||
python openbb_platform/core/openbb_core/provider/utils/update_exchange_data.py
|
||||
python openbb_platform/core/openbb_core/provider/utils/update_exchange_data.py --operating-only
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
import urllib.request
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SCRIPT_DIR = Path(__file__).parent
|
||||
OUTPUT_PATH = SCRIPT_DIR / "exchange_data.json"
|
||||
MIC_CSV_URL = (
|
||||
"https://www.iso20022.org/sites/default/files/ISO10383_MIC/ISO10383_MIC.csv"
|
||||
)
|
||||
|
||||
|
||||
def download_mic_csv() -> list[dict]:
|
||||
"""Download and parse the official ISO 10383 MIC CSV."""
|
||||
logger.info("Downloading MIC data from %s...", MIC_CSV_URL)
|
||||
|
||||
req = urllib.request.Request( # noqa: S310
|
||||
MIC_CSV_URL, headers={"User-Agent": "OpenBB/1.0"}
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=30) as resp: # noqa: S310
|
||||
raw = resp.read().decode("utf-8-sig")
|
||||
|
||||
reader = csv.DictReader(io.StringIO(raw))
|
||||
rows = list(reader)
|
||||
logger.info("Downloaded %d total MIC entries", len(rows))
|
||||
return rows
|
||||
|
||||
|
||||
def process_mic_data(rows: list[dict], operating_only: bool = False) -> list[dict]:
|
||||
"""Filter and transform MIC rows into exchange entries."""
|
||||
# Normalize header keys (strip whitespace, uppercase)
|
||||
active = [r for r in rows if r.get("STATUS", "").strip().upper() == "ACTIVE"]
|
||||
logger.info("Active entries: %d", len(active))
|
||||
|
||||
if operating_only:
|
||||
active = [
|
||||
r for r in active if r.get("OPRT/SGMT", "").strip().upper() in ("OPRT", "O")
|
||||
]
|
||||
logger.info("Operating MICs only: %d", len(active))
|
||||
|
||||
exchanges = []
|
||||
for row in active:
|
||||
mic = row.get("MIC", "").strip()
|
||||
if not mic:
|
||||
continue
|
||||
|
||||
name = row.get("MARKET NAME-INSTITUTION DESCRIPTION", "").strip()
|
||||
if not name:
|
||||
continue
|
||||
|
||||
acronym = row.get("ACRONYM", "").strip() or mic
|
||||
|
||||
entry = {
|
||||
"mic": mic,
|
||||
"acronym": acronym,
|
||||
"name": name,
|
||||
}
|
||||
|
||||
city = row.get("CITY", "").strip()
|
||||
if city:
|
||||
entry["city"] = city.title()
|
||||
|
||||
country_code = row.get("ISO COUNTRY CODE (ISO 3166)", "").strip().upper()
|
||||
if country_code:
|
||||
entry["country"] = country_code
|
||||
|
||||
website = row.get("WEBSITE", "").strip()
|
||||
if website:
|
||||
# Normalize: ensure lowercase, add https:// if missing scheme
|
||||
website = website.lower()
|
||||
if website and not website.startswith(("http://", "https://")):
|
||||
website = f"https://{website}"
|
||||
entry["website"] = website
|
||||
|
||||
entry["_type"] = row.get("OPRT/SGMT", "").strip().upper()
|
||||
exchanges.append(entry)
|
||||
|
||||
# Sort operating MICs before segments so that the lookup in exchange_utils
|
||||
# (first-write-wins) gives priority to operating MICs when acronyms collide.
|
||||
exchanges.sort(key=lambda x: (0 if x["_type"] in ("OPRT", "O") else 1, x["mic"]))
|
||||
|
||||
# Strip internal sort key before output
|
||||
for e in exchanges:
|
||||
del e["_type"]
|
||||
return exchanges
|
||||
|
||||
|
||||
def build_exchange_data(exchanges: list[dict]) -> dict:
|
||||
"""Build the final exchange data structure."""
|
||||
return {
|
||||
"_last_updated": date.today().isoformat(),
|
||||
"_source": {
|
||||
"url": "https://www.iso20022.org/market-identifier-codes",
|
||||
"standard": "ISO 10383",
|
||||
"maintainer": "SWIFT (ISO 20022 Registration Authority)",
|
||||
},
|
||||
"_stats": {
|
||||
"total_exchanges": len(exchanges),
|
||||
},
|
||||
"exchanges": exchanges,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
"""Download ISO 10383 MIC registry and regenerate exchange_data.json."""
|
||||
logging.basicConfig(level=logging.INFO, format="%(message)s")
|
||||
|
||||
parser = argparse.ArgumentParser(description="Update exchange data from ISO 10383")
|
||||
parser.add_argument(
|
||||
"--operating-only",
|
||||
action="store_true",
|
||||
help="Exclude segment MICs (only include operating MICs)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
rows = download_mic_csv()
|
||||
exchanges = process_mic_data(rows, operating_only=args.operating_only)
|
||||
|
||||
if not exchanges:
|
||||
logger.error("No exchanges found after filtering")
|
||||
sys.exit(1)
|
||||
|
||||
data = build_exchange_data(exchanges)
|
||||
|
||||
OUTPUT_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
with open(OUTPUT_PATH, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, indent=2, ensure_ascii=False)
|
||||
f.write("\n")
|
||||
|
||||
logger.info("\nWrote %s", OUTPUT_PATH)
|
||||
logger.info(" Total exchanges: %d", len(exchanges))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
103
openbb_platform/core/tests/provider/utils/test_exchange_utils.py
Normal file
103
openbb_platform/core/tests/provider/utils/test_exchange_utils.py
Normal file
@@ -0,0 +1,103 @@
|
||||
"""Tests for exchange_utils module."""
|
||||
|
||||
import pytest
|
||||
from openbb_core.provider.utils.exchange_utils import Exchange
|
||||
|
||||
|
||||
class TestExchange:
|
||||
"""Tests for the Exchange type."""
|
||||
|
||||
def test_from_mic(self):
|
||||
"""Test creating Exchange from MIC code."""
|
||||
e = Exchange("XNAS")
|
||||
assert str(e) == "XNAS"
|
||||
assert e.mic == "XNAS"
|
||||
assert e.acronym == "NASDAQ"
|
||||
assert e.name == "NASDAQ - ALL MARKETS"
|
||||
|
||||
def test_from_mic_lowercase(self):
|
||||
"""Test creating Exchange from lowercase MIC."""
|
||||
e = Exchange("xnys")
|
||||
assert str(e) == "XNYS"
|
||||
assert e.mic == "XNYS"
|
||||
|
||||
def test_from_acronym(self):
|
||||
"""Test creating Exchange from acronym."""
|
||||
e = Exchange("NASDAQ")
|
||||
assert str(e) == "XNAS"
|
||||
assert e.mic == "XNAS"
|
||||
assert e.acronym == "NASDAQ"
|
||||
|
||||
def test_from_acronym_lowercase(self):
|
||||
"""Test creating Exchange from lowercase acronym."""
|
||||
e = Exchange("nyse")
|
||||
assert str(e) == "XNYS"
|
||||
assert e.acronym == "NYSE"
|
||||
assert e.name == "NEW YORK STOCK EXCHANGE, INC."
|
||||
|
||||
def test_from_name(self):
|
||||
"""Test creating Exchange from full name."""
|
||||
e = Exchange("LONDON STOCK EXCHANGE")
|
||||
assert str(e) == "XLON"
|
||||
assert e.mic == "XLON"
|
||||
assert e.acronym == "LSE"
|
||||
|
||||
def test_from_partial_name(self):
|
||||
"""Test creating Exchange from acronym lookup."""
|
||||
e = Exchange("HKEX")
|
||||
assert str(e) == "XHKG"
|
||||
assert e.mic == "XHKG"
|
||||
assert e.acronym == "HKEX"
|
||||
|
||||
def test_properties(self):
|
||||
"""Test all Exchange properties."""
|
||||
e = Exchange("TSX")
|
||||
assert e.mic == "XTSE"
|
||||
assert e.acronym == "TSX"
|
||||
assert e.name == "TORONTO STOCK EXCHANGE"
|
||||
|
||||
def test_invalid_exchange(self):
|
||||
"""Test that invalid exchanges raise ValueError."""
|
||||
with pytest.raises(ValueError) as exc_info:
|
||||
Exchange("INVALID_EXCHANGE")
|
||||
assert "Invalid exchange" in str(exc_info.value)
|
||||
assert "XNAS" in str(exc_info.value)
|
||||
|
||||
def test_str_inheritance(self):
|
||||
"""Test that Exchange behaves as str."""
|
||||
e = Exchange("NASDAQ")
|
||||
assert isinstance(e, str)
|
||||
assert e == "XNAS"
|
||||
assert e.upper() == "XNAS"
|
||||
|
||||
def test_from_existing_exchange(self):
|
||||
"""Test creating Exchange from another Exchange instance."""
|
||||
e1 = Exchange("NYSE")
|
||||
e2 = Exchange(e1)
|
||||
assert str(e2) == "XNYS"
|
||||
assert e2.acronym == "NYSE"
|
||||
|
||||
def test_various_exchanges(self):
|
||||
"""Test various international exchanges."""
|
||||
test_cases = [
|
||||
("XETRA", "XETR"),
|
||||
("XSWX", "XSWX"),
|
||||
("ASX", "XASX"),
|
||||
("JPX", "XJPX"),
|
||||
("XNSE", "XNSE"),
|
||||
("B3", "BVMF"),
|
||||
]
|
||||
for input_val, expected_mic in test_cases:
|
||||
e = Exchange(input_val)
|
||||
assert e.mic == expected_mic, f"Failed for {input_val}"
|
||||
|
||||
def test_whitespace_handling(self):
|
||||
"""Test that whitespace is stripped from input."""
|
||||
e = Exchange(" NASDAQ ")
|
||||
assert str(e) == "XNAS"
|
||||
|
||||
def test_case_insensitivity(self):
|
||||
"""Test case insensitivity for all input types."""
|
||||
assert Exchange("XNAS").mic == Exchange("xnas").mic
|
||||
assert Exchange("NASDAQ").mic == Exchange("nasdaq").mic
|
||||
assert Exchange("Nasdaq").mic == Exchange("NASDAQ").mic
|
||||
@@ -11,6 +11,7 @@ from openbb_core.provider.standard_models.equity_screener import (
|
||||
)
|
||||
from openbb_core.provider.utils.country_utils import Country
|
||||
from openbb_core.provider.utils.errors import EmptyDataError
|
||||
from openbb_core.provider.utils.exchange_utils import Exchange
|
||||
from openbb_fmp.utils.definitions import (
|
||||
Countries,
|
||||
Exchanges,
|
||||
@@ -127,7 +128,11 @@ class FMPEquityScreenerQueryParams(EquityScreenerQueryParams):
|
||||
description="Filter by country. Accepts ISO 3166-1 alpha-2 codes (e.g., 'US', 'DE'), "
|
||||
"alpha-3 codes (e.g., 'USA'), or country names (e.g., 'United States', 'united_states').",
|
||||
)
|
||||
exchange: Exchanges | None = Field(default=None, description="Filter by exchange.")
|
||||
exchange: Exchange | None = Field(
|
||||
default=None,
|
||||
description="Filter by exchange. Accepts ISO 10383 MIC codes (e.g., 'XNAS', 'XNYS'), "
|
||||
"acronyms (e.g., 'NASDAQ', 'NYSE'), or exchange names (e.g., 'New York Stock Exchange').",
|
||||
)
|
||||
is_etf: bool | None = Field(
|
||||
default=None,
|
||||
description="If true, includes ETFs.",
|
||||
@@ -173,6 +178,22 @@ class FMPEquityScreenerQueryParams(EquityScreenerQueryParams):
|
||||
)
|
||||
return v
|
||||
|
||||
@field_validator("exchange", mode="after")
|
||||
@classmethod
|
||||
def _validate_exchange(cls, v):
|
||||
"""Validate exchange is supported by FMP."""
|
||||
if v is None:
|
||||
return v
|
||||
# Exchange stores MIC, FMP expects lowercase acronym
|
||||
exchange_code = v.acronym.lower()
|
||||
valid_exchanges = list(Exchanges.__args__)
|
||||
if exchange_code not in valid_exchanges:
|
||||
raise ValueError(
|
||||
f"Exchange '{v.name}' ({v.mic}) is not supported by FMP. "
|
||||
f"Valid options: {', '.join(sorted(valid_exchanges)[:20])}..."
|
||||
)
|
||||
return v
|
||||
|
||||
|
||||
class FMPEquityScreenerData(EquityScreenerData):
|
||||
"""FMP Equity Screener Data."""
|
||||
@@ -269,7 +290,7 @@ class FMPEquityScreenerFetcher(
|
||||
.replace("-", "%2D")
|
||||
.replace(",", "%2C")
|
||||
)
|
||||
exchange: str = query.exchange.upper() if query.exchange else ""
|
||||
exchange: str = query.exchange.acronym.upper() if query.exchange else ""
|
||||
country: str = query.country.upper() if query.country else ""
|
||||
query.is_active = True if query.is_active is None else query.is_active
|
||||
query.is_etf = False if query.is_etf is None else query.is_etf
|
||||
|
||||
Reference in New Issue
Block a user