feat(platform): standardize exchange inputs with ISO 10383 MIC codes (#7351)

* feat(platform): standardize exchange inputs with ISO 10383 MIC codes

Add Exchange type in provider utils that standardizes exchange inputs across
providers using ISO 10383 Market Identifier Codes (MIC).

Similar to the Country type from PR #7333, this allows providers to accept:
- ISO 10383 MIC codes (e.g., 'XNAS', 'XNYS')
- Exchange acronyms (e.g., 'NASDAQ', 'NYSE')
- Full exchange names (e.g., 'New York Stock Exchange')
- lower_snake_case names (e.g., 'new_york_stock_exchange')

The Exchange class inherits from str (storing the MIC code) while providing
access to mic, acronym, name, country, and city properties.

Example usage in FMP equity screener demonstrates provider integration pattern:
- exchange: Exchange | None with validator for provider-specific validation
- Maps Exchange.acronym to FMP's expected lowercase exchange codes

Addresses part of #6969

* refactor(platform): use ExchangeData TypedDict for proper typing

- Add ExchangeData TypedDict with explicit field types (consistent with CountryData)
- Update function signatures and class attributes to use ExchangeData
- Add _last_updated metadata to exchange_data.json
- Add Wikipedia reference URL to docstring

Aligns Exchange implementation with Country pattern from PR #7352.

* fix: add missing major exchanges including Cboe (XCBO, XCBT)

* refactor: expand to full ISO 10383 MIC list (2,272 active MICs)

Address review feedback to include all active MICs, not just curated majors.

- Expand exchange_data.json from 66 to 2,272 active MICs (operating + segments)
- Remove country and city fields (out of scope for this PR)
- Add scripts/update_exchange_data.py to regenerate from official ISO 20022
  CSV source using stdlib only (no external dependencies)
- Use first-write-wins lookup with operating MICs ordered before segments
  to resolve acronym collisions (e.g. LSE -> XLON, not XPVT)
- Update tests to reflect simplified data model

* feat: add city, country, and website fields to exchange data

* refactor: move update_exchange_data.py to provider utils

* style: fix black formatting in update_exchange_data.py

---------

Co-authored-by: Danglewood <85772166+deeleeramone@users.noreply.github.com>
This commit is contained in:
John Seong
2026-02-25 11:17:02 -08:00
committed by GitHub
parent e6a3879e98
commit 616c54d79e
5 changed files with 18650 additions and 2 deletions

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,205 @@
"""Utilities for standardizing exchange inputs across providers using ISO 10383 MIC codes.
This module provides an Exchange type that inherits from str for type checker compatibility
while providing full access to ISO 10383 Market Identifier Code (MIC) data via a static dataset.
Providers can access mic, acronym, and name properties as needed.
References:
- ISO 10383: https://www.iso20022.org/market-identifier-codes
- MIC Registry (CSV): https://www.iso20022.org/sites/default/files/ISO10383_MIC/ISO10383_MIC.csv
- Wikipedia: https://en.wikipedia.org/wiki/Market_Identifier_Code
"""
import json
from pathlib import Path
from typing import Any, TypedDict
from pydantic_core import core_schema
class ExchangeData(TypedDict, total=False):
"""Type definition for exchange data dictionary."""
mic: str
acronym: str
name: str
def _load_exchange_data() -> dict[str, ExchangeData]:
"""Load exchange data from JSON and build lookup indices.
Returns a dict with lookup keys (mic, acronym, name variants) mapping to exchange data.
"""
data_path = Path(__file__).parent / "exchange_data.json"
with open(data_path, encoding="utf-8") as f:
data = json.load(f)
lookup: dict[str, ExchangeData] = {}
for exchange in data["exchanges"]:
# Index by MIC (case-insensitive)
lookup[exchange["mic"].upper()] = exchange
lookup[exchange["mic"].lower()] = exchange
# Index by acronym (case-insensitive, first-write-wins to preserve
# operating MIC priority over segment MICs sharing the same acronym)
acronym = exchange["acronym"]
acronym_upper = acronym.upper()
acronym_lower = acronym.lower()
if acronym_upper not in lookup:
lookup[acronym_upper] = exchange
if acronym_lower not in lookup:
lookup[acronym_lower] = exchange
# Index by name (case-insensitive)
name_lower = exchange["name"].lower()
if name_lower not in lookup:
lookup[name_lower] = exchange
if exchange["name"] not in lookup:
lookup[exchange["name"]] = exchange
# Index by snake_case name
snake_name = (
name_lower.replace(" ", "_")
.replace("-", "_")
.replace(",", "")
.replace("'", "")
)
if snake_name not in lookup:
lookup[snake_name] = exchange
return lookup
# Load once at module import
_EXCHANGE_LOOKUP = _load_exchange_data()
class Exchange(str):
"""Exchange string type with full ISO 10383 MIC data access.
Inherits from str (storing MIC code) for type checker compatibility
while providing access to mic, acronym, and name properties.
Accepts:
- ISO 10383 MIC codes (e.g., "XNAS", "XNYS")
- Exchange acronyms (e.g., "NASDAQ", "NYSE")
- Full exchange names (e.g., "New York Stock Exchange")
- lower_snake_case names (e.g., "new_york_stock_exchange")
Examples
--------
>>> e = Exchange("nasdaq")
>>> str(e)
'XNAS'
>>> e.mic
'XNAS'
>>> e.acronym
'NASDAQ'
>>> e.name
'NASDAQ - ALL MARKETS'
"""
_exchange_data: ExchangeData
def __new__(cls, value: Any) -> "Exchange":
"""Create a new Exchange instance.
Parameters
----------
value : Any
Exchange input (MIC, acronym, name, or lower_snake_case).
Returns
-------
Exchange
An Exchange instance storing the MIC code as its string value.
Raises
------
ValueError
If the exchange cannot be resolved.
"""
exchange_data = (
value._exchange_data
if isinstance(value, Exchange)
else cls._lookup_exchange(value)
)
# Create str instance with the MIC code
instance = super().__new__(cls, exchange_data["mic"])
# Store the exchange data dict
instance._exchange_data = exchange_data
return instance
@staticmethod
def _lookup_exchange(value: Any) -> ExchangeData:
"""Look up an exchange from various input formats.
Parameters
----------
value : Any
Exchange input to look up.
Returns
-------
ExchangeData
The exchange data dictionary with mic, acronym, name.
Raises
------
ValueError
If the exchange cannot be found.
"""
val = str(value).strip()
# Convert lower_snake_case to lookup key
if "_" in val:
val = val.replace("_", " ")
# Try direct lookup
lookup_key = val.lower()
if lookup_key in _EXCHANGE_LOOKUP:
return _EXCHANGE_LOOKUP[lookup_key]
# Try uppercase (common for MICs)
if val.upper() in _EXCHANGE_LOOKUP:
return _EXCHANGE_LOOKUP[val.upper()]
# Try original case
if val in _EXCHANGE_LOOKUP:
return _EXCHANGE_LOOKUP[val]
raise ValueError(
f"Invalid exchange: '{value}'. "
"Accepts ISO 10383 MIC codes (e.g., 'XNAS', 'XNYS'), "
"acronyms (e.g., 'NASDAQ', 'NYSE'), "
"or exchange names (e.g., 'New York Stock Exchange')."
)
@property
def mic(self) -> str:
"""ISO 10383 Market Identifier Code (e.g., 'XNAS')."""
return self._exchange_data["mic"]
@property
def acronym(self) -> str:
"""Exchange acronym/short name (e.g., 'NASDAQ')."""
return self._exchange_data["acronym"]
@property
def name(self) -> str:
"""Full exchange name (e.g., 'NASDAQ - ALL MARKETS')."""
return self._exchange_data["name"]
@classmethod
def __get_pydantic_core_schema__(cls, _source_type: Any, _handler: Any) -> Any:
"""Return the Pydantic core schema for validation."""
return core_schema.no_info_after_validator_function(
cls,
core_schema.str_schema(),
)
# Backwards-compatible alias
ExchangeParam = Exchange

View File

@@ -0,0 +1,158 @@
#!/usr/bin/env python3
"""
Update exchange data from ISO 10383 (Market Identifier Codes).
Downloads the official ISO 10383 MIC registry and regenerates exchange_data.json.
Uses only Python standard library (no external dependencies).
Source:
https://www.iso20022.org/market-identifier-codes
Usage:
python openbb_platform/core/openbb_core/provider/utils/update_exchange_data.py
python openbb_platform/core/openbb_core/provider/utils/update_exchange_data.py --operating-only
"""
import argparse
import csv
import io
import json
import logging
import sys
import urllib.request
from datetime import date
from pathlib import Path
logger = logging.getLogger(__name__)
SCRIPT_DIR = Path(__file__).parent
OUTPUT_PATH = SCRIPT_DIR / "exchange_data.json"
MIC_CSV_URL = (
"https://www.iso20022.org/sites/default/files/ISO10383_MIC/ISO10383_MIC.csv"
)
def download_mic_csv() -> list[dict]:
"""Download and parse the official ISO 10383 MIC CSV."""
logger.info("Downloading MIC data from %s...", MIC_CSV_URL)
req = urllib.request.Request( # noqa: S310
MIC_CSV_URL, headers={"User-Agent": "OpenBB/1.0"}
)
with urllib.request.urlopen(req, timeout=30) as resp: # noqa: S310
raw = resp.read().decode("utf-8-sig")
reader = csv.DictReader(io.StringIO(raw))
rows = list(reader)
logger.info("Downloaded %d total MIC entries", len(rows))
return rows
def process_mic_data(rows: list[dict], operating_only: bool = False) -> list[dict]:
"""Filter and transform MIC rows into exchange entries."""
# Normalize header keys (strip whitespace, uppercase)
active = [r for r in rows if r.get("STATUS", "").strip().upper() == "ACTIVE"]
logger.info("Active entries: %d", len(active))
if operating_only:
active = [
r for r in active if r.get("OPRT/SGMT", "").strip().upper() in ("OPRT", "O")
]
logger.info("Operating MICs only: %d", len(active))
exchanges = []
for row in active:
mic = row.get("MIC", "").strip()
if not mic:
continue
name = row.get("MARKET NAME-INSTITUTION DESCRIPTION", "").strip()
if not name:
continue
acronym = row.get("ACRONYM", "").strip() or mic
entry = {
"mic": mic,
"acronym": acronym,
"name": name,
}
city = row.get("CITY", "").strip()
if city:
entry["city"] = city.title()
country_code = row.get("ISO COUNTRY CODE (ISO 3166)", "").strip().upper()
if country_code:
entry["country"] = country_code
website = row.get("WEBSITE", "").strip()
if website:
# Normalize: ensure lowercase, add https:// if missing scheme
website = website.lower()
if website and not website.startswith(("http://", "https://")):
website = f"https://{website}"
entry["website"] = website
entry["_type"] = row.get("OPRT/SGMT", "").strip().upper()
exchanges.append(entry)
# Sort operating MICs before segments so that the lookup in exchange_utils
# (first-write-wins) gives priority to operating MICs when acronyms collide.
exchanges.sort(key=lambda x: (0 if x["_type"] in ("OPRT", "O") else 1, x["mic"]))
# Strip internal sort key before output
for e in exchanges:
del e["_type"]
return exchanges
def build_exchange_data(exchanges: list[dict]) -> dict:
"""Build the final exchange data structure."""
return {
"_last_updated": date.today().isoformat(),
"_source": {
"url": "https://www.iso20022.org/market-identifier-codes",
"standard": "ISO 10383",
"maintainer": "SWIFT (ISO 20022 Registration Authority)",
},
"_stats": {
"total_exchanges": len(exchanges),
},
"exchanges": exchanges,
}
def main():
"""Download ISO 10383 MIC registry and regenerate exchange_data.json."""
logging.basicConfig(level=logging.INFO, format="%(message)s")
parser = argparse.ArgumentParser(description="Update exchange data from ISO 10383")
parser.add_argument(
"--operating-only",
action="store_true",
help="Exclude segment MICs (only include operating MICs)",
)
args = parser.parse_args()
rows = download_mic_csv()
exchanges = process_mic_data(rows, operating_only=args.operating_only)
if not exchanges:
logger.error("No exchanges found after filtering")
sys.exit(1)
data = build_exchange_data(exchanges)
OUTPUT_PATH.parent.mkdir(parents=True, exist_ok=True)
with open(OUTPUT_PATH, "w", encoding="utf-8") as f:
json.dump(data, f, indent=2, ensure_ascii=False)
f.write("\n")
logger.info("\nWrote %s", OUTPUT_PATH)
logger.info(" Total exchanges: %d", len(exchanges))
if __name__ == "__main__":
main()

View File

@@ -0,0 +1,103 @@
"""Tests for exchange_utils module."""
import pytest
from openbb_core.provider.utils.exchange_utils import Exchange
class TestExchange:
"""Tests for the Exchange type."""
def test_from_mic(self):
"""Test creating Exchange from MIC code."""
e = Exchange("XNAS")
assert str(e) == "XNAS"
assert e.mic == "XNAS"
assert e.acronym == "NASDAQ"
assert e.name == "NASDAQ - ALL MARKETS"
def test_from_mic_lowercase(self):
"""Test creating Exchange from lowercase MIC."""
e = Exchange("xnys")
assert str(e) == "XNYS"
assert e.mic == "XNYS"
def test_from_acronym(self):
"""Test creating Exchange from acronym."""
e = Exchange("NASDAQ")
assert str(e) == "XNAS"
assert e.mic == "XNAS"
assert e.acronym == "NASDAQ"
def test_from_acronym_lowercase(self):
"""Test creating Exchange from lowercase acronym."""
e = Exchange("nyse")
assert str(e) == "XNYS"
assert e.acronym == "NYSE"
assert e.name == "NEW YORK STOCK EXCHANGE, INC."
def test_from_name(self):
"""Test creating Exchange from full name."""
e = Exchange("LONDON STOCK EXCHANGE")
assert str(e) == "XLON"
assert e.mic == "XLON"
assert e.acronym == "LSE"
def test_from_partial_name(self):
"""Test creating Exchange from acronym lookup."""
e = Exchange("HKEX")
assert str(e) == "XHKG"
assert e.mic == "XHKG"
assert e.acronym == "HKEX"
def test_properties(self):
"""Test all Exchange properties."""
e = Exchange("TSX")
assert e.mic == "XTSE"
assert e.acronym == "TSX"
assert e.name == "TORONTO STOCK EXCHANGE"
def test_invalid_exchange(self):
"""Test that invalid exchanges raise ValueError."""
with pytest.raises(ValueError) as exc_info:
Exchange("INVALID_EXCHANGE")
assert "Invalid exchange" in str(exc_info.value)
assert "XNAS" in str(exc_info.value)
def test_str_inheritance(self):
"""Test that Exchange behaves as str."""
e = Exchange("NASDAQ")
assert isinstance(e, str)
assert e == "XNAS"
assert e.upper() == "XNAS"
def test_from_existing_exchange(self):
"""Test creating Exchange from another Exchange instance."""
e1 = Exchange("NYSE")
e2 = Exchange(e1)
assert str(e2) == "XNYS"
assert e2.acronym == "NYSE"
def test_various_exchanges(self):
"""Test various international exchanges."""
test_cases = [
("XETRA", "XETR"),
("XSWX", "XSWX"),
("ASX", "XASX"),
("JPX", "XJPX"),
("XNSE", "XNSE"),
("B3", "BVMF"),
]
for input_val, expected_mic in test_cases:
e = Exchange(input_val)
assert e.mic == expected_mic, f"Failed for {input_val}"
def test_whitespace_handling(self):
"""Test that whitespace is stripped from input."""
e = Exchange(" NASDAQ ")
assert str(e) == "XNAS"
def test_case_insensitivity(self):
"""Test case insensitivity for all input types."""
assert Exchange("XNAS").mic == Exchange("xnas").mic
assert Exchange("NASDAQ").mic == Exchange("nasdaq").mic
assert Exchange("Nasdaq").mic == Exchange("NASDAQ").mic

View File

@@ -11,6 +11,7 @@ from openbb_core.provider.standard_models.equity_screener import (
)
from openbb_core.provider.utils.country_utils import Country
from openbb_core.provider.utils.errors import EmptyDataError
from openbb_core.provider.utils.exchange_utils import Exchange
from openbb_fmp.utils.definitions import (
Countries,
Exchanges,
@@ -127,7 +128,11 @@ class FMPEquityScreenerQueryParams(EquityScreenerQueryParams):
description="Filter by country. Accepts ISO 3166-1 alpha-2 codes (e.g., 'US', 'DE'), "
"alpha-3 codes (e.g., 'USA'), or country names (e.g., 'United States', 'united_states').",
)
exchange: Exchanges | None = Field(default=None, description="Filter by exchange.")
exchange: Exchange | None = Field(
default=None,
description="Filter by exchange. Accepts ISO 10383 MIC codes (e.g., 'XNAS', 'XNYS'), "
"acronyms (e.g., 'NASDAQ', 'NYSE'), or exchange names (e.g., 'New York Stock Exchange').",
)
is_etf: bool | None = Field(
default=None,
description="If true, includes ETFs.",
@@ -173,6 +178,22 @@ class FMPEquityScreenerQueryParams(EquityScreenerQueryParams):
)
return v
@field_validator("exchange", mode="after")
@classmethod
def _validate_exchange(cls, v):
"""Validate exchange is supported by FMP."""
if v is None:
return v
# Exchange stores MIC, FMP expects lowercase acronym
exchange_code = v.acronym.lower()
valid_exchanges = list(Exchanges.__args__)
if exchange_code not in valid_exchanges:
raise ValueError(
f"Exchange '{v.name}' ({v.mic}) is not supported by FMP. "
f"Valid options: {', '.join(sorted(valid_exchanges)[:20])}..."
)
return v
class FMPEquityScreenerData(EquityScreenerData):
"""FMP Equity Screener Data."""
@@ -269,7 +290,7 @@ class FMPEquityScreenerFetcher(
.replace("-", "%2D")
.replace(",", "%2C")
)
exchange: str = query.exchange.upper() if query.exchange else ""
exchange: str = query.exchange.acronym.upper() if query.exchange else ""
country: str = query.country.upper() if query.country else ""
query.is_active = True if query.is_active is None else query.is_active
query.is_etf = False if query.is_etf is None else query.is_etf