mirror of
https://github.com/OpenBB-finance/OpenBB.git
synced 2026-11-09 10:51:57 +00:00
2e3d8ef6b2
* stash the work here * finish the job * also add most active * fix tmx etf countries * codespell * fix reference path in codespell * still didn't get the path right * linter * linter * unused-argument * fix polygon * fix tiingo * remove debug print * fix nasdaq * more lint * didn't save that file * fix bad url in error message * some review items --------- Co-authored-by: Theodore Aptekarev <aptekarev@gmail.com>
79 lines
2.4 KiB
Python
79 lines
2.4 KiB
Python
"""Nasdaq Helpers Module."""
|
|
|
|
from functools import lru_cache
|
|
|
|
|
|
def remove_html_tags(text: str):
|
|
"""Remove HTML tags from a string."""
|
|
# pylint: disable=import-outside-toplevel
|
|
import re
|
|
|
|
clean = re.compile("<.*?>")
|
|
return re.sub(clean, " ", text)
|
|
|
|
|
|
def get_random_agent() -> str:
|
|
"""Generate a random user agent for a request."""
|
|
# pylint: disable=import-outside-toplevel
|
|
from random_user_agent.user_agent import UserAgent
|
|
|
|
user_agent_rotator = UserAgent(limit=100)
|
|
user_agent = user_agent_rotator.get_random_user_agent()
|
|
return user_agent
|
|
|
|
|
|
def get_headers(accept_type: str = "json") -> dict:
|
|
"""Get the headers for the request."""
|
|
if accept_type not in ["json", "text"]:
|
|
raise ValueError("Invalid accept_type. Must be either 'json' or 'text'.")
|
|
|
|
return (
|
|
{
|
|
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
|
|
"Accept-Encoding": "gzip",
|
|
"Accept-Language": "en-CA,en-US;q=0.7,en;q=0.3",
|
|
"Host": "api.nasdaq.com",
|
|
"User-Agent": get_random_agent(),
|
|
"Connection": "keep-alive",
|
|
}
|
|
if accept_type == "text"
|
|
else {
|
|
"Accept": "application/json, text/plain, */*",
|
|
"Accept-Encoding": "gzip",
|
|
"Accept-Language": "en-CA,en-US;q=0.7,en;q=0.3",
|
|
"Host": "api.nasdaq.com",
|
|
"Origin": "https://www.nasdaq.com",
|
|
"Referer": "https://www.nasdaq.com/",
|
|
"User-Agent": get_random_agent(),
|
|
"Connection": "keep-alive",
|
|
}
|
|
)
|
|
|
|
|
|
def date_range(start_date, end_date):
|
|
"""Yield dates between start_date and end_date."""
|
|
# pylint: disable=import-outside-toplevel
|
|
from datetime import timedelta
|
|
|
|
for n in range(int((end_date - start_date).days) + 1):
|
|
yield start_date + timedelta(n)
|
|
|
|
|
|
@lru_cache(maxsize=1)
|
|
def get_nasdaq_directory() -> str:
|
|
"""Get the Nasdaq directory from the FTP site."""
|
|
# pylint: disable=import-outside-toplevel
|
|
from openbb_core.app.model.abstract.error import OpenBBError # noqa
|
|
from urllib.error import URLError
|
|
from urllib.request import urlopen
|
|
|
|
url = "ftp://ftp.nasdaqtrader.com/symboldirectory/nasdaqtraded.txt"
|
|
|
|
try:
|
|
with urlopen(url) as response: # noqa: S310
|
|
data = response.read().decode("utf-8")
|
|
except URLError as e:
|
|
raise OpenBBError(e) from e
|
|
|
|
return data
|