mirror of
https://github.com/OpenBB-finance/OpenBB.git
synced 2026-10-30 14:02:06 +00:00
97 lines
2.7 KiB
Python
97 lines
2.7 KiB
Python
"""Substack model"""
|
|
__docformat__ = "numpy"
|
|
|
|
import concurrent.futures
|
|
import logging
|
|
import textwrap
|
|
|
|
import pandas as pd
|
|
import requests
|
|
from bs4 import BeautifulSoup
|
|
from dateutil import parser
|
|
|
|
from gamestonk_terminal.decorators import log_start_end
|
|
from gamestonk_terminal.rich_config import console
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
@log_start_end(log=logger)
|
|
def scrape_substack(url: str) -> list:
|
|
"""Helper method to scrape newsletters from substack.
|
|
[Source: substack.com]
|
|
|
|
Parameters
|
|
----------
|
|
url: str
|
|
Url to newsletter on substack domain.
|
|
|
|
Returns
|
|
-------
|
|
list
|
|
list of news from given newsletter
|
|
"""
|
|
|
|
req = requests.get(url)
|
|
soup = BeautifulSoup(req.text, features="lxml")
|
|
results = []
|
|
posts = soup.find("div", class_="portable-archive-list").find_all(
|
|
"div", class_="post-preview portable-archive-post has-image has-author-line"
|
|
)
|
|
for post in posts:
|
|
title, url, time = (
|
|
post.a.text,
|
|
post.a["href"],
|
|
post.find("time").get("datetime"),
|
|
)
|
|
results.append([title, url, time])
|
|
return results
|
|
|
|
|
|
@log_start_end(log=logger)
|
|
def get_newsletters() -> pd.DataFrame:
|
|
"""Scrape all substack newsletters from url list.
|
|
[Source: substack.com]
|
|
|
|
Returns
|
|
-------
|
|
pd.DataFrame
|
|
DataFrame with recent news from most popular DeFi related newsletters.
|
|
"""
|
|
|
|
urls = [
|
|
"https://defiweekly.substack.com/archive",
|
|
"https://newsletter.thedefiant.io/archive",
|
|
"https://thedailygwei.substack.com/archive",
|
|
"https://todayindefi.substack.com/archive",
|
|
"https://newsletter.banklesshq.com/archive",
|
|
"https://defislate.substack.com/archive",
|
|
]
|
|
|
|
threads = len(urls)
|
|
newsletters = []
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=threads) as executor:
|
|
for newsletter in executor.map(scrape_substack, urls):
|
|
try:
|
|
newsletters.append(pd.DataFrame(newsletter))
|
|
except KeyError as e:
|
|
logger.exception(str(e))
|
|
console.print(e, "\n")
|
|
continue
|
|
|
|
df = pd.concat(newsletters, ignore_index=True)
|
|
df.columns = ["Title", "Link", "Date"]
|
|
|
|
df["Title"] = df["Title"].apply(lambda x: "".join(i for i in x if ord(i) < 128))
|
|
df["Date"] = df["Date"].apply(
|
|
lambda x: parser.parse(x).strftime("%Y-%m-%d %H:%M:%S")
|
|
)
|
|
df["Title"] = df["Title"].apply(
|
|
lambda x: "\n".join(textwrap.wrap(x, width=50)) if isinstance(x, str) else x
|
|
)
|
|
return (
|
|
df[["Title", "Date", "Link"]]
|
|
.sort_values(by="Date", ascending=False)
|
|
.reset_index(drop="index")
|
|
)
|