mirror of
https://github.com/OpenBB-finance/OpenBB.git
synced 2026-10-29 13:32:08 +00:00
59d8b36bb0
* Fixes * Update stocks_helper.py * update git-actions set-output to new format * Update stocks_helper.py * Update terminal_helper.py * removed LineAnnotateDrawer from qa_view * lint * few changes * updates * sdk auto gen modules done * Update stocks_helper.py * updates to changed imports, and remove first sdk_modules * Update generate_sdk.py * Update generate_sdk.py * pylint * revert stocks_helper * Update generate_sdk.py * Update sdk.py * Update generate_sdk.py * full auto generation, added sdk.py/controllers creation * missed enable forecasting * added running black in subprocess after sdk files generation completes * removed deleted sdk_arg_logger * comment out tests * property doc fix * clean up * Update generate_sdk.py * make trailmap classes useable for doc generation * Update generate_sdk.py * added lineon to trailmap class for linking to func in markdown * changed lineon to dict * added full_path to trailmap for linking in docs * updated portfolio * feat: initial files * feat: added meta head * feat: added funcdef * added func_def to trailmap attributes for markdown in docs, added missing type hints to covid functions * feat: added view and merged with jaun * Update generate_sdk.py * Update generate_sdk.py * Update generate_sdk.py * Update generate_sdk.py * init * fix returns * fix: random stuff * fix: random * fixed encoding issue on windows * fix: generate tabs * update * Update generate_sdk_markdown.py * Create .pydocstyle.ini * added type hint classes for views * fixes * alt, ba * alt-economy * Update finviz_compare_model.py * fixs * Update substack_model.py * Update generate_sdk.py * last of my section * porfolio * po * Update optimizer_model.py * fixing more things * few more * keys done * update * fixes * Update generate_sdk_markdown.py * Update generate_sdk_markdown.py * mypy forecast fix * Update generate_sdk_markdown.py * Update generate_sdk_markdown.py * Update generate_sdk_markdown.py * fixes * forecast fixes * one more fix * Update coinbase_model.py * Update generate_sdk_markdown.py Co-authored-by: Colin Delahunty <72827203+colin99d@users.noreply.github.com> Co-authored-by: James Maslek <jmaslek11@gmail.com> Co-authored-by: jose-donato <zmcdonato@gmail.com> Co-authored-by: andrewkenreich <andrew.kenreich@gmail.com>
99 lines
2.8 KiB
Python
99 lines
2.8 KiB
Python
"""Substack model"""
|
|
__docformat__ = "numpy"
|
|
|
|
import concurrent.futures
|
|
import logging
|
|
import textwrap
|
|
from typing import List
|
|
|
|
import pandas as pd
|
|
import requests
|
|
from bs4 import BeautifulSoup
|
|
from dateutil import parser
|
|
|
|
from openbb_terminal.decorators import log_start_end
|
|
from openbb_terminal.rich_config import console
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
@log_start_end(log=logger)
|
|
def scrape_substack(url: str) -> List[List[str]]:
|
|
"""Helper method to scrape newsletters from substack.
|
|
[Source: substack.com]
|
|
|
|
Parameters
|
|
----------
|
|
url: str
|
|
Url to newsletter on substack domain.
|
|
|
|
Returns
|
|
-------
|
|
List[List[str]]
|
|
List of lists containing:
|
|
- title of newsletter
|
|
- url to newsletter
|
|
- str datetime of newsletter [Format: "%Y-%m-%d %H:%M:%S"]
|
|
"""
|
|
|
|
req = requests.get(url)
|
|
soup = BeautifulSoup(req.text, features="lxml")
|
|
results: List[List[str]] = []
|
|
posts = soup.find("div", class_="portable-archive-list").find_all(
|
|
"div", class_="post-preview portable-archive-post has-image has-author-line"
|
|
)
|
|
for post in posts:
|
|
title: str = post.a.text
|
|
post_url: str = post.a["href"]
|
|
time: str = post.find("time").get("datetime")
|
|
results.append([title, post_url, time])
|
|
return results
|
|
|
|
|
|
@log_start_end(log=logger)
|
|
def get_newsletters() -> pd.DataFrame:
|
|
"""Scrape all substack newsletters from url list.
|
|
[Source: substack.com]
|
|
|
|
Returns
|
|
-------
|
|
pd.DataFrame
|
|
DataFrame with recent news from most popular DeFi related newsletters.
|
|
"""
|
|
|
|
urls = [
|
|
"https://defiweekly.substack.com/archive",
|
|
"https://newsletter.thedefiant.io/archive",
|
|
"https://thedailygwei.substack.com/archive",
|
|
"https://todayindefi.substack.com/archive",
|
|
"https://newsletter.banklesshq.com/archive",
|
|
"https://defislate.substack.com/archive",
|
|
]
|
|
|
|
threads = len(urls)
|
|
newsletters = []
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=threads) as executor:
|
|
for newsletter in executor.map(scrape_substack, urls):
|
|
try:
|
|
newsletters.append(pd.DataFrame(newsletter))
|
|
except KeyError as e:
|
|
logger.exception(str(e))
|
|
console.print(e, "\n")
|
|
continue
|
|
|
|
df = pd.concat(newsletters, ignore_index=True)
|
|
df.columns = ["Title", "Link", "Date"]
|
|
|
|
df["Title"] = df["Title"].apply(lambda x: "".join(i for i in x if ord(i) < 128))
|
|
df["Date"] = df["Date"].apply(
|
|
lambda x: parser.parse(x).strftime("%Y-%m-%d %H:%M:%S")
|
|
)
|
|
df["Title"] = df["Title"].apply(
|
|
lambda x: "\n".join(textwrap.wrap(x, width=50)) if isinstance(x, str) else x
|
|
)
|
|
return (
|
|
df[["Title", "Date", "Link"]]
|
|
.sort_values(by="Date", ascending=False)
|
|
.reset_index(drop="index")
|
|
)
|