jerry/OpenBB
1
0
Fork 0
mirror of https://github.com/OpenBB-finance/OpenBB.git synced 2026-10-27 20:42:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

Use get_user_agent for all requests (#914)

Browse Source
This commit is contained in:
DidierRLopes
2021-11-05 01:17:20 +00:00
committed by GitHub
parent c75810aae8
commit 93379e7d6b
11 changed files with 26 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
gamestonk_terminal/cryptocurrency/pycoingecko_helpers.py
View File

@@ -12,6 +12,7 @@ import pandas as pd
from dateutil import parser
from requests.adapters import HTTPAdapter, RetryError
from urllib3.util.retry import Retry
from gamestonk_terminal.helper_funcs import get_user_agent
GECKO_BASE_URL = "https://www.coingecko.com"
@@ -84,7 +85,7 @@ def scrape_gecko_data(url: str) -> BeautifulSoup:
-------
BeautifulSoup object
"""
headers = {"User-Agent": "Mozilla/5.0"}
headers = {"User-Agent": get_user_agent()}
session = _retry_session("https://www.coingecko.com")
try:
req = session.get(url, headers=headers, timeout=5)

13
gamestonk_terminal/economy/wsj_model.py
View File

@@ -2,6 +2,7 @@
__docformat__ = "numpy"
import pandas as pd
import requests
from gamestonk_terminal.helper_funcs import get_user_agent
def us_indices() -> pd.DataFrame:
@@ -23,7 +24,7 @@ def us_indices() -> pd.DataFrame:
"name%22%3A%22CBOE%20Volatility%22%7D%2C%7B%22symbol%22%3A%22FUTURE%2FUS%2F%2FDJIA%20FUTURES%22%2C%22name%22%3A%"
"22DJIA%20Futures%22%7D%2C%7B%22symbol%22%3A%22FUTURE%2FUS%2F%2FS%26P%20500%20FUTURES%22%2C%22name%22%3A%22S%26P"
"%20500%20Futures%22%7D%5D%7D&type=mdc_quotes",
headers={"User-Agent": "Mozilla/5.0"},
headers={"User-Agent": get_user_agent()},
).json()
name, last_price, net_change, percent_change = [], [], [], []
@@ -59,7 +60,7 @@ def market_overview() -> pd.DataFrame:
"%22%3A%22FUTURE%2FUS%2F%2FGOLD%22%2C%22name%22%3A%22Gold%20Futures%22%7D%2C%7B%22symbol%22%3A%22CURRENCY%2FUS%2"
"F%2FUSDJPY%22%2C%22name%22%3A%22Yen%22%7D%2C%7B%22symbol%22%3A%22CURRENCY%2FUS%2F%2FEURUSD%22%2C%22name%22%3A%"
"22Euro%22%7D%5D%7D&type=mdc_quotes",
headers={"User-Agent": "Mozilla/5.0"},
headers={"User-Agent": get_user_agent()},
).json()
name, last_price, net_change, percent_change = [], [], [], []
@@ -95,7 +96,7 @@ def top_commodities() -> pd.DataFrame:
"2FUS%2F%2FCOPPER%22%2C%22name%22%3A%22Copper%20Futures%22%7D%2C%7B%22symbol%22%3A%22FUTURE%2FUS%2F%2FCORN%22%2"
"C%22name%22%3A%22Corn%20Futures%22%7D%2C%7B%22symbol%22%3A%22FUTURE%2FUS%2F%2FWHEAT%22%2C%22name%22%3A%22Wheat"
"%20Futures%22%7D%2C%7B%22symbol%22%3A%22INDEX%2FXX%2F%2FBCOM%22%7D%5D%7D&type=mdc_quotes",
headers={"User-Agent": "Mozilla/5.0"},
headers={"User-Agent": get_user_agent()},
).json()
name, last_price, net_change, percent_change = [], [], [], []
@@ -133,7 +134,7 @@ def us_bonds() -> pd.DataFrame:
"BOND%2FBX%2F%2FTMUBMUSD06M%22%2C%22name%22%3A%226-Month%20Bill%22%7D%2C%7B%22symbol%22%3A%22BON"
"D%2FBX%2F%2FTMUBMUSD03M%22%2C%22name%22%3A%223-Month%20Bill%22%7D%2C%7B%22symbol%22%3A%22BOND%"
"2FBX%2F%2FTMUBMUSD01M%22%2C%22name%22%3A%221-Month%20Bill%22%7D%5D%7D&type=mdc_quotes",
headers={"User-Agent": "Mozilla/5.0"},
headers={"User-Agent": get_user_agent()},
).json()
name, yield_pct, rate, yld_chng = [], [], [], []
@@ -165,7 +166,7 @@ def global_bonds() -> pd.DataFrame:
"22name%22%3A%22Japan%2010%20Year%22%7D%2C%7B%22symbol%22%3A%22TMBMKAU-10Y%22%2C%22name%22%3A%2"
"2Australia%2010%20Year%22%7D%2C%7B%22symbol%22%3A%22AMBMKRM-10Y%22%2C%22name%22%3A%22China%2010"
"%20Year%22%7D%5D%7D&type=mdc_governmentbonds",
headers={"User-Agent": "Mozilla/5.0"},
headers={"User-Agent": get_user_agent()},
).json()
name, yield_pct, rate, yld_chng = [], [], [], []
@@ -202,7 +203,7 @@ def global_currencies() -> pd.DataFrame:
"CURRENCY%2FUS%2F%2FBTCUSD%22%2C%22name%22%3A%22Bitcoin%20(BTC%2FUSD)%22%7D%2C%7B%22symbol%22%3A"
"%22INDEX%2FXX%2F%2FBUXX%22%2C%22name%22%3A%22WSJ%20Dollar%20Index%22%7D%2C%7B%22symbol%22%3A%2"
"2INDEX%2FUS%2F%2FDXY%22%2C%22name%22%3A%22U.S.%20Dollar%20Index%22%7D%5D%7D&type=mdc_quotes",
headers={"User-Agent": "Mozilla/5.0"},
headers={"User-Agent": get_user_agent()},
).json()
name, last_price, price_change, pct_change = [], [], [], []

7
gamestonk_terminal/etf/stockanalysis_model.py
View File

@@ -6,6 +6,7 @@ from typing import List, Tuple
import pandas as pd
import requests
from bs4 import BeautifulSoup as bs
from gamestonk_terminal.helper_funcs import get_user_agent
def get_all_names_symbols() -> Tuple[List[str], List[str]]:
@@ -19,7 +20,7 @@ def get_all_names_symbols() -> Tuple[List[str], List[str]]:
List of all available etf names
"""
r = requests.get(
"https://api.stockanalysis.com/etf/", headers={"User-Agent": "Mozilla/5.0"}
"https://api.stockanalysis.com/etf/", headers={"User-Agent": get_user_agent()}
)
soup = bs(r.text, "html.parser").findAll("ul", {"class": "no-spacing"})
all_links = soup[0].findAll("li")
@@ -47,7 +48,7 @@ def get_etf_overview(etf_symbol: str) -> pd.DataFrame:
"""
r = requests.get(
f"https://stockanalysis.com/etf/{etf_symbol}",
headers={"User-Agent": "Mozilla/5.0"},
headers={"User-Agent": get_user_agent()},
)
soup = bs(r.text, "html.parser") # %%
tables = soup.findAll("table")
@@ -80,7 +81,7 @@ def get_etf_holdings(symbol: str) -> pd.DataFrame:
"""
link = f"https://api.stockanalysis.com/etf/{symbol}/holdings/"
r = requests.get(link, headers={"User-Agent": "Mozilla/5.0"})
r = requests.get(link, headers={"User-Agent": get_user_agent()})
soup = bs(r.text, "html.parser")
soup = soup.find("table")
tds = soup.findAll("td")

3
gamestonk_terminal/etf/wsj_model.py
View File

@@ -2,6 +2,7 @@
__docformat__ = "numpy"
import pandas as pd
import requests
from gamestonk_terminal.helper_funcs import get_user_agent
def etf_movers(sort_type: str = "gainers") -> pd.DataFrame:
@@ -34,7 +35,7 @@ def etf_movers(sort_type: str = "gainers") -> pd.DataFrame:
"%22%3A%22WSJ%22%2C%22etfMover%22%3A%22most_active%22%2C%22count%22%3A25%7D&type=mdc_etfmovers"
)
data = requests.get(url, headers={"User-Agent": "Mozilla/5.0"}).json()
data = requests.get(url, headers={"User-Agent": get_user_agent()}).json()
name, last_price, net_change, percent_change, volume = [], [], [], [], []
for entry in data["data"]["instruments"]:

3
gamestonk_terminal/options/barchart_model.py
View File

@@ -4,6 +4,7 @@ __docformat__ = "numpy"
import pandas as pd
import requests
from bs4 import BeautifulSoup
from gamestonk_terminal.helper_funcs import get_user_agent
def get_options_info(ticker: str) -> pd.DataFrame:
@@ -21,7 +22,7 @@ def get_options_info(ticker: str) -> pd.DataFrame:
"""
page = f"https://www.barchart.com/stocks/quotes/{ticker}/overview"
r = requests.get(page, headers={"User-Agent": "Mozilla/5.0"})
r = requests.get(page, headers={"User-Agent": get_user_agent()})
soup = BeautifulSoup(r.text, "html.parser")
tags = soup.find(
"div",

3
gamestonk_terminal/options/chartexchange_model.py
View File

@@ -6,6 +6,7 @@ import pandas as pd
from bs4 import BeautifulSoup
from gamestonk_terminal.options.op_helpers import convert
from gamestonk_terminal.helper_funcs import get_user_agent
def get_option_history(ticker: str, date: str, call: bool, price: str) -> pd.DataFrame:
@@ -32,7 +33,7 @@ def get_option_history(ticker: str, date: str, call: bool, price: str) -> pd.Dat
)
url += f"{'c' if call else 'p'}{float(price):g}/historical/"
data = requests.get(url, headers={"User-Agent": "test"}).content
data = requests.get(url, headers={"User-Agent": get_user_agent()}).content
soup = BeautifulSoup(data, "html.parser")
table = soup.find("div", attrs={"style": "display: table; font-size: 0.9em; "})
rows = table.find_all("div", attrs={"style": "display: table-row;"})

3
gamestonk_terminal/options/fdscanner_model.py
View File

@@ -4,6 +4,7 @@ __docformat__ = "numpy"
import numpy as np
import pandas as pd
import requests
from gamestonk_terminal.helper_funcs import get_user_agent
def unusual_options(num: int, sort_column: str, ascending: bool):
@@ -31,7 +32,7 @@ def unusual_options(num: int, sort_column: str, ascending: bool):
r = requests.get(
f"https://app.fdscanner.com/api2/unusualvolume?p=0&page_size=20&page={int(page_num)}",
headers={"User-Agent": "Mozilla/5.0"},
headers={"User-Agent": get_user_agent()},
)
if r.status_code != 200:

2
gamestonk_terminal/portfolio/brokers/degiro/degiro_model.py
View File

@@ -5,7 +5,7 @@ from typing import Union
# IMPORTATION THIRDPARTY
import pandas as pd
import degiro_connector.core.helpers.pb_handler as pb_handler
from degiro_connector.core.helpers import pb_handler
from degiro_connector.trading.api import API as TradingAPI
from degiro_connector.trading.models.trading_pb2 import (

2
gamestonk_terminal/portfolio/brokers/degiro/degiro_view.py
View File

@@ -1,7 +1,7 @@
# IMPORTATION THIRDPARTY
from argparse import Namespace
import pandas as pd
import degiro_connector.core.helpers.pb_handler as pb_handler
from degiro_connector.core.helpers import pb_handler
from degiro_connector.trading.models.trading_pb2 import (
Credentials,

4
gamestonk_terminal/stocks/discovery/shortinterest_model.py
View File

@@ -68,9 +68,7 @@ def get_today_hot_penny_stocks() -> DataFrame:
text_soup_penny_stock_stocks = BeautifulSoup(
requests.get(
url_penny_stock_stocks,
headers={
"User-Agent": "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.10; rv:86.1) Gecko/20100101 Firefox/86.1"
},
headers={"User-Agent": get_user_agent()},
).text,
"lxml",
)

8
gamestonk_terminal/stocks/fundamental_analysis/dcf_model.py
View File

@@ -13,6 +13,7 @@ import financedatabase as fd
import yfinance as yf
import pandas as pd
from gamestonk_terminal.helper_funcs import get_user_agent
opts = Union[int, str, float]
@@ -440,12 +441,7 @@ red = Font(color="FF0000")
fmt_acct = "_($* #,##0.00_);[Red]_($* (#,##0.00);_($* -_0_0_);_(@"
headers = {
"User-Agent": (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36"
)
}
headers = {"User-Agent": get_user_agent()}
tickers = [
"AEIS",