jerry/OpenBB
1
0
Fork 0
mirror of https://github.com/OpenBB-finance/OpenBB.git synced 2026-10-27 12:40:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fill missing static data (sector, industry, country and region) on portfolio creation (#2072)

Browse Source 
* pull sector industry country data from yfinance

* fix imports with missing info

* run black on helper

* fix tests

* fix tests

* fix linting

* autofill stock data and display progress bar

* redirect stock data file

* substitute only stocks not provided by user

* reset benchmark when new portfolio is loaded (bug fix)

* rewrite help, will be fixed in previous pr

* fix mypy

* fix linting

* fix linting with open

* format progress bar

* run black

* avoid clearing of all fields

* reformat error message

* improve returns calculation performance

* general linting

Co-authored-by: Jeroen Bouma <jer.bouma@gmail.com>
Co-authored-by: Theodore Aptekarev <aptekarev@gmail.com>
This commit is contained in:
montezdesousa
2022-07-18 18:47:45 +01:00
committed by GitHub
parent 82b6411cab
commit ec9a88e8fa
3 changed files with 281 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
openbb_terminal/portfolio/portfolio_controller.py
View File

@@ -319,6 +319,7 @@ class PortfolioController(BaseController):
str(file_location)
)
self.portfolio = portfolio_model.PortfolioModel(orderbook)
self.benchmark_name = ""
if ns_parser.name:
self.portfolio_name = ns_parser.name

183
openbb_terminal/portfolio/portfolio_helper.py
View File

@@ -2,10 +2,15 @@
__docformat__ = "numpy"
from datetime import datetime
import os
from pathlib import Path
import csv
from dateutil.relativedelta import relativedelta
import yfinance as yf
import pandas as pd
from openbb_terminal.rich_config import console
# pylint: disable=too-many-return-statements
BENCHMARK_LIST = {
@@ -113,6 +118,118 @@ BENCHMARK_LIST = {
PERIODS = ["mtd", "qtd", "ytd", "3m", "6m", "1y", "3y", "5y", "10y", "all"]
REGIONS = {
"Afghanistan": "Middle East",
"Anguilla": "North America",
"Argentina": "Latin America",
"Australia": "Asia",
"Austria": "Europe",
"Azerbaijan": "Europe",
"Bahamas": "North America",
"Bangladesh": "Asia",
"Barbados": "North America",
"Belgium": "Europe",
"Belize": "North America",
"Bermuda": "North America",
"Botswana": "Africa",
"Brazil": "Latin America",
"British Virgin Islands": "North America",
"Cambodia": "Asia",
"Canada": "North America",
"Cayman Islands": "North America",
"Chile": "Latin America",
"China": "Asia",
"Colombia": "Latin America",
"Costa Rica": "North America",
"Cyprus": "Europe",
"Czech Republic": "Europe",
"Denmark": "Europe",
"Dominican Republic": "North America",
"Egypt": "Middle East",
"Estonia": "Europe",
"Falkland Islands": "Latin America",
"Finland": "Europe",
"France": "Europe",
"French Guiana": "Europe",
"Gabon": "Africa",
"Georgia": "Europe",
"Germany": "Europe",
"Ghana": "Africa",
"Gibraltar": "Europe",
"Greece": "Europe",
"Greenland": "North America",
"Guernsey": "Europe",
"Hong Kong": "Asia",
"Hungary": "Europe",
"Iceland": "Europe",
"India": "Asia",
"Indonesia": "Asia",
"Ireland": "Europe",
"Isle of Man": "Europe",
"Israel": "Middle East",
"Italy": "Europe",
"Ivory Coast": "Africa",
"Japan": "Asia",
"Jersey": "Europe",
"Jordan": "Middle East",
"Kazakhstan": "Asia",
"Kyrgyzstan": "Asia",
"Latvia": "Europe",
"Liechtenstein": "Europe",
"Lithuania": "Europe",
"Luxembourg": "Europe",
"Macau": "Asia",
"Macedonia": "Europe",
"Malaysia": "Asia",
"Malta": "Europe",
"Mauritius": "Africa",
"Mexico": "Latin America",
"Monaco": "Europe",
"Mongolia": "Asia",
"Montenegro": "Europe",
"Morocco": "Africa",
"Mozambique": "Africa",
"Myanmar": "Asia",
"Namibia": "Africa",
"Netherlands": "Europe",
"Netherlands Antilles": "Europe",
"New Zealand": "Asia",
"Nigeria": "Africa",
"Norway": "Europe",
"Panama": "North America",
"Papua New Guinea": "Asia",
"Peru": "Latin America",
"Philippines": "Asia",
"Poland": "Europe",
"Portugal": "Europe",
"Qatar": "Middle East",
"Reunion": "Africa",
"Romania": "Europe",
"Russia": "Asia",
"Saudi Arabia": "Middle East",
"Senegal": "Africa",
"Singapore": "Asia",
"Slovakia": "Europe",
"Slovenia": "Europe",
"South Africa": "Africa",
"South Korea": "Asia",
"Spain": "Europe",
"Suriname": "Latin America",
"Sweden": "Europe",
"Switzerland": "Europe",
"Taiwan": "Asia",
"Tanzania": "Africa",
"Thailand": "Asia",
"Turkey": "Middle East",
"Ukraine": "Europe",
"United Arab Emirates": "Middle East",
"United Kingdom": "Europe",
"United States": "North America",
"Uruguay": "Latin America",
"Vietnam": "Asia",
"Zambia": "Africa",
}
now = datetime.now()
PERIODS_DAYS = {
"mtd": (now - datetime(now.year, now.month, 1)).days,
@@ -267,3 +384,69 @@ def make_equal_length(df1: pd.DataFrame, df2: pd.DataFrame):
df2 = df2.loc[df1.index]
return df1, df2
def get_region_from_country(country: str) -> str:
return REGIONS[country]
def get_info_update_file(ticker: str, file_path: Path, writemode: str) -> list:
# Pull ticker info from yf
yf_ticker_info = yf.Ticker(ticker).info
if "sector" in yf_ticker_info.keys():
# Ticker has valid sector
# Replace the dash to UTF-8 readable
ticker_info_list = [
yf_ticker_info["sector"],
yf_ticker_info["industry"].replace("—", "-"),
yf_ticker_info["country"],
get_region_from_country(yf_ticker_info["country"]),
]
with open(file_path, writemode, newline="") as f:
writer = csv.writer(f)
if writemode == "a":
# file already has data, so just append
writer.writerow([ticker] + ticker_info_list)
else:
# file did not exist or as empty, so write headers first
writer.writerow(["Ticker", "Sector", "Industry", "Country", "Region"])
writer.writerow([ticker] + ticker_info_list)
f.close()
return ticker_info_list
# Ticker does not have a valid sector
console.print(f"F:{ticker}", end="")
return ["", "", "", ""]
def get_info_from_ticker(ticker: str) -> list:
filename = "tickers_info.csv"
# Direct file to openbb_terminal\portfolio
DEFAULT_HOLDINGS_PATH = os.path.abspath(
os.path.join(
os.path.dirname(__file__), "..", "..", "openbb_terminal", "portfolio"
)
)
file_path = Path(str(DEFAULT_HOLDINGS_PATH), filename)
if file_path.is_file() and os.stat(file_path).st_size > 0:
# file exists and is not empty, so append if necessary
ticker_info_df = pd.read_csv(file_path)
df_row = ticker_info_df.loc[ticker_info_df["Ticker"] == ticker]
if len(df_row) > 0:
# ticker is in file, just return it
ticker_info_list = list(df_row.iloc[0].drop("Ticker"))
return ticker_info_list
# ticker is not in file, go get it
ticker_info_list = get_info_update_file(ticker, file_path, "a")
return ticker_info_list
# file does not exist or is empty, so write it
ticker_info_list = get_info_update_file(ticker, file_path, "w")
return ticker_info_list

124
openbb_terminal/portfolio/portfolio_model.py
View File

@@ -859,17 +859,21 @@ class PortfolioModel:
# descrbibe outputs
try:
console.print(" Preprocessing orderbook: ", end="")
# Convert Date to datetime
self.__orderbook["Date"] = pd.to_datetime(self.__orderbook["Date"])
console.print(".", end="")
# Sort orderbook by date
self.__orderbook = self.__orderbook.sort_values(by="Date")
console.print(".", end="")
# Capitalize Ticker and Type [of instrument...]
self.__orderbook["Ticker"] = self.__orderbook["Ticker"].map(
lambda x: x.upper()
)
self.__orderbook["Type"] = self.__orderbook["Type"].map(lambda x: x.upper())
console.print(".", end="")
# Translate side: ["deposit", "buy"] -> 1 and ["withdrawal", "sell"] -> -1
self.__orderbook["Side"] = self.__orderbook["Side"].map(
@@ -877,17 +881,20 @@ class PortfolioModel:
if x.lower() in ["deposit", "buy"]
else (-1 if x.lower() in ["withdrawal", "sell"] else 0)
)
console.print(".", end="")
# Convert quantity to signed integer
self.__orderbook["Quantity"] = (
self.__orderbook["Quantity"] * self.__orderbook["Side"]
abs(self.__orderbook["Quantity"]) * self.__orderbook["Side"]
)
console.print(".", end="")
# Determining the investment/divestment value
self.__orderbook["Investment"] = (
self.__orderbook["Quantity"] * self.__orderbook["Price"]
- self.__orderbook["Fees"]
)
console.print(".", end="")
# Reformat crypto tickers to yfinance format (e.g. BTC -> BTC-USD)
crypto_trades = self.__orderbook[self.__orderbook.Type == "CRYPTO"]
@@ -897,6 +904,7 @@ class PortfolioModel:
crypto_trades.Ticker, crypto_trades.Currency
)
]
console.print(".", end="")
# Create tickers dictionary with structure {'Type': [Ticker]}
for ticker_type in set(self.__orderbook["Type"]):
@@ -907,22 +915,77 @@ class PortfolioModel:
]
)
)
console.print(".", end="")
# Create list with tickers except cash
self.tickers_list = list(set(self.__orderbook["Ticker"]))
console.print(".", end="")
# Save orderbook inception date
self.inception_date = self.__orderbook["Date"][0]
console.print(".", end="")
# Save trades static data
self.static_data = self.__orderbook.pivot(
index="Ticker",
columns=[],
values=["Type", "Sector", "Industry", "Country"],
)
# Populate fields Sector, Industry and Country
if not (
{"Sector", "Industry", "Country", "Region"}.issubset(
set(self.__orderbook.columns)
)
):
# if fields not in the orderbook add missing
if "Sector" not in self.__orderbook.columns:
self.__orderbook["Sector"] = np.nan
if "Industry" not in self.__orderbook.columns:
self.__orderbook["Industry"] = np.nan
if "Country" not in self.__orderbook.columns:
self.__orderbook["Country"] = np.nan
if "Region" not in self.__orderbook.columns:
self.__orderbook["Region"] = np.nan
self.load_company_data()
elif (
self.__orderbook.loc[
self.__orderbook["Type"] == "STOCK",
["Sector", "Industry", "Country", "Region"],
]
.isnull()
.values.any()
):
# if any fields is empty for Stocks (overwrites any info there)
self.load_company_data()
except Exception:
console.print("Could not preprocess orderbook.")
console.print("\nCould not preprocess orderbook.")
def load_company_data(self):
console.print("\n Loading company data: ", end="")
for ticker_type, ticker_list in self.tickers.items():
# yfinance only has sector, industry and country for stocks
if ticker_type == "STOCK":
for ticker in ticker_list:
# Only gets fields for tickers with missing data
# TODO: Should only get field missing for tickers with missing data
# now it's taking the 4 of them
if (
self.__orderbook.loc[
self.__orderbook["Ticker"] == ticker,
["Sector", "Industry", "Country", "Region"],
]
.isnull()
.values.any()
):
ticker_info_list = portfolio_helper.get_info_from_ticker(ticker)
# replace fields in orderbook
self.__orderbook.loc[
self.__orderbook.Ticker == ticker,
["Sector", "Industry", "Country", "Region"],
] = ticker_info_list
# Display progress
console.print(".", end="")
def load_benchmark(self, ticker: str = "SPY", full_shares: bool = False):
"""Adds benchmark dataframe
@@ -1004,6 +1067,7 @@ class PortfolioModel:
def generate_portfolio_data(self):
"""Generates portfolio data from orderbook"""
self.load_portfolio_historical_prices()
self.populate_historical_trade_data()
self.calculate_value()
@@ -1062,6 +1126,8 @@ class PortfolioModel:
def load_portfolio_historical_prices(self, use_close: bool = False):
"""Loads historical adj close prices for tickers in list of trades"""
console.print("\n Loading price data: ", end="")
for ticker_type, data in self.tickers.items():
if ticker_type in ["STOCK", "ETF", "CRYPTO"]:
# Download yfinance data
@@ -1079,13 +1145,16 @@ class PortfolioModel:
[self.portfolio_historical_prices, price_data], axis=1
)
else:
console.print("Type not supported in the orderbook.")
console.print(f"Type {ticker_type} not supported.")
console.print(".", end="")
# Fill missing values with last known price
self.portfolio_historical_prices.fillna(method="ffill", inplace=True)
def populate_historical_trade_data(self):
"""Create a new dataframe to store historical prices by ticker"""
trade_data = self.__orderbook.pivot(
index="Date",
columns="Ticker",
@@ -1131,6 +1200,9 @@ class PortfolioModel:
def calculate_value(self):
"""Calculate value from historical data"""
console.print("\n Calculating returns: ", end="")
trade_data = self.historical_trade_data
# For each type [STOCK, ETF, etc] calculate value value by trade date
@@ -1151,31 +1223,29 @@ class PortfolioModel:
for ticker_type, data in self.tickers.items():
self.itemized_value[ticker_type] = trade_data["End Value"][data].sum(axis=1)
# Initial Value = Cumulative Investment - (Previous End Value - Previous Initial Value)
trade_data[
pd.MultiIndex.from_product([["Initial Value"], self.tickers_list])
pd.MultiIndex.from_product(
[["Initial Value"], self.tickers_list + ["Total"]]
)
] = 0
for i, date in enumerate(trade_data.index):
if i == 0:
for t in self.tickers_list:
trade_data.at[date, ("Initial Value", t)] = trade_data.iloc[i][
"Investment"
][t]
else:
for t in self.tickers_list:
trade_data.at[date, ("Initial Value", t)] = (
+trade_data.iloc[i - 1]["End Value"][t]
+ trade_data.iloc[i]["Investment"][t]
- trade_data.iloc[i - 1]["Investment"][t]
)
# Initial Value = Previous End Value + Investment changes
trade_data["Initial Value"] = trade_data["End Value"].shift(1) + trade_data[
"Investment"
].diff(periods=1)
trade_data.loc[:, ("Initial Value", "Total")] = trade_data["Initial Value"][
self.tickers_list
].sum(axis=1)
# Set first day Initial Value as the Investment (NaNs break first period)
for t in self.tickers_list + ["Total"]:
trade_data.at[trade_data.index[0], ("Initial Value", t)] = trade_data.iloc[
0
]["Investment"][t]
console.print(".", end="")
self.historical_trade_data = trade_data
console.print("\n")
def calculate_reserves(self):
"""_summary_"""
# TODO: Add back cash dividends and deduct exchange costs