From 291e7d69914e9ab8b9bf9b20bb44d971bcedc247 Mon Sep 17 00:00:00 2001 From: montezdesousa <79287829+montezdesousa@users.noreply.github.com> Date: Fri, 9 Sep 2022 15:12:59 +0100 Subject: [PATCH] Add ETF support for portfolio allocation command (#2143) * fill etf and prevent different length errors * auto fill etf sectors and allow 0 value portfolios * allow 0 value portfolios * remove unused folder * allow 1 asset portfolios * split allocation calls by category in controller * comment sector allocation model * get country and region allocations for etf * add comments and black * improve comments * improve comments * allow for 1 category in sector, country and region * add progress bars * linting fix * fix mypy * set default date * fix pylint * add isin on paexport * merge main * auto pre load benchmark * fix rich table for np.float64 * refactor portfolio allocs * refactor alloc command * add isins column * format output alloc * rename variable * fix nan bug * black * add ticker conversion to yf format by isin * warn and removed unsupported ISINs * solve same day trades bug * display bench loading progress * portfolio show * check if valid isins on preprocessing * black * fix bug when region empty * warn when category is empty * reformat preprocessing * codespell * check if ticker is valid * flake8 * fix test * fix bug with trades on holidays Co-authored-by: Jeroen Bouma --- openbb_terminal/helper_funcs.py | 2 +- openbb_terminal/portfolio/allocation_model.py | 295 +++++++++--- .../portfolio/brokers/degiro/degiro_model.py | 4 +- .../portfolio/portfolio_controller.py | 53 ++- openbb_terminal/portfolio/portfolio_model.py | 421 +++++++++++++----- openbb_terminal/portfolio/portfolio_view.py | 80 ++-- portfolio/holdings/paexport_degiro.csv | 0 7 files changed, 641 insertions(+), 214 deletions(-) delete mode 100644 portfolio/holdings/paexport_degiro.csv diff --git a/openbb_terminal/helper_funcs.py b/openbb_terminal/helper_funcs.py index 98317d0a347..a99de22f717 100644 --- a/openbb_terminal/helper_funcs.py +++ b/openbb_terminal/helper_funcs.py @@ -271,7 +271,7 @@ def print_rich_table( row = [str(idx)] if show_index else [] row += [ str(x) - if not isinstance(x, float) or not isinstance(x, np.float64) + if not isinstance(x, float) and not isinstance(x, np.float64) else ( f"{x:{floatfmt[idx]}}" if isinstance(floatfmt, list) diff --git a/openbb_terminal/portfolio/allocation_model.py b/openbb_terminal/portfolio/allocation_model.py index c1504fa7397..05e365baf68 100644 --- a/openbb_terminal/portfolio/allocation_model.py +++ b/openbb_terminal/portfolio/allocation_model.py @@ -3,13 +3,15 @@ from typing import Dict import pandas as pd import requests +import yfinance as yf from openbb_terminal.decorators import log_start_end +from openbb_terminal.rich_config import console logger = logging.getLogger(__name__) @log_start_end(log=logger) -def obtain_assets_allocation(benchmark_info: Dict, portfolio_trades: pd.DataFrame): +def get_assets_allocation(benchmark_info: Dict, portfolio_trades: pd.DataFrame): """Obtain the assets allocation of the benchmark and portfolio [Source: Yahoo Finance] Parameters @@ -27,22 +29,28 @@ def obtain_assets_allocation(benchmark_info: Dict, portfolio_trades: pd.DataFram Dictionary with the portfolio's asset allocations """ benchmark_assets_allocation = pd.DataFrame(benchmark_info["holdings"]) - portfolio_assets_allocation = ( - ( - portfolio_trades[portfolio_trades["Type"] != "CASH"] - .groupby(by="Ticker") - .agg({"Portfolio Value": "sum"}) - .div(portfolio_trades["Portfolio Value"].sum()) - ) - .squeeze() - .sort_values(ascending=False) + benchmark_assets_allocation.rename( + columns={"symbol": "Symbol", "holdingPercent": "Benchmark"}, inplace=True ) + benchmark_assets_allocation.drop(columns=["holdingName"], inplace=True) + + portfolio_assets_allocation = ( + portfolio_trades[portfolio_trades["Type"] != "CASH"] + .groupby(by="Ticker") + .agg({"Portfolio Value": "sum"}) + .div(portfolio_trades["Portfolio Value"].sum()) + ).sort_values(by="Portfolio Value", ascending=False) + portfolio_assets_allocation.reset_index(inplace=True) + portfolio_assets_allocation.rename( + columns={"Ticker": "Symbol", "Portfolio Value": "Portfolio"}, inplace=True + ) + portfolio_assets_allocation.fillna(0, inplace=True) return benchmark_assets_allocation, portfolio_assets_allocation @log_start_end(log=logger) -def obtain_sector_allocation(benchmark_info: Dict, portfolio_trades: pd.DataFrame): +def get_sector_allocation(benchmark_info: Dict, portfolio_trades: pd.DataFrame): """Obtain the sector allocation of the benchmark and portfolio [Source: Yahoo Finance] Parameters @@ -59,6 +67,9 @@ def obtain_sector_allocation(benchmark_info: Dict, portfolio_trades: pd.DataFram country_allocation: dict Dictionary with country allocations """ + + console.print(" Loading sector data: ", end="") + benchmark_sectors_allocation = ( pd.DataFrame.from_dict( data={ @@ -80,32 +91,110 @@ def obtain_sector_allocation(benchmark_info: Dict, portfolio_trades: pd.DataFram benchmark_sectors_allocation.index = prettified # Define portfolio sector allocation + # Aggregate sector value for stocks and crypto portfolio_sectors_allocation = ( - ( - portfolio_trades[portfolio_trades["Type"] != "CASH"] - .groupby(by="Sector") - .agg({"Portfolio Value": "sum"}) + portfolio_trades[portfolio_trades["Type"].isin(["STOCK", "CRYPTO"])] + .groupby(by="Sector") + .agg({"Portfolio Value": "sum"}) + ) + + # Aggregate sector value for ETFs + # Start by getting value by isin/ticker + etf_ticker_value = ( + portfolio_trades[portfolio_trades["Type"].isin(["ETF"])] + .groupby(by="Ticker") + .agg({"Portfolio Value": "sum"}) + ) + etf_global_sector_alloc = pd.DataFrame() + + # Loop through each etf and multiply sector weights by current value + for item in etf_ticker_value.index.values: + + # TODO: This can be improved by caching this info similar to what is done in stocks + etf_info = yf.Ticker(item).info + + try: + etf_sector_weight = pd.DataFrame.from_dict( + data={ + sector_name: allocation + for sector in etf_info["sectorWeightings"] + for sector_name, allocation in sector.items() + }, + orient="index", + columns=["Portfolio Value"], + ) + + except Exception: + # If ETF has no sectors like VIX for example or it was not found, add to Other + etf_sector_weight = pd.DataFrame.from_dict( + data={"Other": 1}, orient="index", columns=["Portfolio Value"] + ) + + etf_value = etf_ticker_value["Portfolio Value"][item] + + etf_ticker_sector_alloc = etf_sector_weight * etf_value + + # Aggregate etf sector allocation by value + etf_global_sector_alloc = pd.concat( + [etf_global_sector_alloc, etf_ticker_sector_alloc], axis=1 ) - .div(portfolio_trades["Portfolio Value"].sum()) - .squeeze() + + etf_global_sector_alloc.fillna(0, inplace=True) + + etf_global_sector_alloc = etf_global_sector_alloc.sum(axis=1) + + console.print(".", end="") + console.print("\n") + + etf_global_sector_alloc = pd.DataFrame( + etf_global_sector_alloc, columns=["Portfolio Value"] + ) + + # Rename columns to match stock and crypto classification + etf_global_sector_alloc.index.name = "Sector" + prettified = [] + for sector in etf_global_sector_alloc.index: + prettified.append(sector.replace("_", " ").title()) + + etf_global_sector_alloc.index = prettified + + # Aggregate sector allocation for stocks and crypto with ETFs + portfolio_sectors_allocation = pd.merge( + portfolio_sectors_allocation, + etf_global_sector_alloc, + how="outer", + left_index=True, + right_index=True, + ).sum(axis=1) + + portfolio_sectors_allocation = pd.DataFrame( + portfolio_sectors_allocation, columns=["Portfolio Value"] + ) + + portfolio_sectors_allocation = ( + portfolio_sectors_allocation.div(portfolio_trades["Portfolio Value"].sum()) + .squeeze(axis=1) .sort_values(ascending=False) ) + portfolio_sectors_allocation.fillna(0, inplace=True) + return benchmark_sectors_allocation, portfolio_sectors_allocation @log_start_end(log=logger) -def obtain_benchmark_regional_and_country_allocation( - benchmark_ticker: str, regional_test: list = None, country_test: list = None +def get_region_country_allocation( + ticker: str, region_test: list = None, country_test: list = None ): - """Obtain the regional and country allocation of the benchmark ticker. [Source: Fidelity.com] + + """Obtain the region and country allocation for ETF ticker. [Source: Fidelity.com] Parameters ---------- benchmark_ticker: str - The benchmark ticker, e.g. "SPY" - regional_test: list - This includes a list of regional names that should be used to discover + The ticker, e.g. "SPY" + region_test: list + This includes a list of region names that should be used to discover which list on the Fidelity page is the correct one country_test: list This includes a list of country names that should be used to discover @@ -113,14 +202,15 @@ def obtain_benchmark_regional_and_country_allocation( Returns ------ - benchmark_regional_allocation: dict + region_allocation: dict Dictionary with regional allocations. - benchmark_country_allocation: dict + country_allocation: dict Dictionary with country allocations """ + # Initialize variables - if not regional_test: - regional_test = [ + if not region_test: + region_test = [ "North America", "Europe", "Asia", @@ -137,52 +227,52 @@ def obtain_benchmark_regional_and_country_allocation( "China", ] - regional_list = 0 + region_list = 0 country_list = 0 # Collect data from Fidelity about the portfolio composition of the benchmark - URL = f"https://screener.fidelity.com/ftgw/etf/goto/snapshot/portfolioComposition.jhtml?symbols={benchmark_ticker}" + URL = f"https://screener.fidelity.com/ftgw/etf/goto/snapshot/portfolioComposition.jhtml?symbols={ticker}" html = requests.get(URL).content df_list = pd.read_html(html) # Find the ones that contain regions and countries for index, item in enumerate(df_list): - for region in regional_test: + for region in region_test: if region in item.values: - regional_list = index + region_list = index break for country in country_test: if country in item.values: country_list = index break - if regional_list: - benchmark_regional_allocation = { + if region_list: + region_allocation = { row[1]: float(row[2].strip("%")) / 100 - for _, row in df_list[regional_list].dropna(axis="columns").iterrows() + for _, row in df_list[region_list].dropna(axis="columns").iterrows() } - benchmark_regional_allocation = pd.DataFrame.from_dict( - benchmark_regional_allocation, orient="index" + region_allocation = pd.DataFrame.from_dict( + region_allocation, orient="index" ).squeeze() else: - benchmark_regional_allocation = pd.DataFrame() + region_allocation = pd.DataFrame() if country_list: - benchmark_country_allocation = { + country_allocation = { row[1]: float(row[2].strip("%")) / 100 for _, row in df_list[country_list].dropna(axis="columns").iterrows() } - benchmark_country_allocation = pd.DataFrame.from_dict( - benchmark_country_allocation, orient="index" + country_allocation = pd.DataFrame.from_dict( + country_allocation, orient="index" ).squeeze() else: - benchmark_country_allocation = pd.DataFrame() + country_allocation = pd.DataFrame() - return benchmark_regional_allocation, benchmark_country_allocation + return region_allocation, country_allocation @log_start_end(log=logger) -def obtain_portfolio_regional_and_country_allocation(portfolio_trades: pd.DataFrame): +def get_portfolio_region_country_allocation(portfolio_trades: pd.DataFrame): """Obtain the regional and country allocation of the portfolio. Parameters @@ -197,28 +287,121 @@ def obtain_portfolio_regional_and_country_allocation(portfolio_trades: pd.DataFr portfolio_country_allocation: pd.DataFrame Dictionary with country allocations """ + + console.print(" Loading country/region data: ", end="") + # Define portfolio regional allocation - portfolio_regional_allocation = ( - ( - portfolio_trades[portfolio_trades["Type"] != "CASH"] + if not portfolio_trades["Region"].isnull().any(): + portfolio_region_allocation = ( + portfolio_trades[portfolio_trades["Type"].isin(["STOCK", "CRYPTO"])] .groupby(by="Region") .agg({"Portfolio Value": "sum"}) ) - .div(portfolio_trades["Portfolio Value"].sum()) - .squeeze() - .sort_values(ascending=False) - ) + else: + portfolio_region_allocation = pd.DataFrame() # Define portfolio country allocation - portfolio_country_allocation = ( - ( - portfolio_trades[portfolio_trades["Type"] != "CASH"] + if not portfolio_trades["Country"].isnull().any(): + portfolio_country_allocation = ( + portfolio_trades[portfolio_trades["Type"].isin(["STOCK", "CRYPTO"])] .groupby(by="Country") .agg({"Portfolio Value": "sum"}) ) - .div(portfolio_trades["Portfolio Value"].sum()) - .squeeze() + else: + portfolio_country_allocation = pd.DataFrame() + + # Aggregate sector value for ETFs + # Start by getting value by ticker + etf_ticker_value = ( + portfolio_trades[portfolio_trades["Type"].isin(["ETF"])] + .groupby(by="Ticker") + .agg({"Portfolio Value": "sum"}) + ) + etf_global_region_alloc = pd.DataFrame() + etf_global_country_alloc = pd.DataFrame() + + # Loop through each etf and multiply sector weights by current value + for item in etf_ticker_value.index.values: + + etf_region_weight, etf_country_weight = get_region_country_allocation(item) + + if etf_region_weight.empty: + # If ETF has no sectors like VIX for example or it was not found, add to Other + etf_region_weight = pd.DataFrame.from_dict( + data={"Other": 1}, orient="index", columns=["Portfolio Value"] + ) + + if etf_country_weight.empty: + etf_country_weight = pd.DataFrame.from_dict( + data={"Other": 1}, orient="index", columns=["Portfolio Value"] + ) + + etf_value = etf_ticker_value["Portfolio Value"][item] + + # Aggregate etf region allocation by value + etf_ticker_region_alloc = etf_region_weight * etf_value + etf_global_region_alloc = pd.concat( + [etf_global_region_alloc, etf_ticker_region_alloc], axis=1 + ) + etf_global_region_alloc.fillna(0, inplace=True) + etf_global_region_alloc = etf_global_region_alloc.sum(axis=1) + + # Aggregate etf country allocation by value + etf_ticker_country_alloc = etf_country_weight * etf_value + etf_global_country_alloc = pd.concat( + [etf_global_country_alloc, etf_ticker_country_alloc], axis=1 + ) + etf_global_country_alloc.fillna(0, inplace=True) + etf_global_country_alloc = etf_global_country_alloc.sum(axis=1) + + console.print(".", end="") + console.print("\n") + + etf_global_region_alloc = pd.DataFrame( + etf_global_region_alloc, columns=["Portfolio Value"] + ) + etf_global_country_alloc = pd.DataFrame( + etf_global_country_alloc, columns=["Portfolio Value"] + ) + + # Aggregate region allocation for stocks and crypto with ETFs + portfolio_region_allocation = pd.merge( + portfolio_region_allocation, + etf_global_region_alloc, + how="outer", + left_index=True, + right_index=True, + ).sum(axis=1) + + # Aggregate country allocation for stocks and crypto with ETFs + portfolio_country_allocation = pd.merge( + portfolio_country_allocation, + etf_global_country_alloc, + how="outer", + left_index=True, + right_index=True, + ).sum(axis=1) + + portfolio_region_allocation = pd.DataFrame( + portfolio_region_allocation, columns=["Portfolio Value"] + ) + portfolio_country_allocation = pd.DataFrame( + portfolio_country_allocation, columns=["Portfolio Value"] + ) + + portfolio_region_allocation = ( + portfolio_region_allocation.div(portfolio_trades["Portfolio Value"].sum()) + .squeeze(axis=1) .sort_values(ascending=False) ) - return portfolio_regional_allocation, portfolio_country_allocation + portfolio_country_allocation = ( + portfolio_country_allocation.div(portfolio_trades["Portfolio Value"].sum()) + .squeeze(axis=1) + .sort_values(ascending=False) + ) + + portfolio_region_allocation.fillna(0, inplace=True) + portfolio_country_allocation.fillna(0, inplace=True) + + return portfolio_region_allocation, portfolio_country_allocation diff --git a/openbb_terminal/portfolio/brokers/degiro/degiro_model.py b/openbb_terminal/portfolio/brokers/degiro/degiro_model.py index c91b26d2ff8..ef42d943514 100644 --- a/openbb_terminal/portfolio/brokers/degiro/degiro_model.py +++ b/openbb_terminal/portfolio/brokers/degiro/degiro_model.py @@ -404,13 +404,14 @@ class DegiroModel: products_df["productId"] = products_df["productId"].astype("int") transactions_full_df = pd.merge( transactions_df, - products_df[{"productId", "symbol", "productType"}], + products_df[{"productId", "symbol", "productType", "isin"}], on="productId", ) portfolio_df = transactions_full_df.rename( columns={ "date": "Date", + "isin": "ISIN", "symbol": "Ticker", "productType": "Type", # STOCK or ETF "price": "Price", @@ -428,6 +429,7 @@ class DegiroModel: ).dt.date columns = [ "Date", + "ISIN", "Ticker", "Type", "Price", diff --git a/openbb_terminal/portfolio/portfolio_controller.py b/openbb_terminal/portfolio/portfolio_controller.py index 940594c147a..17e125b67bc 100644 --- a/openbb_terminal/portfolio/portfolio_controller.py +++ b/openbb_terminal/portfolio/portfolio_controller.py @@ -342,16 +342,40 @@ class PortfolioController(BaseController): # Add in the Risk-free rate self.portfolio.set_risk_free_rate(ns_parser.risk_free_rate) - console.print(f"\n[bold]Portfolio:[/bold] {self.portfolio_name}") + # Load benchmark + self.call_bench(["-b", "SPDR S&P 500 ETF Trust (SPY)"]) + console.print( - f"[bold]Risk Free Rate:[/bold] {self.portfolio.risk_free_rate}" + f"\n[bold][param]Portfolio:[/param][/bold] {self.portfolio_name}" + ) + console.print( + f"[bold][param]Risk Free Rate:[/param][/bold] {self.portfolio.risk_free_rate}" + ) + console.print( + f"[bold][param]Benchmark:[/param][/bold] {self.benchmark_name}\n" ) - console.print() @log_start_end(log=logger) - def call_show(self, _): + def call_show(self, other_args: List[str]): """Process show command""" - portfolio_view.display_orderbook(self.portfolio, show_index=False) + parser = argparse.ArgumentParser( + add_help=False, + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + prog="show", + description="Show transactions table", + ) + ns_parser = self.parse_known_args_and_warn( + parser, + other_args, + export_allowed=EXPORT_BOTH_RAW_DATA_AND_FIGURES, + limit=10, + ) + portfolio_view.display_orderbook( + self.portfolio, + show_index=False, + limit=ns_parser.limit, + export=ns_parser.export, + ) @log_start_end(log=logger) def call_bench(self, other_args: List[str]): @@ -382,7 +406,6 @@ class PortfolioController(BaseController): ) if other_args and "-" not in other_args[0][0]: other_args.insert(0, "-b") - ns_parser = self.parse_known_args_and_warn(parser, other_args) if ns_parser and self.portfolio is not None: @@ -409,9 +432,6 @@ class PortfolioController(BaseController): self.portfolio.returns, self.portfolio.benchmark_returns ) - console.print( - f"[bold]\nBenchmark:[/bold] {self.benchmark_name} ({benchmark_ticker})" - ) else: console.print("[red]Please first load orderbook using 'load'[/red]\n") console.print() @@ -454,10 +474,9 @@ class PortfolioController(BaseController): if check_portfolio_benchmark_defined( self.portfolio_name, self.benchmark_name ): - if self.portfolio.portfolio_assets_allocation.empty: - self.portfolio.calculate_allocations() - if ns_parser.agg == "assets": + if self.portfolio.portfolio_assets_allocation.empty: + self.portfolio.calculate_allocations("asset") portfolio_view.display_assets_allocation( self.portfolio.portfolio_assets_allocation, self.portfolio.benchmark_assets_allocation, @@ -465,6 +484,8 @@ class PortfolioController(BaseController): ns_parser.tables, ) elif ns_parser.agg == "sectors": + if self.portfolio.portfolio_sectors_allocation.empty: + self.portfolio.calculate_allocations("sector") portfolio_view.display_category_allocation( ns_parser.agg, self.portfolio.portfolio_sectors_allocation, @@ -473,6 +494,8 @@ class PortfolioController(BaseController): ns_parser.tables, ) elif ns_parser.agg == "countries": + if self.portfolio.portfolio_country_allocation.empty: + self.portfolio.calculate_allocations("country") portfolio_view.display_category_allocation( ns_parser.agg, self.portfolio.portfolio_country_allocation, @@ -481,10 +504,12 @@ class PortfolioController(BaseController): ns_parser.tables, ) elif ns_parser.agg == "regions": + if self.portfolio.portfolio_region_allocation.empty: + self.portfolio.calculate_allocations("region") portfolio_view.display_category_allocation( ns_parser.agg, - self.portfolio.portfolio_regional_allocation, - self.portfolio.benchmark_regional_allocation, + self.portfolio.portfolio_region_allocation, + self.portfolio.benchmark_region_allocation, ns_parser.limit, ns_parser.tables, ) diff --git a/openbb_terminal/portfolio/portfolio_model.py b/openbb_terminal/portfolio/portfolio_model.py index 260401d35ec..2c078f94eff 100644 --- a/openbb_terminal/portfolio/portfolio_model.py +++ b/openbb_terminal/portfolio/portfolio_model.py @@ -1,8 +1,10 @@ """Portfolio Model""" __docformat__ = "numpy" +import contextlib import logging from typing import Dict, Any, Tuple +import datetime import numpy as np import scipy @@ -921,10 +923,6 @@ class PortfolioModel: Class for portfolio analysis in OpenBB Implements a Portfolio and related methods. - Attributes - ------- - - Methods ------- read_orderbook: Class method to read orderbook from file @@ -938,36 +936,31 @@ class PortfolioModel: generate_portfolio_data: Generates portfolio data from orderbook load_portfolio_historical_prices: Loads historical adj close prices for tickers in list of trades populate_historical_trade_data: Create a new dataframe to store historical prices by ticker - calculate_value: Calculate value from historical data + calculate_value: Calculate value end of day from historical data - calculate_reserves: + calculate_reserves: Takes dividends into account for returns calculation - calculate_allocations: + calculate_allocations: Determine allocations based on assets, sectors, countries and region set_risk_free_rate: Sets risk free rate - calculate_metrics: - """ + @log_start_end(log=logger) def __init__(self, orderbook: pd.DataFrame = pd.DataFrame()): - """Initialize Portfolio class""" + """Initialize PortfolioModel class""" + # Portfolio self.tickers_list = None self.tickers: Dict[Any, Any] = {} - self.inception_date = None - self.static_data = pd.DataFrame() + self.inception_date = datetime.date(1970, 1, 1) self.historical_trade_data = pd.DataFrame() self.returns = pd.DataFrame() self.itemized_value = pd.DataFrame() self.portfolio_trades = pd.DataFrame() self.portfolio_value = None - self.portfolio_assets_allocation = pd.DataFrame() - self.portfolio_regional_allocation = pd.DataFrame() - self.portfolio_country_allocation = pd.DataFrame() self.portfolio_historical_prices = pd.DataFrame() self.empty = True - self.risk_free_rate = float(0) # Benchmark @@ -976,8 +969,16 @@ class PortfolioModel: self.benchmark_historical_prices = pd.DataFrame() self.benchmark_returns = pd.DataFrame() self.benchmark_trades = pd.DataFrame() + + # Allocations + self.portfolio_assets_allocation = pd.DataFrame() + self.portfolio_sectors_allocation = pd.DataFrame() + self.portfolio_region_allocation = pd.DataFrame() + self.portfolio_country_allocation = pd.DataFrame() + self.benchmark_assets_allocation = pd.DataFrame() - self.benchmark_regional_allocation = pd.DataFrame() + self.benchmark_sectors_allocation = pd.DataFrame() + self.benchmark_region_allocation = pd.DataFrame() self.benchmark_country_allocation = pd.DataFrame() # Set and preprocess orderbook @@ -990,14 +991,43 @@ class PortfolioModel: self.empty = False def get_orderbook(self): - return self.__orderbook + """Get formatted transactions + + Returns: + pd.DataFrame: formatted transactions + """ + df = self.__orderbook[ + [ + "Date", + "Type", + "Ticker", + "Side", + "Price", + "Quantity", + "Fees", + "Investment", + "Currency", + "Sector", + "Industry", + "Country", + "Region", + ] + ] + df = df.replace(np.nan, "-") + df["Date"] = df["Date"].dt.strftime("%Y-%m-%d") + df.sort_values(by="Date", ascending=False, inplace=True) + return df @staticmethod + @log_start_end(log=logger) def read_orderbook(path: str) -> pd.DataFrame: - """Class method to read orderbook from file + """Static method to read orderbook from file + + Parameters + ---------- + path: str + path to orderbook file - Args: - path (str): path to orderbook file """ # Load orderbook from file if path.endswith(".xlsx"): @@ -1007,15 +1037,30 @@ class PortfolioModel: return orderbook + @log_start_end(log=logger) def preprocess_orderbook(self): """Method to preprocess, format and compute auxiliary fields""" - # descrbibe outputs + # Preprocessing steps: + # 0. If optional fields not in the orderbook add missing + # 1. Convert Date to datetime + # 2. Sort orderbook by date + # 3. Capitalize Ticker and Type [of instrument...] + # 4. Translate side: ["deposit", "buy"] -> 1 and ["withdrawal", "sell"] -> -1 + # 5. Convert quantity to signed integer + # 6. Determining the investment/divestment value + # 7. Reformat crypto tickers to yfinance format (e.g. BTC -> BTC-USD) + # 8. Reformat STOCK/ETF tickers to yfinance format if ISIN provided + # 9. Remove unsupported ISINs that came out empty + # 10. Create tickers dictionary with structure {'Type': [Ticker]} + # 11. Create list with tickers except cash + # 12. Save orderbook inception date + # 13. Populate fields Sector, Industry and Country try: console.print(" Preprocessing orderbook: ", end="") - # if optional fields not in the orderbook add missing + # 0. If optional fields not in the orderbook add missing optional_fields = [ "Sector", "Industry", @@ -1023,61 +1068,50 @@ class PortfolioModel: "Region", "Fees", "Premium", + "ISIN", ] if not set(optional_fields).issubset(set(self.__orderbook.columns)): for field in optional_fields: if field not in self.__orderbook.columns: self.__orderbook[field] = np.nan - # if optional fields for stock data has missing datapoints, fill them - if ( - self.__orderbook.loc[ - self.__orderbook["Type"] == "STOCK", - optional_fields, - ] - .isnull() - .values.any() - ): - # if any fields is empty for Stocks (overwrites any info there) - self.load_company_data() - - # Convert Date to datetime + # 1. Convert Date to datetime self.__orderbook["Date"] = pd.to_datetime(self.__orderbook["Date"]) console.print(".", end="") - # Sort orderbook by date + # 3. Sort orderbook by date self.__orderbook = self.__orderbook.sort_values(by="Date") console.print(".", end="") - # Capitalize Ticker and Type [of instrument...] + # 3. Capitalize Ticker and Type [of instrument...] self.__orderbook["Ticker"] = self.__orderbook["Ticker"].map( lambda x: x.upper() ) self.__orderbook["Type"] = self.__orderbook["Type"].map(lambda x: x.upper()) console.print(".", end="") - # Translate side: ["deposit", "buy"] -> 1 and ["withdrawal", "sell"] -> -1 - self.__orderbook["Side"] = self.__orderbook["Side"].map( + # 4. Translate side: ["deposit", "buy"] -> 1 and ["withdrawal", "sell"] -> -1 + self.__orderbook["Signal"] = self.__orderbook["Side"].map( lambda x: 1 if x.lower() in ["deposit", "buy"] else (-1 if x.lower() in ["withdrawal", "sell"] else 0) ) console.print(".", end="") - # Convert quantity to signed integer + # 5. Convert quantity to signed integer self.__orderbook["Quantity"] = ( - abs(self.__orderbook["Quantity"]) * self.__orderbook["Side"] + abs(self.__orderbook["Quantity"]) * self.__orderbook["Signal"] ) console.print(".", end="") - # Determining the investment/divestment value + # 6. Determining the investment/divestment value self.__orderbook["Investment"] = ( self.__orderbook["Quantity"] * self.__orderbook["Price"] - - self.__orderbook["Fees"] + + self.__orderbook["Fees"] ) console.print(".", end="") - # Reformat crypto tickers to yfinance format (e.g. BTC -> BTC-USD) + # 7. Reformat crypto tickers to yfinance format (e.g. BTC -> BTC-USD) crypto_trades = self.__orderbook[self.__orderbook.Type == "CRYPTO"] self.__orderbook.loc[(self.__orderbook.Type == "CRYPTO"), "Ticker"] = [ f"{crypto}-{currency}" @@ -1087,7 +1121,57 @@ class PortfolioModel: ] console.print(".", end="") - # Create tickers dictionary with structure {'Type': [Ticker]} + # 8. Reformat STOCK/ETF tickers to yfinance format if ISIN provided. + + # If isin not valid ticker is empty + self.__orderbook["yf_Ticker"] = self.__orderbook["ISIN"].apply( + lambda x: yf.utils.get_ticker_by_isin(x) if not pd.isna(x) else np.nan + ) + + empty_tickers = list( + self.__orderbook[ + (self.__orderbook["yf_Ticker"] == "") + | (self.__orderbook["yf_Ticker"].isna()) + ]["Ticker"].unique() + ) + + # If ticker from isin is empty it is not valid in yfinance, so check if user provided ticker is supported + removed_tickers = [] + for item in empty_tickers: + with contextlib.redirect_stdout(None): + # Suppress yfinance failed download message if occurs + valid_ticker = not ( + yf.download( + item, + start=datetime.datetime.now() + datetime.timedelta(days=-5), + progress=False, + ).empty + ) + if valid_ticker: + # Invalid ISIN but valid ticker + self.__orderbook.loc[ + self.__orderbook["Ticker"] == item, "yf_Ticker" + ] = np.nan + else: + self.__orderbook.loc[ + self.__orderbook["Ticker"] == item, "yf_Ticker" + ] = "" + removed_tickers.append(item) + + # Merge reformated tickers into Ticker + self.__orderbook["Ticker"] = self.__orderbook["yf_Ticker"].fillna( + self.__orderbook["Ticker"] + ) + + console.print(".", end="") + + # 9. Remove unsupported ISINs that came out empty + self.__orderbook.drop( + self.__orderbook[self.__orderbook["Ticker"] == ""].index, inplace=True + ) + console.print(".", end="") + + # 10. Create tickers dictionary with structure {'Type': [Ticker]} for ticker_type in set(self.__orderbook["Type"]): self.tickers[ticker_type] = list( set( @@ -1098,27 +1182,49 @@ class PortfolioModel: ) console.print(".", end="") - # Create list with tickers except cash + # 11. Create list with tickers except cash self.tickers_list = list(set(self.__orderbook["Ticker"])) console.print(".", end="") - # Save orderbook inception date + # 12. Save orderbook inception date self.inception_date = self.__orderbook["Date"][0] console.print(".", end="") + # 13. Populate fields Sector, Industry and Country + if ( + self.__orderbook.loc[ + self.__orderbook["Type"] == "STOCK", + optional_fields, + ] + .isnull() + .values.any() + ): + # if any fields is empty for Stocks (overwrites any info there) + self.load_company_data() + console.print(".", end="") + + # Warn user of removed ISINs + if removed_tickers: + console.print( + f"\n\n[red]The following tickers are not supported and were removed: {removed_tickers}." + f"\nManually edit the 'Ticker' field with the proper Yahoo Finance suffix or provide a valid ISIN." + f"\nSuffix info on 'Yahoo Finance market coverage':" + " https://help.yahoo.com/kb/exchanges-data-providers-yahoo-finance-sln2310.html" + f"\nE.g. IWDA -> IWDA.AS[/red]" + ) except Exception: console.print("\nCould not preprocess orderbook.") + @log_start_end(log=logger) def load_company_data(self): + """Method to populate company data for stocks such as sector, industry and country""" console.print("\n Loading company data: ", end="") for ticker_type, ticker_list in self.tickers.items(): - # yfinance only has sector, industry and country for stocks if ticker_type == "STOCK": for ticker in ticker_list: - # Only gets fields for tickers with missing data # TODO: Should only get field missing for tickers with missing data # now it's taking the 4 of them @@ -1130,34 +1236,117 @@ class PortfolioModel: .isnull() .values.any() ): - ticker_info_list = portfolio_helper.get_info_from_ticker(ticker) + # Get ticker info in list ["Sector", "Industry", "Country", "Region"] from isin/ticker + info_list = portfolio_helper.get_info_from_ticker(ticker) - # replace fields in orderbook + # Replace fields in orderbook self.__orderbook.loc[ self.__orderbook.Ticker == ticker, ["Sector", "Industry", "Country", "Region"], - ] = ticker_info_list + ] = info_list + # Display progress console.print(".", end="") + elif ticker_type == "CRYPTO": + for ticker in ticker_list: + if ( + self.__orderbook.loc[ + self.__orderbook["Ticker"] == ticker, + ["Sector", "Industry", "Country", "Region"], + ] + .isnull() + .values.any() + ): + # Get ticker info in list ["Sector", "Industry", "Country", "Region"] + info_list = ["Crypto", "Crypto", "Crypto", "Crypto"] + + # Replace fields in orderbook + self.__orderbook.loc[ + self.__orderbook.Ticker == ticker, + ["Sector", "Industry", "Country", "Region"], + ] = info_list + + # Display progress + console.print(".", end="") + + else: + for ticker in ticker_list: + if ( + self.__orderbook.loc[ + self.__orderbook["Ticker"] == ticker, + ["Sector", "Industry", "Country", "Region"], + ] + .isnull() + .values.any() + ): + # Get ticker info in list ["Sector", "Industry", "Country", "Region"] + info_list = ["-", "-", "-", "-"] + + # Replace fields in orderbook + self.__orderbook.loc[ + self.__orderbook.Ticker == ticker, + ["Sector", "Industry", "Country", "Region"], + ] = info_list + + # Display progress + console.print(".", end="") + + @log_start_end(log=logger) def load_benchmark(self, ticker: str = "SPY", full_shares: bool = False): """Adds benchmark dataframe - Args: - ticker (str): benchmark ticker to download data - full_shares (bool): whether to mimic the portfolio trades exactly (partial shares) or round down the + Parameters + ---------- + ticker: str + benchmark ticker to download data + full_shares: bool + whether to mimic the portfolio trades exactly (partial shares) or round down the quantity to the nearest number. + """ + + console.print("\n Loading benchmark: ", end="") + self.benchmark_ticker = ticker + self.benchmark_historical_prices = yf.download( - ticker, start=self.inception_date, threads=False, progress=False + ticker, + start=self.inception_date - datetime.timedelta(days=1), + threads=False, + progress=False, )["Adj Close"] - self.benchmark_returns = self.benchmark_historical_prices.pct_change().dropna() - self.benchmark_info = yf.Ticker(ticker).info + self.mimic_trades_for_benchmark(full_shares) + # Merge benchmark and portfolio dates to ensure same length + self.benchmark_historical_prices = pd.merge( + self.portfolio_historical_prices["Close"], + self.benchmark_historical_prices, + how="outer", + left_index=True, + right_index=True, + )["Adj Close"] + self.benchmark_historical_prices.fillna(method="ffill", inplace=True) + + self.benchmark_returns = self.benchmark_historical_prices.pct_change().dropna() + self.benchmark_info = yf.Ticker(ticker).info + + # Display progress + console.print(".") + + @log_start_end(log=logger) def mimic_trades_for_benchmark(self, full_shares: bool = False): - """Mimic trades from the orderbook based on chosen benchmark assuming partial shares""" + """Mimic trades from the orderbook based on chosen benchmark assuming partial shares + + Parameters + ---------- + full_shares: bool + whether to mimic the portfolio trades exactly (partial shares) or round down the + quantity to the nearest number. + + """ + # Create dataframe to store benchmark trades self.benchmark_trades = self.__orderbook[["Date", "Type", "Investment"]].copy() @@ -1223,6 +1412,7 @@ class PortfolioModel: f" in the benchmark ({round(sum(self.benchmark_trades['Benchmark Investment']), 2)})." ) + @log_start_end(log=logger) def generate_portfolio_data(self): """Generates portfolio data from orderbook""" @@ -1255,6 +1445,7 @@ class PortfolioModel: self.portfolio_returns = pd.DataFrame(self.__orderbook["Date"]) + # Save portfolio trades to compute allocations later self.portfolio_trades = self.__orderbook.copy() self.portfolio_trades[ [ @@ -1281,8 +1472,16 @@ class PortfolioModel: - self.portfolio_trades["Portfolio Investment"][index] ) + @log_start_end(log=logger) def load_portfolio_historical_prices(self, use_close: bool = False): - """Loads historical adj close prices for tickers in list of trades""" + """Loads historical adj close/close prices for tickers in list of trades + + Parameters + ---------- + use_close: bool + whether to use close or adjusted close prices + + """ console.print("\n Loading price data: ", end="") @@ -1310,25 +1509,18 @@ class PortfolioModel: # Fill missing values with last known price self.portfolio_historical_prices.fillna(method="ffill", inplace=True) + @log_start_end(log=logger) def populate_historical_trade_data(self): """Create a new dataframe to store historical prices by ticker""" - trade_data = self.__orderbook.pivot( + trade_data = self.__orderbook.pivot_table( index="Date", - columns="Ticker", + columns=["Ticker"], values=[ - "Type", - "Sector", - "Industry", - "Country", - "Price", "Quantity", - "Fees", - "Premium", "Investment", - "Side", - "Currency", ], + aggfunc={"Quantity": np.sum, "Investment": np.sum}, ) # Make historical prices columns a multi-index. This helps the merging. @@ -1340,10 +1532,13 @@ class PortfolioModel: trade_data = pd.merge( trade_data, self.portfolio_historical_prices, - how="right", + how="outer", left_index=True, right_index=True, - ).fillna(0) + ) + + trade_data["Close"] = trade_data["Close"].fillna(method="ffill") + trade_data.fillna(0, inplace=True) # Accumulate quantity held by trade date trade_data["Quantity"] = trade_data["Quantity"].cumsum() @@ -1356,8 +1551,9 @@ class PortfolioModel: self.historical_trade_data = trade_data + @log_start_end(log=logger) def calculate_value(self): - """Calculate value from historical data""" + """Calculate end of day value from historical data""" console.print("\n Calculating returns: ", end="") @@ -1402,56 +1598,65 @@ class PortfolioModel: self.historical_trade_data = trade_data - console.print("\n") - + @log_start_end(log=logger) def calculate_reserves(self): - """_summary_""" + """Takes dividends into account for returns calculation""" # TODO: Add back cash dividends and deduct exchange costs console.print("Still has to be build.") - def calculate_allocations(self): - """Determine allocations based on assets, sectors, countries and regional.""" + @log_start_end(log=logger) + def calculate_allocations(self, category: str): + """Determine allocations based on assets, sectors, countries and region. - # Determine asset allocation - ( - self.benchmark_assets_allocation, - self.portfolio_assets_allocation, - ) = allocation_model.obtain_assets_allocation( - self.benchmark_info, self.portfolio_trades - ) + Parameters + ---------- + category: str + chosen allocation category from asset, sector, country or region - # Determine sector allocation - ( - self.benchmark_sectors_allocation, - self.portfolio_sectors_allocation, - ) = allocation_model.obtain_sector_allocation( - self.benchmark_info, self.portfolio_trades - ) + """ - # Determine regional and country allocations - ( - self.benchmark_regional_allocation, - self.benchmark_country_allocation, - ) = allocation_model.obtain_benchmark_regional_and_country_allocation( - self.benchmark_ticker - ) + if category == "asset": + # Determine asset allocation + ( + self.benchmark_assets_allocation, + self.portfolio_assets_allocation, + ) = allocation_model.get_assets_allocation( + self.benchmark_info, self.portfolio_trades + ) + elif category == "sector": + # Determine sector allocation + ( + self.benchmark_sectors_allocation, + self.portfolio_sectors_allocation, + ) = allocation_model.get_sector_allocation( + self.benchmark_info, self.portfolio_trades + ) + elif category in ("country", "region"): + # Determine region and country allocations + ( + self.benchmark_region_allocation, + self.benchmark_country_allocation, + ) = allocation_model.get_region_country_allocation(self.benchmark_ticker) - ( - self.portfolio_regional_allocation, - self.portfolio_country_allocation, - ) = allocation_model.obtain_portfolio_regional_and_country_allocation( - self.portfolio_trades - ) + ( + self.portfolio_region_allocation, + self.portfolio_country_allocation, + ) = allocation_model.get_portfolio_region_country_allocation( + self.portfolio_trades + ) + @log_start_end(log=logger) def set_risk_free_rate(self, risk_free_rate: float): """Sets risk free rate Parameters ---------- - risk_free (float): risk free rate in decimal format + risk_free (float): risk free rate in float format + """ self.risk_free_rate = risk_free_rate + # Metrics @log_start_end(log=logger) def get_r2_score(self) -> pd.DataFrame: """Class method that retrieves R2 Score for portfolio and benchmark selected diff --git a/openbb_terminal/portfolio/portfolio_view.py b/openbb_terminal/portfolio/portfolio_view.py index 6d2fa4c9a65..f6a1a581028 100644 --- a/openbb_terminal/portfolio/portfolio_view.py +++ b/openbb_terminal/portfolio/portfolio_view.py @@ -65,7 +65,12 @@ In order to load a CSV do the following: @log_start_end(log=logger) -def display_orderbook(portfolio=None, show_index=False): +def display_orderbook( + portfolio=None, + show_index=False, + limit: int = 10, + export: str = "", +): """Display portfolio orderbook Parameters @@ -74,13 +79,29 @@ def display_orderbook(portfolio=None, show_index=False): Instance of Portfolio class show_index: bool Defaults to False. + limit: int + Number of rows to display + export : str + Export certain type of data """ if portfolio.empty: logger.warning("No orderbook loaded") console.print("[red]No orderbook loaded.[/red]\n") else: - print_rich_table(portfolio.get_orderbook(), show_index) + df = portfolio.get_orderbook() + print_rich_table( + df=df[:limit], + show_index=show_index, + title=f"Last {limit if limit < len(df) else len(df)} transactions", + ) + + export_data( + export, + os.path.dirname(os.path.abspath(__file__)), + "transactions", + df.set_index("Date"), + ) @log_start_end(log=logger) @@ -106,54 +127,36 @@ def display_assets_allocation( benchmark_allocation = benchmark_allocation.iloc[:limit] portfolio_allocation = portfolio_allocation.iloc[:limit] - combined = pd.DataFrame() - - for ticker, allocation in portfolio_allocation.items(): - if ticker in benchmark_allocation["symbol"].values: - benchmark_allocation_value = float( - benchmark_allocation[benchmark_allocation["symbol"] == ticker][ - "holdingPercent" - ] - ) - else: - benchmark_allocation_value = 0 - - combined = combined.append( - [ - [ - ticker, - allocation, - benchmark_allocation_value, - allocation - benchmark_allocation_value, - ] - ] - ) - - combined.columns = ["Symbol", "Portfolio", "Benchmark", "Difference"] + combined = pd.merge( + portfolio_allocation, benchmark_allocation, on="Symbol", how="left" + ) + combined["Difference"] = combined["Portfolio"] - combined["Benchmark"] + combined = combined.replace(np.nan, "-") + combined = combined.replace(0, "-") print_rich_table( - combined.replace(0, "-"), + combined, headers=list(combined.columns), title=f"Portfolio vs. Benchmark - Top {len(combined) if len(combined) < limit else limit} Assets Allocation", - floatfmt=[".2f", ".2%", ".2%", ".2%"], + floatfmt=["", ".2%", ".2%", ".2%"], show_index=False, ) if include_separate_tables: print_rich_table( - pd.DataFrame(portfolio_allocation), - headers=list(["Allocation"]), + portfolio_allocation, + headers=list(portfolio_allocation.columns), title=f"Portfolio - Top {len(portfolio_allocation) if len(benchmark_allocation) < limit else limit} " f"Assets Allocation", - floatfmt=[".2%"], - show_index=True, + floatfmt=[".2%", ".2%"], + show_index=False, ) print_rich_table( benchmark_allocation, - headers=list(["Symbol", "Name", "Allocation"]), + headers=list(benchmark_allocation.columns), title=f"Benchmark - Top {len(benchmark_allocation) if len(benchmark_allocation) < limit else limit} " f"Assets Allocation", - floatfmt=[".2f", ".2f", ".2%"], + floatfmt=[".2%", ".2%"], show_index=False, ) @@ -181,6 +184,15 @@ def display_category_allocation( include_separate_tables: bool Whether to include separate asset allocation tables """ + + if benchmark_allocation.empty: + console.print(f"[red]Benchmark data for {category} is empty.\n[/red]") + return + + if portfolio_allocation.empty: + console.print(f"[red]Portfolio data for {category} is empty.\n[/red]") + return + benchmark_allocation = benchmark_allocation.iloc[:limit] portfolio_allocation = portfolio_allocation.iloc[:limit] diff --git a/portfolio/holdings/paexport_degiro.csv b/portfolio/holdings/paexport_degiro.csv deleted file mode 100644 index e69de29bb2d..00000000000