From 3b8d6a4e8912141d0bfdc0f4bbf495ce995b50b0 Mon Sep 17 00:00:00 2001 From: jeroen Date: Thu, 23 Feb 2023 01:36:45 +0100 Subject: [PATCH] Add FinanceDatabase v2 --- openbb_terminal/economy/economy_controller.py | 4 +- openbb_terminal/economy/yfinance_model.py | 12 +- openbb_terminal/etf/financedatabase_model.py | 19 ++- openbb_terminal/etf/financedatabase_view.py | 65 ++++----- .../stocks/discovery/financedatabase_view.py | 15 ++- .../stocks/fundamental_analysis/dcf_model.py | 7 +- .../financedatabase_model.py | 120 +++++++++-------- .../financedatabase_view.py | 30 +++-- .../sia_controller.py | 126 ++++++++++-------- .../stockanalysis_view.py | 3 +- openbb_terminal/stocks/stocks_controller.py | 32 ++++- openbb_terminal/stocks/stocks_helper.py | 58 ++++---- .../tradinghours/tradinghours_controller.py | 12 +- .../tradinghours/tradinghours_helper.py | 2 +- requirements-full.txt | 2 +- requirements.txt | 2 +- 16 files changed, 279 insertions(+), 230 deletions(-) diff --git a/openbb_terminal/economy/economy_controller.py b/openbb_terminal/economy/economy_controller.py index c2de753d052..9758079d00a 100644 --- a/openbb_terminal/economy/economy_controller.py +++ b/openbb_terminal/economy/economy_controller.py @@ -841,6 +841,7 @@ class EconomyController(BaseController): "-q", "--query", type=str, + nargs="+", dest="query", help="Search for indices with given keyword", ) @@ -864,7 +865,8 @@ class EconomyController(BaseController): if ns_parser: indices = list_from_str(ns_parser.indices) if ns_parser.query and ns_parser.limit: - yfinance_view.search_indices(ns_parser.query, ns_parser.limit) + query = " ".join(ns_parser.query) + yfinance_view.search_indices(query, ns_parser.limit) return self.queue if ns_parser.show_indices: diff --git a/openbb_terminal/economy/yfinance_model.py b/openbb_terminal/economy/yfinance_model.py index 360662e5487..179dbc29e53 100644 --- a/openbb_terminal/economy/yfinance_model.py +++ b/openbb_terminal/economy/yfinance_model.py @@ -738,10 +738,16 @@ def get_search_indices(keyword: list, limit: int = 10) -> pd.DataFrame: keyword.replace(",", " ") if isinstance(keyword, str) else " ".join(keyword) # type: ignore ) - indices = fd.select_indices() + indices = fd.Indices() - queried_indices = pd.DataFrame.from_dict( - fd.search_products(indices, keyword_adjusted, "short_name"), orient="index" + queried_indices = indices.search( + short_name=keyword_adjusted, exclude_exchanges=True + ) + queried_indices = pd.concat( + [ + queried_indices, + indices.data[indices.data.index.str.contains(keyword_adjusted)], + ] ) queried_indices = queried_indices.iloc[:limit] diff --git a/openbb_terminal/etf/financedatabase_model.py b/openbb_terminal/etf/financedatabase_model.py index f456553d014..d02ff6e4d52 100644 --- a/openbb_terminal/etf/financedatabase_model.py +++ b/openbb_terminal/etf/financedatabase_model.py @@ -5,11 +5,14 @@ import logging from typing import Dict, List import financedatabase as fd +import pandas as pd from openbb_terminal.decorators import log_start_end logger = logging.getLogger(__name__) +etfs = fd.ETFs() + @log_start_end(log=logger) def get_etfs_by_name(name: str) -> Dict: @@ -25,8 +28,8 @@ def get_etfs_by_name(name: str) -> Dict: data : Dict[str, Any] Dictionary with ETFs that match a certain name """ - data = fd.select_etfs() - data = fd.search_products(data, query=name, search="long_name") + data = etfs.search(long_name=name, exclude_exchanges=True) + data = pd.concat([data, etfs.search(index=name, exclude_exchanges=True)]) return data @@ -46,10 +49,7 @@ def get_etfs_by_description(description: str) -> Dict: data: Dict[str, Any] Dictionary with ETFs that match a certain description """ - data = fd.select_etfs() - data = fd.search_products(data, query=description, search="summary") - - return data + return etfs.search(summary=description, exclude_exchanges=True) @log_start_end(log=logger) @@ -67,9 +67,7 @@ def get_etfs_by_category(category: str) -> Dict: data: Dict[str, Any] Dictionary with ETFs that match a certain description """ - data = fd.select_etfs(category=category) - - return data + return etfs.select(category=category, exclude_exchanges=True) @log_start_end(log=logger) @@ -81,5 +79,4 @@ def get_etfs_categories() -> List[str]: List[str] ETF categories """ - - return fd.show_options("etfs") + return etfs.options("category") diff --git a/openbb_terminal/etf/financedatabase_view.py b/openbb_terminal/etf/financedatabase_view.py index 7b9c33f525c..320c1c1a099 100644 --- a/openbb_terminal/etf/financedatabase_view.py +++ b/openbb_terminal/etf/financedatabase_view.py @@ -5,8 +5,6 @@ import logging import os from typing import Optional -import pandas as pd - from openbb_terminal.decorators import log_start_end from openbb_terminal.etf import financedatabase_model from openbb_terminal.helper_funcs import export_data, print_rich_table @@ -22,7 +20,7 @@ def display_etf_by_name( export: str = "", sheet_name: Optional[str] = None, ): - """Display a selection of ETFs based on name filtered by total assets. [Source: Finance Database] + """Display a selection of ETFs based on name. [Source: Finance Database] Parameters ---------- @@ -36,30 +34,25 @@ def display_etf_by_name( Type of format to export data """ data = financedatabase_model.get_etfs_by_name(name) - if not data: + + if data.empty: console.print("No data was found with that name\n") return - table_data = pd.DataFrame(data).T[ - ["long_name", "family", "category", "total_assets"] - ] - - # Sort by total assets but it is then dropped due to not being completely up to date - table_data_sorted = table_data.sort_values(by="total_assets", ascending=False) - table_data_sorted = table_data_sorted.drop("total_assets", axis=1) + table_data = data[["long_name", "family", "category_group", "category"]] print_rich_table( - table_data_sorted.iloc[:limit], + table_data.iloc[:limit], show_index=True, - headers=["Name", "Family", "Category"], - title="ETFs by Total Assets", + headers=["Name", "Family", "Category Group", "Category"], + title="ETFs", ) export_data( export, os.path.dirname(os.path.abspath(__file__)), "ln_fd", - table_data_sorted, + table_data, sheet_name, ) @@ -71,8 +64,7 @@ def display_etf_by_description( export: str = "", sheet_name: Optional[str] = None, ): - """Display a selection of ETFs based on description filtered by total assets. - [Source: Finance Database] + """Display a selection of ETFs based on description. [Source: Finance Database] Parameters ---------- @@ -86,29 +78,25 @@ def display_etf_by_description( Type of format to export data """ data = financedatabase_model.get_etfs_by_description(description) - if not data: + + if data.empty: console.print("No data was found with that description\n") return - table_data = pd.DataFrame(data).T[ - ["long_name", "family", "category", "total_assets"] - ] - # Sort by total assets but it is then dropped due to not being completely up to date - table_data_sorted = table_data.sort_values(by="total_assets", ascending=False) - table_data_sorted = table_data_sorted.drop("total_assets", axis=1) + table_data = data[["long_name", "family", "category_group", "category"]] print_rich_table( - table_data_sorted.iloc[:limit], + table_data.iloc[:limit], show_index=True, - headers=["Name", "Family", "Category"], - title="ETFs by Total Assets", + headers=["Name", "Family", "Category Group", "Category"], + title="ETFs", ) export_data( export, os.path.dirname(os.path.abspath(__file__)), "ld", - data, + table_data, sheet_name, ) @@ -120,8 +108,7 @@ def display_etf_by_category( export: str = "", sheet_name: Optional[str] = None, ): - """Display a selection of ETFs based on a category filtered by total assets. - [Source: Finance Database] + """Display a selection of ETFs based on a category. [Source: Finance Database] Parameters ---------- @@ -135,28 +122,24 @@ def display_etf_by_category( Type of format to export data """ data = financedatabase_model.get_etfs_by_category(category) - if not data: + + if data.empty: console.print("No data was found on that category\n") return - table_data = pd.DataFrame(data).T[ - ["long_name", "family", "category", "total_assets"] - ] - # Sort by total assets but it is then dropped due to not being completely up to date - table_data_sorted = table_data.sort_values(by="total_assets", ascending=False) - table_data_sorted = table_data_sorted.drop("total_assets", axis=1) + table_data = data[["long_name", "family", "category_group", "category"]] print_rich_table( - table_data_sorted.iloc[:limit], + table_data.iloc[:limit], show_index=True, - headers=["Name", "Family", "Category"], - title="ETFs by Category and Total Assets", + headers=["Name", "Family", "Category Group", "Category"], + title="ETFs by Category", ) export_data( export, os.path.join(os.path.dirname(os.path.abspath(__file__)), "screener"), "sbc", - data, + table_data, sheet_name, ) diff --git a/openbb_terminal/stocks/discovery/financedatabase_view.py b/openbb_terminal/stocks/discovery/financedatabase_view.py index 5ff9968c5fe..50f8d60a4f8 100644 --- a/openbb_terminal/stocks/discovery/financedatabase_view.py +++ b/openbb_terminal/stocks/discovery/financedatabase_view.py @@ -13,6 +13,8 @@ from openbb_terminal.rich_config import console logger = logging.getLogger(__name__) +equities = fd.Equities() + @log_start_end(log=logger) def show_equities( @@ -64,7 +66,7 @@ def show_equities( if industry is not None: industry = " ".join(industry).title() - data = fd.select_equities( + data = equities.search( country=country, sector=sector, industry=industry, @@ -72,19 +74,17 @@ def show_equities( ) if name is not None: - data = fd.search_products(data, query=" ".join(name), search="long_name") + data = equities.search(long_name=" ".join(name)) if description is not None: - data = fd.search_products(data, query=" ".join(description), search="summary") + data = equities.search(summary=" ".join(description)) if marketcap is not None: - data = fd.search_products( - data, query=f"{''.join(marketcap)} Cap", search="market_cap" - ) + data = equities.search(market_cap=f"{''.join(marketcap)} Cap") table_data = pd.DataFrame(data).T[ [ "long_name", "sector", - "industry", + "industry_group" "industry", "country", "city", "website", @@ -98,6 +98,7 @@ def show_equities( headers=[ "Name", "Sector", + "Industry Group", "Industry", "Country", "City", diff --git a/openbb_terminal/stocks/fundamental_analysis/dcf_model.py b/openbb_terminal/stocks/fundamental_analysis/dcf_model.py index dacb2d215a9..2b1cb8c6822 100644 --- a/openbb_terminal/stocks/fundamental_analysis/dcf_model.py +++ b/openbb_terminal/stocks/fundamental_analysis/dcf_model.py @@ -355,11 +355,12 @@ def others_in_sector( industry = industry.replace("—", " - ") industry = industry.replace("/", " ") - similars = fd.select_equities(sector=sector, industry=industry) + equities = fd.Equities() + similars = equities.select(sector=sector, industry=industry) # This filters similars to match market cap and removes ticker analyzed - if symbol in similars: - market_cap = similars[symbol]["market_cap"] + if symbol in similars.index: + market_cap = similars.loc[symbol]["market_cap"] similars.pop(symbol, None) if not no_filter: similars = { diff --git a/openbb_terminal/stocks/sector_industry_analysis/financedatabase_model.py b/openbb_terminal/stocks/sector_industry_analysis/financedatabase_model.py index be428d59927..42e6cd2ef8c 100644 --- a/openbb_terminal/stocks/sector_industry_analysis/financedatabase_model.py +++ b/openbb_terminal/stocks/sector_industry_analysis/financedatabase_model.py @@ -5,7 +5,6 @@ __docformat__ = "numpy" import logging from typing import Optional -import financedatabase as fd import yahooquery as yq from requests.exceptions import ReadTimeout from tqdm import tqdm @@ -17,7 +16,7 @@ logger = logging.getLogger(__name__) @log_start_end(log=logger) -def get_countries(industry: str = "", sector: str = "") -> list: +def get_countries(equities, industry: str = "", sector: str = "") -> list: """Get all countries in Yahoo Finance data based on sector or industry. [Source: Finance Database] Parameters @@ -34,15 +33,15 @@ def get_countries(industry: str = "", sector: str = "") -> list: """ # industry takes priority since there's 1 sector per industry, but multiple industries per sector if industry: - return fd.show_options("equities", industry=True)[industry]["Countries"] + return equities.search(industry=industry)["country"].dropna().unique() if sector: - return fd.show_options("equities", sector=sector)["Countries"] + return equities.search(sector=sector)["country"].dropna().unique() - return [count for count in fd.show_options("equities", "countries") if count] + return equities.select()["country"].dropna().unique() @log_start_end(log=logger) -def get_sectors(industry: str = "", country: str = "") -> list: +def get_sectors(equities, industry: str = "", country: str = "") -> list: """Get all sectors in Yahoo Finance data based on country or industry. [Source: Finance Database] Parameters @@ -59,21 +58,21 @@ def get_sectors(industry: str = "", country: str = "") -> list: """ # industry takes priority since there's 1 sector per industry, but multiple industries per country if industry: - return [fd.show_options("equities", industry=True)[industry]["Sector"]] + return equities.search(industry=industry)["sector"].dropna().unique() if country: - return fd.show_options("equities", country=country)["Sectors"] + return equities.search(country=country)["sector"].dropna().unique() - return [sect for sect in fd.show_options("equities", "sectors") if sect] + return equities.select()["sector"].dropna().unique() @log_start_end(log=logger) -def get_industries(country: str = "", sector: str = "") -> list: +def get_industries(equities, country: str = "", sector: str = "") -> list: """Get all industries in Yahoo Finance data based on country or sector. [Source: Finance Database] Parameters ---------- country : str - Filter retrieved industries by country + Filter retrieved industries by countrys sector : str Filter retrieved industries by sector @@ -83,15 +82,19 @@ def get_industries(country: str = "", sector: str = "") -> list: List of possible industries """ if country and sector: - return fd.show_options("equities", country=country, sector=sector) + return ( + equities.select(country=country, sector=sector)["industry"] + .dropna() + .unique() + ) if country: - return fd.show_options("equities", country=country)["Industries"] + return equities.select(country=country)["industry"].dropna().unique() if sector: - return fd.show_options("equities", sector=sector)["Industries"] + return equities.select(sector=sector)["industry"].dropna().unique() - return [ind for ind in fd.show_options("equities", "industries") if ind] + return equities.select()["industry"].dropna().unique() @log_start_end(log=logger) @@ -108,6 +111,7 @@ def get_marketcap() -> list: @log_start_end(log=logger) def filter_stocks( + equities, country: Optional[str] = None, sector: Optional[str] = None, industry: Optional[str] = None, @@ -142,13 +146,14 @@ def filter_stocks( kwargs["sector"] = sector if industry: kwargs["industry"] = industry + if exclude_exchanges: + kwargs["exclude_exchanges"] = exclude_exchanges + if marketcap: + kwargs["market_cap"] = marketcap try: - data = fd.select_equities(exclude_exchanges=exclude_exchanges, **kwargs) + data = equities.search(**kwargs) - if marketcap: - data = fd.search_products(data, query=marketcap, search="market_cap") - - return list(data.keys()) + return list(data.index) except ValueError as e: logger.exception(str(e)) @@ -189,10 +194,11 @@ def get_json(symbol: str) -> dict: @log_start_end(log=logger) def get_stocks_data( + equities, country: str = "United States", - sector: str = "Communication Services", - industry: str = "Internet Content & Information", - marketcap: str = "Mega Cap", + sector: str = "Materials", + industry: str = "Metals & Mining", + marketcap: str = "Large Cap", exclude_exchanges: bool = True, ) -> dict: """Get stocks data based on country, sector, industry, market cap and exclude exchanges. @@ -216,7 +222,9 @@ def get_stocks_data( dict Dictionary of filtered stocks data """ - stocks = filter_stocks(country, sector, industry, marketcap, exclude_exchanges) + stocks = filter_stocks( + equities, country, sector, industry, marketcap, exclude_exchanges + ) stocks_data = {symbol: get_json(symbol=symbol) for symbol in tqdm(stocks)} @@ -225,6 +233,7 @@ def get_stocks_data( @log_start_end(log=logger) def get_companies_per_sector_in_country( + equities, country: str = "United States", mktcap: str = "Large", exclude_exchanges: bool = True, @@ -247,16 +256,15 @@ def get_companies_per_sector_in_country( """ companies_per_sector = {} - for sector in tqdm(get_sectors(country=country)): + for sector in tqdm(get_sectors(equities, country=country)): if sector: try: - companies = fd.select_equities( - country=country, sector=sector, exclude_exchanges=exclude_exchanges + companies = equities.search( + country=country, + sector=sector, + exclude_exchanges=exclude_exchanges, + market_cap=f"{mktcap} Cap", ) - if mktcap: - companies = fd.search_products( - companies, query=mktcap + " Cap", search="market_cap" - ) companies_per_sector[sector] = len(companies) except ValueError as e: @@ -275,6 +283,7 @@ def get_companies_per_sector_in_country( @log_start_end(log=logger) def get_companies_per_industry_in_country( + equities, country: str = "United States", mktcap: str = "Large", exclude_exchanges: bool = True, @@ -298,18 +307,15 @@ def get_companies_per_industry_in_country( """ companies_per_industry = {} - for industry in tqdm(get_industries(country=country)): + for industry in tqdm(get_industries(equities, country=country)): if industry: try: - companies = fd.select_equities( + companies = equities.search( country=country, industry=industry, exclude_exchanges=exclude_exchanges, + market_cap=f"{mktcap} Cap", ) - if mktcap: - companies = fd.search_products( - companies, query=mktcap + " Cap", search="market_cap" - ) companies_per_industry[industry] = len(companies) @@ -329,7 +335,10 @@ def get_companies_per_industry_in_country( @log_start_end(log=logger) def get_companies_per_industry_in_sector( - sector: str = "Technology", mktcap: str = "Large", exclude_exchanges: bool = True + equities, + sector: str = "Technology", + mktcap: str = "Large", + exclude_exchanges: bool = True, ) -> dict: """Get number of companies per industry in a specific sector (and specific market cap). [Source: Finance Database] @@ -349,18 +358,15 @@ def get_companies_per_industry_in_sector( Dictionary of industries and number of companies in a specific sector """ companies_per_industry = {} - for industry in tqdm(get_industries(sector=sector)): + for industry in tqdm(get_industries(equities, sector=sector)): if industry: try: - companies = fd.select_equities( + companies = equities.search( sector=sector, industry=industry, exclude_exchanges=exclude_exchanges, + market_cap=f"{mktcap} Cap", ) - if mktcap: - companies = fd.search_products( - companies, query=mktcap + " Cap", search="market_cap" - ) companies_per_industry[industry] = len(companies) @@ -380,7 +386,10 @@ def get_companies_per_industry_in_sector( @log_start_end(log=logger) def get_companies_per_country_in_sector( - sector: str = "Technology", mktcap: str = "Large", exclude_exchanges: bool = True + equities, + sector: str = "Technology", + mktcap: str = "Large", + exclude_exchanges: bool = True, ) -> dict: """Get number of companies per country in a specific sector (and specific market cap). [Source: Finance Database] @@ -400,18 +409,15 @@ def get_companies_per_country_in_sector( Dictionary of countries and number of companies in a specific sector """ companies_per_country = {} - for country in tqdm(get_countries(sector=sector)): + for country in tqdm(get_countries(equities, sector=sector)): if country: try: - companies = fd.select_equities( + companies = equities.search( sector=sector, country=country, exclude_exchanges=exclude_exchanges, + market_cap=f"{mktcap} Cap", ) - if mktcap: - companies = fd.search_products( - companies, query=mktcap + " Cap", search="market_cap" - ) companies_per_country[country] = len(companies) @@ -431,7 +437,8 @@ def get_companies_per_country_in_sector( @log_start_end(log=logger) def get_companies_per_country_in_industry( - industry: str = "Internet Content & Information", + equities, + industry: str = "Metals & Mining", mktcap: str = "Large", exclude_exchanges: bool = True, ) -> dict: @@ -453,18 +460,15 @@ def get_companies_per_country_in_industry( Dictionary of countries and number of companies in a specific sector """ companies_per_country = {} - for country in tqdm(get_countries(industry=industry)): + for country in tqdm(get_countries(equities, industry=industry)): if country: try: - companies = fd.select_equities( + companies = equities.search( industry=industry, country=country, exclude_exchanges=exclude_exchanges, + market_cap=f"{mktcap} Cap", ) - if mktcap: - companies = fd.search_products( - companies, query=mktcap + " Cap", search="market_cap" - ) companies_per_country[country] = len(companies) diff --git a/openbb_terminal/stocks/sector_industry_analysis/financedatabase_view.py b/openbb_terminal/stocks/sector_industry_analysis/financedatabase_view.py index 03787121316..6605d682cd7 100644 --- a/openbb_terminal/stocks/sector_industry_analysis/financedatabase_view.py +++ b/openbb_terminal/stocks/sector_industry_analysis/financedatabase_view.py @@ -28,12 +28,13 @@ logger = logging.getLogger(__name__) @log_start_end(log=logger) def display_bars_financials( + equities, finance_key: str = "financialData", finance_metric: str = "ebitda", country: str = "United States", - sector: str = "Communication Services", - industry: str = "Internet Content & Information", - marketcap: str = "Mega Cap", + sector: str = "Materials", + industry: str = "Metals & Mining", + marketcap: str = "Large Cap", exclude_exchanges: bool = True, limit: int = 10, export: str = "", @@ -84,7 +85,7 @@ def display_bars_financials( stocks_data = already_loaded_stocks_data else: stocks_data = financedatabase_model.get_stocks_data( - country, sector, industry, marketcap, exclude_exchanges + equities, country, sector, industry, marketcap, exclude_exchanges ) metric_data = {} @@ -238,6 +239,7 @@ def display_bars_financials( @log_start_end(log=logger) def display_companies_per_sector_in_country( + equities, country: str = "United States", mktcap: str = "Large", exclude_exchanges: bool = True, @@ -272,7 +274,7 @@ def display_companies_per_sector_in_country( External axes (1 axis is expected in the list), by default None """ companies_per_sector = financedatabase_model.get_companies_per_sector_in_country( - country, mktcap, exclude_exchanges + equities, country, mktcap, exclude_exchanges ) companies_per_sector = dict( @@ -379,6 +381,7 @@ def display_companies_per_sector_in_country( @log_start_end(log=logger) def display_companies_per_industry_in_country( + equities, country: str = "United States", mktcap: str = "Large", exclude_exchanges: bool = True, @@ -415,7 +418,7 @@ def display_companies_per_industry_in_country( """ companies_per_industry = ( financedatabase_model.get_companies_per_industry_in_country( - country, mktcap, exclude_exchanges + equities, country, mktcap, exclude_exchanges ) ) @@ -530,7 +533,8 @@ def display_companies_per_industry_in_country( @log_start_end(log=logger) def display_companies_per_industry_in_sector( - sector: str = "Technology", + equities, + sector: str = "Materials", mktcap: str = "Large", exclude_exchanges: bool = True, export: str = "", @@ -564,7 +568,7 @@ def display_companies_per_industry_in_sector( External axes (1 axis is expected in the list), by default None """ companies_per_industry = financedatabase_model.get_companies_per_industry_in_sector( - sector, mktcap, exclude_exchanges + equities, sector, mktcap, exclude_exchanges ) companies_per_industry = dict( @@ -682,7 +686,8 @@ def display_companies_per_industry_in_sector( @log_start_end(log=logger) def display_companies_per_country_in_sector( - sector: str = "Technology", + equities, + sector: str = "Materials", mktcap: str = "Large", exclude_exchanges: bool = True, export: str = "", @@ -716,7 +721,7 @@ def display_companies_per_country_in_sector( External axes (1 axis is expected in the list), by default None """ companies_per_country = financedatabase_model.get_companies_per_country_in_sector( - sector, mktcap, exclude_exchanges + equities, sector, mktcap, exclude_exchanges ) companies_per_country = dict( @@ -827,7 +832,8 @@ def display_companies_per_country_in_sector( @log_start_end(log=logger) def display_companies_per_country_in_industry( - industry: str = "Internet Content & Information", + equities, + industry: str = "Metals & Mining", mktcap: str = "Large", exclude_exchanges: bool = True, export: str = "", @@ -861,7 +867,7 @@ def display_companies_per_country_in_industry( External axes (1 axis is expected in the list), by default None """ companies_per_country = financedatabase_model.get_companies_per_country_in_industry( - industry, mktcap, exclude_exchanges + equities, industry, mktcap, exclude_exchanges ) companies_per_country = dict( diff --git a/openbb_terminal/stocks/sector_industry_analysis/sia_controller.py b/openbb_terminal/stocks/sector_industry_analysis/sia_controller.py index 5cdaed300ca..871002f4e1c 100644 --- a/openbb_terminal/stocks/sector_industry_analysis/sia_controller.py +++ b/openbb_terminal/stocks/sector_industry_analysis/sia_controller.py @@ -6,6 +6,7 @@ import difflib import logging from typing import List, Optional +import financedatabase as fd import yfinance as yf from openbb_terminal import feature_flags as obbff @@ -167,6 +168,9 @@ class SectorIndustryAnalysisController(BaseController): """Constructor""" super().__init__(queue) + # This obtains all equities + self.equities = fd.Equities() + self.country = "" self.sector = "" self.industry = "" @@ -195,19 +199,19 @@ class SectorIndustryAnalysisController(BaseController): self.choices["industry"] = { i: {} for i in financedatabase_model.get_industries( - country=self.country, sector=self.sector + country=self.country, sector=self.sector, equities=self.equities ) } self.choices["sector"] = { s: {} for s in financedatabase_model.get_sectors( - industry=self.industry, country=self.country + industry=self.industry, country=self.country, equities=self.equities ) } self.choices["country"] = { c: {} for c in financedatabase_model.get_countries( - industry=self.industry, sector=self.sector + industry=self.industry, sector=self.sector, equities=self.equities ) } self.completer = NestedCompleter.from_nested_dict(self.choices) @@ -314,8 +318,10 @@ class SectorIndustryAnalysisController(BaseController): f"Failed to load {self.ticker} Industry from Yahoo Finance" ) - self.country = data["summaryProfile"]["country"] - if self.country not in financedatabase_model.get_countries(): + self.country = self.equities.select().loc[self.ticker, "country"] + if self.country not in financedatabase_model.get_countries( + equities=self.equities + ): similar_cmd = difflib.get_close_matches( self.country, financedatabase_model.get_countries(), @@ -324,21 +330,27 @@ class SectorIndustryAnalysisController(BaseController): ) if similar_cmd: self.country = similar_cmd[0] - self.sector = data["summaryProfile"]["sector"] - if self.sector not in financedatabase_model.get_sectors(): + self.sector = self.equities.select().loc[self.ticker, "sector"] + if self.sector not in financedatabase_model.get_sectors( + equities=self.equities + ): + print(self.sector) + print(financedatabase_model.get_sectors(equities=self.equities)) similar_cmd = difflib.get_close_matches( self.sector, - financedatabase_model.get_sectors(), + financedatabase_model.get_sectors(equities=self.equities), n=1, cutoff=0.7, ) if similar_cmd: self.sector = similar_cmd[0] - self.industry = data["summaryProfile"]["industry"] - if self.industry not in financedatabase_model.get_industries(): + self.industry = self.equities.select().loc[self.ticker, "industry"] + if self.industry not in financedatabase_model.get_industries( + equities=self.equities + ): similar_cmd = difflib.get_close_matches( self.industry, - financedatabase_model.get_industries(), + financedatabase_model.get_industries(equities=self.equities), n=1, cutoff=0.7, ) @@ -387,15 +399,14 @@ class SectorIndustryAnalysisController(BaseController): if ns_parser: possible_industries = financedatabase_model.get_industries( - country=self.country, - sector=self.sector, + country=self.country, sector=self.sector, equities=self.equities ) if ns_parser.name: if " ".join(ns_parser.name) in possible_industries: self.industry = " ".join(ns_parser.name) # if we get the industry, then we also automatically know the sector self.sector = financedatabase_model.get_sectors( - industry=self.industry + industry=self.industry, equities=self.equities )[0] self.update_runtime_choices() else: @@ -413,7 +424,7 @@ class SectorIndustryAnalysisController(BaseController): self.industry = similar_cmd[0] # if we get the industry, then we also automatically know the sector self.sector = financedatabase_model.get_sectors( - industry=self.industry + industry=self.industry, equities=self.equities )[0] self.update_runtime_choices() else: @@ -454,7 +465,7 @@ class SectorIndustryAnalysisController(BaseController): ns_parser = self.parse_known_args_and_warn(parser, other_args) if ns_parser: possible_sectors = financedatabase_model.get_sectors( - self.industry, self.country + self.industry, self.country, equities=self.equities ) if ns_parser.name: if " ".join(ns_parser.name) in possible_sectors: @@ -515,7 +526,7 @@ class SectorIndustryAnalysisController(BaseController): ns_parser = self.parse_known_args_and_warn(parser, other_args) if ns_parser: possible_countries = financedatabase_model.get_countries( - industry=self.industry, sector=self.sector + industry=self.industry, sector=self.sector, equities=self.equities ) if ns_parser.name: if " ".join(ns_parser.name) in possible_countries: @@ -759,6 +770,7 @@ class SectorIndustryAnalysisController(BaseController): self.stocks_data, self.tickers, ) = financedatabase_view.display_bars_financials( + self.equities, self.metric_yf_keys[ns_parser.metric][0], self.metric_yf_keys[ns_parser.metric][1], self.country, @@ -849,6 +861,7 @@ class SectorIndustryAnalysisController(BaseController): self.stocks_data, self.tickers, ) = stockanalysis_view.display_plots_financials( + equities=self.equities, finance_key=ns_parser.metric, country=self.country, sector=self.sector, @@ -916,14 +929,17 @@ class SectorIndustryAnalysisController(BaseController): console.print("The country parameter needs to be selected!\n") else: financedatabase_view.display_companies_per_sector_in_country( - self.country, - self.mktcap, - self.exclude_exchanges, - ns_parser.export, - " ".join(ns_parser.sheet_name) if ns_parser.sheet_name else None, - ns_parser.raw, - ns_parser.max_sectors_to_display, - ns_parser.min_pct_to_display_sector, + equities=self.equities, + country=self.country, + mktcap=self.mktcap, + exclude_exchanges=self.exclude_exchanges, + export=ns_parser.export, + sheet_name=" ".join(ns_parser.sheet_name) + if ns_parser.sheet_name + else None, + raw=ns_parser.raw, + max_sectors_to_display=ns_parser.max_sectors_to_display, + min_pct_to_display_sector=ns_parser.min_pct_to_display_sector, ) @log_start_end(log=logger) @@ -968,13 +984,14 @@ class SectorIndustryAnalysisController(BaseController): console.print("The country parameter needs to be selected!\n") else: financedatabase_view.display_companies_per_industry_in_country( - self.country, - self.mktcap, - self.exclude_exchanges, - ns_parser.export, - ns_parser.raw, - ns_parser.max_industries_to_display, - ns_parser.min_pct_to_display_industry, + equities=self.equities, + country=self.country, + mktcap=self.mktcap, + exclude_exchanges=self.exclude_exchanges, + export=ns_parser.export, + raw=ns_parser.raw, + max_industries_to_display=ns_parser.max_industries_to_display, + min_pct_to_display_industry=ns_parser.min_pct_to_display_industry, ) @log_start_end(log=logger) @@ -1019,13 +1036,14 @@ class SectorIndustryAnalysisController(BaseController): console.print("The sector parameter needs to be selected!\n") else: financedatabase_view.display_companies_per_industry_in_sector( - self.sector, - self.mktcap, - self.exclude_exchanges, - ns_parser.export, - ns_parser.raw, - ns_parser.max_industries_to_display, - ns_parser.min_pct_to_display_industry, + equities=self.equities, + sector=self.sector, + mktcap=self.mktcap, + exclude_exchanges=self.exclude_exchanges, + export=ns_parser.export, + raw=ns_parser.raw, + max_industries_to_display=ns_parser.max_industries_to_display, + min_pct_to_display_industry=ns_parser.min_pct_to_display_industry, ) @log_start_end(log=logger) @@ -1070,13 +1088,14 @@ class SectorIndustryAnalysisController(BaseController): console.print("The sector parameter needs to be selected!\n") else: financedatabase_view.display_companies_per_country_in_sector( - self.sector, - self.mktcap, - self.exclude_exchanges, - ns_parser.export, - ns_parser.raw, - ns_parser.max_countries_to_display, - ns_parser.min_pct_to_display_country, + equities=self.equities, + sector=self.sector, + mktcap=self.mktcap, + exclude_exchanges=self.exclude_exchanges, + export=ns_parser.export, + raw=ns_parser.raw, + max_countries_to_display=ns_parser.max_countries_to_display, + min_pct_to_display_country=ns_parser.min_pct_to_display_country, ) @log_start_end(log=logger) @@ -1121,13 +1140,14 @@ class SectorIndustryAnalysisController(BaseController): console.print("The industry parameter needs to be selected!\n") else: financedatabase_view.display_companies_per_country_in_industry( - self.industry, - self.mktcap, - self.exclude_exchanges, - ns_parser.export, - ns_parser.raw, - ns_parser.max_countries_to_display, - ns_parser.min_pct_to_display_country, + equities=self.equities, + industry=self.industry, + mktcap=self.mktcap, + exclude_exchanges=self.exclude_exchanges, + export=ns_parser.export, + raw=ns_parser.raw, + max_countries_to_display=ns_parser.max_countries_to_display, + min_pct_to_display_country=ns_parser.min_pct_to_display_country, ) @log_start_end(log=logger) diff --git a/openbb_terminal/stocks/sector_industry_analysis/stockanalysis_view.py b/openbb_terminal/stocks/sector_industry_analysis/stockanalysis_view.py index 175a162d94f..72f82858887 100644 --- a/openbb_terminal/stocks/sector_industry_analysis/stockanalysis_view.py +++ b/openbb_terminal/stocks/sector_industry_analysis/stockanalysis_view.py @@ -32,6 +32,7 @@ logger = logging.getLogger(__name__) @log_start_end(log=logger) def display_plots_financials( + equities, finance_key: str = "re", country: str = "United States", sector: str = "Communication Services", @@ -103,7 +104,7 @@ def display_plots_financials( stocks_data = already_loaded_stocks_data else: company_tickers = filter_stocks( - country, sector, industry, marketcap, exclude_exchanges + equities, country, sector, industry, marketcap, exclude_exchanges ) if len(company_tickers) <= 1: diff --git a/openbb_terminal/stocks/stocks_controller.py b/openbb_terminal/stocks/stocks_controller.py index f11a4386909..a550c106a7a 100644 --- a/openbb_terminal/stocks/stocks_controller.py +++ b/openbb_terminal/stocks/stocks_controller.py @@ -7,7 +7,7 @@ import os from datetime import datetime, timedelta from typing import List, Optional -import financedatabase +import pandas as pd import yfinance as yf from openbb_terminal import feature_flags as obbff @@ -69,9 +69,19 @@ class StocksController(StockBaseController): FILE_PATH = os.path.join(os.path.dirname(__file__), "README.md") try: - country = financedatabase.show_options("equities", "countries") - sector = financedatabase.show_options("equities", "sectors") - industry = financedatabase.show_options("equities", "industries") + database_url = "https://raw.githubusercontent.com/JerBouma/FinanceDatabase/main/Database/Categories" + country = pd.read_csv(f"{database_url}/equities_countries.csv", header=None)[ + 0 + ].values.tolist() + sector = pd.read_csv(f"{database_url}/equities_sectors.csv", header=None)[ + 0 + ].values.tolist() + industry_group = pd.read_csv( + f"{database_url}/equities_industry_groups.csv", header=None + )[0].values.tolist() + industry = pd.read_csv(f"{database_url}/equities_industries.csv", header=None)[ + 0 + ].values.tolist() except Exception: country, sector, industry = {}, {}, {} console.print( @@ -191,6 +201,16 @@ class StocksController(StockBaseController): dest="sector", help="Search by sector to find stocks matching the criteria", ) + parser.add_argument( + "-g", + "--industry-group", + default="", + choices=stocks_helper.format_parse_choices(self.industry_group), + type=str.lower, + metavar="industry_group", + dest="industry_group", + help="Search by industry group to find stocks matching the criteria", + ) parser.add_argument( "-i", "--industry", @@ -236,6 +256,9 @@ class StocksController(StockBaseController): industry = stocks_helper.map_parse_choices(self.industry)[ ns_parser.industry ] + industry_group = stocks_helper.map_parse_choices(self.industry_group)[ + ns_parser.industry_group + ] exchange = stocks_helper.map_parse_choices( list(stocks_helper.market_coverage_suffix.keys()) )[ns_parser.exchange_country] @@ -244,6 +267,7 @@ class StocksController(StockBaseController): query=" ".join(ns_parser.query), country=ns_parser.country, sector=sector, + industry_group=industry_group, industry=industry, exchange_country=exchange, all_exchanges=ns_parser.all_exchanges, diff --git a/openbb_terminal/stocks/stocks_helper.py b/openbb_terminal/stocks/stocks_helper.py index 5c39d7e8cba..d08b5ce4be2 100644 --- a/openbb_terminal/stocks/stocks_helper.py +++ b/openbb_terminal/stocks/stocks_helper.py @@ -104,6 +104,7 @@ def search( query: str = "", country: str = "", sector: str = "", + industry_group: str = "", industry: str = "", exchange_country: str = "", all_exchanges: bool = False, @@ -146,10 +147,17 @@ def search( kwargs["sector"] = sector if industry: kwargs["industry"] = industry + if industry_group: + kwargs["industry_group"] = industry_group kwargs["exclude_exchanges"] = False if exchange_country else not all_exchanges try: - data = fd.select_equities(**kwargs) + equities = fd.Equities() + if query: + data = equities.search(**kwargs, long_name=query) + data.update(equities.search(**kwargs, short_name=query)) + else: + data = equities.search(**kwargs) except ReadTimeout: console.print( "[red]Unable to retrieve company data from GitHub which limits the search" @@ -162,33 +170,23 @@ def search( "[red]No companies were found that match the given criteria.[/red]\n" ) return pd.DataFrame() - if not data: + + if data.empty: console.print("No companies found.\n") return pd.DataFrame() - if query: - d = fd.search_products( - data, query, search="long_name", case_sensitive=False, new_database=None - ) - d.update( - fd.search_products( - data, - query, - search="short_name", - case_sensitive=False, - new_database=None, - ) - ) - else: - d = data - - if not d: - console.print("No companies found.\n") - return pd.DataFrame() - - df = pd.DataFrame.from_dict(d).T[ - ["long_name", "short_name", "country", "sector", "industry", "exchange"] + df = data[ + [ + "long_name", + "short_name", + "country", + "sector", + "industry_group", + "industry", + "exchange", + ] ] + if exchange_country and exchange_country in market_coverage_suffix: suffix_tickers = [ ticker.split(".")[1] if "." in ticker else "" for ticker in list(df.index) @@ -202,8 +200,10 @@ def search( for x in v: exchange_suffix[x] = k - df["name"] = df["long_name"].combine_first(df["short_name"]) - df = df[["name", "country", "sector", "industry", "exchange"]] + with pd.option_context("mode.chained_assignment", None): + df["name"] = df["long_name"].combine_first(df["short_name"]) + + df = df[["name", "country", "sector", "industry_group", "industry", "exchange"]] title = "Companies found" if query: @@ -214,8 +214,12 @@ def search( title += f" in {country.replace('_', ' ').title()}" if sector: title += f" within {sector}" + if industry_group: + title += f" and {industry_group}" if industry: title += f" and {industry}" + if not sector and industry_group: + title += f" within {industry_group}" if not sector and industry: title += f" within {industry}" @@ -225,7 +229,7 @@ def search( print_rich_table( df.iloc[:limit] if limit else df, show_index=True, - headers=["Name", "Country", "Sector", "Industry", "Exchange"], + headers=["Name", "Country", "Sector", "Industry Group", "Industry", "Exchange"], title=title, ) diff --git a/openbb_terminal/stocks/tradinghours/tradinghours_controller.py b/openbb_terminal/stocks/tradinghours/tradinghours_controller.py index 6831bebaefc..29f5f4af1bc 100644 --- a/openbb_terminal/stocks/tradinghours/tradinghours_controller.py +++ b/openbb_terminal/stocks/tradinghours/tradinghours_controller.py @@ -62,10 +62,10 @@ class TradingHoursController(BaseController): self.exchange = None if ticker: - if ticker in self.equities: + if ticker in self.equities.index: self.symbol = ticker - self.symbol_name = self.equities[ticker]["short_name"] - self.exchange = self.equities[ticker]["exchange"] + self.symbol_name = self.equities.loc[ticker]["short_name"] + self.exchange = self.equities.loc[ticker]["exchange"] open_ex = get_open() if self.exchange in open_ex.index: self.symbol_market_open = True @@ -132,9 +132,9 @@ class TradingHoursController(BaseController): ns_parser = self.parse_known_args_and_warn(parser, other_args) if ns_parser: self.symbol = ns_parser.symbol - if ns_parser.symbol in self.equities: - self.symbol_name = self.equities[self.symbol]["short_name"] - self.exchange = self.equities[self.symbol]["exchange"] + if ns_parser.symbol in self.equities.index: + self.symbol_name = self.equities.loc[self.symbol]["short_name"] + self.exchange = self.equities.loc[self.symbol]["exchange"] open_ex = get_open() if self.exchange in open_ex.index: self.symbol_market_open = True diff --git a/openbb_terminal/stocks/tradinghours/tradinghours_helper.py b/openbb_terminal/stocks/tradinghours/tradinghours_helper.py index 5d65081d2b8..76b93774f1b 100644 --- a/openbb_terminal/stocks/tradinghours/tradinghours_helper.py +++ b/openbb_terminal/stocks/tradinghours/tradinghours_helper.py @@ -12,7 +12,7 @@ logger = logging.getLogger(__name__) @log_start_end(log=logger) def get_fd_equities_list() -> List: """Load FD list of equity symbols.""" - equities = fd.select_equities(exclude_exchanges=False) + equities = fd.Equities().select(exclude_exchanges=False) return equities diff --git a/requirements-full.txt b/requirements-full.txt index 964d0656c20..2eb2abac67a 100644 --- a/requirements-full.txt +++ b/requirements-full.txt @@ -78,7 +78,7 @@ fastjsonschema==2.16.2 ; python_version >= "3.8" and python_full_version != "3.9 feedparser==6.0.10 ; python_version >= "3.8" and python_full_version != "3.9.7" and python_version < "3.11" ffn==0.3.6 ; python_version >= "3.8" and python_full_version != "3.9.7" and python_version < "3.11" filelock==3.9.0 ; python_version >= "3.8" and python_full_version != "3.9.7" and python_version < "3.11" -financedatabase==1.0.2 ; python_version >= "3.8" and python_full_version != "3.9.7" and python_version < "3.11" +financedatabase==2.0.2 ; python_version >= "3.8" and python_full_version != "3.9.7" and python_version < "3.11" finnhub-python==2.4.16 ; python_version >= "3.8" and python_full_version != "3.9.7" and python_version < "3.11" finviz==1.4.4 ; python_version >= "3.8" and python_full_version != "3.9.7" and python_version < "3.11" finvizfinance==0.14.5 ; python_version >= "3.8" and python_full_version != "3.9.7" and python_version < "3.11" diff --git a/requirements.txt b/requirements.txt index a40de683e77..456efcfb8db 100644 --- a/requirements.txt +++ b/requirements.txt @@ -54,7 +54,7 @@ executing==1.2.0 ; python_version >= "3.8" and python_full_version != "3.9.7" an fastjsonschema==2.16.2 ; python_version >= "3.8" and python_full_version != "3.9.7" and python_version < "3.11" feedparser==6.0.10 ; python_version >= "3.8" and python_full_version != "3.9.7" and python_version < "3.11" ffn==0.3.6 ; python_version >= "3.8" and python_full_version != "3.9.7" and python_version < "3.11" -financedatabase==1.0.2 ; python_version >= "3.8" and python_full_version != "3.9.7" and python_version < "3.11" +financedatabase==2.0.2 ; python_version >= "3.8" and python_full_version != "3.9.7" and python_version < "3.11" finnhub-python==2.4.16 ; python_version >= "3.8" and python_full_version != "3.9.7" and python_version < "3.11" finviz==1.4.4 ; python_version >= "3.8" and python_full_version != "3.9.7" and python_version < "3.11" finvizfinance==0.14.5 ; python_version >= "3.8" and python_full_version != "3.9.7" and python_version < "3.11"