"""Regression Model""" __docformat__ = "numpy" import logging import os import warnings from typing import Any, Dict, List, Tuple import pandas as pd import statsmodels import statsmodels.api as sm from linearmodels import PooledOLS from linearmodels.panel import ( BetweenOLS, FirstDifferenceOLS, PanelOLS, RandomEffects, compare, ) from pandas import DataFrame from statsmodels.api import add_constant from statsmodels.stats.api import het_breuschpagan from statsmodels.stats.diagnostic import acorr_breusch_godfrey from statsmodels.stats.stattools import durbin_watson from openbb_terminal.decorators import log_start_end from openbb_terminal.econometrics.econometrics_helpers import get_datasets from openbb_terminal.helper_funcs import export_data from openbb_terminal.rich_config import console logger = logging.getLogger(__name__) def get_regressions_results( Y: pd.DataFrame, X: pd.DataFrame, regression_type: str = "OLS", entity_effects: bool = False, time_effects: bool = False, ) -> Any: """Based on the regression type, this function decides what regression to run. Parameters ---------- Y: pd.DataFrame Dataframe containing the dependent variable. X: pd.DataFrame Dataframe containing the independent variables. regression_type: str The type of regression you wish to execute. entity_effects: bool Whether to apply Fixed Effects on entities. time_effects: bool Whether to apply Fixed Effects on time. Returns ------- Any A regression model Examples -------- >>> from openbb_terminal.sdk import openbb >>> df = openbb.econometrics.load("wage_panel") >>> df = df.set_index(["nr","year"]) >>> X = df[["exper","educ","union"]] >>> Y = df["lwage"] >>> pooled_ols_model = openbb.econometrics.panel(Y,X,"POLS") >>> print(pooled_ols_model.summary) PooledOLS Estimation Summary ================================================================================ Dep. Variable: lwage R-squared: 0.1634 Estimator: PooledOLS R-squared (Between): 0.1686 No. Observations: 4360 R-squared (Within): 0.1575 Date: Sun, Nov 13 2022 R-squared (Overall): 0.1634 Time: 13:04:02 Log-likelihood -3050.4 Cov. Estimator: Unadjusted F-statistic: 283.68 Entities: 545 P-value 0.0000 Avg Obs: 8.0000 Distribution: F(3,4356) Min Obs: 8.0000 Max Obs: 8.0000 F-statistic (robust): 283.68 P-value 0.0000 Time periods: 8 Distribution: F(3,4356) Avg Obs: 545.00 Min Obs: 545.00 Max Obs: 545.00 Parameter Estimates ============================================================================== Parameter Std. Err. T-stat P-value Lower CI Upper CI ------------------------------------------------------------------------------ const -0.0308 0.0620 -0.4965 0.6196 -0.1523 0.0908 exper 0.0561 0.0028 20.220 0.0000 0.0507 0.0616 educ 0.1080 0.0045 24.034 0.0000 0.0992 0.1168 union 0.1777 0.0172 10.344 0.0000 0.1441 0.2114 ============================================================================== """ regressions = { "OLS": lambda: get_ols(Y, X), "POLS": lambda: get_pols(Y, X), "RE": lambda: get_re(Y, X), "BOLS": lambda: get_bols(Y, X), "FE": lambda: get_fe(Y, X, entity_effects, time_effects), "FDOLS": lambda: get_fdols(Y, X), } if regression_type in regressions: return regressions[regression_type]() return console.print(f"{regression_type} is not an option.") def get_regression_data( regression_variables: List[tuple], data: Dict[str, pd.DataFrame], regression_type: str = "", ) -> Tuple[DataFrame, Any, List[Any]]: """This function creates a DataFrame with the required regression data as well sets up the dependent and independent variables. Parameters ---------- regression_variables : list The regressions variables entered where the first variable is the dependent variable. data : dict A dictionary containing the datasets. regression_type: str The type of regression that is executed. Returns ------- Tuple[DataFrame, Any, List[Any]] The dataset used, Dependent variable, Independent variable, OLS model. """ datasets = get_datasets(data) regression = {} independent_variables = [] dependent_variable = None for variable in regression_variables: column, dataset = datasets[variable].keys() regression[f"{dataset}.{column}"] = data[dataset][column] if variable == regression_variables[0]: dependent_variable = f"{dataset}.{column}" elif variable in regression_variables[1:]: independent_variables.append(f"{dataset}.{column}") regression_df = pd.DataFrame(regression) nan_values = regression_df.isnull().sum().sum() if nan_values > 0: regression_df = regression_df.dropna(how="any", axis="index") if regression_df.empty: console.print( f"The resulting DataFrame only consists of NaN values. This is usually due to an index " f"mismatch. Therefore, no {regression_type} regression can be performed. Consider revisiting your " "dataset(s) and adjust accordingly." ) else: console.print( f"The resulting DataFrame has {nan_values} NaN values. This is usually due to " f"an index mismatch. Rows that contain NaNs are dropped to be able to perform the {regression_type} " f"regression. Consider revisiting your dataset(s) and adjust accordingly." ) return regression_df, dependent_variable, independent_variables @log_start_end(log=logger) def get_ols(Y: pd.DataFrame, X: pd.DataFrame) -> Any: """Performs an OLS regression on timeseries data. [Source: Statsmodels] Parameters ---------- Y: pd.DataFrame Dependent variable series. X: pd.DataFrame Dataframe of independent variables series. Returns ------- statsmodels.regression.linear_model.RegressionResultsWrapper Regression model wrapper from statsmodels. Examples -------- >>> from openbb_terminal.sdk import openbb >>> df = openbb.econometrics.load("wage_panel") >>> OLS_model = openbb.econometrics.ols(df["lwage"], df[["educ", "exper", "expersq"]]) >>> print(OLS_model.summary())` OLS Regression Results ======================================================================================= Dep. Variable: lwage R-squared (uncentered): 0.920 Model: OLS Adj. R-squared (uncentered): 0.919 Method: Least Squares F-statistic: 1.659e+04 Date: Thu, 10 Nov 2022 Prob (F-statistic): 0.00 Time: 15:28:11 Log-Likelihood: -3091.3 No. Observations: 4360 AIC: 6189. Df Residuals: 4357 BIC: 6208. Df Model: 3 Covariance Type: nonrobust ============================================================================== coef std err t P>|t| [0.025 0.975] ------------------------------------------------------------------------------ educ 0.0986 0.002 39.879 0.000 0.094 0.103 exper 0.1018 0.009 10.737 0.000 0.083 0.120 expersq -0.0034 0.001 -4.894 0.000 -0.005 -0.002 ============================================================================== Omnibus: 1249.642 Durbin-Watson: 0.954 Prob(Omnibus): 0.000 Jarque-Bera (JB): 9627.436 Skew: -1.152 Prob(JB): 0.00 Kurtosis: 9.905 Cond. No. 86.4 ============================================================================== Notes: [1] R² is computed without centering (uncentered) since the model does not contain a constant. [2] Standard Errors assume that the covariance matrix of the errors is correctly specified. """ if X.empty or Y.empty: model = None else: with warnings.catch_warnings(record=True) as warning_messages: model = sm.OLS(Y, X).fit() if len(warning_messages) > 0: console.print("Warnings:") for warning in warning_messages: console.print(f"[red]{warning.message}[/red]".replace("\n", "")) return model @log_start_end(log=logger) def get_pols(Y: pd.DataFrame, X: pd.DataFrame) -> Any: """PooledOLS is just plain OLS that understands that various panel data structures. It is useful as a base model. [Source: LinearModels] Parameters ---------- regression_variables : list The regressions variables entered where the first variable is the dependent variable. data : dict A dictionary containing the datasets. Returns ------- Tuple[DataFrame, Any, List[Any], Any] The dataset used, Dependent variable, Independent variable, PooledOLS model """ if Y.empty or X.empty: model = None else: with warnings.catch_warnings(record=True) as warning_messages: exogenous = add_constant(X) model = PooledOLS(Y, exogenous).fit() if len(warning_messages) > 0: console.print("Warnings:") for warning in warning_messages: console.print(f"[red]{warning.message}[/red]".replace("\n", "")) return model @log_start_end(log=logger) def get_re(Y: pd.DataFrame, X: pd.DataFrame) -> Any: """The random effects model is virtually identical to the pooled OLS model except that is accounts for the structure of the model and so is more efficient. Random effects uses a quasi-demeaning strategy which subtracts the time average of the within entity values to account for the common shock. [Source: LinearModels] Parameters ---------- regression_variables : list The regressions variables entered where the first variable is the dependent variable. data : dict A dictionary containing the datasets. Returns ------- Tuple[DataFrame, Any, List[Any], Any] The dataset used, Dependent variable, Independent variable, RandomEffects model """ if X.empty or Y.empty: model = None else: with warnings.catch_warnings(record=True) as warning_messages: exogenous = add_constant(X) model = RandomEffects(Y, exogenous).fit() if len(warning_messages) > 0: console.print("Warnings:") for warning in warning_messages: console.print(f"[red]{warning.message}[/red]".replace("\n", "")) return model @log_start_end(log=logger) def get_bols(Y: pd.DataFrame, X: pd.DataFrame) -> Any: """The between estimator is an alternative, usually less efficient estimator, can can be used to estimate model parameters. It is particular simple since it first computes the time averages of y and x and then runs a simple regression using these averages. [Source: LinearModels] Parameters ---------- regression_variables : list The regressions variables entered where the first variable is the dependent variable. data : dict A dictionary containing the datasets. Returns ------- Tuple[DataFrame, Any, List[Any], Any] The dataset used, Dependent variable, Independent variable, Between OLS model. """ if Y.empty or X.empty: model = None else: with warnings.catch_warnings(record=True) as warning_messages: exogenous = add_constant(X) model = BetweenOLS(Y, exogenous).fit() if len(warning_messages) > 0: console.print("Warnings:") for warning in warning_messages: console.print(f"[red]{warning.message}[/red]".replace("\n", "")) return model @log_start_end(log=logger) def get_fe( Y: pd.DataFrame, X: pd.DataFrame, entity_effects: bool = False, time_effects: bool = False, ) -> Any: """When effects are correlated with the regressors the RE and BE estimators are not consistent. The usual solution is to use Fixed Effects which are called entity_effects when applied to entities and time_effects when applied to the time dimension. [Source: LinearModels] Parameters ---------- regression_variables : list The regressions variables entered where the first variable is the dependent variable. data : dict A dictionary containing the datasets. entity_effects : bool Whether to include entity effects time_effects : bool Whether to include time effects Returns ------- Tuple[DataFrame, Any, List[Any], Any] The dataset used, Dependent variable, Independent variable, PanelOLS model with Fixed Effects """ if X.empty or Y.empty: model = None else: with warnings.catch_warnings(record=True) as warning_messages: exogenous = add_constant(X) model = PanelOLS( Y, exogenous, entity_effects=entity_effects, time_effects=time_effects ).fit() if len(warning_messages) > 0: console.print("Warnings:") for warning in warning_messages: console.print(f"[red]{warning.message}[/red]".replace("\n", "")) return model @log_start_end(log=logger) def get_fdols(Y: pd.DataFrame, X: pd.DataFrame) -> Any: """First differencing is an alternative to using fixed effects when there might be correlation. When using first differences, time-invariant variables must be excluded. Additionally, only one linear time-trending variable can be included since this will look like a constant. This variable will soak up all time-trends in the data, and so interpretations of these variable can be challenging. [Source: LinearModels] Parameters ---------- regression_variables : list The regressions variables entered where the first variable is the dependent variable. data : dict A dictionary containing the datasets. Returns ------- Tuple[DataFrame, Any, List[Any], Any] The dataset used, Dependent variable, Independent variable, First Difference OLS model """ if X.empty or Y.empty: model = None else: with warnings.catch_warnings(record=True) as warning_messages: model = FirstDifferenceOLS(Y, X).fit() if len(warning_messages) > 0: console.print("Warnings:") for warning in warning_messages: console.print(f"[red]{warning.message}[/red]".replace("\n", "")) return model @log_start_end(log=logger) def get_comparison(regressions: Dict, export: str = ""): """Compare regression results between Panel Data regressions. Parameters ---------- regressions : Dict Dictionary with regression results. export : str Format to export data Returns ------- dict Returns a PanelModelComparison which shows an overview of the different regression results. """ comparison = {} for regression_type, data in regressions.items(): if regression_type == "OLS": continue if data["model"]: comparison[regression_type] = data["model"] if not comparison: # When the dictionary is empty, it means no Panel regression # estimates are available and thus the function will have no output return console.print( "No Panel regression estimates available. Please use the " "command 'panel' before using this command." ) comparison_result = compare(comparison) console.print(comparison_result) if export: results_as_html = comparison_result.summary.tables[0].as_html() df = pd.read_html(results_as_html, header=0, index_col=0)[0] export_data( export, os.path.dirname(os.path.abspath(__file__)), "regressions_compare", df, ) return comparison_result @log_start_end(log=logger) def get_dwat( model: statsmodels.regression.linear_model.RegressionResultsWrapper, ) -> float: """Calculate test statistics for Durbin Watson autocorrelation Parameters ---------- model : statsmodels.regression.linear_model.RegressionResultsWrapper Previously fit statsmodels OLS. Returns ------- float Test statistic of the Durbin Watson test. Notes ------ When using chart = True, the dependent variable in the regression must be passed to view the residuals Example ------- >>> from openbb_terminal.sdk import openbb >>> df = openbb.econometrics.load("wage_panel") >>> Y, X = df["lwage"], df[["exper","educ"]] >>> model = openbb.econometrics.ols(Y,X) >>> durbin_watson_value = openbb.econometrics.dwat(model) 0.96 """ # Durbin Watson test: The test statistic is approximately equal to 2*(1-r) where r is the # sample autocorrelation of the residuals. Thus, for r == 0, indicating no serial correlation, # the test statistic equals 2. This statistic will always be between 0 and 4. The closer # to 0 the statistic, the more evidence for positive serial correlation. The closer to 4, # the more evidence for negative serial correlation. result = durbin_watson(model.resid) return round(result, 2) @log_start_end(log=logger) def get_bgod(model: pd.DataFrame, lags: int = 3) -> Tuple[float, float, float, float]: """Calculate test statistics for autocorrelation Parameters ---------- model : OLS Model OLS model that has been fit. lags : int The amount of lags. Returns ------- pd.DataFrame Test results from the Breusch-Godfrey Test """ lm_stat, p_value, f_stat, fp_value = acorr_breusch_godfrey(model, nlags=lags) return pd.DataFrame( [lm_stat, p_value, f_stat, fp_value], index=["lm-stat", "p-value", "f-stat", "fp-value"], ) @log_start_end(log=logger) def get_bpag( model: statsmodels.regression.linear_model.RegressionResultsWrapper, ) -> pd.DataFrame: """Calculate test statistics for heteroscedasticity Parameters ---------- model : OLS Model Model containing residual values. Returns ------- pd.DataFrame Test results from the Breusch-Pagan Test """ lm_stat, p_value, f_stat, fp_value = het_breuschpagan(model.resid, model.model.exog) return pd.DataFrame( [lm_stat, p_value, f_stat, fp_value], index=["lm-stat", "p-value", "f-stat", "fp-value"], )