jerry/OpenBB
1
0
Fork 0
mirror of https://github.com/OpenBB-finance/OpenBB.git synced 2026-10-30 05:52:23 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
07e8faee5dc00aebf448d5ee35914d8ecb86c674
OpenBB/openbb_terminal/econometrics/regression_model.py
james d73dfd3980 fixes for mypy 1.0
2023-02-08 19:14:25 -05:00

574 lines
20 KiB
Python
Raw Blame History

"""Regression Model"""
__docformat__ = "numpy"
import logging
import os
import warnings
from typing import Any, Dict, List, Optional, Tuple
import pandas as pd
import statsmodels
import statsmodels.api as sm
from linearmodels import PooledOLS
from linearmodels.panel import (
BetweenOLS,
FirstDifferenceOLS,
PanelOLS,
RandomEffects,
compare,
)
from pandas import DataFrame
from statsmodels.api import add_constant
from statsmodels.stats.api import het_breuschpagan
from statsmodels.stats.diagnostic import acorr_breusch_godfrey
from statsmodels.stats.stattools import durbin_watson
from openbb_terminal.decorators import log_start_end
from openbb_terminal.econometrics.econometrics_helpers import get_datasets
from openbb_terminal.helper_funcs import export_data
from openbb_terminal.rich_config import console
logger = logging.getLogger(__name__)
def get_regressions_results(
Y: pd.DataFrame,
X: pd.DataFrame,
regression_type: str = "OLS",
entity_effects: bool = False,
time_effects: bool = False,
) -> Any:
"""Based on the regression type, this function decides what regression to run.
Parameters
----------
Y: pd.DataFrame
Dataframe containing the dependent variable.
X: pd.DataFrame
Dataframe containing the independent variables.
regression_type: str
The type of regression you wish to execute.
entity_effects: bool
Whether to apply Fixed Effects on entities.
time_effects: bool
Whether to apply Fixed Effects on time.
Returns
-------
Any
A regression model
Examples
--------
>>> from openbb_terminal.sdk import openbb
>>> df = openbb.econometrics.load("wage_panel")
>>> df = df.set_index(["nr","year"])
>>> X = df[["exper","educ","union"]]
>>> Y = df["lwage"]
>>> pooled_ols_model = openbb.econometrics.panel(Y,X,"POLS")
>>> print(pooled_ols_model.summary)
PooledOLS Estimation Summary
================================================================================
Dep. Variable: lwage R-squared: 0.1634
Estimator: PooledOLS R-squared (Between): 0.1686
No. Observations: 4360 R-squared (Within): 0.1575
Date: Sun, Nov 13 2022 R-squared (Overall): 0.1634
Time: 13:04:02 Log-likelihood -3050.4
Cov. Estimator: Unadjusted
F-statistic: 283.68
Entities: 545 P-value 0.0000
Avg Obs: 8.0000 Distribution: F(3,4356)
Min Obs: 8.0000
Max Obs: 8.0000 F-statistic (robust): 283.68
P-value 0.0000
Time periods: 8 Distribution: F(3,4356)
Avg Obs: 545.00
Min Obs: 545.00
Max Obs: 545.00
Parameter Estimates
==============================================================================
Parameter Std. Err. T-stat P-value Lower CI Upper CI
------------------------------------------------------------------------------
const -0.0308 0.0620 -0.4965 0.6196 -0.1523 0.0908
exper 0.0561 0.0028 20.220 0.0000 0.0507 0.0616
educ 0.1080 0.0045 24.034 0.0000 0.0992 0.1168
union 0.1777 0.0172 10.344 0.0000 0.1441 0.2114
==============================================================================
"""
regressions = {
"OLS": lambda: get_ols(Y, X),
"POLS": lambda: get_pols(Y, X),
"RE": lambda: get_re(Y, X),
"BOLS": lambda: get_bols(Y, X),
"FE": lambda: get_fe(Y, X, entity_effects, time_effects),
"FDOLS": lambda: get_fdols(Y, X),
}
if regression_type in regressions:
return regressions[regression_type]()
return console.print(f"{regression_type} is not an option.")
def get_regression_data(
regression_variables: List[tuple],
data: Dict[str, pd.DataFrame],
regression_type: str = "",
) -> Tuple[DataFrame, Any, List[Any]]:
"""This function creates a DataFrame with the required regression data as
well sets up the dependent and independent variables.
Parameters
----------
regression_variables : list
The regressions variables entered where the first variable is
the dependent variable.
data : dict
A dictionary containing the datasets.
regression_type: str
The type of regression that is executed.
Returns
-------
Tuple[DataFrame, Any, List[Any]]
The dataset used,
Dependent variable,
Independent variable,
OLS model.
"""
datasets = get_datasets(data)
regression = {}
independent_variables = []
dependent_variable = None
for variable in regression_variables:
column, dataset = datasets[variable].keys()
regression[f"{dataset}.{column}"] = data[dataset][column]
if variable == regression_variables[0]:
dependent_variable = f"{dataset}.{column}"
elif variable in regression_variables[1:]:
independent_variables.append(f"{dataset}.{column}")
regression_df = pd.DataFrame(regression)
nan_values = regression_df.isnull().sum().sum()
if nan_values > 0:
regression_df = regression_df.dropna(how="any", axis="index")
if regression_df.empty:
console.print(
f"The resulting DataFrame only consists of NaN values. This is usually due to an index "
f"mismatch. Therefore, no {regression_type} regression can be performed. Consider revisiting your "
"dataset(s) and adjust accordingly."
)
else:
console.print(
f"The resulting DataFrame has {nan_values} NaN values. This is usually due to "
f"an index mismatch. Rows that contain NaNs are dropped to be able to perform the {regression_type} "
f"regression. Consider revisiting your dataset(s) and adjust accordingly."
)
return regression_df, dependent_variable, independent_variables
@log_start_end(log=logger)
def get_ols(Y: pd.DataFrame, X: pd.DataFrame) -> Any:
"""Performs an OLS regression on timeseries data. [Source: Statsmodels]
Parameters
----------
Y: pd.DataFrame
Dependent variable series.
X: pd.DataFrame
Dataframe of independent variables series.
Returns
-------
statsmodels.regression.linear_model.RegressionResultsWrapper
Regression model wrapper from statsmodels.
Examples
--------
>>> from openbb_terminal.sdk import openbb
>>> df = openbb.econometrics.load("wage_panel")
>>> OLS_model = openbb.econometrics.ols(df["lwage"], df[["educ", "exper", "expersq"]])
>>> print(OLS_model.summary())`
OLS Regression Results
=======================================================================================
Dep. Variable: lwage R-squared (uncentered): 0.920
Model: OLS Adj. R-squared (uncentered): 0.919
Method: Least Squares F-statistic: 1.659e+04
Date: Thu, 10 Nov 2022 Prob (F-statistic): 0.00
Time: 15:28:11 Log-Likelihood: -3091.3
No. Observations: 4360 AIC: 6189.
Df Residuals: 4357 BIC: 6208.
Df Model: 3
Covariance Type: nonrobust
==============================================================================
coef std err t P>|t| [0.025 0.975]
------------------------------------------------------------------------------
educ 0.0986 0.002 39.879 0.000 0.094 0.103
exper 0.1018 0.009 10.737 0.000 0.083 0.120
expersq -0.0034 0.001 -4.894 0.000 -0.005 -0.002
==============================================================================
Omnibus: 1249.642 Durbin-Watson: 0.954
Prob(Omnibus): 0.000 Jarque-Bera (JB): 9627.436
Skew: -1.152 Prob(JB): 0.00
Kurtosis: 9.905 Cond. No. 86.4
==============================================================================
Notes:
[1] R² is computed without centering (uncentered) since the model does not contain a constant.
[2] Standard Errors assume that the covariance matrix of the errors is correctly specified.
"""
if X.empty or Y.empty:
model = None
else:
with warnings.catch_warnings(record=True) as warning_messages:
model = sm.OLS(Y, X).fit()
if len(warning_messages) > 0:
console.print("Warnings:")
for warning in warning_messages:
console.print(f"[red]{warning.message}[/red]".replace("\n", ""))
return model
@log_start_end(log=logger)
def get_pols(Y: pd.DataFrame, X: pd.DataFrame) -> Any:
"""PooledOLS is just plain OLS that understands that various panel data structures.
It is useful as a base model. [Source: LinearModels]
Parameters
----------
regression_variables : list
The regressions variables entered where the first variable is
the dependent variable.
data : dict
A dictionary containing the datasets.
Returns
-------
Tuple[DataFrame, Any, List[Any], Any]
The dataset used,
Dependent variable,
Independent variable,
PooledOLS model
"""
if Y.empty or X.empty:
model = None
else:
with warnings.catch_warnings(record=True) as warning_messages:
exogenous = add_constant(X)
model = PooledOLS(Y, exogenous).fit()
if len(warning_messages) > 0:
console.print("Warnings:")
for warning in warning_messages:
console.print(f"[red]{warning.message}[/red]".replace("\n", ""))
return model
@log_start_end(log=logger)
def get_re(Y: pd.DataFrame, X: pd.DataFrame) -> Any:
"""The random effects model is virtually identical to the pooled OLS model except that is accounts for the
structure of the model and so is more efficient. Random effects uses a quasi-demeaning strategy which
subtracts the time average of the within entity values to account for the common shock. [Source: LinearModels]
Parameters
----------
regression_variables : list
The regressions variables entered where the first variable is
the dependent variable.
data : dict
A dictionary containing the datasets.
Returns
-------
Tuple[DataFrame, Any, List[Any], Any]
The dataset used,
Dependent variable,
Independent variable,
RandomEffects model
"""
if X.empty or Y.empty:
model = None
else:
with warnings.catch_warnings(record=True) as warning_messages:
exogenous = add_constant(X)
model = RandomEffects(Y, exogenous).fit()
if len(warning_messages) > 0:
console.print("Warnings:")
for warning in warning_messages:
console.print(f"[red]{warning.message}[/red]".replace("\n", ""))
return model
@log_start_end(log=logger)
def get_bols(Y: pd.DataFrame, X: pd.DataFrame) -> Any:
"""The between estimator is an alternative, usually less efficient estimator, can can be used to
estimate model parameters. It is particular simple since it first computes the time averages of
y and x and then runs a simple regression using these averages. [Source: LinearModels]
Parameters
----------
regression_variables : list
The regressions variables entered where the first variable is
the dependent variable.
data : dict
A dictionary containing the datasets.
Returns
-------
Tuple[DataFrame, Any, List[Any], Any]
The dataset used,
Dependent variable,
Independent variable,
Between OLS model.
"""
if Y.empty or X.empty:
model = None
else:
with warnings.catch_warnings(record=True) as warning_messages:
exogenous = add_constant(X)
model = BetweenOLS(Y, exogenous).fit()
if len(warning_messages) > 0:
console.print("Warnings:")
for warning in warning_messages:
console.print(f"[red]{warning.message}[/red]".replace("\n", ""))
return model
@log_start_end(log=logger)
def get_fe(
Y: pd.DataFrame,
X: pd.DataFrame,
entity_effects: bool = False,
time_effects: bool = False,
) -> Any:
"""When effects are correlated with the regressors the RE and BE estimators are not consistent.
The usual solution is to use Fixed Effects which are called entity_effects when applied to
entities and time_effects when applied to the time dimension. [Source: LinearModels]
Parameters
----------
regression_variables : list
The regressions variables entered where the first variable is
the dependent variable.
data : dict
A dictionary containing the datasets.
entity_effects : bool
Whether to include entity effects
time_effects : bool
Whether to include time effects
Returns
-------
Tuple[DataFrame, Any, List[Any], Any]
The dataset used,
Dependent variable,
Independent variable,
PanelOLS model with Fixed Effects
"""
if X.empty or Y.empty:
model = None
else:
with warnings.catch_warnings(record=True) as warning_messages:
exogenous = add_constant(X)
model = PanelOLS(
Y, exogenous, entity_effects=entity_effects, time_effects=time_effects
).fit()
if len(warning_messages) > 0:
console.print("Warnings:")
for warning in warning_messages:
console.print(f"[red]{warning.message}[/red]".replace("\n", ""))
return model
@log_start_end(log=logger)
def get_fdols(Y: pd.DataFrame, X: pd.DataFrame) -> Any:
"""First differencing is an alternative to using fixed effects when there might be correlation.
When using first differences, time-invariant variables must be excluded. Additionally,
only one linear time-trending variable can be included since this will look like a constant.
This variable will soak up all time-trends in the data, and so interpretations of
these variable can be challenging. [Source: LinearModels]
Parameters
----------
regression_variables : list
The regressions variables entered where the first variable is
the dependent variable.
data : dict
A dictionary containing the datasets.
Returns
-------
Tuple[DataFrame, Any, List[Any], Any]
The dataset used,
Dependent variable,
Independent variable,
First Difference OLS model
"""
if X.empty or Y.empty:
model = None
else:
with warnings.catch_warnings(record=True) as warning_messages:
model = FirstDifferenceOLS(Y, X).fit()
if len(warning_messages) > 0:
console.print("Warnings:")
for warning in warning_messages:
console.print(f"[red]{warning.message}[/red]".replace("\n", ""))
return model
@log_start_end(log=logger)
def get_comparison(
regressions: Dict, export: str = "", sheet_name: Optional[str] = None
):
"""Compare regression results between Panel Data regressions.
Parameters
----------
regressions : Dict
Dictionary with regression results.
export : str
Format to export data
Returns
-------
dict
Returns a PanelModelComparison which shows an overview of the different regression results.
"""
comparison = {}
for regression_type, data in regressions.items():
if regression_type == "OLS":
continue
if data["model"]:
comparison[regression_type] = data["model"]
if not comparison:
# When the dictionary is empty, it means no Panel regression
# estimates are available and thus the function will have no output
return console.print(
"No Panel regression estimates available. Please use the "
"command 'panel' before using this command."
)
comparison_result = compare(comparison)
console.print(comparison_result)
if export:
results_as_html = comparison_result.summary.tables[0].as_html()
df = pd.read_html(results_as_html, header=0, index_col=0)[0]
export_data(
export,
os.path.dirname(os.path.abspath(__file__)),
"regressions_compare",
df,
sheet_name,
)
return comparison_result
@log_start_end(log=logger)
def get_dwat(
model: statsmodels.regression.linear_model.RegressionResultsWrapper,
) -> float:
"""Calculate test statistics for Durbin Watson autocorrelation
Parameters
----------
model : statsmodels.regression.linear_model.RegressionResultsWrapper
Previously fit statsmodels OLS.
Returns
-------
float
Test statistic of the Durbin Watson test.
Notes
------
When using chart = True, the dependent variable in the regression must be passed to view the residuals
Example
-------
>>> from openbb_terminal.sdk import openbb
>>> df = openbb.econometrics.load("wage_panel")
>>> Y, X = df["lwage"], df[["exper","educ"]]
>>> model = openbb.econometrics.ols(Y,X)
>>> durbin_watson_value = openbb.econometrics.dwat(model)
0.96
"""
# Durbin Watson test: The test statistic is approximately equal to 2*(1-r) where r is the
# sample autocorrelation of the residuals. Thus, for r == 0, indicating no serial correlation,
# the test statistic equals 2. This statistic will always be between 0 and 4. The closer
# to 0 the statistic, the more evidence for positive serial correlation. The closer to 4,
# the more evidence for negative serial correlation.
result = durbin_watson(model.resid)
return round(result, 2)
@log_start_end(log=logger)
def get_bgod(model: pd.DataFrame, lags: int = 3) -> Tuple[float, float, float, float]:
"""Calculate test statistics for autocorrelation
Parameters
----------
model : OLS Model
OLS model that has been fit.
lags : int
The amount of lags.
Returns
-------
pd.DataFrame
Test results from the Breusch-Godfrey Test
"""
lm_stat, p_value, f_stat, fp_value = acorr_breusch_godfrey(model, nlags=lags)
return pd.DataFrame(
[lm_stat, p_value, f_stat, fp_value],
index=["lm-stat", "p-value", "f-stat", "fp-value"],
)
@log_start_end(log=logger)
def get_bpag(
model: statsmodels.regression.linear_model.RegressionResultsWrapper,
) -> pd.DataFrame:
"""Calculate test statistics for heteroscedasticity
Parameters
----------
model : OLS Model
Model containing residual values.
Returns
-------
pd.DataFrame
Test results from the Breusch-Pagan Test
"""
lm_stat, p_value, f_stat, fp_value = het_breuschpagan(model.resid, model.model.exog)
return pd.DataFrame(
[lm_stat, p_value, f_stat, fp_value],
index=["lm-stat", "p-value", "f-stat", "fp-value"],
)
Reference in New Issue View Git Blame Copy Permalink