From e367530d44c9c5d50dbfe6a404cf726935bc5204 Mon Sep 17 00:00:00 2001 From: James Maslek Date: Sat, 21 Jan 2023 09:17:20 -0500 Subject: [PATCH] Feature/excel sheets (#4023) * bump * Add sheet name to export for base load function * Disable pylint due to known issue with pylint and pandas * Add robust option if file already exists * Abstract the sheet name argument into the export data helper * Feature flag for auto overwrite * add to featflag menu * Add featflag to builds * Remove safety check * Fix test --- .github/workflows/intel_macos_build.yml | 1 + .github/workflows/linting.yml | 2 +- .github/workflows/m1_macos_build.yml | 5 +- .github/workflows/mac.yml | 1 + .github/workflows/nightly-build.yml | 5 +- .github/workflows/test.yml | 1 + .github/workflows/unit-test.yml | 1 + .github/workflows/windows.yml | 1 + build/conda/conda-3-9-env-full.yaml | 1 + openbb_terminal/featflags_controller.py | 7 ++ openbb_terminal/feature_flags.py | 3 + openbb_terminal/helper_funcs.py | 77 ++++++++++++++++++- openbb_terminal/parent_classes.py | 13 ++++ .../test_tracking_error.csv | 4 +- 14 files changed, 111 insertions(+), 11 deletions(-) diff --git a/.github/workflows/intel_macos_build.yml b/.github/workflows/intel_macos_build.yml index bdb01afcd4e..5e274e00dbe 100644 --- a/.github/workflows/intel_macos_build.yml +++ b/.github/workflows/intel_macos_build.yml @@ -5,6 +5,7 @@ env: OPENBB_LOG_COLLECT: false OPENBB_USE_ION: false OPENBB_USE_PROMPT_TOOLKIT: false + OPENBB_FILE_OVERWRITE: true PIP_DEFAULT_TIMEOUT: 100 PYTHONNOUSERSITE: 1 diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index ba3ccdabc20..d6edf443d61 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -5,6 +5,7 @@ env: OPENBB_LOG_COLLECT: false OPENBB_USE_ION: false OPENBB_USE_PROMPT_TOOLKIT: false + OPENBB_FILE_OVERWRITE: true PIP_DEFAULT_TIMEOUT: 100 on: @@ -48,7 +49,6 @@ jobs: - run: ruff . - run: mypy --ignore-missing-imports openbb_terminal - run: shopt -s globstar && pyupgrade --py38-plus **/*.py - - run: safety check - run: | pylint terminal.py openbb_terminal tests diff --git a/.github/workflows/m1_macos_build.yml b/.github/workflows/m1_macos_build.yml index 19131d86899..24fa3beb712 100644 --- a/.github/workflows/m1_macos_build.yml +++ b/.github/workflows/m1_macos_build.yml @@ -5,6 +5,7 @@ env: OPENBB_LOG_COLLECT: false OPENBB_USE_ION: false OPENBB_USE_PROMPT_TOOLKIT: false + OPENBB_FILE_OVERWRITE: true PIP_DEFAULT_TIMEOUT: 100 PYTHONNOUSERSITE: 1 @@ -58,10 +59,10 @@ jobs: - name: Install Pyinstaller run: poetry install -E installer - + - name: Poetry Install Portfolio Optimization and Forecasting Toolkits run: poetry install -E all - + - name: Install Specific Papermill run: | pip uninstall papermill -y diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index 440df2f0971..2bca5fa0910 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -5,6 +5,7 @@ env: OPENBB_USE_ION: false OPENBB_USE_PROMPT_TOOLKIT: false PIP_DEFAULT_TIMEOUT: 100 + OPENBB_FILE_OVERWRITE: true PYTHONNOUSERSITE: 1 on: push: diff --git a/.github/workflows/nightly-build.yml b/.github/workflows/nightly-build.yml index 42e734dab2e..ce8e87e7d76 100644 --- a/.github/workflows/nightly-build.yml +++ b/.github/workflows/nightly-build.yml @@ -6,6 +6,7 @@ env: OPENBB_USE_ION: false OPENBB_USE_PROMPT_TOOLKIT: false PIP_DEFAULT_TIMEOUT: 100 + OPENBB_FILE_OVERWRITE: true PYTHONNOUSERSITE: 1 on: @@ -24,7 +25,7 @@ jobs: uses: benc-uk/workflow-dispatch@v1 with: workflow: windows.yml - + trigger-macos-build: runs-on: ubuntu-latest steps: @@ -32,7 +33,7 @@ jobs: uses: benc-uk/workflow-dispatch@v1 with: workflow: m1_macos_build.yml - + trigger-intel-build: runs-on: ubuntu-latest steps: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8491355f1c6..c6cd57ebaf5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -4,6 +4,7 @@ env: OPENBB_LOG_COLLECT: false OPENBB_USE_ION: false OPENBB_USE_PROMPT_TOOLKIT: false + OPENBB_FILE_OVERWRITE: true PIP_DEFAULT_TIMEOUT: 100 on: [workflow_dispatch] diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index fd0ae41914a..c0aaa4e260f 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -5,6 +5,7 @@ env: OPENBB_LOG_COLLECT: false OPENBB_USE_ION: false OPENBB_USE_PROMPT_TOOLKIT: false + OPENBB_FILE_OVERWRITE: true PIP_DEFAULT_TIMEOUT: 100 on: diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 5b470bdd0fc..6c5763f28b4 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -4,6 +4,7 @@ env: OPENBB_LOG_COLLECT: false OPENBB_USE_ION: false OPENBB_USE_PROMPT_TOOLKIT: false + OPENBB_FILE_OVERWRITE: true PIP_DEFAULT_TIMEOUT: 100 PYTHONNOUSERSITE: 1 on: diff --git a/build/conda/conda-3-9-env-full.yaml b/build/conda/conda-3-9-env-full.yaml index 33392f4c800..65e559f63a6 100644 --- a/build/conda/conda-3-9-env-full.yaml +++ b/build/conda/conda-3-9-env-full.yaml @@ -6,3 +6,4 @@ dependencies: - pip - poetry=1.1.13 - lightgbm=3.3.3 + - charset-normalizer=2.1.1 diff --git a/openbb_terminal/featflags_controller.py b/openbb_terminal/featflags_controller.py index 8470d349766..b99a7040d83 100644 --- a/openbb_terminal/featflags_controller.py +++ b/openbb_terminal/featflags_controller.py @@ -42,6 +42,7 @@ class FeatureFlagsController(BaseController): "rich", "richpanel", "tbhint", + "overwrite", ] PATH = "/featflags/" @@ -73,9 +74,15 @@ class FeatureFlagsController(BaseController): mt.add_setting("watermark", obbff.USE_WATERMARK) mt.add_setting("cmdloc", obbff.USE_CMD_LOCATION_FIGURE) mt.add_setting("tbhint", obbff.TOOLBAR_HINT) + mt.add_setting("overwrite", obbff.FILE_OVERWITE) console.print(text=mt.menu_text, menu="Feature Flags") + def call_overwrite(self, _): + """Process overwrite command""" + obbff.FILE_OVERWITE = not obbff.FILE_OVERWITE + set_key(USER_ENV_FILE, "OPENBB_FILE_OVERWITE", str(obbff.FILE_OVERWITE)) + def call_retryload(self, _): """Process retryload command""" obbff.RETRY_WITH_LOAD = not obbff.RETRY_WITH_LOAD diff --git a/openbb_terminal/feature_flags.py b/openbb_terminal/feature_flags.py index 4673c17652f..86da3e1787f 100644 --- a/openbb_terminal/feature_flags.py +++ b/openbb_terminal/feature_flags.py @@ -31,6 +31,9 @@ else: WITH_GIT = True +# Always overwrite exported files +FILE_OVERWITE = load_env_vars("OPENBB_FILE_OVERWRITE", strtobool, False, "featflags") + # Retry unknown commands with `load` RETRY_WITH_LOAD = load_env_vars("OPENBB_RETRY_WITH_LOAD", strtobool, False, "featflags") diff --git a/openbb_terminal/helper_funcs.py b/openbb_terminal/helper_funcs.py index 031eaa74442..87f92001c66 100644 --- a/openbb_terminal/helper_funcs.py +++ b/openbb_terminal/helper_funcs.py @@ -5,7 +5,7 @@ import argparse import io import logging from pathlib import Path -from typing import List, Union, Optional, Dict +from typing import List, Union, Optional, Dict, Tuple from functools import lru_cache from datetime import datetime, timedelta from datetime import date as d @@ -1322,8 +1322,34 @@ def compose_export_path(func_name: str, dir_path: str) -> Path: return full_path +def ask_file_overwrite(file_path: str) -> Tuple[bool, bool]: + """Helper to provide a prompt for overwriting existing files. + + Returns two values, the first is a boolean indicating if the file exists and the + second is a boolean indicating if the user wants to overwrite the file. + """ + # Jeroen asked for a flag to overwrite no matter what + if obbff.FILE_OVERWITE: + return False, True + if os.path.exists(file_path): + overwrite = input("\nFile already exists. Overwrite? [y/n]: ").lower() + if overwrite == "y": + # File exists and user wants to overwrite + return True, True + # File exists and user does not want to overwrite + return True, False + # File does not exist + return False, True + + +# This is a false positive on pylint and being tracked in pylint #3060 +# pylint: disable=abstract-class-instantiated def export_data( - export_type: str, dir_path: str, func_name: str, df: pd.DataFrame = pd.DataFrame() + export_type: str, + dir_path: str, + func_name: str, + df: pd.DataFrame = pd.DataFrame(), + sheet_name: str = None, ) -> None: """Export data to a file. @@ -1337,6 +1363,8 @@ def export_data( Name of the command that invokes this function df : pd.Dataframe Dataframe of data to save + sheet_name : str + If provided. The name of the sheet to save in excel file """ if export_type: export_path = compose_export_path(func_name, dir_path) @@ -1372,19 +1400,62 @@ def export_data( ) if exp_type.endswith("csv"): + exists, overwrite = ask_file_overwrite(saved_path) + if exists and not overwrite: + return df.to_csv(saved_path) elif exp_type.endswith("json"): + exists, overwrite = ask_file_overwrite(saved_path) + if exists and not overwrite: + return df.reset_index(drop=True, inplace=True) df.to_json(saved_path) elif exp_type.endswith("xlsx"): - df.to_excel(saved_path, index=True, header=True) + + if sheet_name is None: + exists, overwrite = ask_file_overwrite(saved_path) + if exists and not overwrite: + return + df.to_excel(saved_path, index=True, header=True) + + else: + if os.path.exists(saved_path): + with pd.ExcelWriter( + saved_path, + mode="a", + if_sheet_exists="new", + engine="openpyxl", + ) as writer: + df.to_excel( + writer, sheet_name=sheet_name, index=True, header=True + ) + else: + with pd.ExcelWriter( + saved_path, + engine="openpyxl", + ) as writer: + df.to_excel( + writer, sheet_name=sheet_name, index=True, header=True + ) elif exp_type.endswith("png"): + exists, overwrite = ask_file_overwrite(saved_path) + if exists and not overwrite: + return plt.savefig(saved_path) elif exp_type.endswith("jpg"): + exists, overwrite = ask_file_overwrite(saved_path) + if exists and not overwrite: + return plt.savefig(saved_path) elif exp_type.endswith("pdf"): + exists, overwrite = ask_file_overwrite(saved_path) + if exists and not overwrite: + return plt.savefig(saved_path) elif exp_type.endswith("svg"): + exists, overwrite = ask_file_overwrite(saved_path) + if exists and not overwrite: + return plt.savefig(saved_path) else: console.print("\nWrong export file specified.") diff --git a/openbb_terminal/parent_classes.py b/openbb_terminal/parent_classes.py index c719edccd7f..5f2e06f746c 100644 --- a/openbb_terminal/parent_classes.py +++ b/openbb_terminal/parent_classes.py @@ -753,6 +753,16 @@ class BaseController(metaclass=ABCMeta): help=help_export, ) + # If excel is an option, add the sheet name + if export_allowed == EXPORT_ONLY_RAW_DATA_ALLOWED: + parser.add_argument( + "--sheet-name", + dest="sheet_name", + default=None, + nargs="+", + help="Name of excel sheet to save data to. Only valid for .xlsx files.", + ) + if raw: parser.add_argument( "--raw", @@ -1147,6 +1157,9 @@ class StockBaseController(BaseController, metaclass=ABCMeta): os.path.dirname(os.path.abspath(__file__)), f"load_{self.ticker}", self.stock.copy(), + sheet_name=" ".join(ns_parser.sheet_name) + if ns_parser.sheet_name + else None, ) diff --git a/tests/openbb_terminal/portfolio/csv/test_portfolio_model/test_tracking_error.csv b/tests/openbb_terminal/portfolio/csv/test_portfolio_model/test_tracking_error.csv index 8dec8d7ca01..c2b7ce9b12e 100644 --- a/tests/openbb_terminal/portfolio/csv/test_portfolio_model/test_tracking_error.csv +++ b/tests/openbb_terminal/portfolio/csv/test_portfolio_model/test_tracking_error.csv @@ -5,9 +5,7 @@ Total NaN dtype: float64" ytd,- 3m,- -6m,"Adj Close NaN -Total NaN -dtype: float64" +6m,- 1y,"Adj Close NaN Total NaN dtype: float64"