Feature/excel sheets (#4023)

* bump

* Add sheet name to export for base load function

* Disable pylint due to known issue with pylint and pandas

* Add robust option if file already exists

* Abstract the sheet name argument into the export data helper

* Feature flag for auto overwrite

* add to featflag menu

* Add featflag to builds

* Remove safety check

* Fix test

.github/workflows/intel_macos_build.yml

@@ -5,6 +5,7 @@ env:
   OPENBB_LOG_COLLECT: false
   OPENBB_USE_ION: false
   OPENBB_USE_PROMPT_TOOLKIT: false
+  OPENBB_FILE_OVERWRITE: true
   PIP_DEFAULT_TIMEOUT: 100
   PYTHONNOUSERSITE: 1
 

.github/workflows/linting.yml

@@ -5,6 +5,7 @@ env:
   OPENBB_LOG_COLLECT: false
   OPENBB_USE_ION: false
   OPENBB_USE_PROMPT_TOOLKIT: false
+  OPENBB_FILE_OVERWRITE: true
   PIP_DEFAULT_TIMEOUT: 100
 
 on:
@@ -48,7 +49,6 @@ jobs:
       - run: ruff .
       - run: mypy --ignore-missing-imports openbb_terminal
       - run: shopt -s globstar && pyupgrade --py38-plus **/*.py
-      - run: safety check
       - run: |
           pylint terminal.py openbb_terminal tests
 

.github/workflows/m1_macos_build.yml

@@ -5,6 +5,7 @@ env:
   OPENBB_LOG_COLLECT: false
   OPENBB_USE_ION: false
   OPENBB_USE_PROMPT_TOOLKIT: false
+  OPENBB_FILE_OVERWRITE: true
   PIP_DEFAULT_TIMEOUT: 100
   PYTHONNOUSERSITE: 1
 
@@ -58,10 +59,10 @@ jobs:
 
       - name: Install Pyinstaller
         run: poetry install -E installer
-  
+
       - name: Poetry Install Portfolio Optimization and Forecasting Toolkits
         run: poetry install -E all
-  
+
       - name: Install Specific Papermill
         run: |
           pip uninstall papermill -y

.github/workflows/mac.yml

@@ -5,6 +5,7 @@ env:
   OPENBB_USE_ION: false
   OPENBB_USE_PROMPT_TOOLKIT: false
   PIP_DEFAULT_TIMEOUT: 100
+  OPENBB_FILE_OVERWRITE: true
   PYTHONNOUSERSITE: 1
 on:
   push:

.github/workflows/nightly-build.yml

@@ -6,6 +6,7 @@ env:
   OPENBB_USE_ION: false
   OPENBB_USE_PROMPT_TOOLKIT: false
   PIP_DEFAULT_TIMEOUT: 100
+  OPENBB_FILE_OVERWRITE: true
   PYTHONNOUSERSITE: 1
 
 on:
@@ -24,7 +25,7 @@ jobs:
         uses: benc-uk/workflow-dispatch@v1
         with:
           workflow: windows.yml
-  
+
   trigger-macos-build:
     runs-on: ubuntu-latest
     steps:
@@ -32,7 +33,7 @@ jobs:
         uses: benc-uk/workflow-dispatch@v1
         with:
           workflow: m1_macos_build.yml
-  
+
   trigger-intel-build:
     runs-on: ubuntu-latest
     steps:

.github/workflows/test.yml

@@ -4,6 +4,7 @@ env:
   OPENBB_LOG_COLLECT: false
   OPENBB_USE_ION: false
   OPENBB_USE_PROMPT_TOOLKIT: false
+  OPENBB_FILE_OVERWRITE: true
   PIP_DEFAULT_TIMEOUT: 100
 
 on: [workflow_dispatch]

.github/workflows/unit-test.yml

@@ -5,6 +5,7 @@ env:
   OPENBB_LOG_COLLECT: false
   OPENBB_USE_ION: false
   OPENBB_USE_PROMPT_TOOLKIT: false
+  OPENBB_FILE_OVERWRITE: true
   PIP_DEFAULT_TIMEOUT: 100
 
 on:

.github/workflows/windows.yml

@@ -4,6 +4,7 @@ env:
   OPENBB_LOG_COLLECT: false
   OPENBB_USE_ION: false
   OPENBB_USE_PROMPT_TOOLKIT: false
+  OPENBB_FILE_OVERWRITE: true
   PIP_DEFAULT_TIMEOUT: 100
   PYTHONNOUSERSITE: 1
 on:

build/conda/conda-3-9-env-full.yaml

@@ -6,3 +6,4 @@ dependencies:
   - pip
   - poetry=1.1.13
   - lightgbm=3.3.3
+  - charset-normalizer=2.1.1

openbb_terminal/featflags_controller.py

@@ -42,6 +42,7 @@ class FeatureFlagsController(BaseController):
         "rich",
         "richpanel",
         "tbhint",
+        "overwrite",
     ]
     PATH = "/featflags/"
 
@@ -73,9 +74,15 @@ class FeatureFlagsController(BaseController):
         mt.add_setting("watermark", obbff.USE_WATERMARK)
         mt.add_setting("cmdloc", obbff.USE_CMD_LOCATION_FIGURE)
         mt.add_setting("tbhint", obbff.TOOLBAR_HINT)
+        mt.add_setting("overwrite", obbff.FILE_OVERWITE)
 
         console.print(text=mt.menu_text, menu="Feature Flags")
 
+    def call_overwrite(self, _):
+        """Process overwrite command"""
+        obbff.FILE_OVERWITE = not obbff.FILE_OVERWITE
+        set_key(USER_ENV_FILE, "OPENBB_FILE_OVERWITE", str(obbff.FILE_OVERWITE))
+
     def call_retryload(self, _):
         """Process retryload command"""
         obbff.RETRY_WITH_LOAD = not obbff.RETRY_WITH_LOAD

openbb_terminal/feature_flags.py

@@ -31,6 +31,9 @@ else:
     WITH_GIT = True
 
 
+# Always overwrite exported files
+FILE_OVERWITE = load_env_vars("OPENBB_FILE_OVERWRITE", strtobool, False, "featflags")
+
 # Retry unknown commands with `load`
 RETRY_WITH_LOAD = load_env_vars("OPENBB_RETRY_WITH_LOAD", strtobool, False, "featflags")
 

openbb_terminal/helper_funcs.py

@@ -5,7 +5,7 @@ import argparse
 import io
 import logging
 from pathlib import Path
-from typing import List, Union, Optional, Dict
+from typing import List, Union, Optional, Dict, Tuple
 from functools import lru_cache
 from datetime import datetime, timedelta
 from datetime import date as d
@@ -1322,8 +1322,34 @@ def compose_export_path(func_name: str, dir_path: str) -> Path:
     return full_path
 
 
+def ask_file_overwrite(file_path: str) -> Tuple[bool, bool]:
+    """Helper to provide a prompt for overwriting existing files.
+
+    Returns two values, the first is a boolean indicating if the file exists and the
+    second is a boolean indicating if the user wants to overwrite the file.
+    """
+    # Jeroen asked for a flag to overwrite no matter what
+    if obbff.FILE_OVERWITE:
+        return False, True
+    if os.path.exists(file_path):
+        overwrite = input("\nFile already exists. Overwrite? [y/n]: ").lower()
+        if overwrite == "y":
+            # File exists and user wants to overwrite
+            return True, True
+        # File exists and user does not want to overwrite
+        return True, False
+    # File does not exist
+    return False, True
+
+
+# This is a false positive on pylint and being tracked in pylint #3060
+# pylint: disable=abstract-class-instantiated
 def export_data(
-    export_type: str, dir_path: str, func_name: str, df: pd.DataFrame = pd.DataFrame()
+    export_type: str,
+    dir_path: str,
+    func_name: str,
+    df: pd.DataFrame = pd.DataFrame(),
+    sheet_name: str = None,
 ) -> None:
     """Export data to a file.
 
@@ -1337,6 +1363,8 @@ def export_data(
         Name of the command that invokes this function
     df : pd.Dataframe
         Dataframe of data to save
+    sheet_name : str
+        If provided.  The name of the sheet to save in excel file
     """
     if export_type:
         export_path = compose_export_path(func_name, dir_path)
@@ -1372,19 +1400,62 @@ def export_data(
             )
 
             if exp_type.endswith("csv"):
+                exists, overwrite = ask_file_overwrite(saved_path)
+                if exists and not overwrite:
+                    return
                 df.to_csv(saved_path)
             elif exp_type.endswith("json"):
+                exists, overwrite = ask_file_overwrite(saved_path)
+                if exists and not overwrite:
+                    return
                 df.reset_index(drop=True, inplace=True)
                 df.to_json(saved_path)
             elif exp_type.endswith("xlsx"):
-                df.to_excel(saved_path, index=True, header=True)
+
+                if sheet_name is None:
+                    exists, overwrite = ask_file_overwrite(saved_path)
+                    if exists and not overwrite:
+                        return
+                    df.to_excel(saved_path, index=True, header=True)
+
+                else:
+                    if os.path.exists(saved_path):
+                        with pd.ExcelWriter(
+                            saved_path,
+                            mode="a",
+                            if_sheet_exists="new",
+                            engine="openpyxl",
+                        ) as writer:
+                            df.to_excel(
+                                writer, sheet_name=sheet_name, index=True, header=True
+                            )
+                    else:
+                        with pd.ExcelWriter(
+                            saved_path,
+                            engine="openpyxl",
+                        ) as writer:
+                            df.to_excel(
+                                writer, sheet_name=sheet_name, index=True, header=True
+                            )
             elif exp_type.endswith("png"):
+                exists, overwrite = ask_file_overwrite(saved_path)
+                if exists and not overwrite:
+                    return
                 plt.savefig(saved_path)
             elif exp_type.endswith("jpg"):
+                exists, overwrite = ask_file_overwrite(saved_path)
+                if exists and not overwrite:
+                    return
                 plt.savefig(saved_path)
             elif exp_type.endswith("pdf"):
+                exists, overwrite = ask_file_overwrite(saved_path)
+                if exists and not overwrite:
+                    return
                 plt.savefig(saved_path)
             elif exp_type.endswith("svg"):
+                exists, overwrite = ask_file_overwrite(saved_path)
+                if exists and not overwrite:
+                    return
                 plt.savefig(saved_path)
             else:
                 console.print("\nWrong export file specified.")

openbb_terminal/parent_classes.py

@@ -753,6 +753,16 @@ class BaseController(metaclass=ABCMeta):
                 help=help_export,
             )
 
+            # If excel is an option, add the sheet name
+            if export_allowed == EXPORT_ONLY_RAW_DATA_ALLOWED:
+                parser.add_argument(
+                    "--sheet-name",
+                    dest="sheet_name",
+                    default=None,
+                    nargs="+",
+                    help="Name of excel sheet to save data to. Only valid for .xlsx files.",
+                )
+
         if raw:
             parser.add_argument(
                 "--raw",
@@ -1147,6 +1157,9 @@ class StockBaseController(BaseController, metaclass=ABCMeta):
                     os.path.dirname(os.path.abspath(__file__)),
                     f"load_{self.ticker}",
                     self.stock.copy(),
+                    sheet_name=" ".join(ns_parser.sheet_name)
+                    if ns_parser.sheet_name
+                    else None,
                 )
 
 

tests/openbb_terminal/portfolio/csv/test_portfolio_model/test_tracking_error.csv

@@ -5,9 +5,7 @@ Total       NaN
 dtype: float64"
 ytd,-
 3m,-
-6m,"Adj Close   NaN
-Total       NaN
-dtype: float64"
+6m,-
 1y,"Adj Close   NaN
 Total       NaN
 dtype: float64"