diff --git a/openbb_terminal/config_terminal.py b/openbb_terminal/config_terminal.py index 076aed4801e..33636403802 100644 --- a/openbb_terminal/config_terminal.py +++ b/openbb_terminal/config_terminal.py @@ -35,7 +35,10 @@ PAPERMILL_NOTEBOOK_REPORT_PORT = ( # Logging section # USE IN LOG LINES + FOR FOLDER NAME INSIDE S3 BUCKET -LOGGING_APP_NAME = os.getenv("OPENBB_LOGGING_APP_NAME") or "gst" +if "site-packages" in __file__: + LOGGING_APP_NAME = "gst_packaged_pypi" +else: + LOGGING_APP_NAME = os.getenv("OPENBB_LOGGING_APP_NAME") or "gst" # AWS KEYS LOGGING_AWS_ACCESS_KEY_ID = ( os.getenv("OPENBB_LOGGING_AWS_ACCESS_KEY_ID") or "REPLACE_ME" diff --git a/openbb_terminal/core/log/generation/formatter_with_exceptions.py b/openbb_terminal/core/log/generation/formatter_with_exceptions.py index 5bb0c7df4d8..ae4d4cc55ad 100644 --- a/openbb_terminal/core/log/generation/formatter_with_exceptions.py +++ b/openbb_terminal/core/log/generation/formatter_with_exceptions.py @@ -1,12 +1,11 @@ # IMPORTATION STANDARD import logging -import os import re # IMPORTATION THIRDPARTY # IMPORTATION INTERNAL -from openbb_terminal.core.config.paths import REPOSITORY_DIRECTORY +from openbb_terminal.core.config.paths import HOME_DIRECTORY from openbb_terminal.core.log.generation.settings import AppSettings @@ -20,7 +19,7 @@ class FormatterWithExceptions(logging.Formatter): ) @staticmethod - def calculate_level_name(record: logging.LogRecord): + def calculate_level_name(record: logging.LogRecord) -> str: if record.exc_text: level_name = "X" elif record.levelname: @@ -38,7 +37,7 @@ class FormatterWithExceptions(logging.Formatter): record.funcName = record.func_name_override # type: ignore record.lineno = 0 - log_extra["userId"] = record.user_id if hasattr(record, "user_id") else "NA" # type: ignore + log_extra["userId"] = getattr(record, "user_id", "NA") if hasattr(record, "session_id"): log_extra["sessionId"] = record.session_id # type: ignore @@ -46,68 +45,59 @@ class FormatterWithExceptions(logging.Formatter): return log_extra @staticmethod - def filter_piis(text: str) -> str: - ip_regex = r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}" - s_list = [] - ip_reg = re.compile(ip_regex) + def mock_ipv4(text: str) -> str: + pattern = r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}" + replacement = " FILTERED_IP " + text_mocked = re.sub(pattern, replacement, text) - for word in text.split(): - - if ip_reg.search(word): - s_list.append("suspected_ip") - elif "@" in word and "." in word: - s_list.append("suspected_email") - elif f"{REPOSITORY_DIRECTORY.name}{os.sep}" in word: - s_list.append( - word.split(f"{REPOSITORY_DIRECTORY.name}{os.sep}")[1] - .replace('"', "") - .replace("'", "") - ) - elif os.sep in word and os.sep != word: - s_list.append( - (f"cut{os.sep}file{os.sep}path{os.sep}" + word.split(os.sep)[-1]) - .replace('"', "") - .replace("'", "") - ) - else: - s_list.append(word) - - text = " ".join(s_list) - - return text + return text_mocked @staticmethod - def filter_special_characters(text: str): - filtered_text = ( - text.replace("\\n", " - ") - .replace("\n", " - ") - .replace("\t", " ") - .replace("\r", "") - .replace("'Traceback", "Traceback") - ) - return filtered_text + def mock_email(text: str) -> str: + pattern = r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b" + replacement = " FILTERED_EMAIL " + text_mocked = re.sub(pattern, replacement, text) + + return text_mocked @staticmethod - def detect_terminal_message(text: str): - if "The command " in text and "doesn't exist on the" in text and os.sep == "/": - return True - return False + def mock_home_directory(text: str) -> str: + user_home_directory = str(HOME_DIRECTORY) + text_mocked = text.replace(user_home_directory, "MOCKING_USER_PATH") + + return text_mocked + + @staticmethod + def mock_break_lines(text: str) -> str: + text_mocked = text.replace("\n", " MOCKING_BREAKLINE ") + text_mocked = text_mocked.replace("\r", " MOCKING_CARRIAGE_RETURN ") + text_mocked = text_mocked.replace("\t", " MOCKING_TAB ") + text_mocked = text_mocked.replace("\\n", " MOCKING_BREAKLINE ") + text_mocked = text_mocked.replace("\\r", " MOCKING_CARRIAGE_RETURN ") + + return text_mocked + + @classmethod + def filter_special_tags(cls, text: str) -> str: + text_filtered = cls.mock_break_lines(text=text) + text_filtered = text_filtered.replace("'Traceback", "Traceback") + + return text_filtered + + @classmethod + def filter_piis(cls, text: str) -> str: + text_filtered = cls.mock_ipv4(text=text) + text_filtered = cls.mock_email(text=text_filtered) + text_filtered = cls.mock_home_directory(text=text_filtered) + + return text_filtered @classmethod def filter_log_line(cls, text: str): - text = cls.filter_special_characters(text=text) - contains_terminal_menu = cls.detect_terminal_message(text) - if contains_terminal_menu: - first_message, second_message = text.split("menu. - Traceback") - text = ( - first_message - + "menu. - Traceback" - + cls.filter_piis(text=second_message) - ) - elif "CMD: {" not in text and "QUEUE: {" not in text: - text = cls.filter_piis(text=text) + text_filtered = cls.filter_special_tags(text=text) + text_filtered = cls.filter_piis(text=text_filtered) - return text + return text_filtered # OVERRIDE def __init__( diff --git a/openbb_terminal/custom_reports/lunar_crash/lunatics.ipynb b/openbb_terminal/custom_reports/lunar_crash/lunatics.ipynb index e65688d8673..08df658b0be 100644 --- a/openbb_terminal/custom_reports/lunar_crash/lunatics.ipynb +++ b/openbb_terminal/custom_reports/lunar_crash/lunatics.ipynb @@ -27,7 +27,8 @@ "import matplotlib_inline.backend_inline\n", "\n", "import sys\n", - "sys.path.append('../../../')\n", + "\n", + "sys.path.append(\"../../../\")\n", "\n", "from openbb_terminal import api as openbb\n", "from openbb_terminal.helper_classes import TerminalStyle\n", @@ -63,8 +64,8 @@ "source": [ "author = \"Minh Hoang\"\n", "report_title = f\"LUNATICS REPORT\"\n", - "report_date = datetime.datetime.now().strftime('%d %B, %Y')\n", - "report_time = datetime.datetime.now().strftime('%H:%M') \n", + "report_date = datetime.datetime.now().strftime(\"%d %B, %Y\")\n", + "report_time = datetime.datetime.now().strftime(\"%H:%M\")\n", "report_timezone = datetime.datetime.now(datetime.timezone.utc).astimezone().tzinfo\n", "report_name = \"luna_crash\"" ] @@ -79,7 +80,9 @@ "body = \"\"\n", "\n", "img = f'\"OpenBB\"'\n", - "body += openbb.widgets.header(img, author, report_date, report_time, report_timezone, f'LUNA CRASH EVENTS')\n", + "body += openbb.widgets.header(\n", + " img, author, report_date, report_time, report_timezone, f\"LUNA CRASH EVENTS\"\n", + ")\n", "\n", "body += openbb.widgets.tablinks([\"Overview\", \"Before\", \"During\", \"After\"])" ] @@ -108,18 +111,24 @@ "outputs": [], "source": [ "# BEFORE\n", - "htmlcode = openbb.widgets.add_external_fig(\"./luna_supply.png\", \"width:500px; margin-bottom:16px\")\n", + "htmlcode = openbb.widgets.add_external_fig(\n", + " \"./luna_supply.png\", \"width:500px; margin-bottom:16px\"\n", + ")\n", "text = \"\"\"Just a few weeks ago, LUNA and the Terra ecosystem were the darlings of crypto investors.
\n", "Experiencing a 20,000% increase since the beginning of 2021, LUNA reached its ATH of $119 in January 2021.\n", "\"\"\"\n", "htmlcode += openbb.widgets.p(text, \"width:1200px; margin-bottom:8px\")\n", - "htmlcode += openbb.widgets.add_external_fig(\"./coin_highs.png\", \"width:500px; margin-bottom:16px\")\n", + "htmlcode += openbb.widgets.add_external_fig(\n", + " \"./coin_highs.png\", \"width:500px; margin-bottom:16px\"\n", + ")\n", "text = \"\"\"Overall, before the crisis, Terra was the second biggest smart contract platform in terms of total value locked, with almost $22 billion.
\n", "Its biggest dApp is Anchor, a lending and borrowing protocol with the (in)famous APY of 20%.
\n", "This lucrative return is too hard to ignore, even within the crypto market. Within a year, its TVL has shot up from $500M to almost $17.5B.\n", "\"\"\"\n", "htmlcode += openbb.widgets.p(text, \"width:1200px; margin-bottom:8px\")\n", - "htmlcode += openbb.widgets.add_external_fig(\"./tvl.png\", \"width:500px; margin-bottom:16px\")\n", + "htmlcode += openbb.widgets.add_external_fig(\n", + " \"./tvl.png\", \"width:500px; margin-bottom:16px\"\n", + ")\n", "body += openbb.widgets.add_tab(\"Before\", htmlcode, False)" ] }, @@ -155,7 +164,9 @@ "Whenever there’s a price discrepancy between USD and UST, Terra relies on this genius arbitrage incentives to bring price back to $1.\n", "\"\"\"\n", "htmlcode = openbb.widgets.p(text, \"width:1200px; margin-bottom:8px; margin-top:16px\")\n", - "htmlcode += openbb.widgets.add_external_fig(\"./ust_terra.png\", \"width:500px; margin-left:2em\")\n", + "htmlcode += openbb.widgets.add_external_fig(\n", + " \"./ust_terra.png\", \"width:500px; margin-left:2em\"\n", + ")\n", "\n", "text = \"\"\"\n", "But why can’t the burn-and-mint mechanism save the day? Let’s go back to what happened.\n", diff --git a/openbb_terminal/forecast/forecast.ipynb b/openbb_terminal/forecast/forecast.ipynb index d7dd2a2cae8..3e7d0c013df 100644 --- a/openbb_terminal/forecast/forecast.ipynb +++ b/openbb_terminal/forecast/forecast.ipynb @@ -59,7 +59,9 @@ "metadata": {}, "outputs": [], "source": [ - "api.forecast.brnn(df, target_column=\"MaxTemp\", past_covariates=\"MinTemp,Rainfall,Evaporation\")" + "api.forecast.brnn(\n", + " df, target_column=\"MaxTemp\", past_covariates=\"MinTemp,Rainfall,Evaporation\"\n", + ")" ] } ], diff --git a/openbb_terminal/miscellaneous/i18n/help_translation.ipynb b/openbb_terminal/miscellaneous/i18n/help_translation.ipynb index 206457c8c4f..503a00562a7 100644 --- a/openbb_terminal/miscellaneous/i18n/help_translation.ipynb +++ b/openbb_terminal/miscellaneous/i18n/help_translation.ipynb @@ -42,7 +42,7 @@ "outputs": [], "source": [ "# Select language to translate\n", - "language_to_translate = 'pt'" + "language_to_translate = \"pt\"" ] }, { @@ -56,12 +56,12 @@ "english_dictionary = {}\n", "with open(\"en.yml\", \"r\") as stream:\n", " try:\n", - " english_dictionary = yaml.safe_load(stream)['en']\n", + " english_dictionary = yaml.safe_load(stream)[\"en\"]\n", " except yaml.YAMLError as exc:\n", " print(exc)\n", "\n", "# Translated file\n", - "translated_file = f'{language_to_translate}.yml'" + "translated_file = f\"{language_to_translate}.yml\"" ] }, { @@ -90,27 +90,33 @@ " language_dictionary = yaml.safe_load(stream)[language_to_translate]\n", " except yaml.YAMLError as exc:\n", " print(exc)\n", - " \n", + "\n", " # Iterate through english dictionary\n", " for k in english_dictionary:\n", " # Check if that variable doesn't existed on the already translated dictionary\n", " if k not in language_dictionary:\n", " # If the variable doesn't exist, let's update it to it's default\n", - " language_dictionary[k] = translate(english_dictionary[k], language_to_translate)\n", - " \n", + " language_dictionary[k] = translate(\n", + " english_dictionary[k], language_to_translate\n", + " )\n", + "\n", "else:\n", " # Iterate through english dictionary\n", " for k in english_dictionary:\n", " try:\n", " # Replace english value by its translation\n", - " language_dictionary[k] = translate(english_dictionary[k], language_to_translate)\n", + " language_dictionary[k] = translate(\n", + " english_dictionary[k], language_to_translate\n", + " )\n", " except:\n", " # Keep english language because there was an issue with the convertion\n", " language_dictionary[k] = english_dictionary[k]\n", "\n", "# Save dictionary to another language\n", - "with open(f'{language_to_translate}.yml', 'w') as file:\n", - " yaml.dump({language_to_translate: language_dictionary}, stream=file, allow_unicode=True)" + "with open(f\"{language_to_translate}.yml\", \"w\") as file:\n", + " yaml.dump(\n", + " {language_to_translate: language_dictionary}, stream=file, allow_unicode=True\n", + " )" ] }, { @@ -128,24 +134,32 @@ "metadata": {}, "outputs": [], "source": [ - "for language_to_translate in [val.split(\".\")[0] for val in os.listdir() if val.endswith(\"yml\") and val != \"en.yml\"]:\n", + "for language_to_translate in [\n", + " val.split(\".\")[0] for val in os.listdir() if val.endswith(\"yml\") and val != \"en.yml\"\n", + "]:\n", " # Read already translated dictionary\n", " with open(translated_file, \"r\") as stream:\n", " try:\n", " language_dictionary = yaml.safe_load(stream)[language_to_translate]\n", " except yaml.YAMLError as exc:\n", " print(exc)\n", - " \n", + "\n", " # Iterate through english dictionary\n", " for k in english_dictionary:\n", " # Check if that variable doesn't existed on the already translated dictionary\n", " if k not in language_dictionary:\n", " # If the variable doesn't exist, let's update it to it's default\n", - " language_dictionary[k] = translate(english_dictionary[k], language_to_translate)\n", + " language_dictionary[k] = translate(\n", + " english_dictionary[k], language_to_translate\n", + " )\n", "\n", " # Save dictionary to another language\n", - " with open(f'{language_to_translate}.yml', 'w') as file:\n", - " yaml.dump({language_to_translate: language_dictionary}, stream=file, allow_unicode=True)" + " with open(f\"{language_to_translate}.yml\", \"w\") as file:\n", + " yaml.dump(\n", + " {language_to_translate: language_dictionary},\n", + " stream=file,\n", + " allow_unicode=True,\n", + " )" ] }, { diff --git a/tests/openbb_terminal/core/log/generation/json/test_formatter_with_exceptions/test_extract_log_extra.json b/tests/openbb_terminal/core/log/generation/json/test_formatter_with_exceptions/test_extract_log_extra.json new file mode 100644 index 00000000000..75c63f4834c --- /dev/null +++ b/tests/openbb_terminal/core/log/generation/json/test_formatter_with_exceptions/test_extract_log_extra.json @@ -0,0 +1 @@ +{"userId": "MOCK_USER_ID", "sessionId": "MOCK_SESSION_ID"} \ No newline at end of file diff --git a/tests/openbb_terminal/core/log/generation/test_formatter_with_exceptions.py b/tests/openbb_terminal/core/log/generation/test_formatter_with_exceptions.py index a8b1b592d2b..5c42eb925cb 100644 --- a/tests/openbb_terminal/core/log/generation/test_formatter_with_exceptions.py +++ b/tests/openbb_terminal/core/log/generation/test_formatter_with_exceptions.py @@ -1,8 +1,13 @@ -import os +# IMPORTATION STANDARD + +# IMPORTATION THIRDPARTY import pytest -from openbb_terminal.core.log.generation import formatter_with_exceptions + +# IMPORTATION INTERNAL +from openbb_terminal.core.log.generation.formatter_with_exceptions import ( + FormatterWithExceptions, +) from openbb_terminal.core.log.generation.settings import AppSettings -from openbb_terminal.core.config.paths import REPOSITORY_DIRECTORY app_settings = AppSettings( commit_hash="MOCK_COMMIT_HASH", @@ -10,56 +15,40 @@ app_settings = AppSettings( identifier="MOCK_COMMIT_HASH", session_id="MOCK_SESSION_ID", ) - - -formatter = formatter_with_exceptions.FormatterWithExceptions(app_settings) +formatter = FormatterWithExceptions(app_settings) @pytest.mark.parametrize("exc, level", [(True, True), (False, "name"), (False, False)]) -def test_calculate_level_name(exc, level, mocker): +def test_calculate_level_name(exc, level, mocker, recorder): mock = mocker.Mock() mock.exc_text = exc mock.levelname = level - value = formatter.calculate_level_name(mock) - assert value + text_expected = FormatterWithExceptions.calculate_level_name(mock) + recorder.capture(text_expected) -def test_extract_log_extra(mocker): +def test_extract_log_extra(mocker, recorder): mock = mocker.Mock() mock.exc_text = True - mock.levelname = "name" - value = formatter.extract_log_extra(mock) - assert value + mock.levelname = "MOCK_LEVELNAME" + mock.user_id = "MOCK_USER_ID" + mock.session_id = "MOCK_SESSION_ID" + text_expected = FormatterWithExceptions.extract_log_extra(mock) + recorder.capture(text_expected) -def test_filter_piis(): - text = ( - f"test 1.1.1.1 chavi@chavi.com {REPOSITORY_DIRECTORY.name}{os.sep} dd{os.sep}dd" - ) - value = formatter.filter_piis(text) - assert value +def test_filter_log_line(recorder): + text = "test 1.1.1.1 testestest chavi@chavi.com testestest C:\\Users\\username\\some folder\\some path testestest" + text_expected = FormatterWithExceptions.filter_log_line(text=text) + recorder.capture(text_expected) + text = "test 1.1.1.1 testestest chavi@chavi.com testestest /home/username/some folder/some path testestest" + text_expected = FormatterWithExceptions.filter_log_line(text=text) + recorder.capture(text_expected) -def test_filter_special_characters(): - text = "\nhello \this \rgreetings" - value = formatter.filter_special_characters(text) - assert value == " - hello his greetings" - - -def test_detect_terminal_message(): - value = formatter.detect_terminal_message("The command doesn't exist on the") - assert value or os.sep != "/" - new_val = formatter.detect_terminal_message("boring message") - assert new_val is False - - -def test_filter_log_line(): - value = formatter.filter_log_line("stocks") - assert value - value = formatter.filter_log_line( - "The command doesn't exist on the menu. - Traceback hello" - ) - assert value + text = "\\n\nhello \this\\r \rgreetings" + text_expected = FormatterWithExceptions.filter_log_line(text=text) + recorder.capture(text_expected) def test_formatException_invalid(): diff --git a/tests/openbb_terminal/core/log/generation/txt/test_formatter_with_exceptions/test_calculate_level_name[False-False].txt b/tests/openbb_terminal/core/log/generation/txt/test_formatter_with_exceptions/test_calculate_level_name[False-False].txt new file mode 100644 index 00000000000..4f0734cbe3a --- /dev/null +++ b/tests/openbb_terminal/core/log/generation/txt/test_formatter_with_exceptions/test_calculate_level_name[False-False].txt @@ -0,0 +1 @@ +U \ No newline at end of file diff --git a/tests/openbb_terminal/core/log/generation/txt/test_formatter_with_exceptions/test_calculate_level_name[False-name].txt b/tests/openbb_terminal/core/log/generation/txt/test_formatter_with_exceptions/test_calculate_level_name[False-name].txt new file mode 100644 index 00000000000..ef073cc45cc --- /dev/null +++ b/tests/openbb_terminal/core/log/generation/txt/test_formatter_with_exceptions/test_calculate_level_name[False-name].txt @@ -0,0 +1 @@ +n \ No newline at end of file diff --git a/tests/openbb_terminal/core/log/generation/txt/test_formatter_with_exceptions/test_calculate_level_name[True-True].txt b/tests/openbb_terminal/core/log/generation/txt/test_formatter_with_exceptions/test_calculate_level_name[True-True].txt new file mode 100644 index 00000000000..500c0709ca2 --- /dev/null +++ b/tests/openbb_terminal/core/log/generation/txt/test_formatter_with_exceptions/test_calculate_level_name[True-True].txt @@ -0,0 +1 @@ +X \ No newline at end of file diff --git a/tests/openbb_terminal/core/log/generation/txt/test_formatter_with_exceptions/test_filter_log_line.txt b/tests/openbb_terminal/core/log/generation/txt/test_formatter_with_exceptions/test_filter_log_line.txt new file mode 100644 index 00000000000..1b2294ab972 --- /dev/null +++ b/tests/openbb_terminal/core/log/generation/txt/test_formatter_with_exceptions/test_filter_log_line.txt @@ -0,0 +1 @@ +test FILTERED_IP testestest FILTERED_EMAIL testestest C:\Users\username\some folder\some path testestest \ No newline at end of file diff --git a/tests/openbb_terminal/core/log/generation/txt/test_formatter_with_exceptions/test_filter_log_line_1.txt b/tests/openbb_terminal/core/log/generation/txt/test_formatter_with_exceptions/test_filter_log_line_1.txt new file mode 100644 index 00000000000..1a86ce763c5 --- /dev/null +++ b/tests/openbb_terminal/core/log/generation/txt/test_formatter_with_exceptions/test_filter_log_line_1.txt @@ -0,0 +1 @@ +test FILTERED_IP testestest FILTERED_EMAIL testestest /home/username/some folder/some path testestest \ No newline at end of file diff --git a/tests/openbb_terminal/core/log/generation/txt/test_formatter_with_exceptions/test_filter_log_line_2.txt b/tests/openbb_terminal/core/log/generation/txt/test_formatter_with_exceptions/test_filter_log_line_2.txt new file mode 100644 index 00000000000..644a23d8728 --- /dev/null +++ b/tests/openbb_terminal/core/log/generation/txt/test_formatter_with_exceptions/test_filter_log_line_2.txt @@ -0,0 +1 @@ + MOCKING_BREAKLINE MOCKING_BREAKLINE hello MOCKING_TAB his MOCKING_CARRIAGE_RETURN MOCKING_CARRIAGE_RETURN greetings \ No newline at end of file