jerry/OpenBB
1
0
Fork 0
mirror of https://github.com/OpenBB-finance/OpenBB.git synced 2026-10-27 12:40:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Logging : detect pypi install + PIIS filtering (#2973)

Browse Source 
* LOG : add app_name=gst_packaged_pypi

* LOG : update piis filtering

* Linting

* Log : generation

* Tests : log generation

* Linting

* Tests : piis filtering

Co-authored-by: James Maslek <jmaslek11@gmail.com>
This commit is contained in:
Chavithra
2022-10-24 09:35:02 +02:00
committed by GitHub
parent 90a1469c8b
commit d4a462cbe9
13 changed files with 136 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
openbb_terminal/config_terminal.py
View File

@@ -35,7 +35,10 @@ PAPERMILL_NOTEBOOK_REPORT_PORT = (
# Logging section
# USE IN LOG LINES + FOR FOLDER NAME INSIDE S3 BUCKET
LOGGING_APP_NAME = os.getenv("OPENBB_LOGGING_APP_NAME") or "gst"
if "site-packages" in __file__:
LOGGING_APP_NAME = "gst_packaged_pypi"
else:
LOGGING_APP_NAME = os.getenv("OPENBB_LOGGING_APP_NAME") or "gst"
# AWS KEYS
LOGGING_AWS_ACCESS_KEY_ID = (
os.getenv("OPENBB_LOGGING_AWS_ACCESS_KEY_ID") or "REPLACE_ME"

104
openbb_terminal/core/log/generation/formatter_with_exceptions.py
View File

@@ -1,12 +1,11 @@
# IMPORTATION STANDARD
import logging
import os
import re
# IMPORTATION THIRDPARTY
# IMPORTATION INTERNAL
from openbb_terminal.core.config.paths import REPOSITORY_DIRECTORY
from openbb_terminal.core.config.paths import HOME_DIRECTORY
from openbb_terminal.core.log.generation.settings import AppSettings
@@ -20,7 +19,7 @@ class FormatterWithExceptions(logging.Formatter):
)
@staticmethod
def calculate_level_name(record: logging.LogRecord):
def calculate_level_name(record: logging.LogRecord) -> str:
if record.exc_text:
level_name = "X"
elif record.levelname:
@@ -38,7 +37,7 @@ class FormatterWithExceptions(logging.Formatter):
record.funcName = record.func_name_override # type: ignore
record.lineno = 0
log_extra["userId"] = record.user_id if hasattr(record, "user_id") else "NA" # type: ignore
log_extra["userId"] = getattr(record, "user_id", "NA")
if hasattr(record, "session_id"):
log_extra["sessionId"] = record.session_id # type: ignore
@@ -46,68 +45,59 @@ class FormatterWithExceptions(logging.Formatter):
return log_extra
@staticmethod
def filter_piis(text: str) -> str:
ip_regex = r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}"
s_list = []
ip_reg = re.compile(ip_regex)
def mock_ipv4(text: str) -> str:
pattern = r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}"
replacement = " FILTERED_IP "
text_mocked = re.sub(pattern, replacement, text)
for word in text.split():
if ip_reg.search(word):
s_list.append("suspected_ip")
elif "@" in word and "." in word:
s_list.append("suspected_email")
elif f"{REPOSITORY_DIRECTORY.name}{os.sep}" in word:
s_list.append(
word.split(f"{REPOSITORY_DIRECTORY.name}{os.sep}")[1]
.replace('"', "")
.replace("'", "")
)
elif os.sep in word and os.sep != word:
s_list.append(
(f"cut{os.sep}file{os.sep}path{os.sep}" + word.split(os.sep)[-1])
.replace('"', "")
.replace("'", "")
)
else:
s_list.append(word)
text = " ".join(s_list)
return text
return text_mocked
@staticmethod
def filter_special_characters(text: str):
filtered_text = (
text.replace("\\n", " - ")
.replace("\n", " - ")
.replace("\t", " ")
.replace("\r", "")
.replace("'Traceback", "Traceback")
)
return filtered_text
def mock_email(text: str) -> str:
pattern = r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b"
replacement = " FILTERED_EMAIL "
text_mocked = re.sub(pattern, replacement, text)
return text_mocked
@staticmethod
def detect_terminal_message(text: str):
if "The command " in text and "doesn't exist on the" in text and os.sep == "/":
return True
return False
def mock_home_directory(text: str) -> str:
user_home_directory = str(HOME_DIRECTORY)
text_mocked = text.replace(user_home_directory, "MOCKING_USER_PATH")
return text_mocked
@staticmethod
def mock_break_lines(text: str) -> str:
text_mocked = text.replace("\n", " MOCKING_BREAKLINE ")
text_mocked = text_mocked.replace("\r", " MOCKING_CARRIAGE_RETURN ")
text_mocked = text_mocked.replace("\t", " MOCKING_TAB ")
text_mocked = text_mocked.replace("\\n", " MOCKING_BREAKLINE ")
text_mocked = text_mocked.replace("\\r", " MOCKING_CARRIAGE_RETURN ")
return text_mocked
@classmethod
def filter_special_tags(cls, text: str) -> str:
text_filtered = cls.mock_break_lines(text=text)
text_filtered = text_filtered.replace("'Traceback", "Traceback")
return text_filtered
@classmethod
def filter_piis(cls, text: str) -> str:
text_filtered = cls.mock_ipv4(text=text)
text_filtered = cls.mock_email(text=text_filtered)
text_filtered = cls.mock_home_directory(text=text_filtered)
return text_filtered
@classmethod
def filter_log_line(cls, text: str):
text = cls.filter_special_characters(text=text)
contains_terminal_menu = cls.detect_terminal_message(text)
if contains_terminal_menu:
first_message, second_message = text.split("menu. - Traceback")
text = (
first_message
+ "menu. - Traceback"
+ cls.filter_piis(text=second_message)
)
elif "CMD: {" not in text and "QUEUE: {" not in text:
text = cls.filter_piis(text=text)
text_filtered = cls.filter_special_tags(text=text)
text_filtered = cls.filter_piis(text=text_filtered)
return text
return text_filtered
# OVERRIDE
def __init__(

27
openbb_terminal/custom_reports/lunar_crash/lunatics.ipynb vendored
View File

@@ -27,7 +27,8 @@
"import matplotlib_inline.backend_inline\n",
"\n",
"import sys\n",
"sys.path.append('../../../')\n",
"\n",
"sys.path.append(\"../../../\")\n",
"\n",
"from openbb_terminal import api as openbb\n",
"from openbb_terminal.helper_classes import TerminalStyle\n",
@@ -63,8 +64,8 @@
"source": [
"author = \"Minh Hoang\"\n",
"report_title = f\"LUNATICS REPORT\"\n",
"report_date = datetime.datetime.now().strftime('%d %B, %Y')\n",
"report_time = datetime.datetime.now().strftime('%H:%M') \n",
"report_date = datetime.datetime.now().strftime(\"%d %B, %Y\")\n",
"report_time = datetime.datetime.now().strftime(\"%H:%M\")\n",
"report_timezone = datetime.datetime.now(datetime.timezone.utc).astimezone().tzinfo\n",
"report_name = \"luna_crash\""
]
@@ -79,7 +80,9 @@
"body = \"\"\n",
"\n",
"img = f'<img src=\"data:image/png;base64,{openbb_image_encoded.decode()}\" alt=\"OpenBB\" style=\"width:144px;\">'\n",
"body += openbb.widgets.header(img, author, report_date, report_time, report_timezone, f'<b>LUNA CRASH EVENTS</b>')\n",
"body += openbb.widgets.header(\n",
" img, author, report_date, report_time, report_timezone, f\"<b>LUNA CRASH EVENTS</b>\"\n",
")\n",
"\n",
"body += openbb.widgets.tablinks([\"Overview\", \"Before\", \"During\", \"After\"])"
]
@@ -108,18 +111,24 @@
"outputs": [],
"source": [
"# BEFORE\n",
"htmlcode = openbb.widgets.add_external_fig(\"./luna_supply.png\", \"width:500px; margin-bottom:16px\")\n",
"htmlcode = openbb.widgets.add_external_fig(\n",
" \"./luna_supply.png\", \"width:500px; margin-bottom:16px\"\n",
")\n",
"text = \"\"\"Just a few weeks ago, LUNA and the Terra ecosystem were the darlings of crypto investors.</br> \n",
"Experiencing a 20,000% increase since the beginning of 2021, LUNA reached its ATH of $119 in January 2021.\n",
"\"\"\"\n",
"htmlcode += openbb.widgets.p(text, \"width:1200px; margin-bottom:8px\")\n",
"htmlcode += openbb.widgets.add_external_fig(\"./coin_highs.png\", \"width:500px; margin-bottom:16px\")\n",
"htmlcode += openbb.widgets.add_external_fig(\n",
" \"./coin_highs.png\", \"width:500px; margin-bottom:16px\"\n",
")\n",
"text = \"\"\"Overall, before the crisis, Terra was the second biggest smart contract platform in terms of total value locked, with almost $22 billion.</br>\n",
"Its biggest dApp is Anchor, a lending and borrowing protocol with the (in)famous APY of 20%.</br>\n",
"This lucrative return is too hard to ignore, even within the crypto market. Within a year, its TVL has shot up from $500M to almost $17.5B.\n",
"\"\"\"\n",
"htmlcode += openbb.widgets.p(text, \"width:1200px; margin-bottom:8px\")\n",
"htmlcode += openbb.widgets.add_external_fig(\"./tvl.png\", \"width:500px; margin-bottom:16px\")\n",
"htmlcode += openbb.widgets.add_external_fig(\n",
" \"./tvl.png\", \"width:500px; margin-bottom:16px\"\n",
")\n",
"body += openbb.widgets.add_tab(\"Before\", htmlcode, False)"
]
},
@@ -155,7 +164,9 @@
"Whenever there’s a price discrepancy between USD and UST, Terra relies on this genius arbitrage incentives to bring price back to $1.\n",
"\"\"\"\n",
"htmlcode = openbb.widgets.p(text, \"width:1200px; margin-bottom:8px; margin-top:16px\")\n",
"htmlcode += openbb.widgets.add_external_fig(\"./ust_terra.png\", \"width:500px; margin-left:2em\")\n",
"htmlcode += openbb.widgets.add_external_fig(\n",
" \"./ust_terra.png\", \"width:500px; margin-left:2em\"\n",
")\n",
"\n",
"text = \"\"\"\n",
"But why can’t the burn-and-mint mechanism save the day? Let’s go back to what happened.\n",

4
openbb_terminal/forecast/forecast.ipynb vendored
View File

@@ -59,7 +59,9 @@
"metadata": {},
"outputs": [],
"source": [
"api.forecast.brnn(df, target_column=\"MaxTemp\", past_covariates=\"MinTemp,Rainfall,Evaporation\")"
"api.forecast.brnn(\n",
" df, target_column=\"MaxTemp\", past_covariates=\"MinTemp,Rainfall,Evaporation\"\n",
")"
]
}
],

42
openbb_terminal/miscellaneous/i18n/help_translation.ipynb vendored
View File

@@ -42,7 +42,7 @@
"outputs": [],
"source": [
"# Select language to translate\n",
"language_to_translate = 'pt'"
"language_to_translate = \"pt\""
]
},
{
@@ -56,12 +56,12 @@
"english_dictionary = {}\n",
"with open(\"en.yml\", \"r\") as stream:\n",
" try:\n",
" english_dictionary = yaml.safe_load(stream)['en']\n",
" english_dictionary = yaml.safe_load(stream)[\"en\"]\n",
" except yaml.YAMLError as exc:\n",
" print(exc)\n",
"\n",
"# Translated file\n",
"translated_file = f'{language_to_translate}.yml'"
"translated_file = f\"{language_to_translate}.yml\""
]
},
{
@@ -90,27 +90,33 @@
" language_dictionary = yaml.safe_load(stream)[language_to_translate]\n",
" except yaml.YAMLError as exc:\n",
" print(exc)\n",
" \n",
"\n",
" # Iterate through english dictionary\n",
" for k in english_dictionary:\n",
" # Check if that variable doesn't existed on the already translated dictionary\n",
" if k not in language_dictionary:\n",
" # If the variable doesn't exist, let's update it to it's default\n",
" language_dictionary[k] = translate(english_dictionary[k], language_to_translate)\n",
" \n",
" language_dictionary[k] = translate(\n",
" english_dictionary[k], language_to_translate\n",
" )\n",
"\n",
"else:\n",
" # Iterate through english dictionary\n",
" for k in english_dictionary:\n",
" try:\n",
" # Replace english value by its translation\n",
" language_dictionary[k] = translate(english_dictionary[k], language_to_translate)\n",
" language_dictionary[k] = translate(\n",
" english_dictionary[k], language_to_translate\n",
" )\n",
" except:\n",
" # Keep english language because there was an issue with the convertion\n",
" language_dictionary[k] = english_dictionary[k]\n",
"\n",
"# Save dictionary to another language\n",
"with open(f'{language_to_translate}.yml', 'w') as file:\n",
" yaml.dump({language_to_translate: language_dictionary}, stream=file, allow_unicode=True)"
"with open(f\"{language_to_translate}.yml\", \"w\") as file:\n",
" yaml.dump(\n",
" {language_to_translate: language_dictionary}, stream=file, allow_unicode=True\n",
" )"
]
},
{
@@ -128,24 +134,32 @@
"metadata": {},
"outputs": [],
"source": [
"for language_to_translate in [val.split(\".\")[0] for val in os.listdir() if val.endswith(\"yml\") and val != \"en.yml\"]:\n",
"for language_to_translate in [\n",
" val.split(\".\")[0] for val in os.listdir() if val.endswith(\"yml\") and val != \"en.yml\"\n",
"]:\n",
" # Read already translated dictionary\n",
" with open(translated_file, \"r\") as stream:\n",
" try:\n",
" language_dictionary = yaml.safe_load(stream)[language_to_translate]\n",
" except yaml.YAMLError as exc:\n",
" print(exc)\n",
" \n",
"\n",
" # Iterate through english dictionary\n",
" for k in english_dictionary:\n",
" # Check if that variable doesn't existed on the already translated dictionary\n",
" if k not in language_dictionary:\n",
" # If the variable doesn't exist, let's update it to it's default\n",
" language_dictionary[k] = translate(english_dictionary[k], language_to_translate)\n",
" language_dictionary[k] = translate(\n",
" english_dictionary[k], language_to_translate\n",
" )\n",
"\n",
" # Save dictionary to another language\n",
" with open(f'{language_to_translate}.yml', 'w') as file:\n",
" yaml.dump({language_to_translate: language_dictionary}, stream=file, allow_unicode=True)"
" with open(f\"{language_to_translate}.yml\", \"w\") as file:\n",
" yaml.dump(\n",
" {language_to_translate: language_dictionary},\n",
" stream=file,\n",
" allow_unicode=True,\n",
" )"
]
},
{

1
tests/openbb_terminal/core/log/generation/json/test_formatter_with_exceptions/test_extract_log_extra.json vendored Normal file
View File

@@ -0,0 +1 @@
{"userId": "MOCK_USER_ID", "sessionId": "MOCK_SESSION_ID"}

67
tests/openbb_terminal/core/log/generation/test_formatter_with_exceptions.py
View File

@@ -1,8 +1,13 @@
import os
# IMPORTATION STANDARD
# IMPORTATION THIRDPARTY
import pytest
from openbb_terminal.core.log.generation import formatter_with_exceptions
# IMPORTATION INTERNAL
from openbb_terminal.core.log.generation.formatter_with_exceptions import (
FormatterWithExceptions,
)
from openbb_terminal.core.log.generation.settings import AppSettings
from openbb_terminal.core.config.paths import REPOSITORY_DIRECTORY
app_settings = AppSettings(
commit_hash="MOCK_COMMIT_HASH",
@@ -10,56 +15,40 @@ app_settings = AppSettings(
identifier="MOCK_COMMIT_HASH",
session_id="MOCK_SESSION_ID",
)
formatter = formatter_with_exceptions.FormatterWithExceptions(app_settings)
formatter = FormatterWithExceptions(app_settings)
@pytest.mark.parametrize("exc, level", [(True, True), (False, "name"), (False, False)])
def test_calculate_level_name(exc, level, mocker):
def test_calculate_level_name(exc, level, mocker, recorder):
mock = mocker.Mock()
mock.exc_text = exc
mock.levelname = level
value = formatter.calculate_level_name(mock)
assert value
text_expected = FormatterWithExceptions.calculate_level_name(mock)
recorder.capture(text_expected)
def test_extract_log_extra(mocker):
def test_extract_log_extra(mocker, recorder):
mock = mocker.Mock()
mock.exc_text = True
mock.levelname = "name"
value = formatter.extract_log_extra(mock)
assert value
mock.levelname = "MOCK_LEVELNAME"
mock.user_id = "MOCK_USER_ID"
mock.session_id = "MOCK_SESSION_ID"
text_expected = FormatterWithExceptions.extract_log_extra(mock)
recorder.capture(text_expected)
def test_filter_piis():
text = (
f"test 1.1.1.1 chavi@chavi.com {REPOSITORY_DIRECTORY.name}{os.sep} dd{os.sep}dd"
)
value = formatter.filter_piis(text)
assert value
def test_filter_log_line(recorder):
text = "test 1.1.1.1 testestest chavi@chavi.com testestest C:\\Users\\username\\some folder\\some path testestest"
text_expected = FormatterWithExceptions.filter_log_line(text=text)
recorder.capture(text_expected)
text = "test 1.1.1.1 testestest chavi@chavi.com testestest /home/username/some folder/some path testestest"
text_expected = FormatterWithExceptions.filter_log_line(text=text)
recorder.capture(text_expected)
def test_filter_special_characters():
text = "\nhello \this \rgreetings"
value = formatter.filter_special_characters(text)
assert value == " - hello his greetings"
def test_detect_terminal_message():
value = formatter.detect_terminal_message("The command doesn't exist on the")
assert value or os.sep != "/"
new_val = formatter.detect_terminal_message("boring message")
assert new_val is False
def test_filter_log_line():
value = formatter.filter_log_line("stocks")
assert value
value = formatter.filter_log_line(
"The command doesn't exist on the menu. - Traceback hello"
)
assert value
text = "\\n\nhello \this\\r \rgreetings"
text_expected = FormatterWithExceptions.filter_log_line(text=text)
recorder.capture(text_expected)
def test_formatException_invalid():

1
tests/openbb_terminal/core/log/generation/txt/test_formatter_with_exceptions/test_calculate_level_name[False-False].txt vendored Normal file
View File

@@ -0,0 +1 @@
U

1
tests/openbb_terminal/core/log/generation/txt/test_formatter_with_exceptions/test_calculate_level_name[False-name].txt vendored Normal file
View File

@@ -0,0 +1 @@
n

1
tests/openbb_terminal/core/log/generation/txt/test_formatter_with_exceptions/test_calculate_level_name[True-True].txt vendored Normal file
View File

@@ -0,0 +1 @@
X

1
tests/openbb_terminal/core/log/generation/txt/test_formatter_with_exceptions/test_filter_log_line.txt vendored Normal file
View File

@@ -0,0 +1 @@
test FILTERED_IP testestest FILTERED_EMAIL testestest C:\Users\username\some folder\some path testestest

1
tests/openbb_terminal/core/log/generation/txt/test_formatter_with_exceptions/test_filter_log_line_1.txt vendored Normal file
View File

@@ -0,0 +1 @@
test FILTERED_IP testestest FILTERED_EMAIL testestest /home/username/some folder/some path testestest

1
tests/openbb_terminal/core/log/generation/txt/test_formatter_with_exceptions/test_filter_log_line_2.txt vendored Normal file
View File

@@ -0,0 +1 @@
MOCKING_BREAKLINE MOCKING_BREAKLINE hello MOCKING_TAB his MOCKING_CARRIAGE_RETURN MOCKING_CARRIAGE_RETURN greetings