Feature / Logging (#4923)

* init

* updates

* Update loggers.py

* Update loggers.py

* sdk handling

* feature flag

* fix MOCKING_USER_PATH for windows

* Update loggers.py

* Update plotly.html

* updates

* Update parent_classes.py

* Update plotly.html

* add posthog to deps

* Update loggers.py

frontend-components/tables/index.html

@@ -18,17 +18,8 @@
               const data = JSON.parse(window.json_data);
 
               const log_id = data?.log_id || "";
-              const posthog_obj = data?.posthog || {};
-
               if (log_id != "" && log_id != "REPLACE_ME") {
-                if (posthog_obj?.user_id) {
-                  posthog.identify(posthog_obj.user_id, {
-                    email: posthog_obj.email,
-                  });
-                  posthog.alias(posthog_obj.user_id, log_id);
-                } else {
-                  posthog.identify(log_id);
-                }
+                posthog.identify(log_id);
               }
 
               posthog.capture("table", {

openbb_terminal/base_helpers.py

@@ -4,6 +4,7 @@ import os
 from typing import Any, List, Literal, Optional
 
 from dotenv import load_dotenv
+from posthog import Posthog
 from rich.console import Console
 
 from openbb_terminal.core.config.paths import (
@@ -16,6 +17,11 @@ console = Console()
 
 menus = Literal["", "featflags", "settings"]
 
+openbb_posthog = Posthog(
+    "phc_vhssDAMod5qIplznQ75Kdgz4aB1qPFmeVmfEOZ4hkRw",
+    host="https://app.posthog.com",
+)
+
 
 def handle_error(name: str, default: Any, menu: menus = ""):
     """Handles the error by returning the default value and printing an

openbb_terminal/core/log/generation/formatter_with_exceptions.py

@@ -75,8 +75,10 @@ class FormatterWithExceptions(logging.Formatter):
 
     @staticmethod
     def mock_home_directory(text: str) -> str:
-        user_home_directory = str(HOME_DIRECTORY)
-        text_mocked = text.replace(user_home_directory, "MOCKING_USER_PATH")
+        user_home_directory = str(HOME_DIRECTORY.as_posix())
+        text_mocked = text.replace("\\\\", "/").replace(
+            user_home_directory, "MOCKING_USER_PATH"
+        )
 
         return text_mocked
 

openbb_terminal/core/models/profile_model.py

@@ -2,6 +2,7 @@ from typing import Dict
 
 from pydantic.dataclasses import dataclass
 
+from openbb_terminal.base_helpers import openbb_posthog
 from openbb_terminal.core.models import BaseModel
 
 
@@ -34,6 +35,7 @@ class ProfileModel(BaseModel):
         self.email = email
         self.username = self.email[: self.email.find("@")]
         self.remember = remember
+        openbb_posthog.identify(self.uuid, {"email": self.email})
 
     def get_uuid(self) -> str:
         """Get uuid.

openbb_terminal/core/plots/plotly.html

@@ -29,24 +29,22 @@
           capture_pageview: false,
           loaded: function (posthog) {
             const log_id = window.plotly_figure?.log_id || "";
-            const posthog_obj = window.plotly_figure?.posthog || {};
 
             if (log_id != "" && log_id != "REPLACE_ME") {
-              if (posthog_obj?.user_id) {
-                console.log("aliasing", posthog_obj.user_id, log_id);
-                posthog.identify(posthog_obj.user_id, { email: posthog_obj.email });
-                posthog.alias(posthog_obj.user_id, log_id);
-              } else {
-                posthog.identify(log_id);
-              }
-              console.log("identified", log_id);
+              posthog.identify(log_id);
             }
 
             posthog.onFeatureFlags(function () {
-              if (!posthog.isFeatureEnabled("record-pywry", {send_event: false})) {
+              if (
+                !posthog.isFeatureEnabled("record-pywry", { send_event: false })
+              ) {
                 posthog.stopSessionRecording();
               }
-              if (!posthog.isFeatureEnabled("collect-logs-pywry", {send_event: false})) {
+              if (
+                !posthog.isFeatureEnabled("collect-logs-pywry", {
+                  send_event: false,
+                })
+              ) {
                 posthog.opt_out_capturing();
               } else if (posthog.has_opted_out_capturing()) {
                 posthog.opt_in_capturing();
@@ -288,7 +286,10 @@
               }
             });
           }
-          if (globals.cmd_src != "" && window.plotly_figure?.posthog?.collect_logs) {
+          if (
+            globals.cmd_src != "" &&
+            window.plotly_figure?.posthog?.collect_logs
+          ) {
             posthog.capture("chart", {
               INFO: {
                 command: globals.cmd_src,

openbb_terminal/core/plots/table.html

@@ -18,17 +18,8 @@
               const data = JSON.parse(window.json_data);
 
               const log_id = data?.log_id || "";
-              const posthog_obj = data?.posthog || {};
-
               if (log_id != "" && log_id != "REPLACE_ME") {
-                if (posthog_obj?.user_id) {
-                  posthog.identify(posthog_obj.user_id, {
-                    email: posthog_obj.email,
-                  });
-                  posthog.alias(posthog_obj.user_id, log_id);
-                } else {
-                  posthog.identify(log_id);
-                }
+                posthog.identify(log_id);
               }
 
               posthog.capture("table", {

openbb_terminal/loggers.py

@@ -2,12 +2,16 @@
 __docformat__ = "numpy"
 
 # IMPORTATION STANDARD
+import atexit
+import json
 import logging
+import re
 import sys
 import time
 import uuid
 from pathlib import Path
-from typing import Optional
+from platform import platform, python_version
+from typing import Any, Dict, Optional
 
 # IMPORTATION THIRDPARTY
 try:
@@ -18,6 +22,7 @@ else:
     WITH_GIT = True
 
 # IMPORTATION INTERNAL
+from openbb_terminal.base_helpers import openbb_posthog
 from openbb_terminal.core.log.generation.directories import get_log_dir
 from openbb_terminal.core.log.generation.formatter_with_exceptions import (
     FormatterWithExceptions,
@@ -31,7 +36,10 @@ from openbb_terminal.core.log.generation.settings import (
     LogSettings,
     Settings,
 )
-from openbb_terminal.core.log.generation.user_logger import get_user_uuid
+from openbb_terminal.core.log.generation.user_logger import (
+    NO_USER_PLACEHOLDER,
+    get_user_uuid,
+)
 from openbb_terminal.core.session.current_system import (
     get_current_system,
     set_current_system,
@@ -88,6 +96,104 @@ def get_commit_hash(use_env=True) -> str:
     return commit_hash
 
 
+class PosthogHandler(logging.Handler):
+    """Posthog Handler"""
+
+    def __init__(self, settings: Settings):
+        super().__init__()
+        self.settings = settings
+        self.app_settings = settings.app_settings
+        self.logged_in = False
+        atexit.register(openbb_posthog.shutdown)
+
+    def emit(self, record: logging.LogRecord):
+        try:
+            self.send(record=record)
+        except Exception:
+            self.handleError(record)
+
+    def log_to_dict(self, log_info: str) -> dict:
+        """Log to dict"""
+        log_regex = r"(KEYS|PREFERENCES|SYSTEM|CMD|QUEUE): (.*)"
+        log_dict: Dict[str, Any] = {}
+
+        for log in re.findall(log_regex, log_info):
+            log_dict[log[0]] = json.loads(log[1])
+
+        sdk_regex = r"({\"INPUT\":.*})"
+        if sdk_dict := re.findall(sdk_regex, log_info):
+            log_dict["SDK"] = json.loads(sdk_dict[0])
+
+        return log_dict
+
+    def send(self, record: logging.LogRecord):
+        """Send log record to Posthog"""
+
+        app_settings = self.app_settings
+
+        level_name = logging.getLevelName(record.levelno)
+        log_line = FormatterWithExceptions.filter_log_line(text=record.getMessage())
+
+        log_extra = self.extract_log_extra(record=record)
+        log_extra.update(dict(level=level_name, message=log_line))
+        event_name = f"log_{level_name.lower()}"
+
+        if log_dict := self.log_to_dict(log_info=log_line):
+            event_name = f"log_{list(log_dict.keys())[0].lower()}"
+
+            log_extra = {**log_extra, **log_dict}
+            log_extra.pop("message", None)
+
+        if re.match(r"^(START|END|INPUT:)", log_line):
+            return
+
+        if not self.logged_in and get_user_uuid() != NO_USER_PLACEHOLDER:
+            self.logged_in = True
+            openbb_posthog.alias(get_user_uuid(), app_settings.identifier)
+
+        openbb_posthog.capture(
+            app_settings.identifier,
+            event_name,
+            properties=log_extra,
+        )
+
+    def extract_log_extra(self, record: logging.LogRecord) -> Dict[str, Any]:
+        """Extract log extra from record"""
+
+        log_extra: Dict[str, Any] = {
+            "appName": self.app_settings.name,
+            "appId": self.app_settings.identifier,
+            "sessionId": self.app_settings.session_id,
+            "commitHash": self.app_settings.commit_hash,
+            "platform": platform(),
+            "pythonVersion": python_version(),
+            "terminalVersion": current_system.VERSION,
+        }
+
+        if get_user_uuid() != NO_USER_PLACEHOLDER:
+            log_extra["userId"] = get_user_uuid()
+
+        if hasattr(record, "extra"):
+            log_extra = {**log_extra, **record.extra}
+
+        if record.exc_info:
+            log_extra["exception"] = {
+                "type": str(record.exc_info[0]),
+                "value": str(record.exc_info[1]),
+                "traceback": self.format(record),
+            }
+
+        return log_extra
+
+
+def add_posthog_handler(settings: Settings):
+    app_settings = settings.app_settings
+    handler = PosthogHandler(settings=settings)
+    formatter = FormatterWithExceptions(app_settings=app_settings)
+    handler.setFormatter(formatter)
+    logging.getLogger().addHandler(handler)
+
+
 def add_stdout_handler(settings: Settings):
     app_settings = settings.app_settings
     handler = logging.StreamHandler(sys.stdout)
@@ -153,6 +259,12 @@ def setup_handlers(settings: Settings):
         FormatterWithExceptions.LOGFORMAT.replace("|", "-"),
     )
 
+    if (
+        not any([current_system.TEST_MODE, current_system.LOGGING_SUPPRESS])
+        and current_system.LOG_COLLECT
+    ):
+        add_posthog_handler(settings=settings)
+
 
 def setup_logging(
     app_name: Optional[str] = None,

openbb_terminal/parent_classes.py

@@ -300,10 +300,9 @@ class BaseController(metaclass=ABCMeta):
         if self.queue:
             joined_queue = self.COMMAND_SEPARATOR.join(self.queue)
             if not self.contains_keys(joined_queue):
+                queue = {"path": self.PATH, "queue": joined_queue}
                 logger.info(
-                    "QUEUE: {'path': '%s', 'queue': '%s'}",
-                    self.PATH,
-                    joined_queue,
+                    "QUEUE: %s", json.dumps(queue, default=str, ensure_ascii=False)
                 )
 
     def log_cmd_and_queue(

poetry.lock

@@ -3948,6 +3948,18 @@ build = ["blurb", "twine", "wheel"]
 docs = ["sphinx"]
 test = ["pytest (<5.4)", "pytest-cov"]
 
+[[package]]
+name = "monotonic"
+version = "1.6"
+description = "An implementation of time.monotonic() for Python 2 & < 3.3"
+category = "main"
+optional = false
+python-versions = "*"
+files = [
+    {file = "monotonic-1.6-py2.py3-none-any.whl", hash = "sha256:68687e19a14f11f26d140dd5c86f3dba4bf5df58003000ed467e0e2a69bca96c"},
+    {file = "monotonic-1.6.tar.gz", hash = "sha256:3a55207bcfed53ddd5c5bae174524062935efed17792e9de2ad0205ce9ad63f7"},
+]
+
 [[package]]
 name = "more-itertools"
 version = "9.1.0"
@@ -5111,6 +5123,30 @@ setuptools = ">=38.6.0,<50.0.0 || >50.0.0"
 statsmodels = ">=0.13.2"
 urllib3 = "*"
 
+[[package]]
+name = "posthog"
+version = "3.0.1"
+description = "Integrate PostHog into any python application."
+category = "main"
+optional = false
+python-versions = "*"
+files = [
+    {file = "posthog-3.0.1-py2.py3-none-any.whl", hash = "sha256:9c7f92fecc713257d4b2710d05b456569c9156fbdd3e85655ba7ba5ba6c7b3ae"},
+    {file = "posthog-3.0.1.tar.gz", hash = "sha256:57d2791ff5752ce56ba0f9bb8876faf3ca9208f1c2c6ceaeb5a2504c34493767"},
+]
+
+[package.dependencies]
+backoff = ">=1.10.0"
+monotonic = ">=1.5"
+python-dateutil = ">2.1"
+requests = ">=2.7,<3.0"
+six = ">=1.5"
+
+[package.extras]
+dev = ["black", "flake8", "flake8-print", "isort", "pre-commit"]
+sentry = ["django", "sentry-sdk"]
+test = ["coverage", "flake8", "freezegun (==0.3.15)", "mock (>=2.0.0)", "pylint", "pytest"]
+
 [[package]]
 name = "praw"
 version = "7.7.0"
@@ -9656,4 +9692,4 @@ optimization = ["Riskfolio-Lib"]
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.8,<3.11, !=3.9.7"
-content-hash = "d2bc3a75f2aff5123a842c656d6e8a9d51fa829433dc0cb1a02deb1d5c6967d9"
+content-hash = "29f4161609dbefbe373d25a2f561b1a3d37072236e805195150c802e4eb4108f"

pyproject.toml

@@ -114,6 +114,7 @@ setuptools-rust = {version = "^1.5.2", optional = true}
 transformers = {version = "^4.26.1", optional = true}
 yt-dlp = {version = "^2023.2.17", optional = true}
 pydantic = "^1.10.5"
+posthog = "^3.0.1"
 
 
 

requirements-full.txt

@@ -154,6 +154,7 @@ mccabe==0.7.0 ; python_version >= "3.8" and python_full_version != "3.9.7" and p
 mdit-py-plugins==0.2.8 ; python_version >= "3.8" and python_version < "3.11" and python_full_version != "3.9.7"
 mistune==2.0.5 ; python_version >= "3.8" and python_full_version != "3.9.7" and python_version < "3.11"
 mock==4.0.3 ; python_version >= "3.8" and python_full_version != "3.9.7" and python_version < "3.11"
+monotonic==1.6 ; python_version >= "3.8" and python_full_version != "3.9.7" and python_version < "3.11"
 more-itertools==9.1.0 ; python_version >= "3.8" and python_full_version != "3.9.7" and python_version < "3.11"
 mplfinance==0.12.9b7 ; python_version >= "3.8" and python_full_version != "3.9.7" and python_version < "3.11"
 mstarpy==0.0.4 ; python_version >= "3.8" and python_full_version != "3.9.7" and python_version < "3.11"
@@ -203,6 +204,7 @@ plotly==5.13.1 ; python_version >= "3.8" and python_full_version != "3.9.7" and
 pluggy==1.0.0 ; python_version >= "3.8" and python_full_version != "3.9.7" and python_version < "3.11"
 pmaw==3.0.0 ; python_version >= "3.8" and python_full_version != "3.9.7" and python_version < "3.11"
 pmdarima==2.0.3 ; python_version >= "3.8" and python_full_version != "3.9.7" and python_version < "3.11"
+posthog==3.0.1 ; python_version >= "3.8" and python_full_version != "3.9.7" and python_version < "3.11"
 praw==7.7.0 ; python_version >= "3.8" and python_version < "3.11" and python_full_version != "3.9.7"
 prawcore==2.3.0 ; python_version >= "3.8" and python_version < "3.11" and python_full_version != "3.9.7"
 pre-commit==2.21.0 ; python_version >= "3.8" and python_full_version != "3.9.7" and python_version < "3.11"

requirements.txt

@@ -109,6 +109,7 @@ matplotlib-inline==0.1.6 ; python_version >= "3.8" and python_full_version != "3
 matplotlib==3.7.0 ; python_version >= "3.8" and python_full_version != "3.9.7" and python_version < "3.11"
 mccabe==0.7.0 ; python_version >= "3.8" and python_full_version != "3.9.7" and python_version < "3.11"
 mistune==2.0.5 ; python_version >= "3.8" and python_full_version != "3.9.7" and python_version < "3.11"
+monotonic==1.6 ; python_version >= "3.8" and python_full_version != "3.9.7" and python_version < "3.11"
 more-itertools==9.1.0 ; python_version >= "3.8" and python_full_version != "3.9.7" and python_version < "3.11"
 mplfinance==0.12.9b7 ; python_version >= "3.8" and python_full_version != "3.9.7" and python_version < "3.11"
 mstarpy==0.0.4 ; python_version >= "3.8" and python_full_version != "3.9.7" and python_version < "3.11"
@@ -142,6 +143,7 @@ pkgutil-resolve-name==1.3.10 ; python_version >= "3.8" and python_version < "3.9
 platformdirs==3.1.1 ; python_version >= "3.8" and python_full_version != "3.9.7" and python_version < "3.11"
 plotly==5.13.1 ; python_version >= "3.8" and python_full_version != "3.9.7" and python_version < "3.11"
 pmaw==3.0.0 ; python_version >= "3.8" and python_full_version != "3.9.7" and python_version < "3.11"
+posthog==3.0.1 ; python_version >= "3.8" and python_full_version != "3.9.7" and python_version < "3.11"
 praw==7.7.0 ; python_version >= "3.8" and python_version < "3.11" and python_full_version != "3.9.7"
 prawcore==2.3.0 ; python_version >= "3.8" and python_version < "3.11" and python_full_version != "3.9.7"
 prometheus-client==0.16.0 ; python_version >= "3.8" and python_full_version != "3.9.7" and python_version < "3.11"