From f5bd14c03c5f5e67d48cef59e8b2aed6b4ab3016 Mon Sep 17 00:00:00 2001 From: montezdesousa <79287829+montezdesousa@users.noreply.github.com> Date: Fri, 1 Sep 2023 20:48:09 +0100 Subject: [PATCH] Auto build sdk (#5397) * save map of installed extensions * allow silencing build prints * use console class instead * uncomment * remove menus with empty commands * detect changes * build * change warning * rename variable * uncomment * remove unused variable * check and build * rename params * change func name * Update stocks.py * Update crypto.py * Update economy.py * Update forex.py * cwd * fix test * remove cwd * rename var * linters debug mode * move code to core * change description * add option to avoid auto build * unittests * minor fix * update readme * readme * Update README.md * Update README.md * Update package_builder.py * ruff * fix test --- openbb_sdk/openbb/__init__.py | 42 ++- openbb_sdk/openbb/package/extension_map.json | 22 ++ openbb_sdk/openbb/package/stocks_disc.py | 300 ------------------ openbb_sdk/openbb/package/stocks_dps.py | 204 ------------ openbb_sdk/openbb/package/stocks_gov.py | 192 ----------- openbb_sdk/openbb/package/stocks_ins.py | 272 ---------------- openbb_sdk/sdk/core/README.md | 41 +-- openbb_sdk/sdk/core/openbb_core/app/env.py | 4 + .../openbb_core/app/static/package_builder.py | 97 +++--- .../sdk/core/openbb_core/app/static/utils.py | 107 +++++++ .../tests/app/static/test_package_builder.py | 2 +- .../tests/app/static/test_static_utils.py | 122 +++++++ 12 files changed, 355 insertions(+), 1050 deletions(-) create mode 100644 openbb_sdk/openbb/package/extension_map.json delete mode 100644 openbb_sdk/openbb/package/stocks_disc.py delete mode 100644 openbb_sdk/openbb/package/stocks_dps.py delete mode 100644 openbb_sdk/openbb/package/stocks_gov.py delete mode 100644 openbb_sdk/openbb/package/stocks_ins.py create mode 100644 openbb_sdk/sdk/core/openbb_core/app/static/utils.py create mode 100644 openbb_sdk/sdk/core/tests/app/static/test_static_utils.py diff --git a/openbb_sdk/openbb/__init__.py b/openbb_sdk/openbb/__init__.py index 8c62a0f4993..0c30e50eb83 100644 --- a/openbb_sdk/openbb/__init__.py +++ b/openbb_sdk/openbb/__init__.py @@ -1,27 +1,23 @@ """OpenBB SDK.""" # flake8: noqa +import os +from pathlib import Path from typing import List, Optional, Union +from openbb_core.app.static.utils import auto_build as _auto_build, build as _build from openbb_core.app.static.app_factory import ( create_app as _create_app, BaseApp as _BaseApp, ) -try: - # pylint: disable=import-outside-toplevel - from openbb.package.__extensions__ import Extensions as _Extensions - - obb: Union[_BaseApp, _Extensions] = _create_app(_Extensions) - sdk = obb -except (ImportError, ModuleNotFoundError): - print("Failed to import extensions. Run `openbb.build()` to build extensions code.") - obb = sdk = _create_app() +_this_dir = Path(os.path.dirname(os.path.realpath(__file__))) def build( modules: Optional[Union[str, List[str]]] = None, lint: bool = True, + verbose: bool = False, ) -> None: """Build extension modules. @@ -33,20 +29,20 @@ def build( If None, all modules are rebuilt. lint : bool, optional Whether to lint the code, by default True + verbose : bool, optional + Enable/disable verbose mode """ + _build(directory=_this_dir, modules=modules, lint=lint, verbose=verbose) + + +_auto_build(directory=_this_dir) + +try: # pylint: disable=import-outside-toplevel - import os - from pathlib import Path - from multiprocessing import Pool - from openbb_core.app.static.package_builder import PackageBuilder + from openbb.package.__extensions__ import Extensions as _Extensions - current_dir = Path(os.path.dirname(os.path.realpath(__file__))) - - # `build` is running in a separate process. This avoids consecutive calls to this - # function in the same interpreter to reuse objects already in memory. Not doing - # this was causing docstrings to have repeated sections, for example. - with Pool(processes=1) as pool: - pool.apply( - PackageBuilder(current_dir).build, - args=(modules, lint), - ) + obb: Union[_BaseApp, _Extensions] = _create_app(_Extensions) + sdk = obb +except (ImportError, ModuleNotFoundError): + print("Failed to import extensions.") + obb = sdk = _create_app() diff --git a/openbb_sdk/openbb/package/extension_map.json b/openbb_sdk/openbb/package/extension_map.json new file mode 100644 index 00000000000..22a9b30f21e --- /dev/null +++ b/openbb_sdk/openbb/package/extension_map.json @@ -0,0 +1,22 @@ +{ + "openbb_core_extension": [ + "charting", + "crypto", + "economy", + "fixedincome", + "forex", + "futures", + "news", + "qa", + "stocks", + "ta" + ], + "openbb_provider_extension": [ + "benzinga", + "cboe", + "fmp", + "fred", + "polygon", + "yfinance" + ] +} \ No newline at end of file diff --git a/openbb_sdk/openbb/package/stocks_disc.py b/openbb_sdk/openbb/package/stocks_disc.py deleted file mode 100644 index 6b31898d451..00000000000 --- a/openbb_sdk/openbb/package/stocks_disc.py +++ /dev/null @@ -1,300 +0,0 @@ -### THIS FILE IS AUTO-GENERATED. DO NOT EDIT. ### - - -import openbb_core.app.model.results.empty -from openbb_core.app.model.obbject import OBBject -from openbb_core.app.static.container import Container -from openbb_core.app.static.filters import filter_inputs -from pydantic import validate_arguments - - -class CLASS_stocks_disc(Container): - """/stocks/disc - active - arkord - asc - dividends - filings - fipo - gainers - gtech - hotpenny - ipo - losers - lowfloat - pipo - rtat - trending - ugs - ulc - upcoming - """ - - def __repr__(self) -> str: - return self.__doc__ or "" - - @validate_arguments - def active( - self, chart: bool = False - ) -> OBBject[openbb_core.app.model.results.empty.Empty]: - """Most active stocks by intraday trade volumes.""" # noqa: E501 - - inputs = filter_inputs( - chart=chart, - ) - - return self._command_runner.run( - "/stocks/disc/active", - **inputs, - ) - - @validate_arguments - def arkord( - self, chart: bool = False - ) -> OBBject[openbb_core.app.model.results.empty.Empty]: - """Order by ARK INvestment Management LLC.""" # noqa: E501 - - inputs = filter_inputs( - chart=chart, - ) - - return self._command_runner.run( - "/stocks/disc/arkord", - **inputs, - ) - - @validate_arguments - def asc( - self, chart: bool = False - ) -> OBBject[openbb_core.app.model.results.empty.Empty]: - """Small cap stocks with revenue and earnings growth more than 25%.""" # noqa: E501 - - inputs = filter_inputs( - chart=chart, - ) - - return self._command_runner.run( - "/stocks/disc/asc", - **inputs, - ) - - @validate_arguments - def dividends( - self, chart: bool = False - ) -> OBBject[openbb_core.app.model.results.empty.Empty]: - inputs = filter_inputs( - chart=chart, - ) - - return self._command_runner.run( - "/stocks/disc/dividends", - **inputs, - ) - - @validate_arguments - def filings( - self, chart: bool = False - ) -> OBBject[openbb_core.app.model.results.empty.Empty]: - """The most-recent form submissions to the SEC.""" # noqa: E501 - - inputs = filter_inputs( - chart=chart, - ) - - return self._command_runner.run( - "/stocks/disc/filings", - **inputs, - ) - - @validate_arguments - def fipo( - self, chart: bool = False - ) -> OBBject[openbb_core.app.model.results.empty.Empty]: - """Future IPOs dates.""" # noqa: E501 - - inputs = filter_inputs( - chart=chart, - ) - - return self._command_runner.run( - "/stocks/disc/fipo", - **inputs, - ) - - @validate_arguments - def gainers( - self, chart: bool = False - ) -> OBBject[openbb_core.app.model.results.empty.Empty]: - """Show latest top gainers.""" # noqa: E501 - - inputs = filter_inputs( - chart=chart, - ) - - return self._command_runner.run( - "/stocks/disc/gainers", - **inputs, - ) - - @validate_arguments - def gtech( - self, chart: bool = False - ) -> OBBject[openbb_core.app.model.results.empty.Empty]: - """Tech stocks with revenue and earnings growth more than 25%.""" # noqa: E501 - - inputs = filter_inputs( - chart=chart, - ) - - return self._command_runner.run( - "/stocks/disc/gtech", - **inputs, - ) - - @validate_arguments - def hotpenny( - self, chart: bool = False - ) -> OBBject[openbb_core.app.model.results.empty.Empty]: - """Today's hot penny stocks.""" # noqa: E501 - - inputs = filter_inputs( - chart=chart, - ) - - return self._command_runner.run( - "/stocks/disc/hotpenny", - **inputs, - ) - - @validate_arguments - def ipo( - self, chart: bool = False - ) -> OBBject[openbb_core.app.model.results.empty.Empty]: - inputs = filter_inputs( - chart=chart, - ) - - return self._command_runner.run( - "/stocks/disc/ipo", - **inputs, - ) - - @validate_arguments - def losers( - self, chart: bool = False - ) -> OBBject[openbb_core.app.model.results.empty.Empty]: - """Show latest top losers.""" # noqa: E501 - - inputs = filter_inputs( - chart=chart, - ) - - return self._command_runner.run( - "/stocks/disc/losers", - **inputs, - ) - - @validate_arguments - def lowfloat( - self, chart: bool = False - ) -> OBBject[openbb_core.app.model.results.empty.Empty]: - """Low float stocks under 10M shares float.""" # noqa: E501 - - inputs = filter_inputs( - chart=chart, - ) - - return self._command_runner.run( - "/stocks/disc/lowfloat", - **inputs, - ) - - @validate_arguments - def pipo( - self, chart: bool = False - ) -> OBBject[openbb_core.app.model.results.empty.Empty]: - """Past IPOs dates.""" # noqa: E501 - - inputs = filter_inputs( - chart=chart, - ) - - return self._command_runner.run( - "/stocks/disc/pipo", - **inputs, - ) - - @validate_arguments - def rtat( - self, chart: bool = False - ) -> OBBject[openbb_core.app.model.results.empty.Empty]: - """Top 10 retail traded stocks per day.""" # noqa: E501 - - inputs = filter_inputs( - chart=chart, - ) - - return self._command_runner.run( - "/stocks/disc/rtat", - **inputs, - ) - - @validate_arguments - def trending( - self, chart: bool = False - ) -> OBBject[openbb_core.app.model.results.empty.Empty]: - """Trending news.""" # noqa: E501 - - inputs = filter_inputs( - chart=chart, - ) - - return self._command_runner.run( - "/stocks/disc/trending", - **inputs, - ) - - @validate_arguments - def ugs( - self, chart: bool = False - ) -> OBBject[openbb_core.app.model.results.empty.Empty]: - """Undervalued stocks with revenue and earnings growth above 25%.""" # noqa: E501 - - inputs = filter_inputs( - chart=chart, - ) - - return self._command_runner.run( - "/stocks/disc/ugs", - **inputs, - ) - - @validate_arguments - def ulc( - self, chart: bool = False - ) -> OBBject[openbb_core.app.model.results.empty.Empty]: - """Potentially undervalued large cap stocks.""" # noqa: E501 - - inputs = filter_inputs( - chart=chart, - ) - - return self._command_runner.run( - "/stocks/disc/ulc", - **inputs, - ) - - @validate_arguments - def upcoming( - self, chart: bool = False - ) -> OBBject[openbb_core.app.model.results.empty.Empty]: - """Upcoming earnings release dates.""" # noqa: E501 - - inputs = filter_inputs( - chart=chart, - ) - - return self._command_runner.run( - "/stocks/disc/upcoming", - **inputs, - ) diff --git a/openbb_sdk/openbb/package/stocks_dps.py b/openbb_sdk/openbb/package/stocks_dps.py deleted file mode 100644 index 5ee280f4c35..00000000000 --- a/openbb_sdk/openbb/package/stocks_dps.py +++ /dev/null @@ -1,204 +0,0 @@ -### THIS FILE IS AUTO-GENERATED. DO NOT EDIT. ### - - -import openbb_core.app.model.results.empty -from openbb_core.app.model.obbject import OBBject -from openbb_core.app.static.container import Container -from openbb_core.app.static.filters import filter_inputs -from pydantic import validate_arguments - - -class CLASS_stocks_dps(Container): - """/stocks/dps - ctb - dpotc - ftd - hsi - pos - prom - psi - psi_q - psi_sg - shorted - sidtc - spos - """ - - def __repr__(self) -> str: - return self.__doc__ or "" - - @validate_arguments - def ctb( - self, chart: bool = False - ) -> OBBject[openbb_core.app.model.results.empty.Empty]: - """Cost to borrow of stocks.""" # noqa: E501 - - inputs = filter_inputs( - chart=chart, - ) - - return self._command_runner.run( - "/stocks/dps/ctb", - **inputs, - ) - - @validate_arguments - def dpotc( - self, chart: bool = False - ) -> OBBject[openbb_core.app.model.results.empty.Empty]: - """Dark pools (ATS) vs OTC data.""" # noqa: E501 - - inputs = filter_inputs( - chart=chart, - ) - - return self._command_runner.run( - "/stocks/dps/dpotc", - **inputs, - ) - - @validate_arguments - def ftd( - self, chart: bool = False - ) -> OBBject[openbb_core.app.model.results.empty.Empty]: - """Fails-to-deliver data.""" # noqa: E501 - - inputs = filter_inputs( - chart=chart, - ) - - return self._command_runner.run( - "/stocks/dps/ftd", - **inputs, - ) - - @validate_arguments - def hsi( - self, chart: bool = False - ) -> OBBject[openbb_core.app.model.results.empty.Empty]: - """Show top high short interest stocks of over 20% ratio.""" # noqa: E501 - - inputs = filter_inputs( - chart=chart, - ) - - return self._command_runner.run( - "/stocks/dps/hsi", - **inputs, - ) - - @validate_arguments - def pos( - self, chart: bool = False - ) -> OBBject[openbb_core.app.model.results.empty.Empty]: - """Dark pool short position.""" # noqa: E501 - - inputs = filter_inputs( - chart=chart, - ) - - return self._command_runner.run( - "/stocks/dps/pos", - **inputs, - ) - - @validate_arguments - def prom( - self, chart: bool = False - ) -> OBBject[openbb_core.app.model.results.empty.Empty]: - """Promising tickers based on dark pool shares regression.""" # noqa: E501 - - inputs = filter_inputs( - chart=chart, - ) - - return self._command_runner.run( - "/stocks/dps/prom", - **inputs, - ) - - @validate_arguments - def psi( - self, chart: bool = False - ) -> OBBject[openbb_core.app.model.results.empty.Empty]: - """Price vs short interest volume""" # noqa: E501 - - inputs = filter_inputs( - chart=chart, - ) - - return self._command_runner.run( - "/stocks/dps/psi", - **inputs, - ) - - @validate_arguments - def psi_q( - self, chart: bool = False - ) -> OBBject[openbb_core.app.model.results.empty.Empty]: - inputs = filter_inputs( - chart=chart, - ) - - return self._command_runner.run( - "/stocks/dps/psi_q", - **inputs, - ) - - @validate_arguments - def psi_sg( - self, chart: bool = False - ) -> OBBject[openbb_core.app.model.results.empty.Empty]: - inputs = filter_inputs( - chart=chart, - ) - - return self._command_runner.run( - "/stocks/dps/psi_sg", - **inputs, - ) - - @validate_arguments - def shorted( - self, chart: bool = False - ) -> OBBject[openbb_core.app.model.results.empty.Empty]: - """Most shorted stocks.""" # noqa: E501 - - inputs = filter_inputs( - chart=chart, - ) - - return self._command_runner.run( - "/stocks/dps/shorted", - **inputs, - ) - - @validate_arguments - def sidtc( - self, chart: bool = False - ) -> OBBject[openbb_core.app.model.results.empty.Empty]: - """Short interest and days to cover.""" # noqa: E501 - - inputs = filter_inputs( - chart=chart, - ) - - return self._command_runner.run( - "/stocks/dps/sidtc", - **inputs, - ) - - @validate_arguments - def spos( - self, chart: bool = False - ) -> OBBject[openbb_core.app.model.results.empty.Empty]: - """Net short vs position.""" # noqa: E501 - - inputs = filter_inputs( - chart=chart, - ) - - return self._command_runner.run( - "/stocks/dps/spos", - **inputs, - ) diff --git a/openbb_sdk/openbb/package/stocks_gov.py b/openbb_sdk/openbb/package/stocks_gov.py deleted file mode 100644 index edd640d2313..00000000000 --- a/openbb_sdk/openbb/package/stocks_gov.py +++ /dev/null @@ -1,192 +0,0 @@ -### THIS FILE IS AUTO-GENERATED. DO NOT EDIT. ### - - -import openbb_core.app.model.results.empty -from openbb_core.app.model.obbject import OBBject -from openbb_core.app.static.container import Container -from openbb_core.app.static.filters import filter_inputs -from pydantic import validate_arguments - - -class CLASS_stocks_gov(Container): - """/stocks/gov - contracts - government_trading - gtrades - histcont - lastcontracts - lasttrades - lobbying - qtrcontracts - topbuys - toplobbying - topsells - """ - - def __repr__(self) -> str: - return self.__doc__ or "" - - @validate_arguments - def contracts( - self, chart: bool = False - ) -> OBBject[openbb_core.app.model.results.empty.Empty]: - """Return government contracts.""" # noqa: E501 - - inputs = filter_inputs( - chart=chart, - ) - - return self._command_runner.run( - "/stocks/gov/contracts", - **inputs, - ) - - @validate_arguments - def government_trading( - self, chart: bool = False - ) -> OBBject[openbb_core.app.model.results.empty.Empty]: - """Return government trading.""" # noqa: E501 - - inputs = filter_inputs( - chart=chart, - ) - - return self._command_runner.run( - "/stocks/gov/government_trading", - **inputs, - ) - - @validate_arguments - def gtrades( - self, chart: bool = False - ) -> OBBject[openbb_core.app.model.results.empty.Empty]: - """Return government trades.""" # noqa: E501 - - inputs = filter_inputs( - chart=chart, - ) - - return self._command_runner.run( - "/stocks/gov/gtrades", - **inputs, - ) - - @validate_arguments - def histcont( - self, chart: bool = False - ) -> OBBject[openbb_core.app.model.results.empty.Empty]: - """Historical quarterly government contracts.""" # noqa: E501 - - inputs = filter_inputs( - chart=chart, - ) - - return self._command_runner.run( - "/stocks/gov/histcont", - **inputs, - ) - - @validate_arguments - def lastcontracts( - self, chart: bool = False - ) -> OBBject[openbb_core.app.model.results.empty.Empty]: - """Return last government contracts given out.""" # noqa: E501 - - inputs = filter_inputs( - chart=chart, - ) - - return self._command_runner.run( - "/stocks/gov/lastcontracts", - **inputs, - ) - - @validate_arguments - def lasttrades( - self, chart: bool = False - ) -> OBBject[openbb_core.app.model.results.empty.Empty]: - """Last trades.""" # noqa: E501 - - inputs = filter_inputs( - chart=chart, - ) - - return self._command_runner.run( - "/stocks/gov/lasttrades", - **inputs, - ) - - @validate_arguments - def lobbying( - self, chart: bool = False - ) -> OBBject[openbb_core.app.model.results.empty.Empty]: - """Corporate lobbying details.""" # noqa: E501 - - inputs = filter_inputs( - chart=chart, - ) - - return self._command_runner.run( - "/stocks/gov/lobbying", - **inputs, - ) - - @validate_arguments - def qtrcontracts( - self, chart: bool = False - ) -> OBBject[openbb_core.app.model.results.empty.Empty]: - """Quarterly government contracts analysis.""" # noqa: E501 - - inputs = filter_inputs( - chart=chart, - ) - - return self._command_runner.run( - "/stocks/gov/qtrcontracts", - **inputs, - ) - - @validate_arguments - def topbuys( - self, chart: bool = False - ) -> OBBject[openbb_core.app.model.results.empty.Empty]: - """Show most purchased stocks.""" # noqa: E501 - - inputs = filter_inputs( - chart=chart, - ) - - return self._command_runner.run( - "/stocks/gov/topbuys", - **inputs, - ) - - @validate_arguments - def toplobbying( - self, chart: bool = False - ) -> OBBject[openbb_core.app.model.results.empty.Empty]: - """Top corporate lobbying tickers.""" # noqa: E501 - - inputs = filter_inputs( - chart=chart, - ) - - return self._command_runner.run( - "/stocks/gov/toplobbying", - **inputs, - ) - - @validate_arguments - def topsells( - self, chart: bool = False - ) -> OBBject[openbb_core.app.model.results.empty.Empty]: - """Show most sold stocks.""" # noqa: E501 - - inputs = filter_inputs( - chart=chart, - ) - - return self._command_runner.run( - "/stocks/gov/topsells", - **inputs, - ) diff --git a/openbb_sdk/openbb/package/stocks_ins.py b/openbb_sdk/openbb/package/stocks_ins.py deleted file mode 100644 index 5727de2cc3a..00000000000 --- a/openbb_sdk/openbb/package/stocks_ins.py +++ /dev/null @@ -1,272 +0,0 @@ -### THIS FILE IS AUTO-GENERATED. DO NOT EDIT. ### - - -import openbb_core.app.model.results.empty -from openbb_core.app.model.obbject import OBBject -from openbb_core.app.static.container import Container -from openbb_core.app.static.filters import filter_inputs -from pydantic import validate_arguments - - -class CLASS_stocks_ins(Container): - """/stocks/ins - act - blcp - blcs - blip - blis - blop - blos - filt - lcb - lins - lip - lis - lit - lpsb - print_insider_data - stats - """ - - def __repr__(self) -> str: - return self.__doc__ or "" - - @validate_arguments - def act( - self, chart: bool = False - ) -> OBBject[openbb_core.app.model.results.empty.Empty]: - """Insider activity over time.""" # noqa: E501 - - inputs = filter_inputs( - chart=chart, - ) - - return self._command_runner.run( - "/stocks/ins/act", - **inputs, - ) - - @validate_arguments - def blcp( - self, chart: bool = False - ) -> OBBject[openbb_core.app.model.results.empty.Empty]: - """Big latest CEO/CFO purchaces ($25k+).""" # noqa: E501 - - inputs = filter_inputs( - chart=chart, - ) - - return self._command_runner.run( - "/stocks/ins/blcp", - **inputs, - ) - - @validate_arguments - def blcs( - self, chart: bool = False - ) -> OBBject[openbb_core.app.model.results.empty.Empty]: - """Big latest CEO/CFO sales ($100k+).""" # noqa: E501 - - inputs = filter_inputs( - chart=chart, - ) - - return self._command_runner.run( - "/stocks/ins/blcs", - **inputs, - ) - - @validate_arguments - def blip( - self, chart: bool = False - ) -> OBBject[openbb_core.app.model.results.empty.Empty]: - """Big latest insider purchaces ($25+).""" # noqa: E501 - - inputs = filter_inputs( - chart=chart, - ) - - return self._command_runner.run( - "/stocks/ins/blip", - **inputs, - ) - - @validate_arguments - def blis( - self, chart: bool = False - ) -> OBBject[openbb_core.app.model.results.empty.Empty]: - """Big latest insider sales ($100k+).""" # noqa: E501 - - inputs = filter_inputs( - chart=chart, - ) - - return self._command_runner.run( - "/stocks/ins/blis", - **inputs, - ) - - @validate_arguments - def blop( - self, chart: bool = False - ) -> OBBject[openbb_core.app.model.results.empty.Empty]: - """Big latest officer purchaces ($25k+).""" # noqa: E501 - - inputs = filter_inputs( - chart=chart, - ) - - return self._command_runner.run( - "/stocks/ins/blop", - **inputs, - ) - - @validate_arguments - def blos( - self, chart: bool = False - ) -> OBBject[openbb_core.app.model.results.empty.Empty]: - """Big latest officer sales ($100k+).""" # noqa: E501 - - inputs = filter_inputs( - chart=chart, - ) - - return self._command_runner.run( - "/stocks/ins/blos", - **inputs, - ) - - @validate_arguments - def filt( - self, chart: bool = False - ) -> OBBject[openbb_core.app.model.results.empty.Empty]: - """Filter insiders based on preset.""" # noqa: E501 - - inputs = filter_inputs( - chart=chart, - ) - - return self._command_runner.run( - "/stocks/ins/filt", - **inputs, - ) - - @validate_arguments - def lcb( - self, chart: bool = False - ) -> OBBject[openbb_core.app.model.results.empty.Empty]: - """Latest cluster buys.""" # noqa: E501 - - inputs = filter_inputs( - chart=chart, - ) - - return self._command_runner.run( - "/stocks/ins/lcb", - **inputs, - ) - - @validate_arguments - def lins( - self, chart: bool = False - ) -> OBBject[openbb_core.app.model.results.empty.Empty]: - """Last insider trading of the company.""" # noqa: E501 - - inputs = filter_inputs( - chart=chart, - ) - - return self._command_runner.run( - "/stocks/ins/lins", - **inputs, - ) - - @validate_arguments - def lip( - self, chart: bool = False - ) -> OBBject[openbb_core.app.model.results.empty.Empty]: - """Latest insider purchaces.""" # noqa: E501 - - inputs = filter_inputs( - chart=chart, - ) - - return self._command_runner.run( - "/stocks/ins/lip", - **inputs, - ) - - @validate_arguments - def lis( - self, chart: bool = False - ) -> OBBject[openbb_core.app.model.results.empty.Empty]: - """Latest insider sales.""" # noqa: E501 - - inputs = filter_inputs( - chart=chart, - ) - - return self._command_runner.run( - "/stocks/ins/lis", - **inputs, - ) - - @validate_arguments - def lit( - self, chart: bool = False - ) -> OBBject[openbb_core.app.model.results.empty.Empty]: - """Latest insider trading (all filings).""" # noqa: E501 - - inputs = filter_inputs( - chart=chart, - ) - - return self._command_runner.run( - "/stocks/ins/lit", - **inputs, - ) - - @validate_arguments - def lpsb( - self, chart: bool = False - ) -> OBBject[openbb_core.app.model.results.empty.Empty]: - """Latest penny stock buys.""" # noqa: E501 - - inputs = filter_inputs( - chart=chart, - ) - - return self._command_runner.run( - "/stocks/ins/lpsb", - **inputs, - ) - - @validate_arguments - def print_insider_data( - self, chart: bool = False - ) -> OBBject[openbb_core.app.model.results.empty.Empty]: - """Print insider data.""" # noqa: E501 - - inputs = filter_inputs( - chart=chart, - ) - - return self._command_runner.run( - "/stocks/ins/print_insider_data", - **inputs, - ) - - @validate_arguments - def stats( - self, chart: bool = False - ) -> OBBject[openbb_core.app.model.results.empty.Empty]: - """Insider stats of the company.""" # noqa: E501 - - inputs = filter_inputs( - chart=chart, - ) - - return self._command_runner.run( - "/stocks/ins/stats", - **inputs, - ) diff --git a/openbb_sdk/sdk/core/README.md b/openbb_sdk/sdk/core/README.md index bdd6ca19706..e0d5d22f367 100644 --- a/openbb_sdk/sdk/core/README.md +++ b/openbb_sdk/sdk/core/README.md @@ -20,8 +20,7 @@ - [Coverage](#coverage) - [4.1.3. OpenBB Hub Account](#413-openbb-hub-account) - [4.1.4. Command execution](#414-command-execution) - - [4.1.5. Debug mode](#415-debug-mode) - - [4.1.6. Develop mode](#416-develop-mode) + - [4.1.5. Environment variables](#415-environment-variables) - [4.2 Dynamic version](#42-dynamic-version) - [5. REST API](#5-rest-api) - [5.1 Test users](#51-test-users) @@ -330,27 +329,33 @@ obb.stocks.load( ) ``` -### 4.1.5. Debug mode +### 4.1.5. Environment variables -To run the app in debug mode you can use the following code or create a .env file inside -`openbb_sdk/sdk/core/openbb_core/app`. -```python -import os -os.environ["OPENBB_DEBUG_MODE"] = "True" -from openbb import sdk -``` -### 4.1.6. Develop mode +The OS environment is only read once before the program starts, so make sure you change the variable before importing the SDK. We use the prefix "OPENBB_" to avoid polluting the environment (no pun intended). -To run the app in develop mode you can use the following code or create a .env file inside -`openbb_sdk/sdk/core/openbb_core/app`. +To apply an environment variable use one of the following: -```python -import os -os.environ["OPENBB_DEV_MODE"] = "True" -from openbb import sdk -``` +1. Temporary: use this snippet + + ```python + import os + os.environ["OPENBB_DEBUG_MODE"] = "True" + from openbb import sdk + ``` + +2. Persistent: create a `.env` file inside `openbb_sdk/sdk/core/openbb_core/app` with + + ```text + OPENBB_DEBUG_MODE="False" + ``` + +The variables we use are: + +- `OPENBB_DEBUG_MODE`: enable/disable debug mode +- `OPENBB_DEVELOP_MODE`: enable/disable develop mode +- `OPENBB_AUTO_BUILD`: enable/disable automatic build for python package ## 4.2 Dynamic version diff --git a/openbb_sdk/sdk/core/openbb_core/app/env.py b/openbb_sdk/sdk/core/openbb_core/app/env.py index eaad6a8b574..9a034002fcf 100644 --- a/openbb_sdk/sdk/core/openbb_core/app/env.py +++ b/openbb_sdk/sdk/core/openbb_core/app/env.py @@ -24,6 +24,10 @@ class Env(metaclass=SingletonMeta): # TODO: Change default to false when ready to deploy return self.str_to_bool(self._environ.get("OPENBB_DEV_MODE", True)) + @property + def AUTO_BUILD(self) -> bool: + return self.str_to_bool(self._environ.get("OPENBB_AUTO_BUILD", True)) + @staticmethod def str_to_bool(value) -> bool: """Match a string to a boolean value.""" diff --git a/openbb_sdk/sdk/core/openbb_core/app/static/package_builder.py b/openbb_sdk/sdk/core/openbb_core/app/static/package_builder.py index 342c9e7eeed..4ab678cc623 100644 --- a/openbb_sdk/sdk/core/openbb_core/app/static/package_builder.py +++ b/openbb_sdk/sdk/core/openbb_core/app/static/package_builder.py @@ -23,6 +23,7 @@ from typing import ( ) import pandas as pd +from importlib_metadata import entry_points from pydantic.fields import ModelField from starlette.routing import BaseRoute from typing_extensions import Annotated, _AnnotatedAlias @@ -32,28 +33,51 @@ from openbb_core.app.model.custom_parameter import OpenBBCustomParameter from openbb_core.app.provider_interface import ProviderInterface from openbb_core.app.router import RouterLoader -# ruff: noqa: T201 + +class Console: + """Console to be used by builder and linters.""" + + def __init__(self, verbose: bool): + self.verbose = verbose + + def log(self, message: str, **kwargs): + """Console log method""" + if self.verbose or Env().DEBUG_MODE: + print(message, **kwargs) # noqa: T201 class PackageBuilder: """Build the extension package for the SDK.""" - def __init__(self, directory: Optional[Path] = None) -> None: + def __init__( + self, directory: Optional[Path] = None, lint: bool = True, verbose: bool = False + ) -> None: self.directory = directory or Path(__file__).parent + self.lint = lint + self.verbose = verbose + self.console = Console(verbose) def build( self, modules: Optional[Union[str, List[str]]] = None, - lint: bool = True, ) -> None: """Build the extensions for the SDK.""" - print("\nBuilding extensions package...\n") + self.console.log("\nBuilding extensions package...\n") + self.save_extension_map() self.save_module_map() self.save_modules(modules) self.save_package() - if lint: - self.run_linters(self.directory) + if self.lint: + self.run_linters() + + def save_extension_map(self): + """Save the map of extensions available at build time""" + groups = ("openbb_core_extension", "openbb_provider_extension") + ext_map = {g: sorted(list(entry_points(group=g).names)) for g in groups} + code = dumps(obj=dict(sorted(ext_map.items())), indent=4) + self.console.log("Writing extension map...") + self.write_to_package(code=code, name="extension_map", extension="json") def save_module_map(self): """Save the module map.""" @@ -62,21 +86,18 @@ class PackageBuilder: module_map = { PathHandler.build_module_name(path=path): path for path in path_list } - module_code = dumps(obj=dict(sorted(module_map.items())), indent=4) - module_name = "module_map" - print("Writing module map...") - self.write_to_package( - module_code=module_code, module_name=module_name, extension="json" - ) + code = dumps(obj=dict(sorted(module_map.items())), indent=4) + self.console.log("\nWriting module map...") + self.write_to_package(code=code, name="module_map", extension="json") def save_modules(self, modules: Optional[Union[str, List[str]]] = None): """Save the modules.""" - print("\nWriting modules...") + self.console.log("\nWriting modules...") route_map = PathHandler.build_route_map() path_list = PathHandler.build_path_list(route_map=route_map) if not path_list: - print("\nThere is nothing to write.") + self.console.log("\nThere is nothing to write.") return MAX_LEN = max([len(path) for path in path_list if path != "/"]) @@ -89,33 +110,32 @@ class PackageBuilder: if route is None: module_code = ModuleBuilder.build(path=path) module_name = PathHandler.build_module_name(path=path) - print(f"({path})", end=" " * (MAX_LEN - len(path))) - self.write_to_package(module_code=module_code, module_name=module_name) + self.console.log(f"({path})", end=" " * (MAX_LEN - len(path))) + self.write_to_package(code=module_code, name=module_name) def save_package(self): """Save the package.""" - print("\nWriting package __init__...") + self.console.log("\nWriting package __init__...") code = "### THIS FILE IS AUTO-GENERATED. DO NOT EDIT. ###\n" - self.write_to_package(module_code=code, module_name="__init__") + self.write_to_package(code=code, name="__init__") - @staticmethod - def run_linters(directory: Path): + def run_linters(self): """Run the linters.""" - print("\nRunning linters...") - linters = Linters(directory) + self.console.log("\nRunning linters...") + linters = Linters(self.directory, self.verbose) linters.ruff() linters.black() - def write_to_package(self, module_code: str, module_name, extension="py") -> None: + def write_to_package(self, code: str, name: str, extension="py") -> None: """Write the module to the package.""" package_folder = self.directory / "package" - package_path = package_folder / f"{module_name}.{extension}" + package_path = package_folder / f"{name}.{extension}" package_folder.mkdir(exist_ok=True) - print(package_path) + self.console.log(str(package_path)) with package_path.open("w", encoding="utf-8", newline="\n") as file: - file.write(module_code.replace("typing.", "")) + file.write(code.replace("typing.", "")) class ModuleBuilder: @@ -768,15 +788,14 @@ class PathHandler: class Linters: """Run the linters for the SDK.""" - debug_mode = Env().DEBUG_MODE - - def __init__(self, directory: Path) -> None: + def __init__(self, directory: Path, verbose: bool = False) -> None: self.directory = directory + self.verbose = verbose + self.console = Console(verbose) - @staticmethod - def print_separator(symbol: str, length: int = 160): + def print_separator(self, symbol: str, length: int = 160): """Print a separator.""" - print(symbol * length) + self.console.log(symbol * length) def run( self, @@ -785,30 +804,28 @@ class Linters: ): """Run linter with flags.""" if shutil.which(linter): - print(f"\n* {linter}") - if self.debug_mode: - Linters.print_separator("^") + self.console.log(f"\n* {linter}") + self.print_separator("^") command = [linter, self.directory] if flags: command.extend(flags) subprocess.run(command, check=False) # noqa: S603 - if self.debug_mode: - Linters.print_separator("-") + self.print_separator("-") else: - print(f"\n* {linter} not found") + self.console.log(f"\n* {linter} not found") def black(self): """Run black.""" flags = [] - if not self.debug_mode: + if not self.verbose and not Env().DEBUG_MODE: flags.append("--quiet") self.run(linter="black", flags=flags) def ruff(self): """Run ruff.""" flags = ["--fix"] - if not self.debug_mode: + if not self.verbose and not Env().DEBUG_MODE: flags.append("--silent") self.run(linter="ruff", flags=flags) diff --git a/openbb_sdk/sdk/core/openbb_core/app/static/utils.py b/openbb_sdk/sdk/core/openbb_core/app/static/utils.py new file mode 100644 index 00000000000..329fc0f75e9 --- /dev/null +++ b/openbb_sdk/sdk/core/openbb_core/app/static/utils.py @@ -0,0 +1,107 @@ +from json import load +from pathlib import Path +from typing import List, Optional, Set, Tuple, Union + +from importlib_metadata import entry_points + +from openbb_core.app.env import Env + + +def get_ext_map(package: Path) -> dict: + """Get extension map from package folder""" + ext_map_file = Path(package, "extension_map.json") + try: + with open(ext_map_file) as fp: + ext_map = load(fp) + except Exception: + ext_map = {} + + return ext_map + + +def package_diff(package: Path) -> Tuple[Set[str], Set[str]]: + """Check differences between built and installed extensions. + + Parameters + ---------- + package: Path + The path to the package + + Returns + ------- + Tuple[Set[str], Set[str]] + First element: set of installed extensions that are not in the package. + Second element: set of extensions in the package that are not installed. + """ + + ext_map = get_ext_map(package) + + add: Set[str] = set() + remove: Set[str] = set() + groups = ("openbb_core_extension", "openbb_provider_extension") + for g in groups: + built = set(ext_map.get(g, {})) + installed = entry_points(group=g).names + add = add.union(installed - built) + remove = remove.union(built - installed) + + return add, remove + + +def build( + directory: Path, + modules: Optional[Union[str, List[str]]] = None, + lint: bool = True, + verbose: bool = False, +) -> None: + """Build extension modules in a separate process. + + Parameters + ---------- + directory: Path + The path of directory where package lives + modules : Optional[List[str]], optional + The modules to rebuild, by default None + For example: "/news" or ["/news", "/crypto"] + If None, all modules are rebuilt. + lint : bool, optional + Whether to lint the code, by default True + verbose : bool, optional + Enable/disable verbose mode + """ + # pylint: disable=import-outside-toplevel + from multiprocessing import Pool + + from openbb_core.app.static.package_builder import PackageBuilder + + # `build` is running in a separate process. This avoids consecutive calls to this + # function in the same interpreter to reuse objects already in memory. Not doing + # this was causing docstrings to have repeated sections, for example. + with Pool(processes=1) as pool: + pool.apply( + PackageBuilder(directory, lint, verbose).build, + args=(modules,), + ) + + +def auto_build(directory: Path): + """Trigger build if there are differences between built and installed extensions. + + Parameters + ---------- + directory: Path + The path of directory where package lives + """ + if Env().AUTO_BUILD: + add, remove = package_diff(Path(directory, "package")) + if add: + a = ", ".join(add) + print(f"Extensions to add: {a}") # noqa: T201 + + if remove: + r = ", ".join(remove) + print(f"Extensions to remove: {r}") # noqa: T201 + + if add or remove: + print("\nBuilding...") # noqa: T201 + build(directory) diff --git a/openbb_sdk/sdk/core/tests/app/static/test_package_builder.py b/openbb_sdk/sdk/core/tests/app/static/test_package_builder.py index 42df27a3019..fccc75c3700 100644 --- a/openbb_sdk/sdk/core/tests/app/static/test_package_builder.py +++ b/openbb_sdk/sdk/core/tests/app/static/test_package_builder.py @@ -63,7 +63,7 @@ def test_save_package(package_builder): def test_run_linters(package_builder): """Test run linters.""" - package_builder.run_linters(package_builder.directory) + package_builder.run_linters() @pytest.mark.skip("We avoid writing to the package.") diff --git a/openbb_sdk/sdk/core/tests/app/static/test_static_utils.py b/openbb_sdk/sdk/core/tests/app/static/test_static_utils.py new file mode 100644 index 00000000000..c4b8e9de4c1 --- /dev/null +++ b/openbb_sdk/sdk/core/tests/app/static/test_static_utils.py @@ -0,0 +1,122 @@ +"""Test the utils.py file.""" +# pylint: disable=redefined-outer-name + + +from pathlib import Path +from unittest.mock import mock_open, patch + +import pytest +from importlib_metadata import EntryPoint, EntryPoints +from openbb_core.app.env import Env +from openbb_core.app.static.utils import auto_build, get_ext_map, package_diff + +PATH = "openbb_core.app.static.utils." + + +@pytest.fixture(scope="function") +def tmp_dir(tmp_path_factory): + return tmp_path_factory.mktemp("tmp_dir") + + +def test_get_ext_map(tmp_dir): + """Test extensions map.""" + + open_mock = mock_open() + with patch(PATH + "open", open_mock), patch(PATH + "load") as mock_load: + get_ext_map(package=tmp_dir) + open_mock.assert_called_once_with(Path(tmp_dir, "extension_map.json")) + mock_load.assert_called_once() + + +@pytest.mark.parametrize( + "ext_built, ext_installed, expected_add, expected_remove", + [ + ( + { + "openbb_core_extension": [ + "ext_1", + "ext_2", + ], + "openbb_provider_extension": [ + "prov_1", + "prov_2", + ], + }, + EntryPoints( + ( + EntryPoint( + name="ext_2", value="...", group="openbb_core_extension" + ), + EntryPoint( + name="prov_2", value="...", group="openbb_provider_extension" + ), + ) + ), + set(), + {"ext_1", "prov_1"}, + ), + ( + { + "openbb_core_extension": ["ext_1"], + "openbb_provider_extension": ["prov_2"], + }, + EntryPoints( + ( + EntryPoint( + name="ext_2", value="...", group="openbb_core_extension" + ), + EntryPoint( + name="prov_1", value="...", group="openbb_provider_extension" + ), + ) + ), + {"ext_2", "prov_1"}, + {"ext_1", "prov_2"}, + ), + ], +) +def test_package_diff(tmp_dir, ext_built, ext_installed, expected_add, expected_remove): + """Test package differences.""" + + def mock_entry_points(group): + return ext_installed.select(**{"group": group}) + + with patch(PATH + "get_ext_map") as mock_get_ext_map, patch( + PATH + "entry_points", mock_entry_points + ): + mock_get_ext_map.return_value = ext_built + + add, remove = package_diff(tmp_dir) + + # We add whatever is not built, but is installed + assert add == expected_add + # We remove whatever is built, but is not installed + assert remove == expected_remove + + +@pytest.mark.parametrize( + "add, remove, openbb_auto_build", + [ + (set(), set(), True), + ({"this"}, set(), True), + (set(), {"that"}, True), + ({"this"}, {"that"}, True), + ({"this"}, {"that"}, False), + ], +) +def test_auto_build(tmp_dir, add, remove, openbb_auto_build): + """Test auto build.""" + + with patch(PATH + "package_diff") as mock_package_diff, patch( + PATH + "build" + ) as mock_build, patch.object(Env, "AUTO_BUILD", openbb_auto_build): + mock_package_diff.return_value = add, remove + auto_build(tmp_dir) + + if openbb_auto_build: + mock_package_diff.assert_called_once_with(Path(tmp_dir, "package")) + if add or remove: + mock_build.assert_called_once_with(tmp_dir) + else: + mock_package_diff.assert_not_called() + mock_build.assert_not_called()