jerry/OpenBB
1
0
Fork 0
mirror of https://github.com/OpenBB-finance/OpenBB.git synced 2026-11-06 09:22:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

[Feature] - Merge reference.json and extension_map.json, plus documentation generator fixes (#6268)

Browse Source 
* Merge reference.json and extension_map.json

* feat: check all packages are built and installed

* rebuild

* fix

* small fix

* Update package_builder.py

* move core into info

* move extensions into info

* fix test

---------

Co-authored-by: Igor Radovanovic <74266147+IgorWounds@users.noreply.github.com>
This commit is contained in:
montezdesousa
2024-04-04 10:56:37 +01:00
committed by GitHub
parent 57deb8eeab
commit daa4520fea
12 changed files with 26862 additions and 26895 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
openbb_platform/core/openbb_core/app/model/system_settings.py
View File

@@ -15,7 +15,7 @@ from openbb_core.app.constants import (
)
from openbb_core.app.model.abstract.tagged import Tagged
from openbb_core.app.model.fast_api_settings import FastAPISettings
from openbb_core.app.version import VERSION
from openbb_core.app.version import CORE_VERSION, VERSION
class SystemSettings(Tagged):
@@ -28,6 +28,7 @@ class SystemSettings(Tagged):
# OpenBB section
version: str = VERSION
core: str = CORE_VERSION
home_directory: str = str(HOME_DIRECTORY)
openbb_directory: str = str(OPENBB_DIRECTORY)
user_settings_path: str = str(USER_SETTINGS_PATH)

9
openbb_platform/core/openbb_core/app/static/app_factory.py
View File

@@ -1,6 +1,6 @@
"""App factory."""
from typing import Optional, Type, TypeVar
from typing import Dict, Optional, Type, TypeVar
from openbb_core.app.command_runner import CommandRunner
from openbb_core.app.model.system_settings import SystemSettings
@@ -8,6 +8,7 @@ from openbb_core.app.model.user_settings import UserSettings
from openbb_core.app.static.account import Account
from openbb_core.app.static.container import Container
from openbb_core.app.static.coverage import Coverage
from openbb_core.app.static.reference_loader import ReferenceLoader
from openbb_core.app.version import VERSION
E = TypeVar("E", bound=Type[Container])
@@ -29,6 +30,7 @@ class BaseApp:
self._command_runner = command_runner
self._account = Account(self)
self._coverage = Coverage(self)
self._reference = ReferenceLoader().reference
@property
def account(self) -> Account:
@@ -50,6 +52,11 @@ class BaseApp:
"""Coverage menu."""
return self._coverage
@property
def reference(self) -> Dict[str, Dict]:
"""Return reference data."""
return self._reference
def create_app(extensions: Optional[E] = None) -> Type[BaseApp]:
"""Create the app."""

7
openbb_platform/core/openbb_core/app/static/coverage.py
View File

@@ -5,7 +5,6 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional
from openbb_core.api.router.helpers.coverage_helpers import get_route_schema_map
from openbb_core.app.provider_interface import ProviderInterface
from openbb_core.app.router import CommandMap
from openbb_core.app.static.reference_loader import ReferenceLoader
if TYPE_CHECKING:
from openbb_core.app.static.app_factory import BaseApp
@@ -28,7 +27,6 @@ class Coverage:
self._app = app
self._command_map = CommandMap(coverage_sep=".")
self._provider_interface = ProviderInterface()
self._reference_loader = ReferenceLoader()
def __repr__(self) -> str:
"""Return docstring."""
@@ -52,11 +50,6 @@ class Coverage:
for command, value in self._command_map.commands_model.items()
}
@property
def reference(self) -> Dict[str, Dict]:
"""Return reference data."""
return self._reference_loader.reference
def command_schemas(self, filter_by_provider: Optional[str] = None):
"""Return route schema for a command."""
return get_route_schema_map(

55
openbb_platform/core/openbb_core/app/static/package_builder.py
View File

@@ -45,6 +45,7 @@ from openbb_core.app.provider_interface import ProviderInterface
from openbb_core.app.router import RouterLoader
from openbb_core.app.static.utils.console import Console
from openbb_core.app.static.utils.linters import Linters
from openbb_core.app.version import CORE_VERSION, VERSION
from openbb_core.env import Env
from openbb_core.provider.abstract.data import Data
@@ -90,9 +91,11 @@ class PackageBuilder:
def auto_build(self) -> None:
"""Trigger build if there are differences between built and installed extensions."""
if Env().AUTO_BUILD:
add, remove = PackageBuilder._diff(
self.directory / "assets" / "extension_map.json"
reference = PackageBuilder._read(
self.directory / "assets" / "reference.json"
)
ext_map = reference.get("info", {}).get("extensions", {})
add, remove = PackageBuilder._diff(ext_map)
if add:
a = ", ".join(sorted(add))
print(f"Extensions to add: {a}") # noqa: T201
@@ -113,10 +116,9 @@ class PackageBuilder:
self.console.log("\nBuilding extensions package...\n")
self._clean(modules)
ext_map = self._get_extension_map()
self._save_extension_map(ext_map)
self._save_modules(modules, ext_map)
self._save_package()
self._save_reference_file()
self._save_reference_file(ext_map)
if self.lint:
self._run_linters()
@@ -143,12 +145,6 @@ class PackageBuilder:
]
return ext_map
def _save_extension_map(self, ext_map: Dict[str, List[str]]) -> None:
"""Save the map of extensions available at build time."""
code = dumps(obj=dict(sorted(ext_map.items())), indent=4)
self.console.log("Writing extension map...")
self._write(code=code, name="extension_map", extension="json", folder="assets")
def _save_modules(
self,
modules: Optional[Union[str, List[str]]] = None,
@@ -185,11 +181,23 @@ class PackageBuilder:
code = "### THIS FILE IS AUTO-GENERATED. DO NOT EDIT. ###\n"
self._write(code=code, name="__init__")
def _save_reference_file(self):
def _save_reference_file(self, ext_map: Optional[Dict[str, List[str]]] = None):
"""Save the reference.json file."""
self.console.log("\nWriting reference file...")
data = ReferenceGenerator.get_reference_data()
code = dumps(obj=data, indent=4)
code = dumps(
obj={
"openbb": VERSION.replace("dev", ""),
"info": {
"title": "OpenBB Platform (Python)",
"description": "This is the OpenBB Platform (Python).",
"core": CORE_VERSION.replace("dev", ""),
"extensions": ext_map,
},
"paths": data,
},
indent=4,
)
self._write(code=code, name="reference", extension="json", folder="assets")
def _run_linters(self):
@@ -224,13 +232,28 @@ class PackageBuilder:
return content
@staticmethod
def _diff(path: Path) -> Tuple[Set[str], Set[str]]:
def _diff(ext_map: Dict[str, List[str]]) -> Tuple[Set[str], Set[str]]:
"""Check differences between built and installed extensions.
Parameters
----------
path: Path
The path to the folder where the extension map is stored.
ext_map: Dict[str, List[str]]
Dictionary containing the extensions.
Example:
{
"openbb_core_extension": [
"commodity@1.0.1",
...
],
"openbb_provider_extension": [
"benzinga@1.1.3",
...
],
"openbb_obbject_extension": [
"openbb_charting@1.0.0",
...
]
}
Returns
-------
@@ -238,8 +261,6 @@ class PackageBuilder:
First element: set of installed extensions that are not in the package.
Second element: set of extensions in the package that are not installed.
"""
ext_map = PackageBuilder._read(path)
add: Set[str] = set()
remove: Set[str] = set()
groups = OpenBBGroups.groups()

5
openbb_platform/core/openbb_core/app/version.py
View File

@@ -58,3 +58,8 @@ try:
VERSION = get_package_version(PACKAGE)
except pkg_resources.DistributionNotFound:
VERSION = "unknown"
try:
CORE_VERSION = get_package_version("openbb-core")
except pkg_resources.DistributionNotFound:
CORE_VERSION = "unknown"

7
openbb_platform/core/tests/app/static/test_app_factory.py
View File

@@ -47,3 +47,10 @@ def test_app_coverage(app_factory):
coverage = app_factory.coverage
assert coverage
assert isinstance(coverage, Coverage)
def test_app_reference(app_factory):
"""Test app reference."""
reference = app_factory.reference
assert reference
assert isinstance(reference, dict)

7
openbb_platform/core/tests/app/static/test_coverage.py
View File

@@ -37,10 +37,3 @@ def test_coverage_commands(coverage):
command_coverage = coverage.commands
assert command_coverage
assert isinstance(command_coverage, dict)
def test_coverage_reference(coverage):
"""Test coverage reference."""
reference = coverage.reference
assert reference
assert isinstance(reference, dict)

25
openbb_platform/core/tests/app/static/test_package_builder.py
View File

@@ -582,15 +582,15 @@ def test_generate(docstring_generator):
assert "Returns" in doc
def test_read_extension_map(package_builder, tmp_openbb_dir):
"""Test read extension map."""
def test__read(package_builder, tmp_openbb_dir):
"""Test read."""
PATH = "openbb_core.app.static.package_builder."
open_mock = mock_open()
with patch(PATH + "open", open_mock), patch(PATH + "load") as mock_load:
package_builder._read(Path(tmp_openbb_dir / "assets" / "extension_map.json"))
package_builder._read(Path(tmp_openbb_dir / "assets" / "reference.json"))
open_mock.assert_called_once_with(
Path(tmp_openbb_dir / "assets" / "extension_map.json")
Path(tmp_openbb_dir / "assets" / "reference.json")
)
mock_load.assert_called_once()
@@ -646,7 +646,6 @@ def test_read_extension_map(package_builder, tmp_openbb_dir):
)
def test_package_diff(
package_builder,
tmp_openbb_dir,
ext_built,
ext_installed,
ext_inst_version,
@@ -659,19 +658,16 @@ def test_package_diff(
return ext_installed.select(**{"group": group})
PATH = "openbb_core.app.static.package_builder."
with patch.object(PackageBuilder, "_read") as mock_read, patch(
PATH + "entry_points", mock_entry_points
), patch.object(EntryPoint, "dist", new_callable=PropertyMock) as mock_obj:
with patch(PATH + "entry_points", mock_entry_points), patch.object(
EntryPoint, "dist", new_callable=PropertyMock
) as mock_obj:
class MockPathDistribution:
version = ext_inst_version
mock_obj.return_value = MockPathDistribution()
mock_read.return_value = ext_built
add, remove = package_builder._diff(
Path(tmp_openbb_dir, "assets", "extension_map.json")
)
add, remove = package_builder._diff(ext_built)
# We add whatever is not built, but is installed
assert add == expected_add
@@ -689,7 +685,7 @@ def test_package_diff(
({"this"}, {"that"}, False),
],
)
def test_auto_build(package_builder, tmp_openbb_dir, add, remove, openbb_auto_build):
def test_auto_build(package_builder, add, remove, openbb_auto_build):
"""Test auto build."""
with patch.object(PackageBuilder, "_diff") as mock_assets_diff, patch.object(
@@ -700,9 +696,6 @@ def test_auto_build(package_builder, tmp_openbb_dir, add, remove, openbb_auto_bu
package_builder.auto_build()
if openbb_auto_build:
mock_assets_diff.assert_called_once_with(
Path(tmp_openbb_dir, "assets", "extension_map.json")
)
if add or remove:
mock_build.assert_called_once()
else:

29
openbb_platform/openbb/assets/extension_map.json vendored
View File

@@ -1,29 +0,0 @@
{
"openbb_core_extension": [
"commodity@1.0.2",
"crypto@1.1.4",
"currency@1.1.4",
"derivatives@1.1.4",
"economy@1.1.4",
"equity@1.1.4",
"etf@1.1.4",
"fixedincome@1.1.4",
"index@1.1.4",
"news@1.1.4",
"regulators@1.1.4"
],
"openbb_obbject_extension": [],
"openbb_provider_extension": [
"benzinga@1.1.4",
"federal_reserve@1.1.4",
"fmp@1.1.4",
"fred@1.1.4",
"intrinio@1.1.4",
"oecd@1.1.4",
"polygon@1.1.4",
"sec@1.1.4",
"tiingo@1.1.4",
"tradingeconomics@1.1.4",
"yfinance@1.1.4"
]
}

53197
openbb_platform/openbb/assets/reference.json vendored
View File

File diff suppressed because it is too large Load Diff

14
openbb_platform/tests/test_extension_map.py
View File

@@ -6,12 +6,10 @@ from poetry.core.constraints.version import Version, VersionConstraint, parse_co
from poetry.core.pyproject.toml import PyProjectTOML
def load_ext_map(file: Path) -> Dict[str, Version]:
"""Load the extension map from extension_map.json."""
def create_ext_map(extensions: dict) -> Dict[str, Version]:
"""Create the extension map from extension."""
ext_map = {}
with open(file) as f:
ext_map_json = json.load(f)
for _, v in ext_map_json.items():
for _, v in extensions.items():
for value in v:
name, version = value.split("@")
ext_map[name] = Version.parse(version)
@@ -36,9 +34,9 @@ def load_req_ext(file: Path) -> Dict[str, VersionConstraint]:
def test_extension_map():
"""Ensure only required extensions are built and versions respect pyproject.toml"""
this_dir = Path(__file__).parent
ext_map = load_ext_map(
Path(this_dir, "..", "openbb", "assets", "extension_map.json")
)
with open(Path(this_dir, "..", "openbb", "assets", "reference.json")) as f:
reference = json.load(f)
ext_map = create_ext_map(reference.get("info", {}).get("extensions", {}))
req_ext = load_req_ext(Path(this_dir, "..", "pyproject.toml"))
for ext in req_ext: