mirror of
https://github.com/OpenBB-finance/OpenBB.git
synced 2026-10-31 06:22:18 +00:00
[Release] openbb-v5.0.0 (#7741)
* merge branch develop with main * bump desktop version * fix the merge * sync Cargo.lock to desktop 2.0.0 * fix cli conftest.py --------- Co-authored-by: deeleeramone <>
This commit is contained in:
1
cli/openbb_cli/codegen/__init__.py
Normal file
1
cli/openbb_cli/codegen/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
"""Codegen pipeline that turns a ``.spec`` document into a full extension package."""
|
||||
98
cli/openbb_cli/codegen/credentials.py
Normal file
98
cli/openbb_cli/codegen/credentials.py
Normal file
@@ -0,0 +1,98 @@
|
||||
"""Credential detection — separate user-facing params from auth secrets."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Literal
|
||||
|
||||
CredentialLocation = Literal["query", "header", "cookie"]
|
||||
|
||||
_CREDENTIAL_NAMES: frozenset[str] = frozenset(
|
||||
{
|
||||
"api_key",
|
||||
"api_token",
|
||||
"apikey",
|
||||
"x_api_key",
|
||||
"x_apikey",
|
||||
"key",
|
||||
"subscription_key",
|
||||
"ocp_apim_subscription_key",
|
||||
"app_id",
|
||||
"app_key",
|
||||
"app_token",
|
||||
"appid",
|
||||
"client_id",
|
||||
"client_secret",
|
||||
"client_token",
|
||||
"token",
|
||||
"access_token",
|
||||
"auth_token",
|
||||
"x_auth_token",
|
||||
"bearer_token",
|
||||
"secret",
|
||||
"secret_key",
|
||||
"authorization",
|
||||
"consumer_key",
|
||||
"consumer_secret",
|
||||
"private_key",
|
||||
"session_token",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def normalize_credential_key(name: str) -> str:
|
||||
"""Canonicalize a parameter name for credential lookup."""
|
||||
return name.lower().replace("-", "_").strip("_")
|
||||
|
||||
|
||||
def is_credential_name(name: str) -> bool:
|
||||
"""Return ``True`` if ``name`` looks like a credential (not a request param)."""
|
||||
return normalize_credential_key(name) in _CREDENTIAL_NAMES
|
||||
|
||||
|
||||
def classify_parameter(param: dict[str, Any]) -> CredentialLocation | None:
|
||||
"""Return ``"query"`` / ``"header"`` / ``"cookie"`` for a credential param.
|
||||
|
||||
A parameter is a credential when its name is a known secret alias
|
||||
(``api_key``, ``token``, ...), or when it's a *required* ``header``/
|
||||
``cookie`` parameter regardless of name — that's the profile of a
|
||||
per-integration value (a tenant id, a session cookie) declared once on a
|
||||
path item for every operation under it, not something request-specific
|
||||
like ``If-Modified-Since``. Returns ``None`` for anything else.
|
||||
"""
|
||||
name = param.get("name") if isinstance(param, dict) else None
|
||||
if not isinstance(name, str) or not name:
|
||||
return None
|
||||
location = param.get("in", "query")
|
||||
if is_credential_name(name):
|
||||
return "header" if location == "header" else "query"
|
||||
if param.get("required") and location in ("header", "cookie"):
|
||||
return location
|
||||
return None
|
||||
|
||||
|
||||
def credentials_from_command(
|
||||
cmd_spec: dict[str, Any],
|
||||
) -> dict[str, dict[str, str]]:
|
||||
"""Walk a command spec entry; return ``{canonical_key: {name, in}}``."""
|
||||
out: dict[str, dict[str, str]] = {}
|
||||
for param in cmd_spec.get("parameters") or []:
|
||||
loc = classify_parameter(param)
|
||||
if loc is None:
|
||||
continue
|
||||
canonical = normalize_credential_key(param["name"])
|
||||
out[canonical] = {"name": param["name"], "in": loc}
|
||||
return out
|
||||
|
||||
|
||||
def credentials_from_spec(spec_doc: dict[str, Any]) -> dict[str, dict[str, str]]:
|
||||
"""Aggregate every credential entry across an entire spec doc."""
|
||||
out: dict[str, dict[str, str]] = {}
|
||||
for cmd in (spec_doc.get("commands") or {}).values():
|
||||
for canonical, entry in credentials_from_command(cmd).items():
|
||||
out.setdefault(canonical, entry)
|
||||
return out
|
||||
|
||||
|
||||
def filter_user_params(parameters: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
"""Strip credential params from a parameter list, leaving user-facing ones."""
|
||||
return [p for p in parameters or [] if classify_parameter(p) is None]
|
||||
1129
cli/openbb_cli/codegen/fetcher_gen.py
Normal file
1129
cli/openbb_cli/codegen/fetcher_gen.py
Normal file
File diff suppressed because it is too large
Load Diff
158
cli/openbb_cli/codegen/namespace_tree.py
Normal file
158
cli/openbb_cli/codegen/namespace_tree.py
Normal file
@@ -0,0 +1,158 @@
|
||||
"""Group spec commands into a hierarchical namespace tree."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
|
||||
@dataclass
|
||||
class NamespaceNode:
|
||||
"""One node in the per-spec namespace tree.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
name : str
|
||||
The leaf segment of this node's dotted path.
|
||||
full_path : str
|
||||
The full dotted path from the root.
|
||||
cmd_spec : dict, optional
|
||||
The spec command entry when this node is a leaf command.
|
||||
children : dict[str, NamespaceNode]
|
||||
Sub-nodes keyed by their ``name``.
|
||||
"""
|
||||
|
||||
name: str
|
||||
full_path: str
|
||||
cmd_spec: dict[str, Any] | None = None
|
||||
children: dict[str, NamespaceNode] = field(default_factory=dict)
|
||||
|
||||
@property
|
||||
def is_command(self) -> bool:
|
||||
"""Whether this node corresponds to an actual spec command."""
|
||||
return self.cmd_spec is not None
|
||||
|
||||
@property
|
||||
def is_namespace(self) -> bool:
|
||||
"""Whether this node has nested children (acts as a menu / sub-router)."""
|
||||
return bool(self.children)
|
||||
|
||||
|
||||
def build_namespace_tree(commands: dict[str, dict[str, Any]]) -> NamespaceNode:
|
||||
"""Build a hierarchical tree from the spec's flat command dict.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
commands : dict
|
||||
Mapping of dotted command path to spec entry.
|
||||
|
||||
Returns
|
||||
-------
|
||||
NamespaceNode
|
||||
Synthetic root node whose ``children`` are the top-level namespaces.
|
||||
"""
|
||||
root = NamespaceNode(name="", full_path="")
|
||||
for dotted, cmd_spec in commands.items():
|
||||
if not dotted:
|
||||
continue
|
||||
parts = dotted.split(".")
|
||||
node = root
|
||||
path_so_far: list[str] = []
|
||||
for segment in parts:
|
||||
path_so_far.append(segment)
|
||||
child = node.children.get(segment)
|
||||
if child is None:
|
||||
child = NamespaceNode(name=segment, full_path=".".join(path_so_far))
|
||||
node.children[segment] = child
|
||||
node = child
|
||||
node.cmd_spec = cmd_spec
|
||||
return root
|
||||
|
||||
|
||||
def filter_tree_by_provider(root: NamespaceNode, provider: str) -> NamespaceNode:
|
||||
"""Return a copy of ``root`` keeping only commands that include ``provider``.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
root : NamespaceNode
|
||||
The full namespace tree.
|
||||
provider : str
|
||||
Provider identifier to filter by.
|
||||
|
||||
Returns
|
||||
-------
|
||||
NamespaceNode
|
||||
New tree containing only the surviving commands.
|
||||
"""
|
||||
return _filter_node(root, lambda providers: provider in providers) or NamespaceNode(
|
||||
name="", full_path=""
|
||||
)
|
||||
|
||||
|
||||
def filter_tree_local_only(root: NamespaceNode) -> NamespaceNode:
|
||||
"""Return a copy of ``root`` keeping only commands with no provider list.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
root : NamespaceNode
|
||||
The full namespace tree.
|
||||
|
||||
Returns
|
||||
-------
|
||||
NamespaceNode
|
||||
New tree containing only commands whose ``providers`` list is empty.
|
||||
"""
|
||||
return _filter_node(root, lambda providers: not providers) or NamespaceNode(
|
||||
name="", full_path=""
|
||||
)
|
||||
|
||||
|
||||
def _filter_node(node: NamespaceNode, predicate) -> NamespaceNode | None:
|
||||
"""Recursive helper that keeps commands matching ``predicate(providers)``."""
|
||||
keep_self = False
|
||||
new_cmd: dict[str, Any] | None = None
|
||||
if node.cmd_spec is not None:
|
||||
providers_list = node.cmd_spec.get("providers") or []
|
||||
if predicate(providers_list):
|
||||
keep_self = True
|
||||
new_cmd = node.cmd_spec
|
||||
|
||||
new_children: dict[str, NamespaceNode] = {}
|
||||
for name, child in node.children.items():
|
||||
kept = _filter_node(child, predicate)
|
||||
if kept is not None:
|
||||
new_children[name] = kept
|
||||
|
||||
if not keep_self and not new_children:
|
||||
return None
|
||||
|
||||
return NamespaceNode(
|
||||
name=node.name,
|
||||
full_path=node.full_path,
|
||||
cmd_spec=new_cmd,
|
||||
children=new_children,
|
||||
)
|
||||
|
||||
|
||||
def iter_commands(root: NamespaceNode) -> list[tuple[str, dict[str, Any]]]:
|
||||
"""Walk the tree and return ``[(dotted_path, cmd_spec), ...]`` for every command."""
|
||||
out: list[tuple[str, dict[str, Any]]] = []
|
||||
_walk_commands(root, out)
|
||||
return out
|
||||
|
||||
|
||||
def _walk_commands(node: NamespaceNode, out: list[tuple[str, dict[str, Any]]]) -> None:
|
||||
"""Depth-first command collector."""
|
||||
if node.cmd_spec is not None:
|
||||
out.append((node.full_path, node.cmd_spec))
|
||||
for child in node.children.values():
|
||||
_walk_commands(child, out)
|
||||
|
||||
|
||||
def providers_from_tree(root: NamespaceNode) -> set[str]:
|
||||
"""Collect every provider mentioned across the tree's commands."""
|
||||
out: set[str] = set()
|
||||
for _, cmd_spec in iter_commands(root):
|
||||
for p in cmd_spec.get("providers") or []:
|
||||
out.add(str(p))
|
||||
return out
|
||||
605
cli/openbb_cli/codegen/package_gen.py
Normal file
605
cli/openbb_cli/codegen/package_gen.py
Normal file
@@ -0,0 +1,605 @@
|
||||
"""Top-level codegen orchestrator: spec doc to one installable project."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from openbb_cli.codegen.fetcher_gen import (
|
||||
FetcherCommandSpec,
|
||||
GeneratedFetcher,
|
||||
generate_fetcher_module,
|
||||
)
|
||||
from openbb_cli.codegen.namespace_tree import (
|
||||
build_namespace_tree,
|
||||
iter_commands,
|
||||
providers_from_tree,
|
||||
)
|
||||
from openbb_cli.codegen.post_gen import (
|
||||
GeneratedPostCommand,
|
||||
PostCommandSpec,
|
||||
generate_post_command_module,
|
||||
)
|
||||
from openbb_cli.codegen.project_gen import GeneratedProject, generate_pyproject
|
||||
from openbb_cli.codegen.provider_gen import GeneratedProvider, generate_provider_module
|
||||
from openbb_cli.codegen.router_gen import (
|
||||
GeneratedRouter,
|
||||
GeneratedRouters,
|
||||
generate_routers,
|
||||
)
|
||||
from openbb_cli.codegen.test_gen import GeneratedTestModule, generate_provider_tests
|
||||
|
||||
|
||||
@dataclass
|
||||
class GeneratedPackage:
|
||||
"""One installable project that bundles every provider in the spec.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
root : Path
|
||||
Project root directory.
|
||||
project : GeneratedProject
|
||||
``pyproject.toml`` source.
|
||||
providers : list of GeneratedProvider
|
||||
One provider registration module per declared spec provider.
|
||||
routers : GeneratedRouters
|
||||
Shared router modules.
|
||||
fetchers_by_provider : dict
|
||||
``{provider_name: [GeneratedFetcher, ...]}``.
|
||||
post_commands : list of GeneratedPostCommand
|
||||
Local-compute POST endpoints.
|
||||
top_level_routers : list of str
|
||||
Top-level namespace names registered as ``openbb_core_extension`` entry points.
|
||||
"""
|
||||
|
||||
root: Path
|
||||
project: GeneratedProject
|
||||
providers: list[GeneratedProvider]
|
||||
routers: GeneratedRouters
|
||||
fetchers_by_provider: dict[str, list[GeneratedFetcher]]
|
||||
post_commands: list[GeneratedPostCommand] = field(default_factory=list)
|
||||
top_level_routers: list[str] = field(default_factory=list)
|
||||
root_namespace: str = ""
|
||||
base_url: str = ""
|
||||
commands_by_provider: dict[str, list[str]] = field(default_factory=dict)
|
||||
test_modules: list[GeneratedTestModule] = field(default_factory=list)
|
||||
|
||||
def write(self) -> Path:
|
||||
"""Materialize the project on disk under ``self.root``.
|
||||
|
||||
Returns
|
||||
-------
|
||||
Path
|
||||
``self.root``.
|
||||
"""
|
||||
pkg = self.root / self.project.package_name
|
||||
routers_dir = pkg / "routers"
|
||||
providers_dir = pkg / "providers"
|
||||
routers_dir.mkdir(parents=True, exist_ok=True)
|
||||
providers_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
for path in (pkg, routers_dir, providers_dir):
|
||||
init = path / "__init__.py"
|
||||
if not init.exists():
|
||||
init.write_text(f'"""{path.name} subpackage."""\n')
|
||||
|
||||
(pkg / "utils.py").write_text(RUNTIME_UTILS_SOURCE)
|
||||
|
||||
for r in self.routers.routers:
|
||||
(routers_dir / f"{r.module_name}.py").write_text(r.source)
|
||||
|
||||
for prov in self.providers:
|
||||
prov_dir = providers_dir / prov.provider_name
|
||||
models_dir = prov_dir / "models"
|
||||
models_dir.mkdir(parents=True, exist_ok=True)
|
||||
for path in (prov_dir, models_dir):
|
||||
init = path / "__init__.py"
|
||||
if not init.exists():
|
||||
init.write_text(f'"""{path.name} subpackage."""\n')
|
||||
(prov_dir / "__init__.py").write_text(prov.source)
|
||||
for fetcher in self.fetchers_by_provider.get(prov.provider_name, []):
|
||||
(models_dir / f"{fetcher.module_name}.py").write_text(fetcher.source)
|
||||
|
||||
if self.post_commands:
|
||||
tools_models = providers_dir / "tools" / "models"
|
||||
tools_models.mkdir(parents=True, exist_ok=True)
|
||||
for path in (providers_dir / "tools", tools_models):
|
||||
init = path / "__init__.py"
|
||||
if not init.exists(): # pragma: no cover
|
||||
init.write_text(f'"""{path.name} subpackage."""\n')
|
||||
for post in self.post_commands:
|
||||
(tools_models / f"{post.module_name}.py").write_text(post.source)
|
||||
|
||||
if self.test_modules:
|
||||
tests_dir = self.root / "tests"
|
||||
tests_dir.mkdir(parents=True, exist_ok=True)
|
||||
init = tests_dir / "__init__.py"
|
||||
if not init.exists():
|
||||
init.write_text('"""Auto-generated tests."""\n')
|
||||
for tm in self.test_modules:
|
||||
(tests_dir / f"{tm.module_name}.py").write_text(tm.source)
|
||||
|
||||
(self.root / "pyproject.toml").write_text(self.project.source)
|
||||
(self.root / "README.md").write_text(self._readme())
|
||||
_apply_ruff(self.root)
|
||||
return self.root
|
||||
|
||||
def _readme(self) -> str:
|
||||
"""Render a README that describes the actual generated surface."""
|
||||
ns = self.root_namespace or self.project.package_name
|
||||
total_get = sum(len(v) for v in self.fetchers_by_provider.values())
|
||||
total_post = len(self.post_commands)
|
||||
source = self.base_url or "the source OpenAPI spec"
|
||||
|
||||
sections: list[str] = []
|
||||
sections.append(f"# {self.project.project_name}\n")
|
||||
sections.append(
|
||||
f"OpenBB Platform extension generated from {source}. "
|
||||
f"Adds {total_get} GET command{'s' if total_get != 1 else ''}"
|
||||
+ (
|
||||
f" and {total_post} POST command{'s' if total_post != 1 else ''}"
|
||||
if total_post
|
||||
else ""
|
||||
)
|
||||
+ f" under `obb.{ns}.*`.\n"
|
||||
)
|
||||
|
||||
sections.append("## Install\n")
|
||||
sections.append("```bash")
|
||||
sections.append("pip install -e .")
|
||||
sections.append("openbb-build")
|
||||
sections.append("```\n")
|
||||
|
||||
sections.append("## Quick start\n")
|
||||
first_cmd = self._first_command_example(ns)
|
||||
if first_cmd is not None:
|
||||
dotted, model = first_cmd
|
||||
sections.append("```python")
|
||||
sections.append("from openbb import obb")
|
||||
sections.append("")
|
||||
sections.append(f"# {model}")
|
||||
sections.append(f"result = obb.{ns}.{dotted}()")
|
||||
sections.append("print(result.to_df())")
|
||||
sections.append("```\n")
|
||||
|
||||
sections.append(self._providers_section(ns))
|
||||
sections.append(self._commands_section(ns))
|
||||
|
||||
creds = self._unique_credentials()
|
||||
if creds:
|
||||
sections.append(self._credentials_section(creds))
|
||||
|
||||
sections.append(self._notes_section(ns))
|
||||
|
||||
return "\n".join(sections).rstrip() + "\n"
|
||||
|
||||
def _first_command_example(self, ns: str) -> tuple[str, str] | None:
|
||||
"""Pick a representative command to seed the Quick start snippet."""
|
||||
for prov in sorted(self.commands_by_provider):
|
||||
cmds = sorted(self.commands_by_provider[prov])
|
||||
if not cmds:
|
||||
continue
|
||||
dotted = cmds[0]
|
||||
fetcher = next(
|
||||
(
|
||||
f
|
||||
for f in self.fetchers_by_provider.get(prov, [])
|
||||
if f"{ns}_{f.module_name}".endswith(dotted.replace(".", "_"))
|
||||
or f.module_name == dotted.replace(".", "_")
|
||||
),
|
||||
None,
|
||||
)
|
||||
model = fetcher.model_name if fetcher else "Returns OBBject"
|
||||
return dotted, model
|
||||
return None
|
||||
|
||||
def _providers_section(self, ns: str) -> str:
|
||||
lines = ["## Providers\n"]
|
||||
for p in self.providers:
|
||||
cmd_count = len(self.commands_by_provider.get(p.provider_name, []))
|
||||
label = "command" if cmd_count == 1 else "commands"
|
||||
cred_note = (
|
||||
f" — credentials: {', '.join(f'`{k}`' for k in p.credential_keys)}"
|
||||
if p.credential_keys
|
||||
else ""
|
||||
)
|
||||
lines.append(f"- **{p.provider_name}** ({cmd_count} {label}){cred_note}")
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
def _commands_section(self, ns: str) -> str:
|
||||
lines = ["## Commands\n"]
|
||||
lines.append(
|
||||
f"Every command lands at `obb.{ns}.<dotted.path>` with a typed "
|
||||
"signature, full Pydantic validation, and `OBBject` results.\n"
|
||||
)
|
||||
groups: dict[str, list[tuple[str, str]]] = {}
|
||||
for prov, dotted_paths in self.commands_by_provider.items():
|
||||
for dotted in sorted(dotted_paths):
|
||||
top = dotted.split(".", 1)[0]
|
||||
groups.setdefault(top, []).append((dotted, prov))
|
||||
for top in sorted(groups):
|
||||
lines.append(f"### `obb.{ns}.{top}`\n")
|
||||
for dotted, prov in groups[top]:
|
||||
lines.append(f"- `obb.{ns}.{dotted}` — provider: `{prov}`")
|
||||
lines.append("")
|
||||
return "\n".join(lines).rstrip() + "\n"
|
||||
|
||||
def _unique_credentials(self) -> list[str]:
|
||||
seen: set[str] = set()
|
||||
for p in self.providers:
|
||||
seen.update(p.credential_keys)
|
||||
return sorted(seen)
|
||||
|
||||
def _credentials_section(self, creds: list[str]) -> str:
|
||||
lines = ["## Credentials\n"]
|
||||
lines.append(
|
||||
"Set these in `~/.openbb_platform/user_settings.json` under the "
|
||||
"matching `<provider>_<key>` field, or pass via the `OBB_USER_SETTINGS` "
|
||||
"env var:\n"
|
||||
)
|
||||
for c in creds:
|
||||
lines.append(f"- `{c}`")
|
||||
lines.append("")
|
||||
return "\n".join(lines).rstrip() + "\n"
|
||||
|
||||
def _notes_section(self, ns: str) -> str:
|
||||
return (
|
||||
"## Notes\n\n"
|
||||
"- Routers mount under a single root namespace "
|
||||
f"(`obb.{ns}.*`) so the extension never shadows OpenBB's own "
|
||||
"first-party routers.\n"
|
||||
"- Each `aextract_data` strips single-key envelopes and unpacks "
|
||||
"single-element lists at the response boundary; sibling scalar "
|
||||
"fields surface as `AnnotatedResult` metadata in `OBBject.extra`.\n"
|
||||
"- Non-JSON responses (XML, CSV, plain text) come back as a "
|
||||
"single-row dict with `content` / `content_type` rather than "
|
||||
"raising on parse.\n"
|
||||
)
|
||||
|
||||
|
||||
def _slugify(text: str) -> str:
|
||||
"""Lower-case + collapse non-alphanumerics to single underscores."""
|
||||
cleaned = re.sub(r"[^0-9a-zA-Z]+", "_", text).strip("_").lower()
|
||||
return cleaned or "extension"
|
||||
|
||||
|
||||
def _apply_ruff(root: Path) -> None:
|
||||
"""Run ``ruff check --fix --unsafe-fixes`` and ``ruff format`` over ``root``."""
|
||||
ruff = _find_ruff()
|
||||
if ruff is None:
|
||||
return
|
||||
target = str(root)
|
||||
subprocess.run( # noqa: S603
|
||||
[ruff, "check", "--fix", "--unsafe-fixes", "--quiet", target],
|
||||
check=False,
|
||||
)
|
||||
subprocess.run( # noqa: S603
|
||||
[ruff, "format", "--quiet", target],
|
||||
check=False,
|
||||
)
|
||||
|
||||
|
||||
def _find_ruff() -> str | None:
|
||||
"""Locate the ``ruff`` binary; checks the running Python env first."""
|
||||
sibling = Path(sys.executable).parent / "ruff"
|
||||
if sibling.exists():
|
||||
return str(sibling)
|
||||
return shutil.which("ruff")
|
||||
|
||||
|
||||
def _build_root_router(
|
||||
*,
|
||||
package_name: str,
|
||||
root_namespace: str,
|
||||
sub_routers: list[GeneratedRouter],
|
||||
root_commands: list[str] | None = None,
|
||||
root_post_imports: list[tuple[str, str]] | None = None,
|
||||
root_has_stream: bool = False,
|
||||
) -> GeneratedRouter:
|
||||
"""Emit a top-level router that mounts every namespace under ``root_namespace``.
|
||||
|
||||
Top-level leaf commands (no namespace of their own) are rendered directly
|
||||
on this router; unused imports are cleaned up by the ruff pass.
|
||||
"""
|
||||
commands = root_commands or []
|
||||
post_imports = root_post_imports or []
|
||||
parts: list[str] = [
|
||||
f'"""Root router for {root_namespace} — generated from spec."""',
|
||||
"",
|
||||
]
|
||||
if commands:
|
||||
parts.append("from openbb_core.app.model.command_context import CommandContext")
|
||||
parts.append("from openbb_core.app.model.obbject import OBBject")
|
||||
if root_has_stream:
|
||||
parts.append("from openbb_core.app.model.stream import OBBStream")
|
||||
parts.append(
|
||||
"from openbb_core.app.provider_interface import "
|
||||
"ExtraParams, ProviderChoices, StandardParams"
|
||||
)
|
||||
parts.append("from openbb_core.app.query import Query")
|
||||
parts.append("from openbb_core.app.router import Router")
|
||||
if post_imports:
|
||||
parts.append("")
|
||||
for module_path, function_name in post_imports:
|
||||
parts.append(
|
||||
f"from {module_path} import {function_name} as _{function_name}"
|
||||
)
|
||||
if sub_routers:
|
||||
parts.append("")
|
||||
for r in sub_routers:
|
||||
parts.append(
|
||||
f"from {package_name}.routers.{r.module_name} import "
|
||||
f"router as _{r.module_name}_router"
|
||||
)
|
||||
parts.append("")
|
||||
parts.append("")
|
||||
parts.append('router = Router(prefix="")')
|
||||
if sub_routers:
|
||||
parts.append("")
|
||||
for r in sub_routers:
|
||||
parts.append(
|
||||
f"router.include_router(_{r.module_name}_router, "
|
||||
f'prefix="/{r.module_name}")'
|
||||
)
|
||||
if commands:
|
||||
parts.append("")
|
||||
parts.extend(commands)
|
||||
return GeneratedRouter(
|
||||
module_name=root_namespace,
|
||||
entry_point_name=root_namespace,
|
||||
source="\n".join(parts).rstrip() + "\n",
|
||||
)
|
||||
|
||||
|
||||
_SKIP_TOP_NAMESPACES: frozenset[str] = frozenset({"coverage"})
|
||||
|
||||
|
||||
def _runtime_utils_source() -> str:
|
||||
"""Read the canonical ``unpack_response`` source from ``_unpack.py``."""
|
||||
from pathlib import Path
|
||||
|
||||
text = (
|
||||
Path(__file__)
|
||||
.resolve()
|
||||
.parent.parent.joinpath("dispatchers", "_unpack.py")
|
||||
.read_text()
|
||||
)
|
||||
return text
|
||||
|
||||
|
||||
RUNTIME_UTILS_SOURCE = _runtime_utils_source()
|
||||
|
||||
|
||||
def generate_packages(
|
||||
spec_doc: dict[str, Any],
|
||||
*,
|
||||
output_root: Path,
|
||||
provider_name: str | None = None,
|
||||
project_name: str | None = None,
|
||||
package_name: str | None = None,
|
||||
router_name: str | None = None,
|
||||
description: str | None = None,
|
||||
website: str | None = None,
|
||||
version: str = "0.1.0",
|
||||
) -> GeneratedPackageSet:
|
||||
"""Build ONE ``GeneratedPackage`` bundling every provider in ``spec_doc``.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
spec_doc : dict
|
||||
Loaded ``.spec`` document.
|
||||
output_root : Path
|
||||
Directory under which the project is created.
|
||||
provider_name : str, optional
|
||||
Project / package slug.
|
||||
project_name, package_name, description, website : str, optional
|
||||
Project-level overrides.
|
||||
router_name : str, optional
|
||||
Overrides the generated top-level router's module/entry-point name.
|
||||
Defaults to ``provider_name``.
|
||||
version : str
|
||||
Initial version string for ``pyproject.toml``.
|
||||
|
||||
Returns
|
||||
-------
|
||||
GeneratedPackageSet
|
||||
Single-element wrapper around the project.
|
||||
"""
|
||||
base_url = (spec_doc.get("base_url") or "").rstrip("/")
|
||||
api_prefix = spec_doc.get("api_prefix") or ""
|
||||
filtered_commands = {
|
||||
name: cmd
|
||||
for name, cmd in (spec_doc.get("commands") or {}).items()
|
||||
if name.split(".", 1)[0] not in _SKIP_TOP_NAMESPACES
|
||||
}
|
||||
full_tree = build_namespace_tree(filtered_commands)
|
||||
declared_providers = providers_from_tree(full_tree)
|
||||
|
||||
project_slug = _slugify(provider_name or "codegen")
|
||||
project_name = project_name or f"openbb-{project_slug.replace('_', '-')}"
|
||||
package_name = package_name or f"openbb_{project_slug}"
|
||||
description = (
|
||||
description or f"Generated OpenBB Platform extension bundle ({project_slug})."
|
||||
)
|
||||
website = website or base_url
|
||||
|
||||
fetchers_by_provider: dict[str, list[GeneratedFetcher]] = {}
|
||||
post_commands: list[GeneratedPostCommand] = []
|
||||
fetchers_index: dict[str, GeneratedFetcher] = {}
|
||||
post_index: dict[str, GeneratedPostCommand] = {}
|
||||
|
||||
if not declared_providers:
|
||||
target_providers: list[str] = [_slugify(provider_name or "default")]
|
||||
else:
|
||||
target_providers = sorted(declared_providers)
|
||||
|
||||
commands_by_provider: dict[str, list[str]] = {}
|
||||
commands_by_dotted: dict[str, dict[str, Any]] = {}
|
||||
|
||||
for dotted, cmd_spec in iter_commands(full_tree):
|
||||
commands_by_dotted[dotted] = cmd_spec
|
||||
method = (cmd_spec.get("method") or "get").lower()
|
||||
body = cmd_spec.get("request_body_schema")
|
||||
has_body = isinstance(body, dict) and (
|
||||
body.get("properties") or body.get("type") == "array"
|
||||
)
|
||||
cmd_providers = cmd_spec.get("providers") or []
|
||||
|
||||
if method == "post" and has_body:
|
||||
post = generate_post_command_module(
|
||||
PostCommandSpec(
|
||||
name=dotted,
|
||||
cmd_spec=cmd_spec,
|
||||
base_url=base_url,
|
||||
api_prefix=api_prefix,
|
||||
provider_name="tools",
|
||||
)
|
||||
)
|
||||
post_commands.append(post)
|
||||
post_index[dotted] = post
|
||||
commands_by_provider.setdefault("tools", []).append(dotted)
|
||||
continue
|
||||
|
||||
owners = (
|
||||
[p for p in target_providers if p in cmd_providers]
|
||||
if cmd_providers
|
||||
else target_providers
|
||||
)
|
||||
if not owners: # pragma: no cover
|
||||
continue
|
||||
for prov in owners:
|
||||
fetcher = generate_fetcher_module(
|
||||
FetcherCommandSpec(
|
||||
name=dotted,
|
||||
cmd_spec=cmd_spec,
|
||||
base_url=base_url,
|
||||
api_prefix=api_prefix,
|
||||
provider_name=prov,
|
||||
)
|
||||
)
|
||||
fetchers_by_provider.setdefault(prov, []).append(fetcher)
|
||||
commands_by_provider.setdefault(prov, []).append(dotted)
|
||||
fetchers_index.setdefault(dotted, fetcher)
|
||||
|
||||
routers = generate_routers(
|
||||
full_tree,
|
||||
package_name=package_name,
|
||||
provider_name="",
|
||||
fetchers_by_command=fetchers_index,
|
||||
post_commands_by_command=post_index,
|
||||
)
|
||||
for r in routers.routers:
|
||||
r.source = r.source.replace(
|
||||
f"{package_name}.providers..models.",
|
||||
f"{package_name}.providers.tools.models.",
|
||||
)
|
||||
routers.root_post_imports = [
|
||||
(
|
||||
module_path.replace(
|
||||
f"{package_name}.providers..models.",
|
||||
f"{package_name}.providers.tools.models.",
|
||||
),
|
||||
function_name,
|
||||
)
|
||||
for module_path, function_name in routers.root_post_imports
|
||||
]
|
||||
|
||||
root_namespace = _slugify(router_name or provider_name or project_slug)
|
||||
sub_routers = [r for r in routers.routers if r.entry_point_name]
|
||||
for r in sub_routers:
|
||||
r.entry_point_name = None
|
||||
root_router = _build_root_router(
|
||||
package_name=package_name,
|
||||
root_namespace=root_namespace,
|
||||
sub_routers=sub_routers,
|
||||
root_commands=routers.root_commands,
|
||||
root_post_imports=routers.root_post_imports,
|
||||
root_has_stream=routers.root_has_stream,
|
||||
)
|
||||
routers.routers.append(root_router)
|
||||
|
||||
providers: list[GeneratedProvider] = []
|
||||
for prov in target_providers:
|
||||
providers.append(
|
||||
generate_provider_module(
|
||||
package_name=package_name,
|
||||
provider_name=prov,
|
||||
description=f"{prov} provider proxied through {base_url}.",
|
||||
website=website,
|
||||
fetchers=fetchers_by_provider.get(prov, []),
|
||||
)
|
||||
)
|
||||
if post_commands:
|
||||
providers.append(
|
||||
generate_provider_module(
|
||||
package_name=package_name,
|
||||
provider_name="tools",
|
||||
description=(
|
||||
"Local-compute commands (econometrics, quantitative, "
|
||||
f"technical) proxied through {base_url}."
|
||||
),
|
||||
website=website,
|
||||
fetchers=[],
|
||||
)
|
||||
)
|
||||
|
||||
project = generate_pyproject(
|
||||
project_name=project_name,
|
||||
package_name=package_name,
|
||||
providers=[p.provider_name for p in providers],
|
||||
routers=[r for r in routers.routers if r.entry_point_name],
|
||||
description=description,
|
||||
version=version,
|
||||
spec_provenance={
|
||||
"source_url": str(spec_doc.get("source_url") or ""),
|
||||
"api_version": str(spec_doc.get("api_version") or ""),
|
||||
"generator": str(spec_doc.get("generator") or ""),
|
||||
"generated_at": str(spec_doc.get("generated_at") or ""),
|
||||
"spec_version": str(spec_doc.get("version") or ""),
|
||||
"spec_sha256": str(spec_doc.get("content_sha256") or ""),
|
||||
},
|
||||
)
|
||||
|
||||
test_modules: list[GeneratedTestModule] = []
|
||||
for prov in target_providers:
|
||||
tm = generate_provider_tests(
|
||||
package_name=package_name,
|
||||
provider_name=prov,
|
||||
fetchers=fetchers_by_provider.get(prov, []),
|
||||
commands_by_dotted=commands_by_dotted,
|
||||
)
|
||||
if tm is not None:
|
||||
test_modules.append(tm)
|
||||
|
||||
package = GeneratedPackage(
|
||||
root=Path(output_root) / project_name,
|
||||
project=project,
|
||||
providers=providers,
|
||||
routers=routers,
|
||||
fetchers_by_provider=fetchers_by_provider,
|
||||
post_commands=post_commands,
|
||||
top_level_routers=[
|
||||
r.entry_point_name for r in routers.routers if r.entry_point_name
|
||||
],
|
||||
root_namespace=root_namespace,
|
||||
base_url=base_url,
|
||||
commands_by_provider=commands_by_provider,
|
||||
test_modules=test_modules,
|
||||
)
|
||||
return GeneratedPackageSet(packages=[package])
|
||||
|
||||
|
||||
@dataclass
|
||||
class GeneratedPackageSet:
|
||||
"""Wrapper around the generated package list (kept for caller symmetry)."""
|
||||
|
||||
packages: list[GeneratedPackage] = field(default_factory=list)
|
||||
|
||||
def write(self) -> list[Path]:
|
||||
"""Materialize every package; returns the list of written roots."""
|
||||
return [pkg.write() for pkg in self.packages]
|
||||
782
cli/openbb_cli/codegen/post_gen.py
Normal file
782
cli/openbb_cli/codegen/post_gen.py
Normal file
@@ -0,0 +1,782 @@
|
||||
"""Generate free-form router modules for POST endpoints with request bodies."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, cast
|
||||
|
||||
from openbb_cli.codegen.credentials import (
|
||||
credentials_from_command,
|
||||
filter_user_params,
|
||||
)
|
||||
from openbb_cli.codegen.pydantic_gen import (
|
||||
GeneratedClass,
|
||||
class_name_from,
|
||||
consolidate_imports,
|
||||
generate_class,
|
||||
safe_field_name,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class PostCommandSpec:
|
||||
"""Per-command inputs for a POST router emitter.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
name : str
|
||||
Dotted command path from the spec.
|
||||
cmd_spec : dict
|
||||
The command's spec entry.
|
||||
base_url : str
|
||||
Upstream API root, no trailing slash.
|
||||
api_prefix : str
|
||||
Path prefix shared across every command in the spec.
|
||||
provider_name : str
|
||||
Snake-case provider identifier.
|
||||
"""
|
||||
|
||||
name: str
|
||||
cmd_spec: dict[str, Any]
|
||||
base_url: str
|
||||
api_prefix: str
|
||||
provider_name: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class GeneratedPostCommand:
|
||||
"""Output of ``generate_post_command_module``.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
module_name : str
|
||||
Snake-case module filename (without ``.py``).
|
||||
function_name : str
|
||||
The router function name.
|
||||
body_class : str, optional
|
||||
Name of the emitted body element class, if a body is present.
|
||||
data_class : str
|
||||
Name of the emitted response-row class.
|
||||
source : str
|
||||
Full module source ready to write to disk.
|
||||
credentials_used : dict
|
||||
Same shape as ``GeneratedFetcher.credentials_used``.
|
||||
"""
|
||||
|
||||
module_name: str
|
||||
function_name: str
|
||||
data_class: str
|
||||
source: str
|
||||
body_class: str | None = None
|
||||
credentials_used: dict[str, dict[str, str]] = field(default_factory=dict)
|
||||
|
||||
|
||||
_PATH_TEMPLATE_RE = re.compile(r"\{([^}]+)\}")
|
||||
|
||||
|
||||
def _module_name_from_command(name: str) -> str:
|
||||
"""Convert a dotted command path to a snake_case module identifier."""
|
||||
safe = re.sub(r"[^0-9a-zA-Z]+", "_", name).strip("_")
|
||||
if not safe:
|
||||
safe = "command"
|
||||
if safe[0].isdigit():
|
||||
safe = f"_{safe}"
|
||||
return safe.lower()
|
||||
|
||||
|
||||
def _path_template_keys(url_path: str) -> list[str]:
|
||||
"""Extract ``{placeholder}`` names from a URL template, in declaration order."""
|
||||
return [m.group(1) for m in _PATH_TEMPLATE_RE.finditer(url_path)]
|
||||
|
||||
|
||||
def _resolve_url_path(api_prefix: str, url_path: str) -> str:
|
||||
"""Combine api_prefix with url_path; the prefix is dropped if already present."""
|
||||
prefix = ("/" + api_prefix.strip("/")) if api_prefix.strip("/") else ""
|
||||
path = url_path if url_path.startswith("/") else "/" + url_path
|
||||
if prefix and path.startswith(prefix + "/"):
|
||||
return path
|
||||
if prefix:
|
||||
return prefix + path if not path.startswith(prefix) else path
|
||||
return path
|
||||
|
||||
|
||||
def _python_type_from_param(param: dict[str, Any]) -> str:
|
||||
"""Render a Python type annotation for a normalized parameter entry.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
param : dict
|
||||
Spec parameter entry.
|
||||
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
Annotation text suitable for the generated function signature.
|
||||
"""
|
||||
base_type = {
|
||||
"string": "str",
|
||||
"integer": "int",
|
||||
"number": "float",
|
||||
"boolean": "bool",
|
||||
}.get(param.get("type", "string"), "str")
|
||||
if param.get("choices"):
|
||||
rendered = ", ".join(repr(c) for c in param["choices"])
|
||||
base_type = f"Literal[{rendered}]"
|
||||
if param.get("is_list"):
|
||||
return f"list[{base_type}]"
|
||||
return base_type
|
||||
|
||||
|
||||
def _array_item_class(
|
||||
body_schema: dict[str, Any],
|
||||
*,
|
||||
parent_class_name: str,
|
||||
) -> tuple[str | None, GeneratedClass | None, str | None]:
|
||||
"""Detect a body's array-of-objects field and emit its item class.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
body_schema : dict
|
||||
The request_body_schema.
|
||||
parent_class_name : str
|
||||
Camel-case base for the generated item class name.
|
||||
|
||||
Returns
|
||||
-------
|
||||
tuple
|
||||
``(field_name, item_class, type_annotation)``.
|
||||
"""
|
||||
if body_schema.get("type") == "array":
|
||||
items = body_schema.get("items")
|
||||
if isinstance(items, dict):
|
||||
item_class_name = class_name_from(parent_class_name, "BodyItem")
|
||||
item_class = generate_class(
|
||||
items,
|
||||
class_name=item_class_name,
|
||||
base_class="Data",
|
||||
docstring=f"Body item for {parent_class_name}.",
|
||||
)
|
||||
return ("data", item_class, f"list[{item_class_name}]")
|
||||
return (None, None, None)
|
||||
|
||||
properties = body_schema.get("properties") or {}
|
||||
for name, schema in properties.items():
|
||||
if not isinstance(schema, dict):
|
||||
continue
|
||||
if schema.get("type") != "array":
|
||||
continue
|
||||
items = schema.get("items")
|
||||
if not isinstance(items, dict):
|
||||
continue
|
||||
if items.get("type") != "object" and "properties" not in items:
|
||||
continue
|
||||
item_class_name = class_name_from(parent_class_name, "BodyItem")
|
||||
item_class = generate_class(
|
||||
items,
|
||||
class_name=item_class_name,
|
||||
base_class="Data",
|
||||
docstring=f"Body item for {parent_class_name}.",
|
||||
)
|
||||
return (name, item_class, f"list[{item_class_name}]")
|
||||
return (None, None, None)
|
||||
|
||||
|
||||
def _data_schema(cmd_spec: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Pull the per-row data schema out of a response_schema (same logic as fetcher_gen)."""
|
||||
response = cmd_spec.get("response_schema") or {}
|
||||
if not isinstance(response, dict):
|
||||
return {}
|
||||
properties = response.get("properties") or {}
|
||||
results = properties.get("results")
|
||||
if isinstance(results, dict):
|
||||
non_null = [
|
||||
v
|
||||
for v in results.get("anyOf") or [results]
|
||||
if isinstance(v, dict) and v.get("type") != "null"
|
||||
]
|
||||
if non_null:
|
||||
inner = non_null[0]
|
||||
if inner.get("type") == "array" and isinstance(inner.get("items"), dict):
|
||||
inner = inner["items"]
|
||||
if isinstance(inner.get("oneOf"), list):
|
||||
concrete = [c for c in inner["oneOf"] if isinstance(c, dict)]
|
||||
if concrete:
|
||||
inner = concrete[0]
|
||||
return inner
|
||||
return _unwrap_schema_envelopes(response)
|
||||
|
||||
|
||||
def _unwrap_schema_envelopes(schema: Any) -> dict[str, Any]:
|
||||
"""Strip schema envelopes that ``unpack_response`` strips at runtime."""
|
||||
if not isinstance(schema, dict):
|
||||
return {}
|
||||
typed: dict[str, Any] = schema
|
||||
if typed.get("type") == "array":
|
||||
return _unwrap_schema_envelopes(typed.get("items"))
|
||||
combinator_target = _first_non_null_combinator_variant(typed)
|
||||
if combinator_target is not None:
|
||||
return _unwrap_schema_envelopes(combinator_target)
|
||||
if _is_scalar_schema(typed):
|
||||
return _wrap_scalar_as_value(typed)
|
||||
props = typed.get("properties")
|
||||
if not isinstance(props, dict) or not props:
|
||||
return typed
|
||||
inner = _envelope_inner_schema(typed, props)
|
||||
if inner is not None:
|
||||
return _unwrap_schema_envelopes(inner)
|
||||
return typed
|
||||
|
||||
|
||||
def _first_non_null_combinator_variant(schema: dict[str, Any]) -> dict[str, Any] | None:
|
||||
"""Return the first non-null variant of a top-level ``oneOf`` / ``anyOf``."""
|
||||
for combinator in ("oneOf", "anyOf"):
|
||||
variants = schema.get(combinator)
|
||||
if not isinstance(variants, list):
|
||||
continue
|
||||
for variant in variants:
|
||||
if isinstance(variant, dict) and variant.get("type") != "null":
|
||||
return cast(dict[str, Any], variant)
|
||||
return None
|
||||
|
||||
|
||||
def _envelope_inner_schema(
|
||||
typed: dict[str, Any], # noqa: ARG001
|
||||
props: dict[str, Any],
|
||||
) -> dict[str, Any] | None:
|
||||
"""Resolve the inner row schema when ``typed`` is a recognized envelope."""
|
||||
if len(props) == 1:
|
||||
only_value = next(iter(props.values()))
|
||||
if isinstance(only_value, dict):
|
||||
only_dict = cast(dict[str, Any], only_value)
|
||||
if only_dict.get("type") == "array":
|
||||
items = only_dict.get("items")
|
||||
return cast(dict[str, Any], items) if isinstance(items, dict) else None
|
||||
if (
|
||||
only_dict.get("properties")
|
||||
or only_dict.get("oneOf")
|
||||
or only_dict.get("anyOf")
|
||||
):
|
||||
return only_dict
|
||||
array_keys = [
|
||||
k for k, v in props.items() if isinstance(v, dict) and v.get("type") == "array"
|
||||
]
|
||||
# Only resolve as an envelope if there is exactly one list of all dicts,
|
||||
# or if the list is the only element present.
|
||||
if len(array_keys) == 1:
|
||||
items = props[array_keys[0]].get("items")
|
||||
if isinstance(items, dict) and not _is_scalar_schema(items):
|
||||
return cast(dict[str, Any], items)
|
||||
return None
|
||||
|
||||
|
||||
def _is_scalar_schema(schema: dict[str, Any]) -> bool:
|
||||
"""Return True when the schema represents a primitive (string/number/etc.)."""
|
||||
return schema.get("type") in {
|
||||
"string",
|
||||
"integer",
|
||||
"number",
|
||||
"boolean",
|
||||
} and not schema.get("properties")
|
||||
|
||||
|
||||
def _wrap_scalar_as_value(schema: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Promote a scalar schema to ``{type: object, properties: {value: scalar}}``."""
|
||||
return {
|
||||
"type": "object",
|
||||
"properties": {"value": schema},
|
||||
"required": ["value"],
|
||||
}
|
||||
|
||||
|
||||
def _credential_lookup_lines(
|
||||
creds: dict[str, dict[str, str]], provider_name: str
|
||||
) -> list[str]:
|
||||
"""Render the credential extraction block for the POST router body.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
creds : dict
|
||||
Mapping from canonical credential names to ``{name, in}`` info.
|
||||
provider_name : str
|
||||
Snake-case provider name.
|
||||
|
||||
Returns
|
||||
-------
|
||||
list of str
|
||||
Indented lines that pull each credential into ``_cred_<canonical>`` locals.
|
||||
"""
|
||||
if not creds:
|
||||
return []
|
||||
lines: list[str] = [" _user_creds = cc.user_settings.credentials"]
|
||||
for canonical in sorted(creds):
|
||||
var = f"_cred_{canonical}"
|
||||
attr = f"{provider_name}_{canonical}"
|
||||
lines.append(f' {var} = (getattr(_user_creds, "{attr}", "") or "")')
|
||||
return lines
|
||||
|
||||
|
||||
def _signature_params(
|
||||
cmd_spec: dict[str, Any],
|
||||
array_field: str | None,
|
||||
array_annotation: str | None,
|
||||
has_credentials: bool,
|
||||
) -> list[tuple[str, str, str | None, bool, Any]]:
|
||||
"""Build the function parameter list (name, annotation, description, required, default).
|
||||
|
||||
Parameters
|
||||
----------
|
||||
cmd_spec : dict
|
||||
Spec command entry.
|
||||
array_field : str, optional
|
||||
Name of the body's array-of-objects field.
|
||||
array_annotation : str, optional
|
||||
The Python annotation for that field.
|
||||
has_credentials : bool
|
||||
Whether the command needs credentials.
|
||||
|
||||
Returns
|
||||
-------
|
||||
list of tuple
|
||||
Per-parameter tuples in declaration order.
|
||||
"""
|
||||
out: list[tuple[str, str, str | None, bool, Any]] = []
|
||||
if has_credentials:
|
||||
out.append(
|
||||
(
|
||||
"cc",
|
||||
"CommandContext",
|
||||
"Provides access to user-configured credentials.",
|
||||
True,
|
||||
None,
|
||||
)
|
||||
)
|
||||
if array_field and array_annotation:
|
||||
out.append((array_field, array_annotation, "Body data rows.", True, None))
|
||||
body_props = (cmd_spec.get("request_body_schema") or {}).get("properties") or {}
|
||||
body_required = set(
|
||||
(cmd_spec.get("request_body_schema") or {}).get("required") or []
|
||||
)
|
||||
# A name can be declared both as an operation parameter and as a request-body
|
||||
# property — Codat's push endpoints put ``accountId`` in the URL template and
|
||||
# in the body. Emitting both produces two parameters with the same name, which
|
||||
# is a SyntaxError. The operation parameter wins: it carries the path
|
||||
# placeholder and its own ``required``. The payload is unaffected, because
|
||||
# ``_render_body_block`` reads the body fields from ``request_body_schema``
|
||||
# rather than from this list.
|
||||
#
|
||||
# Two wire names can also collapse onto one identifier once sanitized
|
||||
# (``from`` and ``from_``, ``Organization-Id`` and ``Organization_Id``), and
|
||||
# ``operation_parameters`` keeps a name declared twice under different ``in``
|
||||
# locations. Collisions are therefore resolved on the emitted identifier, not
|
||||
# on the wire name, so a duplicate argument cannot be generated at all.
|
||||
operation_params = [
|
||||
p for p in filter_user_params(cmd_spec.get("parameters") or []) if p.get("name")
|
||||
]
|
||||
operation_param_names = {p["name"] for p in operation_params}
|
||||
operation_param_names |= {safe_field_name(p["name"])[0] for p in operation_params}
|
||||
emitted: set[str] = {entry[0] for entry in out}
|
||||
|
||||
for name, schema in body_props.items():
|
||||
if name == array_field:
|
||||
continue
|
||||
if not isinstance(schema, dict):
|
||||
continue
|
||||
safe = safe_field_name(name)[0]
|
||||
if safe in operation_param_names or safe in emitted:
|
||||
continue
|
||||
emitted.add(safe)
|
||||
ann = _python_type_from_param(
|
||||
{
|
||||
"type": schema.get("type"),
|
||||
"is_list": schema.get("type") == "array",
|
||||
"choices": schema.get("enum"),
|
||||
}
|
||||
)
|
||||
out.append(
|
||||
(
|
||||
# ``_render_body_block`` already writes ``{'from': from_}``, keying the
|
||||
# payload by the wire name and reading the safe identifier, so the
|
||||
# signature has to declare the safe one. Codat's Transfer body has a
|
||||
# property named ``from``, which is a keyword.
|
||||
safe,
|
||||
ann,
|
||||
schema.get("description") or schema.get("title"),
|
||||
name in body_required,
|
||||
schema.get("default"),
|
||||
)
|
||||
)
|
||||
for raw in operation_params:
|
||||
name = raw["name"]
|
||||
safe = safe_field_name(name)[0]
|
||||
if safe in emitted:
|
||||
continue
|
||||
emitted.add(safe)
|
||||
ann = _python_type_from_param(raw)
|
||||
out.append(
|
||||
(
|
||||
# Wire names are not always identifiers. Rebilly declares
|
||||
# ``Organization-Id`` and a ``REB-APIKEY`` security header, which
|
||||
# emitted ``REB-APIKEY: str = None``. The fetcher path already
|
||||
# sanitizes these; this one has to match it.
|
||||
safe,
|
||||
ann,
|
||||
raw.get("help"),
|
||||
bool(raw.get("required")),
|
||||
raw.get("default"),
|
||||
)
|
||||
)
|
||||
out.sort(key=lambda entry: (0 if entry[0] == "cc" else 1, 0 if entry[3] else 1))
|
||||
return out
|
||||
|
||||
|
||||
def _render_signature(
|
||||
func_name: str,
|
||||
params: list[tuple[str, str, str | None, bool, Any]],
|
||||
return_annotation: str,
|
||||
) -> str:
|
||||
"""Render ``async def func_name(...) -> return_annotation:`` lines.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
func_name : str
|
||||
Generated function identifier.
|
||||
params : list of tuple
|
||||
Output of ``_signature_params``.
|
||||
return_annotation : str
|
||||
The return type annotation text.
|
||||
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
Multi-line signature ready for the function body to follow.
|
||||
"""
|
||||
if not params:
|
||||
return f"async def {func_name}() -> {return_annotation}:"
|
||||
lines = [f"async def {func_name}("]
|
||||
for name, annotation, _, required, default in params:
|
||||
if required:
|
||||
lines.append(f" {name}: {annotation},")
|
||||
else:
|
||||
default_repr = render_default(default)
|
||||
lines.append(f" {name}: {annotation} = {default_repr},")
|
||||
lines.append(f") -> {return_annotation}:")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def render_default(default: Any) -> str:
|
||||
"""Render a Python literal for a default value.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
default : Any
|
||||
The literal default.
|
||||
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
The literal text usable on the right-hand side of ``= default``.
|
||||
"""
|
||||
if default is None:
|
||||
return "None"
|
||||
if isinstance(default, bool):
|
||||
return "True" if default else "False"
|
||||
return repr(default)
|
||||
|
||||
|
||||
def generate_post_command_module(spec: PostCommandSpec) -> GeneratedPostCommand:
|
||||
"""Render a POST command module for one spec command.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
spec : PostCommandSpec
|
||||
Per-command inputs (see ``PostCommandSpec``).
|
||||
|
||||
Returns
|
||||
-------
|
||||
GeneratedPostCommand
|
||||
The emitted module source plus metadata the package builder
|
||||
needs to register the command in the router.
|
||||
"""
|
||||
module_name = _module_name_from_command(spec.name)
|
||||
base_class_name = class_name_from(spec.name)
|
||||
function_name = module_name
|
||||
data_class = f"{base_class_name}Data"
|
||||
|
||||
body_field, body_item_class, body_annotation = _array_item_class(
|
||||
spec.cmd_spec.get("request_body_schema") or {},
|
||||
parent_class_name=base_class_name,
|
||||
)
|
||||
|
||||
data_schema = _data_schema(spec.cmd_spec) or {"type": "object"}
|
||||
data_generated = generate_class(
|
||||
data_schema,
|
||||
class_name=data_class,
|
||||
base_class="BaseModel",
|
||||
docstring=f"Response row for {spec.name}.",
|
||||
)
|
||||
|
||||
creds = credentials_from_command(spec.cmd_spec)
|
||||
cred_lines = _credential_lookup_lines(creds, spec.provider_name)
|
||||
|
||||
params = _signature_params(
|
||||
spec.cmd_spec, body_field, body_annotation, has_credentials=bool(creds)
|
||||
)
|
||||
return_annotation = f"OBBject[list[{data_class}]]"
|
||||
signature = _render_signature(function_name, params, return_annotation)
|
||||
|
||||
path_params = _path_template_keys(spec.cmd_spec.get("url_path") or "")
|
||||
full_url_path = _resolve_url_path(
|
||||
spec.api_prefix, spec.cmd_spec.get("url_path") or ""
|
||||
)
|
||||
body_block = _render_body_block(
|
||||
cmd_spec=spec.cmd_spec,
|
||||
body_field=body_field,
|
||||
creds=creds,
|
||||
path_params=path_params,
|
||||
url_path_template=full_url_path,
|
||||
base_url=spec.base_url.rstrip("/"),
|
||||
cred_lines=cred_lines,
|
||||
data_class=data_class,
|
||||
params=params,
|
||||
)
|
||||
|
||||
imports = {
|
||||
"import json as _json",
|
||||
"from typing import Any",
|
||||
"from openbb_core.app.model.abstract.error import OpenBBError",
|
||||
"from openbb_core.app.model.obbject import OBBject",
|
||||
"from openbb_core.provider.abstract.data import Data",
|
||||
"from openbb_core.provider.utils.helpers import "
|
||||
"get_async_requests_session, get_querystring",
|
||||
"from pydantic import BaseModel, Field",
|
||||
"from ....utils import unpack_response",
|
||||
}
|
||||
if creds:
|
||||
imports.add("from openbb_core.app.model.command_context import CommandContext")
|
||||
imports.update(data_generated.collect_imports())
|
||||
if body_item_class:
|
||||
imports.update(body_item_class.collect_imports())
|
||||
if any(p[1].startswith("Literal[") for p in params):
|
||||
imports.add("from typing import Literal")
|
||||
|
||||
sorted_imports = consolidate_imports(imports)
|
||||
|
||||
description = (spec.cmd_spec.get("description") or f"POST {spec.name}.").strip()
|
||||
summary = description.split("\n", 1)[0]
|
||||
parts: list[str] = [
|
||||
f'"""POST router function for {spec.name} — generated from spec.\n\n'
|
||||
f"Hits ``{spec.base_url.rstrip('/')}{full_url_path}`` via HTTP POST.\n"
|
||||
f'"""',
|
||||
"",
|
||||
]
|
||||
stdlib = [
|
||||
i
|
||||
for i in sorted_imports
|
||||
if i.startswith(
|
||||
("from typing", "from datetime", "import typing", "import datetime")
|
||||
)
|
||||
]
|
||||
third_party = [i for i in sorted_imports if i not in stdlib]
|
||||
if stdlib:
|
||||
parts.extend(stdlib)
|
||||
parts.append("")
|
||||
if third_party:
|
||||
parts.extend(third_party)
|
||||
parts.append("")
|
||||
parts.append("")
|
||||
|
||||
if body_item_class:
|
||||
for cls in body_item_class.flatten():
|
||||
parts.append(cls.source)
|
||||
for cls in data_generated.flatten():
|
||||
parts.append(cls.source)
|
||||
|
||||
cleaned_summary = (description or summary).strip()
|
||||
if not cleaned_summary:
|
||||
cleaned_summary = summary
|
||||
if not cleaned_summary.endswith((".", "!", "?")):
|
||||
cleaned_summary += "."
|
||||
if "\n" in cleaned_summary:
|
||||
first, _, rest = cleaned_summary.partition("\n")
|
||||
rest_indented = "\n".join(
|
||||
f" {line}" if line.strip() else "" for line in rest.split("\n")
|
||||
)
|
||||
func_doc = f' """{first}\n\n{rest_indented}\n """'
|
||||
else:
|
||||
func_doc = f' """{cleaned_summary}"""'
|
||||
parts.append(f"{signature}\n{func_doc}\n{body_block}\n")
|
||||
|
||||
return GeneratedPostCommand(
|
||||
module_name=module_name,
|
||||
function_name=function_name,
|
||||
body_class=body_item_class.name if body_item_class else None,
|
||||
data_class=data_class,
|
||||
source="\n".join(parts).rstrip() + "\n",
|
||||
credentials_used=creds,
|
||||
)
|
||||
|
||||
|
||||
def _render_body_block(
|
||||
*,
|
||||
cmd_spec: dict[str, Any],
|
||||
body_field: str | None,
|
||||
creds: dict[str, dict[str, str]],
|
||||
path_params: list[str],
|
||||
url_path_template: str,
|
||||
base_url: str,
|
||||
cred_lines: list[str],
|
||||
data_class: str,
|
||||
params: list[tuple[str, str, str | None, bool, Any]],
|
||||
) -> str:
|
||||
"""Render the function body for a POST command.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
cmd_spec : dict
|
||||
The spec command entry.
|
||||
body_field : str, optional
|
||||
Name of the array-of-objects body field, if any.
|
||||
creds : dict
|
||||
Credential entries from ``credentials_from_command``.
|
||||
path_params : list of str
|
||||
URL-template placeholders to substitute.
|
||||
url_path_template : str
|
||||
URL path with placeholders intact.
|
||||
base_url : str
|
||||
Upstream API root, no trailing slash.
|
||||
cred_lines : list of str
|
||||
Credential-extraction source lines.
|
||||
data_class : str
|
||||
Name of the response Data class.
|
||||
params : list of tuple
|
||||
Function parameters.
|
||||
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
Full function body.
|
||||
"""
|
||||
body_props = (cmd_spec.get("request_body_schema") or {}).get("properties") or {}
|
||||
body_field_names = list(body_props)
|
||||
# The signature declares the safe identifier for a body property, so the
|
||||
# body/query split has to compare against those, not the wire names.
|
||||
# Otherwise a keyword-named property such as ``from`` is not recognized as a
|
||||
# body field and leaks into the query string as ``from_``.
|
||||
body_safe_names = {safe_field_name(n)[0] for n in body_field_names}
|
||||
query_field_names = [
|
||||
n for n, _, _, _, _ in params if n != "cc" and n not in body_safe_names
|
||||
]
|
||||
# The signature carries safe identifiers, but the request has to go out under
|
||||
# the wire names, so keep a map back for the query keys and the path check.
|
||||
wire_by_safe = {
|
||||
safe_field_name(p["name"])[0]: p["name"]
|
||||
for p in filter_user_params(cmd_spec.get("parameters") or [])
|
||||
if p.get("name")
|
||||
}
|
||||
|
||||
lines: list[str] = []
|
||||
lines.extend(cred_lines)
|
||||
|
||||
if path_params:
|
||||
sub_args = ", ".join(f"{p}={p}" for p in path_params)
|
||||
lines.append(f" _path = {url_path_template!r}.format({sub_args})")
|
||||
else:
|
||||
lines.append(f" _path = {url_path_template!r}")
|
||||
|
||||
lines.append(" _query_dict: dict[str, Any] = {}")
|
||||
for name in query_field_names:
|
||||
wire = wire_by_safe.get(name, name)
|
||||
if wire in path_params or name in path_params:
|
||||
continue
|
||||
lines.append(f" if {name} is not None:")
|
||||
lines.append(f" _query_dict[{wire!r}] = {name}")
|
||||
for canonical, info in creds.items():
|
||||
if info["in"] != "query":
|
||||
continue
|
||||
wire_name = info["name"]
|
||||
var = f"_cred_{canonical}"
|
||||
lines.append(f" if {var}:")
|
||||
lines.append(f" _query_dict[{wire_name!r}] = {var}")
|
||||
lines.append(" _query_string = get_querystring(_query_dict, [])")
|
||||
lines.append(
|
||||
f' _url = f"{base_url}{{_path}}" + ("?" + _query_string if _query_string else "")'
|
||||
)
|
||||
|
||||
lines.append(" _headers: dict[str, str] = {}")
|
||||
for canonical, info in creds.items():
|
||||
if info["in"] != "header":
|
||||
continue
|
||||
wire_name = info["name"]
|
||||
var = f"_cred_{canonical}"
|
||||
lines.append(f" if {var}:")
|
||||
lines.append(f" _headers[{wire_name!r}] = {var}")
|
||||
|
||||
lines.append(" _cookies: dict[str, str] = {}")
|
||||
for canonical, info in creds.items():
|
||||
if info["in"] != "cookie":
|
||||
continue
|
||||
wire_name = info["name"]
|
||||
var = f"_cred_{canonical}"
|
||||
lines.append(f" if {var}:")
|
||||
lines.append(f" _cookies[{wire_name!r}] = {var}")
|
||||
|
||||
body_top_is_array = (cmd_spec.get("request_body_schema") or {}).get(
|
||||
"type"
|
||||
) == "array"
|
||||
if body_top_is_array and body_field:
|
||||
safe = safe_field_name(body_field)[0]
|
||||
lines.append(
|
||||
" _body: list[dict[str, Any]] = "
|
||||
f'[_item.model_dump(mode="json") for _item in {safe}]'
|
||||
)
|
||||
else:
|
||||
body_entries: list[str] = []
|
||||
for name in body_field_names:
|
||||
safe = safe_field_name(name)[0]
|
||||
if name == body_field:
|
||||
body_entries.append(
|
||||
f' {name!r}: [_item.model_dump(mode="json") for _item in {safe}],'
|
||||
)
|
||||
else:
|
||||
body_entries.append(f" {name!r}: {safe},")
|
||||
if body_entries:
|
||||
body_lines = "\n".join(body_entries)
|
||||
lines.append(f" _body: dict[str, Any] = {{\n{body_lines}\n }}")
|
||||
else:
|
||||
lines.append(" _body: dict[str, Any] = {}")
|
||||
|
||||
lines.append(" _rows: list[dict[str, Any]] = []")
|
||||
lines.append(" _metadata: dict[str, Any] = {}")
|
||||
lines.append(" async with await get_async_requests_session() as _session:")
|
||||
lines.append(
|
||||
" async with await _session.request("
|
||||
'"POST", _url, headers=_headers, cookies=_cookies, json=_body) as _resp:'
|
||||
)
|
||||
lines.append(" _ct = (_resp.headers.get('Content-Type') or '').lower()")
|
||||
lines.append(" _text = await _resp.text()")
|
||||
lines.append(" if _resp.status >= 400:")
|
||||
lines.append(
|
||||
" raise OpenBBError("
|
||||
"f'HTTP {_resp.status} from {_url}: {_text[:500]}')"
|
||||
)
|
||||
lines.append(
|
||||
" if 'json' in _ct or _text.lstrip().startswith(('{', '[')):"
|
||||
)
|
||||
lines.append(" try:")
|
||||
lines.append(" _payload = _json.loads(_text)")
|
||||
lines.append(" except ValueError as _exc:")
|
||||
lines.append(
|
||||
" raise OpenBBError("
|
||||
"f'Upstream {_url} returned malformed JSON: {_exc}') from _exc"
|
||||
)
|
||||
lines.append(" _rows, _metadata = unpack_response(_payload)")
|
||||
lines.append(" else:")
|
||||
lines.append(" _rows = [{'content': _text, 'content_type': _ct}]")
|
||||
lines.append(f" _typed = [{data_class}(**row) for row in _rows]")
|
||||
lines.append(" if _metadata:")
|
||||
lines.append(
|
||||
' return OBBject(results=_typed, extra={"results_metadata": _metadata})'
|
||||
)
|
||||
lines.append(" return OBBject(results=_typed)")
|
||||
return "\n".join(lines)
|
||||
130
cli/openbb_cli/codegen/project_gen.py
Normal file
130
cli/openbb_cli/codegen/project_gen.py
Normal file
@@ -0,0 +1,130 @@
|
||||
"""Generate ``pyproject.toml`` and the package skeleton files."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
@dataclass
|
||||
class GeneratedProject:
|
||||
"""Output of ``generate_pyproject``.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
project_name : str
|
||||
PyPI distribution name.
|
||||
package_name : str
|
||||
Snake-case Python package name.
|
||||
source : str
|
||||
Full ``pyproject.toml`` contents.
|
||||
"""
|
||||
|
||||
project_name: str
|
||||
package_name: str
|
||||
source: str
|
||||
|
||||
|
||||
def generate_pyproject(
|
||||
*,
|
||||
project_name: str,
|
||||
package_name: str,
|
||||
providers: list[str],
|
||||
routers: list,
|
||||
description: str,
|
||||
version: str = "0.1.0",
|
||||
spec_provenance: dict[str, str] | None = None,
|
||||
) -> GeneratedProject:
|
||||
"""Render a PEP 621 + Hatchling ``pyproject.toml`` for the bundled extension.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
project_name : str
|
||||
PyPI / distribution name.
|
||||
package_name : str
|
||||
Python package directory name (snake_case).
|
||||
providers : list of str
|
||||
Provider identifiers.
|
||||
routers : list of GeneratedRouter
|
||||
Top-level routers.
|
||||
description : str
|
||||
Free-form description for ``[project] description``.
|
||||
version : str
|
||||
Initial version string.
|
||||
|
||||
Returns
|
||||
-------
|
||||
GeneratedProject
|
||||
Project metadata plus the ``pyproject.toml`` source.
|
||||
"""
|
||||
provider_entries = "\n".join(
|
||||
f'{p} = "{package_name}.providers.{p}:{p}_provider"' for p in sorted(providers)
|
||||
)
|
||||
router_entries = "\n".join(
|
||||
f'{r.entry_point_name} = "{package_name}.routers.{r.module_name}:router"'
|
||||
for r in routers
|
||||
if r.entry_point_name
|
||||
)
|
||||
provenance_block = _render_provenance_block(spec_provenance or {})
|
||||
body = f"""[build-system]
|
||||
requires = ["hatchling"]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
|
||||
[project]
|
||||
name = "{project_name}"
|
||||
version = "{version}"
|
||||
description = {description!r}
|
||||
requires-python = ">=3.10,<4"
|
||||
license = {{ text = "Apache-2.0" }}
|
||||
dependencies = [
|
||||
"openbb-core>=2.0.0",
|
||||
]
|
||||
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
packages = ["{package_name}"]
|
||||
|
||||
|
||||
[project.entry-points."openbb_provider_extension"]
|
||||
{provider_entries}
|
||||
|
||||
[project.entry-points."openbb_core_extension"]
|
||||
{router_entries}
|
||||
{provenance_block}
|
||||
[tool.ruff]
|
||||
line-length = 100
|
||||
target-version = "py310"
|
||||
|
||||
[tool.ruff.lint]
|
||||
select = ["E", "F", "W", "I", "B", "UP", "D"]
|
||||
ignore = [
|
||||
"D100", "D101", "D102", "D103", "D104", "D105", "D107",
|
||||
"D200", "D205", "D212", "D401", "D415",
|
||||
"E501", "B008",
|
||||
]
|
||||
|
||||
[tool.ruff.lint.pydocstyle]
|
||||
convention = "numpy"
|
||||
"""
|
||||
return GeneratedProject(
|
||||
project_name=project_name,
|
||||
package_name=package_name,
|
||||
source=body,
|
||||
)
|
||||
|
||||
|
||||
def _render_provenance_block(provenance: dict[str, str]) -> str:
|
||||
"""Render ``[tool.openbb-codegen]`` (empty string when no provenance)."""
|
||||
keys = (
|
||||
"source_url",
|
||||
"api_version",
|
||||
"generator",
|
||||
"generated_at",
|
||||
"spec_version",
|
||||
"spec_sha256",
|
||||
)
|
||||
entries = [(k, provenance[k]) for k in keys if provenance.get(k)]
|
||||
if not entries:
|
||||
return ""
|
||||
body = "\n".join(f"{k} = {v!r}" for k, v in entries)
|
||||
return f"\n[tool.openbb-codegen]\n{body}\n"
|
||||
107
cli/openbb_cli/codegen/provider_gen.py
Normal file
107
cli/openbb_cli/codegen/provider_gen.py
Normal file
@@ -0,0 +1,107 @@
|
||||
"""Generate the ``providers/<provider_name>/__init__.py`` registration module."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from openbb_cli.codegen.fetcher_gen import GeneratedFetcher
|
||||
|
||||
|
||||
@dataclass
|
||||
class GeneratedProvider:
|
||||
"""Output of ``generate_provider_module``.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
package_name : str
|
||||
Snake-case package name.
|
||||
provider_name : str
|
||||
Snake-case provider identifier.
|
||||
source : str
|
||||
Full source for ``providers/<provider_name>/__init__.py``.
|
||||
credential_keys : list of str
|
||||
Canonical credential names declared on the provider.
|
||||
"""
|
||||
|
||||
package_name: str
|
||||
provider_name: str
|
||||
source: str
|
||||
credential_keys: list[str]
|
||||
|
||||
|
||||
def generate_provider_module(
|
||||
package_name: str,
|
||||
provider_name: str,
|
||||
description: str,
|
||||
website: str,
|
||||
fetchers: list[GeneratedFetcher],
|
||||
) -> GeneratedProvider:
|
||||
"""Render the provider's ``__init__.py``.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
package_name : str
|
||||
Snake-case top-level package name.
|
||||
provider_name : str
|
||||
Snake-case provider identifier.
|
||||
description : str
|
||||
Provider description shown in OpenBB introspection.
|
||||
website : str
|
||||
Provider home page URL.
|
||||
fetchers : list of GeneratedFetcher
|
||||
Per-command fetchers.
|
||||
|
||||
Returns
|
||||
-------
|
||||
GeneratedProvider
|
||||
Module source plus the credential-key list.
|
||||
"""
|
||||
cred_set: set[str] = set()
|
||||
for f in fetchers:
|
||||
cred_set.update(f.credentials_used)
|
||||
credential_keys = sorted(cred_set)
|
||||
|
||||
parts: list[str] = []
|
||||
parts.append(
|
||||
f'"""Provider registration for {provider_name} — generated from spec."""'
|
||||
)
|
||||
parts.append("")
|
||||
parts.append("from openbb_core.provider.abstract.provider import Provider")
|
||||
parts.append("")
|
||||
|
||||
for f in fetchers:
|
||||
parts.append(
|
||||
f"from {package_name}.providers.{provider_name}.models."
|
||||
f"{f.module_name} import {f.fetcher_class}"
|
||||
)
|
||||
parts.append("")
|
||||
parts.append("")
|
||||
|
||||
fetcher_dict_lines = [f' "{f.model_name}": {f.fetcher_class},' for f in fetchers]
|
||||
if credential_keys:
|
||||
cred_repr = (
|
||||
"[\n "
|
||||
+ ",\n ".join(repr(k) for k in credential_keys)
|
||||
+ ",\n ]"
|
||||
)
|
||||
cred_block = f" credentials={cred_repr},\n"
|
||||
else:
|
||||
cred_block = ""
|
||||
|
||||
parts.append(
|
||||
f"{provider_name}_provider = Provider(\n"
|
||||
f' name="{provider_name}",\n'
|
||||
f" description={description!r},\n"
|
||||
f"{cred_block}"
|
||||
f' website="{website}",\n'
|
||||
" fetcher_dict={\n" + "\n".join(fetcher_dict_lines) + "\n },\n)\n"
|
||||
)
|
||||
|
||||
return GeneratedProvider(
|
||||
package_name=package_name,
|
||||
provider_name=provider_name,
|
||||
source="\n".join(parts),
|
||||
credential_keys=credential_keys,
|
||||
)
|
||||
710
cli/openbb_cli/codegen/pydantic_gen.py
Normal file
710
cli/openbb_cli/codegen/pydantic_gen.py
Normal file
@@ -0,0 +1,710 @@
|
||||
"""Convert OpenAPI JSON-schema fragments into Pydantic class source code."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import keyword
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
_PRIMITIVE_TYPES: dict[str, str] = {
|
||||
"string": "str",
|
||||
"integer": "int",
|
||||
"number": "float",
|
||||
"boolean": "bool",
|
||||
}
|
||||
|
||||
_STRING_FORMATS: dict[str, str] = {
|
||||
"date": "datetime.date",
|
||||
"date-time": "datetime.datetime",
|
||||
}
|
||||
|
||||
_PYTHON_RESERVED_FIELD_NAMES = frozenset(
|
||||
{"model_config", "model_fields", "model_dump", "model_validate"}
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class GeneratedClass:
|
||||
"""One emitted Pydantic class plus any nested classes it depends on."""
|
||||
|
||||
name: str
|
||||
source: str
|
||||
nested: list[GeneratedClass] = field(default_factory=list)
|
||||
imports: set[str] = field(default_factory=set)
|
||||
|
||||
def flatten(self) -> list[GeneratedClass]:
|
||||
"""Return ``[nested..., self]`` in topological order, deduped by name."""
|
||||
seen: set[str] = set()
|
||||
out: list[GeneratedClass] = []
|
||||
for child in self.nested:
|
||||
for grandchild in child.flatten():
|
||||
if grandchild.name not in seen:
|
||||
seen.add(grandchild.name)
|
||||
out.append(grandchild)
|
||||
if self.name not in seen:
|
||||
out.append(self)
|
||||
return out
|
||||
|
||||
def collect_imports(self) -> set[str]:
|
||||
"""Union of ``imports`` across this class and every descendant."""
|
||||
out = set(self.imports)
|
||||
for child in self.nested:
|
||||
out.update(child.collect_imports())
|
||||
return out
|
||||
|
||||
|
||||
def safe_field_name(name: str) -> tuple[str, bool]:
|
||||
"""Normalize a JSON property name to a valid Python identifier.
|
||||
|
||||
Returns ``(safe_name, needs_alias)``.
|
||||
"""
|
||||
cleaned = re.sub(r"[^0-9a-zA-Z_]", "_", name).lstrip("_")
|
||||
if not cleaned:
|
||||
cleaned = "field"
|
||||
if cleaned[0].isdigit():
|
||||
cleaned = f"f_{cleaned}"
|
||||
if keyword.iskeyword(cleaned) or cleaned in _PYTHON_RESERVED_FIELD_NAMES:
|
||||
cleaned = f"{cleaned}_"
|
||||
return (cleaned, cleaned != name)
|
||||
|
||||
|
||||
def class_name_from(*parts: str) -> str:
|
||||
"""Join + camel-case a set of name parts into a Python class identifier."""
|
||||
words: list[str] = []
|
||||
for part in parts:
|
||||
if not part:
|
||||
continue
|
||||
for word in re.split(r"[^0-9a-zA-Z]+", part):
|
||||
if word:
|
||||
words.append(word[:1].upper() + word[1:])
|
||||
name = "".join(words) or "Anonymous"
|
||||
if name[0].isdigit():
|
||||
name = f"_{name}"
|
||||
return name
|
||||
|
||||
|
||||
def schema_to_type( # noqa: PLR0911
|
||||
schema: Any,
|
||||
*,
|
||||
parent_class_name: str,
|
||||
field_name: str,
|
||||
nested_classes: list[GeneratedClass],
|
||||
imports: set[str],
|
||||
) -> str:
|
||||
"""Resolve a JSON schema to a Python type annotation string."""
|
||||
if not isinstance(schema, dict):
|
||||
imports.add("from typing import Any")
|
||||
return "Any"
|
||||
|
||||
if "$ref" in schema:
|
||||
imports.add("from typing import Any")
|
||||
return "dict[str, Any]"
|
||||
|
||||
any_of = schema.get("anyOf") or schema.get("oneOf")
|
||||
if any_of:
|
||||
return _resolve_union(
|
||||
any_of,
|
||||
parent_class_name=parent_class_name,
|
||||
field_name=field_name,
|
||||
nested_classes=nested_classes,
|
||||
imports=imports,
|
||||
)
|
||||
|
||||
enum = schema.get("enum")
|
||||
if enum:
|
||||
return _enum_primitive(enum, schema.get("type"), imports)
|
||||
|
||||
if "const" in schema:
|
||||
return _enum_primitive([schema["const"]], schema.get("type"), imports)
|
||||
|
||||
schema_type = schema.get("type")
|
||||
if schema_type == "array":
|
||||
inner = schema_to_type(
|
||||
schema.get("items") or {},
|
||||
parent_class_name=parent_class_name,
|
||||
field_name=field_name + "_item",
|
||||
nested_classes=nested_classes,
|
||||
imports=imports,
|
||||
)
|
||||
return f"list[{inner}]"
|
||||
|
||||
if schema_type == "object" or "properties" in schema:
|
||||
return _resolve_object(
|
||||
schema,
|
||||
parent_class_name=parent_class_name,
|
||||
field_name=field_name,
|
||||
nested_classes=nested_classes,
|
||||
imports=imports,
|
||||
)
|
||||
|
||||
if schema_type in _PRIMITIVE_TYPES:
|
||||
py_type = _PRIMITIVE_TYPES[schema_type]
|
||||
if py_type == "str":
|
||||
fmt = schema.get("format")
|
||||
if fmt in _STRING_FORMATS:
|
||||
qualified = _STRING_FORMATS[fmt]
|
||||
imports.add("import datetime")
|
||||
return qualified
|
||||
return py_type
|
||||
|
||||
imports.add("from typing import Any")
|
||||
return "Any"
|
||||
|
||||
|
||||
def _enum_primitive(
|
||||
values: list[Any],
|
||||
declared_type: Any, # noqa: ARG001
|
||||
imports: set[str],
|
||||
) -> str:
|
||||
"""Map an ``enum`` / ``const`` to a ``Literal[...]`` annotation."""
|
||||
if not values:
|
||||
imports.add("from typing import Any")
|
||||
return "Any"
|
||||
rendered_members = ", ".join(repr(v) for v in values)
|
||||
imports.add("from typing import Literal")
|
||||
return f"Literal[{rendered_members}]"
|
||||
|
||||
|
||||
def _resolve_union(
|
||||
variants: list[Any],
|
||||
*,
|
||||
parent_class_name: str,
|
||||
field_name: str,
|
||||
nested_classes: list[GeneratedClass],
|
||||
imports: set[str],
|
||||
) -> str:
|
||||
"""Resolve an ``anyOf`` / ``oneOf`` list into a Python annotation."""
|
||||
non_null = [
|
||||
v for v in variants if not (isinstance(v, dict) and v.get("type") == "null")
|
||||
]
|
||||
is_optional = len(non_null) != len(variants)
|
||||
|
||||
if not non_null:
|
||||
imports.add("from typing import Any")
|
||||
return "Any"
|
||||
|
||||
rendered: list[str] = []
|
||||
for variant in non_null:
|
||||
rendered.append(
|
||||
schema_to_type(
|
||||
variant,
|
||||
parent_class_name=parent_class_name,
|
||||
field_name=field_name,
|
||||
nested_classes=nested_classes,
|
||||
imports=imports,
|
||||
)
|
||||
)
|
||||
seen: set[str] = set()
|
||||
unique: list[str] = []
|
||||
for r in rendered:
|
||||
if r not in seen:
|
||||
seen.add(r)
|
||||
unique.append(r)
|
||||
|
||||
inner = unique[0] if len(unique) == 1 else " | ".join(unique)
|
||||
|
||||
if is_optional:
|
||||
return f"{inner} | None"
|
||||
return inner
|
||||
|
||||
|
||||
def _resolve_object(
|
||||
schema: dict[str, Any],
|
||||
*,
|
||||
parent_class_name: str,
|
||||
field_name: str,
|
||||
nested_classes: list[GeneratedClass],
|
||||
imports: set[str],
|
||||
) -> str:
|
||||
"""Either generate a nested class or fall back to ``dict[str, Any]``."""
|
||||
properties = schema.get("properties") or {}
|
||||
if not properties:
|
||||
ap = schema.get("additionalProperties")
|
||||
if isinstance(ap, dict):
|
||||
value_type = schema_to_type(
|
||||
ap,
|
||||
parent_class_name=parent_class_name,
|
||||
field_name=field_name + "_value",
|
||||
nested_classes=nested_classes,
|
||||
imports=imports,
|
||||
)
|
||||
return f"dict[str, {value_type}]"
|
||||
imports.add("from typing import Any")
|
||||
return "dict[str, Any]"
|
||||
|
||||
nested_name = schema.get("title") or class_name_from(parent_class_name, field_name)
|
||||
nested_name = class_name_from(nested_name)
|
||||
nested = generate_class(
|
||||
schema,
|
||||
class_name=nested_name,
|
||||
docstring=schema.get("description"),
|
||||
)
|
||||
nested_classes.append(nested)
|
||||
return nested_name
|
||||
|
||||
|
||||
def render_field_default(default: Any) -> str:
|
||||
"""Render a JSON default value as a Python literal for ``Field(default=...)``."""
|
||||
if isinstance(default, bool):
|
||||
return "True" if default else "False"
|
||||
if isinstance(default, (int, float)):
|
||||
return repr(default)
|
||||
if isinstance(default, str):
|
||||
return repr(default)
|
||||
if default is None:
|
||||
return "None"
|
||||
if isinstance(default, (list, tuple, dict)):
|
||||
return repr(default)
|
||||
return repr(default)
|
||||
|
||||
|
||||
_LINE_LENGTH_BUDGET = 88
|
||||
|
||||
|
||||
def _build_field_call(
|
||||
*,
|
||||
description: str | None,
|
||||
default: Any,
|
||||
has_default: bool,
|
||||
required: bool,
|
||||
alias: str | None,
|
||||
leading_width: int = 0,
|
||||
) -> str:
|
||||
"""Compose the ``Field(...)`` text used after ``field_name: type =``.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
description : str, optional
|
||||
The field description.
|
||||
default : Any
|
||||
The literal default value, or ``None`` when ``has_default`` is False.
|
||||
has_default : bool
|
||||
Whether the schema specified an explicit default.
|
||||
required : bool
|
||||
Whether the field is required.
|
||||
alias : str, optional
|
||||
On-the-wire JSON name when it differs from the Python identifier.
|
||||
leading_width : int
|
||||
Number of characters consumed before the ``Field(...)`` call on the line.
|
||||
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
The full ``Field(...)`` source.
|
||||
"""
|
||||
args: list[str] = []
|
||||
if has_default and not required:
|
||||
args.append(f"default={render_field_default(default)}")
|
||||
elif not required:
|
||||
args.append("default=None")
|
||||
if alias is not None:
|
||||
args.append(f"alias={alias!r}")
|
||||
if description is not None:
|
||||
args.append(f"description={_python_string_literal(description)}")
|
||||
if not args:
|
||||
return ""
|
||||
single_line = "Field(" + ", ".join(args) + ")"
|
||||
if leading_width + len(single_line) <= _LINE_LENGTH_BUDGET:
|
||||
return single_line
|
||||
inner = ",\n ".join(args)
|
||||
return f"Field(\n {inner},\n )"
|
||||
|
||||
|
||||
def _resolve_field_description(
|
||||
prop: dict[str, Any],
|
||||
field_name: str | None = None, # noqa: ARG001
|
||||
) -> str:
|
||||
"""Pick the description text for a Pydantic Field."""
|
||||
text = prop.get("description") or prop.get("title")
|
||||
parts: list[str] = []
|
||||
if text:
|
||||
parts.append(text.rstrip(". ") + ".")
|
||||
elif prop.get("example") is not None:
|
||||
parts.append(f"Example: {prop['example']}.")
|
||||
enum = prop.get("enum") or (
|
||||
[prop["const"]]
|
||||
if isinstance(prop.get("const"), (str, int, float, bool))
|
||||
else None
|
||||
)
|
||||
if enum:
|
||||
rendered = ", ".join(str(v) for v in enum)
|
||||
parts.append(f"Choices: {rendered}.")
|
||||
return " ".join(parts).rstrip()
|
||||
|
||||
|
||||
def _nested_object_breakout(prop: dict[str, Any]) -> str:
|
||||
"""Render an inline breakout of a nested object's fields."""
|
||||
target = prop
|
||||
if isinstance(prop.get("items"), dict):
|
||||
target = prop["items"]
|
||||
for combinator in ("oneOf", "anyOf"):
|
||||
variants = target.get(combinator)
|
||||
if isinstance(variants, list):
|
||||
for variant in variants:
|
||||
if isinstance(variant, dict) and isinstance(
|
||||
variant.get("properties"), dict
|
||||
):
|
||||
target = variant
|
||||
break
|
||||
properties = target.get("properties")
|
||||
if not isinstance(properties, dict) or not properties:
|
||||
return ""
|
||||
summaries: list[str] = []
|
||||
for inner_name, inner_sch in properties.items():
|
||||
if not isinstance(inner_sch, dict):
|
||||
summaries.append(inner_name)
|
||||
continue
|
||||
inner_type = _quick_schema_type(inner_sch)
|
||||
summaries.append(f"{inner_name} ({inner_type})")
|
||||
label = (
|
||||
"Inner item fields" if isinstance(prop.get("items"), dict) else "Inner fields"
|
||||
)
|
||||
return f"{label}: {', '.join(summaries)}."
|
||||
|
||||
|
||||
def _quick_schema_type(schema: dict[str, Any]) -> str:
|
||||
"""Best-effort shorthand type label for breakout text."""
|
||||
if isinstance(schema.get("enum"), list):
|
||||
return "str"
|
||||
schema_type = schema.get("type")
|
||||
if schema_type == "array":
|
||||
items = schema.get("items")
|
||||
inner = "Any"
|
||||
if isinstance(items, dict):
|
||||
inner = _quick_schema_type(items)
|
||||
return f"list[{inner}]"
|
||||
if schema_type == "object" or "properties" in schema:
|
||||
title = schema.get("title")
|
||||
return title or "object"
|
||||
if isinstance(schema_type, str):
|
||||
primitive_type = _PRIMITIVE_TYPES.get(schema_type)
|
||||
if primitive_type is not None:
|
||||
if schema_type == "string":
|
||||
fmt = schema.get("format")
|
||||
if fmt == "date":
|
||||
return "date"
|
||||
if fmt == "date-time":
|
||||
return "datetime"
|
||||
return primitive_type
|
||||
return "Any"
|
||||
|
||||
|
||||
def _python_string_literal(text: str) -> str:
|
||||
"""Render a docstring-safe string literal that survives multi-line text.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
text : str
|
||||
The raw string to embed in generated source.
|
||||
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
A quoted Python literal.
|
||||
"""
|
||||
if "\n" in text or "'" in text and '"' in text:
|
||||
escaped = text.replace("\\", "\\\\").replace('"""', '\\"\\"\\"')
|
||||
return f'"""{escaped}"""'
|
||||
return repr(text)
|
||||
|
||||
|
||||
def _annotation_for_docstring(annotation: str) -> tuple[str, bool]:
|
||||
"""Strip ``| None`` from a type annotation for Numpy-style ``:`` lines.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
annotation : str
|
||||
The Python type annotation.
|
||||
|
||||
Returns
|
||||
-------
|
||||
tuple of (str, bool)
|
||||
The bare type without ``| None`` suffix, and an optional flag.
|
||||
"""
|
||||
suffix = " | None"
|
||||
if annotation.endswith(suffix):
|
||||
return (annotation[: -len(suffix)].strip(), True)
|
||||
return (annotation, False)
|
||||
|
||||
|
||||
def _render_numpy_parameters(
|
||||
fields: list[tuple[str, str, str | None, bool, Any, bool]],
|
||||
) -> str:
|
||||
"""Build a Numpy-style ``Parameters`` block for a class / function docstring.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
fields : list of tuple
|
||||
Each tuple is ``(name, annotation, description, has_default, default, optional)``.
|
||||
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
The full block.
|
||||
"""
|
||||
if not fields:
|
||||
return ""
|
||||
lines = ["Parameters", "----------"]
|
||||
for name, annotation, description, has_default, default, optional in fields:
|
||||
bare, _ = _annotation_for_docstring(annotation)
|
||||
type_label = f"{bare}, optional" if optional else bare
|
||||
lines.append(f"{name} : {type_label}")
|
||||
body = description or ""
|
||||
if has_default and default is not None:
|
||||
suffix = f"(default: {render_field_default(default)})"
|
||||
body = f"{body} {suffix}".strip() if body else suffix
|
||||
if body:
|
||||
for piece in body.split("\n"):
|
||||
lines.append(f" {piece}")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _render_class_docstring(
|
||||
summary: str,
|
||||
fields: list[tuple[str, str, str | None, bool, Any, bool]],
|
||||
) -> str:
|
||||
"""Render a Numpy-style class docstring and indent every line by four spaces.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
summary : str
|
||||
The summary sentence shown immediately after the opening quotes.
|
||||
fields : list of tuple
|
||||
Tuples consumed by ``_render_numpy_parameters``.
|
||||
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
The full triple-double-quoted block.
|
||||
"""
|
||||
summary_clean = summary.strip()
|
||||
if not summary_clean.endswith("."):
|
||||
summary_clean += "."
|
||||
lines = [f' """{summary_clean}']
|
||||
params_block = _render_numpy_parameters(fields)
|
||||
if params_block:
|
||||
lines.append("")
|
||||
for line in params_block.split("\n"):
|
||||
lines.append(f" {line}" if line else "")
|
||||
lines.append(' """')
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def render_function_docstring(
|
||||
summary: str,
|
||||
*,
|
||||
parameters: list[tuple[str, str, str | None]] | None = None,
|
||||
returns: tuple[str, str | None] | None = None,
|
||||
indent: str = " ",
|
||||
) -> str:
|
||||
"""Render a Numpy-style function / method docstring.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
summary : str
|
||||
The one-line summary sentence.
|
||||
parameters : list of (name, type, description) tuples, optional
|
||||
Each tuple becomes one entry under the ``Parameters`` section.
|
||||
returns : tuple of (type, description), optional
|
||||
When supplied, emits a ``Returns`` section.
|
||||
indent : str
|
||||
The indentation prefix for every line of the rendered block.
|
||||
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
The full triple-double-quoted block.
|
||||
"""
|
||||
summary_clean = summary.strip()
|
||||
if not summary_clean.endswith("."):
|
||||
summary_clean += "."
|
||||
lines = [f'{indent}"""{summary_clean}']
|
||||
if parameters:
|
||||
lines.append("")
|
||||
lines.append(f"{indent}Parameters")
|
||||
lines.append(f"{indent}----------")
|
||||
for name, type_label, description in parameters:
|
||||
lines.append(f"{indent}{name} : {type_label}")
|
||||
if description:
|
||||
for piece in description.split("\n"):
|
||||
lines.append(f"{indent} {piece}")
|
||||
if returns is not None:
|
||||
return_type, return_desc = returns
|
||||
lines.append("")
|
||||
lines.append(f"{indent}Returns")
|
||||
lines.append(f"{indent}-------")
|
||||
lines.append(f"{indent}{return_type}")
|
||||
if return_desc:
|
||||
for piece in return_desc.split("\n"):
|
||||
lines.append(f"{indent} {piece}")
|
||||
lines.append(f'{indent}"""')
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def generate_class(
|
||||
schema: dict[str, Any],
|
||||
*,
|
||||
class_name: str,
|
||||
base_class: str = "BaseModel",
|
||||
docstring: str | None = None,
|
||||
) -> GeneratedClass:
|
||||
"""Build a Pydantic class definition from an OpenAPI ``object`` schema."""
|
||||
properties: dict[str, Any] = schema.get("properties") or {}
|
||||
required_set: set[str] = set(schema.get("required") or [])
|
||||
nested_classes: list[GeneratedClass] = []
|
||||
imports: set[str] = {"from pydantic import BaseModel, Field"}
|
||||
body_lines: list[str] = []
|
||||
doc_fields: list[tuple[str, str, str | None, bool, Any, bool]] = []
|
||||
optional_object_fields: list[str] = []
|
||||
|
||||
for raw_name, prop_schema in properties.items():
|
||||
safe_name, needs_alias = safe_field_name(raw_name)
|
||||
annotation = schema_to_type(
|
||||
prop_schema if isinstance(prop_schema, dict) else {},
|
||||
parent_class_name=class_name,
|
||||
field_name=safe_name,
|
||||
nested_classes=nested_classes,
|
||||
imports=imports,
|
||||
)
|
||||
is_required = raw_name in required_set
|
||||
prop = prop_schema if isinstance(prop_schema, dict) else {}
|
||||
if not is_required and (prop.get("type") == "object" or "properties" in prop):
|
||||
optional_object_fields.append(safe_name)
|
||||
has_default = "default" in prop
|
||||
if not is_required and "None" not in annotation:
|
||||
annotation = f"{annotation} | None"
|
||||
description = _resolve_field_description(prop, field_name=raw_name)
|
||||
breakout = _nested_object_breakout(prop)
|
||||
if breakout:
|
||||
description = (
|
||||
f"{description} {breakout}".strip() if description else breakout
|
||||
)
|
||||
prefix = f" {safe_name}: {annotation} = "
|
||||
field_call = _build_field_call(
|
||||
description=description,
|
||||
default=prop.get("default"),
|
||||
has_default=has_default,
|
||||
required=is_required,
|
||||
alias=raw_name if needs_alias else None,
|
||||
leading_width=len(prefix),
|
||||
)
|
||||
line = f" {safe_name}: {annotation}"
|
||||
if field_call:
|
||||
line += f" = {field_call}"
|
||||
body_lines.append(line)
|
||||
|
||||
doc_fields.append(
|
||||
(
|
||||
safe_name,
|
||||
annotation,
|
||||
description,
|
||||
has_default,
|
||||
prop.get("default"),
|
||||
not is_required,
|
||||
)
|
||||
)
|
||||
|
||||
if not body_lines:
|
||||
body_lines.append(" pass")
|
||||
|
||||
if optional_object_fields:
|
||||
# PHP-style serializers emit [] where the schema declares an object
|
||||
# (empty associative array quirk); [] is never a valid JSON-object value.
|
||||
imports.add("from pydantic import field_validator")
|
||||
imports.add("from typing import Any")
|
||||
names = ", ".join(repr(n) for n in optional_object_fields)
|
||||
body_lines.append(
|
||||
f'\n @field_validator({names}, mode="before")\n'
|
||||
" @classmethod\n"
|
||||
" def _empty_list_to_none(cls, v: Any) -> Any:\n"
|
||||
' """Coerce `[]` to None."""\n'
|
||||
" return None if v == [] else v"
|
||||
)
|
||||
|
||||
summary = docstring or class_name
|
||||
doc_block = _render_class_docstring(summary, doc_fields)
|
||||
|
||||
source = (
|
||||
f"class {class_name}({base_class}):\n"
|
||||
f"{doc_block}\n\n" + "\n".join(body_lines) + "\n"
|
||||
)
|
||||
|
||||
return GeneratedClass(
|
||||
name=class_name,
|
||||
source=source,
|
||||
nested=nested_classes,
|
||||
imports=imports,
|
||||
)
|
||||
|
||||
|
||||
def consolidate_imports(imports: set[str]) -> list[str]:
|
||||
"""Merge multiple ``from X import A`` / ``from X import B`` into one line.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
imports : set of str
|
||||
Raw import statements.
|
||||
|
||||
Returns
|
||||
-------
|
||||
list of str
|
||||
Sorted list with same-module ``from`` imports collapsed.
|
||||
"""
|
||||
from_imports: dict[str, set[str]] = {}
|
||||
bare_imports: set[str] = set()
|
||||
for raw_line in imports:
|
||||
line = raw_line.strip()
|
||||
if line.startswith("from ") and " import " in line:
|
||||
module, _, names = line[len("from ") :].partition(" import ")
|
||||
for raw_name in names.split(","):
|
||||
name = raw_name.strip()
|
||||
if name:
|
||||
from_imports.setdefault(module.strip(), set()).add(name)
|
||||
else:
|
||||
bare_imports.add(line)
|
||||
out: list[str] = sorted(bare_imports)
|
||||
for module in sorted(from_imports):
|
||||
names = ", ".join(sorted(from_imports[module]))
|
||||
out.append(f"from {module} import {names}")
|
||||
return out
|
||||
|
||||
|
||||
def render_module(
|
||||
cls: GeneratedClass,
|
||||
*,
|
||||
module_docstring: str | None = None,
|
||||
) -> str:
|
||||
"""Render a ``cls`` and its nested classes as a single importable module."""
|
||||
classes = cls.flatten()
|
||||
imports = consolidate_imports(cls.collect_imports())
|
||||
|
||||
stdlib: list[str] = []
|
||||
third_party: list[str] = []
|
||||
for imp in imports:
|
||||
if (
|
||||
imp.startswith("from typing")
|
||||
or imp.startswith("from datetime")
|
||||
or imp.startswith("import typing")
|
||||
or imp.startswith("import datetime")
|
||||
):
|
||||
stdlib.append(imp)
|
||||
else:
|
||||
third_party.append(imp)
|
||||
|
||||
parts: list[str] = []
|
||||
if module_docstring:
|
||||
summary = module_docstring.strip()
|
||||
if not summary.endswith("."):
|
||||
summary += "."
|
||||
parts.append(f'"""{summary}"""')
|
||||
parts.append("")
|
||||
if stdlib:
|
||||
parts.extend(stdlib)
|
||||
parts.append("")
|
||||
if third_party:
|
||||
parts.extend(third_party)
|
||||
parts.append("")
|
||||
parts.append("")
|
||||
parts.extend(c.source for c in classes)
|
||||
return "\n".join(parts).rstrip() + "\n"
|
||||
401
cli/openbb_cli/codegen/router_gen.py
Normal file
401
cli/openbb_cli/codegen/router_gen.py
Normal file
@@ -0,0 +1,401 @@
|
||||
"""Generate router modules that mirror the spec's hierarchical namespace."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from openbb_cli.codegen.namespace_tree import NamespaceNode
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from openbb_cli.codegen.fetcher_gen import GeneratedFetcher
|
||||
from openbb_cli.codegen.post_gen import GeneratedPostCommand
|
||||
|
||||
|
||||
@dataclass
|
||||
class GeneratedRouter:
|
||||
"""One router module emitted by ``generate_routers``.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
module_name : str
|
||||
Snake-case module filename without ``.py``.
|
||||
entry_point_name : str | None
|
||||
When set, this router is registered as an ``openbb_core_extension`` entry point.
|
||||
source : str
|
||||
Full module source ready to write to ``routers/<module_name>.py``.
|
||||
"""
|
||||
|
||||
module_name: str
|
||||
entry_point_name: str | None
|
||||
source: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class GeneratedRouters:
|
||||
"""Collection of router modules covering every namespace in the tree.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
routers : list of GeneratedRouter
|
||||
All emitted router modules.
|
||||
root_commands : list of str
|
||||
Rendered command blocks for top-level leaf commands; they have no
|
||||
namespace router of their own and mount directly on the root router.
|
||||
root_post_imports : list of (str, str)
|
||||
``(module_path, function_name)`` imports the root command blocks need.
|
||||
root_has_stream : bool
|
||||
Whether any root command streams (root router imports ``OBBStream``).
|
||||
"""
|
||||
|
||||
routers: list[GeneratedRouter] = field(default_factory=list)
|
||||
root_commands: list[str] = field(default_factory=list)
|
||||
root_post_imports: list[tuple[str, str]] = field(default_factory=list)
|
||||
root_has_stream: bool = False
|
||||
|
||||
|
||||
def _safe_segment(name: str) -> str:
|
||||
"""Lower-case + replace non-alphanumerics with underscores."""
|
||||
cleaned = re.sub(r"[^0-9a-zA-Z]+", "_", name).strip("_")
|
||||
return cleaned.lower() or "ns"
|
||||
|
||||
|
||||
def _module_name_for(path: str) -> str:
|
||||
"""Convert a dotted namespace path to a router module name.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
path : str
|
||||
Dotted namespace path.
|
||||
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
Snake-case module identifier.
|
||||
"""
|
||||
return "_".join(_safe_segment(p) for p in path.split(".") if p)
|
||||
|
||||
|
||||
def _model_name_for(path: str) -> str:
|
||||
"""Convert a dotted command path to a CamelCase model identifier."""
|
||||
return "".join(
|
||||
seg[:1].upper() + seg[1:]
|
||||
for seg in (re.split(r"[^0-9a-zA-Z]+", part) for part in path.split("."))
|
||||
for seg in seg
|
||||
if seg
|
||||
)
|
||||
|
||||
|
||||
def generate_routers(
|
||||
root: NamespaceNode,
|
||||
*,
|
||||
package_name: str,
|
||||
provider_name: str,
|
||||
fetchers_by_command: dict[str, GeneratedFetcher],
|
||||
post_commands_by_command: dict[str, GeneratedPostCommand],
|
||||
) -> GeneratedRouters:
|
||||
"""Emit router modules for every namespace in ``root``.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
root : NamespaceNode
|
||||
The synthetic root node returned by ``build_namespace_tree``.
|
||||
package_name : str
|
||||
Top-level Python package name.
|
||||
provider_name : str
|
||||
Snake-case provider identifier.
|
||||
fetchers_by_command : dict
|
||||
Map ``{dotted_command: GeneratedFetcher}`` for every GET command.
|
||||
post_commands_by_command : dict
|
||||
Map ``{dotted_command: GeneratedPostCommand}`` for every POST command.
|
||||
|
||||
Returns
|
||||
-------
|
||||
GeneratedRouters
|
||||
One ``GeneratedRouter`` per namespace node.
|
||||
"""
|
||||
out = GeneratedRouters()
|
||||
for top_name, top_node in sorted(root.children.items()):
|
||||
if top_node.is_namespace:
|
||||
_emit_router(
|
||||
top_node,
|
||||
package_name=package_name,
|
||||
provider_name=provider_name,
|
||||
fetchers_by_command=fetchers_by_command,
|
||||
post_commands_by_command=post_commands_by_command,
|
||||
collected=out,
|
||||
is_top_level=True,
|
||||
)
|
||||
if top_node.cmd_spec is None:
|
||||
continue
|
||||
function_name = _safe_segment(top_name)
|
||||
description = (top_node.cmd_spec.get("description") or "").strip()
|
||||
fetcher = fetchers_by_command.get(top_node.full_path)
|
||||
if fetcher is not None:
|
||||
renderer = (
|
||||
_render_stream_command if fetcher.is_streaming else _render_get_command
|
||||
)
|
||||
out.root_has_stream = out.root_has_stream or fetcher.is_streaming
|
||||
out.root_commands.append(
|
||||
renderer(fetcher, function_name=function_name, description=description)
|
||||
)
|
||||
continue
|
||||
post = post_commands_by_command.get(top_node.full_path)
|
||||
if post is not None:
|
||||
out.root_post_imports.append(
|
||||
(
|
||||
f"{package_name}.providers.{provider_name}.models."
|
||||
f"{post.module_name}",
|
||||
post.function_name,
|
||||
)
|
||||
)
|
||||
out.root_commands.append(
|
||||
f'router.command(methods=["POST"])(_{post.function_name})\n'
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def _emit_router(
|
||||
node: NamespaceNode,
|
||||
*,
|
||||
package_name: str,
|
||||
provider_name: str,
|
||||
fetchers_by_command: dict[str, GeneratedFetcher],
|
||||
post_commands_by_command: dict[str, GeneratedPostCommand],
|
||||
collected: GeneratedRouters,
|
||||
is_top_level: bool,
|
||||
) -> str:
|
||||
"""Recursively emit one router module per namespace node.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
node : NamespaceNode
|
||||
The namespace this call emits a router for.
|
||||
package_name : str
|
||||
Top-level Python package name.
|
||||
provider_name : str
|
||||
Provider identifier for ``providers/<name>/`` imports.
|
||||
fetchers_by_command : dict
|
||||
GET-command fetcher modules.
|
||||
post_commands_by_command : dict
|
||||
POST-command modules.
|
||||
collected : GeneratedRouters
|
||||
Output accumulator.
|
||||
is_top_level : bool
|
||||
Whether this namespace is at the top of the tree.
|
||||
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
The module name of the emitted router.
|
||||
"""
|
||||
module_name = _module_name_for(node.full_path)
|
||||
parts: list[str] = []
|
||||
parts.append(f'"""Router for {node.full_path} commands — generated from spec."""')
|
||||
parts.append("")
|
||||
parts.append("from openbb_core.app.model.command_context import CommandContext")
|
||||
parts.append("from openbb_core.app.model.obbject import OBBject")
|
||||
parts.append(
|
||||
"from openbb_core.app.provider_interface import "
|
||||
"ExtraParams, ProviderChoices, StandardParams"
|
||||
)
|
||||
parts.append("from openbb_core.app.query import Query")
|
||||
parts.append("from openbb_core.app.router import Router")
|
||||
|
||||
has_stream = any(
|
||||
(fetcher := fetchers_by_command.get(child.full_path)) is not None
|
||||
and fetcher.is_streaming
|
||||
for child in node.children.values()
|
||||
if child.is_command and child.cmd_spec is not None
|
||||
)
|
||||
if has_stream:
|
||||
parts.append("from openbb_core.app.model.stream import OBBStream")
|
||||
|
||||
sub_imports: list[tuple[str, str, str]] = []
|
||||
for sub_name, sub_node in sorted(node.children.items()):
|
||||
if not sub_node.is_namespace:
|
||||
continue
|
||||
sub_module = _emit_router(
|
||||
sub_node,
|
||||
package_name=package_name,
|
||||
provider_name=provider_name,
|
||||
fetchers_by_command=fetchers_by_command,
|
||||
post_commands_by_command=post_commands_by_command,
|
||||
collected=collected,
|
||||
is_top_level=False,
|
||||
)
|
||||
sub_imports.append((sub_module, _safe_segment(sub_name), sub_node.full_path))
|
||||
|
||||
post_imports: list[tuple[str, str]] = []
|
||||
for child in node.children.values():
|
||||
if child.is_command and child.cmd_spec is not None:
|
||||
post = post_commands_by_command.get(child.full_path)
|
||||
if post is not None:
|
||||
post_module = (
|
||||
f"{package_name}.providers.{provider_name}.models."
|
||||
f"{post.module_name}"
|
||||
)
|
||||
post_imports.append((post_module, post.function_name))
|
||||
|
||||
if post_imports:
|
||||
parts.append("")
|
||||
for module_path, function_name in post_imports:
|
||||
parts.append(
|
||||
f"from {module_path} import {function_name} as _{function_name}"
|
||||
)
|
||||
|
||||
if sub_imports:
|
||||
parts.append("")
|
||||
for sub_module, sub_segment, _full in sub_imports:
|
||||
parts.append(
|
||||
f"from {package_name}.routers.{sub_module} import "
|
||||
f"router as _{sub_segment}_router"
|
||||
)
|
||||
|
||||
parts.append("")
|
||||
parts.append("")
|
||||
parts.append('router = Router(prefix="")')
|
||||
parts.append("")
|
||||
|
||||
for _, sub_segment, _full in sub_imports:
|
||||
parts.append(
|
||||
f'router.include_router(_{sub_segment}_router, prefix="/{sub_segment}")'
|
||||
)
|
||||
if sub_imports:
|
||||
parts.append("")
|
||||
|
||||
for child_name, child in sorted(node.children.items()):
|
||||
if not child.is_command or child.cmd_spec is None:
|
||||
continue
|
||||
function_name = _safe_segment(child_name)
|
||||
description = (child.cmd_spec.get("description") or "").strip()
|
||||
fetcher = fetchers_by_command.get(child.full_path)
|
||||
if fetcher is not None:
|
||||
renderer = (
|
||||
_render_stream_command if fetcher.is_streaming else _render_get_command
|
||||
)
|
||||
parts.append(
|
||||
renderer(
|
||||
fetcher,
|
||||
function_name=function_name,
|
||||
description=description,
|
||||
)
|
||||
)
|
||||
else:
|
||||
post = post_commands_by_command.get(child.full_path)
|
||||
if post is not None:
|
||||
parts.append(
|
||||
f'router.command(methods=["POST"])(_{post.function_name})\n'
|
||||
)
|
||||
|
||||
source = "\n".join(parts).rstrip() + "\n"
|
||||
collected.routers.append(
|
||||
GeneratedRouter(
|
||||
module_name=module_name,
|
||||
entry_point_name=node.name if is_top_level else None,
|
||||
source=source,
|
||||
)
|
||||
)
|
||||
return module_name
|
||||
|
||||
|
||||
def _render_get_command(
|
||||
fetcher: GeneratedFetcher,
|
||||
*,
|
||||
function_name: str,
|
||||
description: str,
|
||||
) -> str:
|
||||
"""Render one ``@router.command(model=...)`` block for a GET endpoint.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
fetcher : GeneratedFetcher
|
||||
Per-command fetcher metadata.
|
||||
function_name : str
|
||||
Leaf segment of the dotted path used as the function identifier.
|
||||
description : str
|
||||
Spec-supplied command description.
|
||||
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
The decorator + ``async def`` block.
|
||||
"""
|
||||
summary = description or f"{fetcher.model_name} command."
|
||||
docstring = _format_docstring(summary)
|
||||
return (
|
||||
f'@router.command(model="{fetcher.model_name}")\n'
|
||||
f"async def {function_name}(\n"
|
||||
" cc: CommandContext,\n"
|
||||
" provider_choices: ProviderChoices,\n"
|
||||
" standard_params: StandardParams,\n"
|
||||
" extra_params: ExtraParams,\n"
|
||||
") -> OBBject:\n"
|
||||
f"{docstring}\n"
|
||||
" return await OBBject.from_query(Query(**locals()))\n"
|
||||
)
|
||||
|
||||
|
||||
def _render_stream_command(
|
||||
fetcher: GeneratedFetcher,
|
||||
*,
|
||||
function_name: str,
|
||||
description: str,
|
||||
) -> str:
|
||||
"""Render one ``@router.command(model=...)`` block for a streaming endpoint.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
fetcher : GeneratedFetcher
|
||||
Per-command fetcher metadata.
|
||||
function_name : str
|
||||
Leaf segment of the dotted path used as the function identifier.
|
||||
description : str
|
||||
Spec-supplied command description.
|
||||
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
The decorator and ``async def`` block returning an OBBStream.
|
||||
"""
|
||||
summary = description or f"{fetcher.model_name} stream."
|
||||
docstring = _format_docstring(summary)
|
||||
return (
|
||||
f'@router.command(model="{fetcher.model_name}")\n'
|
||||
f"async def {function_name}(\n"
|
||||
" cc: CommandContext,\n"
|
||||
" provider_choices: ProviderChoices,\n"
|
||||
" standard_params: StandardParams,\n"
|
||||
" extra_params: ExtraParams,\n"
|
||||
") -> OBBStream:\n"
|
||||
f"{docstring}\n"
|
||||
" return await OBBStream.from_query(Query(**locals()))\n"
|
||||
)
|
||||
|
||||
|
||||
def _format_docstring(text: str) -> str:
|
||||
"""Render ``text`` as an indented function docstring.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
text : str
|
||||
Source description from the spec.
|
||||
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
Triple-quoted docstring body indented four spaces.
|
||||
"""
|
||||
cleaned = text.strip()
|
||||
if not cleaned:
|
||||
cleaned = "."
|
||||
if not cleaned.endswith((".", "!", "?")):
|
||||
cleaned += "."
|
||||
if "\n" not in cleaned:
|
||||
return f' """{cleaned}"""'
|
||||
first, _, rest = cleaned.partition("\n")
|
||||
rest_indented = "\n".join(
|
||||
f" {line}" if line.strip() else "" for line in rest.split("\n")
|
||||
)
|
||||
return f' """{first}\n\n{rest_indented}\n """'
|
||||
314
cli/openbb_cli/codegen/test_gen.py
Normal file
314
cli/openbb_cli/codegen/test_gen.py
Normal file
@@ -0,0 +1,314 @@
|
||||
"""Emit per-provider unit-test modules for a generated extension."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from openbb_cli.codegen.pydantic_gen import safe_field_name
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from openbb_cli.codegen.fetcher_gen import GeneratedFetcher
|
||||
|
||||
|
||||
_STANDARD_HEADER_REDACTIONS: tuple[tuple[str, str | None], ...] = (
|
||||
("User-Agent", None),
|
||||
("Authorization", "Bearer MOCK_TOKEN"),
|
||||
("X-API-Key", "MOCK_API_KEY"),
|
||||
("X-Api-Key", "MOCK_API_KEY"),
|
||||
("apikey", "MOCK_API_KEY"),
|
||||
("api-key", "MOCK_API_KEY"),
|
||||
("X-Auth-Token", "MOCK_TOKEN"),
|
||||
("X-Access-Token", "MOCK_TOKEN"),
|
||||
("Cookie", "MOCK_COOKIE"),
|
||||
("Set-Cookie", "MOCK_COOKIE"),
|
||||
)
|
||||
|
||||
_STANDARD_QUERY_REDACTIONS: tuple[tuple[str, str], ...] = (
|
||||
("apikey", "MOCK_API_KEY"),
|
||||
("api_key", "MOCK_API_KEY"),
|
||||
("api-key", "MOCK_API_KEY"),
|
||||
("key", "MOCK_API_KEY"),
|
||||
("subscription_key", "MOCK_API_KEY"),
|
||||
("ocp_apim_subscription_key", "MOCK_API_KEY"),
|
||||
("token", "MOCK_TOKEN"),
|
||||
("access_token", "MOCK_TOKEN"),
|
||||
("auth_token", "MOCK_TOKEN"),
|
||||
("bearer_token", "MOCK_TOKEN"),
|
||||
("session_token", "MOCK_TOKEN"),
|
||||
("client_id", "MOCK_CLIENT_ID"),
|
||||
("client_secret", "MOCK_CLIENT_SECRET"),
|
||||
("client_token", "MOCK_TOKEN"),
|
||||
("app_id", "MOCK_APP_ID"),
|
||||
("app_key", "MOCK_APP_KEY"),
|
||||
("app_token", "MOCK_TOKEN"),
|
||||
("appid", "MOCK_APP_ID"),
|
||||
("secret", "MOCK_SECRET"),
|
||||
("secret_key", "MOCK_SECRET"),
|
||||
("private_key", "MOCK_SECRET"),
|
||||
("consumer_key", "MOCK_API_KEY"),
|
||||
("consumer_secret", "MOCK_SECRET"),
|
||||
)
|
||||
|
||||
|
||||
def _mock_value_for(name: str) -> str:
|
||||
"""Pick a mock placeholder that matches what kind of credential ``name`` is."""
|
||||
lower = name.lower().replace("-", "_")
|
||||
if "secret" in lower:
|
||||
return "MOCK_SECRET"
|
||||
if "token" in lower:
|
||||
return "MOCK_TOKEN"
|
||||
if "client_id" in lower or lower.endswith("_id") or lower == "appid":
|
||||
return "MOCK_CLIENT_ID"
|
||||
return "MOCK_API_KEY"
|
||||
|
||||
|
||||
@dataclass
|
||||
class GeneratedTestModule:
|
||||
"""One ``tests/test_<provider>_fetchers.py`` ready to write to disk.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
module_name : str
|
||||
Filename without ``.py``.
|
||||
source : str
|
||||
Full module source.
|
||||
"""
|
||||
|
||||
module_name: str
|
||||
source: str
|
||||
|
||||
|
||||
def generate_provider_tests(
|
||||
*,
|
||||
package_name: str,
|
||||
provider_name: str,
|
||||
fetchers: list[GeneratedFetcher],
|
||||
commands_by_dotted: dict[str, dict[str, Any]],
|
||||
) -> GeneratedTestModule | None:
|
||||
"""Render a unit-test module exercising every fetcher in ``fetchers``.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
package_name : str
|
||||
Snake-case top-level package.
|
||||
provider_name : str
|
||||
Snake-case provider identifier.
|
||||
fetchers : list of GeneratedFetcher
|
||||
The fetchers to test.
|
||||
commands_by_dotted : dict
|
||||
Map ``{dotted_command: cmd_spec}``.
|
||||
|
||||
Returns
|
||||
-------
|
||||
GeneratedTestModule or None
|
||||
``None`` when there's nothing to test.
|
||||
"""
|
||||
if not fetchers:
|
||||
return None
|
||||
|
||||
testable: list[tuple[GeneratedFetcher, dict[str, Any]]] = []
|
||||
for f in sorted(fetchers, key=lambda x: x.fetcher_class):
|
||||
dotted = _dotted_for_fetcher(f, commands_by_dotted)
|
||||
cmd_spec = commands_by_dotted.get(dotted, {}) if dotted else {}
|
||||
params = _derive_test_params(cmd_spec, provider_name)
|
||||
if params is None:
|
||||
continue
|
||||
testable.append((f, params))
|
||||
|
||||
if not testable:
|
||||
return None
|
||||
|
||||
parts: list[str] = [
|
||||
f'"""Auto-generated unit tests for the {provider_name} provider."""',
|
||||
"",
|
||||
"import pytest",
|
||||
"from openbb_core.app.service.user_service import UserService",
|
||||
"",
|
||||
]
|
||||
for f, _ in testable:
|
||||
parts.append(
|
||||
f"from {package_name}.providers.{provider_name}.models."
|
||||
f"{f.module_name} import {f.fetcher_class}"
|
||||
)
|
||||
parts.append("")
|
||||
parts.append("")
|
||||
parts.append(
|
||||
"test_credentials = UserService()"
|
||||
".default_user_settings.credentials.model_dump(mode='json')"
|
||||
)
|
||||
parts.append("")
|
||||
parts.append("")
|
||||
header_filters, query_filters = _merge_redactions([f for f, _ in testable])
|
||||
parts.append('@pytest.fixture(scope="module")')
|
||||
parts.append("def vcr_config():")
|
||||
parts.append(' """Scrub credential fields from recorded cassettes."""')
|
||||
parts.append(" return {")
|
||||
parts.append(f' "filter_headers": {header_filters!r},')
|
||||
parts.append(f' "filter_query_parameters": {query_filters!r},')
|
||||
parts.append(" }")
|
||||
parts.append("")
|
||||
parts.append("")
|
||||
for f, params in testable:
|
||||
parts.append("@pytest.mark.record_http")
|
||||
parts.append(
|
||||
f"def test_{_snake(f.fetcher_class)}(credentials=test_credentials):"
|
||||
)
|
||||
parts.append(f" params = {params!r}")
|
||||
parts.append(f" fetcher = {f.fetcher_class}()")
|
||||
parts.append(" result = fetcher.test(params, credentials)")
|
||||
parts.append(" assert result is None")
|
||||
parts.append("")
|
||||
parts.append("")
|
||||
|
||||
source = "\n".join(parts).rstrip() + "\n"
|
||||
return GeneratedTestModule(
|
||||
module_name=f"test_{provider_name}_fetchers",
|
||||
source=source,
|
||||
)
|
||||
|
||||
|
||||
def _merge_redactions(
|
||||
fetchers: list[GeneratedFetcher],
|
||||
) -> tuple[list[tuple[str, str | None]], list[tuple[str, str]]]:
|
||||
"""Build (header_filters, query_filters) for the ``vcr_config`` fixture."""
|
||||
headers: dict[str, str | None] = {}
|
||||
queries: dict[str, str] = {}
|
||||
for f in fetchers:
|
||||
for info in f.credentials_used.values():
|
||||
wire = info.get("name")
|
||||
location = info.get("in", "query")
|
||||
if not wire:
|
||||
continue
|
||||
if location == "header":
|
||||
headers[wire] = _mock_value_for(wire)
|
||||
elif location == "cookie":
|
||||
# Cookies travel on the wire as the ``Cookie`` header, which
|
||||
# ``_STANDARD_HEADER_REDACTIONS`` already blanket-redacts below
|
||||
# — VCR has no way to filter a single cookie within it.
|
||||
continue
|
||||
else:
|
||||
queries[wire] = _mock_value_for(wire)
|
||||
for n, v in _STANDARD_HEADER_REDACTIONS:
|
||||
headers[n] = v
|
||||
for n, v in _STANDARD_QUERY_REDACTIONS:
|
||||
queries[n] = v
|
||||
return (
|
||||
sorted(headers.items()),
|
||||
sorted(queries.items()),
|
||||
)
|
||||
|
||||
|
||||
def _dotted_for_fetcher(
|
||||
fetcher: GeneratedFetcher,
|
||||
commands_by_dotted: dict[str, dict[str, Any]],
|
||||
) -> str | None:
|
||||
"""Find the dotted command path that produced ``fetcher``."""
|
||||
target = fetcher.module_name
|
||||
for dotted in commands_by_dotted:
|
||||
if _normalize_to_module(dotted) == target:
|
||||
return dotted
|
||||
return None
|
||||
|
||||
|
||||
def _normalize_to_module(dotted: str) -> str:
|
||||
"""Match ``fetcher_gen._module_name_from_command`` exactly."""
|
||||
import re
|
||||
|
||||
safe = re.sub(r"[^0-9a-zA-Z]+", "_", dotted).strip("_").lower()
|
||||
if not safe:
|
||||
safe = "command"
|
||||
if safe[0].isdigit():
|
||||
safe = f"_{safe}"
|
||||
return safe
|
||||
|
||||
|
||||
_DATE_PARAM_NAMES: frozenset[str] = frozenset(
|
||||
{
|
||||
"date",
|
||||
"from",
|
||||
"to",
|
||||
"fromdate",
|
||||
"todate",
|
||||
"start_date",
|
||||
"end_date",
|
||||
"startdate",
|
||||
"enddate",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _is_date_param(name: str) -> bool:
|
||||
"""Heuristic: this param's name implies a date / time range filter."""
|
||||
lower = name.lower().replace("-", "_")
|
||||
return lower in _DATE_PARAM_NAMES or "date" in lower
|
||||
|
||||
|
||||
def _derive_test_params(
|
||||
cmd_spec: dict[str, Any], provider_name: str
|
||||
) -> dict[str, Any] | None:
|
||||
"""Build a kwargs dict for the command's testable params, or ``None``."""
|
||||
out: dict[str, Any] = {}
|
||||
for raw in cmd_spec.get("parameters") or []:
|
||||
if not isinstance(raw, dict):
|
||||
continue
|
||||
name = raw.get("name")
|
||||
if not name or name == "provider":
|
||||
continue
|
||||
param_providers = raw.get("providers") or []
|
||||
if param_providers and provider_name not in param_providers:
|
||||
continue
|
||||
field_name, _ = safe_field_name(name)
|
||||
if raw.get("required"):
|
||||
value = _spec_supplied_value(raw)
|
||||
if value is None:
|
||||
return None
|
||||
out[field_name] = value
|
||||
elif _is_date_param(name):
|
||||
out[field_name] = _refresh_date_shape(name, "1970-01-01")
|
||||
return out
|
||||
|
||||
|
||||
def _spec_supplied_value(param: dict[str, Any]) -> Any:
|
||||
"""Return a value the spec itself vouches for, or ``None`` to abort the test."""
|
||||
raw_value: Any = None
|
||||
if param.get("example") is not None:
|
||||
raw_value = param["example"]
|
||||
elif param.get("default") is not None:
|
||||
raw_value = param["default"]
|
||||
else:
|
||||
choices = param.get("choices") or []
|
||||
if choices:
|
||||
raw_value = choices[0]
|
||||
if raw_value is None:
|
||||
return None
|
||||
return _refresh_date_shape(param.get("name") or "", raw_value)
|
||||
|
||||
|
||||
def _refresh_date_shape(name: str, value: Any) -> Any:
|
||||
"""Replace an aged date-shaped string with an equivalent current date."""
|
||||
import re
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
|
||||
if not isinstance(value, str):
|
||||
return value
|
||||
lower = name.lower()
|
||||
is_start = "start" in lower or lower in {"from", "fromdate"}
|
||||
if re.fullmatch(r"\d{4}-\d{2}-\d{2}", value):
|
||||
today = date.today()
|
||||
return str(today - timedelta(days=30) if is_start else today)
|
||||
if re.fullmatch(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z?", value):
|
||||
now = datetime.now(timezone.utc).replace(microsecond=0)
|
||||
chosen = (now - timedelta(days=30)) if is_start else now
|
||||
return chosen.isoformat().replace("+00:00", "Z")
|
||||
return value
|
||||
|
||||
|
||||
def _snake(camel: str) -> str:
|
||||
"""``CamelCase`` -> ``snake_case`` for test function names."""
|
||||
out: list[str] = []
|
||||
for i, ch in enumerate(camel):
|
||||
if ch.isupper() and i > 0 and not camel[i - 1].isupper():
|
||||
out.append("_")
|
||||
out.append(ch.lower())
|
||||
return "".join(out)
|
||||
Reference in New Issue
Block a user