[Release] openbb-v5.0.0 (#7741)

* merge branch develop with main

* bump desktop version

* fix the merge

* sync Cargo.lock to desktop 2.0.0

* fix cli conftest.py

---------

Co-authored-by: deeleeramone <>
This commit is contained in:
Danglewood
2026-09-29 10:58:44 -07:00
committed by GitHub
parent aca5e6a73d
commit 869cfed125
4364 changed files with 1360878 additions and 1408198 deletions

View File

@@ -0,0 +1 @@
"""Codegen pipeline that turns a ``.spec`` document into a full extension package."""

View File

@@ -0,0 +1,98 @@
"""Credential detection — separate user-facing params from auth secrets."""
from __future__ import annotations
from typing import Any, Literal
CredentialLocation = Literal["query", "header", "cookie"]
_CREDENTIAL_NAMES: frozenset[str] = frozenset(
{
"api_key",
"api_token",
"apikey",
"x_api_key",
"x_apikey",
"key",
"subscription_key",
"ocp_apim_subscription_key",
"app_id",
"app_key",
"app_token",
"appid",
"client_id",
"client_secret",
"client_token",
"token",
"access_token",
"auth_token",
"x_auth_token",
"bearer_token",
"secret",
"secret_key",
"authorization",
"consumer_key",
"consumer_secret",
"private_key",
"session_token",
}
)
def normalize_credential_key(name: str) -> str:
"""Canonicalize a parameter name for credential lookup."""
return name.lower().replace("-", "_").strip("_")
def is_credential_name(name: str) -> bool:
"""Return ``True`` if ``name`` looks like a credential (not a request param)."""
return normalize_credential_key(name) in _CREDENTIAL_NAMES
def classify_parameter(param: dict[str, Any]) -> CredentialLocation | None:
"""Return ``"query"`` / ``"header"`` / ``"cookie"`` for a credential param.
A parameter is a credential when its name is a known secret alias
(``api_key``, ``token``, ...), or when it's a *required* ``header``/
``cookie`` parameter regardless of name — that's the profile of a
per-integration value (a tenant id, a session cookie) declared once on a
path item for every operation under it, not something request-specific
like ``If-Modified-Since``. Returns ``None`` for anything else.
"""
name = param.get("name") if isinstance(param, dict) else None
if not isinstance(name, str) or not name:
return None
location = param.get("in", "query")
if is_credential_name(name):
return "header" if location == "header" else "query"
if param.get("required") and location in ("header", "cookie"):
return location
return None
def credentials_from_command(
cmd_spec: dict[str, Any],
) -> dict[str, dict[str, str]]:
"""Walk a command spec entry; return ``{canonical_key: {name, in}}``."""
out: dict[str, dict[str, str]] = {}
for param in cmd_spec.get("parameters") or []:
loc = classify_parameter(param)
if loc is None:
continue
canonical = normalize_credential_key(param["name"])
out[canonical] = {"name": param["name"], "in": loc}
return out
def credentials_from_spec(spec_doc: dict[str, Any]) -> dict[str, dict[str, str]]:
"""Aggregate every credential entry across an entire spec doc."""
out: dict[str, dict[str, str]] = {}
for cmd in (spec_doc.get("commands") or {}).values():
for canonical, entry in credentials_from_command(cmd).items():
out.setdefault(canonical, entry)
return out
def filter_user_params(parameters: list[dict[str, Any]]) -> list[dict[str, Any]]:
"""Strip credential params from a parameter list, leaving user-facing ones."""
return [p for p in parameters or [] if classify_parameter(p) is None]

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,158 @@
"""Group spec commands into a hierarchical namespace tree."""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Any
@dataclass
class NamespaceNode:
"""One node in the per-spec namespace tree.
Parameters
----------
name : str
The leaf segment of this node's dotted path.
full_path : str
The full dotted path from the root.
cmd_spec : dict, optional
The spec command entry when this node is a leaf command.
children : dict[str, NamespaceNode]
Sub-nodes keyed by their ``name``.
"""
name: str
full_path: str
cmd_spec: dict[str, Any] | None = None
children: dict[str, NamespaceNode] = field(default_factory=dict)
@property
def is_command(self) -> bool:
"""Whether this node corresponds to an actual spec command."""
return self.cmd_spec is not None
@property
def is_namespace(self) -> bool:
"""Whether this node has nested children (acts as a menu / sub-router)."""
return bool(self.children)
def build_namespace_tree(commands: dict[str, dict[str, Any]]) -> NamespaceNode:
"""Build a hierarchical tree from the spec's flat command dict.
Parameters
----------
commands : dict
Mapping of dotted command path to spec entry.
Returns
-------
NamespaceNode
Synthetic root node whose ``children`` are the top-level namespaces.
"""
root = NamespaceNode(name="", full_path="")
for dotted, cmd_spec in commands.items():
if not dotted:
continue
parts = dotted.split(".")
node = root
path_so_far: list[str] = []
for segment in parts:
path_so_far.append(segment)
child = node.children.get(segment)
if child is None:
child = NamespaceNode(name=segment, full_path=".".join(path_so_far))
node.children[segment] = child
node = child
node.cmd_spec = cmd_spec
return root
def filter_tree_by_provider(root: NamespaceNode, provider: str) -> NamespaceNode:
"""Return a copy of ``root`` keeping only commands that include ``provider``.
Parameters
----------
root : NamespaceNode
The full namespace tree.
provider : str
Provider identifier to filter by.
Returns
-------
NamespaceNode
New tree containing only the surviving commands.
"""
return _filter_node(root, lambda providers: provider in providers) or NamespaceNode(
name="", full_path=""
)
def filter_tree_local_only(root: NamespaceNode) -> NamespaceNode:
"""Return a copy of ``root`` keeping only commands with no provider list.
Parameters
----------
root : NamespaceNode
The full namespace tree.
Returns
-------
NamespaceNode
New tree containing only commands whose ``providers`` list is empty.
"""
return _filter_node(root, lambda providers: not providers) or NamespaceNode(
name="", full_path=""
)
def _filter_node(node: NamespaceNode, predicate) -> NamespaceNode | None:
"""Recursive helper that keeps commands matching ``predicate(providers)``."""
keep_self = False
new_cmd: dict[str, Any] | None = None
if node.cmd_spec is not None:
providers_list = node.cmd_spec.get("providers") or []
if predicate(providers_list):
keep_self = True
new_cmd = node.cmd_spec
new_children: dict[str, NamespaceNode] = {}
for name, child in node.children.items():
kept = _filter_node(child, predicate)
if kept is not None:
new_children[name] = kept
if not keep_self and not new_children:
return None
return NamespaceNode(
name=node.name,
full_path=node.full_path,
cmd_spec=new_cmd,
children=new_children,
)
def iter_commands(root: NamespaceNode) -> list[tuple[str, dict[str, Any]]]:
"""Walk the tree and return ``[(dotted_path, cmd_spec), ...]`` for every command."""
out: list[tuple[str, dict[str, Any]]] = []
_walk_commands(root, out)
return out
def _walk_commands(node: NamespaceNode, out: list[tuple[str, dict[str, Any]]]) -> None:
"""Depth-first command collector."""
if node.cmd_spec is not None:
out.append((node.full_path, node.cmd_spec))
for child in node.children.values():
_walk_commands(child, out)
def providers_from_tree(root: NamespaceNode) -> set[str]:
"""Collect every provider mentioned across the tree's commands."""
out: set[str] = set()
for _, cmd_spec in iter_commands(root):
for p in cmd_spec.get("providers") or []:
out.add(str(p))
return out

View File

@@ -0,0 +1,605 @@
"""Top-level codegen orchestrator: spec doc to one installable project."""
from __future__ import annotations
import re
import shutil
import subprocess
import sys
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
from openbb_cli.codegen.fetcher_gen import (
FetcherCommandSpec,
GeneratedFetcher,
generate_fetcher_module,
)
from openbb_cli.codegen.namespace_tree import (
build_namespace_tree,
iter_commands,
providers_from_tree,
)
from openbb_cli.codegen.post_gen import (
GeneratedPostCommand,
PostCommandSpec,
generate_post_command_module,
)
from openbb_cli.codegen.project_gen import GeneratedProject, generate_pyproject
from openbb_cli.codegen.provider_gen import GeneratedProvider, generate_provider_module
from openbb_cli.codegen.router_gen import (
GeneratedRouter,
GeneratedRouters,
generate_routers,
)
from openbb_cli.codegen.test_gen import GeneratedTestModule, generate_provider_tests
@dataclass
class GeneratedPackage:
"""One installable project that bundles every provider in the spec.
Parameters
----------
root : Path
Project root directory.
project : GeneratedProject
``pyproject.toml`` source.
providers : list of GeneratedProvider
One provider registration module per declared spec provider.
routers : GeneratedRouters
Shared router modules.
fetchers_by_provider : dict
``{provider_name: [GeneratedFetcher, ...]}``.
post_commands : list of GeneratedPostCommand
Local-compute POST endpoints.
top_level_routers : list of str
Top-level namespace names registered as ``openbb_core_extension`` entry points.
"""
root: Path
project: GeneratedProject
providers: list[GeneratedProvider]
routers: GeneratedRouters
fetchers_by_provider: dict[str, list[GeneratedFetcher]]
post_commands: list[GeneratedPostCommand] = field(default_factory=list)
top_level_routers: list[str] = field(default_factory=list)
root_namespace: str = ""
base_url: str = ""
commands_by_provider: dict[str, list[str]] = field(default_factory=dict)
test_modules: list[GeneratedTestModule] = field(default_factory=list)
def write(self) -> Path:
"""Materialize the project on disk under ``self.root``.
Returns
-------
Path
``self.root``.
"""
pkg = self.root / self.project.package_name
routers_dir = pkg / "routers"
providers_dir = pkg / "providers"
routers_dir.mkdir(parents=True, exist_ok=True)
providers_dir.mkdir(parents=True, exist_ok=True)
for path in (pkg, routers_dir, providers_dir):
init = path / "__init__.py"
if not init.exists():
init.write_text(f'"""{path.name} subpackage."""\n')
(pkg / "utils.py").write_text(RUNTIME_UTILS_SOURCE)
for r in self.routers.routers:
(routers_dir / f"{r.module_name}.py").write_text(r.source)
for prov in self.providers:
prov_dir = providers_dir / prov.provider_name
models_dir = prov_dir / "models"
models_dir.mkdir(parents=True, exist_ok=True)
for path in (prov_dir, models_dir):
init = path / "__init__.py"
if not init.exists():
init.write_text(f'"""{path.name} subpackage."""\n')
(prov_dir / "__init__.py").write_text(prov.source)
for fetcher in self.fetchers_by_provider.get(prov.provider_name, []):
(models_dir / f"{fetcher.module_name}.py").write_text(fetcher.source)
if self.post_commands:
tools_models = providers_dir / "tools" / "models"
tools_models.mkdir(parents=True, exist_ok=True)
for path in (providers_dir / "tools", tools_models):
init = path / "__init__.py"
if not init.exists(): # pragma: no cover
init.write_text(f'"""{path.name} subpackage."""\n')
for post in self.post_commands:
(tools_models / f"{post.module_name}.py").write_text(post.source)
if self.test_modules:
tests_dir = self.root / "tests"
tests_dir.mkdir(parents=True, exist_ok=True)
init = tests_dir / "__init__.py"
if not init.exists():
init.write_text('"""Auto-generated tests."""\n')
for tm in self.test_modules:
(tests_dir / f"{tm.module_name}.py").write_text(tm.source)
(self.root / "pyproject.toml").write_text(self.project.source)
(self.root / "README.md").write_text(self._readme())
_apply_ruff(self.root)
return self.root
def _readme(self) -> str:
"""Render a README that describes the actual generated surface."""
ns = self.root_namespace or self.project.package_name
total_get = sum(len(v) for v in self.fetchers_by_provider.values())
total_post = len(self.post_commands)
source = self.base_url or "the source OpenAPI spec"
sections: list[str] = []
sections.append(f"# {self.project.project_name}\n")
sections.append(
f"OpenBB Platform extension generated from {source}. "
f"Adds {total_get} GET command{'s' if total_get != 1 else ''}"
+ (
f" and {total_post} POST command{'s' if total_post != 1 else ''}"
if total_post
else ""
)
+ f" under `obb.{ns}.*`.\n"
)
sections.append("## Install\n")
sections.append("```bash")
sections.append("pip install -e .")
sections.append("openbb-build")
sections.append("```\n")
sections.append("## Quick start\n")
first_cmd = self._first_command_example(ns)
if first_cmd is not None:
dotted, model = first_cmd
sections.append("```python")
sections.append("from openbb import obb")
sections.append("")
sections.append(f"# {model}")
sections.append(f"result = obb.{ns}.{dotted}()")
sections.append("print(result.to_df())")
sections.append("```\n")
sections.append(self._providers_section(ns))
sections.append(self._commands_section(ns))
creds = self._unique_credentials()
if creds:
sections.append(self._credentials_section(creds))
sections.append(self._notes_section(ns))
return "\n".join(sections).rstrip() + "\n"
def _first_command_example(self, ns: str) -> tuple[str, str] | None:
"""Pick a representative command to seed the Quick start snippet."""
for prov in sorted(self.commands_by_provider):
cmds = sorted(self.commands_by_provider[prov])
if not cmds:
continue
dotted = cmds[0]
fetcher = next(
(
f
for f in self.fetchers_by_provider.get(prov, [])
if f"{ns}_{f.module_name}".endswith(dotted.replace(".", "_"))
or f.module_name == dotted.replace(".", "_")
),
None,
)
model = fetcher.model_name if fetcher else "Returns OBBject"
return dotted, model
return None
def _providers_section(self, ns: str) -> str:
lines = ["## Providers\n"]
for p in self.providers:
cmd_count = len(self.commands_by_provider.get(p.provider_name, []))
label = "command" if cmd_count == 1 else "commands"
cred_note = (
f" — credentials: {', '.join(f'`{k}`' for k in p.credential_keys)}"
if p.credential_keys
else ""
)
lines.append(f"- **{p.provider_name}** ({cmd_count} {label}){cred_note}")
return "\n".join(lines) + "\n"
def _commands_section(self, ns: str) -> str:
lines = ["## Commands\n"]
lines.append(
f"Every command lands at `obb.{ns}.<dotted.path>` with a typed "
"signature, full Pydantic validation, and `OBBject` results.\n"
)
groups: dict[str, list[tuple[str, str]]] = {}
for prov, dotted_paths in self.commands_by_provider.items():
for dotted in sorted(dotted_paths):
top = dotted.split(".", 1)[0]
groups.setdefault(top, []).append((dotted, prov))
for top in sorted(groups):
lines.append(f"### `obb.{ns}.{top}`\n")
for dotted, prov in groups[top]:
lines.append(f"- `obb.{ns}.{dotted}` — provider: `{prov}`")
lines.append("")
return "\n".join(lines).rstrip() + "\n"
def _unique_credentials(self) -> list[str]:
seen: set[str] = set()
for p in self.providers:
seen.update(p.credential_keys)
return sorted(seen)
def _credentials_section(self, creds: list[str]) -> str:
lines = ["## Credentials\n"]
lines.append(
"Set these in `~/.openbb_platform/user_settings.json` under the "
"matching `<provider>_<key>` field, or pass via the `OBB_USER_SETTINGS` "
"env var:\n"
)
for c in creds:
lines.append(f"- `{c}`")
lines.append("")
return "\n".join(lines).rstrip() + "\n"
def _notes_section(self, ns: str) -> str:
return (
"## Notes\n\n"
"- Routers mount under a single root namespace "
f"(`obb.{ns}.*`) so the extension never shadows OpenBB's own "
"first-party routers.\n"
"- Each `aextract_data` strips single-key envelopes and unpacks "
"single-element lists at the response boundary; sibling scalar "
"fields surface as `AnnotatedResult` metadata in `OBBject.extra`.\n"
"- Non-JSON responses (XML, CSV, plain text) come back as a "
"single-row dict with `content` / `content_type` rather than "
"raising on parse.\n"
)
def _slugify(text: str) -> str:
"""Lower-case + collapse non-alphanumerics to single underscores."""
cleaned = re.sub(r"[^0-9a-zA-Z]+", "_", text).strip("_").lower()
return cleaned or "extension"
def _apply_ruff(root: Path) -> None:
"""Run ``ruff check --fix --unsafe-fixes`` and ``ruff format`` over ``root``."""
ruff = _find_ruff()
if ruff is None:
return
target = str(root)
subprocess.run( # noqa: S603
[ruff, "check", "--fix", "--unsafe-fixes", "--quiet", target],
check=False,
)
subprocess.run( # noqa: S603
[ruff, "format", "--quiet", target],
check=False,
)
def _find_ruff() -> str | None:
"""Locate the ``ruff`` binary; checks the running Python env first."""
sibling = Path(sys.executable).parent / "ruff"
if sibling.exists():
return str(sibling)
return shutil.which("ruff")
def _build_root_router(
*,
package_name: str,
root_namespace: str,
sub_routers: list[GeneratedRouter],
root_commands: list[str] | None = None,
root_post_imports: list[tuple[str, str]] | None = None,
root_has_stream: bool = False,
) -> GeneratedRouter:
"""Emit a top-level router that mounts every namespace under ``root_namespace``.
Top-level leaf commands (no namespace of their own) are rendered directly
on this router; unused imports are cleaned up by the ruff pass.
"""
commands = root_commands or []
post_imports = root_post_imports or []
parts: list[str] = [
f'"""Root router for {root_namespace} — generated from spec."""',
"",
]
if commands:
parts.append("from openbb_core.app.model.command_context import CommandContext")
parts.append("from openbb_core.app.model.obbject import OBBject")
if root_has_stream:
parts.append("from openbb_core.app.model.stream import OBBStream")
parts.append(
"from openbb_core.app.provider_interface import "
"ExtraParams, ProviderChoices, StandardParams"
)
parts.append("from openbb_core.app.query import Query")
parts.append("from openbb_core.app.router import Router")
if post_imports:
parts.append("")
for module_path, function_name in post_imports:
parts.append(
f"from {module_path} import {function_name} as _{function_name}"
)
if sub_routers:
parts.append("")
for r in sub_routers:
parts.append(
f"from {package_name}.routers.{r.module_name} import "
f"router as _{r.module_name}_router"
)
parts.append("")
parts.append("")
parts.append('router = Router(prefix="")')
if sub_routers:
parts.append("")
for r in sub_routers:
parts.append(
f"router.include_router(_{r.module_name}_router, "
f'prefix="/{r.module_name}")'
)
if commands:
parts.append("")
parts.extend(commands)
return GeneratedRouter(
module_name=root_namespace,
entry_point_name=root_namespace,
source="\n".join(parts).rstrip() + "\n",
)
_SKIP_TOP_NAMESPACES: frozenset[str] = frozenset({"coverage"})
def _runtime_utils_source() -> str:
"""Read the canonical ``unpack_response`` source from ``_unpack.py``."""
from pathlib import Path
text = (
Path(__file__)
.resolve()
.parent.parent.joinpath("dispatchers", "_unpack.py")
.read_text()
)
return text
RUNTIME_UTILS_SOURCE = _runtime_utils_source()
def generate_packages(
spec_doc: dict[str, Any],
*,
output_root: Path,
provider_name: str | None = None,
project_name: str | None = None,
package_name: str | None = None,
router_name: str | None = None,
description: str | None = None,
website: str | None = None,
version: str = "0.1.0",
) -> GeneratedPackageSet:
"""Build ONE ``GeneratedPackage`` bundling every provider in ``spec_doc``.
Parameters
----------
spec_doc : dict
Loaded ``.spec`` document.
output_root : Path
Directory under which the project is created.
provider_name : str, optional
Project / package slug.
project_name, package_name, description, website : str, optional
Project-level overrides.
router_name : str, optional
Overrides the generated top-level router's module/entry-point name.
Defaults to ``provider_name``.
version : str
Initial version string for ``pyproject.toml``.
Returns
-------
GeneratedPackageSet
Single-element wrapper around the project.
"""
base_url = (spec_doc.get("base_url") or "").rstrip("/")
api_prefix = spec_doc.get("api_prefix") or ""
filtered_commands = {
name: cmd
for name, cmd in (spec_doc.get("commands") or {}).items()
if name.split(".", 1)[0] not in _SKIP_TOP_NAMESPACES
}
full_tree = build_namespace_tree(filtered_commands)
declared_providers = providers_from_tree(full_tree)
project_slug = _slugify(provider_name or "codegen")
project_name = project_name or f"openbb-{project_slug.replace('_', '-')}"
package_name = package_name or f"openbb_{project_slug}"
description = (
description or f"Generated OpenBB Platform extension bundle ({project_slug})."
)
website = website or base_url
fetchers_by_provider: dict[str, list[GeneratedFetcher]] = {}
post_commands: list[GeneratedPostCommand] = []
fetchers_index: dict[str, GeneratedFetcher] = {}
post_index: dict[str, GeneratedPostCommand] = {}
if not declared_providers:
target_providers: list[str] = [_slugify(provider_name or "default")]
else:
target_providers = sorted(declared_providers)
commands_by_provider: dict[str, list[str]] = {}
commands_by_dotted: dict[str, dict[str, Any]] = {}
for dotted, cmd_spec in iter_commands(full_tree):
commands_by_dotted[dotted] = cmd_spec
method = (cmd_spec.get("method") or "get").lower()
body = cmd_spec.get("request_body_schema")
has_body = isinstance(body, dict) and (
body.get("properties") or body.get("type") == "array"
)
cmd_providers = cmd_spec.get("providers") or []
if method == "post" and has_body:
post = generate_post_command_module(
PostCommandSpec(
name=dotted,
cmd_spec=cmd_spec,
base_url=base_url,
api_prefix=api_prefix,
provider_name="tools",
)
)
post_commands.append(post)
post_index[dotted] = post
commands_by_provider.setdefault("tools", []).append(dotted)
continue
owners = (
[p for p in target_providers if p in cmd_providers]
if cmd_providers
else target_providers
)
if not owners: # pragma: no cover
continue
for prov in owners:
fetcher = generate_fetcher_module(
FetcherCommandSpec(
name=dotted,
cmd_spec=cmd_spec,
base_url=base_url,
api_prefix=api_prefix,
provider_name=prov,
)
)
fetchers_by_provider.setdefault(prov, []).append(fetcher)
commands_by_provider.setdefault(prov, []).append(dotted)
fetchers_index.setdefault(dotted, fetcher)
routers = generate_routers(
full_tree,
package_name=package_name,
provider_name="",
fetchers_by_command=fetchers_index,
post_commands_by_command=post_index,
)
for r in routers.routers:
r.source = r.source.replace(
f"{package_name}.providers..models.",
f"{package_name}.providers.tools.models.",
)
routers.root_post_imports = [
(
module_path.replace(
f"{package_name}.providers..models.",
f"{package_name}.providers.tools.models.",
),
function_name,
)
for module_path, function_name in routers.root_post_imports
]
root_namespace = _slugify(router_name or provider_name or project_slug)
sub_routers = [r for r in routers.routers if r.entry_point_name]
for r in sub_routers:
r.entry_point_name = None
root_router = _build_root_router(
package_name=package_name,
root_namespace=root_namespace,
sub_routers=sub_routers,
root_commands=routers.root_commands,
root_post_imports=routers.root_post_imports,
root_has_stream=routers.root_has_stream,
)
routers.routers.append(root_router)
providers: list[GeneratedProvider] = []
for prov in target_providers:
providers.append(
generate_provider_module(
package_name=package_name,
provider_name=prov,
description=f"{prov} provider proxied through {base_url}.",
website=website,
fetchers=fetchers_by_provider.get(prov, []),
)
)
if post_commands:
providers.append(
generate_provider_module(
package_name=package_name,
provider_name="tools",
description=(
"Local-compute commands (econometrics, quantitative, "
f"technical) proxied through {base_url}."
),
website=website,
fetchers=[],
)
)
project = generate_pyproject(
project_name=project_name,
package_name=package_name,
providers=[p.provider_name for p in providers],
routers=[r for r in routers.routers if r.entry_point_name],
description=description,
version=version,
spec_provenance={
"source_url": str(spec_doc.get("source_url") or ""),
"api_version": str(spec_doc.get("api_version") or ""),
"generator": str(spec_doc.get("generator") or ""),
"generated_at": str(spec_doc.get("generated_at") or ""),
"spec_version": str(spec_doc.get("version") or ""),
"spec_sha256": str(spec_doc.get("content_sha256") or ""),
},
)
test_modules: list[GeneratedTestModule] = []
for prov in target_providers:
tm = generate_provider_tests(
package_name=package_name,
provider_name=prov,
fetchers=fetchers_by_provider.get(prov, []),
commands_by_dotted=commands_by_dotted,
)
if tm is not None:
test_modules.append(tm)
package = GeneratedPackage(
root=Path(output_root) / project_name,
project=project,
providers=providers,
routers=routers,
fetchers_by_provider=fetchers_by_provider,
post_commands=post_commands,
top_level_routers=[
r.entry_point_name for r in routers.routers if r.entry_point_name
],
root_namespace=root_namespace,
base_url=base_url,
commands_by_provider=commands_by_provider,
test_modules=test_modules,
)
return GeneratedPackageSet(packages=[package])
@dataclass
class GeneratedPackageSet:
"""Wrapper around the generated package list (kept for caller symmetry)."""
packages: list[GeneratedPackage] = field(default_factory=list)
def write(self) -> list[Path]:
"""Materialize every package; returns the list of written roots."""
return [pkg.write() for pkg in self.packages]

View File

@@ -0,0 +1,782 @@
"""Generate free-form router modules for POST endpoints with request bodies."""
from __future__ import annotations
import re
from dataclasses import dataclass, field
from typing import Any, cast
from openbb_cli.codegen.credentials import (
credentials_from_command,
filter_user_params,
)
from openbb_cli.codegen.pydantic_gen import (
GeneratedClass,
class_name_from,
consolidate_imports,
generate_class,
safe_field_name,
)
@dataclass
class PostCommandSpec:
"""Per-command inputs for a POST router emitter.
Parameters
----------
name : str
Dotted command path from the spec.
cmd_spec : dict
The command's spec entry.
base_url : str
Upstream API root, no trailing slash.
api_prefix : str
Path prefix shared across every command in the spec.
provider_name : str
Snake-case provider identifier.
"""
name: str
cmd_spec: dict[str, Any]
base_url: str
api_prefix: str
provider_name: str
@dataclass
class GeneratedPostCommand:
"""Output of ``generate_post_command_module``.
Parameters
----------
module_name : str
Snake-case module filename (without ``.py``).
function_name : str
The router function name.
body_class : str, optional
Name of the emitted body element class, if a body is present.
data_class : str
Name of the emitted response-row class.
source : str
Full module source ready to write to disk.
credentials_used : dict
Same shape as ``GeneratedFetcher.credentials_used``.
"""
module_name: str
function_name: str
data_class: str
source: str
body_class: str | None = None
credentials_used: dict[str, dict[str, str]] = field(default_factory=dict)
_PATH_TEMPLATE_RE = re.compile(r"\{([^}]+)\}")
def _module_name_from_command(name: str) -> str:
"""Convert a dotted command path to a snake_case module identifier."""
safe = re.sub(r"[^0-9a-zA-Z]+", "_", name).strip("_")
if not safe:
safe = "command"
if safe[0].isdigit():
safe = f"_{safe}"
return safe.lower()
def _path_template_keys(url_path: str) -> list[str]:
"""Extract ``{placeholder}`` names from a URL template, in declaration order."""
return [m.group(1) for m in _PATH_TEMPLATE_RE.finditer(url_path)]
def _resolve_url_path(api_prefix: str, url_path: str) -> str:
"""Combine api_prefix with url_path; the prefix is dropped if already present."""
prefix = ("/" + api_prefix.strip("/")) if api_prefix.strip("/") else ""
path = url_path if url_path.startswith("/") else "/" + url_path
if prefix and path.startswith(prefix + "/"):
return path
if prefix:
return prefix + path if not path.startswith(prefix) else path
return path
def _python_type_from_param(param: dict[str, Any]) -> str:
"""Render a Python type annotation for a normalized parameter entry.
Parameters
----------
param : dict
Spec parameter entry.
Returns
-------
str
Annotation text suitable for the generated function signature.
"""
base_type = {
"string": "str",
"integer": "int",
"number": "float",
"boolean": "bool",
}.get(param.get("type", "string"), "str")
if param.get("choices"):
rendered = ", ".join(repr(c) for c in param["choices"])
base_type = f"Literal[{rendered}]"
if param.get("is_list"):
return f"list[{base_type}]"
return base_type
def _array_item_class(
body_schema: dict[str, Any],
*,
parent_class_name: str,
) -> tuple[str | None, GeneratedClass | None, str | None]:
"""Detect a body's array-of-objects field and emit its item class.
Parameters
----------
body_schema : dict
The request_body_schema.
parent_class_name : str
Camel-case base for the generated item class name.
Returns
-------
tuple
``(field_name, item_class, type_annotation)``.
"""
if body_schema.get("type") == "array":
items = body_schema.get("items")
if isinstance(items, dict):
item_class_name = class_name_from(parent_class_name, "BodyItem")
item_class = generate_class(
items,
class_name=item_class_name,
base_class="Data",
docstring=f"Body item for {parent_class_name}.",
)
return ("data", item_class, f"list[{item_class_name}]")
return (None, None, None)
properties = body_schema.get("properties") or {}
for name, schema in properties.items():
if not isinstance(schema, dict):
continue
if schema.get("type") != "array":
continue
items = schema.get("items")
if not isinstance(items, dict):
continue
if items.get("type") != "object" and "properties" not in items:
continue
item_class_name = class_name_from(parent_class_name, "BodyItem")
item_class = generate_class(
items,
class_name=item_class_name,
base_class="Data",
docstring=f"Body item for {parent_class_name}.",
)
return (name, item_class, f"list[{item_class_name}]")
return (None, None, None)
def _data_schema(cmd_spec: dict[str, Any]) -> dict[str, Any]:
"""Pull the per-row data schema out of a response_schema (same logic as fetcher_gen)."""
response = cmd_spec.get("response_schema") or {}
if not isinstance(response, dict):
return {}
properties = response.get("properties") or {}
results = properties.get("results")
if isinstance(results, dict):
non_null = [
v
for v in results.get("anyOf") or [results]
if isinstance(v, dict) and v.get("type") != "null"
]
if non_null:
inner = non_null[0]
if inner.get("type") == "array" and isinstance(inner.get("items"), dict):
inner = inner["items"]
if isinstance(inner.get("oneOf"), list):
concrete = [c for c in inner["oneOf"] if isinstance(c, dict)]
if concrete:
inner = concrete[0]
return inner
return _unwrap_schema_envelopes(response)
def _unwrap_schema_envelopes(schema: Any) -> dict[str, Any]:
"""Strip schema envelopes that ``unpack_response`` strips at runtime."""
if not isinstance(schema, dict):
return {}
typed: dict[str, Any] = schema
if typed.get("type") == "array":
return _unwrap_schema_envelopes(typed.get("items"))
combinator_target = _first_non_null_combinator_variant(typed)
if combinator_target is not None:
return _unwrap_schema_envelopes(combinator_target)
if _is_scalar_schema(typed):
return _wrap_scalar_as_value(typed)
props = typed.get("properties")
if not isinstance(props, dict) or not props:
return typed
inner = _envelope_inner_schema(typed, props)
if inner is not None:
return _unwrap_schema_envelopes(inner)
return typed
def _first_non_null_combinator_variant(schema: dict[str, Any]) -> dict[str, Any] | None:
"""Return the first non-null variant of a top-level ``oneOf`` / ``anyOf``."""
for combinator in ("oneOf", "anyOf"):
variants = schema.get(combinator)
if not isinstance(variants, list):
continue
for variant in variants:
if isinstance(variant, dict) and variant.get("type") != "null":
return cast(dict[str, Any], variant)
return None
def _envelope_inner_schema(
typed: dict[str, Any], # noqa: ARG001
props: dict[str, Any],
) -> dict[str, Any] | None:
"""Resolve the inner row schema when ``typed`` is a recognized envelope."""
if len(props) == 1:
only_value = next(iter(props.values()))
if isinstance(only_value, dict):
only_dict = cast(dict[str, Any], only_value)
if only_dict.get("type") == "array":
items = only_dict.get("items")
return cast(dict[str, Any], items) if isinstance(items, dict) else None
if (
only_dict.get("properties")
or only_dict.get("oneOf")
or only_dict.get("anyOf")
):
return only_dict
array_keys = [
k for k, v in props.items() if isinstance(v, dict) and v.get("type") == "array"
]
# Only resolve as an envelope if there is exactly one list of all dicts,
# or if the list is the only element present.
if len(array_keys) == 1:
items = props[array_keys[0]].get("items")
if isinstance(items, dict) and not _is_scalar_schema(items):
return cast(dict[str, Any], items)
return None
def _is_scalar_schema(schema: dict[str, Any]) -> bool:
"""Return True when the schema represents a primitive (string/number/etc.)."""
return schema.get("type") in {
"string",
"integer",
"number",
"boolean",
} and not schema.get("properties")
def _wrap_scalar_as_value(schema: dict[str, Any]) -> dict[str, Any]:
"""Promote a scalar schema to ``{type: object, properties: {value: scalar}}``."""
return {
"type": "object",
"properties": {"value": schema},
"required": ["value"],
}
def _credential_lookup_lines(
creds: dict[str, dict[str, str]], provider_name: str
) -> list[str]:
"""Render the credential extraction block for the POST router body.
Parameters
----------
creds : dict
Mapping from canonical credential names to ``{name, in}`` info.
provider_name : str
Snake-case provider name.
Returns
-------
list of str
Indented lines that pull each credential into ``_cred_<canonical>`` locals.
"""
if not creds:
return []
lines: list[str] = [" _user_creds = cc.user_settings.credentials"]
for canonical in sorted(creds):
var = f"_cred_{canonical}"
attr = f"{provider_name}_{canonical}"
lines.append(f' {var} = (getattr(_user_creds, "{attr}", "") or "")')
return lines
def _signature_params(
cmd_spec: dict[str, Any],
array_field: str | None,
array_annotation: str | None,
has_credentials: bool,
) -> list[tuple[str, str, str | None, bool, Any]]:
"""Build the function parameter list (name, annotation, description, required, default).
Parameters
----------
cmd_spec : dict
Spec command entry.
array_field : str, optional
Name of the body's array-of-objects field.
array_annotation : str, optional
The Python annotation for that field.
has_credentials : bool
Whether the command needs credentials.
Returns
-------
list of tuple
Per-parameter tuples in declaration order.
"""
out: list[tuple[str, str, str | None, bool, Any]] = []
if has_credentials:
out.append(
(
"cc",
"CommandContext",
"Provides access to user-configured credentials.",
True,
None,
)
)
if array_field and array_annotation:
out.append((array_field, array_annotation, "Body data rows.", True, None))
body_props = (cmd_spec.get("request_body_schema") or {}).get("properties") or {}
body_required = set(
(cmd_spec.get("request_body_schema") or {}).get("required") or []
)
# A name can be declared both as an operation parameter and as a request-body
# property — Codat's push endpoints put ``accountId`` in the URL template and
# in the body. Emitting both produces two parameters with the same name, which
# is a SyntaxError. The operation parameter wins: it carries the path
# placeholder and its own ``required``. The payload is unaffected, because
# ``_render_body_block`` reads the body fields from ``request_body_schema``
# rather than from this list.
#
# Two wire names can also collapse onto one identifier once sanitized
# (``from`` and ``from_``, ``Organization-Id`` and ``Organization_Id``), and
# ``operation_parameters`` keeps a name declared twice under different ``in``
# locations. Collisions are therefore resolved on the emitted identifier, not
# on the wire name, so a duplicate argument cannot be generated at all.
operation_params = [
p for p in filter_user_params(cmd_spec.get("parameters") or []) if p.get("name")
]
operation_param_names = {p["name"] for p in operation_params}
operation_param_names |= {safe_field_name(p["name"])[0] for p in operation_params}
emitted: set[str] = {entry[0] for entry in out}
for name, schema in body_props.items():
if name == array_field:
continue
if not isinstance(schema, dict):
continue
safe = safe_field_name(name)[0]
if safe in operation_param_names or safe in emitted:
continue
emitted.add(safe)
ann = _python_type_from_param(
{
"type": schema.get("type"),
"is_list": schema.get("type") == "array",
"choices": schema.get("enum"),
}
)
out.append(
(
# ``_render_body_block`` already writes ``{'from': from_}``, keying the
# payload by the wire name and reading the safe identifier, so the
# signature has to declare the safe one. Codat's Transfer body has a
# property named ``from``, which is a keyword.
safe,
ann,
schema.get("description") or schema.get("title"),
name in body_required,
schema.get("default"),
)
)
for raw in operation_params:
name = raw["name"]
safe = safe_field_name(name)[0]
if safe in emitted:
continue
emitted.add(safe)
ann = _python_type_from_param(raw)
out.append(
(
# Wire names are not always identifiers. Rebilly declares
# ``Organization-Id`` and a ``REB-APIKEY`` security header, which
# emitted ``REB-APIKEY: str = None``. The fetcher path already
# sanitizes these; this one has to match it.
safe,
ann,
raw.get("help"),
bool(raw.get("required")),
raw.get("default"),
)
)
out.sort(key=lambda entry: (0 if entry[0] == "cc" else 1, 0 if entry[3] else 1))
return out
def _render_signature(
func_name: str,
params: list[tuple[str, str, str | None, bool, Any]],
return_annotation: str,
) -> str:
"""Render ``async def func_name(...) -> return_annotation:`` lines.
Parameters
----------
func_name : str
Generated function identifier.
params : list of tuple
Output of ``_signature_params``.
return_annotation : str
The return type annotation text.
Returns
-------
str
Multi-line signature ready for the function body to follow.
"""
if not params:
return f"async def {func_name}() -> {return_annotation}:"
lines = [f"async def {func_name}("]
for name, annotation, _, required, default in params:
if required:
lines.append(f" {name}: {annotation},")
else:
default_repr = render_default(default)
lines.append(f" {name}: {annotation} = {default_repr},")
lines.append(f") -> {return_annotation}:")
return "\n".join(lines)
def render_default(default: Any) -> str:
"""Render a Python literal for a default value.
Parameters
----------
default : Any
The literal default.
Returns
-------
str
The literal text usable on the right-hand side of ``= default``.
"""
if default is None:
return "None"
if isinstance(default, bool):
return "True" if default else "False"
return repr(default)
def generate_post_command_module(spec: PostCommandSpec) -> GeneratedPostCommand:
"""Render a POST command module for one spec command.
Parameters
----------
spec : PostCommandSpec
Per-command inputs (see ``PostCommandSpec``).
Returns
-------
GeneratedPostCommand
The emitted module source plus metadata the package builder
needs to register the command in the router.
"""
module_name = _module_name_from_command(spec.name)
base_class_name = class_name_from(spec.name)
function_name = module_name
data_class = f"{base_class_name}Data"
body_field, body_item_class, body_annotation = _array_item_class(
spec.cmd_spec.get("request_body_schema") or {},
parent_class_name=base_class_name,
)
data_schema = _data_schema(spec.cmd_spec) or {"type": "object"}
data_generated = generate_class(
data_schema,
class_name=data_class,
base_class="BaseModel",
docstring=f"Response row for {spec.name}.",
)
creds = credentials_from_command(spec.cmd_spec)
cred_lines = _credential_lookup_lines(creds, spec.provider_name)
params = _signature_params(
spec.cmd_spec, body_field, body_annotation, has_credentials=bool(creds)
)
return_annotation = f"OBBject[list[{data_class}]]"
signature = _render_signature(function_name, params, return_annotation)
path_params = _path_template_keys(spec.cmd_spec.get("url_path") or "")
full_url_path = _resolve_url_path(
spec.api_prefix, spec.cmd_spec.get("url_path") or ""
)
body_block = _render_body_block(
cmd_spec=spec.cmd_spec,
body_field=body_field,
creds=creds,
path_params=path_params,
url_path_template=full_url_path,
base_url=spec.base_url.rstrip("/"),
cred_lines=cred_lines,
data_class=data_class,
params=params,
)
imports = {
"import json as _json",
"from typing import Any",
"from openbb_core.app.model.abstract.error import OpenBBError",
"from openbb_core.app.model.obbject import OBBject",
"from openbb_core.provider.abstract.data import Data",
"from openbb_core.provider.utils.helpers import "
"get_async_requests_session, get_querystring",
"from pydantic import BaseModel, Field",
"from ....utils import unpack_response",
}
if creds:
imports.add("from openbb_core.app.model.command_context import CommandContext")
imports.update(data_generated.collect_imports())
if body_item_class:
imports.update(body_item_class.collect_imports())
if any(p[1].startswith("Literal[") for p in params):
imports.add("from typing import Literal")
sorted_imports = consolidate_imports(imports)
description = (spec.cmd_spec.get("description") or f"POST {spec.name}.").strip()
summary = description.split("\n", 1)[0]
parts: list[str] = [
f'"""POST router function for {spec.name} — generated from spec.\n\n'
f"Hits ``{spec.base_url.rstrip('/')}{full_url_path}`` via HTTP POST.\n"
f'"""',
"",
]
stdlib = [
i
for i in sorted_imports
if i.startswith(
("from typing", "from datetime", "import typing", "import datetime")
)
]
third_party = [i for i in sorted_imports if i not in stdlib]
if stdlib:
parts.extend(stdlib)
parts.append("")
if third_party:
parts.extend(third_party)
parts.append("")
parts.append("")
if body_item_class:
for cls in body_item_class.flatten():
parts.append(cls.source)
for cls in data_generated.flatten():
parts.append(cls.source)
cleaned_summary = (description or summary).strip()
if not cleaned_summary:
cleaned_summary = summary
if not cleaned_summary.endswith((".", "!", "?")):
cleaned_summary += "."
if "\n" in cleaned_summary:
first, _, rest = cleaned_summary.partition("\n")
rest_indented = "\n".join(
f" {line}" if line.strip() else "" for line in rest.split("\n")
)
func_doc = f' """{first}\n\n{rest_indented}\n """'
else:
func_doc = f' """{cleaned_summary}"""'
parts.append(f"{signature}\n{func_doc}\n{body_block}\n")
return GeneratedPostCommand(
module_name=module_name,
function_name=function_name,
body_class=body_item_class.name if body_item_class else None,
data_class=data_class,
source="\n".join(parts).rstrip() + "\n",
credentials_used=creds,
)
def _render_body_block(
*,
cmd_spec: dict[str, Any],
body_field: str | None,
creds: dict[str, dict[str, str]],
path_params: list[str],
url_path_template: str,
base_url: str,
cred_lines: list[str],
data_class: str,
params: list[tuple[str, str, str | None, bool, Any]],
) -> str:
"""Render the function body for a POST command.
Parameters
----------
cmd_spec : dict
The spec command entry.
body_field : str, optional
Name of the array-of-objects body field, if any.
creds : dict
Credential entries from ``credentials_from_command``.
path_params : list of str
URL-template placeholders to substitute.
url_path_template : str
URL path with placeholders intact.
base_url : str
Upstream API root, no trailing slash.
cred_lines : list of str
Credential-extraction source lines.
data_class : str
Name of the response Data class.
params : list of tuple
Function parameters.
Returns
-------
str
Full function body.
"""
body_props = (cmd_spec.get("request_body_schema") or {}).get("properties") or {}
body_field_names = list(body_props)
# The signature declares the safe identifier for a body property, so the
# body/query split has to compare against those, not the wire names.
# Otherwise a keyword-named property such as ``from`` is not recognized as a
# body field and leaks into the query string as ``from_``.
body_safe_names = {safe_field_name(n)[0] for n in body_field_names}
query_field_names = [
n for n, _, _, _, _ in params if n != "cc" and n not in body_safe_names
]
# The signature carries safe identifiers, but the request has to go out under
# the wire names, so keep a map back for the query keys and the path check.
wire_by_safe = {
safe_field_name(p["name"])[0]: p["name"]
for p in filter_user_params(cmd_spec.get("parameters") or [])
if p.get("name")
}
lines: list[str] = []
lines.extend(cred_lines)
if path_params:
sub_args = ", ".join(f"{p}={p}" for p in path_params)
lines.append(f" _path = {url_path_template!r}.format({sub_args})")
else:
lines.append(f" _path = {url_path_template!r}")
lines.append(" _query_dict: dict[str, Any] = {}")
for name in query_field_names:
wire = wire_by_safe.get(name, name)
if wire in path_params or name in path_params:
continue
lines.append(f" if {name} is not None:")
lines.append(f" _query_dict[{wire!r}] = {name}")
for canonical, info in creds.items():
if info["in"] != "query":
continue
wire_name = info["name"]
var = f"_cred_{canonical}"
lines.append(f" if {var}:")
lines.append(f" _query_dict[{wire_name!r}] = {var}")
lines.append(" _query_string = get_querystring(_query_dict, [])")
lines.append(
f' _url = f"{base_url}{{_path}}" + ("?" + _query_string if _query_string else "")'
)
lines.append(" _headers: dict[str, str] = {}")
for canonical, info in creds.items():
if info["in"] != "header":
continue
wire_name = info["name"]
var = f"_cred_{canonical}"
lines.append(f" if {var}:")
lines.append(f" _headers[{wire_name!r}] = {var}")
lines.append(" _cookies: dict[str, str] = {}")
for canonical, info in creds.items():
if info["in"] != "cookie":
continue
wire_name = info["name"]
var = f"_cred_{canonical}"
lines.append(f" if {var}:")
lines.append(f" _cookies[{wire_name!r}] = {var}")
body_top_is_array = (cmd_spec.get("request_body_schema") or {}).get(
"type"
) == "array"
if body_top_is_array and body_field:
safe = safe_field_name(body_field)[0]
lines.append(
" _body: list[dict[str, Any]] = "
f'[_item.model_dump(mode="json") for _item in {safe}]'
)
else:
body_entries: list[str] = []
for name in body_field_names:
safe = safe_field_name(name)[0]
if name == body_field:
body_entries.append(
f' {name!r}: [_item.model_dump(mode="json") for _item in {safe}],'
)
else:
body_entries.append(f" {name!r}: {safe},")
if body_entries:
body_lines = "\n".join(body_entries)
lines.append(f" _body: dict[str, Any] = {{\n{body_lines}\n }}")
else:
lines.append(" _body: dict[str, Any] = {}")
lines.append(" _rows: list[dict[str, Any]] = []")
lines.append(" _metadata: dict[str, Any] = {}")
lines.append(" async with await get_async_requests_session() as _session:")
lines.append(
" async with await _session.request("
'"POST", _url, headers=_headers, cookies=_cookies, json=_body) as _resp:'
)
lines.append(" _ct = (_resp.headers.get('Content-Type') or '').lower()")
lines.append(" _text = await _resp.text()")
lines.append(" if _resp.status >= 400:")
lines.append(
" raise OpenBBError("
"f'HTTP {_resp.status} from {_url}: {_text[:500]}')"
)
lines.append(
" if 'json' in _ct or _text.lstrip().startswith(('{', '[')):"
)
lines.append(" try:")
lines.append(" _payload = _json.loads(_text)")
lines.append(" except ValueError as _exc:")
lines.append(
" raise OpenBBError("
"f'Upstream {_url} returned malformed JSON: {_exc}') from _exc"
)
lines.append(" _rows, _metadata = unpack_response(_payload)")
lines.append(" else:")
lines.append(" _rows = [{'content': _text, 'content_type': _ct}]")
lines.append(f" _typed = [{data_class}(**row) for row in _rows]")
lines.append(" if _metadata:")
lines.append(
' return OBBject(results=_typed, extra={"results_metadata": _metadata})'
)
lines.append(" return OBBject(results=_typed)")
return "\n".join(lines)

View File

@@ -0,0 +1,130 @@
"""Generate ``pyproject.toml`` and the package skeleton files."""
from __future__ import annotations
from dataclasses import dataclass
@dataclass
class GeneratedProject:
"""Output of ``generate_pyproject``.
Parameters
----------
project_name : str
PyPI distribution name.
package_name : str
Snake-case Python package name.
source : str
Full ``pyproject.toml`` contents.
"""
project_name: str
package_name: str
source: str
def generate_pyproject(
*,
project_name: str,
package_name: str,
providers: list[str],
routers: list,
description: str,
version: str = "0.1.0",
spec_provenance: dict[str, str] | None = None,
) -> GeneratedProject:
"""Render a PEP 621 + Hatchling ``pyproject.toml`` for the bundled extension.
Parameters
----------
project_name : str
PyPI / distribution name.
package_name : str
Python package directory name (snake_case).
providers : list of str
Provider identifiers.
routers : list of GeneratedRouter
Top-level routers.
description : str
Free-form description for ``[project] description``.
version : str
Initial version string.
Returns
-------
GeneratedProject
Project metadata plus the ``pyproject.toml`` source.
"""
provider_entries = "\n".join(
f'{p} = "{package_name}.providers.{p}:{p}_provider"' for p in sorted(providers)
)
router_entries = "\n".join(
f'{r.entry_point_name} = "{package_name}.routers.{r.module_name}:router"'
for r in routers
if r.entry_point_name
)
provenance_block = _render_provenance_block(spec_provenance or {})
body = f"""[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[project]
name = "{project_name}"
version = "{version}"
description = {description!r}
requires-python = ">=3.10,<4"
license = {{ text = "Apache-2.0" }}
dependencies = [
"openbb-core>=2.0.0",
]
[tool.hatch.build.targets.wheel]
packages = ["{package_name}"]
[project.entry-points."openbb_provider_extension"]
{provider_entries}
[project.entry-points."openbb_core_extension"]
{router_entries}
{provenance_block}
[tool.ruff]
line-length = 100
target-version = "py310"
[tool.ruff.lint]
select = ["E", "F", "W", "I", "B", "UP", "D"]
ignore = [
"D100", "D101", "D102", "D103", "D104", "D105", "D107",
"D200", "D205", "D212", "D401", "D415",
"E501", "B008",
]
[tool.ruff.lint.pydocstyle]
convention = "numpy"
"""
return GeneratedProject(
project_name=project_name,
package_name=package_name,
source=body,
)
def _render_provenance_block(provenance: dict[str, str]) -> str:
"""Render ``[tool.openbb-codegen]`` (empty string when no provenance)."""
keys = (
"source_url",
"api_version",
"generator",
"generated_at",
"spec_version",
"spec_sha256",
)
entries = [(k, provenance[k]) for k in keys if provenance.get(k)]
if not entries:
return ""
body = "\n".join(f"{k} = {v!r}" for k, v in entries)
return f"\n[tool.openbb-codegen]\n{body}\n"

View File

@@ -0,0 +1,107 @@
"""Generate the ``providers/<provider_name>/__init__.py`` registration module."""
from __future__ import annotations
from dataclasses import dataclass
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from openbb_cli.codegen.fetcher_gen import GeneratedFetcher
@dataclass
class GeneratedProvider:
"""Output of ``generate_provider_module``.
Parameters
----------
package_name : str
Snake-case package name.
provider_name : str
Snake-case provider identifier.
source : str
Full source for ``providers/<provider_name>/__init__.py``.
credential_keys : list of str
Canonical credential names declared on the provider.
"""
package_name: str
provider_name: str
source: str
credential_keys: list[str]
def generate_provider_module(
package_name: str,
provider_name: str,
description: str,
website: str,
fetchers: list[GeneratedFetcher],
) -> GeneratedProvider:
"""Render the provider's ``__init__.py``.
Parameters
----------
package_name : str
Snake-case top-level package name.
provider_name : str
Snake-case provider identifier.
description : str
Provider description shown in OpenBB introspection.
website : str
Provider home page URL.
fetchers : list of GeneratedFetcher
Per-command fetchers.
Returns
-------
GeneratedProvider
Module source plus the credential-key list.
"""
cred_set: set[str] = set()
for f in fetchers:
cred_set.update(f.credentials_used)
credential_keys = sorted(cred_set)
parts: list[str] = []
parts.append(
f'"""Provider registration for {provider_name} — generated from spec."""'
)
parts.append("")
parts.append("from openbb_core.provider.abstract.provider import Provider")
parts.append("")
for f in fetchers:
parts.append(
f"from {package_name}.providers.{provider_name}.models."
f"{f.module_name} import {f.fetcher_class}"
)
parts.append("")
parts.append("")
fetcher_dict_lines = [f' "{f.model_name}": {f.fetcher_class},' for f in fetchers]
if credential_keys:
cred_repr = (
"[\n "
+ ",\n ".join(repr(k) for k in credential_keys)
+ ",\n ]"
)
cred_block = f" credentials={cred_repr},\n"
else:
cred_block = ""
parts.append(
f"{provider_name}_provider = Provider(\n"
f' name="{provider_name}",\n'
f" description={description!r},\n"
f"{cred_block}"
f' website="{website}",\n'
" fetcher_dict={\n" + "\n".join(fetcher_dict_lines) + "\n },\n)\n"
)
return GeneratedProvider(
package_name=package_name,
provider_name=provider_name,
source="\n".join(parts),
credential_keys=credential_keys,
)

View File

@@ -0,0 +1,710 @@
"""Convert OpenAPI JSON-schema fragments into Pydantic class source code."""
from __future__ import annotations
import keyword
import re
from dataclasses import dataclass, field
from typing import Any
_PRIMITIVE_TYPES: dict[str, str] = {
"string": "str",
"integer": "int",
"number": "float",
"boolean": "bool",
}
_STRING_FORMATS: dict[str, str] = {
"date": "datetime.date",
"date-time": "datetime.datetime",
}
_PYTHON_RESERVED_FIELD_NAMES = frozenset(
{"model_config", "model_fields", "model_dump", "model_validate"}
)
@dataclass
class GeneratedClass:
"""One emitted Pydantic class plus any nested classes it depends on."""
name: str
source: str
nested: list[GeneratedClass] = field(default_factory=list)
imports: set[str] = field(default_factory=set)
def flatten(self) -> list[GeneratedClass]:
"""Return ``[nested..., self]`` in topological order, deduped by name."""
seen: set[str] = set()
out: list[GeneratedClass] = []
for child in self.nested:
for grandchild in child.flatten():
if grandchild.name not in seen:
seen.add(grandchild.name)
out.append(grandchild)
if self.name not in seen:
out.append(self)
return out
def collect_imports(self) -> set[str]:
"""Union of ``imports`` across this class and every descendant."""
out = set(self.imports)
for child in self.nested:
out.update(child.collect_imports())
return out
def safe_field_name(name: str) -> tuple[str, bool]:
"""Normalize a JSON property name to a valid Python identifier.
Returns ``(safe_name, needs_alias)``.
"""
cleaned = re.sub(r"[^0-9a-zA-Z_]", "_", name).lstrip("_")
if not cleaned:
cleaned = "field"
if cleaned[0].isdigit():
cleaned = f"f_{cleaned}"
if keyword.iskeyword(cleaned) or cleaned in _PYTHON_RESERVED_FIELD_NAMES:
cleaned = f"{cleaned}_"
return (cleaned, cleaned != name)
def class_name_from(*parts: str) -> str:
"""Join + camel-case a set of name parts into a Python class identifier."""
words: list[str] = []
for part in parts:
if not part:
continue
for word in re.split(r"[^0-9a-zA-Z]+", part):
if word:
words.append(word[:1].upper() + word[1:])
name = "".join(words) or "Anonymous"
if name[0].isdigit():
name = f"_{name}"
return name
def schema_to_type( # noqa: PLR0911
schema: Any,
*,
parent_class_name: str,
field_name: str,
nested_classes: list[GeneratedClass],
imports: set[str],
) -> str:
"""Resolve a JSON schema to a Python type annotation string."""
if not isinstance(schema, dict):
imports.add("from typing import Any")
return "Any"
if "$ref" in schema:
imports.add("from typing import Any")
return "dict[str, Any]"
any_of = schema.get("anyOf") or schema.get("oneOf")
if any_of:
return _resolve_union(
any_of,
parent_class_name=parent_class_name,
field_name=field_name,
nested_classes=nested_classes,
imports=imports,
)
enum = schema.get("enum")
if enum:
return _enum_primitive(enum, schema.get("type"), imports)
if "const" in schema:
return _enum_primitive([schema["const"]], schema.get("type"), imports)
schema_type = schema.get("type")
if schema_type == "array":
inner = schema_to_type(
schema.get("items") or {},
parent_class_name=parent_class_name,
field_name=field_name + "_item",
nested_classes=nested_classes,
imports=imports,
)
return f"list[{inner}]"
if schema_type == "object" or "properties" in schema:
return _resolve_object(
schema,
parent_class_name=parent_class_name,
field_name=field_name,
nested_classes=nested_classes,
imports=imports,
)
if schema_type in _PRIMITIVE_TYPES:
py_type = _PRIMITIVE_TYPES[schema_type]
if py_type == "str":
fmt = schema.get("format")
if fmt in _STRING_FORMATS:
qualified = _STRING_FORMATS[fmt]
imports.add("import datetime")
return qualified
return py_type
imports.add("from typing import Any")
return "Any"
def _enum_primitive(
values: list[Any],
declared_type: Any, # noqa: ARG001
imports: set[str],
) -> str:
"""Map an ``enum`` / ``const`` to a ``Literal[...]`` annotation."""
if not values:
imports.add("from typing import Any")
return "Any"
rendered_members = ", ".join(repr(v) for v in values)
imports.add("from typing import Literal")
return f"Literal[{rendered_members}]"
def _resolve_union(
variants: list[Any],
*,
parent_class_name: str,
field_name: str,
nested_classes: list[GeneratedClass],
imports: set[str],
) -> str:
"""Resolve an ``anyOf`` / ``oneOf`` list into a Python annotation."""
non_null = [
v for v in variants if not (isinstance(v, dict) and v.get("type") == "null")
]
is_optional = len(non_null) != len(variants)
if not non_null:
imports.add("from typing import Any")
return "Any"
rendered: list[str] = []
for variant in non_null:
rendered.append(
schema_to_type(
variant,
parent_class_name=parent_class_name,
field_name=field_name,
nested_classes=nested_classes,
imports=imports,
)
)
seen: set[str] = set()
unique: list[str] = []
for r in rendered:
if r not in seen:
seen.add(r)
unique.append(r)
inner = unique[0] if len(unique) == 1 else " | ".join(unique)
if is_optional:
return f"{inner} | None"
return inner
def _resolve_object(
schema: dict[str, Any],
*,
parent_class_name: str,
field_name: str,
nested_classes: list[GeneratedClass],
imports: set[str],
) -> str:
"""Either generate a nested class or fall back to ``dict[str, Any]``."""
properties = schema.get("properties") or {}
if not properties:
ap = schema.get("additionalProperties")
if isinstance(ap, dict):
value_type = schema_to_type(
ap,
parent_class_name=parent_class_name,
field_name=field_name + "_value",
nested_classes=nested_classes,
imports=imports,
)
return f"dict[str, {value_type}]"
imports.add("from typing import Any")
return "dict[str, Any]"
nested_name = schema.get("title") or class_name_from(parent_class_name, field_name)
nested_name = class_name_from(nested_name)
nested = generate_class(
schema,
class_name=nested_name,
docstring=schema.get("description"),
)
nested_classes.append(nested)
return nested_name
def render_field_default(default: Any) -> str:
"""Render a JSON default value as a Python literal for ``Field(default=...)``."""
if isinstance(default, bool):
return "True" if default else "False"
if isinstance(default, (int, float)):
return repr(default)
if isinstance(default, str):
return repr(default)
if default is None:
return "None"
if isinstance(default, (list, tuple, dict)):
return repr(default)
return repr(default)
_LINE_LENGTH_BUDGET = 88
def _build_field_call(
*,
description: str | None,
default: Any,
has_default: bool,
required: bool,
alias: str | None,
leading_width: int = 0,
) -> str:
"""Compose the ``Field(...)`` text used after ``field_name: type =``.
Parameters
----------
description : str, optional
The field description.
default : Any
The literal default value, or ``None`` when ``has_default`` is False.
has_default : bool
Whether the schema specified an explicit default.
required : bool
Whether the field is required.
alias : str, optional
On-the-wire JSON name when it differs from the Python identifier.
leading_width : int
Number of characters consumed before the ``Field(...)`` call on the line.
Returns
-------
str
The full ``Field(...)`` source.
"""
args: list[str] = []
if has_default and not required:
args.append(f"default={render_field_default(default)}")
elif not required:
args.append("default=None")
if alias is not None:
args.append(f"alias={alias!r}")
if description is not None:
args.append(f"description={_python_string_literal(description)}")
if not args:
return ""
single_line = "Field(" + ", ".join(args) + ")"
if leading_width + len(single_line) <= _LINE_LENGTH_BUDGET:
return single_line
inner = ",\n ".join(args)
return f"Field(\n {inner},\n )"
def _resolve_field_description(
prop: dict[str, Any],
field_name: str | None = None, # noqa: ARG001
) -> str:
"""Pick the description text for a Pydantic Field."""
text = prop.get("description") or prop.get("title")
parts: list[str] = []
if text:
parts.append(text.rstrip(". ") + ".")
elif prop.get("example") is not None:
parts.append(f"Example: {prop['example']}.")
enum = prop.get("enum") or (
[prop["const"]]
if isinstance(prop.get("const"), (str, int, float, bool))
else None
)
if enum:
rendered = ", ".join(str(v) for v in enum)
parts.append(f"Choices: {rendered}.")
return " ".join(parts).rstrip()
def _nested_object_breakout(prop: dict[str, Any]) -> str:
"""Render an inline breakout of a nested object's fields."""
target = prop
if isinstance(prop.get("items"), dict):
target = prop["items"]
for combinator in ("oneOf", "anyOf"):
variants = target.get(combinator)
if isinstance(variants, list):
for variant in variants:
if isinstance(variant, dict) and isinstance(
variant.get("properties"), dict
):
target = variant
break
properties = target.get("properties")
if not isinstance(properties, dict) or not properties:
return ""
summaries: list[str] = []
for inner_name, inner_sch in properties.items():
if not isinstance(inner_sch, dict):
summaries.append(inner_name)
continue
inner_type = _quick_schema_type(inner_sch)
summaries.append(f"{inner_name} ({inner_type})")
label = (
"Inner item fields" if isinstance(prop.get("items"), dict) else "Inner fields"
)
return f"{label}: {', '.join(summaries)}."
def _quick_schema_type(schema: dict[str, Any]) -> str:
"""Best-effort shorthand type label for breakout text."""
if isinstance(schema.get("enum"), list):
return "str"
schema_type = schema.get("type")
if schema_type == "array":
items = schema.get("items")
inner = "Any"
if isinstance(items, dict):
inner = _quick_schema_type(items)
return f"list[{inner}]"
if schema_type == "object" or "properties" in schema:
title = schema.get("title")
return title or "object"
if isinstance(schema_type, str):
primitive_type = _PRIMITIVE_TYPES.get(schema_type)
if primitive_type is not None:
if schema_type == "string":
fmt = schema.get("format")
if fmt == "date":
return "date"
if fmt == "date-time":
return "datetime"
return primitive_type
return "Any"
def _python_string_literal(text: str) -> str:
"""Render a docstring-safe string literal that survives multi-line text.
Parameters
----------
text : str
The raw string to embed in generated source.
Returns
-------
str
A quoted Python literal.
"""
if "\n" in text or "'" in text and '"' in text:
escaped = text.replace("\\", "\\\\").replace('"""', '\\"\\"\\"')
return f'"""{escaped}"""'
return repr(text)
def _annotation_for_docstring(annotation: str) -> tuple[str, bool]:
"""Strip ``| None`` from a type annotation for Numpy-style ``:`` lines.
Parameters
----------
annotation : str
The Python type annotation.
Returns
-------
tuple of (str, bool)
The bare type without ``| None`` suffix, and an optional flag.
"""
suffix = " | None"
if annotation.endswith(suffix):
return (annotation[: -len(suffix)].strip(), True)
return (annotation, False)
def _render_numpy_parameters(
fields: list[tuple[str, str, str | None, bool, Any, bool]],
) -> str:
"""Build a Numpy-style ``Parameters`` block for a class / function docstring.
Parameters
----------
fields : list of tuple
Each tuple is ``(name, annotation, description, has_default, default, optional)``.
Returns
-------
str
The full block.
"""
if not fields:
return ""
lines = ["Parameters", "----------"]
for name, annotation, description, has_default, default, optional in fields:
bare, _ = _annotation_for_docstring(annotation)
type_label = f"{bare}, optional" if optional else bare
lines.append(f"{name} : {type_label}")
body = description or ""
if has_default and default is not None:
suffix = f"(default: {render_field_default(default)})"
body = f"{body} {suffix}".strip() if body else suffix
if body:
for piece in body.split("\n"):
lines.append(f" {piece}")
return "\n".join(lines)
def _render_class_docstring(
summary: str,
fields: list[tuple[str, str, str | None, bool, Any, bool]],
) -> str:
"""Render a Numpy-style class docstring and indent every line by four spaces.
Parameters
----------
summary : str
The summary sentence shown immediately after the opening quotes.
fields : list of tuple
Tuples consumed by ``_render_numpy_parameters``.
Returns
-------
str
The full triple-double-quoted block.
"""
summary_clean = summary.strip()
if not summary_clean.endswith("."):
summary_clean += "."
lines = [f' """{summary_clean}']
params_block = _render_numpy_parameters(fields)
if params_block:
lines.append("")
for line in params_block.split("\n"):
lines.append(f" {line}" if line else "")
lines.append(' """')
return "\n".join(lines)
def render_function_docstring(
summary: str,
*,
parameters: list[tuple[str, str, str | None]] | None = None,
returns: tuple[str, str | None] | None = None,
indent: str = " ",
) -> str:
"""Render a Numpy-style function / method docstring.
Parameters
----------
summary : str
The one-line summary sentence.
parameters : list of (name, type, description) tuples, optional
Each tuple becomes one entry under the ``Parameters`` section.
returns : tuple of (type, description), optional
When supplied, emits a ``Returns`` section.
indent : str
The indentation prefix for every line of the rendered block.
Returns
-------
str
The full triple-double-quoted block.
"""
summary_clean = summary.strip()
if not summary_clean.endswith("."):
summary_clean += "."
lines = [f'{indent}"""{summary_clean}']
if parameters:
lines.append("")
lines.append(f"{indent}Parameters")
lines.append(f"{indent}----------")
for name, type_label, description in parameters:
lines.append(f"{indent}{name} : {type_label}")
if description:
for piece in description.split("\n"):
lines.append(f"{indent} {piece}")
if returns is not None:
return_type, return_desc = returns
lines.append("")
lines.append(f"{indent}Returns")
lines.append(f"{indent}-------")
lines.append(f"{indent}{return_type}")
if return_desc:
for piece in return_desc.split("\n"):
lines.append(f"{indent} {piece}")
lines.append(f'{indent}"""')
return "\n".join(lines)
def generate_class(
schema: dict[str, Any],
*,
class_name: str,
base_class: str = "BaseModel",
docstring: str | None = None,
) -> GeneratedClass:
"""Build a Pydantic class definition from an OpenAPI ``object`` schema."""
properties: dict[str, Any] = schema.get("properties") or {}
required_set: set[str] = set(schema.get("required") or [])
nested_classes: list[GeneratedClass] = []
imports: set[str] = {"from pydantic import BaseModel, Field"}
body_lines: list[str] = []
doc_fields: list[tuple[str, str, str | None, bool, Any, bool]] = []
optional_object_fields: list[str] = []
for raw_name, prop_schema in properties.items():
safe_name, needs_alias = safe_field_name(raw_name)
annotation = schema_to_type(
prop_schema if isinstance(prop_schema, dict) else {},
parent_class_name=class_name,
field_name=safe_name,
nested_classes=nested_classes,
imports=imports,
)
is_required = raw_name in required_set
prop = prop_schema if isinstance(prop_schema, dict) else {}
if not is_required and (prop.get("type") == "object" or "properties" in prop):
optional_object_fields.append(safe_name)
has_default = "default" in prop
if not is_required and "None" not in annotation:
annotation = f"{annotation} | None"
description = _resolve_field_description(prop, field_name=raw_name)
breakout = _nested_object_breakout(prop)
if breakout:
description = (
f"{description} {breakout}".strip() if description else breakout
)
prefix = f" {safe_name}: {annotation} = "
field_call = _build_field_call(
description=description,
default=prop.get("default"),
has_default=has_default,
required=is_required,
alias=raw_name if needs_alias else None,
leading_width=len(prefix),
)
line = f" {safe_name}: {annotation}"
if field_call:
line += f" = {field_call}"
body_lines.append(line)
doc_fields.append(
(
safe_name,
annotation,
description,
has_default,
prop.get("default"),
not is_required,
)
)
if not body_lines:
body_lines.append(" pass")
if optional_object_fields:
# PHP-style serializers emit [] where the schema declares an object
# (empty associative array quirk); [] is never a valid JSON-object value.
imports.add("from pydantic import field_validator")
imports.add("from typing import Any")
names = ", ".join(repr(n) for n in optional_object_fields)
body_lines.append(
f'\n @field_validator({names}, mode="before")\n'
" @classmethod\n"
" def _empty_list_to_none(cls, v: Any) -> Any:\n"
' """Coerce `[]` to None."""\n'
" return None if v == [] else v"
)
summary = docstring or class_name
doc_block = _render_class_docstring(summary, doc_fields)
source = (
f"class {class_name}({base_class}):\n"
f"{doc_block}\n\n" + "\n".join(body_lines) + "\n"
)
return GeneratedClass(
name=class_name,
source=source,
nested=nested_classes,
imports=imports,
)
def consolidate_imports(imports: set[str]) -> list[str]:
"""Merge multiple ``from X import A`` / ``from X import B`` into one line.
Parameters
----------
imports : set of str
Raw import statements.
Returns
-------
list of str
Sorted list with same-module ``from`` imports collapsed.
"""
from_imports: dict[str, set[str]] = {}
bare_imports: set[str] = set()
for raw_line in imports:
line = raw_line.strip()
if line.startswith("from ") and " import " in line:
module, _, names = line[len("from ") :].partition(" import ")
for raw_name in names.split(","):
name = raw_name.strip()
if name:
from_imports.setdefault(module.strip(), set()).add(name)
else:
bare_imports.add(line)
out: list[str] = sorted(bare_imports)
for module in sorted(from_imports):
names = ", ".join(sorted(from_imports[module]))
out.append(f"from {module} import {names}")
return out
def render_module(
cls: GeneratedClass,
*,
module_docstring: str | None = None,
) -> str:
"""Render a ``cls`` and its nested classes as a single importable module."""
classes = cls.flatten()
imports = consolidate_imports(cls.collect_imports())
stdlib: list[str] = []
third_party: list[str] = []
for imp in imports:
if (
imp.startswith("from typing")
or imp.startswith("from datetime")
or imp.startswith("import typing")
or imp.startswith("import datetime")
):
stdlib.append(imp)
else:
third_party.append(imp)
parts: list[str] = []
if module_docstring:
summary = module_docstring.strip()
if not summary.endswith("."):
summary += "."
parts.append(f'"""{summary}"""')
parts.append("")
if stdlib:
parts.extend(stdlib)
parts.append("")
if third_party:
parts.extend(third_party)
parts.append("")
parts.append("")
parts.extend(c.source for c in classes)
return "\n".join(parts).rstrip() + "\n"

View File

@@ -0,0 +1,401 @@
"""Generate router modules that mirror the spec's hierarchical namespace."""
from __future__ import annotations
import re
from dataclasses import dataclass, field
from typing import TYPE_CHECKING
from openbb_cli.codegen.namespace_tree import NamespaceNode
if TYPE_CHECKING:
from openbb_cli.codegen.fetcher_gen import GeneratedFetcher
from openbb_cli.codegen.post_gen import GeneratedPostCommand
@dataclass
class GeneratedRouter:
"""One router module emitted by ``generate_routers``.
Parameters
----------
module_name : str
Snake-case module filename without ``.py``.
entry_point_name : str | None
When set, this router is registered as an ``openbb_core_extension`` entry point.
source : str
Full module source ready to write to ``routers/<module_name>.py``.
"""
module_name: str
entry_point_name: str | None
source: str
@dataclass
class GeneratedRouters:
"""Collection of router modules covering every namespace in the tree.
Parameters
----------
routers : list of GeneratedRouter
All emitted router modules.
root_commands : list of str
Rendered command blocks for top-level leaf commands; they have no
namespace router of their own and mount directly on the root router.
root_post_imports : list of (str, str)
``(module_path, function_name)`` imports the root command blocks need.
root_has_stream : bool
Whether any root command streams (root router imports ``OBBStream``).
"""
routers: list[GeneratedRouter] = field(default_factory=list)
root_commands: list[str] = field(default_factory=list)
root_post_imports: list[tuple[str, str]] = field(default_factory=list)
root_has_stream: bool = False
def _safe_segment(name: str) -> str:
"""Lower-case + replace non-alphanumerics with underscores."""
cleaned = re.sub(r"[^0-9a-zA-Z]+", "_", name).strip("_")
return cleaned.lower() or "ns"
def _module_name_for(path: str) -> str:
"""Convert a dotted namespace path to a router module name.
Parameters
----------
path : str
Dotted namespace path.
Returns
-------
str
Snake-case module identifier.
"""
return "_".join(_safe_segment(p) for p in path.split(".") if p)
def _model_name_for(path: str) -> str:
"""Convert a dotted command path to a CamelCase model identifier."""
return "".join(
seg[:1].upper() + seg[1:]
for seg in (re.split(r"[^0-9a-zA-Z]+", part) for part in path.split("."))
for seg in seg
if seg
)
def generate_routers(
root: NamespaceNode,
*,
package_name: str,
provider_name: str,
fetchers_by_command: dict[str, GeneratedFetcher],
post_commands_by_command: dict[str, GeneratedPostCommand],
) -> GeneratedRouters:
"""Emit router modules for every namespace in ``root``.
Parameters
----------
root : NamespaceNode
The synthetic root node returned by ``build_namespace_tree``.
package_name : str
Top-level Python package name.
provider_name : str
Snake-case provider identifier.
fetchers_by_command : dict
Map ``{dotted_command: GeneratedFetcher}`` for every GET command.
post_commands_by_command : dict
Map ``{dotted_command: GeneratedPostCommand}`` for every POST command.
Returns
-------
GeneratedRouters
One ``GeneratedRouter`` per namespace node.
"""
out = GeneratedRouters()
for top_name, top_node in sorted(root.children.items()):
if top_node.is_namespace:
_emit_router(
top_node,
package_name=package_name,
provider_name=provider_name,
fetchers_by_command=fetchers_by_command,
post_commands_by_command=post_commands_by_command,
collected=out,
is_top_level=True,
)
if top_node.cmd_spec is None:
continue
function_name = _safe_segment(top_name)
description = (top_node.cmd_spec.get("description") or "").strip()
fetcher = fetchers_by_command.get(top_node.full_path)
if fetcher is not None:
renderer = (
_render_stream_command if fetcher.is_streaming else _render_get_command
)
out.root_has_stream = out.root_has_stream or fetcher.is_streaming
out.root_commands.append(
renderer(fetcher, function_name=function_name, description=description)
)
continue
post = post_commands_by_command.get(top_node.full_path)
if post is not None:
out.root_post_imports.append(
(
f"{package_name}.providers.{provider_name}.models."
f"{post.module_name}",
post.function_name,
)
)
out.root_commands.append(
f'router.command(methods=["POST"])(_{post.function_name})\n'
)
return out
def _emit_router(
node: NamespaceNode,
*,
package_name: str,
provider_name: str,
fetchers_by_command: dict[str, GeneratedFetcher],
post_commands_by_command: dict[str, GeneratedPostCommand],
collected: GeneratedRouters,
is_top_level: bool,
) -> str:
"""Recursively emit one router module per namespace node.
Parameters
----------
node : NamespaceNode
The namespace this call emits a router for.
package_name : str
Top-level Python package name.
provider_name : str
Provider identifier for ``providers/<name>/`` imports.
fetchers_by_command : dict
GET-command fetcher modules.
post_commands_by_command : dict
POST-command modules.
collected : GeneratedRouters
Output accumulator.
is_top_level : bool
Whether this namespace is at the top of the tree.
Returns
-------
str
The module name of the emitted router.
"""
module_name = _module_name_for(node.full_path)
parts: list[str] = []
parts.append(f'"""Router for {node.full_path} commands — generated from spec."""')
parts.append("")
parts.append("from openbb_core.app.model.command_context import CommandContext")
parts.append("from openbb_core.app.model.obbject import OBBject")
parts.append(
"from openbb_core.app.provider_interface import "
"ExtraParams, ProviderChoices, StandardParams"
)
parts.append("from openbb_core.app.query import Query")
parts.append("from openbb_core.app.router import Router")
has_stream = any(
(fetcher := fetchers_by_command.get(child.full_path)) is not None
and fetcher.is_streaming
for child in node.children.values()
if child.is_command and child.cmd_spec is not None
)
if has_stream:
parts.append("from openbb_core.app.model.stream import OBBStream")
sub_imports: list[tuple[str, str, str]] = []
for sub_name, sub_node in sorted(node.children.items()):
if not sub_node.is_namespace:
continue
sub_module = _emit_router(
sub_node,
package_name=package_name,
provider_name=provider_name,
fetchers_by_command=fetchers_by_command,
post_commands_by_command=post_commands_by_command,
collected=collected,
is_top_level=False,
)
sub_imports.append((sub_module, _safe_segment(sub_name), sub_node.full_path))
post_imports: list[tuple[str, str]] = []
for child in node.children.values():
if child.is_command and child.cmd_spec is not None:
post = post_commands_by_command.get(child.full_path)
if post is not None:
post_module = (
f"{package_name}.providers.{provider_name}.models."
f"{post.module_name}"
)
post_imports.append((post_module, post.function_name))
if post_imports:
parts.append("")
for module_path, function_name in post_imports:
parts.append(
f"from {module_path} import {function_name} as _{function_name}"
)
if sub_imports:
parts.append("")
for sub_module, sub_segment, _full in sub_imports:
parts.append(
f"from {package_name}.routers.{sub_module} import "
f"router as _{sub_segment}_router"
)
parts.append("")
parts.append("")
parts.append('router = Router(prefix="")')
parts.append("")
for _, sub_segment, _full in sub_imports:
parts.append(
f'router.include_router(_{sub_segment}_router, prefix="/{sub_segment}")'
)
if sub_imports:
parts.append("")
for child_name, child in sorted(node.children.items()):
if not child.is_command or child.cmd_spec is None:
continue
function_name = _safe_segment(child_name)
description = (child.cmd_spec.get("description") or "").strip()
fetcher = fetchers_by_command.get(child.full_path)
if fetcher is not None:
renderer = (
_render_stream_command if fetcher.is_streaming else _render_get_command
)
parts.append(
renderer(
fetcher,
function_name=function_name,
description=description,
)
)
else:
post = post_commands_by_command.get(child.full_path)
if post is not None:
parts.append(
f'router.command(methods=["POST"])(_{post.function_name})\n'
)
source = "\n".join(parts).rstrip() + "\n"
collected.routers.append(
GeneratedRouter(
module_name=module_name,
entry_point_name=node.name if is_top_level else None,
source=source,
)
)
return module_name
def _render_get_command(
fetcher: GeneratedFetcher,
*,
function_name: str,
description: str,
) -> str:
"""Render one ``@router.command(model=...)`` block for a GET endpoint.
Parameters
----------
fetcher : GeneratedFetcher
Per-command fetcher metadata.
function_name : str
Leaf segment of the dotted path used as the function identifier.
description : str
Spec-supplied command description.
Returns
-------
str
The decorator + ``async def`` block.
"""
summary = description or f"{fetcher.model_name} command."
docstring = _format_docstring(summary)
return (
f'@router.command(model="{fetcher.model_name}")\n'
f"async def {function_name}(\n"
" cc: CommandContext,\n"
" provider_choices: ProviderChoices,\n"
" standard_params: StandardParams,\n"
" extra_params: ExtraParams,\n"
") -> OBBject:\n"
f"{docstring}\n"
" return await OBBject.from_query(Query(**locals()))\n"
)
def _render_stream_command(
fetcher: GeneratedFetcher,
*,
function_name: str,
description: str,
) -> str:
"""Render one ``@router.command(model=...)`` block for a streaming endpoint.
Parameters
----------
fetcher : GeneratedFetcher
Per-command fetcher metadata.
function_name : str
Leaf segment of the dotted path used as the function identifier.
description : str
Spec-supplied command description.
Returns
-------
str
The decorator and ``async def`` block returning an OBBStream.
"""
summary = description or f"{fetcher.model_name} stream."
docstring = _format_docstring(summary)
return (
f'@router.command(model="{fetcher.model_name}")\n'
f"async def {function_name}(\n"
" cc: CommandContext,\n"
" provider_choices: ProviderChoices,\n"
" standard_params: StandardParams,\n"
" extra_params: ExtraParams,\n"
") -> OBBStream:\n"
f"{docstring}\n"
" return await OBBStream.from_query(Query(**locals()))\n"
)
def _format_docstring(text: str) -> str:
"""Render ``text`` as an indented function docstring.
Parameters
----------
text : str
Source description from the spec.
Returns
-------
str
Triple-quoted docstring body indented four spaces.
"""
cleaned = text.strip()
if not cleaned:
cleaned = "."
if not cleaned.endswith((".", "!", "?")):
cleaned += "."
if "\n" not in cleaned:
return f' """{cleaned}"""'
first, _, rest = cleaned.partition("\n")
rest_indented = "\n".join(
f" {line}" if line.strip() else "" for line in rest.split("\n")
)
return f' """{first}\n\n{rest_indented}\n """'

View File

@@ -0,0 +1,314 @@
"""Emit per-provider unit-test modules for a generated extension."""
from __future__ import annotations
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any
from openbb_cli.codegen.pydantic_gen import safe_field_name
if TYPE_CHECKING:
from openbb_cli.codegen.fetcher_gen import GeneratedFetcher
_STANDARD_HEADER_REDACTIONS: tuple[tuple[str, str | None], ...] = (
("User-Agent", None),
("Authorization", "Bearer MOCK_TOKEN"),
("X-API-Key", "MOCK_API_KEY"),
("X-Api-Key", "MOCK_API_KEY"),
("apikey", "MOCK_API_KEY"),
("api-key", "MOCK_API_KEY"),
("X-Auth-Token", "MOCK_TOKEN"),
("X-Access-Token", "MOCK_TOKEN"),
("Cookie", "MOCK_COOKIE"),
("Set-Cookie", "MOCK_COOKIE"),
)
_STANDARD_QUERY_REDACTIONS: tuple[tuple[str, str], ...] = (
("apikey", "MOCK_API_KEY"),
("api_key", "MOCK_API_KEY"),
("api-key", "MOCK_API_KEY"),
("key", "MOCK_API_KEY"),
("subscription_key", "MOCK_API_KEY"),
("ocp_apim_subscription_key", "MOCK_API_KEY"),
("token", "MOCK_TOKEN"),
("access_token", "MOCK_TOKEN"),
("auth_token", "MOCK_TOKEN"),
("bearer_token", "MOCK_TOKEN"),
("session_token", "MOCK_TOKEN"),
("client_id", "MOCK_CLIENT_ID"),
("client_secret", "MOCK_CLIENT_SECRET"),
("client_token", "MOCK_TOKEN"),
("app_id", "MOCK_APP_ID"),
("app_key", "MOCK_APP_KEY"),
("app_token", "MOCK_TOKEN"),
("appid", "MOCK_APP_ID"),
("secret", "MOCK_SECRET"),
("secret_key", "MOCK_SECRET"),
("private_key", "MOCK_SECRET"),
("consumer_key", "MOCK_API_KEY"),
("consumer_secret", "MOCK_SECRET"),
)
def _mock_value_for(name: str) -> str:
"""Pick a mock placeholder that matches what kind of credential ``name`` is."""
lower = name.lower().replace("-", "_")
if "secret" in lower:
return "MOCK_SECRET"
if "token" in lower:
return "MOCK_TOKEN"
if "client_id" in lower or lower.endswith("_id") or lower == "appid":
return "MOCK_CLIENT_ID"
return "MOCK_API_KEY"
@dataclass
class GeneratedTestModule:
"""One ``tests/test_<provider>_fetchers.py`` ready to write to disk.
Parameters
----------
module_name : str
Filename without ``.py``.
source : str
Full module source.
"""
module_name: str
source: str
def generate_provider_tests(
*,
package_name: str,
provider_name: str,
fetchers: list[GeneratedFetcher],
commands_by_dotted: dict[str, dict[str, Any]],
) -> GeneratedTestModule | None:
"""Render a unit-test module exercising every fetcher in ``fetchers``.
Parameters
----------
package_name : str
Snake-case top-level package.
provider_name : str
Snake-case provider identifier.
fetchers : list of GeneratedFetcher
The fetchers to test.
commands_by_dotted : dict
Map ``{dotted_command: cmd_spec}``.
Returns
-------
GeneratedTestModule or None
``None`` when there's nothing to test.
"""
if not fetchers:
return None
testable: list[tuple[GeneratedFetcher, dict[str, Any]]] = []
for f in sorted(fetchers, key=lambda x: x.fetcher_class):
dotted = _dotted_for_fetcher(f, commands_by_dotted)
cmd_spec = commands_by_dotted.get(dotted, {}) if dotted else {}
params = _derive_test_params(cmd_spec, provider_name)
if params is None:
continue
testable.append((f, params))
if not testable:
return None
parts: list[str] = [
f'"""Auto-generated unit tests for the {provider_name} provider."""',
"",
"import pytest",
"from openbb_core.app.service.user_service import UserService",
"",
]
for f, _ in testable:
parts.append(
f"from {package_name}.providers.{provider_name}.models."
f"{f.module_name} import {f.fetcher_class}"
)
parts.append("")
parts.append("")
parts.append(
"test_credentials = UserService()"
".default_user_settings.credentials.model_dump(mode='json')"
)
parts.append("")
parts.append("")
header_filters, query_filters = _merge_redactions([f for f, _ in testable])
parts.append('@pytest.fixture(scope="module")')
parts.append("def vcr_config():")
parts.append(' """Scrub credential fields from recorded cassettes."""')
parts.append(" return {")
parts.append(f' "filter_headers": {header_filters!r},')
parts.append(f' "filter_query_parameters": {query_filters!r},')
parts.append(" }")
parts.append("")
parts.append("")
for f, params in testable:
parts.append("@pytest.mark.record_http")
parts.append(
f"def test_{_snake(f.fetcher_class)}(credentials=test_credentials):"
)
parts.append(f" params = {params!r}")
parts.append(f" fetcher = {f.fetcher_class}()")
parts.append(" result = fetcher.test(params, credentials)")
parts.append(" assert result is None")
parts.append("")
parts.append("")
source = "\n".join(parts).rstrip() + "\n"
return GeneratedTestModule(
module_name=f"test_{provider_name}_fetchers",
source=source,
)
def _merge_redactions(
fetchers: list[GeneratedFetcher],
) -> tuple[list[tuple[str, str | None]], list[tuple[str, str]]]:
"""Build (header_filters, query_filters) for the ``vcr_config`` fixture."""
headers: dict[str, str | None] = {}
queries: dict[str, str] = {}
for f in fetchers:
for info in f.credentials_used.values():
wire = info.get("name")
location = info.get("in", "query")
if not wire:
continue
if location == "header":
headers[wire] = _mock_value_for(wire)
elif location == "cookie":
# Cookies travel on the wire as the ``Cookie`` header, which
# ``_STANDARD_HEADER_REDACTIONS`` already blanket-redacts below
# — VCR has no way to filter a single cookie within it.
continue
else:
queries[wire] = _mock_value_for(wire)
for n, v in _STANDARD_HEADER_REDACTIONS:
headers[n] = v
for n, v in _STANDARD_QUERY_REDACTIONS:
queries[n] = v
return (
sorted(headers.items()),
sorted(queries.items()),
)
def _dotted_for_fetcher(
fetcher: GeneratedFetcher,
commands_by_dotted: dict[str, dict[str, Any]],
) -> str | None:
"""Find the dotted command path that produced ``fetcher``."""
target = fetcher.module_name
for dotted in commands_by_dotted:
if _normalize_to_module(dotted) == target:
return dotted
return None
def _normalize_to_module(dotted: str) -> str:
"""Match ``fetcher_gen._module_name_from_command`` exactly."""
import re
safe = re.sub(r"[^0-9a-zA-Z]+", "_", dotted).strip("_").lower()
if not safe:
safe = "command"
if safe[0].isdigit():
safe = f"_{safe}"
return safe
_DATE_PARAM_NAMES: frozenset[str] = frozenset(
{
"date",
"from",
"to",
"fromdate",
"todate",
"start_date",
"end_date",
"startdate",
"enddate",
}
)
def _is_date_param(name: str) -> bool:
"""Heuristic: this param's name implies a date / time range filter."""
lower = name.lower().replace("-", "_")
return lower in _DATE_PARAM_NAMES or "date" in lower
def _derive_test_params(
cmd_spec: dict[str, Any], provider_name: str
) -> dict[str, Any] | None:
"""Build a kwargs dict for the command's testable params, or ``None``."""
out: dict[str, Any] = {}
for raw in cmd_spec.get("parameters") or []:
if not isinstance(raw, dict):
continue
name = raw.get("name")
if not name or name == "provider":
continue
param_providers = raw.get("providers") or []
if param_providers and provider_name not in param_providers:
continue
field_name, _ = safe_field_name(name)
if raw.get("required"):
value = _spec_supplied_value(raw)
if value is None:
return None
out[field_name] = value
elif _is_date_param(name):
out[field_name] = _refresh_date_shape(name, "1970-01-01")
return out
def _spec_supplied_value(param: dict[str, Any]) -> Any:
"""Return a value the spec itself vouches for, or ``None`` to abort the test."""
raw_value: Any = None
if param.get("example") is not None:
raw_value = param["example"]
elif param.get("default") is not None:
raw_value = param["default"]
else:
choices = param.get("choices") or []
if choices:
raw_value = choices[0]
if raw_value is None:
return None
return _refresh_date_shape(param.get("name") or "", raw_value)
def _refresh_date_shape(name: str, value: Any) -> Any:
"""Replace an aged date-shaped string with an equivalent current date."""
import re
from datetime import date, datetime, timedelta, timezone
if not isinstance(value, str):
return value
lower = name.lower()
is_start = "start" in lower or lower in {"from", "fromdate"}
if re.fullmatch(r"\d{4}-\d{2}-\d{2}", value):
today = date.today()
return str(today - timedelta(days=30) if is_start else today)
if re.fullmatch(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z?", value):
now = datetime.now(timezone.utc).replace(microsecond=0)
chosen = (now - timedelta(days=30)) if is_start else now
return chosen.isoformat().replace("+00:00", "Z")
return value
def _snake(camel: str) -> str:
"""``CamelCase`` -> ``snake_case`` for test function names."""
out: list[str] = []
for i, ch in enumerate(camel):
if ch.isupper() and i > 0 and not camel[i - 1].isupper():
out.append("_")
out.append(ch.lower())
return "".join(out)