jerry/OpenBB
1
0
Fork 0
mirror of https://github.com/OpenBB-finance/OpenBB.git synced 2026-11-11 20:01:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

[BugFix] openbb-cli: Add api-prefix and overlay support to spec generation (#7811)

Browse Source 
* feat(cli): add api-prefix and overlay support to spec generation

Add --api-prefix and --overlay options to the spec-generation command so
commands can be named from a URL path prefix and OpenAPI Overlay 1.0
documents can be applied before building the spec. Normalize router
segments so a leading digit gets an underscore and a Python keyword gets
a trailing one, and add mounted_command_paths to map spec paths to the
paths they mount at under obb.

* fix: remove local-only entries from .gitignore

Remove .claude/ and github_metadata/ patterns that only exist locally
and have no business in the shared repository.

---------

Co-authored-by: dlee-bq <325002271+dlee-bq@users.noreply.github.com>
This commit is contained in:
dlee-bq
2026-11-02 04:56:46 +00:00
committed by GitHub
parent 53d4dcc46d
commit 2a392e890f
30 changed files with 304136 additions and 150 deletions
Download Patch File Download Diff File Expand all files Collapse all files

559
cli/openbb_cli/codegen/fetcher_gen.py
View File

@@ -2,6 +2,8 @@
from __future__ import annotations
import json
import keyword
import re
from dataclasses import dataclass, field
from typing import Any, cast
@@ -16,6 +18,7 @@ from openbb_cli.codegen.pydantic_gen import (
consolidate_imports,
generate_class,
query_field_name,
query_params_field_names,
render_function_docstring,
)
@@ -121,7 +124,8 @@ def _module_name_from_command(name: str) -> str:
safe = "command"
if safe[0].isdigit():
safe = f"_{safe}"
return safe.lower()
safe = safe.lower()
return f"{safe}_" if keyword.iskeyword(safe) else safe
_COORDINATE_NAME_RE = re.compile(r"(^|[^a-z])(lat|lon|lng|latitude|longitude)", re.I)
@@ -142,14 +146,14 @@ def _mark_null_empty(node: Any) -> Any:
return [_mark_null_empty(n) for n in node]
if not isinstance(node, dict):
return node
out = {
k: (
v
if k in ("enum", "const", "default", "example", "examples")
else _mark_null_empty(v)
)
for k, v in node.items()
}
out: dict[str, Any] = {}
for k, v in node.items():
if k in ("enum", "const", "default", "example", "examples"):
out[k] = v
elif k in ("properties", "patternProperties") and isinstance(v, dict):
out[k] = {name: _mark_null_empty(schema) for name, schema in v.items()}
else:
out[k] = _mark_null_empty(v)
if out.get("type") == "object" or "properties" in out:
out["x-null-empty"] = True
return out
@@ -209,7 +213,7 @@ def _query_params_schema(
prop_schema = {"type": "string", "x-multiple-items": True}
if description:
prop_schema["description"] = description
if raw.get("choices"):
if raw.get("choices") and raw.get("list_style") != "json":
prop_schema["x-choices"] = list(raw["choices"])
if raw.get("default") is not None:
prop_schema["default"] = raw["default"]
@@ -231,10 +235,16 @@ def _query_params_schema(
properties[body_name] = body_prop
if body_name in body_required:
required.append(body_name)
required_groups = [
group
for group in cmd_spec.get("required_groups") or []
if all(name in properties for alt in group["alternatives"] for name in alt)
]
return {
"type": "object",
"properties": properties,
"required": required,
**({"x-required-groups": required_groups} if required_groups else {}),
}
@@ -329,6 +339,11 @@ def _unwrap_schema_envelopes(schema: Any) -> dict[str, Any]:
return {}
typed: dict[str, Any] = schema
if typed.get("type") == "array":
kind = _response_shape(typed)[1]
if kind == "tuples":
return _wrap_scalar_as_value(typed["items"])
if kind == "records":
return _final_row_schema(typed["items"])
return _unwrap_schema_envelopes(typed.get("items"))
choice_rows = _wrapper_choice_rows(typed)
if choice_rows is not None:
@@ -520,11 +535,16 @@ def _response_shape(response: Any) -> tuple[str | None, str | None]:
only array, or the only one whose items are objects; every other property
is metadata), ``"map"`` when the only property is a free-form
``{key: value}`` map (one row per entry), ``"single"`` when the object is
itself the row, else ``(None, None)``. The data model and the generated
``unpack_response`` call both use this, so they cannot disagree.
itself the row, ``"tuples"`` when the response is an array of arrays (each
inner array is one positional row), ``"records"`` when it is an array of
objects (each object is one row), else ``(None, None)``. The data model
and the generated ``unpack_response`` call both use this, so they cannot
disagree.
"""
if not isinstance(response, dict) or response.get("type") == "array":
if not isinstance(response, dict):
return None, None
if response.get("type") == "array":
return None, _array_row_kind(response.get("items"))
props = response.get("properties")
if not isinstance(props, dict) or not props or "results" in props:
return None, None
@@ -549,6 +569,17 @@ def _response_shape(response: Any) -> tuple[str | None, str | None]:
return None, "single"
def _array_row_kind(items: Any) -> str | None:
"""``"tuples"`` for array items that are arrays, ``"records"`` for objects."""
if not isinstance(items, dict):
return None
if items.get("type") == "array" or "items" in items:
return "tuples"
if items.get("type") == "object" or "properties" in items:
return "records"
return None
def _envelope_inner_schema(
typed: dict[str, Any],
props: dict[str, Any], # noqa: ARG001
@@ -599,13 +630,124 @@ def _single_prop_inner_schema(props: dict[str, Any]) -> dict[str, Any] | None:
return None
def _merged_data_schema(cmd_spec: dict[str, Any]) -> dict[str, Any]:
"""Row schema of a command, covering every URL of a merged command.
The fields of each URL's row schema are combined. A field declared
differently by several URLs takes every distinct declaration
(``_merge_prop_schemas``) and keeps the first one's docs; declarations
that differ only in docs or nullability count as one (``_shape_key``). A
field stays required only when every URL's row schema requires it.
"""
base = _data_schema(cmd_spec)
variants = cmd_spec.get("template_response_schemas") or {}
if not variants:
return base
schemas = [base] + [
_data_schema({"response_schema": schema}) for schema in variants.values()
]
declarations: dict[str, list[Any]] = {}
for schema in schemas:
for name, prop in (schema.get("properties") or {}).items():
seen = declarations.setdefault(name, [])
if all(_shape_key(prop) != _shape_key(other) for other in seen):
seen.append(prop)
properties: dict[str, Any] = {}
for name, distinct in declarations.items():
first = distinct[0]
if len(distinct) == 1 or not all(isinstance(d, dict) for d in distinct):
properties[name] = first
continue
merged = first
for other in distinct[1:]:
merged = _merge_prop_schemas(merged, other)
docs = {k: first[k] for k in ("description", "title", "example") if k in first}
properties[name] = {**merged, **docs}
if not properties:
return base
required = set.intersection(*(set(s.get("required") or []) for s in schemas))
return {
**base,
"type": "object",
"properties": properties,
"required": [name for name in properties if name in required],
}
_DOC_KEYS = frozenset({"description", "example", "examples", "title"})
def _shape(schema: Any) -> Any:
"""Return ``schema`` without docs or nullability, at every level.
Descriptions, examples, titles and ``nullable`` are dropped, and an
``anyOf`` / ``oneOf`` of one schema and ``null`` becomes that schema.
"""
if isinstance(schema, list):
return [_shape(v) for v in schema]
if not isinstance(schema, dict):
return schema
out: dict[str, Any] = {}
for key, value in schema.items():
if key in ("properties", "patternProperties") and isinstance(value, dict):
out[key] = {name: _shape(sub) for name, sub in value.items()}
elif key not in _DOC_KEYS and key != "nullable":
out[key] = _shape(value)
for combinator in ("anyOf", "oneOf"):
variants = out.get(combinator)
if not isinstance(variants, list):
continue
non_null = [
v for v in variants if not (isinstance(v, dict) and v.get("type") == "null")
]
if len(non_null) == 1 and len(non_null) < len(variants):
return non_null[0]
return out
def _shape_key(schema: Any) -> str:
"""Compare key for a field schema (see ``_shape``)."""
return json.dumps(_shape(schema), sort_keys=True, default=str)
def _unpack_kwargs(response_schema: Any) -> dict[str, Any]:
"""``unpack_response`` keyword arguments for one response schema."""
key, kind = _response_shape(response_schema)
if kind is None:
return {}
if kind in ("single", "tuples", "records"):
return {"kind": kind}
return {"rows_key": key, "kind": kind}
def _template_unpack_hints(
cmd_spec: dict[str, Any], url_choices: UrlChoices | None
) -> dict[tuple[str, ...], dict[str, Any]] | None:
"""Per-URL ``unpack_response`` hints, keyed by each URL's sorted placeholders.
``None`` unless the URLs of a merged command declare different response
shapes, in which case the generated code picks the hint for the URL it used.
"""
variants = cmd_spec.get("template_response_schemas") or {}
if not url_choices or not variants:
return None
base = cmd_spec.get("response_schema")
hints: dict[tuple[str, ...], dict[str, Any]] = {}
for template in cmd_spec.get("url_templates") or []:
names = tuple(sorted(_path_template_keys(template)))
hints[names] = _unpack_kwargs(variants.get(template, base))
if len({repr(h) for h in hints.values()}) < 2:
return None
return hints
def _unpack_hint(cmd_spec: dict[str, Any]) -> str:
"""Extra ``unpack_response`` arguments derived from the response schema."""
key, kind = _response_shape(cmd_spec.get("response_schema"))
if kind is None:
return ""
if kind == "single":
return ', kind="single"'
if kind in ("single", "tuples", "records"):
return f', kind="{kind}"'
return f", rows_key={key!r}, kind={kind!r}"
@@ -635,7 +777,7 @@ def _is_scalar_schema(schema: dict[str, Any]) -> bool:
def _wrap_scalar_as_value(schema: dict[str, Any]) -> dict[str, Any]:
"""Promote a scalar schema to ``{type: object, properties: {value: scalar}}``."""
"""Promote a schema to ``{type: object, properties: {value: schema}}``."""
return {
"type": "object",
"properties": {"value": schema},
@@ -673,6 +815,7 @@ def _query_dict_construction(
creds: dict[str, dict[str, str]],
path_params: list[str],
provider_name: str,
field_names: dict[str, str] | None = None,
) -> str:
"""Compose the ``query_dict = {...}`` block emitted in ``aextract_data``.
@@ -691,13 +834,13 @@ def _query_dict_construction(
The Python source for the ``query_dict`` assignment.
"""
body_props = (cmd_spec.get("request_body_schema") or {}).get("properties") or {}
excluded = {query_field_name(n) for n in (*path_params, *body_props)}
excluded = {_field_name(field_names, n) for n in (*path_params, *body_props)}
range_params = [
p
for p in (cmd_spec.get("parameters") or [])
if isinstance(p, dict) and p.get("_socrata_op") in {"date_min", "date_max"}
]
excluded.update(query_field_name(p["name"]) for p in range_params)
excluded.update(_field_name(field_names, p["name"]) for p in range_params)
if excluded:
excluded_literal = (
"{" + ", ".join(repr(name) for name in sorted(excluded)) + "}"
@@ -725,7 +868,7 @@ def _query_dict_construction(
lines.append(f" if {var}:")
lines.append(f" _query_dict[{wire_name!r}] = {var}")
if range_params:
lines.extend(_socrata_where_clause_lines(range_params))
lines.extend(_socrata_where_clause_lines(range_params, field_names))
date_column = next(
(p["_socrata_column"] for p in range_params if p.get("_socrata_column")),
None,
@@ -741,8 +884,23 @@ def _query_dict_construction(
and param.get("in", "query") == "query"
):
lines.extend(
_list_param_lines(param["name"], param.get("list_style") or "multi")
_list_param_lines(
param["name"],
param.get("list_style") or "multi",
param.get("type") or "string",
)
)
group_members = list(
dict.fromkeys(
name
for group in cmd_spec.get("required_groups") or []
for name in group.get("members") or []
)
)
if group_members:
lines.append(f" for _name in {tuple(group_members)!r}:")
lines.append(' if _query_dict.get(_name) in ("", []):')
lines.append(" _query_dict.pop(_name, None)")
rename_pairs: list[tuple[str, str]] = []
for param in cmd_spec.get("parameters") or []:
if not isinstance(param, dict):
@@ -763,13 +921,22 @@ def _query_dict_construction(
_LIST_DELIMITERS = {"csv": ",", "space": " ", "pipe": "|"}
def _list_param_lines(wire_name: str, style: str) -> list[str]:
def _list_param_lines(
wire_name: str, style: str, item_type: str = "string"
) -> list[str]:
"""Emit lines turning the model's comma-joined string into the wire format.
``multi`` becomes a list (``get_querystring`` repeats the key); the other
styles are re-joined with the delimiter the upstream API expects.
``multi`` becomes a list (``encode_query`` repeats the key), ``json``
becomes a JSON array, and the other styles are re-joined with the
delimiter the upstream API expects.
"""
key = repr(wire_name)
if style == "json":
type_arg = "" if item_type == "string" else f", {item_type!r}"
return [
f" if isinstance(_query_dict.get({key}), str):",
f" _query_dict[{key}] = json_array(_query_dict[{key}]{type_arg})",
]
lines = [
f" if isinstance(_query_dict.get({key}), str):",
f" _items = [i.strip() for i in _query_dict[{key}].split(',') if i.strip()]",
@@ -782,14 +949,238 @@ def _list_param_lines(wire_name: str, style: str) -> list[str]:
return lines
def _socrata_where_clause_lines(range_params: list[dict[str, Any]]) -> list[str]:
@dataclass
class UrlChoices:
"""The URLs of a merged command, as generated code needs them.
Parameters
----------
templates : list of tuple
``(template, path_names, query_names)`` per URL, most placeholders
first, as ``choose_url_template`` takes them.
names : list of str
Every placeholder and declared query parameter across the URLs.
attributes : dict[str, str]
Wire name to the attribute holding its value.
defaults : dict[str, Any]
Placeholder defaults that must not pick a URL by themselves.
"""
templates: list[tuple[str, tuple[str, ...], tuple[str, ...]]]
names: list[str]
attributes: dict[str, str]
defaults: dict[str, Any]
@property
def placeholders(self) -> list[str]:
"""Every placeholder name, in first-seen order."""
return list(
dict.fromkeys(name for _, names, _ in self.templates for name in names)
)
def url_choices_for(
cmd_spec: dict[str, Any],
api_prefix: str,
attributes: dict[str, str],
) -> UrlChoices | None:
"""Collect the URLs of a merged command whose placeholders all have attributes.
``attributes`` maps each wire name the generated code can read to its
attribute name. URLs with a placeholder outside it (a credential-named or
undeclared segment) are left out, since no value can fill them. ``None``
when fewer than two URLs remain. A spec written before
``template_query_params`` existed lets every URL take every query
parameter, as before.
"""
templates = cmd_spec.get("url_templates") or []
if len(templates) < 2:
return None
params = filter_user_params(cmd_spec.get("parameters") or [])
declared = cmd_spec.get("template_query_params")
legacy = tuple(p["name"] for p in params if p.get("in", "query") != "path")
choices = []
for template in templates:
path_names = tuple(_path_template_keys(template))
if not all(name in attributes for name in path_names):
continue
query_names = (
tuple(n for n in declared.get(template) or () if n in attributes)
if isinstance(declared, dict)
else tuple(n for n in legacy if n in attributes)
)
choices.append(
(_resolve_url_path(api_prefix, template), path_names, query_names)
)
if len(choices) < 2:
return None
names = list(
dict.fromkeys(
name
for _, path_names, query_names in choices
for name in (*path_names, *query_names)
)
)
placeholders = {name for _, path_names, _ in choices for name in path_names}
defaults = {
p["name"]: p["default"]
for p in params
if p.get("name") in placeholders and p.get("default") is not None
}
return UrlChoices(
templates=choices,
names=names,
attributes={name: attributes[name] for name in names},
defaults=defaults,
)
def query_params_fields(
cmd_spec: dict[str, Any], provider_name: str, class_name: str
) -> dict[str, str]:
"""Each parameter's field name in a command's generated ``QueryParams``."""
return query_params_field_names(
_query_params_schema(cmd_spec, provider_name)["properties"], class_name
)
def _query_field_names(spec: FetcherCommandSpec) -> dict[str, str]:
"""``query_params_fields`` for the ``QueryParams`` class of ``spec``."""
return query_params_fields(
spec.cmd_spec, spec.provider_name, f"{class_name_from(spec.name)}QueryParams"
)
def _field_name(field_names: dict[str, str] | None, name: str) -> str:
"""Return the ``QueryParams`` field that holds parameter ``name``."""
return (field_names or {}).get(name) or query_field_name(name)
def boolean_params(cmd_spec: dict[str, Any]) -> set[str]:
"""Names of the command's boolean parameters."""
return {
p["name"]
for p in cmd_spec.get("parameters") or []
if isinstance(p, dict) and p.get("name") and p.get("type") == "boolean"
}
def _url_template_choices(spec: FetcherCommandSpec) -> UrlChoices | None:
"""``url_choices_for`` with the generated ``QueryParams`` fields as attributes."""
return url_choices_for(spec.cmd_spec, spec.api_prefix, _query_field_names(spec))
def render_choose_url_call(
choices: UrlChoices, owner: str | None, indent: str
) -> list[str]:
"""Render a ``choose_url_template(...)`` call.
Values are read off ``owner`` (``query``, ``self``), or from local
variables named after the attributes when ``owner`` is ``None``.
"""
values = ", ".join(
f"{name!r}: "
+ (
f"getattr({owner}, {choices.attributes[name]!r})"
if owner
else choices.attributes[name]
)
for name in choices.placeholders
)
labels = {name: choices.attributes[name] for name in choices.placeholders}
return [
f"{indent}choose_url_template(",
f"{indent} {tuple(choices.templates)!r},",
f"{indent} {{{values}}},",
f"{indent} {labels!r},",
f"{indent} {choices.defaults!r},",
f"{indent})",
]
def _path_lines(
url_path_template: str,
path_params: list[str],
url_choices: UrlChoices | None,
field_names: dict[str, str] | None = None,
booleans: set[str] | frozenset[str] = frozenset(),
) -> list[str]:
"""Emit the ``_path = ...`` assignment for ``aextract_data``.
A command with several URLs picks one at call time with
``choose_url_template``; ``_used`` and ``_query_names`` record the names its
placeholders used and the query parameters it declares. A boolean path
value goes through ``wire_value``, so it is sent as ``true`` / ``false``.
"""
if url_choices:
call = render_choose_url_call(url_choices, "query", " ")
call[0] = " _path, _used, _query_names = choose_url_template("
return call
if path_params:
sub_args = ", ".join(
(
f"{p}=wire_value(getattr(query, {_field_name(field_names, p)!r}))"
if p in booleans
else f"{p}=getattr(query, {_field_name(field_names, p)!r})"
)
for p in path_params
)
return [f" _path = {url_path_template!r}.format({sub_args})"]
return [f" _path = {url_path_template!r}"]
def _consumed_path_lines(url_choices: UrlChoices | None) -> list[str]:
"""Emit lines dropping what the chosen URL does not send as query parameters.
That is its own path values, other URLs' parameters it does not declare
(a list URL's ``limit`` default on an item URL), and empty values.
"""
if not url_choices:
return []
return [
f" for _name in {tuple(url_choices.names)!r}:",
" if (",
" _name in _used",
" or _name not in _query_names",
' or _query_dict.get(_name) in ("", [])',
" ):",
" _query_dict.pop(_name, None)",
]
def _url_template_validator(url_choices: UrlChoices) -> str:
"""Render a ``QueryParams`` validator rejecting path values that form no URL."""
call = "\n".join(render_choose_url_call(url_choices, "self", " "))
return (
'\n @model_validator(mode="after")\n'
" def _check_url_template(self) -> Any:\n"
' """Allow only path-parameter combinations that form one of the URLs."""\n'
f"{call}\n"
" return self\n"
)
def _has_json_list_param(cmd_spec: dict[str, Any]) -> bool:
"""Whether any query parameter is sent as a JSON-encoded array."""
return any(
isinstance(p, dict)
and p.get("is_list")
and p.get("list_style") == "json"
and p.get("in", "query") == "query"
for p in filter_user_params(cmd_spec.get("parameters") or [])
)
def _socrata_where_clause_lines(
range_params: list[dict[str, Any]], field_names: dict[str, str] | None = None
) -> list[str]:
"""Compose runtime lines that fold Socrata range params into ``$where``."""
field_lookups: list[str] = []
for param in range_params:
wire_name = param["name"]
column = param["_socrata_column"]
op = ">=" if param["_socrata_op"] == "date_min" else "<="
py_field = query_field_name(wire_name)
py_field = _field_name(field_names, wire_name)
var = f"_range_{py_field}"
field_lookups.append(
f" {var} = getattr(query, {py_field!r}, None)\n"
@@ -892,7 +1283,9 @@ def generate_fetcher_module(spec: FetcherCommandSpec) -> GeneratedFetcher:
fetcher_class = f"{model_name}Fetcher"
qp_schema = _query_params_schema(spec.cmd_spec, spec.provider_name)
data_schema = _with_body_field(_data_schema(spec.cmd_spec) or {"type": "object"})
data_schema = _with_body_field(
_merged_data_schema(spec.cmd_spec) or {"type": "object"}
)
qp_generated = generate_class(
qp_schema,
@@ -909,6 +1302,20 @@ def generate_fetcher_module(spec: FetcherCommandSpec) -> GeneratedFetcher:
creds = credentials_from_command(spec.cmd_spec)
path_params = _path_template_keys(spec.cmd_spec.get("url_path") or "")
url_choices = _url_template_choices(spec)
if url_choices:
qp_generated.source = (
qp_generated.source.rstrip("\n")
+ "\n"
+ _url_template_validator(url_choices)
)
qp_generated.imports.update(
{
"from pydantic import model_validator",
"from typing import Any",
"from ....utils import choose_url_template",
}
)
imports = {
"import base64",
@@ -918,11 +1325,14 @@ def generate_fetcher_module(spec: FetcherCommandSpec) -> GeneratedFetcher:
"from openbb_core.provider.abstract.data import Data",
"from openbb_core.provider.abstract.fetcher import Fetcher",
"from openbb_core.provider.abstract.query_params import QueryParams",
"from openbb_core.provider.utils.helpers import "
"get_async_requests_session, get_querystring",
"from openbb_core.provider.utils.helpers import get_async_requests_session",
"from pydantic import BaseModel, Field",
"from ....utils import safe_json_loads, unpack_response",
"from ....utils import encode_query, safe_json_loads, unpack_response",
}
if _has_json_list_param(spec.cmd_spec):
imports.add("from ....utils import json_array")
if not url_choices and boolean_params(spec.cmd_spec) & set(path_params):
imports.add("from ....utils import wire_value")
imports.update(qp_generated.collect_imports())
imports.update(data_generated.collect_imports())
@@ -934,8 +1344,13 @@ def generate_fetcher_module(spec: FetcherCommandSpec) -> GeneratedFetcher:
description = (spec.cmd_spec.get("description") or f"Fetch {spec.name}.").strip()
cred_lines = _credential_lookup_lines(creds, spec.provider_name)
field_names = _query_field_names(spec)
query_block = _query_dict_construction(
spec.cmd_spec, creds, path_params, spec.provider_name
spec.cmd_spec,
creds,
[] if url_choices else path_params,
spec.provider_name,
field_names,
)
header_block = _header_dict_construction(creds, spec.headers)
cookie_block = _cookie_dict_construction(creds)
@@ -950,6 +1365,7 @@ def generate_fetcher_module(spec: FetcherCommandSpec) -> GeneratedFetcher:
method=method,
url_path_template=full_url_path,
base_url=base_url,
path_params=path_params,
cred_lines=cred_lines,
query_block=query_block,
header_block=header_block,
@@ -989,7 +1405,13 @@ def generate_fetcher_module(spec: FetcherCommandSpec) -> GeneratedFetcher:
imports=consolidate_imports(imports),
column_metadata=_column_metadata_from_data_schema(data_schema),
)
if hint:
template_hints = _template_unpack_hints(spec.cmd_spec, url_choices)
if template_hints:
source = source.replace(
"unpack_response(_payload)",
f"unpack_response(_payload, **{template_hints!r}.get(tuple(sorted(_used)), {{}}))",
)
elif hint:
source = source.replace(
"unpack_response(_payload)", f"unpack_response(_payload{hint})"
)
@@ -1015,6 +1437,7 @@ def _render_streaming_fetcher_source(
method: str,
url_path_template: str,
base_url: str,
path_params: list[str],
cred_lines: list[str],
query_block: str,
header_block: str,
@@ -1041,6 +1464,8 @@ def _render_streaming_fetcher_source(
URL path with placeholders intact.
base_url : str
Upstream API root, no trailing slash.
path_params : list of str
Placeholder names in ``url_path_template``.
cred_lines : list of str
Credential-extraction source lines.
query_block : str
@@ -1087,15 +1512,24 @@ def _render_streaming_fetcher_source(
parts: list[str] = []
parts.append(f'"""Streaming fetcher for {spec.name} — generated from spec."""')
parts.append("")
parts.append("from collections.abc import AsyncIterator")
parts.append("from typing import Any")
parts.append("")
parts.append("import httpx")
parts.append("")
parts.append("from openbb_core.provider.abstract.data import Data")
parts.append("from openbb_core.provider.abstract.fetcher import Fetcher")
parts.append("from openbb_core.provider.abstract.query_params import QueryParams")
parts.append("from openbb_core.provider.utils.helpers import get_querystring")
imports = {
"from collections.abc import AsyncIterator",
"from typing import Any",
"import httpx",
"from openbb_core.provider.abstract.data import Data",
"from openbb_core.provider.abstract.fetcher import Fetcher",
"from openbb_core.provider.abstract.query_params import QueryParams",
"from ....utils import encode_query",
}
url_choices = _url_template_choices(spec)
booleans = boolean_params(spec.cmd_spec)
if _has_json_list_param(spec.cmd_spec):
imports.add("from ....utils import json_array")
if not url_choices and booleans & set(path_params):
imports.add("from ....utils import wire_value")
for cls in qp_classes:
imports.update(cls.imports)
parts.extend(consolidate_imports(imports))
parts.append("")
parts.append("")
for cls in qp_classes:
@@ -1123,10 +1557,20 @@ def _render_streaming_fetcher_source(
)
parts.append(aextract_doc)
parts.extend(cred_lines)
parts.extend(
_path_lines(
url_path_template,
path_params,
url_choices,
_query_field_names(spec),
booleans,
)
)
parts.append(query_block)
parts.append(" _query_string = get_querystring(_query_dict, [])")
parts.extend(_consumed_path_lines(url_choices))
parts.append(" _query_string = encode_query(_query_dict)")
parts.append(
f' _url = f"{base_url}{url_path_template}"'
f' _url = f"{base_url}{{_path}}"'
' + ("?" + _query_string if _query_string else "")'
)
parts.append(header_block)
@@ -1315,6 +1759,9 @@ def _render_fetcher_source(
),
has_path_params=has_path_params,
has_query_creds=has_query_creds,
url_choices=_url_template_choices(spec),
field_names=_query_field_names(spec),
booleans=boolean_params(spec.cmd_spec),
)
fetcher_source_lines = [
@@ -1381,6 +1828,9 @@ def _aextract_body(
body_field_names: list[str],
has_path_params: bool,
has_query_creds: bool,
url_choices: UrlChoices | None = None,
field_names: dict[str, str] | None = None,
booleans: set[str] | frozenset[str] = frozenset(),
) -> list[str]:
"""Render the body of ``aextract_data`` as a list of source lines.
@@ -1408,6 +1858,12 @@ def _aextract_body(
Whether the URL has placeholders to substitute.
has_query_creds : bool
Whether any credential lives in the query string.
url_choices : list, optional
``_url_template_choices`` output for a command with several URLs.
field_names : dict, optional
Parameter name to ``QueryParams`` field name (``_query_field_names``).
booleans : set of str, optional
Names of boolean parameters.
Returns
-------
@@ -1415,16 +1871,19 @@ def _aextract_body(
Indented lines forming the body of ``aextract_data``.
"""
lines: list[str] = list(cred_lines)
if has_path_params:
sub_args = ", ".join(
f"{p}=getattr(query, {query_field_name(p)!r})" for p in path_params
lines.extend(
_path_lines(
url_path_template,
path_params if has_path_params else [],
url_choices,
field_names,
booleans,
)
lines.append(f" _path = {url_path_template!r}.format({sub_args})")
else:
lines.append(f" _path = {url_path_template!r}")
)
lines.append("")
lines.append(query_block)
lines.append(" _query_string = get_querystring(_query_dict, [])")
lines.extend(_consumed_path_lines(url_choices))
lines.append(" _query_string = encode_query(_query_dict)")
lines.append(
f' _url = f"{base_url}{{_path}}" + ("?" + _query_string if _query_string else "")'
)
@@ -1437,7 +1896,7 @@ def _aextract_body(
body_literal = (
"{"
+ ", ".join(
f"{name!r}: getattr(query, {query_field_name(name)!r}, None)"
f"{name!r}: getattr(query, {_field_name(field_names, name)!r}, None)"
for name in body_field_names
)
+ "}"

10
cli/openbb_cli/codegen/package_gen.py
View File

@@ -31,6 +31,7 @@ from openbb_cli.codegen.router_gen import (
GeneratedRouter,
GeneratedRouters,
generate_routers,
mounted_command_paths,
)
from openbb_cli.codegen.test_gen import GeneratedTestModule, generate_provider_tests
@@ -68,6 +69,7 @@ class GeneratedPackage:
base_url: str = ""
commands_by_provider: dict[str, list[str]] = field(default_factory=dict)
test_modules: list[GeneratedTestModule] = field(default_factory=list)
command_paths: dict[str, str] = field(default_factory=dict)
def write(self) -> Path:
"""Materialize the project on disk under ``self.root``.
@@ -195,7 +197,7 @@ class GeneratedPackage:
None,
)
model = fetcher.model_name if fetcher else "Returns OBBject"
return dotted, model
return self.command_paths.get(dotted, dotted), model
return None
def _providers_section(self, ns: str) -> str:
@@ -220,8 +222,9 @@ class GeneratedPackage:
groups: dict[str, list[tuple[str, str]]] = {}
for prov, dotted_paths in self.commands_by_provider.items():
for dotted in sorted(dotted_paths):
top = dotted.split(".", 1)[0]
groups.setdefault(top, []).append((dotted, prov))
mounted = self.command_paths.get(dotted, dotted)
top = mounted.split(".", 1)[0]
groups.setdefault(top, []).append((mounted, prov))
for top in sorted(groups):
lines.append(f"### `obb.{ns}.{top}`\n")
for dotted, prov in groups[top]:
@@ -647,6 +650,7 @@ def generate_packages(
base_url=base_url,
commands_by_provider=commands_by_provider,
test_modules=test_modules,
command_paths=mounted_command_paths(full_tree),
)
return GeneratedPackageSet(packages=[package])

76
cli/openbb_cli/codegen/post_gen.py
View File

@@ -2,6 +2,7 @@
from __future__ import annotations
import keyword
import re
from dataclasses import dataclass, field
from typing import Any, cast
@@ -10,6 +11,12 @@ from openbb_cli.codegen.credentials import (
credentials_from_command,
filter_user_params,
)
from openbb_cli.codegen.fetcher_gen import (
UrlChoices,
boolean_params,
render_choose_url_call,
url_choices_for,
)
from openbb_cli.codegen.pydantic_gen import (
GeneratedClass,
class_name_from,
@@ -83,7 +90,8 @@ def _module_name_from_command(name: str) -> str:
safe = "command"
if safe[0].isdigit():
safe = f"_{safe}"
return safe.lower()
safe = safe.lower()
return f"{safe}_" if keyword.iskeyword(safe) else safe
def _path_template_keys(url_path: str) -> list[str]:
@@ -530,6 +538,16 @@ def generate_post_command_module(spec: PostCommandSpec) -> GeneratedPostCommand:
full_url_path = _resolve_url_path(
spec.api_prefix, spec.cmd_spec.get("url_path") or ""
)
signature_names = {name for name, *_ in params}
url_choices = url_choices_for(
spec.cmd_spec,
spec.api_prefix,
{
p["name"]: safe_field_name(p["name"])[0]
for p in filter_user_params(spec.cmd_spec.get("parameters") or [])
if p.get("name") and safe_field_name(p["name"])[0] in signature_names
},
)
body_block = _render_body_block(
cmd_spec=spec.cmd_spec,
body_field=body_field,
@@ -541,6 +559,7 @@ def generate_post_command_module(spec: PostCommandSpec) -> GeneratedPostCommand:
data_class=data_class,
params=params,
fixed_headers=spec.headers,
url_choices=url_choices,
)
imports = {
@@ -549,11 +568,19 @@ def generate_post_command_module(spec: PostCommandSpec) -> GeneratedPostCommand:
"from openbb_core.app.model.abstract.error import OpenBBError",
"from openbb_core.app.model.obbject import OBBject",
"from openbb_core.provider.abstract.data import Data",
"from openbb_core.provider.utils.helpers import "
"get_async_requests_session, get_querystring",
"from openbb_core.provider.utils.helpers import get_async_requests_session",
"from pydantic import BaseModel, Field",
"from ....utils import unpack_response",
"from ....utils import encode_query, unpack_response",
}
if url_choices:
imports.add("from ....utils import choose_url_template")
elif boolean_params(spec.cmd_spec) & set(path_params):
imports.add("from ....utils import wire_value")
if any(
p.get("is_list") and p.get("list_style") == "json"
for p in filter_user_params(spec.cmd_spec.get("parameters") or [])
):
imports.add("from ....utils import json_array")
if creds:
imports.add("from openbb_core.app.model.command_context import CommandContext")
imports.update(data_generated.collect_imports())
@@ -631,6 +658,7 @@ def _render_body_block(
data_class: str,
params: list[tuple[str, str, str | None, bool, Any]],
fixed_headers: dict[str, str] | None = None,
url_choices: UrlChoices | None = None,
) -> str:
"""Render the function body for a POST command.
@@ -656,6 +684,8 @@ def _render_body_block(
Function parameters.
fixed_headers : dict, optional
Headers sent with every request (from ``--header`` at spec generation).
url_choices : UrlChoices, optional
The URLs of a merged command, picked between at call time.
Returns
-------
@@ -683,19 +713,47 @@ def _render_body_block(
lines: list[str] = []
lines.extend(cred_lines)
if path_params:
sub_args = ", ".join(f"{p}={p}" for p in path_params)
if url_choices:
call = render_choose_url_call(url_choices, None, " ")
call[0] = " _path, _used, _query_names = choose_url_template("
lines.extend(call)
elif path_params:
booleans = boolean_params(cmd_spec)
sub_args = ", ".join(
f"{p}=wire_value({p})" if p in booleans else f"{p}={p}" for p in path_params
)
lines.append(f" _path = {url_path_template!r}.format({sub_args})")
else:
lines.append(f" _path = {url_path_template!r}")
json_lists = {
p["name"]: p.get("type") or "string"
for p in filter_user_params(cmd_spec.get("parameters") or [])
if p.get("is_list") and p.get("list_style") == "json"
}
lines.append(" _query_dict: dict[str, Any] = {}")
for name in query_field_names:
wire = wire_by_safe.get(name, name)
if wire in path_params or name in path_params:
if not url_choices and (wire in path_params or name in path_params):
continue
lines.append(f" if {name} is not None:")
lines.append(f" _query_dict[{wire!r}] = {name}")
if wire in json_lists:
type_arg = "" if json_lists[wire] == "string" else f", {json_lists[wire]!r}"
lines.append(
f" _value = json_array({name} if isinstance({name}, str) "
f"else _json.dumps(list({name})){type_arg})"
)
lines.append(" if _value is not None:")
lines.append(f" _query_dict[{wire!r}] = _value")
else:
lines.append(f" _query_dict[{wire!r}] = {name}")
if url_choices:
lines.append(f" for _name in {tuple(url_choices.names)!r}:")
lines.append(
" if _name in _used or _name not in _query_names "
'or _query_dict.get(_name) in ("", []):'
)
lines.append(" _query_dict.pop(_name, None)")
for canonical, info in creds.items():
if info["in"] != "query":
continue
@@ -703,7 +761,7 @@ def _render_body_block(
var = f"_cred_{canonical}"
lines.append(f" if {var}:")
lines.append(f" _query_dict[{wire_name!r}] = {var}")
lines.append(" _query_string = get_querystring(_query_dict, [])")
lines.append(" _query_string = encode_query(_query_dict)")
lines.append(
f' _url = f"{base_url}{{_path}}" + ("?" + _query_string if _query_string else "")'
)

2
cli/openbb_cli/codegen/project_gen.py
View File

@@ -100,7 +100,7 @@ select = ["E", "F", "W", "I", "B", "UP", "D"]
ignore = [
"D100", "D101", "D102", "D103", "D104", "D105", "D107",
"D200", "D205", "D212", "D401", "D415",
"E501", "B008",
"E501", "E741", "B008",
]
[tool.ruff.lint.pydocstyle]

202
cli/openbb_cli/codegen/pydantic_gen.py
View File

@@ -565,6 +565,87 @@ def render_function_docstring(
return "\n".join(lines)
_BUILTIN_TYPE_NAMES = (
"list",
"dict",
"set",
"tuple",
"str",
"int",
"float",
"bool",
"bytes",
)
_QUOTED = re.compile(r"'(?:[^'\\]|\\.)*'|\"(?:[^\"\\]|\\.)*\"")
def _shadowed_builtins(properties: dict[str, Any], class_name: str) -> set[str]:
"""Builtin type names that a field would shadow inside its class's annotations.
A field named ``list`` binds ``list`` in the class body, so a later
``list[...]`` annotation (its own included) fails when the class is built.
Returns the builtin names some field is called and some annotation uses
outside a string (``Literal['list']`` uses none).
"""
names = {safe_field_name(raw)[0] for raw in properties} | {
query_field_name(raw) for raw in properties
}
candidates = names & set(_BUILTIN_TYPE_NAMES)
if not candidates:
return set()
annotations = " ".join(
schema_to_type(
prop if isinstance(prop, dict) else {},
parent_class_name=class_name,
field_name=safe_field_name(raw)[0],
nested_classes=[],
imports=set(),
)
for raw, prop in properties.items()
)
annotations = _QUOTED.sub("''", annotations)
return {name for name in candidates if re.search(rf"\b{name}\b", annotations)}
def _unique_field_names(
properties: dict[str, Any], class_name: str, *, query_params: bool
) -> dict[str, str]:
"""Map each property name to a field name no other property of the class uses.
Names are made safe the usual way (``query_field_name`` for query
parameters), and a builtin some annotation needs gets a ``_`` suffix. When
two properties land on one name (``@id`` and ``id``), the property already
spelled that way keeps it and the others get ``_`` suffixes.
"""
shadowed = _shadowed_builtins(properties, class_name)
wanted: dict[str, str] = {}
for raw in properties:
safe = query_field_name(raw) if query_params else safe_field_name(raw)[0]
wanted[raw] = f"{safe}_" if safe in shadowed else safe
taken: set[str] = set()
out: dict[str, str] = {}
for raw in sorted(properties, key=lambda r: wanted[r] != r):
name = wanted[raw]
while name in taken:
name = f"{name}_"
taken.add(name)
out[raw] = name
return {raw: out[raw] for raw in properties}
def query_params_field_names(
properties: dict[str, Any], class_name: str
) -> dict[str, str]:
"""Map each parameter to its field name in a ``QueryParams`` class.
The names ``generate_class`` gives the fields of ``class_name`` built from
``properties``, for code that reads those fields back.
"""
return _unique_field_names(properties, class_name, query_params=True)
def generate_class(
schema: dict[str, Any],
*,
@@ -581,12 +662,15 @@ def generate_class(
doc_fields: list[tuple[str, str, str | None, bool, Any, bool]] = []
optional_object_fields: list[str] = []
multiple_items: dict[str, dict[str, Any]] = {}
field_names: dict[str, str] = {}
unique_names = _unique_field_names(
properties, class_name, query_params=base_class == "QueryParams"
)
for raw_name, prop_schema in properties.items():
safe_name, needs_alias = safe_field_name(raw_name)
if base_class == "QueryParams":
safe_name = query_field_name(raw_name)
needs_alias = safe_name != raw_name
safe_name = unique_names[raw_name]
needs_alias = safe_name != raw_name
field_names[raw_name] = safe_name
annotation = schema_to_type(
prop_schema if isinstance(prop_schema, dict) else {},
parent_class_name=class_name,
@@ -687,6 +771,23 @@ def generate_class(
" return values"
)
required_groups = [
group
for group in schema.get("x-required-groups") or []
if _group_maps_to_distinct_fields(group, field_names)
]
if required_groups:
imports.add("from pydantic import model_validator")
imports.add("from typing import Any")
defaults = {
field_names[raw]: prop["default"]
for raw, prop in properties.items()
if isinstance(prop, dict) and prop.get("default") is not None
}
body_lines.append(
_render_required_groups_validator(required_groups, field_names, defaults)
)
summary = docstring or class_name
doc_block = _render_class_docstring(summary, doc_fields)
@@ -703,6 +804,99 @@ def generate_class(
)
def _group_maps_to_distinct_fields(
group: dict[str, Any], field_names: dict[str, str]
) -> bool:
"""Whether every name in a required group is a field of its own.
A name with no field, or two names that collapse to one field (``Symbol``
and ``symbol``), would make the group impossible to satisfy.
"""
alternatives = group.get("alternatives") or []
names = {name for alt in alternatives for name in alt}
names.update(name for name in group.get("members") or [] if name in field_names)
if not alternatives or any(name not in field_names for name in names):
return False
return len({field_names[name] for name in names}) == len(names)
def _render_required_groups_validator(
groups: list[dict[str, Any]],
field_names: dict[str, str],
defaults: dict[str, Any] | None = None,
) -> str:
"""Render an ``after`` model validator enforcing ``oneOf`` / ``anyOf`` groups.
Parameters
----------
groups : list[dict]
Required groups in the spec form (``combinator``, ``alternatives``,
``members``, ``required``), naming properties by their schema name.
field_names : dict[str, str]
Schema property name to generated field name.
defaults : dict[str, Any], optional
Field name to default value. An optional group is only checked when a
member is passed, and a member left at its default was not passed.
Returns
-------
str
The indented validator method source. ``None``, ``""`` and ``[]``
count as not passed.
"""
defaults = defaults or {}
lines = [
'\n @model_validator(mode="after")',
" def _check_required_groups(self) -> Any:",
' """Enforce the parameter combinations the API requires."""',
]
for group in groups:
alternatives = tuple(
tuple(field_names[name] for name in alt) for alt in group["alternatives"]
)
exactly_one = group.get("combinator") == "oneOf"
described = " | ".join(
alt[0] if len(alt) == 1 else f"({', '.join(alt)})" for alt in alternatives
)
message = f"Pass {'exactly' if exactly_one else 'at least'} one of: {described}"
indent = " "
if not group.get("required", True):
members = tuple(
field_names[name]
for name in group.get("members") or []
if name in field_names
)
if any(name in defaults for name in members):
pairs = tuple((name, defaults.get(name)) for name in members)
lines.extend(
[
f"{indent}if any(",
f"{indent} getattr(self, _name) not in (None, '', [])",
f"{indent} and getattr(self, _name) != _default",
f"{indent} for _name, _default in {pairs!r}",
f"{indent}):",
]
)
else:
lines.append(
f"{indent}if any(getattr(self, _name) not in (None, '', []) "
f"for _name in {members!r}):"
)
indent += " "
lines.extend(
[
f"{indent}_satisfied = sum(",
f"{indent} all(getattr(self, _name) not in (None, '', []) for _name in _names)",
f"{indent} for _names in {alternatives!r}",
f"{indent})",
f"{indent}if _satisfied {'!= 1' if exactly_one else '< 1'}:",
f"{indent} raise ValueError({message!r})",
]
)
lines.append(" return self")
return "\n".join(lines)
def consolidate_imports(imports: set[str]) -> list[str]:
"""Merge multiple ``from X import A`` / ``from X import B`` into one line.

30
cli/openbb_cli/codegen/router_gen.py
View File

@@ -97,6 +97,36 @@ def _model_name_for(path: str) -> str:
)
def mounted_command_paths(root: NamespaceNode) -> dict[str, str]:
"""Map each command's dotted spec path to the path it mounts at under ``obb``.
Segments are normalized the same way as router modules and functions, and a
command that is also a namespace mounts on its own router, named after itself.
Parameters
----------
root : NamespaceNode
The synthetic root node returned by ``build_namespace_tree``.
Returns
-------
dict[str, str]
``{dotted_spec_path: dotted_mounted_path}`` for every command in the tree.
"""
out: dict[str, str] = {}
pending = list(root.children.values())
while pending:
node = pending.pop()
pending.extend(node.children.values())
if node.cmd_spec is None:
continue
parts = [_safe_segment(p) for p in node.full_path.split(".") if p]
if node.is_namespace:
parts.append(_safe_segment(node.name))
out[node.full_path] = ".".join(parts)
return out
def generate_routers(
root: NamespaceNode,
*,

225
cli/openbb_cli/codegen/test_gen.py
View File

@@ -2,14 +2,17 @@
from __future__ import annotations
import keyword
import re
from dataclasses import dataclass
from datetime import UTC
from itertools import islice, product
from typing import TYPE_CHECKING, Any
from openbb_cli.codegen.pydantic_gen import safe_field_name
from openbb_cli.codegen.pydantic_gen import query_field_name
if TYPE_CHECKING:
from openbb_cli.codegen.fetcher_gen import GeneratedFetcher
from openbb_cli.codegen.fetcher_gen import GeneratedFetcher, UrlChoices
_STANDARD_HEADER_REDACTIONS: tuple[tuple[str, str | None], ...] = (
@@ -112,7 +115,11 @@ def generate_provider_tests(
for f in sorted(fetchers, key=lambda x: x.fetcher_class):
dotted = _dotted_for_fetcher(f, commands_by_dotted)
cmd_spec = commands_by_dotted.get(dotted, {}) if dotted else {}
params = _derive_test_params(cmd_spec, provider_name)
params = _derive_test_params(
cmd_spec,
provider_name,
class_name=f.fetcher_class.removesuffix("Fetcher") + "QueryParams",
)
if params is None:
continue
testable.append((f, params))
@@ -214,14 +221,12 @@ def _dotted_for_fetcher(
def _normalize_to_module(dotted: str) -> str:
"""Match ``fetcher_gen._module_name_from_command`` exactly."""
import re
safe = re.sub(r"[^0-9a-zA-Z]+", "_", dotted).strip("_").lower()
if not safe:
safe = "command"
if safe[0].isdigit():
safe = f"_{safe}"
return safe
return f"{safe}_" if keyword.iskeyword(safe) else safe
_DATE_PARAM_NAMES: frozenset[str] = frozenset(
@@ -246,10 +251,24 @@ def _is_date_param(name: str) -> bool:
def _derive_test_params(
cmd_spec: dict[str, Any], provider_name: str
cmd_spec: dict[str, Any], provider_name: str, class_name: str = "QueryParams"
) -> dict[str, Any] | None:
"""Build a kwargs dict for the command's testable params, or ``None``."""
"""Build a kwargs dict for the command's testable params, or ``None``.
Keys are the generated ``QueryParams`` field names, the attribute names
``Fetcher.test`` reads back from the validated query. The params satisfy
the command's required groups and, for a command with several URLs, form
one of them; when that is impossible the test is skipped.
"""
out: dict[str, Any] = {}
params_by_name: dict[str, dict[str, Any]] = {}
heuristic: set[str] = set()
placeholders = {
name
for template in cmd_spec.get("url_templates")
or [cmd_spec.get("url_path") or ""]
for name in re.findall(r"\{([^}]+)\}", template)
}
for raw in cmd_spec.get("parameters") or []:
if not isinstance(raw, dict):
continue
@@ -259,30 +278,190 @@ def _derive_test_params(
param_providers = raw.get("providers") or []
if param_providers and provider_name not in param_providers:
continue
field_name, _ = safe_field_name(name)
params_by_name[name] = raw
if raw.get("required"):
value = _spec_supplied_value(raw)
if value is None:
return None
out[field_name] = value
elif _is_date_param(name):
out[field_name] = _refresh_date_shape(name, "1970-01-01")
return out
out[name] = value
elif (
_is_date_param(name)
and not raw.get("choices")
and raw.get("in", "query") != "path"
and name not in placeholders
):
out[name] = _refresh_date_shape(name, "1970-01-01")
heuristic.add(name)
groups = cmd_spec.get("required_groups") or []
satisfied = _satisfy_groups(groups, out, params_by_name, heuristic)
if satisfied is None:
return None
formed = _satisfy_url_templates(cmd_spec, satisfied, params_by_name)
if formed is None:
return None
from openbb_cli.codegen.fetcher_gen import query_params_fields
field_names = query_params_fields(
{**cmd_spec, "parameters": list(params_by_name.values())},
provider_name,
class_name,
)
return {
field_names.get(name) or query_field_name(name): value
for name, value in formed.items()
}
def _is_present(out: dict[str, Any], name: str) -> bool:
"""Whether ``out`` passes a value for ``name``, by the validator's rule."""
return out.get(name) not in (None, "", [])
def _group_holds(group: dict[str, Any], out: dict[str, Any]) -> bool:
"""Whether ``out`` passes the generated ``_check_required_groups`` for ``group``."""
if not group.get("required", True) and not any(
_is_present(out, name) for name in group.get("members") or []
):
return True
met = sum(
all(_is_present(out, name) for name in alt)
for alt in group.get("alternatives") or []
)
return met == 1 if group.get("combinator") == "oneOf" else met >= 1
_MAX_GROUP_COMBINATIONS = 512
def _satisfy_groups(
groups: list[dict[str, Any]],
out: dict[str, Any],
params_by_name: dict[str, dict[str, Any]],
heuristic: set[str],
) -> dict[str, Any] | None:
"""Return ``out`` changed so every required group passes, or ``None``.
Optional groups set off only by params the date heuristic added are
disarmed by removing those params. Then every combination of one
alternative per group (an optional group may also be left out) is tried in
order, up to a bound: missing values come from the spec, and for ``oneOf``
the params only unchosen alternatives need are removed. Required params
are never removed.
"""
if all(_group_holds(group, out) for group in groups):
return out
base = dict(out)
for group in groups:
if group.get("required", True) or _group_holds(group, base):
continue
triggered = [n for n in group.get("members") or [] if _is_present(base, n)]
if triggered and set(triggered) <= heuristic:
for name in triggered:
base.pop(name, None)
if all(_group_holds(group, base) for group in groups):
return base
required = {name for name, raw in params_by_name.items() if raw.get("required")}
options = [
list(group.get("alternatives") or [])
+ ([] if group.get("required", True) else [None])
for group in groups
]
for combo in islice(product(*options), _MAX_GROUP_COMBINATIONS):
candidate = _fill(
base, [n for alt in combo if alt for n in alt], params_by_name
)
if candidate is None:
continue
chosen = {name for alt in combo if alt for name in alt}
for group, alt in zip(groups, combo, strict=True):
if alt is None or group.get("combinator") != "oneOf":
continue
for other in group.get("alternatives") or []:
for name in other:
if name not in chosen and name not in required:
candidate.pop(name, None)
if all(_group_holds(group, candidate) for group in groups):
return candidate
return None
def _fill(
out: dict[str, Any],
names: list[str],
params_by_name: dict[str, dict[str, Any]],
) -> dict[str, Any] | None:
"""Copy ``out`` with spec-supplied values for any of ``names`` it lacks."""
candidate = dict(out)
for name in names:
if _is_present(candidate, name):
continue
raw = params_by_name.get(name)
value = _spec_supplied_value(raw) if raw is not None else None
if value is None:
return None
candidate[name] = value
return candidate
def _satisfy_url_templates(
cmd_spec: dict[str, Any],
out: dict[str, Any],
params_by_name: dict[str, dict[str, Any]],
) -> dict[str, Any] | None:
"""Make ``out`` form one of a merged command's URLs, as its validator checks.
When the params form none, the first URL whose placeholders can all be
filled from the spec is filled; ``None`` when no URL can be.
"""
from openbb_cli.codegen.fetcher_gen import url_choices_for
choices = url_choices_for(cmd_spec, "", {name: name for name in params_by_name})
if choices is None or _forms_url(choices, out):
return out
for _, path_names, _ in choices.templates:
candidate = _fill(out, list(path_names), params_by_name)
if candidate is not None and _forms_url(choices, candidate):
return candidate
return None
def _forms_url(choices: UrlChoices, out: dict[str, Any]) -> bool:
"""Whether ``out`` passes the generated ``_check_url_template``."""
from openbb_cli.dispatchers._unpack import choose_url_template
values = {
name: out.get(name, choices.defaults.get(name)) for name in choices.placeholders
}
try:
choose_url_template(choices.templates, values, None, choices.defaults)
except ValueError:
return False
return True
def _spec_supplied_value(param: dict[str, Any]) -> Any:
"""Return a value the spec itself vouches for, or ``None`` to abort the test."""
raw_value: Any = None
if param.get("example") is not None:
raw_value = param["example"]
elif param.get("default") is not None:
raw_value = param["default"]
else:
choices = param.get("choices") or []
if choices:
raw_value = choices[0]
"""Return a value the spec itself vouches for, or ``None`` to abort the test.
The example wins, then the default, then the first choice; when the param
declares ``choices``, an example or default outside them is skipped.
"""
choices = param.get("choices") or []
raw_value: Any = next(
(
value
for value in (param.get("example"), param.get("default"))
if value is not None and (not choices or value in choices)
),
choices[0] if choices else None,
)
if raw_value is None:
return None
if (
isinstance(raw_value, list)
and param.get("is_list")
and param.get("in", "query") == "query"
):
raw_value = ",".join(str(item) for item in raw_value)
return _refresh_date_shape(param.get("name") or "", raw_value)