OpenBB/cli/openbb_cli/codegen/fetcher_gen.py

"""Generate ``QueryParams`` + ``Data`` + ``Fetcher`` modules for one spec command."""

from __future__ import annotations

import re
from dataclasses import dataclass, field
from typing import Any, cast

from openbb_cli.codegen.credentials import (
    credentials_from_command,
    filter_user_params,
)
from openbb_cli.codegen.pydantic_gen import (
    GeneratedClass,
    class_name_from,
    consolidate_imports,
    generate_class,
    query_field_name,
    render_function_docstring,
)

_PATH_TEMPLATE_RE = re.compile(r"\{([^}]+)\}")


@dataclass
class FetcherCommandSpec:
    """Per-command inputs to the fetcher emitter.

    Parameters
    ----------
    name : str
        Dotted command path as it appears in ``spec_doc["commands"]``.
    cmd_spec : dict
        The spec entry.
    base_url : str
        The upstream API root.
    api_prefix : str
        Path prefix shared across every command in the spec.
    provider_name : str
        Snake-case provider name.
    """

    name: str
    cmd_spec: dict[str, Any]
    base_url: str
    api_prefix: str
    provider_name: str
    headers: dict[str, str] = field(default_factory=dict)


@dataclass
class GeneratedFetcher:
    """The output of ``generate_fetcher_module``.

    Parameters
    ----------
    module_name : str
        Snake-case module filename (without ``.py``).
    model_name : str
        The Pydantic model name registered in the provider's ``fetcher_dict``.
    query_params_class : str
        Name of the emitted ``QueryParams`` subclass.
    data_class : str
        Name of the emitted ``Data`` subclass.
    fetcher_class : str
        Name of the emitted ``Fetcher`` subclass.
    source : str
        Full module source ready to write to disk.
    credentials_used : dict
        ``{canonical_name: {"name": wire_name, "in": "query"|"header"|"cookie"}}``.
    """

    module_name: str
    model_name: str
    query_params_class: str
    data_class: str
    fetcher_class: str
    source: str
    credentials_used: dict[str, dict[str, str]] = field(default_factory=dict)
    is_streaming: bool = False


def _command_is_streaming(cmd_spec: dict[str, Any]) -> bool:
    """Return whether a command's success response is a stream.

    Parameters
    ----------
    cmd_spec : dict
        The spec command entry.

    Returns
    -------
    bool
        True when a 2xx response advertises ``text/event-stream``.
    """
    for status, content in (cmd_spec.get("response_schemas") or {}).items():
        if (
            str(status).startswith("2")
            and isinstance(content, dict)
            and any("event-stream" in str(ct) for ct in content)
        ):
            return True
    return False


def _module_name_from_command(name: str) -> str:
    """Convert a dotted command path to a snake_case module identifier.

    Parameters
    ----------
    name : str
        Dotted command path.

    Returns
    -------
    str
        Snake-case identifier.
    """
    safe = re.sub(r"[^0-9a-zA-Z]+", "_", name).strip("_")
    if not safe:
        safe = "command"
    if safe[0].isdigit():
        safe = f"_{safe}"
    return safe.lower()


_COORDINATE_NAME_RE = re.compile(r"(^|[^a-z])(lat|lon|lng|latitude|longitude)", re.I)


def _is_coordinate_name(name: str) -> bool:
    """Whether a path parameter is a decimal coordinate (never an integer id)."""
    return bool(_COORDINATE_NAME_RE.search(name))


def _mark_null_empty(node: Any) -> Any:
    """Flag every object schema so its generated class treats ``''`` as null.

    The marker is a vendor extension read by ``generate_class``; nested object
    schemas are visited too, so nested classes get the same behavior.
    """
    if isinstance(node, list):
        return [_mark_null_empty(n) for n in node]
    if not isinstance(node, dict):
        return node
    out = {
        k: (
            v
            if k in ("enum", "const", "default", "example", "examples")
            else _mark_null_empty(v)
        )
        for k, v in node.items()
    }
    if out.get("type") == "object" or "properties" in out:
        out["x-null-empty"] = True
    return out


def _path_template_keys(url_path: str) -> list[str]:
    """Extract ``{placeholder}`` names from a URL template, in declaration order."""
    return [m.group(1) for m in _PATH_TEMPLATE_RE.finditer(url_path)]


def _query_params_schema(
    cmd_spec: dict[str, Any], provider_name: str
) -> dict[str, Any]:
    """Build a JSON-schema dict combining query, path, and body fields.

    Parameters
    ----------
    cmd_spec : dict
        The spec command entry.
    provider_name : str
        Provider this fetcher is being generated for.

    Returns
    -------
    dict
        A synthetic ``{"type": "object", "properties": {...}, "required": [...]}``.
    """
    properties: dict[str, Any] = {}
    required: list[str] = []
    for raw in filter_user_params(cmd_spec.get("parameters") or []):
        name = raw.get("name")
        if not name:
            continue
        if name == "provider":
            continue
        param_providers = raw.get("providers") or []
        if param_providers and provider_name not in param_providers:
            continue
        description = raw.get("help") or raw.get("description")
        if description:
            description = re.sub(r"\s*\(provider:[^)]*\)\s*$", "", description).strip()
        prop_schema: dict[str, Any] = {}
        for key in ("type", "default", "choices"):
            if key in raw and raw[key] is not None:
                prop_schema[key] = raw[key]
        if (
            raw.get("in") == "path"
            and prop_schema.get("type") == "number"
            and not _is_coordinate_name(name)
        ):
            prop_schema["type"] = "integer"
        if description:
            prop_schema["description"] = description
        if raw.get("is_list") and raw.get("in", "query") == "query":
            # Platform convention: a ``str`` field flagged ``multiple_items_allowed``
            # (comma-separated over REST, list-or-str in Python).
            prop_schema = {"type": "string", "x-multiple-items": True}
            if description:
                prop_schema["description"] = description
            if raw.get("choices"):
                prop_schema["x-choices"] = list(raw["choices"])
            if raw.get("default") is not None:
                prop_schema["default"] = raw["default"]
        elif raw.get("is_list"):
            inner = {"type": prop_schema.get("type", "string")}
            prop_schema = {"type": "array", "items": inner}
            if description:
                prop_schema["description"] = description
        if raw.get("choices") and "x-multiple-items" not in prop_schema:
            prop_schema["enum"] = list(raw["choices"])
        properties[name] = prop_schema
        if raw.get("required"):
            required.append(name)
    body_schema = cmd_spec.get("request_body_schema") or {}
    body_props = body_schema.get("properties") or {}
    body_required = body_schema.get("required") or []
    for body_name, body_prop in body_props.items():
        if body_name not in properties:
            properties[body_name] = body_prop
            if body_name in body_required:
                required.append(body_name)
    return {
        "type": "object",
        "properties": properties,
        "required": required,
    }


def _column_metadata_from_data_schema(
    data_schema: dict[str, Any],
) -> dict[str, dict[str, Any]]:
    """Pull per-column ``description`` / ``socrata_format`` out of the row schema."""
    properties = (data_schema or {}).get("properties") or {}
    out: dict[str, dict[str, Any]] = {}
    for field_name, schema in properties.items():
        if not isinstance(schema, dict):
            continue
        meta: dict[str, Any] = {}
        if schema.get("description"):
            meta["description"] = schema["description"]
        if schema.get("format"):
            meta["format"] = schema["format"]
        if schema.get("socrata_format"):
            meta["socrata_format"] = schema["socrata_format"]
        if meta:
            out[field_name] = meta
    return out


def _data_schema(cmd_spec: dict[str, Any]) -> dict[str, Any]:
    """Pull the per-row data schema out of a response_schema.

    Parameters
    ----------
    cmd_spec : dict
        The spec command entry.

    Returns
    -------
    dict
        The schema that should drive the ``Data`` Pydantic class.
    """
    response = cmd_spec.get("response_schema") or {}
    if not isinstance(response, dict):
        return {}
    properties = response.get("properties") or {}
    results = properties.get("results")
    if isinstance(results, dict):
        non_null = [
            v
            for v in results.get("anyOf") or [results]
            if isinstance(v, dict) and v.get("type") != "null"
        ]
        if non_null:
            inner = non_null[0]
            if inner.get("type") == "array" and isinstance(inner.get("items"), dict):
                inner = inner["items"]
            if isinstance(inner.get("oneOf"), list):
                merged = _merge_object_variants(inner)
                concrete = [c for c in inner["oneOf"] if isinstance(c, dict)]
                if merged is not None:
                    inner = merged
                elif concrete:
                    inner = concrete[0]
            return inner
    return _unwrap_schema_envelopes(response)


def _with_body_field(schema: dict[str, Any]) -> dict[str, Any]:
    """Give a schema-less row a ``value`` field for the raw body.

    A text or PDF response has no row schema; the fetcher puts the body in
    ``value``. Declaring it keeps the row from being a field-less model, which
    the platform's ``to_df`` cannot tabulate.
    """
    structured = (
        schema.get("properties")
        or schema.get("additionalProperties")
        or any(schema.get(k) for k in ("anyOf", "oneOf", "allOf", "items"))
        or schema.get("type") not in (None, "object")
    )
    if structured:
        return schema
    return {
        "type": "object",
        "properties": {
            "value": {
                "description": "Response body; the API defines no response schema."
            }
        },
    }


def _unwrap_schema_envelopes(schema: Any) -> dict[str, Any]:
    """Strip schema envelopes that ``unpack_response`` strips at runtime."""
    if not isinstance(schema, dict):
        return {}
    typed: dict[str, Any] = schema
    if typed.get("type") == "array":
        return _unwrap_schema_envelopes(typed.get("items"))
    choice_rows = _wrapper_choice_rows(typed)
    if choice_rows is not None:
        return _final_row_schema(choice_rows)
    merged = _merge_object_variants(typed)
    if merged is not None:
        return _unwrap_schema_envelopes(merged)
    combinator_target = _first_non_null_combinator_variant(typed)
    if combinator_target is not None:
        return _unwrap_schema_envelopes(combinator_target)
    if _is_scalar_schema(typed):
        return _wrap_scalar_as_value(typed)
    return _unwrap_object_schema(typed)


def _unwrap_object_schema(typed: dict[str, Any]) -> dict[str, Any]:
    """Resolve the row schema for an object schema that has ``properties``."""
    props = typed.get("properties")
    if not isinstance(props, dict) or not props:
        return typed
    if _response_shape(typed)[1] in ("list", "map"):
        # The chosen array's items (or a map's entries) *are* the rows: never
        # look for another envelope inside a row (an alert with one array of
        # records among its fields is still an alert, not that array).
        row = _envelope_inner_schema(typed, props)
        return typed if row is None else _final_row_schema(row)
    inner = _envelope_inner_schema(typed, props)
    if inner is not None:
        return _unwrap_schema_envelopes(inner)
    return typed


def _merge_prop_schemas(a: dict[str, Any], b: dict[str, Any]) -> dict[str, Any]:
    """Combine one property's schema from two variants of a union."""
    if a == b:
        return a
    if a.get("type") == "array" and b.get("type") == "array":
        ia, ib = a.get("items"), b.get("items")
        if isinstance(ia, dict) and isinstance(ib, dict):
            return {**a, "items": _merge_prop_schemas(ia, ib)}
    if (
        isinstance(a.get("enum"), list)
        and isinstance(b.get("enum"), list)
        and a.get("type") == b.get("type")
    ):
        merged = {
            **a,
            "enum": [*a["enum"], *[e for e in b["enum"] if e not in a["enum"]]],
        }
        return merged

    # An enum in one variant and the plain type in another: the plain type admits both.
    def strip(s):
        return {
            k: v for k, v in s.items() if k not in ("enum", "example", "description")
        }

    if strip(a) == strip(b):
        return {k: v for k, v in a.items() if k != "enum"}
    variants: list[dict[str, Any]] = []
    for s in (a, b):
        for v in s.get("anyOf") or [s]:
            if v not in variants:
                variants.append(v)
    return {"anyOf": variants}


def _wrapper_choice_rows(schema: dict[str, Any]) -> dict[str, Any] | None:
    """Row schema of a ``oneOf`` whose variants each wrap one container.

    ``{si: {...}}`` | ``{wi: {...}}`` | ``{fs: {...}}`` means "the response is
    whichever of these keys is present", so each variant's single property is
    unwrapped like any envelope and the rows are the union of what they yield.
    Only containers (arrays or objects) count; a one-field *row* is not a wrapper.
    """
    for comb in ("oneOf", "anyOf"):
        variants = schema.get(comb)
        if not isinstance(variants, list) or len(variants) < 2:
            continue
        rows: list[dict[str, Any]] = []
        for v in variants:
            props = v.get("properties") if isinstance(v, dict) else None
            if not isinstance(props, dict) or len(props) != 1:
                break
            only = next(iter(props.values()))
            if not (
                isinstance(only, dict)
                and (_is_array_schema(only) or only.get("properties"))
            ):
                break
            rows.append(_unwrap_schema_envelopes(only))
        else:
            return {"anyOf": rows}
    return None


def _merge_object_variants(schema: dict[str, Any]) -> dict[str, Any] | None:
    """Merge a ``oneOf`` / ``anyOf`` of object shapes into one object schema.

    The union means "any of these shapes", so the row type carries every
    variant's fields; a field is required only if all variants require it, and
    a property declared differently across variants takes the union of them.
    Returns ``None`` unless the union is made only of two or more objects.
    """
    for comb in ("oneOf", "anyOf"):
        variants = schema.get(comb)
        if not isinstance(variants, list):
            continue
        objs = [v for v in variants if isinstance(v, dict) and v.get("type") != "null"]
        if len(objs) < 2 or not all(v.get("properties") for v in objs):
            continue
        props: dict[str, Any] = {}
        for v in objs:
            for name, prop in v["properties"].items():
                props[name] = (
                    _merge_prop_schemas(props[name], prop)
                    if name in props
                    and isinstance(prop, dict)
                    and isinstance(props[name], dict)
                    else props.get(name, prop)
                )
        required = set.intersection(*(set(v.get("required") or []) for v in objs))
        out: dict[str, Any] = {"type": "object", "properties": props}
        if required:
            out["required"] = sorted(required)
        return out
    return None


def _first_non_null_combinator_variant(schema: dict[str, Any]) -> dict[str, Any] | None:
    """Return the first non-null variant of a top-level ``oneOf`` / ``anyOf``."""
    for combinator in ("oneOf", "anyOf"):
        variants = schema.get(combinator)
        if not isinstance(variants, list):
            continue
        for variant in variants:
            if isinstance(variant, dict) and variant.get("type") != "null":
                return cast(dict[str, Any], variant)
    return None


_LD_KEYS = frozenset({"@context", "@id", "@type", "@vocab", "@version"})


def _is_array_schema(schema: Any) -> bool:
    """Whether a property schema describes an array (directly or via a union)."""
    if not isinstance(schema, dict):
        return False
    if schema.get("type") == "array" or "items" in schema:
        return True
    return any(
        _is_array_schema(v)
        for comb in ("anyOf", "oneOf")
        for v in schema.get(comb) or []
    )


def _array_items(schema: dict[str, Any]) -> dict[str, Any] | None:
    """Return the ``items`` schema of an array property (unwrapping unions)."""
    if isinstance(schema.get("items"), dict):
        return cast(dict[str, Any], schema["items"])
    for comb in ("anyOf", "oneOf"):
        for variant in schema.get(comb) or []:
            if _is_array_schema(variant):
                return _array_items(variant)
    return None


def _is_map_schema(schema: Any) -> bool:
    """Whether a property is a free-form ``{key: value}`` map."""
    return (
        isinstance(schema, dict)
        and schema.get("type") == "object"
        and not schema.get("properties")
        and bool(schema.get("additionalProperties"))
    )


def _has_record_items(schema: dict[str, Any]) -> bool:
    """Whether an array property's items are objects (records), per the schema."""
    items = _array_items(schema)
    return isinstance(items, dict) and not _is_scalar_schema(items)


def _response_shape(response: Any) -> tuple[str | None, str | None]:
    """Decide how a response object maps to rows, from the response schema alone.

    Returns ``(key, kind)``: ``"list"`` when one property is the data array (the
    only array, or the only one whose items are objects; every other property
    is metadata), ``"map"`` when the only property is a free-form
    ``{key: value}`` map (one row per entry), ``"single"`` when the object is
    itself the row, else ``(None, None)``. The data model and the generated
    ``unpack_response`` call both use this, so they cannot disagree.
    """
    if not isinstance(response, dict) or response.get("type") == "array":
        return None, None
    props = response.get("properties")
    if not isinstance(props, dict) or not props or "results" in props:
        return None, None
    if _first_non_null_combinator_variant(response) is not None:
        return None, None
    eff = {k: v for k, v in props.items() if k not in _LD_KEYS}
    arrays = [k for k, v in eff.items() if _is_array_schema(v)]
    if len(eff) > 1:
        # Beside other fields an array of scalars is a field of the row
        # (``securityTypes``), not the rows; it only is the rows when it stands alone.
        arrays = [k for k in arrays if _has_record_items(eff[k])]
    if len(arrays) > 1:
        # e.g. ``@graph`` (station objects) beside ``observationStations`` (the
        # same stations as URL strings): the schema's item types say which.
        arrays = [k for k in arrays if _has_record_items(eff[k])]
    if len(arrays) == 1:
        return arrays[0], "list"
    if len(eff) == 1 and _is_map_schema(next(iter(eff.values()))):
        return next(iter(eff)), "map"
    if _single_prop_inner_schema(props) is not None:
        return None, None  # a one-property wrapper object: unwrapped by the model too
    return None, "single"


def _envelope_inner_schema(
    typed: dict[str, Any],
    props: dict[str, Any],  # noqa: ARG001
) -> dict[str, Any] | None:
    """Resolve the inner row schema when ``typed`` is a recognized envelope."""
    key, kind = _response_shape(typed)
    if key is None or kind is None:
        return _single_prop_inner_schema(props)
    value = props[key]
    if kind == "map":
        extra = value.get("additionalProperties")
        extra = extra if isinstance(extra, dict) else {}
        variants = [
            v
            for v in extra.get("anyOf") or [extra]
            if isinstance(v, dict) and v.get("type") != "null"
        ]
        inner = variants[0] if variants else {}
        row_props: dict[str, Any] = {"key": {"type": "string"}}
        if inner.get("properties"):
            row_props.update(inner["properties"])
        else:
            row_props["value"] = inner
        return {"type": "object", "properties": row_props}
    items = _array_items(value)
    if isinstance(items, dict) and not _is_scalar_schema(items):
        return items
    if isinstance(items, dict):
        return _wrap_scalar_as_value(items)
    return None


def _single_prop_inner_schema(props: dict[str, Any]) -> dict[str, Any] | None:
    """Inner schema of a one-property envelope that is not an array or map."""
    if len(props) == 1:
        only_value = next(iter(props.values()))
        if isinstance(only_value, dict):
            only_dict = cast(dict[str, Any], only_value)
            if _is_array_schema(only_dict):
                items = _array_items(only_dict)
                return items if isinstance(items, dict) else None
            if (
                only_dict.get("properties")
                or only_dict.get("oneOf")
                or only_dict.get("anyOf")
            ):
                return only_dict
    return None


def _unpack_hint(cmd_spec: dict[str, Any]) -> str:
    """Extra ``unpack_response`` arguments derived from the response schema."""
    key, kind = _response_shape(cmd_spec.get("response_schema"))
    if kind is None:
        return ""
    if kind == "single":
        return ', kind="single"'
    return f", rows_key={key!r}, kind={kind!r}"


def _final_row_schema(row: dict[str, Any] | None) -> dict[str, Any]:
    """Resolve the row schema without unwrapping it any further."""
    if not isinstance(row, dict):
        return {}
    if _is_scalar_schema(row):
        return _wrap_scalar_as_value(row)
    merged = _merge_object_variants(row)
    if merged is not None:
        return merged
    variant = _first_non_null_combinator_variant(row)
    if variant is not None:
        return _final_row_schema(variant)
    return row


def _is_scalar_schema(schema: dict[str, Any]) -> bool:
    """Return True when the schema represents a primitive (string/number/etc.)."""
    return schema.get("type") in {
        "string",
        "integer",
        "number",
        "boolean",
    } and not schema.get("properties")


def _wrap_scalar_as_value(schema: dict[str, Any]) -> dict[str, Any]:
    """Promote a scalar schema to ``{type: object, properties: {value: scalar}}``."""
    return {
        "type": "object",
        "properties": {"value": schema},
        "required": ["value"],
    }


def _credential_lookup_lines(
    creds: dict[str, dict[str, str]], provider_name: str
) -> list[str]:
    """Render ``api_key = creds.get("<provider>_api_key", "")`` lines.

    Parameters
    ----------
    creds : dict
        Mapping returned by ``credentials_from_command``.
    provider_name : str
        Snake-case provider name used as the key prefix.

    Returns
    -------
    list of str
        One assignment per credential, in canonical-name order.
    """
    lines: list[str] = []
    for canonical in sorted(creds):
        var = f"_cred_{canonical}"
        full_key = f"{provider_name}_{canonical}"
        lines.append(f'        {var} = _creds.get("{full_key}", "")')
    return lines


def _query_dict_construction(
    cmd_spec: dict[str, Any],
    creds: dict[str, dict[str, str]],
    path_params: list[str],
    provider_name: str,
) -> str:
    """Compose the ``query_dict = {...}`` block emitted in ``aextract_data``.

    Parameters
    ----------
    cmd_spec : dict
        The spec command entry.
    creds : dict
        Credential entries.
    path_params : list of str
        URL-template placeholders.

    Returns
    -------
    str
        The Python source for the ``query_dict`` assignment.
    """
    body_props = (cmd_spec.get("request_body_schema") or {}).get("properties") or {}
    excluded = {query_field_name(n) for n in (*path_params, *body_props)}
    range_params = [
        p
        for p in (cmd_spec.get("parameters") or [])
        if isinstance(p, dict) and p.get("_socrata_op") in {"date_min", "date_max"}
    ]
    excluded.update(query_field_name(p["name"]) for p in range_params)
    if excluded:
        excluded_literal = (
            "{" + ", ".join(repr(name) for name in sorted(excluded)) + "}"
        )
        dump = (
            f"        _query_dict = query.model_dump("
            f"by_alias=True, exclude={excluded_literal}, exclude_none=True)"
        )
    else:
        dump = (
            "        _query_dict = query.model_dump(by_alias=True, exclude_none=True)"
        )
    lines = [dump]
    declares_provider = any(
        isinstance(p, dict) and p.get("name") == "provider"
        for p in (cmd_spec.get("parameters") or [])
    )
    if declares_provider:
        lines.append(f'        _query_dict["provider"] = {provider_name!r}')
    for canonical, info in creds.items():
        if info["in"] != "query":
            continue
        var = f"_cred_{canonical}"
        wire_name = info["name"]
        lines.append(f"        if {var}:")
        lines.append(f"            _query_dict[{wire_name!r}] = {var}")
    if range_params:
        lines.extend(_socrata_where_clause_lines(range_params))
        date_column = next(
            (p["_socrata_column"] for p in range_params if p.get("_socrata_column")),
            None,
        )
        if date_column:
            lines.append(
                f"        _query_dict.setdefault('$order', {date_column!r} + ' DESC')"
            )
    for param in filter_user_params(cmd_spec.get("parameters") or []):
        if (
            isinstance(param, dict)
            and param.get("is_list")
            and param.get("in", "query") == "query"
        ):
            lines.extend(
                _list_param_lines(param["name"], param.get("list_style") or "multi")
            )
    rename_pairs: list[tuple[str, str]] = []
    for param in cmd_spec.get("parameters") or []:
        if not isinstance(param, dict):
            continue
        wire_name = param.get("wire_name")
        if wire_name and wire_name != param.get("name"):
            rename_pairs.append((param["name"], wire_name))
    if rename_pairs:
        lines.append("        for _src, _dst in (")
        for src, dst in rename_pairs:
            lines.append(f"            ({src!r}, {dst!r}),")
        lines.append("        ):")
        lines.append("            if _src in _query_dict:")
        lines.append("                _query_dict[_dst] = _query_dict.pop(_src)")
    return "\n".join(lines)


_LIST_DELIMITERS = {"csv": ",", "space": " ", "pipe": "|"}


def _list_param_lines(wire_name: str, style: str) -> list[str]:
    """Emit lines turning the model's comma-joined string into the wire format.

    ``multi`` becomes a list (``get_querystring`` repeats the key); the other
    styles are re-joined with the delimiter the upstream API expects.
    """
    key = repr(wire_name)
    lines = [
        f"        if isinstance(_query_dict.get({key}), str):",
        f"            _items = [i.strip() for i in _query_dict[{key}].split(',') if i.strip()]",
    ]
    if style == "multi":
        lines.append(f"            _query_dict[{key}] = _items")
    else:
        delimiter = _LIST_DELIMITERS.get(style, ",")
        lines.append(f"            _query_dict[{key}] = {delimiter!r}.join(_items)")
    return lines


def _socrata_where_clause_lines(range_params: list[dict[str, Any]]) -> list[str]:
    """Compose runtime lines that fold Socrata range params into ``$where``."""
    field_lookups: list[str] = []
    for param in range_params:
        wire_name = param["name"]
        column = param["_socrata_column"]
        op = ">=" if param["_socrata_op"] == "date_min" else "<="
        py_field = query_field_name(wire_name)
        var = f"_range_{py_field}"
        field_lookups.append(
            f"        {var} = getattr(query, {py_field!r}, None)\n"
            f"        if {var} is not None:\n"
            f"            _socrata_where_parts.append("
            f'f"{column} {op} \'" + str({var}) + "\'")'
        )
    out: list[str] = ["        _socrata_where_parts: list[str] = []"]
    out.extend(field_lookups)
    out.append("        if _socrata_where_parts:")
    out.append("            _existing_where = _query_dict.pop('$where', None)")
    out.append("            _socrata_where = ' AND '.join(_socrata_where_parts)")
    out.append("            if _existing_where:")
    out.append(
        "                _socrata_where = f'({_existing_where}) AND ({_socrata_where})'"
    )
    out.append("            _query_dict['$where'] = _socrata_where")
    return out


def _header_dict_construction(
    creds: dict[str, dict[str, str]], fixed: dict[str, str] | None = None
) -> str:
    """Compose the ``_headers = {...}`` block for credential headers.

    Parameters
    ----------
    creds : dict
        Credential entries.

    Returns
    -------
    str
        Python source for the headers dict.
    """
    lines = [f"        _headers: dict[str, str] = {dict(fixed or {})!r}"]
    for canonical, info in creds.items():
        if info["in"] != "header":
            continue
        var = f"_cred_{canonical}"
        wire_name = info["name"]
        lines.append(f"        if {var}:")
        lines.append(f"            _headers[{wire_name!r}] = {var}")
    return "\n".join(lines)


def _cookie_dict_construction(creds: dict[str, dict[str, str]]) -> str:
    """Compose the ``_cookies = {...}`` block for credential cookies.

    Parameters
    ----------
    creds : dict
        Credential entries.

    Returns
    -------
    str
        Python source for the cookies dict.
    """
    lines = ["        _cookies: dict[str, str] = {}"]
    for canonical, info in creds.items():
        if info["in"] != "cookie":
            continue
        var = f"_cred_{canonical}"
        wire_name = info["name"]
        lines.append(f"        if {var}:")
        lines.append(f"            _cookies[{wire_name!r}] = {var}")
    return "\n".join(lines)


def _resolve_url_path(api_prefix: str, url_path: str) -> str:
    """Combine api_prefix with url_path; the prefix is dropped if already present."""
    prefix = ("/" + api_prefix.strip("/")) if api_prefix.strip("/") else ""
    path = url_path if url_path.startswith("/") else "/" + url_path
    if prefix and path.startswith(prefix + "/"):
        return path
    if prefix:
        return prefix + path if not path.startswith(prefix) else path
    return path


def generate_fetcher_module(spec: FetcherCommandSpec) -> GeneratedFetcher:
    """Render the full three-class fetcher module for one spec command.

    Parameters
    ----------
    spec : FetcherCommandSpec
        The command-level inputs (see ``FetcherCommandSpec``).

    Returns
    -------
    GeneratedFetcher
        The emitted source plus metadata the package builder needs to
        register the fetcher in the provider's ``fetcher_dict``.
    """
    module_name = _module_name_from_command(spec.name)
    model_name = class_name_from(spec.name)
    qp_class = f"{model_name}QueryParams"
    data_class = f"{model_name}Data"
    fetcher_class = f"{model_name}Fetcher"

    qp_schema = _query_params_schema(spec.cmd_spec, spec.provider_name)
    data_schema = _with_body_field(_data_schema(spec.cmd_spec) or {"type": "object"})

    qp_generated = generate_class(
        qp_schema,
        class_name=qp_class,
        base_class="QueryParams",
        docstring=f"Query parameters for {spec.name}.",
    )
    data_generated = generate_class(
        _mark_null_empty(data_schema),
        class_name=data_class,
        base_class="Data",
        docstring=f"Response row for {spec.name}.",
    )

    creds = credentials_from_command(spec.cmd_spec)
    path_params = _path_template_keys(spec.cmd_spec.get("url_path") or "")

    imports = {
        "import base64",
        "from typing import Any",
        "from openbb_core.app.model.abstract.error import OpenBBError",
        "from openbb_core.provider.abstract.annotated_result import AnnotatedResult",
        "from openbb_core.provider.abstract.data import Data",
        "from openbb_core.provider.abstract.fetcher import Fetcher",
        "from openbb_core.provider.abstract.query_params import QueryParams",
        "from openbb_core.provider.utils.helpers import "
        "get_async_requests_session, get_querystring",
        "from pydantic import BaseModel, Field",
        "from ....utils import safe_json_loads, unpack_response",
    }
    imports.update(qp_generated.collect_imports())
    imports.update(data_generated.collect_imports())

    method = (spec.cmd_spec.get("method") or "get").lower()
    full_url_path = _resolve_url_path(
        spec.api_prefix, spec.cmd_spec.get("url_path") or ""
    )
    base_url = spec.base_url.rstrip("/")
    description = (spec.cmd_spec.get("description") or f"Fetch {spec.name}.").strip()

    cred_lines = _credential_lookup_lines(creds, spec.provider_name)
    query_block = _query_dict_construction(
        spec.cmd_spec, creds, path_params, spec.provider_name
    )
    header_block = _header_dict_construction(creds, spec.headers)
    cookie_block = _cookie_dict_construction(creds)

    if _command_is_streaming(spec.cmd_spec):
        source = _render_streaming_fetcher_source(
            spec=spec,
            qp_class=qp_class,
            qp_classes=qp_generated.flatten(),
            data_class=data_class,
            fetcher_class=fetcher_class,
            method=method,
            url_path_template=full_url_path,
            base_url=base_url,
            cred_lines=cred_lines,
            query_block=query_block,
            header_block=header_block,
            cookie_block=cookie_block,
            description=description,
        )
        return GeneratedFetcher(
            module_name=module_name,
            model_name=model_name,
            query_params_class=qp_class,
            data_class=data_class,
            fetcher_class=fetcher_class,
            source=source,
            credentials_used=creds,
            is_streaming=True,
        )

    hint = _unpack_hint(spec.cmd_spec)
    source = _render_fetcher_source(
        spec=spec,
        qp_class=qp_class,
        qp_classes=qp_generated.flatten(),
        data_class=data_class,
        data_classes=data_generated.flatten(),
        fetcher_class=fetcher_class,
        model_name=model_name,
        method=method,
        url_path_template=full_url_path,
        base_url=base_url,
        path_params=path_params,
        cred_lines=cred_lines,
        query_block=query_block,
        header_block=header_block,
        cookie_block=cookie_block,
        creds=creds,
        description=description,
        imports=consolidate_imports(imports),
        column_metadata=_column_metadata_from_data_schema(data_schema),
    )
    if hint:
        source = source.replace(
            "unpack_response(_payload)", f"unpack_response(_payload{hint})"
        )

    return GeneratedFetcher(
        module_name=module_name,
        model_name=model_name,
        query_params_class=qp_class,
        data_class=data_class,
        fetcher_class=fetcher_class,
        source=source,
        credentials_used=creds,
    )


def _render_streaming_fetcher_source(
    *,
    spec: FetcherCommandSpec,
    qp_class: str,
    qp_classes: list[GeneratedClass],
    data_class: str,
    fetcher_class: str,
    method: str,
    url_path_template: str,
    base_url: str,
    cred_lines: list[str],
    query_block: str,
    header_block: str,
    cookie_block: str,
    description: str,
) -> str:
    """Format a streaming fetcher module source.

    Parameters
    ----------
    spec : FetcherCommandSpec
        Original command input.
    qp_class : str
        Query parameters class name.
    qp_classes : list of GeneratedClass
        Pre-flattened query parameter classes.
    data_class : str
        Streamed row class name.
    fetcher_class : str
        Fetcher class name.
    method : str
        HTTP verb.
    url_path_template : str
        URL path with placeholders intact.
    base_url : str
        Upstream API root, no trailing slash.
    cred_lines : list of str
        Credential-extraction source lines.
    query_block : str
        Pre-rendered query-dict source.
    header_block : str
        Pre-rendered headers-dict source.
    cookie_block : str
        Pre-rendered cookies-dict source.
    description : str
        First paragraph of the spec command's description.

    Returns
    -------
    str
        The full streaming fetcher module source.
    """
    transform_query_doc = render_function_docstring(
        "Validate raw input into typed query parameters.",
        parameters=[("params", "dict[str, Any]", "Raw user input.")],
        returns=(qp_class, "Validated query parameters."),
        indent="        ",
    )
    aextract_doc = render_function_docstring(
        f"Open the stream at {base_url}{url_path_template}.",
        parameters=[
            ("query", qp_class, "Validated query parameters."),
            ("credentials", "dict | None", "Provider credentials."),
            ("**kwargs", "Any", "Forwarded by the provider runtime."),
        ],
        returns=("AsyncIterator[str]", "Async iterator of raw stream lines."),
        indent="        ",
    )
    stream_data_doc = render_function_docstring(
        f"Type each stream line as {data_class}.",
        parameters=[
            ("query", qp_class, "Validated query parameters."),
            ("data", "Any", "Async iterator of raw stream lines."),
            ("**kwargs", "Any", "Forwarded by the provider runtime."),
        ],
        returns=(f"AsyncIterator[{data_class}]", "Async iterator of typed rows."),
        indent="        ",
    )

    parts: list[str] = []
    parts.append(f'"""Streaming fetcher for {spec.name} — generated from spec."""')
    parts.append("")
    parts.append("from collections.abc import AsyncIterator")
    parts.append("from typing import Any")
    parts.append("")
    parts.append("import httpx")
    parts.append("")
    parts.append("from openbb_core.provider.abstract.data import Data")
    parts.append("from openbb_core.provider.abstract.fetcher import Fetcher")
    parts.append("from openbb_core.provider.abstract.query_params import QueryParams")
    parts.append("from openbb_core.provider.utils.helpers import get_querystring")
    parts.append("")
    parts.append("")
    for cls in qp_classes:
        parts.append(cls.source)
    parts.append(f"class {data_class}(Data):")
    parts.append(f'    """Streamed row for {spec.name}."""')
    parts.append("")
    parts.append("    data: str")
    parts.append("")
    parts.append("")
    parts.append(
        f"class {fetcher_class}(Fetcher[{qp_class}, AsyncIterator[{data_class}]]):"
    )
    parts.append(f'    """Streaming fetcher for {spec.name}."""')
    parts.append("")
    parts.append("    @staticmethod")
    parts.append(f"    def transform_query(params: dict) -> {qp_class}:")
    parts.append(transform_query_doc)
    parts.append(f"        return {qp_class}(**params)")
    parts.append("")
    parts.append("    @staticmethod")
    parts.append(
        f"    async def aextract_data(query: {qp_class}, "
        "credentials: dict | None, **kwargs) -> AsyncIterator[str]:"
    )
    parts.append(aextract_doc)
    parts.extend(cred_lines)
    parts.append(query_block)
    parts.append("        _query_string = get_querystring(_query_dict, [])")
    parts.append(
        f'        _url = f"{base_url}{url_path_template}"'
        ' + ("?" + _query_string if _query_string else "")'
    )
    parts.append(header_block)
    parts.append(cookie_block)
    parts.append("")
    parts.append("        async def _stream() -> AsyncIterator[str]:")
    parts.append(
        "            async with httpx.AsyncClient(timeout=None) as _client:  # noqa: S113"
    )
    parts.append(
        f'                async with _client.stream("{method.upper()}", '
        "_url, headers=_headers, cookies=_cookies) as _resp:"
    )
    parts.append("                    _resp.raise_for_status()")
    parts.append("                    async for _line in _resp.aiter_lines():")
    parts.append("                        if _line:")
    parts.append("                            yield _line")
    parts.append("")
    parts.append("        return _stream()")
    parts.append("")
    parts.append("    @staticmethod")
    parts.append(
        f"    async def stream_data(query: {qp_class}, data: Any, **kwargs)"
        f" -> AsyncIterator[{data_class}]:"
    )
    parts.append(stream_data_doc)
    parts.append("        async for _line in data:")
    parts.append(f"            yield {data_class}(data=_line)")
    return "\n".join(parts).rstrip() + "\n"


def _render_fetcher_source(
    *,
    spec: FetcherCommandSpec,
    qp_class: str,
    qp_classes: list[GeneratedClass],
    data_class: str,
    data_classes: list[GeneratedClass],
    fetcher_class: str,
    model_name: str,
    method: str,
    url_path_template: str,
    base_url: str,
    path_params: list[str],
    cred_lines: list[str],
    query_block: str,
    header_block: str,
    cookie_block: str,
    creds: dict[str, dict[str, str]],
    description: str,
    imports: list[str],
    column_metadata: dict[str, dict[str, Any]] | None = None,
) -> str:
    """Format the final fetcher module source.

    Parameters
    ----------
    spec : FetcherCommandSpec
        Original command input.
    qp_class, data_class, fetcher_class : str
        Class names emitted in the module.
    qp_classes, data_classes : list of GeneratedClass
        Pre-flattened lists of emitted classes.
    model_name : str
        The Pydantic model key referenced by ``@router.command(model=...)``.
    method : str
        HTTP verb.
    url_path_template : str
        URL path with placeholders intact.
    base_url : str
        Upstream API root, no trailing slash.
    path_params : list of str
        Names of placeholders in ``url_path_template``.
    cred_lines : list of str
        Credential-extraction source lines.
    query_block, header_block, cookie_block : str
        Pre-rendered query-dict, headers-dict, and cookies-dict source.
    creds : dict
        Credential entries.
    description : str
        First paragraph of the spec command's description.
    imports : list of str
        All imports the module needs, already sorted.

    Returns
    -------
    str
        The full module source text.
    """
    stdlib_imports = [
        i
        for i in imports
        if i.startswith(
            ("from typing", "from datetime", "import typing", "import datetime")
        )
    ]
    third_party_imports = [i for i in imports if i not in stdlib_imports]

    parts: list[str] = []
    parts.append(
        f'"""Fetcher for {spec.name} — generated from spec.\n\n'
        f"Hits ``{base_url}{url_path_template}`` via HTTP "
        f'{method.upper()}.\n"""'
    )
    parts.append("")
    if stdlib_imports:
        parts.extend(stdlib_imports)
        parts.append("")
    if third_party_imports:
        parts.extend(third_party_imports)
        parts.append("")
    parts.append("")

    for cls in qp_classes:
        parts.append(cls.source)
    for cls in data_classes:
        parts.append(cls.source)

    fetcher_doc = render_function_docstring(
        description.split("\n", 1)[0],
    )

    transform_query_doc = render_function_docstring(
        "Validate raw input into typed query parameters.",
        parameters=[
            ("params", "dict[str, Any]", "Raw user input (CLI flags / API body)."),
        ],
        returns=(qp_class, "Validated query parameters."),
        indent="        ",
    )

    aextract_doc_params: list[tuple[str, str, str | None]] = [
        ("query", qp_class, "Validated query parameters."),
        (
            "credentials",
            "dict[str, str] | None",
            "Provider credentials registered on the ``Provider`` instance.",
        ),
        ("**kwargs", "Any", "Forwarded by the provider runtime; unused."),
    ]
    aextract_doc = render_function_docstring(
        f"Fetch from {base_url}{url_path_template} and split rows from metadata.",
        parameters=aextract_doc_params,
        returns=(
            "dict[str, Any]",
            (
                "``{'rows': [...], 'metadata': {...}}`` — single-element list "
                "and single-key envelope wrappers are stripped at the response "
                "boundary, scalar fields beside the data array become metadata."
            ),
        ),
        indent="        ",
    )

    transform_data_doc = render_function_docstring(
        f"Type the unpacked rows as {data_class}; surface metadata via AnnotatedResult.",
        parameters=[
            ("query", qp_class, "The validated query (unused but provided)."),
            ("data", "dict[str, Any]", "Output of ``aextract_data``."),
            ("**kwargs", "Any", "Forwarded by the provider runtime; unused."),
        ],
        returns=(
            f"list[{data_class}] | AnnotatedResult[list[{data_class}]]",
            (
                "Typed rows; wrapped in ``AnnotatedResult`` when the response "
                "carried metadata alongside the data array."
            ),
        ),
        indent="        ",
    )

    has_path_params = bool(path_params)
    has_query_creds = any(info["in"] == "query" for info in creds.values())

    body_construction = _aextract_body(
        method=method,
        base_url=base_url,
        url_path_template=url_path_template,
        path_params=path_params,
        cred_lines=cred_lines,
        query_block=query_block,
        header_block=header_block,
        cookie_block=cookie_block,
        body_field_names=list(
            (spec.cmd_spec.get("request_body_schema") or {}).get("properties") or {}
        ),
        has_path_params=has_path_params,
        has_query_creds=has_query_creds,
    )

    fetcher_source_lines = [
        f"class {fetcher_class}(Fetcher[{qp_class}, list[{data_class}]]):",
        fetcher_doc,
        "",
        "    @staticmethod",
        f"    def transform_query(params: dict[str, Any]) -> {qp_class}:",
        transform_query_doc,
        f"        return {qp_class}(**params)",
        "",
        "    @staticmethod",
        "    async def aextract_data(",
        f"        query: {qp_class},",
        "        credentials: dict[str, str] | None,",
        "        **kwargs: Any,",
        "    ) -> dict[str, Any]:",
        aextract_doc,
        "        _creds = credentials or {}",
    ]
    fetcher_source_lines.extend(body_construction)
    column_meta_literal = repr(column_metadata) if column_metadata else None
    transform_data_lines = [
        "",
        "    @staticmethod",
        "    def transform_data(",
        f"        query: {qp_class},",
        "        data: dict[str, Any],",
        "        **kwargs: Any,",
        f"    ) -> list[{data_class}] | AnnotatedResult[list[{data_class}]]:",
        transform_data_doc,
        f"        _typed = [{data_class}(**row) for row in data['rows']]",
        "        _metadata = dict(data.get('metadata') or {})",
    ]
    if column_meta_literal:
        transform_data_lines.append(f"        _column_metadata = {column_meta_literal}")
        transform_data_lines.append(
            "        _metadata.setdefault('columns', _column_metadata)"
        )
    transform_data_lines.extend(
        [
            "        if _metadata:",
            "            return AnnotatedResult(result=_typed, metadata=_metadata)",
            "        return _typed",
            "",
        ]
    )
    fetcher_source_lines.extend(transform_data_lines)
    parts.append("\n".join(fetcher_source_lines))

    return "\n".join(parts).rstrip() + "\n"


def _aextract_body(
    *,
    method: str,
    base_url: str,
    url_path_template: str,
    path_params: list[str],
    cred_lines: list[str],
    query_block: str,
    header_block: str,
    cookie_block: str,
    body_field_names: list[str],
    has_path_params: bool,
    has_query_creds: bool,
) -> list[str]:
    """Render the body of ``aextract_data`` as a list of source lines.

    Parameters
    ----------
    method : str
        ``"get"`` or ``"post"``.
    base_url : str
        Upstream API root, no trailing slash.
    url_path_template : str
        URL path with ``{placeholder}`` substitutions intact.
    path_params : list of str
        Placeholder names in ``url_path_template``.
    cred_lines : list of str
        Credential-extraction source lines.
    query_block : str
        Pre-rendered query-dict construction block.
    header_block : str
        Pre-rendered headers-dict construction block.
    cookie_block : str
        Pre-rendered cookies-dict construction block.
    body_field_names : list of str
        Names of fields that should travel in the JSON request body.
    has_path_params : bool
        Whether the URL has placeholders to substitute.
    has_query_creds : bool
        Whether any credential lives in the query string.

    Returns
    -------
    list of str
        Indented lines forming the body of ``aextract_data``.
    """
    lines: list[str] = list(cred_lines)
    if has_path_params:
        sub_args = ", ".join(
            f"{p}=getattr(query, {query_field_name(p)!r})" for p in path_params
        )
        lines.append(f"        _path = {url_path_template!r}.format({sub_args})")
    else:
        lines.append(f"        _path = {url_path_template!r}")
    lines.append("")
    lines.append(query_block)
    lines.append("        _query_string = get_querystring(_query_dict, [])")
    lines.append(
        f'        _url = f"{base_url}{{_path}}" + ("?" + _query_string if _query_string else "")'
    )
    lines.append("")
    lines.append(header_block)
    lines.append("")
    lines.append(cookie_block)
    lines.append("")
    if method == "post":
        body_literal = (
            "{"
            + ", ".join(
                f"{name!r}: getattr(query, {query_field_name(name)!r}, None)"
                for name in body_field_names
            )
            + "}"
        )
        lines.append(f"        _body = {body_literal}")
        lines.append('        _method = "POST"')
        lines.append('        _request_kwargs: dict[str, Any] = {"json": _body}')
    else:
        lines.append('        _method = "GET"')
        lines.append("        _request_kwargs: dict[str, Any] = {}")
    lines.append("        async with await get_async_requests_session() as _session:")
    lines.append(
        "            async with await _session.request("
        "_method, _url, headers=_headers, cookies=_cookies, **_request_kwargs) as _resp:"
    )
    lines.append(
        "                _ct = (_resp.headers.get('Content-Type') or '').lower()"
    )
    lines.append("                _raw = await _resp.read()")
    lines.append(
        "                _textual = not _ct or any("
        "_t in _ct for _t in ('json', 'text', 'xml', 'csv', 'javascript', 'yaml'))"
    )
    lines.append("                if _resp.status < 400 and not _textual:")
    lines.append(
        "                    return {'rows': [{'value': "
        "base64.b64encode(_raw).decode('ascii')}], "
        "'metadata': {'content_type': _ct, 'encoding': 'base64'}}"
    )
    lines.append(
        "                _text = _raw.decode(_resp.get_encoding() or 'utf-8', errors='replace')"
    )
    lines.append("                if _resp.status >= 400:")
    lines.append(
        "                    raise OpenBBError("
        "f'HTTP {_resp.status} from {_url}: {_text[:500]}')"
    )
    lines.append(
        "                if 'json' in _ct or _text.lstrip().startswith(('{', '[')):"
    )
    lines.append("                    try:")
    lines.append("                        _payload = safe_json_loads(_text)")
    lines.append("                    except ValueError as _exc:")
    lines.append(
        "                        raise OpenBBError("
        "f'Upstream {_url} returned malformed JSON: {_exc}') from _exc"
    )
    lines.append("                else:")
    lines.append(
        "                    return {'rows': [{'value': _text}], "
        "'metadata': {'content_type': _ct}}"
    )
    lines.append("        _rows, _metadata = unpack_response(_payload)")
    lines.append("        return {'rows': _rows, 'metadata': _metadata}")
    return lines