STT-Assistant/src/stt_daemon/assistant/triage.py

"""Triage analyzer: pure, import-light classification helpers.

Stdlib only. No ML imports here (not even lazily): the dedicated lane module
owns the model import, background thread, and inference. This module only
validates the questions file, condenses transcript windows, normalizes raw
answers into display items, decides escalation, and builds/parses/finalizes
the plain-text brief. The only I/O is the one-time questions file read in the
constructor.

Public API (consumed by the assistant service layer):

    TRIAGE_REASONS = ("ok", "no_questions", "loading", "laya_missing",
                      "disabled", "error")

    def load_questions(path: str) -> dict | None

    class TriageAnalyzer:
        def __init__(self, cfg) -> None
        @property
        def questions(self) -> dict | None
        def build_state(self, window) -> str
        def normalize(self, payload) -> dict
        def should_escalate(self, signals) -> bool
        def build_brief_prompt(self, deltas, signals) -> str
        def build_ledger_prompt(self, narrative, ledger, deltas, signals) -> str
        def parse_brief(self, text) -> str | None
        def parse_ledger_reply(self, text, truncated=False) -> tuple[str, list[str]] | None
        def parse_facts(self, block) -> list[str]
        def should_update_brief(self, signals) -> bool
        def finalize_brief(self, text, finish_reason) -> str | None
        def state(self) -> tuple[str, str]
        def refresh(self, state: str, reason: str = "") -> None

    class FactLedger:
        def __init__(self, max_facts=200, fact_max_chars=240) -> None
        def render(self) -> str
        def merge(self, new_facts) -> int
        def facts(self) -> list[str]
        def clear(self) -> None
        @classmethod
        def from_brief(cls, brief, ...) -> "FactLedger"

``cfg`` is duck-typed (attribute access only, never imported): it provides
``questions_file``, ``state_char_budget`` (default 3000),
``confidence_field`` (default ``"answer_confidence"``),
``escalate_threshold`` (default 0.6), ``choice_min_confidence``
(default 0.85), ``brief_max_chars`` (default 4000) and
``brief_gate_threshold`` (default 0.5).
"""

import json
import logging
import re
import unicodedata

logger = logging.getLogger(__name__)

TRIAGE_REASONS = ("ok", "no_questions", "loading", "laya_missing", "disabled", "error")

_VALID_TYPES = ("choice", "score", "noul")


def _invalid(path, qid, why):
    logger.warning("ignoring questions file %r: question %r: %s", path, qid, why)
    return None


def load_questions(path: str) -> dict | None:
    """Load and validate a bare Laya questions dict; None (never raise) on failure."""
    try:
        with open(path, encoding="utf-8") as handle:
            data = json.load(handle)
    except (OSError, ValueError, UnicodeError) as exc:
        logger.warning("ignoring questions file %r: cannot read/parse (%s)", path, exc)
        return None
    if not isinstance(data, dict) or not data:
        logger.warning("ignoring questions file %r: top level must be a non-empty object", path)
        return None
    for qid, qdef in data.items():
        if not isinstance(qdef, dict):
            return _invalid(path, qid, "definition must be an object")
        qtype = qdef.get("type")
        if qtype not in _VALID_TYPES:
            return _invalid(
                path, qid, "unknown type %r; use one of %s" % (qtype, sorted(_VALID_TYPES))
            )
        if "instructions" not in qdef:
            return _invalid(path, qid, "no 'instructions'; add the text the model should answer")
        crit = qdef.get("criteria")
        if qtype == "choice":
            if not isinstance(crit, (dict, list)):
                return _invalid(path, qid, "a choice question takes 'criteria' as a dict or list")
            if not crit:
                return _invalid(path, qid, "a choice question needs at least one criterion")
        elif qtype == "score":
            if not isinstance(crit, list):
                return _invalid(path, qid, "a score question takes 'criteria' as a list")
            if not crit:
                return _invalid(path, qid, "a score question needs at least one level")
            if None in crit:
                return _invalid(path, qid, "score criteria must not contain null")
        elif crit is not None:
            if not isinstance(crit, dict):
                return _invalid(path, qid, "a noul question takes 'criteria' as a dict or omits it")
            keys = {str(key).lower() for key in crit}
            if not keys <= {"true", "false"}:
                return _invalid(
                    path, qid, "a noul question takes 'criteria' keyed only 'true'/'false'"
                )
    return data


def _number(value):
    if isinstance(value, bool):
        return None
    try:
        result = float(value)
    except (TypeError, ValueError):
        return None
    return result


class TriageAnalyzer:
    """Pure analyzer over a duck-typed triage config (see module docstring)."""

    def __init__(self, cfg) -> None:
        self._cfg = cfg
        path = getattr(cfg, "questions_file", "questions.json")
        try:
            questions = load_questions(path)
        except Exception as exc:  # load_questions never raises; belt and braces
            logger.warning("ignoring questions file %r: unexpected error (%s)", path, exc)
            questions = None
        self._questions = questions
        if questions is None:
            self._state = "no_questions"
            self._reason = "no questions loaded from %s" % (path,)
        else:
            self._state = "loading"
            self._reason = ""

    @property
    def questions(self) -> dict | None:
        return self._questions

    def _budget(self) -> int:
        try:
            return int(getattr(self._cfg, "state_char_budget", 3000))
        except (TypeError, ValueError):
            return 3000

    def _confidence_field(self) -> str:
        return getattr(self._cfg, "confidence_field", "answer_confidence")

    def _escalate_threshold(self) -> float:
        try:
            return float(getattr(self._cfg, "escalate_threshold", 0.6))
        except (TypeError, ValueError):
            return 0.6

    def _choice_min_confidence(self) -> float:
        try:
            return float(getattr(self._cfg, "choice_min_confidence", 0.85))
        except (TypeError, ValueError):
            return 0.85

    def build_state(self, window) -> str:
        """Condense oldest->newest utterances newest-first, restore order, "- " lines."""
        utterances = [str(item) for item in (window or [])]
        if not utterances:
            return ""
        budget = self._budget()
        rendered = ["- " + text for text in utterances]
        kept = []
        total = 0
        for line in reversed(rendered):
            extra = len(line) + (1 if kept else 0)
            if kept and total + extra > budget:
                break
            kept.append(line)
            total += extra
        kept.reverse()
        return "\n".join(kept)

    def _normalize_item(self, qid, qdef, answer):
        if not isinstance(answer, dict):
            return None
        qtype = qdef.get("type")
        field = self._confidence_field()
        if qtype == "choice":
            label = answer.get("choice")
            if not isinstance(label, str):
                return None
            crit = qdef.get("criteria")
            if isinstance(crit, dict):
                options = list(crit.keys())
            elif isinstance(crit, list):
                options = list(crit)
            else:
                return None
            if options and label not in options:
                return None
            confidence = _number(answer.get("confidence"))
            answer_confidence = _number(answer.get("answer_confidence"))
            if confidence is None or answer_confidence is None:
                return None
            gate = confidence if field == "confidence" else answer_confidence
            return {
                "kind": "choice",
                "choice": label,
                "confidence": confidence,
                "answer_confidence": answer_confidence,
                "gate": gate,
                "flagged": gate < self._choice_min_confidence(),
            }
        if qtype == "score":
            expected = _number(answer.get("score"))
            crit = qdef.get("criteria")
            if expected is None or not isinstance(crit, list) or not crit:
                return None
            level = max(0, min(len(crit) - 1, round(expected)))
            label = None
            legend = answer.get("legend")
            if isinstance(legend, dict) and str(level) in legend:
                label = legend[str(level)]
            else:
                label = crit[level]
            confidence = _number(answer.get("confidence"))
            answer_confidence = _number(answer.get("answer_confidence"))
            if confidence is None or answer_confidence is None:
                return None
            gate = confidence if field == "confidence" else answer_confidence
            return {
                "kind": "score",
                "expected": expected,
                "level": level,
                "label": str(label),
                "confidence": confidence,
                "answer_confidence": answer_confidence,
                "gate": gate,
                "flagged": expected >= (len(crit) - 1) * 0.5,
            }
        if qtype == "noul":
            prob = _number(answer.get("noul"))
            if prob is None:
                return None
            confidence = _number(answer.get("confidence"))
            answer_confidence = _number(answer.get("answer_confidence"))
            if confidence is None or answer_confidence is None:
                return None
            gate = confidence if field == "confidence" else answer_confidence
            return {
                "kind": "noul",
                "p": prob,
                "confidence": confidence,
                "answer_confidence": answer_confidence,
                "gate": gate,
                "flagged": prob >= self._escalate_threshold(),
            }
        return None

    def normalize(self, payload) -> dict:
        """Normalize a full laya result or a bare answers dict into signals."""
        result = {"order": [], "items": {}, "escalated": False, "input_tokens": 0}
        if not isinstance(payload, dict):
            return result
        raw_answers = payload.get("answers")
        if isinstance(raw_answers, dict):
            answers = raw_answers
            usage = payload.get("usage")
            if isinstance(usage, dict):
                tokens = _number(usage.get("input_tokens", 0))
                result["input_tokens"] = int(tokens) if tokens is not None else 0
        else:
            answers = payload
        questions = self._questions or {}
        for qid in questions:
            if qid not in answers:
                continue
            try:
                item = self._normalize_item(qid, questions[qid], answers[qid])
            except Exception:
                continue
            if item is None:
                continue
            result["order"].append(qid)
            result["items"][qid] = item
        result["escalated"] = self.should_escalate(result)
        return result

    def _score_levels(self, qid):
        questions = self._questions or {}
        qdef = questions.get(qid)
        if not isinstance(qdef, dict):
            return None
        crit = qdef.get("criteria")
        if not isinstance(crit, list) or not crit:
            return None
        return len(crit)

    def should_escalate(self, signals) -> bool:
        """True iff any arm fires: noul p, low choice gate, or high score expected."""
        items = signals.get("items", {}) if isinstance(signals, dict) else {}
        if not isinstance(items, dict):
            return False
        for qid, item in items.items():
            if not isinstance(item, dict):
                continue
            kind = item.get("kind")
            if kind == "noul":
                prob = _number(item.get("p"))
                if prob is not None and prob >= self._escalate_threshold():
                    return True
            elif kind == "choice":
                gate = _number(item.get("gate"))
                if gate is not None and gate < self._choice_min_confidence():
                    return True
            elif kind == "score":
                expected = _number(item.get("expected"))
                if expected is None:
                    continue
                levels = self._score_levels(qid)
                if levels is None:
                    if item.get("flagged") is True:
                        return True
                elif expected >= (levels - 1) * 0.5:
                    return True
        return False

    @staticmethod
    def _clip(text, limit=160) -> str:
        """Collapse whitespace and truncate to ~limit chars with an ellipsis."""
        try:
            collapsed = " ".join(str(text).split())
        except Exception:
            return ""
        if len(collapsed) <= limit:
            return collapsed
        return collapsed[: max(0, limit - 1)].rstrip() + "…"

    def _signal_words(self, qid, item, qdef=None, escalated=False) -> str:
        try:
            kind = item.get("kind") if isinstance(item, dict) else None
        except Exception:
            return "%s: flagged" % (qid,)
        try:
            questions = self._questions or {}
            if qdef is None and isinstance(questions, dict):
                candidate = questions.get(qid)
                if isinstance(candidate, dict):
                    qdef = candidate
            instructions = ""
            if isinstance(qdef, dict):
                raw = qdef.get("instructions")
                if isinstance(raw, str) and raw.strip():
                    instructions = self._clip(raw.strip())
            suffix = " [escalated]" if (escalated and isinstance(item, dict)
                                        and item.get("flagged") is True) else ""
            if kind == "choice":
                choice = item.get("choice")
                gate = _number(item.get("gate"))
                flagged = item.get("flagged") is True
                conf = "%.2f" % gate if gate is not None else "?"
                head = "%s (choice): selected %s (conf %s%s)" % (
                    qid, choice, conf, ", flagged as uncertain" if flagged else "")
                label_def = ""
                try:
                    crit = qdef.get("criteria") if isinstance(qdef, dict) else None
                    if isinstance(crit, dict) and isinstance(choice, str):
                        definition = crit.get(choice)
                        if isinstance(definition, str) and definition.strip():
                            label_def = self._clip(definition.strip())
                except Exception:
                    label_def = ""
                tail = " — %s" % instructions if instructions else ""
                if label_def:
                    tail += " Options: %s" % label_def if not tail else " Options: %s" % label_def
                    if not instructions:
                        tail = " Options: %s" % label_def
                return head + tail + suffix
            if kind == "score":
                expected = _number(item.get("expected"))
                level = item.get("level")
                label = item.get("label")
                flagged = item.get("flagged") is True
                exp = "%.2f" % expected if expected is not None else "?"
                head = "%s (score): expected %s, level %s (%s)%s" % (
                    qid, exp, level, label, ", flagged" if flagged else "")
                if instructions:
                    head += " — %s" % instructions
                return head + suffix
            if kind == "noul":
                prob = _number(item.get("p"))
                gate = _number(item.get("gate"))
                flagged = item.get("flagged") is True
                prob_s = "%.2f" % prob if prob is not None else "?"
                gate_s = ", gate=%.2f" % gate if gate is not None else ""
                head = "%s (p=%s%s%s)" % (
                    qid, prob_s, ", flagged" if flagged else "", gate_s)
                if instructions:
                    head += ": %s" % instructions
                return head + suffix
            return "%s: flagged" % (qid,)
        except Exception:
            try:
                return "%s: flagged" % (qid,)
            except Exception:
                return "signal: flagged"

    def _brief_max_chars(self) -> int:
        try:
            value = int(getattr(self._cfg, "brief_max_chars", 4000))
        except (TypeError, ValueError):
            return 4000
        return value if value > 0 else 4000

    def _brief_gate_threshold(self) -> float:
        try:
            return float(getattr(self._cfg, "brief_gate_threshold", 0.5))
        except (TypeError, ValueError):
            return 0.5

    def _signal_lines(self, signals) -> list[str]:
        """Render each ordered signal as a "- " words line; never raises.

        Filters low-value noise: noul items with p < 0.30 are omitted,
        except the ``substantive_update`` gate arm which is always kept for
        threshold context. Choice/score items are always kept (flagged
        marker explains uncertainty). Original order is preserved.
        """
        lines = []
        try:
            items = signals.get("items", {}) if isinstance(signals, dict) else {}
            order = signals.get("order", []) if isinstance(signals, dict) else []
            escalated = signals.get("escalated") is True if isinstance(signals, dict) else False
            if not isinstance(items, dict):
                return []
            if not isinstance(order, list):
                return []
            questions = self._questions if isinstance(self._questions, dict) else {}
            for qid in order:
                try:
                    item = items.get(qid)
                    if not isinstance(item, dict):
                        continue
                    if item.get("kind") == "noul" and qid != "substantive_update":
                        prob = _number(item.get("p"))
                        if prob is not None and prob < 0.30:
                            continue
                    qdef = questions.get(qid) if isinstance(questions, dict) else None
                    lines.append(
                        "- " + self._signal_words(qid, item, qdef=qdef,
                                                  escalated=escalated))
                except Exception:
                    continue
        except Exception:
            return []
        return lines

    def build_brief_prompt(self, deltas, signals) -> str:
        """Render new speaker-labeled utterances + focus words; no JSON keys."""
        try:
            body = "\n".join("- " + delta for delta in (deltas or []))
        except Exception:
            body = ""
        parts = ["New utterances:\n" + body]
        focus = self._signal_lines(signals)
        if focus:
            parts.append("Focus:\n" + "\n".join(focus))
        parts.append("Return only the updated brief as plain text.")
        return "\n\n".join(parts)

    def build_ledger_prompt(self, narrative, ledger, deltas, signals) -> str:
        """Ledger-aware single-turn prompt: volatile narrative + code-owned ledger.

        The model is shown the current narrative (the sections it owns) and the
        current ledger (facts it must NOT repeat) and asked for the two halves
        back in fixed machine-parseable sections. ``NEW FACTS:`` is placed first
        so a length cutoff loses narrative (regenerated every pass) rather than a
        durable fact.
        """
        try:
            body = "\n".join("- " + delta for delta in (deltas or []))
        except Exception:
            body = ""
        ledger_text = "\n".join("- " + fact for fact in (ledger or [])) or "(none yet)"
        parts = [
            "CURRENT LEDGER (facts already recorded - never repeat these):\n" + ledger_text,
            "CURRENT BRIEF:\n" + (narrative or "(none yet)"),
            "NEW UTTERANCES:\n" + body,
        ]
        focus = self._signal_lines(signals)
        if focus:
            parts.append("Focus:\n" + "\n".join(focus))
        parts.append(
            "Reply with exactly these two sections and nothing else:\n"
            "NEW FACTS:\n"
            "- only genuinely new durable facts, one per line; write 'none' if there are none\n"
            "BRIEF:\n"
            "Summary:\n...\nOpportunities:\n...\nBlockers/Risks:\n...\n"
            "Decisions:\n...\nOpen Questions:\n..."
        )
        return "\n\n".join(parts)

    def parse_ledger_reply(self, text, truncated: bool = False) -> tuple[str, list[str]] | None:
        """Split a ledger reply into ``(narrative, new_facts)``.

        Both halves are optional and fail open: a missing ``NEW FACTS:`` block
        yields no facts, and a missing ``BRIEF:`` block yields an empty
        narrative (the caller then keeps the prior narrative). Any ``Prior
        Details:`` the model volunteers is dropped from the narrative so the
        code-rendered ledger is never duplicated. When ``truncated`` the reply hit
        the token cap, so the final fact line may be cut mid-sentence and is
        discarded. Returns None only when the reply carries no usable text at all.
        """
        cleaned = self.parse_brief(text)
        if cleaned is None:
            return None
        facts_at = cleaned.upper().find("NEW FACTS:")
        brief_at = cleaned.upper().find("BRIEF:")
        if facts_at == -1 and brief_at == -1:
            return None
        if facts_at != -1 and (brief_at == -1 or facts_at < brief_at):
            facts_end = brief_at if brief_at != -1 else len(cleaned)
            facts_block = cleaned[facts_at + len("NEW FACTS:"): facts_end]
            narrative_block = cleaned[brief_at + len("BRIEF:"):] if brief_at != -1 else ""
        else:
            facts_block = ""
            narrative_block = cleaned[brief_at + len("BRIEF:"):]
        facts = self.parse_facts(facts_block)
        if truncated and facts:
            facts = facts[:-1]
        narrative = self._strip_prior_details(narrative_block).strip()
        return narrative, facts

    def parse_facts(self, block) -> list[str]:
        """Extract durable facts from a ``NEW FACTS:`` block; never treats prose as a fact."""
        if not isinstance(block, str):
            return []
        facts = []
        for line in block.splitlines():
            stripped = line.strip()
            if not stripped or stripped.startswith("#"):
                continue
            marker = next((m for m in ("- ", "* ", "• ") if stripped.startswith(m)), None)
            if marker is not None:
                candidate = stripped[len(marker):]
            elif stripped[:2].rstrip(".").isdigit() or stripped[:3].rstrip(".").isdigit():
                candidate = stripped.split(".", 1)[1] if "." in stripped else stripped
            else:
                continue
            candidate = candidate.strip().strip("`").strip()
            if not candidate or candidate.lower() in ("none", "-"):
                continue
            if ":" in candidate and candidate.lower().startswith("new facts"):
                continue
            facts.append(candidate)
        return facts

    @staticmethod
    def _strip_prior_details(text: str) -> str:
        """Drop a model-supplied ``Prior Details:`` section from the narrative."""
        if not isinstance(text, str):
            return ""
        heads = (
            "Summary:", "Opportunities:", "Blockers/Risks:", "Responsibilities:",
            "Decisions:", "Action Items:", "Deadlines:", "Open Questions:",
        )
        at = text.find("Prior Details:")
        if at == -1:
            return text
        rest = text[at + len("Prior Details:"):]
        cuts = [rest.find(h) for h in heads if rest.find(h) != -1]
        end = min(cuts) if cuts else len(rest)
        return (text[:at] + rest[end:]).strip()

    @staticmethod
    def _normalize_bullet(text: str) -> str:
        """Collapse a bullet to a comparison key; empty when it carries no words."""
        lowered = " ".join(str(text).strip().lower().split())
        for prefix in ("- **new:**", "- new:", "**new:**", "new:"):
            if lowered.startswith(prefix):
                lowered = lowered[len(prefix):].strip()
                break
        return lowered.strip("-* ").strip()

    def extract_bullets(self, brief) -> list[str]:
        """Return the normalized bullet keys of every '- ' line in ``brief``."""
        if not isinstance(brief, str):
            return []
        keys = []
        for line in brief.splitlines():
            stripped = line.strip()
            if not stripped.startswith("- "):
                continue
            key = self._normalize_bullet(stripped)
            if key:
                keys.append(key)
        return keys

    def restore_missing_bullets(self, new_brief, prior_brief) -> str:
        """Re-append any prior bullet that the model dropped from ``new_brief``.

        Deterministic retention guard kept as a belt-and-braces layer beneath the
        code-owned ledger: a previously-recorded bullet must never vanish because
        the model re-emitted the document without it. Bullets the new brief merely
        reworded still match by normalized key, so only a genuinely absent bullet
        is restored. Restoration appends under the brief's final ``Prior
        Details:`` section, or at the end when absent.
        """
        if not isinstance(new_brief, str) or not new_brief.strip():
            return new_brief
        if not isinstance(prior_brief, str) or not prior_brief.strip():
            return new_brief
        present = set(self.extract_bullets(new_brief))
        missing = []
        for line in prior_brief.splitlines():
            stripped = line.strip()
            if not stripped.startswith("- "):
                continue
            key = self._normalize_bullet(stripped)
            if not key or key in present:
                continue
            present.add(key)
            missing.append(stripped[2:].strip())
        if not missing:
            return new_brief
        restored = "\n".join("- " + bullet for bullet in missing)
        marker = "Prior Details:"
        if marker in new_brief:
            head, _, tail = new_brief.rpartition(marker)
            return f"{head}{marker}{tail.rstrip()}\n{restored}"
        return f"{new_brief.rstrip()}\n\nPrior Details:\n{restored}"

    @staticmethod
    def _first_arg_start(text: str) -> int:
        name_at = text.find("updated_brief")
        if name_at != -1:
            cursor = name_at + len("updated_brief")
            while cursor < len(text) and text[cursor] == " ":
                cursor += 1
            if cursor < len(text) and text[cursor] == "=":
                return cursor + 1
        call_at = text.find("update_brief(")
        if call_at == -1:
            call_at = text.find("update_brief (")
        return call_at + len("update_brief(") if call_at != -1 else 0

    @staticmethod
    def _first_quote(text: str, start: int) -> tuple[int, str]:
        opening = -1
        quote = "'"
        for candidate in ("'", '"'):
            index = text.find(candidate, start)
            if index != -1 and (opening == -1 or index < opening):
                opening, quote = index, candidate
        return opening, quote

    @staticmethod
    def _arg_boundary_quote(text: str, opening: int, quote: str) -> int:
        index = opening + 1
        while True:
            candidate = text.find(quote, index)
            if candidate == -1:
                return -1
            after = candidate + 1
            while after < len(text) and text[after] == " ":
                after += 1
            if after >= len(text) or text[after] in ",)]<":
                return candidate
            index = candidate + 1

    def _unwrap_tool_call(self, text: str) -> str | None:
        """Unwrap a stray tool-call envelope into its first plain argument value.

        Only fires when an explicit wrapper marker is present so clean markdown
        containing ``[``, ``(``, or ``'`` is passed through untouched. The first
        argument is taken up to its closing quote boundary, so a multi-argument
        call never swallows the separators; an envelope that cannot be reduced to
        a clean payload is rejected (None) rather than stored verbatim.
        """
        markers = ("<|tool_call_start|>", "<|tool_call_end|>", "update_brief(", "update_brief (")
        if not any(marker in text for marker in markers):
            return text
        if "<|tool_call_start|>" in text:
            text = text.split("<|tool_call_start|>", 1)[1]
        if "<|tool_call_end|>" in text:
            text = text.split("<|tool_call_end|>", 1)[0]
        text = text.strip()
        if "update_brief(" not in text and "update_brief (" not in text:
            if any(token in text for token in ("<|", "previous_brief=", "mode=")):
                return None
            return text
        opening, quote = self._first_quote(text, self._first_arg_start(text))
        if opening == -1:
            return None
        closing = self._arg_boundary_quote(text, opening, quote)
        if closing == -1:
            return None
        value = text[opening + 1 : closing].strip()
        if any(token in value for token in ("<|", "previous_brief=", "mode=")):
            return None
        return value

    def parse_brief(self, text) -> str | None:
        """Strip a code fence and any stray tool-call envelope; None when empty."""
        if not isinstance(text, str):
            return None
        cleaned = text.strip()
        if not cleaned:
            return None
        if cleaned.startswith("```"):
            _, _, rest = cleaned.partition("\n")
            cleaned = rest if rest else ""
            end = cleaned.rfind("```")
            if end != -1:
                cleaned = cleaned[:end]
            cleaned = cleaned.strip()
        if not cleaned:
            return None
        cleaned = self._unwrap_tool_call(cleaned)
        if not cleaned:
            return None
        # Some models emit escaped newlines (literal backslash-n) instead of
        # real line breaks; unescape only when no real newline is present so
        # genuine backslashes in clean markdown are never mangled.
        if "\n" not in cleaned and ("\\n" in cleaned or "\\t" in cleaned):
            cleaned = (
                cleaned.replace("\\r\\n", "\n")
                .replace("\\n", "\n")
                .replace("\\t", "\t")
                .strip()
            )
            if not cleaned:
                return None
        limit = self._brief_max_chars()
        if len(cleaned) > limit:
            logger.warning("truncating brief from %d to %d chars", len(cleaned), limit)
            cleaned = cleaned[:limit]
        return cleaned

    def should_update_brief(self, signals) -> bool:
        """Fail-open gate on the substantive_update noul arm."""
        if not isinstance(signals, dict):
            return True
        items = signals.get("items", {})
        if not isinstance(items, dict):
            return True
        item = items.get("substantive_update")
        if not isinstance(item, dict):
            return True
        prob = _number(item.get("p"))
        if prob is None:
            return True
        return prob >= self._brief_gate_threshold()

    def finalize_brief(self, text, finish_reason) -> str | None:
        """Accept any parseable brief; a length cutoff truncates, never rejects."""
        if finish_reason == "length":
            logger.warning("brief reply hit the length limit; keeping the truncated text")
        return self.parse_brief(text)

    def state(self) -> tuple[str, str]:
        """Return the cached (state, reason) pair; performs no I/O."""
        return (self._state, self._reason)

    def refresh(self, state: str, reason: str = "") -> None:
        """Explicit transition; keeps no_questions; ignores unknown states."""
        if self._questions is None:
            return
        if state not in TRIAGE_REASONS:
            logger.warning("ignoring unknown triage state %r", state)
            return
        self._state = state
        self._reason = reason if isinstance(reason, str) else str(reason)


def _accents_folded(text: str) -> str:
    """Casefold, strip combining accents, and collapse to plain words."""
    decomposed = unicodedata.normalize("NFKD", str(text))
    without_marks = "".join(ch for ch in decomposed if not unicodedata.combining(ch))
    return " ".join(without_marks.casefold().split())


def _dedupe_key(text: str) -> str:
    """Accent- and punctuation-insensitive key for near-verbatim comparisons."""
    return " ".join(t for t in re.split(r"[^0-9a-z$%]+", _accents_folded(text)) if t)


def _number_tokens(text: str) -> frozenset:
    """Digit-bearing tokens (currency/percent stripped) for strict dedupe checks."""
    tokens = set()
    for raw in _accents_folded(text).split():
        digits = "".join(ch for ch in raw if ch.isdigit())
        if digits:
            tokens.add(digits)
    return frozenset(tokens)


def _entity_tokens(text: str) -> frozenset:
    """Capitalized (proper-noun) tokens, accent-folded, for strict dedupe checks."""
    tokens = set()
    for raw in str(text).split():
        stripped = raw.strip(".,;:!?()[]\"'")
        if stripped[:1].isupper() and stripped[1:].islower():
            tokens.add(_accents_folded(stripped))
    return frozenset(tokens)


class FactLedger:
    """Code-owned, append-only ledger of durable facts.

    The model proposes new facts per pass; this class owns dedupe, the size cap,
    eviction, and rendering, so a recorded fact can only disappear via a logged,
    deterministic eviction (never because the model dropped it). Pure and
    stdlib-only, matching the analyzer.

    Dedupe biasses toward *keeping* a fact: an exact normalized match is dropped,
    and a fuzzy near-match is dropped only when its number and entity tokens are
    identical. A redundant bullet is cosmetic; a false merge would silently lose a
    fact and break the retention guarantee.
    """

    def __init__(self, max_facts: int = 200, fact_max_chars: int = 240) -> None:
        self.max_facts = max(1, int(max_facts))
        self.fact_max_chars = max(1, int(fact_max_chars))
        self._facts: list[str] = []

    def __len__(self) -> int:
        return len(self._facts)

    def facts(self) -> list[str]:
        """Return a copy of the recorded facts, in insertion order."""
        return list(self._facts)

    def clear(self) -> None:
        """Drop every recorded fact."""
        self._facts = []

    def render(self) -> str:
        """Render the ledger as a ``Prior Details:`` section ("" when empty)."""
        if not self._facts:
            return ""
        return "Prior Details:\n" + "\n".join("- " + fact for fact in self._facts)

    def merge(self, new_facts) -> int:
        """Append genuinely-new facts, dedupe, then evict oldest over the cap.

        Returns the number of facts added. Eviction is oldest-first and logged;
        a fact added in this call is never evicted by this call.
        """
        added = 0
        for raw in new_facts or []:
            fact = self._clean(raw)
            if fact is None:
                continue
            if self._is_duplicate(fact):
                continue
            self._facts.append(fact)
            added += 1
        if added:
            self._evict()
        return added

    def _clean(self, raw):
        if not isinstance(raw, str):
            return None
        fact = " ".join(raw.strip().split())
        if not fact:
            return None
        if len(fact) > self.fact_max_chars:
            fact = fact[: self.fact_max_chars].rstrip()
        return fact or None

    @staticmethod
    def _tokens(text: str) -> list[str]:
        normalized = _accents_folded(text)
        return [t for t in re.split(r"[^0-9a-z$%]+", normalized) if t]

    def _is_duplicate(self, fact: str) -> bool:
        key = _dedupe_key(fact)
        cand_tokens = self._tokens(fact)
        cand_numbers = _number_tokens(fact)
        cand_entities = _entity_tokens(fact)
        for existing in self._facts:
            if _dedupe_key(existing) == key:
                return True
            if cand_numbers != _number_tokens(existing):
                continue
            if cand_entities != _entity_tokens(existing):
                continue
            existing_tokens = self._tokens(existing)
            if not cand_tokens or not existing_tokens:
                continue
            overlap = len(set(cand_tokens) & set(existing_tokens)) / len(set(cand_tokens))
            if overlap >= 0.85:
                return True
        return False

    def _evict(self) -> None:
        overflow = len(self._facts) - self.max_facts
        if overflow <= 0:
            return
        evicted = self._facts[:overflow]
        self._facts = self._facts[overflow:]
        logger.info(
            "fact ledger at cap %d; evicted %d oldest fact(s): %s",
            self.max_facts,
            len(evicted),
            "; ".join(evicted),
        )

    @classmethod
    def from_brief(cls, brief, max_facts: int = 200, fact_max_chars: int = 240):
        """Rebuild a ledger by harvesting ``- `` bullets from an existing brief."""
        ledger = cls(max_facts=max_facts, fact_max_chars=fact_max_chars)
        if isinstance(brief, str):
            for line in brief.splitlines():
                stripped = line.strip()
                if stripped.startswith("- "):
                    ledger.merge([stripped[2:]])
        return ledger