|
|
@@ -22,12 +22,24 @@ Public API (consumed by the assistant service layer):
|
|
|
def normalize(self, payload) -> dict
|
|
|
def should_escalate(self, signals) -> bool
|
|
|
def build_brief_prompt(self, deltas, signals) -> str
|
|
|
+ def build_ledger_prompt(self, narrative, ledger, deltas, signals) -> str
|
|
|
def parse_brief(self, text) -> str | None
|
|
|
+ def parse_ledger_reply(self, text, truncated=False) -> tuple[str, list[str]] | None
|
|
|
+ def parse_facts(self, block) -> list[str]
|
|
|
def should_update_brief(self, signals) -> bool
|
|
|
def finalize_brief(self, text, finish_reason) -> str | None
|
|
|
def state(self) -> tuple[str, str]
|
|
|
def refresh(self, state: str, reason: str = "") -> None
|
|
|
|
|
|
+ class FactLedger:
|
|
|
+ def __init__(self, max_facts=200, fact_max_chars=240) -> None
|
|
|
+ def render(self) -> str
|
|
|
+ def merge(self, new_facts) -> int
|
|
|
+ def facts(self) -> list[str]
|
|
|
+ def clear(self) -> None
|
|
|
+ @classmethod
|
|
|
+ def from_brief(cls, brief, ...) -> "FactLedger"
|
|
|
+
|
|
|
``cfg`` is duck-typed (attribute access only, never imported): it provides
|
|
|
``questions_file``, ``state_char_budget`` (default 3000),
|
|
|
``confidence_field`` (default ``"answer_confidence"``),
|
|
|
@@ -38,6 +50,8 @@ Public API (consumed by the assistant service layer):
|
|
|
|
|
|
import json
|
|
|
import logging
|
|
|
+import re
|
|
|
+import unicodedata
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
@@ -379,27 +393,110 @@ class TriageAnalyzer:
|
|
|
parts.append("Return only the updated brief as plain text.")
|
|
|
return "\n\n".join(parts)
|
|
|
|
|
|
- def build_pinned_brief_prompt(self, current_brief, deltas, signals) -> str:
|
|
|
- """Single-turn brief prompt: pin the current brief + new utterances.
|
|
|
+ def build_ledger_prompt(self, narrative, ledger, deltas, signals) -> str:
|
|
|
+ """Ledger-aware single-turn prompt: volatile narrative + code-owned ledger.
|
|
|
|
|
|
- Unlike ``build_brief_prompt`` this carries the full prior brief inline so
|
|
|
- the engine can send one user turn with no conversation history, keeping
|
|
|
- the prompt flat and the previous brief the single source of truth.
|
|
|
+ The model is shown the current narrative (the sections it owns) and the
|
|
|
+ current ledger (facts it must NOT repeat) and asked for the two halves
|
|
|
+ back in fixed machine-parseable sections. ``NEW FACTS:`` is placed first
|
|
|
+ so a length cutoff loses narrative (regenerated every pass) rather than a
|
|
|
+ durable fact.
|
|
|
"""
|
|
|
try:
|
|
|
body = "\n".join("- " + delta for delta in (deltas or []))
|
|
|
except Exception:
|
|
|
body = ""
|
|
|
+ ledger_text = "\n".join("- " + fact for fact in (ledger or [])) or "(none yet)"
|
|
|
parts = [
|
|
|
- "CURRENT BRIEF:\n" + (current_brief or "(none yet)"),
|
|
|
+ "CURRENT LEDGER (facts already recorded - never repeat these):\n" + ledger_text,
|
|
|
+ "CURRENT BRIEF:\n" + (narrative or "(none yet)"),
|
|
|
"NEW UTTERANCES:\n" + body,
|
|
|
]
|
|
|
focus = self._signal_lines(signals)
|
|
|
if focus:
|
|
|
parts.append("Focus:\n" + "\n".join(focus))
|
|
|
- parts.append("Return only the updated brief as plain text.")
|
|
|
+ parts.append(
|
|
|
+ "Reply with exactly these two sections and nothing else:\n"
|
|
|
+ "NEW FACTS:\n"
|
|
|
+ "- only genuinely new durable facts, one per line; write 'none' if there are none\n"
|
|
|
+ "BRIEF:\n"
|
|
|
+ "Summary:\n...\nOpportunities:\n...\nBlockers/Risks:\n...\n"
|
|
|
+ "Decisions:\n...\nOpen Questions:\n..."
|
|
|
+ )
|
|
|
return "\n\n".join(parts)
|
|
|
|
|
|
+ def parse_ledger_reply(self, text, truncated: bool = False) -> tuple[str, list[str]] | None:
|
|
|
+ """Split a ledger reply into ``(narrative, new_facts)``.
|
|
|
+
|
|
|
+ Both halves are optional and fail open: a missing ``NEW FACTS:`` block
|
|
|
+ yields no facts, and a missing ``BRIEF:`` block yields an empty
|
|
|
+ narrative (the caller then keeps the prior narrative). Any ``Prior
|
|
|
+ Details:`` the model volunteers is dropped from the narrative so the
|
|
|
+ code-rendered ledger is never duplicated. When ``truncated`` the reply hit
|
|
|
+ the token cap, so the final fact line may be cut mid-sentence and is
|
|
|
+ discarded. Returns None only when the reply carries no usable text at all.
|
|
|
+ """
|
|
|
+ cleaned = self.parse_brief(text)
|
|
|
+ if cleaned is None:
|
|
|
+ return None
|
|
|
+ facts_at = cleaned.upper().find("NEW FACTS:")
|
|
|
+ brief_at = cleaned.upper().find("BRIEF:")
|
|
|
+ if facts_at == -1 and brief_at == -1:
|
|
|
+ return None
|
|
|
+ if facts_at != -1 and (brief_at == -1 or facts_at < brief_at):
|
|
|
+ facts_end = brief_at if brief_at != -1 else len(cleaned)
|
|
|
+ facts_block = cleaned[facts_at + len("NEW FACTS:"): facts_end]
|
|
|
+ narrative_block = cleaned[brief_at + len("BRIEF:"):] if brief_at != -1 else ""
|
|
|
+ else:
|
|
|
+ facts_block = ""
|
|
|
+ narrative_block = cleaned[brief_at + len("BRIEF:"):]
|
|
|
+ facts = self.parse_facts(facts_block)
|
|
|
+ if truncated and facts:
|
|
|
+ facts = facts[:-1]
|
|
|
+ narrative = self._strip_prior_details(narrative_block).strip()
|
|
|
+ return narrative, facts
|
|
|
+
|
|
|
+ def parse_facts(self, block) -> list[str]:
|
|
|
+ """Extract durable facts from a ``NEW FACTS:`` block; never treats prose as a fact."""
|
|
|
+ if not isinstance(block, str):
|
|
|
+ return []
|
|
|
+ facts = []
|
|
|
+ for line in block.splitlines():
|
|
|
+ stripped = line.strip()
|
|
|
+ if not stripped or stripped.startswith("#"):
|
|
|
+ continue
|
|
|
+ marker = next((m for m in ("- ", "* ", "• ") if stripped.startswith(m)), None)
|
|
|
+ if marker is not None:
|
|
|
+ candidate = stripped[len(marker):]
|
|
|
+ elif stripped[:2].rstrip(".").isdigit() or stripped[:3].rstrip(".").isdigit():
|
|
|
+ candidate = stripped.split(".", 1)[1] if "." in stripped else stripped
|
|
|
+ else:
|
|
|
+ continue
|
|
|
+ candidate = candidate.strip().strip("`").strip()
|
|
|
+ if not candidate or candidate.lower() in ("none", "-"):
|
|
|
+ continue
|
|
|
+ if ":" in candidate and candidate.lower().startswith("new facts"):
|
|
|
+ continue
|
|
|
+ facts.append(candidate)
|
|
|
+ return facts
|
|
|
+
|
|
|
+ @staticmethod
|
|
|
+ def _strip_prior_details(text: str) -> str:
|
|
|
+ """Drop a model-supplied ``Prior Details:`` section from the narrative."""
|
|
|
+ if not isinstance(text, str):
|
|
|
+ return ""
|
|
|
+ heads = (
|
|
|
+ "Summary:", "Opportunities:", "Blockers/Risks:", "Responsibilities:",
|
|
|
+ "Decisions:", "Action Items:", "Deadlines:", "Open Questions:",
|
|
|
+ )
|
|
|
+ at = text.find("Prior Details:")
|
|
|
+ if at == -1:
|
|
|
+ return text
|
|
|
+ rest = text[at + len("Prior Details:"):]
|
|
|
+ cuts = [rest.find(h) for h in heads if rest.find(h) != -1]
|
|
|
+ end = min(cuts) if cuts else len(rest)
|
|
|
+ return (text[:at] + rest[end:]).strip()
|
|
|
+
|
|
|
@staticmethod
|
|
|
def _normalize_bullet(text: str) -> str:
|
|
|
"""Collapse a bullet to a comparison key; empty when it carries no words."""
|
|
|
@@ -427,11 +524,12 @@ class TriageAnalyzer:
|
|
|
def restore_missing_bullets(self, new_brief, prior_brief) -> str:
|
|
|
"""Re-append any prior bullet that the model dropped from ``new_brief``.
|
|
|
|
|
|
- Deterministic retention guard: a previously-recorded fact must never
|
|
|
- vanish because the model re-emitted the document without it. Bullets
|
|
|
- the new brief merely reworded still match by normalized key, so only a
|
|
|
- genuinely absent bullet is restored. Restoration appends under the
|
|
|
- brief's final ``Prior Details:`` section, or at the end when absent.
|
|
|
+ Deterministic retention guard kept as a belt-and-braces layer beneath the
|
|
|
+ code-owned ledger: a previously-recorded bullet must never vanish because
|
|
|
+ the model re-emitted the document without it. Bullets the new brief merely
|
|
|
+ reworded still match by normalized key, so only a genuinely absent bullet
|
|
|
+ is restored. Restoration appends under the brief's final ``Prior
|
|
|
+ Details:`` section, or at the end when absent.
|
|
|
"""
|
|
|
if not isinstance(new_brief, str) or not new_brief.strip():
|
|
|
return new_brief
|
|
|
@@ -598,3 +696,150 @@ class TriageAnalyzer:
|
|
|
return
|
|
|
self._state = state
|
|
|
self._reason = reason if isinstance(reason, str) else str(reason)
|
|
|
+
|
|
|
+
|
|
|
+def _accents_folded(text: str) -> str:
|
|
|
+ """Casefold, strip combining accents, and collapse to plain words."""
|
|
|
+ decomposed = unicodedata.normalize("NFKD", str(text))
|
|
|
+ without_marks = "".join(ch for ch in decomposed if not unicodedata.combining(ch))
|
|
|
+ return " ".join(without_marks.casefold().split())
|
|
|
+
|
|
|
+
|
|
|
+def _dedupe_key(text: str) -> str:
|
|
|
+ """Accent- and punctuation-insensitive key for near-verbatim comparisons."""
|
|
|
+ return " ".join(t for t in re.split(r"[^0-9a-z$%]+", _accents_folded(text)) if t)
|
|
|
+
|
|
|
+
|
|
|
+def _number_tokens(text: str) -> frozenset:
|
|
|
+ """Digit-bearing tokens (currency/percent stripped) for strict dedupe checks."""
|
|
|
+ tokens = set()
|
|
|
+ for raw in _accents_folded(text).split():
|
|
|
+ digits = "".join(ch for ch in raw if ch.isdigit())
|
|
|
+ if digits:
|
|
|
+ tokens.add(digits)
|
|
|
+ return frozenset(tokens)
|
|
|
+
|
|
|
+
|
|
|
+def _entity_tokens(text: str) -> frozenset:
|
|
|
+ """Capitalized (proper-noun) tokens, accent-folded, for strict dedupe checks."""
|
|
|
+ tokens = set()
|
|
|
+ for raw in str(text).split():
|
|
|
+ stripped = raw.strip(".,;:!?()[]\"'")
|
|
|
+ if stripped[:1].isupper() and stripped[1:].islower():
|
|
|
+ tokens.add(_accents_folded(stripped))
|
|
|
+ return frozenset(tokens)
|
|
|
+
|
|
|
+
|
|
|
+class FactLedger:
|
|
|
+ """Code-owned, append-only ledger of durable facts.
|
|
|
+
|
|
|
+ The model proposes new facts per pass; this class owns dedupe, the size cap,
|
|
|
+ eviction, and rendering, so a recorded fact can only disappear via a logged,
|
|
|
+ deterministic eviction (never because the model dropped it). Pure and
|
|
|
+ stdlib-only, matching the analyzer.
|
|
|
+
|
|
|
+ Dedupe biasses toward *keeping* a fact: an exact normalized match is dropped,
|
|
|
+ and a fuzzy near-match is dropped only when its number and entity tokens are
|
|
|
+ identical. A redundant bullet is cosmetic; a false merge would silently lose a
|
|
|
+ fact and break the retention guarantee.
|
|
|
+ """
|
|
|
+
|
|
|
+ def __init__(self, max_facts: int = 200, fact_max_chars: int = 240) -> None:
|
|
|
+ self.max_facts = max(1, int(max_facts))
|
|
|
+ self.fact_max_chars = max(1, int(fact_max_chars))
|
|
|
+ self._facts: list[str] = []
|
|
|
+
|
|
|
+ def __len__(self) -> int:
|
|
|
+ return len(self._facts)
|
|
|
+
|
|
|
+ def facts(self) -> list[str]:
|
|
|
+ """Return a copy of the recorded facts, in insertion order."""
|
|
|
+ return list(self._facts)
|
|
|
+
|
|
|
+ def clear(self) -> None:
|
|
|
+ """Drop every recorded fact."""
|
|
|
+ self._facts = []
|
|
|
+
|
|
|
+ def render(self) -> str:
|
|
|
+ """Render the ledger as a ``Prior Details:`` section ("" when empty)."""
|
|
|
+ if not self._facts:
|
|
|
+ return ""
|
|
|
+ return "Prior Details:\n" + "\n".join("- " + fact for fact in self._facts)
|
|
|
+
|
|
|
+ def merge(self, new_facts) -> int:
|
|
|
+ """Append genuinely-new facts, dedupe, then evict oldest over the cap.
|
|
|
+
|
|
|
+ Returns the number of facts added. Eviction is oldest-first and logged;
|
|
|
+ a fact added in this call is never evicted by this call.
|
|
|
+ """
|
|
|
+ added = 0
|
|
|
+ for raw in new_facts or []:
|
|
|
+ fact = self._clean(raw)
|
|
|
+ if fact is None:
|
|
|
+ continue
|
|
|
+ if self._is_duplicate(fact):
|
|
|
+ continue
|
|
|
+ self._facts.append(fact)
|
|
|
+ added += 1
|
|
|
+ if added:
|
|
|
+ self._evict()
|
|
|
+ return added
|
|
|
+
|
|
|
+ def _clean(self, raw):
|
|
|
+ if not isinstance(raw, str):
|
|
|
+ return None
|
|
|
+ fact = " ".join(raw.strip().split())
|
|
|
+ if not fact:
|
|
|
+ return None
|
|
|
+ if len(fact) > self.fact_max_chars:
|
|
|
+ fact = fact[: self.fact_max_chars].rstrip()
|
|
|
+ return fact or None
|
|
|
+
|
|
|
+ @staticmethod
|
|
|
+ def _tokens(text: str) -> list[str]:
|
|
|
+ normalized = _accents_folded(text)
|
|
|
+ return [t for t in re.split(r"[^0-9a-z$%]+", normalized) if t]
|
|
|
+
|
|
|
+ def _is_duplicate(self, fact: str) -> bool:
|
|
|
+ key = _dedupe_key(fact)
|
|
|
+ cand_tokens = self._tokens(fact)
|
|
|
+ cand_numbers = _number_tokens(fact)
|
|
|
+ cand_entities = _entity_tokens(fact)
|
|
|
+ for existing in self._facts:
|
|
|
+ if _dedupe_key(existing) == key:
|
|
|
+ return True
|
|
|
+ if cand_numbers != _number_tokens(existing):
|
|
|
+ continue
|
|
|
+ if cand_entities != _entity_tokens(existing):
|
|
|
+ continue
|
|
|
+ existing_tokens = self._tokens(existing)
|
|
|
+ if not cand_tokens or not existing_tokens:
|
|
|
+ continue
|
|
|
+ overlap = len(set(cand_tokens) & set(existing_tokens)) / len(set(cand_tokens))
|
|
|
+ if overlap >= 0.85:
|
|
|
+ return True
|
|
|
+ return False
|
|
|
+
|
|
|
+ def _evict(self) -> None:
|
|
|
+ overflow = len(self._facts) - self.max_facts
|
|
|
+ if overflow <= 0:
|
|
|
+ return
|
|
|
+ evicted = self._facts[:overflow]
|
|
|
+ self._facts = self._facts[overflow:]
|
|
|
+ logger.info(
|
|
|
+ "fact ledger at cap %d; evicted %d oldest fact(s): %s",
|
|
|
+ self.max_facts,
|
|
|
+ len(evicted),
|
|
|
+ "; ".join(evicted),
|
|
|
+ )
|
|
|
+
|
|
|
+ @classmethod
|
|
|
+ def from_brief(cls, brief, max_facts: int = 200, fact_max_chars: int = 240):
|
|
|
+ """Rebuild a ledger by harvesting ``- `` bullets from an existing brief."""
|
|
|
+ ledger = cls(max_facts=max_facts, fact_max_chars=fact_max_chars)
|
|
|
+ if isinstance(brief, str):
|
|
|
+ for line in brief.splitlines():
|
|
|
+ stripped = line.strip()
|
|
|
+ if stripped.startswith("- "):
|
|
|
+ ledger.merge([stripped[2:]])
|
|
|
+ return ledger
|