test_triage.py 53 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466
  1. """Tests for the triage analyzer (todos 3, 9, 10, 11). Pure, hardware-free."""
  2. import json
  3. import logging
  4. from pathlib import Path
  5. from types import SimpleNamespace
  6. import pytest
  7. from stt_daemon import events
  8. from stt_daemon.assistant import triage
  9. from stt_daemon.assistant.service import AssistantService
  10. from stt_daemon.assistant.triage import FactLedger, TriageAnalyzer, load_questions
  11. from stt_daemon.config import Config
  12. from stt_daemon.tui.state import ViewState
  13. EXAMPLE_PATH = str(Path(__file__).resolve().parent.parent / "questions.example.json")
  14. def _cfg(path=EXAMPLE_PATH, **overrides):
  15. base = {
  16. "questions_file": path,
  17. "state_char_budget": 1200,
  18. "confidence_field": "answer_confidence",
  19. "escalate_threshold": 0.6,
  20. "choice_min_confidence": 0.85,
  21. }
  22. base.update(overrides)
  23. return SimpleNamespace(**base)
  24. def _write_questions(tmp_path, questions):
  25. path = tmp_path / "questions.json"
  26. path.write_text(json.dumps(questions), encoding="utf-8")
  27. return str(path)
  28. def _analyzer(tmp_path, questions=None, **overrides):
  29. if questions is None:
  30. return TriageAnalyzer(_cfg(**overrides))
  31. return TriageAnalyzer(_cfg(_write_questions(tmp_path, questions), **overrides))
  32. # --- todo 3: load_questions -------------------------------------------------
  33. def test_load_example_order_preserved():
  34. raw = json.loads(Path(EXAMPLE_PATH).read_text(encoding="utf-8"))
  35. questions = load_questions(EXAMPLE_PATH)
  36. assert questions is not None
  37. assert questions
  38. assert list(questions.keys()) == list(raw.keys())
  39. for definition in questions.values():
  40. assert definition["type"] in {"choice", "score", "noul"}
  41. assert isinstance(definition["instructions"], str)
  42. assert definition["instructions"].strip()
  43. def test_example_questions_match_design():
  44. questions = load_questions(EXAMPLE_PATH)
  45. assert questions is not None
  46. assert list(questions.keys()) == [
  47. "substantive_update",
  48. "opportunity",
  49. "blocker",
  50. "decision_status",
  51. "action_item",
  52. "responsibility_clarity",
  53. "deadline",
  54. "open_question",
  55. ]
  56. for qid, definition in questions.items():
  57. assert isinstance(definition["instructions"], str)
  58. assert definition["instructions"].strip()
  59. if definition["type"] == "choice":
  60. criteria = definition["criteria"]
  61. options = list(criteria.keys()) if isinstance(criteria, dict) else list(criteria)
  62. assert "other" in options
  63. assert len(options) <= 7
  64. if definition["type"] == "noul":
  65. assert isinstance(definition.get("instructions"), str)
  66. assert definition["instructions"].strip()
  67. def test_load_missing_file_returns_none_with_warning(caplog):
  68. with caplog.at_level(logging.WARNING, logger="stt_daemon.assistant.triage"):
  69. assert load_questions("/nonexistent/questions.json") is None
  70. assert caplog.records, "expected a warning for the missing file"
  71. def test_load_bad_json_returns_none_with_warning(tmp_path, caplog):
  72. path = tmp_path / "questions.json"
  73. path.write_text("{not valid json", encoding="utf-8")
  74. with caplog.at_level(logging.WARNING, logger="stt_daemon.assistant.triage"):
  75. assert load_questions(str(path)) is None
  76. assert caplog.records
  77. @pytest.mark.parametrize(
  78. "bad_entry",
  79. [
  80. {"type": "mystery", "instructions": "do something"}, # unknown type
  81. {"type": "score", "instructions": "rate it", "criteria": {"0": "low"}}, # score dict
  82. { # noul criteria with a bad key
  83. "type": "noul",
  84. "instructions": "holds?",
  85. "criteria": {"true": "yes", "maybe": "hmm"},
  86. },
  87. "just a string", # non-object entry
  88. {"type": "choice", "instructions": "pick one", "criteria": []}, # empty choice
  89. {"type": "choice", "instructions": "pick one", "criteria": {}}, # empty choice dict
  90. {"type": "choice", "criteria": ["a", "b"]}, # missing instructions
  91. {"type": "score", "instructions": "rate it", "criteria": ["a", None]}, # None level
  92. {"type": "noul", "instructions": "holds?", "criteria": ["true"]}, # noul list
  93. ],
  94. )
  95. def test_load_malformed_returns_none_with_warning(tmp_path, caplog, bad_entry):
  96. path = _write_questions(
  97. tmp_path, {"good": {"type": "noul", "instructions": "ok?"}, "bad": bad_entry}
  98. )
  99. with caplog.at_level(logging.WARNING, logger="stt_daemon.assistant.triage"):
  100. assert load_questions(str(path)) is None
  101. assert caplog.records
  102. # --- todo 9: normalize ------------------------------------------------------
  103. def _score_questions():
  104. return {
  105. "severity": {
  106. "type": "score",
  107. "instructions": "How severe is the issue?",
  108. "criteria": ["none", "low", "high", "critical"],
  109. },
  110. "category": {
  111. "type": "choice",
  112. "instructions": "What is this about?",
  113. "criteria": {"support": "help needed", "sales": "buying"},
  114. },
  115. "churn_risk": {"type": "noul", "instructions": "User may leave."},
  116. }
  117. def test_normalize_choice_exact(tmp_path):
  118. analyzer = _analyzer(tmp_path, _score_questions())
  119. payload = {
  120. "model": "multilingual",
  121. "answers": {
  122. "category": {
  123. "type": "choice",
  124. "choice": "support",
  125. "confidence": 0.7,
  126. "answer_confidence": 0.4,
  127. }
  128. },
  129. "usage": {"input_tokens": 123, "output_tokens": 0},
  130. }
  131. result = analyzer.normalize(payload)
  132. assert result["order"] == ["category"]
  133. assert result["items"]["category"] == {
  134. "kind": "choice",
  135. "choice": "support",
  136. "confidence": 0.7,
  137. "answer_confidence": 0.4,
  138. "gate": 0.4,
  139. "flagged": True,
  140. }
  141. assert result["input_tokens"] == 123
  142. assert result["escalated"] is True
  143. def test_normalize_score_with_legend(tmp_path):
  144. analyzer = _analyzer(tmp_path, _score_questions())
  145. payload = {
  146. "answers": {
  147. "severity": {
  148. "type": "score",
  149. "score": 2.6,
  150. "legend": {"0": "none", "1": "low", "2": "high", "3": "critical"},
  151. "confidence": 0.5,
  152. "answer_confidence": 0.6,
  153. }
  154. },
  155. "usage": {"input_tokens": 7, "output_tokens": 0},
  156. }
  157. item = analyzer.normalize(payload)["items"]["severity"]
  158. assert item["kind"] == "score"
  159. assert item["expected"] == 2.6
  160. assert item["level"] == 3
  161. assert item["label"] == "critical"
  162. assert item["confidence"] == 0.5
  163. assert item["answer_confidence"] == 0.6
  164. assert item["gate"] == 0.6
  165. assert item["flagged"] is True
  166. def test_normalize_score_legend_fallback(tmp_path):
  167. analyzer = _analyzer(tmp_path, _score_questions())
  168. payload = {
  169. "answers": {
  170. "severity": {"type": "score", "score": 1.2, "confidence": 0.9, "answer_confidence": 0.9}
  171. }
  172. }
  173. item = analyzer.normalize(payload)["items"]["severity"]
  174. assert item["level"] == 1
  175. assert item["label"] == "low" # falls back to criteria[level]
  176. assert item["flagged"] is False # 1.2 < (4 - 1) * 0.5
  177. def test_normalize_noul_and_bare_payload(tmp_path):
  178. analyzer = _analyzer(tmp_path, _score_questions())
  179. bare = {
  180. "churn_risk": {"type": "noul", "noul": 0.9, "confidence": 0.9, "answer_confidence": 0.85}
  181. }
  182. result = analyzer.normalize(bare)
  183. assert result["order"] == ["churn_risk"]
  184. assert result["items"]["churn_risk"] == {
  185. "kind": "noul",
  186. "p": 0.9,
  187. "confidence": 0.9,
  188. "answer_confidence": 0.85,
  189. "gate": 0.85,
  190. "threshold": 0.6,
  191. "flagged": True,
  192. }
  193. assert result["input_tokens"] == 0
  194. def test_normalize_skips_unknown_qid_and_malformed(tmp_path):
  195. analyzer = _analyzer(tmp_path, _score_questions())
  196. payload = {
  197. "answers": {
  198. "ghost": {"type": "noul", "noul": 0.99, "confidence": 0.9, "answer_confidence": 0.9},
  199. "category": {"type": "choice", "choice": 42}, # malformed: no confidences
  200. "churn_risk": "nonsense", # malformed: not a dict
  201. },
  202. "usage": {"input_tokens": 3, "output_tokens": 0},
  203. }
  204. result = analyzer.normalize(payload)
  205. assert result["order"] == []
  206. assert result["items"] == {}
  207. assert result["escalated"] is False
  208. def test_normalize_ignores_side_channel_fields(tmp_path):
  209. analyzer = _analyzer(tmp_path, _score_questions())
  210. payload = {
  211. "answers": {
  212. "category": {
  213. "type": "choice",
  214. "choice": "sales",
  215. "confidence": 0.95,
  216. "answer_confidence": 0.9,
  217. "action": {"act" + "_probability": 0.99},
  218. }
  219. },
  220. "usage": {"input_tokens": 1, "output_tokens": 0},
  221. }
  222. result = analyzer.normalize(payload)
  223. assert result["items"]["category"]["flagged"] is False
  224. assert "act" + "_probability" not in json.dumps(result)
  225. source = Path(triage.__file__).read_text(encoding="utf-8")
  226. assert "act" + "_probability" not in source
  227. def test_normalize_gating_field_selectable(tmp_path):
  228. analyzer = _analyzer(tmp_path, _score_questions(), confidence_field="confidence")
  229. payload = {
  230. "answers": {
  231. "category": {
  232. "type": "choice",
  233. "choice": "sales",
  234. "confidence": 0.5,
  235. "answer_confidence": 0.99,
  236. }
  237. }
  238. }
  239. item = analyzer.normalize(payload)["items"]["category"]
  240. assert item["gate"] == 0.5 # gates on confidence, keeps both
  241. assert item["answer_confidence"] == 0.99
  242. assert item["flagged"] is True
  243. # --- todo 9: condensation ---------------------------------------------------
  244. def test_condense_keeps_newest_chronological_no_numerals(tmp_path):
  245. analyzer = _analyzer(tmp_path, _score_questions(), state_char_budget=40)
  246. window = ["alpha first", "beta second", "gamma third newest here"]
  247. state = analyzer.build_state(window)
  248. assert "gamma third newest here" in state
  249. assert "alpha first" not in state # oldest dropped under the budget
  250. lines = state.splitlines()
  251. assert all(line.startswith("- ") for line in lines)
  252. assert not any(ch.isdigit() for ch in state) # no numerals injected
  253. # chronological restore: kept lines stay in original order
  254. kept = [u for u in window if u in state]
  255. assert [line[2:] for line in lines] == kept
  256. def test_condense_empty_window(tmp_path):
  257. analyzer = _analyzer(tmp_path, _score_questions())
  258. assert analyzer.build_state([]) == ""
  259. # --- todo 10: escalation + extraction ---------------------------------------
  260. def _signals(choice_gate=0.9, noul_p=0.1, score_expected=0.2):
  261. return {
  262. "order": ["category", "churn_risk", "severity"],
  263. "items": {
  264. "category": {
  265. "kind": "choice",
  266. "choice": "sales",
  267. "confidence": choice_gate,
  268. "answer_confidence": choice_gate,
  269. "gate": choice_gate,
  270. "flagged": choice_gate < 0.85,
  271. },
  272. "churn_risk": {
  273. "kind": "noul",
  274. "p": noul_p,
  275. "confidence": noul_p,
  276. "answer_confidence": noul_p,
  277. "gate": noul_p,
  278. "flagged": noul_p >= 0.6,
  279. },
  280. "severity": {
  281. "kind": "score",
  282. "expected": score_expected,
  283. "level": 0,
  284. "label": "none",
  285. "confidence": 0.9,
  286. "answer_confidence": 0.9,
  287. "gate": 0.9,
  288. "flagged": score_expected >= 1.5,
  289. },
  290. },
  291. "escalated": False,
  292. "input_tokens": 0,
  293. }
  294. def test_escalation_arms_fire_independently(tmp_path):
  295. analyzer = _analyzer(tmp_path, _score_questions())
  296. assert analyzer.should_escalate(_signals(choice_gate=0.5)) is True # choice arm
  297. assert analyzer.should_escalate(_signals(noul_p=0.8)) is True # noul arm
  298. assert analyzer.should_escalate(_signals(score_expected=2.0)) is True # score arm
  299. assert analyzer.should_escalate(_signals()) is False
  300. def test_normalize_escalated_matches_should_escalate(tmp_path):
  301. analyzer = _analyzer(tmp_path, _score_questions())
  302. payload = {
  303. "answers": {
  304. "category": {
  305. "type": "choice",
  306. "choice": "sales",
  307. "confidence": 0.9,
  308. "answer_confidence": 0.9,
  309. },
  310. "churn_risk": {
  311. "type": "noul",
  312. "noul": 0.1,
  313. "confidence": 0.9,
  314. "answer_confidence": 0.9,
  315. },
  316. "severity": {
  317. "type": "score",
  318. "score": 0.1,
  319. "confidence": 0.9,
  320. "answer_confidence": 0.9,
  321. },
  322. }
  323. }
  324. result = analyzer.normalize(payload)
  325. assert result["escalated"] == analyzer.should_escalate(result) is False
  326. # --- todo 5: brief prompt / parse / gate / finalize --------------------------
  327. def test_build_brief_prompt_order_and_focus(tmp_path):
  328. analyzer = _analyzer(tmp_path, _score_questions())
  329. prompt = analyzer.build_brief_prompt(
  330. ["Alice: let's ship Friday", "Bob: agreed"], _signals(noul_p=0.8)
  331. )
  332. assert prompt.startswith("New utterances:\n- Alice: let's ship Friday\n- Bob: agreed")
  333. assert "Focus:" in prompt
  334. assert "churn_risk" in prompt # signal named in words
  335. assert prompt.index("New utterances:") < prompt.index("Focus:")
  336. assert prompt.index("Focus:") < prompt.index("Return only the updated brief")
  337. # no JSON extraction keys or raw dict shapes leak into the brief prompt
  338. for key in ("summary", "opportunities", "action_items", "risks", "doubts"):
  339. assert key not in prompt
  340. assert "{" not in prompt and "}" not in prompt
  341. def test_build_brief_prompt_no_focus_without_signals(tmp_path):
  342. analyzer = _analyzer(tmp_path, _score_questions())
  343. for signals in (None, {}, {"order": [], "items": {}}):
  344. prompt = analyzer.build_brief_prompt(["solo utterance"], signals)
  345. assert "solo utterance" in prompt
  346. assert "Focus:" not in prompt
  347. assert prompt.endswith("Return only the updated brief as plain text.")
  348. def test_build_ledger_prompt_inlines_ledger_narrative_and_deltas(tmp_path):
  349. analyzer = _analyzer(tmp_path, _score_questions())
  350. prompt = analyzer.build_ledger_prompt(
  351. "Summary: shipped\n\nOpportunities: expand",
  352. ["fact one", "fact two"],
  353. ["Alice: let's ship Friday"],
  354. _signals(noul_p=0.8),
  355. )
  356. assert "- fact one" in prompt and "- fact two" in prompt
  357. assert "CURRENT LEDGER" in prompt
  358. assert "Summary: shipped" in prompt
  359. assert "NEW UTTERANCES:\n- Alice: let's ship Friday" in prompt
  360. assert prompt.index("CURRENT LEDGER") < prompt.index("CURRENT BRIEF:")
  361. assert prompt.index("CURRENT BRIEF:") < prompt.index("NEW UTTERANCES:")
  362. assert prompt.index("NEW UTTERANCES:") < prompt.index("Focus")
  363. assert "churn_risk" in prompt
  364. assert "NEW FACTS:" in prompt and "BRIEF:" in prompt
  365. def test_build_ledger_prompt_never_leaks_question_definitions(tmp_path):
  366. """The brief model used to copy Focus wording and numbers into the brief."""
  367. analyzer = _analyzer(tmp_path, _score_questions())
  368. prompt = analyzer.build_ledger_prompt("Summary: x", [], ["Alice: hi"], _signals(noul_p=0.8))
  369. focus = prompt.split("Focus (classifier verdicts, not meeting content):\n", 1)[1]
  370. focus = focus.split("\n\n", 1)[0]
  371. assert "churn_risk: 0.80" in focus
  372. assert "instruction" not in focus.lower()
  373. assert "gate=" not in focus
  374. assert "Does the customer" not in prompt
  375. def test_build_ledger_prompt_empty_ledger_and_narrative(tmp_path):
  376. analyzer = _analyzer(tmp_path, _score_questions())
  377. prompt = analyzer.build_ledger_prompt("", [], ["solo"], None)
  378. assert "CURRENT LEDGER" in prompt and "(none yet)" in prompt
  379. assert "CURRENT BRIEF:\n(none yet)" in prompt
  380. assert "NEW UTTERANCES:\n- solo" in prompt
  381. assert "Focus:" not in prompt
  382. def test_extract_bullets_strips_new_marker_and_normalizes(tmp_path):
  383. analyzer = _analyzer(tmp_path, _score_questions())
  384. brief = (
  385. "Summary: things\n"
  386. "- top level bullet\n"
  387. " - indented child\n"
  388. "- **New:** tagged fact\n"
  389. "not a bullet\n"
  390. "\n"
  391. )
  392. assert analyzer.extract_bullets(brief) == [
  393. "top level bullet",
  394. "indented child",
  395. "tagged fact",
  396. ]
  397. def test_restore_missing_bullets_reappends_dropped_fact(tmp_path):
  398. analyzer = _analyzer(tmp_path, _score_questions())
  399. prior = (
  400. "Summary: old\n\nPrior Details:\n"
  401. "- The budget was $305 million.\n"
  402. "- Sense could find prey.\n"
  403. )
  404. new = "Summary: newer\n\nPrior Details:\n- Sense could find prey.\n"
  405. restored = analyzer.restore_missing_bullets(new, prior)
  406. assert "- Sense could find prey." in restored
  407. assert "- The budget was $305 million." in restored
  408. assert restored.count("- Sense could find prey.") == 1
  409. def test_restore_missing_bullets_matches_reworded_bullet(tmp_path):
  410. analyzer = _analyzer(tmp_path, _score_questions())
  411. prior = "Prior Details:\n- **New:** Sense could find prey.\n"
  412. new = "Prior Details:\n- Sense could find prey.\n"
  413. restored = analyzer.restore_missing_bullets(new, prior)
  414. assert restored == new
  415. def test_restore_missing_bullets_restores_reworded_as_new_fact(tmp_path):
  416. analyzer = _analyzer(tmp_path, _score_questions())
  417. prior = "Prior Details:\n- The budget was $305 million.\n"
  418. new = "Prior Details:\n- Budget: $305 million (nearly Spider-Man 2).\n"
  419. restored = analyzer.restore_missing_bullets(new, prior)
  420. assert restored.count("$305 million") == 2
  421. assert restored.rstrip().endswith("- The budget was $305 million.")
  422. def test_restore_missing_bullets_noop_without_prior_or_new(tmp_path):
  423. analyzer = _analyzer(tmp_path, _score_questions())
  424. new = "Summary: only\nPrior Details:\n- keep me\n"
  425. assert analyzer.restore_missing_bullets(new, "") == new
  426. assert analyzer.restore_missing_bullets(new, None) == new
  427. assert analyzer.restore_missing_bullets("", new) == ""
  428. assert analyzer.restore_missing_bullets(None, new) is None
  429. def test_restore_missing_bullets_appends_section_when_absent(tmp_path):
  430. analyzer = _analyzer(tmp_path, _score_questions())
  431. prior = "- orphan fact\n"
  432. new = "Summary: no prior details header\n"
  433. restored = analyzer.restore_missing_bullets(new, prior)
  434. assert "- orphan fact" in restored
  435. assert "Prior Details:" in restored
  436. def test_build_ledger_prompt_contains_resolution_rule(tmp_path):
  437. analyzer = _analyzer(tmp_path, _score_questions())
  438. prompt = analyzer.build_ledger_prompt("Summary: x", [], ["hi"], None)
  439. assert "Never silently delete Action Items, Open Questions" in prompt
  440. assert "Done: <original>" in prompt
  441. assert "NEW FACTS bullet" in prompt
  442. def test_restore_missing_bullets_reappends_dropped_action_item(tmp_path):
  443. analyzer = _analyzer(tmp_path, _score_questions())
  444. prior = (
  445. "Summary: s\n\nAction Items:\n- Call Maria about pricing\n\n"
  446. "Open Questions:\n- Who owns the launch checklist?\n"
  447. )
  448. new = "Summary: s\n\nDecisions:\n- nothing yet\n"
  449. restored = analyzer.restore_missing_bullets(new, prior)
  450. assert "- Call Maria about pricing" in restored
  451. assert "- Who owns the launch checklist?" in restored
  452. assert "Prior Details:" in restored
  453. def test_parse_brief_strips_fences_and_rejects_empty(tmp_path):
  454. analyzer = _analyzer(tmp_path, _score_questions())
  455. assert analyzer.parse_brief("```\nhello brief\n```") == "hello brief"
  456. assert analyzer.parse_brief("```markdown\nhello brief\n```") == "hello brief"
  457. assert analyzer.parse_brief("plain brief") == "plain brief"
  458. assert analyzer.parse_brief(" padded ") == "padded"
  459. assert analyzer.parse_brief("") is None
  460. assert analyzer.parse_brief(" ") is None
  461. assert analyzer.parse_brief(None) is None
  462. assert analyzer.parse_brief(42) is None
  463. assert analyzer.parse_brief("```") is None
  464. def test_parse_brief_unescapes_literal_backslash_n(tmp_path):
  465. analyzer = _analyzer(tmp_path, _score_questions())
  466. assert analyzer.parse_brief("## Title\\n\\nBody line") == "## Title\n\nBody line"
  467. assert analyzer.parse_brief("a\\tb") == "a\tb"
  468. assert analyzer.parse_brief("line1\nline2\\nline3") == "line1\nline2\\nline3"
  469. def test_parse_brief_never_truncates_accepted_text(tmp_path):
  470. """The accepted brief is never cut: a long reply is returned whole."""
  471. analyzer = _analyzer(tmp_path, _score_questions())
  472. parsed = analyzer.parse_brief("x" * 5000)
  473. assert parsed == "x" * 5000
  474. wrapped = "[update_brief(updated_brief='" + "x" * 5000 + "')]"
  475. assert analyzer.parse_brief(wrapped) == "x" * 5000
  476. def test_build_ledger_prompt_clips_current_brief_on_line_boundary(tmp_path):
  477. analyzer = _analyzer(tmp_path, _score_questions(), brief_max_chars=40)
  478. narrative = "Summary: first line\nOpportunities:\n- keep this out"
  479. prompt = analyzer.build_ledger_prompt(narrative, [], ["hi"], None)
  480. assert "Summary: first line" in prompt
  481. assert "- keep this out" not in prompt
  482. assert "The CURRENT BRIEF was trimmed" in prompt
  483. def test_build_ledger_prompt_trims_long_paragraph_at_a_word(tmp_path):
  484. """A single long Summary paragraph must keep most of the budget, not 8 chars."""
  485. analyzer = _analyzer(tmp_path, _score_questions(), brief_max_chars=400)
  486. narrative = "Summary:\n" + ("word " * 200).strip()
  487. prompt = analyzer.build_ledger_prompt(narrative, [], ["hi"], None)
  488. view = prompt.split("CURRENT BRIEF:\n", 1)[1].split("\n\nNEW UTTERANCES:", 1)[0]
  489. assert len(view) > 200
  490. assert not view.endswith("wor")
  491. def test_build_ledger_prompt_keeps_short_brief_verbatim(tmp_path):
  492. analyzer = _analyzer(tmp_path, _score_questions())
  493. prompt = analyzer.build_ledger_prompt("Summary: x", [], ["hi"], None)
  494. assert "CURRENT BRIEF:\nSummary: x" in prompt
  495. assert "was trimmed" not in prompt
  496. def test_parse_ledger_reply_keeps_narrative_past_the_brief_cap(tmp_path):
  497. """Regression: the cap used to slice the reply before the BRIEF split."""
  498. analyzer = _analyzer(tmp_path, _score_questions(), brief_max_chars=50)
  499. narrative = "Summary: " + ("word " * 400).strip() + "\nOpen Questions:\n- who owns it?"
  500. reply = "NEW FACTS:\n- durable fact\n\nBRIEF:\n" + narrative
  501. parsed = analyzer.parse_ledger_reply(reply)
  502. assert parsed is not None
  503. got_narrative, facts = parsed
  504. assert facts == ["durable fact"]
  505. assert got_narrative.endswith("- who owns it?")
  506. assert len(got_narrative) > 50
  507. def test_restore_missing_bullets_caps_prior_details_block(tmp_path):
  508. analyzer = _analyzer(tmp_path, _score_questions(), brief_max_chars=80)
  509. prior = "Summary: s\n\nPrior Details:\n" + "\n".join(f"- bullet {i}" for i in range(20))
  510. restored = analyzer.restore_missing_bullets("Summary: s2\n", prior)
  511. tail = restored.split("Prior Details:", 1)[1]
  512. assert len(tail) <= 80
  513. assert "- bullet 19" in tail
  514. assert "- bullet 0" not in tail
  515. def test_is_regression_flags_lost_section_only(tmp_path):
  516. analyzer = _analyzer(tmp_path, _score_questions())
  517. prior = "Summary: a\n\nOpen Questions:\n- who owns it?\n"
  518. assert analyzer.is_regression(prior, "Summary: b\n") is True
  519. assert analyzer.is_regression(prior, "Summary: b\n\nOpen Questions:\n- who?\n") is False
  520. assert analyzer.is_regression("", "anything") is False
  521. assert analyzer.is_regression(None, "x") is False
  522. def test_build_ledger_prompt_lists_every_canonical_section(tmp_path):
  523. analyzer = _analyzer(tmp_path, _score_questions())
  524. prompt = analyzer.build_ledger_prompt("Summary: x", [], ["hi"], None)
  525. for heading in triage.BRIEF_HEADINGS:
  526. assert heading in prompt
  527. def test_parse_brief_unwraps_single_quoted_tool_call_with_apostrophes(tmp_path):
  528. analyzer = _analyzer(tmp_path, _score_questions())
  529. observed = "<|tool_call_start|>[update_brief(updated_brief='the company's game failed')]"
  530. assert analyzer.parse_brief(observed) == "the company's game failed"
  531. def test_parse_brief_unwraps_double_quoted_tool_call(tmp_path):
  532. analyzer = _analyzer(tmp_path, _score_questions())
  533. wrapped = '[update_brief(updated_brief="the company\'s plan")]'
  534. assert analyzer.parse_brief(wrapped) == "the company's plan"
  535. def test_parse_brief_unwraps_trailing_end_marker(tmp_path):
  536. analyzer = _analyzer(tmp_path, _score_questions())
  537. wrapped = "[update_brief(updated_brief='ship friday')]<|tool_call_end|>"
  538. assert analyzer.parse_brief(wrapped) == "ship friday"
  539. def test_parse_brief_unwraps_fenced_and_wrapped(tmp_path):
  540. analyzer = _analyzer(tmp_path, _score_questions())
  541. wrapped = "```\n<|tool_call_start|>[update_brief(updated_brief='fenced payload')]\n```"
  542. assert analyzer.parse_brief(wrapped) == "fenced payload"
  543. def test_parse_brief_does_not_mangle_clean_markdown(tmp_path):
  544. analyzer = _analyzer(tmp_path, _score_questions())
  545. clean = "Here's the [brief]: it's the company's plan - don't mangle [it]."
  546. assert analyzer.parse_brief(clean) == clean
  547. prose = "We should update_brief the notes, here's [why]."
  548. assert analyzer.parse_brief(prose) == prose
  549. def test_parse_brief_empty_tool_call_payload_is_none(tmp_path):
  550. analyzer = _analyzer(tmp_path, _score_questions())
  551. assert analyzer.parse_brief("[update_brief(updated_brief='')]") is None
  552. def test_parse_brief_unwraps_multi_arg_tool_call(tmp_path):
  553. analyzer = _analyzer(tmp_path, _score_questions())
  554. wrapped = (
  555. "[update_brief(updated_brief='## Opportunities\n- ship Friday', "
  556. "previous_brief='## old\n- y', mode='read')]"
  557. )
  558. assert analyzer.parse_brief(wrapped) == "## Opportunities\n- ship Friday"
  559. def test_parse_brief_unwraps_multi_arg_with_markers(tmp_path):
  560. analyzer = _analyzer(tmp_path, _score_questions())
  561. wrapped = (
  562. "<|tool_call_start|>[update_brief(updated_brief='## Opportunities\n- ship Friday', "
  563. "previous_brief='## old\n- y', mode='read')]<|tool_call_end|>"
  564. )
  565. assert analyzer.parse_brief(wrapped) == "## Opportunities\n- ship Friday"
  566. def test_parse_brief_unwraps_multi_arg_apostrophes_preserved(tmp_path):
  567. analyzer = _analyzer(tmp_path, _score_questions())
  568. wrapped = (
  569. "[update_brief(updated_brief='the company's game failed', "
  570. "previous_brief='## old', mode='read')]"
  571. )
  572. assert analyzer.parse_brief(wrapped) == "the company's game failed"
  573. def test_parse_brief_unwraps_positional_tool_call(tmp_path):
  574. analyzer = _analyzer(tmp_path, _score_questions())
  575. assert analyzer.parse_brief("[update_brief('payload here', mode='read')]") == "payload here"
  576. def test_parse_brief_rejects_polluted_tool_call(tmp_path):
  577. analyzer = _analyzer(tmp_path, _score_questions())
  578. polluted = "[update_brief(updated_brief='oops, previous_brief=', mode='read')]"
  579. assert analyzer.parse_brief(polluted) is None
  580. no_arg = "<|tool_call_start|>[update_brief(previous_brief=old, mode=read)]"
  581. assert analyzer.parse_brief(no_arg) is None
  582. def test_parse_brief_clean_mode_text_not_mangled(tmp_path):
  583. analyzer = _analyzer(tmp_path, _score_questions())
  584. clean = "Set mode=read, then read [the brief] - it's fine (really)."
  585. assert analyzer.parse_brief(clean) == clean
  586. def test_parse_brief_rejects_marker_with_artifacts_but_no_call(tmp_path):
  587. analyzer = _analyzer(tmp_path, _score_questions())
  588. assert analyzer.parse_brief("<|tool_call_start|>previous_brief='x', mode='read'") is None
  589. assert analyzer.parse_brief("<|tool_call_start|>mode='read'") is None
  590. def _brief_signals(p=None):
  591. item = {"kind": "noul"}
  592. if p is not None:
  593. item["p"] = p
  594. return {"order": ["substantive_update"], "items": {"substantive_update": item}}
  595. def test_should_update_brief_gate(tmp_path):
  596. analyzer = _analyzer(tmp_path, _score_questions())
  597. assert analyzer.should_update_brief(_brief_signals(0.9)) is True
  598. assert analyzer.should_update_brief(_brief_signals(0.5)) is True # boundary
  599. assert analyzer.should_update_brief(_brief_signals(0.2)) is False
  600. # absent update / no numeric p / not a dict -> fail open
  601. assert analyzer.should_update_brief({"order": [], "items": {}}) is True
  602. assert analyzer.should_update_brief(_brief_signals(None)) is True
  603. assert analyzer.should_update_brief(None) is True
  604. assert analyzer.should_update_brief({"items": {"substantive_update": "nope"}}) is True
  605. def test_should_update_brief_threshold_configured(tmp_path):
  606. analyzer = _analyzer(tmp_path, _score_questions(), brief_gate_threshold=0.8)
  607. assert analyzer.should_update_brief(_brief_signals(0.6)) is False
  608. assert analyzer.should_update_brief(_brief_signals(0.8)) is True
  609. def test_finalize_brief_length_rejects_cut_reply(tmp_path, caplog):
  610. analyzer = _analyzer(tmp_path, _score_questions())
  611. with caplog.at_level(logging.WARNING, logger="stt_daemon.assistant.triage"):
  612. assert analyzer.finalize_brief("y" * 5000, "length") is None
  613. assert caplog.records
  614. def test_finalize_brief_stop_and_empty(tmp_path):
  615. analyzer = _analyzer(tmp_path, _score_questions())
  616. assert analyzer.finalize_brief("a brief", "stop") == "a brief"
  617. assert analyzer.finalize_brief("", "stop") is None
  618. assert analyzer.finalize_brief("```\nfenced\n```", "stop") == "fenced"
  619. assert analyzer.finalize_brief(None, "length") is None
  620. # --- todo 11: cached state() ------------------------------------------------
  621. def test_state_initial_loading_with_good_questions(tmp_path):
  622. analyzer = _analyzer(tmp_path, _score_questions())
  623. assert analyzer.state() == ("loading", "")
  624. def test_state_initial_no_questions_with_bad_path(tmp_path):
  625. missing = str(tmp_path / "missing.json")
  626. analyzer = TriageAnalyzer(_cfg(missing))
  627. state, reason = analyzer.state()
  628. assert state == "no_questions"
  629. assert missing in reason
  630. def test_state_vocabulary_has_no_extra_member():
  631. assert set(triage.TRIAGE_REASONS) == {
  632. "ok",
  633. "no_questions",
  634. "loading",
  635. "laya_missing",
  636. "disabled",
  637. "error",
  638. }
  639. assert "service" + "_down" not in triage.TRIAGE_REASONS
  640. def test_refresh_transitions(tmp_path):
  641. analyzer = _analyzer(tmp_path, _score_questions())
  642. analyzer.refresh("ok")
  643. assert analyzer.state() == ("ok", "")
  644. analyzer.refresh("laya_missing", "run: uv sync --extra triage")
  645. assert analyzer.state() == ("laya_missing", "run: uv sync --extra triage")
  646. analyzer.refresh("error", "boom")
  647. assert analyzer.state() == ("error", "boom")
  648. def test_refresh_unknown_state_ignored(tmp_path, caplog):
  649. analyzer = _analyzer(tmp_path, _score_questions())
  650. analyzer.refresh("ok")
  651. with caplog.at_level(logging.WARNING, logger="stt_daemon.assistant.triage"):
  652. analyzer.refresh("bogus_state")
  653. assert analyzer.state() == ("ok", "")
  654. assert caplog.records
  655. def test_no_questions_state_cannot_be_overridden(tmp_path):
  656. analyzer = TriageAnalyzer(_cfg(str(tmp_path / "missing.json")))
  657. assert analyzer.state()[0] == "no_questions"
  658. analyzer.refresh("ok")
  659. analyzer.refresh("error", "boom")
  660. assert analyzer.state()[0] == "no_questions"
  661. def test_state_does_not_touch_lane(tmp_path):
  662. analyzer = _analyzer(tmp_path, _score_questions())
  663. class StrictLane:
  664. def __init__(self):
  665. self.calls = []
  666. def __getattr__(self, name):
  667. def record(*args, **kwargs):
  668. self.calls.append(name)
  669. return record
  670. lane = StrictLane()
  671. for _ in range(3):
  672. analyzer.state()
  673. analyzer.refresh("ok")
  674. analyzer.state()
  675. assert lane.calls == []
  676. # --- F3: end-to-end triage flow on fakes ------------------------------------
  677. EXTRACT_JSON = (
  678. '{"summary":"s","opportunities":[],"doubts":[],"action_items":[],"risks":[]}'
  679. )
  680. class _Clock:
  681. """Injected clock: no sleeps, fully deterministic."""
  682. def __init__(self, start=1000.0):
  683. self.t = float(start)
  684. def __call__(self):
  685. return self.t
  686. def advance(self, dt):
  687. self.t += float(dt)
  688. class _FakeSubscriber:
  689. def events(self):
  690. return iter(())
  691. def close(self):
  692. pass
  693. class _FakeLane:
  694. """Duck-typed LayaLane: canned-ready, records submits, counts drops."""
  695. def __init__(self):
  696. self.submits: list[tuple] = []
  697. self.drops = 0
  698. def ready(self) -> bool:
  699. return True
  700. def state(self):
  701. return ("ok", "")
  702. def submit(self, state, questions, lang, generation):
  703. self.submits.append((state, questions, lang, generation))
  704. return True
  705. def drop_pending(self):
  706. self.drops += 1
  707. return 1
  708. def _answers(noul_p=0.1):
  709. """A full laya-shaped answers payload for the example questions.
  710. `noul_p` drives every noul arm, including the `substantive_update` gate:
  711. below `brief_gate_threshold` (0.5) the turn is gated OUT (no brief job),
  712. at or above it the turn gates IN.
  713. """
  714. answers = {
  715. "decision_status": {
  716. "type": "choice",
  717. "choice": "made",
  718. "confidence": 0.9,
  719. "answer_confidence": 0.9,
  720. },
  721. "responsibility_clarity": {
  722. "type": "choice",
  723. "choice": "explicit",
  724. "confidence": 0.9,
  725. "answer_confidence": 0.9,
  726. },
  727. }
  728. for qid in (
  729. "substantive_update",
  730. "opportunity",
  731. "blocker",
  732. "action_item",
  733. "deadline",
  734. "open_question",
  735. ):
  736. answers[qid] = {
  737. "type": "noul",
  738. "noul": noul_p,
  739. "confidence": 0.9,
  740. "answer_confidence": 0.9,
  741. }
  742. return {"model": "multilingual", "answers": answers, "usage": {"input_tokens": 42}}
  743. def test_end_to_end_triage_flow():
  744. """signals -> content -> cleared across service, analyzer, lane, and ViewState.
  745. The lane result is wired by calling `service._on_triage_result(gen, payload)`
  746. directly (the runner does exactly this in production); the brief job is
  747. drained with `job.run()` so no worker thread is started. The
  748. single-outstanding guard plus the shared `_pending_brief` collapse settled
  749. turns into ONE job consuming every buffered delta, and the
  750. `substantive_update` gate can reduce that to ZERO jobs.
  751. """
  752. cfg = Config()
  753. cfg.assistant.turn_gap_s = 0.5
  754. cfg.triage.enabled = True
  755. cfg.triage.questions_file = EXAMPLE_PATH
  756. clock = _Clock(1000.0)
  757. lane = _FakeLane()
  758. emitted: list[dict] = []
  759. prompts: list[str] = []
  760. def emit(type_, **fields):
  761. emitted.append({"type": type_, **fields})
  762. analyzer = TriageAnalyzer(cfg.triage)
  763. def brief_fn(prompt, on_io=None, finalize=None):
  764. prompts.append(prompt)
  765. return "NEW FACTS:\nnone\n\nBRIEF:\nSummary: brief\n"
  766. service = AssistantService(
  767. cfg,
  768. _FakeSubscriber(),
  769. emit=emit,
  770. time_fn=clock,
  771. chat_fn=lambda text: "reply",
  772. triage=analyzer,
  773. laya_lane=lane,
  774. brief_fn=brief_fn,
  775. )
  776. def triage_events():
  777. return [e for e in emitted if e["type"] == "triage"]
  778. def brief_jobs():
  779. return [j for j in service._inference._queue if j.name == "brief-update"]
  780. # 1. settled turn -> exactly one lane submit, lang + generation recorded
  781. assert service.toggle_chat() is True
  782. service.handle_event(
  783. {"type": events.TRANSCRIPT, "text": "hola, necesito ayuda", "lang": "es"}
  784. )
  785. clock.advance(1.0)
  786. service.tick()
  787. assert len(lane.submits) == 1
  788. state_text, questions, lang, gen1 = lane.submits[0]
  789. assert "hola, necesito ayuda" in state_text
  790. assert questions is analyzer.questions
  791. assert lang == "es"
  792. assert gen1 == 0
  793. # 2. low-p turn -> signals only: the gate (p=0.1 < 0.5) dispatches ZERO jobs
  794. service._on_triage_result(gen1, _answers(noul_p=0.1))
  795. assert [e["stage"] for e in triage_events()] == ["signals"]
  796. assert triage_events()[0]["triage"]["escalated"] is False
  797. assert service.last_triage is not None
  798. assert service.last_triage["escalated"] is False
  799. assert brief_jobs() == []
  800. assert prompts == []
  801. # 3. high-p turn -> second submit, then ONE brief job (the only one outstanding)
  802. service.handle_event(
  803. {"type": events.TRANSCRIPT, "text": "quiero cancelar mi suscripcion", "lang": "es"}
  804. )
  805. clock.advance(1.0)
  806. service.tick()
  807. assert len(lane.submits) == 2
  808. gen2 = lane.submits[1][3]
  809. service._on_triage_result(gen2, _answers(noul_p=0.9))
  810. assert [e["stage"] for e in triage_events()] == ["signals", "signals"]
  811. assert triage_events()[1]["triage"]["escalated"] is True
  812. jobs = brief_jobs()
  813. assert len(jobs) == 1
  814. # 4. run the brief job directly (deterministic, no worker thread): it consumes
  815. # both buffered deltas and emits ONE content with the brief merged over signals
  816. jobs[0].run()
  817. assert len(prompts) == 1
  818. assert "hola, necesito ayuda" in prompts[0]
  819. assert "quiero cancelar mi suscripcion" in prompts[0]
  820. content_events = [e for e in triage_events() if e["stage"] == "content"]
  821. assert len(content_events) == 1
  822. merged = content_events[0]["triage"]
  823. assert merged["brief"] == "Summary: brief"
  824. assert merged["escalated"] is True # signals preserved under the brief
  825. assert merged["order"] == triage_events()[1]["triage"]["order"]
  826. # 5. drive the emitted stream through a fresh ViewState
  827. view = ViewState()
  828. for event in emitted:
  829. view.ingest(event)
  830. analyzer.refresh("ok") # the lane callback does this in production
  831. view.ingest({"type": "status", **service.status()})
  832. assert view.triage is not None
  833. assert view.triage["brief"] == "Summary: brief"
  834. lines = view.triage_lines()
  835. assert lines
  836. # 6. clear -> cleared event empties the panel and bumps the epoch
  837. baseline_epoch = view.triage_epoch
  838. already = len(emitted)
  839. service.clear()
  840. for event in emitted[already:]:
  841. view.ingest(event)
  842. assert view.triage is None
  843. assert view.triage_epoch > baseline_epoch
  844. # a stale result after clear must not re-populate the panel
  845. service._on_triage_result(gen2, _answers(noul_p=0.9))
  846. assert [e["stage"] for e in triage_events()] == [
  847. "signals",
  848. "signals",
  849. "content",
  850. "cleared",
  851. ]
  852. assert view.triage is None
  853. assert lane.drops == 1
  854. # 7. disabled control: no triage/lane -> status disabled, clear emits nothing
  855. cfg2 = Config()
  856. clock2 = _Clock(2000.0)
  857. emitted2: list[dict] = []
  858. def emit2(type_, **fields):
  859. emitted2.append({"type": type_, **fields})
  860. plain = AssistantService(cfg2, _FakeSubscriber(), emit=emit2, time_fn=clock2)
  861. assert plain.status()["triage_state"] == "disabled"
  862. assert plain.clear() is True
  863. assert [e for e in emitted2 if e["type"] == "triage"] == []
  864. def test_ledger_merge_dedupes_exact_and_punctuation_variants():
  865. ledger = FactLedger()
  866. assert ledger.merge(["The budget was $305 million.", "Sense could find prey."]) == 2
  867. assert ledger.merge(["sense could find prey", "The budget was $305 million"]) == 0
  868. assert ledger.facts() == [
  869. "The budget was $305 million.",
  870. "Sense could find prey.",
  871. ]
  872. def test_ledger_merge_keeps_near_match_with_different_number():
  873. ledger = FactLedger()
  874. ledger.merge(["Sold 1.9 million units."])
  875. assert ledger.merge(["Sold 10 million units."]) == 1
  876. assert len(ledger) == 2
  877. def test_ledger_merge_keeps_near_match_with_different_entity():
  878. ledger = FactLedger()
  879. ledger.merge(["Marcus Smith directed the game."])
  880. assert ledger.merge(["Mike Daly directed the game."]) == 1
  881. assert len(ledger) == 2
  882. def test_ledger_merge_ignores_empty_and_non_string():
  883. ledger = FactLedger()
  884. assert ledger.merge(["", " ", None, 123]) == 0
  885. assert ledger.render() == ""
  886. def test_ledger_render_has_prior_details_section():
  887. ledger = FactLedger()
  888. ledger.merge(["fact one", "fact two"])
  889. assert ledger.render() == "Prior Details:\n- fact one\n- fact two"
  890. def test_ledger_evicts_oldest_over_cap_and_never_new():
  891. ledger = FactLedger(max_facts=2)
  892. ledger.merge(["alpha", "beta"])
  893. ledger.merge(["gamma"])
  894. assert ledger.facts() == ["beta", "gamma"]
  895. def test_ledger_truncates_overlong_fact():
  896. ledger = FactLedger(fact_max_chars=10)
  897. ledger.merge(["x" * 50])
  898. assert ledger.facts() == ["x" * 10]
  899. def test_ledger_clear_empties():
  900. ledger = FactLedger()
  901. ledger.merge(["fact"])
  902. ledger.clear()
  903. assert ledger.facts() == []
  904. assert ledger.render() == ""
  905. def test_ledger_from_brief_harvests_bullets():
  906. ledger = FactLedger.from_brief("Summary: s\n\nPrior Details:\n- one\n- two\n")
  907. assert ledger.facts() == ["one", "two"]
  908. def test_parse_ledger_reply_splits_narrative_and_facts(tmp_path):
  909. analyzer = _analyzer(tmp_path, _score_questions())
  910. narrative, facts = analyzer.parse_ledger_reply(
  911. "NEW FACTS:\n- budget is $305m\n- sense finds prey\n\nBRIEF:\nSummary: ok"
  912. )
  913. assert facts == ["budget is $305m", "sense finds prey"]
  914. assert narrative == "Summary: ok"
  915. def test_parse_ledger_reply_drops_trailing_fact_when_truncated(tmp_path):
  916. analyzer = _analyzer(tmp_path, _score_questions())
  917. _, facts = analyzer.parse_ledger_reply(
  918. "NEW FACTS:\n- complete fact\n- half a fac", truncated=True
  919. )
  920. assert facts == ["complete fact"]
  921. def test_parse_ledger_reply_missing_facts_block(tmp_path):
  922. analyzer = _analyzer(tmp_path, _score_questions())
  923. narrative, facts = analyzer.parse_ledger_reply("BRIEF:\nSummary: only narrative")
  924. assert facts == []
  925. assert narrative == "Summary: only narrative"
  926. def test_parse_ledger_reply_strips_model_prior_details(tmp_path):
  927. analyzer = _analyzer(tmp_path, _score_questions())
  928. narrative, _ = analyzer.parse_ledger_reply(
  929. "NEW FACTS:\nnone\n\nBRIEF:\nSummary: s\n\nPrior Details:\n- model echo\n"
  930. )
  931. assert "model echo" not in narrative
  932. assert "Summary: s" in narrative
  933. def test_parse_ledger_reply_none_on_unusable(tmp_path):
  934. analyzer = _analyzer(tmp_path, _score_questions())
  935. assert analyzer.parse_ledger_reply("") is None
  936. assert analyzer.parse_ledger_reply("just some prose with no sections") is None
  937. def test_parse_facts_ignores_prose_and_handles_markers(tmp_path):
  938. analyzer = _analyzer(tmp_path, _score_questions())
  939. block = "- bullet fact\n* star fact\n• dot fact\n1. numbered fact\nplain prose\nnone"
  940. assert analyzer.parse_facts(block) == [
  941. "bullet fact",
  942. "star fact",
  943. "dot fact",
  944. "numbered fact",
  945. ]
  946. def _rich_questions():
  947. return {
  948. "substantive_update": {
  949. "type": "noul",
  950. "instructions": "This recent conversation segment adds something a running brief "
  951. "should record.",
  952. },
  953. "blocker": {
  954. "type": "noul",
  955. "instructions": "An obstacle, dependency, objection, or risk that could block "
  956. "or delay the work was raised.",
  957. },
  958. "churn_risk": {"type": "noul", "instructions": "User may leave."},
  959. "category": {
  960. "type": "choice",
  961. "instructions": "What is this about?",
  962. "criteria": {"support": "help needed", "sales": "buying"},
  963. },
  964. "severity": {
  965. "type": "score",
  966. "instructions": "How severe is the issue?",
  967. "criteria": ["none", "low", "high", "critical"],
  968. },
  969. }
  970. def _rich_signals(**overrides):
  971. items = {
  972. "blocker": {
  973. "kind": "noul",
  974. "p": 0.91,
  975. "confidence": 0.9,
  976. "answer_confidence": 0.9,
  977. "gate": 0.9,
  978. "flagged": True,
  979. },
  980. "churn_risk": {
  981. "kind": "noul",
  982. "p": 0.10,
  983. "confidence": 0.9,
  984. "answer_confidence": 0.9,
  985. "gate": 0.9,
  986. "flagged": False,
  987. },
  988. "substantive_update": {
  989. "kind": "noul",
  990. "p": 0.10,
  991. "confidence": 0.9,
  992. "answer_confidence": 0.9,
  993. "gate": 0.9,
  994. "flagged": False,
  995. },
  996. "category": {
  997. "kind": "choice",
  998. "choice": "support",
  999. "confidence": 0.4,
  1000. "answer_confidence": 0.4,
  1001. "gate": 0.4,
  1002. "flagged": True,
  1003. },
  1004. "severity": {
  1005. "kind": "score",
  1006. "expected": 2.6,
  1007. "level": 3,
  1008. "label": "critical",
  1009. "confidence": 0.9,
  1010. "answer_confidence": 0.9,
  1011. "gate": 0.9,
  1012. "flagged": True,
  1013. },
  1014. }
  1015. items.update(overrides.get("items", {}))
  1016. order = overrides.get(
  1017. "order", ["blocker", "churn_risk", "substantive_update", "category", "severity"])
  1018. return {"order": order, "items": items, "escalated": overrides.get("escalated", False),
  1019. "input_tokens": 0}
  1020. def test_signal_lines_filters_low_noul_but_keeps_gate_qid(tmp_path):
  1021. analyzer = _analyzer(tmp_path, _rich_questions())
  1022. lines = analyzer._signal_lines(_rich_signals())
  1023. text = "\n".join(lines)
  1024. assert not any("churn_risk" in line for line in lines) # p=0.10 filtered
  1025. assert any("substantive_update" in line for line in lines) # gate qid kept
  1026. assert any("blocker" in line for line in lines) # high-signal kept
  1027. assert "p=0.91" in text and "flagged" in text
  1028. assert "could block or delay" in text # instructions carried
  1029. assert all(line.startswith("- ") for line in lines)
  1030. # original order preserved among kept lines
  1031. kept = [line for line in lines]
  1032. assert [i for i, line in enumerate(kept) if "blocker" in line][0] < [
  1033. i for i, line in enumerate(kept) if "category" in line][0]
  1034. def test_signal_lines_choice_carries_instructions_and_flag(tmp_path):
  1035. analyzer = _analyzer(tmp_path, _rich_questions())
  1036. lines = analyzer._signal_lines(_rich_signals(order=["category"], escalated=False))
  1037. assert len(lines) == 1
  1038. assert "category (choice): selected support (conf 0.40, flagged as uncertain)" in lines[0]
  1039. assert "What is this about?" in lines[0]
  1040. assert "help needed" in lines[0] # criteria[selected] definition
  1041. def test_signal_lines_escalated_marker_only_on_flagged(tmp_path):
  1042. analyzer = _analyzer(tmp_path, _rich_questions())
  1043. lines = analyzer._signal_lines(_rich_signals(
  1044. order=["blocker", "substantive_update"], escalated=True))
  1045. by_qid = {line.split(" ")[1]: line for line in lines}
  1046. assert by_qid["blocker"].endswith(" [escalated]")
  1047. assert not by_qid["substantive_update"].endswith(" [escalated]")
  1048. plain = analyzer._signal_lines(_rich_signals(
  1049. order=["blocker", "substantive_update"], escalated=False))
  1050. assert not any("[escalated]" in line for line in plain)
  1051. def test_signal_lines_score_format_and_instruction_cap(tmp_path):
  1052. questions = _rich_questions()
  1053. questions["severity"]["instructions"] = "x " * 200 # force long instructions
  1054. analyzer = _analyzer(tmp_path, questions)
  1055. lines = analyzer._signal_lines(_rich_signals(order=["severity"]))
  1056. assert len(lines) == 1
  1057. assert "severity (score): expected 2.60, level 3 (critical), flagged" in lines[0]
  1058. assert len(lines[0]) <= len("- severity (score): expected 2.60, level 3 "
  1059. "(critical), flagged — ") + 160 + len(" [escalated]")
  1060. assert lines[0].rstrip().endswith("…")
  1061. def test_signal_lines_never_raises_on_malformed(tmp_path):
  1062. analyzer = _analyzer(tmp_path, _rich_questions())
  1063. assert analyzer._signal_lines(None) == [] # type: ignore[arg-type]
  1064. assert analyzer._signal_lines({}) == [] # type: ignore[dict-item]
  1065. assert analyzer._signal_lines({"order": "nope", "items": "nope"}) == [] # type: ignore[dict-item]
  1066. assert analyzer._signal_lines({"order": ["ghost"], "items": {}}) == []
  1067. def test_signal_lines_opportunity_kept_below_threshold(tmp_path):
  1068. questions = _rich_questions()
  1069. questions["opportunity"] = {"type": "noul", "instructions": "A new lead."}
  1070. analyzer = _analyzer(tmp_path, questions)
  1071. items = _rich_signals()["items"]
  1072. items["opportunity"] = {
  1073. "kind": "noul",
  1074. "p": 0.10,
  1075. "confidence": 0.9,
  1076. "answer_confidence": 0.9,
  1077. "gate": 0.9,
  1078. "flagged": False,
  1079. }
  1080. signals = {"order": ["blocker", "opportunity", "churn_risk"],
  1081. "items": items, "escalated": False}
  1082. lines = analyzer._signal_lines(signals)
  1083. assert any("opportunity" in line for line in lines)
  1084. assert not any("churn_risk" in line for line in lines)
  1085. def test_signal_lines_opportunity_ordered_first(tmp_path):
  1086. questions = _rich_questions()
  1087. questions["opportunity"] = {"type": "noul", "instructions": "A new lead."}
  1088. analyzer = _analyzer(tmp_path, questions)
  1089. items = _rich_signals()["items"]
  1090. items["opportunity"] = {
  1091. "kind": "noul",
  1092. "p": 0.80,
  1093. "confidence": 0.9,
  1094. "answer_confidence": 0.9,
  1095. "gate": 0.9,
  1096. "flagged": False,
  1097. }
  1098. signals = {"order": ["blocker", "severity", "opportunity"],
  1099. "items": items, "escalated": False}
  1100. lines = analyzer._signal_lines(signals)
  1101. assert lines[0].startswith("- opportunity")
  1102. # relative order of the rest preserved
  1103. rest_idx = [i for i, line in enumerate(lines) if "blocker" in line][0]
  1104. sev_idx = [i for i, line in enumerate(lines) if "severity" in line][0]
  1105. assert rest_idx < sev_idx
  1106. def test_build_ledger_prompt_contains_opportunity_lead_rule(tmp_path):
  1107. analyzer = _analyzer(tmp_path, _rich_questions())
  1108. prompt = analyzer.build_ledger_prompt("Summary: x", [], ["hi"], None)
  1109. assert "currently-open opportunities first" in prompt
  1110. assert "Opportunities section the most complete" in prompt
  1111. def test_normalize_uses_per_question_threshold(tmp_path):
  1112. questions = {
  1113. "strict": {"type": "noul", "instructions": "a", "threshold": 0.9},
  1114. "lax": {"type": "noul", "instructions": "b", "threshold": 0.2},
  1115. "default": {"type": "noul", "instructions": "c"},
  1116. }
  1117. analyzer = _analyzer(tmp_path, questions)
  1118. payload = {
  1119. qid: {"noul": 0.5, "confidence": 0.5, "answer_confidence": 0.5}
  1120. for qid in questions
  1121. }
  1122. result = analyzer.normalize(payload)
  1123. items = result["items"]
  1124. assert items["strict"]["threshold"] == 0.9
  1125. assert items["strict"]["flagged"] is False
  1126. assert items["lax"]["threshold"] == 0.2
  1127. assert items["lax"]["flagged"] is True
  1128. assert items["default"]["threshold"] == 0.6
  1129. def test_normalize_counts_hidden_low_signals(tmp_path):
  1130. analyzer = _analyzer(tmp_path, _score_questions())
  1131. payload = {
  1132. "severity": {"score": 0.1, "confidence": 0.9, "answer_confidence": 0.9},
  1133. "category": {"choice": "sales", "confidence": 0.9, "answer_confidence": 0.9},
  1134. "churn_risk": {"noul": 0.05, "confidence": 0.95, "answer_confidence": 0.95},
  1135. }
  1136. result = analyzer.normalize(payload)
  1137. assert result["hidden"] == 1
  1138. assert "churn_risk" in result["items"]
  1139. def _gate_signals(choice="sales", hot=False):
  1140. return {
  1141. "order": ["substantive_update", "category", "churn_risk"],
  1142. "items": {
  1143. "substantive_update": {
  1144. "kind": "noul",
  1145. "p": 0.1,
  1146. "threshold": 0.5,
  1147. "flagged": False,
  1148. },
  1149. "category": {
  1150. "kind": "choice",
  1151. "choice": choice,
  1152. "gate": 0.9,
  1153. "flagged": False,
  1154. },
  1155. "churn_risk": {
  1156. "kind": "noul",
  1157. "p": 0.9 if hot else 0.1,
  1158. "threshold": 0.5,
  1159. "flagged": hot,
  1160. },
  1161. },
  1162. "escalated": False,
  1163. }
  1164. def test_should_update_brief_fires_on_a_choice_transition(tmp_path):
  1165. analyzer = _analyzer(tmp_path, _score_questions())
  1166. assert analyzer.should_update_brief(_gate_signals("support"), _gate_signals("sales")) is True
  1167. assert analyzer.should_update_brief(_gate_signals("sales"), _gate_signals("sales")) is False
  1168. def test_should_update_brief_fires_on_a_newly_hot_signal(tmp_path):
  1169. analyzer = _analyzer(tmp_path, _score_questions())
  1170. assert analyzer.should_update_brief(_gate_signals(hot=True), _gate_signals()) is True
  1171. current = _gate_signals(hot=True)
  1172. assert analyzer.should_update_brief(current, current) is False
  1173. def test_should_update_brief_ignores_transitions_without_prior(tmp_path):
  1174. analyzer = _analyzer(tmp_path, _score_questions())
  1175. assert analyzer.should_update_brief(_gate_signals(hot=True), None) is False
  1176. def test_should_update_brief_fail_open_when_arm_absent(tmp_path):
  1177. analyzer = _analyzer(tmp_path, _score_questions())
  1178. assert analyzer.should_update_brief(_signals(noul_p=0.1), _signals(noul_p=0.1)) is True
  1179. def test_window_line_shortens_and_respects_toggle():
  1180. assert triage.window_line("Elvis Avelardo García", "hola") == "Elvis: hola"
  1181. assert triage.window_line("", "hola") == "Unknown: hola"
  1182. assert triage.window_line("Elvis", "hola", include_speaker=False) == "hola"
  1183. def test_load_questions_accepts_and_validates_threshold(tmp_path):
  1184. good = {"q": {"type": "noul", "instructions": "x", "threshold": 0.85}}
  1185. analyzer = _analyzer(tmp_path, good)
  1186. assert analyzer.questions is not None
  1187. assert analyzer.questions["q"]["threshold"] == 0.85
  1188. for bad in (0, -0.1, 1.5, "high", True):
  1189. definition = {"type": "noul", "instructions": "x", "threshold": bad}
  1190. path = _write_questions(tmp_path, {"q": definition})
  1191. assert load_questions(path) is None