test_triage.py 34 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970
  1. """Tests for the triage analyzer (todos 3, 9, 10, 11). Pure, hardware-free."""
  2. import json
  3. import logging
  4. from pathlib import Path
  5. from types import SimpleNamespace
  6. import pytest
  7. from stt_daemon import events
  8. from stt_daemon.assistant import triage
  9. from stt_daemon.assistant.service import AssistantService
  10. from stt_daemon.assistant.triage import TriageAnalyzer, load_questions
  11. from stt_daemon.config import Config
  12. from stt_daemon.tui.state import ViewState
  13. EXAMPLE_PATH = str(Path(__file__).resolve().parent.parent / "questions.example.json")
  14. def _cfg(path=EXAMPLE_PATH, **overrides):
  15. base = {
  16. "questions_file": path,
  17. "state_char_budget": 1200,
  18. "confidence_field": "answer_confidence",
  19. "escalate_threshold": 0.6,
  20. "choice_min_confidence": 0.85,
  21. }
  22. base.update(overrides)
  23. return SimpleNamespace(**base)
  24. def _write_questions(tmp_path, questions):
  25. path = tmp_path / "questions.json"
  26. path.write_text(json.dumps(questions), encoding="utf-8")
  27. return str(path)
  28. def _analyzer(tmp_path, questions=None, **overrides):
  29. if questions is None:
  30. return TriageAnalyzer(_cfg(**overrides))
  31. return TriageAnalyzer(_cfg(_write_questions(tmp_path, questions), **overrides))
  32. # --- todo 3: load_questions -------------------------------------------------
  33. def test_load_example_order_preserved():
  34. raw = json.loads(Path(EXAMPLE_PATH).read_text(encoding="utf-8"))
  35. questions = load_questions(EXAMPLE_PATH)
  36. assert questions is not None
  37. assert questions
  38. assert list(questions.keys()) == list(raw.keys())
  39. for definition in questions.values():
  40. assert definition["type"] in {"choice", "score", "noul"}
  41. assert isinstance(definition["instructions"], str)
  42. assert definition["instructions"].strip()
  43. def test_example_questions_match_design():
  44. questions = load_questions(EXAMPLE_PATH)
  45. assert questions is not None
  46. assert list(questions.keys()) == [
  47. "substantive_update",
  48. "opportunity",
  49. "blocker",
  50. "decision_status",
  51. "action_item",
  52. "responsibility_clarity",
  53. "deadline",
  54. "open_question",
  55. ]
  56. for qid, definition in questions.items():
  57. assert isinstance(definition["instructions"], str)
  58. assert definition["instructions"].strip()
  59. if definition["type"] == "choice":
  60. criteria = definition["criteria"]
  61. options = list(criteria.keys()) if isinstance(criteria, dict) else list(criteria)
  62. assert "other" in options
  63. assert len(options) <= 7
  64. if definition["type"] == "noul":
  65. assert isinstance(definition.get("instructions"), str)
  66. assert definition["instructions"].strip()
  67. def test_load_missing_file_returns_none_with_warning(caplog):
  68. with caplog.at_level(logging.WARNING, logger="stt_daemon.assistant.triage"):
  69. assert load_questions("/nonexistent/questions.json") is None
  70. assert caplog.records, "expected a warning for the missing file"
  71. def test_load_bad_json_returns_none_with_warning(tmp_path, caplog):
  72. path = tmp_path / "questions.json"
  73. path.write_text("{not valid json", encoding="utf-8")
  74. with caplog.at_level(logging.WARNING, logger="stt_daemon.assistant.triage"):
  75. assert load_questions(str(path)) is None
  76. assert caplog.records
  77. @pytest.mark.parametrize(
  78. "bad_entry",
  79. [
  80. {"type": "mystery", "instructions": "do something"}, # unknown type
  81. {"type": "score", "instructions": "rate it", "criteria": {"0": "low"}}, # score dict
  82. { # noul criteria with a bad key
  83. "type": "noul",
  84. "instructions": "holds?",
  85. "criteria": {"true": "yes", "maybe": "hmm"},
  86. },
  87. "just a string", # non-object entry
  88. {"type": "choice", "instructions": "pick one", "criteria": []}, # empty choice
  89. {"type": "choice", "instructions": "pick one", "criteria": {}}, # empty choice dict
  90. {"type": "choice", "criteria": ["a", "b"]}, # missing instructions
  91. {"type": "score", "instructions": "rate it", "criteria": ["a", None]}, # None level
  92. {"type": "noul", "instructions": "holds?", "criteria": ["true"]}, # noul list
  93. ],
  94. )
  95. def test_load_malformed_returns_none_with_warning(tmp_path, caplog, bad_entry):
  96. path = _write_questions(
  97. tmp_path, {"good": {"type": "noul", "instructions": "ok?"}, "bad": bad_entry}
  98. )
  99. with caplog.at_level(logging.WARNING, logger="stt_daemon.assistant.triage"):
  100. assert load_questions(str(path)) is None
  101. assert caplog.records
  102. # --- todo 9: normalize ------------------------------------------------------
  103. def _score_questions():
  104. return {
  105. "severity": {
  106. "type": "score",
  107. "instructions": "How severe is the issue?",
  108. "criteria": ["none", "low", "high", "critical"],
  109. },
  110. "category": {
  111. "type": "choice",
  112. "instructions": "What is this about?",
  113. "criteria": {"support": "help needed", "sales": "buying"},
  114. },
  115. "churn_risk": {"type": "noul", "instructions": "User may leave."},
  116. }
  117. def test_normalize_choice_exact(tmp_path):
  118. analyzer = _analyzer(tmp_path, _score_questions())
  119. payload = {
  120. "model": "multilingual",
  121. "answers": {
  122. "category": {
  123. "type": "choice",
  124. "choice": "support",
  125. "confidence": 0.7,
  126. "answer_confidence": 0.4,
  127. }
  128. },
  129. "usage": {"input_tokens": 123, "output_tokens": 0},
  130. }
  131. result = analyzer.normalize(payload)
  132. assert result["order"] == ["category"]
  133. assert result["items"]["category"] == {
  134. "kind": "choice",
  135. "choice": "support",
  136. "confidence": 0.7,
  137. "answer_confidence": 0.4,
  138. "gate": 0.4,
  139. "flagged": True,
  140. }
  141. assert result["input_tokens"] == 123
  142. assert result["escalated"] is True
  143. def test_normalize_score_with_legend(tmp_path):
  144. analyzer = _analyzer(tmp_path, _score_questions())
  145. payload = {
  146. "answers": {
  147. "severity": {
  148. "type": "score",
  149. "score": 2.6,
  150. "legend": {"0": "none", "1": "low", "2": "high", "3": "critical"},
  151. "confidence": 0.5,
  152. "answer_confidence": 0.6,
  153. }
  154. },
  155. "usage": {"input_tokens": 7, "output_tokens": 0},
  156. }
  157. item = analyzer.normalize(payload)["items"]["severity"]
  158. assert item["kind"] == "score"
  159. assert item["expected"] == 2.6
  160. assert item["level"] == 3
  161. assert item["label"] == "critical"
  162. assert item["confidence"] == 0.5
  163. assert item["answer_confidence"] == 0.6
  164. assert item["gate"] == 0.6
  165. assert item["flagged"] is True
  166. def test_normalize_score_legend_fallback(tmp_path):
  167. analyzer = _analyzer(tmp_path, _score_questions())
  168. payload = {
  169. "answers": {
  170. "severity": {"type": "score", "score": 1.2, "confidence": 0.9, "answer_confidence": 0.9}
  171. }
  172. }
  173. item = analyzer.normalize(payload)["items"]["severity"]
  174. assert item["level"] == 1
  175. assert item["label"] == "low" # falls back to criteria[level]
  176. assert item["flagged"] is False # 1.2 < (4 - 1) * 0.5
  177. def test_normalize_noul_and_bare_payload(tmp_path):
  178. analyzer = _analyzer(tmp_path, _score_questions())
  179. bare = {
  180. "churn_risk": {"type": "noul", "noul": 0.9, "confidence": 0.9, "answer_confidence": 0.85}
  181. }
  182. result = analyzer.normalize(bare)
  183. assert result["order"] == ["churn_risk"]
  184. assert result["items"]["churn_risk"] == {
  185. "kind": "noul",
  186. "p": 0.9,
  187. "confidence": 0.9,
  188. "answer_confidence": 0.85,
  189. "gate": 0.85,
  190. "flagged": True,
  191. }
  192. assert result["input_tokens"] == 0
  193. def test_normalize_skips_unknown_qid_and_malformed(tmp_path):
  194. analyzer = _analyzer(tmp_path, _score_questions())
  195. payload = {
  196. "answers": {
  197. "ghost": {"type": "noul", "noul": 0.99, "confidence": 0.9, "answer_confidence": 0.9},
  198. "category": {"type": "choice", "choice": 42}, # malformed: no confidences
  199. "churn_risk": "nonsense", # malformed: not a dict
  200. },
  201. "usage": {"input_tokens": 3, "output_tokens": 0},
  202. }
  203. result = analyzer.normalize(payload)
  204. assert result["order"] == []
  205. assert result["items"] == {}
  206. assert result["escalated"] is False
  207. def test_normalize_ignores_side_channel_fields(tmp_path):
  208. analyzer = _analyzer(tmp_path, _score_questions())
  209. payload = {
  210. "answers": {
  211. "category": {
  212. "type": "choice",
  213. "choice": "sales",
  214. "confidence": 0.95,
  215. "answer_confidence": 0.9,
  216. "action": {"act" + "_probability": 0.99},
  217. }
  218. },
  219. "usage": {"input_tokens": 1, "output_tokens": 0},
  220. }
  221. result = analyzer.normalize(payload)
  222. assert result["items"]["category"]["flagged"] is False
  223. assert "act" + "_probability" not in json.dumps(result)
  224. source = Path(triage.__file__).read_text(encoding="utf-8")
  225. assert "act" + "_probability" not in source
  226. def test_normalize_gating_field_selectable(tmp_path):
  227. analyzer = _analyzer(tmp_path, _score_questions(), confidence_field="confidence")
  228. payload = {
  229. "answers": {
  230. "category": {
  231. "type": "choice",
  232. "choice": "sales",
  233. "confidence": 0.5,
  234. "answer_confidence": 0.99,
  235. }
  236. }
  237. }
  238. item = analyzer.normalize(payload)["items"]["category"]
  239. assert item["gate"] == 0.5 # gates on confidence, keeps both
  240. assert item["answer_confidence"] == 0.99
  241. assert item["flagged"] is True
  242. # --- todo 9: condensation ---------------------------------------------------
  243. def test_condense_keeps_newest_chronological_no_numerals(tmp_path):
  244. analyzer = _analyzer(tmp_path, _score_questions(), state_char_budget=40)
  245. window = ["alpha first", "beta second", "gamma third newest here"]
  246. state = analyzer.build_state(window)
  247. assert "gamma third newest here" in state
  248. assert "alpha first" not in state # oldest dropped under the budget
  249. lines = state.splitlines()
  250. assert all(line.startswith("- ") for line in lines)
  251. assert not any(ch.isdigit() for ch in state) # no numerals injected
  252. # chronological restore: kept lines stay in original order
  253. kept = [u for u in window if u in state]
  254. assert [line[2:] for line in lines] == kept
  255. def test_condense_empty_window(tmp_path):
  256. analyzer = _analyzer(tmp_path, _score_questions())
  257. assert analyzer.build_state([]) == ""
  258. # --- todo 10: escalation + extraction ---------------------------------------
  259. def _signals(choice_gate=0.9, noul_p=0.1, score_expected=0.2):
  260. return {
  261. "order": ["category", "churn_risk", "severity"],
  262. "items": {
  263. "category": {
  264. "kind": "choice",
  265. "choice": "sales",
  266. "confidence": choice_gate,
  267. "answer_confidence": choice_gate,
  268. "gate": choice_gate,
  269. "flagged": choice_gate < 0.85,
  270. },
  271. "churn_risk": {
  272. "kind": "noul",
  273. "p": noul_p,
  274. "confidence": noul_p,
  275. "answer_confidence": noul_p,
  276. "gate": noul_p,
  277. "flagged": noul_p >= 0.6,
  278. },
  279. "severity": {
  280. "kind": "score",
  281. "expected": score_expected,
  282. "level": 0,
  283. "label": "none",
  284. "confidence": 0.9,
  285. "answer_confidence": 0.9,
  286. "gate": 0.9,
  287. "flagged": score_expected >= 1.5,
  288. },
  289. },
  290. "escalated": False,
  291. "input_tokens": 0,
  292. }
  293. def test_escalation_arms_fire_independently(tmp_path):
  294. analyzer = _analyzer(tmp_path, _score_questions())
  295. assert analyzer.should_escalate(_signals(choice_gate=0.5)) is True # choice arm
  296. assert analyzer.should_escalate(_signals(noul_p=0.8)) is True # noul arm
  297. assert analyzer.should_escalate(_signals(score_expected=2.0)) is True # score arm
  298. assert analyzer.should_escalate(_signals()) is False
  299. def test_normalize_escalated_matches_should_escalate(tmp_path):
  300. analyzer = _analyzer(tmp_path, _score_questions())
  301. payload = {
  302. "answers": {
  303. "category": {
  304. "type": "choice",
  305. "choice": "sales",
  306. "confidence": 0.9,
  307. "answer_confidence": 0.9,
  308. },
  309. "churn_risk": {
  310. "type": "noul",
  311. "noul": 0.1,
  312. "confidence": 0.9,
  313. "answer_confidence": 0.9,
  314. },
  315. "severity": {
  316. "type": "score",
  317. "score": 0.1,
  318. "confidence": 0.9,
  319. "answer_confidence": 0.9,
  320. },
  321. }
  322. }
  323. result = analyzer.normalize(payload)
  324. assert result["escalated"] == analyzer.should_escalate(result) is False
  325. # --- todo 5: brief prompt / parse / gate / finalize --------------------------
  326. def test_build_brief_prompt_order_and_focus(tmp_path):
  327. analyzer = _analyzer(tmp_path, _score_questions())
  328. prompt = analyzer.build_brief_prompt(
  329. ["Alice: let's ship Friday", "Bob: agreed"], _signals(noul_p=0.8)
  330. )
  331. assert prompt.startswith("New utterances:\n- Alice: let's ship Friday\n- Bob: agreed")
  332. assert "Focus:" in prompt
  333. assert "churn_risk" in prompt # signal named in words
  334. assert prompt.index("New utterances:") < prompt.index("Focus:")
  335. assert prompt.index("Focus:") < prompt.index("Return only the updated brief")
  336. # no JSON extraction keys or raw dict shapes leak into the brief prompt
  337. for key in ("summary", "opportunities", "action_items", "risks", "doubts"):
  338. assert key not in prompt
  339. assert "{" not in prompt and "}" not in prompt
  340. def test_build_brief_prompt_no_focus_without_signals(tmp_path):
  341. analyzer = _analyzer(tmp_path, _score_questions())
  342. for signals in (None, {}, {"order": [], "items": {}}):
  343. prompt = analyzer.build_brief_prompt(["solo utterance"], signals)
  344. assert "solo utterance" in prompt
  345. assert "Focus:" not in prompt
  346. assert prompt.endswith("Return only the updated brief as plain text.")
  347. def test_build_pinned_brief_prompt_inlines_current_brief(tmp_path):
  348. analyzer = _analyzer(tmp_path, _score_questions())
  349. prompt = analyzer.build_pinned_brief_prompt(
  350. "Summary: shipped\n\nPrior Details:\n- fact one",
  351. ["Alice: let's ship Friday"],
  352. _signals(noul_p=0.8),
  353. )
  354. assert prompt.startswith("CURRENT BRIEF:\nSummary: shipped")
  355. assert "- fact one" in prompt
  356. assert "NEW UTTERANCES:\n- Alice: let's ship Friday" in prompt
  357. assert prompt.index("CURRENT BRIEF:") < prompt.index("NEW UTTERANCES:")
  358. assert prompt.index("NEW UTTERANCES:") < prompt.index("Focus:")
  359. assert "churn_risk" in prompt
  360. assert prompt.endswith("Return only the updated brief as plain text.")
  361. def test_build_pinned_brief_prompt_none_brief_is_placeholder(tmp_path):
  362. analyzer = _analyzer(tmp_path, _score_questions())
  363. prompt = analyzer.build_pinned_brief_prompt(None, ["solo"], None)
  364. assert "CURRENT BRIEF:\n(none yet)" in prompt
  365. assert "NEW UTTERANCES:\n- solo" in prompt
  366. assert "Focus:" not in prompt
  367. def test_extract_bullets_strips_new_marker_and_normalizes(tmp_path):
  368. analyzer = _analyzer(tmp_path, _score_questions())
  369. brief = (
  370. "Summary: things\n"
  371. "- top level bullet\n"
  372. " - indented child\n"
  373. "- **New:** tagged fact\n"
  374. "not a bullet\n"
  375. "\n"
  376. )
  377. assert analyzer.extract_bullets(brief) == [
  378. "top level bullet",
  379. "indented child",
  380. "tagged fact",
  381. ]
  382. def test_restore_missing_bullets_reappends_dropped_fact(tmp_path):
  383. analyzer = _analyzer(tmp_path, _score_questions())
  384. prior = (
  385. "Summary: old\n\nPrior Details:\n"
  386. "- The budget was $305 million.\n"
  387. "- Sense could find prey.\n"
  388. )
  389. new = "Summary: newer\n\nPrior Details:\n- Sense could find prey.\n"
  390. restored = analyzer.restore_missing_bullets(new, prior)
  391. assert "- Sense could find prey." in restored
  392. assert "- The budget was $305 million." in restored
  393. assert restored.count("- Sense could find prey.") == 1
  394. def test_restore_missing_bullets_matches_reworded_bullet(tmp_path):
  395. analyzer = _analyzer(tmp_path, _score_questions())
  396. prior = "Prior Details:\n- **New:** Sense could find prey.\n"
  397. new = "Prior Details:\n- Sense could find prey.\n"
  398. restored = analyzer.restore_missing_bullets(new, prior)
  399. assert restored == new
  400. def test_restore_missing_bullets_restores_reworded_as_new_fact(tmp_path):
  401. analyzer = _analyzer(tmp_path, _score_questions())
  402. prior = "Prior Details:\n- The budget was $305 million.\n"
  403. new = "Prior Details:\n- Budget: $305 million (nearly Spider-Man 2).\n"
  404. restored = analyzer.restore_missing_bullets(new, prior)
  405. assert restored.count("$305 million") == 2
  406. assert restored.rstrip().endswith("- The budget was $305 million.")
  407. def test_restore_missing_bullets_noop_without_prior_or_new(tmp_path):
  408. analyzer = _analyzer(tmp_path, _score_questions())
  409. new = "Summary: only\nPrior Details:\n- keep me\n"
  410. assert analyzer.restore_missing_bullets(new, "") == new
  411. assert analyzer.restore_missing_bullets(new, None) == new
  412. assert analyzer.restore_missing_bullets("", new) == ""
  413. assert analyzer.restore_missing_bullets(None, new) is None
  414. def test_restore_missing_bullets_appends_section_when_absent(tmp_path):
  415. analyzer = _analyzer(tmp_path, _score_questions())
  416. prior = "- orphan fact\n"
  417. new = "Summary: no prior details header\n"
  418. restored = analyzer.restore_missing_bullets(new, prior)
  419. assert "- orphan fact" in restored
  420. assert "Prior Details:" in restored
  421. def test_parse_brief_strips_fences_and_rejects_empty(tmp_path):
  422. analyzer = _analyzer(tmp_path, _score_questions())
  423. assert analyzer.parse_brief("```\nhello brief\n```") == "hello brief"
  424. assert analyzer.parse_brief("```markdown\nhello brief\n```") == "hello brief"
  425. assert analyzer.parse_brief("plain brief") == "plain brief"
  426. assert analyzer.parse_brief(" padded ") == "padded"
  427. assert analyzer.parse_brief("") is None
  428. assert analyzer.parse_brief(" ") is None
  429. assert analyzer.parse_brief(None) is None
  430. assert analyzer.parse_brief(42) is None
  431. assert analyzer.parse_brief("```") is None
  432. def test_parse_brief_unescapes_literal_backslash_n(tmp_path):
  433. analyzer = _analyzer(tmp_path, _score_questions())
  434. assert analyzer.parse_brief("## Title\\n\\nBody line") == "## Title\n\nBody line"
  435. assert analyzer.parse_brief("a\\tb") == "a\tb"
  436. assert analyzer.parse_brief("line1\nline2\\nline3") == "line1\nline2\\nline3"
  437. def test_parse_brief_truncates_not_rejects(tmp_path):
  438. analyzer = _analyzer(tmp_path, _score_questions())
  439. parsed = analyzer.parse_brief("x" * 5000)
  440. assert parsed is not None
  441. assert parsed == "x" * 4000
  442. assert len(parsed) == 4000
  443. wrapped = "[update_brief(updated_brief='" + "x" * 5000 + "')]"
  444. assert analyzer.parse_brief(wrapped) == "x" * 4000
  445. def test_parse_brief_honors_configured_cap(tmp_path):
  446. analyzer = _analyzer(tmp_path, _score_questions(), brief_max_chars=10)
  447. assert analyzer.parse_brief("x" * 50) == "x" * 10
  448. def test_parse_brief_unwraps_single_quoted_tool_call_with_apostrophes(tmp_path):
  449. analyzer = _analyzer(tmp_path, _score_questions())
  450. observed = "<|tool_call_start|>[update_brief(updated_brief='the company's game failed')]"
  451. assert analyzer.parse_brief(observed) == "the company's game failed"
  452. def test_parse_brief_unwraps_double_quoted_tool_call(tmp_path):
  453. analyzer = _analyzer(tmp_path, _score_questions())
  454. wrapped = '[update_brief(updated_brief="the company\'s plan")]'
  455. assert analyzer.parse_brief(wrapped) == "the company's plan"
  456. def test_parse_brief_unwraps_trailing_end_marker(tmp_path):
  457. analyzer = _analyzer(tmp_path, _score_questions())
  458. wrapped = "[update_brief(updated_brief='ship friday')]<|tool_call_end|>"
  459. assert analyzer.parse_brief(wrapped) == "ship friday"
  460. def test_parse_brief_unwraps_fenced_and_wrapped(tmp_path):
  461. analyzer = _analyzer(tmp_path, _score_questions())
  462. wrapped = "```\n<|tool_call_start|>[update_brief(updated_brief='fenced payload')]\n```"
  463. assert analyzer.parse_brief(wrapped) == "fenced payload"
  464. def test_parse_brief_does_not_mangle_clean_markdown(tmp_path):
  465. analyzer = _analyzer(tmp_path, _score_questions())
  466. clean = "Here's the [brief]: it's the company's plan - don't mangle [it]."
  467. assert analyzer.parse_brief(clean) == clean
  468. prose = "We should update_brief the notes, here's [why]."
  469. assert analyzer.parse_brief(prose) == prose
  470. def test_parse_brief_empty_tool_call_payload_is_none(tmp_path):
  471. analyzer = _analyzer(tmp_path, _score_questions())
  472. assert analyzer.parse_brief("[update_brief(updated_brief='')]") is None
  473. def test_parse_brief_unwraps_multi_arg_tool_call(tmp_path):
  474. analyzer = _analyzer(tmp_path, _score_questions())
  475. wrapped = (
  476. "[update_brief(updated_brief='## Opportunities\n- ship Friday', "
  477. "previous_brief='## old\n- y', mode='read')]"
  478. )
  479. assert analyzer.parse_brief(wrapped) == "## Opportunities\n- ship Friday"
  480. def test_parse_brief_unwraps_multi_arg_with_markers(tmp_path):
  481. analyzer = _analyzer(tmp_path, _score_questions())
  482. wrapped = (
  483. "<|tool_call_start|>[update_brief(updated_brief='## Opportunities\n- ship Friday', "
  484. "previous_brief='## old\n- y', mode='read')]<|tool_call_end|>"
  485. )
  486. assert analyzer.parse_brief(wrapped) == "## Opportunities\n- ship Friday"
  487. def test_parse_brief_unwraps_multi_arg_apostrophes_preserved(tmp_path):
  488. analyzer = _analyzer(tmp_path, _score_questions())
  489. wrapped = (
  490. "[update_brief(updated_brief='the company's game failed', "
  491. "previous_brief='## old', mode='read')]"
  492. )
  493. assert analyzer.parse_brief(wrapped) == "the company's game failed"
  494. def test_parse_brief_unwraps_positional_tool_call(tmp_path):
  495. analyzer = _analyzer(tmp_path, _score_questions())
  496. assert analyzer.parse_brief("[update_brief('payload here', mode='read')]") == "payload here"
  497. def test_parse_brief_rejects_polluted_tool_call(tmp_path):
  498. analyzer = _analyzer(tmp_path, _score_questions())
  499. polluted = "[update_brief(updated_brief='oops, previous_brief=', mode='read')]"
  500. assert analyzer.parse_brief(polluted) is None
  501. no_arg = "<|tool_call_start|>[update_brief(previous_brief=old, mode=read)]"
  502. assert analyzer.parse_brief(no_arg) is None
  503. def test_parse_brief_clean_mode_text_not_mangled(tmp_path):
  504. analyzer = _analyzer(tmp_path, _score_questions())
  505. clean = "Set mode=read, then read [the brief] - it's fine (really)."
  506. assert analyzer.parse_brief(clean) == clean
  507. def test_parse_brief_rejects_marker_with_artifacts_but_no_call(tmp_path):
  508. analyzer = _analyzer(tmp_path, _score_questions())
  509. assert analyzer.parse_brief("<|tool_call_start|>previous_brief='x', mode='read'") is None
  510. assert analyzer.parse_brief("<|tool_call_start|>mode='read'") is None
  511. def _brief_signals(p=None):
  512. item = {"kind": "noul"}
  513. if p is not None:
  514. item["p"] = p
  515. return {"order": ["substantive_update"], "items": {"substantive_update": item}}
  516. def test_should_update_brief_gate(tmp_path):
  517. analyzer = _analyzer(tmp_path, _score_questions())
  518. assert analyzer.should_update_brief(_brief_signals(0.9)) is True
  519. assert analyzer.should_update_brief(_brief_signals(0.5)) is True # boundary
  520. assert analyzer.should_update_brief(_brief_signals(0.2)) is False
  521. # absent update / no numeric p / not a dict -> fail open
  522. assert analyzer.should_update_brief({"order": [], "items": {}}) is True
  523. assert analyzer.should_update_brief(_brief_signals(None)) is True
  524. assert analyzer.should_update_brief(None) is True
  525. assert analyzer.should_update_brief({"items": {"substantive_update": "nope"}}) is True
  526. def test_should_update_brief_threshold_configured(tmp_path):
  527. analyzer = _analyzer(tmp_path, _score_questions(), brief_gate_threshold=0.8)
  528. assert analyzer.should_update_brief(_brief_signals(0.6)) is False
  529. assert analyzer.should_update_brief(_brief_signals(0.8)) is True
  530. def test_finalize_brief_length_truncates_not_rejects(tmp_path, caplog):
  531. analyzer = _analyzer(tmp_path, _score_questions())
  532. with caplog.at_level(logging.WARNING, logger="stt_daemon.assistant.triage"):
  533. result = analyzer.finalize_brief("y" * 5000, "length")
  534. assert result is not None
  535. assert result == "y" * 4000
  536. assert caplog.records # warned about the length cutoff but still accepted
  537. def test_finalize_brief_stop_and_empty(tmp_path):
  538. analyzer = _analyzer(tmp_path, _score_questions())
  539. assert analyzer.finalize_brief("a brief", "stop") == "a brief"
  540. assert analyzer.finalize_brief("", "stop") is None
  541. assert analyzer.finalize_brief("```\nfenced\n```", "stop") == "fenced"
  542. assert analyzer.finalize_brief(None, "length") is None
  543. # --- todo 11: cached state() ------------------------------------------------
  544. def test_state_initial_loading_with_good_questions(tmp_path):
  545. analyzer = _analyzer(tmp_path, _score_questions())
  546. assert analyzer.state() == ("loading", "")
  547. def test_state_initial_no_questions_with_bad_path(tmp_path):
  548. missing = str(tmp_path / "missing.json")
  549. analyzer = TriageAnalyzer(_cfg(missing))
  550. state, reason = analyzer.state()
  551. assert state == "no_questions"
  552. assert missing in reason
  553. def test_state_vocabulary_has_no_extra_member():
  554. assert set(triage.TRIAGE_REASONS) == {
  555. "ok",
  556. "no_questions",
  557. "loading",
  558. "laya_missing",
  559. "disabled",
  560. "error",
  561. }
  562. assert "service" + "_down" not in triage.TRIAGE_REASONS
  563. def test_refresh_transitions(tmp_path):
  564. analyzer = _analyzer(tmp_path, _score_questions())
  565. analyzer.refresh("ok")
  566. assert analyzer.state() == ("ok", "")
  567. analyzer.refresh("laya_missing", "run: uv sync --extra triage")
  568. assert analyzer.state() == ("laya_missing", "run: uv sync --extra triage")
  569. analyzer.refresh("error", "boom")
  570. assert analyzer.state() == ("error", "boom")
  571. def test_refresh_unknown_state_ignored(tmp_path, caplog):
  572. analyzer = _analyzer(tmp_path, _score_questions())
  573. analyzer.refresh("ok")
  574. with caplog.at_level(logging.WARNING, logger="stt_daemon.assistant.triage"):
  575. analyzer.refresh("bogus_state")
  576. assert analyzer.state() == ("ok", "")
  577. assert caplog.records
  578. def test_no_questions_state_cannot_be_overridden(tmp_path):
  579. analyzer = TriageAnalyzer(_cfg(str(tmp_path / "missing.json")))
  580. assert analyzer.state()[0] == "no_questions"
  581. analyzer.refresh("ok")
  582. analyzer.refresh("error", "boom")
  583. assert analyzer.state()[0] == "no_questions"
  584. def test_state_does_not_touch_lane(tmp_path):
  585. analyzer = _analyzer(tmp_path, _score_questions())
  586. class StrictLane:
  587. def __init__(self):
  588. self.calls = []
  589. def __getattr__(self, name):
  590. def record(*args, **kwargs):
  591. self.calls.append(name)
  592. return record
  593. lane = StrictLane()
  594. for _ in range(3):
  595. analyzer.state()
  596. analyzer.refresh("ok")
  597. analyzer.state()
  598. assert lane.calls == []
  599. # --- F3: end-to-end triage flow on fakes ------------------------------------
  600. EXTRACT_JSON = (
  601. '{"summary":"s","opportunities":[],"doubts":[],"action_items":[],"risks":[]}'
  602. )
  603. class _Clock:
  604. """Injected clock: no sleeps, fully deterministic."""
  605. def __init__(self, start=1000.0):
  606. self.t = float(start)
  607. def __call__(self):
  608. return self.t
  609. def advance(self, dt):
  610. self.t += float(dt)
  611. class _FakeSubscriber:
  612. def events(self):
  613. return iter(())
  614. def close(self):
  615. pass
  616. class _FakeLane:
  617. """Duck-typed LayaLane: canned-ready, records submits, counts drops."""
  618. def __init__(self):
  619. self.submits: list[tuple] = []
  620. self.drops = 0
  621. def ready(self) -> bool:
  622. return True
  623. def state(self):
  624. return ("ok", "")
  625. def submit(self, state, questions, lang, generation):
  626. self.submits.append((state, questions, lang, generation))
  627. return True
  628. def drop_pending(self):
  629. self.drops += 1
  630. return 1
  631. def _answers(noul_p=0.1):
  632. """A full laya-shaped answers payload for the example questions.
  633. `noul_p` drives every noul arm, including the `substantive_update` gate:
  634. below `brief_gate_threshold` (0.5) the turn is gated OUT (no brief job),
  635. at or above it the turn gates IN.
  636. """
  637. answers = {
  638. "decision_status": {
  639. "type": "choice",
  640. "choice": "made",
  641. "confidence": 0.9,
  642. "answer_confidence": 0.9,
  643. },
  644. "responsibility_clarity": {
  645. "type": "choice",
  646. "choice": "explicit",
  647. "confidence": 0.9,
  648. "answer_confidence": 0.9,
  649. },
  650. }
  651. for qid in (
  652. "substantive_update",
  653. "opportunity",
  654. "blocker",
  655. "action_item",
  656. "deadline",
  657. "open_question",
  658. ):
  659. answers[qid] = {
  660. "type": "noul",
  661. "noul": noul_p,
  662. "confidence": 0.9,
  663. "answer_confidence": 0.9,
  664. }
  665. return {"model": "multilingual", "answers": answers, "usage": {"input_tokens": 42}}
  666. def test_end_to_end_triage_flow():
  667. """signals -> content -> cleared across service, analyzer, lane, and ViewState.
  668. The lane result is wired by calling `service._on_triage_result(gen, payload)`
  669. directly (the runner does exactly this in production); the brief job is
  670. drained with `job.run()` so no worker thread is started. The
  671. single-outstanding guard plus the shared `_pending_brief` collapse settled
  672. turns into ONE job consuming every buffered delta, and the
  673. `substantive_update` gate can reduce that to ZERO jobs.
  674. """
  675. cfg = Config()
  676. cfg.assistant.turn_gap_s = 0.5
  677. cfg.triage.enabled = True
  678. cfg.triage.questions_file = EXAMPLE_PATH
  679. clock = _Clock(1000.0)
  680. lane = _FakeLane()
  681. emitted: list[dict] = []
  682. prompts: list[str] = []
  683. def emit(type_, **fields):
  684. emitted.append({"type": type_, **fields})
  685. analyzer = TriageAnalyzer(cfg.triage)
  686. service = AssistantService(
  687. cfg,
  688. _FakeSubscriber(),
  689. emit=emit,
  690. time_fn=clock,
  691. chat_fn=lambda text: "reply",
  692. triage=analyzer,
  693. laya_lane=lane,
  694. brief_fn=lambda prompt: prompts.append(prompt) or "BRIEF-TEXT",
  695. )
  696. def triage_events():
  697. return [e for e in emitted if e["type"] == "triage"]
  698. def brief_jobs():
  699. return [j for j in service._inference._queue if j.name == "brief-update"]
  700. # 1. settled turn -> exactly one lane submit, lang + generation recorded
  701. assert service.toggle_chat() is True
  702. service.handle_event(
  703. {"type": events.TRANSCRIPT, "text": "hola, necesito ayuda", "lang": "es"}
  704. )
  705. clock.advance(1.0)
  706. service.tick()
  707. assert len(lane.submits) == 1
  708. state_text, questions, lang, gen1 = lane.submits[0]
  709. assert "hola, necesito ayuda" in state_text
  710. assert questions is analyzer.questions
  711. assert lang == "es"
  712. assert gen1 == 0
  713. # 2. low-p turn -> signals only: the gate (p=0.1 < 0.5) dispatches ZERO jobs
  714. service._on_triage_result(gen1, _answers(noul_p=0.1))
  715. assert [e["stage"] for e in triage_events()] == ["signals"]
  716. assert triage_events()[0]["triage"]["escalated"] is False
  717. assert service.last_triage is not None
  718. assert service.last_triage["escalated"] is False
  719. assert brief_jobs() == []
  720. assert prompts == []
  721. # 3. high-p turn -> second submit, then ONE brief job (the only one outstanding)
  722. service.handle_event(
  723. {"type": events.TRANSCRIPT, "text": "quiero cancelar mi suscripcion", "lang": "es"}
  724. )
  725. clock.advance(1.0)
  726. service.tick()
  727. assert len(lane.submits) == 2
  728. gen2 = lane.submits[1][3]
  729. service._on_triage_result(gen2, _answers(noul_p=0.9))
  730. assert [e["stage"] for e in triage_events()] == ["signals", "signals"]
  731. assert triage_events()[1]["triage"]["escalated"] is True
  732. jobs = brief_jobs()
  733. assert len(jobs) == 1
  734. # 4. run the brief job directly (deterministic, no worker thread): it consumes
  735. # both buffered deltas and emits ONE content with the brief merged over signals
  736. jobs[0].run()
  737. assert len(prompts) == 1
  738. assert "hola, necesito ayuda" in prompts[0]
  739. assert "quiero cancelar mi suscripcion" in prompts[0]
  740. content_events = [e for e in triage_events() if e["stage"] == "content"]
  741. assert len(content_events) == 1
  742. merged = content_events[0]["triage"]
  743. assert merged["brief"] == "BRIEF-TEXT"
  744. assert merged["escalated"] is True # signals preserved under the brief
  745. assert merged["order"] == triage_events()[1]["triage"]["order"]
  746. # 5. drive the emitted stream through a fresh ViewState
  747. view = ViewState()
  748. for event in emitted:
  749. view.ingest(event)
  750. analyzer.refresh("ok") # the lane callback does this in production
  751. view.ingest({"type": "status", **service.status()})
  752. assert view.triage is not None
  753. assert view.triage["brief"] == "BRIEF-TEXT"
  754. lines = view.triage_lines()
  755. assert lines
  756. # 6. clear -> cleared event empties the panel and bumps the epoch
  757. baseline_epoch = view.triage_epoch
  758. already = len(emitted)
  759. service.clear()
  760. for event in emitted[already:]:
  761. view.ingest(event)
  762. assert view.triage is None
  763. assert view.triage_epoch > baseline_epoch
  764. # a stale result after clear must not re-populate the panel
  765. service._on_triage_result(gen2, _answers(noul_p=0.9))
  766. assert [e["stage"] for e in triage_events()] == [
  767. "signals",
  768. "signals",
  769. "content",
  770. "cleared",
  771. ]
  772. assert view.triage is None
  773. assert lane.drops == 1
  774. # 7. disabled control: no triage/lane -> status disabled, clear emits nothing
  775. cfg2 = Config()
  776. clock2 = _Clock(2000.0)
  777. emitted2: list[dict] = []
  778. def emit2(type_, **fields):
  779. emitted2.append({"type": type_, **fields})
  780. plain = AssistantService(cfg2, _FakeSubscriber(), emit=emit2, time_fn=clock2)
  781. assert plain.status()["triage_state"] == "disabled"
  782. assert plain.clear() is True
  783. assert [e for e in emitted2 if e["type"] == "triage"] == []