|
@@ -1,6 +1,8 @@
|
|
|
"""Tests for the assistant service (Phase 2) using synthetic event streams."""
|
|
"""Tests for the assistant service (Phase 2) using synthetic event streams."""
|
|
|
|
|
|
|
|
import itertools
|
|
import itertools
|
|
|
|
|
+import json
|
|
|
|
|
+import re
|
|
|
import threading
|
|
import threading
|
|
|
import time
|
|
import time
|
|
|
from typing import Any
|
|
from typing import Any
|
|
@@ -8,6 +10,7 @@ from typing import Any
|
|
|
from stt_daemon import events
|
|
from stt_daemon import events
|
|
|
from stt_daemon.assistant.inference import InferenceQueue
|
|
from stt_daemon.assistant.inference import InferenceQueue
|
|
|
from stt_daemon.assistant.service import EXECUTE_BUSY, AssistantService
|
|
from stt_daemon.assistant.service import EXECUTE_BUSY, AssistantService
|
|
|
|
|
+from stt_daemon.assistant.triage_dump import TriageDumper
|
|
|
from stt_daemon.config import Config
|
|
from stt_daemon.config import Config
|
|
|
|
|
|
|
|
_sequence = itertools.count(1)
|
|
_sequence = itertools.count(1)
|
|
@@ -1343,3 +1346,179 @@ def test_tick_heals_triage_panel_after_lane_ready():
|
|
|
assert not any("loading" in line for line in lines)
|
|
assert not any("loading" in line for line in lines)
|
|
|
finally:
|
|
finally:
|
|
|
lane.stop()
|
|
lane.stop()
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def _dump_extract(prompt, on_io=None):
|
|
|
|
|
+ """Extract fake that mimics LlmClient.chat's on_io phases."""
|
|
|
|
|
+ if on_io is not None:
|
|
|
|
|
+ on_io({"phase": "request", "url": "http://llm/v1", "body": {"prompt": prompt}})
|
|
|
|
|
+ on_io({"phase": "response", "payload": {"content": "parsed"}, "error": None})
|
|
|
|
|
+ return "RAW"
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def _run_triage_flow(service, emitted, clock, lane, payload):
|
|
|
|
|
+ service.toggle_chat()
|
|
|
|
|
+ service.handle_event(_transcript(clock, "hello"))
|
|
|
|
|
+ clock.advance(service.cfg.assistant.turn_gap_s + 0.1)
|
|
|
|
|
+ service.tick()
|
|
|
|
|
+ _, _, _, gen = lane.submits[0]
|
|
|
|
|
+ service._on_triage_result(gen, payload)
|
|
|
|
|
+ return gen
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def _dump_files(tmp_path):
|
|
|
|
|
+ return list(tmp_path.rglob("*.json"))
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def test_dump_writes_full_laya_and_llm_io(tmp_path):
|
|
|
|
|
+ service, emitted, clock, triage, lane = _triage_service(extract_fn=_dump_extract)
|
|
|
|
|
+ service._triage_dump = TriageDumper(True, directory=str(tmp_path))
|
|
|
|
|
+ service.cfg.triage.dump_io = True
|
|
|
|
|
+
|
|
|
|
|
+ _run_triage_flow(service, emitted, clock, lane, _escalated_payload())
|
|
|
|
|
+ jobs = [j for j in service._inference._queue if j.name == "triage-extract"]
|
|
|
|
|
+ assert len(jobs) == 1
|
|
|
|
|
+ jobs[0].run()
|
|
|
|
|
+
|
|
|
|
|
+ files = _dump_files(tmp_path)
|
|
|
|
|
+ assert len(files) == 1
|
|
|
|
|
+ assert re.fullmatch(r"\d{6}-\d{3}\.json", files[0].name)
|
|
|
|
|
+ data = json.loads(files[0].read_text(encoding="utf-8"))
|
|
|
|
|
+ assert data["ts"] == clock()
|
|
|
|
|
+ assert isinstance(data["iso"], str) and data["iso"]
|
|
|
|
|
+ assert data["laya"]["request"]["state"] == "- hello"
|
|
|
|
|
+ assert data["laya"]["request"]["questions"] == triage.questions
|
|
|
|
|
+ assert data["laya"]["request"]["model"] == "multilingual"
|
|
|
|
|
+ assert data["laya"]["request"]["max_len"] == 1024
|
|
|
|
|
+ assert data["laya"]["response"] == _escalated_payload()
|
|
|
|
|
+ assert data["laya"]["error"] is None
|
|
|
|
|
+ assert data["llm"]["request"]["url"] == "http://llm/v1"
|
|
|
|
|
+ assert data["llm"]["request"]["body"] == {"prompt": "EXTRACT"}
|
|
|
|
|
+ assert data["llm"]["response"] == {"content": "parsed"}
|
|
|
|
|
+ assert data["llm"]["error"] is None
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def test_dump_laya_error_keeps_state_only_llm_half(tmp_path):
|
|
|
|
|
+ service, emitted, clock, _, lane = _triage_service(extract_fn=_dump_extract)
|
|
|
|
|
+ service._triage_dump = TriageDumper(True, directory=str(tmp_path))
|
|
|
|
|
+ service.cfg.triage.dump_io = True
|
|
|
|
|
+
|
|
|
|
|
+ _run_triage_flow(service, emitted, clock, lane, None)
|
|
|
|
|
+ jobs = [j for j in service._inference._queue if j.name == "triage-extract"]
|
|
|
|
|
+ assert len(jobs) == 1
|
|
|
|
|
+ jobs[0].run()
|
|
|
|
|
+
|
|
|
|
|
+ files = _dump_files(tmp_path)
|
|
|
|
|
+ assert len(files) == 1
|
|
|
|
|
+ data = json.loads(files[0].read_text(encoding="utf-8"))
|
|
|
|
|
+ assert data["laya"]["response"] is None
|
|
|
|
|
+ assert data["laya"]["error"] == "predict failed"
|
|
|
|
|
+ assert data["llm"]["request"]["url"] == "http://llm/v1"
|
|
|
|
|
+ assert data["llm"]["response"] == {"content": "parsed"}
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def test_dump_without_extract_has_laya_only(tmp_path):
|
|
|
|
|
+ service, emitted, clock, _, lane = _triage_service(extract_fn=_dump_extract)
|
|
|
|
|
+ service._triage_dump = TriageDumper(True, directory=str(tmp_path))
|
|
|
|
|
+ service.cfg.triage.dump_io = True
|
|
|
|
|
+ service.cfg.triage.extract_enabled = False
|
|
|
|
|
+
|
|
|
|
|
+ _run_triage_flow(service, emitted, clock, lane, _escalated_payload())
|
|
|
|
|
+
|
|
|
|
|
+ files = _dump_files(tmp_path)
|
|
|
|
|
+ assert len(files) == 1
|
|
|
|
|
+ data = json.loads(files[0].read_text(encoding="utf-8"))
|
|
|
|
|
+ assert data["laya"]["response"] == _escalated_payload()
|
|
|
|
|
+ assert data["llm"] is None
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def test_dump_queue_full_has_laya_only(tmp_path):
|
|
|
|
|
+ service, emitted, clock, _, lane = _triage_service(extract_fn=_dump_extract)
|
|
|
|
|
+ service._triage_dump = TriageDumper(True, directory=str(tmp_path))
|
|
|
|
|
+ service.cfg.triage.dump_io = True
|
|
|
|
|
+ service._inference = InferenceQueue(size=2)
|
|
|
|
|
+
|
|
|
|
|
+ _run_triage_flow(service, emitted, clock, lane, _escalated_payload())
|
|
|
|
|
+
|
|
|
|
|
+ assert service._inference.depth() == 1
|
|
|
|
|
+ files = _dump_files(tmp_path)
|
|
|
|
|
+ assert len(files) == 1
|
|
|
|
|
+ data = json.loads(files[0].read_text(encoding="utf-8"))
|
|
|
|
|
+ assert data["laya"]["response"] == _escalated_payload()
|
|
|
|
|
+ assert data["llm"] is None
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def test_dump_disabled_writes_nothing(tmp_path):
|
|
|
|
|
+ service, emitted, clock, _, lane = _triage_service(
|
|
|
|
|
+ extract_fn=lambda prompt: "RAW"
|
|
|
|
|
+ )
|
|
|
|
|
+ assert service._triage_dump.enabled is False
|
|
|
|
|
+ service._triage_dump = TriageDumper(False, directory=str(tmp_path))
|
|
|
|
|
+
|
|
|
|
|
+ _run_triage_flow(service, emitted, clock, lane, _escalated_payload())
|
|
|
|
|
+ jobs = [j for j in service._inference._queue if j.name == "triage-extract"]
|
|
|
|
|
+ assert len(jobs) == 1
|
|
|
|
|
+ jobs[0].run()
|
|
|
|
|
+
|
|
|
|
|
+ assert _dump_files(tmp_path) == []
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def test_dump_stale_result_discards_without_file(tmp_path):
|
|
|
|
|
+ service, emitted, clock, triage, lane = _triage_service(extract_fn=_dump_extract)
|
|
|
|
|
+ service._triage_dump = TriageDumper(True, directory=str(tmp_path))
|
|
|
|
|
+ service.cfg.triage.dump_io = True
|
|
|
|
|
+ service.toggle_chat()
|
|
|
|
|
+ service.handle_event(_transcript(clock, "hello"))
|
|
|
|
|
+ clock.advance(service.cfg.assistant.turn_gap_s + 0.1)
|
|
|
|
|
+ service.tick()
|
|
|
|
|
+ _, _, _, gen = lane.submits[0]
|
|
|
|
|
+
|
|
|
|
|
+ real_normalize = triage.normalize
|
|
|
|
|
+
|
|
|
|
|
+ def stale_normalize(payload):
|
|
|
|
|
+ # Simulate a clear landing between the first check/begin and the re-check.
|
|
|
|
|
+ service._triage_generation += 1
|
|
|
|
|
+ return real_normalize(payload)
|
|
|
|
|
+
|
|
|
|
|
+ triage.normalize = stale_normalize
|
|
|
|
|
+ service._on_triage_result(gen, _escalated_payload())
|
|
|
|
|
+
|
|
|
|
|
+ assert _dump_files(tmp_path) == []
|
|
|
|
|
+ assert service._triage_dump._pending == {}
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def test_dump_stale_extract_discards_without_file(tmp_path):
|
|
|
|
|
+ service, _, clock, _, _ = _triage_service(extract_fn=_dump_extract)
|
|
|
|
|
+ service._triage_dump = TriageDumper(True, directory=str(tmp_path))
|
|
|
|
|
+ service.cfg.triage.dump_io = True
|
|
|
|
|
+ ts = clock()
|
|
|
|
|
+ service._triage_dump.begin(ts, laya={"request": {}, "response": {}, "error": None})
|
|
|
|
|
+ service._triage_generation += 1 # stale before the extract even starts
|
|
|
|
|
+
|
|
|
|
|
+ service._run_triage_extract(0, ("hello",), _escalated_payload(), ts)
|
|
|
|
|
+
|
|
|
|
|
+ assert _dump_files(tmp_path) == []
|
|
|
|
|
+ assert service._triage_dump._pending == {}
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def test_dump_extract_cleared_mid_flight_writes_nothing(tmp_path):
|
|
|
|
|
+ service, _, clock, _, _ = _triage_service(extract_fn=_dump_extract)
|
|
|
|
|
+ service._triage_dump = TriageDumper(True, directory=str(tmp_path))
|
|
|
|
|
+ service.cfg.triage.dump_io = True
|
|
|
|
|
+
|
|
|
|
|
+ def stale_extract(prompt, on_io=None):
|
|
|
|
|
+ # Simulate a clear landing mid-LLM-call (after the request phase fired).
|
|
|
|
|
+ service._triage_generation += 1
|
|
|
|
|
+ if on_io is not None:
|
|
|
|
|
+ on_io({"phase": "request", "url": "http://llm/v1", "body": {}})
|
|
|
|
|
+ on_io({"phase": "response", "payload": {"c": 1}, "error": None})
|
|
|
|
|
+ return "RAW"
|
|
|
|
|
+
|
|
|
|
|
+ service._triage_extract_fn = stale_extract
|
|
|
|
|
+ ts = clock()
|
|
|
|
|
+ service._triage_dump.begin(ts, laya={"request": {}, "response": {}, "error": None})
|
|
|
|
|
+
|
|
|
|
|
+ service._run_triage_extract(0, ("hello",), _escalated_payload(), ts)
|
|
|
|
|
+
|
|
|
|
|
+ assert _dump_files(tmp_path) == []
|
|
|
|
|
+ assert service._triage_dump._pending == {}
|