test_daemon_events.py 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899
  1. """Tests for the daemon's publish-only event stream (Phase 1)."""
  2. import concurrent.futures
  3. import logging
  4. import os
  5. import shutil
  6. import socket
  7. import tempfile
  8. import threading
  9. import time
  10. from types import SimpleNamespace
  11. import numpy as np
  12. import pytest
  13. from stt_daemon import events
  14. from stt_daemon.bus import EventSubscriber
  15. from stt_daemon.config import Config
  16. from stt_daemon.daemon import Daemon
  17. from stt_daemon.segmenter import Utterance
  18. from stt_daemon.supervisor import Supervisor
  19. from stt_daemon.transcriber import TranscriptResult
  20. FRAME_SAMPLES = 480
  21. FRAME_BYTES = FRAME_SAMPLES * 2
  22. @pytest.fixture
  23. def sock_path():
  24. directory = tempfile.mkdtemp(prefix="sttb-")
  25. try:
  26. yield os.path.join(directory, "s.sock")
  27. finally:
  28. shutil.rmtree(directory, ignore_errors=True)
  29. def _frame(value=0):
  30. return np.full(FRAME_SAMPLES, value, dtype=np.int16).tobytes()
  31. class FakeVad:
  32. def __init__(self, decisions):
  33. self._decisions = list(decisions)
  34. self._index = 0
  35. def is_speech(self, frame, sample_rate):
  36. if self._index < len(self._decisions):
  37. decision = self._decisions[self._index]
  38. self._index += 1
  39. return decision
  40. return False
  41. class FakeCapture:
  42. def __init__(self, frames, delay=0.0, stall_after=None, healthy=True):
  43. self._frames = list(frames)
  44. self._delay = delay
  45. self._stall_after = stall_after
  46. self._healthy = healthy
  47. self._closed = threading.Event()
  48. self.frame_bytes = FRAME_BYTES
  49. def frames(self):
  50. for index, frame in enumerate(self._frames):
  51. if self._closed.is_set():
  52. return
  53. if self._delay:
  54. time.sleep(self._delay)
  55. yield frame
  56. if self._stall_after is not None and index + 1 >= self._stall_after:
  57. self._closed.wait()
  58. return
  59. def healthy(self):
  60. return self._healthy
  61. def close(self):
  62. self._closed.set()
  63. class FakeTranscriber:
  64. def __init__(self, text="hello", language="en", probability=0.99):
  65. self.text = text
  66. self.language = language
  67. self.probability = probability
  68. def transcribe(self, audio):
  69. return TranscriptResult(
  70. language=self.language,
  71. language_probability=self.probability,
  72. text=self.text,
  73. )
  74. def close(self):
  75. pass
  76. class FakeSink:
  77. def __init__(self):
  78. self.emitted = []
  79. self.transcribe_times = []
  80. def emit(
  81. self,
  82. result,
  83. duration_s,
  84. rms=0.0,
  85. reason="unknown",
  86. audio=None,
  87. transcribe_s=None,
  88. **kwargs,
  89. ):
  90. if not result.text:
  91. return None
  92. self.emitted.append(result.text)
  93. self.transcribe_times.append(transcribe_s)
  94. self.last_kwargs = dict(kwargs)
  95. return None
  96. def close(self):
  97. pass
  98. class _Collector:
  99. def __init__(self, subscriber):
  100. self.events = []
  101. self._thread = threading.Thread(target=self._run, args=(subscriber,), daemon=True)
  102. self._thread.start()
  103. def _run(self, subscriber):
  104. try:
  105. for event in subscriber.events():
  106. self.events.append(event)
  107. except Exception: # noqa: BLE001 - collector must not crash the test thread
  108. pass
  109. def types(self):
  110. return [e["type"] for e in self.events]
  111. def of(self, type_):
  112. return [e for e in self.events if e["type"] == type_]
  113. def wait_for(self, predicate, timeout=5.0):
  114. deadline = time.time() + timeout
  115. while time.time() < deadline:
  116. if predicate(self.events):
  117. return True
  118. time.sleep(0.01)
  119. return predicate(self.events)
  120. def _base_config(sock_path):
  121. cfg = Config()
  122. cfg.events.socket = sock_path
  123. cfg.events.heartbeat_ms = 50
  124. cfg.audio.frame_ms = 30
  125. cfg.vad.pre_roll_ms = 60
  126. cfg.vad.speech_start_ms = 30
  127. cfg.vad.silence_ms = 60
  128. cfg.vad.min_utterance_ms = 30
  129. cfg.vad.chunk_pause_ms = 300
  130. cfg.vad.soft_max_s = 10.0
  131. cfg.vad.max_utterance_s = 30.0
  132. cfg.vad.rms_floor = 0.0
  133. cfg.daemon.queue_size = 50
  134. cfg.output.stdout = False
  135. cfg.output.jsonl = False
  136. cfg.output.save_audio = False
  137. return cfg
  138. def _start_daemon(cfg, capture, vad, transcriber, debug_sink_factory=None):
  139. sink = FakeSink()
  140. daemon = Daemon(
  141. cfg,
  142. log_level=logging.WARNING,
  143. capture_factory=lambda: capture,
  144. vad_factory=lambda _aggr: vad,
  145. transcriber_factory=lambda: transcriber,
  146. sink_factory=lambda: sink,
  147. debug_sink_factory=debug_sink_factory,
  148. )
  149. daemon._initialize()
  150. subscriber = EventSubscriber(cfg.events.socket, reconnect_delay=0.05)
  151. collector = _Collector(subscriber)
  152. daemon._start_workers()
  153. return daemon, collector, subscriber, sink
  154. def test_transcript_and_segment_done_share_seg_id(sock_path):
  155. cfg = _base_config(sock_path)
  156. frames = [_frame() for _ in range(5)]
  157. vad = FakeVad([False, True, True, False, False])
  158. transcriber = FakeTranscriber(text="hello")
  159. daemon, collector, subscriber, _ = _start_daemon(cfg, FakeCapture(frames), vad, transcriber)
  160. collector.wait_for(lambda e: any(x["type"] == events.SEGMENT_DONE for x in e))
  161. starts = collector.of(events.SPEECH_START)
  162. stops = collector.of(events.SPEECH_STOP)
  163. transcripts = collector.of(events.TRANSCRIPT)
  164. dones = collector.of(events.SEGMENT_DONE)
  165. assert len(starts) == 1
  166. assert "seg_id" not in starts[0] # speech_start carries no seg_id
  167. assert len(stops) == 1
  168. assert len(transcripts) == 1
  169. assert len(dones) == 1
  170. assert stops[0]["seg_id"] == transcripts[0]["seg_id"] == dones[0]["seg_id"] == 1
  171. assert transcripts[0]["text"] == "hello"
  172. assert dones[0]["emitted"] is True
  173. assert dones[0]["end_reason"] == "end_of_turn"
  174. subscriber.close()
  175. daemon.stop()
  176. def test_transcribe_seconds_is_forwarded_to_sink(sock_path):
  177. """The worker measures wall time around transcribe() and passes it to emit()."""
  178. cfg = _base_config(sock_path)
  179. frames = [_frame() for _ in range(5)]
  180. vad = FakeVad([False, True, True, False, False])
  181. transcriber = FakeTranscriber(text="hello")
  182. daemon, collector, subscriber, sink = _start_daemon(
  183. cfg, FakeCapture(frames), vad, transcriber
  184. )
  185. collector.wait_for(lambda e: any(x["type"] == events.SEGMENT_DONE for x in e))
  186. assert len(sink.transcribe_times) == 1
  187. assert sink.transcribe_times[0] is not None
  188. assert sink.transcribe_times[0] >= 0.0
  189. subscriber.close()
  190. daemon.stop()
  191. def test_gated_segment_reports_emitted_false(sock_path):
  192. cfg = _base_config(sock_path)
  193. frames = [_frame() for _ in range(5)]
  194. vad = FakeVad([False, True, True, False, False])
  195. transcriber = FakeTranscriber(text="") # gated / no text
  196. daemon, collector, subscriber, _ = _start_daemon(cfg, FakeCapture(frames), vad, transcriber)
  197. collector.wait_for(lambda e: any(x["type"] == events.SEGMENT_DONE for x in e))
  198. assert collector.of(events.TRANSCRIPT) == []
  199. dones = collector.of(events.SEGMENT_DONE)
  200. assert len(dones) == 1
  201. assert dones[0]["emitted"] is False
  202. assert len(collector.of(events.SPEECH_STOP)) == 1
  203. subscriber.close()
  204. daemon.stop()
  205. def test_chunked_utterance_yields_one_pair_per_chunk(sock_path):
  206. cfg = _base_config(sock_path)
  207. cfg.vad.soft_max_s = 0.3 # 10 frames
  208. cfg.vad.chunk_pause_ms = 60 # 2 frames
  209. cfg.vad.silence_ms = 600 # 20 frames (never reached here)
  210. frames = [_frame() for _ in range(47)]
  211. decisions = [False] + [True] * 12 + [False] * 2 + [True] * 12 + [False] * 20
  212. transient = FakeTranscriber(text="chunk")
  213. daemon, collector, subscriber, _ = _start_daemon(
  214. cfg, FakeCapture(frames), FakeVad(decisions), transient
  215. )
  216. collector.wait_for(
  217. lambda e: len([x for x in e if x["type"] == events.SEGMENT_DONE]) >= 2
  218. )
  219. stops = collector.of(events.SPEECH_STOP)
  220. dones = collector.of(events.SEGMENT_DONE)
  221. assert len(stops) == len(dones) >= 2
  222. stop_ids = [s["seg_id"] for s in stops]
  223. done_ids = [d["seg_id"] for d in dones]
  224. assert stop_ids == sorted(stop_ids)
  225. assert len(set(stop_ids)) == len(stop_ids) # unique per chunk
  226. assert set(stop_ids) == set(done_ids)
  227. assert all(d["end_reason"] == "pause" for d in dones)
  228. subscriber.close()
  229. daemon.stop()
  230. def test_heartbeats_advance_and_carry_health(sock_path):
  231. cfg = _base_config(sock_path)
  232. cfg.events.heartbeat_ms = 40
  233. frames = [_frame() for _ in range(40)]
  234. daemon, collector, subscriber, _ = _start_daemon(
  235. cfg, FakeCapture(frames, delay=0.02), FakeVad([]), FakeTranscriber()
  236. )
  237. collector.wait_for(lambda e: len([x for x in e if x["type"] == events.HEARTBEAT]) >= 3)
  238. heartbeats = collector.of(events.HEARTBEAT)
  239. assert len(heartbeats) >= 3
  240. seqs = [h["seq"] for h in heartbeats]
  241. assert seqs == sorted(seqs)
  242. assert len(set(seqs)) == len(seqs)
  243. last = heartbeats[-1]
  244. assert last["model_loaded"] is True
  245. assert last["device_ok"] is True
  246. assert last["capture"] == cfg.audio.device
  247. for field in ("in_flight", "segments_done", "worker_stall_s", "queue_depth",
  248. "frames_since_last", "rms", "uptime_s", "last_transcript_age_s"):
  249. assert field in last
  250. subscriber.close()
  251. daemon.stop()
  252. def test_stalled_capture_stops_heartbeats(sock_path):
  253. cfg = _base_config(sock_path)
  254. cfg.events.heartbeat_ms = 60
  255. frames = [_frame() for _ in range(30)]
  256. capture = FakeCapture(frames, delay=0.01, stall_after=10)
  257. daemon, collector, subscriber, _ = _start_daemon(
  258. cfg, capture, FakeVad([]), FakeTranscriber()
  259. )
  260. assert collector.wait_for(
  261. lambda e: any(x["type"] == events.HEARTBEAT for x in e), timeout=3.0
  262. )
  263. time.sleep(0.2) # let it stall
  264. before = len(collector.of(events.HEARTBEAT))
  265. time.sleep(0.45) # ~7 heartbeat intervals at 60ms
  266. after = len(collector.of(events.HEARTBEAT))
  267. assert after == before, f"heartbeats continued during a stall ({before} -> {after})"
  268. subscriber.close()
  269. daemon.stop()
  270. def test_discarded_segment_still_emits_speech_stop(sock_path):
  271. """A turn that is discarded (blip / too quiet) must still emit speech_stop.
  272. Without it, consumers that track speech_open from speech_start/speech_stop
  273. latch true forever, which blocks readiness and chat dispatch.
  274. """
  275. cfg = _base_config(sock_path)
  276. cfg.vad.min_utterance_ms = 300 # a 1-frame blip is discarded
  277. frames = [_frame() for _ in range(4)]
  278. vad = FakeVad([True, False, False, False]) # confirm, then silence -> discard
  279. daemon, collector, subscriber, _ = _start_daemon(
  280. cfg, FakeCapture(frames), vad, FakeTranscriber()
  281. )
  282. collector.wait_for(lambda e: any(x["type"] == events.SPEECH_STOP for x in e))
  283. time.sleep(0.2)
  284. starts = collector.of(events.SPEECH_START)
  285. stops = collector.of(events.SPEECH_STOP)
  286. assert len(starts) == 1
  287. assert len(stops) == 1
  288. assert stops[0]["seg_id"] is None # discarded: no segment assigned
  289. assert stops[0]["end_reason"] == "discarded"
  290. assert collector.of(events.SEGMENT_DONE) == []
  291. assert collector.of(events.TRANSCRIPT) == []
  292. subscriber.close()
  293. daemon.stop()
  294. def test_inbound_data_on_subscriber_socket_is_ignored(sock_path):
  295. cfg = _base_config(sock_path)
  296. cfg.events.heartbeat_ms = 40
  297. frames = [_frame() for _ in range(60)]
  298. daemon, collector, subscriber, _ = _start_daemon(
  299. cfg, FakeCapture(frames, delay=0.02), FakeVad([]), FakeTranscriber()
  300. )
  301. # The bus is publish-only: a client writing data must have no effect.
  302. raw = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
  303. raw.connect(sock_path)
  304. raw.sendall(b'{"v":1,"type":"evil","seq":1,"ts":0}\n')
  305. time.sleep(0.1)
  306. assert collector.wait_for(
  307. lambda e: len([x for x in e if x["type"] == events.HEARTBEAT]) >= 2
  308. )
  309. assert all(x["type"] != "evil" for x in collector.events)
  310. raw.close()
  311. subscriber.close()
  312. daemon.stop()
  313. class FakeDebugSink:
  314. """In-memory DebugSink stand-in (duck-typed emit/close, thread-safe)."""
  315. def __init__(self, dir="debug", proc="daemon"):
  316. self.dir = dir
  317. self.proc = proc
  318. self.emitted = []
  319. self.closed = False
  320. self._lock = threading.Lock()
  321. def emit(self, type_, **fields):
  322. with self._lock:
  323. self.emitted.append({"type": type_, **fields})
  324. def types(self):
  325. with self._lock:
  326. return [e["type"] for e in self.emitted]
  327. def of(self, type_):
  328. with self._lock:
  329. return [e for e in self.emitted if e["type"] == type_]
  330. def close(self):
  331. self.closed = True
  332. class BoomDebugSink(FakeDebugSink):
  333. """Debug sink whose emit always throws (total-swallow check)."""
  334. def emit(self, type_, **fields):
  335. raise RuntimeError("disk gone")
  336. def _enable_debug(cfg, enabled=True, heartbeat=True):
  337. cfg.debug = SimpleNamespace(enabled=enabled, heartbeat=heartbeat, dir="debug")
  338. def _wait_until(predicate, timeout=5.0):
  339. deadline = time.time() + timeout
  340. while time.time() < deadline:
  341. if predicate():
  342. return True
  343. time.sleep(0.01)
  344. return predicate()
  345. def test_debug_sink_none_when_disabled(sock_path):
  346. cfg = _base_config(sock_path)
  347. daemon = Daemon(
  348. cfg,
  349. log_level=logging.WARNING,
  350. capture_factory=lambda: FakeCapture([_frame()]),
  351. vad_factory=lambda _aggr: FakeVad([]),
  352. transcriber_factory=lambda: FakeTranscriber(),
  353. sink_factory=lambda: FakeSink(),
  354. )
  355. daemon._initialize()
  356. assert daemon.debug_sink is None
  357. daemon.stop()
  358. def test_debug_heartbeat_lines_carry_health_payload(sock_path):
  359. cfg = _base_config(sock_path)
  360. cfg.events.heartbeat_ms = 40
  361. _enable_debug(cfg)
  362. debug = FakeDebugSink()
  363. frames = [_frame() for _ in range(40)]
  364. daemon, collector, subscriber, _ = _start_daemon(
  365. cfg,
  366. FakeCapture(frames, delay=0.02),
  367. FakeVad([]),
  368. FakeTranscriber(),
  369. debug_sink_factory=lambda: debug,
  370. )
  371. assert daemon.debug_sink is debug
  372. assert _wait_until(lambda: bool(debug.of("heartbeat")), timeout=5.0)
  373. beat = debug.of("heartbeat")[0]
  374. assert beat["model_loaded"] is True
  375. for field in ("queue_depth", "device_ok", "worker_stall_s", "rms", "uptime_s"):
  376. assert field in beat
  377. subscriber.close()
  378. daemon.stop()
  379. assert debug.closed is True
  380. def test_debug_heartbeat_gate_off_leaves_only_anomalies(sock_path):
  381. cfg = _base_config(sock_path)
  382. cfg.events.heartbeat_ms = 40
  383. _enable_debug(cfg, heartbeat=False)
  384. debug = FakeDebugSink()
  385. daemon, collector, subscriber, _ = _start_daemon(
  386. cfg,
  387. FakeCapture([_frame() for _ in range(20)], delay=0.02),
  388. FakeVad([]),
  389. FakeTranscriber(),
  390. debug_sink_factory=lambda: debug,
  391. )
  392. assert _wait_until(
  393. lambda: any(e["type"] == events.HEARTBEAT for e in collector.events),
  394. timeout=5.0,
  395. )
  396. assert daemon.debug_sink is debug
  397. assert debug.of("heartbeat") == []
  398. subscriber.close()
  399. daemon.stop()
  400. def test_debug_reopen_attempt_success_and_fail_lines(sock_path):
  401. cfg = _base_config(sock_path)
  402. _enable_debug(cfg)
  403. debug = FakeDebugSink()
  404. daemon = Daemon(
  405. cfg,
  406. log_level=logging.WARNING,
  407. capture_factory=lambda: FakeCapture([_frame()]),
  408. vad_factory=lambda _aggr: FakeVad([]),
  409. transcriber_factory=lambda: FakeTranscriber(),
  410. sink_factory=lambda: FakeSink(),
  411. debug_sink_factory=lambda: debug,
  412. )
  413. daemon._initialize()
  414. calls = []
  415. def flaky_factory():
  416. calls.append(1)
  417. if len(calls) == 1:
  418. raise RuntimeError("no device")
  419. return FakeCapture([_frame()])
  420. daemon._capture_factory = flaky_factory
  421. assert daemon._reopen_capture(attempts=2, delay_s=0.0) is True
  422. assert debug.types().count("capture_reopen_attempt") == 2
  423. fails = debug.of("capture_reopen_fail")
  424. assert len(fails) == 1
  425. assert "no device" in fails[0]["error"]
  426. assert len(debug.of("capture_reopen_ok")) == 1
  427. def dead_factory():
  428. raise RuntimeError("still gone")
  429. daemon._capture_factory = dead_factory
  430. assert daemon._reopen_capture(attempts=2, delay_s=0.0) is False
  431. assert debug.types().count("capture_reopen_attempt") == 4
  432. assert len(debug.of("capture_reopen_fail")) == 3
  433. daemon.stop()
  434. def test_debug_capture_failure_emits_fatal_and_stops(sock_path):
  435. cfg = _base_config(sock_path)
  436. _enable_debug(cfg)
  437. debug = FakeDebugSink()
  438. class BoomCapture(FakeCapture):
  439. def frames(self):
  440. raise RuntimeError("device lost")
  441. yield
  442. daemon = Daemon(
  443. cfg,
  444. log_level=logging.WARNING,
  445. capture_factory=lambda: BoomCapture([]),
  446. vad_factory=lambda _aggr: FakeVad([]),
  447. transcriber_factory=lambda: FakeTranscriber(),
  448. sink_factory=lambda: FakeSink(),
  449. debug_sink_factory=lambda: debug,
  450. )
  451. daemon._initialize()
  452. def dead_factory():
  453. raise RuntimeError("device lost")
  454. daemon._capture_factory = dead_factory
  455. daemon._start_workers()
  456. assert daemon._stop_event.wait(timeout=10.0)
  457. daemon.stop()
  458. assert "capture_reopen_attempt" in debug.types()
  459. assert "reader_fatal" in debug.types()
  460. def test_debug_clean_exhaustion_emits_no_fatal(sock_path):
  461. cfg = _base_config(sock_path)
  462. _enable_debug(cfg)
  463. debug = FakeDebugSink()
  464. daemon, collector, subscriber, _ = _start_daemon(
  465. cfg,
  466. FakeCapture([_frame()]),
  467. FakeVad([False]),
  468. FakeTranscriber(),
  469. debug_sink_factory=lambda: debug,
  470. )
  471. reader = daemon._reader_thread
  472. assert reader is not None
  473. assert _wait_until(lambda: not reader.is_alive(), timeout=5.0)
  474. assert not daemon._stop_event.is_set()
  475. assert "reader_fatal" not in debug.types()
  476. assert "worker_fatal" not in debug.types()
  477. subscriber.close()
  478. daemon.stop()
  479. def test_debug_queue_drop_and_backlog_warn(sock_path):
  480. cfg = _base_config(sock_path)
  481. cfg.daemon.queue_size = 2
  482. cfg.daemon.backlog_warn_at = 1
  483. _enable_debug(cfg)
  484. debug = FakeDebugSink()
  485. daemon = Daemon(
  486. cfg,
  487. log_level=logging.WARNING,
  488. capture_factory=lambda: FakeCapture([_frame()]),
  489. vad_factory=lambda _aggr: FakeVad([]),
  490. transcriber_factory=lambda: FakeTranscriber(),
  491. sink_factory=lambda: FakeSink(),
  492. debug_sink_factory=lambda: debug,
  493. )
  494. daemon._initialize()
  495. def _utt(i):
  496. return Utterance(
  497. audio=np.zeros(160, dtype=np.float32),
  498. duration_s=0.01,
  499. started_at=time.time(),
  500. rms=0.0,
  501. reason="end_of_turn",
  502. seg_id=i,
  503. )
  504. assert daemon._enqueue(_utt(1)) is True
  505. assert daemon._enqueue(_utt(2)) is True
  506. assert daemon._enqueue(_utt(3)) is True # drops seg 1
  507. daemon._check_backlog()
  508. drops = debug.of("queue_drop")
  509. assert len(drops) == 1
  510. assert drops[0]["dropped_seg_id"] == 1
  511. warns = debug.of("backlog_warn")
  512. assert len(warns) == 1
  513. assert warns[0]["size"] == 2
  514. daemon.stop()
  515. def test_debug_speaker_fallback_on_lane_failure(sock_path):
  516. cfg = _base_config(sock_path)
  517. cfg.speaker.join_timeout_s = 0.05
  518. _enable_debug(cfg)
  519. debug = FakeDebugSink()
  520. daemon = Daemon(
  521. cfg,
  522. log_level=logging.WARNING,
  523. capture_factory=lambda: FakeCapture([_frame()]),
  524. vad_factory=lambda _aggr: FakeVad([]),
  525. transcriber_factory=lambda: FakeTranscriber(),
  526. sink_factory=lambda: FakeSink(),
  527. debug_sink_factory=lambda: debug,
  528. )
  529. daemon._initialize()
  530. failed: concurrent.futures.Future = concurrent.futures.Future()
  531. failed.set_exception(RuntimeError("lane died"))
  532. assert daemon._resolve_speaker(failed) == (None, "none", "?", None)
  533. pending: concurrent.futures.Future = concurrent.futures.Future()
  534. assert daemon._resolve_speaker(pending) == (None, "none", "?", None)
  535. fallbacks = debug.of("speaker_fallback")
  536. assert len(fallbacks) == 2
  537. assert "lane died" in fallbacks[0]["error"]
  538. daemon.stop()
  539. def test_debug_sink_closed_last_in_stop(sock_path):
  540. cfg = _base_config(sock_path)
  541. _enable_debug(cfg)
  542. order = []
  543. transcriber = FakeTranscriber()
  544. transcriber.close = lambda: order.append("transcriber")
  545. sink = FakeSink()
  546. sink.close = lambda: order.append("sink")
  547. debug = FakeDebugSink()
  548. orig_debug_close = debug.close
  549. def tracked_debug_close() -> None:
  550. order.append("debug")
  551. orig_debug_close()
  552. debug.close = tracked_debug_close
  553. daemon = Daemon(
  554. cfg,
  555. log_level=logging.WARNING,
  556. capture_factory=lambda: FakeCapture([_frame()]),
  557. vad_factory=lambda _aggr: FakeVad([]),
  558. transcriber_factory=lambda: transcriber,
  559. sink_factory=lambda: sink,
  560. debug_sink_factory=lambda: debug,
  561. )
  562. daemon._initialize()
  563. assert daemon.publisher is not None
  564. orig_pub_close = daemon.publisher.close
  565. def tracked_pub_close() -> None:
  566. order.append("publisher")
  567. orig_pub_close()
  568. daemon.publisher.close = tracked_pub_close
  569. daemon.stop()
  570. assert order == ["transcriber", "sink", "publisher", "debug"]
  571. assert debug.closed is True
  572. assert daemon.debug_sink is None
  573. def test_debug_factory_throw_disables_sink(sock_path):
  574. cfg = _base_config(sock_path)
  575. _enable_debug(cfg)
  576. def boom_factory():
  577. raise RuntimeError("no dir")
  578. daemon = Daemon(
  579. cfg,
  580. log_level=logging.WARNING,
  581. capture_factory=lambda: FakeCapture([_frame()]),
  582. vad_factory=lambda _aggr: FakeVad([]),
  583. transcriber_factory=lambda: FakeTranscriber(),
  584. sink_factory=lambda: FakeSink(),
  585. debug_sink_factory=boom_factory,
  586. )
  587. daemon._initialize()
  588. assert daemon.debug_sink is None
  589. daemon.stop()
  590. class FakeProc:
  591. """Minimal Popen stand-in for supervisor debug tests."""
  592. _pid_counter = 20000
  593. def __init__(self, exit_on_poll=None, stubborn=False):
  594. self.pid = FakeProc._pid_counter
  595. FakeProc._pid_counter += 1
  596. self._exit_on_poll = exit_on_poll
  597. self._stubborn = stubborn
  598. self.returncode = None
  599. self.terminated = False
  600. self.killed = False
  601. def poll(self):
  602. if self.returncode is None and self._exit_on_poll is not None:
  603. self.returncode = self._exit_on_poll
  604. return self.returncode
  605. def terminate(self):
  606. self.terminated = True
  607. if not self._stubborn and self.returncode is None:
  608. self.returncode = -15
  609. def kill(self):
  610. self.killed = True
  611. if self.returncode is None:
  612. self.returncode = -9
  613. def wait(self, timeout=None):
  614. if self.returncode is None:
  615. self.returncode = 0
  616. return self.returncode
  617. def _supervisor_spawn(procs):
  618. def fake_spawn(argv, *, redirect_output=False):
  619. joined = " ".join(argv)
  620. if "stt_daemon.assistant" in joined:
  621. return procs["assistant"]
  622. return procs["daemon"]
  623. return fake_spawn
  624. def test_supervisor_debug_spawn_exit_shutdown_lines():
  625. cfg = Config()
  626. _enable_debug(cfg)
  627. debug = FakeDebugSink(proc="supervisor")
  628. procs = {"daemon": FakeProc(exit_on_poll=3), "assistant": FakeProc()}
  629. supervisor = Supervisor(
  630. cfg,
  631. with_tui=False,
  632. spawn=_supervisor_spawn(procs),
  633. sleep=lambda s: None,
  634. debug_sink_factory=lambda: debug,
  635. )
  636. assert supervisor.run() == 3
  637. assert debug.types() == ["spawn", "spawn", "child_exit", "shutdown"]
  638. assert debug.emitted[2]["code"] == 3
  639. assert debug.emitted[3]["codes"] == [3, -15]
  640. def test_supervisor_debug_kill_escalation_line():
  641. cfg = Config()
  642. _enable_debug(cfg)
  643. debug = FakeDebugSink(proc="supervisor")
  644. daemon_proc = FakeProc(stubborn=True)
  645. procs = {"daemon": daemon_proc, "assistant": FakeProc(exit_on_poll=1)}
  646. supervisor = Supervisor(
  647. cfg,
  648. with_tui=False,
  649. spawn=_supervisor_spawn(procs),
  650. sleep=lambda s: None,
  651. grace_s=0,
  652. debug_sink_factory=lambda: debug,
  653. )
  654. assert supervisor.run() == 1
  655. assert daemon_proc.killed is True
  656. kills = debug.of("kill_escalation")
  657. assert len(kills) == 1
  658. assert kills[0]["pids"] == [daemon_proc.pid]
  659. def test_supervisor_debug_socket_timeout_line(tmp_path):
  660. cfg = Config()
  661. cfg.assistant.control_socket = str(tmp_path / "missing.sock")
  662. _enable_debug(cfg)
  663. debug = FakeDebugSink(proc="supervisor")
  664. procs = {"daemon": FakeProc(), "assistant": FakeProc()}
  665. supervisor = Supervisor(
  666. cfg,
  667. with_tui=True,
  668. spawn=_supervisor_spawn(procs),
  669. sleep=lambda s: None,
  670. wait_timeout_s=0,
  671. debug_sink_factory=lambda: debug,
  672. )
  673. assert supervisor.run() == 1
  674. assert len(debug.of("socket_timeout")) == 1
  675. assert debug.of("shutdown")
  676. def test_supervisor_debug_disabled_never_builds_sink():
  677. cfg = Config()
  678. calls = []
  679. procs = {"daemon": FakeProc(exit_on_poll=2), "assistant": FakeProc()}
  680. supervisor = Supervisor(
  681. cfg,
  682. with_tui=False,
  683. spawn=_supervisor_spawn(procs),
  684. sleep=lambda s: None,
  685. debug_sink_factory=lambda: calls.append(1),
  686. )
  687. assert supervisor.run() == 2
  688. assert calls == []
  689. def test_supervisor_debug_emit_throw_never_breaks_shutdown():
  690. cfg = Config()
  691. _enable_debug(cfg)
  692. procs = {"daemon": FakeProc(exit_on_poll=4), "assistant": FakeProc()}
  693. supervisor = Supervisor(
  694. cfg,
  695. with_tui=False,
  696. spawn=_supervisor_spawn(procs),
  697. sleep=lambda s: None,
  698. debug_sink_factory=lambda: BoomDebugSink(proc="supervisor"),
  699. )
  700. assert supervisor.run() == 4
  701. def test_request_stop_shuts_down_run_loop(tmp_path):
  702. """request_stop() (the signal-handler path) must unwind run() cleanly."""
  703. sock_path = str(tmp_path / "events.sock")
  704. cfg = _base_config(sock_path)
  705. capture = FakeCapture([_frame(0)] * 5, stall_after=1)
  706. sink = FakeSink()
  707. daemon = Daemon(
  708. cfg,
  709. log_level=logging.WARNING,
  710. capture_factory=lambda: capture,
  711. vad_factory=lambda _aggr: FakeVad([]),
  712. transcriber_factory=lambda: FakeTranscriber(),
  713. sink_factory=lambda: sink,
  714. )
  715. thread = threading.Thread(target=daemon.run, daemon=True)
  716. thread.start()
  717. time.sleep(0.2)
  718. daemon.request_stop()
  719. thread.join(timeout=5)
  720. assert not thread.is_alive()
  721. assert capture._closed.is_set()