daemon.py 35 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883
  1. """Main daemon that orchestrates capture, segmentation, and transcription."""
  2. import collections
  3. import concurrent.futures
  4. import itertools
  5. import json
  6. import logging
  7. import math
  8. import os
  9. import queue
  10. import shutil
  11. import stat
  12. import threading
  13. import time
  14. from pathlib import Path
  15. from typing import Any, Callable, Optional
  16. import numpy as np
  17. from . import events
  18. from .audio import AudioCapture, open_capture
  19. from .bus import EventPublisher
  20. from .config import Config
  21. from .output import TranscriptSink
  22. from .segmenter import FrameSegmenter, Utterance, make_vad
  23. from .speakers import SpeakerStore
  24. from .transcriber import TranscriberLike, build_transcriber
  25. class _LaneJob:
  26. """One embedding request: audio in, Outcome (or exception) out via future."""
  27. __slots__ = ("seg_id", "audio", "dur_s", "rms", "future")
  28. def __init__(
  29. self,
  30. seg_id: int,
  31. audio: np.ndarray,
  32. dur_s: float,
  33. rms: float,
  34. future: concurrent.futures.Future,
  35. ) -> None:
  36. self.seg_id = seg_id
  37. self.audio = audio
  38. self.dur_s = dur_s
  39. self.rms = rms
  40. self.future = future
  41. _REQUEST_MAX_BYTES = 65536
  42. class _EmbeddingLane:
  43. """Single-worker speaker-embedding lane.
  44. Confinement rule: ALL SpeakerStore access (match/process, inbox rename,
  45. prune-persisted saves) happens on this lane's one worker thread. The
  46. transcription worker only submits jobs and awaits the returned future, so
  47. the lane never touches faster-whisper and the daemon socket stays
  48. publish-only. Startup-time load()/prune() in Daemon._initialize runs
  49. single-threaded before this lane starts, so it cannot race the worker.
  50. """
  51. def __init__(
  52. self,
  53. store: SpeakerStore,
  54. embedder: Any,
  55. publish: Callable[..., None],
  56. inbox_dir: Path,
  57. logger: logging.Logger,
  58. ) -> None:
  59. self._store = store
  60. self._embedder = embedder
  61. self._publish = publish
  62. self._inbox_dir = Path(inbox_dir)
  63. self._logger = logger
  64. self._queue: queue.Queue = queue.Queue()
  65. self._stop_event = threading.Event()
  66. self._thread: threading.Thread | None = None
  67. def start(self) -> None:
  68. """Start the single worker thread (non-daemon, like the other lanes)."""
  69. self._thread = threading.Thread(target=self._run, daemon=False)
  70. self._thread.start()
  71. def stop(self) -> None:
  72. """Signal the worker and join it."""
  73. self._stop_event.set()
  74. if self._thread is not None:
  75. self._thread.join(timeout=5)
  76. self._thread = None
  77. def submit(
  78. self, seg_id: int, audio: np.ndarray, dur_s: float, rms: float
  79. ) -> concurrent.futures.Future:
  80. """Enqueue one embedding job; the transcription lane awaits the future."""
  81. future: concurrent.futures.Future = concurrent.futures.Future()
  82. self._queue.put(_LaneJob(seg_id, audio, dur_s, rms, future))
  83. return future
  84. def drain_inbox(self) -> None:
  85. """Apply pending rename requests; quarantine bad ones. Never raises."""
  86. try:
  87. files = sorted(self._inbox_dir.glob("*.json"))
  88. except OSError as e:
  89. self._logger.warning("speaker inbox unreadable (%s)", e)
  90. return
  91. for path in files:
  92. try:
  93. self._apply_request(path)
  94. except Exception as e: # noqa: BLE001 -- boundary: drain must never raise
  95. self._logger.warning("speaker inbox request %s failed: %s", path.name, e)
  96. self._quarantine(path)
  97. def _apply_request(self, path: Path) -> None:
  98. try:
  99. st = os.lstat(path)
  100. except OSError as e:
  101. self._logger.warning("ignoring unreadable speaker request %s (%s)", path.name, e)
  102. self._quarantine(path)
  103. return
  104. if not stat.S_ISREG(st.st_mode):
  105. self._logger.warning(
  106. "ignoring non-regular speaker request %s (not a regular file)", path.name
  107. )
  108. self._quarantine(path)
  109. return
  110. if st.st_size > _REQUEST_MAX_BYTES:
  111. self._logger.warning(
  112. "ignoring oversized speaker request %s (%d bytes)", path.name, st.st_size
  113. )
  114. self._quarantine(path)
  115. return
  116. try:
  117. fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK)
  118. except OSError as e:
  119. self._logger.warning("ignoring unreadable speaker request %s (%s)", path.name, e)
  120. self._quarantine(path)
  121. return
  122. try:
  123. chunks: list[bytes] = []
  124. remaining = _REQUEST_MAX_BYTES + 1
  125. while remaining > 0:
  126. try:
  127. chunk = os.read(fd, remaining)
  128. except OSError as e:
  129. self._logger.warning(
  130. "ignoring unreadable speaker request %s (%s)", path.name, e
  131. )
  132. self._quarantine(path)
  133. return
  134. if not chunk:
  135. break
  136. chunks.append(chunk)
  137. remaining -= len(chunk)
  138. raw = b"".join(chunks)
  139. finally:
  140. try:
  141. os.close(fd)
  142. except OSError:
  143. pass
  144. if len(raw) > _REQUEST_MAX_BYTES:
  145. self._logger.warning(
  146. "ignoring oversized speaker request %s (%d bytes)", path.name, len(raw)
  147. )
  148. self._quarantine(path)
  149. return
  150. try:
  151. payload = json.loads(raw.decode("utf-8"))
  152. except (OSError, ValueError) as e:
  153. self._logger.warning("ignoring malformed speaker request %s (%s)", path.name, e)
  154. self._quarantine(path)
  155. return
  156. if (
  157. not isinstance(payload, dict)
  158. or not isinstance(payload.get("speaker_id"), str)
  159. or not isinstance(payload.get("name"), str)
  160. ):
  161. self._logger.warning("ignoring malformed speaker request %s (bad shape)", path.name)
  162. self._quarantine(path)
  163. return
  164. if not self._store.rename(payload["speaker_id"], payload["name"]):
  165. self._logger.warning(
  166. "ignoring speaker request %s (unknown id %r)", path.name, payload["speaker_id"]
  167. )
  168. self._quarantine(path)
  169. return
  170. self._store.save()
  171. self._publish(events.SPEAKERS, speakers=self._store.snapshot())
  172. try:
  173. path.unlink()
  174. except OSError as e:
  175. self._logger.warning("failed to delete applied speaker request %s (%s)", path.name, e)
  176. def _quarantine(self, path: Path) -> None:
  177. try:
  178. rejected = self._inbox_dir / "rejected"
  179. rejected.mkdir(parents=True, exist_ok=True)
  180. shutil.move(str(path), str(rejected / path.name))
  181. except OSError as e:
  182. self._logger.warning("failed to quarantine speaker request %s (%s)", path.name, e)
  183. def _run(self) -> None:
  184. # A request file may have landed while the daemon was down; apply it
  185. # on startup before the first job so renames never wait a full tick.
  186. self.drain_inbox()
  187. while not self._stop_event.is_set():
  188. try:
  189. job = self._queue.get(timeout=0.5)
  190. except queue.Empty:
  191. self.drain_inbox()
  192. continue
  193. try:
  194. emb = self._embedder.embed(job.audio)
  195. outcome = self._store.process(emb, job.dur_s, job.rms, job.audio)
  196. if outcome.event in ("mint", "confirm"):
  197. self._store.save()
  198. self._publish(events.SPEAKERS, speakers=self._store.snapshot())
  199. if not job.future.done():
  200. job.future.set_result(outcome)
  201. except Exception as e: # noqa: BLE001 -- boundary: lane must resolve every future
  202. if not job.future.done():
  203. job.future.set_exception(e)
  204. self.drain_inbox()
  205. class Daemon:
  206. """Always-on STT daemon with worker threads."""
  207. def __init__(
  208. self,
  209. cfg: Config,
  210. log_level: int = logging.INFO,
  211. *,
  212. capture_factory: Callable[[], Any] | None = None,
  213. vad_factory: Callable[[int], Any] | None = None,
  214. transcriber_factory: Callable[[], Any] | None = None,
  215. sink_factory: Callable[[], Any] | None = None,
  216. embedder_factory: Callable[[], Any] | None = None,
  217. debug_sink_factory: Callable[[], Any] | None = None,
  218. ):
  219. """Initialize the daemon.
  220. Args:
  221. cfg: Configuration object.
  222. log_level: Logging level (e.g., logging.INFO, logging.DEBUG).
  223. capture_factory: Override audio capture creation (used by tests).
  224. vad_factory: Override VAD creation (used by tests).
  225. transcriber_factory: Override transcriber creation (used by tests).
  226. sink_factory: Override transcript sink creation (used by tests).
  227. embedder_factory: Override speaker embedder creation (used by tests).
  228. debug_sink_factory: Override debug signal sink creation (used by
  229. tests). Distinct from sink_factory, which builds TranscriptSink.
  230. """
  231. self.cfg = cfg
  232. self._capture_factory = capture_factory or (lambda: open_capture(self.cfg.audio))
  233. self._vad_factory = vad_factory or make_vad
  234. self._transcriber_factory = transcriber_factory or self._build_transcriber
  235. self._sink_factory = sink_factory or self._build_sink
  236. self._embedder_factory = embedder_factory or self._build_embedder
  237. self._debug_sink_factory = debug_sink_factory
  238. self.debug_sink: Any = None
  239. # Set up logging
  240. self.logger = logging.getLogger("stt_daemon")
  241. self.logger.setLevel(log_level)
  242. if not self.logger.handlers:
  243. handler = logging.StreamHandler()
  244. handler.setFormatter(logging.Formatter("[%(levelname)s] %(message)s"))
  245. self.logger.addHandler(handler)
  246. # Initialize components
  247. self.capture: Optional[AudioCapture] = None
  248. self.segmenter: Optional[FrameSegmenter] = None
  249. self.transcriber: Optional[TranscriberLike] = None
  250. self.sink: Optional[TranscriptSink] = None
  251. self.publisher: Optional[EventPublisher] = None
  252. self.speaker_store: Optional[SpeakerStore] = None
  253. self._speaker_lane: Optional[_EmbeddingLane] = None
  254. # Threading
  255. self.segment_queue: queue.Queue = queue.Queue(maxsize=cfg.daemon.queue_size)
  256. self._stop_event = threading.Event()
  257. self._stopped = False
  258. self._backlog_warned = False
  259. self._reader_thread: Optional[threading.Thread] = None
  260. self._worker_thread: Optional[threading.Thread] = None
  261. # Event bus state (shared across reader/worker threads)
  262. self._seg_counter = itertools.count(1)
  263. self._segments_done = 0
  264. self._last_segment_done_ts = 0.0
  265. self._last_worker_activity_ts = 0.0
  266. self._last_transcript_ts = 0.0
  267. self._in_flight = False
  268. self._frames_since_heartbeat = 0
  269. self._started_at = 0.0
  270. rms_window = max(1, round(1000 / max(1, cfg.audio.frame_ms)))
  271. self._rms_window: collections.deque[float] = collections.deque(maxlen=rms_window)
  272. def run(self) -> None:
  273. """Run the daemon (blocks until stopped)."""
  274. try:
  275. self._initialize()
  276. self._start_workers()
  277. self._stop_event.wait()
  278. finally:
  279. self.stop()
  280. def request_stop(self) -> None:
  281. """Ask the run loop to stop. Signal-safe: cleanup happens in stop()."""
  282. self._stop_event.set()
  283. def _build_transcriber(self) -> TranscriberLike:
  284. return build_transcriber(self.cfg.whisper)
  285. def _build_sink(self) -> TranscriptSink:
  286. return TranscriptSink(
  287. stdout=self.cfg.output.stdout,
  288. jsonl_dir=self.cfg.output.jsonl_dir,
  289. jsonl=self.cfg.output.jsonl,
  290. save_audio=self.cfg.output.save_audio,
  291. audio_dir=self.cfg.output.audio_dir,
  292. sample_rate=self.cfg.audio.sample_rate,
  293. )
  294. def _build_embedder(self) -> Any:
  295. from .embedder import build_embedder
  296. return build_embedder(self.cfg)
  297. def _debug_config(self) -> Any:
  298. """Return the [debug] section, or None when the config predates it."""
  299. return getattr(self.cfg, "debug", None)
  300. def _build_debug_sink(self) -> Any:
  301. """Default debug sink builder (lazy import; duck-typed by callers)."""
  302. from .debug import DebugSink
  303. return DebugSink(getattr(self._debug_config(), "dir", "debug"), "daemon")
  304. def _debug_emit(self, type_: str, **fields: Any) -> None:
  305. """Best-effort debug signal line; never raises, never blocks shutdown."""
  306. sink = self.debug_sink
  307. if sink is None:
  308. return
  309. try:
  310. sink.emit(type_, **fields)
  311. except Exception as e: # noqa: BLE001 -- debug must never break the daemon
  312. self.logger.warning("Debug sink emit failed (%s)", e)
  313. def _initialize(self) -> None:
  314. """Initialize all components and start publishing events."""
  315. self.logger.info("Initializing audio capture...")
  316. self.capture = self._capture_factory()
  317. self.logger.info("Initializing VAD...")
  318. vad = self._vad_factory(self.cfg.vad.aggressiveness)
  319. self.logger.info("Initializing segmenter...")
  320. self.segmenter = FrameSegmenter(
  321. sample_rate=self.cfg.audio.sample_rate,
  322. frame_ms=self.cfg.audio.frame_ms,
  323. vad=vad,
  324. pre_roll_ms=self.cfg.vad.pre_roll_ms,
  325. speech_start_ms=self.cfg.vad.speech_start_ms,
  326. silence_ms=self.cfg.vad.silence_ms,
  327. min_utterance_ms=self.cfg.vad.min_utterance_ms,
  328. max_utterance_s=self.cfg.vad.max_utterance_s,
  329. chunk_pause_ms=self.cfg.vad.chunk_pause_ms,
  330. soft_max_s=self.cfg.vad.soft_max_s,
  331. rms_floor=self.cfg.vad.rms_floor,
  332. trailing_trim_ms=self.cfg.vad.trailing_trim_ms,
  333. )
  334. self.logger.info(f"Loading Whisper model '{self.cfg.whisper.model}'...")
  335. self.transcriber = self._transcriber_factory()
  336. self.logger.info("Initializing output sink...")
  337. self.sink = self._sink_factory()
  338. self._started_at = time.time()
  339. self._start_publisher()
  340. self._start_speaker_lane()
  341. self._start_debug_sink()
  342. self.logger.info("Daemon ready. Listening...")
  343. def _start_debug_sink(self) -> None:
  344. """Construct the debug signal sink when [debug] is enabled (None otherwise)."""
  345. self.debug_sink = None
  346. debug_cfg = self._debug_config()
  347. if debug_cfg is None or not getattr(debug_cfg, "enabled", False):
  348. return
  349. try:
  350. factory = self._debug_sink_factory or self._build_debug_sink
  351. self.debug_sink = factory()
  352. except Exception as e: # noqa: BLE001 -- boundary: debug must not crash init
  353. self.logger.warning("Debug sink disabled (%s)", e)
  354. self.debug_sink = None
  355. def _start_speaker_lane(self) -> None:
  356. """Build the speaker store + embedding lane (no-op when disabled).
  357. Runs single-threaded before any lane thread starts; creates nothing
  358. on disk when the embedder resolves to None.
  359. """
  360. try:
  361. embedder = self._embedder_factory()
  362. except Exception as e: # noqa: BLE001 -- boundary: a bad factory must not crash
  363. self.logger.warning("Speaker identification disabled (%s)", e)
  364. return
  365. if embedder is None:
  366. return
  367. speaker_cfg = self.cfg.speaker
  368. store = SpeakerStore(
  369. Path(speaker_cfg.store_dir),
  370. threshold_id=speaker_cfg.threshold_id,
  371. threshold_new=speaker_cfg.threshold_new,
  372. min_enroll_s=speaker_cfg.min_enroll_s,
  373. rms_floor=speaker_cfg.rms_floor,
  374. margin=speaker_cfg.margin,
  375. mint_cooldown_s=speaker_cfg.mint_cooldown_s,
  376. confirm_s=speaker_cfg.confirm_s,
  377. max_prototypes=speaker_cfg.max_prototypes,
  378. provisional_ttl_h=speaker_cfg.provisional_ttl_h,
  379. )
  380. store.load()
  381. removed = store.prune()
  382. if removed:
  383. store.save()
  384. self._publish(events.SPEAKERS, speakers=store.snapshot())
  385. self.speaker_store = store
  386. self._speaker_lane = _EmbeddingLane(
  387. store,
  388. embedder,
  389. self._publish,
  390. Path(speaker_cfg.store_dir) / "inbox",
  391. self.logger,
  392. )
  393. def _start_publisher(self) -> None:
  394. """Start the publish-only event bus (best effort; never fatal)."""
  395. publisher = EventPublisher(
  396. path=self.cfg.events.socket or None,
  397. ring_size=self.cfg.events.ring_size,
  398. status_provider=self._status_fields,
  399. )
  400. try:
  401. publisher.start()
  402. except OSError as e:
  403. self.logger.warning("Event bus disabled (%s)", e)
  404. self.publisher = None
  405. return
  406. self.publisher = publisher
  407. def _status_fields(self) -> dict:
  408. return {
  409. "capture": self.cfg.audio.device,
  410. "model_loaded": self.transcriber is not None,
  411. "segments_done": self._segments_done,
  412. "heartbeat_ms": self.cfg.events.heartbeat_ms,
  413. }
  414. def _publish(self, type_: str, **fields) -> None:
  415. if self.publisher is not None:
  416. self.publisher.emit(type_, **fields)
  417. def _start_workers(self) -> None:
  418. """Start reader and worker threads."""
  419. self._reader_thread = threading.Thread(target=self._reader_loop, daemon=False)
  420. self._worker_thread = threading.Thread(target=self._worker_loop, daemon=False)
  421. self._reader_thread.start()
  422. self._worker_thread.start()
  423. if self._speaker_lane is not None:
  424. self._speaker_lane.start()
  425. def _reopen_capture(self, attempts: int = 3, delay_s: float = 1.0) -> bool:
  426. """Close and recreate the capture backend with bounded retries.
  427. Returns True on success (self.capture replaced), False when every
  428. attempt failed or shutdown was requested. Never raises.
  429. """
  430. for attempt in range(1, attempts + 1):
  431. if self._stop_event.is_set():
  432. return False
  433. self._debug_emit(
  434. "capture_reopen_attempt", attempt=attempt, attempts=attempts
  435. )
  436. try:
  437. if self.capture is not None:
  438. try:
  439. self.capture.close()
  440. except Exception as e: # noqa: BLE001 -- close must not block reopen
  441. self.logger.debug("Capture close during reopen failed (%s)", e)
  442. self.capture = self._capture_factory()
  443. self.logger.warning(
  444. "Capture reopened on attempt %d/%d", attempt, attempts
  445. )
  446. self._debug_emit(
  447. "capture_reopen_ok", attempt=attempt, attempts=attempts
  448. )
  449. return True
  450. except Exception as e: # noqa: BLE001 -- boundary: retry, then give up
  451. self.logger.warning(
  452. "Capture reopen attempt %d/%d failed (%s)", attempt, attempts, e
  453. )
  454. self._debug_emit(
  455. "capture_reopen_fail",
  456. attempt=attempt,
  457. attempts=attempts,
  458. error=str(e),
  459. )
  460. if attempt < attempts:
  461. self._stop_event.wait(delay_s)
  462. return False
  463. def _reader_loop(self) -> None:
  464. """Reader thread: capture frames, segment them, and publish events."""
  465. failed = False
  466. try:
  467. if not self.capture or not self.segmenter:
  468. self.logger.error("Reader cannot start: capture or segmenter missing")
  469. failed = True
  470. return
  471. heartbeat_s = max(0.05, self.cfg.events.heartbeat_ms / 1000.0)
  472. next_heartbeat = time.time() + heartbeat_s
  473. while not self._stop_event.is_set():
  474. try:
  475. for frame in self.capture.frames():
  476. if self._stop_event.is_set():
  477. break
  478. self._track_level(frame)
  479. self._frames_since_heartbeat += 1
  480. was_speaking = self.segmenter.speaking
  481. utt = self.segmenter.push(frame)
  482. if not was_speaking and self.segmenter.speaking:
  483. self._publish(events.SPEECH_START)
  484. if was_speaking and not self.segmenter.speaking:
  485. # A turn ended. It may have been discarded (blip / too quiet),
  486. # in which case there is no utterance - but speech_stop must
  487. # still be published or consumers latch `speech_open` true.
  488. if utt is not None:
  489. self._on_segment_finalized(utt)
  490. else:
  491. self._publish(
  492. events.SPEECH_STOP, seg_id=None, end_reason="discarded"
  493. )
  494. now = time.time()
  495. if now >= next_heartbeat:
  496. self._publish_heartbeat()
  497. next_heartbeat = now + heartbeat_s
  498. # Generator exhausted cleanly (or stop requested): leave the
  499. # retry loop and run the shutdown flush below.
  500. break
  501. except Exception as e: # noqa: BLE001 -- boundary: EOF/short-read land here
  502. if self._stop_event.is_set():
  503. self.logger.debug(f"Reader stopped: {e}")
  504. break
  505. self.logger.error("Capture failure: %s", e)
  506. if self._stop_event.is_set():
  507. break
  508. if not self._reopen_capture():
  509. self.logger.error(
  510. "Capture unrecoverable after retries; requesting shutdown"
  511. )
  512. failed = True
  513. break
  514. next_heartbeat = time.time() + heartbeat_s
  515. continue
  516. # Shutdown: flush any in-progress utterance
  517. if self.segmenter:
  518. was_speaking = self.segmenter.speaking
  519. utt = self.segmenter.flush()
  520. if utt is not None:
  521. self._on_segment_finalized(utt)
  522. elif was_speaking:
  523. self._publish(
  524. events.SPEECH_STOP, seg_id=None, end_reason="discarded"
  525. )
  526. self._publish_heartbeat()
  527. except Exception as e: # noqa: BLE001 -- boundary: reader must never die silent
  528. if self._stop_event.is_set():
  529. self.logger.debug(f"Reader stopped: {e}")
  530. else:
  531. self.logger.error(f"Reader thread error: {e}")
  532. failed = True
  533. finally:
  534. if self.capture:
  535. try:
  536. self.capture.close()
  537. except Exception as e: # noqa: BLE001 -- shutdown path must not raise
  538. self.logger.debug("Capture close on reader exit failed (%s)", e)
  539. if failed and not self._stop_event.is_set():
  540. self.logger.error("Reader thread exited unexpectedly; requesting shutdown")
  541. self._debug_emit("reader_fatal")
  542. self.request_stop()
  543. def _on_segment_finalized(self, utt: Utterance) -> None:
  544. """Assign a seg_id at finalization, announce the stop, and enqueue."""
  545. seg_id = next(self._seg_counter)
  546. utt.seg_id = seg_id
  547. self._publish(events.SPEECH_STOP, seg_id=seg_id, end_reason=utt.reason)
  548. if not self._enqueue(utt):
  549. self._publish(
  550. events.SEGMENT_DONE, seg_id=seg_id, emitted=False, end_reason=utt.reason
  551. )
  552. self._check_backlog()
  553. def _enqueue(self, utt: Utterance) -> bool:
  554. """Queue a segment, dropping the oldest if full. False if it could not be queued."""
  555. try:
  556. self.segment_queue.put(utt, block=False)
  557. return True
  558. except queue.Full:
  559. self.logger.warning("Segment queue full, dropping oldest")
  560. try:
  561. dropped = self.segment_queue.get_nowait()
  562. except queue.Empty:
  563. dropped = None
  564. if dropped is not None:
  565. self._publish(
  566. events.SEGMENT_DONE,
  567. seg_id=dropped.seg_id,
  568. emitted=False,
  569. end_reason=dropped.reason,
  570. )
  571. self._debug_emit(
  572. "queue_drop",
  573. dropped_seg_id=dropped.seg_id,
  574. end_reason=dropped.reason,
  575. queue_size=self.segment_queue.maxsize,
  576. )
  577. try:
  578. self.segment_queue.put(utt, block=False)
  579. return True
  580. except queue.Full:
  581. return False
  582. def _track_level(self, frame: bytes) -> None:
  583. samples = np.frombuffer(frame, dtype=np.int16).astype(np.float32) / 32768.0
  584. if samples.size:
  585. self._rms_window.append(float(np.mean(samples * samples)))
  586. def _publish_heartbeat(self) -> None:
  587. now = time.time()
  588. rms = math.sqrt(sum(self._rms_window) / len(self._rms_window)) if self._rms_window else 0.0
  589. busy = self._in_flight or self.segment_queue.qsize() > 0
  590. payload = {
  591. "in_flight": self._in_flight,
  592. "segments_done": self._segments_done,
  593. "worker_stall_s": (
  594. round(now - self._last_worker_activity_ts, 3)
  595. if busy and self._last_worker_activity_ts
  596. else 0.0
  597. ),
  598. "queue_depth": self.segment_queue.qsize(),
  599. "frames_since_last": self._frames_since_heartbeat,
  600. "rms": round(rms, 5),
  601. "capture": self.cfg.audio.device,
  602. "device_ok": self.capture.healthy() if self.capture else False,
  603. "model_loaded": self.transcriber is not None,
  604. "uptime_s": round(now - self._started_at, 3),
  605. "last_transcript_age_s": (
  606. round(now - self._last_transcript_ts, 3)
  607. if self._last_transcript_ts
  608. else 0.0
  609. ),
  610. }
  611. self._publish(events.HEARTBEAT, **payload)
  612. debug_cfg = self._debug_config()
  613. if debug_cfg is not None and getattr(debug_cfg, "heartbeat", True):
  614. self._debug_emit("heartbeat", **payload)
  615. self._frames_since_heartbeat = 0
  616. def _check_backlog(self) -> None:
  617. """Warn once per overload episode; re-arm only after the queue fully drains."""
  618. size = self.segment_queue.qsize()
  619. if size >= self.cfg.daemon.backlog_warn_at and not self._backlog_warned:
  620. self.logger.warning(
  621. "Transcription backlog reached %d of %d - worker is falling behind",
  622. size,
  623. self.segment_queue.maxsize,
  624. )
  625. self._debug_emit(
  626. "backlog_warn", size=size, maxsize=self.segment_queue.maxsize
  627. )
  628. self._backlog_warned = True
  629. elif size == 0 and self._backlog_warned:
  630. self._backlog_warned = False
  631. def _worker_loop(self) -> None:
  632. """Worker thread: transcribe segments and publish results."""
  633. failed = False
  634. try:
  635. while not self._stop_event.is_set():
  636. try:
  637. utt = self.segment_queue.get(timeout=1)
  638. except queue.Empty:
  639. continue
  640. self._process(utt)
  641. except Exception as e: # noqa: BLE001 -- boundary: worker must never die silent
  642. if self._stop_event.is_set():
  643. self.logger.debug(f"Worker stopped: {e}")
  644. else:
  645. self.logger.error(f"Worker thread error: {e}")
  646. failed = True
  647. finally:
  648. try:
  649. self._drain_remaining()
  650. except Exception as e: # noqa: BLE001 -- shutdown path must not raise
  651. self.logger.debug("Drain on worker exit failed (%s)", e)
  652. if failed and not self._stop_event.is_set():
  653. self.logger.error("Worker thread exited unexpectedly; requesting shutdown")
  654. self._debug_emit("worker_fatal")
  655. self.request_stop()
  656. def _process(self, utt: Utterance) -> None:
  657. """Transcribe one segment, then always publish its segment_done."""
  658. emitted = False
  659. self._in_flight = True
  660. self._last_worker_activity_ts = time.time()
  661. try:
  662. if self.transcriber and self.sink:
  663. speaker_future = None
  664. if self._speaker_lane is not None:
  665. speaker_future = self._speaker_lane.submit(
  666. utt.seg_id, utt.audio, utt.duration_s, utt.rms
  667. )
  668. started = time.perf_counter()
  669. result = self.transcriber.transcribe(utt.audio)
  670. transcribe_s = time.perf_counter() - started
  671. seg_id = utt.seg_id
  672. speaker_id: str | None = None
  673. speaker_status = "none"
  674. speaker: str | None = "?"
  675. speaker_score: float | None = None
  676. if speaker_future is not None:
  677. speaker_id, speaker_status, speaker, speaker_score = (
  678. self._resolve_speaker(speaker_future)
  679. )
  680. try:
  681. audio_path = self.sink.emit(
  682. result,
  683. utt.duration_s,
  684. utt.rms,
  685. utt.reason,
  686. utt.audio,
  687. transcribe_s=transcribe_s,
  688. seg_id=seg_id,
  689. speaker_id=speaker_id,
  690. speaker_status=speaker_status,
  691. speaker=speaker,
  692. speaker_score=speaker_score,
  693. )
  694. except Exception as e:
  695. # A sink failure must not drop the transcript from the bus.
  696. self.logger.error("Output error: %s", e)
  697. audio_path = None
  698. if result.text:
  699. emitted = True
  700. self._last_transcript_ts = time.time()
  701. self._publish(
  702. events.TRANSCRIPT,
  703. seg_id=seg_id,
  704. text=result.text,
  705. lang=result.language,
  706. lang_prob=result.language_probability,
  707. duration_s=utt.duration_s,
  708. rms=utt.rms,
  709. end_reason=utt.reason,
  710. audio=audio_path,
  711. speaker_id=speaker_id,
  712. speaker_status=speaker_status,
  713. speaker=speaker,
  714. speaker_score=speaker_score,
  715. )
  716. except Exception as e:
  717. self.logger.error(f"Transcription error: {e}")
  718. finally:
  719. self._in_flight = False
  720. self._segments_done += 1
  721. self._last_segment_done_ts = time.time()
  722. self._publish(
  723. events.SEGMENT_DONE,
  724. seg_id=utt.seg_id,
  725. emitted=emitted,
  726. end_reason=utt.reason,
  727. )
  728. def _resolve_speaker(
  729. self, future: concurrent.futures.Future
  730. ) -> tuple[str | None, str, str | None, float | None]:
  731. """Await one embedding outcome; degrade to "none" on any failure."""
  732. try:
  733. outcome = future.result(timeout=self.cfg.speaker.join_timeout_s)
  734. except Exception as e: # noqa: BLE001 -- TimeoutError or lane failure -> "none"
  735. self.logger.debug("Speaker lookup failed (%s)", e)
  736. self._debug_emit("speaker_fallback", error=str(e))
  737. return None, "none", "?", None
  738. try:
  739. return self._speaker_fields(outcome)
  740. except Exception as e: # noqa: BLE001 -- mapping must never break emit
  741. self.logger.warning("Speaker mapping failed (%s)", e)
  742. return None, "none", "?", None
  743. def _speaker_fields(self, outcome) -> tuple[str | None, str, str | None, float | None]:
  744. """Map a lane outcome to (speaker_id, speaker_status, speaker, speaker_score)."""
  745. match = outcome.match
  746. status = match.status
  747. speakers = self.speaker_store.speakers if self.speaker_store is not None else {}
  748. if status == "matched":
  749. name = speakers.get(match.speaker_id, {}).get("name") if match.speaker_id else None
  750. return match.speaker_id, "matched", name or match.speaker_id, match.score
  751. if status == "uncertain":
  752. top = match.top_id
  753. name = speakers.get(top, {}).get("name") if top else None
  754. return None, "uncertain", f"{(name or top)}?", match.score
  755. if status == "unknown":
  756. minted = match.speaker_id if outcome.event == "mint" else None
  757. return minted, "unknown", "?", None
  758. return None, "none", "?", None
  759. def _drain_remaining(self) -> None:
  760. """Announce any queued-but-unprocessed segments on shutdown."""
  761. while True:
  762. try:
  763. utt = self.segment_queue.get_nowait()
  764. except queue.Empty:
  765. break
  766. self._publish(
  767. events.SEGMENT_DONE,
  768. seg_id=utt.seg_id,
  769. emitted=False,
  770. end_reason=utt.reason,
  771. )
  772. def stop(self) -> None:
  773. """Stop the daemon. Safe to call more than once."""
  774. if self._stopped:
  775. return
  776. self._stopped = True
  777. self.logger.info("Stopping daemon...")
  778. self._stop_event.set()
  779. # Close capture FIRST so the reader thread's blocking read unblocks
  780. # (otherwise a non-daemon reader thread can hang process shutdown).
  781. if self.capture:
  782. self.capture.close()
  783. if self._reader_thread:
  784. self._reader_thread.join(timeout=5)
  785. if self._worker_thread:
  786. self._worker_thread.join(timeout=5)
  787. if self._speaker_lane is not None:
  788. self._speaker_lane.stop()
  789. self._speaker_lane = None
  790. if self.transcriber:
  791. self.transcriber.close()
  792. if self.sink:
  793. self.sink.close()
  794. if self.publisher:
  795. self.publisher.close()
  796. if self.debug_sink is not None:
  797. try:
  798. self.debug_sink.close()
  799. except Exception as e: # noqa: BLE001 -- shutdown path must not raise
  800. self.logger.debug("Debug sink close failed (%s)", e)
  801. finally:
  802. self.debug_sink = None
  803. self.logger.info("Daemon stopped.")