Improved to split long utterances at pauses and skip silent segments

README.md

@@ -92,9 +92,13 @@ gain        = 1.0                     # Amplitude scaling (1.0 = no change, >1 =
 aggressiveness   = 2                  # 0–3: higher = fewer false positives (speech misses more)
 pre_roll_ms      = 400                # Ring buffer duration: keep this many ms of pre-speech
 speech_start_ms  = 90                 # Consecutive speech frames to confirm speech start
-silence_ms       = 700                # Consecutive silence to end an utterance
-min_utterance_ms = 300                # Discard utterances shorter than this
-max_utterance_s  = 30.0               # Force-finalize if utterance exceeds this duration
+silence_ms       = 700                # Consecutive silence to end a turn
+min_utterance_ms = 300                # Discard utterances with less actual speech than this
+max_utterance_s  = 30.0               # Hard cap: force-finalize a very long utterance
+chunk_pause_ms   = 350                # Once soft_max_s is reached, end a chunk at a pause this long
+soft_max_s       = 10.0               # After this much speech, chunk-ends become eligible at pauses
+rms_floor        = 0.008              # Drop segments quieter than this RMS (noise-triggered blips)
+trailing_trim_ms = 200                # Keep only this much trailing silence (less hallucination-prone padding)
 
 [whisper]
 model        = "small"                # "tiny", "base", "small", "medium", "large"
@@ -103,6 +107,8 @@ compute_type = "int8"                 # "int8" (CPU) or "float16" (CUDA, slower
 language     = "auto"                 # "auto" (detect), "en", "es", or None/""/"none"
 beam_size    = 1                      # 1 = fast greedy, >1 = slower beam search
 vad_filter   = false                  # Use Whisper's built-in VAD (we already have webrtcvad)
+no_speech_prob_max = 0.6              # Drop segments Whisper scores as non-speech above this
+avg_logprob_min    = -1.0             # Drop segments with confidence below this
 
 [sink]                                # Virtual audio sink configuration
 sink_name             = "VirtualMicSink"       # null-sink name
@@ -378,7 +384,21 @@ The **pre-roll buffer** is a ring buffer that keeps the last N milliseconds (def
 1. **Silence phase**: Frames are pushed into a ring buffer (maxlen = pre_roll_ms / frame_ms). Old frames fall off.
 2. **Speech detected**: VAD returns True for consecutive `speech_start_ms` worth of frames.
 3. **Speech confirmed**: `_begin()` is called. The entire ring buffer contents are copied into the segment, then regular frames append to it.
-4. **Silence finalized**: After `silence_ms` of silence, the segment is converted to an Utterance and emitted.
+4. **End of turn**: after `silence_ms` of silence the segment is finalized and transcribed.
+
+### Long speech and silence
+
+Two extra rules keep long monologues from becoming one huge blob, and keep
+noise from being transcribed:
+
+- **Pause-aligned chunking**: once a segment reaches `soft_max_s` (10s) of speech,
+  a shorter `chunk_pause_ms` (350ms) pause ends a *chunk*. The cut lands on a
+  natural phrase boundary, so no word is split and no overlap/duplication occurs.
+  `max_utterance_s` (30s) remains the last-resort cap for someone who never pauses.
+- **Noise suppression**: `min_utterance_ms` drops blips, `rms_floor` drops
+  segments too quiet to be speech, `trailing_trim_ms` strips hallucination-prone
+  trailing silence before transcription, and Whisper segments below
+  `no_speech_prob_max` / `avg_logprob_min` are discarded.
 
 ## Troubleshooting
 

config.example.toml

@@ -10,9 +10,13 @@ gain        = 1.0
 aggressiveness   = 2                  # webrtcvad 0-3
 pre_roll_ms      = 400                # ring buffer kept BEFORE speech is confirmed
 speech_start_ms  = 90                 # consecutive speech frames needed to confirm start
-silence_ms       = 700                # trailing silence that ends an utterance
+silence_ms       = 700                # trailing silence that ends a turn
 min_utterance_ms = 300                # discard utterances with less actual speech than this
-max_utterance_s  = 30.0               # force-finalize a very long utterance
+max_utterance_s  = 30.0               # hard cap: force-finalize a very long utterance
+chunk_pause_ms   = 350                # once soft_max_s is reached, end a chunk at a pause this long
+soft_max_s       = 10.0               # after this much speech, chunk-ends become eligible at pauses
+rms_floor        = 0.008              # drop segments quieter than this RMS (kills noise-triggered blips)
+trailing_trim_ms = 200                # keep only this much trailing silence (less hallucination-prone padding)
 
 [whisper]
 model        = "small"                # multilingual (DO NOT use .en suffix)
@@ -21,6 +25,8 @@ compute_type = "int8"                 # "int8" cpu, "float16" cuda
 language     = "auto"                 # "auto" | "en" | "es"
 beam_size    = 1
 vad_filter   = false
+no_speech_prob_max = 0.6              # drop segments Whisper thinks are non-speech above this
+avg_logprob_min    = -1.0             # drop segments with confidence below this
 
 [output]
 stdout    = true

src/stt_daemon/cli.py

@@ -427,6 +427,8 @@ def _check(args) -> int:
             language=cfg.whisper.language,
             beam_size=cfg.whisper.beam_size,
             vad_filter=cfg.whisper.vad_filter,
+            no_speech_prob_max=cfg.whisper.no_speech_prob_max,
+            avg_logprob_min=cfg.whisper.avg_logprob_min,
         )
         transcriber.close()
         print("✓ Model ready", flush=True)

src/stt_daemon/config.py

@@ -25,6 +25,10 @@ class VadConfig:
     silence_ms: int = 700
     min_utterance_ms: int = 300
     max_utterance_s: float = 30.0
+    chunk_pause_ms: int = 350
+    soft_max_s: float = 10.0
+    rms_floor: float = 0.008
+    trailing_trim_ms: int = 200
 
 
 @dataclass
@@ -36,6 +40,8 @@ class WhisperConfig:
     language: str | None = None
     beam_size: int = 1
     vad_filter: bool = False
+    no_speech_prob_max: float = 0.6
+    avg_logprob_min: float = -1.0
 
 
 @dataclass
@@ -111,6 +117,10 @@ def load_config(path: str | None = None) -> Config:
                 silence_ms=vad_data.get("silence_ms", cfg.vad.silence_ms),
                 min_utterance_ms=vad_data.get("min_utterance_ms", cfg.vad.min_utterance_ms),
                 max_utterance_s=vad_data.get("max_utterance_s", cfg.vad.max_utterance_s),
+                chunk_pause_ms=vad_data.get("chunk_pause_ms", cfg.vad.chunk_pause_ms),
+                soft_max_s=vad_data.get("soft_max_s", cfg.vad.soft_max_s),
+                rms_floor=vad_data.get("rms_floor", cfg.vad.rms_floor),
+                trailing_trim_ms=vad_data.get("trailing_trim_ms", cfg.vad.trailing_trim_ms),
             )
 
         # Whisper section
@@ -127,6 +137,12 @@ def load_config(path: str | None = None) -> Config:
                 language=language,
                 beam_size=whisper_data.get("beam_size", cfg.whisper.beam_size),
                 vad_filter=whisper_data.get("vad_filter", cfg.whisper.vad_filter),
+                no_speech_prob_max=whisper_data.get(
+                    "no_speech_prob_max", cfg.whisper.no_speech_prob_max
+                ),
+                avg_logprob_min=whisper_data.get(
+                    "avg_logprob_min", cfg.whisper.avg_logprob_min
+                ),
             )
 
         # Output section

src/stt_daemon/daemon.py

@@ -72,6 +72,10 @@ class Daemon:
             silence_ms=self.cfg.vad.silence_ms,
             min_utterance_ms=self.cfg.vad.min_utterance_ms,
             max_utterance_s=self.cfg.vad.max_utterance_s,
+            chunk_pause_ms=self.cfg.vad.chunk_pause_ms,
+            soft_max_s=self.cfg.vad.soft_max_s,
+            rms_floor=self.cfg.vad.rms_floor,
+            trailing_trim_ms=self.cfg.vad.trailing_trim_ms,
         )
 
         self.logger.info(f"Loading Whisper model '{self.cfg.whisper.model}'...")
@@ -82,6 +86,8 @@ class Daemon:
             language=self.cfg.whisper.language,
             beam_size=self.cfg.whisper.beam_size,
             vad_filter=self.cfg.whisper.vad_filter,
+            no_speech_prob_max=self.cfg.whisper.no_speech_prob_max,
+            avg_logprob_min=self.cfg.whisper.avg_logprob_min,
         )
 
         self.logger.info("Initializing output sink...")

src/stt_daemon/segmenter.py

@@ -53,6 +53,10 @@ class FrameSegmenter:
         silence_ms: int,
         min_utterance_ms: int,
         max_utterance_s: float,
+        chunk_pause_ms: int = 350,
+        soft_max_s: float = 10.0,
+        rms_floor: float = 0.008,
+        trailing_trim_ms: int = 200,
     ):
         """Initialize the segmenter.
 
@@ -62,9 +66,13 @@ class FrameSegmenter:
             vad: webrtcvad.Vad instance (already created).
             pre_roll_ms: Duration of pre-roll buffer in ms.
             speech_start_ms: Consecutive speech frames to confirm speech start.
-            silence_ms: Consecutive silence frames to end an utterance.
-            min_utterance_ms: Minimum utterance duration to keep (discard blips).
-            max_utterance_s: Maximum utterance duration before force-finalize.
+            silence_ms: Consecutive silence frames that end a turn.
+            min_utterance_ms: Minimum actual speech to keep (discard blips).
+            max_utterance_s: Hard cap; force-finalize a very long utterance.
+            chunk_pause_ms: Shorter pause that may end a chunk once soft_max_s is reached.
+            soft_max_s: Speech duration after which chunk-ending pauses become eligible.
+            rms_floor: Discard utterances quieter than this RMS (noise that tripped the VAD).
+            trailing_trim_ms: Trailing silence to keep (the rest is trimmed before transcription).
         """
         self.sample_rate = sample_rate
         self.frame_ms = frame_ms
@@ -72,11 +80,16 @@ class FrameSegmenter:
 
         # Frame-based counters and buffers
         self.frame_bytes = round(sample_rate * frame_ms / 1000) * 2  # s16le = 2 bytes/sample
+        self.frame_samples = self.frame_bytes // 2
         self.pre_roll_frames = max(1, round(pre_roll_ms / frame_ms))
         self.start_frames = max(1, round(speech_start_ms / frame_ms))
         self.silence_frames = max(1, round(silence_ms / frame_ms))
         self.min_frames = max(1, round(min_utterance_ms / frame_ms))
         self.max_frames = max(1, round(max_utterance_s * 1000 / frame_ms))
+        self.chunk_pause_frames = max(1, round(chunk_pause_ms / frame_ms))
+        self.soft_max_frames = max(1, round(soft_max_s * 1000 / frame_ms))
+        self.trailing_trim_frames = max(0, round(trailing_trim_ms / frame_ms))
+        self.rms_floor = rms_floor
 
         # Ring buffer for pre-speech frames
         self.ring: collections.deque[bytes] = collections.deque(maxlen=self.pre_roll_frames)
@@ -123,13 +136,20 @@ class FrameSegmenter:
             self.silence_run += 1
 
         if self.silence_run >= self.silence_frames:
-            # Silence finalized
+            # End of turn: the speaker stopped.
             return self._finish(keep_tail=True)
 
         if len(self.segment) >= self.max_frames:
-            # Max duration exceeded
+            # Hard cap (speaker never paused) - must cut somewhere.
             return self._finish(keep_tail=False)
 
+        if (
+            len(self.segment) >= self.soft_max_frames
+            and self.silence_run >= self.chunk_pause_frames
+        ):
+            # Long speech: end a chunk at a natural pause instead of shipping a huge blob.
+            return self._finish(keep_tail=True)
+
         return None
 
     def _begin(self) -> None:
@@ -149,12 +169,13 @@ class FrameSegmenter:
         return None
 
     def _finish(self, keep_tail: bool) -> Utterance | None:
-        """Finalize and emit an utterance."""
+        """Finalize and emit an utterance, or discard it if it is a blip or too quiet."""
         frames = self.segment
+        trailing_silence = self.silence_run
 
         # Optionally keep trailing silence frames to seed next pre-roll
-        if keep_tail and self.silence_run > 0:
-            tail = frames[-self.silence_run :]
+        if keep_tail and trailing_silence > 0:
+            tail = frames[-trailing_silence:]
         else:
             tail = []
 
@@ -175,8 +196,20 @@ class FrameSegmenter:
             return None
 
         # Convert frames to audio
-        raw = b"".join(frames)
-        audio = np.frombuffer(raw, dtype=np.int16).astype(np.float32) / 32768.0
+        audio = np.frombuffer(b"".join(frames), dtype=np.int16).astype(np.float32) / 32768.0
+
+        # Trim most of the trailing silence: pure padding invites Whisper hallucinations.
+        dropped = max(0, trailing_silence - self.trailing_trim_frames)
+        if dropped:
+            audio = audio[: max(0, len(audio) - dropped * self.frame_samples)]
+
+        if audio.size == 0:
+            return None
+
+        # Drop segments too quiet to be real speech (noise that tripped the VAD).
+        rms = float(np.sqrt(np.mean(audio**2)))
+        if rms < self.rms_floor:
+            return None
 
         return Utterance(
             audio=audio,

src/stt_daemon/transcriber.py

@@ -14,6 +14,18 @@ class TranscriptResult:
     text: str
 
 
+def accept_segment(segment, no_speech_prob_max: float, avg_logprob_min: float) -> bool:
+    """Keep a Whisper segment only when it is confidently speech.
+
+    Drops the hallucinated text Whisper produces on noise/silence.
+    """
+    if getattr(segment, "no_speech_prob", 0.0) > no_speech_prob_max:
+        return False
+    if getattr(segment, "avg_logprob", 0.0) < avg_logprob_min:
+        return False
+    return True
+
+
 class Transcriber:
     """Wrapper around Faster-Whisper for transcription."""
 
@@ -25,6 +37,8 @@ class Transcriber:
         language: str | None,
         beam_size: int = 1,
         vad_filter: bool = False,
+        no_speech_prob_max: float = 0.6,
+        avg_logprob_min: float = -1.0,
     ):
         """Initialize the transcriber.
 
@@ -35,12 +49,16 @@ class Transcriber:
             language: Language code (e.g., "en", "es") or None for auto-detect.
             beam_size: Beam search width (1 = greedy, fastest).
             vad_filter: Enable Whisper's built-in VAD (we already run webrtcvad).
+            no_speech_prob_max: Drop segments Whisper scores as non-speech above this.
+            avg_logprob_min: Drop segments with average log-probability below this.
         """
         self.language = language
         self.device = device
         self.compute_type = compute_type
         self.beam_size = beam_size
         self.vad_filter = vad_filter
+        self.no_speech_prob_max = no_speech_prob_max
+        self.avg_logprob_min = avg_logprob_min
 
         # Attempt CUDA if requested, fall back to CPU
         if device == "cuda":
@@ -77,8 +95,12 @@ class Transcriber:
             condition_on_previous_text=False,
         )
 
-        # Join all segments
-        text = "".join(seg.text for seg in segments).strip()
+        # Join the segments Whisper is confident about
+        text = "".join(
+            seg.text
+            for seg in segments
+            if accept_segment(seg, self.no_speech_prob_max, self.avg_logprob_min)
+        ).strip()
 
         return TranscriptResult(
             language=info.language,

tests/test_config.py

@@ -140,3 +140,49 @@ device = "custom_device"
         assert cfg.audio.backend == "parec"
         assert cfg.audio.sample_rate == 16000
         assert cfg.vad.aggressiveness == 2
+
+
+def test_chunking_and_confidence_fields():
+    """New chunking/gate fields load from TOML."""
+    config_toml = """
+[vad]
+chunk_pause_ms = 500
+soft_max_s = 8.0
+rms_floor = 0.01
+trailing_trim_ms = 150
+
+[whisper]
+no_speech_prob_max = 0.7
+avg_logprob_min = -0.8
+"""
+    with tempfile.TemporaryDirectory() as tmpdir:
+        config_path = Path(tmpdir) / "config.toml"
+        config_path.write_text(config_toml)
+
+        cfg = load_config(str(config_path))
+
+    assert cfg.vad.chunk_pause_ms == 500
+    assert cfg.vad.soft_max_s == 8.0
+    assert cfg.vad.rms_floor == 0.01
+    assert cfg.vad.trailing_trim_ms == 150
+    assert cfg.whisper.no_speech_prob_max == 0.7
+    assert cfg.whisper.avg_logprob_min == -0.8
+
+
+def test_chunking_defaults():
+    """New fields have sane defaults."""
+    with tempfile.TemporaryDirectory() as tmpdir:
+        import os
+        old_cwd = os.getcwd()
+        try:
+            os.chdir(tmpdir)
+            cfg = load_config()
+        finally:
+            os.chdir(old_cwd)
+
+    assert cfg.vad.chunk_pause_ms == 350
+    assert cfg.vad.soft_max_s == 10.0
+    assert cfg.vad.rms_floor == 0.008
+    assert cfg.vad.trailing_trim_ms == 200
+    assert cfg.whisper.no_speech_prob_max == 0.6
+    assert cfg.whisper.avg_logprob_min == -1.0

tests/test_segmenter.py

@@ -408,3 +408,92 @@ def test_real_speech_kept_with_realistic_pre_roll():
 
     assert utt is not None
     assert len(utt.audio) >= frame_samples * 12
+
+
+def _segmenter(vad, **overrides):
+    params: dict = dict(
+        sample_rate=16000,
+        frame_ms=30,
+        vad=vad,
+        pre_roll_ms=60,
+        speech_start_ms=30,
+        silence_ms=600,
+        min_utterance_ms=30,
+        max_utterance_s=30.0,
+        chunk_pause_ms=300,
+        soft_max_s=1.0,
+        rms_floor=0.0,
+        trailing_trim_ms=200,
+    )
+    params.update(overrides)
+    return FrameSegmenter(**params)
+
+
+def test_long_speech_ends_chunk_at_pause_after_soft_max():
+    """A long utterance must split at a pause once soft_max_s is reached."""
+    frame_samples = 480
+    # 35 speech frames (> soft_max of 33) then a 10-frame (300ms) pause.
+    seg = _segmenter(FakeVad([True] * 35 + [False] * 10))
+
+    for _ in range(35):
+        assert seg.push(make_speech_frame(frame_samples)) is None
+
+    utt = None
+    for _ in range(10):
+        utt = seg.push(make_silent_frame(frame_samples))
+        if utt is not None:
+            break
+
+    assert utt is not None, "chunk should end at the pause (silence_ms is 600, so not end-of-turn)"
+    # 45 frames total; trailing 10 silence trimmed to round(200/30)=7 kept -> 3 dropped
+    assert len(utt.audio) == (45 - 3) * frame_samples
+
+
+def test_short_pause_before_soft_max_does_not_split():
+    """A pause shorter than the threshold must not split a short utterance."""
+    frame_samples = 480
+    # soft_max is 10s here, so 50 frames (1.5s) + a 300ms pause must NOT finalize.
+    seg = _segmenter(FakeVad([True] * 50 + [False] * 10), soft_max_s=10.0)
+
+    for _ in range(50):
+        assert seg.push(make_speech_frame(frame_samples)) is None
+    for _ in range(10):
+        assert seg.push(make_silent_frame(frame_samples)) is None
+
+    # Still speaking; flush emits the whole thing.
+    assert seg.flush() is not None
+
+
+def test_quiet_segment_discarded_by_rms_floor():
+    """Noise that trips the VAD but is too quiet should be dropped."""
+    frame_samples = 480
+
+    # High floor: the quiet utterance is dropped at finalization.
+    strict = _segmenter(FakeVad([True] * 5 + [False] * 2), silence_ms=60, rms_floor=0.5)
+    for _ in range(5):
+        strict.push(make_speech_frame(frame_samples, value=1000))
+    strict.push(make_silent_frame(frame_samples))
+    assert strict.push(make_silent_frame(frame_samples)) is None
+
+    # Same audio with a permissive floor is kept -> proves the floor is the cause.
+    lenient = _segmenter(FakeVad([True] * 5 + [False] * 2), silence_ms=60, rms_floor=0.0)
+    for _ in range(5):
+        lenient.push(make_speech_frame(frame_samples, value=1000))
+    lenient.push(make_silent_frame(frame_samples))
+    assert lenient.push(make_silent_frame(frame_samples)) is not None
+
+
+def test_trailing_silence_trimmed():
+    """Trailing silence is trimmed to the configured padding."""
+    frame_samples = 480
+    # 4 speech frames then 10 silence; silence_ms=600 keeps it speaking until flush.
+    seg = _segmenter(FakeVad([True] * 4 + [False] * 10))
+    for _ in range(4):
+        seg.push(make_speech_frame(frame_samples))
+    for _ in range(10):
+        assert seg.push(make_silent_frame(frame_samples)) is None
+
+    utt = seg.flush()
+    assert utt is not None
+    # 14 frames; trailing 10 silence trimmed to 7 kept -> 3 dropped
+    assert len(utt.audio) == (14 - 3) * frame_samples

tests/test_transcriber.py

@@ -0,0 +1,33 @@
+"""Tests for the Whisper segment confidence gate."""
+
+from stt_daemon.transcriber import accept_segment
+
+
+class _Seg:
+    def __init__(self, text="hello", no_speech_prob=0.1, avg_logprob=-0.3):
+        self.text = text
+        self.no_speech_prob = no_speech_prob
+        self.avg_logprob = avg_logprob
+
+
+def test_accepts_confident_speech():
+    assert accept_segment(_Seg(), 0.6, -1.0)
+
+
+def test_rejects_high_no_speech_prob():
+    assert not accept_segment(_Seg(no_speech_prob=0.9), 0.6, -1.0)
+
+
+def test_rejects_low_avg_logprob():
+    assert not accept_segment(_Seg(avg_logprob=-1.5), 0.6, -1.0)
+
+
+def test_threshold_boundary_is_accepted():
+    assert accept_segment(_Seg(no_speech_prob=0.6, avg_logprob=-1.0), 0.6, -1.0)
+
+
+def test_missing_attributes_default_to_accept():
+    class Bare:
+        text = "x"
+
+    assert accept_segment(Bare(), 0.6, -1.0)