feat: OpenVINO CPU MetricGAN mask; 100 ms hop

Per-worker OpenVINO CPU runs the MetricGAN BLSTM mask; SpeechBrain
keeps STFT/ISTFT. ENHANCE_INFER=auto with torch fallback. Hop is
0.1 s after isolated last_ms ~6 ms (model+ov+beamform).
This commit is contained in:
root
2026-08-28 07:25:30 +00:00
parent 42443420be
commit ef9573c294
9 changed files with 211 additions and 26 deletions
+66 -14
View File
@@ -65,7 +65,8 @@ class AudioCleaner:
hop_s: float = 0.2,
beamform: str = "auto",
torch_num_threads: int = 1,
tdoa_every: int = 5):
tdoa_every: int = 5,
enhance_infer: str = "auto"):
self.mode = mode
self.beamform_mode = beamform
self.sample_rate = sample_rate
@@ -87,6 +88,11 @@ class AudioCleaner:
self._istft = None
self._last_tdoas = None
self._tdoa_age = 0
self.enhance_infer = (enhance_infer or "auto").lower()
self.infer_backend = "torch"
self._ov_compiled = None
self._ov_request = None
self._ov_input = None
self._load_torch()
if beamform == "force":
@@ -107,12 +113,20 @@ class AudioCleaner:
elif mode == "auto":
self._load_model(model_source)
if self._enhancer is not None and self.enhance_infer != "torch":
self._load_openvino()
if self.enhance_infer == "openvino" and self._ov_compiled is None:
raise RuntimeError(
"ENHANCE_INFER=openvino but OpenVINO CPU mask failed to load")
self._refresh_backend()
def _refresh_backend(self) -> None:
parts = []
if self._enhancer is not None:
parts.append("model")
if self._ov_compiled is not None:
parts.append("ov")
if self._beamformer is not None:
parts.append("beamform")
self.backend = "+".join(parts) if parts else "off"
@@ -188,6 +202,28 @@ class AudioCleaner:
"(using fallback)", source, e)
self._enhancer = None
def _load_openvino(self) -> None:
"""Compile MetricGAN mask on OpenVINO CPU. STFT/ISTFT stay in Torch."""
try:
from ov_metricgan import compile_cpu, ensure_onnx
except Exception as e: # noqa: BLE001
log.info("openvino helper unavailable (%s)", e)
return
try:
onnx_path = ensure_onnx(self._enhancer, self._torch)
compiled, request, ov_input = compile_cpu(
onnx_path, num_threads=self.torch_num_threads)
self._ov_compiled = compiled
self._ov_request = request
self._ov_input = ov_input
self.infer_backend = "openvino"
except Exception as e: # noqa: BLE001
log.warning("OpenVINO CPU mask failed (%s); using torch enhance", e)
self._ov_compiled = None
self._ov_request = None
self._ov_input = None
self.infer_backend = "torch"
# -- DSP fallback ------------------------------------------------------
@staticmethod
@@ -258,6 +294,19 @@ class AudioCleaner:
out = np.pad(out, (0, n - out.shape[0]))
return out
def _enhance_openvino(self, w):
"""w: torch [1, T] on CPU. Returns numpy [T]."""
from ov_metricgan import infer_mask
torch = self._torch
with torch.no_grad():
feats = self._enhancer.compute_features(w)
feats_np = np.ascontiguousarray(feats.detach().cpu().numpy())
mask_np = infer_mask(self._ov_request, self._ov_input, feats_np)
mask = torch.from_numpy(mask_np)
enhanced = torch.mul(mask, feats)
wav = self._enhancer.hparams.resynth(torch.expm1(enhanced), w)
return wav.squeeze(0).detach().cpu().numpy()
def _to_int16(self, y: np.ndarray) -> np.ndarray:
return (np.clip(y, -1.0, 1.0) * 32767.0).astype(np.int16)
@@ -309,20 +358,23 @@ class AudioCleaner:
w = torch.from_numpy(self._ctx.copy()).unsqueeze(0)
if hasattr(self._enhancer, "device"):
w = w.to(self._enhancer.device)
# speechbrain >= 1.1 (e.g. metricGAN-plus) requires `lengths`
# on enhance_batch. The value is RELATIVE (multiplied by the
# number of frames internally), so 1.0 = "the whole sequence".
# Older versions accept the call without it (TypeError fallback).
lengths = torch.tensor([1.0]).to(w.device)
try:
with torch.no_grad():
out = self._enhancer.enhance_batch(w, lengths=lengths)
except TypeError:
with torch.no_grad():
out = self._enhancer.enhance_batch(w)
enhanced = out.squeeze(0).detach().cpu().numpy()
if self._ov_compiled is not None:
enhanced = self._enhance_openvino(w)
else:
# speechbrain >= 1.1 (e.g. metricGAN-plus) requires `lengths`
# on enhance_batch. The value is RELATIVE (multiplied by the
# number of frames internally), so 1.0 = "the whole sequence".
# Older versions accept the call without it (TypeError fallback).
lengths = torch.tensor([1.0]).to(w.device)
try:
with torch.no_grad():
out = self._enhancer.enhance_batch(w, lengths=lengths)
except TypeError:
with torch.no_grad():
out = self._enhancer.enhance_batch(w)
enhanced = out.squeeze(0).detach().cpu().numpy()
except Exception as e: # noqa: BLE001
log.warning("enhance_batch failed (%s); using fallback", e)
log.warning("enhance failed (%s); using fallback", e)
return self._to_int16(self._simple_clean(mono_hop))
hop_out = enhanced[-n:] if len(enhanced) >= n else np.pad(