feat: OpenVINO CPU MetricGAN mask; 100 ms hop
Per-worker OpenVINO CPU runs the MetricGAN BLSTM mask; SpeechBrain keeps STFT/ISTFT. ENHANCE_INFER=auto with torch fallback. Hop is 0.1 s after isolated last_ms ~6 ms (model+ov+beamform).
This commit is contained in:
+66
-14
@@ -65,7 +65,8 @@ class AudioCleaner:
|
||||
hop_s: float = 0.2,
|
||||
beamform: str = "auto",
|
||||
torch_num_threads: int = 1,
|
||||
tdoa_every: int = 5):
|
||||
tdoa_every: int = 5,
|
||||
enhance_infer: str = "auto"):
|
||||
self.mode = mode
|
||||
self.beamform_mode = beamform
|
||||
self.sample_rate = sample_rate
|
||||
@@ -87,6 +88,11 @@ class AudioCleaner:
|
||||
self._istft = None
|
||||
self._last_tdoas = None
|
||||
self._tdoa_age = 0
|
||||
self.enhance_infer = (enhance_infer or "auto").lower()
|
||||
self.infer_backend = "torch"
|
||||
self._ov_compiled = None
|
||||
self._ov_request = None
|
||||
self._ov_input = None
|
||||
|
||||
self._load_torch()
|
||||
if beamform == "force":
|
||||
@@ -107,12 +113,20 @@ class AudioCleaner:
|
||||
elif mode == "auto":
|
||||
self._load_model(model_source)
|
||||
|
||||
if self._enhancer is not None and self.enhance_infer != "torch":
|
||||
self._load_openvino()
|
||||
if self.enhance_infer == "openvino" and self._ov_compiled is None:
|
||||
raise RuntimeError(
|
||||
"ENHANCE_INFER=openvino but OpenVINO CPU mask failed to load")
|
||||
|
||||
self._refresh_backend()
|
||||
|
||||
def _refresh_backend(self) -> None:
|
||||
parts = []
|
||||
if self._enhancer is not None:
|
||||
parts.append("model")
|
||||
if self._ov_compiled is not None:
|
||||
parts.append("ov")
|
||||
if self._beamformer is not None:
|
||||
parts.append("beamform")
|
||||
self.backend = "+".join(parts) if parts else "off"
|
||||
@@ -188,6 +202,28 @@ class AudioCleaner:
|
||||
"(using fallback)", source, e)
|
||||
self._enhancer = None
|
||||
|
||||
def _load_openvino(self) -> None:
|
||||
"""Compile MetricGAN mask on OpenVINO CPU. STFT/ISTFT stay in Torch."""
|
||||
try:
|
||||
from ov_metricgan import compile_cpu, ensure_onnx
|
||||
except Exception as e: # noqa: BLE001
|
||||
log.info("openvino helper unavailable (%s)", e)
|
||||
return
|
||||
try:
|
||||
onnx_path = ensure_onnx(self._enhancer, self._torch)
|
||||
compiled, request, ov_input = compile_cpu(
|
||||
onnx_path, num_threads=self.torch_num_threads)
|
||||
self._ov_compiled = compiled
|
||||
self._ov_request = request
|
||||
self._ov_input = ov_input
|
||||
self.infer_backend = "openvino"
|
||||
except Exception as e: # noqa: BLE001
|
||||
log.warning("OpenVINO CPU mask failed (%s); using torch enhance", e)
|
||||
self._ov_compiled = None
|
||||
self._ov_request = None
|
||||
self._ov_input = None
|
||||
self.infer_backend = "torch"
|
||||
|
||||
# -- DSP fallback ------------------------------------------------------
|
||||
|
||||
@staticmethod
|
||||
@@ -258,6 +294,19 @@ class AudioCleaner:
|
||||
out = np.pad(out, (0, n - out.shape[0]))
|
||||
return out
|
||||
|
||||
def _enhance_openvino(self, w):
|
||||
"""w: torch [1, T] on CPU. Returns numpy [T]."""
|
||||
from ov_metricgan import infer_mask
|
||||
torch = self._torch
|
||||
with torch.no_grad():
|
||||
feats = self._enhancer.compute_features(w)
|
||||
feats_np = np.ascontiguousarray(feats.detach().cpu().numpy())
|
||||
mask_np = infer_mask(self._ov_request, self._ov_input, feats_np)
|
||||
mask = torch.from_numpy(mask_np)
|
||||
enhanced = torch.mul(mask, feats)
|
||||
wav = self._enhancer.hparams.resynth(torch.expm1(enhanced), w)
|
||||
return wav.squeeze(0).detach().cpu().numpy()
|
||||
|
||||
def _to_int16(self, y: np.ndarray) -> np.ndarray:
|
||||
return (np.clip(y, -1.0, 1.0) * 32767.0).astype(np.int16)
|
||||
|
||||
@@ -309,20 +358,23 @@ class AudioCleaner:
|
||||
w = torch.from_numpy(self._ctx.copy()).unsqueeze(0)
|
||||
if hasattr(self._enhancer, "device"):
|
||||
w = w.to(self._enhancer.device)
|
||||
# speechbrain >= 1.1 (e.g. metricGAN-plus) requires `lengths`
|
||||
# on enhance_batch. The value is RELATIVE (multiplied by the
|
||||
# number of frames internally), so 1.0 = "the whole sequence".
|
||||
# Older versions accept the call without it (TypeError fallback).
|
||||
lengths = torch.tensor([1.0]).to(w.device)
|
||||
try:
|
||||
with torch.no_grad():
|
||||
out = self._enhancer.enhance_batch(w, lengths=lengths)
|
||||
except TypeError:
|
||||
with torch.no_grad():
|
||||
out = self._enhancer.enhance_batch(w)
|
||||
enhanced = out.squeeze(0).detach().cpu().numpy()
|
||||
if self._ov_compiled is not None:
|
||||
enhanced = self._enhance_openvino(w)
|
||||
else:
|
||||
# speechbrain >= 1.1 (e.g. metricGAN-plus) requires `lengths`
|
||||
# on enhance_batch. The value is RELATIVE (multiplied by the
|
||||
# number of frames internally), so 1.0 = "the whole sequence".
|
||||
# Older versions accept the call without it (TypeError fallback).
|
||||
lengths = torch.tensor([1.0]).to(w.device)
|
||||
try:
|
||||
with torch.no_grad():
|
||||
out = self._enhancer.enhance_batch(w, lengths=lengths)
|
||||
except TypeError:
|
||||
with torch.no_grad():
|
||||
out = self._enhancer.enhance_batch(w)
|
||||
enhanced = out.squeeze(0).detach().cpu().numpy()
|
||||
except Exception as e: # noqa: BLE001
|
||||
log.warning("enhance_batch failed (%s); using fallback", e)
|
||||
log.warning("enhance failed (%s); using fallback", e)
|
||||
return self._to_int16(self._simple_clean(mono_hop))
|
||||
|
||||
hop_out = enhanced[-n:] if len(enhanced) >= n else np.pad(
|
||||
|
||||
Reference in New Issue
Block a user