diff --git a/configs/acoustic.yaml b/configs/acoustic.yaml index e49eb114..499cd3f2 100644 --- a/configs/acoustic.yaml +++ b/configs/acoustic.yaml @@ -47,6 +47,7 @@ mel_base: 'e' energy_smooth_width: 0.06 breathiness_smooth_width: 0.06 voicing_smooth_width: 0.06 +voicing_domain: 'mulaw' tension_smooth_width: 0.06 use_lang_id: false diff --git a/configs/templates/config_acoustic.yaml b/configs/templates/config_acoustic.yaml index 68dabb3a..a3582fb0 100644 --- a/configs/templates/config_acoustic.yaml +++ b/configs/templates/config_acoustic.yaml @@ -52,6 +52,7 @@ use_energy_embed: false use_breathiness_embed: false use_voicing_embed: false use_tension_embed: false +voicing_domain: 'mulaw' use_key_shift_embed: true use_speed_embed: true diff --git a/configs/templates/config_variance.yaml b/configs/templates/config_variance.yaml index 7c5af0ea..2f9eee2f 100644 --- a/configs/templates/config_variance.yaml +++ b/configs/templates/config_variance.yaml @@ -59,6 +59,7 @@ breathiness_db_max: -20.0 voicing_db_min: -96.0 voicing_db_max: -12.0 +voicing_domain: 'mulaw' tension_logit_min: -10.0 tension_logit_max: 10.0 diff --git a/configs/variance.yaml b/configs/variance.yaml index 2547e47a..b18517bb 100644 --- a/configs/variance.yaml +++ b/configs/variance.yaml @@ -89,6 +89,7 @@ breathiness_smooth_width: 0.06 voicing_db_min: -96.0 voicing_db_max: -12.0 voicing_smooth_width: 0.06 +voicing_domain: 'mulaw' tension_logit_min: -10.0 tension_logit_max: 10.0 diff --git a/docs/ConfigurationSchemas.md b/docs/ConfigurationSchemas.md index 449e9e3c..3ec18fb3 100644 --- a/docs/ConfigurationSchemas.md +++ b/docs/ConfigurationSchemas.md @@ -2284,6 +2284,24 @@ Minimum voicing value in dB used for normalization to [-1, 1]. Note that with [d default-96.0 +### voicing_domain + +Domain of the extracted voicing curve and of the RMS energy used to compute it. The following values are currently available: + +- `'db'`: the raw RMS of the harmonic part converted to decibels. +- `'mulaw'`: mu-law compression of the raw RMS values with `mu = 255`, rescaled to a `[-96, 0]` range. This keeps the values compatible with the dB-like range expected by the rest of the pipeline, while allocating more of the available range to low-energy frames, which gives finer resolution on unvoiced and weakly voiced segments. + +The value is read when extracting voicing during preprocessing, and by the variance parameter adaptor that builds the normalization range of the voicing branch. It must therefore stay consistent between the binarized dataset and the training/inference configuration, and switching it requires re-running binarization and training from scratch. When it is `'mulaw'`, the upper normalization bound of the voicing branch is clamped to `0` regardless of [voicing_db_max](#voicing_db_max). + + + + + + + + +
visibilityacoustic, variance
scopepreprocessing, training, inference
customizabilitynormal
typestr
defaultdb
constraintsChoose from 'db' or 'mulaw'.
+ ### voicing_smooth_width Length of sinusoidal smoothing convolution kernel (in seconds) on the extracted voicing curve. diff --git a/modules/fastspeech/param_adaptor.py b/modules/fastspeech/param_adaptor.py index 77ebb833..28735c5c 100644 --- a/modules/fastspeech/param_adaptor.py +++ b/modules/fastspeech/param_adaptor.py @@ -49,7 +49,7 @@ def build_adaptor(self, cls=MultiVarianceDiffusion): if self.predict_voicing: ranges.append(( hparams['voicing_db_min'], - hparams['voicing_db_max'] + 0. if hparams.get('voicing_domain', 'db') == 'mulaw' else hparams['voicing_db_max'] )) clamps.append((hparams['voicing_db_min'], 0.)) diff --git a/preprocessing/acoustic_binarizer.py b/preprocessing/acoustic_binarizer.py index dff5e679..ba8e04fc 100644 --- a/preprocessing/acoustic_binarizer.py +++ b/preprocessing/acoustic_binarizer.py @@ -193,7 +193,7 @@ def process_item(self, item_name, meta_data, binarization_args): if self.need_voicing: # get ground truth voicing voicing = get_voicing( - dec_waveform, None, None, length=length + dec_waveform, None, None, length=length, domain=hparams.get('voicing_domain', 'db') ) global voicing_smooth diff --git a/preprocessing/variance_binarizer.py b/preprocessing/variance_binarizer.py index a20160ca..59371904 100644 --- a/preprocessing/variance_binarizer.py +++ b/preprocessing/variance_binarizer.py @@ -492,7 +492,7 @@ def process_item(self, item_name, meta_data, binarization_args): ) if voicing is None: voicing = get_voicing( - dec_waveform, None, None, length=length + dec_waveform, None, None, length=length, domain=hparams.get('voicing_domain', 'db') ) voicing_from_wav = True diff --git a/utils/binarizer_utils.py b/utils/binarizer_utils.py index de11dd03..4d99ddf5 100644 --- a/utils/binarizer_utils.py +++ b/utils/binarizer_utils.py @@ -79,14 +79,15 @@ def get_pitch_parselmouth( return f0, uv -def get_energy_librosa(waveform, length, *, hop_size, win_size, domain='db'): +def get_energy_librosa(waveform, length, *, hop_size, win_size, domain='db', mu=255.0): """ Definition of energy: RMS of the waveform, in dB representation :param waveform: [T] :param length: Expected number of frames :param hop_size: Frame width, in number of samples :param win_size: Window size, in number of samples - :param domain: db or amplitude + :param domain: 'db', 'amplitude', or 'mulaw' + :param mu: mu parameter for mu-law compression :return: energy """ energy = librosa.feature.rms(y=waveform, frame_length=win_size, hop_length=hop_size)[0] @@ -97,6 +98,10 @@ def get_energy_librosa(waveform, length, *, hop_size, win_size, domain='db'): energy = librosa.amplitude_to_db(energy, top_db=None) elif domain == 'amplitude': pass + elif domain == 'mulaw': + energy = np.log1p(mu * energy) / np.log1p(mu) + # Since modifications to the API have been frozen, this approach is adopted for compatibility. + energy = energy * 96 - 96 else: raise ValueError(f'Invalid domain: {domain}') return energy @@ -134,7 +139,8 @@ def get_breathiness( def get_voicing( waveform: Union[np.ndarray, DecomposedWaveform], samplerate, f0, length, - *, hop_size=None, fft_size=None, win_size=None + *, hop_size=None, fft_size=None, win_size=None, + domain='db', mu=255.0 ): """ Definition of voicing: RMS of the harmonic part, in dB representation @@ -145,6 +151,8 @@ def get_voicing( :param hop_size: Frame width, in number of samples :param fft_size: Number of fft bins :param win_size: Window size, in number of samples + :param domain: 'db', 'amplitude', or 'mulaw' + :param mu: mu parameter for mu-law compression :return: voicing """ if not isinstance(waveform, DecomposedWaveform): @@ -155,7 +163,8 @@ def get_voicing( waveform_sp = waveform.harmonic() voicing = get_energy_librosa( waveform_sp, length=length, - hop_size=waveform.hop_size, win_size=waveform.win_size + hop_size=waveform.hop_size, win_size=waveform.win_size, + domain=domain, mu=mu ) return voicing