from abc import ABC import torch import torch.nn.functional as F from modules.diffusion_transformer import DiT from modules.commons import sequence_mask from tqdm import tqdm class BASECFM(torch.nn.Module, ABC): def __init__( self, args, ): super().__init__() self.sigma_min = 1e-6 self.estimator = None self.in_channels = args.DiT.in_channels self.criterion = torch.nn.MSELoss() if args.reg_loss_type == "l2" else torch.nn.L1Loss() if hasattr(args.DiT, 'zero_prompt_speech_token'): self.zero_prompt_speech_token = args.DiT.zero_prompt_speech_token else: self.zero_prompt_speech_token = False @torch.inference_mode() def inference(self, mu, x_lens, prompt, style, f0, n_timesteps, temperature=1.0, inference_cfg_rate=0.5): """Forward diffusion Args: mu (torch.Tensor): output of encoder shape: (batch_size, n_feats, mel_timesteps) mask (torch.Tensor): output_mask shape: (batch_size, 1, mel_timesteps) n_timesteps (int): number of diffusion steps temperature (float, optional): temperature for scaling noise. Defaults to 1.0. spks (torch.Tensor, optional): speaker ids. Defaults to None. shape: (batch_size, spk_emb_dim) cond: Not used but kept for future purposes Returns: sample: generated mel-spectrogram shape: (batch_size, n_feats, mel_timesteps) """ B, T = mu.size(0), mu.size(1) z = torch.randn([B, self.in_channels, T], device=mu.device) * temperature t_span = torch.linspace(0, 1, n_timesteps + 1, device=mu.device) return self.solve_euler(z, x_lens, prompt, mu, style, f0, t_span, inference_cfg_rate) def solve_euler(self, x, x_lens, prompt, mu, style, f0, t_span, inference_cfg_rate=0.5): """ Fixed euler solver for ODEs. Args: x (torch.Tensor): random noise t_span (torch.Tensor): n_timesteps interpolated shape: (n_timesteps + 1,) mu (torch.Tensor): output of encoder shape: (batch_size, n_feats, mel_timesteps) mask (torch.Tensor): output_mask shape: (batch_size, 1, mel_timesteps) spks (torch.Tensor, optional): speaker ids. Defaults to None. shape: (batch_size, spk_emb_dim) cond: Not used but kept for future purposes """ t, _, dt = t_span[0], t_span[-1], t_span[1] - t_span[0] # I am storing this because I can later plot it by putting a debugger here and saving it to a file # Or in future might add like a return_all_steps flag sol = [] # apply prompt prompt_len = prompt.size(-1) prompt_x = torch.zeros_like(x) prompt_x[..., :prompt_len] = prompt[..., :prompt_len] x[..., :prompt_len] = 0 if self.zero_prompt_speech_token: mu[..., :prompt_len] = 0 for step in tqdm(range(1, len(t_span))): dphi_dt = self.estimator(x, prompt_x, x_lens, t.unsqueeze(0), style, mu, f0) # Classifier-Free Guidance inference introduced in VoiceBox if inference_cfg_rate > 0: cfg_dphi_dt = self.estimator( x, torch.zeros_like(prompt_x), x_lens, t.unsqueeze(0), torch.zeros_like(style), torch.zeros_like(mu), None ) dphi_dt = ((1.0 + inference_cfg_rate) * dphi_dt - inference_cfg_rate * cfg_dphi_dt) x = x + dt * dphi_dt t = t + dt sol.append(x) if step < len(t_span) - 1: dt = t_span[step + 1] - t x[:, :, :prompt_len] = 0 return sol[-1] def forward(self, x1, x_lens, prompt_lens, mu, style, f0=None): """Computes diffusion loss Args: x1 (torch.Tensor): Target shape: (batch_size, n_feats, mel_timesteps) mask (torch.Tensor): target mask shape: (batch_size, 1, mel_timesteps) mu (torch.Tensor): output of encoder shape: (batch_size, n_feats, mel_timesteps) spks (torch.Tensor, optional): speaker embedding. Defaults to None. shape: (batch_size, spk_emb_dim) Returns: loss: conditional flow matching loss y: conditional flow shape: (batch_size, n_feats, mel_timesteps) """ b, _, t = x1.shape # random timestep t = torch.rand([b, 1, 1], device=mu.device, dtype=x1.dtype) # sample noise p(x_0) z = torch.randn_like(x1) y = (1 - (1 - self.sigma_min) * t) * z + t * x1 u = x1 - (1 - self.sigma_min) * z prompt = torch.zeros_like(x1) for bib in range(b): prompt[bib, :, :prompt_lens[bib]] = x1[bib, :, :prompt_lens[bib]] # range covered by prompt are set to 0 y[bib, :, :prompt_lens[bib]] = 0 if self.zero_prompt_speech_token: mu[bib, :, :prompt_lens[bib]] = 0 estimator_out = self.estimator(y, prompt, x_lens, t.squeeze(), style, mu, f0) loss = 0 for bib in range(b): loss += self.criterion(estimator_out[bib, :, prompt_lens[bib]:x_lens[bib]], u[bib, :, prompt_lens[bib]:x_lens[bib]]) loss /= b return loss, y class CFM(BASECFM): def __init__(self, args): super().__init__( args ) if args.dit_type == "DiT": self.estimator = DiT(args) else: raise NotImplementedError(f"Unknown diffusion type {args.dit_type}")