import os from typing import Text import gradio as gr import soundfile as sf from transformers import pipeline import numpy as np import torch import re from speechbrain.pretrained import EncoderClassifier def create_speaker_embedding(speaker_model, waveform: np.ndarray) -> np.ndarray: with torch.no_grad(): speaker_embeddings = speaker_model.encode_batch(torch.tensor(waveform)) speaker_embeddings = torch.nn.functional.normalize(speaker_embeddings, dim=2) if device.type != 'cuda': speaker_embeddings = speaker_embeddings.squeeze().numpy() else: speaker_embeddings = speaker_embeddings.squeeze().cpu().numpy() speaker_embeddings = torch.tensor(speaker_embeddings, dtype=dtype).unsqueeze(0).to(device) return speaker_embeddings def remove_special_characters_s(text: Text) -> Text: chars_to_remove_regex = '[\=\´\–\“\”\…\=]' # remove special characters text = re.sub(chars_to_remove_regex, '', text) text = re.sub("‘", "'", text) text = re.sub("’", "'", text) text = re.sub("´", "'", text) text = text.lower() return text def dutch_to_english(text: Text) -> Text: replacements = [ ("à", "a"), ("ç", "c"), ("è", "e"), ("ë", "e"), ("í", "i"), ("ï", "i"), ("ö", "o"), ("ü", "u"), ('&', "en"), ('á','a'), ('ä','a'), ('î','i'), ('ó','o'), ('ö','o'), ('ú','u'), ('û','u'), ('ă','a'), ('ć','c'), ('đ','d'), ('š','s'), ('ţ','t'), ('j', 'y'), ('k', 'k'), ('ci', 'si'), ('ce', 'se'), ('ca', 'ka'), ('co', 'ko'), ('cu', 'ku'), (' sch', ' sg'), ('sch ', 's '), ('ch', 'g'), ('eeuw', 'eaw'), ('ee', 'ea'), ('aai','ay'), ('oei', 'ooy'), ('ooi', 'oay'), ('ieuw', 'eew'), ('ie', 'ee'), ('oo', 'oa'), ('oe', 'oo'), ('ei', '\\i\\'), ('ij', 'i'), ('\\i\\', 'i') ] for src, dst in replacements: text = text.replace(src, dst) return text device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') if torch.cuda.is_available(): dtype = torch.bfloat16 if torch.cuda.get_device_capability()[0] == 8 else torch.float16 else: dtype = torch.float32 spk_model_name = "speechbrain/spkrec-xvect-voxceleb" speaker_model = EncoderClassifier.from_hparams( source=spk_model_name, run_opts={"device": device}, savedir=os.path.join("/tmp", spk_model_name) ) waveform, samplerate = sf.read("files/speaker.wav") speaker_embeddings = create_speaker_embedding(speaker_model, waveform) transcriber = pipeline("text-to-speech", model="Oysiyl/speecht5_tts_common_voice_nl") def transcribe(text: Text) -> tuple((int, np.ndarray)): text = remove_special_characters_s(text) text = dutch_to_english(text) out = transcriber(text, forward_params={"speaker_embeddings": speaker_embeddings}) audio, sr = out["audio"], out["sampling_rate"] return sr, audio demo = gr.Interface( transcribe, gr.Textbox(), outputs="audio", title="Text to Speech for Dutch language demo", description="Click on the example below or type text!", examples=[["Goedenavond, ik kom uit Oekraïne!"], ["Hallo allemaal, ik praat nederlands. Groetjes aan iedereen!"]], cache_examples=True ) demo.launch()