Create app.py

app.py

@@ -0,0 +1,125 @@
+import os
+from typing import Text
+import gradio as gr
+import soundfile as sf
+from transformers import pipeline
+import numpy as np
+import torch
+import re
+from speechbrain.pretrained import EncoderClassifier
+
+
+def create_speaker_embedding(speaker_model, waveform: np.ndarray) -> np.ndarray:
+    with torch.no_grad():
+        speaker_embeddings = speaker_model.encode_batch(torch.tensor(waveform))
+        speaker_embeddings = torch.nn.functional.normalize(speaker_embeddings, dim=2)
+        if device.type != 'cuda':
+            speaker_embeddings = speaker_embeddings.squeeze().numpy()
+        else:
+            speaker_embeddings = speaker_embeddings.squeeze().cpu().numpy()
+    speaker_embeddings = torch.tensor(speaker_embeddings, dtype=dtype).unsqueeze(0).to(device)
+    return speaker_embeddings
+
+
+def remove_special_characters_s(text: Text) -> Text:
+    chars_to_remove_regex = '[\=\´\–\“\”\…\=]'
+    # remove special characters
+    text = re.sub(chars_to_remove_regex, '', text).lower()
+    text = re.sub("‘", "'", text).lower()
+    text = re.sub("’", "'", text).lower()
+    text = re.sub("´", "'", text).lower()
+    text = text.lower()
+    return text
+
+
+def dutch_to_english(text: Text) -> Text:
+    replacements = [
+    ("à", "a"),
+    ("ç", "c"),
+    ("è", "e"),
+    ("ë", "e"),
+    ("í", "i"),
+    ("ï", "i"),
+    ("ö", "o"),
+    ("ü", "u"),
+    ('&', "en"),
+    ('á','a'),
+    ('ä','a'),
+    ('î','i'),
+    ('ó','o'),
+    ('ö','o'),
+    ('ú','u'),
+    ('û','u'),
+    ('ă','a'),
+    ('ć','c'),
+    ('đ','d'),
+    ('š','s'),
+    ('ţ','t'),
+    ('j', 'y'),
+    ('k', 'k'),
+    ('ci', 'si'),
+    ('ce', 'se'),
+    ('ca', 'ka'),
+    ('co', 'ko'),
+    ('cu', 'ku'),
+    (' sch', ' sg'),
+    ('sch ', 's '),
+    ('ch', 'g'),
+    ('eeuw', 'eaw'),
+    ('ee', 'ea'),
+    ('aai','ay'),
+    ('oei', 'ooy'),
+    ('ooi', 'oay'),
+    ('ieuw', 'eew'),
+    ('ie', 'ee'),
+    ('oo', 'oa'),
+    ('oe', 'oo'),
+    ('ei', '\\i\\'),
+    ('ij', 'i'),
+    ('\\i\\', 'i')
+    ]
+
+    for src, dst in replacements:
+        text = text.replace(src, dst)
+    return text
+
+
+device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+if torch.cuda.is_available():
+    dtype = torch.bfloat16 if torch.cuda.get_device_capability()[0] == 8 else torch.float16
+else:
+    dtype = torch.float32
+
+spk_model_name = "speechbrain/spkrec-xvect-voxceleb"
+
+speaker_model = EncoderClassifier.from_hparams(
+            source=spk_model_name, 
+            run_opts={"device": device}, 
+            savedir=os.path.join("/tmp", spk_model_name)
+            )
+
+waveform, samplerate = sf.read("files/speaker.wav")
+
+speaker_embeddings = create_speaker_embedding(speaker_model, waveform)
+
+transcriber = pipeline("text-to-speech", model="Oysiyl/speecht5_tts_common_voice_nl")
+
+def transcribe(text: Text) -> tuple((int, np.ndarray)):
+    text = remove_special_characters_s(text)
+    text = dutch_to_english(text)
+    out = transcriber(text, forward_params={"speaker_embeddings": speaker_embeddings})
+    audio, sr = out["audio"], out["sampling_rate"]
+    return sr, audio
+
+
+demo = gr.Interface(
+    transcribe,
+    gr.Textbox(),
+    outputs="audio",
+    title="Text to Speech for Dutch language demo",
+    description="Click on the example below or type text!",
+    examples=[["hallo allemaal, ik praat nederlands. groetjes aan iedereen"]],
+    cache_examples=True
+)
+
+demo.launch()