Antoine-caubriere commited on
Commit
b128f21
1 Parent(s): 39d1b9d

add notebook finetuning on FLEURS

Browse files
ASR_FLEURS-swahili_hf.yaml ADDED
@@ -0,0 +1,190 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Seed needs to be set at top of yaml, before objects with parameters are made
2
+ seed: 1986
3
+ __set_seed: !apply:torch.manual_seed [!ref <seed>]
4
+
5
+ lang_csv: Swahili
6
+
7
+ output_folder: !ref results/finetune_hubert_ASR_char/<seed>/<lang_csv>
8
+ output_wer_folder: !ref <output_folder>/
9
+ save_folder: !ref <output_folder>/save
10
+ train_log: !ref <output_folder>/train_log.txt
11
+
12
+ # huggingface format
13
+ hubert_hub: Orange/SSA-HuBERT-base-60k
14
+
15
+ hubert_folder: !ref <save_folder>/hubert_checkpoint
16
+
17
+ # Data files
18
+ data_folder: !ref PATH_TO_YOUR_FOLDER/data_speechbrain/<lang_csv>
19
+
20
+ ckpt_interval_minutes: 10 # save checkpoint every N min
21
+ train_csv: !ref <data_folder>/train.csv
22
+ valid_csv: !ref <data_folder>/validation.csv
23
+ test_csv:
24
+ - !ref <data_folder>/test.csv
25
+
26
+ ####################### Training Parameters ####################################
27
+
28
+ number_of_epochs: 10
29
+ lr: 0.1
30
+ lr_hubert: 0.000005
31
+ sorting: ascending
32
+ precision: fp32 # bf16, fp16 or fp32
33
+ sample_rate: 16000
34
+
35
+ # skip audio file longer than
36
+ avoid_if_longer_than: 60
37
+
38
+ batch_size: 2
39
+ test_batch_size: 2
40
+
41
+ # Dataloader options
42
+ train_dataloader_opts:
43
+ batch_size: !ref <batch_size>
44
+
45
+ valid_dataloader_opts:
46
+ batch_size: !ref <batch_size>
47
+
48
+ test_dataloader_opts:
49
+ batch_size: !ref <test_batch_size>
50
+
51
+ ####################### Model Parameters #######################################
52
+ activation: !name:torch.nn.LeakyReLU
53
+ dnn_layers: 2
54
+ dnn_neurons: 1024
55
+ freeze_hubert: False
56
+
57
+ # Outputs
58
+ output_neurons: 66 # BPE size, index(blank/eos/bos) = 0
59
+ blank_index: 0
60
+
61
+ #
62
+ # Functions and classes
63
+ #
64
+
65
+ label_encoder: !new:speechbrain.dataio.encoder.CTCTextEncoder
66
+
67
+ epoch_counter: !new:speechbrain.utils.epoch_loop.EpochCounter
68
+ limit: !ref <number_of_epochs>
69
+
70
+ hubert: !new:speechbrain.lobes.models.huggingface_transformers.hubert.HuBERT
71
+ source: !ref <hubert_hub>
72
+ output_norm: True
73
+ freeze: !ref <freeze_hubert>
74
+ save_path: !ref <hubert_folder>
75
+
76
+ top_lin: !new:speechbrain.lobes.models.VanillaNN.VanillaNN
77
+ input_shape: [null, null, 768] # 768 == output of hubert base model
78
+ activation: !ref <activation>
79
+ dnn_blocks: !ref <dnn_layers>
80
+ dnn_neurons: !ref <dnn_neurons>
81
+
82
+ ctc_lin: !new:speechbrain.nnet.linear.Linear
83
+ input_size: !ref <dnn_neurons>
84
+ n_neurons: !ref <output_neurons>
85
+
86
+ log_softmax: !new:speechbrain.nnet.activations.Softmax
87
+ apply_log: True
88
+
89
+ ctc_cost: !name:speechbrain.nnet.losses.ctc_loss
90
+ blank_index: !ref <blank_index>
91
+
92
+ modules:
93
+ hubert: !ref <hubert>
94
+ top_lin: !ref <top_lin>
95
+ ctc_lin: !ref <ctc_lin>
96
+
97
+ model: !new:torch.nn.ModuleList
98
+ - [!ref <top_lin>, !ref <ctc_lin>]
99
+
100
+ model_opt_class: !name:torch.optim.Adadelta
101
+ lr: !ref <lr>
102
+ rho: 0.95
103
+ eps: 1.e-8
104
+
105
+ hubert_opt_class: !name:torch.optim.Adam
106
+ lr: !ref <lr_hubert>
107
+
108
+ lr_annealing_model: !new:speechbrain.nnet.schedulers.NewBobScheduler
109
+ initial_value: !ref <lr>
110
+ improvement_threshold: 0.0025
111
+ annealing_factor: 0.8
112
+ patient: 0
113
+
114
+ lr_annealing_hubert: !new:speechbrain.nnet.schedulers.NewBobScheduler
115
+ initial_value: !ref <lr_hubert>
116
+ improvement_threshold: 0.0025
117
+ annealing_factor: 0.9
118
+ patient: 0
119
+
120
+ ############################## Augmentations ###################################
121
+
122
+ # Speed perturbation
123
+ speed_perturb: !new:speechbrain.augment.time_domain.SpeedPerturb
124
+ orig_freq: !ref <sample_rate>
125
+ speeds: [95, 100, 105]
126
+
127
+ # Frequency drop: randomly drops a number of frequency bands to zero.
128
+ drop_freq: !new:speechbrain.augment.time_domain.DropFreq
129
+ drop_freq_low: 0
130
+ drop_freq_high: 1
131
+ drop_freq_count_low: 1
132
+ drop_freq_count_high: 3
133
+ drop_freq_width: 0.05
134
+
135
+ # Time drop: randomly drops a number of temporal chunks.
136
+ drop_chunk: !new:speechbrain.augment.time_domain.DropChunk
137
+ drop_length_low: 1000
138
+ drop_length_high: 2000
139
+ drop_count_low: 1
140
+ drop_count_high: 5
141
+
142
+ # Augmenter: Combines previously defined augmentations to perform data augmentation
143
+ wav_augment: !new:speechbrain.augment.augmenter.Augmenter
144
+ concat_original: True
145
+ min_augmentations: 4
146
+ max_augmentations: 4
147
+ augment_prob: 1.0
148
+ augmentations: [
149
+ !ref <speed_perturb>,
150
+ !ref <drop_freq>,
151
+ !ref <drop_chunk>]
152
+
153
+ ############################## Decoding ########################################
154
+
155
+ # Decoding parameters
156
+ test_beam_search:
157
+ beam_size: 143
158
+ topk: 1
159
+ blank_index: !ref <blank_index>
160
+ space_token: ' ' # make sure this is the same as the one used in the tokenizer
161
+ beam_prune_logp: -12.0
162
+ token_prune_min_logp: -1.20
163
+ prune_history: True
164
+ alpha: 0.8
165
+ beta: 1.2
166
+ # can be downloaded from here https://www.openslr.org/11/ or trained with kenLM
167
+ # It can either be a .bin or .arpa ; note: .arpa is much slower at loading
168
+ # If you don't want to use an LM, comment it out or set it to null
169
+ kenlm_model_path: null
170
+
171
+ ############################## Logging and Pretrainer ##########################
172
+
173
+ checkpointer: !new:speechbrain.utils.checkpoints.Checkpointer
174
+ checkpoints_dir: !ref <save_folder>
175
+ recoverables:
176
+ hubert: !ref <hubert>
177
+ model: !ref <model>
178
+ scheduler_model: !ref <lr_annealing_model>
179
+ scheduler_hubert: !ref <lr_annealing_hubert>
180
+ counter: !ref <epoch_counter>
181
+ tokenizer: !ref <label_encoder>
182
+
183
+ train_logger: !new:speechbrain.utils.train_logger.FileTrainLogger
184
+ save_file: !ref <train_log>
185
+
186
+ error_rate_computer: !name:speechbrain.utils.metric_stats.ErrorRateStats
187
+
188
+ cer_computer: !name:speechbrain.utils.metric_stats.ErrorRateStats
189
+ split_tokens: True
190
+
SB_ASR_finetuning_notebook.ipynb ADDED
@@ -0,0 +1,689 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "cells": [
3
+ {
4
+ "cell_type": "markdown",
5
+ "id": "49b85514-0fb6-49c6-be76-259bfeb638c6",
6
+ "metadata": {},
7
+ "source": [
8
+ "# Introduction\n",
9
+ "N'hésitez pas à nous contacter en cas de questions : antoine.caubriere@orange.com & elodie.gauthier@orange.com\n",
10
+ "\n",
11
+ "Pensez à modifier l'ensemble des PATH dans le fichier de configuration ASR_FLEURS-swahili_hf.yaml et dans le code python ci-dessous (PATH_TO_YOUR_FOLDER).\n",
12
+ "\n",
13
+ "Dans le cas d'un changement de corpus (autre sous partie de FLEURS / vos propres jeux de données), pensez à modifier la taille de la couche de sortie du modèle : ASR_swahili_hf.yaml/output_neurons\n"
14
+ ]
15
+ },
16
+ {
17
+ "cell_type": "markdown",
18
+ "id": "e62faa86-911a-48ce-82bc-8a34e13ffbc4",
19
+ "metadata": {},
20
+ "source": [
21
+ "# Préparation des données FLEURS"
22
+ ]
23
+ },
24
+ {
25
+ "cell_type": "markdown",
26
+ "id": "c6ccf4a5-cad1-4632-8954-f4e454ff3540",
27
+ "metadata": {},
28
+ "source": [
29
+ "### 1. Installation des dépendances"
30
+ ]
31
+ },
32
+ {
33
+ "cell_type": "code",
34
+ "execution_count": null,
35
+ "id": "7bb8b44e-826f-4f13-b128-eebbd18dedc5",
36
+ "metadata": {
37
+ "jupyter": {
38
+ "source_hidden": true
39
+ }
40
+ },
41
+ "outputs": [],
42
+ "source": [
43
+ "pip install datasets librosa soundfile"
44
+ ]
45
+ },
46
+ {
47
+ "cell_type": "markdown",
48
+ "id": "016d7646-bcca-4422-8b28-9d12d4b86c8f",
49
+ "metadata": {},
50
+ "source": [
51
+ "### 2. Téléchargement et formatage du dataset"
52
+ ]
53
+ },
54
+ {
55
+ "cell_type": "code",
56
+ "execution_count": null,
57
+ "id": "da273973-05ee-4de5-830e-34d7f2220353",
58
+ "metadata": {},
59
+ "outputs": [],
60
+ "source": [
61
+ "from datasets import load_dataset\n",
62
+ "from pathlib import Path\n",
63
+ "from collections import OrderedDict\n",
64
+ "from tqdm import tqdm\n",
65
+ "import shutil\n",
66
+ "import os\n",
67
+ "\n",
68
+ "dataset_write_base = \"PATH_TO_YOUR_FOLDER/data_speechbrain\"\n",
69
+ "cache_dir = \"PATH_TO_YOUR_FOLDER/data_huggingface\"\n",
70
+ "\n",
71
+ "if os.path.isdir(cache_dir):\n",
72
+ " print(\"rm -rf \"+cache_dir)\n",
73
+ " os.system(\"rm -rf \"+cache_dir)\n",
74
+ "\n",
75
+ "if os.path.isdir(dataset_write_base):\n",
76
+ " print(\"rm -rf \"+dataset_write_base)\n",
77
+ " os.system(\"rm -rf \"+dataset_write_base)\n",
78
+ "\n",
79
+ "# **************************************\n",
80
+ "# choix des langues à extraire de FLEURS\n",
81
+ "# **************************************\n",
82
+ "lang_dict = OrderedDict([\n",
83
+ " #(\"Afrikaans\",\"af_za\"),\n",
84
+ " #(\"Amharic\", \"am_et\"),\n",
85
+ " #(\"Fula\", \"ff_sn\"),\n",
86
+ " #(\"Ganda\", \"lg_ug\"),\n",
87
+ " #(\"Hausa\", \"ha_ng\"),\n",
88
+ " #(\"Igbo\", \"ig_ng\"),\n",
89
+ " #(\"Kamba\", \"kam_ke\"),\n",
90
+ " #(\"Lingala\", \"ln_cd\"),\n",
91
+ " #(\"Luo\", \"luo_ke\"),\n",
92
+ " #(\"Northern-Sotho\", \"nso_za\"),\n",
93
+ " #(\"Nyanja\", \"ny_mw\"),\n",
94
+ " #(\"Oromo\", \"om_et\"),\n",
95
+ " #(\"Shona\", \"sn_zw\"),\n",
96
+ " #(\"Somali\", \"so_so\"),\n",
97
+ " (\"Swahili\", \"sw_ke\"),\n",
98
+ " #(\"Umbundu\", \"umb_ao\"),\n",
99
+ " #(\"Wolof\", \"wo_sn\"), \n",
100
+ " #(\"Xhosa\", \"xh_za\"), \n",
101
+ " #(\"Yoruba\", \"yo_ng\"), \n",
102
+ " #(\"Zulu\", \"zu_za\")\n",
103
+ " ])\n",
104
+ "\n",
105
+ "# ********************************\n",
106
+ "# choix des sous-parties à traiter\n",
107
+ "# ********************************\n",
108
+ "datasets = [\"train\",\"test\",\"validation\"]\n",
109
+ "\n",
110
+ "for lang in lang_dict:\n",
111
+ " print(\"Prepare --->\", lang)\n",
112
+ " \n",
113
+ " # ********************************\n",
114
+ " # Download FLEURS from huggingface\n",
115
+ " # ********************************\n",
116
+ " fleurs_asr = load_dataset(\"google/fleurs\", lang_dict[lang],cache_dir=cache_dir, trust_remote_code=True)\n",
117
+ "\n",
118
+ " for subparts in datasets:\n",
119
+ " \n",
120
+ " used_ID = []\n",
121
+ " Path(dataset_write_base+\"/\"+lang+\"/wavs/\"+subparts).mkdir(parents=True, exist_ok=True)\n",
122
+ " \n",
123
+ " # csv header\n",
124
+ " f = open(dataset_write_base+\"/\"+lang+\"/\"+subparts+\".csv\", \"w\")\n",
125
+ " f.write(\"ID,duration,wav,spk_id,wrd\\n\")\n",
126
+ "\n",
127
+ " for uid in tqdm(range(len(fleurs_asr[subparts]))):\n",
128
+ "\n",
129
+ " # ***************\n",
130
+ " # format CSV line\n",
131
+ " # ***************\n",
132
+ " text_id = lang+\"_\"+str(fleurs_asr[subparts][uid][\"id\"])\n",
133
+ " \n",
134
+ " # some ID are duplicated (same speaker, same transcription BUT different recording)\n",
135
+ " while(text_id in used_ID):\n",
136
+ " text_id += \"_bis\"\n",
137
+ " used_ID.append(text_id)\n",
138
+ "\n",
139
+ " duration = \"{:.3f}\".format(round(float(fleurs_asr[subparts][uid][\"num_samples\"])/float(fleurs_asr[subparts][uid][\"audio\"][\"sampling_rate\"]),3))\n",
140
+ " wav_path = \"/\".join([dataset_write_base, lang, \"wavs\",subparts, fleurs_asr[subparts][uid][\"audio\"][\"path\"].split('/')[-1]])\n",
141
+ " spk_id = \"spk_\" + text_id\n",
142
+ " # AC : \"pseudo-normalisation\" de cas marginaux -- TODO mieux\n",
143
+ " wrd = fleurs_asr[subparts][uid][\"transcription\"].replace(',','').replace('$',' $ ').replace('\"','').replace('”','').replace(' ',' ')\n",
144
+ "\n",
145
+ " # **************\n",
146
+ " # write CSV line\n",
147
+ " # **************\n",
148
+ " f.write(text_id+\",\"+duration+\",\"+wav_path+\",\"+spk_id+\",\"+wrd+\"\\n\") \n",
149
+ "\n",
150
+ " # *******************\n",
151
+ " # Move wav from cache\n",
152
+ " # *******************\n",
153
+ " previous_path = \"/\".join(fleurs_asr[subparts][uid][\"path\"].split('/')[:-1]) + \"/\" + fleurs_asr[subparts][uid][\"audio\"][\"path\"]\n",
154
+ " new_path = \"/\".join([dataset_write_base,lang,\"wavs\",subparts,fleurs_asr[subparts][uid][\"audio\"][\"path\"].split('/')[-1]])\n",
155
+ " shutil.move(previous_path,new_path)\n",
156
+ " \n",
157
+ " f.close()\n",
158
+ " print(\"--->\", lang, \"done\")"
159
+ ]
160
+ },
161
+ {
162
+ "cell_type": "markdown",
163
+ "id": "4c32e369-f0f9-4695-8c9a-aa3a9de7bf7b",
164
+ "metadata": {},
165
+ "source": [
166
+ "# Recette ASR"
167
+ ]
168
+ },
169
+ {
170
+ "cell_type": "markdown",
171
+ "id": "77fb2c55-3f8c-4f34-81f0-ad48a632e010",
172
+ "metadata": {
173
+ "jp-MarkdownHeadingCollapsed": true
174
+ },
175
+ "source": [
176
+ "## 1. Installation des dépendances"
177
+ ]
178
+ },
179
+ {
180
+ "cell_type": "code",
181
+ "execution_count": null,
182
+ "id": "fbe25635-e765-480c-8416-c48a31ee6140",
183
+ "metadata": {},
184
+ "outputs": [],
185
+ "source": [
186
+ "pip install torch==2.2.2 torchaudio==2.2.2 torchvision==0.17.2 speechbrain transformers jdc"
187
+ ]
188
+ },
189
+ {
190
+ "cell_type": "markdown",
191
+ "id": "6acf1f8c-2cf3-4c9c-8a45-e2580ecbee27",
192
+ "metadata": {},
193
+ "source": [
194
+ "## 2. Mise en place de la recette Speechbrain -- class Brain"
195
+ ]
196
+ },
197
+ {
198
+ "cell_type": "markdown",
199
+ "id": "d5e8884d-3542-40ff-a454-597078fcf97c",
200
+ "metadata": {},
201
+ "source": [
202
+ "### 2.1 Imports & logger"
203
+ ]
204
+ },
205
+ {
206
+ "cell_type": "code",
207
+ "execution_count": null,
208
+ "id": "6c677f9f-6abe-423f-b4dd-fdf5ded357cd",
209
+ "metadata": {},
210
+ "outputs": [],
211
+ "source": [
212
+ "import logging\n",
213
+ "import os\n",
214
+ "import sys\n",
215
+ "from pathlib import Path\n",
216
+ "\n",
217
+ "import torch\n",
218
+ "from hyperpyyaml import load_hyperpyyaml\n",
219
+ "\n",
220
+ "import speechbrain as sb\n",
221
+ "from speechbrain.utils.distributed import if_main_process, run_on_main\n",
222
+ "\n",
223
+ "import jdc\n",
224
+ "\n",
225
+ "logger = logging.getLogger(__name__)"
226
+ ]
227
+ },
228
+ {
229
+ "cell_type": "markdown",
230
+ "id": "9698bb92-16ad-4b61-8938-c74b62ee93b2",
231
+ "metadata": {},
232
+ "source": [
233
+ "### 2.2 Création de notre classe héritant de la classe brain"
234
+ ]
235
+ },
236
+ {
237
+ "cell_type": "code",
238
+ "execution_count": null,
239
+ "id": "7c7cd624-6249-449b-8ee9-d4a73b7b3301",
240
+ "metadata": {},
241
+ "outputs": [],
242
+ "source": [
243
+ "# Define training procedure\n",
244
+ "class MY_SSA_ASR(sb.Brain):\n",
245
+ " print(\"\")\n",
246
+ " # define here"
247
+ ]
248
+ },
249
+ {
250
+ "cell_type": "markdown",
251
+ "id": "ecf31c9c-15dd-4428-aa10-b3cc5e127f0d",
252
+ "metadata": {},
253
+ "source": [
254
+ "### 2.3 Définition de la fonction forward "
255
+ ]
256
+ },
257
+ {
258
+ "cell_type": "code",
259
+ "execution_count": null,
260
+ "id": "4368b488-b9d8-49ff-8ce3-78a12d46be83",
261
+ "metadata": {},
262
+ "outputs": [],
263
+ "source": [
264
+ "%%add_to MY_SSA_ASR\n",
265
+ "def compute_forward(self, batch, stage):\n",
266
+ " \"\"\"Forward computations from the waveform batches to the output probabilities.\"\"\"\n",
267
+ " batch = batch.to(self.device)\n",
268
+ " wavs, wav_lens = batch.sig\n",
269
+ " wavs, wav_lens = wavs.to(self.device), wav_lens.to(self.device)\n",
270
+ "\n",
271
+ " # Downsample the inputs if specified\n",
272
+ " if hasattr(self.modules, \"downsampler\"):\n",
273
+ " wavs = self.modules.downsampler(wavs)\n",
274
+ "\n",
275
+ " # Add waveform augmentation if specified.\n",
276
+ " if stage == sb.Stage.TRAIN and hasattr(self.hparams, \"wav_augment\"):\n",
277
+ " wavs, wav_lens = self.hparams.wav_augment(wavs, wav_lens)\n",
278
+ "\n",
279
+ " # Forward pass\n",
280
+ " feats = self.modules.hubert(wavs, wav_lens)\n",
281
+ " x = self.modules.top_lin(feats)\n",
282
+ "\n",
283
+ " # Compute outputs\n",
284
+ " logits = self.modules.ctc_lin(x)\n",
285
+ " p_ctc = self.hparams.log_softmax(logits)\n",
286
+ "\n",
287
+ "\n",
288
+ " p_tokens = None\n",
289
+ " if stage == sb.Stage.VALID:\n",
290
+ " p_tokens = sb.decoders.ctc_greedy_decode(p_ctc, wav_lens, blank_id=self.hparams.blank_index)\n",
291
+ "\n",
292
+ " elif stage == sb.Stage.TEST:\n",
293
+ " p_tokens = test_searcher(p_ctc, wav_lens)\n",
294
+ "\n",
295
+ " candidates = []\n",
296
+ " scores = []\n",
297
+ "\n",
298
+ " for batch in p_tokens:\n",
299
+ " candidates.append([hyp.text for hyp in batch])\n",
300
+ " scores.append([hyp.score for hyp in batch])\n",
301
+ "\n",
302
+ " if hasattr(self.hparams, \"rescorer\"):\n",
303
+ " p_tokens, _ = self.hparams.rescorer.rescore(candidates, scores)\n",
304
+ "\n",
305
+ " return p_ctc, wav_lens, p_tokens\n"
306
+ ]
307
+ },
308
+ {
309
+ "cell_type": "markdown",
310
+ "id": "f0052b79-5a27-4c4c-8601-7ab064e8c951",
311
+ "metadata": {},
312
+ "source": [
313
+ "### 2.4 Définition de la fonction objectives"
314
+ ]
315
+ },
316
+ {
317
+ "cell_type": "code",
318
+ "execution_count": null,
319
+ "id": "3608aee8-c9c3-4e34-98bc-667513fa7f7b",
320
+ "metadata": {},
321
+ "outputs": [],
322
+ "source": [
323
+ "%%add_to MY_SSA_ASR\n",
324
+ "def compute_objectives(self, predictions, batch, stage):\n",
325
+ " \"\"\"Computes the loss (CTC+NLL) given predictions and targets.\"\"\"\n",
326
+ "\n",
327
+ " p_ctc, wav_lens, predicted_tokens = predictions\n",
328
+ "\n",
329
+ " ids = batch.id\n",
330
+ " tokens, tokens_lens = batch.tokens\n",
331
+ "\n",
332
+ " # Labels must be extended if parallel augmentation or concatenated\n",
333
+ " # augmentation was performed on the input (increasing the time dimension)\n",
334
+ " if stage == sb.Stage.TRAIN and hasattr(self.hparams, \"wav_augment\"):\n",
335
+ " (tokens, tokens_lens) = self.hparams.wav_augment.replicate_multiple_labels(tokens, tokens_lens)\n",
336
+ "\n",
337
+ "\n",
338
+ "\n",
339
+ " # Compute loss\n",
340
+ " loss = self.hparams.ctc_cost(p_ctc, tokens, wav_lens, tokens_lens)\n",
341
+ "\n",
342
+ " if stage == sb.Stage.VALID:\n",
343
+ " # Decode token terms to words\n",
344
+ " predicted_words = [\"\".join(self.tokenizer.decode_ndim(utt_seq)).split(\" \") for utt_seq in predicted_tokens]\n",
345
+ " \n",
346
+ " elif stage == sb.Stage.TEST:\n",
347
+ " predicted_words = [hyp[0].text.split(\" \") for hyp in predicted_tokens]\n",
348
+ "\n",
349
+ " if stage != sb.Stage.TRAIN:\n",
350
+ " target_words = [wrd.split(\" \") for wrd in batch.wrd]\n",
351
+ " self.wer_metric.append(ids, predicted_words, target_words)\n",
352
+ " self.cer_metric.append(ids, predicted_words, target_words)\n",
353
+ "\n",
354
+ " return loss\n"
355
+ ]
356
+ },
357
+ {
358
+ "cell_type": "markdown",
359
+ "id": "9a514c50-89ad-41cb-882a-23daf829a538",
360
+ "metadata": {},
361
+ "source": [
362
+ "### 2.5 définition du comportement au début d'un \"stage\""
363
+ ]
364
+ },
365
+ {
366
+ "cell_type": "code",
367
+ "execution_count": null,
368
+ "id": "609814ce-3ef0-4818-a70f-cadc293c9dd2",
369
+ "metadata": {},
370
+ "outputs": [],
371
+ "source": [
372
+ "%%add_to MY_SSA_ASR\n",
373
+ "# stage gestion\n",
374
+ "def on_stage_start(self, stage, epoch):\n",
375
+ " \"\"\"Gets called at the beginning of each epoch\"\"\"\n",
376
+ " if stage != sb.Stage.TRAIN:\n",
377
+ " self.cer_metric = self.hparams.cer_computer()\n",
378
+ " self.wer_metric = self.hparams.error_rate_computer()\n",
379
+ "\n",
380
+ " if stage == sb.Stage.TEST:\n",
381
+ " if hasattr(self.hparams, \"rescorer\"):\n",
382
+ " self.hparams.rescorer.move_rescorers_to_device()\n",
383
+ "\n"
384
+ ]
385
+ },
386
+ {
387
+ "cell_type": "markdown",
388
+ "id": "55929209-c94a-4f8b-8f2e-9dd5d9de8be9",
389
+ "metadata": {},
390
+ "source": [
391
+ "### 2.6 définition du comportement à la fin d'un \"stage\""
392
+ ]
393
+ },
394
+ {
395
+ "cell_type": "code",
396
+ "execution_count": null,
397
+ "id": "8f297542-10d5-47bf-9938-c141f5a99ab8",
398
+ "metadata": {},
399
+ "outputs": [],
400
+ "source": [
401
+ "%%add_to MY_SSA_ASR\n",
402
+ "def on_stage_end(self, stage, stage_loss, epoch):\n",
403
+ " \"\"\"Gets called at the end of an epoch.\"\"\"\n",
404
+ " # Compute/store important stats\n",
405
+ " stage_stats = {\"loss\": stage_loss}\n",
406
+ " if stage == sb.Stage.TRAIN:\n",
407
+ " self.train_stats = stage_stats\n",
408
+ " else:\n",
409
+ " stage_stats[\"CER\"] = self.cer_metric.summarize(\"error_rate\")\n",
410
+ " stage_stats[\"WER\"] = self.wer_metric.summarize(\"error_rate\")\n",
411
+ "\n",
412
+ " # Perform end-of-iteration things, like annealing, logging, etc.\n",
413
+ " if stage == sb.Stage.VALID:\n",
414
+ " # *******************************\n",
415
+ " # Anneal and update Learning Rate\n",
416
+ " # *******************************\n",
417
+ " old_lr_model, new_lr_model = self.hparams.lr_annealing_model(stage_stats[\"loss\"])\n",
418
+ " old_lr_hubert, new_lr_hubert = self.hparams.lr_annealing_hubert(stage_stats[\"loss\"])\n",
419
+ " sb.nnet.schedulers.update_learning_rate(self.model_optimizer, new_lr_model)\n",
420
+ " sb.nnet.schedulers.update_learning_rate(self.hubert_optimizer, new_lr_hubert)\n",
421
+ "\n",
422
+ " # *****************\n",
423
+ " # Logs informations\n",
424
+ " # *****************\n",
425
+ " self.hparams.train_logger.log_stats(stats_meta={\"epoch\": epoch, \"lr_model\": old_lr_model, \"lr_hubert\": old_lr_hubert}, train_stats=self.train_stats, valid_stats=stage_stats)\n",
426
+ "\n",
427
+ " # ***************\n",
428
+ " # Save checkpoint\n",
429
+ " # ***************\n",
430
+ " self.checkpointer.save_and_keep_only(meta={\"WER\": stage_stats[\"WER\"]},min_keys=[\"WER\"])\n",
431
+ "\n",
432
+ " elif stage == sb.Stage.TEST:\n",
433
+ " self.hparams.train_logger.log_stats(stats_meta={\"Epoch loaded\": self.hparams.epoch_counter.current},test_stats=stage_stats)\n",
434
+ " if if_main_process():\n",
435
+ " with open(self.hparams.test_wer_file, \"w\") as w:\n",
436
+ " self.wer_metric.write_stats(w)\n"
437
+ ]
438
+ },
439
+ {
440
+ "cell_type": "markdown",
441
+ "id": "0c656457-6b61-4316-8199-70021f92babf",
442
+ "metadata": {},
443
+ "source": [
444
+ "### 2.7 définition de l'initialisation des optimizers"
445
+ ]
446
+ },
447
+ {
448
+ "cell_type": "code",
449
+ "execution_count": null,
450
+ "id": "da8d9cb5-c5ad-4e78-83d3-e129e138a741",
451
+ "metadata": {},
452
+ "outputs": [],
453
+ "source": [
454
+ "%%add_to MY_SSA_ASR\n",
455
+ "def init_optimizers(self):\n",
456
+ " \"Initializes the hubert optimizer and model optimizer\"\n",
457
+ " self.hubert_optimizer = self.hparams.hubert_opt_class(self.modules.hubert.parameters())\n",
458
+ " self.model_optimizer = self.hparams.model_opt_class(self.hparams.model.parameters())\n",
459
+ "\n",
460
+ " # save the optimizers in a dictionary\n",
461
+ " # the key will be used in `freeze_optimizers()`\n",
462
+ " self.optimizers_dict = {\"model_optimizer\": self.model_optimizer}\n",
463
+ " if not self.hparams.freeze_hubert:\n",
464
+ " self.optimizers_dict[\"hubert_optimizer\"] = self.hubert_optimizer\n",
465
+ "\n",
466
+ " if self.checkpointer is not None:\n",
467
+ " self.checkpointer.add_recoverable(\"hubert_opt\", self.hubert_optimizer)\n",
468
+ " self.checkpointer.add_recoverable(\"model_opt\", self.model_optimizer)\n"
469
+ ]
470
+ },
471
+ {
472
+ "cell_type": "markdown",
473
+ "id": "cf2e730c-2faa-41f2-b98d-e5fbb2305cc2",
474
+ "metadata": {},
475
+ "source": [
476
+ "## 3 Définition de la lecture des datasets"
477
+ ]
478
+ },
479
+ {
480
+ "cell_type": "code",
481
+ "execution_count": null,
482
+ "id": "c5e667f7-6269-4b49-88bb-5e431762c8fe",
483
+ "metadata": {},
484
+ "outputs": [],
485
+ "source": [
486
+ "def dataio_prepare(hparams):\n",
487
+ " \"\"\"This function prepares the datasets to be used in the brain class.\n",
488
+ " It also defines the data processing pipeline through user-defined functions.\n",
489
+ " \"\"\"\n",
490
+ "\n",
491
+ " # **************\n",
492
+ " # Load CSV files\n",
493
+ " # **************\n",
494
+ " data_folder = hparams[\"data_folder\"]\n",
495
+ "\n",
496
+ " train_data = sb.dataio.dataset.DynamicItemDataset.from_csv(csv_path=hparams[\"train_csv\"],replacements={\"data_root\": data_folder})\n",
497
+ " # we sort training data to speed up training and get better results.\n",
498
+ " train_data = train_data.filtered_sorted(sort_key=\"duration\")\n",
499
+ " hparams[\"train_dataloader_opts\"][\"shuffle\"] = False # when sorting do not shuffle in dataloader ! otherwise is pointless\n",
500
+ "\n",
501
+ " valid_data = sb.dataio.dataset.DynamicItemDataset.from_csv(csv_path=hparams[\"valid_csv\"],replacements={\"data_root\": data_folder})\n",
502
+ " valid_data = valid_data.filtered_sorted(sort_key=\"duration\")\n",
503
+ "\n",
504
+ " # test is separate\n",
505
+ " test_datasets = {}\n",
506
+ " for csv_file in hparams[\"test_csv\"]:\n",
507
+ " name = Path(csv_file).stem\n",
508
+ " test_datasets[name] = sb.dataio.dataset.DynamicItemDataset.from_csv(csv_path=csv_file, replacements={\"data_root\": data_folder})\n",
509
+ " test_datasets[name] = test_datasets[name].filtered_sorted(sort_key=\"duration\")\n",
510
+ "\n",
511
+ " datasets = [train_data, valid_data] + [i for k, i in test_datasets.items()]\n",
512
+ "\n",
513
+ " # *************************\n",
514
+ " # 2. Define audio pipeline:\n",
515
+ " # *************************\n",
516
+ " @sb.utils.data_pipeline.takes(\"wav\")\n",
517
+ " @sb.utils.data_pipeline.provides(\"sig\")\n",
518
+ " def audio_pipeline(wav):\n",
519
+ " sig = sb.dataio.dataio.read_audio(wav)\n",
520
+ " return sig\n",
521
+ "\n",
522
+ " sb.dataio.dataset.add_dynamic_item(datasets, audio_pipeline)\n",
523
+ "\n",
524
+ " # ************************\n",
525
+ " # 3. Define text pipeline:\n",
526
+ " # ************************\n",
527
+ " label_encoder = sb.dataio.encoder.CTCTextEncoder()\n",
528
+ " \n",
529
+ " @sb.utils.data_pipeline.takes(\"wrd\")\n",
530
+ " @sb.utils.data_pipeline.provides(\"wrd\", \"char_list\", \"tokens_list\", \"tokens\")\n",
531
+ " def text_pipeline(wrd):\n",
532
+ " yield wrd\n",
533
+ " char_list = list(wrd)\n",
534
+ " yield char_list\n",
535
+ " tokens_list = label_encoder.encode_sequence(char_list)\n",
536
+ " yield tokens_list\n",
537
+ " tokens = torch.LongTensor(tokens_list)\n",
538
+ " yield tokens\n",
539
+ "\n",
540
+ " sb.dataio.dataset.add_dynamic_item(datasets, text_pipeline)\n",
541
+ "\n",
542
+ "\n",
543
+ " # *******************************\n",
544
+ " # 4. Create or load label encoder\n",
545
+ " # *******************************\n",
546
+ " lab_enc_file = os.path.join(hparams[\"save_folder\"], \"label_encoder.txt\")\n",
547
+ " special_labels = {\"blank_label\": hparams[\"blank_index\"]}\n",
548
+ " label_encoder.add_unk()\n",
549
+ " label_encoder.load_or_create(path=lab_enc_file, from_didatasets=[train_data], output_key=\"char_list\", special_labels=special_labels, sequence_input=True)\n",
550
+ "\n",
551
+ " # **************\n",
552
+ " # 5. Set output:\n",
553
+ " # **************\n",
554
+ " sb.dataio.dataset.set_output_keys(datasets,[\"id\", \"sig\", \"wrd\", \"char_list\", \"tokens\"],)\n",
555
+ "\n",
556
+ " return train_data, valid_data, test_datasets, label_encoder\n"
557
+ ]
558
+ },
559
+ {
560
+ "cell_type": "markdown",
561
+ "id": "e97c4f20-6951-4d12-8e17-9eb818a52bb1",
562
+ "metadata": {},
563
+ "source": [
564
+ "## 4. Utilisation de la recette Créée"
565
+ ]
566
+ },
567
+ {
568
+ "cell_type": "markdown",
569
+ "id": "76b72148-6bd0-48bd-ad40-cb6f8bfd34c0",
570
+ "metadata": {},
571
+ "source": [
572
+ "### 4.1 Préparation au lancement"
573
+ ]
574
+ },
575
+ {
576
+ "cell_type": "code",
577
+ "execution_count": null,
578
+ "id": "d47ec39a-5562-4a63-8243-656c9235b7a2",
579
+ "metadata": {},
580
+ "outputs": [],
581
+ "source": [
582
+ "hparams_file, run_opts, overrides = sb.parse_arguments([\"PATH_TO_YOUR_FOLDER/ASR_FLEURS-swahili_hf.yaml\"])\n",
583
+ "# create ddp_group with the right communication protocol\n",
584
+ "sb.utils.distributed.ddp_init_group(run_opts)\n",
585
+ "\n",
586
+ "# ***********************************\n",
587
+ "# Chargement du fichier de paramètres\n",
588
+ "# ***********************************\n",
589
+ "with open(hparams_file) as fin:\n",
590
+ " hparams = load_hyperpyyaml(fin, overrides)\n",
591
+ "\n",
592
+ "# ***************************\n",
593
+ "# Create experiment directory\n",
594
+ "# ***************************\n",
595
+ "sb.create_experiment_directory(experiment_directory=hparams[\"output_folder\"], hyperparams_to_save=hparams_file, overrides=overrides)\n",
596
+ "\n",
597
+ "# ***************************\n",
598
+ "# Create the datasets objects\n",
599
+ "# ***************************\n",
600
+ "train_data, valid_data, test_datasets, label_encoder = dataio_prepare(hparams)\n",
601
+ "\n",
602
+ "# **********************\n",
603
+ "# Trainer initialization\n",
604
+ "# **********************\n",
605
+ "asr_brain = MY_SSA_ASR(modules=hparams[\"modules\"], hparams=hparams, run_opts=run_opts, checkpointer=hparams[\"checkpointer\"])\n",
606
+ "asr_brain.tokenizer = label_encoder"
607
+ ]
608
+ },
609
+ {
610
+ "cell_type": "markdown",
611
+ "id": "62ae72eb-416c-4ef0-9348-d02bbc268fbd",
612
+ "metadata": {},
613
+ "source": [
614
+ "### 4.2 Apprentissage du modèle"
615
+ ]
616
+ },
617
+ {
618
+ "cell_type": "code",
619
+ "execution_count": null,
620
+ "id": "d3dd30ee-89c0-40ea-a9d2-0e2b9d8c8686",
621
+ "metadata": {},
622
+ "outputs": [],
623
+ "source": [
624
+ "# ********\n",
625
+ "# Training\n",
626
+ "# ********\n",
627
+ "asr_brain.fit(asr_brain.hparams.epoch_counter, \n",
628
+ " train_data, valid_data, \n",
629
+ " train_loader_kwargs=hparams[\"train_dataloader_opts\"], \n",
630
+ " valid_loader_kwargs=hparams[\"valid_dataloader_opts\"],\n",
631
+ " )\n",
632
+ "\n"
633
+ ]
634
+ },
635
+ {
636
+ "cell_type": "markdown",
637
+ "id": "1b55af4c-c544-45ff-8435-58226218328f",
638
+ "metadata": {},
639
+ "source": [
640
+ "### 4.3 Test du Modèle"
641
+ ]
642
+ },
643
+ {
644
+ "cell_type": "code",
645
+ "execution_count": null,
646
+ "id": "9cef9011-1a3e-43a4-ab16-8cfb2b57dbd9",
647
+ "metadata": {},
648
+ "outputs": [],
649
+ "source": [
650
+ "# *******\n",
651
+ "# Testing\n",
652
+ "# *******\n",
653
+ "if not os.path.exists(hparams[\"output_wer_folder\"]):\n",
654
+ " os.makedirs(hparams[\"output_wer_folder\"])\n",
655
+ "\n",
656
+ "from speechbrain.decoders.ctc import CTCBeamSearcher\n",
657
+ "\n",
658
+ "ind2lab = label_encoder.ind2lab\n",
659
+ "vocab_list = [ind2lab[x] for x in range(len(ind2lab))]\n",
660
+ "test_searcher = CTCBeamSearcher(**hparams[\"test_beam_search\"], vocab_list=vocab_list)\n",
661
+ "\n",
662
+ "for k in test_datasets.keys(): # Allow multiple evaluation throught list of test sets\n",
663
+ " asr_brain.hparams.test_wer_file = os.path.join(hparams[\"output_wer_folder\"], f\"wer_{k}.txt\")\n",
664
+ " asr_brain.evaluate(test_datasets[k], test_loader_kwargs=hparams[\"test_dataloader_opts\"], min_key=\"WER\")\n"
665
+ ]
666
+ }
667
+ ],
668
+ "metadata": {
669
+ "kernelspec": {
670
+ "display_name": "Python 3 (ipykernel)",
671
+ "language": "python",
672
+ "name": "python3"
673
+ },
674
+ "language_info": {
675
+ "codemirror_mode": {
676
+ "name": "ipython",
677
+ "version": 3
678
+ },
679
+ "file_extension": ".py",
680
+ "mimetype": "text/x-python",
681
+ "name": "python",
682
+ "nbconvert_exporter": "python",
683
+ "pygments_lexer": "ipython3",
684
+ "version": "3.10.14"
685
+ }
686
+ },
687
+ "nbformat": 4,
688
+ "nbformat_minor": 5
689
+ }