File size: 4,156 Bytes
5cb94f8 bce2bd5 5cb94f8 f1a3370 bce2bd5 5cb94f8 bb0dfd2 5cb94f8 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 |
---
datasets:
- agender
- mozillacommonvoice
- timit
- voxceleb2
inference: true
tags:
- speech
- audio
- wav2vec2
- audio-classification
- age-recognition
- gender-recognition
license: cc-by-nc-sa-4.0
---
# Model for Age and Gender Recognition based on Wav2vec 2.0 (24 layers)
The model expects a raw audio signal as input and outputs predictions
for age in a range of approximately 0...1 (0...100 years)
and gender expressing the probababilty for being child, female, or male.
In addition, it also provides the pooled states of the last transformer layer.
The model was created by fine-tuning [
Wav2Vec2-Large-Robust](https://huggingface.co/facebook/wav2vec2-large-robust)
on [aGender](https://paperswithcode.com/dataset/agender),
[Mozilla Common Voice](https://commonvoice.mozilla.org/),
[Timit](https://catalog.ldc.upenn.edu/LDC93s1) and
[Voxceleb 2](https://www.robots.ox.ac.uk/~vgg/data/voxceleb/vox2.html).
For this version of the model we trained all 24 transformer layers.
An [ONNX](https://onnx.ai/") export of the model is available from
[doi:10.5281/zenodo.7761387](https://doi.org/10.5281/zenodo.7761387).
Further details are given in the associated [paper](https://arxiv.org/abs/2306.16962)
and [tutorial](https://github.com/audeering/w2v2-age-gender-how-to).
# Usage
```python
import numpy as np
import torch
import torch.nn as nn
from transformers import Wav2Vec2Processor
from transformers.models.wav2vec2.modeling_wav2vec2 import (
Wav2Vec2Model,
Wav2Vec2PreTrainedModel,
)
class ModelHead(nn.Module):
r"""Classification head."""
def __init__(self, config, num_labels):
super().__init__()
self.dense = nn.Linear(config.hidden_size, config.hidden_size)
self.dropout = nn.Dropout(config.final_dropout)
self.out_proj = nn.Linear(config.hidden_size, num_labels)
def forward(self, features, **kwargs):
x = features
x = self.dropout(x)
x = self.dense(x)
x = torch.tanh(x)
x = self.dropout(x)
x = self.out_proj(x)
return x
class AgeGenderModel(Wav2Vec2PreTrainedModel):
r"""Speech emotion classifier."""
def __init__(self, config):
super().__init__(config)
self.config = config
self.wav2vec2 = Wav2Vec2Model(config)
self.age = ModelHead(config, 1)
self.gender = ModelHead(config, 3)
self.init_weights()
def forward(
self,
input_values,
):
outputs = self.wav2vec2(input_values)
hidden_states = outputs[0]
hidden_states = torch.mean(hidden_states, dim=1)
logits_age = self.age(hidden_states)
logits_gender = torch.softmax(self.gender(hidden_states), dim=1)
return hidden_states, logits_age, logits_gender
# load model from hub
device = 'cpu'
model_name = 'audeering/wav2vec2-large-robust-24-ft-age-gender'
processor = Wav2Vec2Processor.from_pretrained(model_name)
model = AgeGenderModel.from_pretrained(model_name)
# dummy signal
sampling_rate = 16000
signal = np.zeros((1, sampling_rate), dtype=np.float32)
def process_func(
x: np.ndarray,
sampling_rate: int,
embeddings: bool = False,
) -> np.ndarray:
r"""Predict age and gender or extract embeddings from raw audio signal."""
# run through processor to normalize signal
# always returns a batch, so we just get the first entry
# then we put it on the device
y = processor(x, sampling_rate=sampling_rate)
y = y['input_values'][0]
y = y.reshape(1, -1)
y = torch.from_numpy(y).to(device)
# run through model
with torch.no_grad():
y = model(y)
if embeddings:
y = y[0]
else:
y = torch.hstack([y[1], y[2]])
# convert to numpy
y = y.detach().cpu().numpy()
return y
print(process_func(signal, sampling_rate))
# Age female male child
# [[ 0.33793038 0.2715511 0.2275236 0.5009253 ]]
print(process_func(signal, sampling_rate, embeddings=True))
# Pooled hidden states of last transformer layer
# [[ 0.024444 0.0508722 0.04930823 ... 0.07247854 -0.0697901
# -0.0170537 ]]
```
|