Spaces:
Build error
Build error
File size: 6,635 Bytes
7f17572 640b40a 7f17572 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 |
import os
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from transformers import AutoModel, AutoConfig, AutoTokenizer
import gradio as gr
os.system("gdown https://drive.google.com/uc?id=1whDb0yL_Kqoyx-sIw0sS5xTfb6r_9nlJ")
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
def init_params(module_lst):
for module in module_lst:
for param in module.parameters():
if param.dim() > 1:
torch.nn.init.xavier_uniform_(param)
return
class Custom_bert(nn.Module):
def __init__(self, model_dir):
super().__init__()
# load base model
config = AutoConfig.from_pretrained(model_dir)
config.update({"output_hidden_states": True,
"hidden_dropout_prob": 0.0,
"layer_norm_eps": 1e-7})
self.base = AutoModel.from_pretrained(model_dir, config=config)
dim = self.base.encoder.layer[0].output.dense.bias.shape[0]
self.dropout = nn.Dropout(p=0.2)
self.high_dropout = nn.Dropout(p=0.5)
# weights for weighted layer average
n_weights = 24
weights_init = torch.zeros(n_weights).float()
weights_init.data[:-1] = -3
self.layer_weights = torch.nn.Parameter(weights_init)
# attention head
self.attention = nn.Sequential(
nn.Linear(1024, 1024),
nn.Tanh(),
nn.Linear(1024, 1),
nn.Softmax(dim=1)
)
self.cls = nn.Sequential(
nn.Linear(dim, 1)
)
init_params([self.cls, self.attention])
def reini_head(self):
init_params([self.cls, self.attention])
return
def forward(self, input_ids, attention_mask):
base_output = self.base(input_ids=input_ids,
attention_mask=attention_mask)
# weighted average of all encoder outputs
cls_outputs = torch.stack(
[self.dropout(layer) for layer in base_output['hidden_states'][-24:]], dim=0
)
cls_output = (
torch.softmax(self.layer_weights, dim=0).unsqueeze(1).unsqueeze(1).unsqueeze(1) * cls_outputs).sum(
0)
# multisample dropout
logits = torch.mean(
torch.stack(
[torch.sum(self.attention(self.high_dropout(cls_output)) * cls_output, dim=1) for _ in range(5)],
dim=0,
),
dim=0,
)
return self.cls(logits)
def get_batches(input, tokenizer, batch_size=128, max_length=256, device='cpu'):
out = tokenizer(input, return_tensors='pt', max_length=max_length, padding='max_length')
out['input_ids'], out['attention_mask'] = out['input_ids'].to(device), out['attention_mask'].to(device)
input_id_split = torch.split(out['input_ids'], max_length, dim=1)
attention_split = torch.split(out['attention_mask'], max_length, dim=1)
input_id_batches = []
attention_batches = []
i = 0
input_length = len(input_id_split)
while i * batch_size < input_length:
if i * batch_size + batch_size <= input_length:
input_id_batches.append(list(input_id_split[i * batch_size:(i + 1) * batch_size]))
attention_batches.append(list(attention_split[i * batch_size:(i + 1) * batch_size]))
else:
input_id_batches.append(list(input_id_split[i * batch_size:input_length]))
attention_batches.append(list(attention_split[i * batch_size:input_length]))
i += 1
if input_id_batches[-1][-1].shape[1] < max_length:
input_id_batches[-1][-1] = F.pad(input_id_batches[-1][-1],
(1, max_length - input_id_batches[-1][-1].shape[1] - 1),
value=0)
attention_batches[-1][-1] = F.pad(attention_batches[-1][-1],
(1, max_length - attention_batches[-1][-1].shape[1] - 1),
value=1)
input_id_batches = [torch.cat(batch, dim=0) for batch in input_id_batches]
attention_batches = [torch.cat(batch, dim=0) for batch in attention_batches]
return tuple(zip(input_id_batches, attention_batches))
def predict(input, tokenizer, model, batch_size=128, max_length=256, max_val=-4, min_val=3, score=100):
device = model.base.device
batches = get_batches(input, tokenizer, batch_size, max_length, device)
predictions = []
with torch.no_grad():
for input_ids, attention_mask in batches:
pred = model(input_ids, attention_mask)
pred = score * (pred - min_val) / (max_val - min_val)
predictions.append(pred)
predictions = torch.cat(predictions, dim=0)
mean, std = predictions.mean().cpu().item(), predictions.std().cpu().item()
mean, std = round(mean, 2), round(std, 2)
if np.isnan(std):
return f"The reading difficulty score is {mean}."
else:
return f"""The reading difficulty score is {mean} with a standard deviation of {std}.
\nThe 95% confidence interval of the score is {mean - 2 * std} to {mean + 2 * std}."""
if __name__ == "__main__":
deberta_loc = "deberta_large_0.pt"
deberta_tokenizer = AutoTokenizer.from_pretrained("microsoft/deberta-large", model_max_length=256)
model = Custom_bert("microsoft/deberta-large")
model.load_state_dict(torch.load(deberta_loc, map_location=torch.device(device)))
model.eval().to(device)
description = """
This tool attempts to estimate how difficult a piece of text is to read by a school child.
The underlying model has been developed based on expert ranking of text difficulty for students from grade 3 to 12.
The score has been scaled to range from zero (very easy) to one hundred (very difficult).
Very long passages will be broken up and reported with the average as well as the standard deviation of the difficulty score.
"""
interface = gr.Interface(fn=lambda x: predict(x, deberta_tokenizer, model, batch_size=4),
inputs=gr.inputs.Textbox(lines = 7, label = "Text:",
placeholder = "Insert text to be scored here."),
outputs='text',
title = "Reading Difficulty Analyser",
description = description)
interface.launch()
|