Spaces:
Build error
Build error
File size: 3,230 Bytes
273b73b |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 |
# -*- coding: utf-8 -*-
"""
Created on Mon Aug 24 16:21:23 2020
@author: luol2
"""
import argparse
from ssplit_tokenzier import ssplit_token_pos_lemma
from ml_ner import ml_tagging,ml_tagging_allngram
from combine_result import combine_ml_dict
from restore_index import restore_index_nest_fn
from dic_ner import dic_ont
from post_processing import combine_overlap
from abbre_resolution import postprocess_abbr
import os
import time
import json
#hybrid method
def bioTag(session,text,biotag_dic,ml_model,onlyLongest=False, abbrRecog=False, Threshold=0.95):
# startTime=time.time()
ssplit_token=ssplit_token_pos_lemma(text)
# print(ssplit_token)
# print('ssplit token:',time.time()-startTime)
# startTime=time.time()
dict_tsv=biotag_dic.matching(ssplit_token)
# print('dict tsv:\n',dict_tsv)
# print('dict ner:',time.time()-startTime)
# startTime=time.time()
ml_tsv=ml_tagging(session,ssplit_token,ml_model,Threshold)
#print('ml_tsv:\n',ml_tsv)
# print('ml ner:',time.time()-startTime)
# startTime=time.time()
combine_tsv=combine_ml_dict(dict_tsv,ml_tsv)
#combine_tsv=combine_ml_dict_fn(ml_tsv,dict_tsv)
#print('combine:\n',combine_tsv)
# print('combine:',time.time()-startTime)
# startTime=time.time()
final_result= restore_index_nest_fn(text,combine_tsv)
# print('final ner:',time.time()-startTime)
if onlyLongest==True:
final_result=combine_overlap(final_result)
if abbrRecog==True:
final_result=postprocess_abbr(final_result,text)
# print('final result:')
# print(final_result)
# print('final ner:',time.time()-startTime)
return final_result
# only machine learning-based method
def bioTag_ml(text,ml_model,onlyLongest=False,abbrRecog=False, Threshold=0.95):
# startTime=time.time()
ssplit_token=ssplit_token_pos_lemma(text)
# print(ssplit_token)
# print('ssplit token:',time.time()-startTime)
# startTime=time.time()
ml_tsv=ml_tagging_allngram(ssplit_token,ml_model,Threshold)
# print('ml_tsv:\n',ml_tsv)
# print('ml ner:',time.time()-startTime)
final_result= restore_index_nest_fn(text,ml_tsv)
# print('final ner:',time.time()-startTime)
if onlyLongest==True:
final_result=combine_overlap(final_result)
if abbrRecog==True:
final_result=postprocess_abbr(final_result,text)
return final_result
# only dict method
def bioTag_dic(text,biotag_dic,onlyLongest=False, abbrRecog=False):
# startTime=time.time()
ssplit_token=ssplit_token_pos_lemma(text)
# print(ssplit_token)
# print('ssplit token:',time.time()-startTime)
# startTime=time.time()
dict_tsv=biotag_dic.matching(ssplit_token)
# print('dict tsv:\n',dict_tsv)
# print('dict ner:',time.time()-startTime)
final_result= restore_index_nest_fn(text,dict_tsv)
# print('final ner:',time.time()-startTime)
if onlyLongest==True:
final_result=combine_overlap(final_result)
if abbrRecog==True:
final_result=postprocess_abbr(final_result,text)
return final_result
|