python 利用已有Ner模型进行数据清洗合并代码

Python  /  管理员 发布于 7年前   236

我就废话不多说了，直接上代码吧！

# -*- coding: utf-8 -*-from kashgari.corpus import DataReaderimport refrom tqdm import tqdmdef cut_text(text, lenth):  textArr = re.findall('.{' + str(lenth) + '}', text)  textArr.append(text[(len(textArr) * lenth):])  return textArrdef clean_data(source_file, target_file, ner_model):    data_x, data_y = DataReader().read_conll_format_file(source_file)  with tqdm(total=len(data_x)) as pbar:    for idx, text_array in enumerate(data_x):      if len(text_array) <= 100:        ners = ner_model.predict([text_array])        ner = ners[0]      else:        texts = cut_text(''.join(text_array), 100)        ners = []        for text in texts:          ner = ner_model.predict([[char for char in text]])          ners = ners + ner[0]        ner = ners           # print('[-----------------------', idx, len(data_x))      # print(data_y[idx])      # print(ner)          for jdx, t in enumerate(text_array):        if ner[jdx].startswith('B') or ner[jdx].startswith('I') :          if data_y[idx][jdx] == 'O':data_y[idx][jdx] = ner[jdx]# print(data_y[idx])      # print('-----------------------]')       pbar.update(1)        f = open(target_file, 'a', encoding="utf-8")    for idx, text_array in enumerate(data_x):    if idx != 0:      f.writelines(['\n'])      for jdx, t in enumerate(text_array):      text = t + ' ' + data_y[idx][jdx]       if idx == 0 and jdx == 0:        text = text      else:        text = '\n' + text      f.writelines([text])      f.close()      data_x2, data_y2 = DataReader().read_conll_format_file(source_file)  print(data_x == data_x2, len(data_y) == len(data_y2), '数据清洗完成')       

# -*- coding: utf-8 -*-import kashgarifrom data_tools import clean_datatime_ner = kashgari.utils.load_model('time_ner.h5')clean_data('./data/example.dev', 'example.dev', time_ner)

以上这篇python 利用已有Ner模型进行数据清洗合并代码就是小编分享给大家的全部内容了，希望能给大家一个参考，也希望大家多多支持。