一、句子语义等价识别任务句子语义等价识别任务说白了就是让模型来判断两个句子是不是在说同一个意思。就像我们人类有时候会说两句话来表达同一个想法。本文基于开源的hfl/chinese-roberta-wwm-ext模型微调训练句子语义等价识别任务。实验所使用的主要依赖版本如下torch1.13.1cu116transformers4.37.0预训练模型下载地址https://huggingface.co/hfl/chinese-roberta-wwm-ext二、实验数据集介绍数据集采用modelscope上公开的大规模中文语义相似度数据地址如下https://modelscope.cn/datasets/DAMO_NLP/BQ_Corpus/summary数据量统计训练集train.csv)验证集dev.csv)861939999数据集格式如下所示其中label:0: 语义不相干1: 语义等价train.csv数据集的标签分布为整体标签分布是非常均衡的。下面统计下train.csv的Token数分布以便后面训练时指定一个合理的max_lengthfromtransformersimportBertTokenizerimportmatplotlib.pyplotaspltimportpandasaspd plt.rcParams[font.sans-serif][SimHei]# 获取Token数defget_num_tokens(file_path,tokenizer):input_num_tokens[]dfpd.read_csv(file_path)forindex,rowindf.iterrows():sentence1row[sentence1]sentence2row[sentence2]tokenslen(tokenizer(sentence1,sentence2)[input_ids])## bert最大512将超过512的丢弃iftokens512:continueinput_num_tokens.append(tokens)returninput_num_tokens# 计算分布defcount_intervals(num_tokens,interval):max_valuemax(num_tokens)intervals_count{}forlower_boundinrange(0,max_value1,interval):upper_boundlower_boundinterval countlen([numfornuminnum_tokensiflower_boundnumupper_bound])intervals_count[f{lower_bound}-{upper_bound}]countreturnintervals_countdefmain():model_pathhfl/chinese-roberta-wwm-exttrain_data_pathdataset/train.csvtokenizerBertTokenizer.from_pretrained(model_path)input_num_tokensget_num_tokens(train_data_path,tokenizer)intervals_countcount_intervals(input_num_tokens,20)print(intervals_count)x[kfork,vinintervals_count.items()]y[vfork,vinintervals_count.items()]plt.figure(figsize(8,6))barsplt.bar(x,y)plt.title(训练集Token分布情况)plt.ylabel(数量)forbarinbars:yvalbar.get_height()plt.text(bar.get_x()bar.get_width()/2,yval,int(yval),vabottom)plt.show()if__name____main__:main()数据集Token量主要都分布在20-40之间整体来看后面max_length可以设置为64。三、微调训练构建Datasetequate_dataset.py# -*- coding: utf-8 -*-fromtorch.utils.dataimportDatasetimporttorchimportpandasaspdclassEquateDataset(Dataset):def__init__(self,data_path,tokenizer,max_length)-None:super().__init__()self.tokenizertokenizer self.max_lengthmax_length self.data[]ifdata_path:dfpd.read_csv(data_path)forindex,rowindf.iterrows():sentence1row[sentence1]sentence2row[sentence2]labelrow[label]tokenslen(tokenizer(sentence1,sentence2)[input_ids])iftokens512:continueself.data.append({sentence1:sentence1,sentence2:sentence2,label:label})print(data load size,len(self.data))defpreprocess(self,sentence1,sentence2,label):encodingself.tokenizer.encode_plus(sentence1,sentence2,max_lengthself.max_length,pad_to_max_lengthTrue,truncationTrue,paddingmax_length,return_tensorspt,)input_idsencoding[input_ids].squeeze()attention_maskencoding[attention_mask].squeeze()returninput_ids,attention_mask,labeldef__getitem__(self,index):item_dataself.data[index]input_ids,attention_mask,labelself.preprocess(**item_data)return{input_ids:input_ids.to(dtypetorch.long),attention_mask:attention_mask.to(dtypetorch.long),labels:torch.tensor(label,dtypetorch.long)}def__len__(self):returnlen(self.data)微调训练# -*- coding: utf-8 -*-importos.pathimporttorchfromtorch.utils.dataimportDataLoaderfromtorch.utils.tensorboardimportSummaryWriterimporttransformersfromtransformersimportBertTokenizer,BertForSequenceClassificationfromequate_datasetimportEquateDatasetfromtqdmimporttqdmimporttime,sysfromsklearn.metricsimportf1_score transformers.logging.set_verbosity_error()deftrain_model(model,train_loader,val_loader,optimizer,device,num_epochs,model_output_dir,scheduler,writer):batch_step0best_accuracy0.0forepochinrange(num_epochs):time1time.time()model.train()forindex,datainenumerate(tqdm(train_loader,filesys.stdout,descTrain Epoch: str(epoch))):input_idsdata[input_ids].to(device)attention_maskdata[attention_mask].to(device)labelsdata[labels].to(device)# 清空过往梯度optimizer.zero_grad()# 前向传播outputsmodel(input_idsinput_ids,attention_maskattention_mask,labelslabels)lossoutputs.loss# 反向传播计算当前梯度loss.backward()# 更新网络参数optimizer.step()writer.add_scalar(Loss/train,loss,batch_step)batch_step1# 100轮打印一次 lossifindex%1000orindexlen(train_loader)-1:time2time.time()tqdm.write(f{index}, epoch:{epoch}-loss:{str(loss)}; lr:{optimizer.param_groups[0][lr]};each steps time spent:{(str(float(time2-time1)/float(index0.0001)))})# 验证model.eval()accuracy,val_loss,f1validate_model(model,device,val_loader)writer.add_scalar(Loss/val,val_loss,epoch)writer.add_scalar(Accuracy/val,accuracy,epoch)writer.add_scalar(F1/val,f1,epoch)print(fval loss:{val_loss}, val accuracy:{accuracy}, f1:{f1}, epoch:{epoch})# 学习率调整scheduler.step(accuracy)# 保存最优模型ifaccuracybest_accuracy:best_accuracyaccuracy best_model_pathos.path.join(model_output_dir,best)print(Save Best Model To ,best_model_path,, epoch: ,epoch)model.save_pretrained(best_model_path)# 保存当前模型last_model_pathos.path.join(model_output_dir,last)print(Save Last Model To ,last_model_path,, epoch: ,epoch)model.save_pretrained(last_model_path)defvalidate_model(model,device,val_loader):running_loss0.0correct0total0y_true[]y_pred[]withtorch.no_grad():for_,datainenumerate(tqdm(val_loader,filesys.stdout,descValidation Data)):input_idsdata[input_ids].to(device)attention_maskdata[attention_mask].to(device)labelsdata[labels].to(device)outputsmodel(input_idsinput_ids,attention_maskattention_mask,labelslabels)lossoutputs.loss logitsoutputs[logits]totallabels.size(0)predictedlogits.max(-1,keepdimTrue)[1]correctpredicted.eq(labels.view_as(predicted)).sum().item()running_lossloss.item()y_true.extend(labels.cpu().numpy())y_pred.extend(predicted.cpu().numpy())f1f1_score(y_true,y_pred,averagemacro)returncorrect/total*100,running_loss/len(val_loader),f1*100defmain():# 基础模型位置model_namehfl/chinese-roberta-wwm-ext# 训练集 验证集train_json_pathdataset/train.csvval_json_pathdataset/dev.csvmax_length64num_classes2epochs15batch_size128lr1e-4model_output_diroutputlogs_dirlogs# 设备devicetorch.device(cuda:0iftorch.cuda.is_available()elsecpu)# 加载分词器和模型tokenizerBertTokenizer.from_pretrained(model_name)modelBertForSequenceClassification.from_pretrained(model_name,num_labelsnum_classes)print(Start Load Train Data...)train_params{batch_size:batch_size,shuffle:True,num_workers:4,}training_setEquateDataset(train_json_path,tokenizer,max_length)training_loaderDataLoader(training_set,**train_params)print(Start Load Validation Data...)val_params{batch_size:batch_size,shuffle:False,num_workers:4,}val_setEquateDataset(val_json_path,tokenizer,max_length)val_loaderDataLoader(val_set,**val_params)# 日志记录writerSummaryWriter(logs_dir)# 优化器optimizertorch.optim.AdamW(paramsmodel.parameters(),lrlr)# 学习率调度器连续两个周期没有改进学习率调整为当前的0.8schedulertorch.optim.lr_scheduler.ReduceLROnPlateau(optimizer,max,patience2,factor0.8)modelmodel.to(device)# 开始训练print(Start Training...)train_model(modelmodel,train_loadertraining_loader,val_loaderval_loader,optimizeroptimizer,devicedevice,num_epochsepochs,model_output_dirmodel_output_dir,schedulerscheduler,writerwriter)writer.close()if__name____main__:main()训练过程如下在batch_size在128的情况下显存仅占用约8G训练结束最终在dev验证集上的loss为0.759准确率为85.1485F185.1480下面可以通过tensorboard查看你训练过程趋势tensorboard--logdirlogs--bind_all在 浏览器访问http:ip:6006/四、模型使用测试# -*- coding: utf-8 -*-importjsonfromtransformersimportBertTokenizer,BertForSequenceClassificationimporttorchdefmain():base_pathhfl/chinese-roberta-wwm-extmodel_pathoutput/bestmax_length64num_classes2# 指定设备devicetorch.device(cuda:0iftorch.cuda.is_available()elsecpu)# 加载分词器tokenizerBertTokenizer.from_pretrained(base_path)# 加载基础模型modelBertForSequenceClassification.from_pretrained(model_path,num_labelsnum_classes)model.to(device)classify{0:语义不相关,1:语义等价}tests[[一直都没有电话打来,我很久没有接到电话],[换卡要多久,请问换卡成功没],[安全性可靠吗,这东西安全吗],[不需要帮忙,别帮我,谢谢],[手机停机了怎么办,我停机了该咋办],[为什么不能登录系统总忙。,解绑后不能登录],[年龄没有要求。,不限制年龄],[为什么会审核未通过。,怎么一直失败],]forlineintests:print()encodingtokenizer.encode_plus(line[0],line[1],max_lengthmax_length,return_tensorspt)input_idsencoding[input_ids].to(device)attention_maskencoding[attention_mask].to(device)outputsmodel(input_idsinput_ids,attention_maskattention_mask)logitsoutputs[logits]predictedlogits.max(-1,keepdimTrue)[1].item()print(f{line}{classify[predicted]})if__name____main__:main()