上篇链接看LZ上篇博客的时间竟然是7月18日着实是懈怠了其实有很多东西需要总结归纳这周末就补一下之前欠的债吧上篇主要介绍了DBFace的大体框架这篇主要介绍数据的预处理部分5. 数据预处理pytorch数据一般是要写一个类函数来继承Dataset类的需要定义三个函数__init__(self),len(self),getitem(self)这三个函数在DBFace中的代码如下所示classLDataset(Dataset):def__init__(self,labelfile,imagesdir,numlandmarks,mean,std,width800,height800):self.widthwidth self.heightheight self.numlandmarksnumlandmarks self.itemscommon.load_webface(labelfile,imagesdir,numlandmarks)self.meanmean self.stdstddef__len__(self):returnlen(self.items)def__getitem__(self,index):...可以看到初始化还是比较简单的主要是下载对应的参数这边LZ其实是修改了一部分代码的感兴趣的小伙伴可以在github上下载原始代码。在初始化部分需要看的是这一行self.itemscommon.load_webface(labelfile,imagesdir,numlandmarks)我们来看下load_webface部分defload_webface(labelfile,imagesdir,numlandmarks):withopen(labelfile,r)asf:linesf.readlines()lines[line.replace(\n,)forlineinlines]stage0facials[]fileNonefiles[]forindex,lineinenumerate(lines):ifline.startswith(#):iffileisnotNone:files.append([f{imagesdir}/{file},parse_facials_webface(facials,numlandmarks)])fileline[2:]facials[]else:facials.append([float(item)foriteminline.split( )])iffileisnotNone:files.append([f{imagesdir}/{file},parse_facials_webface(facials,numlandmarks)])returnfileswiderface的label.txt如下所示# 0--Parade/0_Parade_marchingband_1_849.jpg 449 330 122 149 488.906 373.643 0.0 542.089 376.442 0.0 515.031 412.83 0.0 485.174 425.893 0.0 538.357 431.491 0.0 0.82 # 0--Parade/0_Parade_Parade_0_904.jpg 361 98 263 339 424.143 251.656 0.0 547.134 232.571 0.0 494.121 325.875 0.0 453.83 368.286 0.0 561.978 342.839 0.0 0.89 # 0--Parade/0_Parade_marchingband_1_799.jpg ...“#”后面有一个空格后面跟着图片名称第二行分别是xywh也就是人脸框左上角的点坐标和对应框的宽度和高度后面跟着关键点坐标这个应该还是很好理解的在parse_facials_webface 函数中要根据自己的关键点数量进行修改这个看了源码应该很好理解self.items里面就是保存的要读取图片的路径和对应人脸框和关键点的数据# 构建dataset部分,继承torch 的dataset类self.train_datasetLDataset(labelfile,imagesdir,numlandmarks,meanself.mean,stdself.std,widthself.width,heightself.height)self.train_loaderDataLoader(datasetself.train_dataset,batch_sizeself.batch_size,shuffleTrue,num_workers24)# 优化器adam,使用默认的weight_decay0self.optimizertorch.optim.Adam(self.model.parameters(),lrself.lr)self.per_epoch_batchslen(self.train_loader)self.iter0self.epochs150使用DataLoader类生成一个迭代对象在训练的时候可以给网络送不同的数据优化器是adam学习率是之前设置好的# warm up一下lr_scheduer{1:1e-3,2:2e-3,3:1e-3,60:1e-4,120:1e-5}实际上很多数据的预处理是在训练的阶段进行的DBFace中APP中的两个函数就是用来训练的deftrain_epoch(self,epoch):forindbatch,(images,heatmap_gt,heatmap_posweight,reg_tlrb,reg_mask,landmark_gt,landmark_mask,num_objs,keep_mask)inenumerate(self.train_loader):self.iter1batch_objssum(num_objs)batch_sizeself.batch_sizeifbatch_objs0:batch_objs1heatmap_gtheatmap_gt.to(self.gpu_master)heatmap_posweightheatmap_posweight.to(self.gpu_master)keep_maskkeep_mask.to(self.gpu_master)reg_tlrbreg_tlrb.to(self.gpu_master)reg_maskreg_mask.to(self.gpu_master)landmark_gtlandmark_gt.to(self.gpu_master)landmark_masklandmark_mask.to(self.gpu_master)imagesimages.to(self.gpu_master)hm,tlrb,landmarkself.model(images)# 把数据压到0-1的范围hmhm.sigmoid()hmtorch.clamp(hm,min1e-4,max1-1e-4)# 为什么回归出来框坐标要进行exp处理# 因为使用exp后的结果进行拟合换句话说网络推断出来的是logtlrbtlrbtorch.exp(tlrb)hm_lossself.focal_loss(hm,heatmap_gt,heatmap_posweight,keep_maskkeep_mask)/batch_objs reg_lossself.giou_loss(tlrb,reg_tlrb,reg_mask)*5# 这个权重要改吗?landmark_lossself.landmark_loss(landmark,landmark_gt,landmark_mask)*0.1losshm_lossreg_losslandmark_loss self.optimizer.zero_grad()loss.backward()self.optimizer.step()epoch_fltepochindbatch/self.per_epoch_batchsifindbatch%100:log.info(fiter:{self.iter}, lr:{self.lr:g}, epoch:{epoch_flt:.2f}, loss:{loss.item():.2f}, hm_loss:{hm_loss.item():.2f}, fbox_loss:{reg_loss.item():.2f}, lmdk_loss:{landmark_loss.item():.5f})ifindbatch%10000:log.info(save hm)hm_imagehm[0,0].cpu().data.numpy()common.imwrite(f{jobdir}/imgs/hm_image.jpg,hm_image*255)common.imwrite(f{jobdir}/imgs/hm_image_gt.jpg,heatmap_gt[0,0].cpu().data.numpy()*255)imagenp.clip((images[0].permute(1,2,0).cpu().data.numpy()*self.stdself.mean)*255,0,255).astype(np.uint8)outobjseval_tool.detect_images_giou_with_netout(hm,tlrb,landmark,threshold0.1,ibatch0)im1image.copy()forobjinoutobjs:common.drawbbox(im1,obj)common.imwrite(f{jobdir}/imgs/train_result.jpg,im1)deftrain(self):# warm up?lr_scheduer{1:1e-3,2:2e-3,3:1e-3,60:1e-4,120:1e-5}# trainself.model.train()forepochinrange(self.epochs):ifepochinlr_scheduer:self.set_lr(lr_scheduer[epoch])self.train_epoch(epoch)filef{jobdir}/models/{epoch1}.pthcommon.mkdirs_from_file_path(f让ile)torch.save(self.model.module.state_dict(),file)在训练的的函数中pytorch就会调用LDataset中的__getitem__(self, index)这个其实才是比较关键的数据预处理部分def__getitem__(self,index):# 获取对应的图片的路径objs是对应图片中的人脸框和关键点如果有多个人脸就会有多个listimgfile,objsself.items[index]imagecommon.imread(imgfile)ifimageisNone:log.info({} is empty, index{}.format(imgfile,index))returnself[random.randint(0,len(self.items)-1)]keepsize12# 进行数据增广image,objsaugment.webface(image,objs,self.numlandmarks,self.width,self.height,keepsize0)# norm, 固定值可以放到NNIE上去做进行数据归一化这个可以在生成wk的时候做也可以用网络做当然也可以用cpu做用neon加速# 现在真的每个操作都得节约时间1ms也要节约LZ哭了image((image/255.0-self.mean)/self.std).astype(np.float32)posweight_radius2# 这个有啥用?,后面高斯核的半径# 这个是通过fpn输出的feature map stride 4加速可以是stride8满脑子加速stride4fm_widthself.width//stride fm_heightself.height//stride# 这里需要根据关键点的数量进行修改初始化一些mapheatmap_gtnp.zeros((1,fm_height,fm_width),np.float32)heatmap_posweightnp.zeros((1,fm_height,fm_width),np.float32)keep_masknp.ones((1,fm_height,fm_width),np.float32)reg_tlrbnp.zeros((1*4,fm_height,fm_width),np.float32)reg_masknp.zeros((1,fm_height,fm_width),np.float32)distance_mapnp.zeros((1,fm_height,fm_width),np.float32)1000# 我有25个关键点有xy坐标要改成25×2# landmark_gt np.zeros((1 * 10, fm_height, fm_width), np.float32)# landmark_mask np.zeros((1, fm_height, fm_width), np.float32)landmark_gtnp.zeros((1*50,fm_height,fm_width),np.float32)landmark_masknp.zeros((1,fm_height,fm_width),np.float32)hassmallFalseforobjinobjs:isSmallObjobj.areakeepsize*keepsizeifisSmallObj:cx,cyobj.safe_scale_center(1/stride,fm_width,fm_height)keep_mask[0,cy,cx]0w,hobj.width/stride,obj.height/stride x0int(common.clip_value(cx-w//2,fm_width-1))y0int(common.clip_value(cy-h//2,fm_height-1))x1int(common.clip_value(cxw//2,fm_width-1)1)y1int(common.clip_value(cyh//2,fm_height-1)1)#这个是计算loss的时候的一个参数也就是说只有有人脸的区域才参与loss的计算如果不是人脸区域不参与loss的计算ifx1-x00andy1-y00:keep_mask[0,y0:y1,x0:x1]0hassmallTrueforobjinobjs:classes0cx,cyobj.safe_scale_center(1/stride,fm_width,fm_height)reg_boxnp.array(obj.box)/stride#框的坐标除以对应的strideisSmallObjobj.areakeepsize*keepsizeifisSmallObj:ifobj.area5*5:distance_map[classes,cy,cx]0reg_tlrb[classes*4:(classes1)*4,cy,cx]reg_box# 通道数代表你回归的框的坐标乘以类别reg_mask[classes,cy,cx]1continuew,hobj.width/stride,obj.height/stride x0int(common.clip_value(cx-w//2,fm_width-1))y0int(common.clip_value(cy-h//2,fm_height-1))x1int(common.clip_value(cxw//2,fm_width-1)1)y1int(common.clip_value(cyh//2,fm_height-1)1)ifx1-x00andy1-y00:keep_mask[0,y0:y1,x0:x1]1# 参考cornernetw_radius,h_radiuscommon.truncate_radius((obj.width,obj.height))# size/(4*stride)gaussian_mapcommon.draw_truncate_gaussian(heatmap_gt[classes,:,:],(cx,cy),h_radius,w_radius)mxface300miface25mxlinemax(obj.width,obj.height)gamma(mxline-miface)/(mxface-miface)*10gammamin(max(0,gamma),10)1common.draw_gaussian(heatmap_posweight[classes,:,:],(cx,cy),posweight_radius,kgamma)range_expand_xmath.ceil(w_radius)range_expand_ymath.ceil(h_radius)min_expand_size3range_expand_xmax(min_expand_size,range_expand_x)range_expand_ymax(min_expand_size,range_expand_y)icx,icycx,cy reg_landmarkNonefill_threshold0.3# 这里也需要根据关键点数量进行修改ifobj.haslandmark:reg_landmarknp.array(obj.x5y5_cat_landmark)/stride# x5y5 [cx] * 5 [cy] * 5x5y5[cx]*25[cy]*25rvalue(reg_landmark-x5y5)# landmark_gt[0:10, cy, cx] np.array(common.log(rvalue)) / 4# 注意这里的loglandmark_gt[0:50,cy,cx]np.array(common.log(rvalue))/4landmark_mask[0,cy,cx]1ifnotobj.rotate:forcxinrange(icx-range_expand_x,icxrange_expand_x1):forcyinrange(icy-range_expand_y,icyrange_expand_y1):ifcxfm_widthandcyfm_heightandcx0andcy0:my_gaussian_value0.9gy,gxcy-icyrange_expand_y,cx-icxrange_expand_xifgy0andgygaussian_map.shape[0]andgx0andgxgaussian_map.shape[1]:my_gaussian_valuegaussian_map[gy,gx]distancemath.sqrt((cx-icx)**2(cy-icy)**2)ifmy_gaussian_valuefill_thresholdordistancemin_expand_size:already_distancedistance_map[classes,cy,cx]my_mix_distance(1-my_gaussian_value)*distanceifmy_mix_distancealready_distance:continuedistance_map[classes,cy,cx]my_mix_distance reg_tlrb[classes*4:(classes1)*4,cy,cx]reg_box reg_mask[classes,cy,cx]1# if hassmall:# common.imwrite(test_result/keep_mask.jpg, keep_mask[0]*255)# common.imwrite(test_result/heatmap_gt.jpg, heatmap_gt[0]*255)# common.imwrite(test_result/keep_ori.jpg, (image*self.stdself.mean)*255)returnT.to_tensor(image),heatmap_gt,heatmap_posweight,reg_tlrb,reg_mask,landmark_gt,landmark_mask,len(objs),keep_mask6. 数据增广数据增广函数在argument.py脚本中defwebface(image,objs,numlandmarks,outw800,outh800,keepsize8):funcs[[augmentWithColorJittering,0.7],[augmentWithFlip,0.7]]random.shuffle(funcs)numlen(funcs)forninrange(num):func,freqfuncs[n]ifrandrf(0,1)freq:image,objsfunc(image,objs)ifrandrf(0,1)0.5:image,objscubeTransform(image,objs,outw,outh,keepsizekeepsize)image,objsaugmentWithCropScaleWebface(image,objs,numlandmarks,outw,outh,cube,keepsizekeepsize)else:image,objsaugmentWithCropScaleWebface(image,objs,numlandmarks,outw,outh,keepsizekeepsize)returnimage,objs主要是augmentWithColorJittering对颜色的数据增强包括图像的亮度对比度和饱和度augmentWithFlip水平翻转这个当中需要注意的是关键点要根据水平翻转后也要进行镜像处理augmentWithCropScaleWebface随机裁剪和尺度变换cubeTransform 立方体转换后面一篇准备写下网络推断部分未完待续。。。