Browse Source

compatible with python3

tags/v0.5-sensible
kkkim 7 years ago
parent
commit
d12f97097b
20 changed files with 90 additions and 84 deletions
  1. +10
    -6
      README.md
  2. +1
    -1
      dface/core/detect.py
  3. +1
    -1
      dface/core/imagedb.py
  4. +1
    -1
      dface/prepare_data/assemble_onet_imglist.py
  5. +1
    -1
      dface/prepare_data/assemble_pnet_imglist.py
  6. +1
    -1
      dface/prepare_data/assemble_rnet_imglist.py
  7. +12
    -12
      dface/prepare_data/gen_Onet_train_data.py
  8. +13
    -10
      dface/prepare_data/gen_Pnet_train_data.py
  9. +13
    -13
      dface/prepare_data/gen_Rnet_train_data.py
  10. +5
    -5
      dface/prepare_data/gen_landmark_12.py
  11. +5
    -5
      dface/prepare_data/gen_landmark_24.py
  12. +5
    -5
      dface/prepare_data/gen_landmark_48.py
  13. +6
    -6
      dface/prepare_data/gen_landmark_net_48.py
  14. +0
    -0
      dface/prepare_data/widerface_annotation_gen/__init__.py
  15. +6
    -6
      dface/prepare_data/widerface_annotation_gen/transform.py
  16. +5
    -5
      dface/train_net/train.py
  17. +1
    -1
      dface/train_net/train_o_net.py
  18. +2
    -2
      dface/train_net/train_p_net.py
  19. +2
    -2
      dface/train_net/train_r_net.py
  20. +0
    -1
      test_image.py

+ 10
- 6
README.md View File

@@ -79,15 +79,19 @@ export PYTHONPATH=$PYTHONPATH:{your local DFace root path}


#### 训练mtcnn模型 #### 训练mtcnn模型


MTCNN主要有三个网络,叫做**PNet**, **RNet** 和 **ONet**。因此我们的训练过程也需要分三步先后进行。为了更好的实现效果,当前被训练的网络都将依赖于上一个训练好的网络来生成数据。所有的人脸数据集都来自 **[WIDER FACE](http://mmlab.ie.cuhk.edu.hk/projects/WIDERFace/)** 和 **[CelebA](http://mmlab.ie.cuhk.edu.hk/projects/CelebA.html)**。WIDER FACE仅提供了大量的人脸边框定位数据,而CelebA包含了人脸关键点定位数据。以下训练除了 生成ONet的人脸关键点训练数据和标注文件 该步骤使用CelebA数据集,其他一律使用WIDER FACE。如果使用wider face的 wider_face_train.mat 注解文件需要转换成txt格式的,我这里用h5py写了个 [转换脚本](https://gitee.com/kuaikuaikim/dface/blob/master/src/prepare_data/widerface_annotation_gen/transform.py)
MTCNN主要有三个网络,叫做**PNet**, **RNet** 和 **ONet**。因此我们的训练过程也需要分三步先后进行。为了更好的实现效果,当前被训练的网络都将依赖于上一个训练好的网络来生成数据。所有的人脸数据集都来自 **[WIDER FACE](http://mmlab.ie.cuhk.edu.hk/projects/WIDERFace/)** 和 **[CelebA](http://mmlab.ie.cuhk.edu.hk/projects/CelebA.html)**。WIDER FACE仅提供了大量的人脸边框定位数据,而CelebA包含了人脸关键点定位数据。以下训练除了 生成ONet的人脸关键点训练数据和标注文件 该步骤使用CelebA数据集,其他一律使用WIDER FACE。如果使用wider face的 wider_face_train.mat 注解文件需要转换成txt格式的,我这里用h5py写了个 [转换脚本](https://gitee.com/kuaikuaikim/dface/blob/master/dface/prepare_data/widerface_annotation_gen/transform.py). 这里我提供一个已经转换好的wider face注解文件 [anno_store/wider_origin_anno.txt](https://gitee.com/kuaikuaikim/dface/blob/master/anno_store/wider_origin_anno.txt), 以下训练过程参数名--anno_file默认就是使用该转换好的注解文件。


* 创建 dface 训练数据临时目录,对应于以下所有的参数名 --dface_traindata_store
```shell
mkdir {your dface traindata folder}
```




* 生成PNet训练数据和标注文件 * 生成PNet训练数据和标注文件


```shell ```shell
python dface/prepare_data/gen_Pnet_train_data.py --dataset_path {your dataset path} --anno_file {your dataset original annotation path}
python dface/prepare_data/gen_Pnet_train_data.py --prefix_path {注解文件中图片的目录前缀} --dface_traindata_store {之前创建的dface训练数据临时目录} --anno_file {wider face 注解文件,可以不填,默认使用anno_store/wider_origin_anno.txt}
``` ```
* 乱序合并标注文件 * 乱序合并标注文件


@@ -104,7 +108,7 @@ python dface/train_net/train_p_net.py
* 生成RNet训练数据和标注文件 * 生成RNet训练数据和标注文件


```shell ```shell
python dface/prepare_data/gen_Rnet_train_data.py --dataset_path {your dataset path} --anno_file {your dataset original annotation path} --pmodel_file {yout PNet model file trained before}
python dface/prepare_data/gen_Rnet_train_data.py --dface_traindata_store {之前创建的dface训练数据临时目录} --anno_file {wider face 注解文件,可以不填,默认使用anno_store/wider_origin_anno.txt} --pmodel_file {之前训练的Pnet模型文件}
``` ```
* 乱序合并标注文件 * 乱序合并标注文件


@@ -121,16 +125,16 @@ python dface/train_net/train_r_net.py
* 生成ONet训练数据和标注文件 * 生成ONet训练数据和标注文件


```shell ```shell
python dface/prepare_data/gen_Onet_train_data.py --dataset_path {your dataset path} --anno_file {your dataset original annotation path} --pmodel_file {yout PNet model file trained before} --rmodel_file {yout RNet model file trained before}
python dface/prepare_data/gen_Onet_train_data.py --dface_traindata_store {之前创建的dface训练数据临时目录} --anno_file {wider face 注解文件,可以不填,默认使用anno_store/wider_origin_anno.txt} --pmodel_file {之前训练的Pnet模型文件} --rmodel_file {之前训练的Rnet模型文件}
``` ```


* 生成ONet的人脸关键点训练数据和标注文件
* 生成ONet的人脸五官关键点训练数据和标注文件


```shell ```shell
python dface/prepare_data/gen_landmark_48.py python dface/prepare_data/gen_landmark_48.py
``` ```


* 乱序合并标注文件(包括人脸关键点)
* 乱序合并标注文件(包括人脸五官关键点)


```shell ```shell
python dface/prepare_data/assemble_onet_imglist.py python dface/prepare_data/assemble_onet_imglist.py


+ 1
- 1
dface/core/detect.py View File

@@ -623,7 +623,7 @@ face candidates:%d, current batch_size:%d"%(num_boxes, batch_size)


t3 = time.time() - t t3 = time.time() - t
t = time.time() t = time.time()
print "time cost " + '{:.3f}'.format(t1+t2+t3) + ' pnet {:.3f} rnet {:.3f} onet {:.3f}'.format(t1, t2, t3)
print("time cost " + '{:.3f}'.format(t1+t2+t3) + ' pnet {:.3f} rnet {:.3f} onet {:.3f}'.format(t1, t2, t3))


return boxes_align, landmark_align return boxes_align, landmark_align




+ 1
- 1
dface/core/imagedb.py View File

@@ -137,7 +137,7 @@ class ImageDB(object):
imdb: dict imdb: dict
image database with flipped image annotations added image database with flipped image annotations added
""" """
print 'append flipped images to imdb', len(imdb)
print('append flipped images to imdb', len(imdb))
for i in range(len(imdb)): for i in range(len(imdb)):
imdb_ = imdb[i] imdb_ = imdb[i]
m_bbox = imdb_['bbox_target'].copy() m_bbox = imdb_['bbox_target'].copy()


+ 1
- 1
dface/prepare_data/assemble_onet_imglist.py View File

@@ -22,4 +22,4 @@ if __name__ == '__main__':
imglist_file = os.path.join(anno_dir, imglist_filename) imglist_file = os.path.join(anno_dir, imglist_filename)


chose_count = assemble.assemble_data(imglist_file ,anno_list) chose_count = assemble.assemble_data(imglist_file ,anno_list)
print "PNet train annotation result file path:%s" % imglist_file
print("PNet train annotation result file path:%s" % imglist_file)

+ 1
- 1
dface/prepare_data/assemble_pnet_imglist.py View File

@@ -22,4 +22,4 @@ if __name__ == '__main__':
imglist_file = os.path.join(anno_dir, imglist_filename) imglist_file = os.path.join(anno_dir, imglist_filename)


chose_count = assemble.assemble_data(imglist_file ,anno_list) chose_count = assemble.assemble_data(imglist_file ,anno_list)
print "PNet train annotation result file path:%s" % imglist_file
print("PNet train annotation result file path:%s" % imglist_file)

+ 1
- 1
dface/prepare_data/assemble_rnet_imglist.py View File

@@ -22,4 +22,4 @@ if __name__ == '__main__':
imglist_file = os.path.join(anno_dir, imglist_filename) imglist_file = os.path.join(anno_dir, imglist_filename)


chose_count = assemble.assemble_data(imglist_file ,anno_list) chose_count = assemble.assemble_data(imglist_file ,anno_list)
print "PNet train annotation result file path:%s" % imglist_file
print("PNet train annotation result file path:%s" % imglist_file)

+ 12
- 12
dface/prepare_data/gen_Onet_train_data.py View File

@@ -9,7 +9,7 @@ import time
import os import os
import cPickle import cPickle
from dface.core.utils import convert_to_square,IoU from dface.core.utils import convert_to_square,IoU
import dface.config
import dface.config as config
import dface.core.vision as vision import dface.core.vision as vision


def gen_onet_data(data_dir, anno_file, pnet_model_file, rnet_model_file, prefix_path='', use_cuda=True, vis=False): def gen_onet_data(data_dir, anno_file, pnet_model_file, rnet_model_file, prefix_path='', use_cuda=True, vis=False):
@@ -27,7 +27,7 @@ def gen_onet_data(data_dir, anno_file, pnet_model_file, rnet_model_file, prefix_


for databatch in image_reader: for databatch in image_reader:
if batch_idx % 100 == 0: if batch_idx % 100 == 0:
print "%d images done" % batch_idx
print("%d images done" % batch_idx)
im = databatch im = databatch


t = time.time() t = time.time()
@@ -89,7 +89,7 @@ def gen_onet_sample_data(data_dir,anno_file,det_boxs_file):
im_idx_list = list() im_idx_list = list()
gt_boxes_list = list() gt_boxes_list = list()
num_of_images = len(annotations) num_of_images = len(annotations)
print "processing %d images in total" % num_of_images
print("processing %d images in total" % num_of_images)


for annotation in annotations: for annotation in annotations:
annotation = annotation.strip().split(' ') annotation = annotation.strip().split(' ')
@@ -112,7 +112,7 @@ def gen_onet_sample_data(data_dir,anno_file,det_boxs_file):
det_handle = open(det_boxs_file, 'r') det_handle = open(det_boxs_file, 'r')


det_boxes = cPickle.load(det_handle) det_boxes = cPickle.load(det_handle)
print len(det_boxes), num_of_images
print(len(det_boxes), num_of_images)
assert len(det_boxes) == num_of_images, "incorrect detections or ground truths" assert len(det_boxes) == num_of_images, "incorrect detections or ground truths"


# index of neg, pos and part face, used as their image names # index of neg, pos and part face, used as their image names
@@ -122,7 +122,7 @@ def gen_onet_sample_data(data_dir,anno_file,det_boxs_file):
image_done = 0 image_done = 0
for im_idx, dets, gts in zip(im_idx_list, det_boxes, gt_boxes_list): for im_idx, dets, gts in zip(im_idx_list, det_boxes, gt_boxes_list):
if image_done % 100 == 0: if image_done % 100 == 0:
print "%d images done" % image_done
print("%d images done" % image_done)
image_done += 1 image_done += 1


if dets.shape[0] == 0: if dets.shape[0] == 0:
@@ -194,17 +194,17 @@ def parse_args():
parser = argparse.ArgumentParser(description='Test mtcnn', parser = argparse.ArgumentParser(description='Test mtcnn',
formatter_class=argparse.ArgumentDefaultsHelpFormatter) formatter_class=argparse.ArgumentDefaultsHelpFormatter)


parser.add_argument('--dataset_path', dest='dataset_path', help='dataset folder',
parser.add_argument('--dface_traindata_store', dest='traindata_store', help='dface train data temporary folder,include 12,24,48/postive,negative,part,landmark',
default='../data/wider/', type=str) default='../data/wider/', type=str)
parser.add_argument('--anno_file', dest='annotation_file', help='output data folder',
default='../data/wider/anno.txt', type=str)
parser.add_argument('--anno_file', dest='annotation_file', help='wider face original annotation file',
default=os.path.join(config.ANNO_STORE_DIR,"wider_origin_anno.txt"), type=str)
parser.add_argument('--pmodel_file', dest='pnet_model_file', help='PNet model file path', parser.add_argument('--pmodel_file', dest='pnet_model_file', help='PNet model file path',
default='/idata/workspace/mtcnn/model_store/pnet_epoch_5best.pt', type=str)
default='/idata/workspace/dface/model_store/pnet_epoch.pt', type=str)
parser.add_argument('--rmodel_file', dest='rnet_model_file', help='RNet model file path', parser.add_argument('--rmodel_file', dest='rnet_model_file', help='RNet model file path',
default='/idata/workspace/mtcnn/model_store/rnet_epoch_1.pt', type=str)
default='/idata/workspace/dface/model_store/rnet_epoch.pt', type=str)
parser.add_argument('--gpu', dest='use_cuda', help='with gpu', parser.add_argument('--gpu', dest='use_cuda', help='with gpu',
default=config.USE_CUDA, type=bool) default=config.USE_CUDA, type=bool)
parser.add_argument('--prefix_path', dest='prefix_path', help='image prefix root path',
parser.add_argument('--prefix_path', dest='prefix_path', help='annotation file image prefix root path',
default='', type=str) default='', type=str)


args = parser.parse_args() args = parser.parse_args()
@@ -214,7 +214,7 @@ def parse_args():


if __name__ == '__main__': if __name__ == '__main__':
args = parse_args() args = parse_args()
gen_onet_data(args.dataset_path, args.annotation_file, args.pnet_model_file, args.rnet_model_file, args.prefix_path, args.use_cuda)
gen_onet_data(args.traindata_store, args.annotation_file, args.pnet_model_file, args.rnet_model_file, args.prefix_path, args.use_cuda)







+ 13
- 10
dface/prepare_data/gen_Pnet_train_data.py View File

@@ -6,7 +6,7 @@ import numpy.random as npr
from dface.core.utils import IoU from dface.core.utils import IoU
import dface.config as config import dface.config as config


def gen_pnet_data(data_dir,anno_file):
def gen_pnet_data(data_dir,anno_file,prefix):


neg_save_dir = os.path.join(data_dir,"12/negative") neg_save_dir = os.path.join(data_dir,"12/negative")
pos_save_dir = os.path.join(data_dir,"12/positive") pos_save_dir = os.path.join(data_dir,"12/positive")
@@ -32,7 +32,7 @@ def gen_pnet_data(data_dir,anno_file):
annotations = f.readlines() annotations = f.readlines()


num = len(annotations) num = len(annotations)
print "%d pics in total" % num
print("%d pics in total" % num)
p_idx = 0 p_idx = 0
n_idx = 0 n_idx = 0
d_idx = 0 d_idx = 0
@@ -40,13 +40,13 @@ def gen_pnet_data(data_dir,anno_file):
box_idx = 0 box_idx = 0
for annotation in annotations: for annotation in annotations:
annotation = annotation.strip().split(' ') annotation = annotation.strip().split(' ')
im_path = annotation[0]
im_path = os.path.join(prefix,annotation[0])
bbox = map(float, annotation[1:]) bbox = map(float, annotation[1:])
boxes = np.array(bbox, dtype=np.int32).reshape(-1, 4) boxes = np.array(bbox, dtype=np.int32).reshape(-1, 4)
img = cv2.imread(im_path) img = cv2.imread(im_path)
idx += 1 idx += 1
if idx % 100 == 0: if idx % 100 == 0:
print idx, "images done"
print(idx, "images done")


height, width, channel = img.shape height, width, channel = img.shape


@@ -145,7 +145,7 @@ def gen_pnet_data(data_dir,anno_file):
cv2.imwrite(save_file, resized_im) cv2.imwrite(save_file, resized_im)
d_idx += 1 d_idx += 1
box_idx += 1 box_idx += 1
print "%s images done, pos: %s part: %s neg: %s"%(idx, p_idx, d_idx, n_idx)
print("%s images done, pos: %s part: %s neg: %s"%(idx, p_idx, d_idx, n_idx))


f1.close() f1.close()
f2.close() f2.close()
@@ -157,16 +157,19 @@ def parse_args():
parser = argparse.ArgumentParser(description='Test mtcnn', parser = argparse.ArgumentParser(description='Test mtcnn',
formatter_class=argparse.ArgumentDefaultsHelpFormatter) formatter_class=argparse.ArgumentDefaultsHelpFormatter)


parser.add_argument('--dataset_path', dest='dataset_path', help='dataset folder',
parser.add_argument('--dface_traindata_store', dest='traindata_store', help='dface train data temporary folder,include 12,24,48/postive,negative,part,landmark',
default='../data/wider/', type=str) default='../data/wider/', type=str)
parser.add_argument('--anno_file', dest='annotation_file', help='dataset original annotation file',
default='../data/wider/anno.txt', type=str)
parser.add_argument('--prefix_path', dest='prefix_path', help='image prefix root path',
parser.add_argument('--anno_file', dest='annotation_file', help='wider face original annotation file',
default=os.path.join(config.ANNO_STORE_DIR,"wider_origin_anno.txt"), type=str)
parser.add_argument('--prefix_path', dest='prefix_path', help='annotation file image prefix root path',
default='', type=str) default='', type=str)





args = parser.parse_args() args = parser.parse_args()
return args return args


if __name__ == '__main__': if __name__ == '__main__':
args = parse_args() args = parse_args()
gen_pnet_data(args.dataset_path,args.annotation_file)
gen_pnet_data(args.traindata_store,args.annotation_file,args.prefix_path)

+ 13
- 13
dface/prepare_data/gen_Rnet_train_data.py View File

@@ -28,7 +28,7 @@ def gen_rnet_data(data_dir, anno_file, pnet_model_file, prefix_path='', use_cuda


for databatch in image_reader: for databatch in image_reader:
if batch_idx % 100 == 0: if batch_idx % 100 == 0:
print "%d images done" % batch_idx
print ("%d images done" % batch_idx)
im = databatch im = databatch


t = time.time() t = time.time()
@@ -58,11 +58,11 @@ def gen_rnet_data(data_dir, anno_file, pnet_model_file, prefix_path='', use_cuda
cPickle.dump(all_boxes, f, cPickle.HIGHEST_PROTOCOL) cPickle.dump(all_boxes, f, cPickle.HIGHEST_PROTOCOL)




gen_rnet_sample_data(data_dir,anno_file,save_file)
gen_rnet_sample_data(data_dir,anno_file,save_file,prefix_path)






def gen_rnet_sample_data(data_dir,anno_file,det_boxs_file):
def gen_rnet_sample_data(data_dir,anno_file,det_boxs_file,prefix_path):


neg_save_dir = os.path.join(data_dir, "24/negative") neg_save_dir = os.path.join(data_dir, "24/negative")
pos_save_dir = os.path.join(data_dir, "24/positive") pos_save_dir = os.path.join(data_dir, "24/positive")
@@ -85,11 +85,11 @@ def gen_rnet_sample_data(data_dir,anno_file,det_boxs_file):
im_idx_list = list() im_idx_list = list()
gt_boxes_list = list() gt_boxes_list = list()
num_of_images = len(annotations) num_of_images = len(annotations)
print "processing %d images in total" % num_of_images
print ("processing %d images in total" % num_of_images)


for annotation in annotations: for annotation in annotations:
annotation = annotation.strip().split(' ') annotation = annotation.strip().split(' ')
im_idx = annotation[0]
im_idx = os.path.join(prefix_path,annotation[0])


boxes = map(float, annotation[1:]) boxes = map(float, annotation[1:])
boxes = np.array(boxes, dtype=np.float32).reshape(-1, 4) boxes = np.array(boxes, dtype=np.float32).reshape(-1, 4)
@@ -108,7 +108,7 @@ def gen_rnet_sample_data(data_dir,anno_file,det_boxs_file):
det_handle = open(det_boxs_file, 'r') det_handle = open(det_boxs_file, 'r')


det_boxes = cPickle.load(det_handle) det_boxes = cPickle.load(det_handle)
print len(det_boxes), num_of_images
print(len(det_boxes), num_of_images)
assert len(det_boxes) == num_of_images, "incorrect detections or ground truths" assert len(det_boxes) == num_of_images, "incorrect detections or ground truths"


# index of neg, pos and part face, used as their image names # index of neg, pos and part face, used as their image names
@@ -118,7 +118,7 @@ def gen_rnet_sample_data(data_dir,anno_file,det_boxs_file):
image_done = 0 image_done = 0
for im_idx, dets, gts in zip(im_idx_list, det_boxes, gt_boxes_list): for im_idx, dets, gts in zip(im_idx_list, det_boxes, gt_boxes_list):
if image_done % 100 == 0: if image_done % 100 == 0:
print "%d images done" % image_done
print("%d images done" % image_done)
image_done += 1 image_done += 1


if dets.shape[0] == 0: if dets.shape[0] == 0:
@@ -194,15 +194,15 @@ def parse_args():
parser = argparse.ArgumentParser(description='Test mtcnn', parser = argparse.ArgumentParser(description='Test mtcnn',
formatter_class=argparse.ArgumentDefaultsHelpFormatter) formatter_class=argparse.ArgumentDefaultsHelpFormatter)


parser.add_argument('--dataset_path', dest='dataset_path', help='dataset folder',
parser.add_argument('--dface_traindata_store', dest='traindata_store', help='dface train data temporary folder,include 12,24,48/postive,negative,part,landmark',
default='../data/wider/', type=str) default='../data/wider/', type=str)
parser.add_argument('--anno_file', dest='annotation_file', help='dataset original annotation file',
default='../data/wider/anno.txt', type=str)
parser.add_argument('--anno_file', dest='annotation_file', help='wider face original annotation file',
default=os.path.join(config.ANNO_STORE_DIR,"wider_origin_anno.txt"), type=str)
parser.add_argument('--pmodel_file', dest='pnet_model_file', help='PNet model file path', parser.add_argument('--pmodel_file', dest='pnet_model_file', help='PNet model file path',
default='/idata/workspace/mtcnn/model_store/pnet_epoch_5best.pt', type=str)
default='/idata/workspace/dface/model_store/pnet_epoch.pt', type=str)
parser.add_argument('--gpu', dest='use_cuda', help='with gpu', parser.add_argument('--gpu', dest='use_cuda', help='with gpu',
default=config.USE_CUDA, type=bool) default=config.USE_CUDA, type=bool)
parser.add_argument('--prefix_path', dest='prefix_path', help='image prefix root path',
parser.add_argument('--prefix_path', dest='prefix_path', help='annotation file image prefix root path',
default='', type=str) default='', type=str)




@@ -213,7 +213,7 @@ def parse_args():


if __name__ == '__main__': if __name__ == '__main__':
args = parse_args() args = parse_args()
gen_rnet_data(args.dataset_path, args.annotation_file, args.pnet_model_file, args.prefix_path, args.use_cuda)
gen_rnet_data(args.traindata_store, args.annotation_file, args.pnet_model_file, args.prefix_path, args.use_cuda)







+ 5
- 5
dface/prepare_data/gen_landmark_12.py View File

@@ -34,7 +34,7 @@ def gen_data(anno_file, data_dir, prefix):
annotations = f2.readlines() annotations = f2.readlines()


num = len(annotations) num = len(annotations)
print "%d pics in total" % num
print("%d pics in total" % num)


l_idx =0 l_idx =0
idx = 0 idx = 0
@@ -68,7 +68,7 @@ def gen_data(anno_file, data_dir, prefix):


idx = idx + 1 idx = idx + 1
if idx % 100 == 0: if idx % 100 == 0:
print "%d images done, landmark images: %d"%(idx,l_idx)
print("%d images done, landmark images: %d"%(idx,l_idx))


x1, y1, x2, y2 = gt_box x1, y1, x2, y2 = gt_box


@@ -137,11 +137,11 @@ def parse_args():
parser = argparse.ArgumentParser(description='Test mtcnn', parser = argparse.ArgumentParser(description='Test mtcnn',
formatter_class=argparse.ArgumentDefaultsHelpFormatter) formatter_class=argparse.ArgumentDefaultsHelpFormatter)


parser.add_argument('--dataset_path', dest='dataset_path', help='dataset folder',
parser.add_argument('--dface_traindata_store', dest='traindata_store', help='dataset folder',
default='../data/wider/', type=str) default='../data/wider/', type=str)
parser.add_argument('--anno_file', dest='annotation_file', help='dataset original annotation file', parser.add_argument('--anno_file', dest='annotation_file', help='dataset original annotation file',
default='../data/wider/anno.txt', type=str) default='../data/wider/anno.txt', type=str)
parser.add_argument('--prefix_path', dest='prefix_path', help='image prefix root path',
parser.add_argument('--prefix_path', dest='prefix_path', help='annotation file image prefix root path',
default='../data/', type=str) default='../data/', type=str)




@@ -151,6 +151,6 @@ def parse_args():
if __name__ == '__main__': if __name__ == '__main__':
args = parse_args() args = parse_args()


gen_data(args.annotation_file, args.dataset_path, args.prefix_path)
gen_data(args.annotation_file, args.traindata_store, args.prefix_path)





+ 5
- 5
dface/prepare_data/gen_landmark_24.py View File

@@ -35,7 +35,7 @@ def gen_data(anno_file, data_dir, prefix):
annotations = f2.readlines() annotations = f2.readlines()


num = len(annotations) num = len(annotations)
print "%d total images" % num
print("%d total images" % num)


l_idx =0 l_idx =0
idx = 0 idx = 0
@@ -67,7 +67,7 @@ def gen_data(anno_file, data_dir, prefix):


idx = idx + 1 idx = idx + 1
if idx % 100 == 0: if idx % 100 == 0:
print "%d images done, landmark images: %d"%(idx,l_idx)
print("%d images done, landmark images: %d"%(idx,l_idx))


x1, y1, x2, y2 = gt_box x1, y1, x2, y2 = gt_box


@@ -135,11 +135,11 @@ def parse_args():
parser = argparse.ArgumentParser(description='Test mtcnn', parser = argparse.ArgumentParser(description='Test mtcnn',
formatter_class=argparse.ArgumentDefaultsHelpFormatter) formatter_class=argparse.ArgumentDefaultsHelpFormatter)


parser.add_argument('--dataset_path', dest='dataset_path', help='dataset folder',
parser.add_argument('--dface_traindata_store', dest='traindata_store', help='dataset folder',
default='/idata/data/wider/', type=str) default='/idata/data/wider/', type=str)
parser.add_argument('--anno_file', dest='annotation_file', help='dataset original annotation file', parser.add_argument('--anno_file', dest='annotation_file', help='dataset original annotation file',
default='/idata/data/trainImageList.txt', type=str) default='/idata/data/trainImageList.txt', type=str)
parser.add_argument('--prefix_path', dest='prefix_path', help='image prefix root path',
parser.add_argument('--prefix_path', dest='prefix_path', help='annotation file image prefix root path',
default='/idata/data', type=str) default='/idata/data', type=str)




@@ -149,6 +149,6 @@ def parse_args():
if __name__ == '__main__': if __name__ == '__main__':
args = parse_args() args = parse_args()


gen_data(args.annotation_file, args.dataset_path, args.prefix_path)
gen_data(args.annotation_file, args.traindata_store, args.prefix_path)





+ 5
- 5
dface/prepare_data/gen_landmark_48.py View File

@@ -34,7 +34,7 @@ def gen_data(anno_file, data_dir, prefix):
annotations = f2.readlines() annotations = f2.readlines()


num = len(annotations) num = len(annotations)
print "%d total images" % num
print("%d total images" % num)


l_idx =0 l_idx =0
idx = 0 idx = 0
@@ -66,7 +66,7 @@ def gen_data(anno_file, data_dir, prefix):


idx = idx + 1 idx = idx + 1
if idx % 100 == 0: if idx % 100 == 0:
print "%d images done, landmark images: %d"%(idx,l_idx)
print("%d images done, landmark images: %d"%(idx,l_idx))


x1, y1, x2, y2 = gt_box x1, y1, x2, y2 = gt_box


@@ -134,11 +134,11 @@ def parse_args():
parser = argparse.ArgumentParser(description='Test mtcnn', parser = argparse.ArgumentParser(description='Test mtcnn',
formatter_class=argparse.ArgumentDefaultsHelpFormatter) formatter_class=argparse.ArgumentDefaultsHelpFormatter)


parser.add_argument('--dataset_path', dest='dataset_path', help='dataset folder',
parser.add_argument('--dface_traindata_store', dest='traindata_store', help='dataset folder',
default='/idata/data/wider/', type=str) default='/idata/data/wider/', type=str)
parser.add_argument('--anno_file', dest='annotation_file', help='dataset original annotation file', parser.add_argument('--anno_file', dest='annotation_file', help='dataset original annotation file',
default='/idata/data/trainImageList.txt', type=str) default='/idata/data/trainImageList.txt', type=str)
parser.add_argument('--prefix_path', dest='prefix_path', help='image prefix root path',
parser.add_argument('--prefix_path', dest='prefix_path', help='annotation file image prefix root path',
default='/idata/data', type=str) default='/idata/data', type=str)




@@ -148,6 +148,6 @@ def parse_args():
if __name__ == '__main__': if __name__ == '__main__':
args = parse_args() args = parse_args()


gen_data(args.annotation_file, args.dataset_path, args.prefix_path)
gen_data(args.annotation_file, args.traindata_store, args.prefix_path)





+ 6
- 6
dface/prepare_data/gen_landmark_net_48.py View File

@@ -27,7 +27,7 @@ def gen_landmark48_data(data_dir, anno_file, pnet_model_file, rnet_model_file, p


for databatch in image_reader: for databatch in image_reader:
if batch_idx % 100 == 0: if batch_idx % 100 == 0:
print "%d images done" % batch_idx
print("%d images done" % batch_idx)
im = databatch im = databatch




@@ -91,7 +91,7 @@ def gen_sample_data(data_dir, anno_file, det_boxs_file, prefix_path =''):
gt_boxes_list = list() gt_boxes_list = list()
gt_landmark_list = list() gt_landmark_list = list()
num_of_images = len(annotations) num_of_images = len(annotations)
print "processing %d images in total" % num_of_images
print("processing %d images in total" % num_of_images)


for annotation in annotations: for annotation in annotations:
annotation = annotation.strip().split(' ') annotation = annotation.strip().split(' ')
@@ -117,7 +117,7 @@ def gen_sample_data(data_dir, anno_file, det_boxs_file, prefix_path =''):
det_handle = open(det_boxs_file, 'r') det_handle = open(det_boxs_file, 'r')


det_boxes = cPickle.load(det_handle) det_boxes = cPickle.load(det_handle)
print len(det_boxes), num_of_images
print(len(det_boxes), num_of_images)
assert len(det_boxes) == num_of_images, "incorrect detections or ground truths" assert len(det_boxes) == num_of_images, "incorrect detections or ground truths"


# index of neg, pos and part face, used as their image names # index of neg, pos and part face, used as their image names
@@ -125,7 +125,7 @@ def gen_sample_data(data_dir, anno_file, det_boxs_file, prefix_path =''):
image_done = 0 image_done = 0
for im_idx, dets, gts, landmark in zip(im_idx_list, det_boxes, gt_boxes_list, gt_landmark_list): for im_idx, dets, gts, landmark in zip(im_idx_list, det_boxes, gt_boxes_list, gt_landmark_list):
if image_done % 100 == 0: if image_done % 100 == 0:
print "%d images done" % image_done
print("%d images done" % image_done)
image_done += 1 image_done += 1


if dets.shape[0] == 0: if dets.shape[0] == 0:
@@ -208,7 +208,7 @@ def parse_args():
parser = argparse.ArgumentParser(description='Test mtcnn', parser = argparse.ArgumentParser(description='Test mtcnn',
formatter_class=argparse.ArgumentDefaultsHelpFormatter) formatter_class=argparse.ArgumentDefaultsHelpFormatter)


parser.add_argument('--dataset_path', dest='dataset_path', help='dataset folder',
parser.add_argument('--dface_traindata_store', dest='traindata_store', help='dataset folder',
default='../data/wider/', type=str) default='../data/wider/', type=str)
parser.add_argument('--anno_file', dest='annotation_file', help='output data folder', parser.add_argument('--anno_file', dest='annotation_file', help='output data folder',
default='../data/wider/anno.txt', type=str) default='../data/wider/anno.txt', type=str)
@@ -228,7 +228,7 @@ def parse_args():


if __name__ == '__main__': if __name__ == '__main__':
args = parse_args() args = parse_args()
gen_landmark48_data(args.dataset_path, args.annotation_file, args.pnet_model_file, args.rnet_model_file, args.prefix_path, args.use_cuda)
gen_landmark48_data(args.traindata_store, args.annotation_file, args.pnet_model_file, args.rnet_model_file, args.prefix_path, args.use_cuda)







+ 0
- 0
dface/prepare_data/widerface_annotation_gen/__init__.py View File


+ 6
- 6
dface/prepare_data/widerface_annotation_gen/transform.py View File

@@ -1,4 +1,4 @@
from wider_loader import WIDER
from dface.prepare_data.widerface_annotation_gen.wider_loader import WIDER
import cv2 import cv2
import time import time


@@ -17,7 +17,7 @@ wider = WIDER(file_to_label, path_to_image)
line_count = 0 line_count = 0
box_count = 0 box_count = 0


print 'start transforming....'
print('start transforming....')
t = time.time() t = time.time()


with open(target_file, 'w+') as f: with open(target_file, 'w+') as f:
@@ -37,10 +37,10 @@ with open(target_file, 'w+') as f:
f.write(line_str) f.write(line_str)


st = time.time()-t st = time.time()-t
print 'end transforming'
print('end transforming')


print 'spend time:%ld'%st
print 'total line(images):%d'%line_count
print 'total boxes(faces):%d'%box_count
print('spend time:%ld'%st)
print('total line(images):%d'%line_count)
print('total boxes(faces):%d'%box_count)





+ 5
- 5
dface/train_net/train.py View File

@@ -84,7 +84,7 @@ def train_pnet(model_store_path, end_epoch,imdb,
show3 = box_offset_loss.data.tolist()[0] show3 = box_offset_loss.data.tolist()[0]
show5 = all_loss.data.tolist()[0] show5 = all_loss.data.tolist()[0]


print "%s : Epoch: %d, Step: %d, accuracy: %s, det loss: %s, bbox loss: %s, all_loss: %s, lr:%s "%(datetime.datetime.now(),cur_epoch,batch_idx, show1,show2,show3,show5,base_lr)
print("%s : Epoch: %d, Step: %d, accuracy: %s, det loss: %s, bbox loss: %s, all_loss: %s, lr:%s "%(datetime.datetime.now(),cur_epoch,batch_idx, show1,show2,show3,show5,base_lr))
accuracy_list.append(accuracy) accuracy_list.append(accuracy)
cls_loss_list.append(cls_loss) cls_loss_list.append(cls_loss)
bbox_loss_list.append(box_offset_loss) bbox_loss_list.append(box_offset_loss)
@@ -103,7 +103,7 @@ def train_pnet(model_store_path, end_epoch,imdb,
show7 = cls_loss_avg.data.tolist()[0] show7 = cls_loss_avg.data.tolist()[0]
show8 = bbox_loss_avg.data.tolist()[0] show8 = bbox_loss_avg.data.tolist()[0]


print "Epoch: %d, accuracy: %s, cls loss: %s, bbox loss: %s" % (cur_epoch, show6, show7, show8)
print("Epoch: %d, accuracy: %s, cls loss: %s, bbox loss: %s" % (cur_epoch, show6, show7, show8))
torch.save(net.state_dict(), os.path.join(model_store_path,"pnet_epoch_%d.pt" % cur_epoch)) torch.save(net.state_dict(), os.path.join(model_store_path,"pnet_epoch_%d.pt" % cur_epoch))
torch.save(net, os.path.join(model_store_path,"pnet_epoch_model_%d.pkl" % cur_epoch)) torch.save(net, os.path.join(model_store_path,"pnet_epoch_model_%d.pkl" % cur_epoch))


@@ -169,7 +169,7 @@ def train_rnet(model_store_path, end_epoch,imdb,
# show4 = landmark_loss.data.tolist()[0] # show4 = landmark_loss.data.tolist()[0]
show5 = all_loss.data.tolist()[0] show5 = all_loss.data.tolist()[0]


print "%s : Epoch: %d, Step: %d, accuracy: %s, det loss: %s, bbox loss: %s, all_loss: %s, lr:%s "%(datetime.datetime.now(), cur_epoch, batch_idx, show1, show2, show3, show5, base_lr)
print("%s : Epoch: %d, Step: %d, accuracy: %s, det loss: %s, bbox loss: %s, all_loss: %s, lr:%s "%(datetime.datetime.now(), cur_epoch, batch_idx, show1, show2, show3, show5, base_lr))
accuracy_list.append(accuracy) accuracy_list.append(accuracy)
cls_loss_list.append(cls_loss) cls_loss_list.append(cls_loss)
bbox_loss_list.append(box_offset_loss) bbox_loss_list.append(box_offset_loss)
@@ -190,7 +190,7 @@ def train_rnet(model_store_path, end_epoch,imdb,
show8 = bbox_loss_avg.data.tolist()[0] show8 = bbox_loss_avg.data.tolist()[0]
# show9 = landmark_loss_avg.data.tolist()[0] # show9 = landmark_loss_avg.data.tolist()[0]


print "Epoch: %d, accuracy: %s, cls loss: %s, bbox loss: %s" % (cur_epoch, show6, show7, show8)
print("Epoch: %d, accuracy: %s, cls loss: %s, bbox loss: %s" % (cur_epoch, show6, show7, show8))
torch.save(net.state_dict(), os.path.join(model_store_path,"rnet_epoch_%d.pt" % cur_epoch)) torch.save(net.state_dict(), os.path.join(model_store_path,"rnet_epoch_%d.pt" % cur_epoch))
torch.save(net, os.path.join(model_store_path,"rnet_epoch_model_%d.pkl" % cur_epoch)) torch.save(net, os.path.join(model_store_path,"rnet_epoch_model_%d.pkl" % cur_epoch))


@@ -275,7 +275,7 @@ def train_onet(model_store_path, end_epoch,imdb,
show8 = bbox_loss_avg.data.tolist()[0] show8 = bbox_loss_avg.data.tolist()[0]
show9 = landmark_loss_avg.data.tolist()[0] show9 = landmark_loss_avg.data.tolist()[0]


print "Epoch: %d, accuracy: %s, cls loss: %s, bbox loss: %s, landmark loss: %s " % (cur_epoch, show6, show7, show8, show9)
print("Epoch: %d, accuracy: %s, cls loss: %s, bbox loss: %s, landmark loss: %s " % (cur_epoch, show6, show7, show8, show9))
torch.save(net.state_dict(), os.path.join(model_store_path,"onet_epoch_%d.pt" % cur_epoch)) torch.save(net.state_dict(), os.path.join(model_store_path,"onet_epoch_%d.pt" % cur_epoch))
torch.save(net, os.path.join(model_store_path,"onet_epoch_model_%d.pkl" % cur_epoch)) torch.save(net, os.path.join(model_store_path,"onet_epoch_model_%d.pkl" % cur_epoch))



+ 1
- 1
dface/train_net/train_o_net.py View File

@@ -42,7 +42,7 @@ def parse_args():


if __name__ == '__main__': if __name__ == '__main__':
args = parse_args() args = parse_args()
print 'train ONet argument:'
print('train ONet argument:')
print args print args






+ 2
- 2
dface/train_net/train_p_net.py View File

@@ -2,7 +2,7 @@ import argparse
import sys import sys
from dface.core.imagedb import ImageDB from dface.core.imagedb import ImageDB
from train import train_pnet from train import train_pnet
import dface.config
import dface.config as config
import os import os




@@ -42,7 +42,7 @@ def parse_args():


if __name__ == '__main__': if __name__ == '__main__':
args = parse_args() args = parse_args()
print 'train Pnet argument:'
print('train Pnet argument:')
print args print args


train_net(annotation_file=args.annotation_file, model_store_path=args.model_store_path, train_net(annotation_file=args.annotation_file, model_store_path=args.model_store_path,


+ 2
- 2
dface/train_net/train_r_net.py View File

@@ -2,7 +2,7 @@ import argparse
import sys import sys
from dface.core.imagedb import ImageDB from dface.core.imagedb import ImageDB
import train as train import train as train
import dface.config
import dface.config as config
import os import os




@@ -42,7 +42,7 @@ def parse_args():


if __name__ == '__main__': if __name__ == '__main__':
args = parse_args() args = parse_args()
print 'train Rnet argument:'
print('train Rnet argument:')
print args print args






+ 0
- 1
test_image.py View File

@@ -3,7 +3,6 @@ from dface.core.detect import create_mtcnn_net, MtcnnDetector
from dface.core import vision from dface.core import vision





if __name__ == '__main__': if __name__ == '__main__':


# refer to your local model path # refer to your local model path


Loading…
Cancel
Save