// // Created by Jack Yu on 16/10/2017. // #include "../include/PlateSegmentation.h" #include "../include/niBlackThreshold.h" //#define DEBUG namespace pr{ PlateSegmentation::PlateSegmentation(std::string prototxt,std::string caffemodel) { net = cv::dnn::readNetFromCaffe(prototxt, caffemodel); } cv::Mat PlateSegmentation::classifyResponse(const cv::Mat &cropped){ cv::Mat inputBlob = cv::dnn::blobFromImage(cropped, 1/255.0, cv::Size(22,22), cv::Scalar(0,0,0),false); net.setInput(inputBlob,"data"); return net.forward(); } void drawHist(float* seq,int size,const char* name){ cv::Mat image(300,size,CV_8U); image.setTo(0); float* start =seq; float* end = seq+size; float l = *std::max_element(start,end); for(int i = 0;i>1),rect.y + (rect.height>>1)); int rebuildLeft = (rect.width>>1 )+ left; int rebuildRight = (rect.width>>1 )+ right; int rebuildTop = (rect.height>>1 )+ top; int rebuildBottom = (rect.height>>1 )+ bottom; return boxFromCenter(center,rebuildLeft,rebuildRight,rebuildTop,rebuildBottom,bdSize); } void PlateSegmentation:: refineRegion(cv::Mat &plateImage,const std::vector &candidatePts,const int padding,std::vector &rects){ int w = candidatePts[5] - candidatePts[4]; int cols = plateImage.cols; int rows = plateImage.rows; for(int i = 0 ; i < candidatePts.size() ; i++) { int left = 0; int right = 0 ; if(i == 0 ){ left= candidatePts[i]; right = left+w+padding; } else { left = candidatePts[i] - padding; right = left + w + padding * 2; } computeSafeMargin(right,cols); computeSafeMargin(left,cols); cv::Rect roi(left,0,right - left,rows-1); cv::Mat roiImage; plateImage(roi).copyTo(roiImage); if (i>=1) { cv::Mat roi_thres; // cv::threshold(roiImage,roi_thres,0,255,cv::THRESH_OTSU|cv::THRESH_BINARY); niBlackThreshold(roiImage,roi_thres,255,cv::THRESH_BINARY,15,0.27,BINARIZATION_NIBLACK); std::vector> contours; cv::findContours(roi_thres,contours,cv::RETR_LIST,cv::CHAIN_APPROX_SIMPLE); cv::Point boxCenter(roiImage.cols>>1,roiImage.rows>>1); cv::Rect final_bdbox; cv::Point final_center; int final_dist = INT_MAX; for(auto contour:contours) { cv::Rect bdbox = cv::boundingRect(contour); cv::Point center(bdbox.x+(bdbox.width>>1),bdbox.y + (bdbox.height>>1)); int dist = (center.x - boxCenter.x)*(center.x - boxCenter.x); if(dist rows>>1) { final_dist =dist; final_center = center; final_bdbox = bdbox; } } //rebuild box if(final_bdbox.height/ static_cast(final_bdbox.width) > 3.5 && final_bdbox.width*final_bdbox.height<10) final_bdbox = boxFromCenter(final_center,8,8,(rows>>1)-3 , (rows>>1) - 2,roiImage.size()); else { if(i == candidatePts.size()-1) final_bdbox = boxPadding(final_bdbox, padding/2, padding, padding/2, padding/2, roiImage.size()); else final_bdbox = boxPadding(final_bdbox, padding, padding, padding, padding, roiImage.size()); // std::cout<0&&i+j+r> &candidatePts){ int rows = respones.rows; int cols = respones.cols; float *data = (float*)respones.data; float *engNum_prob = data; float *false_prob = data+cols; float *ch_prob = data+cols*2; avgfilter(engNum_prob,cols,5); avgfilter(false_prob,cols,5); // avgfilter(ch_prob,cols,5); std::vector candidate_pts(7); #ifdef DEBUG drawHist(engNum_prob,cols,"engNum_prob"); drawHist(false_prob,cols,"false_prob"); drawHist(ch_prob,cols,"ch_prob"); cv::waitKey(0); #endif int cp_list[7]; float loss_selected = -10; for(int start = 0 ; start < 20 ; start+=2) for(int width = windowsWidth-5; width < windowsWidth+5 ; width++ ){ for(int interval = windowsWidth/2; interval < windowsWidth; interval++) { int cp1_ch = start; int cp2_p0 = cp1_ch+ width; int cp3_p1 = cp2_p0+ width + interval; int cp4_p2 = cp3_p1 + width; int cp5_p3 = cp4_p2 + width+1; int cp6_p4 = cp5_p3 + width+2; int cp7_p5= cp6_p4+ width+2; int md1 = (cp1_ch+cp2_p0)>>1; int md2 = (cp2_p0+cp3_p1)>>1; int md3 = (cp3_p1+cp4_p2)>>1; int md4 = (cp4_p2+cp5_p3)>>1; int md5 = (cp5_p3+cp6_p4)>>1; int md6 = (cp6_p4+cp7_p5)>>1; if(cp7_p5>=cols) continue; // float loss = ch_prob[cp1_ch]+ // engNum_prob[cp2_p0] +engNum_prob[cp3_p1]+engNum_prob[cp4_p2]+engNum_prob[cp5_p3]+engNum_prob[cp6_p4] +engNum_prob[cp7_p5] // + (false_prob[md2]+false_prob[md3]+false_prob[md4]+false_prob[md5]+false_prob[md5] + false_prob[md6]); float loss = ch_prob[cp1_ch]*3 -(false_prob[cp3_p1]+false_prob[cp4_p2]+false_prob[cp5_p3]+false_prob[cp6_p4]+false_prob[cp7_p5]); if(loss>loss_selected) { loss_selected = loss; cp_list[0]= cp1_ch; cp_list[1]= cp2_p0; cp_list[2]= cp3_p1; cp_list[3]= cp4_p2; cp_list[4]= cp5_p3; cp_list[5]= cp6_p4; cp_list[6]= cp7_p5; } } } candidate_pts[0] = cp_list[0]; candidate_pts[1] = cp_list[1]; candidate_pts[2] = cp_list[2]; candidate_pts[3] = cp_list[3]; candidate_pts[4] = cp_list[4]; candidate_pts[5] = cp_list[5]; candidate_pts[6] = cp_list[6]; candidatePts.first = loss_selected; candidatePts.second = candidate_pts; }; void PlateSegmentation::segmentPlateBySlidingWindows(cv::Mat &plateImage,int windowsWidth,int stride,cv::Mat &respones){ // cv::resize(plateImage,plateImage,cv::Size(136,36)); cv::Mat plateImageGray; cv::cvtColor(plateImage,plateImageGray,cv::COLOR_BGR2GRAY); int padding = plateImage.cols-136 ; // int padding = 0 ; int height = plateImage.rows - 1; int width = plateImage.cols - 1 - padding; for(int i = 0 ; i < width - windowsWidth +1 ; i +=stride) { cv::Rect roi(i,0,windowsWidth,height); cv::Mat roiImage = plateImageGray(roi); cv::Mat response = classifyResponse(roiImage); respones.push_back(response); } respones = respones.t(); // std::pair> images ; // // // std::cout< &Char_rects){ cv::Mat plateImage = plateInfo.getPlateImage(); // get src image . cv::Mat plateImageGray; cv::cvtColor(plateImage,plateImageGray,cv::COLOR_BGR2GRAY); //do binarzation // std::pair> sections ; // segment points variables . cv::Mat respones; //three response of every sub region from origin image . segmentPlateBySlidingWindows(plateImage,DEFAULT_WIDTH,1,respones); templateMatchFinding(respones,DEFAULT_WIDTH/stride,sections); for(int i = 0; i < sections.second.size() ; i++) { sections.second[i]*=stride; } // std::cout< &rects){ cv::Mat plateImage = plateInfo.getPlateImage(); for(int i = 0 ; i < rects.size() ; i++){ cv::Mat charImage; plateImage(rects[i]).copyTo(charImage); if(charImage.channels()) cv::cvtColor(charImage,charImage,cv::COLOR_BGR2GRAY); // cv::imshow("image",charImage); // cv::waitKey(0); cv::equalizeHist(charImage,charImage); // // std::pair char_instance; if(i == 0 ){ char_instance.first = CHINESE; } else if(i == 1){ char_instance.first = LETTER; } else{ char_instance.first = LETTER_NUMS; } char_instance.second = charImage; plateInfo.appendPlateChar(char_instance); } } }//namespace pr