From a5e57fa22ef52262d9dabbf94dd055d78442f3d0 Mon Sep 17 00:00:00 2001 From: nihuini Date: Fri, 16 Nov 2018 11:13:38 +0800 Subject: [PATCH] implement multiscale yolov2, update example model comment --- examples/yolov2.cpp | 17 +-- src/layer/yolodetectionoutput.cpp | 166 ++++++++++++++++-------------- src/layer/yolodetectionoutput.h | 2 +- 3 files changed, 92 insertions(+), 93 deletions(-) diff --git a/examples/yolov2.cpp b/examples/yolov2.cpp index 354aff13b..4be9c9a2f 100644 --- a/examples/yolov2.cpp +++ b/examples/yolov2.cpp @@ -31,18 +31,11 @@ static int detect_yolov2(const cv::Mat& bgr, std::vector& objects) { ncnn::Net yolov2; - // original pretrained model from https://github.com/eric612/Caffe-YOLOv2-Windows - // yolov2_deploy.prototxt - // yolov2_deploy_iter_30000.caffemodel - // https://drive.google.com/file/d/17w7oZBbTHPI5TMuD9DKQzkPhSVDaTlC9/view?usp=sharing - yolov2.load_param("yolov2.param"); - yolov2.load_model("yolov2.bin"); - - // https://github.com/eric612/MobileNet-YOLO - // https://github.com/eric612/MobileNet-YOLO/blob/master/models/yolov2/mobilenet_yolo_deploy%20.prototxt - // https://github.com/eric612/MobileNet-YOLO/blob/master/models/yolov2/mobilenet_yolo_deploy_iter_57000.caffemodel -// yolov2.load_param("mobilenet_yolo.param"); -// yolov2.load_model("mobilenet_yolo.bin"); + // original pretrained model from https://github.com/eric612/MobileNet-YOLO + // https://github.com/eric612/MobileNet-YOLO/blob/master/models/yolov2/mobilenet_yolo_deploy.prototxt + // https://github.com/eric612/MobileNet-YOLO/blob/master/models/yolov2/mobilenet_yolo_deploy_iter_80000.caffemodel + yolov2.load_param("mobilenet_yolo.param"); + yolov2.load_model("mobilenet_yolo.bin"); const int target_size = 416; diff --git a/src/layer/yolodetectionoutput.cpp b/src/layer/yolodetectionoutput.cpp index 0d4f0e67d..faf7ba056 100644 --- a/src/layer/yolodetectionoutput.cpp +++ b/src/layer/yolodetectionoutput.cpp @@ -23,7 +23,7 @@ DEFINE_LAYER_CREATOR(YoloDetectionOutput) YoloDetectionOutput::YoloDetectionOutput() { - one_blob_only = true; + one_blob_only = false; support_inplace = true; softmax = ncnn::create_layer(ncnn::LayerType::Softmax); @@ -160,104 +160,109 @@ static inline float sigmoid(float x) return 1.f / (1.f + exp(-x)); } -int YoloDetectionOutput::forward_inplace(Mat& bottom_top_blob, const Option& opt) const +int YoloDetectionOutput::forward_inplace(std::vector& bottom_top_blobs, const Option& opt) const { - int w = bottom_top_blob.w; - int h = bottom_top_blob.h; - int channels = bottom_top_blob.c; + // gather all box + std::vector all_bbox_rects; + std::vector all_bbox_scores; - const int channels_per_box = channels / num_box; + for (size_t b=0; b > all_box_bbox_rects; - std::vector< std::vector > all_box_bbox_scores; - all_box_bbox_rects.resize(num_box); - all_box_bbox_scores.resize(num_box); + const int channels_per_box = channels / num_box; - #pragma omp parallel for num_threads(opt.num_threads) - for (int pp = 0; pp < num_box; pp++) - { - int p = pp * channels_per_box; + // anchor coord + box score + num_class + if (channels_per_box != 4 + 1 + num_class) + return -1; + + std::vector< std::vector > all_box_bbox_rects; + std::vector< std::vector > all_box_bbox_scores; + all_box_bbox_rects.resize(num_box); + all_box_bbox_scores.resize(num_box); - const float bias_w = biases[pp*2]; - const float bias_h = biases[pp*2+1]; + #pragma omp parallel for num_threads(opt.num_threads) + for (int pp = 0; pp < num_box; pp++) + { + int p = pp * channels_per_box; - const float* xptr = bottom_top_blob.channel(p); - const float* yptr = bottom_top_blob.channel(p+1); - const float* wptr = bottom_top_blob.channel(p+2); - const float* hptr = bottom_top_blob.channel(p+3); + const float bias_w = biases[pp*2]; + const float bias_h = biases[pp*2+1]; - const float* box_score_ptr = bottom_top_blob.channel(p+4); + const float* xptr = bottom_top_blob.channel(p); + const float* yptr = bottom_top_blob.channel(p+1); + const float* wptr = bottom_top_blob.channel(p+2); + const float* hptr = bottom_top_blob.channel(p+3); - // softmax class scores - Mat scores = bottom_top_blob.channel_range(p+5, num_class); - softmax->forward_inplace(scores, opt); + const float* box_score_ptr = bottom_top_blob.channel(p+4); - for (int i = 0; i < h; i++) - { - for (int j = 0; j < w; j++) + // softmax class scores + Mat scores = bottom_top_blob.channel_range(p+5, num_class); + softmax->forward_inplace(scores, opt); + + for (int i = 0; i < h; i++) { - // region box - float bbox_cx = (j + sigmoid(xptr[0])) / w; - float bbox_cy = (i + sigmoid(yptr[0])) / h; - float bbox_w = exp(wptr[0]) * bias_w / w; - float bbox_h = exp(hptr[0]) * bias_h / h; - - float bbox_xmin = bbox_cx - bbox_w * 0.5f; - float bbox_ymin = bbox_cy - bbox_h * 0.5f; - float bbox_xmax = bbox_cx + bbox_w * 0.5f; - float bbox_ymax = bbox_cy + bbox_h * 0.5f; - - // box score - float box_score = sigmoid(box_score_ptr[0]); - - // find class index with max class score - int class_index = 0; - float class_score = 0.f; - for (int q = 0; q < num_class; q++) + for (int j = 0; j < w; j++) { - float score = scores.channel(q).row(i)[j]; - if (score > class_score) + // region box + float bbox_cx = (j + sigmoid(xptr[0])) / w; + float bbox_cy = (i + sigmoid(yptr[0])) / h; + float bbox_w = exp(wptr[0]) * bias_w / w; + float bbox_h = exp(hptr[0]) * bias_h / h; + + float bbox_xmin = bbox_cx - bbox_w * 0.5f; + float bbox_ymin = bbox_cy - bbox_h * 0.5f; + float bbox_xmax = bbox_cx + bbox_w * 0.5f; + float bbox_ymax = bbox_cy + bbox_h * 0.5f; + + // box score + float box_score = sigmoid(box_score_ptr[0]); + + // find class index with max class score + int class_index = 0; + float class_score = 0.f; + for (int q = 0; q < num_class; q++) { - class_index = q; - class_score = score; + float score = scores.channel(q).row(i)[j]; + if (score > class_score) + { + class_index = q; + class_score = score; + } } - } -// fprintf(stderr, "%d %f %f\n", class_index, box_score, class_score); + // fprintf(stderr, "%d %f %f\n", class_index, box_score, class_score); - float confidence = box_score * class_score; - if (confidence >= confidence_threshold) - { - BBoxRect c = { bbox_xmin, bbox_ymin, bbox_xmax, bbox_ymax, class_index }; - all_box_bbox_rects[pp].push_back(c); - all_box_bbox_scores[pp].push_back(confidence); - } + float confidence = box_score * class_score; + if (confidence >= confidence_threshold) + { + BBoxRect c = { bbox_xmin, bbox_ymin, bbox_xmax, bbox_ymax, class_index }; + all_box_bbox_rects[pp].push_back(c); + all_box_bbox_scores[pp].push_back(confidence); + } - xptr++; - yptr++; - wptr++; - hptr++; + xptr++; + yptr++; + wptr++; + hptr++; - box_score_ptr++; + box_score_ptr++; + } } } - } - // gather all box - std::vector all_bbox_rects; - std::vector all_bbox_scores; - - for (int i = 0; i < num_box; i++) - { - const std::vector& box_bbox_rects = all_box_bbox_rects[i]; - const std::vector& box_bbox_scores = all_box_bbox_scores[i]; + for (int i = 0; i < num_box; i++) + { + const std::vector& box_bbox_rects = all_box_bbox_rects[i]; + const std::vector& box_bbox_scores = all_box_bbox_scores[i]; - all_bbox_rects.insert(all_bbox_rects.end(), box_bbox_rects.begin(), box_bbox_rects.end()); - all_bbox_scores.insert(all_bbox_scores.end(), box_bbox_scores.begin(), box_bbox_scores.end()); + all_bbox_rects.insert(all_bbox_rects.end(), box_bbox_rects.begin(), box_bbox_rects.end()); + all_bbox_scores.insert(all_bbox_scores.end(), box_bbox_scores.begin(), box_bbox_scores.end()); + } } // global sort inplace @@ -281,15 +286,16 @@ int YoloDetectionOutput::forward_inplace(Mat& bottom_top_blob, const Option& opt // fill result int num_detected = bbox_rects.size(); - bottom_top_blob.create(6, num_detected, 4u, opt.blob_allocator); - if (bottom_top_blob.empty()) + Mat& top_blob = bottom_top_blobs[0]; + top_blob.create(6, num_detected, 4u, opt.blob_allocator); + if (top_blob.empty()) return -100; for (int i = 0; i < num_detected; i++) { const BBoxRect& r = bbox_rects[i]; float score = bbox_scores[i]; - float* outptr = bottom_top_blob.row(i); + float* outptr = top_blob.row(i); outptr[0] = r.label + 1;// +1 for prepend background class outptr[1] = score; diff --git a/src/layer/yolodetectionoutput.h b/src/layer/yolodetectionoutput.h index 8513ca30d..652692c5b 100644 --- a/src/layer/yolodetectionoutput.h +++ b/src/layer/yolodetectionoutput.h @@ -27,7 +27,7 @@ public: virtual int load_param(const ParamDict& pd); - virtual int forward_inplace(Mat& bottom_top_blob, const Option& opt) const; + virtual int forward_inplace(std::vector& bottom_top_blobs, const Option& opt) const; public: int num_class;