implement multiscale yolov2, update example model comment

examples/yolov2.cpp

@@ -31,18 +31,11 @@ static int detect_yolov2(const cv::Mat& bgr, std::vector<Object>& objects)
 {
     ncnn::Net yolov2;
 
-    // original pretrained model from https://github.com/eric612/Caffe-YOLOv2-Windows
-    // yolov2_deploy.prototxt
-    // yolov2_deploy_iter_30000.caffemodel
-    // https://drive.google.com/file/d/17w7oZBbTHPI5TMuD9DKQzkPhSVDaTlC9/view?usp=sharing
-    yolov2.load_param("yolov2.param");
-    yolov2.load_model("yolov2.bin");
-
-    // https://github.com/eric612/MobileNet-YOLO
-    // https://github.com/eric612/MobileNet-YOLO/blob/master/models/yolov2/mobilenet_yolo_deploy%20.prototxt
-    // https://github.com/eric612/MobileNet-YOLO/blob/master/models/yolov2/mobilenet_yolo_deploy_iter_57000.caffemodel
-//     yolov2.load_param("mobilenet_yolo.param");
-//     yolov2.load_model("mobilenet_yolo.bin");
+    // original pretrained model from https://github.com/eric612/MobileNet-YOLO
+    // https://github.com/eric612/MobileNet-YOLO/blob/master/models/yolov2/mobilenet_yolo_deploy.prototxt
+    // https://github.com/eric612/MobileNet-YOLO/blob/master/models/yolov2/mobilenet_yolo_deploy_iter_80000.caffemodel
+    yolov2.load_param("mobilenet_yolo.param");
+    yolov2.load_model("mobilenet_yolo.bin");
 
     const int target_size = 416;
 

src/layer/yolodetectionoutput.cpp

@@ -23,7 +23,7 @@ DEFINE_LAYER_CREATOR(YoloDetectionOutput)
 
 YoloDetectionOutput::YoloDetectionOutput()
 {
-    one_blob_only = true;
+    one_blob_only = false;
     support_inplace = true;
 
     softmax = ncnn::create_layer(ncnn::LayerType::Softmax);
@@ -160,104 +160,109 @@ static inline float sigmoid(float x)
     return 1.f / (1.f + exp(-x));
 }
 
-int YoloDetectionOutput::forward_inplace(Mat& bottom_top_blob, const Option& opt) const
+int YoloDetectionOutput::forward_inplace(std::vector<Mat>& bottom_top_blobs, const Option& opt) const
 {
-    int w = bottom_top_blob.w;
-    int h = bottom_top_blob.h;
-    int channels = bottom_top_blob.c;
+    // gather all box
+    std::vector<BBoxRect> all_bbox_rects;
+    std::vector<float> all_bbox_scores;
 
-    const int channels_per_box = channels / num_box;
+    for (size_t b=0; b<bottom_top_blobs.size(); b++)
+    {
+        Mat& bottom_top_blob = bottom_top_blobs[b];
 
-    // anchor coord + box score + num_class
-    if (channels_per_box != 4 + 1 + num_class)
-        return -1;
+        int w = bottom_top_blob.w;
+        int h = bottom_top_blob.h;
+        int channels = bottom_top_blob.c;
 
-    std::vector< std::vector<BBoxRect> > all_box_bbox_rects;
-    std::vector< std::vector<float> > all_box_bbox_scores;
-    all_box_bbox_rects.resize(num_box);
-    all_box_bbox_scores.resize(num_box);
+        const int channels_per_box = channels / num_box;
 
-    #pragma omp parallel for num_threads(opt.num_threads)
-    for (int pp = 0; pp < num_box; pp++)
-    {
-        int p = pp * channels_per_box;
+        // anchor coord + box score + num_class
+        if (channels_per_box != 4 + 1 + num_class)
+            return -1;
+
+        std::vector< std::vector<BBoxRect> > all_box_bbox_rects;
+        std::vector< std::vector<float> > all_box_bbox_scores;
+        all_box_bbox_rects.resize(num_box);
+        all_box_bbox_scores.resize(num_box);
 
-        const float bias_w = biases[pp*2];
-        const float bias_h = biases[pp*2+1];
+        #pragma omp parallel for num_threads(opt.num_threads)
+        for (int pp = 0; pp < num_box; pp++)
+        {
+            int p = pp * channels_per_box;
 
-        const float* xptr = bottom_top_blob.channel(p);
-        const float* yptr = bottom_top_blob.channel(p+1);
-        const float* wptr = bottom_top_blob.channel(p+2);
-        const float* hptr = bottom_top_blob.channel(p+3);
+            const float bias_w = biases[pp*2];
+            const float bias_h = biases[pp*2+1];
 
-        const float* box_score_ptr = bottom_top_blob.channel(p+4);
+            const float* xptr = bottom_top_blob.channel(p);
+            const float* yptr = bottom_top_blob.channel(p+1);
+            const float* wptr = bottom_top_blob.channel(p+2);
+            const float* hptr = bottom_top_blob.channel(p+3);
 
-        // softmax class scores
-        Mat scores = bottom_top_blob.channel_range(p+5, num_class);
-        softmax->forward_inplace(scores, opt);
+            const float* box_score_ptr = bottom_top_blob.channel(p+4);
 
-        for (int i = 0; i < h; i++)
-        {
-            for (int j = 0; j < w; j++)
+            // softmax class scores
+            Mat scores = bottom_top_blob.channel_range(p+5, num_class);
+            softmax->forward_inplace(scores, opt);
+
+            for (int i = 0; i < h; i++)
             {
-                // region box
-                float bbox_cx = (j + sigmoid(xptr[0])) / w;
-                float bbox_cy = (i + sigmoid(yptr[0])) / h;
-                float bbox_w = exp(wptr[0]) * bias_w / w;
-                float bbox_h = exp(hptr[0]) * bias_h / h;
-
-                float bbox_xmin = bbox_cx - bbox_w * 0.5f;
-                float bbox_ymin = bbox_cy - bbox_h * 0.5f;
-                float bbox_xmax = bbox_cx + bbox_w * 0.5f;
-                float bbox_ymax = bbox_cy + bbox_h * 0.5f;
-
-                // box score
-                float box_score = sigmoid(box_score_ptr[0]);
-
-                // find class index with max class score
-                int class_index = 0;
-                float class_score = 0.f;
-                for (int q = 0; q < num_class; q++)
+                for (int j = 0; j < w; j++)
                 {
-                    float score = scores.channel(q).row(i)[j];
-                    if (score > class_score)
+                    // region box
+                    float bbox_cx = (j + sigmoid(xptr[0])) / w;
+                    float bbox_cy = (i + sigmoid(yptr[0])) / h;
+                    float bbox_w = exp(wptr[0]) * bias_w / w;
+                    float bbox_h = exp(hptr[0]) * bias_h / h;
+
+                    float bbox_xmin = bbox_cx - bbox_w * 0.5f;
+                    float bbox_ymin = bbox_cy - bbox_h * 0.5f;
+                    float bbox_xmax = bbox_cx + bbox_w * 0.5f;
+                    float bbox_ymax = bbox_cy + bbox_h * 0.5f;
+
+                    // box score
+                    float box_score = sigmoid(box_score_ptr[0]);
+
+                    // find class index with max class score
+                    int class_index = 0;
+                    float class_score = 0.f;
+                    for (int q = 0; q < num_class; q++)
                     {
-                        class_index = q;
-                        class_score = score;
+                        float score = scores.channel(q).row(i)[j];
+                        if (score > class_score)
+                        {
+                            class_index = q;
+                            class_score = score;
+                        }
                     }
-                }
 
-//                 fprintf(stderr, "%d %f %f\n", class_index, box_score, class_score);
+    //                 fprintf(stderr, "%d %f %f\n", class_index, box_score, class_score);
 
-                float confidence = box_score * class_score;
-                if (confidence >= confidence_threshold)
-                {
-                    BBoxRect c = { bbox_xmin, bbox_ymin, bbox_xmax, bbox_ymax, class_index };
-                    all_box_bbox_rects[pp].push_back(c);
-                    all_box_bbox_scores[pp].push_back(confidence);
-                }
+                    float confidence = box_score * class_score;
+                    if (confidence >= confidence_threshold)
+                    {
+                        BBoxRect c = { bbox_xmin, bbox_ymin, bbox_xmax, bbox_ymax, class_index };
+                        all_box_bbox_rects[pp].push_back(c);
+                        all_box_bbox_scores[pp].push_back(confidence);
+                    }
 
-                xptr++;
-                yptr++;
-                wptr++;
-                hptr++;
+                    xptr++;
+                    yptr++;
+                    wptr++;
+                    hptr++;
 
-                box_score_ptr++;
+                    box_score_ptr++;
+                }
             }
         }
-    }
 
-    // gather all box
-    std::vector<BBoxRect> all_bbox_rects;
-    std::vector<float> all_bbox_scores;
-
-    for (int i = 0; i < num_box; i++)
-    {
-        const std::vector<BBoxRect>& box_bbox_rects = all_box_bbox_rects[i];
-        const std::vector<float>& box_bbox_scores = all_box_bbox_scores[i];
+        for (int i = 0; i < num_box; i++)
+        {
+            const std::vector<BBoxRect>& box_bbox_rects = all_box_bbox_rects[i];
+            const std::vector<float>& box_bbox_scores = all_box_bbox_scores[i];
 
-        all_bbox_rects.insert(all_bbox_rects.end(), box_bbox_rects.begin(), box_bbox_rects.end());
-        all_bbox_scores.insert(all_bbox_scores.end(), box_bbox_scores.begin(), box_bbox_scores.end());
+            all_bbox_rects.insert(all_bbox_rects.end(), box_bbox_rects.begin(), box_bbox_rects.end());
+            all_bbox_scores.insert(all_bbox_scores.end(), box_bbox_scores.begin(), box_bbox_scores.end());
+        }
     }
 
     // global sort inplace
@@ -281,15 +286,16 @@ int YoloDetectionOutput::forward_inplace(Mat& bottom_top_blob, const Option& opt
     // fill result
     int num_detected = bbox_rects.size();
 
-    bottom_top_blob.create(6, num_detected, 4u, opt.blob_allocator);
-    if (bottom_top_blob.empty())
+    Mat& top_blob = bottom_top_blobs[0];
+    top_blob.create(6, num_detected, 4u, opt.blob_allocator);
+    if (top_blob.empty())
         return -100;
 
     for (int i = 0; i < num_detected; i++)
     {
         const BBoxRect& r = bbox_rects[i];
         float score = bbox_scores[i];
-        float* outptr = bottom_top_blob.row(i);
+        float* outptr = top_blob.row(i);
 
         outptr[0] = r.label + 1;// +1 for prepend background class
         outptr[1] = score;

src/layer/yolodetectionoutput.h

@@ -27,7 +27,7 @@ public:
 
     virtual int load_param(const ParamDict& pd);
 
-    virtual int forward_inplace(Mat& bottom_top_blob, const Option& opt) const;
+    virtual int forward_inplace(std::vector<Mat>& bottom_top_blobs, const Option& opt) const;
 
 public:
     int num_class;