mips msa optimization for convolutiondepthwise innerproduct int8 (#3679)

src/layer/mips/convolutiondepthwise_mips.cpp

@@ -46,6 +46,13 @@ int ConvolutionDepthWise_mips::create_pipeline(const Option& opt)
 
     activation = create_activation_layer(activation_type, activation_params, opt);
 
+#if NCNN_INT8
+    if (opt.use_int8_inference && weight_data.elemsize == (size_t)1u)
+    {
+        return create_pipeline_int8_mips(opt);
+    }
+#endif
+
     const int maxk = kernel_w * kernel_h;
     int channels = (weight_data_size / group) / maxk / (num_output / group) * group;
 
@@ -203,27 +210,7 @@ int ConvolutionDepthWise_mips::forward(const Mat& bottom_blob, Mat& top_blob, co
 #if NCNN_INT8
     if (opt.use_int8_inference && weight_data.elemsize == (size_t)1u)
     {
-        Mat bottom_blob_unpacked = bottom_blob;
-        if (bottom_blob.elempack != 1)
-        {
-            Option opt_pack1 = opt;
-            opt_pack1.blob_allocator = opt.workspace_allocator;
-
-            convert_packing(bottom_blob, bottom_blob_unpacked, 1, opt_pack1);
-        }
-
-        Mat bottom_blob_unpacked_fp32 = bottom_blob_unpacked;
-        if (bottom_blob_unpacked.elembits() == 16)
-        {
-            Option opt_pack1 = opt;
-            opt_pack1.blob_allocator = opt.workspace_allocator;
-
-            cast_float16_to_float32(bottom_blob_unpacked, bottom_blob_unpacked_fp32, opt_pack1);
-        }
-
-        Option opt_unpacked = opt;
-        opt_unpacked.use_packing_layout = false;
-        return ConvolutionDepthWise::forward_int8(bottom_blob_unpacked_fp32, top_blob, opt_unpacked);
+        return forward_int8_mips(bottom_blob, top_blob, opt);
     }
 #endif
 
@@ -559,4 +546,379 @@ int ConvolutionDepthWise_mips::forward(const std::vector<Mat>& bottom_blobs, std
     return 0;
 }
 
+#if NCNN_INT8
+int ConvolutionDepthWise_mips::create_pipeline_int8_mips(const Option& opt)
+{
+    const int maxk = kernel_w * kernel_h;
+    int channels = (weight_data_size / group) / maxk / (num_output / group) * group;
+
+    // depth-wise
+    if (channels == group && group == num_output)
+    {
+        int elempack = 1;
+#if __mips_msa
+        if (opt.use_packing_layout)
+        {
+            elempack = channels % 8 == 0 ? 8 : 1;
+        }
+#endif // __mips_msa
+
+        if (elempack == 8)
+        {
+            Mat weight_data_r2 = weight_data.reshape(maxk, group);
+            convert_packing(weight_data_r2, weight_data_int8, 8, opt);
+        }
+
+        return 0;
+    }
+
+    // group convolution
+    create_group_ops(opt);
+
+    return 0;
+}
+
+int ConvolutionDepthWise_mips::forward_int8_mips(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const
+{
+    int w = bottom_blob.w;
+    int h = bottom_blob.h;
+    int channels = bottom_blob.c;
+    int elempack = bottom_blob.elempack;
+
+    int elembits = bottom_blob.elembits();
+
+    const int kernel_extent_w = dilation_w * (kernel_w - 1) + 1;
+    const int kernel_extent_h = dilation_h * (kernel_h - 1) + 1;
+
+    Mat bottom_blob_int8 = bottom_blob;
+    if (elembits != 8)
+    {
+        const int channels_g = channels * elempack / group;
+
+        Mat scales(channels * elempack);
+        {
+            float* ps = scales;
+            for (int g = 0; g < group; g++)
+            {
+                float scale = bottom_blob_int8_scales[g];
+                for (int q = 0; q < channels_g; q++)
+                {
+                    *ps++ = scale;
+                }
+            }
+        }
+
+        Option opt_q = opt;
+        opt_q.blob_allocator = opt.workspace_allocator;
+        quantize_to_int8(bottom_blob, bottom_blob_int8, scales, opt_q);
+    }
+
+    Mat bottom_blob_bordered;
+    make_padding(bottom_blob_int8, bottom_blob_bordered, opt);
+    if (bottom_blob_bordered.empty())
+        return -100;
+
+    w = bottom_blob_bordered.w;
+    h = bottom_blob_bordered.h;
+    channels = bottom_blob_bordered.c;
+    elempack = bottom_blob_bordered.elempack;
+
+    int outw = (w - kernel_extent_w) / stride_w + 1;
+    int outh = (h - kernel_extent_h) / stride_h + 1;
+
+    // depth-wise
+    if (channels * elempack == group && group == num_output)
+    {
+        int out_elempack = 1;
+#if __mips_msa
+        if (opt.use_packing_layout)
+        {
+            out_elempack = num_output % 8 == 0 ? 8 : 1;
+        }
+#endif // __mips_msa
+        bool use_int8_requantize = int8_scale_term > 100;
+        size_t out_elemsize = use_int8_requantize ? 1u * out_elempack : 4u * out_elempack;
+
+        top_blob.create(outw, outh, num_output / out_elempack, out_elemsize, out_elempack, opt.blob_allocator);
+        if (top_blob.empty())
+            return -100;
+
+#if __mips_msa
+        if (elempack == 8)
+        {
+            {
+                const int maxk = kernel_w * kernel_h;
+
+                // kernel offsets
+                std::vector<int> _space_ofs(maxk);
+                int* space_ofs = &_space_ofs[0];
+                {
+                    int p1 = 0;
+                    int p2 = 0;
+                    int gap = w * dilation_h - kernel_w * dilation_w;
+                    for (int i = 0; i < kernel_h; i++)
+                    {
+                        for (int j = 0; j < kernel_w; j++)
+                        {
+                            space_ofs[p1] = p2;
+                            p1++;
+                            p2 += dilation_w;
+                        }
+                        p2 += gap;
+                    }
+                }
+
+                #pragma omp parallel for num_threads(opt.num_threads)
+                for (int g = 0; g < channels; g++)
+                {
+                    signed char* outptr_s8 = top_blob.channel(g);
+                    float* outptr_f32 = top_blob.channel(g);
+                    const signed char* kptr = (const signed char*)weight_data_int8 + maxk * g * 8;
+                    const Mat m = bottom_blob_bordered.channel(g);
+
+                    for (int i = 0; i < outh; i++)
+                    {
+                        for (int j = 0; j < outw; j++)
+                        {
+                            v4i32 _sum0 = __msa_fill_w(0);
+                            v4i32 _sum1 = __msa_fill_w(0);
+
+                            const signed char* sptr = m.row<const signed char>(i * stride_h) + j * stride_w * 8;
+
+                            for (int k = 0; k < maxk; k++)
+                            {
+                                v16i8 _val = __msa_ld_b(sptr + space_ofs[k] * 8, 0);
+                                v8i16 _val16 = (v8i16)__msa_ilvr_b(__msa_clti_s_b(_val, 0), _val);
+
+                                v16i8 _w = __msa_ld_b(kptr + k * 8, 0);
+                                v8i16 _w16 = (v8i16)__msa_ilvr_b(__msa_clti_s_b(_w, 0), _w);
+
+                                v8i16 _s0 = __msa_mulv_h(_val16, _w16);
+                                v8i16 _exts0 = __msa_clti_s_h(_s0, 0);
+                                v4i32 _s0l = (v4i32)__msa_ilvr_h(_exts0, _s0);
+                                v4i32 _s0h = (v4i32)__msa_ilvl_h(_exts0, _s0);
+
+                                _sum0 = __msa_addv_w(_sum0, _s0l);
+                                _sum1 = __msa_addv_w(_sum1, _s0h);
+                            }
+
+                            v4f32 _scale_in0;
+                            v4f32 _scale_in1;
+                            {
+                                v4f32 _bottom_blob_int8_scales0 = (v4f32)__msa_ld_w((const float*)bottom_blob_int8_scales + g * 8, 0);
+                                v4f32 _bottom_blob_int8_scales1 = (v4f32)__msa_ld_w((const float*)bottom_blob_int8_scales + g * 8 + 4, 0);
+                                v4f32 _weight_data_int8_scales0 = (v4f32)__msa_ld_w((const float*)weight_data_int8_scales + g * 8, 0);
+                                v4f32 _weight_data_int8_scales1 = (v4f32)__msa_ld_w((const float*)weight_data_int8_scales + g * 8 + 4, 0);
+                                _scale_in0 = __msa_frcp_w(__msa_fmul_w(_bottom_blob_int8_scales0, _weight_data_int8_scales0));
+                                _scale_in1 = __msa_frcp_w(__msa_fmul_w(_bottom_blob_int8_scales1, _weight_data_int8_scales1));
+
+                                v4i32 _m0 = __msa_fcne_w(_weight_data_int8_scales0, __msa_fill_w_f32(0.f));
+                                v4i32 _m1 = __msa_fcne_w(_weight_data_int8_scales1, __msa_fill_w_f32(0.f));
+                                _scale_in0 = (v4f32)__msa_and_v((v16u8)_scale_in0, (v16u8)_m0);
+                                _scale_in1 = (v4f32)__msa_and_v((v16u8)_scale_in1, (v16u8)_m1);
+                            }
+
+                            v4f32 _sumfp32_0 = __msa_fmul_w(__msa_ffint_s_w(_sum0), _scale_in0);
+                            v4f32 _sumfp32_1 = __msa_fmul_w(__msa_ffint_s_w(_sum1), _scale_in1);
+
+                            if (bias_term)
+                            {
+                                v4f32 _bias0 = (v4f32)__msa_ld_w((const float*)bias_data + g * 8, 0);
+                                v4f32 _bias1 = (v4f32)__msa_ld_w((const float*)bias_data + g * 8 + 4, 0);
+                                _sumfp32_0 = __msa_fadd_w(_sumfp32_0, _bias0);
+                                _sumfp32_1 = __msa_fadd_w(_sumfp32_1, _bias1);
+                            }
+
+                            _sumfp32_0 = activation_ps(_sumfp32_0, activation_type, activation_params);
+                            _sumfp32_1 = activation_ps(_sumfp32_1, activation_type, activation_params);
+
+                            if (use_int8_requantize)
+                            {
+                                // requantize and relu
+                                v4f32 _scale_out0 = (v4f32)__msa_ld_w((const float*)top_blob_int8_scales + g * 8, 0);
+                                v4f32 _scale_out1 = (v4f32)__msa_ld_w((const float*)top_blob_int8_scales + g * 8 + 4, 0);
+                                _sumfp32_0 = __msa_fmul_w(_sumfp32_0, _scale_out0);
+                                _sumfp32_1 = __msa_fmul_w(_sumfp32_1, _scale_out1);
+                                int64_t _sum8 = float2int8(_sumfp32_0, _sumfp32_1);
+
+                                *(int64_t*)outptr_s8 = _sum8;
+                                outptr_s8 += 8;
+                            }
+                            else
+                            {
+                                // dequantize and relu
+                                __msa_st_w((v4i32)_sumfp32_0, outptr_f32, 0);
+                                __msa_st_w((v4i32)_sumfp32_1, outptr_f32 + 4, 0);
+                                outptr_f32 += 8;
+                            }
+                        }
+                    }
+                }
+            }
+        }
+#endif // __mips_msa
+
+        if (elempack == 1)
+        {
+            {
+                const int maxk = kernel_w * kernel_h;
+
+                // kernel offsets
+                std::vector<int> _space_ofs(maxk);
+                int* space_ofs = &_space_ofs[0];
+                {
+                    int p1 = 0;
+                    int p2 = 0;
+                    int gap = w * dilation_h - kernel_w * dilation_w;
+                    for (int i = 0; i < kernel_h; i++)
+                    {
+                        for (int j = 0; j < kernel_w; j++)
+                        {
+                            space_ofs[p1] = p2;
+                            p1++;
+                            p2 += dilation_w;
+                        }
+                        p2 += gap;
+                    }
+                }
+
+                #pragma omp parallel for num_threads(opt.num_threads)
+                for (int g = 0; g < group; g++)
+                {
+                    signed char* outptr_s8 = top_blob.channel(g);
+                    float* outptr_f32 = top_blob.channel(g);
+                    const signed char* kptr = (const signed char*)weight_data + maxk * g;
+                    const Mat m = bottom_blob_bordered.channel(g);
+
+                    for (int i = 0; i < outh; i++)
+                    {
+                        for (int j = 0; j < outw; j++)
+                        {
+                            int sum = 0;
+
+                            const signed char* sptr = m.row<const signed char>(i * stride_h) + j * stride_w;
+
+                            for (int k = 0; k < maxk; k++)
+                            {
+                                signed char val = sptr[space_ofs[k]];
+                                signed char w = kptr[k];
+                                sum += val * w;
+                            }
+
+                            float scale_in;
+                            if (weight_data_int8_scales[g] == 0)
+                                scale_in = 0;
+                            else
+                                scale_in = 1.f / (bottom_blob_int8_scales[g] * weight_data_int8_scales[g]);
+
+                            float sumfp32 = sum * scale_in;
+
+                            if (bias_term)
+                                sumfp32 += bias_data[g];
+
+                            sumfp32 = activation_ss(sumfp32, activation_type, activation_params);
+
+                            if (use_int8_requantize)
+                            {
+                                // requantize
+                                float scale_out = top_blob_int8_scales[g];
+                                signed char sums8 = float2int8(sumfp32 * scale_out);
+                                outptr_s8[0] = sums8;
+                                outptr_s8 += 1;
+                            }
+                            else
+                            {
+                                // dequantize
+                                outptr_f32[0] = sumfp32;
+                                outptr_f32 += 1;
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        return 0;
+    }
+
+    bool use_int8_requantize = int8_scale_term > 100;
+    int out_elempack = 1;
+#if __mips_msa
+    if (opt.use_packing_layout)
+    {
+        if (use_int8_requantize)
+            out_elempack = num_output % 8 == 0 ? 8 : 1;
+        else
+            out_elempack = num_output % 4 == 0 ? 4 : 1;
+    }
+#endif // __mips_msa
+    size_t out_elemsize = use_int8_requantize ? 1u * out_elempack : 4u * out_elempack;
+
+    top_blob.create(outw, outh, num_output / out_elempack, out_elemsize, out_elempack, opt.blob_allocator);
+    if (top_blob.empty())
+        return -100;
+
+    // group convolution
+    const int channels_g = channels * elempack / group;
+    const int num_output_g = num_output / group;
+
+    int g_elempack = 1;
+    int out_g_elempack = 1;
+#if __mips_msa
+    if (opt.use_packing_layout)
+    {
+        g_elempack = channels_g % 8 == 0 ? 8 : 1;
+        if (use_int8_requantize)
+            out_g_elempack = num_output_g % 8 == 0 ? 8 : 1;
+        else
+            out_g_elempack = num_output_g % 4 == 0 ? 4 : 1;
+    }
+#endif // __mips_msa
+
+    // unpacking
+    Mat bottom_blob_bordered_unpacked = bottom_blob_bordered;
+    if (elempack > g_elempack)
+    {
+        Option opt_p = opt;
+        opt_p.blob_allocator = opt.workspace_allocator;
+        convert_packing(bottom_blob_bordered, bottom_blob_bordered_unpacked, g_elempack, opt_p);
+    }
+
+    Mat top_blob_unpacked = top_blob;
+    if (out_g_elempack < out_elempack)
+    {
+        top_blob_unpacked.create(outw, outh, num_output / out_g_elempack, out_elemsize / out_elempack * out_g_elempack, out_g_elempack, opt.workspace_allocator);
+        if (top_blob_unpacked.empty())
+            return -100;
+    }
+
+    #pragma omp parallel for num_threads(opt.num_threads)
+    for (int g = 0; g < group; g++)
+    {
+        const Mat bottom_blob_bordered_g = bottom_blob_bordered_unpacked.channel_range(channels_g * g / g_elempack, channels_g / g_elempack);
+        Mat top_blob_g = top_blob_unpacked.channel_range(num_output_g * g / out_g_elempack, num_output_g / out_g_elempack);
+
+        const ncnn::Layer* op = group_ops[g];
+
+        Option opt_g = opt;
+        opt_g.blob_allocator = top_blob.allocator;
+
+        // forward
+        op->forward(bottom_blob_bordered_g, top_blob_g, opt_g);
+    }
+
+    // packing
+    if (out_g_elempack < out_elempack)
+    {
+        convert_packing(top_blob_unpacked, top_blob, out_elempack, opt);
+    }
+    else
+    {
+        top_blob = top_blob_unpacked;
+    }
+
+    return 0;
+}
+#endif // NCNN_INT8
+
 } // namespace ncnn

src/layer/mips/convolutiondepthwise_mips.h

@@ -33,6 +33,10 @@ public:
 
 protected:
     int create_group_ops(const Option& opt);
+#if NCNN_INT8
+    int create_pipeline_int8_mips(const Option& opt);
+    int forward_int8_mips(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const;
+#endif
 
 public:
     Layer* activation;
@@ -40,6 +44,11 @@ public:
 
     // packing
     Mat weight_data_packed;
+
+#if NCNN_INT8
+    // int8
+    Mat weight_data_int8;
+#endif
 };
 
 } // namespace ncnn

src/layer/mips/innerproduct_mips.cpp

@@ -32,12 +32,11 @@ InnerProduct_mips::InnerProduct_mips()
 #endif // __mips_msa
 
     flatten = 0;
+    activation = 0;
 }
 
 int InnerProduct_mips::create_pipeline(const Option& opt)
 {
-#if __mips_msa
-    if (opt.use_packing_layout || opt.use_int8_inference)
     {
         flatten = ncnn::create_layer(ncnn::LayerType::Flatten);
 
@@ -47,7 +46,13 @@ int InnerProduct_mips::create_pipeline(const Option& opt)
 
         flatten->create_pipeline(opt);
     }
-#endif // __mips_msa
+
+#if NCNN_INT8
+    if (opt.use_int8_inference && weight_data.elemsize == (size_t)1u)
+    {
+        return create_pipeline_int8_mips(opt);
+    }
+#endif
 
     return 0;
 }
@@ -61,6 +66,13 @@ int InnerProduct_mips::destroy_pipeline(const Option& opt)
         flatten = 0;
     }
 
+    if (activation)
+    {
+        activation->destroy_pipeline(opt);
+        delete activation;
+        activation = 0;
+    }
+
     return 0;
 }
 
@@ -69,27 +81,7 @@ int InnerProduct_mips::forward(const Mat& bottom_blob, Mat& top_blob, const Opti
 #if NCNN_INT8
     if (opt.use_int8_inference && weight_data.elemsize == (size_t)1u)
     {
-        Mat bottom_blob_unpacked = bottom_blob;
-        if (bottom_blob.elempack != 1)
-        {
-            Option opt_pack1 = opt;
-            opt_pack1.blob_allocator = opt.workspace_allocator;
-
-            convert_packing(bottom_blob, bottom_blob_unpacked, 1, opt_pack1);
-        }
-
-        Mat bottom_blob_unpacked_fp32 = bottom_blob_unpacked;
-        if (bottom_blob_unpacked.elembits() == 16)
-        {
-            Option opt_pack1 = opt;
-            opt_pack1.blob_allocator = opt.workspace_allocator;
-
-            cast_float16_to_float32(bottom_blob_unpacked, bottom_blob_unpacked_fp32, opt_pack1);
-        }
-
-        Option opt_unpacked = opt;
-        opt_unpacked.use_packing_layout = false;
-        return InnerProduct::forward_int8(bottom_blob_unpacked_fp32, top_blob, opt_unpacked);
+        return forward_int8_mips(bottom_blob, top_blob, opt);
     }
 #endif
 
@@ -350,4 +342,189 @@ int InnerProduct_mips::forward(const Mat& bottom_blob, Mat& top_blob, const Opti
     return 0;
 }
 
+#if NCNN_INT8
+int InnerProduct_mips::create_pipeline_int8_mips(const Option& opt)
+{
+    activation = create_activation_layer(activation_type, activation_params, opt);
+
+    const int num_input = weight_data_size / num_output;
+
+    int out_elempack = 1;
+#if __mips_msa
+    if (opt.use_packing_layout)
+    {
+        out_elempack = num_output % 8 == 0 ? 8 : 1;
+    }
+#endif // __mips_msa
+
+    // src = inch-outch
+    // dst = pb-inch-outch/pb
+    {
+        Mat weight_data_r2 = weight_data.reshape(num_input, num_output);
+
+        weight_data_int8.create(num_input, num_output / out_elempack, (size_t)out_elempack, out_elempack);
+
+        for (int q = 0; q + (out_elempack - 1) < num_output; q += out_elempack)
+        {
+            signed char* g0 = weight_data_int8.row<signed char>(q / out_elempack);
+
+            for (int p = 0; p < num_input; p++)
+            {
+                for (int j = 0; j < out_elempack; j++)
+                {
+                    *g0++ = weight_data_r2.row<signed char>(q + j)[p];
+                }
+            }
+        }
+    }
+
+    return 0;
+}
+
+int InnerProduct_mips::forward_int8_mips(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const
+{
+    const int num_input = weight_data_size / num_output;
+
+    if (bottom_blob.dims == 2 && bottom_blob.w == num_input && bottom_blob.h * bottom_blob.elempack > 1)
+    {
+        // gemm
+        Mat bottom_blob_unpacked;
+        Option opt_unpack = opt;
+        opt_unpack.blob_allocator = opt.workspace_allocator;
+        convert_packing(bottom_blob, bottom_blob_unpacked, 1, opt_unpack);
+
+        return forward_int8(bottom_blob_unpacked, top_blob, opt);
+    }
+
+    int elembits = bottom_blob.elembits();
+
+    Mat bottom_blob_int8 = bottom_blob;
+    if (elembits != 8)
+    {
+        Option opt_q = opt;
+        opt_q.blob_allocator = opt.workspace_allocator;
+        quantize_to_int8(bottom_blob, bottom_blob_int8, bottom_blob_int8_scales, opt_q);
+    }
+
+    Mat bottom_blob_int8_flattened = bottom_blob_int8;
+    if (bottom_blob_int8.dims != 1)
+    {
+        Option opt_flatten = opt;
+        opt_flatten.blob_allocator = opt.workspace_allocator;
+        flatten->forward(bottom_blob_int8, bottom_blob_int8_flattened, opt_flatten);
+    }
+
+    //     int elempack = bottom_blob_int8_flattened.elempack;
+
+    int out_elempack = 1;
+#if __mips_msa
+    if (opt.use_packing_layout)
+    {
+        out_elempack = num_output % 8 == 0 ? 8 : 1;
+    }
+#endif // __mips_msa
+    //     size_t out_elemsize = elemsize / elempack * out_elempack;
+
+    top_blob.create(num_output / out_elempack, (size_t)(4u * out_elempack), out_elempack, opt.blob_allocator);
+    if (top_blob.empty())
+        return -100;
+
+    Mat top_blob_int32;
+    top_blob_int32.create(num_output / out_elempack, (size_t)(4u * out_elempack), out_elempack, opt.workspace_allocator);
+    if (top_blob_int32.empty())
+        return -100;
+
+#if __mips_msa
+    if (out_elempack == 8)
+    {
+        #pragma omp parallel for num_threads(opt.num_threads)
+        for (int p = 0; p < num_output / out_elempack; p++)
+        {
+            v4i32 _sum0 = __msa_fill_w(0);
+            v4i32 _sum1 = __msa_fill_w(0);
+
+            const signed char* kptr = weight_data_int8.row<const signed char>(p);
+            const signed char* sptr = bottom_blob_int8_flattened;
+
+            int i = 0;
+            for (; i < num_input; i++)
+            {
+                __builtin_prefetch(sptr + 4);
+                __builtin_prefetch(kptr + 32);
+                v8i16 _val = __msa_fill_h((short)sptr[0]);
+
+                v16i8 _w = __msa_ld_b(kptr, 0);
+                v8i16 _w16 = (v8i16)__msa_ilvr_b(__msa_clti_s_b(_w, 0), _w);
+
+                v8i16 _s0 = __msa_mulv_h(_val, _w16);
+                v8i16 _exts0 = __msa_clti_s_h(_s0, 0);
+                v4i32 _s0l = (v4i32)__msa_ilvr_h(_exts0, _s0);
+                v4i32 _s0h = (v4i32)__msa_ilvl_h(_exts0, _s0);
+
+                _sum0 = __msa_addv_w(_sum0, _s0l);
+                _sum1 = __msa_addv_w(_sum1, _s0h);
+
+                sptr += 1;
+                kptr += 8;
+            }
+
+            int* outptr = (int*)top_blob_int32;
+            __msa_st_w((v4i32)_sum0, outptr + p * 8, 0);
+            __msa_st_w((v4i32)_sum1, outptr + p * 8 + 4, 0);
+        }
+    }
+#endif // __mips_msa
+
+    if (out_elempack == 1)
+    {
+        #pragma omp parallel for num_threads(opt.num_threads)
+        for (int p = 0; p < num_output / out_elempack; p++)
+        {
+            int sum = 0;
+
+            const signed char* kptr = weight_data_int8.row<const signed char>(p);
+            const signed char* sptr = bottom_blob_int8_flattened;
+
+            int i = 0;
+            for (; i < num_input; i++)
+            {
+                signed char val = sptr[0];
+
+                signed char w = kptr[0];
+
+                sum += val * w;
+
+                sptr += 1;
+                kptr += 1;
+            }
+
+            int* outptr = (int*)top_blob_int32;
+            outptr[p] = sum;
+        }
+    }
+
+    Mat scale_data(num_output);
+    for (int p = 0; p < num_output; p++)
+    {
+        // dequantize
+        float scale_in;
+        if (weight_data_int8_scales[p] == 0)
+            scale_in = 0;
+        else
+            scale_in = 1.f / (bottom_blob_int8_scales[0] * weight_data_int8_scales[p]);
+
+        scale_data[p] = scale_in;
+    }
+
+    dequantize_from_int32(top_blob_int32, top_blob, scale_data, bias_data, opt);
+
+    if (activation)
+    {
+        activation->forward_inplace(top_blob, opt);
+    }
+
+    return 0;
+}
+#endif // NCNN_INT8
+
 } // namespace ncnn

src/layer/mips/innerproduct_mips.h

@@ -29,8 +29,21 @@ public:
 
     virtual int forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const;
 
+protected:
+#if NCNN_INT8
+    int create_pipeline_int8_mips(const Option& opt);
+    int forward_int8_mips(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const;
+#endif
+
 public:
     Layer* flatten;
+    Layer* activation;
+
+#if NCNN_INT8
+    // int8
+    Mat weight_data_int8;
+    Mat scales_in;
+#endif
 };
 
 } // namespace ncnn

src/layer/x86/innerproduct_x86.cpp

@@ -109,6 +109,13 @@ int InnerProduct_x86::destroy_pipeline(const Option& opt)
         flatten = 0;
     }
 
+    if (activation)
+    {
+        activation->destroy_pipeline(opt);
+        delete activation;
+        activation = 0;
+    }
+
     return 0;
 }