nihui
/
ncnn
Not watched
1
0
Fork 0
Code
Releases 46 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
Browse Source

arm optimization for convolution int8 winograd unified elempack (#5087) 

* enable out elempack 8 for winograd and sgemm
tags/20231027
nihui GitHub 2 years ago
parent
3f437d3f3d
commit
80b3b9c6f0
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 5809 additions and 4866 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +0
    -229
      src/layer/arm/convolution_3x3_int8.h
  2. +0
    -185
      src/layer/arm/convolution_3x3_pack8to1_int8.h
  3. +0
    -205
      src/layer/arm/convolution_3x3_pack8to4_int8.h
  4. +5719
    -0
      src/layer/arm/convolution_3x3_winograd_int8.h
  5. +41
    -55
      src/layer/arm/convolution_arm.cpp
  6. +0
    -1005
      src/layer/arm/convolution_winograd_dot_int8.h
  7. +0
    -774
      src/layer/arm/convolution_winograd_dot_pack8to1_int8.h
  8. +0
    -1835
      src/layer/arm/convolution_winograd_dot_pack8to4_int8.h
  9. +0
    -230
      src/layer/arm/convolution_winograd_transform_int8.h
  10. +0
    -178
      src/layer/arm/convolution_winograd_transform_pack4_int8.h
  11. +0
    -131
      src/layer/arm/convolution_winograd_transform_pack8_int8.h
  12. +49
    -39
      src/net.cpp

+ 0
- 229
src/layer/arm/convolution_3x3_int8.h View File

@@ -12,235 +12,6 @@
// CONDITIONS OF ANY KIND, either express or implied. See the License for the
// specific language governing permissions and limitations under the License.

static void conv3x3s1_winograd43_transform_kernel_int8_neon(const Mat& kernel, Mat& kernel_tm_packed, int inch, int outch, const Option& opt)
{
// winograd43 transform kernel
Mat kernel_tm(6 * 6, inch, outch, (size_t)2u);

const short ktm[6][3] = {
{6, 0, 0},
{-4, -4, -4},
{-4, 4, -4},
{1, 2, 4},
{1, -2, 4},
{0, 0, 6}
};

#pragma omp parallel for num_threads(opt.num_threads)
for (int p = 0; p < outch; p++)
{
for (int q = 0; q < inch; q++)
{
const signed char* kernel0 = (const signed char*)kernel + p * inch * 9 + q * 9;
short* kernel_tm0 = kernel_tm.channel(p).row<short>(q);

// transform kernel
const signed char* k0 = kernel0;
const signed char* k1 = kernel0 + 3;
const signed char* k2 = kernel0 + 6;

// h
short tmp[6][3];
for (int i = 0; i < 6; i++)
{
tmp[i][0] = k0[0] * ktm[i][0] + k0[1] * ktm[i][1] + k0[2] * ktm[i][2];
tmp[i][1] = k1[0] * ktm[i][0] + k1[1] * ktm[i][1] + k1[2] * ktm[i][2];
tmp[i][2] = k2[0] * ktm[i][0] + k2[1] * ktm[i][1] + k2[2] * ktm[i][2];
}

// U
for (int j = 0; j < 6; j++)
{
short* tmpp = &tmp[j][0];

for (int i = 0; i < 6; i++)
{
kernel_tm0[j * 6 + i] = tmpp[0] * ktm[i][0] + tmpp[1] * ktm[i][1] + tmpp[2] * ktm[i][2];
}
}
}
}

// interleave
// src = 36-inch-outch
// dst = 8a-8b-inch/8a-36-outch/8b
#if __ARM_NEON
if (outch >= 8)
{
kernel_tm_packed.create(inch, 36, outch / 8 + (outch % 8) / 4 + outch % 4, (size_t)2u * 8, 8);
}
else if (outch >= 4)
{
kernel_tm_packed.create(inch, 36, outch / 4 + outch % 4, (size_t)2u * 4, 4);
}
#else // __ARM_NEON
if (outch >= 2)
{
kernel_tm_packed.create(inch, 36, outch / 2 + outch % 2, (size_t)2u * 2, 2);
}
#endif // __ARM_NEON
else
{
kernel_tm_packed.create(inch, 36, outch, (size_t)2u, 1);
}

int p = 0;
#if __ARM_NEON
for (; p + 7 < outch; p += 8)
{
Mat g0 = kernel_tm_packed.channel(p / 8);

for (int k = 0; k < 36; k++)
{
short* g00 = g0.row<short>(k);

for (int q = 0; q < inch; q++)
{
for (int i = 0; i < 8; i++)
{
g00[0] = kernel_tm.channel(p + i).row<const short>(q)[k];
g00++;
}
}
}
}
for (; p + 3 < outch; p += 4)
{
const Mat k0 = kernel_tm.channel(p);
const Mat k1 = kernel_tm.channel(p + 1);
const Mat k2 = kernel_tm.channel(p + 2);
const Mat k3 = kernel_tm.channel(p + 3);

Mat g0 = kernel_tm_packed.channel(p / 8 + (p % 8) / 4);

for (int k = 0; k < 36; k++)
{
short* g00 = g0.row<short>(k);

for (int q = 0; q < inch; q++)
{
g00[0] = k0.row<const short>(q)[k];
g00[1] = k1.row<const short>(q)[k];
g00[2] = k2.row<const short>(q)[k];
g00[3] = k3.row<const short>(q)[k];
g00 += 4;
}
}
}
#else // __ARM_NEON
for (; p + 1 < outch; p += 2)
{
const Mat k0 = kernel_tm.channel(p);
const Mat k1 = kernel_tm.channel(p + 1);

Mat g0 = kernel_tm_packed.channel(p / 2);

for (int k = 0; k < 36; k++)
{
short* g00 = g0.row<short>(k);

int q = 0;
#if __ARM_FEATURE_SIMD32
for (; q + 1 < inch; q += 2)
{
g00[0] = k0.row<const short>(q)[k];
g00[2] = k1.row<const short>(q)[k];
g00[1] = k0.row<const short>(q + 1)[k];
g00[3] = k1.row<const short>(q + 1)[k];
g00 += 4;
}
#endif // __ARM_FEATURE_SIMD32
for (; q < inch; q++)
{
g00[0] = k0.row<const short>(q)[k];
g00[1] = k1.row<const short>(q)[k];
g00 += 2;
}
}
}
#endif // __ARM_NEON
for (; p < outch; p++)
{
const Mat k0 = kernel_tm.channel(p);

#if __ARM_NEON
Mat g0 = kernel_tm_packed.channel(p / 8 + (p % 8) / 4 + p % 4);
#else
Mat g0 = kernel_tm_packed.channel(p / 2 + p % 2);
#endif

for (int k = 0; k < 36; k++)
{
short* g00 = g0.row<short>(k);

for (int q = 0; q < inch; q++)
{
g00[0] = k0.row<const short>(q)[k];
g00 += 1;
}
}
}
}

static void conv3x3s1_winograd43_int8_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& kernel_tm, const Option& opt)
{
int w = bottom_blob.w;
int h = bottom_blob.h;
int inch = bottom_blob.c;
// size_t elemsize = bottom_blob.elemsize;
int elempack = bottom_blob.elempack;

int outw = top_blob.w;
int outh = top_blob.h;
int outch = top_blob.c;

// pad to 4n+2
Mat bottom_blob_bordered = bottom_blob;

outw = (outw + 3) / 4 * 4;
outh = (outh + 3) / 4 * 4;

w = outw + 2;
h = outh + 2;
copy_make_border(bottom_blob, bottom_blob_bordered, 0, h - bottom_blob.h, 0, w - bottom_blob.w, BORDER_CONSTANT, 0.f, opt);

// BEGIN transform input
Mat bottom_blob_tm;
{
int w_tiles = outw / 4;
int h_tiles = outh / 4;
const int tiles = w_tiles * h_tiles;

bottom_blob_tm.create(tiles, 36, inch, 2u * elempack, elempack, opt.workspace_allocator);
conv3x3s1_winograd43_transform_input_int8_neon(bottom_blob_bordered, bottom_blob_tm, opt);
}
bottom_blob_bordered = Mat();
// END transform input

// BEGIN dot
Mat top_blob_tm;
convolution_winograd_dot_int8_neon(bottom_blob_tm, outch, kernel_tm, top_blob_tm, opt);
// END dot

// BEGIN transform output
Mat top_blob_bordered;
if (outw == top_blob.w && outh == top_blob.h)
{
top_blob_bordered = top_blob;
}
else
{
top_blob_bordered.create(outw, outh, outch, 4u, 1, opt.workspace_allocator);
}
{
conv3x3s1_winograd43_transform_output_int8_neon(top_blob_tm, top_blob_bordered, opt);
}
// END transform output

// cut result pad
copy_cut_border(top_blob_bordered, top_blob, 0, top_blob_bordered.h - top_blob.h, 0, top_blob_bordered.w - top_blob.w, opt);
}

static void conv3x3s2_transform_kernel_int8_neon(const Mat& _kernel, Mat& kernel_tm, int inch, int outch)
{
kernel_tm.create(8 * 9, inch, outch / 8 + outch % 8, (size_t)1u);


+ 0
- 185
src/layer/arm/convolution_3x3_pack8to1_int8.h View File

@@ -1,185 +0,0 @@
// Tencent is pleased to support the open source community by making ncnn available.
//
// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved.
//
// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
// in compliance with the License. You may obtain a copy of the License at
//
// https://opensource.org/licenses/BSD-3-Clause
//
// Unless required by applicable law or agreed to in writing, software distributed
// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
// CONDITIONS OF ANY KIND, either express or implied. See the License for the
// specific language governing permissions and limitations under the License.

static void conv3x3s1_winograd43_transform_kernel_pack8to1_int8_neon(const Mat& kernel, Mat& kernel_tm_pack8to1, int inch, int outch, const Option& opt)
{
// winograd43 transform kernel
Mat kernel_tm(6 * 6, inch, outch, (size_t)2u);

const short ktm[6][3] = {
{6, 0, 0},
{-4, -4, -4},
{-4, 4, -4},
{1, 2, 4},
{1, -2, 4},
{0, 0, 6}
};

#pragma omp parallel for num_threads(opt.num_threads)
for (int p = 0; p < outch; p++)
{
for (int q = 0; q < inch; q++)
{
const signed char* kernel0 = (const signed char*)kernel + p * inch * 9 + q * 9;
short* kernel_tm0 = kernel_tm.channel(p).row<short>(q);

// transform kernel
const signed char* k0 = kernel0;
const signed char* k1 = kernel0 + 3;
const signed char* k2 = kernel0 + 6;

// h
short tmp[6][3];
for (int i = 0; i < 6; i++)
{
tmp[i][0] = k0[0] * ktm[i][0] + k0[1] * ktm[i][1] + k0[2] * ktm[i][2];
tmp[i][1] = k1[0] * ktm[i][0] + k1[1] * ktm[i][1] + k1[2] * ktm[i][2];
tmp[i][2] = k2[0] * ktm[i][0] + k2[1] * ktm[i][1] + k2[2] * ktm[i][2];
}

// U
for (int j = 0; j < 6; j++)
{
short* tmpp = &tmp[j][0];

for (int i = 0; i < 6; i++)
{
kernel_tm0[j * 6 + i] = tmpp[0] * ktm[i][0] + tmpp[1] * ktm[i][1] + tmpp[2] * ktm[i][2];
}
}
}
}

// interleave
// src = 36-inch-outch
// dst = 8a-inch/8a-36-outch
kernel_tm_pack8to1.create(8 * inch / 8, 36, outch / 8 + outch % 8, (size_t)2u * 8, 8);

int p = 0;
for (; p + 7 < outch; p += 8)
{
const Mat k0 = kernel_tm.channel(p);
const Mat k1 = kernel_tm.channel(p + 1);
const Mat k2 = kernel_tm.channel(p + 2);
const Mat k3 = kernel_tm.channel(p + 3);
const Mat k4 = kernel_tm.channel(p + 4);
const Mat k5 = kernel_tm.channel(p + 5);
const Mat k6 = kernel_tm.channel(p + 6);
const Mat k7 = kernel_tm.channel(p + 7);

Mat g0 = kernel_tm_pack8to1.channel(p / 8);

for (int k = 0; k < 36; k++)
{
short* g00 = g0.row<short>(k);

for (int q = 0; q + 7 < inch; q += 8)
{
for (int i = 0; i < 8; i++)
{
g00[0] = k0.row<const short>(q + i)[k];
g00[1] = k1.row<const short>(q + i)[k];
g00[2] = k2.row<const short>(q + i)[k];
g00[3] = k3.row<const short>(q + i)[k];
g00[4] = k4.row<const short>(q + i)[k];
g00[5] = k5.row<const short>(q + i)[k];
g00[6] = k6.row<const short>(q + i)[k];
g00[7] = k7.row<const short>(q + i)[k];

g00 += 8;
}
}
}
}
for (; p < outch; p++)
{
const Mat k0 = kernel_tm.channel(p);

Mat g0 = kernel_tm_pack8to1.channel(p / 8 + p % 8);

for (int k = 0; k < 36; k++)
{
short* g00 = g0.row<short>(k);

for (int q = 0; q + 7 < inch; q += 8)
{
for (int i = 0; i < 8; i++)
{
g00[0] = k0.row<const short>(q + i)[k];

g00 += 1;
}
}
}
}
}

static void conv3x3s1_winograd43_pack8to1_int8_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& kernel_tm, const Option& opt)
{
int w = bottom_blob.w;
int h = bottom_blob.h;
int inch = bottom_blob.c;
// size_t elemsize = bottom_blob.elemsize;
int elempack = bottom_blob.elempack;

int outw = top_blob.w;
int outh = top_blob.h;
int outch = top_blob.c;

// pad to 4n+2
Mat bottom_blob_bordered = bottom_blob;

outw = (outw + 3) / 4 * 4;
outh = (outh + 3) / 4 * 4;

w = outw + 2;
h = outh + 2;
copy_make_border(bottom_blob, bottom_blob_bordered, 0, h - bottom_blob.h, 0, w - bottom_blob.w, BORDER_CONSTANT, 0.f, opt);

// BEGIN transform input
Mat bottom_blob_tm;
{
int w_tiles = outw / 4;
int h_tiles = outh / 4;
const int tiles = w_tiles * h_tiles;

bottom_blob_tm.create(tiles, 36, inch, 2u * elempack, elempack, opt.workspace_allocator);
conv3x3s1_winograd43_transform_input_pack8_int8_neon(bottom_blob_bordered, bottom_blob_tm, opt);
}
bottom_blob_bordered = Mat();
// END transform input

// BEGIN dot
Mat top_blob_tm;
convolution_winograd_dot_pack8to1_int8_neon(bottom_blob_tm, outch, kernel_tm, top_blob_tm, opt);
// END dot

// BEGIN transform output
Mat top_blob_bordered;
if (outw == top_blob.w && outh == top_blob.h)
{
top_blob_bordered = top_blob;
}
else
{
top_blob_bordered.create(outw, outh, outch, 4u, 1, opt.workspace_allocator);
}
{
conv3x3s1_winograd43_transform_output_int8_neon(top_blob_tm, top_blob_bordered, opt);
}
// END transform output

// cut result pad
copy_cut_border(top_blob_bordered, top_blob, 0, top_blob_bordered.h - top_blob.h, 0, top_blob_bordered.w - top_blob.w, opt);
}

+ 0
- 205
src/layer/arm/convolution_3x3_pack8to4_int8.h View File

@@ -1,205 +0,0 @@
// Tencent is pleased to support the open source community by making ncnn available.
//
// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved.
//
// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
// in compliance with the License. You may obtain a copy of the License at
//
// https://opensource.org/licenses/BSD-3-Clause
//
// Unless required by applicable law or agreed to in writing, software distributed
// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
// CONDITIONS OF ANY KIND, either express or implied. See the License for the
// specific language governing permissions and limitations under the License.

static void conv3x3s1_winograd43_transform_kernel_pack8to4_int8_neon(const Mat& kernel, Mat& kernel_tm_pack8, int inch, int outch, const Option& opt)
{
// winograd43 transform kernel
Mat kernel_tm(6 * 6, inch, outch, (size_t)2u);

const short ktm[6][3] = {
{6, 0, 0},
{-4, -4, -4},
{-4, 4, -4},
{1, 2, 4},
{1, -2, 4},
{0, 0, 6}
};

#pragma omp parallel for num_threads(opt.num_threads)
for (int p = 0; p < outch; p++)
{
for (int q = 0; q < inch; q++)
{
const signed char* kernel0 = (const signed char*)kernel + p * inch * 9 + q * 9;
short* kernel_tm0 = kernel_tm.channel(p).row<short>(q);

// transform kernel
const signed char* k0 = kernel0;
const signed char* k1 = kernel0 + 3;
const signed char* k2 = kernel0 + 6;

// h
short tmp[6][3];
for (int i = 0; i < 6; i++)
{
tmp[i][0] = k0[0] * ktm[i][0] + k0[1] * ktm[i][1] + k0[2] * ktm[i][2];
tmp[i][1] = k1[0] * ktm[i][0] + k1[1] * ktm[i][1] + k1[2] * ktm[i][2];
tmp[i][2] = k2[0] * ktm[i][0] + k2[1] * ktm[i][1] + k2[2] * ktm[i][2];
}

// U
for (int j = 0; j < 6; j++)
{
short* tmpp = &tmp[j][0];

for (int i = 0; i < 6; i++)
{
kernel_tm0[j * 6 + i] = tmpp[0] * ktm[i][0] + tmpp[1] * ktm[i][1] + tmpp[2] * ktm[i][2];
}
}
}
}

// interleave
// src = 36-inch-outch
// dst = 4b-8a-inch/8a-36-outch/4b
kernel_tm_pack8.create(inch / 8, 36, outch / 8 + (outch % 8) / 4, (size_t)2u * 64, 64);

int q = 0;
for (; q + 7 < outch; q += 8)
{
const Mat k0 = kernel_tm.channel(q);
const Mat k1 = kernel_tm.channel(q + 1);
const Mat k2 = kernel_tm.channel(q + 2);
const Mat k3 = kernel_tm.channel(q + 3);
const Mat k4 = kernel_tm.channel(q + 4);
const Mat k5 = kernel_tm.channel(q + 5);
const Mat k6 = kernel_tm.channel(q + 6);
const Mat k7 = kernel_tm.channel(q + 7);

Mat kernel_tm = kernel_tm_pack8.channel(q / 8);

for (int k = 0; k < 36; k++)
{
short* g00 = kernel_tm.row<short>(k);

for (int p = 0; p + 7 < inch; p += 8)
{
for (int i = 0; i < 8; i++)
{
const short* k00 = k0.row<const short>(p + i);
const short* k10 = k1.row<const short>(p + i);
const short* k20 = k2.row<const short>(p + i);
const short* k30 = k3.row<const short>(p + i);
const short* k40 = k4.row<const short>(p + i);
const short* k50 = k5.row<const short>(p + i);
const short* k60 = k6.row<const short>(p + i);
const short* k70 = k7.row<const short>(p + i);

g00[0] = k00[k];
g00[1] = k10[k];
g00[2] = k20[k];
g00[3] = k30[k];
g00[4] = k40[k];
g00[5] = k50[k];
g00[6] = k60[k];
g00[7] = k70[k];

g00 += 8;
}
}
}
}
for (; q + 3 < outch; q += 4)
{
const Mat k0 = kernel_tm.channel(q);
const Mat k1 = kernel_tm.channel(q + 1);
const Mat k2 = kernel_tm.channel(q + 2);
const Mat k3 = kernel_tm.channel(q + 3);

Mat kernel_tm = kernel_tm_pack8.channel(q / 8 + (q % 8) / 4);

for (int k = 0; k < 36; k++)
{
short* g00 = kernel_tm.row<short>(k);

for (int p = 0; p + 7 < inch; p += 8)
{
for (int i = 0; i < 8; i++)
{
const short* k00 = k0.row<const short>(p + i);
const short* k10 = k1.row<const short>(p + i);
const short* k20 = k2.row<const short>(p + i);
const short* k30 = k3.row<const short>(p + i);

g00[0] = k00[k];
g00[1] = k10[k];
g00[2] = k20[k];
g00[3] = k30[k];

g00 += 4;
}
}
}
}
}

static void conv3x3s1_winograd43_pack8to4_int8_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& kernel_tm, const Option& opt)
{
int w = bottom_blob.w;
int h = bottom_blob.h;
int inch = bottom_blob.c;
// size_t elemsize = bottom_blob.elemsize;
int elempack = bottom_blob.elempack;

int outw = top_blob.w;
int outh = top_blob.h;
int outch = top_blob.c;

// pad to 4n+2
Mat bottom_blob_bordered = bottom_blob;

outw = (outw + 3) / 4 * 4;
outh = (outh + 3) / 4 * 4;

w = outw + 2;
h = outh + 2;
copy_make_border(bottom_blob, bottom_blob_bordered, 0, h - bottom_blob.h, 0, w - bottom_blob.w, BORDER_CONSTANT, 0.f, opt);

// BEGIN transform input
Mat bottom_blob_tm;
{
int w_tiles = outw / 4;
int h_tiles = outh / 4;
const int tiles = w_tiles * h_tiles;

bottom_blob_tm.create(tiles, 36, inch, 2u * elempack, elempack, opt.workspace_allocator);
conv3x3s1_winograd43_transform_input_pack8_int8_neon(bottom_blob_bordered, bottom_blob_tm, opt);
}
bottom_blob_bordered = Mat();
// END transform input

// BEGIN dot
Mat top_blob_tm;
convolution_winograd_dot_pack8to4_int8_neon(bottom_blob_tm, outch, kernel_tm, top_blob_tm, opt);
// END dot

// BEGIN transform output
Mat top_blob_bordered;
if (outw == top_blob.w && outh == top_blob.h)
{
top_blob_bordered = top_blob;
}
else
{
top_blob_bordered.create(outw, outh, outch, 4u * 4, 4, opt.workspace_allocator);
}
{
conv3x3s1_winograd43_transform_output_pack4_int8_neon(top_blob_tm, top_blob_bordered, opt);
}
// END transform output

// cut result pad
copy_cut_border(top_blob_bordered, top_blob, 0, top_blob_bordered.h - top_blob.h, 0, top_blob_bordered.w - top_blob.w, opt);
}

+ 5719
- 0
src/layer/arm/convolution_3x3_winograd_int8.h
File diff suppressed because it is too large
View File


+ 41
- 55
src/layer/arm/convolution_arm.cpp View File

@@ -49,10 +49,9 @@ namespace ncnn {

#if NCNN_INT8
#include "convolution_im2col_gemm_int8.h"
#include "convolution_3x3_winograd_int8.h"

#include "convolution_winograd_transform_int8.h"
#include "convolution_winograd_dot_int8.h"
#include "convolution_3x3_int8.h"
// #include "convolution_3x3_int8.h"
#include "convolution_int8.h"
#endif // NCNN_INT8

@@ -74,12 +73,6 @@ namespace ncnn {
#include "convolution_pack8to4_int8.h"
#include "convolution_pack1to4_int8.h"
#include "convolution_pack8to1_int8.h"
#include "convolution_winograd_transform_pack4_int8.h"
#include "convolution_winograd_transform_pack8_int8.h"
#include "convolution_winograd_dot_pack8to4_int8.h"
#include "convolution_winograd_dot_pack8to1_int8.h"
#include "convolution_3x3_pack8to4_int8.h"
#include "convolution_3x3_pack8to1_int8.h"
#endif // NCNN_INT8
#endif // __ARM_NEON

@@ -1285,6 +1278,14 @@ int Convolution_arm::create_pipeline_int8_arm(const Option& opt)
const int maxk = kernel_w * kernel_h;
const int num_input = weight_data_size / maxk / num_output;

bool prefer_winograd = (opt.use_winograd23_convolution || opt.use_winograd43_convolution) && (num_input >= 8 && num_output >= 8) && kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1;
#if NCNN_ARM82DOT
if (ncnn::cpu_support_arm_asimddp())
{
prefer_winograd = false;
}
#endif

int elempack = 1;
int out_elempack = 1;
#if __ARM_NEON
@@ -1295,25 +1296,12 @@ int Convolution_arm::create_pipeline_int8_arm(const Option& opt)
}
#endif // __ARM_NEON

#if NCNN_ARM82DOT
if (elempack == 8 && out_elempack == 4 && opt.use_winograd_convolution && opt.use_winograd43_convolution && kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1 && (!ncnn::cpu_support_arm_asimddp() || (ncnn::cpu_support_arm_asimddp() && num_input >= 256 && num_output >= 256)))
#else
if (elempack == 8 && out_elempack == 4 && opt.use_winograd_convolution && opt.use_winograd43_convolution && kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
#endif
{
#if __ARM_NEON
conv3x3s1_winograd43_transform_kernel_pack8to4_int8_neon(weight_data, weight_winograd43_data, num_input, num_output, opt);
#endif // __ARM_NEON
}
else if (elempack == 8 && out_elempack == 1 && opt.use_winograd_convolution && opt.use_winograd43_convolution && kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
if (opt.use_winograd_convolution && prefer_winograd)
{
#if __ARM_NEON
conv3x3s1_winograd43_transform_kernel_pack8to1_int8_neon(weight_data, weight_winograd43_data, num_input, num_output, opt);
#endif // __ARM_NEON
}
else if (elempack == 1 && out_elempack == 1 && opt.use_winograd_convolution && opt.use_winograd43_convolution && kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
{
conv3x3s1_winograd43_transform_kernel_int8_neon(weight_data, weight_winograd43_data, num_input, num_output, opt);
if (opt.use_winograd43_convolution)
conv3x3s1_winograd43_transform_kernel_int8(weight_data, weight_winograd43_data, num_input, num_output, opt);
else
conv3x3s1_winograd23_transform_kernel_int8(weight_data, weight_winograd23_data, num_input, num_output, opt);
}
else if (opt.use_sgemm_convolution)
{
@@ -1321,10 +1309,6 @@ int Convolution_arm::create_pipeline_int8_arm(const Option& opt)
}
else if (elempack == 1 && out_elempack == 1)
{
// if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
// {
// conv3x3s2_transform_kernel_int8_neon(weight_data, weight_3x3s2_data_int8, num_input, num_output);
// }
weight_data_tm = weight_data;
}
else
@@ -1405,20 +1389,29 @@ int Convolution_arm::forward_int8_arm(const Mat& bottom_blob, Mat& top_blob, con

// NCNN_LOGE("forward_int8_arm %d %d %d %d %d", w, h, bottom_blob_bordered.c, elempack, out_elempack);

top_blob.create(outw, outh, num_output / out_elempack, out_elemsize, out_elempack, opt.blob_allocator);
if (top_blob.empty())
return -100;

#if NCNN_ARM82DOT
int channels = bottom_blob_bordered.c;
const int num_input = channels * elempack;

bool prefer_winograd = (opt.use_winograd23_convolution || opt.use_winograd43_convolution) && (num_input >= 8 && num_output >= 8) && kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1;
#if NCNN_ARM82DOT
if (ncnn::cpu_support_arm_asimddp())
{
prefer_winograd = false;
}
#endif

int out_elempack_int32 = 1;
#if __ARM_NEON
if (opt.use_packing_layout)
{
out_elempack_int32 = num_output % 4 == 0 ? 4 : 1;
if ((opt.use_winograd_convolution && prefer_winograd) || opt.use_sgemm_convolution)
{
out_elempack_int32 = num_output % 8 == 0 ? 8 : num_output % 4 == 0 ? 4 : 1;
}
else
{
out_elempack_int32 = num_output % 4 == 0 ? 4 : 1;
}
}
#endif // __ARM_NEON

@@ -1435,25 +1428,12 @@ int Convolution_arm::forward_int8_arm(const Mat& bottom_blob, Mat& top_blob, con
NCNN_LOGE("opt.num_threads %d changed, convolution gemm will use load-time value %d", opt.num_threads, nT);
}

#if NCNN_ARM82DOT
if (elempack == 8 && out_elempack_int32 == 4 && opt.use_winograd_convolution && opt.use_winograd43_convolution && kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1 && (!ncnn::cpu_support_arm_asimddp() || (ncnn::cpu_support_arm_asimddp() && num_input >= 256 && num_output >= 256)))
#else
if (elempack == 8 && out_elempack_int32 == 4 && opt.use_winograd_convolution && opt.use_winograd43_convolution && kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
#endif
if (opt.use_winograd_convolution && prefer_winograd)
{
#if __ARM_NEON
conv3x3s1_winograd43_pack8to4_int8_neon(bottom_blob_bordered, top_blob_int32, weight_winograd43_data, opt);
#endif // __ARM_NEON
}
else if (elempack == 8 && out_elempack_int32 == 1 && opt.use_winograd_convolution && opt.use_winograd43_convolution && kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
{
#if __ARM_NEON
conv3x3s1_winograd43_pack8to1_int8_neon(bottom_blob_bordered, top_blob_int32, weight_winograd43_data, opt);
#endif // __ARM_NEON
}
else if (elempack == 1 && out_elempack_int32 == 1 && opt.use_winograd_convolution && opt.use_winograd43_convolution && kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
{
conv3x3s1_winograd43_int8_neon(bottom_blob_bordered, top_blob_int32, weight_winograd43_data, opt);
if (opt.use_winograd43_convolution && !weight_winograd43_data.empty())
conv3x3s1_winograd43_int8(bottom_blob_bordered, top_blob_int32, weight_winograd43_data, _nT, opt);
else
conv3x3s1_winograd23_int8(bottom_blob_bordered, top_blob_int32, weight_winograd23_data, _nT, opt);
}
else if (opt.use_sgemm_convolution)
{
@@ -1478,6 +1458,12 @@ int Convolution_arm::forward_int8_arm(const Mat& bottom_blob, Mat& top_blob, con
convolution_int8(bottom_blob_bordered, top_blob_int32, weight_data_tm, kernel_w, kernel_h, dilation_w, dilation_h, stride_w, stride_h, opt);
}

bottom_blob_bordered.release();

top_blob.create(outw, outh, num_output / out_elempack, out_elemsize, out_elempack, opt.blob_allocator);
if (top_blob.empty())
return -100;

if (use_int8_requantize)
{
requantize_from_int32_to_int8(top_blob_int32, top_blob, scale_in_data, top_blob_int8_scales, bias_data, activation_type, activation_params, opt);


+ 0
- 1005
src/layer/arm/convolution_winograd_dot_int8.h
File diff suppressed because it is too large
View File


+ 0
- 774
src/layer/arm/convolution_winograd_dot_pack8to1_int8.h View File

@@ -1,774 +0,0 @@
// Tencent is pleased to support the open source community by making ncnn available.
//
// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved.
//
// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
// in compliance with the License. You may obtain a copy of the License at
//
// https://opensource.org/licenses/BSD-3-Clause
//
// Unless required by applicable law or agreed to in writing, software distributed
// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
// CONDITIONS OF ANY KIND, either express or implied. See the License for the
// specific language governing permissions and limitations under the License.

static void convolution_winograd_dot_pack8to1_int8_neon(Mat& bottom_blob_tm, int outch, const Mat& kernel_tm, Mat& top_blob_tm, const Option& opt)
{
// Mat bottom_blob_tm(tiles, 16/36/64, inch, 16u, 8, opt.workspace_allocator);

const int tiles = bottom_blob_tm.w;
const int batch = bottom_blob_tm.h;
const int inch = bottom_blob_tm.c;

// permute
Mat bottom_blob_tm2;
#if __aarch64__
if (tiles >= 8)
bottom_blob_tm2.create(8 * inch, tiles / 8 + (tiles % 8) / 4 + tiles % 4, batch, 16u, 8, opt.workspace_allocator);
else if (tiles >= 4)
bottom_blob_tm2.create(4 * inch, tiles / 4 + tiles % 4, batch, 16u, 8, opt.workspace_allocator);
else // if (tiles >= 1)
bottom_blob_tm2.create(1 * inch, tiles, batch, 16u, 8, opt.workspace_allocator);
#else
if (tiles >= 4)
bottom_blob_tm2.create(4 * inch, tiles / 4 + tiles % 4, batch, 16u, 8, opt.workspace_allocator);
else // if (tiles >= 1)
bottom_blob_tm2.create(1 * inch, tiles, batch, 16u, 8, opt.workspace_allocator);
#endif // __aarch64__

#pragma omp parallel for num_threads(opt.num_threads)
for (int r = 0; r < batch; r++)
{
Mat tm2 = bottom_blob_tm2.channel(r);

// tile
int i = 0;
#if __aarch64__
for (; i + 7 < tiles; i += 8)
{
short* tm2p = tm2.row<short>(i / 8);

const short* r0 = bottom_blob_tm;

r0 += (r * tiles + i) * 8;

for (int q = 0; q < inch; q++)
{
// transpose 8x8
asm volatile(
"prfm pldl1keep, [%0, #512] \n"
"ld4 {v0.8h, v1.8h, v2.8h, v3.8h}, [%0], #64 \n"
"ld4 {v4.8h, v5.8h, v6.8h, v7.8h}, [%0] \n"
"sub %0, %0, #64 \n"

"uzp1 v16.8h, v0.8h, v4.8h \n"
"uzp2 v20.8h, v0.8h, v4.8h \n"
"uzp1 v17.8h, v1.8h, v5.8h \n"
"uzp2 v21.8h, v1.8h, v5.8h \n"
"uzp1 v18.8h, v2.8h, v6.8h \n"
"uzp2 v22.8h, v2.8h, v6.8h \n"
"uzp1 v19.8h, v3.8h, v7.8h \n"
"uzp2 v23.8h, v3.8h, v7.8h \n"

"st1 {v16.8h, v17.8h, v18.8h, v19.8h}, [%1], #64 \n"
"st1 {v20.8h, v21.8h, v22.8h, v23.8h}, [%1], #64 \n"
: "=r"(r0), // %0
"=r"(tm2p) // %1
: "0"(r0),
"1"(tm2p)
: "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23");

r0 += bottom_blob_tm.cstep * 8;
}
}
#endif
for (; i + 3 < tiles; i += 4)
{
#if __aarch64__
short* tm2p = tm2.row<short>(i / 8 + (i % 8) / 4);
#else
short* tm2p = tm2.row<short>(i / 4);
#endif

const short* r0 = bottom_blob_tm;

r0 += (r * tiles + i) * 8;

for (int q = 0; q < inch; q++)
{
// transpose 8x4
#if __aarch64__
asm volatile(
"prfm pldl1keep, [%0, #512] \n"
"ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [%0] \n"
"st4 {v0.8h, v1.8h, v2.8h, v3.8h}, [%1], #64 \n"
: "=r"(r0), // %0
"=r"(tm2p) // %1
: "0"(r0),
"1"(tm2p)
: "memory", "v0", "v1", "v2", "v3");
#else
asm volatile(
"pld [%0, #512] \n"
"vldm %0, {d0-d7} \n"
"vswp d1, d2 \n"
"vswp d5, d6 \n"
"vswp q1, q2 \n"
"vst4.s16 {d0-d3}, [%1 :64]! \n"
"vst4.s16 {d4-d7}, [%1 :64]! \n"
: "=r"(r0), // %0
"=r"(tm2p) // %1
: "0"(r0),
"1"(tm2p)
: "memory", "q0", "q1", "q2", "q3");
#endif // __aarch64__
r0 += bottom_blob_tm.cstep * 8;
}
}
for (; i < tiles; i++)
{
#if __aarch64__
short* tm2p = tm2.row<short>(i / 8 + (i % 8) / 4 + i % 4);
#else
short* tm2p = tm2.row<short>(i / 4 + i % 4);
#endif

const short* r0 = bottom_blob_tm;

r0 += (r * tiles + i) * 8;

for (int q = 0; q < inch; q++)
{
#if __aarch64__
asm volatile(
"prfm pldl1keep, [%0, #128] \n"
"ld1 {v0.8h}, [%0] \n"
"st1 {v0.8h}, [%1], #16 \n"
: "=r"(r0), // %0
"=r"(tm2p) // %1
: "0"(r0),
"1"(tm2p)
: "memory", "v0");
#else
asm volatile(
"pld [%0, #128] \n"
"vld1.s16 {d0-d1}, [%0 :64] \n"
"vst1.s16 {d0-d1}, [%1 :64]! \n"
: "=r"(r0), // %0
"=r"(tm2p) // %1
: "0"(r0),
"1"(tm2p)
: "memory", "q0");
#endif // __aarch64__
r0 += bottom_blob_tm.cstep * 8;
}
}
}

bottom_blob_tm = Mat();
// permute end

top_blob_tm.create(tiles, batch, outch, 4u, 1, opt.workspace_allocator);

int nn_outch = 0;
int remain_outch_start = 0;

nn_outch = outch >> 3;

#pragma omp parallel for num_threads(opt.num_threads)
for (int pp = 0; pp < nn_outch; pp++)
{
int p = pp * 8;

int* output0_tm = top_blob_tm.channel(p);
int* output1_tm = top_blob_tm.channel(p + 1);
int* output2_tm = top_blob_tm.channel(p + 2);
int* output3_tm = top_blob_tm.channel(p + 3);
int* output4_tm = top_blob_tm.channel(p + 4);
int* output5_tm = top_blob_tm.channel(p + 5);
int* output6_tm = top_blob_tm.channel(p + 6);
int* output7_tm = top_blob_tm.channel(p + 7);

const Mat kernel01_tm = kernel_tm.channel(p / 8);

for (int r = 0; r < batch; r++)
{
const Mat bb2 = bottom_blob_tm2.channel(r);

int i = 0;
#if __aarch64__
for (; i + 7 < tiles; i += 8)
{
const short* r0 = bb2.row<const short>(i / 8);
const short* kptr = kernel01_tm.row<const short>(r);

int nn = inch; // inch always > 0

asm volatile(
"eor v16.16b, v16.16b, v16.16b \n"
"eor v17.16b, v17.16b, v17.16b \n"
"eor v18.16b, v18.16b, v18.16b \n"
"eor v19.16b, v19.16b, v19.16b \n"
"eor v20.16b, v20.16b, v20.16b \n"
"eor v21.16b, v21.16b, v21.16b \n"
"eor v22.16b, v22.16b, v22.16b \n"
"eor v23.16b, v23.16b, v23.16b \n"
"eor v24.16b, v24.16b, v24.16b \n"
"eor v25.16b, v25.16b, v25.16b \n"
"eor v26.16b, v26.16b, v26.16b \n"
"eor v27.16b, v27.16b, v27.16b \n"
"eor v28.16b, v28.16b, v28.16b \n"
"eor v29.16b, v29.16b, v29.16b \n"
"eor v30.16b, v30.16b, v30.16b \n"
"eor v31.16b, v31.16b, v31.16b \n"

"0: \n"

"prfm pldl1keep, [%9, #512] \n"
"ld1 {v8.8h, v9.8h, v10.8h, v11.8h}, [%9], #64 \n"

"prfm pldl1keep, [%10, #512] \n"
"ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [%10], #64 \n"

"smlal v16.4s, v8.4h, v0.h[0] \n"
"smlal2 v17.4s, v8.8h, v0.h[0] \n"
"smlal v18.4s, v8.4h, v0.h[1] \n"
"smlal2 v19.4s, v8.8h, v0.h[1] \n"
"smlal v20.4s, v8.4h, v0.h[2] \n"
"smlal2 v21.4s, v8.8h, v0.h[2] \n"
"smlal v22.4s, v8.4h, v0.h[3] \n"
"smlal2 v23.4s, v8.8h, v0.h[3] \n"
"smlal v24.4s, v8.4h, v0.h[4] \n"
"smlal2 v25.4s, v8.8h, v0.h[4] \n"
"smlal v26.4s, v8.4h, v0.h[5] \n"
"smlal2 v27.4s, v8.8h, v0.h[5] \n"
"smlal v28.4s, v8.4h, v0.h[6] \n"
"smlal2 v29.4s, v8.8h, v0.h[6] \n"
"smlal v30.4s, v8.4h, v0.h[7] \n"
"smlal2 v31.4s, v8.8h, v0.h[7] \n"

"smlal v16.4s, v9.4h, v1.h[0] \n"
"smlal2 v17.4s, v9.8h, v1.h[0] \n"
"smlal v18.4s, v9.4h, v1.h[1] \n"
"smlal2 v19.4s, v9.8h, v1.h[1] \n"
"smlal v20.4s, v9.4h, v1.h[2] \n"
"smlal2 v21.4s, v9.8h, v1.h[2] \n"
"smlal v22.4s, v9.4h, v1.h[3] \n"
"smlal2 v23.4s, v9.8h, v1.h[3] \n"
"smlal v24.4s, v9.4h, v1.h[4] \n"
"smlal2 v25.4s, v9.8h, v1.h[4] \n"
"smlal v26.4s, v9.4h, v1.h[5] \n"
"smlal2 v27.4s, v9.8h, v1.h[5] \n"
"smlal v28.4s, v9.4h, v1.h[6] \n"
"smlal2 v29.4s, v9.8h, v1.h[6] \n"
"smlal v30.4s, v9.4h, v1.h[7] \n"
"smlal2 v31.4s, v9.8h, v1.h[7] \n"

"prfm pldl1keep, [%9, #512] \n"
"ld1 {v12.8h, v13.8h, v14.8h, v15.8h}, [%9], #64 \n"

"smlal v16.4s, v10.4h, v2.h[0] \n"
"smlal2 v17.4s, v10.8h, v2.h[0] \n"
"smlal v18.4s, v10.4h, v2.h[1] \n"
"smlal2 v19.4s, v10.8h, v2.h[1] \n"
"smlal v20.4s, v10.4h, v2.h[2] \n"
"smlal2 v21.4s, v10.8h, v2.h[2] \n"
"smlal v22.4s, v10.4h, v2.h[3] \n"
"smlal2 v23.4s, v10.8h, v2.h[3] \n"
"smlal v24.4s, v10.4h, v2.h[4] \n"
"smlal2 v25.4s, v10.8h, v2.h[4] \n"
"smlal v26.4s, v10.4h, v2.h[5] \n"
"smlal2 v27.4s, v10.8h, v2.h[5] \n"
"smlal v28.4s, v10.4h, v2.h[6] \n"
"smlal2 v29.4s, v10.8h, v2.h[6] \n"
"smlal v30.4s, v10.4h, v2.h[7] \n"
"smlal2 v31.4s, v10.8h, v2.h[7] \n"

"prfm pldl1keep, [%10, #512] \n"
"ld1 {v4.8h, v5.8h, v6.8h, v7.8h}, [%10], #64 \n"

"smlal v16.4s, v11.4h, v3.h[0] \n"
"smlal2 v17.4s, v11.8h, v3.h[0] \n"
"smlal v18.4s, v11.4h, v3.h[1] \n"
"smlal2 v19.4s, v11.8h, v3.h[1] \n"
"smlal v20.4s, v11.4h, v3.h[2] \n"
"smlal2 v21.4s, v11.8h, v3.h[2] \n"
"smlal v22.4s, v11.4h, v3.h[3] \n"
"smlal2 v23.4s, v11.8h, v3.h[3] \n"
"smlal v24.4s, v11.4h, v3.h[4] \n"
"smlal2 v25.4s, v11.8h, v3.h[4] \n"
"smlal v26.4s, v11.4h, v3.h[5] \n"
"smlal2 v27.4s, v11.8h, v3.h[5] \n"
"smlal v28.4s, v11.4h, v3.h[6] \n"
"smlal2 v29.4s, v11.8h, v3.h[6] \n"
"smlal v30.4s, v11.4h, v3.h[7] \n"
"smlal2 v31.4s, v11.8h, v3.h[7] \n"

"smlal v16.4s, v12.4h, v4.h[0] \n"
"smlal2 v17.4s, v12.8h, v4.h[0] \n"
"smlal v18.4s, v12.4h, v4.h[1] \n"
"smlal2 v19.4s, v12.8h, v4.h[1] \n"
"smlal v20.4s, v12.4h, v4.h[2] \n"
"smlal2 v21.4s, v12.8h, v4.h[2] \n"
"smlal v22.4s, v12.4h, v4.h[3] \n"
"smlal2 v23.4s, v12.8h, v4.h[3] \n"
"smlal v24.4s, v12.4h, v4.h[4] \n"
"smlal2 v25.4s, v12.8h, v4.h[4] \n"
"smlal v26.4s, v12.4h, v4.h[5] \n"
"smlal2 v27.4s, v12.8h, v4.h[5] \n"
"smlal v28.4s, v12.4h, v4.h[6] \n"
"smlal2 v29.4s, v12.8h, v4.h[6] \n"
"smlal v30.4s, v12.4h, v4.h[7] \n"
"smlal2 v31.4s, v12.8h, v4.h[7] \n"

"smlal v16.4s, v13.4h, v5.h[0] \n"
"smlal2 v17.4s, v13.8h, v5.h[0] \n"
"smlal v18.4s, v13.4h, v5.h[1] \n"
"smlal2 v19.4s, v13.8h, v5.h[1] \n"
"smlal v20.4s, v13.4h, v5.h[2] \n"
"smlal2 v21.4s, v13.8h, v5.h[2] \n"
"smlal v22.4s, v13.4h, v5.h[3] \n"
"smlal2 v23.4s, v13.8h, v5.h[3] \n"
"smlal v24.4s, v13.4h, v5.h[4] \n"
"smlal2 v25.4s, v13.8h, v5.h[4] \n"
"smlal v26.4s, v13.4h, v5.h[5] \n"
"smlal2 v27.4s, v13.8h, v5.h[5] \n"
"smlal v28.4s, v13.4h, v5.h[6] \n"
"smlal2 v29.4s, v13.8h, v5.h[6] \n"
"smlal v30.4s, v13.4h, v5.h[7] \n"
"smlal2 v31.4s, v13.8h, v5.h[7] \n"

"smlal v16.4s, v14.4h, v6.h[0] \n"
"smlal2 v17.4s, v14.8h, v6.h[0] \n"
"smlal v18.4s, v14.4h, v6.h[1] \n"
"smlal2 v19.4s, v14.8h, v6.h[1] \n"
"smlal v20.4s, v14.4h, v6.h[2] \n"
"smlal2 v21.4s, v14.8h, v6.h[2] \n"
"smlal v22.4s, v14.4h, v6.h[3] \n"
"smlal2 v23.4s, v14.8h, v6.h[3] \n"
"smlal v24.4s, v14.4h, v6.h[4] \n"
"smlal2 v25.4s, v14.8h, v6.h[4] \n"
"smlal v26.4s, v14.4h, v6.h[5] \n"
"smlal2 v27.4s, v14.8h, v6.h[5] \n"
"smlal v28.4s, v14.4h, v6.h[6] \n"
"smlal2 v29.4s, v14.8h, v6.h[6] \n"
"smlal v30.4s, v14.4h, v6.h[7] \n"
"smlal2 v31.4s, v14.8h, v6.h[7] \n"

"subs %w0, %w0, #1 \n"

"smlal v16.4s, v15.4h, v7.h[0] \n"
"smlal2 v17.4s, v15.8h, v7.h[0] \n"
"smlal v18.4s, v15.4h, v7.h[1] \n"
"smlal2 v19.4s, v15.8h, v7.h[1] \n"
"smlal v20.4s, v15.4h, v7.h[2] \n"
"smlal2 v21.4s, v15.8h, v7.h[2] \n"
"smlal v22.4s, v15.4h, v7.h[3] \n"
"smlal2 v23.4s, v15.8h, v7.h[3] \n"
"smlal v24.4s, v15.4h, v7.h[4] \n"
"smlal2 v25.4s, v15.8h, v7.h[4] \n"
"smlal v26.4s, v15.4h, v7.h[5] \n"
"smlal2 v27.4s, v15.8h, v7.h[5] \n"
"smlal v28.4s, v15.4h, v7.h[6] \n"
"smlal2 v29.4s, v15.8h, v7.h[6] \n"
"smlal v30.4s, v15.4h, v7.h[7] \n"
"smlal2 v31.4s, v15.8h, v7.h[7] \n"

"bne 0b \n"

"st1 {v16.4s, v17.4s}, [%1], #32 \n"
"st1 {v18.4s, v19.4s}, [%2], #32 \n"
"st1 {v20.4s, v21.4s}, [%3], #32 \n"
"st1 {v22.4s, v23.4s}, [%4], #32 \n"
"st1 {v24.4s, v25.4s}, [%5], #32 \n"
"st1 {v26.4s, v27.4s}, [%6], #32 \n"
"st1 {v28.4s, v29.4s}, [%7], #32 \n"
"st1 {v30.4s, v31.4s}, [%8], #32 \n"

: "=r"(nn), // %0
"=r"(output0_tm), // %1
"=r"(output1_tm), // %2
"=r"(output2_tm), // %3
"=r"(output3_tm), // %4
"=r"(output4_tm), // %5
"=r"(output5_tm), // %6
"=r"(output6_tm), // %7
"=r"(output7_tm), // %8
"=r"(r0), // %9
"=r"(kptr) // %10
: "0"(nn),
"1"(output0_tm),
"2"(output1_tm),
"3"(output2_tm),
"4"(output3_tm),
"5"(output4_tm),
"6"(output5_tm),
"7"(output6_tm),
"8"(output7_tm),
"9"(r0),
"10"(kptr)
: "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31");
}
#endif
for (; i + 3 < tiles; i += 4)
{
#if __aarch64__
const short* r0 = bb2.row<const short>(i / 8 + (i % 8) / 4);
#else
const short* r0 = bb2.row<const short>(i / 4);
#endif
const short* k0 = kernel01_tm.row<const short>(r);

int nn = inch; // inch always > 0

int32x4_t _sum0 = vdupq_n_s32(0);
int32x4_t _sum1 = vdupq_n_s32(0);
int32x4_t _sum2 = vdupq_n_s32(0);
int32x4_t _sum3 = vdupq_n_s32(0);
int32x4_t _sum4 = vdupq_n_s32(0);
int32x4_t _sum5 = vdupq_n_s32(0);
int32x4_t _sum6 = vdupq_n_s32(0);
int32x4_t _sum7 = vdupq_n_s32(0);

for (int j = 0; j < nn; j++)
{
int16x8_t _val0 = vld1q_s16(r0);
int16x8_t _val1 = vld1q_s16(r0 + 8);
int16x8_t _val2 = vld1q_s16(r0 + 16);
int16x8_t _val3 = vld1q_s16(r0 + 24);

int16x8_t _w0 = vld1q_s16(k0);

_sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_val0), vget_low_s16(_w0), 0);
_sum1 = vmlal_lane_s16(_sum1, vget_low_s16(_val0), vget_low_s16(_w0), 1);
_sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_val0), vget_low_s16(_w0), 2);
_sum3 = vmlal_lane_s16(_sum3, vget_low_s16(_val0), vget_low_s16(_w0), 3);
_sum4 = vmlal_lane_s16(_sum4, vget_low_s16(_val0), vget_high_s16(_w0), 0);
_sum5 = vmlal_lane_s16(_sum5, vget_low_s16(_val0), vget_high_s16(_w0), 1);
_sum6 = vmlal_lane_s16(_sum6, vget_low_s16(_val0), vget_high_s16(_w0), 2);
_sum7 = vmlal_lane_s16(_sum7, vget_low_s16(_val0), vget_high_s16(_w0), 3);

int16x8_t _w1 = vld1q_s16(k0 + 8);

_sum0 = vmlal_lane_s16(_sum0, vget_high_s16(_val0), vget_low_s16(_w1), 0);
_sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_val0), vget_low_s16(_w1), 1);
_sum2 = vmlal_lane_s16(_sum2, vget_high_s16(_val0), vget_low_s16(_w1), 2);
_sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_val0), vget_low_s16(_w1), 3);
_sum4 = vmlal_lane_s16(_sum4, vget_high_s16(_val0), vget_high_s16(_w1), 0);
_sum5 = vmlal_lane_s16(_sum5, vget_high_s16(_val0), vget_high_s16(_w1), 1);
_sum6 = vmlal_lane_s16(_sum6, vget_high_s16(_val0), vget_high_s16(_w1), 2);
_sum7 = vmlal_lane_s16(_sum7, vget_high_s16(_val0), vget_high_s16(_w1), 3);

int16x8_t _w2 = vld1q_s16(k0 + 16);

_sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_val1), vget_low_s16(_w2), 0);
_sum1 = vmlal_lane_s16(_sum1, vget_low_s16(_val1), vget_low_s16(_w2), 1);
_sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_val1), vget_low_s16(_w2), 2);
_sum3 = vmlal_lane_s16(_sum3, vget_low_s16(_val1), vget_low_s16(_w2), 3);
_sum4 = vmlal_lane_s16(_sum4, vget_low_s16(_val1), vget_high_s16(_w2), 0);
_sum5 = vmlal_lane_s16(_sum5, vget_low_s16(_val1), vget_high_s16(_w2), 1);
_sum6 = vmlal_lane_s16(_sum6, vget_low_s16(_val1), vget_high_s16(_w2), 2);
_sum7 = vmlal_lane_s16(_sum7, vget_low_s16(_val1), vget_high_s16(_w2), 3);

int16x8_t _w3 = vld1q_s16(k0 + 24);

_sum0 = vmlal_lane_s16(_sum0, vget_high_s16(_val1), vget_low_s16(_w3), 0);
_sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_val1), vget_low_s16(_w3), 1);
_sum2 = vmlal_lane_s16(_sum2, vget_high_s16(_val1), vget_low_s16(_w3), 2);
_sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_val1), vget_low_s16(_w3), 3);
_sum4 = vmlal_lane_s16(_sum4, vget_high_s16(_val1), vget_high_s16(_w3), 0);
_sum5 = vmlal_lane_s16(_sum5, vget_high_s16(_val1), vget_high_s16(_w3), 1);
_sum6 = vmlal_lane_s16(_sum6, vget_high_s16(_val1), vget_high_s16(_w3), 2);
_sum7 = vmlal_lane_s16(_sum7, vget_high_s16(_val1), vget_high_s16(_w3), 3);

int16x8_t _w4 = vld1q_s16(k0 + 32);

_sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_val2), vget_low_s16(_w4), 0);
_sum1 = vmlal_lane_s16(_sum1, vget_low_s16(_val2), vget_low_s16(_w4), 1);
_sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_val2), vget_low_s16(_w4), 2);
_sum3 = vmlal_lane_s16(_sum3, vget_low_s16(_val2), vget_low_s16(_w4), 3);
_sum4 = vmlal_lane_s16(_sum4, vget_low_s16(_val2), vget_high_s16(_w4), 0);
_sum5 = vmlal_lane_s16(_sum5, vget_low_s16(_val2), vget_high_s16(_w4), 1);
_sum6 = vmlal_lane_s16(_sum6, vget_low_s16(_val2), vget_high_s16(_w4), 2);
_sum7 = vmlal_lane_s16(_sum7, vget_low_s16(_val2), vget_high_s16(_w4), 3);

int16x8_t _w5 = vld1q_s16(k0 + 40);

_sum0 = vmlal_lane_s16(_sum0, vget_high_s16(_val2), vget_low_s16(_w5), 0);
_sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_val2), vget_low_s16(_w5), 1);
_sum2 = vmlal_lane_s16(_sum2, vget_high_s16(_val2), vget_low_s16(_w5), 2);
_sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_val2), vget_low_s16(_w5), 3);
_sum4 = vmlal_lane_s16(_sum4, vget_high_s16(_val2), vget_high_s16(_w5), 0);
_sum5 = vmlal_lane_s16(_sum5, vget_high_s16(_val2), vget_high_s16(_w5), 1);
_sum6 = vmlal_lane_s16(_sum6, vget_high_s16(_val2), vget_high_s16(_w5), 2);
_sum7 = vmlal_lane_s16(_sum7, vget_high_s16(_val2), vget_high_s16(_w5), 3);

int16x8_t _w6 = vld1q_s16(k0 + 48);

_sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_val3), vget_low_s16(_w6), 0);
_sum1 = vmlal_lane_s16(_sum1, vget_low_s16(_val3), vget_low_s16(_w6), 1);
_sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_val3), vget_low_s16(_w6), 2);
_sum3 = vmlal_lane_s16(_sum3, vget_low_s16(_val3), vget_low_s16(_w6), 3);
_sum4 = vmlal_lane_s16(_sum4, vget_low_s16(_val3), vget_high_s16(_w6), 0);
_sum5 = vmlal_lane_s16(_sum5, vget_low_s16(_val3), vget_high_s16(_w6), 1);
_sum6 = vmlal_lane_s16(_sum6, vget_low_s16(_val3), vget_high_s16(_w6), 2);
_sum7 = vmlal_lane_s16(_sum7, vget_low_s16(_val3), vget_high_s16(_w6), 3);

int16x8_t _w7 = vld1q_s16(k0 + 56);

_sum0 = vmlal_lane_s16(_sum0, vget_high_s16(_val3), vget_low_s16(_w7), 0);
_sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_val3), vget_low_s16(_w7), 1);
_sum2 = vmlal_lane_s16(_sum2, vget_high_s16(_val3), vget_low_s16(_w7), 2);
_sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_val3), vget_low_s16(_w7), 3);
_sum4 = vmlal_lane_s16(_sum4, vget_high_s16(_val3), vget_high_s16(_w7), 0);
_sum5 = vmlal_lane_s16(_sum5, vget_high_s16(_val3), vget_high_s16(_w7), 1);
_sum6 = vmlal_lane_s16(_sum6, vget_high_s16(_val3), vget_high_s16(_w7), 2);
_sum7 = vmlal_lane_s16(_sum7, vget_high_s16(_val3), vget_high_s16(_w7), 3);

r0 += 32;
k0 += 64;
}

vst1q_s32(output0_tm, _sum0);
vst1q_s32(output1_tm, _sum1);
vst1q_s32(output2_tm, _sum2);
vst1q_s32(output3_tm, _sum3);
vst1q_s32(output4_tm, _sum4);
vst1q_s32(output5_tm, _sum5);
vst1q_s32(output6_tm, _sum6);
vst1q_s32(output7_tm, _sum7);

output0_tm += 4;
output1_tm += 4;
output2_tm += 4;
output3_tm += 4;
output4_tm += 4;
output5_tm += 4;
output6_tm += 4;
output7_tm += 4;
}
for (; i < tiles; i++)
{
#if __aarch64__
const short* r0 = bb2.row<const short>(i / 8 + (i % 8) / 4 + i % 4);
#else
const short* r0 = bb2.row<const short>(i / 4 + i % 4);
#endif
const short* k0 = kernel01_tm.row<const short>(r);

int nn = inch; // inch always > 0

int32x4_t _sum0 = vdupq_n_s32(0);
int32x4_t _sum1 = vdupq_n_s32(0);

for (int j = 0; j < nn; j++)
{
int16x8_t _val0 = vld1q_s16(r0);

int16x8_t _w0 = vld1q_s16(k0);
int16x8_t _w1 = vld1q_s16(k0 + 8);
int16x8_t _w2 = vld1q_s16(k0 + 16);
int16x8_t _w3 = vld1q_s16(k0 + 24);
int16x8_t _w4 = vld1q_s16(k0 + 32);
int16x8_t _w5 = vld1q_s16(k0 + 40);
int16x8_t _w6 = vld1q_s16(k0 + 48);
int16x8_t _w7 = vld1q_s16(k0 + 56);

_sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w0), vget_low_s16(_val0), 0);
_sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w0), vget_low_s16(_val0), 0);

_sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w1), vget_low_s16(_val0), 1);
_sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w1), vget_low_s16(_val0), 1);

_sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w2), vget_low_s16(_val0), 2);
_sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w2), vget_low_s16(_val0), 2);

_sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w3), vget_low_s16(_val0), 3);
_sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w3), vget_low_s16(_val0), 3);

_sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w4), vget_high_s16(_val0), 0);
_sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w4), vget_high_s16(_val0), 0);

_sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w5), vget_high_s16(_val0), 1);
_sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w5), vget_high_s16(_val0), 1);

_sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w6), vget_high_s16(_val0), 2);
_sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w6), vget_high_s16(_val0), 2);

_sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_w7), vget_high_s16(_val0), 3);
_sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_w7), vget_high_s16(_val0), 3);

r0 += 8;
k0 += 64;
}

output0_tm[0] = vgetq_lane_s32(_sum0, 0);
output1_tm[0] = vgetq_lane_s32(_sum0, 1);
output2_tm[0] = vgetq_lane_s32(_sum0, 2);
output3_tm[0] = vgetq_lane_s32(_sum0, 3);
output4_tm[0] = vgetq_lane_s32(_sum1, 0);
output5_tm[0] = vgetq_lane_s32(_sum1, 1);
output6_tm[0] = vgetq_lane_s32(_sum1, 2);
output7_tm[0] = vgetq_lane_s32(_sum1, 3);
output0_tm += 1;
output1_tm += 1;
output2_tm += 1;
output3_tm += 1;
output4_tm += 1;
output5_tm += 1;
output6_tm += 1;
output7_tm += 1;
}
}
}

remain_outch_start += nn_outch << 3;

#pragma omp parallel for num_threads(opt.num_threads)
for (int p = remain_outch_start; p < outch; p++)
{
int* output0_tm = top_blob_tm.channel(p);

const Mat kernel0_tm = kernel_tm.channel(p / 8 + p % 8);

for (int r = 0; r < batch; r++)
{
const Mat bb2 = bottom_blob_tm2.channel(r);

int i = 0;
#if __aarch64__
for (; i + 7 < tiles; i += 8)
{
const short* r0 = bb2.row<const short>(i / 8);

const short* kptr = kernel0_tm.row<const short>(r);

int32x4_t _sum0 = vdupq_n_s32(0);
int32x4_t _sum1 = vdupq_n_s32(0);
int32x4_t _sum2 = vdupq_n_s32(0);
int32x4_t _sum3 = vdupq_n_s32(0);

for (int q = 0; q < inch; q++)
{
int16x8_t _r0 = vld1q_s16(r0);
int16x8_t _r1 = vld1q_s16(r0 + 8);
int16x8_t _r2 = vld1q_s16(r0 + 16);
int16x8_t _r3 = vld1q_s16(r0 + 24);
int16x8_t _r4 = vld1q_s16(r0 + 32);
int16x8_t _r5 = vld1q_s16(r0 + 40);
int16x8_t _r6 = vld1q_s16(r0 + 48);
int16x8_t _r7 = vld1q_s16(r0 + 56);

int16x8_t _k0 = vld1q_s16(kptr);

_sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_r0), vget_low_s16(_k0), 0);
_sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_r0), vget_low_s16(_k0), 0);
_sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_r1), vget_low_s16(_k0), 1);
_sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_r1), vget_low_s16(_k0), 1);
_sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_r2), vget_low_s16(_k0), 2);
_sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_r2), vget_low_s16(_k0), 2);
_sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_r3), vget_low_s16(_k0), 3);
_sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_r3), vget_low_s16(_k0), 3);
_sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_r4), vget_high_s16(_k0), 0);
_sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_r4), vget_high_s16(_k0), 0);
_sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_r5), vget_high_s16(_k0), 1);
_sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_r5), vget_high_s16(_k0), 1);
_sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_r6), vget_high_s16(_k0), 2);
_sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_r6), vget_high_s16(_k0), 2);
_sum2 = vmlal_lane_s16(_sum2, vget_low_s16(_r7), vget_high_s16(_k0), 3);
_sum3 = vmlal_lane_s16(_sum3, vget_high_s16(_r7), vget_high_s16(_k0), 3);

kptr += 8;
r0 += 64;
}

_sum0 = vaddq_s32(_sum0, _sum2);
_sum1 = vaddq_s32(_sum1, _sum3);

vst1q_s32(output0_tm, _sum0);
vst1q_s32(output0_tm + 4, _sum1);

output0_tm += 8;
}
#endif
for (; i + 3 < tiles; i += 4)
{
#if __aarch64__
const short* r0 = bb2.row<const short>(i / 8 + (i % 8) / 4);
#else
const short* r0 = bb2.row<const short>(i / 4);
#endif
const short* kptr = kernel0_tm.row<const short>(r);

int32x4_t _sum0 = vdupq_n_s32(0);
int32x4_t _sum1 = vdupq_n_s32(0);

for (int q = 0; q < inch; q++)
{
int16x8_t _r0 = vld1q_s16(r0);
int16x8_t _r1 = vld1q_s16(r0 + 8);
int16x8_t _r2 = vld1q_s16(r0 + 16);
int16x8_t _r3 = vld1q_s16(r0 + 24);

int16x8_t _k0 = vld1q_s16(kptr);

_sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_r0), vget_low_s16(_k0), 0);
_sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_r0), vget_low_s16(_k0), 1);
_sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_r1), vget_low_s16(_k0), 2);
_sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_r1), vget_low_s16(_k0), 3);
_sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_r2), vget_high_s16(_k0), 0);
_sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_r2), vget_high_s16(_k0), 1);
_sum0 = vmlal_lane_s16(_sum0, vget_low_s16(_r3), vget_high_s16(_k0), 2);
_sum1 = vmlal_lane_s16(_sum1, vget_high_s16(_r3), vget_high_s16(_k0), 3);

kptr += 8;
r0 += 32;
}

int32x4_t _sum01 = vaddq_s32(_sum0, _sum1);

vst1q_s32(output0_tm, _sum01);

output0_tm += 4;
}
for (; i < tiles; i++)
{
#if __aarch64__
const short* r0 = bb2.row<const short>(i / 8 + (i % 8) / 4 + i % 4);
#else
const short* r0 = bb2.row<const short>(i / 4 + i % 4);
#endif
const short* kptr = kernel0_tm.row<const short>(r);

int32x4_t _sum0 = vdupq_n_s32(0);
int32x4_t _sum1 = vdupq_n_s32(0);

for (int q = 0; q < inch; q++)
{
int16x8_t _r0 = vld1q_s16(r0);

int16x8_t _k0 = vld1q_s16(kptr);

_sum0 = vmlal_s16(_sum0, vget_low_s16(_r0), vget_low_s16(_k0));
_sum1 = vmlal_s16(_sum1, vget_high_s16(_r0), vget_high_s16(_k0));

kptr += 8;
r0 += 8;
}

int32x4_t _sum = vaddq_s32(_sum0, _sum1);
#if __aarch64__
int sum = vaddvq_s32(_sum); // dot
#else
int32x2_t _ss = vadd_s32(vget_low_s32(_sum), vget_high_s32(_sum));
_ss = vpadd_s32(_ss, _ss);
int sum = vget_lane_s32(_ss, 0);
#endif

output0_tm[0] = sum;

output0_tm++;
}
}
}
}

+ 0
- 1835
src/layer/arm/convolution_winograd_dot_pack8to4_int8.h
File diff suppressed because it is too large
View File


+ 0
- 230
src/layer/arm/convolution_winograd_transform_int8.h View File

@@ -1,230 +0,0 @@
// Tencent is pleased to support the open source community by making ncnn available.
//
// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved.
//
// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
// in compliance with the License. You may obtain a copy of the License at
//
// https://opensource.org/licenses/BSD-3-Clause
//
// Unless required by applicable law or agreed to in writing, software distributed
// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
// CONDITIONS OF ANY KIND, either express or implied. See the License for the
// specific language governing permissions and limitations under the License.

static void conv3x3s1_winograd43_transform_input_int8_neon(const Mat& bottom_blob, Mat& bottom_blob_tm, const Option& opt)
{
const int w = bottom_blob.w;
const int h = bottom_blob.h;
const int inch = bottom_blob.c;

const int w_tiles = (w - 2) / 4;
const int h_tiles = (h - 2) / 4;
const int tiles = w_tiles * h_tiles;

// const float itm[6][6] = {
// {4.0f, 0.0f, -5.0f, 0.0f, 1.0f, 0.0f},
// {0.0f,-4.0f, -4.0f, 1.0f, 1.0f, 0.0f},
// {0.0f, 4.0f, -4.0f,-1.0f, 1.0f, 0.0f},
// {0.0f,-2.0f, -1.0f, 2.0f, 1.0f, 0.0f},
// {0.0f, 2.0f, -1.0f,-2.0f, 1.0f, 0.0f},
// {0.0f, 4.0f, 0.0f,-5.0f, 0.0f, 1.0f}
// };

// 0 = 4 * r00 - 5 * r02 + r04
// 1 = -4 * (r01 + r02) + r04 + r03
// 2 = 4 * (r01 - r02) + r04 - r03
// 3 = -2 * (r01 - r03) + r04 - r02
// 4 = 2 * (r01 - r03) + r04 - r02
// 5 = 4 * r01 - 5 * r03 + r05

#pragma omp parallel for num_threads(opt.num_threads)
for (int q = 0; q < inch; q++)
{
const Mat img0 = bottom_blob.channel(q);
Mat img0_tm = bottom_blob_tm.channel(q);

short tmp[6][6];

// tile
for (int i = 0; i < h_tiles; i++)
{
for (int j = 0; j < w_tiles; j++)
{
const signed char* r0 = img0.row<const signed char>(i * 4) + (j * 4);

for (int m = 0; m < 6; m++)
{
signed char r00 = r0[0];
signed char r01 = r0[1];
signed char r02 = r0[2];
signed char r03 = r0[3];
signed char r04 = r0[4];
signed char r05 = r0[5];

short tmp0m = 4 * r00 - 5 * r02 + r04;
short tmp1m = -4 * (r01 + r02) + r04 + r03;
short tmp2m = 4 * (r01 - r02) + r04 - r03;
short tmp3m = -2 * (r01 - r03) + r04 - r02;
short tmp4m = 2 * (r01 - r03) + r04 - r02;
short tmp5m = 4 * r01 - 5 * r03 + r05;

tmp[0][m] = tmp0m;
tmp[1][m] = tmp1m;
tmp[2][m] = tmp2m;
tmp[3][m] = tmp3m;
tmp[4][m] = tmp4m;
tmp[5][m] = tmp5m;

r0 += w;
}

short* r0_tm_0 = (short*)img0_tm + (i * w_tiles + j);
short* r0_tm_1 = r0_tm_0 + tiles;
short* r0_tm_2 = r0_tm_0 + tiles * 2;
short* r0_tm_3 = r0_tm_0 + tiles * 3;
short* r0_tm_4 = r0_tm_0 + tiles * 4;
short* r0_tm_5 = r0_tm_0 + tiles * 5;

for (int m = 0; m < 6; m++)
{
short tmp00 = tmp[m][0];
short tmp01 = tmp[m][1];
short tmp02 = tmp[m][2];
short tmp03 = tmp[m][3];
short tmp04 = tmp[m][4];
short tmp05 = tmp[m][5];

short r0tm0 = 4 * tmp00 - 5 * tmp02 + tmp04;
short r0tm1 = -4 * (tmp01 + tmp02) + tmp04 + tmp03;
short r0tm2 = 4 * (tmp01 - tmp02) + tmp04 - tmp03;
short r0tm3 = -2 * (tmp01 - tmp03) + tmp04 - tmp02;
short r0tm4 = 2 * (tmp01 - tmp03) + tmp04 - tmp02;
short r0tm5 = 4 * tmp01 - 5 * tmp03 + tmp05;

r0_tm_0[0] = r0tm0;
r0_tm_1[0] = r0tm1;
r0_tm_2[0] = r0tm2;
r0_tm_3[0] = r0tm3;
r0_tm_4[0] = r0tm4;
r0_tm_5[0] = r0tm5;

r0_tm_0 += tiles * 6;
r0_tm_1 += tiles * 6;
r0_tm_2 += tiles * 6;
r0_tm_3 += tiles * 6;
r0_tm_4 += tiles * 6;
r0_tm_5 += tiles * 6;
}
}
}
}
}

static void conv3x3s1_winograd43_transform_output_int8_neon(const Mat& top_blob_tm, Mat& top_blob, const Option& opt)
{
const int outw = top_blob.w;
const int outh = top_blob.h;
const int outch = top_blob.c;

const int w_tiles = outw / 4;
const int h_tiles = outh / 4;
const int tiles = w_tiles * h_tiles;

// const float otm[4][6] = {
// {1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 0.0f},
// {0.0f, 1.0f, -1.0f, 2.0f, -2.0f, 0.0f},
// {0.0f, 1.0f, 1.0f, 4.0f, 4.0f, 0.0f},
// {0.0f, 1.0f, -1.0f, 8.0f, -8.0f, 1.0f}
// };

// 0 = r00 + (r01 + r02) + (r03 + r04)
// 1 = (r01 - r02) + (r03 - r04) * 2
// 2 = (r01 + r02) + (r03 + r04) * 4
// 3 = r05 + (r01 - r02) + (r03 - r04) * 8

#pragma omp parallel for num_threads(opt.num_threads)
for (int p = 0; p < outch; p++)
{
const Mat out0_tm = top_blob_tm.channel(p);
Mat out0 = top_blob.channel(p);

int tmp[4][6];

// tile
for (int i = 0; i < h_tiles; i++)
{
for (int j = 0; j < w_tiles; j++)
{
const int* output0_tm_0 = (const int*)out0_tm + (i * w_tiles + j) * 1;
const int* output0_tm_1 = output0_tm_0 + tiles * 1;
const int* output0_tm_2 = output0_tm_0 + tiles * 2;
const int* output0_tm_3 = output0_tm_0 + tiles * 3;
const int* output0_tm_4 = output0_tm_0 + tiles * 4;
const int* output0_tm_5 = output0_tm_0 + tiles * 5;

int* output0 = out0.row<int>(i * 4) + j * 4;

// TODO neon optimize
for (int m = 0; m < 5; m++)
{
int tmp02a = output0_tm_1[0] + output0_tm_2[0];
int tmp13a = output0_tm_1[0] - output0_tm_2[0];

int tmp02b = output0_tm_3[0] + output0_tm_4[0];
int tmp13b = output0_tm_3[0] - output0_tm_4[0];

tmp[0][m] = output0_tm_0[0] + tmp02a + tmp02b;
tmp[1][m] = tmp13a + tmp13b * 2;
tmp[2][m] = tmp02a + tmp02b * 4;
tmp[3][m] = output0_tm_5[0] * 4 + tmp13a + tmp13b * 8;

output0_tm_0 += tiles * 6;
output0_tm_1 += tiles * 6;
output0_tm_2 += tiles * 6;
output0_tm_3 += tiles * 6;
output0_tm_4 += tiles * 6;
output0_tm_5 += tiles * 6;
}
for (int m = 5; m < 6; m++)
{
int tmp02a = output0_tm_1[0] + output0_tm_2[0];
int tmp13a = output0_tm_1[0] - output0_tm_2[0];

int tmp02b = output0_tm_3[0] + output0_tm_4[0];
int tmp13b = output0_tm_3[0] - output0_tm_4[0];

tmp[0][m] = (output0_tm_0[0] + tmp02a + tmp02b) * 4;
tmp[1][m] = (tmp13a + tmp13b * 2) * 4;
tmp[2][m] = (tmp02a + tmp02b * 4) * 4;
tmp[3][m] = (output0_tm_5[0] * 4 + tmp13a + tmp13b * 8) * 4;

output0_tm_0 += tiles * 6;
output0_tm_1 += tiles * 6;
output0_tm_2 += tiles * 6;
output0_tm_3 += tiles * 6;
output0_tm_4 += tiles * 6;
output0_tm_5 += tiles * 6;
}

for (int m = 0; m < 4; m++)
{
const int* tmp0 = tmp[m];

int tmp02a = tmp0[1] + tmp0[2];
int tmp13a = tmp0[1] - tmp0[2];

int tmp02b = tmp0[3] + tmp0[4];
int tmp13b = tmp0[3] - tmp0[4];

output0[0] = (tmp0[0] + tmp02a + tmp02b) / 576;
output0[1] = (tmp13a + tmp13b * 2) / 576;
output0[2] = (tmp02a + tmp02b * 4) / 576;
output0[3] = (tmp0[5] + tmp13a + tmp13b * 8) / 576;

output0 += outw;
}
}
}
}
}

+ 0
- 178
src/layer/arm/convolution_winograd_transform_pack4_int8.h View File

@@ -1,178 +0,0 @@
// Tencent is pleased to support the open source community by making ncnn available.
//
// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved.
//
// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
// in compliance with the License. You may obtain a copy of the License at
//
// https://opensource.org/licenses/BSD-3-Clause
//
// Unless required by applicable law or agreed to in writing, software distributed
// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
// CONDITIONS OF ANY KIND, either express or implied. See the License for the
// specific language governing permissions and limitations under the License.

static void conv3x3s1_winograd43_transform_output_pack4_int8_neon(const Mat& top_blob_tm, Mat& top_blob, const Option& opt)
{
const int outw = top_blob.w;
const int outh = top_blob.h;
const int outch = top_blob.c;

const int w_tiles = outw / 4;
const int h_tiles = outh / 4;
const int tiles = w_tiles * h_tiles;

// const float otm[4][6] = {
// {1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 0.0f},
// {0.0f, 1.0f, -1.0f, 2.0f, -2.0f, 0.0f},
// {0.0f, 1.0f, 1.0f, 4.0f, 4.0f, 0.0f},
// {0.0f, 1.0f, -1.0f, 8.0f, -8.0f, 1.0f}
// };

// 0 = r00 + (r01 + r02) + (r03 + r04)
// 1 = (r01 - r02) + (r03 - r04) * 2
// 2 = (r01 + r02) + (r03 + r04) * 4
// 3 = r05 + (r01 - r02) + (r03 - r04) * 8

#pragma omp parallel for num_threads(opt.num_threads)
for (int p = 0; p < outch; p++)
{
const Mat out0_tm = top_blob_tm.channel(p);
Mat out0 = top_blob.channel(p);

int tmp[4][6][4];

// tile
for (int i = 0; i < h_tiles; i++)
{
for (int j = 0; j < w_tiles; j++)
{
const int* output0_tm_0 = (const int*)out0_tm + (i * w_tiles + j) * 4;
const int* output0_tm_1 = output0_tm_0 + tiles * 4;
const int* output0_tm_2 = output0_tm_0 + tiles * 8;
const int* output0_tm_3 = output0_tm_0 + tiles * 12;
const int* output0_tm_4 = output0_tm_0 + tiles * 16;
const int* output0_tm_5 = output0_tm_0 + tiles * 20;

int* output0 = out0.row<int>(i * 4) + (j * 4) * 4;

for (int m = 0; m < 5; m++)
{
int32x4_t _out0tm0 = vld1q_s32(output0_tm_0);
int32x4_t _out0tm1 = vld1q_s32(output0_tm_1);
int32x4_t _out0tm2 = vld1q_s32(output0_tm_2);
int32x4_t _out0tm3 = vld1q_s32(output0_tm_3);
int32x4_t _out0tm4 = vld1q_s32(output0_tm_4);
int32x4_t _out0tm5 = vld1q_s32(output0_tm_5);

int32x4_t _tmp02a = vaddq_s32(_out0tm1, _out0tm2);
int32x4_t _tmp13a = vsubq_s32(_out0tm1, _out0tm2);

int32x4_t _tmp02b = vaddq_s32(_out0tm3, _out0tm4);
int32x4_t _tmp13b = vsubq_s32(_out0tm3, _out0tm4);

int32x4_t _v2 = vdupq_n_s32(2);
int32x4_t _v4 = vdupq_n_s32(4);
int32x4_t _v8 = vdupq_n_s32(8);

int32x4_t _tmp0m = vaddq_s32(vaddq_s32(_out0tm0, _tmp02a), _tmp02b);
int32x4_t _tmp1m = vmlaq_s32(_tmp13a, _tmp13b, _v2);
int32x4_t _tmp2m = vmlaq_s32(_tmp02a, _tmp02b, _v4);
int32x4_t _tmp3m = vmlaq_s32(vmlaq_s32(_tmp13a, _out0tm5, _v4), _tmp13b, _v8);

vst1q_s32(tmp[0][m], _tmp0m);
vst1q_s32(tmp[1][m], _tmp1m);
vst1q_s32(tmp[2][m], _tmp2m);
vst1q_s32(tmp[3][m], _tmp3m);

output0_tm_0 += tiles * 24;
output0_tm_1 += tiles * 24;
output0_tm_2 += tiles * 24;
output0_tm_3 += tiles * 24;
output0_tm_4 += tiles * 24;
output0_tm_5 += tiles * 24;
}
for (int m = 5; m < 6; m++)
{
int32x4_t _out0tm0 = vld1q_s32(output0_tm_0);
int32x4_t _out0tm1 = vld1q_s32(output0_tm_1);
int32x4_t _out0tm2 = vld1q_s32(output0_tm_2);
int32x4_t _out0tm3 = vld1q_s32(output0_tm_3);
int32x4_t _out0tm4 = vld1q_s32(output0_tm_4);
int32x4_t _out0tm5 = vld1q_s32(output0_tm_5);

int32x4_t _tmp02a = vaddq_s32(_out0tm1, _out0tm2);
int32x4_t _tmp13a = vsubq_s32(_out0tm1, _out0tm2);

int32x4_t _tmp02b = vaddq_s32(_out0tm3, _out0tm4);
int32x4_t _tmp13b = vsubq_s32(_out0tm3, _out0tm4);

int32x4_t _v2 = vdupq_n_s32(2);
int32x4_t _v4 = vdupq_n_s32(4);
int32x4_t _v8 = vdupq_n_s32(8);

int32x4_t _tmp0m = vaddq_s32(vaddq_s32(_out0tm0, _tmp02a), _tmp02b);
int32x4_t _tmp1m = vmlaq_s32(_tmp13a, _tmp13b, _v2);
int32x4_t _tmp2m = vmlaq_s32(_tmp02a, _tmp02b, _v4);
int32x4_t _tmp3m = vmlaq_s32(vmlaq_s32(_tmp13a, _out0tm5, _v4), _tmp13b, _v8);

_tmp0m = vmulq_s32(_tmp0m, _v4);
_tmp1m = vmulq_s32(_tmp1m, _v4);
_tmp2m = vmulq_s32(_tmp2m, _v4);
_tmp3m = vmulq_s32(_tmp3m, _v4);

vst1q_s32(tmp[0][m], _tmp0m);
vst1q_s32(tmp[1][m], _tmp1m);
vst1q_s32(tmp[2][m], _tmp2m);
vst1q_s32(tmp[3][m], _tmp3m);

output0_tm_0 += tiles * 24;
output0_tm_1 += tiles * 24;
output0_tm_2 += tiles * 24;
output0_tm_3 += tiles * 24;
output0_tm_4 += tiles * 24;
output0_tm_5 += tiles * 24;
}

for (int m = 0; m < 4; m++)
{
int32x4_t _tmp00 = vld1q_s32(tmp[m][0]);
int32x4_t _tmp01 = vld1q_s32(tmp[m][1]);
int32x4_t _tmp02 = vld1q_s32(tmp[m][2]);
int32x4_t _tmp03 = vld1q_s32(tmp[m][3]);
int32x4_t _tmp04 = vld1q_s32(tmp[m][4]);
int32x4_t _tmp05 = vld1q_s32(tmp[m][5]);

int32x4_t _tmp02a = vaddq_s32(_tmp01, _tmp02);
int32x4_t _tmp13a = vsubq_s32(_tmp01, _tmp02);

int32x4_t _tmp02b = vaddq_s32(_tmp03, _tmp04);
int32x4_t _tmp13b = vsubq_s32(_tmp03, _tmp04);

int32x4_t _v2 = vdupq_n_s32(2);
int32x4_t _v4 = vdupq_n_s32(4);
int32x4_t _v8 = vdupq_n_s32(8);

int32x4_t _out00 = vaddq_s32(vaddq_s32(_tmp00, _tmp02a), _tmp02b);
int32x4_t _out01 = vmlaq_s32(_tmp13a, _tmp13b, _v2);
int32x4_t _out02 = vmlaq_s32(_tmp02a, _tmp02b, _v4);
int32x4_t _out03 = vmlaq_s32(vaddq_s32(_tmp05, _tmp13a), _tmp13b, _v8);

// TODO use integer trick for division by 576
float32x4_t _v576 = vdupq_n_f32(1.0 / 576);
_out00 = vcvtq_s32_f32(vmulq_f32(vcvtq_f32_s32(_out00), _v576));
_out01 = vcvtq_s32_f32(vmulq_f32(vcvtq_f32_s32(_out01), _v576));
_out02 = vcvtq_s32_f32(vmulq_f32(vcvtq_f32_s32(_out02), _v576));
_out03 = vcvtq_s32_f32(vmulq_f32(vcvtq_f32_s32(_out03), _v576));

vst1q_s32(output0, _out00);
vst1q_s32(output0 + 4, _out01);
vst1q_s32(output0 + 8, _out02);
vst1q_s32(output0 + 12, _out03);

output0 += outw * 4;
}
}
}
}
}

+ 0
- 131
src/layer/arm/convolution_winograd_transform_pack8_int8.h View File

@@ -1,131 +0,0 @@
// Tencent is pleased to support the open source community by making ncnn available.
//
// Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved.
//
// Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
// in compliance with the License. You may obtain a copy of the License at
//
// https://opensource.org/licenses/BSD-3-Clause
//
// Unless required by applicable law or agreed to in writing, software distributed
// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
// CONDITIONS OF ANY KIND, either express or implied. See the License for the
// specific language governing permissions and limitations under the License.

static void conv3x3s1_winograd43_transform_input_pack8_int8_neon(const Mat& bottom_blob, Mat& bottom_blob_tm, const Option& opt)
{
const int w = bottom_blob.w;
const int h = bottom_blob.h;
const int inch = bottom_blob.c;

const int w_tiles = (w - 2) / 4;
const int h_tiles = (h - 2) / 4;
const int tiles = w_tiles * h_tiles;

// const float itm[6][6] = {
// {4.0f, 0.0f, -5.0f, 0.0f, 1.0f, 0.0f},
// {0.0f,-4.0f, -4.0f, 1.0f, 1.0f, 0.0f},
// {0.0f, 4.0f, -4.0f,-1.0f, 1.0f, 0.0f},
// {0.0f,-2.0f, -1.0f, 2.0f, 1.0f, 0.0f},
// {0.0f, 2.0f, -1.0f,-2.0f, 1.0f, 0.0f},
// {0.0f, 4.0f, 0.0f,-5.0f, 0.0f, 1.0f}
// };

// 0 = 4 * r00 - 5 * r02 + r04
// 1 = -4 * (r01 + r02) + r04 + r03
// 2 = 4 * (r01 - r02) + r04 - r03
// 3 = -2 * (r01 - r03) + r04 - r02
// 4 = 2 * (r01 - r03) + r04 - r02
// 5 = 4 * r01 - 5 * r03 + r05

#pragma omp parallel for num_threads(opt.num_threads)
for (int q = 0; q < inch; q++)
{
const Mat img0 = bottom_blob.channel(q);
Mat img0_tm = bottom_blob_tm.channel(q);

short tmp[6][6][8];

// tile
for (int i = 0; i < h_tiles; i++)
{
for (int j = 0; j < w_tiles; j++)
{
const signed char* r0 = img0.row<const signed char>(i * 4) + (j * 4) * 8;

for (int m = 0; m < 6; m++)
{
int8x8_t _r00 = vld1_s8(r0);
int8x8_t _r01 = vld1_s8(r0 + 8);
int8x8_t _r02 = vld1_s8(r0 + 16);
int8x8_t _r03 = vld1_s8(r0 + 24);
int8x8_t _r04 = vld1_s8(r0 + 32);
int8x8_t _r05 = vld1_s8(r0 + 40);

int8x8_t _v4s8 = vdup_n_s8(4);
int8x8_t _v5s8 = vdup_n_s8(5);
int16x8_t _v2 = vdupq_n_s16(2);
int16x8_t _v4 = vdupq_n_s16(4);

int16x8_t _tmp0m = vsubq_s16(vaddw_s8(vmull_s8(_r00, _v4s8), _r04), vmull_s8(_r02, _v5s8));
int16x8_t _tmp1m = vmlsq_s16(vaddl_s8(_r04, _r03), vaddl_s8(_r01, _r02), _v4);
int16x8_t _tmp2m = vmlaq_s16(vsubl_s8(_r04, _r03), vsubl_s8(_r01, _r02), _v4);
int16x8_t _tmp3m = vmlsq_s16(vsubl_s8(_r04, _r02), vsubl_s8(_r01, _r03), _v2);
int16x8_t _tmp4m = vmlaq_s16(vsubl_s8(_r04, _r02), vsubl_s8(_r01, _r03), _v2);
int16x8_t _tmp5m = vsubq_s16(vaddw_s8(vmull_s8(_r01, _v4s8), _r05), vmull_s8(_r03, _v5s8));

vst1q_s16(tmp[0][m], _tmp0m);
vst1q_s16(tmp[1][m], _tmp1m);
vst1q_s16(tmp[2][m], _tmp2m);
vst1q_s16(tmp[3][m], _tmp3m);
vst1q_s16(tmp[4][m], _tmp4m);
vst1q_s16(tmp[5][m], _tmp5m);

r0 += w * 8;
}

short* r0_tm_0 = (short*)img0_tm + (i * w_tiles + j) * 8;
short* r0_tm_1 = r0_tm_0 + tiles * 8;
short* r0_tm_2 = r0_tm_0 + tiles * 16;
short* r0_tm_3 = r0_tm_0 + tiles * 24;
short* r0_tm_4 = r0_tm_0 + tiles * 32;
short* r0_tm_5 = r0_tm_0 + tiles * 40;

for (int m = 0; m < 6; m++)
{
int16x8_t _tmp00 = vld1q_s16(tmp[m][0]);
int16x8_t _tmp01 = vld1q_s16(tmp[m][1]);
int16x8_t _tmp02 = vld1q_s16(tmp[m][2]);
int16x8_t _tmp03 = vld1q_s16(tmp[m][3]);
int16x8_t _tmp04 = vld1q_s16(tmp[m][4]);
int16x8_t _tmp05 = vld1q_s16(tmp[m][5]);

int16x8_t _v2 = vdupq_n_s16(2);
int16x8_t _v4 = vdupq_n_s16(4);
int16x8_t _v5 = vdupq_n_s16(5);

int16x8_t _r0tm0 = vmlsq_s16(vmlaq_s16(_tmp04, _tmp00, _v4), _tmp02, _v5);
int16x8_t _r0tm1 = vmlsq_s16(vaddq_s16(_tmp04, _tmp03), vaddq_s16(_tmp01, _tmp02), _v4);
int16x8_t _r0tm2 = vmlaq_s16(vsubq_s16(_tmp04, _tmp03), vsubq_s16(_tmp01, _tmp02), _v4);
int16x8_t _r0tm3 = vmlsq_s16(vsubq_s16(_tmp04, _tmp02), vsubq_s16(_tmp01, _tmp03), _v2);
int16x8_t _r0tm4 = vmlaq_s16(vsubq_s16(_tmp04, _tmp02), vsubq_s16(_tmp01, _tmp03), _v2);
int16x8_t _r0tm5 = vmlsq_s16(vmlaq_s16(_tmp05, _tmp01, _v4), _tmp03, _v5);

vst1q_s16(r0_tm_0, _r0tm0);
vst1q_s16(r0_tm_1, _r0tm1);
vst1q_s16(r0_tm_2, _r0tm2);
vst1q_s16(r0_tm_3, _r0tm3);
vst1q_s16(r0_tm_4, _r0tm4);
vst1q_s16(r0_tm_5, _r0tm5);

r0_tm_0 += tiles * 48;
r0_tm_1 += tiles * 48;
r0_tm_2 += tiles * 48;
r0_tm_3 += tiles * 48;
r0_tm_4 += tiles * 48;
r0_tm_5 += tiles * 48;
}
}
}
}
}

+ 49
- 39
src/net.cpp View File

@@ -610,67 +610,41 @@ IMAGE_ALLOCATION_FAILED:

int NetPrivate::convert_layout(Mat& bottom_blob, const Layer* layer, const Option& opt) const
{
// clang-format off
// *INDENT-OFF*
#if NCNN_ARM82
if (opt.use_fp16_storage && cpu_support_arm_asimdhp())
if (bottom_blob.elembits() == 32)
{
if (bottom_blob.elembits() == 32 && layer->support_fp16_storage)
// clang-format off
// *INDENT-OFF*

#if NCNN_ARM82
if (opt.use_fp16_storage && cpu_support_arm_asimdhp() && layer->support_fp16_storage)
{
Mat bottom_blob_fp16;
cast_float32_to_float16(bottom_blob, bottom_blob_fp16, opt);
bottom_blob = bottom_blob_fp16;
}
if (bottom_blob.elembits() == 16 && !layer->support_fp16_storage)
{
Mat bottom_blob_fp32;
cast_float16_to_float32(bottom_blob, bottom_blob_fp32, opt);
bottom_blob = bottom_blob_fp32;
}
}
else
else
#endif // NCNN_ARM82
#if NCNN_RVV
if (opt.use_fp16_storage && cpu_support_riscv_v() && cpu_support_riscv_zfh())
{
if (bottom_blob.elembits() == 32 && layer->support_fp16_storage)
if (opt.use_fp16_storage && cpu_support_riscv_v() && cpu_support_riscv_zfh() && layer->support_fp16_storage)
{
Mat bottom_blob_fp16;
cast_float32_to_float16(bottom_blob, bottom_blob_fp16, opt);
bottom_blob = bottom_blob_fp16;
}
if (bottom_blob.elembits() == 16 && !layer->support_fp16_storage)
{
Mat bottom_blob_fp32;
cast_float16_to_float32(bottom_blob, bottom_blob_fp32, opt);
bottom_blob = bottom_blob_fp32;
}
}
else
else
#endif // NCNN_RVV
#if NCNN_BF16
if (opt.use_bf16_storage)
{
if (bottom_blob.elembits() == 32 && layer->support_bf16_storage)
if (opt.use_bf16_storage && layer->support_bf16_storage)
{
Mat bottom_blob_bf16;
cast_float32_to_bfloat16(bottom_blob, bottom_blob_bf16, opt);
bottom_blob = bottom_blob_bf16;
}
if (bottom_blob.elembits() == 16 && !layer->support_bf16_storage)
{
Mat bottom_blob_fp32;
cast_bfloat16_to_float32(bottom_blob, bottom_blob_fp32, opt);
bottom_blob = bottom_blob_fp32;
}
}
else
#endif // NCNN_BF16
{
// no type conversion

// *INDENT-ON*
// clang-format on
}
// *INDENT-ON*
// clang-format on

int dst_elempack = 1;
if (opt.use_packing_layout)
@@ -746,6 +720,42 @@ int NetPrivate::convert_layout(Mat& bottom_blob, const Layer* layer, const Optio
bottom_blob = bottom_blob_packed;
}

if (bottom_blob.elembits() == 16)
{
// clang-format off
// *INDENT-OFF*

#if NCNN_ARM82
if (opt.use_fp16_storage && cpu_support_arm_asimdhp() && !layer->support_fp16_storage)
{
Mat bottom_blob_fp32;
cast_float16_to_float32(bottom_blob, bottom_blob_fp32, opt);
bottom_blob = bottom_blob_fp32;
}
else
#endif // NCNN_ARM82
#if NCNN_RVV
if (opt.use_fp16_storage && cpu_support_riscv_v() && cpu_support_riscv_zfh() && !layer->support_fp16_storage)
{
Mat bottom_blob_fp32;
cast_float16_to_float32(bottom_blob, bottom_blob_fp32, opt);
bottom_blob = bottom_blob_fp32;
}
else
#endif // NCNN_RVV
#if NCNN_BF16
if (opt.use_bf16_storage && !layer->support_bf16_storage)
{
Mat bottom_blob_fp32;
cast_bfloat16_to_float32(bottom_blob, bottom_blob_fp32, opt);
bottom_blob = bottom_blob_fp32;
}
#endif // NCNN_BF16

// *INDENT-ON*
// clang-format on
}

return 0;
}



Write Preview
Loading…
Cancel
Save