nihui
/
ncnn
Not watched
1
0
Fork 0
Code
Releases 46 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
Browse Source

conv3x3s1 conv3x3s2 pack1to8, padding pack8, relu pack8 arm neon fp16sa assmebly optimization

tags/20200916
nihui 6 years ago
parent
076b509bb9
commit
d8e9fc1443
4 changed files with 756 additions and 174 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +566
    -126
      src/layer/arm/convolution_3x3_pack1to8_fp16s.h
  2. +10
    -1
      src/layer/arm/convolution_arm.cpp
  3. +139
    -39
      src/layer/arm/padding_pack8_fp16s.h
  4. +41
    -8
      src/layer/arm/relu_arm.cpp

+ 566
- 126
src/layer/arm/convolution_3x3_pack1to8_fp16s.h View File

@@ -12,6 +12,385 @@
// CONDITIONS OF ANY KIND, either express or implied. See the License for the
// specific language governing permissions and limitations under the License.

static void conv3x3s1_pack1to8_fp16sa_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& kernel, const Mat& _bias, const Option& opt)
{
int inch = bottom_blob.c;
int outw = top_blob.w;
int outh = top_blob.h;
int outch = top_blob.c;

const __fp16* bias = _bias;

#pragma omp parallel for num_threads(opt.num_threads)
for (int p = 0; p < outch; p++)
{
Mat out0 = top_blob.channel(p);

float16x8_t _bias0 = bias ? vld1q_f16(bias + p * 8) : vdupq_n_f16((__fp16)0.f);
out0.fill(_bias0);

const __fp16* k0 = kernel.channel(p);

int q = 0;
for (; q < inch; q++)
{
__fp16* outptr0 = out0;

const Mat img0 = bottom_blob.channel(q);

const __fp16* r0 = img0.row<const __fp16>(0);
const __fp16* r1 = img0.row<const __fp16>(1);
const __fp16* r2 = img0.row<const __fp16>(2);

float16x8_t _k00 = vld1q_f16(k0);
float16x8_t _k01 = vld1q_f16(k0 + 8);
float16x8_t _k02 = vld1q_f16(k0 + 16);
float16x8_t _k10 = vld1q_f16(k0 + 24);
float16x8_t _k11 = vld1q_f16(k0 + 32);
float16x8_t _k12 = vld1q_f16(k0 + 40);
float16x8_t _k20 = vld1q_f16(k0 + 48);
float16x8_t _k21 = vld1q_f16(k0 + 56);
float16x8_t _k22 = vld1q_f16(k0 + 64);

int i = 0;
for (; i < outh; i++)
{
int j = 0;
for (; j + 7 < outw; j += 8)
{
asm volatile(
"prfm pldl1keep, [%0, #512] \n"
"ld1 {v24.8h, v25.8h, v26.8h, v27.8h}, [%0], #64 \n" // sum0 sum1 sum2 sum3

"prfm pldl1keep, [%0, #512] \n"
"ld1 {v28.8h, v29.8h, v30.8h, v31.8h}, [%0] \n" // sum4 sum5 sum6 sum7

"sub %0, %0, #64 \n"

"prfm pldl1keep, [%1, #128] \n"
"ld1 {v0.8h}, [%1], #16 \n" // r0
"ld1 {v1.4h}, [%1] \n"

"fmla v24.8h, %8.8h, v0.h[0] \n"
"fmla v25.8h, %8.8h, v0.h[1] \n"
"fmla v26.8h, %8.8h, v0.h[2] \n"
"fmla v27.8h, %8.8h, v0.h[3] \n"
"fmla v28.8h, %8.8h, v0.h[4] \n"
"fmla v29.8h, %8.8h, v0.h[5] \n"
"fmla v30.8h, %8.8h, v0.h[6] \n"
"fmla v31.8h, %8.8h, v0.h[7] \n"

"fmla v24.8h, %9.8h, v0.h[1] \n"
"fmla v25.8h, %9.8h, v0.h[2] \n"
"fmla v26.8h, %9.8h, v0.h[3] \n"
"fmla v27.8h, %9.8h, v0.h[4] \n"
"fmla v28.8h, %9.8h, v0.h[5] \n"
"fmla v29.8h, %9.8h, v0.h[6] \n"
"fmla v30.8h, %9.8h, v0.h[7] \n"
"fmla v31.8h, %9.8h, v1.h[0] \n"

"fmla v24.8h, %10.8h, v0.h[2] \n"
"fmla v25.8h, %10.8h, v0.h[3] \n"
"fmla v26.8h, %10.8h, v0.h[4] \n"
"fmla v27.8h, %10.8h, v0.h[5] \n"
"fmla v28.8h, %10.8h, v0.h[6] \n"
"fmla v29.8h, %10.8h, v0.h[7] \n"
"fmla v30.8h, %10.8h, v1.h[0] \n"
"fmla v31.8h, %10.8h, v1.h[1] \n"

"prfm pldl1keep, [%2, #128] \n"
"ld1 {v2.8h}, [%2], #16 \n" // r1
"ld1 {v3.4h}, [%2] \n"

"fmla v24.8h, %11.8h, v2.h[0] \n"
"fmla v25.8h, %11.8h, v2.h[1] \n"
"fmla v26.8h, %11.8h, v2.h[2] \n"
"fmla v27.8h, %11.8h, v2.h[3] \n"
"fmla v28.8h, %11.8h, v2.h[4] \n"
"fmla v29.8h, %11.8h, v2.h[5] \n"
"fmla v30.8h, %11.8h, v2.h[6] \n"
"fmla v31.8h, %11.8h, v2.h[7] \n"

"fmla v24.8h, %12.8h, v2.h[1] \n"
"fmla v25.8h, %12.8h, v2.h[2] \n"
"fmla v26.8h, %12.8h, v2.h[3] \n"
"fmla v27.8h, %12.8h, v2.h[4] \n"
"fmla v28.8h, %12.8h, v2.h[5] \n"
"fmla v29.8h, %12.8h, v2.h[6] \n"
"fmla v30.8h, %12.8h, v2.h[7] \n"
"fmla v31.8h, %12.8h, v3.h[0] \n"

"fmla v24.8h, %13.8h, v2.h[2] \n"
"fmla v25.8h, %13.8h, v2.h[3] \n"
"fmla v26.8h, %13.8h, v2.h[4] \n"
"fmla v27.8h, %13.8h, v2.h[5] \n"
"fmla v28.8h, %13.8h, v2.h[6] \n"
"fmla v29.8h, %13.8h, v2.h[7] \n"
"fmla v30.8h, %13.8h, v3.h[0] \n"
"fmla v31.8h, %13.8h, v3.h[1] \n"

"prfm pldl1keep, [%3, #128] \n"
"ld1 {v4.8h}, [%3], #16 \n" // r2
"ld1 {v5.4h}, [%3] \n"

"fmla v24.8h, %14.8h, v4.h[0] \n"
"fmla v25.8h, %14.8h, v4.h[1] \n"
"fmla v26.8h, %14.8h, v4.h[2] \n"
"fmla v27.8h, %14.8h, v4.h[3] \n"
"fmla v28.8h, %14.8h, v4.h[4] \n"
"fmla v29.8h, %14.8h, v4.h[5] \n"
"fmla v30.8h, %14.8h, v4.h[6] \n"
"fmla v31.8h, %14.8h, v4.h[7] \n"

"fmla v24.8h, %15.8h, v4.h[1] \n"
"fmla v25.8h, %15.8h, v4.h[2] \n"
"fmla v26.8h, %15.8h, v4.h[3] \n"
"fmla v27.8h, %15.8h, v4.h[4] \n"
"fmla v28.8h, %15.8h, v4.h[5] \n"
"fmla v29.8h, %15.8h, v4.h[6] \n"
"fmla v30.8h, %15.8h, v4.h[7] \n"
"fmla v31.8h, %15.8h, v5.h[0] \n"

"fmla v24.8h, %16.8h, v4.h[2] \n"
"fmla v25.8h, %16.8h, v4.h[3] \n"
"fmla v26.8h, %16.8h, v4.h[4] \n"
"fmla v27.8h, %16.8h, v4.h[5] \n"
"fmla v28.8h, %16.8h, v4.h[6] \n"
"fmla v29.8h, %16.8h, v4.h[7] \n"
"fmla v30.8h, %16.8h, v5.h[0] \n"
"fmla v31.8h, %16.8h, v5.h[1] \n"

"st1 {v24.8h, v25.8h, v26.8h, v27.8h}, [%0], #64 \n"
"st1 {v28.8h, v29.8h, v30.8h, v31.8h}, [%0], #64 \n"

: "=r"(outptr0), // %0
"=r"(r0), // %1
"=r"(r1), // %2
"=r"(r2) // %3
: "0"(outptr0),
"1"(r0),
"2"(r1),
"3"(r2),
"w"(_k00), // %8
"w"(_k01), // %9
"w"(_k02), // %10
"w"(_k10), // %11
"w"(_k11), // %12
"w"(_k12), // %13
"w"(_k20), // %14
"w"(_k21), // %15
"w"(_k22) // %16
: "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31");
}
for (; j + 3 < outw; j += 4)
{
asm volatile(
"prfm pldl1keep, [%0, #512] \n"
"ld1 {v28.8h, v29.8h, v30.8h, v31.8h}, [%0] \n" // sum0 sum1 sum2 sum3

"prfm pldl1keep, [%1, #128] \n"
"ld1 {v0.8h}, [%1] \n" // r0

"fmla v28.8h, %8.8h, v0.h[0] \n"
"fmla v29.8h, %8.8h, v0.h[1] \n"
"fmla v30.8h, %8.8h, v0.h[2] \n"
"fmla v31.8h, %8.8h, v0.h[3] \n"

"fmla v28.8h, %9.8h, v0.h[1] \n"
"fmla v29.8h, %9.8h, v0.h[2] \n"
"fmla v30.8h, %9.8h, v0.h[3] \n"
"fmla v31.8h, %9.8h, v0.h[4] \n"

"fmla v28.8h, %10.8h, v0.h[2] \n"
"fmla v29.8h, %10.8h, v0.h[3] \n"
"fmla v30.8h, %10.8h, v0.h[4] \n"
"fmla v31.8h, %10.8h, v0.h[5] \n"

"prfm pldl1keep, [%2, #128] \n"
"ld1 {v1.8h}, [%2] \n" // r1

"fmla v28.8h, %11.8h, v1.h[0] \n"
"fmla v29.8h, %11.8h, v1.h[1] \n"
"fmla v30.8h, %11.8h, v1.h[2] \n"
"fmla v31.8h, %11.8h, v1.h[3] \n"

"fmla v28.8h, %12.8h, v1.h[1] \n"
"fmla v29.8h, %12.8h, v1.h[2] \n"
"fmla v30.8h, %12.8h, v1.h[3] \n"
"fmla v31.8h, %12.8h, v1.h[4] \n"

"fmla v28.8h, %13.8h, v1.h[2] \n"
"fmla v29.8h, %13.8h, v1.h[3] \n"
"fmla v30.8h, %13.8h, v1.h[4] \n"
"fmla v31.8h, %13.8h, v1.h[5] \n"

"prfm pldl1keep, [%3, #128] \n"
"ld1 {v2.8h}, [%3] \n" // r2

"fmla v28.8h, %14.8h, v2.h[0] \n"
"fmla v29.8h, %14.8h, v2.h[1] \n"
"fmla v30.8h, %14.8h, v2.h[2] \n"
"fmla v31.8h, %14.8h, v2.h[3] \n"

"fmla v28.8h, %15.8h, v2.h[1] \n"
"fmla v29.8h, %15.8h, v2.h[2] \n"
"fmla v30.8h, %15.8h, v2.h[3] \n"
"fmla v31.8h, %15.8h, v2.h[4] \n"

"fmla v28.8h, %16.8h, v2.h[2] \n"
"fmla v29.8h, %16.8h, v2.h[3] \n"
"fmla v30.8h, %16.8h, v2.h[4] \n"
"fmla v31.8h, %16.8h, v2.h[5] \n"

"add %1, %1, #8 \n"
"add %2, %2, #8 \n"
"add %3, %3, #8 \n"

"st1 {v28.8h, v29.8h, v30.8h, v31.8h}, [%0], #64 \n"

: "=r"(outptr0), // %0
"=r"(r0), // %1
"=r"(r1), // %2
"=r"(r2) // %3
: "0"(outptr0),
"1"(r0),
"2"(r1),
"3"(r2),
"w"(_k00), // %8
"w"(_k01), // %9
"w"(_k02), // %10
"w"(_k10), // %11
"w"(_k11), // %12
"w"(_k12), // %13
"w"(_k20), // %14
"w"(_k21), // %15
"w"(_k22) // %16
: "cc", "memory", "v0", "v1", "v2", "v28", "v29", "v30", "v31");
}
for (; j + 1 < outw; j += 2)
{
asm volatile(
"prfm pldl1keep, [%0, #256] \n"
"ld1 {v30.8h, v31.8h}, [%0] \n" // sum0 sum1

"prfm pldl1keep, [%1, #64] \n"
"ld1 {v0.4h}, [%1] \n" // r0

"fmla v30.8h, %8.8h, v0.h[0] \n"
"fmla v31.8h, %8.8h, v0.h[1] \n"
"fmla v30.8h, %9.8h, v0.h[1] \n"
"fmla v31.8h, %9.8h, v0.h[2] \n"
"fmla v30.8h, %10.8h, v0.h[2] \n"
"fmla v31.8h, %10.8h, v0.h[3] \n"

"prfm pldl1keep, [%2, #64] \n"
"ld1 {v1.4h}, [%2] \n" // r1

"fmla v30.8h, %11.8h, v1.h[0] \n"
"fmla v31.8h, %11.8h, v1.h[1] \n"
"fmla v30.8h, %12.8h, v1.h[1] \n"
"fmla v31.8h, %12.8h, v1.h[2] \n"
"fmla v30.8h, %13.8h, v1.h[2] \n"
"fmla v31.8h, %13.8h, v1.h[3] \n"

"prfm pldl1keep, [%3, #64] \n"
"ld1 {v2.4h}, [%3] \n" // r2

"fmla v30.8h, %14.8h, v2.h[0] \n"
"fmla v31.8h, %14.8h, v2.h[1] \n"
"fmla v30.8h, %15.8h, v2.h[1] \n"
"fmla v31.8h, %15.8h, v2.h[2] \n"
"fmla v30.8h, %16.8h, v2.h[2] \n"
"fmla v31.8h, %16.8h, v2.h[3] \n"

"add %1, %1, #4 \n"
"add %2, %2, #4 \n"
"add %3, %3, #4 \n"

"st1 {v30.8h, v31.8h}, [%0], #32 \n"

: "=r"(outptr0), // %0
"=r"(r0), // %1
"=r"(r1), // %2
"=r"(r2) // %3
: "0"(outptr0),
"1"(r0),
"2"(r1),
"3"(r2),
"w"(_k00), // %8
"w"(_k01), // %9
"w"(_k02), // %10
"w"(_k10), // %11
"w"(_k11), // %12
"w"(_k12), // %13
"w"(_k20), // %14
"w"(_k21), // %15
"w"(_k22) // %16
: "cc", "memory", "v0", "v1", "v2", "v30", "v31");
}
for (; j < outw; j++)
{
asm volatile(
"prfm pldl1keep, [%0, #128] \n"
"ld1 {v30.8h}, [%0] \n" // sum0

"prfm pldl1keep, [%1, #64] \n"
"ld1 {v0.4h}, [%1] \n" // r0

"fmla v30.8h, %8.8h, v0.h[0] \n"
"fmla v30.8h, %9.8h, v0.h[1] \n"
"fmla v30.8h, %10.8h, v0.h[2] \n"

"prfm pldl1keep, [%2, #64] \n"
"ld1 {v1.4h}, [%2] \n" // r1

"fmla v30.8h, %11.8h, v1.h[0] \n"
"fmla v30.8h, %12.8h, v1.h[1] \n"
"fmla v30.8h, %13.8h, v1.h[2] \n"

"prfm pldl1keep, [%3, #64] \n"
"ld1 {v2.4h}, [%3] \n" // r2

"fmla v30.8h, %14.8h, v2.h[0] \n"
"fmla v30.8h, %15.8h, v2.h[1] \n"
"fmla v30.8h, %16.8h, v2.h[2] \n"

"add %1, %1, #2 \n"
"add %2, %2, #2 \n"
"add %3, %3, #2 \n"

"st1 {v30.8h}, [%0], #16 \n"

: "=r"(outptr0), // %0
"=r"(r0), // %1
"=r"(r1), // %2
"=r"(r2) // %3
: "0"(outptr0),
"1"(r0),
"2"(r1),
"3"(r2),
"w"(_k00), // %8
"w"(_k01), // %9
"w"(_k02), // %10
"w"(_k10), // %11
"w"(_k11), // %12
"w"(_k12), // %13
"w"(_k20), // %14
"w"(_k21), // %15
"w"(_k22) // %16
: "cc", "memory", "v0", "v1", "v2", "v30");
}

r0 += 2;
r1 += 2;
r2 += 2;
}

k0 += 9 * 8;
}
}
}

static void conv3x3s2_pack1to8_fp16sa_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& kernel, const Mat& _bias, const Option& opt)
{
int w = bottom_blob.w;
@@ -56,143 +435,204 @@ static void conv3x3s2_pack1to8_fp16sa_neon(const Mat& bottom_blob, Mat& top_blob
float16x8_t _k22 = vld1q_f16(k0 + 64);

int i = 0;

for (; i < outh; i++)
{
int j = 0;

for (; j + 3 < outw; j += 4)
{
float16x8_t _sum0 = vld1q_f16(outptr0);
float16x8_t _sum1 = vld1q_f16(outptr0 + 8);
float16x8_t _sum2 = vld1q_f16(outptr0 + 16);
float16x8_t _sum3 = vld1q_f16(outptr0 + 24);

float16x8_t _r0 = vld1q_f16(r0);
float16x8_t _r1 = vld1q_f16(r1);
float16x8_t _r2 = vld1q_f16(r2);

r0 += 8;
r1 += 8;
r2 += 8;

float16x4_t _r0n = vld1_dup_f16(r0);
float16x4_t _r1n = vld1_dup_f16(r1);
float16x4_t _r2n = vld1_dup_f16(r2);

_sum0 = vfmaq_laneq_f16(_sum0, _k00, _r0, 0);
_sum0 = vfmaq_laneq_f16(_sum0, _k01, _r0, 1);
_sum0 = vfmaq_laneq_f16(_sum0, _k02, _r0, 2);
_sum0 = vfmaq_laneq_f16(_sum0, _k10, _r1, 0);
_sum0 = vfmaq_laneq_f16(_sum0, _k11, _r1, 1);
_sum0 = vfmaq_laneq_f16(_sum0, _k12, _r1, 2);
_sum0 = vfmaq_laneq_f16(_sum0, _k20, _r2, 0);
_sum0 = vfmaq_laneq_f16(_sum0, _k21, _r2, 1);
_sum0 = vfmaq_laneq_f16(_sum0, _k22, _r2, 2);

_sum1 = vfmaq_laneq_f16(_sum1, _k00, _r0, 2);
_sum1 = vfmaq_laneq_f16(_sum1, _k01, _r0, 3);
_sum1 = vfmaq_laneq_f16(_sum1, _k02, _r0, 4);
_sum1 = vfmaq_laneq_f16(_sum1, _k10, _r1, 2);
_sum1 = vfmaq_laneq_f16(_sum1, _k11, _r1, 3);
_sum1 = vfmaq_laneq_f16(_sum1, _k12, _r1, 4);
_sum1 = vfmaq_laneq_f16(_sum1, _k20, _r2, 2);
_sum1 = vfmaq_laneq_f16(_sum1, _k21, _r2, 3);
_sum1 = vfmaq_laneq_f16(_sum1, _k22, _r2, 4);

_sum2 = vfmaq_laneq_f16(_sum2, _k00, _r0, 4);
_sum2 = vfmaq_laneq_f16(_sum2, _k01, _r0, 5);
_sum2 = vfmaq_laneq_f16(_sum2, _k02, _r0, 6);
_sum2 = vfmaq_laneq_f16(_sum2, _k10, _r1, 4);
_sum2 = vfmaq_laneq_f16(_sum2, _k11, _r1, 5);
_sum2 = vfmaq_laneq_f16(_sum2, _k12, _r1, 6);
_sum2 = vfmaq_laneq_f16(_sum2, _k20, _r2, 4);
_sum2 = vfmaq_laneq_f16(_sum2, _k21, _r2, 5);
_sum2 = vfmaq_laneq_f16(_sum2, _k22, _r2, 6);

_sum3 = vfmaq_laneq_f16(_sum3, _k00, _r0, 6);
_sum3 = vfmaq_laneq_f16(_sum3, _k01, _r0, 7);
_sum3 = vfmaq_lane_f16(_sum3, _k02, _r0n, 0);
_sum3 = vfmaq_laneq_f16(_sum3, _k10, _r1, 6);
_sum3 = vfmaq_laneq_f16(_sum3, _k11, _r1, 7);
_sum3 = vfmaq_lane_f16(_sum3, _k12, _r1n, 0);
_sum3 = vfmaq_laneq_f16(_sum3, _k20, _r2, 6);
_sum3 = vfmaq_laneq_f16(_sum3, _k21, _r2, 7);
_sum3 = vfmaq_lane_f16(_sum3, _k22, _r2n, 0);

vst1q_f16(outptr0, _sum0);
vst1q_f16(outptr0 + 8, _sum1);
vst1q_f16(outptr0 + 16, _sum2);
vst1q_f16(outptr0 + 24, _sum3);

outptr0 += 32;
asm volatile(
"prfm pldl1keep, [%0, #512] \n"
"ld1 {v28.8h, v29.8h, v30.8h, v31.8h}, [%0] \n" // sum0 sum1 sum2 sum3

"prfm pldl1keep, [%1, #128] \n"
"ld1 {v0.8h}, [%1], #16 \n" // r0
"ld1 {v1.h}[0], [%1] \n"

"fmla v28.8h, %8.8h, v0.h[0] \n"
"fmla v29.8h, %8.8h, v0.h[2] \n"
"fmla v30.8h, %8.8h, v0.h[4] \n"
"fmla v31.8h, %8.8h, v0.h[6] \n"

"fmla v28.8h, %9.8h, v0.h[1] \n"
"fmla v29.8h, %9.8h, v0.h[3] \n"
"fmla v30.8h, %9.8h, v0.h[5] \n"
"fmla v31.8h, %9.8h, v0.h[7] \n"

"fmla v28.8h, %10.8h, v0.h[2] \n"
"fmla v29.8h, %10.8h, v0.h[4] \n"
"fmla v30.8h, %10.8h, v0.h[6] \n"
"fmla v31.8h, %10.8h, v1.h[0] \n"

"prfm pldl1keep, [%2, #128] \n"
"ld1 {v2.8h}, [%2], #16 \n" // r1
"ld1 {v3.h}[0], [%2] \n"

"fmla v28.8h, %11.8h, v2.h[0] \n"
"fmla v29.8h, %11.8h, v2.h[2] \n"
"fmla v30.8h, %11.8h, v2.h[4] \n"
"fmla v31.8h, %11.8h, v2.h[6] \n"

"fmla v28.8h, %12.8h, v2.h[1] \n"
"fmla v29.8h, %12.8h, v2.h[3] \n"
"fmla v30.8h, %12.8h, v2.h[5] \n"
"fmla v31.8h, %12.8h, v2.h[7] \n"

"fmla v28.8h, %13.8h, v2.h[2] \n"
"fmla v29.8h, %13.8h, v2.h[4] \n"
"fmla v30.8h, %13.8h, v2.h[6] \n"
"fmla v31.8h, %13.8h, v3.h[0] \n"

"prfm pldl1keep, [%3, #128] \n"
"ld1 {v4.8h}, [%3], #16 \n" // r2
"ld1 {v5.h}[0], [%3] \n"

"fmla v28.8h, %14.8h, v4.h[0] \n"
"fmla v29.8h, %14.8h, v4.h[2] \n"
"fmla v30.8h, %14.8h, v4.h[4] \n"
"fmla v31.8h, %14.8h, v4.h[6] \n"

"fmla v28.8h, %15.8h, v4.h[1] \n"
"fmla v29.8h, %15.8h, v4.h[3] \n"
"fmla v30.8h, %15.8h, v4.h[5] \n"
"fmla v31.8h, %15.8h, v4.h[7] \n"

"fmla v28.8h, %16.8h, v4.h[2] \n"
"fmla v29.8h, %16.8h, v4.h[4] \n"
"fmla v30.8h, %16.8h, v4.h[6] \n"
"fmla v31.8h, %16.8h, v5.h[0] \n"

"st1 {v28.8h, v29.8h, v30.8h, v31.8h}, [%0], #64 \n"

: "=r"(outptr0), // %0
"=r"(r0), // %1
"=r"(r1), // %2
"=r"(r2) // %3
: "0"(outptr0),
"1"(r0),
"2"(r1),
"3"(r2),
"w"(_k00), // %8
"w"(_k01), // %9
"w"(_k02), // %10
"w"(_k10), // %11
"w"(_k11), // %12
"w"(_k12), // %13
"w"(_k20), // %14
"w"(_k21), // %15
"w"(_k22) // %16
: "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v28", "v29", "v30", "v31");
}
for (; j + 1 < outw; j += 2)
{
float16x8_t _sum0 = vld1q_f16(outptr0);
float16x8_t _sum1 = vld1q_f16(outptr0 + 8);

float16x4_t _r0 = vld1_f16(r0);
float16x4_t _r1 = vld1_f16(r1);
float16x4_t _r2 = vld1_f16(r2);

r0 += 4;
r1 += 4;
r2 += 4;

float16x4_t _r0n = vld1_dup_f16(r0);
float16x4_t _r1n = vld1_dup_f16(r1);
float16x4_t _r2n = vld1_dup_f16(r2);

_sum0 = vfmaq_lane_f16(_sum0, _k00, _r0, 0);
_sum0 = vfmaq_lane_f16(_sum0, _k01, _r0, 1);
_sum0 = vfmaq_lane_f16(_sum0, _k02, _r0, 2);
_sum0 = vfmaq_lane_f16(_sum0, _k10, _r1, 0);
_sum0 = vfmaq_lane_f16(_sum0, _k11, _r1, 1);
_sum0 = vfmaq_lane_f16(_sum0, _k12, _r1, 2);
_sum0 = vfmaq_lane_f16(_sum0, _k20, _r2, 0);
_sum0 = vfmaq_lane_f16(_sum0, _k21, _r2, 1);
_sum0 = vfmaq_lane_f16(_sum0, _k22, _r2, 2);

_sum1 = vfmaq_lane_f16(_sum1, _k00, _r0, 2);
_sum1 = vfmaq_lane_f16(_sum1, _k01, _r0, 3);
_sum1 = vfmaq_lane_f16(_sum1, _k02, _r0n, 0);
_sum1 = vfmaq_lane_f16(_sum1, _k10, _r1, 2);
_sum1 = vfmaq_lane_f16(_sum1, _k11, _r1, 3);
_sum1 = vfmaq_lane_f16(_sum1, _k12, _r1n, 0);
_sum1 = vfmaq_lane_f16(_sum1, _k20, _r2, 2);
_sum1 = vfmaq_lane_f16(_sum1, _k21, _r2, 3);
_sum1 = vfmaq_lane_f16(_sum1, _k22, _r2n, 0);

vst1q_f16(outptr0, _sum0);
vst1q_f16(outptr0 + 8, _sum1);

outptr0 += 16;
asm volatile(
"prfm pldl1keep, [%0, #256] \n"
"ld1 {v30.8h, v31.8h}, [%0] \n" // sum0 sum1

"prfm pldl1keep, [%1, #64] \n"
"ld1 {v0.4h}, [%1], #8 \n" // r0
"ld1 {v1.h}[0], [%1] \n"

"fmla v30.8h, %8.8h, v0.h[0] \n"
"fmla v31.8h, %8.8h, v0.h[2] \n"
"fmla v30.8h, %9.8h, v0.h[1] \n"
"fmla v31.8h, %9.8h, v0.h[3] \n"
"fmla v30.8h, %10.8h, v0.h[2] \n"
"fmla v31.8h, %10.8h, v1.h[0] \n"

"prfm pldl1keep, [%2, #64] \n"
"ld1 {v2.4h}, [%2], #8 \n" // r1
"ld1 {v3.h}[0], [%2] \n"

"fmla v30.8h, %11.8h, v2.h[0] \n"
"fmla v31.8h, %11.8h, v2.h[2] \n"
"fmla v30.8h, %12.8h, v2.h[1] \n"
"fmla v31.8h, %12.8h, v2.h[3] \n"
"fmla v30.8h, %13.8h, v2.h[2] \n"
"fmla v31.8h, %13.8h, v3.h[0] \n"

"prfm pldl1keep, [%3, #64] \n"
"ld1 {v4.4h}, [%3], #8 \n" // r2
"ld1 {v5.h}[0], [%3] \n"

"fmla v30.8h, %14.8h, v4.h[0] \n"
"fmla v31.8h, %14.8h, v4.h[2] \n"
"fmla v30.8h, %15.8h, v4.h[1] \n"
"fmla v31.8h, %15.8h, v4.h[3] \n"
"fmla v30.8h, %16.8h, v4.h[2] \n"
"fmla v31.8h, %16.8h, v5.h[0] \n"

"st1 {v30.8h, v31.8h}, [%0], #32 \n"

: "=r"(outptr0), // %0
"=r"(r0), // %1
"=r"(r1), // %2
"=r"(r2) // %3
: "0"(outptr0),
"1"(r0),
"2"(r1),
"3"(r2),
"w"(_k00), // %8
"w"(_k01), // %9
"w"(_k02), // %10
"w"(_k10), // %11
"w"(_k11), // %12
"w"(_k12), // %13
"w"(_k20), // %14
"w"(_k21), // %15
"w"(_k22) // %16
: "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v30", "v31");
}
for (; j < outw; j++)
{
float16x8_t _sum0 = vld1q_f16(outptr0);

float16x4_t _r0 = vld1_f16(r0);
float16x4_t _r1 = vld1_f16(r1);
float16x4_t _r2 = vld1_f16(r2);

_sum0 = vfmaq_lane_f16(_sum0, _k00, _r0, 0);
_sum0 = vfmaq_lane_f16(_sum0, _k01, _r0, 1);
_sum0 = vfmaq_lane_f16(_sum0, _k02, _r0, 2);
_sum0 = vfmaq_lane_f16(_sum0, _k10, _r1, 0);
_sum0 = vfmaq_lane_f16(_sum0, _k11, _r1, 1);
_sum0 = vfmaq_lane_f16(_sum0, _k12, _r1, 2);
_sum0 = vfmaq_lane_f16(_sum0, _k20, _r2, 0);
_sum0 = vfmaq_lane_f16(_sum0, _k21, _r2, 1);
_sum0 = vfmaq_lane_f16(_sum0, _k22, _r2, 2);

vst1q_f16(outptr0, _sum0);

r0 += 2;
r1 += 2;
r2 += 2;
outptr0 += 8;
asm volatile(
"prfm pldl1keep, [%0, #128] \n"
"ld1 {v30.8h}, [%0] \n" // sum0

"prfm pldl1keep, [%1, #64] \n"
"ld1 {v0.4h}, [%1] \n" // r0

"fmla v30.8h, %8.8h, v0.h[0] \n"
"fmla v30.8h, %9.8h, v0.h[1] \n"
"fmla v30.8h, %10.8h, v0.h[2] \n"

"prfm pldl1keep, [%2, #64] \n"
"ld1 {v1.4h}, [%2] \n" // r1

"fmla v30.8h, %11.8h, v1.h[0] \n"
"fmla v30.8h, %12.8h, v1.h[1] \n"
"fmla v30.8h, %13.8h, v1.h[2] \n"

"prfm pldl1keep, [%3, #64] \n"
"ld1 {v2.4h}, [%3] \n" // r2

"fmla v30.8h, %14.8h, v2.h[0] \n"
"fmla v30.8h, %15.8h, v2.h[1] \n"
"fmla v30.8h, %16.8h, v2.h[2] \n"

"add %1, %1, #4 \n"
"add %2, %2, #4 \n"
"add %3, %3, #4 \n"

"st1 {v30.8h}, [%0], #16 \n"

: "=r"(outptr0), // %0
"=r"(r0), // %1
"=r"(r1), // %2
"=r"(r2) // %3
: "0"(outptr0),
"1"(r0),
"2"(r1),
"3"(r2),
"w"(_k00), // %8
"w"(_k01), // %9
"w"(_k02), // %10
"w"(_k10), // %11
"w"(_k11), // %12
"w"(_k12), // %13
"w"(_k20), // %14
"w"(_k21), // %15
"w"(_k22) // %16
: "cc", "memory", "v0", "v1", "v2", "v30");
}

r0 += tailstep;


+ 10
- 1
src/layer/arm/convolution_arm.cpp View File

@@ -1533,7 +1533,16 @@ int Convolution_arm::forward_fp16sa(const Mat& bottom_blob, Mat& top_blob, const

if (elempack == 1 && out_elempack == 8)
{
if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1)
{
conv3x3s1_pack1to8_fp16sa_neon(bottom_blob_bordered, top_blob, weight_data_fp16, bias_data_fp16, opt);

if (activation)
{
activation->forward_inplace(top_blob, opt);
}
}
else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2)
{
conv3x3s2_pack1to8_fp16sa_neon(bottom_blob_bordered, top_blob, weight_data_fp16, bias_data_fp16, opt);



+ 139
- 39
src/layer/arm/padding_pack8_fp16s.h View File

@@ -17,45 +17,145 @@ static void padding_constant_pack8_fp16s_neon(const Mat& src, Mat& dst, int top,
const __fp16* ptr = src;
__fp16* outptr = dst;

// fill top
for (int y = 0; y < top; y++)
{
for (int x = 0; x < dst.w; x++)
{
vst1q_f16(outptr, _v);
outptr += 8;
}
}
// fill center
for (int y = 0; y < src.h; y++)
{
for (int x = 0; x < left; x++)
{
vst1q_f16(outptr, _v);
outptr += 8;
}
for (int x = 0; x < src.w; x++)
{
float16x8_t _p = vld1q_f16(ptr);
vst1q_f16(outptr, _p);
ptr += 8;
outptr += 8;
}
for (int x = 0; x < right; x++)
{
vst1q_f16(outptr, _v);
outptr += 8;
}
}
// fill bottom
for (int y = 0; y < bottom; y++)
{
for (int x = 0; x < dst.w; x++)
{
vst1q_f16(outptr, _v);
outptr += 8;
}
}
int w = src.w;
int h = src.h;

int top_size = top * dst.w;
int bottom_size = bottom * dst.w;

asm volatile(
"mov v0.16b, %10.16b \n"
"mov v1.16b, %10.16b \n"
"mov v2.16b, %10.16b \n"
"mov v3.16b, %10.16b \n"

// fill top
"lsr w4, %w8, #2 \n" // w4 = nn = top_size >> 2
"cmp w4, #0 \n"
"beq 1f \n"

"0: \n"
"st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [%0], #64 \n"
"subs w4, w4, #1 \n"
"bne 0b \n"

"1: \n"

// fill top remain
"and w4, %w8, #3 \n" // w4 = remain = top_size & 3

"cmp w4, #2 \n" // w4 >= 2
"blt 2f \n"
"sub w4, w4, #2 \n"
"st1 {v0.8h, v1.8h}, [%0], #32 \n"
"2: \n"

"cmp w4, #0 \n" // w4 > 0
"beq 3f \n"
"st1 {v0.8h}, [%0], #16 \n"
"3: \n"

// fill center h loop
"cmp %w5, #0 \n"
"beq 13f \n"
"4: \n"

// fill left
"mov w4, %w6 \n" // w4 = left
"cmp w4, #0 \n"
"beq 6f \n"

"5: \n"
"st1 {v0.8h}, [%0], #16 \n"
"subs w4, w4, #1 \n"
"bne 5b \n"

"6: \n"

// fill middle
"lsr w4, %w4, #2 \n" // w4 = nn = w >> 2
"cmp w4, #0 \n"
"beq 8f \n"

"7: \n"
"prfm pldl1keep, [%1, #512] \n"
"ld1 {v16.8h, v17.8h, v18.8h, v19.8h}, [%1], #64 \n"
"subs w4, w4, #1 \n"
"st1 {v16.8h, v17.8h, v18.8h, v19.8h}, [%0], #64 \n"
"bne 7b \n"

"8: \n"

"and w4, %w4, #3 \n" // w4 = remain = w & 3

"cmp w4, #2 \n" // w4 >= 2
"blt 9f \n"
"prfm pldl1keep, [%1, #256] \n"
"ld1 {v16.8h, v17.8h}, [%1], #32 \n"
"sub w4, w4, #2 \n"
"st1 {v16.8h, v17.8h}, [%0], #32 \n"
"9: \n"

"cmp w4, #0 \n" // w4 > 0
"beq 10f \n"
"prfm pldl1keep, [%1, #128] \n"
"ld1 {v16.8h}, [%1], #16 \n"
"st1 {v16.8h}, [%0], #16 \n"
"10: \n"

// fill right
"mov w4, %w7 \n" // w4 = right
"cmp w4, #0 \n"
"beq 12f \n"

"11: \n"
"subs w4, w4, #1 \n"
"st1 {v0.8h}, [%0], #16 \n"
"bne 11b \n"
"12: \n"

"subs %w5, %w5, #1 \n"
"bne 4b \n"

"13: \n"

// fill bottom
"lsr w4, %w9, #2 \n" // w4 = nn = bottom_size >> 2
"cmp w4, #0 \n"
"beq 15f \n"

"14: \n"
"st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [%0], #64 \n"
"subs w4, w4, #1 \n"
"bne 14b \n"
"15: \n"

// fill bottom remain
"and w4, %w9, #3 \n" // w4 = remain = bottom_size & 3

"cmp w4, #2 \n" // w4 >= 2
"blt 16f \n"
"sub w4, w4, #2 \n"
"st1 {v0.8h, v1.8h}, [%0], #32 \n"
"16: \n"

"cmp w4, #0 \n" // w4 > 0
"beq 17f \n"
"st1 {v0.8h}, [%0], #16 \n"
"17: \n"

: "=r"(outptr), // %0
"=r"(ptr) // %1
: "0"(outptr),
"1"(ptr),
"r"(w), // %4
"r"(h), // %5
"r"(left), // %6
"r"(right), // %7
"r"(top_size), // %8
"r"(bottom_size), // %9
"w"(_v) // %10
: "cc", "memory", "x4", "v0", "v1", "v2", "v3", "v16", "v17", "v18", "v19");
}

static void padding_replicate_pack8_fp16s_neon(const Mat& src, Mat& dst, int top, int bottom, int left, int right)


+ 41
- 8
src/layer/arm/relu_arm.cpp View File

@@ -358,16 +358,49 @@ int ReLU_arm::forward_inplace_fp16s(Mat& bottom_top_blob, const Option& opt) con
{
__fp16* ptr = bottom_top_blob.channel(q);

float16x8_t _zero = vdupq_n_f16((__fp16)0.f);
asm volatile(
"eor v16.16b, v16.16b, v16.16b \n"

for (int i = 0; i < size; i++)
{
float16x8_t _p = vld1q_f16(ptr);
_p = vmaxq_f16(_p, _zero);
vst1q_f16(ptr, _p);
"lsr w4, %w2, #2 \n" // w4 = nn = size >> 2
"cmp w4, #0 \n"
"beq 1f \n"

ptr += 8;
}
"0: \n"
"prfm pldl1keep, [%0, #512] \n"
"ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [%0] \n"
"fmax v0.8h, v0.8h, v16.8h \n"
"fmax v1.8h, v1.8h, v16.8h \n"
"fmax v2.8h, v2.8h, v16.8h \n"
"fmax v3.8h, v3.8h, v16.8h \n"
"subs w4, w4, #1 \n"
"st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [%0], #64 \n"
"bne 0b \n"
"1: \n"

"and w4, %w2, #3 \n" // w4 = remain = size & 3

"cmp w4, #2 \n" // w4 >= 2
"blt 2f \n"
"prfm pldl1keep, [%0, #256] \n"
"ld1 {v0.8h, v1.8h}, [%0] \n"
"fmax v0.8h, v0.8h, v16.8h \n"
"fmax v1.8h, v1.8h, v16.8h \n"
"sub w4, w4, #2 \n"
"st1 {v0.8h, v1.8h}, [%0], #32 \n"
"2: \n"

"cmp w4, #0 \n" // w4 > 0
"beq 3f \n"
"prfm pldl1keep, [%0, #128] \n"
"ld1 {v0.8h}, [%0] \n"
"fmax v0.8h, v0.8h, v16.8h \n"
"st1 {v0.8h}, [%0], #16 \n"
"3: \n"

: "=r"(ptr) // %0
: "0"(ptr),
"r"(size) // %2
: "cc", "memory", "x4", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16");
}
}
else


Write Preview
Loading…
Cancel
Save