nihui
/
ncnn
Not watched
1
0
Fork 0
Code
Releases 46 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
Browse Source

fix mha gemm allocator race (#4611)

tags/20230517
nihui GitHub 3 years ago
parent
804ac3421d
commit
2ce77ba918
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 71 additions and 71 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +1
    -1
      src/layer/arm/convolution_im2col_gemm.h
  2. +1
    -1
      src/layer/arm/convolution_im2col_gemm_bf16s.h
  3. +1
    -1
      src/layer/arm/convolution_im2col_gemm_fp16s.h
  4. +20
    -20
      src/layer/arm/gemm_arm.cpp
  5. +10
    -10
      src/layer/arm/gemm_arm_asimdhp.cpp
  6. +10
    -10
      src/layer/arm/gemm_arm_vfpv4.cpp
  7. +18
    -18
      src/layer/x86/convolution_3x3_winograd.h
  8. +10
    -10
      src/layer/x86/gemm_x86.cpp

+ 1
- 1
src/layer/arm/convolution_im2col_gemm.h View File

@@ -6843,7 +6843,7 @@ static void convolution_im2col_gemm(const Mat& bottom_blob, Mat& top_blob, const

// NCNN_LOGE("TILE M/N/K = %d %d %d -> %d %d %d", M, N, K, TILE_M, TILE_N, TILE_K);

Mat BT(TILE_K * TILE_N, (K + TILE_K - 1) / TILE_K, (N + TILE_N - 1) / TILE_N, 4u, opt.blob_allocator);
Mat BT(TILE_K * TILE_N, (K + TILE_K - 1) / TILE_K, (N + TILE_N - 1) / TILE_N, 4u, opt.workspace_allocator);

const int nn_NK = nn_N * nn_K;



+ 1
- 1
src/layer/arm/convolution_im2col_gemm_bf16s.h View File

@@ -6024,7 +6024,7 @@ static void convolution_im2col_gemm_bf16s(const Mat& bottom_blob, Mat& top_blob,

// NCNN_LOGE("TILE M/N/K = %d %d %d -> %d %d %d", M, N, K, TILE_M, TILE_N, TILE_K);

Mat BT(TILE_K * TILE_N, (K + TILE_K - 1) / TILE_K, (N + TILE_N - 1) / TILE_N, 2u, opt.blob_allocator);
Mat BT(TILE_K * TILE_N, (K + TILE_K - 1) / TILE_K, (N + TILE_N - 1) / TILE_N, 2u, opt.workspace_allocator);

const int nn_NK = nn_N * nn_K;



+ 1
- 1
src/layer/arm/convolution_im2col_gemm_fp16s.h View File

@@ -3160,7 +3160,7 @@ static void convolution_im2col_gemm_fp16sa(const Mat& bottom_blob, Mat& top_blob

// NCNN_LOGE("TILE M/N/K = %d %d %d -> %d %d %d", M, N, K, TILE_M, TILE_N, TILE_K);

Mat BT(TILE_K * TILE_N, (K + TILE_K - 1) / TILE_K, (N + TILE_N - 1) / TILE_N, 2u, opt.blob_allocator);
Mat BT(TILE_K * TILE_N, (K + TILE_K - 1) / TILE_K, (N + TILE_N - 1) / TILE_N, 2u, opt.workspace_allocator);

const int nn_NK = nn_N * nn_K;



+ 20
- 20
src/layer/arm/gemm_arm.cpp View File

@@ -3796,8 +3796,8 @@ static int gemm_arm(const Mat& A, const Mat& B, const Mat& C, Mat& top_blob, int
int nn_N = (N + TILE_N - 1) / TILE_N;
int nn_K = (K + TILE_K - 1) / TILE_K;

Mat ATX(TILE_K * TILE_M, (K + TILE_K - 1) / TILE_K, nT, 4u, opt.blob_allocator);
Mat BT(TILE_K * TILE_N, (K + TILE_K - 1) / TILE_K, (N + TILE_N - 1) / TILE_N, 4u, opt.blob_allocator);
Mat ATX(TILE_K * TILE_M, (K + TILE_K - 1) / TILE_K, nT, 4u, opt.workspace_allocator);
Mat BT(TILE_K * TILE_N, (K + TILE_K - 1) / TILE_K, (N + TILE_N - 1) / TILE_N, 4u, opt.workspace_allocator);

const int nn_NK = nn_N * nn_K;

@@ -3828,7 +3828,7 @@ static int gemm_arm(const Mat& A, const Mat& B, const Mat& C, Mat& top_blob, int

Mat topT;
if (K > TILE_K || broadcast_type_C == 3 || output_transpose)
topT.create(TILE_N * TILE_M, 1, nT, 4u, opt.blob_allocator);
topT.create(TILE_N * TILE_M, 1, nT, 4u, opt.workspace_allocator);

#pragma omp parallel for num_threads(nT)
for (int ppi = 0; ppi < nn_M; ppi++)
@@ -3904,7 +3904,7 @@ static int gemm_AT_arm(const Mat& AT, const Mat& B, const Mat& C, Mat& top_blob,
int nn_N = (N + TILE_N - 1) / TILE_N;
int nn_K = (K + TILE_K - 1) / TILE_K;

Mat BT(TILE_K * TILE_N, (K + TILE_K - 1) / TILE_K, (N + TILE_N - 1) / TILE_N, 4u, opt.blob_allocator);
Mat BT(TILE_K * TILE_N, (K + TILE_K - 1) / TILE_K, (N + TILE_N - 1) / TILE_N, 4u, opt.workspace_allocator);

const int nn_NK = nn_N * nn_K;

@@ -3935,7 +3935,7 @@ static int gemm_AT_arm(const Mat& AT, const Mat& B, const Mat& C, Mat& top_blob,

Mat topT;
if (K > TILE_K || broadcast_type_C == 3 || output_transpose)
topT.create(TILE_N * TILE_M, 1, nT, 4u, opt.blob_allocator);
topT.create(TILE_N * TILE_M, 1, nT, 4u, opt.workspace_allocator);

#pragma omp parallel for num_threads(nT)
for (int ppi = 0; ppi < nn_M; ppi++)
@@ -3998,11 +3998,11 @@ static int gemm_BT_arm(const Mat& A, const Mat& BT, const Mat& C, Mat& top_blob,
int nn_M = (M + TILE_M - 1) / TILE_M;
// int nn_N = (N + TILE_N - 1) / TILE_N;

Mat ATX(TILE_K * TILE_M, (K + TILE_K - 1) / TILE_K, nT, 4u, opt.blob_allocator);
Mat ATX(TILE_K * TILE_M, (K + TILE_K - 1) / TILE_K, nT, 4u, opt.workspace_allocator);

Mat topT;
if (K > TILE_K || broadcast_type_C == 3 || output_transpose)
topT.create(TILE_N * TILE_M, 1, nT, 4u, opt.blob_allocator);
topT.create(TILE_N * TILE_M, 1, nT, 4u, opt.workspace_allocator);

#pragma omp parallel for num_threads(nT)
for (int ppi = 0; ppi < nn_M; ppi++)
@@ -4077,7 +4077,7 @@ static int gemm_AT_BT_arm(const Mat& AT, const Mat& BT, const Mat& C, Mat& top_b

Mat topT;
if (K > TILE_K || broadcast_type_C == 3 || output_transpose)
topT.create(TILE_N * TILE_M, 1, nT, 4u, opt.blob_allocator);
topT.create(TILE_N * TILE_M, 1, nT, 4u, opt.workspace_allocator);

#pragma omp parallel for num_threads(nT)
for (int ppi = 0; ppi < nn_M; ppi++)
@@ -4162,7 +4162,7 @@ int Gemm_arm::create_pipeline(const Option& opt)

const int nn_M = (M + TILE_M - 1) / TILE_M;

AT_data.create(TILE_K * TILE_M, (K + TILE_K - 1) / TILE_K, (M + TILE_M - 1) / TILE_M, 4u, opt.blob_allocator);
AT_data.create(TILE_K * TILE_M, (K + TILE_K - 1) / TILE_K, (M + TILE_M - 1) / TILE_M, 4u, (Allocator*)0);
if (AT_data.empty())
return -100;

@@ -4205,7 +4205,7 @@ int Gemm_arm::create_pipeline(const Option& opt)

const int nn_N = (N + TILE_N - 1) / TILE_N;

BT_data.create(TILE_K * TILE_N, (K + TILE_K - 1) / TILE_K, (N + TILE_N - 1) / TILE_N, 4u, opt.blob_allocator);
BT_data.create(TILE_K * TILE_N, (K + TILE_K - 1) / TILE_K, (N + TILE_N - 1) / TILE_N, 4u, (Allocator*)0);
if (BT_data.empty())
return -100;

@@ -4505,8 +4505,8 @@ static int gemm_arm_bf16s(const Mat& A, const Mat& B, const Mat& C, Mat& top_blo
int nn_N = (N + TILE_N - 1) / TILE_N;
int nn_K = (K + TILE_K - 1) / TILE_K;

Mat ATX(TILE_K * TILE_M, (K + TILE_K - 1) / TILE_K, nT, 2u, opt.blob_allocator);
Mat BT(TILE_K * TILE_N, (K + TILE_K - 1) / TILE_K, (N + TILE_N - 1) / TILE_N, 2u, opt.blob_allocator);
Mat ATX(TILE_K * TILE_M, (K + TILE_K - 1) / TILE_K, nT, 2u, opt.workspace_allocator);
Mat BT(TILE_K * TILE_N, (K + TILE_K - 1) / TILE_K, (N + TILE_N - 1) / TILE_N, 2u, opt.workspace_allocator);

const int nn_NK = nn_N * nn_K;

@@ -4537,7 +4537,7 @@ static int gemm_arm_bf16s(const Mat& A, const Mat& B, const Mat& C, Mat& top_blo

Mat topT;
if (K > TILE_K || broadcast_type_C == 3 || output_transpose)
topT.create(TILE_N * TILE_M, 1, nT, 4u, opt.blob_allocator);
topT.create(TILE_N * TILE_M, 1, nT, 4u, opt.workspace_allocator);

#pragma omp parallel for num_threads(nT)
for (int ppi = 0; ppi < nn_M; ppi++)
@@ -4614,7 +4614,7 @@ static int gemm_AT_arm_bf16s(const Mat& AT, const Mat& B, const Mat& C, Mat& top
int nn_N = (N + TILE_N - 1) / TILE_N;
int nn_K = (K + TILE_K - 1) / TILE_K;

Mat BT(TILE_K * TILE_N, (K + TILE_K - 1) / TILE_K, (N + TILE_N - 1) / TILE_N, 2u, opt.blob_allocator);
Mat BT(TILE_K * TILE_N, (K + TILE_K - 1) / TILE_K, (N + TILE_N - 1) / TILE_N, 2u, opt.workspace_allocator);

const int nn_NK = nn_N * nn_K;

@@ -4645,7 +4645,7 @@ static int gemm_AT_arm_bf16s(const Mat& AT, const Mat& B, const Mat& C, Mat& top

Mat topT;
if (K > TILE_K || broadcast_type_C == 3 || output_transpose)
topT.create(TILE_N * TILE_M, 1, nT, 4u, opt.blob_allocator);
topT.create(TILE_N * TILE_M, 1, nT, 4u, opt.workspace_allocator);

#pragma omp parallel for num_threads(nT)
for (int ppi = 0; ppi < nn_M; ppi++)
@@ -4709,11 +4709,11 @@ static int gemm_BT_arm_bf16s(const Mat& A, const Mat& BT, const Mat& C, Mat& top
int nn_M = (M + TILE_M - 1) / TILE_M;
// int nn_N = (N + TILE_N - 1) / TILE_N;

Mat ATX(TILE_K * TILE_M, (K + TILE_K - 1) / TILE_K, nT, 2u, opt.blob_allocator);
Mat ATX(TILE_K * TILE_M, (K + TILE_K - 1) / TILE_K, nT, 2u, opt.workspace_allocator);

Mat topT;
if (K > TILE_K || broadcast_type_C == 3 || output_transpose)
topT.create(TILE_N * TILE_M, 1, nT, 4u, opt.blob_allocator);
topT.create(TILE_N * TILE_M, 1, nT, 4u, opt.workspace_allocator);

#pragma omp parallel for num_threads(nT)
for (int ppi = 0; ppi < nn_M; ppi++)
@@ -4789,7 +4789,7 @@ static int gemm_AT_BT_arm_bf16s(const Mat& AT, const Mat& BT, const Mat& C, Mat&

Mat topT;
if (K > TILE_K || broadcast_type_C == 3 || output_transpose)
topT.create(TILE_N * TILE_M, 1, nT, 4u, opt.blob_allocator);
topT.create(TILE_N * TILE_M, 1, nT, 4u, opt.workspace_allocator);

#pragma omp parallel for num_threads(nT)
for (int ppi = 0; ppi < nn_M; ppi++)
@@ -4851,7 +4851,7 @@ int Gemm_arm::create_pipeline_bf16s(const Option& opt)

const int nn_M = (M + TILE_M - 1) / TILE_M;

AT_data.create(TILE_K * TILE_M, (K + TILE_K - 1) / TILE_K, (M + TILE_M - 1) / TILE_M, 2u, opt.blob_allocator);
AT_data.create(TILE_K * TILE_M, (K + TILE_K - 1) / TILE_K, (M + TILE_M - 1) / TILE_M, 2u, (Allocator*)0);
if (AT_data.empty())
return -100;

@@ -4894,7 +4894,7 @@ int Gemm_arm::create_pipeline_bf16s(const Option& opt)

const int nn_N = (N + TILE_N - 1) / TILE_N;

BT_data.create(TILE_K * TILE_N, (K + TILE_K - 1) / TILE_K, (N + TILE_N - 1) / TILE_N, 2u, opt.blob_allocator);
BT_data.create(TILE_K * TILE_N, (K + TILE_K - 1) / TILE_K, (N + TILE_N - 1) / TILE_N, 2u, (Allocator*)0);
if (BT_data.empty())
return -100;



+ 10
- 10
src/layer/arm/gemm_arm_asimdhp.cpp View File

@@ -2355,8 +2355,8 @@ static int gemm_arm_fp16sa(const Mat& A, const Mat& B, const Mat& C, Mat& top_bl
int nn_N = (N + TILE_N - 1) / TILE_N;
int nn_K = (K + TILE_K - 1) / TILE_K;

Mat ATX(TILE_K * TILE_M, (K + TILE_K - 1) / TILE_K, nT, 2u, opt.blob_allocator);
Mat BT(TILE_K * TILE_N, (K + TILE_K - 1) / TILE_K, (N + TILE_N - 1) / TILE_N, 2u, opt.blob_allocator);
Mat ATX(TILE_K * TILE_M, (K + TILE_K - 1) / TILE_K, nT, 2u, opt.workspace_allocator);
Mat BT(TILE_K * TILE_N, (K + TILE_K - 1) / TILE_K, (N + TILE_N - 1) / TILE_N, 2u, opt.workspace_allocator);

const int nn_NK = nn_N * nn_K;

@@ -2387,7 +2387,7 @@ static int gemm_arm_fp16sa(const Mat& A, const Mat& B, const Mat& C, Mat& top_bl

Mat topT;
if (K > TILE_K || broadcast_type_C == 3 || output_transpose)
topT.create(TILE_N * TILE_M, 1, nT, 2u, opt.blob_allocator);
topT.create(TILE_N * TILE_M, 1, nT, 2u, opt.workspace_allocator);

#pragma omp parallel for num_threads(nT)
for (int ppi = 0; ppi < nn_M; ppi++)
@@ -2463,7 +2463,7 @@ static int gemm_AT_arm_fp16sa(const Mat& AT, const Mat& B, const Mat& C, Mat& to
int nn_N = (N + TILE_N - 1) / TILE_N;
int nn_K = (K + TILE_K - 1) / TILE_K;

Mat BT(TILE_K * TILE_N, (K + TILE_K - 1) / TILE_K, (N + TILE_N - 1) / TILE_N, 2u, opt.blob_allocator);
Mat BT(TILE_K * TILE_N, (K + TILE_K - 1) / TILE_K, (N + TILE_N - 1) / TILE_N, 2u, opt.workspace_allocator);

const int nn_NK = nn_N * nn_K;

@@ -2494,7 +2494,7 @@ static int gemm_AT_arm_fp16sa(const Mat& AT, const Mat& B, const Mat& C, Mat& to

Mat topT;
if (K > TILE_K || broadcast_type_C == 3 || output_transpose)
topT.create(TILE_N * TILE_M, 1, nT, 2u, opt.blob_allocator);
topT.create(TILE_N * TILE_M, 1, nT, 2u, opt.workspace_allocator);

#pragma omp parallel for num_threads(nT)
for (int ppi = 0; ppi < nn_M; ppi++)
@@ -2557,11 +2557,11 @@ static int gemm_BT_arm_fp16sa(const Mat& A, const Mat& BT, const Mat& C, Mat& to
int nn_M = (M + TILE_M - 1) / TILE_M;
// int nn_N = (N + TILE_N - 1) / TILE_N;

Mat ATX(TILE_K * TILE_M, (K + TILE_K - 1) / TILE_K, nT, 2u, opt.blob_allocator);
Mat ATX(TILE_K * TILE_M, (K + TILE_K - 1) / TILE_K, nT, 2u, opt.workspace_allocator);

Mat topT;
if (K > TILE_K || broadcast_type_C == 3 || output_transpose)
topT.create(TILE_N * TILE_M, 1, nT, 2u, opt.blob_allocator);
topT.create(TILE_N * TILE_M, 1, nT, 2u, opt.workspace_allocator);

#pragma omp parallel for num_threads(nT)
for (int ppi = 0; ppi < nn_M; ppi++)
@@ -2636,7 +2636,7 @@ static int gemm_AT_BT_arm_fp16sa(const Mat& AT, const Mat& BT, const Mat& C, Mat

Mat topT;
if (K > TILE_K || broadcast_type_C == 3 || output_transpose)
topT.create(TILE_N * TILE_M, 1, nT, 2u, opt.blob_allocator);
topT.create(TILE_N * TILE_M, 1, nT, 2u, opt.workspace_allocator);

#pragma omp parallel for num_threads(nT)
for (int ppi = 0; ppi < nn_M; ppi++)
@@ -2697,7 +2697,7 @@ int Gemm_arm::create_pipeline_fp16sa(const Option& opt)

const int nn_M = (M + TILE_M - 1) / TILE_M;

AT_data.create(TILE_K * TILE_M, (K + TILE_K - 1) / TILE_K, (M + TILE_M - 1) / TILE_M, 2u, opt.blob_allocator);
AT_data.create(TILE_K * TILE_M, (K + TILE_K - 1) / TILE_K, (M + TILE_M - 1) / TILE_M, 2u, (Allocator*)0);
if (AT_data.empty())
return -100;

@@ -2740,7 +2740,7 @@ int Gemm_arm::create_pipeline_fp16sa(const Option& opt)

const int nn_N = (N + TILE_N - 1) / TILE_N;

BT_data.create(TILE_K * TILE_N, (K + TILE_K - 1) / TILE_K, (N + TILE_N - 1) / TILE_N, 2u, opt.blob_allocator);
BT_data.create(TILE_K * TILE_N, (K + TILE_K - 1) / TILE_K, (N + TILE_N - 1) / TILE_N, 2u, (Allocator*)0);
if (BT_data.empty())
return -100;



+ 10
- 10
src/layer/arm/gemm_arm_vfpv4.cpp View File

@@ -46,8 +46,8 @@ static int gemm_arm_fp16s(const Mat& A, const Mat& B, const Mat& C, Mat& top_blo
int nn_N = (N + TILE_N - 1) / TILE_N;
int nn_K = (K + TILE_K - 1) / TILE_K;

Mat ATX(TILE_K * TILE_M, (K + TILE_K - 1) / TILE_K, nT, 2u, opt.blob_allocator);
Mat BT(TILE_K * TILE_N, (K + TILE_K - 1) / TILE_K, (N + TILE_N - 1) / TILE_N, 2u, opt.blob_allocator);
Mat ATX(TILE_K * TILE_M, (K + TILE_K - 1) / TILE_K, nT, 2u, opt.workspace_allocator);
Mat BT(TILE_K * TILE_N, (K + TILE_K - 1) / TILE_K, (N + TILE_N - 1) / TILE_N, 2u, opt.workspace_allocator);

const int nn_NK = nn_N * nn_K;

@@ -78,7 +78,7 @@ static int gemm_arm_fp16s(const Mat& A, const Mat& B, const Mat& C, Mat& top_blo

Mat topT;
if (K > TILE_K || broadcast_type_C == 3 || output_transpose)
topT.create(TILE_N * TILE_M, 1, nT, 4u, opt.blob_allocator);
topT.create(TILE_N * TILE_M, 1, nT, 4u, opt.workspace_allocator);

#pragma omp parallel for num_threads(nT)
for (int ppi = 0; ppi < nn_M; ppi++)
@@ -155,7 +155,7 @@ static int gemm_AT_arm_fp16s(const Mat& AT, const Mat& B, const Mat& C, Mat& top
int nn_N = (N + TILE_N - 1) / TILE_N;
int nn_K = (K + TILE_K - 1) / TILE_K;

Mat BT(TILE_K * TILE_N, (K + TILE_K - 1) / TILE_K, (N + TILE_N - 1) / TILE_N, 2u, opt.blob_allocator);
Mat BT(TILE_K * TILE_N, (K + TILE_K - 1) / TILE_K, (N + TILE_N - 1) / TILE_N, 2u, opt.workspace_allocator);

const int nn_NK = nn_N * nn_K;

@@ -186,7 +186,7 @@ static int gemm_AT_arm_fp16s(const Mat& AT, const Mat& B, const Mat& C, Mat& top

Mat topT;
if (K > TILE_K || broadcast_type_C == 3 || output_transpose)
topT.create(TILE_N * TILE_M, 1, nT, 4u, opt.blob_allocator);
topT.create(TILE_N * TILE_M, 1, nT, 4u, opt.workspace_allocator);

#pragma omp parallel for num_threads(nT)
for (int ppi = 0; ppi < nn_M; ppi++)
@@ -250,11 +250,11 @@ static int gemm_BT_arm_fp16s(const Mat& A, const Mat& BT, const Mat& C, Mat& top
int nn_M = (M + TILE_M - 1) / TILE_M;
// int nn_N = (N + TILE_N - 1) / TILE_N;

Mat ATX(TILE_K * TILE_M, (K + TILE_K - 1) / TILE_K, nT, 2u, opt.blob_allocator);
Mat ATX(TILE_K * TILE_M, (K + TILE_K - 1) / TILE_K, nT, 2u, opt.workspace_allocator);

Mat topT;
if (K > TILE_K || broadcast_type_C == 3 || output_transpose)
topT.create(TILE_N * TILE_M, 1, nT, 4u, opt.blob_allocator);
topT.create(TILE_N * TILE_M, 1, nT, 4u, opt.workspace_allocator);

#pragma omp parallel for num_threads(nT)
for (int ppi = 0; ppi < nn_M; ppi++)
@@ -330,7 +330,7 @@ static int gemm_AT_BT_arm_fp16s(const Mat& AT, const Mat& BT, const Mat& C, Mat&

Mat topT;
if (K > TILE_K || broadcast_type_C == 3 || output_transpose)
topT.create(TILE_N * TILE_M, 1, nT, 4u, opt.blob_allocator);
topT.create(TILE_N * TILE_M, 1, nT, 4u, opt.workspace_allocator);

#pragma omp parallel for num_threads(nT)
for (int ppi = 0; ppi < nn_M; ppi++)
@@ -392,7 +392,7 @@ int Gemm_arm::create_pipeline_fp16s(const Option& opt)

const int nn_M = (M + TILE_M - 1) / TILE_M;

AT_data.create(TILE_K * TILE_M, (K + TILE_K - 1) / TILE_K, (M + TILE_M - 1) / TILE_M, 2u, opt.blob_allocator);
AT_data.create(TILE_K * TILE_M, (K + TILE_K - 1) / TILE_K, (M + TILE_M - 1) / TILE_M, 2u, (Allocator*)0);
if (AT_data.empty())
return -100;

@@ -435,7 +435,7 @@ int Gemm_arm::create_pipeline_fp16s(const Option& opt)

const int nn_N = (N + TILE_N - 1) / TILE_N;

BT_data.create(TILE_K * TILE_N, (K + TILE_K - 1) / TILE_K, (N + TILE_N - 1) / TILE_N, 2u, opt.blob_allocator);
BT_data.create(TILE_K * TILE_N, (K + TILE_K - 1) / TILE_K, (N + TILE_N - 1) / TILE_N, 2u, (Allocator*)0);
if (BT_data.empty())
return -100;



+ 18
- 18
src/layer/x86/convolution_3x3_winograd.h View File

@@ -1983,9 +1983,9 @@ static void conv3x3s1_winograd23_transform_kernel(const Mat& kernel, Mat& AT, in

const int nn_M = (M + TILE_M - 1) / TILE_M;

Mat A_tileX(B * TILE_M * TILE_K, 1, opt.num_threads, 4u, opt.blob_allocator);
Mat A_tileX(B * TILE_M * TILE_K, 1, opt.num_threads, 4u, (Allocator*)0);

AT.create(TILE_K * TILE_M, B, (K + TILE_K - 1) / TILE_K, (M + TILE_M - 1) / TILE_M, 4u, opt.blob_allocator);
AT.create(TILE_K * TILE_M, B, (K + TILE_K - 1) / TILE_K, (M + TILE_M - 1) / TILE_M, 4u, (Allocator*)0);

#pragma omp parallel for num_threads(opt.num_threads)
for (int ppj = 0; ppj < nn_M; ppj++)
@@ -3158,13 +3158,13 @@ static void conv3x3s1_winograd23(const Mat& bottom_blob, Mat& top_blob, const Ma

// NCNN_LOGE("TILE M/N/K = %d %d %d -> %d %d %d", M, N, K, TILE_M, TILE_N, TILE_K);

Mat BT(TILE_K * TILE_N, B, (K + TILE_K - 1) / TILE_K, (N + TILE_N - 1) / TILE_N, 4u, opt.blob_allocator);
Mat BT(TILE_K * TILE_N, B, (K + TILE_K - 1) / TILE_K, (N + TILE_N - 1) / TILE_N, 4u, opt.workspace_allocator);

const int nn_NK = nn_N * nn_K;

if (nT > 1 && nn_NK < nT)
{
Mat B_tile(TILE_N * B * TILE_K, 4u, opt.blob_allocator);
Mat B_tile(TILE_N * B * TILE_K, 4u, opt.workspace_allocator);

for (int ppjk = 0; ppjk < nn_NK; ppjk++)
{
@@ -3187,7 +3187,7 @@ static void conv3x3s1_winograd23(const Mat& bottom_blob, Mat& top_blob, const Ma
}
else
{
Mat B_tileX(TILE_N * B * TILE_K, 1, nT, 4u, opt.blob_allocator);
Mat B_tileX(TILE_N * B * TILE_K, 1, nT, 4u, opt.workspace_allocator);

#pragma omp parallel for num_threads(nT)
for (int ppjk = 0; ppjk < nn_NK; ppjk++)
@@ -3212,7 +3212,7 @@ static void conv3x3s1_winograd23(const Mat& bottom_blob, Mat& top_blob, const Ma
}
}

Mat top_tileX(TILE_N * B * TILE_M, 1, nT, 4u, opt.blob_allocator);
Mat top_tileX(TILE_N * B * TILE_M, 1, nT, 4u, opt.workspace_allocator);

#pragma omp parallel for num_threads(nT)
for (int ppj = 0; ppj < nn_M; ppj++)
@@ -3325,9 +3325,9 @@ static void conv3x3s1_winograd43_transform_kernel(const Mat& kernel, Mat& AT, in

const int nn_M = (M + TILE_M - 1) / TILE_M;

Mat A_tileX(B * TILE_M * TILE_K, 1, opt.num_threads, 4u, opt.blob_allocator);
Mat A_tileX(B * TILE_M * TILE_K, 1, opt.num_threads, 4u, (Allocator*)0);

AT.create(TILE_K * TILE_M, B, (K + TILE_K - 1) / TILE_K, (M + TILE_M - 1) / TILE_M, 4u, opt.blob_allocator);
AT.create(TILE_K * TILE_M, B, (K + TILE_K - 1) / TILE_K, (M + TILE_M - 1) / TILE_M, 4u, (Allocator*)0);

#pragma omp parallel for num_threads(opt.num_threads)
for (int ppj = 0; ppj < nn_M; ppj++)
@@ -5000,13 +5000,13 @@ static void conv3x3s1_winograd43(const Mat& bottom_blob, Mat& top_blob, const Ma

// NCNN_LOGE("TILE M/N/K = %d %d %d -> %d %d %d", M, N, K, TILE_M, TILE_N, TILE_K);

Mat BT(TILE_K * TILE_N, B, (K + TILE_K - 1) / TILE_K, (N + TILE_N - 1) / TILE_N, 4u, opt.blob_allocator);
Mat BT(TILE_K * TILE_N, B, (K + TILE_K - 1) / TILE_K, (N + TILE_N - 1) / TILE_N, 4u, opt.workspace_allocator);

const int nn_NK = nn_N * nn_K;

if (nT > 1 && nn_NK < nT)
{
Mat B_tile(TILE_N * B * TILE_K, 4u, opt.blob_allocator);
Mat B_tile(TILE_N * B * TILE_K, 4u, opt.workspace_allocator);

for (int ppjk = 0; ppjk < nn_NK; ppjk++)
{
@@ -5029,7 +5029,7 @@ static void conv3x3s1_winograd43(const Mat& bottom_blob, Mat& top_blob, const Ma
}
else
{
Mat B_tileX(TILE_N * B * TILE_K, 1, nT, 4u, opt.blob_allocator);
Mat B_tileX(TILE_N * B * TILE_K, 1, nT, 4u, opt.workspace_allocator);

#pragma omp parallel for num_threads(nT)
for (int ppjk = 0; ppjk < nn_NK; ppjk++)
@@ -5054,7 +5054,7 @@ static void conv3x3s1_winograd43(const Mat& bottom_blob, Mat& top_blob, const Ma
}
}

Mat top_tileX(TILE_N * B * TILE_M, 1, nT, 4u, opt.blob_allocator);
Mat top_tileX(TILE_N * B * TILE_M, 1, nT, 4u, opt.workspace_allocator);

#pragma omp parallel for num_threads(nT)
for (int ppj = 0; ppj < nn_M; ppj++)
@@ -5174,9 +5174,9 @@ static void conv3x3s1_winograd63_transform_kernel(const Mat& kernel, Mat& AT, in

const int nn_M = (M + TILE_M - 1) / TILE_M;

Mat A_tileX(B * TILE_M * TILE_K, 1, opt.num_threads, 4u, opt.blob_allocator);
Mat A_tileX(B * TILE_M * TILE_K, 1, opt.num_threads, 4u, (Allocator*)0);

AT.create(TILE_K * TILE_M, B, (K + TILE_K - 1) / TILE_K, (M + TILE_M - 1) / TILE_M, 4u, opt.blob_allocator);
AT.create(TILE_K * TILE_M, B, (K + TILE_K - 1) / TILE_K, (M + TILE_M - 1) / TILE_M, 4u, (Allocator*)0);

#pragma omp parallel for num_threads(opt.num_threads)
for (int ppj = 0; ppj < nn_M; ppj++)
@@ -7290,13 +7290,13 @@ static void conv3x3s1_winograd63(const Mat& bottom_blob, Mat& top_blob, const Ma

// NCNN_LOGE("TILE M/N/K = %d %d %d -> %d %d %d", M, N, K, TILE_M, TILE_N, TILE_K);

Mat BT(TILE_K * TILE_N, B, (K + TILE_K - 1) / TILE_K, (N + TILE_N - 1) / TILE_N, 4u, opt.blob_allocator);
Mat BT(TILE_K * TILE_N, B, (K + TILE_K - 1) / TILE_K, (N + TILE_N - 1) / TILE_N, 4u, opt.workspace_allocator);

const int nn_NK = nn_N * nn_K;

if (nT > 1 && nn_NK < nT)
{
Mat B_tile(TILE_N * B * TILE_K, 4u, opt.blob_allocator);
Mat B_tile(TILE_N * B * TILE_K, 4u, opt.workspace_allocator);

for (int ppjk = 0; ppjk < nn_NK; ppjk++)
{
@@ -7319,7 +7319,7 @@ static void conv3x3s1_winograd63(const Mat& bottom_blob, Mat& top_blob, const Ma
}
else
{
Mat B_tileX(TILE_N * B * TILE_K, 1, nT, 4u, opt.blob_allocator);
Mat B_tileX(TILE_N * B * TILE_K, 1, nT, 4u, opt.workspace_allocator);

#pragma omp parallel for num_threads(nT)
for (int ppjk = 0; ppjk < nn_NK; ppjk++)
@@ -7344,7 +7344,7 @@ static void conv3x3s1_winograd63(const Mat& bottom_blob, Mat& top_blob, const Ma
}
}

Mat top_tileX(TILE_N * B * TILE_M, 1, nT, 4u, opt.blob_allocator);
Mat top_tileX(TILE_N * B * TILE_M, 1, nT, 4u, opt.workspace_allocator);

#pragma omp parallel for num_threads(nT)
for (int ppj = 0; ppj < nn_M; ppj++)


+ 10
- 10
src/layer/x86/gemm_x86.cpp View File

@@ -6854,8 +6854,8 @@ static int gemm_x86(const Mat& A, const Mat& B, const Mat& C, Mat& top_blob, int
int nn_N = (N + TILE_N - 1) / TILE_N;
int nn_K = (K + TILE_K - 1) / TILE_K;

Mat ATX(TILE_K * TILE_M, (K + TILE_K - 1) / TILE_K, nT, 4u, opt.blob_allocator);
Mat BT(TILE_K * TILE_N, (K + TILE_K - 1) / TILE_K, (N + TILE_N - 1) / TILE_N, 4u, opt.blob_allocator);
Mat ATX(TILE_K * TILE_M, (K + TILE_K - 1) / TILE_K, nT, 4u, opt.workspace_allocator);
Mat BT(TILE_K * TILE_N, (K + TILE_K - 1) / TILE_K, (N + TILE_N - 1) / TILE_N, 4u, opt.workspace_allocator);

const int nn_NK = nn_N * nn_K;

@@ -6886,7 +6886,7 @@ static int gemm_x86(const Mat& A, const Mat& B, const Mat& C, Mat& top_blob, int

Mat topT;
if (K > TILE_K || broadcast_type_C == 3 || output_transpose)
topT.create(TILE_N * TILE_M, 1, nT, 4u, opt.blob_allocator);
topT.create(TILE_N * TILE_M, 1, nT, 4u, opt.workspace_allocator);

#pragma omp parallel for num_threads(nT)
for (int ppi = 0; ppi < nn_M; ppi++)
@@ -6962,7 +6962,7 @@ static int gemm_AT_x86(const Mat& AT, const Mat& B, const Mat& C, Mat& top_blob,
int nn_N = (N + TILE_N - 1) / TILE_N;
int nn_K = (K + TILE_K - 1) / TILE_K;

Mat BT(TILE_K * TILE_N, (K + TILE_K - 1) / TILE_K, (N + TILE_N - 1) / TILE_N, 4u, opt.blob_allocator);
Mat BT(TILE_K * TILE_N, (K + TILE_K - 1) / TILE_K, (N + TILE_N - 1) / TILE_N, 4u, opt.workspace_allocator);

const int nn_NK = nn_N * nn_K;

@@ -6993,7 +6993,7 @@ static int gemm_AT_x86(const Mat& AT, const Mat& B, const Mat& C, Mat& top_blob,

Mat topT;
if (K > TILE_K || broadcast_type_C == 3 || output_transpose)
topT.create(TILE_N * TILE_M, 1, nT, 4u, opt.blob_allocator);
topT.create(TILE_N * TILE_M, 1, nT, 4u, opt.workspace_allocator);

#pragma omp parallel for num_threads(nT)
for (int ppi = 0; ppi < nn_M; ppi++)
@@ -7056,11 +7056,11 @@ static int gemm_BT_x86(const Mat& A, const Mat& BT, const Mat& C, Mat& top_blob,
int nn_M = (M + TILE_M - 1) / TILE_M;
// int nn_N = (N + TILE_N - 1) / TILE_N;

Mat ATX(TILE_K * TILE_M, (K + TILE_K - 1) / TILE_K, nT, 4u, opt.blob_allocator);
Mat ATX(TILE_K * TILE_M, (K + TILE_K - 1) / TILE_K, nT, 4u, opt.workspace_allocator);

Mat topT;
if (K > TILE_K || broadcast_type_C == 3 || output_transpose)
topT.create(TILE_N * TILE_M, 1, nT, 4u, opt.blob_allocator);
topT.create(TILE_N * TILE_M, 1, nT, 4u, opt.workspace_allocator);

#pragma omp parallel for num_threads(nT)
for (int ppi = 0; ppi < nn_M; ppi++)
@@ -7135,7 +7135,7 @@ static int gemm_AT_BT_x86(const Mat& AT, const Mat& BT, const Mat& C, Mat& top_b

Mat topT;
if (K > TILE_K || broadcast_type_C == 3 || output_transpose)
topT.create(TILE_N * TILE_M, 1, nT, 4u, opt.blob_allocator);
topT.create(TILE_N * TILE_M, 1, nT, 4u, opt.workspace_allocator);

#pragma omp parallel for num_threads(nT)
for (int ppi = 0; ppi < nn_M; ppi++)
@@ -7196,7 +7196,7 @@ int Gemm_x86::create_pipeline(const Option& opt)

const int nn_M = (M + TILE_M - 1) / TILE_M;

AT_data.create(TILE_K * TILE_M, (K + TILE_K - 1) / TILE_K, (M + TILE_M - 1) / TILE_M, 4u, opt.blob_allocator);
AT_data.create(TILE_K * TILE_M, (K + TILE_K - 1) / TILE_K, (M + TILE_M - 1) / TILE_M, 4u, (Allocator*)0);
if (AT_data.empty())
return -100;

@@ -7240,7 +7240,7 @@ int Gemm_x86::create_pipeline(const Option& opt)
const int nn_N = (N + TILE_N - 1) / TILE_N;
const int nn_K = (K + TILE_K - 1) / TILE_K;

BT_data.create(TILE_K * TILE_N, (K + TILE_K - 1) / TILE_K, (N + TILE_N - 1) / TILE_N, 4u, opt.blob_allocator);
BT_data.create(TILE_K * TILE_N, (K + TILE_K - 1) / TILE_K, (N + TILE_N - 1) / TILE_N, 4u, (Allocator*)0);
if (BT_data.empty())
return -100;



Write Preview
Loading…
Cancel
Save