From eaa7e24db6b5c7dbe1d812f24a2f0a80d53ebae3 Mon Sep 17 00:00:00 2001 From: Tijmen Verhulsdonck Date: Wed, 14 Jul 2021 14:28:15 +0200 Subject: [PATCH] Added ability to switch AVX/AVX2 during runtime (#3076) --- .github/workflows/linux-x64-cpu-clang.yml | 20 + .github/workflows/linux-x64-cpu-gcc.yml | 14 + .github/workflows/release.yml | 8 +- .github/workflows/test-coverage.yml | 27 + .github/workflows/web-assembly.yml | 6 +- .github/workflows/windows-x64-cpu-vs2015.yml | 16 +- .github/workflows/windows-x64-cpu-vs2017.yml | 17 +- .github/workflows/windows-x64-cpu-vs2019.yml | 16 +- CMakeLists.txt | 1 + cmake/ncnn_add_layer.cmake | 7 + cmake/ncnn_generate_avx_source.cmake | 14 + .../low-level-operation-api.md | 4 +- docs/how-to-build/how-to-build.md | 8 +- python/src/main.cpp | 1 + src/CMakeLists.txt | 11 +- src/cpu.cpp | 45 + src/cpu.h | 3 + src/layer.cpp | 8 +- src/layer/x86/avx_mathfun.h | 105 +- src/layer/x86/batchnorm_x86.cpp | 9 +- src/layer/x86/cast_x86.cpp | 6 +- src/layer/x86/convolution_1x1_pack4.h | 56 +- src/layer/x86/convolution_1x1_pack8.h | 288 ++-- src/layer/x86/convolution_1x1_pack8_fp16.h | 432 +++--- src/layer/x86/convolution_2x2_pack8.h | 192 +-- src/layer/x86/convolution_2x2_pack8_fp16.h | 192 +-- src/layer/x86/convolution_3x3.h | 206 +-- src/layer/x86/convolution_3x3_pack1to8.h | 1226 ++++++++--------- src/layer/x86/convolution_3x3_pack8.h | 848 ++++++------ src/layer/x86/convolution_3x3_pack8to1.h | 12 +- src/layer/x86/convolution_sgemm.h | 152 +- src/layer/x86/convolution_x86.cpp | 53 +- .../x86/convolutiondepthwise_3x3_pack8.h | 396 +++--- .../x86/convolutiondepthwise_3x3_pack8_fp16.h | 396 +++--- .../x86/convolutiondepthwise_5x5_pack8.h | 100 +- src/layer/x86/convolutiondepthwise_x86.cpp | 18 +- src/layer/x86/dequantize_x86.cpp | 14 +- src/layer/x86/eltwise_x86.cpp | 5 +- src/layer/x86/hardsigmoid_x86.cpp | 6 +- src/layer/x86/hardswish_x86.cpp | 6 +- src/layer/x86/innerproduct_x86.cpp | 274 ++-- src/layer/x86/lstm_x86.cpp | 167 ++- src/layer/x86/quantize_x86.cpp | 53 + src/layer/x86/requantize_x86.cpp | 78 +- src/layer/x86/scale_x86.cpp | 10 +- src/layer/x86/x86_activation.h | 4 + src/layer/x86/x86_usability.h | 53 +- src/layer/x86/yolov3detectionoutput_x86.cpp | 6 +- src/layer_registry.h.in | 5 + src/net.cpp | 4 +- src/platform.h.in | 1 + tests/testutil.h | 8 +- 52 files changed, 2977 insertions(+), 2630 deletions(-) create mode 100644 cmake/ncnn_generate_avx_source.cmake diff --git a/.github/workflows/linux-x64-cpu-clang.yml b/.github/workflows/linux-x64-cpu-clang.yml index 318004fbc..65c768cd8 100644 --- a/.github/workflows/linux-x64-cpu-clang.yml +++ b/.github/workflows/linux-x64-cpu-clang.yml @@ -67,6 +67,26 @@ jobs: cmake --build . -j 2 - name: test-avx2 run: cd build-avx2 && ctest --output-on-failure -j 2 + - name: build-avx + env: + CC: clang + CXX: clang++ + run: | + mkdir build-avx && cd build-avx + cmake -DNCNN_AVX2=OFF -DNCNN_AVX=ON -DNCNN_BUILD_TESTS=ON .. + cmake --build . -j 2 + - name: test-avx + run: cd build-avx && ctest --output-on-failure -j 2 + - name: build-avx1-2 + env: + CC: clang + CXX: clang++ + run: | + mkdir build-avx1-2 && cd build-avx1-2 + cmake -DNCNN_AVX2=ON -DNCNN_AVX=ON -DNCNN_BUILD_TESTS=ON .. + cmake --build . -j 2 + - name: test-avx1-2 + run: cd build-avx1-2 && ctest --output-on-failure -j 2 - name: build-noint8 env: CC: clang diff --git a/.github/workflows/linux-x64-cpu-gcc.yml b/.github/workflows/linux-x64-cpu-gcc.yml index 6849cb7d9..3a2609d7d 100644 --- a/.github/workflows/linux-x64-cpu-gcc.yml +++ b/.github/workflows/linux-x64-cpu-gcc.yml @@ -58,6 +58,20 @@ jobs: cmake --build . -j 2 - name: test-avx2 run: cd build-avx2 && ctest --output-on-failure -j 2 + - name: build-avx + run: | + mkdir build-avx && cd build-avx + cmake -DNCNN_AVX2=OFF -DNCNN_AVX=ON -DNCNN_BUILD_TESTS=ON .. + cmake --build . -j 2 + - name: test-avx + run: cd build-avx && ctest --output-on-failure -j 2 + - name: build-avx1-2 + run: | + mkdir build-avx1-2 && cd build-avx1-2 + cmake -DNCNN_AVX2=ON -DNCNN_AVX=ON -DNCNN_BUILD_TESTS=ON .. + cmake --build . -j 2 + - name: test-avx1-2 + run: cd build-avx1-2 && ctest --output-on-failure -j 2 - name: build-noint8 run: | mkdir build-noint8 && cd build-noint8 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e3e9b679c..dc7ac5bde 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1291,7 +1291,7 @@ jobs: source emsdk/emsdk_env.sh mkdir build && cd build cmake -DCMAKE_TOOLCHAIN_FILE=../emsdk/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=install -DNCNN_VERSION_STRING="${{ needs.setup.outputs.VERSION }}" \ - -DNCNN_THREADS=OFF -DNCNN_OPENMP=OFF -DNCNN_SIMPLEOMP=OFF -DNCNN_SIMPLEOCV=ON -DNCNN_RUNTIME_CPU=OFF -DNCNN_SSE2=OFF -DNCNN_AVX2=OFF \ + -DNCNN_THREADS=OFF -DNCNN_OPENMP=OFF -DNCNN_SIMPLEOMP=OFF -DNCNN_SIMPLEOCV=ON -DNCNN_RUNTIME_CPU=OFF -DNCNN_SSE2=OFF -DNCNN_AVX2=OFF -DNCNN_AV2=OFF \ -DNCNN_BUILD_TOOLS=OFF -DNCNN_BUILD_EXAMPLES=OFF -DNCNN_BUILD_BENCHMARK=OFF .. cmake --build . -j 2 cmake --build . --target install @@ -1300,7 +1300,7 @@ jobs: source emsdk/emsdk_env.sh mkdir build-simd && cd build-simd cmake -DCMAKE_TOOLCHAIN_FILE=../emsdk/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=install -DNCNN_VERSION_STRING="${{ needs.setup.outputs.VERSION }}" \ - -DNCNN_THREADS=OFF -DNCNN_OPENMP=OFF -DNCNN_SIMPLEOMP=OFF -DNCNN_SIMPLEOCV=ON -DNCNN_RUNTIME_CPU=OFF -DNCNN_SSE2=ON -DNCNN_AVX2=OFF \ + -DNCNN_THREADS=OFF -DNCNN_OPENMP=OFF -DNCNN_SIMPLEOMP=OFF -DNCNN_SIMPLEOCV=ON -DNCNN_RUNTIME_CPU=OFF -DNCNN_SSE2=ON -DNCNN_AVX2=OFF -DNCNN_AV2=OFF \ -DNCNN_BUILD_TOOLS=OFF -DNCNN_BUILD_EXAMPLES=OFF -DNCNN_BUILD_BENCHMARK=OFF .. cmake --build . -j 2 cmake --build . --target install @@ -1309,7 +1309,7 @@ jobs: source emsdk/emsdk_env.sh mkdir build-threads && cd build-threads cmake -DCMAKE_TOOLCHAIN_FILE=../emsdk/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=install -DNCNN_VERSION_STRING="${{ needs.setup.outputs.VERSION }}" \ - -DNCNN_THREADS=ON -DNCNN_OPENMP=ON -DNCNN_SIMPLEOMP=ON -DNCNN_SIMPLEOCV=ON -DNCNN_RUNTIME_CPU=OFF -DNCNN_SSE2=OFF -DNCNN_AVX2=OFF \ + -DNCNN_THREADS=ON -DNCNN_OPENMP=ON -DNCNN_SIMPLEOMP=ON -DNCNN_SIMPLEOCV=ON -DNCNN_RUNTIME_CPU=OFF -DNCNN_SSE2=OFF -DNCNN_AVX2=OFF -DNCNN_AV2=OFF \ -DNCNN_BUILD_TOOLS=OFF -DNCNN_BUILD_EXAMPLES=OFF -DNCNN_BUILD_BENCHMARK=OFF .. cmake --build . -j 2 cmake --build . --target install @@ -1318,7 +1318,7 @@ jobs: source emsdk/emsdk_env.sh mkdir build-simd-threads && cd build-simd-threads cmake -DCMAKE_TOOLCHAIN_FILE=../emsdk/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=install -DNCNN_VERSION_STRING="${{ needs.setup.outputs.VERSION }}" \ - -DNCNN_THREADS=ON -DNCNN_OPENMP=ON -DNCNN_SIMPLEOMP=ON -DNCNN_SIMPLEOCV=ON -DNCNN_RUNTIME_CPU=OFF -DNCNN_SSE2=ON -DNCNN_AVX2=OFF \ + -DNCNN_THREADS=ON -DNCNN_OPENMP=ON -DNCNN_SIMPLEOMP=ON -DNCNN_SIMPLEOCV=ON -DNCNN_RUNTIME_CPU=OFF -DNCNN_SSE2=ON -DNCNN_AVX2=OFF -DNCNN_AV2=OFF \ -DNCNN_BUILD_TOOLS=OFF -DNCNN_BUILD_EXAMPLES=OFF -DNCNN_BUILD_BENCHMARK=OFF .. cmake --build . -j 2 cmake --build . --target install diff --git a/.github/workflows/test-coverage.yml b/.github/workflows/test-coverage.yml index 3163c548c..b2c8e989f 100644 --- a/.github/workflows/test-coverage.yml +++ b/.github/workflows/test-coverage.yml @@ -146,6 +146,33 @@ jobs: with: token: ${{ secrets.CODECOV_TOKEN }} file: build/lcov.info + linux-gcc-avx-omp: + runs-on: ubuntu-latest + steps: + - name: cancel-previous-runs + uses: styfle/cancel-workflow-action@0.9.0 + with: + access_token: ${{ secrets.GITHUB_TOKEN }} + - uses: actions/checkout@v2 + - name: lcov + run: sudo apt-get install lcov + - name: configure + run: mkdir build && cd build && cmake -DCMAKE_BUILD_TYPE=debug -DNCNN_COVERAGE=ON -DNCNN_RUNTIME_CPU=ON -DNCNN_AVX2=OFF -DNCNN_AVX=ON -DNCNN_OPENMP=ON -DNCNN_BUILD_TOOLS=OFF -DNCNN_BUILD_EXAMPLES=OFF -DNCNN_BUILD_TESTS=ON .. + - name: build + run: cmake --build build -j 2 + - name: test + run: cd build && ctest --output-on-failure -j 2 + - name: lcov-collect + run: | + cd build + lcov -d ./src -c -o lcov.info + lcov -r lcov.info '/usr/*' -o lcov.info + lcov --list lcov.info + - name: codecov + uses: codecov/codecov-action@v1.5.2 + with: + token: ${{ secrets.CODECOV_TOKEN }} + file: build/lcov.info linux-gcc-arm: runs-on: ubuntu-20.04 diff --git a/.github/workflows/web-assembly.yml b/.github/workflows/web-assembly.yml index c8732d199..5919c6d6c 100644 --- a/.github/workflows/web-assembly.yml +++ b/.github/workflows/web-assembly.yml @@ -39,7 +39,7 @@ jobs: run: | source emsdk/emsdk_env.sh mkdir build-basic && cd build-basic - cmake -DCMAKE_TOOLCHAIN_FILE=../emsdk/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake -DNCNN_THREADS=OFF -DNCNN_OPENMP=OFF -DNCNN_SIMPLEOMP=OFF -DNCNN_SIMPLEOCV=ON -DNCNN_RUNTIME_CPU=OFF -DNCNN_SSE2=OFF -DNCNN_AVX2=OFF -DNCNN_BUILD_TESTS=ON .. + cmake -DCMAKE_TOOLCHAIN_FILE=../emsdk/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake -DNCNN_THREADS=OFF -DNCNN_OPENMP=OFF -DNCNN_SIMPLEOMP=OFF -DNCNN_SIMPLEOCV=ON -DNCNN_RUNTIME_CPU=OFF -DNCNN_SSE2=OFF -DNCNN_AVX2=OFF -DNCNN_AVX=OFF -DNCNN_BUILD_TESTS=ON .. cmake --build . -j 2 - name: test-basic run: | @@ -49,7 +49,7 @@ jobs: run: | source emsdk/emsdk_env.sh mkdir build-simd && cd build-simd - cmake -DCMAKE_TOOLCHAIN_FILE=../emsdk/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake -DNCNN_THREADS=OFF -DNCNN_OPENMP=OFF -DNCNN_SIMPLEOMP=OFF -DNCNN_SIMPLEOCV=ON -DNCNN_RUNTIME_CPU=OFF -DNCNN_SSE2=ON -DNCNN_AVX2=OFF -DNCNN_BUILD_TESTS=ON .. + cmake -DCMAKE_TOOLCHAIN_FILE=../emsdk/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake -DNCNN_THREADS=OFF -DNCNN_OPENMP=OFF -DNCNN_SIMPLEOMP=OFF -DNCNN_SIMPLEOCV=ON -DNCNN_RUNTIME_CPU=OFF -DNCNN_SSE2=ON -DNCNN_AVX2=OFF -DNCNN_AVX=OFF -DNCNN_BUILD_TESTS=ON .. cmake --build . -j 2 - name: test-simd run: | @@ -59,7 +59,7 @@ jobs: run: | source emsdk/emsdk_env.sh mkdir build-simd-omp && cd build-simd-omp - cmake -DCMAKE_TOOLCHAIN_FILE=../emsdk/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake -DNCNN_THREADS=ON -DNCNN_OPENMP=ON -DNCNN_SIMPLEOMP=ON -DNCNN_SIMPLEOCV=ON -DNCNN_RUNTIME_CPU=OFF -DNCNN_SSE2=ON -DNCNN_AVX2=OFF -DNCNN_BUILD_TESTS=ON .. + cmake -DCMAKE_TOOLCHAIN_FILE=../emsdk/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake -DNCNN_THREADS=ON -DNCNN_OPENMP=ON -DNCNN_SIMPLEOMP=ON -DNCNN_SIMPLEOCV=ON -DNCNN_RUNTIME_CPU=OFF -DNCNN_SSE2=ON -DNCNN_AVX2=OFF -DNCNN_AVX=OFF -DNCNN_BUILD_TESTS=ON .. cmake --build . -j 2 - name: test-simd-omp run: | diff --git a/.github/workflows/windows-x64-cpu-vs2015.yml b/.github/workflows/windows-x64-cpu-vs2015.yml index b72380283..40bcd9629 100644 --- a/.github/workflows/windows-x64-cpu-vs2015.yml +++ b/.github/workflows/windows-x64-cpu-vs2015.yml @@ -51,7 +51,7 @@ jobs: - name: build-sse2 run: | mkdir build-sse2; cd build-sse2 - cmake -T v140,host=x64 -A x64 -DProtobuf_INCLUDE_DIR="$env:GITHUB_WORKSPACE\protobuf-install\include" -DProtobuf_LIBRARIES="$env:GITHUB_WORKSPACE\protobuf-install\lib\libprotobuf.lib" -DProtobuf_PROTOC_EXECUTABLE="$env:GITHUB_WORKSPACE\protobuf-install\bin\protoc.exe" -DNCNN_RUNTIME_CPU=OFF -DNCNN_AVX2=OFF -DNCNN_BUILD_TESTS=ON .. + cmake -T v140,host=x64 -A x64 -DProtobuf_INCLUDE_DIR="$env:GITHUB_WORKSPACE\protobuf-install\include" -DProtobuf_LIBRARIES="$env:GITHUB_WORKSPACE\protobuf-install\lib\libprotobuf.lib" -DProtobuf_PROTOC_EXECUTABLE="$env:GITHUB_WORKSPACE\protobuf-install\bin\protoc.exe" -DNCNN_RUNTIME_CPU=OFF -DNCNN_AVX2=OFF -DNCNN_AVX=OFF -DNCNN_BUILD_TESTS=ON .. cmake --build . --config Release -j 2 - name: test-sse2 run: cd build-sse2; ctest -C Release --output-on-failure -j 2 @@ -67,3 +67,17 @@ jobs: cmake --build . --config Release -j 2 - name: test-avx2 run: cd build-avx2; ctest -C Release --output-on-failure -j 2 + - name: build-avx + run: | + mkdir build-avx; cd build-avx + cmake -T v140,host=x64 -A x64 -DProtobuf_INCLUDE_DIR="$env:GITHUB_WORKSPACE\protobuf-install\include" -DProtobuf_LIBRARIES="$env:GITHUB_WORKSPACE\protobuf-install\lib\libprotobuf.lib" -DProtobuf_PROTOC_EXECUTABLE="$env:GITHUB_WORKSPACE\protobuf-install\bin\protoc.exe" -DNCNN_RUNTIME_CPU=OFF -DNCNN_AVX2=OFF -DNCNN_AVX=ON -DNCNN_BUILD_TESTS=ON .. + cmake --build . --config Release -j 2 + - name: test-avx + run: cd build-avx; ctest -C Release --output-on-failure -j 2 + - name: build-avx1-2 + run: | + mkdir build-avx1-2; cd build-avx1-2 + cmake -T v140,host=x64 -A x64 -DProtobuf_INCLUDE_DIR="$env:GITHUB_WORKSPACE\protobuf-install\include" -DProtobuf_LIBRARIES="$env:GITHUB_WORKSPACE\protobuf-install\lib\libprotobuf.lib" -DProtobuf_PROTOC_EXECUTABLE="$env:GITHUB_WORKSPACE\protobuf-install\bin\protoc.exe" -DNCNN_RUNTIME_CPU=OFF -DNCNN_AVX2=ON -DNCNN_AVX=ON -DNCNN_BUILD_TESTS=ON .. + cmake --build . --config Release -j 2 + - name: test-avx1-2 + run: cd build-avx1-2; ctest -C Release --output-on-failure -j 2 diff --git a/.github/workflows/windows-x64-cpu-vs2017.yml b/.github/workflows/windows-x64-cpu-vs2017.yml index abc7e23b1..9d315ba7a 100644 --- a/.github/workflows/windows-x64-cpu-vs2017.yml +++ b/.github/workflows/windows-x64-cpu-vs2017.yml @@ -51,7 +51,7 @@ jobs: - name: build-sse2 run: | mkdir build-sse2; cd build-sse2 - cmake -T v141,host=x64 -A x64 -DProtobuf_INCLUDE_DIR="$env:GITHUB_WORKSPACE\protobuf-install\include" -DProtobuf_LIBRARIES="$env:GITHUB_WORKSPACE\protobuf-install\lib\libprotobuf.lib" -DProtobuf_PROTOC_EXECUTABLE="$env:GITHUB_WORKSPACE\protobuf-install\bin\protoc.exe" -DNCNN_RUNTIME_CPU=OFF -DNCNN_AVX2=OFF -DNCNN_BUILD_TESTS=ON .. + cmake -T v141,host=x64 -A x64 -DProtobuf_INCLUDE_DIR="$env:GITHUB_WORKSPACE\protobuf-install\include" -DProtobuf_LIBRARIES="$env:GITHUB_WORKSPACE\protobuf-install\lib\libprotobuf.lib" -DProtobuf_PROTOC_EXECUTABLE="$env:GITHUB_WORKSPACE\protobuf-install\bin\protoc.exe" -DNCNN_RUNTIME_CPU=OFF -DNCNN_AVX2=OFF -DNCNN_AVX=OFF -DNCNN_BUILD_TESTS=ON .. cmake --build . --config Release -j 2 - name: test-sse2 run: cd build-sse2; ctest -C Release --output-on-failure -j 2 @@ -67,3 +67,18 @@ jobs: cmake --build . --config Release -j 2 - name: test-avx2 run: cd build-avx2; ctest -C Release --output-on-failure -j 2 + - name: build-avx + run: | + mkdir build-avx; cd build-avx + cmake -T v141,host=x64 -A x64 -DProtobuf_INCLUDE_DIR="$env:GITHUB_WORKSPACE\protobuf-install\include" -DProtobuf_LIBRARIES="$env:GITHUB_WORKSPACE\protobuf-install\lib\libprotobuf.lib" -DProtobuf_PROTOC_EXECUTABLE="$env:GITHUB_WORKSPACE\protobuf-install\bin\protoc.exe" -DNCNN_RUNTIME_CPU=OFF -DNCNN_AVX2=OFF -DNCNN_AVX=ON -DNCNN_BUILD_TESTS=ON .. + cmake --build . --config Release -j 2 + - name: test-avx + run: cd build-avx; ctest -C Release --output-on-failure -j 2 + - name: build-avx1-2 + run: | + mkdir build-avx1-2; cd build-avx1-2 + cmake -T v141,host=x64 -A x64 -DProtobuf_INCLUDE_DIR="$env:GITHUB_WORKSPACE\protobuf-install\include" -DProtobuf_LIBRARIES="$env:GITHUB_WORKSPACE\protobuf-install\lib\libprotobuf.lib" -DProtobuf_PROTOC_EXECUTABLE="$env:GITHUB_WORKSPACE\protobuf-install\bin\protoc.exe" -DNCNN_RUNTIME_CPU=OFF -DNCNN_AVX2=ON -DNCNN_AVX=ON -DNCNN_BUILD_TESTS=ON .. + cmake --build . --config Release -j 2 + - name: test-avx1-2 + run: cd build-avx1-2; ctest -C Release --output-on-failure -j 2 + diff --git a/.github/workflows/windows-x64-cpu-vs2019.yml b/.github/workflows/windows-x64-cpu-vs2019.yml index a3263f6a5..1164ea133 100644 --- a/.github/workflows/windows-x64-cpu-vs2019.yml +++ b/.github/workflows/windows-x64-cpu-vs2019.yml @@ -51,7 +51,7 @@ jobs: - name: build-sse2 run: | mkdir build-sse2; cd build-sse2 - cmake -DProtobuf_INCLUDE_DIR="$env:GITHUB_WORKSPACE\protobuf-install\include" -DProtobuf_LIBRARIES="$env:GITHUB_WORKSPACE\protobuf-install\lib\libprotobuf.lib" -DProtobuf_PROTOC_EXECUTABLE="$env:GITHUB_WORKSPACE\protobuf-install\bin\protoc.exe" -DNCNN_RUNTIME_CPU=OFF -DNCNN_AVX2=OFF -DNCNN_BUILD_TESTS=ON .. + cmake -DProtobuf_INCLUDE_DIR="$env:GITHUB_WORKSPACE\protobuf-install\include" -DProtobuf_LIBRARIES="$env:GITHUB_WORKSPACE\protobuf-install\lib\libprotobuf.lib" -DProtobuf_PROTOC_EXECUTABLE="$env:GITHUB_WORKSPACE\protobuf-install\bin\protoc.exe" -DNCNN_RUNTIME_CPU=OFF -DNCNN_AVX2=OFF -DNCNN_AVX=OFF -DNCNN_BUILD_TESTS=ON .. cmake --build . --config Release -j 2 - name: test-sse2 run: cd build-sse2; ctest -C Release --output-on-failure -j 2 @@ -67,3 +67,17 @@ jobs: cmake --build . --config Release -j 2 - name: test-avx2 run: cd build-avx2; ctest -C Release --output-on-failure -j 2 + - name: build-avx + run: | + mkdir build-avx; cd build-avx + cmake -DProtobuf_INCLUDE_DIR="$env:GITHUB_WORKSPACE\protobuf-install\include" -DProtobuf_LIBRARIES="$env:GITHUB_WORKSPACE\protobuf-install\lib\libprotobuf.lib" -DProtobuf_PROTOC_EXECUTABLE="$env:GITHUB_WORKSPACE\protobuf-install\bin\protoc.exe" -DNCNN_RUNTIME_CPU=OFF -DNCNN_AVX=ON -DNCNN_AVX2=OFF -DNCNN_BUILD_TESTS=ON .. + cmake --build . --config Release -j 2 + - name: test-avx + run: cd build-avx; ctest -C Release --output-on-failure -j 2 + - name: build-avx1-2 + run: | + mkdir build-avx1-2; cd build-avx1-2 + cmake -DProtobuf_INCLUDE_DIR="$env:GITHUB_WORKSPACE\protobuf-install\include" -DProtobuf_LIBRARIES="$env:GITHUB_WORKSPACE\protobuf-install\lib\libprotobuf.lib" -DProtobuf_PROTOC_EXECUTABLE="$env:GITHUB_WORKSPACE\protobuf-install\bin\protoc.exe" -DNCNN_RUNTIME_CPU=OFF -DNCNN_AVX=ON -DNCNN_AVX2=ON -DNCNN_BUILD_TESTS=ON .. + cmake --build . --config Release -j 2 + - name: test-avx1-2 + run: cd build-avx1-2; ctest -C Release --output-on-failure -j 2 diff --git a/CMakeLists.txt b/CMakeLists.txt index 6d2c3de41..5309c5b40 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -183,6 +183,7 @@ else() option(NCNN_SSE2 "optimize x86 platform with sse2" ON) if(NOT CMAKE_SYSTEM_NAME STREQUAL "Emscripten" AND NOT (CMAKE_CXX_COMPILER_ID MATCHES "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4.7)) option(NCNN_AVX2 "optimize x86 platform with avx2" ON) + option(NCNN_AVX "optimize x86 platform with avx" ON) endif() endif() diff --git a/cmake/ncnn_add_layer.cmake b/cmake/ncnn_add_layer.cmake index 9affe3241..a6ee33b20 100644 --- a/cmake/ncnn_add_layer.cmake +++ b/cmake/ncnn_add_layer.cmake @@ -156,6 +156,13 @@ macro(ncnn_add_layer class) ncnn_add_arch_opt_layer(${class} avx2 "-mfma -mf16c -mavx2") endif() endif() + if(NCNN_RUNTIME_CPU AND NCNN_AVX AND NCNN_TARGET_ARCH STREQUAL "x86") + if(CMAKE_CXX_COMPILER_ID MATCHES "MSVC" OR (CMAKE_CXX_COMPILER_ID MATCHES "Clang" AND CMAKE_CXX_SIMULATE_ID MATCHES "MSVC" AND CMAKE_CXX_COMPILER_FRONTEND_VARIANT MATCHES "MSVC")) + ncnn_add_arch_opt_layer(${class} avx "/arch:AVX /DAVX /fp:strict") + else() + ncnn_add_arch_opt_layer(${class} avx "-mavx") + endif() + endif() if(NCNN_RUNTIME_CPU AND NCNN_ARM82 AND (NOT IOS) AND (NOT APPLE) AND (CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm64|aarch64)")) if(NCNN_COMPILER_SUPPORT_ARM82_FP16) diff --git a/cmake/ncnn_generate_avx_source.cmake b/cmake/ncnn_generate_avx_source.cmake new file mode 100644 index 000000000..9a359857f --- /dev/null +++ b/cmake/ncnn_generate_avx_source.cmake @@ -0,0 +1,14 @@ + +# must define SRC DST CLASS + +file(READ ${SRC} source_data) + +# replace +string(TOUPPER ${CLASS} CLASS_UPPER) +string(TOLOWER ${CLASS} CLASS_LOWER) + +string(REGEX REPLACE "LAYER_${CLASS_UPPER}_X86_H" "LAYER_${CLASS_UPPER}_X86_AVX_H" source_data "${source_data}") +string(REGEX REPLACE "${CLASS}_x86" "${CLASS}_x86_avx" source_data "${source_data}") +string(REGEX REPLACE "#include \"${CLASS_LOWER}_x86.h\"" "#include \"${CLASS_LOWER}_x86_avx.h\"" source_data "${source_data}") + +file(WRITE ${DST} "${source_data}") diff --git a/docs/developer-guide/low-level-operation-api.md b/docs/developer-guide/low-level-operation-api.md index 524afeae3..768b94492 100644 --- a/docs/developer-guide/low-level-operation-api.md +++ b/docs/developer-guide/low-level-operation-api.md @@ -118,7 +118,7 @@ void transpose(const ncnn::Mat& in, ncnn::Mat& out) int dst_elempack = 1; if (layer->support_packing) { - if (elemcount % 8 == 0 && ncnn::cpu_support_x86_avx2()) + if (elemcount % 8 == 0 && (ncnn::cpu_support_x86_avx2() || ncnn::cpu_support_x86_avx())) dst_elempack = 8; else if (elemcount % 4 == 0) dst_elempack = 4; @@ -195,7 +195,7 @@ void normalize(const ncnn::Mat& in, ncnn::Mat& out) int dst_elempack = 1; if (layer->support_packing) { - if (elemcount % 8 == 0 && ncnn::cpu_support_x86_avx2()) + if (elemcount % 8 == 0 && (ncnn::cpu_support_x86_avx2() || ncnn::cpu_support_x86_avx())) dst_elempack = 8; else if (elemcount % 4 == 0) dst_elempack = 4; diff --git a/docs/how-to-build/how-to-build.md b/docs/how-to-build/how-to-build.md index 9cd4b2b3a..e79430afe 100644 --- a/docs/how-to-build/how-to-build.md +++ b/docs/how-to-build/how-to-build.md @@ -515,7 +515,7 @@ Build without any extension for general compatibility: mkdir -p build cd build cmake -DCMAKE_TOOLCHAIN_FILE=../emsdk/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake \ - -DNCNN_THREADS=OFF -DNCNN_OPENMP=OFF -DNCNN_SIMPLEOMP=OFF -DNCNN_RUNTIME_CPU=OFF -DNCNN_SSE2=OFF -DNCNN_AVX2=OFF \ + -DNCNN_THREADS=OFF -DNCNN_OPENMP=OFF -DNCNN_SIMPLEOMP=OFF -DNCNN_RUNTIME_CPU=OFF -DNCNN_SSE2=OFF -DNCNN_AVX2=OFF -DNCNN_AVX=OFF \ -DNCNN_BUILD_TOOLS=OFF -DNCNN_BUILD_EXAMPLES=OFF -DNCNN_BUILD_BENCHMARK=OFF .. cmake --build . -j 4 cmake --build . --target install @@ -526,7 +526,7 @@ Build with WASM SIMD extension: mkdir -p build-simd cd build-simd cmake -DCMAKE_TOOLCHAIN_FILE=../emsdk/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake \ - -DNCNN_THREADS=OFF -DNCNN_OPENMP=OFF -DNCNN_SIMPLEOMP=OFF -DNCNN_RUNTIME_CPU=OFF -DNCNN_SSE2=ON -DNCNN_AVX2=OFF \ + -DNCNN_THREADS=OFF -DNCNN_OPENMP=OFF -DNCNN_SIMPLEOMP=OFF -DNCNN_RUNTIME_CPU=OFF -DNCNN_SSE2=ON -DNCNN_AVX2=OFF -DNCNN_AVX=OFF \ -DNCNN_BUILD_TOOLS=OFF -DNCNN_BUILD_EXAMPLES=OFF -DNCNN_BUILD_BENCHMARK=OFF .. cmake --build . -j 4 cmake --build . --target install @@ -537,7 +537,7 @@ Build with WASM Thread extension: mkdir -p build-threads cd build-threads cmake -DCMAKE_TOOLCHAIN_FILE=../emsdk/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake \ - -DNCNN_THREADS=ON -DNCNN_OPENMP=ON -DNCNN_SIMPLEOMP=ON -DNCNN_RUNTIME_CPU=OFF -DNCNN_SSE2=OFF -DNCNN_AVX2=OFF \ + -DNCNN_THREADS=ON -DNCNN_OPENMP=ON -DNCNN_SIMPLEOMP=ON -DNCNN_RUNTIME_CPU=OFF -DNCNN_SSE2=OFF -DNCNN_AVX2=OFF -DNCNN_AVX=OFF \ -DNCNN_BUILD_TOOLS=OFF -DNCNN_BUILD_EXAMPLES=OFF -DNCNN_BUILD_BENCHMARK=OFF .. cmake --build . -j 4 cmake --build . --target install @@ -548,7 +548,7 @@ Build with WASM SIMD and Thread extension: mkdir -p build-simd-threads cd build-simd-threads cmake -DCMAKE_TOOLCHAIN_FILE=../emsdk/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake \ - -DNCNN_THREADS=ON -DNCNN_OPENMP=ON -DNCNN_SIMPLEOMP=ON -DNCNN_RUNTIME_CPU=OFF -DNCNN_SSE2=ON -DNCNN_AVX2=OFF \ + -DNCNN_THREADS=ON -DNCNN_OPENMP=ON -DNCNN_SIMPLEOMP=ON -DNCNN_RUNTIME_CPU=OFF -DNCNN_SSE2=ON -DNCNN_AVX2=OFF -DNCNN_AVX=OFF \ -DNCNN_BUILD_TOOLS=OFF -DNCNN_BUILD_EXAMPLES=OFF -DNCNN_BUILD_BENCHMARK=OFF .. cmake --build . -j 4 cmake --build . --target install diff --git a/python/src/main.cpp b/python/src/main.cpp index cb39cc287..f18135f2c 100644 --- a/python/src/main.cpp +++ b/python/src/main.cpp @@ -903,6 +903,7 @@ PYBIND11_MODULE(ncnn, m) m.def("cpu_support_arm_vfpv4", &cpu_support_arm_vfpv4); m.def("cpu_support_arm_asimdhp", &cpu_support_arm_asimdhp); m.def("cpu_support_x86_avx2", &cpu_support_x86_avx2); + m.def("cpu_support_x86_avx", &cpu_support_x86_avx); m.def("get_cpu_count", &get_cpu_count); m.def("get_little_cpu_count", &get_little_cpu_count); m.def("get_big_cpu_count", &get_big_cpu_count); diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 28c827dbf..90007b24f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -173,7 +173,7 @@ generate_export_header(ncnn) if(NCNN_SHARED_LIB) if(CMAKE_CXX_COMPILER_ID MATCHES "GNU") - # for cpu_support_x86_avx2 + # for cpu_support_x86_avx(2) target_link_libraries(ncnn PRIVATE gcc) endif() else() @@ -186,7 +186,7 @@ if(NCNN_SIMPLESTL) # do not link gcc lib on darwin and emscripten if(CMAKE_CXX_COMPILER_ID MATCHES "(Clang|GNU)" AND NOT CMAKE_SYSTEM_NAME MATCHES "(Darwin|Emscripten)") - # for cpu_support_x86_avx2 + # for cpu_support_x86_avx(2) target_link_libraries(ncnn PUBLIC gcc) endif() endif() @@ -342,6 +342,13 @@ if(NCNN_TARGET_ARCH STREQUAL "x86") target_compile_options(ncnn PRIVATE -mfma -mf16c -mavx2) endif() endif() + if(NOT NCNN_RUNTIME_CPU AND NCNN_AVX) + if(CMAKE_CXX_COMPILER_ID MATCHES "MSVC" OR (CMAKE_CXX_COMPILER_ID MATCHES "Clang" AND CMAKE_CXX_SIMULATE_ID MATCHES "MSVC" AND CMAKE_CXX_COMPILER_FRONTEND_VARIANT MATCHES "MSVC")) + target_compile_options(ncnn PRIVATE /arch:AVX) + else() + target_compile_options(ncnn PRIVATE -mavx) + endif() + endif() endif() if(((IOS AND CMAKE_OSX_ARCHITECTURES MATCHES "arm64") OR (APPLE AND CMAKE_OSX_ARCHITECTURES MATCHES "arm64") OR (CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm64|aarch64)"))) diff --git a/src/cpu.cpp b/src/cpu.cpp index 732380a5c..7b76f90d8 100644 --- a/src/cpu.cpp +++ b/src/cpu.cpp @@ -365,6 +365,9 @@ int cpu_support_arm_asimddp() int cpu_support_x86_avx2() { +#if !NCNN_AVX2 + return 0; +#endif #if (_M_AMD64 || __x86_64__) || (_M_IX86 || __i386__) #if defined(_MSC_VER) // TODO move to init function @@ -404,6 +407,48 @@ int cpu_support_x86_avx2() #endif } +int cpu_support_x86_avx() +{ +#if !NCNN_AVX + return 0; +#endif +#if (_M_AMD64 || __x86_64__) || (_M_IX86 || __i386__) +#if defined(_MSC_VER) + // TODO move to init function + int cpu_info[4]; + __cpuid(cpu_info, 0); + + int nIds = cpu_info[0]; + if (nIds < 7) + return 0; + + __cpuid(cpu_info, 1); + // check AVX XSAVE OSXSAVE + if (!(cpu_info[2] & 0x10000000) || !(cpu_info[2] & 0x04000000) || !(cpu_info[2] & 0x08000000)) + return 0; + + // check XSAVE enabled by kernel + if ((_xgetbv(0) & 6) != 6) + return 0; + return 1; +#elif defined(__clang__) +#if __clang_major__ >= 6 + __builtin_cpu_init(); +#endif + return __builtin_cpu_supports("avx"); +#elif __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8) + __builtin_cpu_init(); + return __builtin_cpu_supports("avx"); +#else + // TODO: other x86 compilers checking avx here + NCNN_LOGE("AVX detection method is unknown for current compiler"); + return 0; +#endif +#else + return 0; +#endif +} + int cpu_support_mips_msa() { #if defined __ANDROID__ || defined __linux__ diff --git a/src/cpu.h b/src/cpu.h index 723a2fc56..90f3a9381 100644 --- a/src/cpu.h +++ b/src/cpu.h @@ -57,6 +57,9 @@ NCNN_EXPORT int cpu_support_arm_asimddp(); // avx2 = x86_64 avx2 + fma + f16c NCNN_EXPORT int cpu_support_x86_avx2(); +// avx = x86_64 avx +NCNN_EXPORT int cpu_support_x86_avx(); + // msa = mips mas NCNN_EXPORT int cpu_support_mips_msa(); // mmi = loongson mmi diff --git a/src/layer.cpp b/src/layer.cpp index 367abebb5..945a96b3f 100644 --- a/src/layer.cpp +++ b/src/layer.cpp @@ -236,9 +236,15 @@ Layer* create_layer(int index) if (ncnn::cpu_support_x86_avx2()) { layer_creator = layer_registry_avx2[index].creator; + } else +#endif// NCNN_RUNTIME_CPU && NCNN_AVX2 +#if NCNN_RUNTIME_CPU && NCNN_AVX + if (ncnn::cpu_support_x86_avx()) + { + layer_creator = layer_registry_avx[index].creator; } else -#endif // NCNN_RUNTIME_CPU && NCNN_AVX2 +#endif // NCNN_RUNTIME_CPU && NCNN_AVX #if NCNN_RUNTIME_CPU && NCNN_ARM82DOT if (ncnn::cpu_support_arm_asimdhp() && ncnn::cpu_support_arm_asimddp()) { diff --git a/src/layer/x86/avx_mathfun.h b/src/layer/x86/avx_mathfun.h index c978d9178..9f1a68e43 100644 --- a/src/layer/x86/avx_mathfun.h +++ b/src/layer/x86/avx_mathfun.h @@ -91,6 +91,45 @@ _PS256_CONST(cephes_log_p8, +3.3333331174E-1f); _PS256_CONST(cephes_log_q1, -2.12194440e-4f); _PS256_CONST(cephes_log_q2, 0.693359375f); +#ifndef __AVX2__ +#define AVX2_BITOP_USING_SSE2(fn) \ + static inline __m256i _mm256_comp_##fn(__m256i x, int a) \ + { \ + /* use SSE2 instruction to perform the bitop AVX2 */ \ + __m128i x1, x2; \ + __m256i ret; \ + COPY_IMM_TO_XMM(x, x1, x2); \ + x1 = _mm_##fn(x1, a); \ + x2 = _mm_##fn(x2, a); \ + COPY_XMM_TO_IMM(x1, x2, ret); \ + return (ret); \ + } +#define AVX2_INTOP_USING_SSE2(fn) \ + static inline __m256i _mm256_comp_##fn(__m256i x, __m256i y) \ + { \ + /* use SSE2 instructions to perform the AVX2 integer operation */ \ + __m128i x1, x2; \ + __m128i y1, y2; \ + __m256i ret; \ + COPY_IMM_TO_XMM(x, x1, x2); \ + COPY_IMM_TO_XMM(y, y1, y2); \ + x1 = _mm_##fn(x1, y1); \ + x2 = _mm_##fn(x2, y2); \ + COPY_XMM_TO_IMM(x1, x2, ret); \ + return (ret); \ + } +#else +#define AVX2_BITOP_USING_SSE2(fn) \ + static inline __m256i _mm256_comp_##fn(__m256i x, int a) \ + { \ + return _mm256_##fn(x, a); \ + } +#define AVX2_INTOP_USING_SSE2(fn) \ + static inline __m256i _mm256_comp_##fn(__m256i x, __m256i y) \ + { \ + return _mm256_##fn(x, y); \ + } +#endif #ifndef __AVX2__ typedef union imm_xmm_union @@ -115,54 +154,28 @@ typedef union imm_xmm_union imm_ = u.imm; \ } -#define AVX2_BITOP_USING_SSE2(fn) \ - static inline __m256i _mm256_##fn(__m256i x, int a) \ - { \ - /* use SSE2 instruction to perform the bitop AVX2 */ \ - __m128i x1, x2; \ - __m256i ret; \ - COPY_IMM_TO_XMM(x, x1, x2); \ - x1 = _mm_##fn(x1, a); \ - x2 = _mm_##fn(x2, a); \ - COPY_XMM_TO_IMM(x1, x2, ret); \ - return (ret); \ - } - #if _MSC_VER #pragma WARNING(Using SSE2 to perform AVX2 bitshift ops) #else #warning "Using SSE2 to perform AVX2 bitshift ops" #endif -AVX2_BITOP_USING_SSE2(slli_epi32) -AVX2_BITOP_USING_SSE2(srli_epi32) - -#define AVX2_INTOP_USING_SSE2(fn) \ - static inline __m256i _mm256_##fn(__m256i x, __m256i y) \ - { \ - /* use SSE2 instructions to perform the AVX2 integer operation */ \ - __m128i x1, x2; \ - __m128i y1, y2; \ - __m256i ret; \ - COPY_IMM_TO_XMM(x, x1, x2); \ - COPY_IMM_TO_XMM(y, y1, y2); \ - x1 = _mm_##fn(x1, y1); \ - x2 = _mm_##fn(x2, y2); \ - COPY_XMM_TO_IMM(x1, x2, ret); \ - return (ret); \ - } #if _MSC_VER #pragma WARNING(Using SSE2 to perform AVX2 bitshift ops) #else #warning "Using SSE2 to perform AVX2 integer ops" #endif -AVX2_INTOP_USING_SSE2(and_si128) -AVX2_INTOP_USING_SSE2(andnot_si128) +#endif /* __AVX2__ */ +AVX2_BITOP_USING_SSE2(slli_epi32) +AVX2_BITOP_USING_SSE2(srli_epi32) AVX2_INTOP_USING_SSE2(cmpeq_epi32) AVX2_INTOP_USING_SSE2(sub_epi32) AVX2_INTOP_USING_SSE2(add_epi32) -#endif /* __AVX2__ */ +#ifndef __AVX2__ +AVX2_INTOP_USING_SSE2(and_si128) +AVX2_INTOP_USING_SSE2(andnot_si128) +#endif /* natural logarithm computed for 8 simultaneous float return NaN for x <= 0 @@ -178,14 +191,14 @@ static inline __m256 log256_ps(__m256 x) x = _mm256_max_ps(x, *(__m256*)_ps256_min_norm_pos); /* cut off denormalized stuff */ // can be done with AVX2 - imm0 = _mm256_srli_epi32(_mm256_castps_si256(x), 23); + imm0 = _mm256_comp_srli_epi32(_mm256_castps_si256(x), 23); /* keep only the fractional part */ x = _mm256_and_ps(x, *(__m256*)_ps256_inv_mant_mask); x = _mm256_or_ps(x, *(__m256*)_ps256_0p5); // this is again another AVX2 instruction - imm0 = _mm256_sub_epi32(imm0, *(__m256i*)_pi32_256_0x7f); + imm0 = _mm256_comp_sub_epi32(imm0, *(__m256i*)_pi32_256_0x7f); __m256 e = _mm256_cvtepi32_ps(imm0); e = _mm256_add_ps(e, one); @@ -303,8 +316,8 @@ static inline __m256 exp256_ps(__m256 x) /* build 2^n */ imm0 = _mm256_cvttps_epi32(fx); // another two AVX2 instructions - imm0 = _mm256_add_epi32(imm0, *(__m256i*)_pi32_256_0x7f); - imm0 = _mm256_slli_epi32(imm0, 23); + imm0 = _mm256_comp_add_epi32(imm0, *(__m256i*)_pi32_256_0x7f); + imm0 = _mm256_comp_slli_epi32(imm0, 23); __m256 pow2n = _mm256_castsi256_ps(imm0); y = _mm256_mul_ps(y, pow2n); return y; @@ -363,13 +376,13 @@ static inline __m256 sin256_ps(__m256 x) imm2 = _mm256_cvttps_epi32(y); /* j=(j+1) & (~1) (see the cephes sources) */ // another two AVX2 instruction - imm2 = _mm256_add_epi32(imm2, *(__m256i*)_pi32_256_1); + imm2 = _mm256_comp_add_epi32(imm2, *(__m256i*)_pi32_256_1); imm2 = _mm256_and_si256(imm2, *(__m256i*)_pi32_256_inv1); y = _mm256_cvtepi32_ps(imm2); /* get the swap sign flag */ imm0 = _mm256_and_si256(imm2, *(__m256i*)_pi32_256_4); - imm0 = _mm256_slli_epi32(imm0, 29); + imm0 = _mm256_comp_slli_epi32(imm0, 29); /* get the polynom selection mask there is one polynom for 0 <= x <= Pi/4 and another one for Pi/4 #endif // __AVX__ #endif // __SSE2__ +#include "x86_usability.h" namespace ncnn { @@ -53,7 +54,7 @@ int BatchNorm_x86::forward_inplace(Mat& bottom_top_blob, const Option& opt) cons __m256 _b = _mm256_loadu_ps((const float*)b_data + i * 8); __m256 _p = _mm256_loadu_ps(ptr); - _p = _mm256_fmadd_ps(_p, _b, _a); + _p = _mm256_comp_fmadd_ps(_p, _b, _a); _mm256_storeu_ps(ptr, _p); } } @@ -74,7 +75,7 @@ int BatchNorm_x86::forward_inplace(Mat& bottom_top_blob, const Option& opt) cons for (int j = 0; j < w; j++) { __m256 _p = _mm256_loadu_ps(ptr); - _p = _mm256_fmadd_ps(_p, _b, _a); + _p = _mm256_comp_fmadd_ps(_p, _b, _a); _mm256_storeu_ps(ptr, _p); ptr += 8; @@ -100,7 +101,7 @@ int BatchNorm_x86::forward_inplace(Mat& bottom_top_blob, const Option& opt) cons for (int i = 0; i < size; i++) { __m256 _p = _mm256_loadu_ps(ptr); - _p = _mm256_fmadd_ps(_p, _b, _a); + _p = _mm256_comp_fmadd_ps(_p, _b, _a); _mm256_storeu_ps(ptr, _p); ptr += 8; @@ -213,7 +214,7 @@ int BatchNorm_x86::forward_inplace(Mat& bottom_top_blob, const Option& opt) cons for (; i + 7 < size; i += 8) { __m256 _p = _mm256_loadu_ps(ptr); - _p = _mm256_fmadd_ps(_p, _b256, _a256); + _p = _mm256_comp_fmadd_ps(_p, _b256, _a256); _mm256_storeu_ps(ptr, _p); ptr += 8; } diff --git a/src/layer/x86/cast_x86.cpp b/src/layer/x86/cast_x86.cpp index 141b154c2..5148206f0 100644 --- a/src/layer/x86/cast_x86.cpp +++ b/src/layer/x86/cast_x86.cpp @@ -44,6 +44,8 @@ static inline __m256 bfloat2float_avx(__m128i v0) ab = _mm256_insertf128_si256(ab, b, 1); // insert in high 128-bit lane return _mm256_castsi256_ps(ab); } +#if __AVX2__ + static inline __m256i float2bfloat_avx(__m256 v0, __m256 v1) { __m256i a = _mm256_castps_si256(v0); @@ -60,6 +62,7 @@ static inline __m128i float2bfloat_avx(__m256 v0) __m256i aaaa = _mm256_packus_epi32(a, a); return _mm256_castsi256_si128(_mm256_permutevar8x32_epi32(aaaa, _mm256_setr_epi32(0, 1, 4, 5, 2, 3, 6, 7))); } +#endif #endif // __AVX__ namespace ncnn { @@ -71,7 +74,7 @@ Cast_x86::Cast_x86() int Cast_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const { -#if __AVX__ +#if __AVX2__ if (type_from == type_to) { top_blob = bottom_blob; @@ -128,7 +131,6 @@ int Cast_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) return -100; int size = w * h * elempack; - if (type_from == 1 && type_to == 2) { int nn = size >> 3; diff --git a/src/layer/x86/convolution_1x1_pack4.h b/src/layer/x86/convolution_1x1_pack4.h index 78f136d4e..63a49624a 100644 --- a/src/layer/x86/convolution_1x1_pack4.h +++ b/src/layer/x86/convolution_1x1_pack4.h @@ -199,22 +199,22 @@ static void conv1x1s1_sgemm_pack4_sse(const Mat& bottom_blob, Mat& top_blob, con __m128 _w3 = _mm_load_ps(kptr0 + 12); #if __AVX__ - _sum0 = _mm_fmadd_ps(_w0, _val00, _sum0); - _sum0 = _mm_fmadd_ps(_w1, _val01, _sum0); - _sum0 = _mm_fmadd_ps(_w2, _val02, _sum0); - _sum0 = _mm_fmadd_ps(_w3, _val03, _sum0); - _sum1 = _mm_fmadd_ps(_w0, _val10, _sum1); - _sum1 = _mm_fmadd_ps(_w1, _val11, _sum1); - _sum1 = _mm_fmadd_ps(_w2, _val12, _sum1); - _sum1 = _mm_fmadd_ps(_w3, _val13, _sum1); - _sum2 = _mm_fmadd_ps(_w0, _val20, _sum2); - _sum2 = _mm_fmadd_ps(_w1, _val21, _sum2); - _sum2 = _mm_fmadd_ps(_w2, _val22, _sum2); - _sum2 = _mm_fmadd_ps(_w3, _val23, _sum2); - _sum3 = _mm_fmadd_ps(_w0, _val30, _sum3); - _sum3 = _mm_fmadd_ps(_w1, _val31, _sum3); - _sum3 = _mm_fmadd_ps(_w2, _val32, _sum3); - _sum3 = _mm_fmadd_ps(_w3, _val33, _sum3); + _sum0 = _mm_comp_fmadd_ps(_w0, _val00, _sum0); + _sum0 = _mm_comp_fmadd_ps(_w1, _val01, _sum0); + _sum0 = _mm_comp_fmadd_ps(_w2, _val02, _sum0); + _sum0 = _mm_comp_fmadd_ps(_w3, _val03, _sum0); + _sum1 = _mm_comp_fmadd_ps(_w0, _val10, _sum1); + _sum1 = _mm_comp_fmadd_ps(_w1, _val11, _sum1); + _sum1 = _mm_comp_fmadd_ps(_w2, _val12, _sum1); + _sum1 = _mm_comp_fmadd_ps(_w3, _val13, _sum1); + _sum2 = _mm_comp_fmadd_ps(_w0, _val20, _sum2); + _sum2 = _mm_comp_fmadd_ps(_w1, _val21, _sum2); + _sum2 = _mm_comp_fmadd_ps(_w2, _val22, _sum2); + _sum2 = _mm_comp_fmadd_ps(_w3, _val23, _sum2); + _sum3 = _mm_comp_fmadd_ps(_w0, _val30, _sum3); + _sum3 = _mm_comp_fmadd_ps(_w1, _val31, _sum3); + _sum3 = _mm_comp_fmadd_ps(_w2, _val32, _sum3); + _sum3 = _mm_comp_fmadd_ps(_w3, _val33, _sum3); #else _sum0 = _mm_add_ps(_mm_mul_ps(_w0, _val00), _sum0); _sum0 = _mm_add_ps(_mm_mul_ps(_w1, _val01), _sum0); @@ -269,14 +269,14 @@ static void conv1x1s1_sgemm_pack4_sse(const Mat& bottom_blob, Mat& top_blob, con __m128 _w3 = _mm_load_ps(kptr0 + 12); #if __AVX__ - _sum0 = _mm_fmadd_ps(_w0, _val00, _sum0); - _sum0 = _mm_fmadd_ps(_w1, _val01, _sum0); - _sum0 = _mm_fmadd_ps(_w2, _val02, _sum0); - _sum0 = _mm_fmadd_ps(_w3, _val03, _sum0); - _sum1 = _mm_fmadd_ps(_w0, _val10, _sum1); - _sum1 = _mm_fmadd_ps(_w1, _val11, _sum1); - _sum1 = _mm_fmadd_ps(_w2, _val12, _sum1); - _sum1 = _mm_fmadd_ps(_w3, _val13, _sum1); + _sum0 = _mm_comp_fmadd_ps(_w0, _val00, _sum0); + _sum0 = _mm_comp_fmadd_ps(_w1, _val01, _sum0); + _sum0 = _mm_comp_fmadd_ps(_w2, _val02, _sum0); + _sum0 = _mm_comp_fmadd_ps(_w3, _val03, _sum0); + _sum1 = _mm_comp_fmadd_ps(_w0, _val10, _sum1); + _sum1 = _mm_comp_fmadd_ps(_w1, _val11, _sum1); + _sum1 = _mm_comp_fmadd_ps(_w2, _val12, _sum1); + _sum1 = _mm_comp_fmadd_ps(_w3, _val13, _sum1); #else _sum0 = _mm_add_ps(_mm_mul_ps(_w0, _val00), _sum0); _sum0 = _mm_add_ps(_mm_mul_ps(_w1, _val01), _sum0); @@ -316,10 +316,10 @@ static void conv1x1s1_sgemm_pack4_sse(const Mat& bottom_blob, Mat& top_blob, con __m128 _w3 = _mm_load_ps(kptr0 + 12); #if __AVX__ - _sum = _mm_fmadd_ps(_w0, _val0, _sum); - _sum = _mm_fmadd_ps(_w1, _val1, _sum); - _sum = _mm_fmadd_ps(_w2, _val2, _sum); - _sum = _mm_fmadd_ps(_w3, _val3, _sum); + _sum = _mm_comp_fmadd_ps(_w0, _val0, _sum); + _sum = _mm_comp_fmadd_ps(_w1, _val1, _sum); + _sum = _mm_comp_fmadd_ps(_w2, _val2, _sum); + _sum = _mm_comp_fmadd_ps(_w3, _val3, _sum); #else _sum = _mm_add_ps(_mm_mul_ps(_w0, _val0), _sum); _sum = _mm_add_ps(_mm_mul_ps(_w1, _val1), _sum); diff --git a/src/layer/x86/convolution_1x1_pack8.h b/src/layer/x86/convolution_1x1_pack8.h index da5b0970e..3ed3e29ae 100644 --- a/src/layer/x86/convolution_1x1_pack8.h +++ b/src/layer/x86/convolution_1x1_pack8.h @@ -390,7 +390,7 @@ static void conv1x1s1_sgemm_pack8_avx(const Mat& bottom_blob, Mat& top_blob, con __m256 _val02 = _mm256_broadcast_ss(tmpptr + 2); __m256 _val03 = _mm256_broadcast_ss(tmpptr + 3); - _mm256_fmadd_ps4(_sum0, _w0, _w1, _w2, _w3, _val00, _val01, _val02, _val03); + _mm256_comp_fmadd_ps4(_sum0, _w0, _w1, _w2, _w3, _val00, _val01, _val02, _val03); __m256 _w4 = _mm256_loadu_ps(kptr + 32); __m256 _w5 = _mm256_loadu_ps(kptr + 40); @@ -402,161 +402,161 @@ static void conv1x1s1_sgemm_pack8_avx(const Mat& bottom_blob, Mat& top_blob, con __m256 _val06 = _mm256_broadcast_ss(tmpptr + 6); __m256 _val07 = _mm256_broadcast_ss(tmpptr + 7); - _mm256_fmadd_ps4(_sum0, _w4, _w5, _w6, _w7, _val04, _val05, _val06, _val07); + _mm256_comp_fmadd_ps4(_sum0, _w4, _w5, _w6, _w7, _val04, _val05, _val06, _val07); __m256 _val10 = _mm256_broadcast_ss(tmpptr + 8); __m256 _val11 = _mm256_broadcast_ss(tmpptr + 9); __m256 _val12 = _mm256_broadcast_ss(tmpptr + 10); __m256 _val13 = _mm256_broadcast_ss(tmpptr + 11); - _mm256_fmadd_ps4(_sum1, _w0, _w1, _w2, _w3, _val10, _val11, _val12, _val13); + _mm256_comp_fmadd_ps4(_sum1, _w0, _w1, _w2, _w3, _val10, _val11, _val12, _val13); __m256 _val14 = _mm256_broadcast_ss(tmpptr + 12); __m256 _val15 = _mm256_broadcast_ss(tmpptr + 13); __m256 _val16 = _mm256_broadcast_ss(tmpptr + 14); __m256 _val17 = _mm256_broadcast_ss(tmpptr + 15); - _mm256_fmadd_ps4(_sum1, _w4, _w5, _w6, _w7, _val14, _val15, _val16, _val17); + _mm256_comp_fmadd_ps4(_sum1, _w4, _w5, _w6, _w7, _val14, _val15, _val16, _val17); __m256 _val20 = _mm256_broadcast_ss(tmpptr + 16); __m256 _val21 = _mm256_broadcast_ss(tmpptr + 17); __m256 _val22 = _mm256_broadcast_ss(tmpptr + 18); __m256 _val23 = _mm256_broadcast_ss(tmpptr + 19); - _mm256_fmadd_ps4(_sum2, _w0, _w1, _w2, _w3, _val20, _val21, _val22, _val23); + _mm256_comp_fmadd_ps4(_sum2, _w0, _w1, _w2, _w3, _val20, _val21, _val22, _val23); __m256 _val24 = _mm256_broadcast_ss(tmpptr + 20); __m256 _val25 = _mm256_broadcast_ss(tmpptr + 21); __m256 _val26 = _mm256_broadcast_ss(tmpptr + 22); __m256 _val27 = _mm256_broadcast_ss(tmpptr + 23); - _mm256_fmadd_ps4(_sum2, _w4, _w5, _w6, _w7, _val24, _val25, _val26, _val27); + _mm256_comp_fmadd_ps4(_sum2, _w4, _w5, _w6, _w7, _val24, _val25, _val26, _val27); __m256 _val30 = _mm256_broadcast_ss(tmpptr + 24); __m256 _val31 = _mm256_broadcast_ss(tmpptr + 25); __m256 _val32 = _mm256_broadcast_ss(tmpptr + 26); __m256 _val33 = _mm256_broadcast_ss(tmpptr + 27); - _mm256_fmadd_ps4(_sum3, _w0, _w1, _w2, _w3, _val30, _val31, _val32, _val33); + _mm256_comp_fmadd_ps4(_sum3, _w0, _w1, _w2, _w3, _val30, _val31, _val32, _val33); __m256 _val34 = _mm256_broadcast_ss(tmpptr + 28); __m256 _val35 = _mm256_broadcast_ss(tmpptr + 29); __m256 _val36 = _mm256_broadcast_ss(tmpptr + 30); __m256 _val37 = _mm256_broadcast_ss(tmpptr + 31); - _mm256_fmadd_ps4(_sum3, _w4, _w5, _w6, _w7, _val34, _val35, _val36, _val37); + _mm256_comp_fmadd_ps4(_sum3, _w4, _w5, _w6, _w7, _val34, _val35, _val36, _val37); __m256 _val40 = _mm256_broadcast_ss(tmpptr + 32); __m256 _val41 = _mm256_broadcast_ss(tmpptr + 33); __m256 _val42 = _mm256_broadcast_ss(tmpptr + 34); __m256 _val43 = _mm256_broadcast_ss(tmpptr + 35); - _mm256_fmadd_ps4(_sum4, _w0, _w1, _w2, _w3, _val40, _val41, _val42, _val43); + _mm256_comp_fmadd_ps4(_sum4, _w0, _w1, _w2, _w3, _val40, _val41, _val42, _val43); __m256 _val44 = _mm256_broadcast_ss(tmpptr + 36); __m256 _val45 = _mm256_broadcast_ss(tmpptr + 37); __m256 _val46 = _mm256_broadcast_ss(tmpptr + 38); __m256 _val47 = _mm256_broadcast_ss(tmpptr + 39); - _mm256_fmadd_ps4(_sum4, _w4, _w5, _w6, _w7, _val44, _val45, _val46, _val47); + _mm256_comp_fmadd_ps4(_sum4, _w4, _w5, _w6, _w7, _val44, _val45, _val46, _val47); __m256 _val50 = _mm256_broadcast_ss(tmpptr + 40); __m256 _val51 = _mm256_broadcast_ss(tmpptr + 41); __m256 _val52 = _mm256_broadcast_ss(tmpptr + 42); __m256 _val53 = _mm256_broadcast_ss(tmpptr + 43); - _mm256_fmadd_ps4(_sum5, _w0, _w1, _w2, _w3, _val50, _val51, _val52, _val53); + _mm256_comp_fmadd_ps4(_sum5, _w0, _w1, _w2, _w3, _val50, _val51, _val52, _val53); __m256 _val54 = _mm256_broadcast_ss(tmpptr + 44); __m256 _val55 = _mm256_broadcast_ss(tmpptr + 45); __m256 _val56 = _mm256_broadcast_ss(tmpptr + 46); __m256 _val57 = _mm256_broadcast_ss(tmpptr + 47); - _mm256_fmadd_ps4(_sum5, _w4, _w5, _w6, _w7, _val54, _val55, _val56, _val57); + _mm256_comp_fmadd_ps4(_sum5, _w4, _w5, _w6, _w7, _val54, _val55, _val56, _val57); __m256 _val60 = _mm256_broadcast_ss(tmpptr + 48); __m256 _val61 = _mm256_broadcast_ss(tmpptr + 49); __m256 _val62 = _mm256_broadcast_ss(tmpptr + 50); __m256 _val63 = _mm256_broadcast_ss(tmpptr + 51); - _mm256_fmadd_ps4(_sum6, _w0, _w1, _w2, _w3, _val60, _val61, _val62, _val63); + _mm256_comp_fmadd_ps4(_sum6, _w0, _w1, _w2, _w3, _val60, _val61, _val62, _val63); __m256 _val64 = _mm256_broadcast_ss(tmpptr + 52); __m256 _val65 = _mm256_broadcast_ss(tmpptr + 53); __m256 _val66 = _mm256_broadcast_ss(tmpptr + 54); __m256 _val67 = _mm256_broadcast_ss(tmpptr + 55); - _mm256_fmadd_ps4(_sum6, _w4, _w5, _w6, _w7, _val64, _val65, _val66, _val67); + _mm256_comp_fmadd_ps4(_sum6, _w4, _w5, _w6, _w7, _val64, _val65, _val66, _val67); __m256 _val70 = _mm256_broadcast_ss(tmpptr + 56); __m256 _val71 = _mm256_broadcast_ss(tmpptr + 57); __m256 _val72 = _mm256_broadcast_ss(tmpptr + 58); __m256 _val73 = _mm256_broadcast_ss(tmpptr + 59); - _mm256_fmadd_ps4(_sum7, _w0, _w1, _w2, _w3, _val70, _val71, _val72, _val73); + _mm256_comp_fmadd_ps4(_sum7, _w0, _w1, _w2, _w3, _val70, _val71, _val72, _val73); __m256 _val74 = _mm256_broadcast_ss(tmpptr + 60); __m256 _val75 = _mm256_broadcast_ss(tmpptr + 61); __m256 _val76 = _mm256_broadcast_ss(tmpptr + 62); __m256 _val77 = _mm256_broadcast_ss(tmpptr + 63); - _mm256_fmadd_ps4(_sum7, _w4, _w5, _w6, _w7, _val74, _val75, _val76, _val77); + _mm256_comp_fmadd_ps4(_sum7, _w4, _w5, _w6, _w7, _val74, _val75, _val76, _val77); __m256 _val80 = _mm256_broadcast_ss(tmpptr + 64); __m256 _val81 = _mm256_broadcast_ss(tmpptr + 65); __m256 _val82 = _mm256_broadcast_ss(tmpptr + 66); __m256 _val83 = _mm256_broadcast_ss(tmpptr + 67); - _mm256_fmadd_ps4(_sum8, _w0, _w1, _w2, _w3, _val80, _val81, _val82, _val83); + _mm256_comp_fmadd_ps4(_sum8, _w0, _w1, _w2, _w3, _val80, _val81, _val82, _val83); __m256 _val84 = _mm256_broadcast_ss(tmpptr + 68); __m256 _val85 = _mm256_broadcast_ss(tmpptr + 69); __m256 _val86 = _mm256_broadcast_ss(tmpptr + 70); __m256 _val87 = _mm256_broadcast_ss(tmpptr + 71); - _mm256_fmadd_ps4(_sum8, _w4, _w5, _w6, _w7, _val84, _val85, _val86, _val87); + _mm256_comp_fmadd_ps4(_sum8, _w4, _w5, _w6, _w7, _val84, _val85, _val86, _val87); __m256 _val90 = _mm256_broadcast_ss(tmpptr + 72); __m256 _val91 = _mm256_broadcast_ss(tmpptr + 73); __m256 _val92 = _mm256_broadcast_ss(tmpptr + 74); __m256 _val93 = _mm256_broadcast_ss(tmpptr + 75); - _mm256_fmadd_ps4(_sum9, _w0, _w1, _w2, _w3, _val90, _val91, _val92, _val93); + _mm256_comp_fmadd_ps4(_sum9, _w0, _w1, _w2, _w3, _val90, _val91, _val92, _val93); __m256 _val94 = _mm256_broadcast_ss(tmpptr + 76); __m256 _val95 = _mm256_broadcast_ss(tmpptr + 77); __m256 _val96 = _mm256_broadcast_ss(tmpptr + 78); __m256 _val97 = _mm256_broadcast_ss(tmpptr + 79); - _mm256_fmadd_ps4(_sum9, _w4, _w5, _w6, _w7, _val94, _val95, _val96, _val97); + _mm256_comp_fmadd_ps4(_sum9, _w4, _w5, _w6, _w7, _val94, _val95, _val96, _val97); __m256 _val100 = _mm256_broadcast_ss(tmpptr + 80); __m256 _val101 = _mm256_broadcast_ss(tmpptr + 81); __m256 _val102 = _mm256_broadcast_ss(tmpptr + 82); __m256 _val103 = _mm256_broadcast_ss(tmpptr + 83); - _mm256_fmadd_ps4(_sum10, _w0, _w1, _w2, _w3, _val100, _val101, _val102, _val103); + _mm256_comp_fmadd_ps4(_sum10, _w0, _w1, _w2, _w3, _val100, _val101, _val102, _val103); __m256 _val104 = _mm256_broadcast_ss(tmpptr + 84); __m256 _val105 = _mm256_broadcast_ss(tmpptr + 85); __m256 _val106 = _mm256_broadcast_ss(tmpptr + 86); __m256 _val107 = _mm256_broadcast_ss(tmpptr + 87); - _mm256_fmadd_ps4(_sum10, _w4, _w5, _w6, _w7, _val104, _val105, _val106, _val107); + _mm256_comp_fmadd_ps4(_sum10, _w4, _w5, _w6, _w7, _val104, _val105, _val106, _val107); __m256 _val110 = _mm256_broadcast_ss(tmpptr + 88); __m256 _val111 = _mm256_broadcast_ss(tmpptr + 89); __m256 _val112 = _mm256_broadcast_ss(tmpptr + 90); __m256 _val113 = _mm256_broadcast_ss(tmpptr + 91); - _mm256_fmadd_ps4(_sum11, _w0, _w1, _w2, _w3, _val110, _val111, _val112, _val113); + _mm256_comp_fmadd_ps4(_sum11, _w0, _w1, _w2, _w3, _val110, _val111, _val112, _val113); __m256 _val114 = _mm256_broadcast_ss(tmpptr + 92); __m256 _val115 = _mm256_broadcast_ss(tmpptr + 93); __m256 _val116 = _mm256_broadcast_ss(tmpptr + 94); __m256 _val117 = _mm256_broadcast_ss(tmpptr + 95); - _mm256_fmadd_ps4(_sum11, _w4, _w5, _w6, _w7, _val114, _val115, _val116, _val117); + _mm256_comp_fmadd_ps4(_sum11, _w4, _w5, _w6, _w7, _val114, _val115, _val116, _val117); tmpptr += 96; @@ -619,22 +619,22 @@ static void conv1x1s1_sgemm_pack8_avx(const Mat& bottom_blob, Mat& top_blob, con __m256 _val16 = _mm256_broadcast_ss(tmpptr + 14); __m256 _val17 = _mm256_broadcast_ss(tmpptr + 15); - _sum0 = _mm256_fmadd_ps(_w0, _val00, _sum0); - _sum0 = _mm256_fmadd_ps(_w1, _val01, _sum0); - _sum0 = _mm256_fmadd_ps(_w2, _val02, _sum0); - _sum0 = _mm256_fmadd_ps(_w3, _val03, _sum0); - _sum0 = _mm256_fmadd_ps(_w4, _val04, _sum0); - _sum0 = _mm256_fmadd_ps(_w5, _val05, _sum0); - _sum0 = _mm256_fmadd_ps(_w6, _val06, _sum0); - _sum0 = _mm256_fmadd_ps(_w7, _val07, _sum0); - _sum1 = _mm256_fmadd_ps(_w0, _val10, _sum1); - _sum1 = _mm256_fmadd_ps(_w1, _val11, _sum1); - _sum1 = _mm256_fmadd_ps(_w2, _val12, _sum1); - _sum1 = _mm256_fmadd_ps(_w3, _val13, _sum1); - _sum1 = _mm256_fmadd_ps(_w4, _val14, _sum1); - _sum1 = _mm256_fmadd_ps(_w5, _val15, _sum1); - _sum1 = _mm256_fmadd_ps(_w6, _val16, _sum1); - _sum1 = _mm256_fmadd_ps(_w7, _val17, _sum1); + _sum0 = _mm256_comp_fmadd_ps(_w0, _val00, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_w1, _val01, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_w2, _val02, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_w3, _val03, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_w4, _val04, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_w5, _val05, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_w6, _val06, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_w7, _val07, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_w0, _val10, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_w1, _val11, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_w2, _val12, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_w3, _val13, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_w4, _val14, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_w5, _val15, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_w6, _val16, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_w7, _val17, _sum1); __m256 _val20 = _mm256_broadcast_ss(tmpptr + 16); __m256 _val21 = _mm256_broadcast_ss(tmpptr + 17); @@ -653,22 +653,22 @@ static void conv1x1s1_sgemm_pack8_avx(const Mat& bottom_blob, Mat& top_blob, con __m256 _val36 = _mm256_broadcast_ss(tmpptr + 30); __m256 _val37 = _mm256_broadcast_ss(tmpptr + 31); - _sum2 = _mm256_fmadd_ps(_w0, _val20, _sum2); - _sum2 = _mm256_fmadd_ps(_w1, _val21, _sum2); - _sum2 = _mm256_fmadd_ps(_w2, _val22, _sum2); - _sum2 = _mm256_fmadd_ps(_w3, _val23, _sum2); - _sum2 = _mm256_fmadd_ps(_w4, _val24, _sum2); - _sum2 = _mm256_fmadd_ps(_w5, _val25, _sum2); - _sum2 = _mm256_fmadd_ps(_w6, _val26, _sum2); - _sum2 = _mm256_fmadd_ps(_w7, _val27, _sum2); - _sum3 = _mm256_fmadd_ps(_w0, _val30, _sum3); - _sum3 = _mm256_fmadd_ps(_w1, _val31, _sum3); - _sum3 = _mm256_fmadd_ps(_w2, _val32, _sum3); - _sum3 = _mm256_fmadd_ps(_w3, _val33, _sum3); - _sum3 = _mm256_fmadd_ps(_w4, _val34, _sum3); - _sum3 = _mm256_fmadd_ps(_w5, _val35, _sum3); - _sum3 = _mm256_fmadd_ps(_w6, _val36, _sum3); - _sum3 = _mm256_fmadd_ps(_w7, _val37, _sum3); + _sum2 = _mm256_comp_fmadd_ps(_w0, _val20, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_w1, _val21, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_w2, _val22, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_w3, _val23, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_w4, _val24, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_w5, _val25, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_w6, _val26, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_w7, _val27, _sum2); + _sum3 = _mm256_comp_fmadd_ps(_w0, _val30, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_w1, _val31, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_w2, _val32, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_w3, _val33, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_w4, _val34, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_w5, _val35, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_w6, _val36, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_w7, _val37, _sum3); __m256 _val40 = _mm256_broadcast_ss(tmpptr + 32); __m256 _val41 = _mm256_broadcast_ss(tmpptr + 33); @@ -687,22 +687,22 @@ static void conv1x1s1_sgemm_pack8_avx(const Mat& bottom_blob, Mat& top_blob, con __m256 _val56 = _mm256_broadcast_ss(tmpptr + 46); __m256 _val57 = _mm256_broadcast_ss(tmpptr + 47); - _sum4 = _mm256_fmadd_ps(_w0, _val40, _sum4); - _sum4 = _mm256_fmadd_ps(_w1, _val41, _sum4); - _sum4 = _mm256_fmadd_ps(_w2, _val42, _sum4); - _sum4 = _mm256_fmadd_ps(_w3, _val43, _sum4); - _sum4 = _mm256_fmadd_ps(_w4, _val44, _sum4); - _sum4 = _mm256_fmadd_ps(_w5, _val45, _sum4); - _sum4 = _mm256_fmadd_ps(_w6, _val46, _sum4); - _sum4 = _mm256_fmadd_ps(_w7, _val47, _sum4); - _sum5 = _mm256_fmadd_ps(_w0, _val50, _sum5); - _sum5 = _mm256_fmadd_ps(_w1, _val51, _sum5); - _sum5 = _mm256_fmadd_ps(_w2, _val52, _sum5); - _sum5 = _mm256_fmadd_ps(_w3, _val53, _sum5); - _sum5 = _mm256_fmadd_ps(_w4, _val54, _sum5); - _sum5 = _mm256_fmadd_ps(_w5, _val55, _sum5); - _sum5 = _mm256_fmadd_ps(_w6, _val56, _sum5); - _sum5 = _mm256_fmadd_ps(_w7, _val57, _sum5); + _sum4 = _mm256_comp_fmadd_ps(_w0, _val40, _sum4); + _sum4 = _mm256_comp_fmadd_ps(_w1, _val41, _sum4); + _sum4 = _mm256_comp_fmadd_ps(_w2, _val42, _sum4); + _sum4 = _mm256_comp_fmadd_ps(_w3, _val43, _sum4); + _sum4 = _mm256_comp_fmadd_ps(_w4, _val44, _sum4); + _sum4 = _mm256_comp_fmadd_ps(_w5, _val45, _sum4); + _sum4 = _mm256_comp_fmadd_ps(_w6, _val46, _sum4); + _sum4 = _mm256_comp_fmadd_ps(_w7, _val47, _sum4); + _sum5 = _mm256_comp_fmadd_ps(_w0, _val50, _sum5); + _sum5 = _mm256_comp_fmadd_ps(_w1, _val51, _sum5); + _sum5 = _mm256_comp_fmadd_ps(_w2, _val52, _sum5); + _sum5 = _mm256_comp_fmadd_ps(_w3, _val53, _sum5); + _sum5 = _mm256_comp_fmadd_ps(_w4, _val54, _sum5); + _sum5 = _mm256_comp_fmadd_ps(_w5, _val55, _sum5); + _sum5 = _mm256_comp_fmadd_ps(_w6, _val56, _sum5); + _sum5 = _mm256_comp_fmadd_ps(_w7, _val57, _sum5); __m256 _val60 = _mm256_broadcast_ss(tmpptr + 48); __m256 _val61 = _mm256_broadcast_ss(tmpptr + 49); @@ -721,22 +721,22 @@ static void conv1x1s1_sgemm_pack8_avx(const Mat& bottom_blob, Mat& top_blob, con __m256 _val76 = _mm256_broadcast_ss(tmpptr + 62); __m256 _val77 = _mm256_broadcast_ss(tmpptr + 63); - _sum6 = _mm256_fmadd_ps(_w0, _val60, _sum6); - _sum6 = _mm256_fmadd_ps(_w1, _val61, _sum6); - _sum6 = _mm256_fmadd_ps(_w2, _val62, _sum6); - _sum6 = _mm256_fmadd_ps(_w3, _val63, _sum6); - _sum6 = _mm256_fmadd_ps(_w4, _val64, _sum6); - _sum6 = _mm256_fmadd_ps(_w5, _val65, _sum6); - _sum6 = _mm256_fmadd_ps(_w6, _val66, _sum6); - _sum6 = _mm256_fmadd_ps(_w7, _val67, _sum6); - _sum7 = _mm256_fmadd_ps(_w0, _val70, _sum7); - _sum7 = _mm256_fmadd_ps(_w1, _val71, _sum7); - _sum7 = _mm256_fmadd_ps(_w2, _val72, _sum7); - _sum7 = _mm256_fmadd_ps(_w3, _val73, _sum7); - _sum7 = _mm256_fmadd_ps(_w4, _val74, _sum7); - _sum7 = _mm256_fmadd_ps(_w5, _val75, _sum7); - _sum7 = _mm256_fmadd_ps(_w6, _val76, _sum7); - _sum7 = _mm256_fmadd_ps(_w7, _val77, _sum7); + _sum6 = _mm256_comp_fmadd_ps(_w0, _val60, _sum6); + _sum6 = _mm256_comp_fmadd_ps(_w1, _val61, _sum6); + _sum6 = _mm256_comp_fmadd_ps(_w2, _val62, _sum6); + _sum6 = _mm256_comp_fmadd_ps(_w3, _val63, _sum6); + _sum6 = _mm256_comp_fmadd_ps(_w4, _val64, _sum6); + _sum6 = _mm256_comp_fmadd_ps(_w5, _val65, _sum6); + _sum6 = _mm256_comp_fmadd_ps(_w6, _val66, _sum6); + _sum6 = _mm256_comp_fmadd_ps(_w7, _val67, _sum6); + _sum7 = _mm256_comp_fmadd_ps(_w0, _val70, _sum7); + _sum7 = _mm256_comp_fmadd_ps(_w1, _val71, _sum7); + _sum7 = _mm256_comp_fmadd_ps(_w2, _val72, _sum7); + _sum7 = _mm256_comp_fmadd_ps(_w3, _val73, _sum7); + _sum7 = _mm256_comp_fmadd_ps(_w4, _val74, _sum7); + _sum7 = _mm256_comp_fmadd_ps(_w5, _val75, _sum7); + _sum7 = _mm256_comp_fmadd_ps(_w6, _val76, _sum7); + _sum7 = _mm256_comp_fmadd_ps(_w7, _val77, _sum7); tmpptr += 64; @@ -791,22 +791,22 @@ static void conv1x1s1_sgemm_pack8_avx(const Mat& bottom_blob, Mat& top_blob, con __m256 _val16 = _mm256_broadcast_ss(tmpptr + 14); __m256 _val17 = _mm256_broadcast_ss(tmpptr + 15); - _sum0 = _mm256_fmadd_ps(_w0, _val00, _sum0); - _sum0 = _mm256_fmadd_ps(_w1, _val01, _sum0); - _sum0 = _mm256_fmadd_ps(_w2, _val02, _sum0); - _sum0 = _mm256_fmadd_ps(_w3, _val03, _sum0); - _sum0 = _mm256_fmadd_ps(_w4, _val04, _sum0); - _sum0 = _mm256_fmadd_ps(_w5, _val05, _sum0); - _sum0 = _mm256_fmadd_ps(_w6, _val06, _sum0); - _sum0 = _mm256_fmadd_ps(_w7, _val07, _sum0); - _sum1 = _mm256_fmadd_ps(_w0, _val10, _sum1); - _sum1 = _mm256_fmadd_ps(_w1, _val11, _sum1); - _sum1 = _mm256_fmadd_ps(_w2, _val12, _sum1); - _sum1 = _mm256_fmadd_ps(_w3, _val13, _sum1); - _sum1 = _mm256_fmadd_ps(_w4, _val14, _sum1); - _sum1 = _mm256_fmadd_ps(_w5, _val15, _sum1); - _sum1 = _mm256_fmadd_ps(_w6, _val16, _sum1); - _sum1 = _mm256_fmadd_ps(_w7, _val17, _sum1); + _sum0 = _mm256_comp_fmadd_ps(_w0, _val00, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_w1, _val01, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_w2, _val02, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_w3, _val03, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_w4, _val04, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_w5, _val05, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_w6, _val06, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_w7, _val07, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_w0, _val10, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_w1, _val11, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_w2, _val12, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_w3, _val13, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_w4, _val14, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_w5, _val15, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_w6, _val16, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_w7, _val17, _sum1); __m256 _val20 = _mm256_broadcast_ss(tmpptr + 16); __m256 _val21 = _mm256_broadcast_ss(tmpptr + 17); @@ -825,22 +825,22 @@ static void conv1x1s1_sgemm_pack8_avx(const Mat& bottom_blob, Mat& top_blob, con __m256 _val36 = _mm256_broadcast_ss(tmpptr + 30); __m256 _val37 = _mm256_broadcast_ss(tmpptr + 31); - _sum2 = _mm256_fmadd_ps(_w0, _val20, _sum2); - _sum2 = _mm256_fmadd_ps(_w1, _val21, _sum2); - _sum2 = _mm256_fmadd_ps(_w2, _val22, _sum2); - _sum2 = _mm256_fmadd_ps(_w3, _val23, _sum2); - _sum2 = _mm256_fmadd_ps(_w4, _val24, _sum2); - _sum2 = _mm256_fmadd_ps(_w5, _val25, _sum2); - _sum2 = _mm256_fmadd_ps(_w6, _val26, _sum2); - _sum2 = _mm256_fmadd_ps(_w7, _val27, _sum2); - _sum3 = _mm256_fmadd_ps(_w0, _val30, _sum3); - _sum3 = _mm256_fmadd_ps(_w1, _val31, _sum3); - _sum3 = _mm256_fmadd_ps(_w2, _val32, _sum3); - _sum3 = _mm256_fmadd_ps(_w3, _val33, _sum3); - _sum3 = _mm256_fmadd_ps(_w4, _val34, _sum3); - _sum3 = _mm256_fmadd_ps(_w5, _val35, _sum3); - _sum3 = _mm256_fmadd_ps(_w6, _val36, _sum3); - _sum3 = _mm256_fmadd_ps(_w7, _val37, _sum3); + _sum2 = _mm256_comp_fmadd_ps(_w0, _val20, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_w1, _val21, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_w2, _val22, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_w3, _val23, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_w4, _val24, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_w5, _val25, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_w6, _val26, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_w7, _val27, _sum2); + _sum3 = _mm256_comp_fmadd_ps(_w0, _val30, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_w1, _val31, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_w2, _val32, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_w3, _val33, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_w4, _val34, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_w5, _val35, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_w6, _val36, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_w7, _val37, _sum3); tmpptr += 32; @@ -889,22 +889,22 @@ static void conv1x1s1_sgemm_pack8_avx(const Mat& bottom_blob, Mat& top_blob, con __m256 _w6 = _mm256_loadu_ps(kptr + 48); __m256 _w7 = _mm256_loadu_ps(kptr + 56); - _sum0 = _mm256_fmadd_ps(_w0, _val00, _sum0); - _sum0 = _mm256_fmadd_ps(_w1, _val01, _sum0); - _sum0 = _mm256_fmadd_ps(_w2, _val02, _sum0); - _sum0 = _mm256_fmadd_ps(_w3, _val03, _sum0); - _sum0 = _mm256_fmadd_ps(_w4, _val04, _sum0); - _sum0 = _mm256_fmadd_ps(_w5, _val05, _sum0); - _sum0 = _mm256_fmadd_ps(_w6, _val06, _sum0); - _sum0 = _mm256_fmadd_ps(_w7, _val07, _sum0); - _sum1 = _mm256_fmadd_ps(_w0, _val10, _sum1); - _sum1 = _mm256_fmadd_ps(_w1, _val11, _sum1); - _sum1 = _mm256_fmadd_ps(_w2, _val12, _sum1); - _sum1 = _mm256_fmadd_ps(_w3, _val13, _sum1); - _sum1 = _mm256_fmadd_ps(_w4, _val14, _sum1); - _sum1 = _mm256_fmadd_ps(_w5, _val15, _sum1); - _sum1 = _mm256_fmadd_ps(_w6, _val16, _sum1); - _sum1 = _mm256_fmadd_ps(_w7, _val17, _sum1); + _sum0 = _mm256_comp_fmadd_ps(_w0, _val00, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_w1, _val01, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_w2, _val02, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_w3, _val03, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_w4, _val04, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_w5, _val05, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_w6, _val06, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_w7, _val07, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_w0, _val10, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_w1, _val11, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_w2, _val12, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_w3, _val13, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_w4, _val14, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_w5, _val15, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_w6, _val16, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_w7, _val17, _sum1); tmpptr += 16; @@ -942,14 +942,14 @@ static void conv1x1s1_sgemm_pack8_avx(const Mat& bottom_blob, Mat& top_blob, con __m256 _w6 = _mm256_loadu_ps(kptr + 48); __m256 _w7 = _mm256_loadu_ps(kptr + 56); - _sum = _mm256_fmadd_ps(_w0, _val0, _sum); - _sum = _mm256_fmadd_ps(_w1, _val1, _sum); - _sum = _mm256_fmadd_ps(_w2, _val2, _sum); - _sum = _mm256_fmadd_ps(_w3, _val3, _sum); - _sum = _mm256_fmadd_ps(_w4, _val4, _sum); - _sum = _mm256_fmadd_ps(_w5, _val5, _sum); - _sum = _mm256_fmadd_ps(_w6, _val6, _sum); - _sum = _mm256_fmadd_ps(_w7, _val7, _sum); + _sum = _mm256_comp_fmadd_ps(_w0, _val0, _sum); + _sum = _mm256_comp_fmadd_ps(_w1, _val1, _sum); + _sum = _mm256_comp_fmadd_ps(_w2, _val2, _sum); + _sum = _mm256_comp_fmadd_ps(_w3, _val3, _sum); + _sum = _mm256_comp_fmadd_ps(_w4, _val4, _sum); + _sum = _mm256_comp_fmadd_ps(_w5, _val5, _sum); + _sum = _mm256_comp_fmadd_ps(_w6, _val6, _sum); + _sum = _mm256_comp_fmadd_ps(_w7, _val7, _sum); tmpptr += 8; diff --git a/src/layer/x86/convolution_1x1_pack8_fp16.h b/src/layer/x86/convolution_1x1_pack8_fp16.h index 8202d8680..dab96b441 100644 --- a/src/layer/x86/convolution_1x1_pack8_fp16.h +++ b/src/layer/x86/convolution_1x1_pack8_fp16.h @@ -406,22 +406,22 @@ static void conv1x1s1_sgemm_fp16_pack8_avx(const Mat& bottom_blob, Mat& top_blob __m256 _val16 = _mm256_broadcast_ss(tmpptr + 14); __m256 _val17 = _mm256_broadcast_ss(tmpptr + 15); - _sum0 = _mm256_fmadd_ps(_w0, _val00, _sum0); - _sum0 = _mm256_fmadd_ps(_w1, _val01, _sum0); - _sum0 = _mm256_fmadd_ps(_w2, _val02, _sum0); - _sum0 = _mm256_fmadd_ps(_w3, _val03, _sum0); - _sum0 = _mm256_fmadd_ps(_w4, _val04, _sum0); - _sum0 = _mm256_fmadd_ps(_w5, _val05, _sum0); - _sum0 = _mm256_fmadd_ps(_w6, _val06, _sum0); - _sum0 = _mm256_fmadd_ps(_w7, _val07, _sum0); - _sum1 = _mm256_fmadd_ps(_w0, _val10, _sum1); - _sum1 = _mm256_fmadd_ps(_w1, _val11, _sum1); - _sum1 = _mm256_fmadd_ps(_w2, _val12, _sum1); - _sum1 = _mm256_fmadd_ps(_w3, _val13, _sum1); - _sum1 = _mm256_fmadd_ps(_w4, _val14, _sum1); - _sum1 = _mm256_fmadd_ps(_w5, _val15, _sum1); - _sum1 = _mm256_fmadd_ps(_w6, _val16, _sum1); - _sum1 = _mm256_fmadd_ps(_w7, _val17, _sum1); + _sum0 = _mm256_comp_fmadd_ps(_w0, _val00, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_w1, _val01, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_w2, _val02, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_w3, _val03, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_w4, _val04, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_w5, _val05, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_w6, _val06, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_w7, _val07, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_w0, _val10, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_w1, _val11, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_w2, _val12, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_w3, _val13, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_w4, _val14, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_w5, _val15, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_w6, _val16, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_w7, _val17, _sum1); __m256 _val20 = _mm256_broadcast_ss(tmpptr + 16); __m256 _val21 = _mm256_broadcast_ss(tmpptr + 17); @@ -440,22 +440,22 @@ static void conv1x1s1_sgemm_fp16_pack8_avx(const Mat& bottom_blob, Mat& top_blob __m256 _val36 = _mm256_broadcast_ss(tmpptr + 30); __m256 _val37 = _mm256_broadcast_ss(tmpptr + 31); - _sum2 = _mm256_fmadd_ps(_w0, _val20, _sum2); - _sum2 = _mm256_fmadd_ps(_w1, _val21, _sum2); - _sum2 = _mm256_fmadd_ps(_w2, _val22, _sum2); - _sum2 = _mm256_fmadd_ps(_w3, _val23, _sum2); - _sum2 = _mm256_fmadd_ps(_w4, _val24, _sum2); - _sum2 = _mm256_fmadd_ps(_w5, _val25, _sum2); - _sum2 = _mm256_fmadd_ps(_w6, _val26, _sum2); - _sum2 = _mm256_fmadd_ps(_w7, _val27, _sum2); - _sum3 = _mm256_fmadd_ps(_w0, _val30, _sum3); - _sum3 = _mm256_fmadd_ps(_w1, _val31, _sum3); - _sum3 = _mm256_fmadd_ps(_w2, _val32, _sum3); - _sum3 = _mm256_fmadd_ps(_w3, _val33, _sum3); - _sum3 = _mm256_fmadd_ps(_w4, _val34, _sum3); - _sum3 = _mm256_fmadd_ps(_w5, _val35, _sum3); - _sum3 = _mm256_fmadd_ps(_w6, _val36, _sum3); - _sum3 = _mm256_fmadd_ps(_w7, _val37, _sum3); + _sum2 = _mm256_comp_fmadd_ps(_w0, _val20, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_w1, _val21, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_w2, _val22, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_w3, _val23, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_w4, _val24, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_w5, _val25, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_w6, _val26, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_w7, _val27, _sum2); + _sum3 = _mm256_comp_fmadd_ps(_w0, _val30, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_w1, _val31, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_w2, _val32, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_w3, _val33, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_w4, _val34, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_w5, _val35, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_w6, _val36, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_w7, _val37, _sum3); __m256 _val40 = _mm256_broadcast_ss(tmpptr + 32); __m256 _val41 = _mm256_broadcast_ss(tmpptr + 33); @@ -474,22 +474,22 @@ static void conv1x1s1_sgemm_fp16_pack8_avx(const Mat& bottom_blob, Mat& top_blob __m256 _val56 = _mm256_broadcast_ss(tmpptr + 46); __m256 _val57 = _mm256_broadcast_ss(tmpptr + 47); - _sum4 = _mm256_fmadd_ps(_w0, _val40, _sum4); - _sum4 = _mm256_fmadd_ps(_w1, _val41, _sum4); - _sum4 = _mm256_fmadd_ps(_w2, _val42, _sum4); - _sum4 = _mm256_fmadd_ps(_w3, _val43, _sum4); - _sum4 = _mm256_fmadd_ps(_w4, _val44, _sum4); - _sum4 = _mm256_fmadd_ps(_w5, _val45, _sum4); - _sum4 = _mm256_fmadd_ps(_w6, _val46, _sum4); - _sum4 = _mm256_fmadd_ps(_w7, _val47, _sum4); - _sum5 = _mm256_fmadd_ps(_w0, _val50, _sum5); - _sum5 = _mm256_fmadd_ps(_w1, _val51, _sum5); - _sum5 = _mm256_fmadd_ps(_w2, _val52, _sum5); - _sum5 = _mm256_fmadd_ps(_w3, _val53, _sum5); - _sum5 = _mm256_fmadd_ps(_w4, _val54, _sum5); - _sum5 = _mm256_fmadd_ps(_w5, _val55, _sum5); - _sum5 = _mm256_fmadd_ps(_w6, _val56, _sum5); - _sum5 = _mm256_fmadd_ps(_w7, _val57, _sum5); + _sum4 = _mm256_comp_fmadd_ps(_w0, _val40, _sum4); + _sum4 = _mm256_comp_fmadd_ps(_w1, _val41, _sum4); + _sum4 = _mm256_comp_fmadd_ps(_w2, _val42, _sum4); + _sum4 = _mm256_comp_fmadd_ps(_w3, _val43, _sum4); + _sum4 = _mm256_comp_fmadd_ps(_w4, _val44, _sum4); + _sum4 = _mm256_comp_fmadd_ps(_w5, _val45, _sum4); + _sum4 = _mm256_comp_fmadd_ps(_w6, _val46, _sum4); + _sum4 = _mm256_comp_fmadd_ps(_w7, _val47, _sum4); + _sum5 = _mm256_comp_fmadd_ps(_w0, _val50, _sum5); + _sum5 = _mm256_comp_fmadd_ps(_w1, _val51, _sum5); + _sum5 = _mm256_comp_fmadd_ps(_w2, _val52, _sum5); + _sum5 = _mm256_comp_fmadd_ps(_w3, _val53, _sum5); + _sum5 = _mm256_comp_fmadd_ps(_w4, _val54, _sum5); + _sum5 = _mm256_comp_fmadd_ps(_w5, _val55, _sum5); + _sum5 = _mm256_comp_fmadd_ps(_w6, _val56, _sum5); + _sum5 = _mm256_comp_fmadd_ps(_w7, _val57, _sum5); __m256 _val60 = _mm256_broadcast_ss(tmpptr + 48); __m256 _val61 = _mm256_broadcast_ss(tmpptr + 49); @@ -508,22 +508,22 @@ static void conv1x1s1_sgemm_fp16_pack8_avx(const Mat& bottom_blob, Mat& top_blob __m256 _val76 = _mm256_broadcast_ss(tmpptr + 62); __m256 _val77 = _mm256_broadcast_ss(tmpptr + 63); - _sum6 = _mm256_fmadd_ps(_w0, _val60, _sum6); - _sum6 = _mm256_fmadd_ps(_w1, _val61, _sum6); - _sum6 = _mm256_fmadd_ps(_w2, _val62, _sum6); - _sum6 = _mm256_fmadd_ps(_w3, _val63, _sum6); - _sum6 = _mm256_fmadd_ps(_w4, _val64, _sum6); - _sum6 = _mm256_fmadd_ps(_w5, _val65, _sum6); - _sum6 = _mm256_fmadd_ps(_w6, _val66, _sum6); - _sum6 = _mm256_fmadd_ps(_w7, _val67, _sum6); - _sum7 = _mm256_fmadd_ps(_w0, _val70, _sum7); - _sum7 = _mm256_fmadd_ps(_w1, _val71, _sum7); - _sum7 = _mm256_fmadd_ps(_w2, _val72, _sum7); - _sum7 = _mm256_fmadd_ps(_w3, _val73, _sum7); - _sum7 = _mm256_fmadd_ps(_w4, _val74, _sum7); - _sum7 = _mm256_fmadd_ps(_w5, _val75, _sum7); - _sum7 = _mm256_fmadd_ps(_w6, _val76, _sum7); - _sum7 = _mm256_fmadd_ps(_w7, _val77, _sum7); + _sum6 = _mm256_comp_fmadd_ps(_w0, _val60, _sum6); + _sum6 = _mm256_comp_fmadd_ps(_w1, _val61, _sum6); + _sum6 = _mm256_comp_fmadd_ps(_w2, _val62, _sum6); + _sum6 = _mm256_comp_fmadd_ps(_w3, _val63, _sum6); + _sum6 = _mm256_comp_fmadd_ps(_w4, _val64, _sum6); + _sum6 = _mm256_comp_fmadd_ps(_w5, _val65, _sum6); + _sum6 = _mm256_comp_fmadd_ps(_w6, _val66, _sum6); + _sum6 = _mm256_comp_fmadd_ps(_w7, _val67, _sum6); + _sum7 = _mm256_comp_fmadd_ps(_w0, _val70, _sum7); + _sum7 = _mm256_comp_fmadd_ps(_w1, _val71, _sum7); + _sum7 = _mm256_comp_fmadd_ps(_w2, _val72, _sum7); + _sum7 = _mm256_comp_fmadd_ps(_w3, _val73, _sum7); + _sum7 = _mm256_comp_fmadd_ps(_w4, _val74, _sum7); + _sum7 = _mm256_comp_fmadd_ps(_w5, _val75, _sum7); + _sum7 = _mm256_comp_fmadd_ps(_w6, _val76, _sum7); + _sum7 = _mm256_comp_fmadd_ps(_w7, _val77, _sum7); __m256 _val80 = _mm256_broadcast_ss(tmpptr + 64); __m256 _val81 = _mm256_broadcast_ss(tmpptr + 65); @@ -542,22 +542,22 @@ static void conv1x1s1_sgemm_fp16_pack8_avx(const Mat& bottom_blob, Mat& top_blob __m256 _val96 = _mm256_broadcast_ss(tmpptr + 78); __m256 _val97 = _mm256_broadcast_ss(tmpptr + 79); - _sum8 = _mm256_fmadd_ps(_w0, _val80, _sum8); - _sum8 = _mm256_fmadd_ps(_w1, _val81, _sum8); - _sum8 = _mm256_fmadd_ps(_w2, _val82, _sum8); - _sum8 = _mm256_fmadd_ps(_w3, _val83, _sum8); - _sum8 = _mm256_fmadd_ps(_w4, _val84, _sum8); - _sum8 = _mm256_fmadd_ps(_w5, _val85, _sum8); - _sum8 = _mm256_fmadd_ps(_w6, _val86, _sum8); - _sum8 = _mm256_fmadd_ps(_w7, _val87, _sum8); - _sum9 = _mm256_fmadd_ps(_w0, _val90, _sum9); - _sum9 = _mm256_fmadd_ps(_w1, _val91, _sum9); - _sum9 = _mm256_fmadd_ps(_w2, _val92, _sum9); - _sum9 = _mm256_fmadd_ps(_w3, _val93, _sum9); - _sum9 = _mm256_fmadd_ps(_w4, _val94, _sum9); - _sum9 = _mm256_fmadd_ps(_w5, _val95, _sum9); - _sum9 = _mm256_fmadd_ps(_w6, _val96, _sum9); - _sum9 = _mm256_fmadd_ps(_w7, _val97, _sum9); + _sum8 = _mm256_comp_fmadd_ps(_w0, _val80, _sum8); + _sum8 = _mm256_comp_fmadd_ps(_w1, _val81, _sum8); + _sum8 = _mm256_comp_fmadd_ps(_w2, _val82, _sum8); + _sum8 = _mm256_comp_fmadd_ps(_w3, _val83, _sum8); + _sum8 = _mm256_comp_fmadd_ps(_w4, _val84, _sum8); + _sum8 = _mm256_comp_fmadd_ps(_w5, _val85, _sum8); + _sum8 = _mm256_comp_fmadd_ps(_w6, _val86, _sum8); + _sum8 = _mm256_comp_fmadd_ps(_w7, _val87, _sum8); + _sum9 = _mm256_comp_fmadd_ps(_w0, _val90, _sum9); + _sum9 = _mm256_comp_fmadd_ps(_w1, _val91, _sum9); + _sum9 = _mm256_comp_fmadd_ps(_w2, _val92, _sum9); + _sum9 = _mm256_comp_fmadd_ps(_w3, _val93, _sum9); + _sum9 = _mm256_comp_fmadd_ps(_w4, _val94, _sum9); + _sum9 = _mm256_comp_fmadd_ps(_w5, _val95, _sum9); + _sum9 = _mm256_comp_fmadd_ps(_w6, _val96, _sum9); + _sum9 = _mm256_comp_fmadd_ps(_w7, _val97, _sum9); __m256 _val100 = _mm256_broadcast_ss(tmpptr + 80); __m256 _val101 = _mm256_broadcast_ss(tmpptr + 81); @@ -576,22 +576,22 @@ static void conv1x1s1_sgemm_fp16_pack8_avx(const Mat& bottom_blob, Mat& top_blob __m256 _val116 = _mm256_broadcast_ss(tmpptr + 94); __m256 _val117 = _mm256_broadcast_ss(tmpptr + 95); - _sum10 = _mm256_fmadd_ps(_w0, _val100, _sum10); - _sum10 = _mm256_fmadd_ps(_w1, _val101, _sum10); - _sum10 = _mm256_fmadd_ps(_w2, _val102, _sum10); - _sum10 = _mm256_fmadd_ps(_w3, _val103, _sum10); - _sum10 = _mm256_fmadd_ps(_w4, _val104, _sum10); - _sum10 = _mm256_fmadd_ps(_w5, _val105, _sum10); - _sum10 = _mm256_fmadd_ps(_w6, _val106, _sum10); - _sum10 = _mm256_fmadd_ps(_w7, _val107, _sum10); - _sum11 = _mm256_fmadd_ps(_w0, _val110, _sum11); - _sum11 = _mm256_fmadd_ps(_w1, _val111, _sum11); - _sum11 = _mm256_fmadd_ps(_w2, _val112, _sum11); - _sum11 = _mm256_fmadd_ps(_w3, _val113, _sum11); - _sum11 = _mm256_fmadd_ps(_w4, _val114, _sum11); - _sum11 = _mm256_fmadd_ps(_w5, _val115, _sum11); - _sum11 = _mm256_fmadd_ps(_w6, _val116, _sum11); - _sum11 = _mm256_fmadd_ps(_w7, _val117, _sum11); + _sum10 = _mm256_comp_fmadd_ps(_w0, _val100, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_w1, _val101, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_w2, _val102, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_w3, _val103, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_w4, _val104, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_w5, _val105, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_w6, _val106, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_w7, _val107, _sum10); + _sum11 = _mm256_comp_fmadd_ps(_w0, _val110, _sum11); + _sum11 = _mm256_comp_fmadd_ps(_w1, _val111, _sum11); + _sum11 = _mm256_comp_fmadd_ps(_w2, _val112, _sum11); + _sum11 = _mm256_comp_fmadd_ps(_w3, _val113, _sum11); + _sum11 = _mm256_comp_fmadd_ps(_w4, _val114, _sum11); + _sum11 = _mm256_comp_fmadd_ps(_w5, _val115, _sum11); + _sum11 = _mm256_comp_fmadd_ps(_w6, _val116, _sum11); + _sum11 = _mm256_comp_fmadd_ps(_w7, _val117, _sum11); tmpptr += 96; @@ -654,22 +654,22 @@ static void conv1x1s1_sgemm_fp16_pack8_avx(const Mat& bottom_blob, Mat& top_blob __m256 _val16 = _mm256_broadcast_ss(tmpptr + 14); __m256 _val17 = _mm256_broadcast_ss(tmpptr + 15); - _sum0 = _mm256_fmadd_ps(_w0, _val00, _sum0); - _sum0 = _mm256_fmadd_ps(_w1, _val01, _sum0); - _sum0 = _mm256_fmadd_ps(_w2, _val02, _sum0); - _sum0 = _mm256_fmadd_ps(_w3, _val03, _sum0); - _sum0 = _mm256_fmadd_ps(_w4, _val04, _sum0); - _sum0 = _mm256_fmadd_ps(_w5, _val05, _sum0); - _sum0 = _mm256_fmadd_ps(_w6, _val06, _sum0); - _sum0 = _mm256_fmadd_ps(_w7, _val07, _sum0); - _sum1 = _mm256_fmadd_ps(_w0, _val10, _sum1); - _sum1 = _mm256_fmadd_ps(_w1, _val11, _sum1); - _sum1 = _mm256_fmadd_ps(_w2, _val12, _sum1); - _sum1 = _mm256_fmadd_ps(_w3, _val13, _sum1); - _sum1 = _mm256_fmadd_ps(_w4, _val14, _sum1); - _sum1 = _mm256_fmadd_ps(_w5, _val15, _sum1); - _sum1 = _mm256_fmadd_ps(_w6, _val16, _sum1); - _sum1 = _mm256_fmadd_ps(_w7, _val17, _sum1); + _sum0 = _mm256_comp_fmadd_ps(_w0, _val00, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_w1, _val01, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_w2, _val02, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_w3, _val03, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_w4, _val04, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_w5, _val05, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_w6, _val06, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_w7, _val07, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_w0, _val10, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_w1, _val11, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_w2, _val12, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_w3, _val13, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_w4, _val14, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_w5, _val15, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_w6, _val16, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_w7, _val17, _sum1); __m256 _val20 = _mm256_broadcast_ss(tmpptr + 16); __m256 _val21 = _mm256_broadcast_ss(tmpptr + 17); @@ -688,22 +688,22 @@ static void conv1x1s1_sgemm_fp16_pack8_avx(const Mat& bottom_blob, Mat& top_blob __m256 _val36 = _mm256_broadcast_ss(tmpptr + 30); __m256 _val37 = _mm256_broadcast_ss(tmpptr + 31); - _sum2 = _mm256_fmadd_ps(_w0, _val20, _sum2); - _sum2 = _mm256_fmadd_ps(_w1, _val21, _sum2); - _sum2 = _mm256_fmadd_ps(_w2, _val22, _sum2); - _sum2 = _mm256_fmadd_ps(_w3, _val23, _sum2); - _sum2 = _mm256_fmadd_ps(_w4, _val24, _sum2); - _sum2 = _mm256_fmadd_ps(_w5, _val25, _sum2); - _sum2 = _mm256_fmadd_ps(_w6, _val26, _sum2); - _sum2 = _mm256_fmadd_ps(_w7, _val27, _sum2); - _sum3 = _mm256_fmadd_ps(_w0, _val30, _sum3); - _sum3 = _mm256_fmadd_ps(_w1, _val31, _sum3); - _sum3 = _mm256_fmadd_ps(_w2, _val32, _sum3); - _sum3 = _mm256_fmadd_ps(_w3, _val33, _sum3); - _sum3 = _mm256_fmadd_ps(_w4, _val34, _sum3); - _sum3 = _mm256_fmadd_ps(_w5, _val35, _sum3); - _sum3 = _mm256_fmadd_ps(_w6, _val36, _sum3); - _sum3 = _mm256_fmadd_ps(_w7, _val37, _sum3); + _sum2 = _mm256_comp_fmadd_ps(_w0, _val20, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_w1, _val21, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_w2, _val22, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_w3, _val23, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_w4, _val24, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_w5, _val25, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_w6, _val26, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_w7, _val27, _sum2); + _sum3 = _mm256_comp_fmadd_ps(_w0, _val30, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_w1, _val31, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_w2, _val32, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_w3, _val33, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_w4, _val34, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_w5, _val35, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_w6, _val36, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_w7, _val37, _sum3); __m256 _val40 = _mm256_broadcast_ss(tmpptr + 32); __m256 _val41 = _mm256_broadcast_ss(tmpptr + 33); @@ -722,22 +722,22 @@ static void conv1x1s1_sgemm_fp16_pack8_avx(const Mat& bottom_blob, Mat& top_blob __m256 _val56 = _mm256_broadcast_ss(tmpptr + 46); __m256 _val57 = _mm256_broadcast_ss(tmpptr + 47); - _sum4 = _mm256_fmadd_ps(_w0, _val40, _sum4); - _sum4 = _mm256_fmadd_ps(_w1, _val41, _sum4); - _sum4 = _mm256_fmadd_ps(_w2, _val42, _sum4); - _sum4 = _mm256_fmadd_ps(_w3, _val43, _sum4); - _sum4 = _mm256_fmadd_ps(_w4, _val44, _sum4); - _sum4 = _mm256_fmadd_ps(_w5, _val45, _sum4); - _sum4 = _mm256_fmadd_ps(_w6, _val46, _sum4); - _sum4 = _mm256_fmadd_ps(_w7, _val47, _sum4); - _sum5 = _mm256_fmadd_ps(_w0, _val50, _sum5); - _sum5 = _mm256_fmadd_ps(_w1, _val51, _sum5); - _sum5 = _mm256_fmadd_ps(_w2, _val52, _sum5); - _sum5 = _mm256_fmadd_ps(_w3, _val53, _sum5); - _sum5 = _mm256_fmadd_ps(_w4, _val54, _sum5); - _sum5 = _mm256_fmadd_ps(_w5, _val55, _sum5); - _sum5 = _mm256_fmadd_ps(_w6, _val56, _sum5); - _sum5 = _mm256_fmadd_ps(_w7, _val57, _sum5); + _sum4 = _mm256_comp_fmadd_ps(_w0, _val40, _sum4); + _sum4 = _mm256_comp_fmadd_ps(_w1, _val41, _sum4); + _sum4 = _mm256_comp_fmadd_ps(_w2, _val42, _sum4); + _sum4 = _mm256_comp_fmadd_ps(_w3, _val43, _sum4); + _sum4 = _mm256_comp_fmadd_ps(_w4, _val44, _sum4); + _sum4 = _mm256_comp_fmadd_ps(_w5, _val45, _sum4); + _sum4 = _mm256_comp_fmadd_ps(_w6, _val46, _sum4); + _sum4 = _mm256_comp_fmadd_ps(_w7, _val47, _sum4); + _sum5 = _mm256_comp_fmadd_ps(_w0, _val50, _sum5); + _sum5 = _mm256_comp_fmadd_ps(_w1, _val51, _sum5); + _sum5 = _mm256_comp_fmadd_ps(_w2, _val52, _sum5); + _sum5 = _mm256_comp_fmadd_ps(_w3, _val53, _sum5); + _sum5 = _mm256_comp_fmadd_ps(_w4, _val54, _sum5); + _sum5 = _mm256_comp_fmadd_ps(_w5, _val55, _sum5); + _sum5 = _mm256_comp_fmadd_ps(_w6, _val56, _sum5); + _sum5 = _mm256_comp_fmadd_ps(_w7, _val57, _sum5); __m256 _val60 = _mm256_broadcast_ss(tmpptr + 48); __m256 _val61 = _mm256_broadcast_ss(tmpptr + 49); @@ -756,22 +756,22 @@ static void conv1x1s1_sgemm_fp16_pack8_avx(const Mat& bottom_blob, Mat& top_blob __m256 _val76 = _mm256_broadcast_ss(tmpptr + 62); __m256 _val77 = _mm256_broadcast_ss(tmpptr + 63); - _sum6 = _mm256_fmadd_ps(_w0, _val60, _sum6); - _sum6 = _mm256_fmadd_ps(_w1, _val61, _sum6); - _sum6 = _mm256_fmadd_ps(_w2, _val62, _sum6); - _sum6 = _mm256_fmadd_ps(_w3, _val63, _sum6); - _sum6 = _mm256_fmadd_ps(_w4, _val64, _sum6); - _sum6 = _mm256_fmadd_ps(_w5, _val65, _sum6); - _sum6 = _mm256_fmadd_ps(_w6, _val66, _sum6); - _sum6 = _mm256_fmadd_ps(_w7, _val67, _sum6); - _sum7 = _mm256_fmadd_ps(_w0, _val70, _sum7); - _sum7 = _mm256_fmadd_ps(_w1, _val71, _sum7); - _sum7 = _mm256_fmadd_ps(_w2, _val72, _sum7); - _sum7 = _mm256_fmadd_ps(_w3, _val73, _sum7); - _sum7 = _mm256_fmadd_ps(_w4, _val74, _sum7); - _sum7 = _mm256_fmadd_ps(_w5, _val75, _sum7); - _sum7 = _mm256_fmadd_ps(_w6, _val76, _sum7); - _sum7 = _mm256_fmadd_ps(_w7, _val77, _sum7); + _sum6 = _mm256_comp_fmadd_ps(_w0, _val60, _sum6); + _sum6 = _mm256_comp_fmadd_ps(_w1, _val61, _sum6); + _sum6 = _mm256_comp_fmadd_ps(_w2, _val62, _sum6); + _sum6 = _mm256_comp_fmadd_ps(_w3, _val63, _sum6); + _sum6 = _mm256_comp_fmadd_ps(_w4, _val64, _sum6); + _sum6 = _mm256_comp_fmadd_ps(_w5, _val65, _sum6); + _sum6 = _mm256_comp_fmadd_ps(_w6, _val66, _sum6); + _sum6 = _mm256_comp_fmadd_ps(_w7, _val67, _sum6); + _sum7 = _mm256_comp_fmadd_ps(_w0, _val70, _sum7); + _sum7 = _mm256_comp_fmadd_ps(_w1, _val71, _sum7); + _sum7 = _mm256_comp_fmadd_ps(_w2, _val72, _sum7); + _sum7 = _mm256_comp_fmadd_ps(_w3, _val73, _sum7); + _sum7 = _mm256_comp_fmadd_ps(_w4, _val74, _sum7); + _sum7 = _mm256_comp_fmadd_ps(_w5, _val75, _sum7); + _sum7 = _mm256_comp_fmadd_ps(_w6, _val76, _sum7); + _sum7 = _mm256_comp_fmadd_ps(_w7, _val77, _sum7); tmpptr += 64; @@ -826,22 +826,22 @@ static void conv1x1s1_sgemm_fp16_pack8_avx(const Mat& bottom_blob, Mat& top_blob __m256 _val16 = _mm256_broadcast_ss(tmpptr + 14); __m256 _val17 = _mm256_broadcast_ss(tmpptr + 15); - _sum0 = _mm256_fmadd_ps(_w0, _val00, _sum0); - _sum0 = _mm256_fmadd_ps(_w1, _val01, _sum0); - _sum0 = _mm256_fmadd_ps(_w2, _val02, _sum0); - _sum0 = _mm256_fmadd_ps(_w3, _val03, _sum0); - _sum0 = _mm256_fmadd_ps(_w4, _val04, _sum0); - _sum0 = _mm256_fmadd_ps(_w5, _val05, _sum0); - _sum0 = _mm256_fmadd_ps(_w6, _val06, _sum0); - _sum0 = _mm256_fmadd_ps(_w7, _val07, _sum0); - _sum1 = _mm256_fmadd_ps(_w0, _val10, _sum1); - _sum1 = _mm256_fmadd_ps(_w1, _val11, _sum1); - _sum1 = _mm256_fmadd_ps(_w2, _val12, _sum1); - _sum1 = _mm256_fmadd_ps(_w3, _val13, _sum1); - _sum1 = _mm256_fmadd_ps(_w4, _val14, _sum1); - _sum1 = _mm256_fmadd_ps(_w5, _val15, _sum1); - _sum1 = _mm256_fmadd_ps(_w6, _val16, _sum1); - _sum1 = _mm256_fmadd_ps(_w7, _val17, _sum1); + _sum0 = _mm256_comp_fmadd_ps(_w0, _val00, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_w1, _val01, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_w2, _val02, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_w3, _val03, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_w4, _val04, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_w5, _val05, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_w6, _val06, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_w7, _val07, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_w0, _val10, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_w1, _val11, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_w2, _val12, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_w3, _val13, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_w4, _val14, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_w5, _val15, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_w6, _val16, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_w7, _val17, _sum1); __m256 _val20 = _mm256_broadcast_ss(tmpptr + 16); __m256 _val21 = _mm256_broadcast_ss(tmpptr + 17); @@ -860,22 +860,22 @@ static void conv1x1s1_sgemm_fp16_pack8_avx(const Mat& bottom_blob, Mat& top_blob __m256 _val36 = _mm256_broadcast_ss(tmpptr + 30); __m256 _val37 = _mm256_broadcast_ss(tmpptr + 31); - _sum2 = _mm256_fmadd_ps(_w0, _val20, _sum2); - _sum2 = _mm256_fmadd_ps(_w1, _val21, _sum2); - _sum2 = _mm256_fmadd_ps(_w2, _val22, _sum2); - _sum2 = _mm256_fmadd_ps(_w3, _val23, _sum2); - _sum2 = _mm256_fmadd_ps(_w4, _val24, _sum2); - _sum2 = _mm256_fmadd_ps(_w5, _val25, _sum2); - _sum2 = _mm256_fmadd_ps(_w6, _val26, _sum2); - _sum2 = _mm256_fmadd_ps(_w7, _val27, _sum2); - _sum3 = _mm256_fmadd_ps(_w0, _val30, _sum3); - _sum3 = _mm256_fmadd_ps(_w1, _val31, _sum3); - _sum3 = _mm256_fmadd_ps(_w2, _val32, _sum3); - _sum3 = _mm256_fmadd_ps(_w3, _val33, _sum3); - _sum3 = _mm256_fmadd_ps(_w4, _val34, _sum3); - _sum3 = _mm256_fmadd_ps(_w5, _val35, _sum3); - _sum3 = _mm256_fmadd_ps(_w6, _val36, _sum3); - _sum3 = _mm256_fmadd_ps(_w7, _val37, _sum3); + _sum2 = _mm256_comp_fmadd_ps(_w0, _val20, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_w1, _val21, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_w2, _val22, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_w3, _val23, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_w4, _val24, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_w5, _val25, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_w6, _val26, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_w7, _val27, _sum2); + _sum3 = _mm256_comp_fmadd_ps(_w0, _val30, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_w1, _val31, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_w2, _val32, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_w3, _val33, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_w4, _val34, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_w5, _val35, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_w6, _val36, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_w7, _val37, _sum3); tmpptr += 32; @@ -924,22 +924,22 @@ static void conv1x1s1_sgemm_fp16_pack8_avx(const Mat& bottom_blob, Mat& top_blob __m256 _w6 = loadfp16(kptr + 48); __m256 _w7 = loadfp16(kptr + 56); - _sum0 = _mm256_fmadd_ps(_w0, _val00, _sum0); - _sum0 = _mm256_fmadd_ps(_w1, _val01, _sum0); - _sum0 = _mm256_fmadd_ps(_w2, _val02, _sum0); - _sum0 = _mm256_fmadd_ps(_w3, _val03, _sum0); - _sum0 = _mm256_fmadd_ps(_w4, _val04, _sum0); - _sum0 = _mm256_fmadd_ps(_w5, _val05, _sum0); - _sum0 = _mm256_fmadd_ps(_w6, _val06, _sum0); - _sum0 = _mm256_fmadd_ps(_w7, _val07, _sum0); - _sum1 = _mm256_fmadd_ps(_w0, _val10, _sum1); - _sum1 = _mm256_fmadd_ps(_w1, _val11, _sum1); - _sum1 = _mm256_fmadd_ps(_w2, _val12, _sum1); - _sum1 = _mm256_fmadd_ps(_w3, _val13, _sum1); - _sum1 = _mm256_fmadd_ps(_w4, _val14, _sum1); - _sum1 = _mm256_fmadd_ps(_w5, _val15, _sum1); - _sum1 = _mm256_fmadd_ps(_w6, _val16, _sum1); - _sum1 = _mm256_fmadd_ps(_w7, _val17, _sum1); + _sum0 = _mm256_comp_fmadd_ps(_w0, _val00, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_w1, _val01, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_w2, _val02, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_w3, _val03, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_w4, _val04, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_w5, _val05, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_w6, _val06, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_w7, _val07, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_w0, _val10, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_w1, _val11, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_w2, _val12, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_w3, _val13, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_w4, _val14, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_w5, _val15, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_w6, _val16, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_w7, _val17, _sum1); tmpptr += 16; @@ -977,14 +977,14 @@ static void conv1x1s1_sgemm_fp16_pack8_avx(const Mat& bottom_blob, Mat& top_blob __m256 _w6 = loadfp16(kptr + 48); __m256 _w7 = loadfp16(kptr + 56); - _sum = _mm256_fmadd_ps(_w0, _val0, _sum); - _sum = _mm256_fmadd_ps(_w1, _val1, _sum); - _sum = _mm256_fmadd_ps(_w2, _val2, _sum); - _sum = _mm256_fmadd_ps(_w3, _val3, _sum); - _sum = _mm256_fmadd_ps(_w4, _val4, _sum); - _sum = _mm256_fmadd_ps(_w5, _val5, _sum); - _sum = _mm256_fmadd_ps(_w6, _val6, _sum); - _sum = _mm256_fmadd_ps(_w7, _val7, _sum); + _sum = _mm256_comp_fmadd_ps(_w0, _val0, _sum); + _sum = _mm256_comp_fmadd_ps(_w1, _val1, _sum); + _sum = _mm256_comp_fmadd_ps(_w2, _val2, _sum); + _sum = _mm256_comp_fmadd_ps(_w3, _val3, _sum); + _sum = _mm256_comp_fmadd_ps(_w4, _val4, _sum); + _sum = _mm256_comp_fmadd_ps(_w5, _val5, _sum); + _sum = _mm256_comp_fmadd_ps(_w6, _val6, _sum); + _sum = _mm256_comp_fmadd_ps(_w7, _val7, _sum); tmpptr += 8; diff --git a/src/layer/x86/convolution_2x2_pack8.h b/src/layer/x86/convolution_2x2_pack8.h index 79e6716ac..24812b255 100644 --- a/src/layer/x86/convolution_2x2_pack8.h +++ b/src/layer/x86/convolution_2x2_pack8.h @@ -66,10 +66,10 @@ static void conv2x2s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const Mat __m256 _k03 = _mm256_loadu_ps(kptr + 24); kptr += 32; - _sum0 = _mm256_fmadd_ps(_k00, _r00, _sum0); - _sum0 = _mm256_fmadd_ps(_k01, _r01, _sum0); - _sum0 = _mm256_fmadd_ps(_k02, _r02, _sum0); - _sum0 = _mm256_fmadd_ps(_k03, _r03, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k00, _r00, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k01, _r01, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k02, _r02, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k03, _r03, _sum0); __m256 _k04 = _mm256_loadu_ps(kptr); __m256 _k05 = _mm256_loadu_ps(kptr + 8); @@ -77,10 +77,10 @@ static void conv2x2s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const Mat __m256 _k07 = _mm256_loadu_ps(kptr + 24); kptr += 32; - _sum0 = _mm256_fmadd_ps(_k04, _r04, _sum0); - _sum0 = _mm256_fmadd_ps(_k05, _r05, _sum0); - _sum0 = _mm256_fmadd_ps(_k06, _r06, _sum0); - _sum0 = _mm256_fmadd_ps(_k07, _r07, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k04, _r04, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k05, _r05, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k06, _r06, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k07, _r07, _sum0); //======================================== @@ -94,14 +94,14 @@ static void conv2x2s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const Mat _r07 = _mm256_broadcast_ss(r0 + 7); r0 += 8; - _sum1 = _mm256_fmadd_ps(_k00, _r00, _sum1); - _sum1 = _mm256_fmadd_ps(_k01, _r01, _sum1); - _sum1 = _mm256_fmadd_ps(_k02, _r02, _sum1); - _sum1 = _mm256_fmadd_ps(_k03, _r03, _sum1); - _sum1 = _mm256_fmadd_ps(_k04, _r04, _sum1); - _sum1 = _mm256_fmadd_ps(_k05, _r05, _sum1); - _sum1 = _mm256_fmadd_ps(_k06, _r06, _sum1); - _sum1 = _mm256_fmadd_ps(_k07, _r07, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k00, _r00, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k01, _r01, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k02, _r02, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k03, _r03, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k04, _r04, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k05, _r05, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k06, _r06, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k07, _r07, _sum1); _k00 = _mm256_loadu_ps(kptr); _k01 = _mm256_loadu_ps(kptr + 8); @@ -109,10 +109,10 @@ static void conv2x2s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const Mat _k03 = _mm256_loadu_ps(kptr + 24); kptr += 32; - _sum0 = _mm256_fmadd_ps(_k00, _r00, _sum0); - _sum0 = _mm256_fmadd_ps(_k01, _r01, _sum0); - _sum0 = _mm256_fmadd_ps(_k02, _r02, _sum0); - _sum0 = _mm256_fmadd_ps(_k03, _r03, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k00, _r00, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k01, _r01, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k02, _r02, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k03, _r03, _sum0); _k04 = _mm256_loadu_ps(kptr); _k05 = _mm256_loadu_ps(kptr + 8); @@ -120,10 +120,10 @@ static void conv2x2s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const Mat _k07 = _mm256_loadu_ps(kptr + 24); kptr += 32; - _sum0 = _mm256_fmadd_ps(_k04, _r04, _sum0); - _sum0 = _mm256_fmadd_ps(_k05, _r05, _sum0); - _sum0 = _mm256_fmadd_ps(_k06, _r06, _sum0); - _sum0 = _mm256_fmadd_ps(_k07, _r07, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k04, _r04, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k05, _r05, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k06, _r06, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k07, _r07, _sum0); _r00 = _mm256_broadcast_ss(r0); _r01 = _mm256_broadcast_ss(r0 + 1); @@ -134,14 +134,14 @@ static void conv2x2s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const Mat _r06 = _mm256_broadcast_ss(r0 + 6); _r07 = _mm256_broadcast_ss(r0 + 7); - _sum1 = _mm256_fmadd_ps(_k00, _r00, _sum1); - _sum1 = _mm256_fmadd_ps(_k01, _r01, _sum1); - _sum1 = _mm256_fmadd_ps(_k02, _r02, _sum1); - _sum1 = _mm256_fmadd_ps(_k03, _r03, _sum1); - _sum1 = _mm256_fmadd_ps(_k04, _r04, _sum1); - _sum1 = _mm256_fmadd_ps(_k05, _r05, _sum1); - _sum1 = _mm256_fmadd_ps(_k06, _r06, _sum1); - _sum1 = _mm256_fmadd_ps(_k07, _r07, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k00, _r00, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k01, _r01, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k02, _r02, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k03, _r03, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k04, _r04, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k05, _r05, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k06, _r06, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k07, _r07, _sum1); //=============== __m256 _r10 = _mm256_broadcast_ss(r1); @@ -159,10 +159,10 @@ static void conv2x2s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const Mat __m256 _k13 = _mm256_loadu_ps(kptr + 24); kptr += 32; - _sum0 = _mm256_fmadd_ps(_k10, _r10, _sum0); - _sum0 = _mm256_fmadd_ps(_k11, _r11, _sum0); - _sum0 = _mm256_fmadd_ps(_k12, _r12, _sum0); - _sum0 = _mm256_fmadd_ps(_k13, _r13, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k10, _r10, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k11, _r11, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k12, _r12, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k13, _r13, _sum0); __m256 _k14 = _mm256_loadu_ps(kptr); __m256 _k15 = _mm256_loadu_ps(kptr + 8); @@ -170,10 +170,10 @@ static void conv2x2s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const Mat __m256 _k17 = _mm256_loadu_ps(kptr + 24); kptr += 32; - _sum0 = _mm256_fmadd_ps(_k14, _r14, _sum0); - _sum0 = _mm256_fmadd_ps(_k15, _r15, _sum0); - _sum0 = _mm256_fmadd_ps(_k16, _r16, _sum0); - _sum0 = _mm256_fmadd_ps(_k17, _r17, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k14, _r14, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k15, _r15, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k16, _r16, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k17, _r17, _sum0); //======================================= r1 += 8; @@ -186,14 +186,14 @@ static void conv2x2s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const Mat _r16 = _mm256_broadcast_ss(r1 + 6); _r17 = _mm256_broadcast_ss(r1 + 7); - _sum1 = _mm256_fmadd_ps(_k10, _r10, _sum1); - _sum1 = _mm256_fmadd_ps(_k11, _r11, _sum1); - _sum1 = _mm256_fmadd_ps(_k12, _r12, _sum1); - _sum1 = _mm256_fmadd_ps(_k13, _r13, _sum1); - _sum1 = _mm256_fmadd_ps(_k14, _r14, _sum1); - _sum1 = _mm256_fmadd_ps(_k15, _r15, _sum1); - _sum1 = _mm256_fmadd_ps(_k16, _r16, _sum1); - _sum1 = _mm256_fmadd_ps(_k17, _r17, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k10, _r10, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k11, _r11, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k12, _r12, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k13, _r13, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k14, _r14, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k15, _r15, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k16, _r16, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k17, _r17, _sum1); _k10 = _mm256_loadu_ps(kptr); _k11 = _mm256_loadu_ps(kptr + 8); @@ -201,19 +201,19 @@ static void conv2x2s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const Mat _k13 = _mm256_loadu_ps(kptr + 24); kptr += 32; - _sum0 = _mm256_fmadd_ps(_k10, _r10, _sum0); - _sum0 = _mm256_fmadd_ps(_k11, _r11, _sum0); - _sum0 = _mm256_fmadd_ps(_k12, _r12, _sum0); - _sum0 = _mm256_fmadd_ps(_k13, _r13, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k10, _r10, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k11, _r11, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k12, _r12, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k13, _r13, _sum0); _k14 = _mm256_loadu_ps(kptr); _k15 = _mm256_loadu_ps(kptr + 8); _k16 = _mm256_loadu_ps(kptr + 16); _k17 = _mm256_loadu_ps(kptr + 24); - _sum0 = _mm256_fmadd_ps(_k14, _r14, _sum0); - _sum0 = _mm256_fmadd_ps(_k15, _r15, _sum0); - _sum0 = _mm256_fmadd_ps(_k16, _r16, _sum0); - _sum0 = _mm256_fmadd_ps(_k17, _r17, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k14, _r14, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k15, _r15, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k16, _r16, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k17, _r17, _sum0); r1 += 8; _r10 = _mm256_broadcast_ss(r1); @@ -225,14 +225,14 @@ static void conv2x2s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const Mat _r16 = _mm256_broadcast_ss(r1 + 6); _r17 = _mm256_broadcast_ss(r1 + 7); - _sum1 = _mm256_fmadd_ps(_k10, _r10, _sum1); - _sum1 = _mm256_fmadd_ps(_k11, _r11, _sum1); - _sum1 = _mm256_fmadd_ps(_k12, _r12, _sum1); - _sum1 = _mm256_fmadd_ps(_k13, _r13, _sum1); - _sum1 = _mm256_fmadd_ps(_k14, _r14, _sum1); - _sum1 = _mm256_fmadd_ps(_k15, _r15, _sum1); - _sum1 = _mm256_fmadd_ps(_k16, _r16, _sum1); - _sum1 = _mm256_fmadd_ps(_k17, _r17, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k10, _r10, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k11, _r11, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k12, _r12, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k13, _r13, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k14, _r14, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k15, _r15, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k16, _r16, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k17, _r17, _sum1); kptr -= 224; _mm256_storeu_ps(outptr0, _sum0); @@ -259,10 +259,10 @@ static void conv2x2s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const Mat __m256 _k03 = _mm256_loadu_ps(kptr + 24); kptr += 32; - _sum = _mm256_fmadd_ps(_k00, _r00, _sum); - _sum = _mm256_fmadd_ps(_k01, _r01, _sum); - _sum = _mm256_fmadd_ps(_k02, _r02, _sum); - _sum = _mm256_fmadd_ps(_k03, _r03, _sum); + _sum = _mm256_comp_fmadd_ps(_k00, _r00, _sum); + _sum = _mm256_comp_fmadd_ps(_k01, _r01, _sum); + _sum = _mm256_comp_fmadd_ps(_k02, _r02, _sum); + _sum = _mm256_comp_fmadd_ps(_k03, _r03, _sum); __m256 _k04 = _mm256_loadu_ps(kptr); __m256 _k05 = _mm256_loadu_ps(kptr + 8); @@ -270,10 +270,10 @@ static void conv2x2s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const Mat __m256 _k07 = _mm256_loadu_ps(kptr + 24); kptr += 32; - _sum = _mm256_fmadd_ps(_k04, _r04, _sum); - _sum = _mm256_fmadd_ps(_k05, _r05, _sum); - _sum = _mm256_fmadd_ps(_k06, _r06, _sum); - _sum = _mm256_fmadd_ps(_k07, _r07, _sum); + _sum = _mm256_comp_fmadd_ps(_k04, _r04, _sum); + _sum = _mm256_comp_fmadd_ps(_k05, _r05, _sum); + _sum = _mm256_comp_fmadd_ps(_k06, _r06, _sum); + _sum = _mm256_comp_fmadd_ps(_k07, _r07, _sum); //======================================== r0 += 8; @@ -292,10 +292,10 @@ static void conv2x2s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const Mat _k03 = _mm256_loadu_ps(kptr + 24); kptr += 32; - _sum = _mm256_fmadd_ps(_k00, _r00, _sum); - _sum = _mm256_fmadd_ps(_k01, _r01, _sum); - _sum = _mm256_fmadd_ps(_k02, _r02, _sum); - _sum = _mm256_fmadd_ps(_k03, _r03, _sum); + _sum = _mm256_comp_fmadd_ps(_k00, _r00, _sum); + _sum = _mm256_comp_fmadd_ps(_k01, _r01, _sum); + _sum = _mm256_comp_fmadd_ps(_k02, _r02, _sum); + _sum = _mm256_comp_fmadd_ps(_k03, _r03, _sum); _k04 = _mm256_loadu_ps(kptr); _k05 = _mm256_loadu_ps(kptr + 8); @@ -303,10 +303,10 @@ static void conv2x2s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const Mat _k07 = _mm256_loadu_ps(kptr + 24); kptr += 32; - _sum = _mm256_fmadd_ps(_k04, _r04, _sum); - _sum = _mm256_fmadd_ps(_k05, _r05, _sum); - _sum = _mm256_fmadd_ps(_k06, _r06, _sum); - _sum = _mm256_fmadd_ps(_k07, _r07, _sum); + _sum = _mm256_comp_fmadd_ps(_k04, _r04, _sum); + _sum = _mm256_comp_fmadd_ps(_k05, _r05, _sum); + _sum = _mm256_comp_fmadd_ps(_k06, _r06, _sum); + _sum = _mm256_comp_fmadd_ps(_k07, _r07, _sum); //=============== __m256 _r10 = _mm256_broadcast_ss(r1); @@ -324,10 +324,10 @@ static void conv2x2s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const Mat __m256 _k13 = _mm256_loadu_ps(kptr + 24); kptr += 32; - _sum = _mm256_fmadd_ps(_k10, _r10, _sum); - _sum = _mm256_fmadd_ps(_k11, _r11, _sum); - _sum = _mm256_fmadd_ps(_k12, _r12, _sum); - _sum = _mm256_fmadd_ps(_k13, _r13, _sum); + _sum = _mm256_comp_fmadd_ps(_k10, _r10, _sum); + _sum = _mm256_comp_fmadd_ps(_k11, _r11, _sum); + _sum = _mm256_comp_fmadd_ps(_k12, _r12, _sum); + _sum = _mm256_comp_fmadd_ps(_k13, _r13, _sum); __m256 _k14 = _mm256_loadu_ps(kptr); __m256 _k15 = _mm256_loadu_ps(kptr + 8); @@ -335,10 +335,10 @@ static void conv2x2s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const Mat __m256 _k17 = _mm256_loadu_ps(kptr + 24); kptr += 32; - _sum = _mm256_fmadd_ps(_k14, _r14, _sum); - _sum = _mm256_fmadd_ps(_k15, _r15, _sum); - _sum = _mm256_fmadd_ps(_k16, _r16, _sum); - _sum = _mm256_fmadd_ps(_k17, _r17, _sum); + _sum = _mm256_comp_fmadd_ps(_k14, _r14, _sum); + _sum = _mm256_comp_fmadd_ps(_k15, _r15, _sum); + _sum = _mm256_comp_fmadd_ps(_k16, _r16, _sum); + _sum = _mm256_comp_fmadd_ps(_k17, _r17, _sum); //======================================= r1 += 8; @@ -357,19 +357,19 @@ static void conv2x2s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const Mat _k13 = _mm256_loadu_ps(kptr + 24); kptr += 32; - _sum = _mm256_fmadd_ps(_k10, _r10, _sum); - _sum = _mm256_fmadd_ps(_k11, _r11, _sum); - _sum = _mm256_fmadd_ps(_k12, _r12, _sum); - _sum = _mm256_fmadd_ps(_k13, _r13, _sum); + _sum = _mm256_comp_fmadd_ps(_k10, _r10, _sum); + _sum = _mm256_comp_fmadd_ps(_k11, _r11, _sum); + _sum = _mm256_comp_fmadd_ps(_k12, _r12, _sum); + _sum = _mm256_comp_fmadd_ps(_k13, _r13, _sum); _k14 = _mm256_loadu_ps(kptr); _k15 = _mm256_loadu_ps(kptr + 8); _k16 = _mm256_loadu_ps(kptr + 16); _k17 = _mm256_loadu_ps(kptr + 24); - _sum = _mm256_fmadd_ps(_k14, _r14, _sum); - _sum = _mm256_fmadd_ps(_k15, _r15, _sum); - _sum = _mm256_fmadd_ps(_k16, _r16, _sum); - _sum = _mm256_fmadd_ps(_k17, _r17, _sum); + _sum = _mm256_comp_fmadd_ps(_k14, _r14, _sum); + _sum = _mm256_comp_fmadd_ps(_k15, _r15, _sum); + _sum = _mm256_comp_fmadd_ps(_k16, _r16, _sum); + _sum = _mm256_comp_fmadd_ps(_k17, _r17, _sum); kptr -= 224; _mm256_storeu_ps(outptr0, _sum); diff --git a/src/layer/x86/convolution_2x2_pack8_fp16.h b/src/layer/x86/convolution_2x2_pack8_fp16.h index 68bbcfe02..99eca3a47 100644 --- a/src/layer/x86/convolution_2x2_pack8_fp16.h +++ b/src/layer/x86/convolution_2x2_pack8_fp16.h @@ -247,10 +247,10 @@ static void conv2x2s1_fp16_pack8_avx(const Mat& bottom_blob, Mat& top_blob, cons __m256 _k03 = loadfp16(kptr + 24); kptr += 32; - _sum0 = _mm256_fmadd_ps(_k00, _r00, _sum0); - _sum0 = _mm256_fmadd_ps(_k01, _r01, _sum0); - _sum0 = _mm256_fmadd_ps(_k02, _r02, _sum0); - _sum0 = _mm256_fmadd_ps(_k03, _r03, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k00, _r00, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k01, _r01, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k02, _r02, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k03, _r03, _sum0); __m256 _k04 = loadfp16(kptr); __m256 _k05 = loadfp16(kptr + 8); @@ -258,10 +258,10 @@ static void conv2x2s1_fp16_pack8_avx(const Mat& bottom_blob, Mat& top_blob, cons __m256 _k07 = loadfp16(kptr + 24); kptr += 32; - _sum0 = _mm256_fmadd_ps(_k04, _r04, _sum0); - _sum0 = _mm256_fmadd_ps(_k05, _r05, _sum0); - _sum0 = _mm256_fmadd_ps(_k06, _r06, _sum0); - _sum0 = _mm256_fmadd_ps(_k07, _r07, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k04, _r04, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k05, _r05, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k06, _r06, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k07, _r07, _sum0); //======================================== @@ -275,14 +275,14 @@ static void conv2x2s1_fp16_pack8_avx(const Mat& bottom_blob, Mat& top_blob, cons _r07 = _mm256_broadcast_ss(r0 + 7); r0 += 8; - _sum1 = _mm256_fmadd_ps(_k00, _r00, _sum1); - _sum1 = _mm256_fmadd_ps(_k01, _r01, _sum1); - _sum1 = _mm256_fmadd_ps(_k02, _r02, _sum1); - _sum1 = _mm256_fmadd_ps(_k03, _r03, _sum1); - _sum1 = _mm256_fmadd_ps(_k04, _r04, _sum1); - _sum1 = _mm256_fmadd_ps(_k05, _r05, _sum1); - _sum1 = _mm256_fmadd_ps(_k06, _r06, _sum1); - _sum1 = _mm256_fmadd_ps(_k07, _r07, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k00, _r00, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k01, _r01, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k02, _r02, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k03, _r03, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k04, _r04, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k05, _r05, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k06, _r06, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k07, _r07, _sum1); _k00 = loadfp16(kptr); _k01 = loadfp16(kptr + 8); @@ -290,10 +290,10 @@ static void conv2x2s1_fp16_pack8_avx(const Mat& bottom_blob, Mat& top_blob, cons _k03 = loadfp16(kptr + 24); kptr += 32; - _sum0 = _mm256_fmadd_ps(_k00, _r00, _sum0); - _sum0 = _mm256_fmadd_ps(_k01, _r01, _sum0); - _sum0 = _mm256_fmadd_ps(_k02, _r02, _sum0); - _sum0 = _mm256_fmadd_ps(_k03, _r03, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k00, _r00, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k01, _r01, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k02, _r02, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k03, _r03, _sum0); _k04 = loadfp16(kptr); _k05 = loadfp16(kptr + 8); @@ -301,10 +301,10 @@ static void conv2x2s1_fp16_pack8_avx(const Mat& bottom_blob, Mat& top_blob, cons _k07 = loadfp16(kptr + 24); kptr += 32; - _sum0 = _mm256_fmadd_ps(_k04, _r04, _sum0); - _sum0 = _mm256_fmadd_ps(_k05, _r05, _sum0); - _sum0 = _mm256_fmadd_ps(_k06, _r06, _sum0); - _sum0 = _mm256_fmadd_ps(_k07, _r07, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k04, _r04, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k05, _r05, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k06, _r06, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k07, _r07, _sum0); _r00 = _mm256_broadcast_ss(r0); _r01 = _mm256_broadcast_ss(r0 + 1); @@ -315,14 +315,14 @@ static void conv2x2s1_fp16_pack8_avx(const Mat& bottom_blob, Mat& top_blob, cons _r06 = _mm256_broadcast_ss(r0 + 6); _r07 = _mm256_broadcast_ss(r0 + 7); - _sum1 = _mm256_fmadd_ps(_k00, _r00, _sum1); - _sum1 = _mm256_fmadd_ps(_k01, _r01, _sum1); - _sum1 = _mm256_fmadd_ps(_k02, _r02, _sum1); - _sum1 = _mm256_fmadd_ps(_k03, _r03, _sum1); - _sum1 = _mm256_fmadd_ps(_k04, _r04, _sum1); - _sum1 = _mm256_fmadd_ps(_k05, _r05, _sum1); - _sum1 = _mm256_fmadd_ps(_k06, _r06, _sum1); - _sum1 = _mm256_fmadd_ps(_k07, _r07, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k00, _r00, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k01, _r01, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k02, _r02, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k03, _r03, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k04, _r04, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k05, _r05, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k06, _r06, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k07, _r07, _sum1); //=============== __m256 _r10 = _mm256_broadcast_ss(r1); @@ -340,10 +340,10 @@ static void conv2x2s1_fp16_pack8_avx(const Mat& bottom_blob, Mat& top_blob, cons __m256 _k13 = loadfp16(kptr + 24); kptr += 32; - _sum0 = _mm256_fmadd_ps(_k10, _r10, _sum0); - _sum0 = _mm256_fmadd_ps(_k11, _r11, _sum0); - _sum0 = _mm256_fmadd_ps(_k12, _r12, _sum0); - _sum0 = _mm256_fmadd_ps(_k13, _r13, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k10, _r10, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k11, _r11, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k12, _r12, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k13, _r13, _sum0); __m256 _k14 = loadfp16(kptr); __m256 _k15 = loadfp16(kptr + 8); @@ -351,10 +351,10 @@ static void conv2x2s1_fp16_pack8_avx(const Mat& bottom_blob, Mat& top_blob, cons __m256 _k17 = loadfp16(kptr + 24); kptr += 32; - _sum0 = _mm256_fmadd_ps(_k14, _r14, _sum0); - _sum0 = _mm256_fmadd_ps(_k15, _r15, _sum0); - _sum0 = _mm256_fmadd_ps(_k16, _r16, _sum0); - _sum0 = _mm256_fmadd_ps(_k17, _r17, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k14, _r14, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k15, _r15, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k16, _r16, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k17, _r17, _sum0); //======================================= r1 += 8; @@ -367,14 +367,14 @@ static void conv2x2s1_fp16_pack8_avx(const Mat& bottom_blob, Mat& top_blob, cons _r16 = _mm256_broadcast_ss(r1 + 6); _r17 = _mm256_broadcast_ss(r1 + 7); - _sum1 = _mm256_fmadd_ps(_k10, _r10, _sum1); - _sum1 = _mm256_fmadd_ps(_k11, _r11, _sum1); - _sum1 = _mm256_fmadd_ps(_k12, _r12, _sum1); - _sum1 = _mm256_fmadd_ps(_k13, _r13, _sum1); - _sum1 = _mm256_fmadd_ps(_k14, _r14, _sum1); - _sum1 = _mm256_fmadd_ps(_k15, _r15, _sum1); - _sum1 = _mm256_fmadd_ps(_k16, _r16, _sum1); - _sum1 = _mm256_fmadd_ps(_k17, _r17, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k10, _r10, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k11, _r11, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k12, _r12, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k13, _r13, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k14, _r14, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k15, _r15, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k16, _r16, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k17, _r17, _sum1); _k10 = loadfp16(kptr); _k11 = loadfp16(kptr + 8); @@ -382,19 +382,19 @@ static void conv2x2s1_fp16_pack8_avx(const Mat& bottom_blob, Mat& top_blob, cons _k13 = loadfp16(kptr + 24); kptr += 32; - _sum0 = _mm256_fmadd_ps(_k10, _r10, _sum0); - _sum0 = _mm256_fmadd_ps(_k11, _r11, _sum0); - _sum0 = _mm256_fmadd_ps(_k12, _r12, _sum0); - _sum0 = _mm256_fmadd_ps(_k13, _r13, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k10, _r10, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k11, _r11, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k12, _r12, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k13, _r13, _sum0); _k14 = loadfp16(kptr); _k15 = loadfp16(kptr + 8); _k16 = loadfp16(kptr + 16); _k17 = loadfp16(kptr + 24); - _sum0 = _mm256_fmadd_ps(_k14, _r14, _sum0); - _sum0 = _mm256_fmadd_ps(_k15, _r15, _sum0); - _sum0 = _mm256_fmadd_ps(_k16, _r16, _sum0); - _sum0 = _mm256_fmadd_ps(_k17, _r17, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k14, _r14, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k15, _r15, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k16, _r16, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k17, _r17, _sum0); r1 += 8; _r10 = _mm256_broadcast_ss(r1); @@ -406,14 +406,14 @@ static void conv2x2s1_fp16_pack8_avx(const Mat& bottom_blob, Mat& top_blob, cons _r16 = _mm256_broadcast_ss(r1 + 6); _r17 = _mm256_broadcast_ss(r1 + 7); - _sum1 = _mm256_fmadd_ps(_k10, _r10, _sum1); - _sum1 = _mm256_fmadd_ps(_k11, _r11, _sum1); - _sum1 = _mm256_fmadd_ps(_k12, _r12, _sum1); - _sum1 = _mm256_fmadd_ps(_k13, _r13, _sum1); - _sum1 = _mm256_fmadd_ps(_k14, _r14, _sum1); - _sum1 = _mm256_fmadd_ps(_k15, _r15, _sum1); - _sum1 = _mm256_fmadd_ps(_k16, _r16, _sum1); - _sum1 = _mm256_fmadd_ps(_k17, _r17, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k10, _r10, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k11, _r11, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k12, _r12, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k13, _r13, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k14, _r14, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k15, _r15, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k16, _r16, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k17, _r17, _sum1); kptr -= 224; _mm256_storeu_ps(outptr0, _sum0); @@ -440,10 +440,10 @@ static void conv2x2s1_fp16_pack8_avx(const Mat& bottom_blob, Mat& top_blob, cons __m256 _k03 = loadfp16(kptr + 24); kptr += 32; - _sum = _mm256_fmadd_ps(_k00, _r00, _sum); - _sum = _mm256_fmadd_ps(_k01, _r01, _sum); - _sum = _mm256_fmadd_ps(_k02, _r02, _sum); - _sum = _mm256_fmadd_ps(_k03, _r03, _sum); + _sum = _mm256_comp_fmadd_ps(_k00, _r00, _sum); + _sum = _mm256_comp_fmadd_ps(_k01, _r01, _sum); + _sum = _mm256_comp_fmadd_ps(_k02, _r02, _sum); + _sum = _mm256_comp_fmadd_ps(_k03, _r03, _sum); __m256 _k04 = loadfp16(kptr); __m256 _k05 = loadfp16(kptr + 8); @@ -451,10 +451,10 @@ static void conv2x2s1_fp16_pack8_avx(const Mat& bottom_blob, Mat& top_blob, cons __m256 _k07 = loadfp16(kptr + 24); kptr += 32; - _sum = _mm256_fmadd_ps(_k04, _r04, _sum); - _sum = _mm256_fmadd_ps(_k05, _r05, _sum); - _sum = _mm256_fmadd_ps(_k06, _r06, _sum); - _sum = _mm256_fmadd_ps(_k07, _r07, _sum); + _sum = _mm256_comp_fmadd_ps(_k04, _r04, _sum); + _sum = _mm256_comp_fmadd_ps(_k05, _r05, _sum); + _sum = _mm256_comp_fmadd_ps(_k06, _r06, _sum); + _sum = _mm256_comp_fmadd_ps(_k07, _r07, _sum); //======================================== r0 += 8; @@ -473,10 +473,10 @@ static void conv2x2s1_fp16_pack8_avx(const Mat& bottom_blob, Mat& top_blob, cons _k03 = loadfp16(kptr + 24); kptr += 32; - _sum = _mm256_fmadd_ps(_k00, _r00, _sum); - _sum = _mm256_fmadd_ps(_k01, _r01, _sum); - _sum = _mm256_fmadd_ps(_k02, _r02, _sum); - _sum = _mm256_fmadd_ps(_k03, _r03, _sum); + _sum = _mm256_comp_fmadd_ps(_k00, _r00, _sum); + _sum = _mm256_comp_fmadd_ps(_k01, _r01, _sum); + _sum = _mm256_comp_fmadd_ps(_k02, _r02, _sum); + _sum = _mm256_comp_fmadd_ps(_k03, _r03, _sum); _k04 = loadfp16(kptr); _k05 = loadfp16(kptr + 8); @@ -484,10 +484,10 @@ static void conv2x2s1_fp16_pack8_avx(const Mat& bottom_blob, Mat& top_blob, cons _k07 = loadfp16(kptr + 24); kptr += 32; - _sum = _mm256_fmadd_ps(_k04, _r04, _sum); - _sum = _mm256_fmadd_ps(_k05, _r05, _sum); - _sum = _mm256_fmadd_ps(_k06, _r06, _sum); - _sum = _mm256_fmadd_ps(_k07, _r07, _sum); + _sum = _mm256_comp_fmadd_ps(_k04, _r04, _sum); + _sum = _mm256_comp_fmadd_ps(_k05, _r05, _sum); + _sum = _mm256_comp_fmadd_ps(_k06, _r06, _sum); + _sum = _mm256_comp_fmadd_ps(_k07, _r07, _sum); //=============== __m256 _r10 = _mm256_broadcast_ss(r1); @@ -505,10 +505,10 @@ static void conv2x2s1_fp16_pack8_avx(const Mat& bottom_blob, Mat& top_blob, cons __m256 _k13 = loadfp16(kptr + 24); kptr += 32; - _sum = _mm256_fmadd_ps(_k10, _r10, _sum); - _sum = _mm256_fmadd_ps(_k11, _r11, _sum); - _sum = _mm256_fmadd_ps(_k12, _r12, _sum); - _sum = _mm256_fmadd_ps(_k13, _r13, _sum); + _sum = _mm256_comp_fmadd_ps(_k10, _r10, _sum); + _sum = _mm256_comp_fmadd_ps(_k11, _r11, _sum); + _sum = _mm256_comp_fmadd_ps(_k12, _r12, _sum); + _sum = _mm256_comp_fmadd_ps(_k13, _r13, _sum); __m256 _k14 = loadfp16(kptr); __m256 _k15 = loadfp16(kptr + 8); @@ -516,10 +516,10 @@ static void conv2x2s1_fp16_pack8_avx(const Mat& bottom_blob, Mat& top_blob, cons __m256 _k17 = loadfp16(kptr + 24); kptr += 32; - _sum = _mm256_fmadd_ps(_k14, _r14, _sum); - _sum = _mm256_fmadd_ps(_k15, _r15, _sum); - _sum = _mm256_fmadd_ps(_k16, _r16, _sum); - _sum = _mm256_fmadd_ps(_k17, _r17, _sum); + _sum = _mm256_comp_fmadd_ps(_k14, _r14, _sum); + _sum = _mm256_comp_fmadd_ps(_k15, _r15, _sum); + _sum = _mm256_comp_fmadd_ps(_k16, _r16, _sum); + _sum = _mm256_comp_fmadd_ps(_k17, _r17, _sum); //======================================= r1 += 8; @@ -538,19 +538,19 @@ static void conv2x2s1_fp16_pack8_avx(const Mat& bottom_blob, Mat& top_blob, cons _k13 = loadfp16(kptr + 24); kptr += 32; - _sum = _mm256_fmadd_ps(_k10, _r10, _sum); - _sum = _mm256_fmadd_ps(_k11, _r11, _sum); - _sum = _mm256_fmadd_ps(_k12, _r12, _sum); - _sum = _mm256_fmadd_ps(_k13, _r13, _sum); + _sum = _mm256_comp_fmadd_ps(_k10, _r10, _sum); + _sum = _mm256_comp_fmadd_ps(_k11, _r11, _sum); + _sum = _mm256_comp_fmadd_ps(_k12, _r12, _sum); + _sum = _mm256_comp_fmadd_ps(_k13, _r13, _sum); _k14 = loadfp16(kptr); _k15 = loadfp16(kptr + 8); _k16 = loadfp16(kptr + 16); _k17 = loadfp16(kptr + 24); - _sum = _mm256_fmadd_ps(_k14, _r14, _sum); - _sum = _mm256_fmadd_ps(_k15, _r15, _sum); - _sum = _mm256_fmadd_ps(_k16, _r16, _sum); - _sum = _mm256_fmadd_ps(_k17, _r17, _sum); + _sum = _mm256_comp_fmadd_ps(_k14, _r14, _sum); + _sum = _mm256_comp_fmadd_ps(_k15, _r15, _sum); + _sum = _mm256_comp_fmadd_ps(_k16, _r16, _sum); + _sum = _mm256_comp_fmadd_ps(_k17, _r17, _sum); kptr -= 224; _mm256_storeu_ps(outptr0, _sum); diff --git a/src/layer/x86/convolution_3x3.h b/src/layer/x86/convolution_3x3.h index 07273acd9..7124e3521 100644 --- a/src/layer/x86/convolution_3x3.h +++ b/src/layer/x86/convolution_3x3.h @@ -417,14 +417,14 @@ static void conv3x3s1_winograd23_sse(const Mat& bottom_blob, Mat& top_blob, cons __m256 _k2n = _mm256_loadu_ps(k2 + 8); __m256 _k3 = _mm256_loadu_ps(k3); __m256 _k3n = _mm256_loadu_ps(k3 + 8); - _sum0 = _mm256_fmadd_ps(_r0, _k0, _sum0); - _sum0n = _mm256_fmadd_ps(_r0n, _k0n, _sum0n); - _sum1 = _mm256_fmadd_ps(_r0, _k1, _sum1); - _sum1n = _mm256_fmadd_ps(_r0n, _k1n, _sum1n); - _sum2 = _mm256_fmadd_ps(_r0, _k2, _sum2); - _sum2n = _mm256_fmadd_ps(_r0n, _k2n, _sum2n); - _sum3 = _mm256_fmadd_ps(_r0, _k3, _sum3); - _sum3n = _mm256_fmadd_ps(_r0n, _k3n, _sum3n); + _sum0 = _mm256_comp_fmadd_ps(_r0, _k0, _sum0); + _sum0n = _mm256_comp_fmadd_ps(_r0n, _k0n, _sum0n); + _sum1 = _mm256_comp_fmadd_ps(_r0, _k1, _sum1); + _sum1n = _mm256_comp_fmadd_ps(_r0n, _k1n, _sum1n); + _sum2 = _mm256_comp_fmadd_ps(_r0, _k2, _sum2); + _sum2n = _mm256_comp_fmadd_ps(_r0n, _k2n, _sum2n); + _sum3 = _mm256_comp_fmadd_ps(_r0, _k3, _sum3); + _sum3n = _mm256_comp_fmadd_ps(_r0n, _k3n, _sum3n); // k1 _r0 = _mm256_loadu_ps(r1); @@ -437,14 +437,14 @@ static void conv3x3s1_winograd23_sse(const Mat& bottom_blob, Mat& top_blob, cons _k2n = _mm256_loadu_ps(k2 + 24); _k3 = _mm256_loadu_ps(k3 + 16); _k3n = _mm256_loadu_ps(k3 + 24); - _sum0 = _mm256_fmadd_ps(_r0, _k0, _sum0); - _sum0n = _mm256_fmadd_ps(_r0n, _k0n, _sum0n); - _sum1 = _mm256_fmadd_ps(_r0, _k1, _sum1); - _sum1n = _mm256_fmadd_ps(_r0n, _k1n, _sum1n); - _sum2 = _mm256_fmadd_ps(_r0, _k2, _sum2); - _sum2n = _mm256_fmadd_ps(_r0n, _k2n, _sum2n); - _sum3 = _mm256_fmadd_ps(_r0, _k3, _sum3); - _sum3n = _mm256_fmadd_ps(_r0n, _k3n, _sum3n); + _sum0 = _mm256_comp_fmadd_ps(_r0, _k0, _sum0); + _sum0n = _mm256_comp_fmadd_ps(_r0n, _k0n, _sum0n); + _sum1 = _mm256_comp_fmadd_ps(_r0, _k1, _sum1); + _sum1n = _mm256_comp_fmadd_ps(_r0n, _k1n, _sum1n); + _sum2 = _mm256_comp_fmadd_ps(_r0, _k2, _sum2); + _sum2n = _mm256_comp_fmadd_ps(_r0n, _k2n, _sum2n); + _sum3 = _mm256_comp_fmadd_ps(_r0, _k3, _sum3); + _sum3n = _mm256_comp_fmadd_ps(_r0n, _k3n, _sum3n); // k2 _r0 = _mm256_loadu_ps(r2); _r0n = _mm256_loadu_ps(r2 + 8); @@ -456,14 +456,14 @@ static void conv3x3s1_winograd23_sse(const Mat& bottom_blob, Mat& top_blob, cons _k2n = _mm256_loadu_ps(k2 + 40); _k3 = _mm256_loadu_ps(k3 + 32); _k3n = _mm256_loadu_ps(k3 + 40); - _sum0 = _mm256_fmadd_ps(_r0, _k0, _sum0); - _sum0n = _mm256_fmadd_ps(_r0n, _k0n, _sum0n); - _sum1 = _mm256_fmadd_ps(_r0, _k1, _sum1); - _sum1n = _mm256_fmadd_ps(_r0n, _k1n, _sum1n); - _sum2 = _mm256_fmadd_ps(_r0, _k2, _sum2); - _sum2n = _mm256_fmadd_ps(_r0n, _k2n, _sum2n); - _sum3 = _mm256_fmadd_ps(_r0, _k3, _sum3); - _sum3n = _mm256_fmadd_ps(_r0n, _k3n, _sum3n); + _sum0 = _mm256_comp_fmadd_ps(_r0, _k0, _sum0); + _sum0n = _mm256_comp_fmadd_ps(_r0n, _k0n, _sum0n); + _sum1 = _mm256_comp_fmadd_ps(_r0, _k1, _sum1); + _sum1n = _mm256_comp_fmadd_ps(_r0n, _k1n, _sum1n); + _sum2 = _mm256_comp_fmadd_ps(_r0, _k2, _sum2); + _sum2n = _mm256_comp_fmadd_ps(_r0n, _k2n, _sum2n); + _sum3 = _mm256_comp_fmadd_ps(_r0, _k3, _sum3); + _sum3n = _mm256_comp_fmadd_ps(_r0n, _k3n, _sum3n); // k3 _r0 = _mm256_loadu_ps(r3); _r0n = _mm256_loadu_ps(r3 + 8); @@ -475,14 +475,14 @@ static void conv3x3s1_winograd23_sse(const Mat& bottom_blob, Mat& top_blob, cons _k2n = _mm256_loadu_ps(k2 + 56); _k3 = _mm256_loadu_ps(k3 + 48); _k3n = _mm256_loadu_ps(k3 + 56); - _sum0 = _mm256_fmadd_ps(_r0, _k0, _sum0); - _sum0n = _mm256_fmadd_ps(_r0n, _k0n, _sum0n); - _sum1 = _mm256_fmadd_ps(_r0, _k1, _sum1); - _sum1n = _mm256_fmadd_ps(_r0n, _k1n, _sum1n); - _sum2 = _mm256_fmadd_ps(_r0, _k2, _sum2); - _sum2n = _mm256_fmadd_ps(_r0n, _k2n, _sum2n); - _sum3 = _mm256_fmadd_ps(_r0, _k3, _sum3); - _sum3n = _mm256_fmadd_ps(_r0n, _k3n, _sum3n); + _sum0 = _mm256_comp_fmadd_ps(_r0, _k0, _sum0); + _sum0n = _mm256_comp_fmadd_ps(_r0n, _k0n, _sum0n); + _sum1 = _mm256_comp_fmadd_ps(_r0, _k1, _sum1); + _sum1n = _mm256_comp_fmadd_ps(_r0n, _k1n, _sum1n); + _sum2 = _mm256_comp_fmadd_ps(_r0, _k2, _sum2); + _sum2n = _mm256_comp_fmadd_ps(_r0n, _k2n, _sum2n); + _sum3 = _mm256_comp_fmadd_ps(_r0, _k3, _sum3); + _sum3n = _mm256_comp_fmadd_ps(_r0n, _k3n, _sum3n); } for (; q < inch; q++) @@ -505,14 +505,14 @@ static void conv3x3s1_winograd23_sse(const Mat& bottom_blob, Mat& top_blob, cons __m256 _k3 = _mm256_loadu_ps(k3); __m256 _k3n = _mm256_loadu_ps(k3 + 8); - _sum0 = _mm256_fmadd_ps(_r0, _k0, _sum0); - _sum0n = _mm256_fmadd_ps(_r0n, _k0n, _sum0n); - _sum1 = _mm256_fmadd_ps(_r0, _k1, _sum1); - _sum1n = _mm256_fmadd_ps(_r0n, _k1n, _sum1n); - _sum2 = _mm256_fmadd_ps(_r0, _k2, _sum2); - _sum2n = _mm256_fmadd_ps(_r0n, _k2n, _sum2n); - _sum3 = _mm256_fmadd_ps(_r0, _k3, _sum3); - _sum3n = _mm256_fmadd_ps(_r0n, _k3n, _sum3n); + _sum0 = _mm256_comp_fmadd_ps(_r0, _k0, _sum0); + _sum0n = _mm256_comp_fmadd_ps(_r0n, _k0n, _sum0n); + _sum1 = _mm256_comp_fmadd_ps(_r0, _k1, _sum1); + _sum1n = _mm256_comp_fmadd_ps(_r0n, _k1n, _sum1n); + _sum2 = _mm256_comp_fmadd_ps(_r0, _k2, _sum2); + _sum2n = _mm256_comp_fmadd_ps(_r0n, _k2n, _sum2n); + _sum3 = _mm256_comp_fmadd_ps(_r0, _k3, _sum3); + _sum3n = _mm256_comp_fmadd_ps(_r0n, _k3n, _sum3n); } _mm256_storeu_ps(output0_tm, _sum0); @@ -1052,31 +1052,31 @@ static void conv3x3s1_winograd43_sse(const Mat& bottom_blob, Mat& top_blob, cons // w = B_t * d _w0 = _mm256_mul_ps(_d0, _4_p); - _w0 = _mm256_fmadd_ps(_d2, _5_n, _w0); + _w0 = _mm256_comp_fmadd_ps(_d2, _5_n, _w0); _w0 = _mm256_add_ps(_w0, _d4); _w1 = _mm256_mul_ps(_d1, _4_n); - _w1 = _mm256_fmadd_ps(_d2, _4_n, _w1); + _w1 = _mm256_comp_fmadd_ps(_d2, _4_n, _w1); _w1 = _mm256_add_ps(_w1, _d3); _w1 = _mm256_add_ps(_w1, _d4); _w2 = _mm256_mul_ps(_d1, _4_p); - _w2 = _mm256_fmadd_ps(_d2, _4_n, _w2); - _w2 = _mm256_fmadd_ps(_d3, _1_n, _w2); + _w2 = _mm256_comp_fmadd_ps(_d2, _4_n, _w2); + _w2 = _mm256_comp_fmadd_ps(_d3, _1_n, _w2); _w2 = _mm256_add_ps(_w2, _d4); _w3 = _mm256_mul_ps(_d1, _2_n); - _w3 = _mm256_fmadd_ps(_d2, _1_n, _w3); - _w3 = _mm256_fmadd_ps(_d3, _2_p, _w3); + _w3 = _mm256_comp_fmadd_ps(_d2, _1_n, _w3); + _w3 = _mm256_comp_fmadd_ps(_d3, _2_p, _w3); _w3 = _mm256_add_ps(_w3, _d4); _w4 = _mm256_mul_ps(_d1, _2_p); - _w4 = _mm256_fmadd_ps(_d2, _1_n, _w4); - _w4 = _mm256_fmadd_ps(_d3, _2_n, _w4); + _w4 = _mm256_comp_fmadd_ps(_d2, _1_n, _w4); + _w4 = _mm256_comp_fmadd_ps(_d3, _2_n, _w4); _w4 = _mm256_add_ps(_w4, _d4); _w5 = _mm256_mul_ps(_d1, _4_p); - _w5 = _mm256_fmadd_ps(_d3, _5_n, _w5); + _w5 = _mm256_comp_fmadd_ps(_d3, _5_n, _w5); _w5 = _mm256_add_ps(_w5, _d5); // transpose d to d_t #if (defined _WIN32 && !(defined __MINGW32__) && !__clang__) @@ -1160,31 +1160,31 @@ static void conv3x3s1_winograd43_sse(const Mat& bottom_blob, Mat& top_blob, cons #endif // d = B_t * d_t _n0 = _mm256_mul_ps(_t0, _4_p); - _n0 = _mm256_fmadd_ps(_t2, _5_n, _n0); + _n0 = _mm256_comp_fmadd_ps(_t2, _5_n, _n0); _n0 = _mm256_add_ps(_n0, _t4); _n1 = _mm256_mul_ps(_t1, _4_n); - _n1 = _mm256_fmadd_ps(_t2, _4_n, _n1); + _n1 = _mm256_comp_fmadd_ps(_t2, _4_n, _n1); _n1 = _mm256_add_ps(_n1, _t3); _n1 = _mm256_add_ps(_n1, _t4); _n2 = _mm256_mul_ps(_t1, _4_p); - _n2 = _mm256_fmadd_ps(_t2, _4_n, _n2); - _n2 = _mm256_fmadd_ps(_t3, _1_n, _n2); + _n2 = _mm256_comp_fmadd_ps(_t2, _4_n, _n2); + _n2 = _mm256_comp_fmadd_ps(_t3, _1_n, _n2); _n2 = _mm256_add_ps(_n2, _t4); _n3 = _mm256_mul_ps(_t1, _2_n); - _n3 = _mm256_fmadd_ps(_t2, _1_n, _n3); - _n3 = _mm256_fmadd_ps(_t3, _2_p, _n3); + _n3 = _mm256_comp_fmadd_ps(_t2, _1_n, _n3); + _n3 = _mm256_comp_fmadd_ps(_t3, _2_p, _n3); _n3 = _mm256_add_ps(_n3, _t4); _n4 = _mm256_mul_ps(_t1, _2_p); - _n4 = _mm256_fmadd_ps(_t2, _1_n, _n4); - _n4 = _mm256_fmadd_ps(_t3, _2_n, _n4); + _n4 = _mm256_comp_fmadd_ps(_t2, _1_n, _n4); + _n4 = _mm256_comp_fmadd_ps(_t3, _2_n, _n4); _n4 = _mm256_add_ps(_n4, _t4); _n5 = _mm256_mul_ps(_t1, _4_p); - _n5 = _mm256_fmadd_ps(_t3, _5_n, _n5); + _n5 = _mm256_comp_fmadd_ps(_t3, _5_n, _n5); _n5 = _mm256_add_ps(_n5, _t5); // save to out_tm float output_n0[8] = {0.f}; @@ -1452,14 +1452,14 @@ static void conv3x3s1_winograd43_sse(const Mat& bottom_blob, Mat& top_blob, cons __m128 _k6 = _mm_loadu_ps(kptr + 24); __m128 _k7 = _mm_loadu_ps(kptr + 28); #if __AVX__ - _sum0 = _mm_fmadd_ps(_r0, _k0, _sum0); - _sum1 = _mm_fmadd_ps(_r0, _k1, _sum1); - _sum2 = _mm_fmadd_ps(_r0, _k2, _sum2); - _sum3 = _mm_fmadd_ps(_r0, _k3, _sum3); - _sum4 = _mm_fmadd_ps(_r0, _k4, _sum4); - _sum5 = _mm_fmadd_ps(_r0, _k5, _sum5); - _sum6 = _mm_fmadd_ps(_r0, _k6, _sum6); - _sum7 = _mm_fmadd_ps(_r0, _k7, _sum7); + _sum0 = _mm_comp_fmadd_ps(_r0, _k0, _sum0); + _sum1 = _mm_comp_fmadd_ps(_r0, _k1, _sum1); + _sum2 = _mm_comp_fmadd_ps(_r0, _k2, _sum2); + _sum3 = _mm_comp_fmadd_ps(_r0, _k3, _sum3); + _sum4 = _mm_comp_fmadd_ps(_r0, _k4, _sum4); + _sum5 = _mm_comp_fmadd_ps(_r0, _k5, _sum5); + _sum6 = _mm_comp_fmadd_ps(_r0, _k6, _sum6); + _sum7 = _mm_comp_fmadd_ps(_r0, _k7, _sum7); #else _sum0 = _mm_add_ps(_sum0, _mm_mul_ps(_r0, _k0)); _sum1 = _mm_add_ps(_sum1, _mm_mul_ps(_r0, _k1)); @@ -1480,14 +1480,14 @@ static void conv3x3s1_winograd43_sse(const Mat& bottom_blob, Mat& top_blob, cons _k6 = _mm_loadu_ps(kptr + 24); _k7 = _mm_loadu_ps(kptr + 28); #if __AVX__ - _sum0 = _mm_fmadd_ps(_r1, _k0, _sum0); - _sum1 = _mm_fmadd_ps(_r1, _k1, _sum1); - _sum2 = _mm_fmadd_ps(_r1, _k2, _sum2); - _sum3 = _mm_fmadd_ps(_r1, _k3, _sum3); - _sum4 = _mm_fmadd_ps(_r1, _k4, _sum4); - _sum5 = _mm_fmadd_ps(_r1, _k5, _sum5); - _sum6 = _mm_fmadd_ps(_r1, _k6, _sum6); - _sum7 = _mm_fmadd_ps(_r1, _k7, _sum7); + _sum0 = _mm_comp_fmadd_ps(_r1, _k0, _sum0); + _sum1 = _mm_comp_fmadd_ps(_r1, _k1, _sum1); + _sum2 = _mm_comp_fmadd_ps(_r1, _k2, _sum2); + _sum3 = _mm_comp_fmadd_ps(_r1, _k3, _sum3); + _sum4 = _mm_comp_fmadd_ps(_r1, _k4, _sum4); + _sum5 = _mm_comp_fmadd_ps(_r1, _k5, _sum5); + _sum6 = _mm_comp_fmadd_ps(_r1, _k6, _sum6); + _sum7 = _mm_comp_fmadd_ps(_r1, _k7, _sum7); #else _sum0 = _mm_add_ps(_sum0, _mm_mul_ps(_r1, _k0)); _sum1 = _mm_add_ps(_sum1, _mm_mul_ps(_r1, _k1)); @@ -1509,14 +1509,14 @@ static void conv3x3s1_winograd43_sse(const Mat& bottom_blob, Mat& top_blob, cons _k6 = _mm_loadu_ps(kptr + 24); _k7 = _mm_loadu_ps(kptr + 28); #if __AVX__ - _sum0 = _mm_fmadd_ps(_r2, _k0, _sum0); - _sum1 = _mm_fmadd_ps(_r2, _k1, _sum1); - _sum2 = _mm_fmadd_ps(_r2, _k2, _sum2); - _sum3 = _mm_fmadd_ps(_r2, _k3, _sum3); - _sum4 = _mm_fmadd_ps(_r2, _k4, _sum4); - _sum5 = _mm_fmadd_ps(_r2, _k5, _sum5); - _sum6 = _mm_fmadd_ps(_r2, _k6, _sum6); - _sum7 = _mm_fmadd_ps(_r2, _k7, _sum7); + _sum0 = _mm_comp_fmadd_ps(_r2, _k0, _sum0); + _sum1 = _mm_comp_fmadd_ps(_r2, _k1, _sum1); + _sum2 = _mm_comp_fmadd_ps(_r2, _k2, _sum2); + _sum3 = _mm_comp_fmadd_ps(_r2, _k3, _sum3); + _sum4 = _mm_comp_fmadd_ps(_r2, _k4, _sum4); + _sum5 = _mm_comp_fmadd_ps(_r2, _k5, _sum5); + _sum6 = _mm_comp_fmadd_ps(_r2, _k6, _sum6); + _sum7 = _mm_comp_fmadd_ps(_r2, _k7, _sum7); #else _sum0 = _mm_add_ps(_sum0, _mm_mul_ps(_r2, _k0)); _sum1 = _mm_add_ps(_sum1, _mm_mul_ps(_r2, _k1)); @@ -1537,14 +1537,14 @@ static void conv3x3s1_winograd43_sse(const Mat& bottom_blob, Mat& top_blob, cons _k6 = _mm_loadu_ps(kptr + 24); _k7 = _mm_loadu_ps(kptr + 28); #if __AVX__ - _sum0 = _mm_fmadd_ps(_r3, _k0, _sum0); - _sum1 = _mm_fmadd_ps(_r3, _k1, _sum1); - _sum2 = _mm_fmadd_ps(_r3, _k2, _sum2); - _sum3 = _mm_fmadd_ps(_r3, _k3, _sum3); - _sum4 = _mm_fmadd_ps(_r3, _k4, _sum4); - _sum5 = _mm_fmadd_ps(_r3, _k5, _sum5); - _sum6 = _mm_fmadd_ps(_r3, _k6, _sum6); - _sum7 = _mm_fmadd_ps(_r3, _k7, _sum7); + _sum0 = _mm_comp_fmadd_ps(_r3, _k0, _sum0); + _sum1 = _mm_comp_fmadd_ps(_r3, _k1, _sum1); + _sum2 = _mm_comp_fmadd_ps(_r3, _k2, _sum2); + _sum3 = _mm_comp_fmadd_ps(_r3, _k3, _sum3); + _sum4 = _mm_comp_fmadd_ps(_r3, _k4, _sum4); + _sum5 = _mm_comp_fmadd_ps(_r3, _k5, _sum5); + _sum6 = _mm_comp_fmadd_ps(_r3, _k6, _sum6); + _sum7 = _mm_comp_fmadd_ps(_r3, _k7, _sum7); #else _sum0 = _mm_add_ps(_sum0, _mm_mul_ps(_r3, _k0)); _sum1 = _mm_add_ps(_sum1, _mm_mul_ps(_r3, _k1)); @@ -1572,14 +1572,14 @@ static void conv3x3s1_winograd43_sse(const Mat& bottom_blob, Mat& top_blob, cons __m128 _k7 = _mm_loadu_ps(kptr + 28); #if __AVX__ - _sum0 = _mm_fmadd_ps(_r0, _k0, _sum0); - _sum1 = _mm_fmadd_ps(_r0, _k1, _sum1); - _sum2 = _mm_fmadd_ps(_r0, _k2, _sum2); - _sum3 = _mm_fmadd_ps(_r0, _k3, _sum3); - _sum4 = _mm_fmadd_ps(_r0, _k4, _sum4); - _sum5 = _mm_fmadd_ps(_r0, _k5, _sum5); - _sum6 = _mm_fmadd_ps(_r0, _k6, _sum6); - _sum7 = _mm_fmadd_ps(_r0, _k7, _sum7); + _sum0 = _mm_comp_fmadd_ps(_r0, _k0, _sum0); + _sum1 = _mm_comp_fmadd_ps(_r0, _k1, _sum1); + _sum2 = _mm_comp_fmadd_ps(_r0, _k2, _sum2); + _sum3 = _mm_comp_fmadd_ps(_r0, _k3, _sum3); + _sum4 = _mm_comp_fmadd_ps(_r0, _k4, _sum4); + _sum5 = _mm_comp_fmadd_ps(_r0, _k5, _sum5); + _sum6 = _mm_comp_fmadd_ps(_r0, _k6, _sum6); + _sum7 = _mm_comp_fmadd_ps(_r0, _k7, _sum7); #else _sum0 = _mm_add_ps(_sum0, _mm_mul_ps(_r0, _k0)); _sum1 = _mm_add_ps(_sum1, _mm_mul_ps(_r0, _k1)); @@ -1694,10 +1694,10 @@ static void conv3x3s1_winograd43_sse(const Mat& bottom_blob, Mat& top_blob, cons __m128 _k2 = _mm_loadu_ps(kptr + 8); __m128 _k3 = _mm_loadu_ps(kptr + 12); #if __AVX__ - _sum0 = _mm_fmadd_ps(_r0, _k0, _sum0); - _sum1 = _mm_fmadd_ps(_r0, _k1, _sum1); - _sum2 = _mm_fmadd_ps(_r0, _k2, _sum2); - _sum3 = _mm_fmadd_ps(_r0, _k3, _sum3); + _sum0 = _mm_comp_fmadd_ps(_r0, _k0, _sum0); + _sum1 = _mm_comp_fmadd_ps(_r0, _k1, _sum1); + _sum2 = _mm_comp_fmadd_ps(_r0, _k2, _sum2); + _sum3 = _mm_comp_fmadd_ps(_r0, _k3, _sum3); #else _sum0 = _mm_add_ps(_sum0, _mm_mul_ps(_r0, _k0)); _sum1 = _mm_add_ps(_sum1, _mm_mul_ps(_r0, _k1)); @@ -1771,7 +1771,7 @@ static void conv3x3s1_winograd43_sse(const Mat& bottom_blob, Mat& top_blob, cons __m128 _r0 = _mm_loadu_ps(r0); __m128 _k0 = _mm_loadu_ps(kptr); #if __AVX__ - _sum0 = _mm_fmadd_ps(_r0, _k0, _sum0); + _sum0 = _mm_comp_fmadd_ps(_r0, _k0, _sum0); #else _sum0 = _mm_add_ps(_sum0, _mm_mul_ps(_r0, _k0)); #endif diff --git a/src/layer/x86/convolution_3x3_pack1to8.h b/src/layer/x86/convolution_3x3_pack1to8.h index b2b7b788a..3c4dcccee 100644 --- a/src/layer/x86/convolution_3x3_pack1to8.h +++ b/src/layer/x86/convolution_3x3_pack1to8.h @@ -90,25 +90,25 @@ static void conv3x3s1_pack1to8_avx(const Mat& bottom_blob, Mat& top_blob, const __m256 _r22 = _mm256_broadcast_ss(r2 + 1); __m256 _r23 = _mm256_broadcast_ss(r2 + 2); - _sum00 = _mm256_fmadd_ps(_r01, _k00_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r02, _k01_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r03, _k02_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r11, _k10_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r12, _k11_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r13, _k12_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r21, _k20_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r22, _k21_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r23, _k22_0, _sum00); - - _sum10 = _mm256_fmadd_ps(_r01, _k00_1, _sum10); - _sum10 = _mm256_fmadd_ps(_r02, _k01_1, _sum10); - _sum10 = _mm256_fmadd_ps(_r03, _k02_1, _sum10); - _sum10 = _mm256_fmadd_ps(_r11, _k10_1, _sum10); - _sum10 = _mm256_fmadd_ps(_r12, _k11_1, _sum10); - _sum10 = _mm256_fmadd_ps(_r13, _k12_1, _sum10); - _sum10 = _mm256_fmadd_ps(_r21, _k20_1, _sum10); - _sum10 = _mm256_fmadd_ps(_r22, _k21_1, _sum10); - _sum10 = _mm256_fmadd_ps(_r23, _k22_1, _sum10); + _sum00 = _mm256_comp_fmadd_ps(_r01, _k00_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r02, _k01_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r03, _k02_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r11, _k10_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r12, _k11_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r13, _k12_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r21, _k20_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r22, _k21_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r23, _k22_0, _sum00); + + _sum10 = _mm256_comp_fmadd_ps(_r01, _k00_1, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_r02, _k01_1, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_r03, _k02_1, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_r11, _k10_1, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_r12, _k11_1, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_r13, _k12_1, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_r21, _k20_1, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_r22, _k21_1, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_r23, _k22_1, _sum10); _mm256_storeu_ps(outptr0, _sum00); _mm256_storeu_ps(outptr1, _sum10); @@ -120,25 +120,25 @@ static void conv3x3s1_pack1to8_avx(const Mat& bottom_blob, Mat& top_blob, const __m256 _r14 = _mm256_broadcast_ss(r1 + 3); __m256 _r24 = _mm256_broadcast_ss(r2 + 3); - _sum01 = _mm256_fmadd_ps(_r02, _k00_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r03, _k01_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r04, _k02_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r12, _k10_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r13, _k11_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r14, _k12_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r22, _k20_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r23, _k21_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r24, _k22_0, _sum01); - - _sum11 = _mm256_fmadd_ps(_r02, _k00_1, _sum11); - _sum11 = _mm256_fmadd_ps(_r03, _k01_1, _sum11); - _sum11 = _mm256_fmadd_ps(_r04, _k02_1, _sum11); - _sum11 = _mm256_fmadd_ps(_r12, _k10_1, _sum11); - _sum11 = _mm256_fmadd_ps(_r13, _k11_1, _sum11); - _sum11 = _mm256_fmadd_ps(_r14, _k12_1, _sum11); - _sum11 = _mm256_fmadd_ps(_r22, _k20_1, _sum11); - _sum11 = _mm256_fmadd_ps(_r23, _k21_1, _sum11); - _sum11 = _mm256_fmadd_ps(_r24, _k22_1, _sum11); + _sum01 = _mm256_comp_fmadd_ps(_r02, _k00_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r03, _k01_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r04, _k02_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r12, _k10_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r13, _k11_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r14, _k12_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r22, _k20_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r23, _k21_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r24, _k22_0, _sum01); + + _sum11 = _mm256_comp_fmadd_ps(_r02, _k00_1, _sum11); + _sum11 = _mm256_comp_fmadd_ps(_r03, _k01_1, _sum11); + _sum11 = _mm256_comp_fmadd_ps(_r04, _k02_1, _sum11); + _sum11 = _mm256_comp_fmadd_ps(_r12, _k10_1, _sum11); + _sum11 = _mm256_comp_fmadd_ps(_r13, _k11_1, _sum11); + _sum11 = _mm256_comp_fmadd_ps(_r14, _k12_1, _sum11); + _sum11 = _mm256_comp_fmadd_ps(_r22, _k20_1, _sum11); + _sum11 = _mm256_comp_fmadd_ps(_r23, _k21_1, _sum11); + _sum11 = _mm256_comp_fmadd_ps(_r24, _k22_1, _sum11); _mm256_storeu_ps(outptr0 + 8, _sum01); _mm256_storeu_ps(outptr1 + 8, _sum11); @@ -150,25 +150,25 @@ static void conv3x3s1_pack1to8_avx(const Mat& bottom_blob, Mat& top_blob, const __m256 _r15 = _mm256_broadcast_ss(r1 + 4); __m256 _r25 = _mm256_broadcast_ss(r2 + 4); - _sum02 = _mm256_fmadd_ps(_r03, _k00_0, _sum02); - _sum02 = _mm256_fmadd_ps(_r04, _k01_0, _sum02); - _sum02 = _mm256_fmadd_ps(_r05, _k02_0, _sum02); - _sum02 = _mm256_fmadd_ps(_r13, _k10_0, _sum02); - _sum02 = _mm256_fmadd_ps(_r14, _k11_0, _sum02); - _sum02 = _mm256_fmadd_ps(_r15, _k12_0, _sum02); - _sum02 = _mm256_fmadd_ps(_r23, _k20_0, _sum02); - _sum02 = _mm256_fmadd_ps(_r24, _k21_0, _sum02); - _sum02 = _mm256_fmadd_ps(_r25, _k22_0, _sum02); - - _sum12 = _mm256_fmadd_ps(_r03, _k00_1, _sum12); - _sum12 = _mm256_fmadd_ps(_r04, _k01_1, _sum12); - _sum12 = _mm256_fmadd_ps(_r05, _k02_1, _sum12); - _sum12 = _mm256_fmadd_ps(_r13, _k10_1, _sum12); - _sum12 = _mm256_fmadd_ps(_r14, _k11_1, _sum12); - _sum12 = _mm256_fmadd_ps(_r15, _k12_1, _sum12); - _sum12 = _mm256_fmadd_ps(_r23, _k20_1, _sum12); - _sum12 = _mm256_fmadd_ps(_r24, _k21_1, _sum12); - _sum12 = _mm256_fmadd_ps(_r25, _k22_1, _sum12); + _sum02 = _mm256_comp_fmadd_ps(_r03, _k00_0, _sum02); + _sum02 = _mm256_comp_fmadd_ps(_r04, _k01_0, _sum02); + _sum02 = _mm256_comp_fmadd_ps(_r05, _k02_0, _sum02); + _sum02 = _mm256_comp_fmadd_ps(_r13, _k10_0, _sum02); + _sum02 = _mm256_comp_fmadd_ps(_r14, _k11_0, _sum02); + _sum02 = _mm256_comp_fmadd_ps(_r15, _k12_0, _sum02); + _sum02 = _mm256_comp_fmadd_ps(_r23, _k20_0, _sum02); + _sum02 = _mm256_comp_fmadd_ps(_r24, _k21_0, _sum02); + _sum02 = _mm256_comp_fmadd_ps(_r25, _k22_0, _sum02); + + _sum12 = _mm256_comp_fmadd_ps(_r03, _k00_1, _sum12); + _sum12 = _mm256_comp_fmadd_ps(_r04, _k01_1, _sum12); + _sum12 = _mm256_comp_fmadd_ps(_r05, _k02_1, _sum12); + _sum12 = _mm256_comp_fmadd_ps(_r13, _k10_1, _sum12); + _sum12 = _mm256_comp_fmadd_ps(_r14, _k11_1, _sum12); + _sum12 = _mm256_comp_fmadd_ps(_r15, _k12_1, _sum12); + _sum12 = _mm256_comp_fmadd_ps(_r23, _k20_1, _sum12); + _sum12 = _mm256_comp_fmadd_ps(_r24, _k21_1, _sum12); + _sum12 = _mm256_comp_fmadd_ps(_r25, _k22_1, _sum12); _mm256_storeu_ps(outptr0 + 16, _sum02); _mm256_storeu_ps(outptr1 + 16, _sum12); @@ -180,25 +180,25 @@ static void conv3x3s1_pack1to8_avx(const Mat& bottom_blob, Mat& top_blob, const __m256 _sum03 = _mm256_loadu_ps(outptr0 + 24); __m256 _sum13 = _mm256_loadu_ps(outptr1 + 24); - _sum03 = _mm256_fmadd_ps(_r04, _k00_0, _sum03); - _sum03 = _mm256_fmadd_ps(_r05, _k01_0, _sum03); - _sum03 = _mm256_fmadd_ps(_r06, _k02_0, _sum03); - _sum03 = _mm256_fmadd_ps(_r14, _k10_0, _sum03); - _sum03 = _mm256_fmadd_ps(_r15, _k11_0, _sum03); - _sum03 = _mm256_fmadd_ps(_r16, _k12_0, _sum03); - _sum03 = _mm256_fmadd_ps(_r24, _k20_0, _sum03); - _sum03 = _mm256_fmadd_ps(_r25, _k21_0, _sum03); - _sum03 = _mm256_fmadd_ps(_r26, _k22_0, _sum03); - - _sum13 = _mm256_fmadd_ps(_r04, _k00_1, _sum13); - _sum13 = _mm256_fmadd_ps(_r05, _k01_1, _sum13); - _sum13 = _mm256_fmadd_ps(_r06, _k02_1, _sum13); - _sum13 = _mm256_fmadd_ps(_r14, _k10_1, _sum13); - _sum13 = _mm256_fmadd_ps(_r15, _k11_1, _sum13); - _sum13 = _mm256_fmadd_ps(_r16, _k12_1, _sum13); - _sum13 = _mm256_fmadd_ps(_r24, _k20_1, _sum13); - _sum13 = _mm256_fmadd_ps(_r25, _k21_1, _sum13); - _sum13 = _mm256_fmadd_ps(_r26, _k22_1, _sum13); + _sum03 = _mm256_comp_fmadd_ps(_r04, _k00_0, _sum03); + _sum03 = _mm256_comp_fmadd_ps(_r05, _k01_0, _sum03); + _sum03 = _mm256_comp_fmadd_ps(_r06, _k02_0, _sum03); + _sum03 = _mm256_comp_fmadd_ps(_r14, _k10_0, _sum03); + _sum03 = _mm256_comp_fmadd_ps(_r15, _k11_0, _sum03); + _sum03 = _mm256_comp_fmadd_ps(_r16, _k12_0, _sum03); + _sum03 = _mm256_comp_fmadd_ps(_r24, _k20_0, _sum03); + _sum03 = _mm256_comp_fmadd_ps(_r25, _k21_0, _sum03); + _sum03 = _mm256_comp_fmadd_ps(_r26, _k22_0, _sum03); + + _sum13 = _mm256_comp_fmadd_ps(_r04, _k00_1, _sum13); + _sum13 = _mm256_comp_fmadd_ps(_r05, _k01_1, _sum13); + _sum13 = _mm256_comp_fmadd_ps(_r06, _k02_1, _sum13); + _sum13 = _mm256_comp_fmadd_ps(_r14, _k10_1, _sum13); + _sum13 = _mm256_comp_fmadd_ps(_r15, _k11_1, _sum13); + _sum13 = _mm256_comp_fmadd_ps(_r16, _k12_1, _sum13); + _sum13 = _mm256_comp_fmadd_ps(_r24, _k20_1, _sum13); + _sum13 = _mm256_comp_fmadd_ps(_r25, _k21_1, _sum13); + _sum13 = _mm256_comp_fmadd_ps(_r26, _k22_1, _sum13); _mm256_storeu_ps(outptr0 + 24, _sum03); _mm256_storeu_ps(outptr1 + 24, _sum13); @@ -225,25 +225,25 @@ static void conv3x3s1_pack1to8_avx(const Mat& bottom_blob, Mat& top_blob, const __m256 _r22 = _mm256_broadcast_ss(r2 + 1); __m256 _r23 = _mm256_broadcast_ss(r2 + 2); - _sum00 = _mm256_fmadd_ps(_r01, _k00_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r02, _k01_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r03, _k02_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r11, _k10_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r12, _k11_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r13, _k12_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r21, _k20_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r22, _k21_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r23, _k22_0, _sum00); - - _sum10 = _mm256_fmadd_ps(_r01, _k00_1, _sum10); - _sum10 = _mm256_fmadd_ps(_r02, _k01_1, _sum10); - _sum10 = _mm256_fmadd_ps(_r03, _k02_1, _sum10); - _sum10 = _mm256_fmadd_ps(_r11, _k10_1, _sum10); - _sum10 = _mm256_fmadd_ps(_r12, _k11_1, _sum10); - _sum10 = _mm256_fmadd_ps(_r13, _k12_1, _sum10); - _sum10 = _mm256_fmadd_ps(_r21, _k20_1, _sum10); - _sum10 = _mm256_fmadd_ps(_r22, _k21_1, _sum10); - _sum10 = _mm256_fmadd_ps(_r23, _k22_1, _sum10); + _sum00 = _mm256_comp_fmadd_ps(_r01, _k00_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r02, _k01_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r03, _k02_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r11, _k10_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r12, _k11_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r13, _k12_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r21, _k20_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r22, _k21_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r23, _k22_0, _sum00); + + _sum10 = _mm256_comp_fmadd_ps(_r01, _k00_1, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_r02, _k01_1, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_r03, _k02_1, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_r11, _k10_1, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_r12, _k11_1, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_r13, _k12_1, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_r21, _k20_1, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_r22, _k21_1, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_r23, _k22_1, _sum10); _mm256_storeu_ps(outptr0, _sum00); _mm256_storeu_ps(outptr1, _sum10); @@ -255,25 +255,25 @@ static void conv3x3s1_pack1to8_avx(const Mat& bottom_blob, Mat& top_blob, const __m256 _r14 = _mm256_broadcast_ss(r1 + 3); __m256 _r24 = _mm256_broadcast_ss(r2 + 3); - _sum01 = _mm256_fmadd_ps(_r02, _k00_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r03, _k01_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r04, _k02_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r12, _k10_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r13, _k11_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r14, _k12_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r22, _k20_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r23, _k21_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r24, _k22_0, _sum01); - - _sum11 = _mm256_fmadd_ps(_r02, _k00_1, _sum11); - _sum11 = _mm256_fmadd_ps(_r03, _k01_1, _sum11); - _sum11 = _mm256_fmadd_ps(_r04, _k02_1, _sum11); - _sum11 = _mm256_fmadd_ps(_r12, _k10_1, _sum11); - _sum11 = _mm256_fmadd_ps(_r13, _k11_1, _sum11); - _sum11 = _mm256_fmadd_ps(_r14, _k12_1, _sum11); - _sum11 = _mm256_fmadd_ps(_r22, _k20_1, _sum11); - _sum11 = _mm256_fmadd_ps(_r23, _k21_1, _sum11); - _sum11 = _mm256_fmadd_ps(_r24, _k22_1, _sum11); + _sum01 = _mm256_comp_fmadd_ps(_r02, _k00_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r03, _k01_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r04, _k02_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r12, _k10_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r13, _k11_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r14, _k12_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r22, _k20_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r23, _k21_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r24, _k22_0, _sum01); + + _sum11 = _mm256_comp_fmadd_ps(_r02, _k00_1, _sum11); + _sum11 = _mm256_comp_fmadd_ps(_r03, _k01_1, _sum11); + _sum11 = _mm256_comp_fmadd_ps(_r04, _k02_1, _sum11); + _sum11 = _mm256_comp_fmadd_ps(_r12, _k10_1, _sum11); + _sum11 = _mm256_comp_fmadd_ps(_r13, _k11_1, _sum11); + _sum11 = _mm256_comp_fmadd_ps(_r14, _k12_1, _sum11); + _sum11 = _mm256_comp_fmadd_ps(_r22, _k20_1, _sum11); + _sum11 = _mm256_comp_fmadd_ps(_r23, _k21_1, _sum11); + _sum11 = _mm256_comp_fmadd_ps(_r24, _k22_1, _sum11); _mm256_storeu_ps(outptr0 + 8, _sum01); _mm256_storeu_ps(outptr1 + 8, _sum11); @@ -300,25 +300,25 @@ static void conv3x3s1_pack1to8_avx(const Mat& bottom_blob, Mat& top_blob, const __m256 _r22 = _mm256_broadcast_ss(r2 + 1); __m256 _r23 = _mm256_broadcast_ss(r2 + 2); - _sum00 = _mm256_fmadd_ps(_r01, _k00_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r02, _k01_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r03, _k02_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r11, _k10_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r12, _k11_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r13, _k12_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r21, _k20_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r22, _k21_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r23, _k22_0, _sum00); - - _sum10 = _mm256_fmadd_ps(_r01, _k00_1, _sum10); - _sum10 = _mm256_fmadd_ps(_r02, _k01_1, _sum10); - _sum10 = _mm256_fmadd_ps(_r03, _k02_1, _sum10); - _sum10 = _mm256_fmadd_ps(_r11, _k10_1, _sum10); - _sum10 = _mm256_fmadd_ps(_r12, _k11_1, _sum10); - _sum10 = _mm256_fmadd_ps(_r13, _k12_1, _sum10); - _sum10 = _mm256_fmadd_ps(_r21, _k20_1, _sum10); - _sum10 = _mm256_fmadd_ps(_r22, _k21_1, _sum10); - _sum10 = _mm256_fmadd_ps(_r23, _k22_1, _sum10); + _sum00 = _mm256_comp_fmadd_ps(_r01, _k00_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r02, _k01_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r03, _k02_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r11, _k10_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r12, _k11_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r13, _k12_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r21, _k20_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r22, _k21_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r23, _k22_0, _sum00); + + _sum10 = _mm256_comp_fmadd_ps(_r01, _k00_1, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_r02, _k01_1, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_r03, _k02_1, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_r11, _k10_1, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_r12, _k11_1, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_r13, _k12_1, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_r21, _k20_1, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_r22, _k21_1, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_r23, _k22_1, _sum10); _mm256_storeu_ps(outptr0, _sum00); _mm256_storeu_ps(outptr1, _sum10); @@ -389,15 +389,15 @@ static void conv3x3s1_pack1to8_avx(const Mat& bottom_blob, Mat& top_blob, const __m256 _r22 = _mm256_broadcast_ss(r2 + 1); __m256 _r23 = _mm256_broadcast_ss(r2 + 2); - _sum0 = _mm256_fmadd_ps(_r01, _k00, _sum0); - _sum0 = _mm256_fmadd_ps(_r02, _k01, _sum0); - _sum0 = _mm256_fmadd_ps(_r03, _k02, _sum0); - _sum0 = _mm256_fmadd_ps(_r11, _k10, _sum0); - _sum0 = _mm256_fmadd_ps(_r12, _k11, _sum0); - _sum0 = _mm256_fmadd_ps(_r13, _k12, _sum0); - _sum0 = _mm256_fmadd_ps(_r21, _k20, _sum0); - _sum0 = _mm256_fmadd_ps(_r22, _k21, _sum0); - _sum0 = _mm256_fmadd_ps(_r23, _k22, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_r01, _k00, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_r02, _k01, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_r03, _k02, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_r11, _k10, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_r12, _k11, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_r13, _k12, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_r21, _k20, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_r22, _k21, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_r23, _k22, _sum0); __m256 _sum1 = _mm256_loadu_ps(outptr0 + 8); __m256 _r04 = _mm256_broadcast_ss(r0 + 3); @@ -405,15 +405,15 @@ static void conv3x3s1_pack1to8_avx(const Mat& bottom_blob, Mat& top_blob, const __m256 _r24 = _mm256_broadcast_ss(r2 + 3); _mm256_storeu_ps(outptr0, _sum0); - _sum1 = _mm256_fmadd_ps(_r02, _k00, _sum1); - _sum1 = _mm256_fmadd_ps(_r03, _k01, _sum1); - _sum1 = _mm256_fmadd_ps(_r04, _k02, _sum1); - _sum1 = _mm256_fmadd_ps(_r12, _k10, _sum1); - _sum1 = _mm256_fmadd_ps(_r13, _k11, _sum1); - _sum1 = _mm256_fmadd_ps(_r14, _k12, _sum1); - _sum1 = _mm256_fmadd_ps(_r22, _k20, _sum1); - _sum1 = _mm256_fmadd_ps(_r23, _k21, _sum1); - _sum1 = _mm256_fmadd_ps(_r24, _k22, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_r02, _k00, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_r03, _k01, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_r04, _k02, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_r12, _k10, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_r13, _k11, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_r14, _k12, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_r22, _k20, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_r23, _k21, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_r24, _k22, _sum1); __m256 _sum2 = _mm256_loadu_ps(outptr0 + 16); __m256 _r05 = _mm256_broadcast_ss(r0 + 4); @@ -421,15 +421,15 @@ static void conv3x3s1_pack1to8_avx(const Mat& bottom_blob, Mat& top_blob, const __m256 _r25 = _mm256_broadcast_ss(r2 + 4); _mm256_storeu_ps(outptr0 + 8, _sum1); - _sum2 = _mm256_fmadd_ps(_r03, _k00, _sum2); - _sum2 = _mm256_fmadd_ps(_r04, _k01, _sum2); - _sum2 = _mm256_fmadd_ps(_r05, _k02, _sum2); - _sum2 = _mm256_fmadd_ps(_r13, _k10, _sum2); - _sum2 = _mm256_fmadd_ps(_r14, _k11, _sum2); - _sum2 = _mm256_fmadd_ps(_r15, _k12, _sum2); - _sum2 = _mm256_fmadd_ps(_r23, _k20, _sum2); - _sum2 = _mm256_fmadd_ps(_r24, _k21, _sum2); - _sum2 = _mm256_fmadd_ps(_r25, _k22, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_r03, _k00, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_r04, _k01, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_r05, _k02, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_r13, _k10, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_r14, _k11, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_r15, _k12, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_r23, _k20, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_r24, _k21, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_r25, _k22, _sum2); __m256 _sum3 = _mm256_loadu_ps(outptr0 + 24); __m256 _r06 = _mm256_broadcast_ss(r0 + 5); @@ -437,15 +437,15 @@ static void conv3x3s1_pack1to8_avx(const Mat& bottom_blob, Mat& top_blob, const __m256 _r26 = _mm256_broadcast_ss(r2 + 5); _mm256_storeu_ps(outptr0 + 16, _sum2); - _sum3 = _mm256_fmadd_ps(_r04, _k00, _sum3); - _sum3 = _mm256_fmadd_ps(_r05, _k01, _sum3); - _sum3 = _mm256_fmadd_ps(_r06, _k02, _sum3); - _sum3 = _mm256_fmadd_ps(_r14, _k10, _sum3); - _sum3 = _mm256_fmadd_ps(_r15, _k11, _sum3); - _sum3 = _mm256_fmadd_ps(_r16, _k12, _sum3); - _sum3 = _mm256_fmadd_ps(_r24, _k20, _sum3); - _sum3 = _mm256_fmadd_ps(_r25, _k21, _sum3); - _sum3 = _mm256_fmadd_ps(_r26, _k22, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_r04, _k00, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_r05, _k01, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_r06, _k02, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_r14, _k10, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_r15, _k11, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_r16, _k12, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_r24, _k20, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_r25, _k21, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_r26, _k22, _sum3); _mm256_storeu_ps(outptr0 + 24, _sum3); @@ -468,15 +468,15 @@ static void conv3x3s1_pack1to8_avx(const Mat& bottom_blob, Mat& top_blob, const __m256 _r22 = _mm256_broadcast_ss(r2 + 1); __m256 _r23 = _mm256_broadcast_ss(r2 + 2); - _sum0 = _mm256_fmadd_ps(_r01, _k00, _sum0); - _sum0 = _mm256_fmadd_ps(_r02, _k01, _sum0); - _sum0 = _mm256_fmadd_ps(_r03, _k02, _sum0); - _sum0 = _mm256_fmadd_ps(_r11, _k10, _sum0); - _sum0 = _mm256_fmadd_ps(_r12, _k11, _sum0); - _sum0 = _mm256_fmadd_ps(_r13, _k12, _sum0); - _sum0 = _mm256_fmadd_ps(_r21, _k20, _sum0); - _sum0 = _mm256_fmadd_ps(_r22, _k21, _sum0); - _sum0 = _mm256_fmadd_ps(_r23, _k22, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_r01, _k00, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_r02, _k01, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_r03, _k02, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_r11, _k10, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_r12, _k11, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_r13, _k12, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_r21, _k20, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_r22, _k21, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_r23, _k22, _sum0); __m256 _sum1 = _mm256_loadu_ps(outptr0 + 8); __m256 _r04 = _mm256_broadcast_ss(r0 + 3); @@ -484,15 +484,15 @@ static void conv3x3s1_pack1to8_avx(const Mat& bottom_blob, Mat& top_blob, const __m256 _r24 = _mm256_broadcast_ss(r2 + 3); _mm256_storeu_ps(outptr0, _sum0); - _sum1 = _mm256_fmadd_ps(_r02, _k00, _sum1); - _sum1 = _mm256_fmadd_ps(_r03, _k01, _sum1); - _sum1 = _mm256_fmadd_ps(_r04, _k02, _sum1); - _sum1 = _mm256_fmadd_ps(_r12, _k10, _sum1); - _sum1 = _mm256_fmadd_ps(_r13, _k11, _sum1); - _sum1 = _mm256_fmadd_ps(_r14, _k12, _sum1); - _sum1 = _mm256_fmadd_ps(_r22, _k20, _sum1); - _sum1 = _mm256_fmadd_ps(_r23, _k21, _sum1); - _sum1 = _mm256_fmadd_ps(_r24, _k22, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_r02, _k00, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_r03, _k01, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_r04, _k02, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_r12, _k10, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_r13, _k11, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_r14, _k12, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_r22, _k20, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_r23, _k21, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_r24, _k22, _sum1); _mm256_storeu_ps(outptr0 + 8, _sum1); @@ -515,15 +515,15 @@ static void conv3x3s1_pack1to8_avx(const Mat& bottom_blob, Mat& top_blob, const __m256 _r22 = _mm256_broadcast_ss(r2 + 1); __m256 _r23 = _mm256_broadcast_ss(r2 + 2); - _sum0 = _mm256_fmadd_ps(_r01, _k00, _sum0); - _sum0 = _mm256_fmadd_ps(_r02, _k01, _sum0); - _sum0 = _mm256_fmadd_ps(_r03, _k02, _sum0); - _sum0 = _mm256_fmadd_ps(_r11, _k10, _sum0); - _sum0 = _mm256_fmadd_ps(_r12, _k11, _sum0); - _sum0 = _mm256_fmadd_ps(_r13, _k12, _sum0); - _sum0 = _mm256_fmadd_ps(_r21, _k20, _sum0); - _sum0 = _mm256_fmadd_ps(_r22, _k21, _sum0); - _sum0 = _mm256_fmadd_ps(_r23, _k22, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_r01, _k00, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_r02, _k01, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_r03, _k02, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_r11, _k10, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_r12, _k11, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_r13, _k12, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_r21, _k20, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_r22, _k21, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_r23, _k22, _sum0); _mm256_storeu_ps(outptr0, _sum0); r0 += 1; @@ -624,25 +624,25 @@ static void conv3x3s2_pack1to8_avx(const Mat& bottom_blob, Mat& top_blob, const __m256 _r22 = _mm256_broadcast_ss(r2 + 1); __m256 _r23 = _mm256_broadcast_ss(r2 + 2); - _sum00 = _mm256_fmadd_ps(_r01, _k00_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r02, _k01_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r03, _k02_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r11, _k10_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r12, _k11_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r13, _k12_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r21, _k20_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r22, _k21_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r23, _k22_0, _sum00); - - _sum10 = _mm256_fmadd_ps(_r01, _k00_1, _sum10); - _sum10 = _mm256_fmadd_ps(_r02, _k01_1, _sum10); - _sum10 = _mm256_fmadd_ps(_r03, _k02_1, _sum10); - _sum10 = _mm256_fmadd_ps(_r11, _k10_1, _sum10); - _sum10 = _mm256_fmadd_ps(_r12, _k11_1, _sum10); - _sum10 = _mm256_fmadd_ps(_r13, _k12_1, _sum10); - _sum10 = _mm256_fmadd_ps(_r21, _k20_1, _sum10); - _sum10 = _mm256_fmadd_ps(_r22, _k21_1, _sum10); - _sum10 = _mm256_fmadd_ps(_r23, _k22_1, _sum10); + _sum00 = _mm256_comp_fmadd_ps(_r01, _k00_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r02, _k01_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r03, _k02_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r11, _k10_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r12, _k11_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r13, _k12_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r21, _k20_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r22, _k21_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r23, _k22_0, _sum00); + + _sum10 = _mm256_comp_fmadd_ps(_r01, _k00_1, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_r02, _k01_1, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_r03, _k02_1, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_r11, _k10_1, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_r12, _k11_1, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_r13, _k12_1, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_r21, _k20_1, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_r22, _k21_1, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_r23, _k22_1, _sum10); _mm256_storeu_ps(outptr0, _sum00); _mm256_storeu_ps(outptr1, _sum10); @@ -657,25 +657,25 @@ static void conv3x3s2_pack1to8_avx(const Mat& bottom_blob, Mat& top_blob, const __m256 _r15 = _mm256_broadcast_ss(r1 + 4); __m256 _r25 = _mm256_broadcast_ss(r2 + 4); - _sum01 = _mm256_fmadd_ps(_r03, _k00_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r04, _k01_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r05, _k02_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r13, _k10_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r14, _k11_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r15, _k12_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r23, _k20_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r24, _k21_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r25, _k22_0, _sum01); - - _sum11 = _mm256_fmadd_ps(_r03, _k00_1, _sum11); - _sum11 = _mm256_fmadd_ps(_r04, _k01_1, _sum11); - _sum11 = _mm256_fmadd_ps(_r05, _k02_1, _sum11); - _sum11 = _mm256_fmadd_ps(_r13, _k10_1, _sum11); - _sum11 = _mm256_fmadd_ps(_r14, _k11_1, _sum11); - _sum11 = _mm256_fmadd_ps(_r15, _k12_1, _sum11); - _sum11 = _mm256_fmadd_ps(_r23, _k20_1, _sum11); - _sum11 = _mm256_fmadd_ps(_r24, _k21_1, _sum11); - _sum11 = _mm256_fmadd_ps(_r25, _k22_1, _sum11); + _sum01 = _mm256_comp_fmadd_ps(_r03, _k00_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r04, _k01_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r05, _k02_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r13, _k10_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r14, _k11_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r15, _k12_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r23, _k20_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r24, _k21_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r25, _k22_0, _sum01); + + _sum11 = _mm256_comp_fmadd_ps(_r03, _k00_1, _sum11); + _sum11 = _mm256_comp_fmadd_ps(_r04, _k01_1, _sum11); + _sum11 = _mm256_comp_fmadd_ps(_r05, _k02_1, _sum11); + _sum11 = _mm256_comp_fmadd_ps(_r13, _k10_1, _sum11); + _sum11 = _mm256_comp_fmadd_ps(_r14, _k11_1, _sum11); + _sum11 = _mm256_comp_fmadd_ps(_r15, _k12_1, _sum11); + _sum11 = _mm256_comp_fmadd_ps(_r23, _k20_1, _sum11); + _sum11 = _mm256_comp_fmadd_ps(_r24, _k21_1, _sum11); + _sum11 = _mm256_comp_fmadd_ps(_r25, _k22_1, _sum11); _mm256_storeu_ps(outptr0 + 8, _sum01); _mm256_storeu_ps(outptr1 + 8, _sum11); @@ -690,25 +690,25 @@ static void conv3x3s2_pack1to8_avx(const Mat& bottom_blob, Mat& top_blob, const __m256 _r17 = _mm256_broadcast_ss(r1 + 6); __m256 _r27 = _mm256_broadcast_ss(r2 + 6); - _sum02 = _mm256_fmadd_ps(_r05, _k00_0, _sum02); - _sum02 = _mm256_fmadd_ps(_r06, _k01_0, _sum02); - _sum02 = _mm256_fmadd_ps(_r07, _k02_0, _sum02); - _sum02 = _mm256_fmadd_ps(_r15, _k10_0, _sum02); - _sum02 = _mm256_fmadd_ps(_r16, _k11_0, _sum02); - _sum02 = _mm256_fmadd_ps(_r17, _k12_0, _sum02); - _sum02 = _mm256_fmadd_ps(_r25, _k20_0, _sum02); - _sum02 = _mm256_fmadd_ps(_r26, _k21_0, _sum02); - _sum02 = _mm256_fmadd_ps(_r27, _k22_0, _sum02); - - _sum12 = _mm256_fmadd_ps(_r05, _k00_1, _sum12); - _sum12 = _mm256_fmadd_ps(_r06, _k01_1, _sum12); - _sum12 = _mm256_fmadd_ps(_r07, _k02_1, _sum12); - _sum12 = _mm256_fmadd_ps(_r15, _k10_1, _sum12); - _sum12 = _mm256_fmadd_ps(_r16, _k11_1, _sum12); - _sum12 = _mm256_fmadd_ps(_r17, _k12_1, _sum12); - _sum12 = _mm256_fmadd_ps(_r25, _k20_1, _sum12); - _sum12 = _mm256_fmadd_ps(_r26, _k21_1, _sum12); - _sum12 = _mm256_fmadd_ps(_r27, _k22_1, _sum12); + _sum02 = _mm256_comp_fmadd_ps(_r05, _k00_0, _sum02); + _sum02 = _mm256_comp_fmadd_ps(_r06, _k01_0, _sum02); + _sum02 = _mm256_comp_fmadd_ps(_r07, _k02_0, _sum02); + _sum02 = _mm256_comp_fmadd_ps(_r15, _k10_0, _sum02); + _sum02 = _mm256_comp_fmadd_ps(_r16, _k11_0, _sum02); + _sum02 = _mm256_comp_fmadd_ps(_r17, _k12_0, _sum02); + _sum02 = _mm256_comp_fmadd_ps(_r25, _k20_0, _sum02); + _sum02 = _mm256_comp_fmadd_ps(_r26, _k21_0, _sum02); + _sum02 = _mm256_comp_fmadd_ps(_r27, _k22_0, _sum02); + + _sum12 = _mm256_comp_fmadd_ps(_r05, _k00_1, _sum12); + _sum12 = _mm256_comp_fmadd_ps(_r06, _k01_1, _sum12); + _sum12 = _mm256_comp_fmadd_ps(_r07, _k02_1, _sum12); + _sum12 = _mm256_comp_fmadd_ps(_r15, _k10_1, _sum12); + _sum12 = _mm256_comp_fmadd_ps(_r16, _k11_1, _sum12); + _sum12 = _mm256_comp_fmadd_ps(_r17, _k12_1, _sum12); + _sum12 = _mm256_comp_fmadd_ps(_r25, _k20_1, _sum12); + _sum12 = _mm256_comp_fmadd_ps(_r26, _k21_1, _sum12); + _sum12 = _mm256_comp_fmadd_ps(_r27, _k22_1, _sum12); _mm256_storeu_ps(outptr0 + 16, _sum02); _mm256_storeu_ps(outptr1 + 16, _sum12); @@ -723,25 +723,25 @@ static void conv3x3s2_pack1to8_avx(const Mat& bottom_blob, Mat& top_blob, const __m256 _sum03 = _mm256_loadu_ps(outptr0 + 24); __m256 _sum13 = _mm256_loadu_ps(outptr1 + 24); - _sum03 = _mm256_fmadd_ps(_r07, _k00_0, _sum03); - _sum03 = _mm256_fmadd_ps(_r08, _k01_0, _sum03); - _sum03 = _mm256_fmadd_ps(_r09, _k02_0, _sum03); - _sum03 = _mm256_fmadd_ps(_r17, _k10_0, _sum03); - _sum03 = _mm256_fmadd_ps(_r18, _k11_0, _sum03); - _sum03 = _mm256_fmadd_ps(_r19, _k12_0, _sum03); - _sum03 = _mm256_fmadd_ps(_r27, _k20_0, _sum03); - _sum03 = _mm256_fmadd_ps(_r28, _k21_0, _sum03); - _sum03 = _mm256_fmadd_ps(_r29, _k22_0, _sum03); - - _sum13 = _mm256_fmadd_ps(_r07, _k00_1, _sum13); - _sum13 = _mm256_fmadd_ps(_r08, _k01_1, _sum13); - _sum13 = _mm256_fmadd_ps(_r09, _k02_1, _sum13); - _sum13 = _mm256_fmadd_ps(_r17, _k10_1, _sum13); - _sum13 = _mm256_fmadd_ps(_r18, _k11_1, _sum13); - _sum13 = _mm256_fmadd_ps(_r19, _k12_1, _sum13); - _sum13 = _mm256_fmadd_ps(_r27, _k20_1, _sum13); - _sum13 = _mm256_fmadd_ps(_r28, _k21_1, _sum13); - _sum13 = _mm256_fmadd_ps(_r29, _k22_1, _sum13); + _sum03 = _mm256_comp_fmadd_ps(_r07, _k00_0, _sum03); + _sum03 = _mm256_comp_fmadd_ps(_r08, _k01_0, _sum03); + _sum03 = _mm256_comp_fmadd_ps(_r09, _k02_0, _sum03); + _sum03 = _mm256_comp_fmadd_ps(_r17, _k10_0, _sum03); + _sum03 = _mm256_comp_fmadd_ps(_r18, _k11_0, _sum03); + _sum03 = _mm256_comp_fmadd_ps(_r19, _k12_0, _sum03); + _sum03 = _mm256_comp_fmadd_ps(_r27, _k20_0, _sum03); + _sum03 = _mm256_comp_fmadd_ps(_r28, _k21_0, _sum03); + _sum03 = _mm256_comp_fmadd_ps(_r29, _k22_0, _sum03); + + _sum13 = _mm256_comp_fmadd_ps(_r07, _k00_1, _sum13); + _sum13 = _mm256_comp_fmadd_ps(_r08, _k01_1, _sum13); + _sum13 = _mm256_comp_fmadd_ps(_r09, _k02_1, _sum13); + _sum13 = _mm256_comp_fmadd_ps(_r17, _k10_1, _sum13); + _sum13 = _mm256_comp_fmadd_ps(_r18, _k11_1, _sum13); + _sum13 = _mm256_comp_fmadd_ps(_r19, _k12_1, _sum13); + _sum13 = _mm256_comp_fmadd_ps(_r27, _k20_1, _sum13); + _sum13 = _mm256_comp_fmadd_ps(_r28, _k21_1, _sum13); + _sum13 = _mm256_comp_fmadd_ps(_r29, _k22_1, _sum13); _mm256_storeu_ps(outptr0 + 24, _sum03); _mm256_storeu_ps(outptr1 + 24, _sum13); @@ -756,25 +756,25 @@ static void conv3x3s2_pack1to8_avx(const Mat& bottom_blob, Mat& top_blob, const __m256 _sum04 = _mm256_loadu_ps(outptr0 + 32); __m256 _sum14 = _mm256_loadu_ps(outptr1 + 32); - _sum04 = _mm256_fmadd_ps(_r09, _k00_0, _sum04); - _sum04 = _mm256_fmadd_ps(_r010, _k01_0, _sum04); - _sum04 = _mm256_fmadd_ps(_r011, _k02_0, _sum04); - _sum04 = _mm256_fmadd_ps(_r19, _k10_0, _sum04); - _sum04 = _mm256_fmadd_ps(_r110, _k11_0, _sum04); - _sum04 = _mm256_fmadd_ps(_r111, _k12_0, _sum04); - _sum04 = _mm256_fmadd_ps(_r29, _k20_0, _sum04); - _sum04 = _mm256_fmadd_ps(_r210, _k21_0, _sum04); - _sum04 = _mm256_fmadd_ps(_r211, _k22_0, _sum04); - - _sum14 = _mm256_fmadd_ps(_r09, _k00_1, _sum14); - _sum14 = _mm256_fmadd_ps(_r010, _k01_1, _sum14); - _sum14 = _mm256_fmadd_ps(_r011, _k02_1, _sum14); - _sum14 = _mm256_fmadd_ps(_r19, _k10_1, _sum14); - _sum14 = _mm256_fmadd_ps(_r110, _k11_1, _sum14); - _sum14 = _mm256_fmadd_ps(_r111, _k12_1, _sum14); - _sum14 = _mm256_fmadd_ps(_r29, _k20_1, _sum14); - _sum14 = _mm256_fmadd_ps(_r210, _k21_1, _sum14); - _sum14 = _mm256_fmadd_ps(_r211, _k22_1, _sum14); + _sum04 = _mm256_comp_fmadd_ps(_r09, _k00_0, _sum04); + _sum04 = _mm256_comp_fmadd_ps(_r010, _k01_0, _sum04); + _sum04 = _mm256_comp_fmadd_ps(_r011, _k02_0, _sum04); + _sum04 = _mm256_comp_fmadd_ps(_r19, _k10_0, _sum04); + _sum04 = _mm256_comp_fmadd_ps(_r110, _k11_0, _sum04); + _sum04 = _mm256_comp_fmadd_ps(_r111, _k12_0, _sum04); + _sum04 = _mm256_comp_fmadd_ps(_r29, _k20_0, _sum04); + _sum04 = _mm256_comp_fmadd_ps(_r210, _k21_0, _sum04); + _sum04 = _mm256_comp_fmadd_ps(_r211, _k22_0, _sum04); + + _sum14 = _mm256_comp_fmadd_ps(_r09, _k00_1, _sum14); + _sum14 = _mm256_comp_fmadd_ps(_r010, _k01_1, _sum14); + _sum14 = _mm256_comp_fmadd_ps(_r011, _k02_1, _sum14); + _sum14 = _mm256_comp_fmadd_ps(_r19, _k10_1, _sum14); + _sum14 = _mm256_comp_fmadd_ps(_r110, _k11_1, _sum14); + _sum14 = _mm256_comp_fmadd_ps(_r111, _k12_1, _sum14); + _sum14 = _mm256_comp_fmadd_ps(_r29, _k20_1, _sum14); + _sum14 = _mm256_comp_fmadd_ps(_r210, _k21_1, _sum14); + _sum14 = _mm256_comp_fmadd_ps(_r211, _k22_1, _sum14); _mm256_storeu_ps(outptr0 + 32, _sum04); _mm256_storeu_ps(outptr1 + 32, _sum14); @@ -789,24 +789,24 @@ static void conv3x3s2_pack1to8_avx(const Mat& bottom_blob, Mat& top_blob, const __m256 _sum05 = _mm256_loadu_ps(outptr0 + 40); __m256 _sum15 = _mm256_loadu_ps(outptr1 + 40); - _sum05 = _mm256_fmadd_ps(_r011, _k00_0, _sum05); - _sum05 = _mm256_fmadd_ps(_r012, _k01_0, _sum05); - _sum05 = _mm256_fmadd_ps(_r013, _k02_0, _sum05); - _sum05 = _mm256_fmadd_ps(_r111, _k10_0, _sum05); - _sum05 = _mm256_fmadd_ps(_r112, _k11_0, _sum05); - _sum05 = _mm256_fmadd_ps(_r113, _k12_0, _sum05); - _sum05 = _mm256_fmadd_ps(_r211, _k20_0, _sum05); - _sum05 = _mm256_fmadd_ps(_r212, _k21_0, _sum05); - _sum05 = _mm256_fmadd_ps(_r213, _k22_0, _sum05); - _sum15 = _mm256_fmadd_ps(_r011, _k00_1, _sum15); - _sum15 = _mm256_fmadd_ps(_r012, _k01_1, _sum15); - _sum15 = _mm256_fmadd_ps(_r013, _k02_1, _sum15); - _sum15 = _mm256_fmadd_ps(_r111, _k10_1, _sum15); - _sum15 = _mm256_fmadd_ps(_r112, _k11_1, _sum15); - _sum15 = _mm256_fmadd_ps(_r113, _k12_1, _sum15); - _sum15 = _mm256_fmadd_ps(_r211, _k20_1, _sum15); - _sum15 = _mm256_fmadd_ps(_r212, _k21_1, _sum15); - _sum15 = _mm256_fmadd_ps(_r213, _k22_1, _sum15); + _sum05 = _mm256_comp_fmadd_ps(_r011, _k00_0, _sum05); + _sum05 = _mm256_comp_fmadd_ps(_r012, _k01_0, _sum05); + _sum05 = _mm256_comp_fmadd_ps(_r013, _k02_0, _sum05); + _sum05 = _mm256_comp_fmadd_ps(_r111, _k10_0, _sum05); + _sum05 = _mm256_comp_fmadd_ps(_r112, _k11_0, _sum05); + _sum05 = _mm256_comp_fmadd_ps(_r113, _k12_0, _sum05); + _sum05 = _mm256_comp_fmadd_ps(_r211, _k20_0, _sum05); + _sum05 = _mm256_comp_fmadd_ps(_r212, _k21_0, _sum05); + _sum05 = _mm256_comp_fmadd_ps(_r213, _k22_0, _sum05); + _sum15 = _mm256_comp_fmadd_ps(_r011, _k00_1, _sum15); + _sum15 = _mm256_comp_fmadd_ps(_r012, _k01_1, _sum15); + _sum15 = _mm256_comp_fmadd_ps(_r013, _k02_1, _sum15); + _sum15 = _mm256_comp_fmadd_ps(_r111, _k10_1, _sum15); + _sum15 = _mm256_comp_fmadd_ps(_r112, _k11_1, _sum15); + _sum15 = _mm256_comp_fmadd_ps(_r113, _k12_1, _sum15); + _sum15 = _mm256_comp_fmadd_ps(_r211, _k20_1, _sum15); + _sum15 = _mm256_comp_fmadd_ps(_r212, _k21_1, _sum15); + _sum15 = _mm256_comp_fmadd_ps(_r213, _k22_1, _sum15); _mm256_storeu_ps(outptr0 + 40, _sum05); _mm256_storeu_ps(outptr1 + 40, _sum15); @@ -821,24 +821,24 @@ static void conv3x3s2_pack1to8_avx(const Mat& bottom_blob, Mat& top_blob, const __m256 _sum06 = _mm256_loadu_ps(outptr0 + 48); __m256 _sum16 = _mm256_loadu_ps(outptr1 + 48); - _sum06 = _mm256_fmadd_ps(_r013, _k00_0, _sum06); - _sum06 = _mm256_fmadd_ps(_r014, _k01_0, _sum06); - _sum06 = _mm256_fmadd_ps(_r015, _k02_0, _sum06); - _sum06 = _mm256_fmadd_ps(_r113, _k10_0, _sum06); - _sum06 = _mm256_fmadd_ps(_r114, _k11_0, _sum06); - _sum06 = _mm256_fmadd_ps(_r115, _k12_0, _sum06); - _sum06 = _mm256_fmadd_ps(_r213, _k20_0, _sum06); - _sum06 = _mm256_fmadd_ps(_r214, _k21_0, _sum06); - _sum06 = _mm256_fmadd_ps(_r215, _k22_0, _sum06); - _sum16 = _mm256_fmadd_ps(_r013, _k00_1, _sum16); - _sum16 = _mm256_fmadd_ps(_r014, _k01_1, _sum16); - _sum16 = _mm256_fmadd_ps(_r015, _k02_1, _sum16); - _sum16 = _mm256_fmadd_ps(_r113, _k10_1, _sum16); - _sum16 = _mm256_fmadd_ps(_r114, _k11_1, _sum16); - _sum16 = _mm256_fmadd_ps(_r115, _k12_1, _sum16); - _sum16 = _mm256_fmadd_ps(_r213, _k20_1, _sum16); - _sum16 = _mm256_fmadd_ps(_r214, _k21_1, _sum16); - _sum16 = _mm256_fmadd_ps(_r215, _k22_1, _sum16); + _sum06 = _mm256_comp_fmadd_ps(_r013, _k00_0, _sum06); + _sum06 = _mm256_comp_fmadd_ps(_r014, _k01_0, _sum06); + _sum06 = _mm256_comp_fmadd_ps(_r015, _k02_0, _sum06); + _sum06 = _mm256_comp_fmadd_ps(_r113, _k10_0, _sum06); + _sum06 = _mm256_comp_fmadd_ps(_r114, _k11_0, _sum06); + _sum06 = _mm256_comp_fmadd_ps(_r115, _k12_0, _sum06); + _sum06 = _mm256_comp_fmadd_ps(_r213, _k20_0, _sum06); + _sum06 = _mm256_comp_fmadd_ps(_r214, _k21_0, _sum06); + _sum06 = _mm256_comp_fmadd_ps(_r215, _k22_0, _sum06); + _sum16 = _mm256_comp_fmadd_ps(_r013, _k00_1, _sum16); + _sum16 = _mm256_comp_fmadd_ps(_r014, _k01_1, _sum16); + _sum16 = _mm256_comp_fmadd_ps(_r015, _k02_1, _sum16); + _sum16 = _mm256_comp_fmadd_ps(_r113, _k10_1, _sum16); + _sum16 = _mm256_comp_fmadd_ps(_r114, _k11_1, _sum16); + _sum16 = _mm256_comp_fmadd_ps(_r115, _k12_1, _sum16); + _sum16 = _mm256_comp_fmadd_ps(_r213, _k20_1, _sum16); + _sum16 = _mm256_comp_fmadd_ps(_r214, _k21_1, _sum16); + _sum16 = _mm256_comp_fmadd_ps(_r215, _k22_1, _sum16); _mm256_storeu_ps(outptr0 + 48, _sum06); _mm256_storeu_ps(outptr1 + 48, _sum16); @@ -853,24 +853,24 @@ static void conv3x3s2_pack1to8_avx(const Mat& bottom_blob, Mat& top_blob, const __m256 _sum07 = _mm256_loadu_ps(outptr0 + 56); __m256 _sum17 = _mm256_loadu_ps(outptr1 + 56); - _sum07 = _mm256_fmadd_ps(_r015, _k00_0, _sum07); - _sum07 = _mm256_fmadd_ps(_r016, _k01_0, _sum07); - _sum07 = _mm256_fmadd_ps(_r017, _k02_0, _sum07); - _sum07 = _mm256_fmadd_ps(_r115, _k10_0, _sum07); - _sum07 = _mm256_fmadd_ps(_r116, _k11_0, _sum07); - _sum07 = _mm256_fmadd_ps(_r117, _k12_0, _sum07); - _sum07 = _mm256_fmadd_ps(_r215, _k20_0, _sum07); - _sum07 = _mm256_fmadd_ps(_r216, _k21_0, _sum07); - _sum07 = _mm256_fmadd_ps(_r217, _k22_0, _sum07); - _sum17 = _mm256_fmadd_ps(_r015, _k00_1, _sum17); - _sum17 = _mm256_fmadd_ps(_r016, _k01_1, _sum17); - _sum17 = _mm256_fmadd_ps(_r017, _k02_1, _sum17); - _sum17 = _mm256_fmadd_ps(_r115, _k10_1, _sum17); - _sum17 = _mm256_fmadd_ps(_r116, _k11_1, _sum17); - _sum17 = _mm256_fmadd_ps(_r117, _k12_1, _sum17); - _sum17 = _mm256_fmadd_ps(_r215, _k20_1, _sum17); - _sum17 = _mm256_fmadd_ps(_r216, _k21_1, _sum17); - _sum17 = _mm256_fmadd_ps(_r217, _k22_1, _sum17); + _sum07 = _mm256_comp_fmadd_ps(_r015, _k00_0, _sum07); + _sum07 = _mm256_comp_fmadd_ps(_r016, _k01_0, _sum07); + _sum07 = _mm256_comp_fmadd_ps(_r017, _k02_0, _sum07); + _sum07 = _mm256_comp_fmadd_ps(_r115, _k10_0, _sum07); + _sum07 = _mm256_comp_fmadd_ps(_r116, _k11_0, _sum07); + _sum07 = _mm256_comp_fmadd_ps(_r117, _k12_0, _sum07); + _sum07 = _mm256_comp_fmadd_ps(_r215, _k20_0, _sum07); + _sum07 = _mm256_comp_fmadd_ps(_r216, _k21_0, _sum07); + _sum07 = _mm256_comp_fmadd_ps(_r217, _k22_0, _sum07); + _sum17 = _mm256_comp_fmadd_ps(_r015, _k00_1, _sum17); + _sum17 = _mm256_comp_fmadd_ps(_r016, _k01_1, _sum17); + _sum17 = _mm256_comp_fmadd_ps(_r017, _k02_1, _sum17); + _sum17 = _mm256_comp_fmadd_ps(_r115, _k10_1, _sum17); + _sum17 = _mm256_comp_fmadd_ps(_r116, _k11_1, _sum17); + _sum17 = _mm256_comp_fmadd_ps(_r117, _k12_1, _sum17); + _sum17 = _mm256_comp_fmadd_ps(_r215, _k20_1, _sum17); + _sum17 = _mm256_comp_fmadd_ps(_r216, _k21_1, _sum17); + _sum17 = _mm256_comp_fmadd_ps(_r217, _k22_1, _sum17); _mm256_storeu_ps(outptr0 + 56, _sum07); _mm256_storeu_ps(outptr1 + 56, _sum17); @@ -897,25 +897,25 @@ static void conv3x3s2_pack1to8_avx(const Mat& bottom_blob, Mat& top_blob, const __m256 _r22 = _mm256_broadcast_ss(r2 + 1); __m256 _r23 = _mm256_broadcast_ss(r2 + 2); - _sum00 = _mm256_fmadd_ps(_r01, _k00_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r02, _k01_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r03, _k02_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r11, _k10_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r12, _k11_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r13, _k12_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r21, _k20_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r22, _k21_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r23, _k22_0, _sum00); - - _sum10 = _mm256_fmadd_ps(_r01, _k00_1, _sum10); - _sum10 = _mm256_fmadd_ps(_r02, _k01_1, _sum10); - _sum10 = _mm256_fmadd_ps(_r03, _k02_1, _sum10); - _sum10 = _mm256_fmadd_ps(_r11, _k10_1, _sum10); - _sum10 = _mm256_fmadd_ps(_r12, _k11_1, _sum10); - _sum10 = _mm256_fmadd_ps(_r13, _k12_1, _sum10); - _sum10 = _mm256_fmadd_ps(_r21, _k20_1, _sum10); - _sum10 = _mm256_fmadd_ps(_r22, _k21_1, _sum10); - _sum10 = _mm256_fmadd_ps(_r23, _k22_1, _sum10); + _sum00 = _mm256_comp_fmadd_ps(_r01, _k00_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r02, _k01_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r03, _k02_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r11, _k10_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r12, _k11_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r13, _k12_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r21, _k20_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r22, _k21_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r23, _k22_0, _sum00); + + _sum10 = _mm256_comp_fmadd_ps(_r01, _k00_1, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_r02, _k01_1, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_r03, _k02_1, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_r11, _k10_1, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_r12, _k11_1, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_r13, _k12_1, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_r21, _k20_1, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_r22, _k21_1, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_r23, _k22_1, _sum10); _mm256_storeu_ps(outptr0, _sum00); _mm256_storeu_ps(outptr1, _sum10); @@ -930,25 +930,25 @@ static void conv3x3s2_pack1to8_avx(const Mat& bottom_blob, Mat& top_blob, const __m256 _r15 = _mm256_broadcast_ss(r1 + 4); __m256 _r25 = _mm256_broadcast_ss(r2 + 4); - _sum01 = _mm256_fmadd_ps(_r03, _k00_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r04, _k01_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r05, _k02_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r13, _k10_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r14, _k11_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r15, _k12_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r23, _k20_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r24, _k21_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r25, _k22_0, _sum01); - - _sum11 = _mm256_fmadd_ps(_r03, _k00_1, _sum11); - _sum11 = _mm256_fmadd_ps(_r04, _k01_1, _sum11); - _sum11 = _mm256_fmadd_ps(_r05, _k02_1, _sum11); - _sum11 = _mm256_fmadd_ps(_r13, _k10_1, _sum11); - _sum11 = _mm256_fmadd_ps(_r14, _k11_1, _sum11); - _sum11 = _mm256_fmadd_ps(_r15, _k12_1, _sum11); - _sum11 = _mm256_fmadd_ps(_r23, _k20_1, _sum11); - _sum11 = _mm256_fmadd_ps(_r24, _k21_1, _sum11); - _sum11 = _mm256_fmadd_ps(_r25, _k22_1, _sum11); + _sum01 = _mm256_comp_fmadd_ps(_r03, _k00_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r04, _k01_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r05, _k02_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r13, _k10_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r14, _k11_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r15, _k12_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r23, _k20_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r24, _k21_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r25, _k22_0, _sum01); + + _sum11 = _mm256_comp_fmadd_ps(_r03, _k00_1, _sum11); + _sum11 = _mm256_comp_fmadd_ps(_r04, _k01_1, _sum11); + _sum11 = _mm256_comp_fmadd_ps(_r05, _k02_1, _sum11); + _sum11 = _mm256_comp_fmadd_ps(_r13, _k10_1, _sum11); + _sum11 = _mm256_comp_fmadd_ps(_r14, _k11_1, _sum11); + _sum11 = _mm256_comp_fmadd_ps(_r15, _k12_1, _sum11); + _sum11 = _mm256_comp_fmadd_ps(_r23, _k20_1, _sum11); + _sum11 = _mm256_comp_fmadd_ps(_r24, _k21_1, _sum11); + _sum11 = _mm256_comp_fmadd_ps(_r25, _k22_1, _sum11); _mm256_storeu_ps(outptr0 + 8, _sum01); _mm256_storeu_ps(outptr1 + 8, _sum11); @@ -963,25 +963,25 @@ static void conv3x3s2_pack1to8_avx(const Mat& bottom_blob, Mat& top_blob, const __m256 _r17 = _mm256_broadcast_ss(r1 + 6); __m256 _r27 = _mm256_broadcast_ss(r2 + 6); - _sum02 = _mm256_fmadd_ps(_r05, _k00_0, _sum02); - _sum02 = _mm256_fmadd_ps(_r06, _k01_0, _sum02); - _sum02 = _mm256_fmadd_ps(_r07, _k02_0, _sum02); - _sum02 = _mm256_fmadd_ps(_r15, _k10_0, _sum02); - _sum02 = _mm256_fmadd_ps(_r16, _k11_0, _sum02); - _sum02 = _mm256_fmadd_ps(_r17, _k12_0, _sum02); - _sum02 = _mm256_fmadd_ps(_r25, _k20_0, _sum02); - _sum02 = _mm256_fmadd_ps(_r26, _k21_0, _sum02); - _sum02 = _mm256_fmadd_ps(_r27, _k22_0, _sum02); - - _sum12 = _mm256_fmadd_ps(_r05, _k00_1, _sum12); - _sum12 = _mm256_fmadd_ps(_r06, _k01_1, _sum12); - _sum12 = _mm256_fmadd_ps(_r07, _k02_1, _sum12); - _sum12 = _mm256_fmadd_ps(_r15, _k10_1, _sum12); - _sum12 = _mm256_fmadd_ps(_r16, _k11_1, _sum12); - _sum12 = _mm256_fmadd_ps(_r17, _k12_1, _sum12); - _sum12 = _mm256_fmadd_ps(_r25, _k20_1, _sum12); - _sum12 = _mm256_fmadd_ps(_r26, _k21_1, _sum12); - _sum12 = _mm256_fmadd_ps(_r27, _k22_1, _sum12); + _sum02 = _mm256_comp_fmadd_ps(_r05, _k00_0, _sum02); + _sum02 = _mm256_comp_fmadd_ps(_r06, _k01_0, _sum02); + _sum02 = _mm256_comp_fmadd_ps(_r07, _k02_0, _sum02); + _sum02 = _mm256_comp_fmadd_ps(_r15, _k10_0, _sum02); + _sum02 = _mm256_comp_fmadd_ps(_r16, _k11_0, _sum02); + _sum02 = _mm256_comp_fmadd_ps(_r17, _k12_0, _sum02); + _sum02 = _mm256_comp_fmadd_ps(_r25, _k20_0, _sum02); + _sum02 = _mm256_comp_fmadd_ps(_r26, _k21_0, _sum02); + _sum02 = _mm256_comp_fmadd_ps(_r27, _k22_0, _sum02); + + _sum12 = _mm256_comp_fmadd_ps(_r05, _k00_1, _sum12); + _sum12 = _mm256_comp_fmadd_ps(_r06, _k01_1, _sum12); + _sum12 = _mm256_comp_fmadd_ps(_r07, _k02_1, _sum12); + _sum12 = _mm256_comp_fmadd_ps(_r15, _k10_1, _sum12); + _sum12 = _mm256_comp_fmadd_ps(_r16, _k11_1, _sum12); + _sum12 = _mm256_comp_fmadd_ps(_r17, _k12_1, _sum12); + _sum12 = _mm256_comp_fmadd_ps(_r25, _k20_1, _sum12); + _sum12 = _mm256_comp_fmadd_ps(_r26, _k21_1, _sum12); + _sum12 = _mm256_comp_fmadd_ps(_r27, _k22_1, _sum12); _mm256_storeu_ps(outptr0 + 16, _sum02); _mm256_storeu_ps(outptr1 + 16, _sum12); @@ -996,25 +996,25 @@ static void conv3x3s2_pack1to8_avx(const Mat& bottom_blob, Mat& top_blob, const __m256 _sum03 = _mm256_loadu_ps(outptr0 + 24); __m256 _sum13 = _mm256_loadu_ps(outptr1 + 24); - _sum03 = _mm256_fmadd_ps(_r07, _k00_0, _sum03); - _sum03 = _mm256_fmadd_ps(_r08, _k01_0, _sum03); - _sum03 = _mm256_fmadd_ps(_r09, _k02_0, _sum03); - _sum03 = _mm256_fmadd_ps(_r17, _k10_0, _sum03); - _sum03 = _mm256_fmadd_ps(_r18, _k11_0, _sum03); - _sum03 = _mm256_fmadd_ps(_r19, _k12_0, _sum03); - _sum03 = _mm256_fmadd_ps(_r27, _k20_0, _sum03); - _sum03 = _mm256_fmadd_ps(_r28, _k21_0, _sum03); - _sum03 = _mm256_fmadd_ps(_r29, _k22_0, _sum03); - - _sum13 = _mm256_fmadd_ps(_r07, _k00_1, _sum13); - _sum13 = _mm256_fmadd_ps(_r08, _k01_1, _sum13); - _sum13 = _mm256_fmadd_ps(_r09, _k02_1, _sum13); - _sum13 = _mm256_fmadd_ps(_r17, _k10_1, _sum13); - _sum13 = _mm256_fmadd_ps(_r18, _k11_1, _sum13); - _sum13 = _mm256_fmadd_ps(_r19, _k12_1, _sum13); - _sum13 = _mm256_fmadd_ps(_r27, _k20_1, _sum13); - _sum13 = _mm256_fmadd_ps(_r28, _k21_1, _sum13); - _sum13 = _mm256_fmadd_ps(_r29, _k22_1, _sum13); + _sum03 = _mm256_comp_fmadd_ps(_r07, _k00_0, _sum03); + _sum03 = _mm256_comp_fmadd_ps(_r08, _k01_0, _sum03); + _sum03 = _mm256_comp_fmadd_ps(_r09, _k02_0, _sum03); + _sum03 = _mm256_comp_fmadd_ps(_r17, _k10_0, _sum03); + _sum03 = _mm256_comp_fmadd_ps(_r18, _k11_0, _sum03); + _sum03 = _mm256_comp_fmadd_ps(_r19, _k12_0, _sum03); + _sum03 = _mm256_comp_fmadd_ps(_r27, _k20_0, _sum03); + _sum03 = _mm256_comp_fmadd_ps(_r28, _k21_0, _sum03); + _sum03 = _mm256_comp_fmadd_ps(_r29, _k22_0, _sum03); + + _sum13 = _mm256_comp_fmadd_ps(_r07, _k00_1, _sum13); + _sum13 = _mm256_comp_fmadd_ps(_r08, _k01_1, _sum13); + _sum13 = _mm256_comp_fmadd_ps(_r09, _k02_1, _sum13); + _sum13 = _mm256_comp_fmadd_ps(_r17, _k10_1, _sum13); + _sum13 = _mm256_comp_fmadd_ps(_r18, _k11_1, _sum13); + _sum13 = _mm256_comp_fmadd_ps(_r19, _k12_1, _sum13); + _sum13 = _mm256_comp_fmadd_ps(_r27, _k20_1, _sum13); + _sum13 = _mm256_comp_fmadd_ps(_r28, _k21_1, _sum13); + _sum13 = _mm256_comp_fmadd_ps(_r29, _k22_1, _sum13); _mm256_storeu_ps(outptr0 + 24, _sum03); _mm256_storeu_ps(outptr1 + 24, _sum13); @@ -1040,25 +1040,25 @@ static void conv3x3s2_pack1to8_avx(const Mat& bottom_blob, Mat& top_blob, const __m256 _r22 = _mm256_broadcast_ss(r2 + 1); __m256 _r23 = _mm256_broadcast_ss(r2 + 2); - _sum00 = _mm256_fmadd_ps(_r01, _k00_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r02, _k01_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r03, _k02_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r11, _k10_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r12, _k11_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r13, _k12_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r21, _k20_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r22, _k21_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r23, _k22_0, _sum00); - - _sum10 = _mm256_fmadd_ps(_r01, _k00_1, _sum10); - _sum10 = _mm256_fmadd_ps(_r02, _k01_1, _sum10); - _sum10 = _mm256_fmadd_ps(_r03, _k02_1, _sum10); - _sum10 = _mm256_fmadd_ps(_r11, _k10_1, _sum10); - _sum10 = _mm256_fmadd_ps(_r12, _k11_1, _sum10); - _sum10 = _mm256_fmadd_ps(_r13, _k12_1, _sum10); - _sum10 = _mm256_fmadd_ps(_r21, _k20_1, _sum10); - _sum10 = _mm256_fmadd_ps(_r22, _k21_1, _sum10); - _sum10 = _mm256_fmadd_ps(_r23, _k22_1, _sum10); + _sum00 = _mm256_comp_fmadd_ps(_r01, _k00_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r02, _k01_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r03, _k02_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r11, _k10_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r12, _k11_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r13, _k12_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r21, _k20_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r22, _k21_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r23, _k22_0, _sum00); + + _sum10 = _mm256_comp_fmadd_ps(_r01, _k00_1, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_r02, _k01_1, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_r03, _k02_1, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_r11, _k10_1, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_r12, _k11_1, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_r13, _k12_1, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_r21, _k20_1, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_r22, _k21_1, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_r23, _k22_1, _sum10); _mm256_storeu_ps(outptr0, _sum00); _mm256_storeu_ps(outptr1, _sum10); @@ -1073,25 +1073,25 @@ static void conv3x3s2_pack1to8_avx(const Mat& bottom_blob, Mat& top_blob, const __m256 _r15 = _mm256_broadcast_ss(r1 + 4); __m256 _r25 = _mm256_broadcast_ss(r2 + 4); - _sum01 = _mm256_fmadd_ps(_r03, _k00_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r04, _k01_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r05, _k02_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r13, _k10_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r14, _k11_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r15, _k12_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r23, _k20_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r24, _k21_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r25, _k22_0, _sum01); - - _sum11 = _mm256_fmadd_ps(_r03, _k00_1, _sum11); - _sum11 = _mm256_fmadd_ps(_r04, _k01_1, _sum11); - _sum11 = _mm256_fmadd_ps(_r05, _k02_1, _sum11); - _sum11 = _mm256_fmadd_ps(_r13, _k10_1, _sum11); - _sum11 = _mm256_fmadd_ps(_r14, _k11_1, _sum11); - _sum11 = _mm256_fmadd_ps(_r15, _k12_1, _sum11); - _sum11 = _mm256_fmadd_ps(_r23, _k20_1, _sum11); - _sum11 = _mm256_fmadd_ps(_r24, _k21_1, _sum11); - _sum11 = _mm256_fmadd_ps(_r25, _k22_1, _sum11); + _sum01 = _mm256_comp_fmadd_ps(_r03, _k00_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r04, _k01_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r05, _k02_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r13, _k10_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r14, _k11_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r15, _k12_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r23, _k20_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r24, _k21_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r25, _k22_0, _sum01); + + _sum11 = _mm256_comp_fmadd_ps(_r03, _k00_1, _sum11); + _sum11 = _mm256_comp_fmadd_ps(_r04, _k01_1, _sum11); + _sum11 = _mm256_comp_fmadd_ps(_r05, _k02_1, _sum11); + _sum11 = _mm256_comp_fmadd_ps(_r13, _k10_1, _sum11); + _sum11 = _mm256_comp_fmadd_ps(_r14, _k11_1, _sum11); + _sum11 = _mm256_comp_fmadd_ps(_r15, _k12_1, _sum11); + _sum11 = _mm256_comp_fmadd_ps(_r23, _k20_1, _sum11); + _sum11 = _mm256_comp_fmadd_ps(_r24, _k21_1, _sum11); + _sum11 = _mm256_comp_fmadd_ps(_r25, _k22_1, _sum11); _mm256_storeu_ps(outptr0 + 8, _sum01); _mm256_storeu_ps(outptr1 + 8, _sum11); @@ -1117,25 +1117,25 @@ static void conv3x3s2_pack1to8_avx(const Mat& bottom_blob, Mat& top_blob, const __m256 _r22 = _mm256_broadcast_ss(r2 + 1); __m256 _r23 = _mm256_broadcast_ss(r2 + 2); - _sum00 = _mm256_fmadd_ps(_r01, _k00_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r02, _k01_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r03, _k02_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r11, _k10_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r12, _k11_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r13, _k12_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r21, _k20_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r22, _k21_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r23, _k22_0, _sum00); - - _sum10 = _mm256_fmadd_ps(_r01, _k00_1, _sum10); - _sum10 = _mm256_fmadd_ps(_r02, _k01_1, _sum10); - _sum10 = _mm256_fmadd_ps(_r03, _k02_1, _sum10); - _sum10 = _mm256_fmadd_ps(_r11, _k10_1, _sum10); - _sum10 = _mm256_fmadd_ps(_r12, _k11_1, _sum10); - _sum10 = _mm256_fmadd_ps(_r13, _k12_1, _sum10); - _sum10 = _mm256_fmadd_ps(_r21, _k20_1, _sum10); - _sum10 = _mm256_fmadd_ps(_r22, _k21_1, _sum10); - _sum10 = _mm256_fmadd_ps(_r23, _k22_1, _sum10); + _sum00 = _mm256_comp_fmadd_ps(_r01, _k00_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r02, _k01_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r03, _k02_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r11, _k10_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r12, _k11_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r13, _k12_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r21, _k20_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r22, _k21_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r23, _k22_0, _sum00); + + _sum10 = _mm256_comp_fmadd_ps(_r01, _k00_1, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_r02, _k01_1, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_r03, _k02_1, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_r11, _k10_1, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_r12, _k11_1, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_r13, _k12_1, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_r21, _k20_1, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_r22, _k21_1, _sum10); + _sum10 = _mm256_comp_fmadd_ps(_r23, _k22_1, _sum10); _mm256_storeu_ps(outptr0, _sum00); _mm256_storeu_ps(outptr1, _sum10); @@ -1205,15 +1205,15 @@ static void conv3x3s2_pack1to8_avx(const Mat& bottom_blob, Mat& top_blob, const __m256 _r22 = _mm256_broadcast_ss(r2 + 1); __m256 _r23 = _mm256_broadcast_ss(r2 + 2); - _sum00 = _mm256_fmadd_ps(_r01, _k00_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r02, _k01_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r03, _k02_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r11, _k10_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r12, _k11_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r13, _k12_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r21, _k20_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r22, _k21_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r23, _k22_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r01, _k00_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r02, _k01_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r03, _k02_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r11, _k10_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r12, _k11_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r13, _k12_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r21, _k20_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r22, _k21_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r23, _k22_0, _sum00); _mm256_storeu_ps(outptr0, _sum00); @@ -1226,15 +1226,15 @@ static void conv3x3s2_pack1to8_avx(const Mat& bottom_blob, Mat& top_blob, const __m256 _r15 = _mm256_broadcast_ss(r1 + 4); __m256 _r25 = _mm256_broadcast_ss(r2 + 4); - _sum01 = _mm256_fmadd_ps(_r03, _k00_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r04, _k01_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r05, _k02_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r13, _k10_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r14, _k11_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r15, _k12_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r23, _k20_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r24, _k21_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r25, _k22_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r03, _k00_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r04, _k01_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r05, _k02_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r13, _k10_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r14, _k11_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r15, _k12_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r23, _k20_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r24, _k21_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r25, _k22_0, _sum01); _mm256_storeu_ps(outptr0 + 8, _sum01); @@ -1247,15 +1247,15 @@ static void conv3x3s2_pack1to8_avx(const Mat& bottom_blob, Mat& top_blob, const __m256 _r17 = _mm256_broadcast_ss(r1 + 6); __m256 _r27 = _mm256_broadcast_ss(r2 + 6); - _sum02 = _mm256_fmadd_ps(_r05, _k00_0, _sum02); - _sum02 = _mm256_fmadd_ps(_r06, _k01_0, _sum02); - _sum02 = _mm256_fmadd_ps(_r07, _k02_0, _sum02); - _sum02 = _mm256_fmadd_ps(_r15, _k10_0, _sum02); - _sum02 = _mm256_fmadd_ps(_r16, _k11_0, _sum02); - _sum02 = _mm256_fmadd_ps(_r17, _k12_0, _sum02); - _sum02 = _mm256_fmadd_ps(_r25, _k20_0, _sum02); - _sum02 = _mm256_fmadd_ps(_r26, _k21_0, _sum02); - _sum02 = _mm256_fmadd_ps(_r27, _k22_0, _sum02); + _sum02 = _mm256_comp_fmadd_ps(_r05, _k00_0, _sum02); + _sum02 = _mm256_comp_fmadd_ps(_r06, _k01_0, _sum02); + _sum02 = _mm256_comp_fmadd_ps(_r07, _k02_0, _sum02); + _sum02 = _mm256_comp_fmadd_ps(_r15, _k10_0, _sum02); + _sum02 = _mm256_comp_fmadd_ps(_r16, _k11_0, _sum02); + _sum02 = _mm256_comp_fmadd_ps(_r17, _k12_0, _sum02); + _sum02 = _mm256_comp_fmadd_ps(_r25, _k20_0, _sum02); + _sum02 = _mm256_comp_fmadd_ps(_r26, _k21_0, _sum02); + _sum02 = _mm256_comp_fmadd_ps(_r27, _k22_0, _sum02); _mm256_storeu_ps(outptr0 + 16, _sum02); @@ -1268,15 +1268,15 @@ static void conv3x3s2_pack1to8_avx(const Mat& bottom_blob, Mat& top_blob, const __m256 _sum03 = _mm256_loadu_ps(outptr0 + 24); - _sum03 = _mm256_fmadd_ps(_r07, _k00_0, _sum03); - _sum03 = _mm256_fmadd_ps(_r08, _k01_0, _sum03); - _sum03 = _mm256_fmadd_ps(_r09, _k02_0, _sum03); - _sum03 = _mm256_fmadd_ps(_r17, _k10_0, _sum03); - _sum03 = _mm256_fmadd_ps(_r18, _k11_0, _sum03); - _sum03 = _mm256_fmadd_ps(_r19, _k12_0, _sum03); - _sum03 = _mm256_fmadd_ps(_r27, _k20_0, _sum03); - _sum03 = _mm256_fmadd_ps(_r28, _k21_0, _sum03); - _sum03 = _mm256_fmadd_ps(_r29, _k22_0, _sum03); + _sum03 = _mm256_comp_fmadd_ps(_r07, _k00_0, _sum03); + _sum03 = _mm256_comp_fmadd_ps(_r08, _k01_0, _sum03); + _sum03 = _mm256_comp_fmadd_ps(_r09, _k02_0, _sum03); + _sum03 = _mm256_comp_fmadd_ps(_r17, _k10_0, _sum03); + _sum03 = _mm256_comp_fmadd_ps(_r18, _k11_0, _sum03); + _sum03 = _mm256_comp_fmadd_ps(_r19, _k12_0, _sum03); + _sum03 = _mm256_comp_fmadd_ps(_r27, _k20_0, _sum03); + _sum03 = _mm256_comp_fmadd_ps(_r28, _k21_0, _sum03); + _sum03 = _mm256_comp_fmadd_ps(_r29, _k22_0, _sum03); _mm256_storeu_ps(outptr0 + 24, _sum03); @@ -1289,15 +1289,15 @@ static void conv3x3s2_pack1to8_avx(const Mat& bottom_blob, Mat& top_blob, const __m256 _sum04 = _mm256_loadu_ps(outptr0 + 32); - _sum04 = _mm256_fmadd_ps(_r09, _k00_0, _sum04); - _sum04 = _mm256_fmadd_ps(_r010, _k01_0, _sum04); - _sum04 = _mm256_fmadd_ps(_r011, _k02_0, _sum04); - _sum04 = _mm256_fmadd_ps(_r19, _k10_0, _sum04); - _sum04 = _mm256_fmadd_ps(_r110, _k11_0, _sum04); - _sum04 = _mm256_fmadd_ps(_r111, _k12_0, _sum04); - _sum04 = _mm256_fmadd_ps(_r29, _k20_0, _sum04); - _sum04 = _mm256_fmadd_ps(_r210, _k21_0, _sum04); - _sum04 = _mm256_fmadd_ps(_r211, _k22_0, _sum04); + _sum04 = _mm256_comp_fmadd_ps(_r09, _k00_0, _sum04); + _sum04 = _mm256_comp_fmadd_ps(_r010, _k01_0, _sum04); + _sum04 = _mm256_comp_fmadd_ps(_r011, _k02_0, _sum04); + _sum04 = _mm256_comp_fmadd_ps(_r19, _k10_0, _sum04); + _sum04 = _mm256_comp_fmadd_ps(_r110, _k11_0, _sum04); + _sum04 = _mm256_comp_fmadd_ps(_r111, _k12_0, _sum04); + _sum04 = _mm256_comp_fmadd_ps(_r29, _k20_0, _sum04); + _sum04 = _mm256_comp_fmadd_ps(_r210, _k21_0, _sum04); + _sum04 = _mm256_comp_fmadd_ps(_r211, _k22_0, _sum04); _mm256_storeu_ps(outptr0 + 32, _sum04); @@ -1310,15 +1310,15 @@ static void conv3x3s2_pack1to8_avx(const Mat& bottom_blob, Mat& top_blob, const __m256 _sum05 = _mm256_loadu_ps(outptr0 + 40); - _sum05 = _mm256_fmadd_ps(_r011, _k00_0, _sum05); - _sum05 = _mm256_fmadd_ps(_r012, _k01_0, _sum05); - _sum05 = _mm256_fmadd_ps(_r013, _k02_0, _sum05); - _sum05 = _mm256_fmadd_ps(_r111, _k10_0, _sum05); - _sum05 = _mm256_fmadd_ps(_r112, _k11_0, _sum05); - _sum05 = _mm256_fmadd_ps(_r113, _k12_0, _sum05); - _sum05 = _mm256_fmadd_ps(_r211, _k20_0, _sum05); - _sum05 = _mm256_fmadd_ps(_r212, _k21_0, _sum05); - _sum05 = _mm256_fmadd_ps(_r213, _k22_0, _sum05); + _sum05 = _mm256_comp_fmadd_ps(_r011, _k00_0, _sum05); + _sum05 = _mm256_comp_fmadd_ps(_r012, _k01_0, _sum05); + _sum05 = _mm256_comp_fmadd_ps(_r013, _k02_0, _sum05); + _sum05 = _mm256_comp_fmadd_ps(_r111, _k10_0, _sum05); + _sum05 = _mm256_comp_fmadd_ps(_r112, _k11_0, _sum05); + _sum05 = _mm256_comp_fmadd_ps(_r113, _k12_0, _sum05); + _sum05 = _mm256_comp_fmadd_ps(_r211, _k20_0, _sum05); + _sum05 = _mm256_comp_fmadd_ps(_r212, _k21_0, _sum05); + _sum05 = _mm256_comp_fmadd_ps(_r213, _k22_0, _sum05); _mm256_storeu_ps(outptr0 + 40, _sum05); @@ -1331,15 +1331,15 @@ static void conv3x3s2_pack1to8_avx(const Mat& bottom_blob, Mat& top_blob, const __m256 _sum06 = _mm256_loadu_ps(outptr0 + 48); - _sum06 = _mm256_fmadd_ps(_r013, _k00_0, _sum06); - _sum06 = _mm256_fmadd_ps(_r014, _k01_0, _sum06); - _sum06 = _mm256_fmadd_ps(_r015, _k02_0, _sum06); - _sum06 = _mm256_fmadd_ps(_r113, _k10_0, _sum06); - _sum06 = _mm256_fmadd_ps(_r114, _k11_0, _sum06); - _sum06 = _mm256_fmadd_ps(_r115, _k12_0, _sum06); - _sum06 = _mm256_fmadd_ps(_r213, _k20_0, _sum06); - _sum06 = _mm256_fmadd_ps(_r214, _k21_0, _sum06); - _sum06 = _mm256_fmadd_ps(_r215, _k22_0, _sum06); + _sum06 = _mm256_comp_fmadd_ps(_r013, _k00_0, _sum06); + _sum06 = _mm256_comp_fmadd_ps(_r014, _k01_0, _sum06); + _sum06 = _mm256_comp_fmadd_ps(_r015, _k02_0, _sum06); + _sum06 = _mm256_comp_fmadd_ps(_r113, _k10_0, _sum06); + _sum06 = _mm256_comp_fmadd_ps(_r114, _k11_0, _sum06); + _sum06 = _mm256_comp_fmadd_ps(_r115, _k12_0, _sum06); + _sum06 = _mm256_comp_fmadd_ps(_r213, _k20_0, _sum06); + _sum06 = _mm256_comp_fmadd_ps(_r214, _k21_0, _sum06); + _sum06 = _mm256_comp_fmadd_ps(_r215, _k22_0, _sum06); _mm256_storeu_ps(outptr0 + 48, _sum06); @@ -1352,15 +1352,15 @@ static void conv3x3s2_pack1to8_avx(const Mat& bottom_blob, Mat& top_blob, const __m256 _sum07 = _mm256_loadu_ps(outptr0 + 56); - _sum07 = _mm256_fmadd_ps(_r015, _k00_0, _sum07); - _sum07 = _mm256_fmadd_ps(_r016, _k01_0, _sum07); - _sum07 = _mm256_fmadd_ps(_r017, _k02_0, _sum07); - _sum07 = _mm256_fmadd_ps(_r115, _k10_0, _sum07); - _sum07 = _mm256_fmadd_ps(_r116, _k11_0, _sum07); - _sum07 = _mm256_fmadd_ps(_r117, _k12_0, _sum07); - _sum07 = _mm256_fmadd_ps(_r215, _k20_0, _sum07); - _sum07 = _mm256_fmadd_ps(_r216, _k21_0, _sum07); - _sum07 = _mm256_fmadd_ps(_r217, _k22_0, _sum07); + _sum07 = _mm256_comp_fmadd_ps(_r015, _k00_0, _sum07); + _sum07 = _mm256_comp_fmadd_ps(_r016, _k01_0, _sum07); + _sum07 = _mm256_comp_fmadd_ps(_r017, _k02_0, _sum07); + _sum07 = _mm256_comp_fmadd_ps(_r115, _k10_0, _sum07); + _sum07 = _mm256_comp_fmadd_ps(_r116, _k11_0, _sum07); + _sum07 = _mm256_comp_fmadd_ps(_r117, _k12_0, _sum07); + _sum07 = _mm256_comp_fmadd_ps(_r215, _k20_0, _sum07); + _sum07 = _mm256_comp_fmadd_ps(_r216, _k21_0, _sum07); + _sum07 = _mm256_comp_fmadd_ps(_r217, _k22_0, _sum07); _mm256_storeu_ps(outptr0 + 56, _sum07); @@ -1383,15 +1383,15 @@ static void conv3x3s2_pack1to8_avx(const Mat& bottom_blob, Mat& top_blob, const __m256 _r22 = _mm256_broadcast_ss(r2 + 1); __m256 _r23 = _mm256_broadcast_ss(r2 + 2); - _sum00 = _mm256_fmadd_ps(_r01, _k00_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r02, _k01_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r03, _k02_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r11, _k10_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r12, _k11_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r13, _k12_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r21, _k20_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r22, _k21_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r23, _k22_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r01, _k00_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r02, _k01_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r03, _k02_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r11, _k10_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r12, _k11_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r13, _k12_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r21, _k20_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r22, _k21_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r23, _k22_0, _sum00); _mm256_storeu_ps(outptr0, _sum00); @@ -1404,15 +1404,15 @@ static void conv3x3s2_pack1to8_avx(const Mat& bottom_blob, Mat& top_blob, const __m256 _r15 = _mm256_broadcast_ss(r1 + 4); __m256 _r25 = _mm256_broadcast_ss(r2 + 4); - _sum01 = _mm256_fmadd_ps(_r03, _k00_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r04, _k01_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r05, _k02_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r13, _k10_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r14, _k11_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r15, _k12_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r23, _k20_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r24, _k21_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r25, _k22_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r03, _k00_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r04, _k01_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r05, _k02_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r13, _k10_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r14, _k11_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r15, _k12_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r23, _k20_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r24, _k21_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r25, _k22_0, _sum01); _mm256_storeu_ps(outptr0 + 8, _sum01); @@ -1425,15 +1425,15 @@ static void conv3x3s2_pack1to8_avx(const Mat& bottom_blob, Mat& top_blob, const __m256 _r17 = _mm256_broadcast_ss(r1 + 6); __m256 _r27 = _mm256_broadcast_ss(r2 + 6); - _sum02 = _mm256_fmadd_ps(_r05, _k00_0, _sum02); - _sum02 = _mm256_fmadd_ps(_r06, _k01_0, _sum02); - _sum02 = _mm256_fmadd_ps(_r07, _k02_0, _sum02); - _sum02 = _mm256_fmadd_ps(_r15, _k10_0, _sum02); - _sum02 = _mm256_fmadd_ps(_r16, _k11_0, _sum02); - _sum02 = _mm256_fmadd_ps(_r17, _k12_0, _sum02); - _sum02 = _mm256_fmadd_ps(_r25, _k20_0, _sum02); - _sum02 = _mm256_fmadd_ps(_r26, _k21_0, _sum02); - _sum02 = _mm256_fmadd_ps(_r27, _k22_0, _sum02); + _sum02 = _mm256_comp_fmadd_ps(_r05, _k00_0, _sum02); + _sum02 = _mm256_comp_fmadd_ps(_r06, _k01_0, _sum02); + _sum02 = _mm256_comp_fmadd_ps(_r07, _k02_0, _sum02); + _sum02 = _mm256_comp_fmadd_ps(_r15, _k10_0, _sum02); + _sum02 = _mm256_comp_fmadd_ps(_r16, _k11_0, _sum02); + _sum02 = _mm256_comp_fmadd_ps(_r17, _k12_0, _sum02); + _sum02 = _mm256_comp_fmadd_ps(_r25, _k20_0, _sum02); + _sum02 = _mm256_comp_fmadd_ps(_r26, _k21_0, _sum02); + _sum02 = _mm256_comp_fmadd_ps(_r27, _k22_0, _sum02); _mm256_storeu_ps(outptr0 + 16, _sum02); @@ -1446,15 +1446,15 @@ static void conv3x3s2_pack1to8_avx(const Mat& bottom_blob, Mat& top_blob, const __m256 _sum03 = _mm256_loadu_ps(outptr0 + 24); - _sum03 = _mm256_fmadd_ps(_r07, _k00_0, _sum03); - _sum03 = _mm256_fmadd_ps(_r08, _k01_0, _sum03); - _sum03 = _mm256_fmadd_ps(_r09, _k02_0, _sum03); - _sum03 = _mm256_fmadd_ps(_r17, _k10_0, _sum03); - _sum03 = _mm256_fmadd_ps(_r18, _k11_0, _sum03); - _sum03 = _mm256_fmadd_ps(_r19, _k12_0, _sum03); - _sum03 = _mm256_fmadd_ps(_r27, _k20_0, _sum03); - _sum03 = _mm256_fmadd_ps(_r28, _k21_0, _sum03); - _sum03 = _mm256_fmadd_ps(_r29, _k22_0, _sum03); + _sum03 = _mm256_comp_fmadd_ps(_r07, _k00_0, _sum03); + _sum03 = _mm256_comp_fmadd_ps(_r08, _k01_0, _sum03); + _sum03 = _mm256_comp_fmadd_ps(_r09, _k02_0, _sum03); + _sum03 = _mm256_comp_fmadd_ps(_r17, _k10_0, _sum03); + _sum03 = _mm256_comp_fmadd_ps(_r18, _k11_0, _sum03); + _sum03 = _mm256_comp_fmadd_ps(_r19, _k12_0, _sum03); + _sum03 = _mm256_comp_fmadd_ps(_r27, _k20_0, _sum03); + _sum03 = _mm256_comp_fmadd_ps(_r28, _k21_0, _sum03); + _sum03 = _mm256_comp_fmadd_ps(_r29, _k22_0, _sum03); _mm256_storeu_ps(outptr0 + 24, _sum03); r0 += 8; @@ -1477,15 +1477,15 @@ static void conv3x3s2_pack1to8_avx(const Mat& bottom_blob, Mat& top_blob, const __m256 _r22 = _mm256_broadcast_ss(r2 + 1); __m256 _r23 = _mm256_broadcast_ss(r2 + 2); - _sum00 = _mm256_fmadd_ps(_r01, _k00_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r02, _k01_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r03, _k02_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r11, _k10_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r12, _k11_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r13, _k12_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r21, _k20_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r22, _k21_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r23, _k22_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r01, _k00_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r02, _k01_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r03, _k02_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r11, _k10_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r12, _k11_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r13, _k12_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r21, _k20_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r22, _k21_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r23, _k22_0, _sum00); _mm256_storeu_ps(outptr0, _sum00); @@ -1498,15 +1498,15 @@ static void conv3x3s2_pack1to8_avx(const Mat& bottom_blob, Mat& top_blob, const __m256 _r15 = _mm256_broadcast_ss(r1 + 4); __m256 _r25 = _mm256_broadcast_ss(r2 + 4); - _sum01 = _mm256_fmadd_ps(_r03, _k00_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r04, _k01_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r05, _k02_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r13, _k10_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r14, _k11_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r15, _k12_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r23, _k20_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r24, _k21_0, _sum01); - _sum01 = _mm256_fmadd_ps(_r25, _k22_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r03, _k00_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r04, _k01_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r05, _k02_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r13, _k10_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r14, _k11_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r15, _k12_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r23, _k20_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r24, _k21_0, _sum01); + _sum01 = _mm256_comp_fmadd_ps(_r25, _k22_0, _sum01); _mm256_storeu_ps(outptr0 + 8, _sum01); @@ -1528,15 +1528,15 @@ static void conv3x3s2_pack1to8_avx(const Mat& bottom_blob, Mat& top_blob, const __m256 _r22 = _mm256_broadcast_ss(r2 + 1); __m256 _r23 = _mm256_broadcast_ss(r2 + 2); - _sum00 = _mm256_fmadd_ps(_r01, _k00_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r02, _k01_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r03, _k02_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r11, _k10_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r12, _k11_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r13, _k12_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r21, _k20_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r22, _k21_0, _sum00); - _sum00 = _mm256_fmadd_ps(_r23, _k22_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r01, _k00_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r02, _k01_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r03, _k02_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r11, _k10_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r12, _k11_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r13, _k12_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r21, _k20_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r22, _k21_0, _sum00); + _sum00 = _mm256_comp_fmadd_ps(_r23, _k22_0, _sum00); _mm256_storeu_ps(outptr0, _sum00); r0 += 2; diff --git a/src/layer/x86/convolution_3x3_pack8.h b/src/layer/x86/convolution_3x3_pack8.h index 2cd5afe96..9b8cc62f6 100644 --- a/src/layer/x86/convolution_3x3_pack8.h +++ b/src/layer/x86/convolution_3x3_pack8.h @@ -73,14 +73,14 @@ static void conv3x3s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const Mat kptr += 64; - _sum00 = _mm256_fmadd_ps(_r000, _k00, _sum00); - _sum01 = _mm256_fmadd_ps(_r001, _k01, _sum01); - _sum00 = _mm256_fmadd_ps(_r002, _k02, _sum00); - _sum01 = _mm256_fmadd_ps(_r003, _k03, _sum01); - _sum00 = _mm256_fmadd_ps(_r004, _k04, _sum00); - _sum01 = _mm256_fmadd_ps(_r005, _k05, _sum01); - _sum00 = _mm256_fmadd_ps(_r006, _k06, _sum00); - _sum01 = _mm256_fmadd_ps(_r007, _k07, _sum01); + _sum00 = _mm256_comp_fmadd_ps(_r000, _k00, _sum00); + _sum01 = _mm256_comp_fmadd_ps(_r001, _k01, _sum01); + _sum00 = _mm256_comp_fmadd_ps(_r002, _k02, _sum00); + _sum01 = _mm256_comp_fmadd_ps(_r003, _k03, _sum01); + _sum00 = _mm256_comp_fmadd_ps(_r004, _k04, _sum00); + _sum01 = _mm256_comp_fmadd_ps(_r005, _k05, _sum01); + _sum00 = _mm256_comp_fmadd_ps(_r006, _k06, _sum00); + _sum01 = _mm256_comp_fmadd_ps(_r007, _k07, _sum01); __m256 _r010 = _mm256_broadcast_ss(r0 + 8); __m256 _r011 = _mm256_broadcast_ss(r0 + 9); @@ -91,14 +91,14 @@ static void conv3x3s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const Mat __m256 _r016 = _mm256_broadcast_ss(r0 + 14); __m256 _r017 = _mm256_broadcast_ss(r0 + 15); - _sum10 = _mm256_fmadd_ps(_r010, _k00, _sum10); - _sum11 = _mm256_fmadd_ps(_r011, _k01, _sum11); - _sum10 = _mm256_fmadd_ps(_r012, _k02, _sum10); - _sum11 = _mm256_fmadd_ps(_r013, _k03, _sum11); - _sum10 = _mm256_fmadd_ps(_r014, _k04, _sum10); - _sum11 = _mm256_fmadd_ps(_r015, _k05, _sum11); - _sum10 = _mm256_fmadd_ps(_r016, _k06, _sum10); - _sum11 = _mm256_fmadd_ps(_r017, _k07, _sum11); + _sum10 = _mm256_comp_fmadd_ps(_r010, _k00, _sum10); + _sum11 = _mm256_comp_fmadd_ps(_r011, _k01, _sum11); + _sum10 = _mm256_comp_fmadd_ps(_r012, _k02, _sum10); + _sum11 = _mm256_comp_fmadd_ps(_r013, _k03, _sum11); + _sum10 = _mm256_comp_fmadd_ps(_r014, _k04, _sum10); + _sum11 = _mm256_comp_fmadd_ps(_r015, _k05, _sum11); + _sum10 = _mm256_comp_fmadd_ps(_r016, _k06, _sum10); + _sum11 = _mm256_comp_fmadd_ps(_r017, _k07, _sum11); __m256 _k10 = _mm256_loadu_ps(kptr); __m256 _k11 = _mm256_loadu_ps(kptr + 8); @@ -111,14 +111,14 @@ static void conv3x3s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const Mat kptr += 64; - _sum00 = _mm256_fmadd_ps(_r010, _k10, _sum00); - _sum01 = _mm256_fmadd_ps(_r011, _k11, _sum01); - _sum00 = _mm256_fmadd_ps(_r012, _k12, _sum00); - _sum01 = _mm256_fmadd_ps(_r013, _k13, _sum01); - _sum00 = _mm256_fmadd_ps(_r014, _k14, _sum00); - _sum01 = _mm256_fmadd_ps(_r015, _k15, _sum01); - _sum00 = _mm256_fmadd_ps(_r016, _k16, _sum00); - _sum01 = _mm256_fmadd_ps(_r017, _k17, _sum01); + _sum00 = _mm256_comp_fmadd_ps(_r010, _k10, _sum00); + _sum01 = _mm256_comp_fmadd_ps(_r011, _k11, _sum01); + _sum00 = _mm256_comp_fmadd_ps(_r012, _k12, _sum00); + _sum01 = _mm256_comp_fmadd_ps(_r013, _k13, _sum01); + _sum00 = _mm256_comp_fmadd_ps(_r014, _k14, _sum00); + _sum01 = _mm256_comp_fmadd_ps(_r015, _k15, _sum01); + _sum00 = _mm256_comp_fmadd_ps(_r016, _k16, _sum00); + _sum01 = _mm256_comp_fmadd_ps(_r017, _k17, _sum01); __m256 _r020 = _mm256_broadcast_ss(r0 + 16); __m256 _r021 = _mm256_broadcast_ss(r0 + 17); @@ -129,14 +129,14 @@ static void conv3x3s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const Mat __m256 _r026 = _mm256_broadcast_ss(r0 + 22); __m256 _r027 = _mm256_broadcast_ss(r0 + 23); - _sum10 = _mm256_fmadd_ps(_r020, _k10, _sum10); - _sum11 = _mm256_fmadd_ps(_r021, _k11, _sum11); - _sum10 = _mm256_fmadd_ps(_r022, _k12, _sum10); - _sum11 = _mm256_fmadd_ps(_r023, _k13, _sum11); - _sum10 = _mm256_fmadd_ps(_r024, _k14, _sum10); - _sum11 = _mm256_fmadd_ps(_r025, _k15, _sum11); - _sum10 = _mm256_fmadd_ps(_r026, _k16, _sum10); - _sum11 = _mm256_fmadd_ps(_r027, _k17, _sum11); + _sum10 = _mm256_comp_fmadd_ps(_r020, _k10, _sum10); + _sum11 = _mm256_comp_fmadd_ps(_r021, _k11, _sum11); + _sum10 = _mm256_comp_fmadd_ps(_r022, _k12, _sum10); + _sum11 = _mm256_comp_fmadd_ps(_r023, _k13, _sum11); + _sum10 = _mm256_comp_fmadd_ps(_r024, _k14, _sum10); + _sum11 = _mm256_comp_fmadd_ps(_r025, _k15, _sum11); + _sum10 = _mm256_comp_fmadd_ps(_r026, _k16, _sum10); + _sum11 = _mm256_comp_fmadd_ps(_r027, _k17, _sum11); __m256 _k20 = _mm256_loadu_ps(kptr); __m256 _k21 = _mm256_loadu_ps(kptr + 8); @@ -149,14 +149,14 @@ static void conv3x3s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const Mat kptr += 64; - _sum00 = _mm256_fmadd_ps(_r020, _k20, _sum00); - _sum01 = _mm256_fmadd_ps(_r021, _k21, _sum01); - _sum00 = _mm256_fmadd_ps(_r022, _k22, _sum00); - _sum01 = _mm256_fmadd_ps(_r023, _k23, _sum01); - _sum00 = _mm256_fmadd_ps(_r024, _k24, _sum00); - _sum01 = _mm256_fmadd_ps(_r025, _k25, _sum01); - _sum00 = _mm256_fmadd_ps(_r026, _k26, _sum00); - _sum01 = _mm256_fmadd_ps(_r027, _k27, _sum01); + _sum00 = _mm256_comp_fmadd_ps(_r020, _k20, _sum00); + _sum01 = _mm256_comp_fmadd_ps(_r021, _k21, _sum01); + _sum00 = _mm256_comp_fmadd_ps(_r022, _k22, _sum00); + _sum01 = _mm256_comp_fmadd_ps(_r023, _k23, _sum01); + _sum00 = _mm256_comp_fmadd_ps(_r024, _k24, _sum00); + _sum01 = _mm256_comp_fmadd_ps(_r025, _k25, _sum01); + _sum00 = _mm256_comp_fmadd_ps(_r026, _k26, _sum00); + _sum01 = _mm256_comp_fmadd_ps(_r027, _k27, _sum01); __m256 _r030 = _mm256_broadcast_ss(r0 + 24); __m256 _r031 = _mm256_broadcast_ss(r0 + 25); @@ -167,14 +167,14 @@ static void conv3x3s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const Mat __m256 _r036 = _mm256_broadcast_ss(r0 + 30); __m256 _r037 = _mm256_broadcast_ss(r0 + 31); - _sum10 = _mm256_fmadd_ps(_r030, _k20, _sum10); - _sum11 = _mm256_fmadd_ps(_r031, _k21, _sum11); - _sum10 = _mm256_fmadd_ps(_r032, _k22, _sum10); - _sum11 = _mm256_fmadd_ps(_r033, _k23, _sum11); - _sum10 = _mm256_fmadd_ps(_r034, _k24, _sum10); - _sum11 = _mm256_fmadd_ps(_r035, _k25, _sum11); - _sum10 = _mm256_fmadd_ps(_r036, _k26, _sum10); - _sum11 = _mm256_fmadd_ps(_r037, _k27, _sum11); + _sum10 = _mm256_comp_fmadd_ps(_r030, _k20, _sum10); + _sum11 = _mm256_comp_fmadd_ps(_r031, _k21, _sum11); + _sum10 = _mm256_comp_fmadd_ps(_r032, _k22, _sum10); + _sum11 = _mm256_comp_fmadd_ps(_r033, _k23, _sum11); + _sum10 = _mm256_comp_fmadd_ps(_r034, _k24, _sum10); + _sum11 = _mm256_comp_fmadd_ps(_r035, _k25, _sum11); + _sum10 = _mm256_comp_fmadd_ps(_r036, _k26, _sum10); + _sum11 = _mm256_comp_fmadd_ps(_r037, _k27, _sum11); __m256 _r100 = _mm256_broadcast_ss(r1 + 0); __m256 _r101 = _mm256_broadcast_ss(r1 + 1); @@ -196,14 +196,14 @@ static void conv3x3s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const Mat kptr += 64; - _sum00 = _mm256_fmadd_ps(_r100, _k30, _sum00); - _sum01 = _mm256_fmadd_ps(_r101, _k31, _sum01); - _sum00 = _mm256_fmadd_ps(_r102, _k32, _sum00); - _sum01 = _mm256_fmadd_ps(_r103, _k33, _sum01); - _sum00 = _mm256_fmadd_ps(_r104, _k34, _sum00); - _sum01 = _mm256_fmadd_ps(_r105, _k35, _sum01); - _sum00 = _mm256_fmadd_ps(_r106, _k36, _sum00); - _sum01 = _mm256_fmadd_ps(_r107, _k37, _sum01); + _sum00 = _mm256_comp_fmadd_ps(_r100, _k30, _sum00); + _sum01 = _mm256_comp_fmadd_ps(_r101, _k31, _sum01); + _sum00 = _mm256_comp_fmadd_ps(_r102, _k32, _sum00); + _sum01 = _mm256_comp_fmadd_ps(_r103, _k33, _sum01); + _sum00 = _mm256_comp_fmadd_ps(_r104, _k34, _sum00); + _sum01 = _mm256_comp_fmadd_ps(_r105, _k35, _sum01); + _sum00 = _mm256_comp_fmadd_ps(_r106, _k36, _sum00); + _sum01 = _mm256_comp_fmadd_ps(_r107, _k37, _sum01); __m256 _r110 = _mm256_broadcast_ss(r1 + 8); __m256 _r111 = _mm256_broadcast_ss(r1 + 9); @@ -214,14 +214,14 @@ static void conv3x3s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const Mat __m256 _r116 = _mm256_broadcast_ss(r1 + 14); __m256 _r117 = _mm256_broadcast_ss(r1 + 15); - _sum10 = _mm256_fmadd_ps(_r110, _k30, _sum10); - _sum11 = _mm256_fmadd_ps(_r111, _k31, _sum11); - _sum10 = _mm256_fmadd_ps(_r112, _k32, _sum10); - _sum11 = _mm256_fmadd_ps(_r113, _k33, _sum11); - _sum10 = _mm256_fmadd_ps(_r114, _k34, _sum10); - _sum11 = _mm256_fmadd_ps(_r115, _k35, _sum11); - _sum10 = _mm256_fmadd_ps(_r116, _k36, _sum10); - _sum11 = _mm256_fmadd_ps(_r117, _k37, _sum11); + _sum10 = _mm256_comp_fmadd_ps(_r110, _k30, _sum10); + _sum11 = _mm256_comp_fmadd_ps(_r111, _k31, _sum11); + _sum10 = _mm256_comp_fmadd_ps(_r112, _k32, _sum10); + _sum11 = _mm256_comp_fmadd_ps(_r113, _k33, _sum11); + _sum10 = _mm256_comp_fmadd_ps(_r114, _k34, _sum10); + _sum11 = _mm256_comp_fmadd_ps(_r115, _k35, _sum11); + _sum10 = _mm256_comp_fmadd_ps(_r116, _k36, _sum10); + _sum11 = _mm256_comp_fmadd_ps(_r117, _k37, _sum11); __m256 _k40 = _mm256_loadu_ps(kptr); __m256 _k41 = _mm256_loadu_ps(kptr + 8); @@ -234,14 +234,14 @@ static void conv3x3s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const Mat kptr += 64; - _sum00 = _mm256_fmadd_ps(_r110, _k40, _sum00); - _sum01 = _mm256_fmadd_ps(_r111, _k41, _sum01); - _sum00 = _mm256_fmadd_ps(_r112, _k42, _sum00); - _sum01 = _mm256_fmadd_ps(_r113, _k43, _sum01); - _sum00 = _mm256_fmadd_ps(_r114, _k44, _sum00); - _sum01 = _mm256_fmadd_ps(_r115, _k45, _sum01); - _sum00 = _mm256_fmadd_ps(_r116, _k46, _sum00); - _sum01 = _mm256_fmadd_ps(_r117, _k47, _sum01); + _sum00 = _mm256_comp_fmadd_ps(_r110, _k40, _sum00); + _sum01 = _mm256_comp_fmadd_ps(_r111, _k41, _sum01); + _sum00 = _mm256_comp_fmadd_ps(_r112, _k42, _sum00); + _sum01 = _mm256_comp_fmadd_ps(_r113, _k43, _sum01); + _sum00 = _mm256_comp_fmadd_ps(_r114, _k44, _sum00); + _sum01 = _mm256_comp_fmadd_ps(_r115, _k45, _sum01); + _sum00 = _mm256_comp_fmadd_ps(_r116, _k46, _sum00); + _sum01 = _mm256_comp_fmadd_ps(_r117, _k47, _sum01); __m256 _r120 = _mm256_broadcast_ss(r1 + 16); __m256 _r121 = _mm256_broadcast_ss(r1 + 17); @@ -252,14 +252,14 @@ static void conv3x3s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const Mat __m256 _r126 = _mm256_broadcast_ss(r1 + 22); __m256 _r127 = _mm256_broadcast_ss(r1 + 23); - _sum10 = _mm256_fmadd_ps(_r120, _k40, _sum10); - _sum11 = _mm256_fmadd_ps(_r121, _k41, _sum11); - _sum10 = _mm256_fmadd_ps(_r122, _k42, _sum10); - _sum11 = _mm256_fmadd_ps(_r123, _k43, _sum11); - _sum10 = _mm256_fmadd_ps(_r124, _k44, _sum10); - _sum11 = _mm256_fmadd_ps(_r125, _k45, _sum11); - _sum10 = _mm256_fmadd_ps(_r126, _k46, _sum10); - _sum11 = _mm256_fmadd_ps(_r127, _k47, _sum11); + _sum10 = _mm256_comp_fmadd_ps(_r120, _k40, _sum10); + _sum11 = _mm256_comp_fmadd_ps(_r121, _k41, _sum11); + _sum10 = _mm256_comp_fmadd_ps(_r122, _k42, _sum10); + _sum11 = _mm256_comp_fmadd_ps(_r123, _k43, _sum11); + _sum10 = _mm256_comp_fmadd_ps(_r124, _k44, _sum10); + _sum11 = _mm256_comp_fmadd_ps(_r125, _k45, _sum11); + _sum10 = _mm256_comp_fmadd_ps(_r126, _k46, _sum10); + _sum11 = _mm256_comp_fmadd_ps(_r127, _k47, _sum11); __m256 _k50 = _mm256_loadu_ps(kptr); __m256 _k51 = _mm256_loadu_ps(kptr + 8); @@ -272,14 +272,14 @@ static void conv3x3s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const Mat kptr += 64; - _sum00 = _mm256_fmadd_ps(_r120, _k50, _sum00); - _sum01 = _mm256_fmadd_ps(_r121, _k51, _sum01); - _sum00 = _mm256_fmadd_ps(_r122, _k52, _sum00); - _sum01 = _mm256_fmadd_ps(_r123, _k53, _sum01); - _sum00 = _mm256_fmadd_ps(_r124, _k54, _sum00); - _sum01 = _mm256_fmadd_ps(_r125, _k55, _sum01); - _sum00 = _mm256_fmadd_ps(_r126, _k56, _sum00); - _sum01 = _mm256_fmadd_ps(_r127, _k57, _sum01); + _sum00 = _mm256_comp_fmadd_ps(_r120, _k50, _sum00); + _sum01 = _mm256_comp_fmadd_ps(_r121, _k51, _sum01); + _sum00 = _mm256_comp_fmadd_ps(_r122, _k52, _sum00); + _sum01 = _mm256_comp_fmadd_ps(_r123, _k53, _sum01); + _sum00 = _mm256_comp_fmadd_ps(_r124, _k54, _sum00); + _sum01 = _mm256_comp_fmadd_ps(_r125, _k55, _sum01); + _sum00 = _mm256_comp_fmadd_ps(_r126, _k56, _sum00); + _sum01 = _mm256_comp_fmadd_ps(_r127, _k57, _sum01); __m256 _r130 = _mm256_broadcast_ss(r1 + 24); __m256 _r131 = _mm256_broadcast_ss(r1 + 25); @@ -290,14 +290,14 @@ static void conv3x3s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const Mat __m256 _r136 = _mm256_broadcast_ss(r1 + 30); __m256 _r137 = _mm256_broadcast_ss(r1 + 31); - _sum10 = _mm256_fmadd_ps(_r130, _k50, _sum10); - _sum11 = _mm256_fmadd_ps(_r131, _k51, _sum11); - _sum10 = _mm256_fmadd_ps(_r132, _k52, _sum10); - _sum11 = _mm256_fmadd_ps(_r133, _k53, _sum11); - _sum10 = _mm256_fmadd_ps(_r134, _k54, _sum10); - _sum11 = _mm256_fmadd_ps(_r135, _k55, _sum11); - _sum10 = _mm256_fmadd_ps(_r136, _k56, _sum10); - _sum11 = _mm256_fmadd_ps(_r137, _k57, _sum11); + _sum10 = _mm256_comp_fmadd_ps(_r130, _k50, _sum10); + _sum11 = _mm256_comp_fmadd_ps(_r131, _k51, _sum11); + _sum10 = _mm256_comp_fmadd_ps(_r132, _k52, _sum10); + _sum11 = _mm256_comp_fmadd_ps(_r133, _k53, _sum11); + _sum10 = _mm256_comp_fmadd_ps(_r134, _k54, _sum10); + _sum11 = _mm256_comp_fmadd_ps(_r135, _k55, _sum11); + _sum10 = _mm256_comp_fmadd_ps(_r136, _k56, _sum10); + _sum11 = _mm256_comp_fmadd_ps(_r137, _k57, _sum11); __m256 _r200 = _mm256_broadcast_ss(r2 + 0); __m256 _r201 = _mm256_broadcast_ss(r2 + 1); @@ -319,14 +319,14 @@ static void conv3x3s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const Mat kptr += 64; - _sum00 = _mm256_fmadd_ps(_r200, _k60, _sum00); - _sum01 = _mm256_fmadd_ps(_r201, _k61, _sum01); - _sum00 = _mm256_fmadd_ps(_r202, _k62, _sum00); - _sum01 = _mm256_fmadd_ps(_r203, _k63, _sum01); - _sum00 = _mm256_fmadd_ps(_r204, _k64, _sum00); - _sum01 = _mm256_fmadd_ps(_r205, _k65, _sum01); - _sum00 = _mm256_fmadd_ps(_r206, _k66, _sum00); - _sum01 = _mm256_fmadd_ps(_r207, _k67, _sum01); + _sum00 = _mm256_comp_fmadd_ps(_r200, _k60, _sum00); + _sum01 = _mm256_comp_fmadd_ps(_r201, _k61, _sum01); + _sum00 = _mm256_comp_fmadd_ps(_r202, _k62, _sum00); + _sum01 = _mm256_comp_fmadd_ps(_r203, _k63, _sum01); + _sum00 = _mm256_comp_fmadd_ps(_r204, _k64, _sum00); + _sum01 = _mm256_comp_fmadd_ps(_r205, _k65, _sum01); + _sum00 = _mm256_comp_fmadd_ps(_r206, _k66, _sum00); + _sum01 = _mm256_comp_fmadd_ps(_r207, _k67, _sum01); __m256 _r210 = _mm256_broadcast_ss(r2 + 8); __m256 _r211 = _mm256_broadcast_ss(r2 + 9); @@ -337,14 +337,14 @@ static void conv3x3s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const Mat __m256 _r216 = _mm256_broadcast_ss(r2 + 14); __m256 _r217 = _mm256_broadcast_ss(r2 + 15); - _sum10 = _mm256_fmadd_ps(_r210, _k60, _sum10); - _sum11 = _mm256_fmadd_ps(_r211, _k61, _sum11); - _sum10 = _mm256_fmadd_ps(_r212, _k62, _sum10); - _sum11 = _mm256_fmadd_ps(_r213, _k63, _sum11); - _sum10 = _mm256_fmadd_ps(_r214, _k64, _sum10); - _sum11 = _mm256_fmadd_ps(_r215, _k65, _sum11); - _sum10 = _mm256_fmadd_ps(_r216, _k66, _sum10); - _sum11 = _mm256_fmadd_ps(_r217, _k67, _sum11); + _sum10 = _mm256_comp_fmadd_ps(_r210, _k60, _sum10); + _sum11 = _mm256_comp_fmadd_ps(_r211, _k61, _sum11); + _sum10 = _mm256_comp_fmadd_ps(_r212, _k62, _sum10); + _sum11 = _mm256_comp_fmadd_ps(_r213, _k63, _sum11); + _sum10 = _mm256_comp_fmadd_ps(_r214, _k64, _sum10); + _sum11 = _mm256_comp_fmadd_ps(_r215, _k65, _sum11); + _sum10 = _mm256_comp_fmadd_ps(_r216, _k66, _sum10); + _sum11 = _mm256_comp_fmadd_ps(_r217, _k67, _sum11); __m256 _k70 = _mm256_loadu_ps(kptr); __m256 _k71 = _mm256_loadu_ps(kptr + 8); @@ -357,14 +357,14 @@ static void conv3x3s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const Mat kptr += 64; - _sum00 = _mm256_fmadd_ps(_r210, _k70, _sum00); - _sum01 = _mm256_fmadd_ps(_r211, _k71, _sum01); - _sum00 = _mm256_fmadd_ps(_r212, _k72, _sum00); - _sum01 = _mm256_fmadd_ps(_r213, _k73, _sum01); - _sum00 = _mm256_fmadd_ps(_r214, _k74, _sum00); - _sum01 = _mm256_fmadd_ps(_r215, _k75, _sum01); - _sum00 = _mm256_fmadd_ps(_r216, _k76, _sum00); - _sum01 = _mm256_fmadd_ps(_r217, _k77, _sum01); + _sum00 = _mm256_comp_fmadd_ps(_r210, _k70, _sum00); + _sum01 = _mm256_comp_fmadd_ps(_r211, _k71, _sum01); + _sum00 = _mm256_comp_fmadd_ps(_r212, _k72, _sum00); + _sum01 = _mm256_comp_fmadd_ps(_r213, _k73, _sum01); + _sum00 = _mm256_comp_fmadd_ps(_r214, _k74, _sum00); + _sum01 = _mm256_comp_fmadd_ps(_r215, _k75, _sum01); + _sum00 = _mm256_comp_fmadd_ps(_r216, _k76, _sum00); + _sum01 = _mm256_comp_fmadd_ps(_r217, _k77, _sum01); __m256 _r220 = _mm256_broadcast_ss(r2 + 16); __m256 _r221 = _mm256_broadcast_ss(r2 + 17); @@ -375,14 +375,14 @@ static void conv3x3s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const Mat __m256 _r226 = _mm256_broadcast_ss(r2 + 22); __m256 _r227 = _mm256_broadcast_ss(r2 + 23); - _sum10 = _mm256_fmadd_ps(_r220, _k70, _sum10); - _sum11 = _mm256_fmadd_ps(_r221, _k71, _sum11); - _sum10 = _mm256_fmadd_ps(_r222, _k72, _sum10); - _sum11 = _mm256_fmadd_ps(_r223, _k73, _sum11); - _sum10 = _mm256_fmadd_ps(_r224, _k74, _sum10); - _sum11 = _mm256_fmadd_ps(_r225, _k75, _sum11); - _sum10 = _mm256_fmadd_ps(_r226, _k76, _sum10); - _sum11 = _mm256_fmadd_ps(_r227, _k77, _sum11); + _sum10 = _mm256_comp_fmadd_ps(_r220, _k70, _sum10); + _sum11 = _mm256_comp_fmadd_ps(_r221, _k71, _sum11); + _sum10 = _mm256_comp_fmadd_ps(_r222, _k72, _sum10); + _sum11 = _mm256_comp_fmadd_ps(_r223, _k73, _sum11); + _sum10 = _mm256_comp_fmadd_ps(_r224, _k74, _sum10); + _sum11 = _mm256_comp_fmadd_ps(_r225, _k75, _sum11); + _sum10 = _mm256_comp_fmadd_ps(_r226, _k76, _sum10); + _sum11 = _mm256_comp_fmadd_ps(_r227, _k77, _sum11); __m256 _k80 = _mm256_loadu_ps(kptr); __m256 _k81 = _mm256_loadu_ps(kptr + 8); @@ -393,14 +393,14 @@ static void conv3x3s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const Mat __m256 _k86 = _mm256_loadu_ps(kptr + 48); __m256 _k87 = _mm256_loadu_ps(kptr + 56); - _sum00 = _mm256_fmadd_ps(_r220, _k80, _sum00); - _sum01 = _mm256_fmadd_ps(_r221, _k81, _sum01); - _sum00 = _mm256_fmadd_ps(_r222, _k82, _sum00); - _sum01 = _mm256_fmadd_ps(_r223, _k83, _sum01); - _sum00 = _mm256_fmadd_ps(_r224, _k84, _sum00); - _sum01 = _mm256_fmadd_ps(_r225, _k85, _sum01); - _sum00 = _mm256_fmadd_ps(_r226, _k86, _sum00); - _sum01 = _mm256_fmadd_ps(_r227, _k87, _sum01); + _sum00 = _mm256_comp_fmadd_ps(_r220, _k80, _sum00); + _sum01 = _mm256_comp_fmadd_ps(_r221, _k81, _sum01); + _sum00 = _mm256_comp_fmadd_ps(_r222, _k82, _sum00); + _sum01 = _mm256_comp_fmadd_ps(_r223, _k83, _sum01); + _sum00 = _mm256_comp_fmadd_ps(_r224, _k84, _sum00); + _sum01 = _mm256_comp_fmadd_ps(_r225, _k85, _sum01); + _sum00 = _mm256_comp_fmadd_ps(_r226, _k86, _sum00); + _sum01 = _mm256_comp_fmadd_ps(_r227, _k87, _sum01); __m256 _r230 = _mm256_broadcast_ss(r2 + 24); __m256 _r231 = _mm256_broadcast_ss(r2 + 25); @@ -411,14 +411,14 @@ static void conv3x3s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const Mat __m256 _r236 = _mm256_broadcast_ss(r2 + 30); __m256 _r237 = _mm256_broadcast_ss(r2 + 31); - _sum10 = _mm256_fmadd_ps(_r230, _k80, _sum10); - _sum11 = _mm256_fmadd_ps(_r231, _k81, _sum11); - _sum10 = _mm256_fmadd_ps(_r232, _k82, _sum10); - _sum11 = _mm256_fmadd_ps(_r233, _k83, _sum11); - _sum10 = _mm256_fmadd_ps(_r234, _k84, _sum10); - _sum11 = _mm256_fmadd_ps(_r235, _k85, _sum11); - _sum10 = _mm256_fmadd_ps(_r236, _k86, _sum10); - _sum11 = _mm256_fmadd_ps(_r237, _k87, _sum11); + _sum10 = _mm256_comp_fmadd_ps(_r230, _k80, _sum10); + _sum11 = _mm256_comp_fmadd_ps(_r231, _k81, _sum11); + _sum10 = _mm256_comp_fmadd_ps(_r232, _k82, _sum10); + _sum11 = _mm256_comp_fmadd_ps(_r233, _k83, _sum11); + _sum10 = _mm256_comp_fmadd_ps(_r234, _k84, _sum10); + _sum11 = _mm256_comp_fmadd_ps(_r235, _k85, _sum11); + _sum10 = _mm256_comp_fmadd_ps(_r236, _k86, _sum10); + _sum11 = _mm256_comp_fmadd_ps(_r237, _k87, _sum11); kptr -= 64 * 8; @@ -458,14 +458,14 @@ static void conv3x3s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const Mat kptr += 64; - _sum0 = _mm256_fmadd_ps(_r000, _k00, _sum0); - _sum1 = _mm256_fmadd_ps(_r001, _k01, _sum1); - _sum0 = _mm256_fmadd_ps(_r002, _k02, _sum0); - _sum1 = _mm256_fmadd_ps(_r003, _k03, _sum1); - _sum0 = _mm256_fmadd_ps(_r004, _k04, _sum0); - _sum1 = _mm256_fmadd_ps(_r005, _k05, _sum1); - _sum0 = _mm256_fmadd_ps(_r006, _k06, _sum0); - _sum1 = _mm256_fmadd_ps(_r007, _k07, _sum1); + _sum0 = _mm256_comp_fmadd_ps(_r000, _k00, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_r001, _k01, _sum1); + _sum0 = _mm256_comp_fmadd_ps(_r002, _k02, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_r003, _k03, _sum1); + _sum0 = _mm256_comp_fmadd_ps(_r004, _k04, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_r005, _k05, _sum1); + _sum0 = _mm256_comp_fmadd_ps(_r006, _k06, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_r007, _k07, _sum1); __m256 _r010 = _mm256_broadcast_ss(r0 + 8); __m256 _r011 = _mm256_broadcast_ss(r0 + 9); @@ -487,14 +487,14 @@ static void conv3x3s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const Mat kptr += 64; - _sum0 = _mm256_fmadd_ps(_r010, _k10, _sum0); - _sum1 = _mm256_fmadd_ps(_r011, _k11, _sum1); - _sum0 = _mm256_fmadd_ps(_r012, _k12, _sum0); - _sum1 = _mm256_fmadd_ps(_r013, _k13, _sum1); - _sum0 = _mm256_fmadd_ps(_r014, _k14, _sum0); - _sum1 = _mm256_fmadd_ps(_r015, _k15, _sum1); - _sum0 = _mm256_fmadd_ps(_r016, _k16, _sum0); - _sum1 = _mm256_fmadd_ps(_r017, _k17, _sum1); + _sum0 = _mm256_comp_fmadd_ps(_r010, _k10, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_r011, _k11, _sum1); + _sum0 = _mm256_comp_fmadd_ps(_r012, _k12, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_r013, _k13, _sum1); + _sum0 = _mm256_comp_fmadd_ps(_r014, _k14, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_r015, _k15, _sum1); + _sum0 = _mm256_comp_fmadd_ps(_r016, _k16, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_r017, _k17, _sum1); __m256 _r020 = _mm256_broadcast_ss(r0 + 16); __m256 _r021 = _mm256_broadcast_ss(r0 + 17); @@ -516,14 +516,14 @@ static void conv3x3s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const Mat kptr += 64; - _sum0 = _mm256_fmadd_ps(_r020, _k20, _sum0); - _sum1 = _mm256_fmadd_ps(_r021, _k21, _sum1); - _sum0 = _mm256_fmadd_ps(_r022, _k22, _sum0); - _sum1 = _mm256_fmadd_ps(_r023, _k23, _sum1); - _sum0 = _mm256_fmadd_ps(_r024, _k24, _sum0); - _sum1 = _mm256_fmadd_ps(_r025, _k25, _sum1); - _sum0 = _mm256_fmadd_ps(_r026, _k26, _sum0); - _sum1 = _mm256_fmadd_ps(_r027, _k27, _sum1); + _sum0 = _mm256_comp_fmadd_ps(_r020, _k20, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_r021, _k21, _sum1); + _sum0 = _mm256_comp_fmadd_ps(_r022, _k22, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_r023, _k23, _sum1); + _sum0 = _mm256_comp_fmadd_ps(_r024, _k24, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_r025, _k25, _sum1); + _sum0 = _mm256_comp_fmadd_ps(_r026, _k26, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_r027, _k27, _sum1); __m256 _r100 = _mm256_broadcast_ss(r1 + 0); __m256 _r101 = _mm256_broadcast_ss(r1 + 1); @@ -545,14 +545,14 @@ static void conv3x3s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const Mat kptr += 64; - _sum0 = _mm256_fmadd_ps(_r100, _k30, _sum0); - _sum1 = _mm256_fmadd_ps(_r101, _k31, _sum1); - _sum0 = _mm256_fmadd_ps(_r102, _k32, _sum0); - _sum1 = _mm256_fmadd_ps(_r103, _k33, _sum1); - _sum0 = _mm256_fmadd_ps(_r104, _k34, _sum0); - _sum1 = _mm256_fmadd_ps(_r105, _k35, _sum1); - _sum0 = _mm256_fmadd_ps(_r106, _k36, _sum0); - _sum1 = _mm256_fmadd_ps(_r107, _k37, _sum1); + _sum0 = _mm256_comp_fmadd_ps(_r100, _k30, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_r101, _k31, _sum1); + _sum0 = _mm256_comp_fmadd_ps(_r102, _k32, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_r103, _k33, _sum1); + _sum0 = _mm256_comp_fmadd_ps(_r104, _k34, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_r105, _k35, _sum1); + _sum0 = _mm256_comp_fmadd_ps(_r106, _k36, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_r107, _k37, _sum1); __m256 _r110 = _mm256_broadcast_ss(r1 + 8); __m256 _r111 = _mm256_broadcast_ss(r1 + 9); @@ -574,14 +574,14 @@ static void conv3x3s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const Mat kptr += 64; - _sum0 = _mm256_fmadd_ps(_r110, _k40, _sum0); - _sum1 = _mm256_fmadd_ps(_r111, _k41, _sum1); - _sum0 = _mm256_fmadd_ps(_r112, _k42, _sum0); - _sum1 = _mm256_fmadd_ps(_r113, _k43, _sum1); - _sum0 = _mm256_fmadd_ps(_r114, _k44, _sum0); - _sum1 = _mm256_fmadd_ps(_r115, _k45, _sum1); - _sum0 = _mm256_fmadd_ps(_r116, _k46, _sum0); - _sum1 = _mm256_fmadd_ps(_r117, _k47, _sum1); + _sum0 = _mm256_comp_fmadd_ps(_r110, _k40, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_r111, _k41, _sum1); + _sum0 = _mm256_comp_fmadd_ps(_r112, _k42, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_r113, _k43, _sum1); + _sum0 = _mm256_comp_fmadd_ps(_r114, _k44, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_r115, _k45, _sum1); + _sum0 = _mm256_comp_fmadd_ps(_r116, _k46, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_r117, _k47, _sum1); __m256 _r120 = _mm256_broadcast_ss(r1 + 16); __m256 _r121 = _mm256_broadcast_ss(r1 + 17); @@ -603,14 +603,14 @@ static void conv3x3s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const Mat kptr += 64; - _sum0 = _mm256_fmadd_ps(_r120, _k50, _sum0); - _sum1 = _mm256_fmadd_ps(_r121, _k51, _sum1); - _sum0 = _mm256_fmadd_ps(_r122, _k52, _sum0); - _sum1 = _mm256_fmadd_ps(_r123, _k53, _sum1); - _sum0 = _mm256_fmadd_ps(_r124, _k54, _sum0); - _sum1 = _mm256_fmadd_ps(_r125, _k55, _sum1); - _sum0 = _mm256_fmadd_ps(_r126, _k56, _sum0); - _sum1 = _mm256_fmadd_ps(_r127, _k57, _sum1); + _sum0 = _mm256_comp_fmadd_ps(_r120, _k50, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_r121, _k51, _sum1); + _sum0 = _mm256_comp_fmadd_ps(_r122, _k52, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_r123, _k53, _sum1); + _sum0 = _mm256_comp_fmadd_ps(_r124, _k54, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_r125, _k55, _sum1); + _sum0 = _mm256_comp_fmadd_ps(_r126, _k56, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_r127, _k57, _sum1); __m256 _r200 = _mm256_broadcast_ss(r2 + 0); __m256 _r201 = _mm256_broadcast_ss(r2 + 1); @@ -632,14 +632,14 @@ static void conv3x3s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const Mat kptr += 64; - _sum0 = _mm256_fmadd_ps(_r200, _k60, _sum0); - _sum1 = _mm256_fmadd_ps(_r201, _k61, _sum1); - _sum0 = _mm256_fmadd_ps(_r202, _k62, _sum0); - _sum1 = _mm256_fmadd_ps(_r203, _k63, _sum1); - _sum0 = _mm256_fmadd_ps(_r204, _k64, _sum0); - _sum1 = _mm256_fmadd_ps(_r205, _k65, _sum1); - _sum0 = _mm256_fmadd_ps(_r206, _k66, _sum0); - _sum1 = _mm256_fmadd_ps(_r207, _k67, _sum1); + _sum0 = _mm256_comp_fmadd_ps(_r200, _k60, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_r201, _k61, _sum1); + _sum0 = _mm256_comp_fmadd_ps(_r202, _k62, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_r203, _k63, _sum1); + _sum0 = _mm256_comp_fmadd_ps(_r204, _k64, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_r205, _k65, _sum1); + _sum0 = _mm256_comp_fmadd_ps(_r206, _k66, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_r207, _k67, _sum1); __m256 _r210 = _mm256_broadcast_ss(r2 + 8); __m256 _r211 = _mm256_broadcast_ss(r2 + 9); @@ -661,14 +661,14 @@ static void conv3x3s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const Mat kptr += 64; - _sum0 = _mm256_fmadd_ps(_r210, _k70, _sum0); - _sum1 = _mm256_fmadd_ps(_r211, _k71, _sum1); - _sum0 = _mm256_fmadd_ps(_r212, _k72, _sum0); - _sum1 = _mm256_fmadd_ps(_r213, _k73, _sum1); - _sum0 = _mm256_fmadd_ps(_r214, _k74, _sum0); - _sum1 = _mm256_fmadd_ps(_r215, _k75, _sum1); - _sum0 = _mm256_fmadd_ps(_r216, _k76, _sum0); - _sum1 = _mm256_fmadd_ps(_r217, _k77, _sum1); + _sum0 = _mm256_comp_fmadd_ps(_r210, _k70, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_r211, _k71, _sum1); + _sum0 = _mm256_comp_fmadd_ps(_r212, _k72, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_r213, _k73, _sum1); + _sum0 = _mm256_comp_fmadd_ps(_r214, _k74, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_r215, _k75, _sum1); + _sum0 = _mm256_comp_fmadd_ps(_r216, _k76, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_r217, _k77, _sum1); __m256 _r220 = _mm256_broadcast_ss(r2 + 16); __m256 _r221 = _mm256_broadcast_ss(r2 + 17); @@ -688,14 +688,14 @@ static void conv3x3s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const Mat __m256 _k86 = _mm256_loadu_ps(kptr + 48); __m256 _k87 = _mm256_loadu_ps(kptr + 56); - _sum0 = _mm256_fmadd_ps(_r220, _k80, _sum0); - _sum1 = _mm256_fmadd_ps(_r221, _k81, _sum1); - _sum0 = _mm256_fmadd_ps(_r222, _k82, _sum0); - _sum1 = _mm256_fmadd_ps(_r223, _k83, _sum1); - _sum0 = _mm256_fmadd_ps(_r224, _k84, _sum0); - _sum1 = _mm256_fmadd_ps(_r225, _k85, _sum1); - _sum0 = _mm256_fmadd_ps(_r226, _k86, _sum0); - _sum1 = _mm256_fmadd_ps(_r227, _k87, _sum1); + _sum0 = _mm256_comp_fmadd_ps(_r220, _k80, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_r221, _k81, _sum1); + _sum0 = _mm256_comp_fmadd_ps(_r222, _k82, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_r223, _k83, _sum1); + _sum0 = _mm256_comp_fmadd_ps(_r224, _k84, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_r225, _k85, _sum1); + _sum0 = _mm256_comp_fmadd_ps(_r226, _k86, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_r227, _k87, _sum1); kptr -= 64 * 8; @@ -1367,225 +1367,225 @@ static void conv3x3s1_winograd64_pack8_avx(const Mat& bottom_blob, Mat& top_blob __m256 _r01 = _mm256_broadcast_ss(r0 + 8); __m256 _r02 = _mm256_broadcast_ss(r0 + 16); __m256 _r03 = _mm256_broadcast_ss(r0 + 24); - _sum0 = _mm256_fmadd_ps(_k01, _r00, _sum0); - _sum1 = _mm256_fmadd_ps(_k01, _r01, _sum1); - _sum2 = _mm256_fmadd_ps(_k01, _r02, _sum2); - _sum3 = _mm256_fmadd_ps(_k01, _r03, _sum3); + _sum0 = _mm256_comp_fmadd_ps(_k01, _r00, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_k01, _r01, _sum1); + _sum2 = _mm256_comp_fmadd_ps(_k01, _r02, _sum2); + _sum3 = _mm256_comp_fmadd_ps(_k01, _r03, _sum3); __m256 _r04 = _mm256_broadcast_ss(r0 + 32); __m256 _r05 = _mm256_broadcast_ss(r0 + 40); __m256 _r06 = _mm256_broadcast_ss(r0 + 48); __m256 _r07 = _mm256_broadcast_ss(r0 + 56); - _sum4 = _mm256_fmadd_ps(_k01, _r04, _sum4); - _sum5 = _mm256_fmadd_ps(_k01, _r05, _sum5); - _sum6 = _mm256_fmadd_ps(_k01, _r06, _sum6); - _sum7 = _mm256_fmadd_ps(_k01, _r07, _sum7); + _sum4 = _mm256_comp_fmadd_ps(_k01, _r04, _sum4); + _sum5 = _mm256_comp_fmadd_ps(_k01, _r05, _sum5); + _sum6 = _mm256_comp_fmadd_ps(_k01, _r06, _sum6); + _sum7 = _mm256_comp_fmadd_ps(_k01, _r07, _sum7); __m256 _r08 = _mm256_broadcast_ss(r0 + 64); __m256 _r09 = _mm256_broadcast_ss(r0 + 72); __m256 _r010 = _mm256_broadcast_ss(r0 + 80); __m256 _r011 = _mm256_broadcast_ss(r0 + 88); - _sum8 = _mm256_fmadd_ps(_k01, _r08, _sum8); - _sum9 = _mm256_fmadd_ps(_k01, _r09, _sum9); - _sum10 = _mm256_fmadd_ps(_k01, _r010, _sum10); - _sum11 = _mm256_fmadd_ps(_k01, _r011, _sum11); + _sum8 = _mm256_comp_fmadd_ps(_k01, _r08, _sum8); + _sum9 = _mm256_comp_fmadd_ps(_k01, _r09, _sum9); + _sum10 = _mm256_comp_fmadd_ps(_k01, _r010, _sum10); + _sum11 = _mm256_comp_fmadd_ps(_k01, _r011, _sum11); _k01 = _mm256_loadu_ps(k01 + 8); _r00 = _mm256_broadcast_ss(r0 + 1); _r01 = _mm256_broadcast_ss(r0 + 9); _r02 = _mm256_broadcast_ss(r0 + 17); _r03 = _mm256_broadcast_ss(r0 + 25); - _sum0 = _mm256_fmadd_ps(_k01, _r00, _sum0); - _sum1 = _mm256_fmadd_ps(_k01, _r01, _sum1); - _sum2 = _mm256_fmadd_ps(_k01, _r02, _sum2); - _sum3 = _mm256_fmadd_ps(_k01, _r03, _sum3); + _sum0 = _mm256_comp_fmadd_ps(_k01, _r00, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_k01, _r01, _sum1); + _sum2 = _mm256_comp_fmadd_ps(_k01, _r02, _sum2); + _sum3 = _mm256_comp_fmadd_ps(_k01, _r03, _sum3); _r04 = _mm256_broadcast_ss(r0 + 33); _r05 = _mm256_broadcast_ss(r0 + 41); _r06 = _mm256_broadcast_ss(r0 + 49); _r07 = _mm256_broadcast_ss(r0 + 57); - _sum4 = _mm256_fmadd_ps(_k01, _r04, _sum4); - _sum5 = _mm256_fmadd_ps(_k01, _r05, _sum5); - _sum6 = _mm256_fmadd_ps(_k01, _r06, _sum6); - _sum7 = _mm256_fmadd_ps(_k01, _r07, _sum7); + _sum4 = _mm256_comp_fmadd_ps(_k01, _r04, _sum4); + _sum5 = _mm256_comp_fmadd_ps(_k01, _r05, _sum5); + _sum6 = _mm256_comp_fmadd_ps(_k01, _r06, _sum6); + _sum7 = _mm256_comp_fmadd_ps(_k01, _r07, _sum7); _r08 = _mm256_broadcast_ss(r0 + 65); _r09 = _mm256_broadcast_ss(r0 + 73); _r010 = _mm256_broadcast_ss(r0 + 81); _r011 = _mm256_broadcast_ss(r0 + 89); - _sum8 = _mm256_fmadd_ps(_k01, _r08, _sum8); - _sum9 = _mm256_fmadd_ps(_k01, _r09, _sum9); - _sum10 = _mm256_fmadd_ps(_k01, _r010, _sum10); - _sum11 = _mm256_fmadd_ps(_k01, _r011, _sum11); + _sum8 = _mm256_comp_fmadd_ps(_k01, _r08, _sum8); + _sum9 = _mm256_comp_fmadd_ps(_k01, _r09, _sum9); + _sum10 = _mm256_comp_fmadd_ps(_k01, _r010, _sum10); + _sum11 = _mm256_comp_fmadd_ps(_k01, _r011, _sum11); _k01 = _mm256_loadu_ps(k01 + 16); _r00 = _mm256_broadcast_ss(r0 + 2); _r01 = _mm256_broadcast_ss(r0 + 10); _r02 = _mm256_broadcast_ss(r0 + 18); _r03 = _mm256_broadcast_ss(r0 + 26); - _sum0 = _mm256_fmadd_ps(_k01, _r00, _sum0); - _sum1 = _mm256_fmadd_ps(_k01, _r01, _sum1); - _sum2 = _mm256_fmadd_ps(_k01, _r02, _sum2); - _sum3 = _mm256_fmadd_ps(_k01, _r03, _sum3); + _sum0 = _mm256_comp_fmadd_ps(_k01, _r00, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_k01, _r01, _sum1); + _sum2 = _mm256_comp_fmadd_ps(_k01, _r02, _sum2); + _sum3 = _mm256_comp_fmadd_ps(_k01, _r03, _sum3); _r04 = _mm256_broadcast_ss(r0 + 34); _r05 = _mm256_broadcast_ss(r0 + 42); _r06 = _mm256_broadcast_ss(r0 + 50); _r07 = _mm256_broadcast_ss(r0 + 58); - _sum4 = _mm256_fmadd_ps(_k01, _r04, _sum4); - _sum5 = _mm256_fmadd_ps(_k01, _r05, _sum5); - _sum6 = _mm256_fmadd_ps(_k01, _r06, _sum6); - _sum7 = _mm256_fmadd_ps(_k01, _r07, _sum7); + _sum4 = _mm256_comp_fmadd_ps(_k01, _r04, _sum4); + _sum5 = _mm256_comp_fmadd_ps(_k01, _r05, _sum5); + _sum6 = _mm256_comp_fmadd_ps(_k01, _r06, _sum6); + _sum7 = _mm256_comp_fmadd_ps(_k01, _r07, _sum7); _r08 = _mm256_broadcast_ss(r0 + 66); _r09 = _mm256_broadcast_ss(r0 + 74); _r010 = _mm256_broadcast_ss(r0 + 82); _r011 = _mm256_broadcast_ss(r0 + 90); - _sum8 = _mm256_fmadd_ps(_k01, _r08, _sum8); - _sum9 = _mm256_fmadd_ps(_k01, _r09, _sum9); - _sum10 = _mm256_fmadd_ps(_k01, _r010, _sum10); - _sum11 = _mm256_fmadd_ps(_k01, _r011, _sum11); + _sum8 = _mm256_comp_fmadd_ps(_k01, _r08, _sum8); + _sum9 = _mm256_comp_fmadd_ps(_k01, _r09, _sum9); + _sum10 = _mm256_comp_fmadd_ps(_k01, _r010, _sum10); + _sum11 = _mm256_comp_fmadd_ps(_k01, _r011, _sum11); _k01 = _mm256_loadu_ps(k01 + 24); _r00 = _mm256_broadcast_ss(r0 + 3); _r01 = _mm256_broadcast_ss(r0 + 11); _r02 = _mm256_broadcast_ss(r0 + 19); _r03 = _mm256_broadcast_ss(r0 + 27); - _sum0 = _mm256_fmadd_ps(_k01, _r00, _sum0); - _sum1 = _mm256_fmadd_ps(_k01, _r01, _sum1); - _sum2 = _mm256_fmadd_ps(_k01, _r02, _sum2); - _sum3 = _mm256_fmadd_ps(_k01, _r03, _sum3); + _sum0 = _mm256_comp_fmadd_ps(_k01, _r00, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_k01, _r01, _sum1); + _sum2 = _mm256_comp_fmadd_ps(_k01, _r02, _sum2); + _sum3 = _mm256_comp_fmadd_ps(_k01, _r03, _sum3); _r04 = _mm256_broadcast_ss(r0 + 35); _r05 = _mm256_broadcast_ss(r0 + 43); _r06 = _mm256_broadcast_ss(r0 + 51); _r07 = _mm256_broadcast_ss(r0 + 59); - _sum4 = _mm256_fmadd_ps(_k01, _r04, _sum4); - _sum5 = _mm256_fmadd_ps(_k01, _r05, _sum5); - _sum6 = _mm256_fmadd_ps(_k01, _r06, _sum6); - _sum7 = _mm256_fmadd_ps(_k01, _r07, _sum7); + _sum4 = _mm256_comp_fmadd_ps(_k01, _r04, _sum4); + _sum5 = _mm256_comp_fmadd_ps(_k01, _r05, _sum5); + _sum6 = _mm256_comp_fmadd_ps(_k01, _r06, _sum6); + _sum7 = _mm256_comp_fmadd_ps(_k01, _r07, _sum7); _r08 = _mm256_broadcast_ss(r0 + 67); _r09 = _mm256_broadcast_ss(r0 + 75); _r010 = _mm256_broadcast_ss(r0 + 83); _r011 = _mm256_broadcast_ss(r0 + 91); - _sum8 = _mm256_fmadd_ps(_k01, _r08, _sum8); - _sum9 = _mm256_fmadd_ps(_k01, _r09, _sum9); - _sum10 = _mm256_fmadd_ps(_k01, _r010, _sum10); - _sum11 = _mm256_fmadd_ps(_k01, _r011, _sum11); + _sum8 = _mm256_comp_fmadd_ps(_k01, _r08, _sum8); + _sum9 = _mm256_comp_fmadd_ps(_k01, _r09, _sum9); + _sum10 = _mm256_comp_fmadd_ps(_k01, _r010, _sum10); + _sum11 = _mm256_comp_fmadd_ps(_k01, _r011, _sum11); _k01 = _mm256_loadu_ps(k01 + 32); _r00 = _mm256_broadcast_ss(r0 + 4); _r01 = _mm256_broadcast_ss(r0 + 12); _r02 = _mm256_broadcast_ss(r0 + 20); _r03 = _mm256_broadcast_ss(r0 + 28); - _sum0 = _mm256_fmadd_ps(_k01, _r00, _sum0); - _sum1 = _mm256_fmadd_ps(_k01, _r01, _sum1); - _sum2 = _mm256_fmadd_ps(_k01, _r02, _sum2); - _sum3 = _mm256_fmadd_ps(_k01, _r03, _sum3); + _sum0 = _mm256_comp_fmadd_ps(_k01, _r00, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_k01, _r01, _sum1); + _sum2 = _mm256_comp_fmadd_ps(_k01, _r02, _sum2); + _sum3 = _mm256_comp_fmadd_ps(_k01, _r03, _sum3); _r04 = _mm256_broadcast_ss(r0 + 36); _r05 = _mm256_broadcast_ss(r0 + 44); _r06 = _mm256_broadcast_ss(r0 + 52); _r07 = _mm256_broadcast_ss(r0 + 60); - _sum4 = _mm256_fmadd_ps(_k01, _r04, _sum4); - _sum5 = _mm256_fmadd_ps(_k01, _r05, _sum5); - _sum6 = _mm256_fmadd_ps(_k01, _r06, _sum6); - _sum7 = _mm256_fmadd_ps(_k01, _r07, _sum7); + _sum4 = _mm256_comp_fmadd_ps(_k01, _r04, _sum4); + _sum5 = _mm256_comp_fmadd_ps(_k01, _r05, _sum5); + _sum6 = _mm256_comp_fmadd_ps(_k01, _r06, _sum6); + _sum7 = _mm256_comp_fmadd_ps(_k01, _r07, _sum7); _r08 = _mm256_broadcast_ss(r0 + 68); _r09 = _mm256_broadcast_ss(r0 + 76); _r010 = _mm256_broadcast_ss(r0 + 84); _r011 = _mm256_broadcast_ss(r0 + 92); - _sum8 = _mm256_fmadd_ps(_k01, _r08, _sum8); - _sum9 = _mm256_fmadd_ps(_k01, _r09, _sum9); - _sum10 = _mm256_fmadd_ps(_k01, _r010, _sum10); - _sum11 = _mm256_fmadd_ps(_k01, _r011, _sum11); + _sum8 = _mm256_comp_fmadd_ps(_k01, _r08, _sum8); + _sum9 = _mm256_comp_fmadd_ps(_k01, _r09, _sum9); + _sum10 = _mm256_comp_fmadd_ps(_k01, _r010, _sum10); + _sum11 = _mm256_comp_fmadd_ps(_k01, _r011, _sum11); _k01 = _mm256_loadu_ps(k01 + 40); _r00 = _mm256_broadcast_ss(r0 + 5); _r01 = _mm256_broadcast_ss(r0 + 13); _r02 = _mm256_broadcast_ss(r0 + 21); _r03 = _mm256_broadcast_ss(r0 + 29); - _sum0 = _mm256_fmadd_ps(_k01, _r00, _sum0); - _sum1 = _mm256_fmadd_ps(_k01, _r01, _sum1); - _sum2 = _mm256_fmadd_ps(_k01, _r02, _sum2); - _sum3 = _mm256_fmadd_ps(_k01, _r03, _sum3); + _sum0 = _mm256_comp_fmadd_ps(_k01, _r00, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_k01, _r01, _sum1); + _sum2 = _mm256_comp_fmadd_ps(_k01, _r02, _sum2); + _sum3 = _mm256_comp_fmadd_ps(_k01, _r03, _sum3); _r04 = _mm256_broadcast_ss(r0 + 37); _r05 = _mm256_broadcast_ss(r0 + 45); _r06 = _mm256_broadcast_ss(r0 + 53); _r07 = _mm256_broadcast_ss(r0 + 61); - _sum4 = _mm256_fmadd_ps(_k01, _r04, _sum4); - _sum5 = _mm256_fmadd_ps(_k01, _r05, _sum5); - _sum6 = _mm256_fmadd_ps(_k01, _r06, _sum6); - _sum7 = _mm256_fmadd_ps(_k01, _r07, _sum7); + _sum4 = _mm256_comp_fmadd_ps(_k01, _r04, _sum4); + _sum5 = _mm256_comp_fmadd_ps(_k01, _r05, _sum5); + _sum6 = _mm256_comp_fmadd_ps(_k01, _r06, _sum6); + _sum7 = _mm256_comp_fmadd_ps(_k01, _r07, _sum7); _r08 = _mm256_broadcast_ss(r0 + 69); _r09 = _mm256_broadcast_ss(r0 + 77); _r010 = _mm256_broadcast_ss(r0 + 85); _r011 = _mm256_broadcast_ss(r0 + 93); - _sum8 = _mm256_fmadd_ps(_k01, _r08, _sum8); - _sum9 = _mm256_fmadd_ps(_k01, _r09, _sum9); - _sum10 = _mm256_fmadd_ps(_k01, _r010, _sum10); - _sum11 = _mm256_fmadd_ps(_k01, _r011, _sum11); + _sum8 = _mm256_comp_fmadd_ps(_k01, _r08, _sum8); + _sum9 = _mm256_comp_fmadd_ps(_k01, _r09, _sum9); + _sum10 = _mm256_comp_fmadd_ps(_k01, _r010, _sum10); + _sum11 = _mm256_comp_fmadd_ps(_k01, _r011, _sum11); _k01 = _mm256_loadu_ps(k01 + 48); _r00 = _mm256_broadcast_ss(r0 + 6); _r01 = _mm256_broadcast_ss(r0 + 14); _r02 = _mm256_broadcast_ss(r0 + 22); _r03 = _mm256_broadcast_ss(r0 + 30); - _sum0 = _mm256_fmadd_ps(_k01, _r00, _sum0); - _sum1 = _mm256_fmadd_ps(_k01, _r01, _sum1); - _sum2 = _mm256_fmadd_ps(_k01, _r02, _sum2); - _sum3 = _mm256_fmadd_ps(_k01, _r03, _sum3); + _sum0 = _mm256_comp_fmadd_ps(_k01, _r00, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_k01, _r01, _sum1); + _sum2 = _mm256_comp_fmadd_ps(_k01, _r02, _sum2); + _sum3 = _mm256_comp_fmadd_ps(_k01, _r03, _sum3); _r04 = _mm256_broadcast_ss(r0 + 38); _r05 = _mm256_broadcast_ss(r0 + 46); _r06 = _mm256_broadcast_ss(r0 + 54); _r07 = _mm256_broadcast_ss(r0 + 62); - _sum4 = _mm256_fmadd_ps(_k01, _r04, _sum4); - _sum5 = _mm256_fmadd_ps(_k01, _r05, _sum5); - _sum6 = _mm256_fmadd_ps(_k01, _r06, _sum6); - _sum7 = _mm256_fmadd_ps(_k01, _r07, _sum7); + _sum4 = _mm256_comp_fmadd_ps(_k01, _r04, _sum4); + _sum5 = _mm256_comp_fmadd_ps(_k01, _r05, _sum5); + _sum6 = _mm256_comp_fmadd_ps(_k01, _r06, _sum6); + _sum7 = _mm256_comp_fmadd_ps(_k01, _r07, _sum7); _r08 = _mm256_broadcast_ss(r0 + 70); _r09 = _mm256_broadcast_ss(r0 + 78); _r010 = _mm256_broadcast_ss(r0 + 86); _r011 = _mm256_broadcast_ss(r0 + 94); - _sum8 = _mm256_fmadd_ps(_k01, _r08, _sum8); - _sum9 = _mm256_fmadd_ps(_k01, _r09, _sum9); - _sum10 = _mm256_fmadd_ps(_k01, _r010, _sum10); - _sum11 = _mm256_fmadd_ps(_k01, _r011, _sum11); + _sum8 = _mm256_comp_fmadd_ps(_k01, _r08, _sum8); + _sum9 = _mm256_comp_fmadd_ps(_k01, _r09, _sum9); + _sum10 = _mm256_comp_fmadd_ps(_k01, _r010, _sum10); + _sum11 = _mm256_comp_fmadd_ps(_k01, _r011, _sum11); _k01 = _mm256_loadu_ps(k01 + 56); _r00 = _mm256_broadcast_ss(r0 + 7); _r01 = _mm256_broadcast_ss(r0 + 15); _r02 = _mm256_broadcast_ss(r0 + 23); _r03 = _mm256_broadcast_ss(r0 + 31); - _sum0 = _mm256_fmadd_ps(_k01, _r00, _sum0); - _sum1 = _mm256_fmadd_ps(_k01, _r01, _sum1); - _sum2 = _mm256_fmadd_ps(_k01, _r02, _sum2); - _sum3 = _mm256_fmadd_ps(_k01, _r03, _sum3); + _sum0 = _mm256_comp_fmadd_ps(_k01, _r00, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_k01, _r01, _sum1); + _sum2 = _mm256_comp_fmadd_ps(_k01, _r02, _sum2); + _sum3 = _mm256_comp_fmadd_ps(_k01, _r03, _sum3); _r04 = _mm256_broadcast_ss(r0 + 39); _r05 = _mm256_broadcast_ss(r0 + 47); _r06 = _mm256_broadcast_ss(r0 + 55); _r07 = _mm256_broadcast_ss(r0 + 63); - _sum4 = _mm256_fmadd_ps(_k01, _r04, _sum4); - _sum5 = _mm256_fmadd_ps(_k01, _r05, _sum5); - _sum6 = _mm256_fmadd_ps(_k01, _r06, _sum6); - _sum7 = _mm256_fmadd_ps(_k01, _r07, _sum7); + _sum4 = _mm256_comp_fmadd_ps(_k01, _r04, _sum4); + _sum5 = _mm256_comp_fmadd_ps(_k01, _r05, _sum5); + _sum6 = _mm256_comp_fmadd_ps(_k01, _r06, _sum6); + _sum7 = _mm256_comp_fmadd_ps(_k01, _r07, _sum7); _r08 = _mm256_broadcast_ss(r0 + 71); _r09 = _mm256_broadcast_ss(r0 + 79); _r010 = _mm256_broadcast_ss(r0 + 87); _r011 = _mm256_broadcast_ss(r0 + 95); - _sum8 = _mm256_fmadd_ps(_k01, _r08, _sum8); - _sum9 = _mm256_fmadd_ps(_k01, _r09, _sum9); - _sum10 = _mm256_fmadd_ps(_k01, _r010, _sum10); - _sum11 = _mm256_fmadd_ps(_k01, _r011, _sum11); + _sum8 = _mm256_comp_fmadd_ps(_k01, _r08, _sum8); + _sum9 = _mm256_comp_fmadd_ps(_k01, _r09, _sum9); + _sum10 = _mm256_comp_fmadd_ps(_k01, _r010, _sum10); + _sum11 = _mm256_comp_fmadd_ps(_k01, _r011, _sum11); k01 += 64; r0 += 96; @@ -1626,18 +1626,18 @@ static void conv3x3s1_winograd64_pack8_avx(const Mat& bottom_blob, Mat& top_blob __m256 _r01 = _mm256_broadcast_ss(r0 + 8); __m256 _r02 = _mm256_broadcast_ss(r0 + 16); __m256 _r03 = _mm256_broadcast_ss(r0 + 24); - _sum0 = _mm256_fmadd_ps(_k01, _r00, _sum0); - _sum1 = _mm256_fmadd_ps(_k01, _r01, _sum1); - _sum2 = _mm256_fmadd_ps(_k01, _r02, _sum2); - _sum3 = _mm256_fmadd_ps(_k01, _r03, _sum3); + _sum0 = _mm256_comp_fmadd_ps(_k01, _r00, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_k01, _r01, _sum1); + _sum2 = _mm256_comp_fmadd_ps(_k01, _r02, _sum2); + _sum3 = _mm256_comp_fmadd_ps(_k01, _r03, _sum3); __m256 _r04 = _mm256_broadcast_ss(r0 + 32); __m256 _r05 = _mm256_broadcast_ss(r0 + 40); __m256 _r06 = _mm256_broadcast_ss(r0 + 48); __m256 _r07 = _mm256_broadcast_ss(r0 + 56); - _sum4 = _mm256_fmadd_ps(_k01, _r04, _sum4); - _sum5 = _mm256_fmadd_ps(_k01, _r05, _sum5); - _sum6 = _mm256_fmadd_ps(_k01, _r06, _sum6); - _sum7 = _mm256_fmadd_ps(_k01, _r07, _sum7); + _sum4 = _mm256_comp_fmadd_ps(_k01, _r04, _sum4); + _sum5 = _mm256_comp_fmadd_ps(_k01, _r05, _sum5); + _sum6 = _mm256_comp_fmadd_ps(_k01, _r06, _sum6); + _sum7 = _mm256_comp_fmadd_ps(_k01, _r07, _sum7); _k01 = _mm256_loadu_ps(k01 + 8); _r00 = _mm256_broadcast_ss(r0 + 1); @@ -1645,132 +1645,132 @@ static void conv3x3s1_winograd64_pack8_avx(const Mat& bottom_blob, Mat& top_blob _r02 = _mm256_broadcast_ss(r0 + 17); _r03 = _mm256_broadcast_ss(r0 + 25); - _sum0 = _mm256_fmadd_ps(_k01, _r00, _sum0); - _sum1 = _mm256_fmadd_ps(_k01, _r01, _sum1); - _sum2 = _mm256_fmadd_ps(_k01, _r02, _sum2); - _sum3 = _mm256_fmadd_ps(_k01, _r03, _sum3); + _sum0 = _mm256_comp_fmadd_ps(_k01, _r00, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_k01, _r01, _sum1); + _sum2 = _mm256_comp_fmadd_ps(_k01, _r02, _sum2); + _sum3 = _mm256_comp_fmadd_ps(_k01, _r03, _sum3); _r04 = _mm256_broadcast_ss(r0 + 33); _r05 = _mm256_broadcast_ss(r0 + 41); _r06 = _mm256_broadcast_ss(r0 + 49); _r07 = _mm256_broadcast_ss(r0 + 57); - _sum4 = _mm256_fmadd_ps(_k01, _r04, _sum4); - _sum5 = _mm256_fmadd_ps(_k01, _r05, _sum5); - _sum6 = _mm256_fmadd_ps(_k01, _r06, _sum6); - _sum7 = _mm256_fmadd_ps(_k01, _r07, _sum7); + _sum4 = _mm256_comp_fmadd_ps(_k01, _r04, _sum4); + _sum5 = _mm256_comp_fmadd_ps(_k01, _r05, _sum5); + _sum6 = _mm256_comp_fmadd_ps(_k01, _r06, _sum6); + _sum7 = _mm256_comp_fmadd_ps(_k01, _r07, _sum7); _k01 = _mm256_loadu_ps(k01 + 16); _r00 = _mm256_broadcast_ss(r0 + 2); _r01 = _mm256_broadcast_ss(r0 + 10); _r02 = _mm256_broadcast_ss(r0 + 18); _r03 = _mm256_broadcast_ss(r0 + 26); - _sum0 = _mm256_fmadd_ps(_k01, _r00, _sum0); - _sum1 = _mm256_fmadd_ps(_k01, _r01, _sum1); - _sum2 = _mm256_fmadd_ps(_k01, _r02, _sum2); - _sum3 = _mm256_fmadd_ps(_k01, _r03, _sum3); + _sum0 = _mm256_comp_fmadd_ps(_k01, _r00, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_k01, _r01, _sum1); + _sum2 = _mm256_comp_fmadd_ps(_k01, _r02, _sum2); + _sum3 = _mm256_comp_fmadd_ps(_k01, _r03, _sum3); _r04 = _mm256_broadcast_ss(r0 + 34); _r05 = _mm256_broadcast_ss(r0 + 42); _r06 = _mm256_broadcast_ss(r0 + 50); _r07 = _mm256_broadcast_ss(r0 + 58); - _sum4 = _mm256_fmadd_ps(_k01, _r04, _sum4); - _sum5 = _mm256_fmadd_ps(_k01, _r05, _sum5); - _sum6 = _mm256_fmadd_ps(_k01, _r06, _sum6); - _sum7 = _mm256_fmadd_ps(_k01, _r07, _sum7); + _sum4 = _mm256_comp_fmadd_ps(_k01, _r04, _sum4); + _sum5 = _mm256_comp_fmadd_ps(_k01, _r05, _sum5); + _sum6 = _mm256_comp_fmadd_ps(_k01, _r06, _sum6); + _sum7 = _mm256_comp_fmadd_ps(_k01, _r07, _sum7); _k01 = _mm256_loadu_ps(k01 + 24); _r00 = _mm256_broadcast_ss(r0 + 3); _r01 = _mm256_broadcast_ss(r0 + 11); _r02 = _mm256_broadcast_ss(r0 + 19); _r03 = _mm256_broadcast_ss(r0 + 27); - _sum0 = _mm256_fmadd_ps(_k01, _r00, _sum0); - _sum1 = _mm256_fmadd_ps(_k01, _r01, _sum1); - _sum2 = _mm256_fmadd_ps(_k01, _r02, _sum2); - _sum3 = _mm256_fmadd_ps(_k01, _r03, _sum3); + _sum0 = _mm256_comp_fmadd_ps(_k01, _r00, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_k01, _r01, _sum1); + _sum2 = _mm256_comp_fmadd_ps(_k01, _r02, _sum2); + _sum3 = _mm256_comp_fmadd_ps(_k01, _r03, _sum3); _r04 = _mm256_broadcast_ss(r0 + 35); _r05 = _mm256_broadcast_ss(r0 + 43); _r06 = _mm256_broadcast_ss(r0 + 51); _r07 = _mm256_broadcast_ss(r0 + 59); - _sum4 = _mm256_fmadd_ps(_k01, _r04, _sum4); - _sum5 = _mm256_fmadd_ps(_k01, _r05, _sum5); - _sum6 = _mm256_fmadd_ps(_k01, _r06, _sum6); - _sum7 = _mm256_fmadd_ps(_k01, _r07, _sum7); + _sum4 = _mm256_comp_fmadd_ps(_k01, _r04, _sum4); + _sum5 = _mm256_comp_fmadd_ps(_k01, _r05, _sum5); + _sum6 = _mm256_comp_fmadd_ps(_k01, _r06, _sum6); + _sum7 = _mm256_comp_fmadd_ps(_k01, _r07, _sum7); _k01 = _mm256_loadu_ps(k01 + 32); _r00 = _mm256_broadcast_ss(r0 + 4); _r01 = _mm256_broadcast_ss(r0 + 12); _r02 = _mm256_broadcast_ss(r0 + 20); _r03 = _mm256_broadcast_ss(r0 + 28); - _sum0 = _mm256_fmadd_ps(_k01, _r00, _sum0); - _sum1 = _mm256_fmadd_ps(_k01, _r01, _sum1); - _sum2 = _mm256_fmadd_ps(_k01, _r02, _sum2); - _sum3 = _mm256_fmadd_ps(_k01, _r03, _sum3); + _sum0 = _mm256_comp_fmadd_ps(_k01, _r00, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_k01, _r01, _sum1); + _sum2 = _mm256_comp_fmadd_ps(_k01, _r02, _sum2); + _sum3 = _mm256_comp_fmadd_ps(_k01, _r03, _sum3); _r04 = _mm256_broadcast_ss(r0 + 36); _r05 = _mm256_broadcast_ss(r0 + 44); _r06 = _mm256_broadcast_ss(r0 + 52); _r07 = _mm256_broadcast_ss(r0 + 60); - _sum4 = _mm256_fmadd_ps(_k01, _r04, _sum4); - _sum5 = _mm256_fmadd_ps(_k01, _r05, _sum5); - _sum6 = _mm256_fmadd_ps(_k01, _r06, _sum6); - _sum7 = _mm256_fmadd_ps(_k01, _r07, _sum7); + _sum4 = _mm256_comp_fmadd_ps(_k01, _r04, _sum4); + _sum5 = _mm256_comp_fmadd_ps(_k01, _r05, _sum5); + _sum6 = _mm256_comp_fmadd_ps(_k01, _r06, _sum6); + _sum7 = _mm256_comp_fmadd_ps(_k01, _r07, _sum7); _k01 = _mm256_loadu_ps(k01 + 40); _r00 = _mm256_broadcast_ss(r0 + 5); _r01 = _mm256_broadcast_ss(r0 + 13); _r02 = _mm256_broadcast_ss(r0 + 21); _r03 = _mm256_broadcast_ss(r0 + 29); - _sum0 = _mm256_fmadd_ps(_k01, _r00, _sum0); - _sum1 = _mm256_fmadd_ps(_k01, _r01, _sum1); - _sum2 = _mm256_fmadd_ps(_k01, _r02, _sum2); - _sum3 = _mm256_fmadd_ps(_k01, _r03, _sum3); + _sum0 = _mm256_comp_fmadd_ps(_k01, _r00, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_k01, _r01, _sum1); + _sum2 = _mm256_comp_fmadd_ps(_k01, _r02, _sum2); + _sum3 = _mm256_comp_fmadd_ps(_k01, _r03, _sum3); _r04 = _mm256_broadcast_ss(r0 + 37); _r05 = _mm256_broadcast_ss(r0 + 45); _r06 = _mm256_broadcast_ss(r0 + 53); _r07 = _mm256_broadcast_ss(r0 + 61); - _sum4 = _mm256_fmadd_ps(_k01, _r04, _sum4); - _sum5 = _mm256_fmadd_ps(_k01, _r05, _sum5); - _sum6 = _mm256_fmadd_ps(_k01, _r06, _sum6); - _sum7 = _mm256_fmadd_ps(_k01, _r07, _sum7); + _sum4 = _mm256_comp_fmadd_ps(_k01, _r04, _sum4); + _sum5 = _mm256_comp_fmadd_ps(_k01, _r05, _sum5); + _sum6 = _mm256_comp_fmadd_ps(_k01, _r06, _sum6); + _sum7 = _mm256_comp_fmadd_ps(_k01, _r07, _sum7); _k01 = _mm256_loadu_ps(k01 + 48); _r00 = _mm256_broadcast_ss(r0 + 6); _r01 = _mm256_broadcast_ss(r0 + 14); _r02 = _mm256_broadcast_ss(r0 + 22); _r03 = _mm256_broadcast_ss(r0 + 30); - _sum0 = _mm256_fmadd_ps(_k01, _r00, _sum0); - _sum1 = _mm256_fmadd_ps(_k01, _r01, _sum1); - _sum2 = _mm256_fmadd_ps(_k01, _r02, _sum2); - _sum3 = _mm256_fmadd_ps(_k01, _r03, _sum3); + _sum0 = _mm256_comp_fmadd_ps(_k01, _r00, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_k01, _r01, _sum1); + _sum2 = _mm256_comp_fmadd_ps(_k01, _r02, _sum2); + _sum3 = _mm256_comp_fmadd_ps(_k01, _r03, _sum3); _r04 = _mm256_broadcast_ss(r0 + 38); _r05 = _mm256_broadcast_ss(r0 + 46); _r06 = _mm256_broadcast_ss(r0 + 54); _r07 = _mm256_broadcast_ss(r0 + 62); - _sum4 = _mm256_fmadd_ps(_k01, _r04, _sum4); - _sum5 = _mm256_fmadd_ps(_k01, _r05, _sum5); - _sum6 = _mm256_fmadd_ps(_k01, _r06, _sum6); - _sum7 = _mm256_fmadd_ps(_k01, _r07, _sum7); + _sum4 = _mm256_comp_fmadd_ps(_k01, _r04, _sum4); + _sum5 = _mm256_comp_fmadd_ps(_k01, _r05, _sum5); + _sum6 = _mm256_comp_fmadd_ps(_k01, _r06, _sum6); + _sum7 = _mm256_comp_fmadd_ps(_k01, _r07, _sum7); _k01 = _mm256_loadu_ps(k01 + 56); _r00 = _mm256_broadcast_ss(r0 + 7); _r01 = _mm256_broadcast_ss(r0 + 15); _r02 = _mm256_broadcast_ss(r0 + 23); _r03 = _mm256_broadcast_ss(r0 + 31); - _sum0 = _mm256_fmadd_ps(_k01, _r00, _sum0); - _sum1 = _mm256_fmadd_ps(_k01, _r01, _sum1); - _sum2 = _mm256_fmadd_ps(_k01, _r02, _sum2); - _sum3 = _mm256_fmadd_ps(_k01, _r03, _sum3); + _sum0 = _mm256_comp_fmadd_ps(_k01, _r00, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_k01, _r01, _sum1); + _sum2 = _mm256_comp_fmadd_ps(_k01, _r02, _sum2); + _sum3 = _mm256_comp_fmadd_ps(_k01, _r03, _sum3); _r04 = _mm256_broadcast_ss(r0 + 39); _r05 = _mm256_broadcast_ss(r0 + 47); _r06 = _mm256_broadcast_ss(r0 + 55); _r07 = _mm256_broadcast_ss(r0 + 63); - _sum4 = _mm256_fmadd_ps(_k01, _r04, _sum4); - _sum5 = _mm256_fmadd_ps(_k01, _r05, _sum5); - _sum6 = _mm256_fmadd_ps(_k01, _r06, _sum6); - _sum7 = _mm256_fmadd_ps(_k01, _r07, _sum7); + _sum4 = _mm256_comp_fmadd_ps(_k01, _r04, _sum4); + _sum5 = _mm256_comp_fmadd_ps(_k01, _r05, _sum5); + _sum6 = _mm256_comp_fmadd_ps(_k01, _r06, _sum6); + _sum7 = _mm256_comp_fmadd_ps(_k01, _r07, _sum7); k01 += 64; r0 += 64; @@ -1804,80 +1804,80 @@ static void conv3x3s1_winograd64_pack8_avx(const Mat& bottom_blob, Mat& top_blob __m256 _r01 = _mm256_broadcast_ss(r0 + 8); __m256 _r02 = _mm256_broadcast_ss(r0 + 16); __m256 _r03 = _mm256_broadcast_ss(r0 + 24); - _sum0 = _mm256_fmadd_ps(_k01, _r00, _sum0); - _sum1 = _mm256_fmadd_ps(_k01, _r01, _sum1); - _sum2 = _mm256_fmadd_ps(_k01, _r02, _sum2); - _sum3 = _mm256_fmadd_ps(_k01, _r03, _sum3); + _sum0 = _mm256_comp_fmadd_ps(_k01, _r00, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_k01, _r01, _sum1); + _sum2 = _mm256_comp_fmadd_ps(_k01, _r02, _sum2); + _sum3 = _mm256_comp_fmadd_ps(_k01, _r03, _sum3); _k01 = _mm256_loadu_ps(k01 + 8); _r00 = _mm256_broadcast_ss(r0 + 1); _r01 = _mm256_broadcast_ss(r0 + 9); _r02 = _mm256_broadcast_ss(r0 + 17); _r03 = _mm256_broadcast_ss(r0 + 25); - _sum0 = _mm256_fmadd_ps(_k01, _r00, _sum0); - _sum1 = _mm256_fmadd_ps(_k01, _r01, _sum1); - _sum2 = _mm256_fmadd_ps(_k01, _r02, _sum2); - _sum3 = _mm256_fmadd_ps(_k01, _r03, _sum3); + _sum0 = _mm256_comp_fmadd_ps(_k01, _r00, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_k01, _r01, _sum1); + _sum2 = _mm256_comp_fmadd_ps(_k01, _r02, _sum2); + _sum3 = _mm256_comp_fmadd_ps(_k01, _r03, _sum3); _k01 = _mm256_loadu_ps(k01 + 16); _r00 = _mm256_broadcast_ss(r0 + 2); _r01 = _mm256_broadcast_ss(r0 + 10); _r02 = _mm256_broadcast_ss(r0 + 18); _r03 = _mm256_broadcast_ss(r0 + 26); - _sum0 = _mm256_fmadd_ps(_k01, _r00, _sum0); - _sum1 = _mm256_fmadd_ps(_k01, _r01, _sum1); - _sum2 = _mm256_fmadd_ps(_k01, _r02, _sum2); - _sum3 = _mm256_fmadd_ps(_k01, _r03, _sum3); + _sum0 = _mm256_comp_fmadd_ps(_k01, _r00, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_k01, _r01, _sum1); + _sum2 = _mm256_comp_fmadd_ps(_k01, _r02, _sum2); + _sum3 = _mm256_comp_fmadd_ps(_k01, _r03, _sum3); _k01 = _mm256_loadu_ps(k01 + 24); _r00 = _mm256_broadcast_ss(r0 + 3); _r01 = _mm256_broadcast_ss(r0 + 11); _r02 = _mm256_broadcast_ss(r0 + 19); _r03 = _mm256_broadcast_ss(r0 + 27); - _sum0 = _mm256_fmadd_ps(_k01, _r00, _sum0); - _sum1 = _mm256_fmadd_ps(_k01, _r01, _sum1); - _sum2 = _mm256_fmadd_ps(_k01, _r02, _sum2); - _sum3 = _mm256_fmadd_ps(_k01, _r03, _sum3); + _sum0 = _mm256_comp_fmadd_ps(_k01, _r00, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_k01, _r01, _sum1); + _sum2 = _mm256_comp_fmadd_ps(_k01, _r02, _sum2); + _sum3 = _mm256_comp_fmadd_ps(_k01, _r03, _sum3); _k01 = _mm256_loadu_ps(k01 + 32); _r00 = _mm256_broadcast_ss(r0 + 4); _r01 = _mm256_broadcast_ss(r0 + 12); _r02 = _mm256_broadcast_ss(r0 + 20); _r03 = _mm256_broadcast_ss(r0 + 28); - _sum0 = _mm256_fmadd_ps(_k01, _r00, _sum0); - _sum1 = _mm256_fmadd_ps(_k01, _r01, _sum1); - _sum2 = _mm256_fmadd_ps(_k01, _r02, _sum2); - _sum3 = _mm256_fmadd_ps(_k01, _r03, _sum3); + _sum0 = _mm256_comp_fmadd_ps(_k01, _r00, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_k01, _r01, _sum1); + _sum2 = _mm256_comp_fmadd_ps(_k01, _r02, _sum2); + _sum3 = _mm256_comp_fmadd_ps(_k01, _r03, _sum3); _k01 = _mm256_loadu_ps(k01 + 40); _r00 = _mm256_broadcast_ss(r0 + 5); _r01 = _mm256_broadcast_ss(r0 + 13); _r02 = _mm256_broadcast_ss(r0 + 21); _r03 = _mm256_broadcast_ss(r0 + 29); - _sum0 = _mm256_fmadd_ps(_k01, _r00, _sum0); - _sum1 = _mm256_fmadd_ps(_k01, _r01, _sum1); - _sum2 = _mm256_fmadd_ps(_k01, _r02, _sum2); - _sum3 = _mm256_fmadd_ps(_k01, _r03, _sum3); + _sum0 = _mm256_comp_fmadd_ps(_k01, _r00, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_k01, _r01, _sum1); + _sum2 = _mm256_comp_fmadd_ps(_k01, _r02, _sum2); + _sum3 = _mm256_comp_fmadd_ps(_k01, _r03, _sum3); _k01 = _mm256_loadu_ps(k01 + 48); _r00 = _mm256_broadcast_ss(r0 + 6); _r01 = _mm256_broadcast_ss(r0 + 14); _r02 = _mm256_broadcast_ss(r0 + 22); _r03 = _mm256_broadcast_ss(r0 + 30); - _sum0 = _mm256_fmadd_ps(_k01, _r00, _sum0); - _sum1 = _mm256_fmadd_ps(_k01, _r01, _sum1); - _sum2 = _mm256_fmadd_ps(_k01, _r02, _sum2); - _sum3 = _mm256_fmadd_ps(_k01, _r03, _sum3); + _sum0 = _mm256_comp_fmadd_ps(_k01, _r00, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_k01, _r01, _sum1); + _sum2 = _mm256_comp_fmadd_ps(_k01, _r02, _sum2); + _sum3 = _mm256_comp_fmadd_ps(_k01, _r03, _sum3); _k01 = _mm256_loadu_ps(k01 + 56); _r00 = _mm256_broadcast_ss(r0 + 7); _r01 = _mm256_broadcast_ss(r0 + 15); _r02 = _mm256_broadcast_ss(r0 + 23); _r03 = _mm256_broadcast_ss(r0 + 31); - _sum0 = _mm256_fmadd_ps(_k01, _r00, _sum0); - _sum1 = _mm256_fmadd_ps(_k01, _r01, _sum1); - _sum2 = _mm256_fmadd_ps(_k01, _r02, _sum2); - _sum3 = _mm256_fmadd_ps(_k01, _r03, _sum3); + _sum0 = _mm256_comp_fmadd_ps(_k01, _r00, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_k01, _r01, _sum1); + _sum2 = _mm256_comp_fmadd_ps(_k01, _r02, _sum2); + _sum3 = _mm256_comp_fmadd_ps(_k01, _r03, _sum3); k01 += 64; r0 += 32; } @@ -1902,50 +1902,50 @@ static void conv3x3s1_winograd64_pack8_avx(const Mat& bottom_blob, Mat& top_blob __m256 _k01 = _mm256_loadu_ps(k01); __m256 _r0 = _mm256_broadcast_ss(r0); __m256 _r01 = _mm256_broadcast_ss(r0 + 8); - _sum0 = _mm256_fmadd_ps(_k01, _r0, _sum0); - _sum1 = _mm256_fmadd_ps(_k01, _r01, _sum1); + _sum0 = _mm256_comp_fmadd_ps(_k01, _r0, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_k01, _r01, _sum1); _k01 = _mm256_loadu_ps(k01 + 8); _r0 = _mm256_broadcast_ss(r0 + 1); _r01 = _mm256_broadcast_ss(r0 + 9); - _sum0 = _mm256_fmadd_ps(_k01, _r0, _sum0); - _sum1 = _mm256_fmadd_ps(_k01, _r01, _sum1); + _sum0 = _mm256_comp_fmadd_ps(_k01, _r0, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_k01, _r01, _sum1); _k01 = _mm256_loadu_ps(k01 + 16); _r0 = _mm256_broadcast_ss(r0 + 2); _r01 = _mm256_broadcast_ss(r0 + 10); - _sum0 = _mm256_fmadd_ps(_k01, _r0, _sum0); - _sum1 = _mm256_fmadd_ps(_k01, _r01, _sum1); + _sum0 = _mm256_comp_fmadd_ps(_k01, _r0, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_k01, _r01, _sum1); _k01 = _mm256_loadu_ps(k01 + 24); _r0 = _mm256_broadcast_ss(r0 + 3); _r01 = _mm256_broadcast_ss(r0 + 11); - _sum0 = _mm256_fmadd_ps(_k01, _r0, _sum0); - _sum1 = _mm256_fmadd_ps(_k01, _r01, _sum1); + _sum0 = _mm256_comp_fmadd_ps(_k01, _r0, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_k01, _r01, _sum1); _k01 = _mm256_loadu_ps(k01 + 32); _r0 = _mm256_broadcast_ss(r0 + 4); _r01 = _mm256_broadcast_ss(r0 + 12); - _sum0 = _mm256_fmadd_ps(_k01, _r0, _sum0); - _sum1 = _mm256_fmadd_ps(_k01, _r01, _sum1); + _sum0 = _mm256_comp_fmadd_ps(_k01, _r0, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_k01, _r01, _sum1); _k01 = _mm256_loadu_ps(k01 + 40); _r0 = _mm256_broadcast_ss(r0 + 5); _r01 = _mm256_broadcast_ss(r0 + 13); - _sum0 = _mm256_fmadd_ps(_k01, _r0, _sum0); - _sum1 = _mm256_fmadd_ps(_k01, _r01, _sum1); + _sum0 = _mm256_comp_fmadd_ps(_k01, _r0, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_k01, _r01, _sum1); _k01 = _mm256_loadu_ps(k01 + 48); _r0 = _mm256_broadcast_ss(r0 + 6); _r01 = _mm256_broadcast_ss(r0 + 14); - _sum0 = _mm256_fmadd_ps(_k01, _r0, _sum0); - _sum1 = _mm256_fmadd_ps(_k01, _r01, _sum1); + _sum0 = _mm256_comp_fmadd_ps(_k01, _r0, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_k01, _r01, _sum1); _k01 = _mm256_loadu_ps(k01 + 56); _r0 = _mm256_broadcast_ss(r0 + 7); _r01 = _mm256_broadcast_ss(r0 + 15); - _sum0 = _mm256_fmadd_ps(_k01, _r0, _sum0); - _sum1 = _mm256_fmadd_ps(_k01, _r01, _sum1); + _sum0 = _mm256_comp_fmadd_ps(_k01, _r0, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_k01, _r01, _sum1); k01 += 64; r0 += 16; diff --git a/src/layer/x86/convolution_3x3_pack8to1.h b/src/layer/x86/convolution_3x3_pack8to1.h index 05d2050a2..f7b5301f8 100644 --- a/src/layer/x86/convolution_3x3_pack8to1.h +++ b/src/layer/x86/convolution_3x3_pack8to1.h @@ -71,17 +71,17 @@ static void conv3x3s1_pack8to1_avx(const Mat& bottom_blob, Mat& top_blob, const __m256 _r11 = _mm256_loadu_ps(r1 + 8); __m256 _r12 = _mm256_loadu_ps(r1 + 16); - _sum0 = _mm256_fmadd_ps(_k10, _r10, _sum0); - _sum1 = _mm256_fmadd_ps(_k11, _r11, _sum1); - _sum2 = _mm256_fmadd_ps(_k12, _r12, _sum2); + _sum0 = _mm256_comp_fmadd_ps(_k10, _r10, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_k11, _r11, _sum1); + _sum2 = _mm256_comp_fmadd_ps(_k12, _r12, _sum2); __m256 _r20 = _mm256_loadu_ps(r2); __m256 _r21 = _mm256_loadu_ps(r2 + 8); __m256 _r22 = _mm256_loadu_ps(r2 + 16); - _sum0 = _mm256_fmadd_ps(_k20, _r20, _sum0); - _sum1 = _mm256_fmadd_ps(_k21, _r21, _sum1); - _sum2 = _mm256_fmadd_ps(_k22, _r22, _sum2); + _sum0 = _mm256_comp_fmadd_ps(_k20, _r20, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_k21, _r21, _sum1); + _sum2 = _mm256_comp_fmadd_ps(_k22, _r22, _sum2); __m128 _sum = HorizontalSums(_sum0, _sum1, _sum2); *outptr0 += _mm_reduce_add_ps(_sum); // dot diff --git a/src/layer/x86/convolution_sgemm.h b/src/layer/x86/convolution_sgemm.h index cc02136a5..4bd801c2e 100644 --- a/src/layer/x86/convolution_sgemm.h +++ b/src/layer/x86/convolution_sgemm.h @@ -265,18 +265,18 @@ static void conv_im2col_sgemm_sse(const Mat& bottom_blob, Mat& top_blob, const M __m256 _vb1 = _mm256_loadu_ps(vb + 8); __m256 _vb2 = _mm256_loadu_ps(vb + 16); __m256 _vb3 = _mm256_loadu_ps(vb + 24); - _sum0 = _mm256_fmadd_ps(_vb0, _va0, _sum0); // sum0 = (a00-a07) * k00 - _sum1 = _mm256_fmadd_ps(_vb0, _va1, _sum1); // sum1 = (a00-a07) * k10 - _sum2 = _mm256_fmadd_ps(_vb0, _va2, _sum2); // sum2 = (a00-a07) * k20 - _sum3 = _mm256_fmadd_ps(_vb0, _va3, _sum3); // sum3 = (a00-a07) * k30 + _sum0 = _mm256_comp_fmadd_ps(_vb0, _va0, _sum0); // sum0 = (a00-a07) * k00 + _sum1 = _mm256_comp_fmadd_ps(_vb0, _va1, _sum1); // sum1 = (a00-a07) * k10 + _sum2 = _mm256_comp_fmadd_ps(_vb0, _va2, _sum2); // sum2 = (a00-a07) * k20 + _sum3 = _mm256_comp_fmadd_ps(_vb0, _va3, _sum3); // sum3 = (a00-a07) * k30 _va0 = _mm256_broadcast_ss(va + 4); _va1 = _mm256_broadcast_ss(va + 5); _va2 = _mm256_broadcast_ss(va + 6); _va3 = _mm256_broadcast_ss(va + 7); - _sum4 = _mm256_fmadd_ps(_vb0, _va0, _sum4); // sum4 = (a00-a07) * k40 - _sum5 = _mm256_fmadd_ps(_vb0, _va1, _sum5); // sum5 = (a00-a07) * k50 - _sum6 = _mm256_fmadd_ps(_vb0, _va2, _sum6); // sum6 = (a00-a07) * k60 - _sum7 = _mm256_fmadd_ps(_vb0, _va3, _sum7); // sum7 = (a00-a07) * k70 + _sum4 = _mm256_comp_fmadd_ps(_vb0, _va0, _sum4); // sum4 = (a00-a07) * k40 + _sum5 = _mm256_comp_fmadd_ps(_vb0, _va1, _sum5); // sum5 = (a00-a07) * k50 + _sum6 = _mm256_comp_fmadd_ps(_vb0, _va2, _sum6); // sum6 = (a00-a07) * k60 + _sum7 = _mm256_comp_fmadd_ps(_vb0, _va3, _sum7); // sum7 = (a00-a07) * k70 va += 8; @@ -285,18 +285,18 @@ static void conv_im2col_sgemm_sse(const Mat& bottom_blob, Mat& top_blob, const M _va1 = _mm256_broadcast_ss(va + 1); _va2 = _mm256_broadcast_ss(va + 2); _va3 = _mm256_broadcast_ss(va + 3); - _sum0 = _mm256_fmadd_ps(_vb1, _va0, _sum0); // sum0 += (a10-a17) * k01 - _sum1 = _mm256_fmadd_ps(_vb1, _va1, _sum1); // sum1 += (a10-a17) * k11 - _sum2 = _mm256_fmadd_ps(_vb1, _va2, _sum2); // sum2 += (a10-a17) * k21 - _sum3 = _mm256_fmadd_ps(_vb1, _va3, _sum3); // sum3 += (a10-a17) * k31 + _sum0 = _mm256_comp_fmadd_ps(_vb1, _va0, _sum0); // sum0 += (a10-a17) * k01 + _sum1 = _mm256_comp_fmadd_ps(_vb1, _va1, _sum1); // sum1 += (a10-a17) * k11 + _sum2 = _mm256_comp_fmadd_ps(_vb1, _va2, _sum2); // sum2 += (a10-a17) * k21 + _sum3 = _mm256_comp_fmadd_ps(_vb1, _va3, _sum3); // sum3 += (a10-a17) * k31 _va0 = _mm256_broadcast_ss(va + 4); _va1 = _mm256_broadcast_ss(va + 5); _va2 = _mm256_broadcast_ss(va + 6); _va3 = _mm256_broadcast_ss(va + 7); - _sum4 = _mm256_fmadd_ps(_vb1, _va0, _sum4); // sum4 += (a10-a17) * k41 - _sum5 = _mm256_fmadd_ps(_vb1, _va1, _sum5); // sum5 += (a10-a17) * k51 - _sum6 = _mm256_fmadd_ps(_vb1, _va2, _sum6); // sum6 += (a10-a17) * k61 - _sum7 = _mm256_fmadd_ps(_vb1, _va3, _sum7); // sum7 += (a10-a17) * k71 + _sum4 = _mm256_comp_fmadd_ps(_vb1, _va0, _sum4); // sum4 += (a10-a17) * k41 + _sum5 = _mm256_comp_fmadd_ps(_vb1, _va1, _sum5); // sum5 += (a10-a17) * k51 + _sum6 = _mm256_comp_fmadd_ps(_vb1, _va2, _sum6); // sum6 += (a10-a17) * k61 + _sum7 = _mm256_comp_fmadd_ps(_vb1, _va3, _sum7); // sum7 += (a10-a17) * k71 va += 8; @@ -305,18 +305,18 @@ static void conv_im2col_sgemm_sse(const Mat& bottom_blob, Mat& top_blob, const M _va1 = _mm256_broadcast_ss(va + 1); _va2 = _mm256_broadcast_ss(va + 2); _va3 = _mm256_broadcast_ss(va + 3); - _sum0 = _mm256_fmadd_ps(_vb2, _va0, _sum0); // sum0 += (a20-a27) * k02 - _sum1 = _mm256_fmadd_ps(_vb2, _va1, _sum1); // sum1 += (a20-a27) * k12 - _sum2 = _mm256_fmadd_ps(_vb2, _va2, _sum2); // sum2 += (a20-a27) * k22 - _sum3 = _mm256_fmadd_ps(_vb2, _va3, _sum3); // sum3 += (a20-a27) * k32 + _sum0 = _mm256_comp_fmadd_ps(_vb2, _va0, _sum0); // sum0 += (a20-a27) * k02 + _sum1 = _mm256_comp_fmadd_ps(_vb2, _va1, _sum1); // sum1 += (a20-a27) * k12 + _sum2 = _mm256_comp_fmadd_ps(_vb2, _va2, _sum2); // sum2 += (a20-a27) * k22 + _sum3 = _mm256_comp_fmadd_ps(_vb2, _va3, _sum3); // sum3 += (a20-a27) * k32 _va0 = _mm256_broadcast_ss(va + 4); _va1 = _mm256_broadcast_ss(va + 5); _va2 = _mm256_broadcast_ss(va + 6); _va3 = _mm256_broadcast_ss(va + 7); - _sum4 = _mm256_fmadd_ps(_vb2, _va0, _sum4); // sum4 += (a20-a27) * k42 - _sum5 = _mm256_fmadd_ps(_vb2, _va1, _sum5); // sum5 += (a20-a27) * k52 - _sum6 = _mm256_fmadd_ps(_vb2, _va2, _sum6); // sum6 += (a20-a27) * k62 - _sum7 = _mm256_fmadd_ps(_vb2, _va3, _sum7); // sum7 += (a20-a27) * k72 + _sum4 = _mm256_comp_fmadd_ps(_vb2, _va0, _sum4); // sum4 += (a20-a27) * k42 + _sum5 = _mm256_comp_fmadd_ps(_vb2, _va1, _sum5); // sum5 += (a20-a27) * k52 + _sum6 = _mm256_comp_fmadd_ps(_vb2, _va2, _sum6); // sum6 += (a20-a27) * k62 + _sum7 = _mm256_comp_fmadd_ps(_vb2, _va3, _sum7); // sum7 += (a20-a27) * k72 va += 8; @@ -325,18 +325,18 @@ static void conv_im2col_sgemm_sse(const Mat& bottom_blob, Mat& top_blob, const M _va1 = _mm256_broadcast_ss(va + 1); _va2 = _mm256_broadcast_ss(va + 2); _va3 = _mm256_broadcast_ss(va + 3); - _sum0 = _mm256_fmadd_ps(_vb3, _va0, _sum0); // sum0 += (a30-a37) * k03 - _sum1 = _mm256_fmadd_ps(_vb3, _va1, _sum1); // sum1 += (a30-a37) * k13 - _sum2 = _mm256_fmadd_ps(_vb3, _va2, _sum2); // sum2 += (a30-a37) * k23 - _sum3 = _mm256_fmadd_ps(_vb3, _va3, _sum3); // sum3 += (a30-a37) * k33 + _sum0 = _mm256_comp_fmadd_ps(_vb3, _va0, _sum0); // sum0 += (a30-a37) * k03 + _sum1 = _mm256_comp_fmadd_ps(_vb3, _va1, _sum1); // sum1 += (a30-a37) * k13 + _sum2 = _mm256_comp_fmadd_ps(_vb3, _va2, _sum2); // sum2 += (a30-a37) * k23 + _sum3 = _mm256_comp_fmadd_ps(_vb3, _va3, _sum3); // sum3 += (a30-a37) * k33 _va0 = _mm256_broadcast_ss(va + 4); _va1 = _mm256_broadcast_ss(va + 5); _va2 = _mm256_broadcast_ss(va + 6); _va3 = _mm256_broadcast_ss(va + 7); - _sum4 = _mm256_fmadd_ps(_vb3, _va0, _sum4); // sum4 += (a30-a37) * k43 - _sum5 = _mm256_fmadd_ps(_vb3, _va1, _sum5); // sum5 += (a30-a37) * k53 - _sum6 = _mm256_fmadd_ps(_vb3, _va2, _sum6); // sum6 += (a30-a37) * k63 - _sum7 = _mm256_fmadd_ps(_vb3, _va3, _sum7); // sum7 += (a30-a37) * k73 + _sum4 = _mm256_comp_fmadd_ps(_vb3, _va0, _sum4); // sum4 += (a30-a37) * k43 + _sum5 = _mm256_comp_fmadd_ps(_vb3, _va1, _sum5); // sum5 += (a30-a37) * k53 + _sum6 = _mm256_comp_fmadd_ps(_vb3, _va2, _sum6); // sum6 += (a30-a37) * k63 + _sum7 = _mm256_comp_fmadd_ps(_vb3, _va3, _sum7); // sum7 += (a30-a37) * k73 va += 8; vb += 32; @@ -354,14 +354,14 @@ static void conv_im2col_sgemm_sse(const Mat& bottom_blob, Mat& top_blob, const M __m256 _va6 = _mm256_broadcast_ss(va + 6); __m256 _va7 = _mm256_broadcast_ss(va + 7); __m256 _vb0 = _mm256_loadu_ps(vb); - _sum0 = _mm256_fmadd_ps(_vb0, _va0, _sum0); // sum0 = (a00-a07) * k00 - _sum1 = _mm256_fmadd_ps(_vb0, _va1, _sum1); // sum1 = (a00-a07) * k10 - _sum2 = _mm256_fmadd_ps(_vb0, _va2, _sum2); // sum2 = (a00-a07) * k20 - _sum3 = _mm256_fmadd_ps(_vb0, _va3, _sum3); // sum3 = (a00-a07) * k30 - _sum4 = _mm256_fmadd_ps(_vb0, _va4, _sum4); // sum4 = (a00-a07) * k40 - _sum5 = _mm256_fmadd_ps(_vb0, _va5, _sum5); // sum5 = (a00-a07) * k50 - _sum6 = _mm256_fmadd_ps(_vb0, _va6, _sum6); // sum6 = (a00-a07) * k60 - _sum7 = _mm256_fmadd_ps(_vb0, _va7, _sum7); // sum7 = (a00-a07) * k70 + _sum0 = _mm256_comp_fmadd_ps(_vb0, _va0, _sum0); // sum0 = (a00-a07) * k00 + _sum1 = _mm256_comp_fmadd_ps(_vb0, _va1, _sum1); // sum1 = (a00-a07) * k10 + _sum2 = _mm256_comp_fmadd_ps(_vb0, _va2, _sum2); // sum2 = (a00-a07) * k20 + _sum3 = _mm256_comp_fmadd_ps(_vb0, _va3, _sum3); // sum3 = (a00-a07) * k30 + _sum4 = _mm256_comp_fmadd_ps(_vb0, _va4, _sum4); // sum4 = (a00-a07) * k40 + _sum5 = _mm256_comp_fmadd_ps(_vb0, _va5, _sum5); // sum5 = (a00-a07) * k50 + _sum6 = _mm256_comp_fmadd_ps(_vb0, _va6, _sum6); // sum6 = (a00-a07) * k60 + _sum7 = _mm256_comp_fmadd_ps(_vb0, _va7, _sum7); // sum7 = (a00-a07) * k70 va += 8; vb += 8; @@ -539,10 +539,10 @@ static void conv_im2col_sgemm_sse(const Mat& bottom_blob, Mat& top_blob, const M __m256 _va2 = _mm256_loadu_ps(va + 16); __m256 _va3 = _mm256_loadu_ps(va + 24); - _sum0 = _mm256_fmadd_ps(_va0, _vb0, _sum0); // sum0 += (k00-k70) * a00 - _sum1 = _mm256_fmadd_ps(_va1, _vb1, _sum1); // sum1 += (k01-k71) * a10 - _sum2 = _mm256_fmadd_ps(_va2, _vb2, _sum2); // sum2 += (k02-k72) * a20 - _sum3 = _mm256_fmadd_ps(_va3, _vb3, _sum3); // sum3 += (k03-k73) * a30 + _sum0 = _mm256_comp_fmadd_ps(_va0, _vb0, _sum0); // sum0 += (k00-k70) * a00 + _sum1 = _mm256_comp_fmadd_ps(_va1, _vb1, _sum1); // sum1 += (k01-k71) * a10 + _sum2 = _mm256_comp_fmadd_ps(_va2, _vb2, _sum2); // sum2 += (k02-k72) * a20 + _sum3 = _mm256_comp_fmadd_ps(_va3, _vb3, _sum3); // sum3 += (k03-k73) * a30 va += 32; vb += 4; @@ -558,7 +558,7 @@ static void conv_im2col_sgemm_sse(const Mat& bottom_blob, Mat& top_blob, const M __m256 _vb0 = _mm256_broadcast_ss(vb); __m256 _va = _mm256_loadu_ps(va); - _sum0_7 = _mm256_fmadd_ps(_va, _vb0, _sum0_7); // sum0 += (k00-k70) * a00 + _sum0_7 = _mm256_comp_fmadd_ps(_va, _vb0, _sum0_7); // sum0 += (k00-k70) * a00 va += 8; vb += 1; @@ -658,10 +658,10 @@ static void conv_im2col_sgemm_sse(const Mat& bottom_blob, Mat& top_blob, const M __m256 _vb1 = _mm256_loadu_ps(vb + 8); __m256 _vb2 = _mm256_loadu_ps(vb + 16); __m256 _vb3 = _mm256_loadu_ps(vb + 24); - _sum0 = _mm256_fmadd_ps(_vb0, _va0, _sum0); // sum0 = (a00-a07) * k00 - _sum1 = _mm256_fmadd_ps(_vb0, _va1, _sum1); // sum1 = (a00-a07) * k10 - _sum2 = _mm256_fmadd_ps(_vb0, _va2, _sum2); // sum2 = (a00-a07) * k20 - _sum3 = _mm256_fmadd_ps(_vb0, _va3, _sum3); // sum3 = (a00-a07) * k30 + _sum0 = _mm256_comp_fmadd_ps(_vb0, _va0, _sum0); // sum0 = (a00-a07) * k00 + _sum1 = _mm256_comp_fmadd_ps(_vb0, _va1, _sum1); // sum1 = (a00-a07) * k10 + _sum2 = _mm256_comp_fmadd_ps(_vb0, _va2, _sum2); // sum2 = (a00-a07) * k20 + _sum3 = _mm256_comp_fmadd_ps(_vb0, _va3, _sum3); // sum3 = (a00-a07) * k30 va += 4; @@ -670,10 +670,10 @@ static void conv_im2col_sgemm_sse(const Mat& bottom_blob, Mat& top_blob, const M _va1 = _mm256_broadcast_ss(va + 1); _va2 = _mm256_broadcast_ss(va + 2); _va3 = _mm256_broadcast_ss(va + 3); - _sum0 = _mm256_fmadd_ps(_vb1, _va0, _sum0); // sum0 += (a10-a17) * k01 - _sum1 = _mm256_fmadd_ps(_vb1, _va1, _sum1); // sum1 += (a10-a17) * k11 - _sum2 = _mm256_fmadd_ps(_vb1, _va2, _sum2); // sum2 += (a10-a17) * k21 - _sum3 = _mm256_fmadd_ps(_vb1, _va3, _sum3); // sum3 += (a10-a17) * k31 + _sum0 = _mm256_comp_fmadd_ps(_vb1, _va0, _sum0); // sum0 += (a10-a17) * k01 + _sum1 = _mm256_comp_fmadd_ps(_vb1, _va1, _sum1); // sum1 += (a10-a17) * k11 + _sum2 = _mm256_comp_fmadd_ps(_vb1, _va2, _sum2); // sum2 += (a10-a17) * k21 + _sum3 = _mm256_comp_fmadd_ps(_vb1, _va3, _sum3); // sum3 += (a10-a17) * k31 va += 4; @@ -682,10 +682,10 @@ static void conv_im2col_sgemm_sse(const Mat& bottom_blob, Mat& top_blob, const M _va1 = _mm256_broadcast_ss(va + 1); _va2 = _mm256_broadcast_ss(va + 2); _va3 = _mm256_broadcast_ss(va + 3); - _sum0 = _mm256_fmadd_ps(_vb2, _va0, _sum0); // sum0 += (a20-a27) * k02 - _sum1 = _mm256_fmadd_ps(_vb2, _va1, _sum1); // sum1 += (a20-a27) * k12 - _sum2 = _mm256_fmadd_ps(_vb2, _va2, _sum2); // sum2 += (a20-a27) * k22 - _sum3 = _mm256_fmadd_ps(_vb2, _va3, _sum3); // sum3 += (a20-a27) * k32 + _sum0 = _mm256_comp_fmadd_ps(_vb2, _va0, _sum0); // sum0 += (a20-a27) * k02 + _sum1 = _mm256_comp_fmadd_ps(_vb2, _va1, _sum1); // sum1 += (a20-a27) * k12 + _sum2 = _mm256_comp_fmadd_ps(_vb2, _va2, _sum2); // sum2 += (a20-a27) * k22 + _sum3 = _mm256_comp_fmadd_ps(_vb2, _va3, _sum3); // sum3 += (a20-a27) * k32 va += 4; @@ -694,10 +694,10 @@ static void conv_im2col_sgemm_sse(const Mat& bottom_blob, Mat& top_blob, const M _va1 = _mm256_broadcast_ss(va + 1); _va2 = _mm256_broadcast_ss(va + 2); _va3 = _mm256_broadcast_ss(va + 3); - _sum0 = _mm256_fmadd_ps(_vb3, _va0, _sum0); // sum0 += (a30-a37) * k03 - _sum1 = _mm256_fmadd_ps(_vb3, _va1, _sum1); // sum1 += (a30-a37) * k13 - _sum2 = _mm256_fmadd_ps(_vb3, _va2, _sum2); // sum2 += (a30-a37) * k23 - _sum3 = _mm256_fmadd_ps(_vb3, _va3, _sum3); // sum3 += (a30-a37) * k33 + _sum0 = _mm256_comp_fmadd_ps(_vb3, _va0, _sum0); // sum0 += (a30-a37) * k03 + _sum1 = _mm256_comp_fmadd_ps(_vb3, _va1, _sum1); // sum1 += (a30-a37) * k13 + _sum2 = _mm256_comp_fmadd_ps(_vb3, _va2, _sum2); // sum2 += (a30-a37) * k23 + _sum3 = _mm256_comp_fmadd_ps(_vb3, _va3, _sum3); // sum3 += (a30-a37) * k33 va += 4; vb += 32; @@ -711,10 +711,10 @@ static void conv_im2col_sgemm_sse(const Mat& bottom_blob, Mat& top_blob, const M __m256 _va2 = _mm256_broadcast_ss(va + 2); __m256 _va3 = _mm256_broadcast_ss(va + 3); __m256 _vb0 = _mm256_loadu_ps(vb); - _sum0 = _mm256_fmadd_ps(_vb0, _va0, _sum0); // sum0 = (a00-a07) * k00 - _sum1 = _mm256_fmadd_ps(_vb0, _va1, _sum1); // sum1 = (a00-a07) * k10 - _sum2 = _mm256_fmadd_ps(_vb0, _va2, _sum2); // sum2 = (a00-a07) * k20 - _sum3 = _mm256_fmadd_ps(_vb0, _va3, _sum3); // sum3 = (a00-a07) * k30 + _sum0 = _mm256_comp_fmadd_ps(_vb0, _va0, _sum0); // sum0 = (a00-a07) * k00 + _sum1 = _mm256_comp_fmadd_ps(_vb0, _va1, _sum1); // sum1 = (a00-a07) * k10 + _sum2 = _mm256_comp_fmadd_ps(_vb0, _va2, _sum2); // sum2 = (a00-a07) * k20 + _sum3 = _mm256_comp_fmadd_ps(_vb0, _va3, _sum3); // sum3 = (a00-a07) * k30 va += 4; vb += 8; @@ -839,10 +839,10 @@ static void conv_im2col_sgemm_sse(const Mat& bottom_blob, Mat& top_blob, const M __m128 _va2 = _mm_loadu_ps(va + 8); __m128 _va3 = _mm_loadu_ps(va + 12); - _sum0 = _mm_fmadd_ps(_va0, _vb0, _sum0); // sum0 += (k00-k30) * a00 - _sum1 = _mm_fmadd_ps(_va1, _vb1, _sum1); // sum1 += (k01-k31) * a10 - _sum2 = _mm_fmadd_ps(_va2, _vb2, _sum2); // sum2 += (k02-k32) * a20 - _sum3 = _mm_fmadd_ps(_va3, _vb3, _sum3); // sum3 += (k03-k33) * a30 + _sum0 = _mm_comp_fmadd_ps(_va0, _vb0, _sum0); // sum0 += (k00-k30) * a00 + _sum1 = _mm_comp_fmadd_ps(_va1, _vb1, _sum1); // sum1 += (k01-k31) * a10 + _sum2 = _mm_comp_fmadd_ps(_va2, _vb2, _sum2); // sum2 += (k02-k32) * a20 + _sum3 = _mm_comp_fmadd_ps(_va3, _vb3, _sum3); // sum3 += (k03-k33) * a30 va += 16; vb += 4; @@ -858,7 +858,7 @@ static void conv_im2col_sgemm_sse(const Mat& bottom_blob, Mat& top_blob, const M __m128 _vb0 = _mm_set1_ps(vb[0]); __m128 _va = _mm_loadu_ps(va); - _sum0_3 = _mm_fmadd_ps(_va, _vb0, _sum0_3); // sum0 += (k00-k30) * a00 + _sum0_3 = _mm_comp_fmadd_ps(_va, _vb0, _sum0_3); // sum0 += (k00-k30) * a00 va += 4; vb += 1; @@ -929,10 +929,10 @@ static void conv_im2col_sgemm_sse(const Mat& bottom_blob, Mat& top_blob, const M __m256 _vb2 = _mm256_loadu_ps(vb + 16); __m256 _vb3 = _mm256_loadu_ps(vb + 24); - _sum0 = _mm256_fmadd_ps(_vb0, _va0, _sum0); // sum0 = (a00-a07) * k00 - _sum0 = _mm256_fmadd_ps(_vb1, _va1, _sum0); // sum0 += (a10-a17) * k01 - _sum0 = _mm256_fmadd_ps(_vb2, _va2, _sum0); // sum0 += (a20-a27) * k02 - _sum0 = _mm256_fmadd_ps(_vb3, _va3, _sum0); // sum0 += (a30-a37) * k03 + _sum0 = _mm256_comp_fmadd_ps(_vb0, _va0, _sum0); // sum0 = (a00-a07) * k00 + _sum0 = _mm256_comp_fmadd_ps(_vb1, _va1, _sum0); // sum0 += (a10-a17) * k01 + _sum0 = _mm256_comp_fmadd_ps(_vb2, _va2, _sum0); // sum0 += (a20-a27) * k02 + _sum0 = _mm256_comp_fmadd_ps(_vb3, _va3, _sum0); // sum0 += (a30-a37) * k03 va += 4; vb += 32; @@ -944,7 +944,7 @@ static void conv_im2col_sgemm_sse(const Mat& bottom_blob, Mat& top_blob, const M __m256 _va0 = _mm256_broadcast_ss(va); __m256 _vb0 = _mm256_loadu_ps(vb); - _sum0 = _mm256_fmadd_ps(_vb0, _va0, _sum0); // sum0 = (a00-a07) * k00 + _sum0 = _mm256_comp_fmadd_ps(_vb0, _va0, _sum0); // sum0 = (a00-a07) * k00 va += 1; vb += 8; @@ -1009,7 +1009,7 @@ static void conv_im2col_sgemm_sse(const Mat& bottom_blob, Mat& top_blob, const M __m128 _k0 = _mm_loadu_ps(va); va += 4; - _sum0 = _mm_fmadd_ps(_p0, _k0, _sum0); + _sum0 = _mm_comp_fmadd_ps(_p0, _k0, _sum0); } float output_sum0[4] = {0.f}; diff --git a/src/layer/x86/convolution_x86.cpp b/src/layer/x86/convolution_x86.cpp index 7e4ddee79..196fbe7de 100644 --- a/src/layer/x86/convolution_x86.cpp +++ b/src/layer/x86/convolution_x86.cpp @@ -58,17 +58,19 @@ namespace ncnn { #include "convolution_3x3_pack8to1.h" #include "convolution_3x3_pack8.h" #include "convolution_2x2_pack8.h" -#include "convolution_2x2_pack8_fp16.h" #include "convolution_1x1_pack8.h" +#if __AVX2__ +#include "convolution_2x2_pack8_fp16.h" #include "convolution_1x1_pack8_fp16.h" #endif +#endif #endif // __SSE2__ Convolution_x86::Convolution_x86() { #if __SSE2__ support_packing = true; -#if __AVX__ +#if __AVX2__ support_weight_fp16_storage = true; #endif #endif // __SSE2__ @@ -264,6 +266,8 @@ int Convolution_x86::create_pipeline(const Option& opt) // pack8 if (elempack == 8 && out_elempack == 8) { +#if __AVX2__ + if (opt.use_weight_fp16_storage && kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { conv1x1s1_sgemm_transform_kernel_fp16_pack8_avx(weight_data, weight_data_packed, num_input, num_output); @@ -277,6 +281,9 @@ int Convolution_x86::create_pipeline(const Option& opt) conv2x2s1_weight_fp16_pack8_avx(weight_data, weight_data_packed, num_input, num_output); } else if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1 && num_input >= 16 && num_output >= 16) +#else + if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1 && num_input >= 16 && num_output >= 16) +#endif { conv3x3s1_winograd64_transform_kernel_pack8_avx(weight_data, weight_data_packed, num_input, num_output); } @@ -414,11 +421,13 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option { if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { +#if __AVX2__ if (opt.use_weight_fp16_storage) { conv1x1s1_sgemm_fp16_pack8_avx(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt); } - else +#endif + if (!opt.use_weight_fp16_storage) { conv1x1s1_sgemm_pack8_avx(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt); } @@ -430,11 +439,15 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option } else if (kernel_w == 1 && kernel_h == 1 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { +#if __AVX2__ + if (opt.use_weight_fp16_storage) { conv1x1s2_fp16_pack8_avx(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt); } - else +#endif + if (!opt.use_weight_fp16_storage) + { conv1x1s2_pack8_avx(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt); } @@ -461,11 +474,15 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option } else if (kernel_w == 2 && kernel_h == 2 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { +#if __AVX2__ + if (opt.use_weight_fp16_storage) { conv2x2s1_fp16_pack8_avx(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt); } - else +#endif + + if (!opt.use_weight_fp16_storage) { conv2x2s1_pack8_avx(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt); } @@ -604,7 +621,7 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option { __m256 _val = _mm256_set1_ps(sptr[space_ofs[k]]); __m256 _w = _mm256_loadu_ps(kptr); - _sum = _mm256_fmadd_ps(_val, _w, _sum); + _sum = _mm256_comp_fmadd_ps(_val, _w, _sum); kptr += 8; } @@ -657,13 +674,13 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option __m256 _val3 = _mm256_broadcast_ss((sptr + space_ofs[k] * 4) + 3); __m256 _w0 = _mm256_loadu_ps(kptr); - _sum = _mm256_fmadd_ps(_val0, _w0, _sum); + _sum = _mm256_comp_fmadd_ps(_val0, _w0, _sum); __m256 _w1 = _mm256_loadu_ps(kptr + 8); - _sum = _mm256_fmadd_ps(_val1, _w1, _sum); + _sum = _mm256_comp_fmadd_ps(_val1, _w1, _sum); __m256 _w2 = _mm256_loadu_ps(kptr + 16); - _sum = _mm256_fmadd_ps(_val2, _w2, _sum); + _sum = _mm256_comp_fmadd_ps(_val2, _w2, _sum); __m256 _w3 = _mm256_loadu_ps(kptr + 24); - _sum = _mm256_fmadd_ps(_val3, _w3, _sum); + _sum = _mm256_comp_fmadd_ps(_val3, _w3, _sum); kptr += 32; } @@ -780,21 +797,21 @@ int Convolution_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option __m128 _val7 = _mm_broadcast_ss((sptr + space_ofs[k] * 8) + 7); __m128 _w0 = _mm_loadu_ps(kptr); - _sum = _mm_fmadd_ps(_val0, _w0, _sum); + _sum = _mm_comp_fmadd_ps(_val0, _w0, _sum); __m128 _w1 = _mm_loadu_ps(kptr + 4); - _sum = _mm_fmadd_ps(_val1, _w1, _sum); + _sum = _mm_comp_fmadd_ps(_val1, _w1, _sum); __m128 _w2 = _mm_loadu_ps(kptr + 8); - _sum = _mm_fmadd_ps(_val2, _w2, _sum); + _sum = _mm_comp_fmadd_ps(_val2, _w2, _sum); __m128 _w3 = _mm_loadu_ps(kptr + 12); - _sum = _mm_fmadd_ps(_val3, _w3, _sum); + _sum = _mm_comp_fmadd_ps(_val3, _w3, _sum); __m128 _w4 = _mm_loadu_ps(kptr + 16); - _sum = _mm_fmadd_ps(_val4, _w4, _sum); + _sum = _mm_comp_fmadd_ps(_val4, _w4, _sum); __m128 _w5 = _mm_loadu_ps(kptr + 20); - _sum = _mm_fmadd_ps(_val5, _w5, _sum); + _sum = _mm_comp_fmadd_ps(_val5, _w5, _sum); __m128 _w6 = _mm_loadu_ps(kptr + 24); - _sum = _mm_fmadd_ps(_val6, _w6, _sum); + _sum = _mm_comp_fmadd_ps(_val6, _w6, _sum); __m128 _w7 = _mm_loadu_ps(kptr + 28); - _sum = _mm_fmadd_ps(_val7, _w7, _sum); + _sum = _mm_comp_fmadd_ps(_val7, _w7, _sum); kptr += 32; } diff --git a/src/layer/x86/convolutiondepthwise_3x3_pack8.h b/src/layer/x86/convolutiondepthwise_3x3_pack8.h index 72c1262db..5712bb44d 100644 --- a/src/layer/x86/convolutiondepthwise_3x3_pack8.h +++ b/src/layer/x86/convolutiondepthwise_3x3_pack8.h @@ -67,15 +67,15 @@ static void convdw3x3s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const M __m256 _r21 = _mm256_loadu_ps(r2 + 8); __m256 _r22 = _mm256_loadu_ps(r2 + 16); - _sum0 = _mm256_fmadd_ps(_k00, _r00, _sum0); - _sum0 = _mm256_fmadd_ps(_k01, _r01, _sum0); - _sum0 = _mm256_fmadd_ps(_k02, _r02, _sum0); - _sum0 = _mm256_fmadd_ps(_k10, _r10, _sum0); - _sum0 = _mm256_fmadd_ps(_k11, _r11, _sum0); - _sum0 = _mm256_fmadd_ps(_k12, _r12, _sum0); - _sum0 = _mm256_fmadd_ps(_k20, _r20, _sum0); - _sum0 = _mm256_fmadd_ps(_k21, _r21, _sum0); - _sum0 = _mm256_fmadd_ps(_k22, _r22, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k00, _r00, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k01, _r01, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k02, _r02, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k10, _r10, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k11, _r11, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k12, _r12, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k20, _r20, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k21, _r21, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k22, _r22, _sum0); __m256 _sum1 = _bias0; __m256 _r03 = _mm256_loadu_ps(r0 + 24); @@ -83,15 +83,15 @@ static void convdw3x3s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const M __m256 _r23 = _mm256_loadu_ps(r2 + 24); _mm256_storeu_ps(outptr0, _sum0); - _sum1 = _mm256_fmadd_ps(_k00, _r01, _sum1); - _sum1 = _mm256_fmadd_ps(_k01, _r02, _sum1); - _sum1 = _mm256_fmadd_ps(_k02, _r03, _sum1); - _sum1 = _mm256_fmadd_ps(_k10, _r11, _sum1); - _sum1 = _mm256_fmadd_ps(_k11, _r12, _sum1); - _sum1 = _mm256_fmadd_ps(_k12, _r13, _sum1); - _sum1 = _mm256_fmadd_ps(_k20, _r21, _sum1); - _sum1 = _mm256_fmadd_ps(_k21, _r22, _sum1); - _sum1 = _mm256_fmadd_ps(_k22, _r23, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k00, _r01, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k01, _r02, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k02, _r03, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k10, _r11, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k11, _r12, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k12, _r13, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k20, _r21, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k21, _r22, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k22, _r23, _sum1); __m256 _sum2 = _bias0; __m256 _r04 = _mm256_loadu_ps(r0 + 32); @@ -99,15 +99,15 @@ static void convdw3x3s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const M __m256 _r24 = _mm256_loadu_ps(r2 + 32); _mm256_storeu_ps(outptr0 + 8, _sum1); - _sum2 = _mm256_fmadd_ps(_k00, _r02, _sum2); - _sum2 = _mm256_fmadd_ps(_k01, _r03, _sum2); - _sum2 = _mm256_fmadd_ps(_k02, _r04, _sum2); - _sum2 = _mm256_fmadd_ps(_k10, _r12, _sum2); - _sum2 = _mm256_fmadd_ps(_k11, _r13, _sum2); - _sum2 = _mm256_fmadd_ps(_k12, _r14, _sum2); - _sum2 = _mm256_fmadd_ps(_k20, _r22, _sum2); - _sum2 = _mm256_fmadd_ps(_k21, _r23, _sum2); - _sum2 = _mm256_fmadd_ps(_k22, _r24, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_k00, _r02, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_k01, _r03, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_k02, _r04, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_k10, _r12, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_k11, _r13, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_k12, _r14, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_k20, _r22, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_k21, _r23, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_k22, _r24, _sum2); __m256 _sum3 = _bias0; __m256 _r05 = _mm256_loadu_ps(r0 + 40); @@ -115,15 +115,15 @@ static void convdw3x3s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const M __m256 _r25 = _mm256_loadu_ps(r2 + 40); _mm256_storeu_ps(outptr0 + 16, _sum2); - _sum3 = _mm256_fmadd_ps(_k00, _r03, _sum3); - _sum3 = _mm256_fmadd_ps(_k01, _r04, _sum3); - _sum3 = _mm256_fmadd_ps(_k02, _r05, _sum3); - _sum3 = _mm256_fmadd_ps(_k10, _r13, _sum3); - _sum3 = _mm256_fmadd_ps(_k11, _r14, _sum3); - _sum3 = _mm256_fmadd_ps(_k12, _r15, _sum3); - _sum3 = _mm256_fmadd_ps(_k20, _r23, _sum3); - _sum3 = _mm256_fmadd_ps(_k21, _r24, _sum3); - _sum3 = _mm256_fmadd_ps(_k22, _r25, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_k00, _r03, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_k01, _r04, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_k02, _r05, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_k10, _r13, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_k11, _r14, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_k12, _r15, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_k20, _r23, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_k21, _r24, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_k22, _r25, _sum3); __m256 _sum4 = _bias0; __m256 _r06 = _mm256_loadu_ps(r0 + 48); @@ -131,15 +131,15 @@ static void convdw3x3s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const M __m256 _r26 = _mm256_loadu_ps(r2 + 48); _mm256_storeu_ps(outptr0 + 24, _sum3); - _sum4 = _mm256_fmadd_ps(_k00, _r04, _sum4); - _sum4 = _mm256_fmadd_ps(_k01, _r05, _sum4); - _sum4 = _mm256_fmadd_ps(_k02, _r06, _sum4); - _sum4 = _mm256_fmadd_ps(_k10, _r14, _sum4); - _sum4 = _mm256_fmadd_ps(_k11, _r15, _sum4); - _sum4 = _mm256_fmadd_ps(_k12, _r16, _sum4); - _sum4 = _mm256_fmadd_ps(_k20, _r24, _sum4); - _sum4 = _mm256_fmadd_ps(_k21, _r25, _sum4); - _sum4 = _mm256_fmadd_ps(_k22, _r26, _sum4); + _sum4 = _mm256_comp_fmadd_ps(_k00, _r04, _sum4); + _sum4 = _mm256_comp_fmadd_ps(_k01, _r05, _sum4); + _sum4 = _mm256_comp_fmadd_ps(_k02, _r06, _sum4); + _sum4 = _mm256_comp_fmadd_ps(_k10, _r14, _sum4); + _sum4 = _mm256_comp_fmadd_ps(_k11, _r15, _sum4); + _sum4 = _mm256_comp_fmadd_ps(_k12, _r16, _sum4); + _sum4 = _mm256_comp_fmadd_ps(_k20, _r24, _sum4); + _sum4 = _mm256_comp_fmadd_ps(_k21, _r25, _sum4); + _sum4 = _mm256_comp_fmadd_ps(_k22, _r26, _sum4); __m256 _sum5 = _bias0; __m256 _r07 = _mm256_loadu_ps(r0 + 56); @@ -147,15 +147,15 @@ static void convdw3x3s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const M __m256 _r27 = _mm256_loadu_ps(r2 + 56); _mm256_storeu_ps(outptr0 + 32, _sum4); - _sum5 = _mm256_fmadd_ps(_k00, _r05, _sum5); - _sum5 = _mm256_fmadd_ps(_k01, _r06, _sum5); - _sum5 = _mm256_fmadd_ps(_k02, _r07, _sum5); - _sum5 = _mm256_fmadd_ps(_k10, _r15, _sum5); - _sum5 = _mm256_fmadd_ps(_k11, _r16, _sum5); - _sum5 = _mm256_fmadd_ps(_k12, _r17, _sum5); - _sum5 = _mm256_fmadd_ps(_k20, _r25, _sum5); - _sum5 = _mm256_fmadd_ps(_k21, _r26, _sum5); - _sum5 = _mm256_fmadd_ps(_k22, _r27, _sum5); + _sum5 = _mm256_comp_fmadd_ps(_k00, _r05, _sum5); + _sum5 = _mm256_comp_fmadd_ps(_k01, _r06, _sum5); + _sum5 = _mm256_comp_fmadd_ps(_k02, _r07, _sum5); + _sum5 = _mm256_comp_fmadd_ps(_k10, _r15, _sum5); + _sum5 = _mm256_comp_fmadd_ps(_k11, _r16, _sum5); + _sum5 = _mm256_comp_fmadd_ps(_k12, _r17, _sum5); + _sum5 = _mm256_comp_fmadd_ps(_k20, _r25, _sum5); + _sum5 = _mm256_comp_fmadd_ps(_k21, _r26, _sum5); + _sum5 = _mm256_comp_fmadd_ps(_k22, _r27, _sum5); __m256 _sum6 = _bias0; __m256 _r08 = _mm256_loadu_ps(r0 + 64); @@ -163,15 +163,15 @@ static void convdw3x3s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const M __m256 _r28 = _mm256_loadu_ps(r2 + 64); _mm256_storeu_ps(outptr0 + 40, _sum5); - _sum6 = _mm256_fmadd_ps(_k00, _r06, _sum6); - _sum6 = _mm256_fmadd_ps(_k01, _r07, _sum6); - _sum6 = _mm256_fmadd_ps(_k02, _r08, _sum6); - _sum6 = _mm256_fmadd_ps(_k10, _r16, _sum6); - _sum6 = _mm256_fmadd_ps(_k11, _r17, _sum6); - _sum6 = _mm256_fmadd_ps(_k12, _r18, _sum6); - _sum6 = _mm256_fmadd_ps(_k20, _r26, _sum6); - _sum6 = _mm256_fmadd_ps(_k21, _r27, _sum6); - _sum6 = _mm256_fmadd_ps(_k22, _r28, _sum6); + _sum6 = _mm256_comp_fmadd_ps(_k00, _r06, _sum6); + _sum6 = _mm256_comp_fmadd_ps(_k01, _r07, _sum6); + _sum6 = _mm256_comp_fmadd_ps(_k02, _r08, _sum6); + _sum6 = _mm256_comp_fmadd_ps(_k10, _r16, _sum6); + _sum6 = _mm256_comp_fmadd_ps(_k11, _r17, _sum6); + _sum6 = _mm256_comp_fmadd_ps(_k12, _r18, _sum6); + _sum6 = _mm256_comp_fmadd_ps(_k20, _r26, _sum6); + _sum6 = _mm256_comp_fmadd_ps(_k21, _r27, _sum6); + _sum6 = _mm256_comp_fmadd_ps(_k22, _r28, _sum6); __m256 _sum7 = _bias0; __m256 _r09 = _mm256_loadu_ps(r0 + 72); @@ -179,15 +179,15 @@ static void convdw3x3s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const M __m256 _r29 = _mm256_loadu_ps(r2 + 72); _mm256_storeu_ps(outptr0 + 48, _sum6); - _sum7 = _mm256_fmadd_ps(_k00, _r07, _sum7); - _sum7 = _mm256_fmadd_ps(_k01, _r08, _sum7); - _sum7 = _mm256_fmadd_ps(_k02, _r09, _sum7); - _sum7 = _mm256_fmadd_ps(_k10, _r17, _sum7); - _sum7 = _mm256_fmadd_ps(_k11, _r18, _sum7); - _sum7 = _mm256_fmadd_ps(_k12, _r19, _sum7); - _sum7 = _mm256_fmadd_ps(_k20, _r27, _sum7); - _sum7 = _mm256_fmadd_ps(_k21, _r28, _sum7); - _sum7 = _mm256_fmadd_ps(_k22, _r29, _sum7); + _sum7 = _mm256_comp_fmadd_ps(_k00, _r07, _sum7); + _sum7 = _mm256_comp_fmadd_ps(_k01, _r08, _sum7); + _sum7 = _mm256_comp_fmadd_ps(_k02, _r09, _sum7); + _sum7 = _mm256_comp_fmadd_ps(_k10, _r17, _sum7); + _sum7 = _mm256_comp_fmadd_ps(_k11, _r18, _sum7); + _sum7 = _mm256_comp_fmadd_ps(_k12, _r19, _sum7); + _sum7 = _mm256_comp_fmadd_ps(_k20, _r27, _sum7); + _sum7 = _mm256_comp_fmadd_ps(_k21, _r28, _sum7); + _sum7 = _mm256_comp_fmadd_ps(_k22, _r29, _sum7); _mm256_storeu_ps(outptr0 + 56, _sum7); r0 += 64; @@ -209,15 +209,15 @@ static void convdw3x3s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const M __m256 _r21 = _mm256_loadu_ps(r2 + 8); __m256 _r22 = _mm256_loadu_ps(r2 + 16); - _sum0 = _mm256_fmadd_ps(_k00, _r00, _sum0); - _sum0 = _mm256_fmadd_ps(_k01, _r01, _sum0); - _sum0 = _mm256_fmadd_ps(_k02, _r02, _sum0); - _sum0 = _mm256_fmadd_ps(_k10, _r10, _sum0); - _sum0 = _mm256_fmadd_ps(_k11, _r11, _sum0); - _sum0 = _mm256_fmadd_ps(_k12, _r12, _sum0); - _sum0 = _mm256_fmadd_ps(_k20, _r20, _sum0); - _sum0 = _mm256_fmadd_ps(_k21, _r21, _sum0); - _sum0 = _mm256_fmadd_ps(_k22, _r22, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k00, _r00, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k01, _r01, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k02, _r02, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k10, _r10, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k11, _r11, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k12, _r12, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k20, _r20, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k21, _r21, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k22, _r22, _sum0); __m256 _sum1 = _bias0; __m256 _r03 = _mm256_loadu_ps(r0 + 24); @@ -225,15 +225,15 @@ static void convdw3x3s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const M __m256 _r23 = _mm256_loadu_ps(r2 + 24); _mm256_storeu_ps(outptr0, _sum0); - _sum1 = _mm256_fmadd_ps(_k00, _r01, _sum1); - _sum1 = _mm256_fmadd_ps(_k01, _r02, _sum1); - _sum1 = _mm256_fmadd_ps(_k02, _r03, _sum1); - _sum1 = _mm256_fmadd_ps(_k10, _r11, _sum1); - _sum1 = _mm256_fmadd_ps(_k11, _r12, _sum1); - _sum1 = _mm256_fmadd_ps(_k12, _r13, _sum1); - _sum1 = _mm256_fmadd_ps(_k20, _r21, _sum1); - _sum1 = _mm256_fmadd_ps(_k21, _r22, _sum1); - _sum1 = _mm256_fmadd_ps(_k22, _r23, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k00, _r01, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k01, _r02, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k02, _r03, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k10, _r11, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k11, _r12, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k12, _r13, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k20, _r21, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k21, _r22, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k22, _r23, _sum1); __m256 _sum2 = _bias0; __m256 _r04 = _mm256_loadu_ps(r0 + 32); @@ -241,15 +241,15 @@ static void convdw3x3s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const M __m256 _r24 = _mm256_loadu_ps(r2 + 32); _mm256_storeu_ps(outptr0 + 8, _sum1); - _sum2 = _mm256_fmadd_ps(_k00, _r02, _sum2); - _sum2 = _mm256_fmadd_ps(_k01, _r03, _sum2); - _sum2 = _mm256_fmadd_ps(_k02, _r04, _sum2); - _sum2 = _mm256_fmadd_ps(_k10, _r12, _sum2); - _sum2 = _mm256_fmadd_ps(_k11, _r13, _sum2); - _sum2 = _mm256_fmadd_ps(_k12, _r14, _sum2); - _sum2 = _mm256_fmadd_ps(_k20, _r22, _sum2); - _sum2 = _mm256_fmadd_ps(_k21, _r23, _sum2); - _sum2 = _mm256_fmadd_ps(_k22, _r24, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_k00, _r02, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_k01, _r03, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_k02, _r04, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_k10, _r12, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_k11, _r13, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_k12, _r14, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_k20, _r22, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_k21, _r23, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_k22, _r24, _sum2); __m256 _sum3 = _bias0; __m256 _r05 = _mm256_loadu_ps(r0 + 40); @@ -257,15 +257,15 @@ static void convdw3x3s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const M __m256 _r25 = _mm256_loadu_ps(r2 + 40); _mm256_storeu_ps(outptr0 + 16, _sum2); - _sum3 = _mm256_fmadd_ps(_k00, _r03, _sum3); - _sum3 = _mm256_fmadd_ps(_k01, _r04, _sum3); - _sum3 = _mm256_fmadd_ps(_k02, _r05, _sum3); - _sum3 = _mm256_fmadd_ps(_k10, _r13, _sum3); - _sum3 = _mm256_fmadd_ps(_k11, _r14, _sum3); - _sum3 = _mm256_fmadd_ps(_k12, _r15, _sum3); - _sum3 = _mm256_fmadd_ps(_k20, _r23, _sum3); - _sum3 = _mm256_fmadd_ps(_k21, _r24, _sum3); - _sum3 = _mm256_fmadd_ps(_k22, _r25, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_k00, _r03, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_k01, _r04, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_k02, _r05, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_k10, _r13, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_k11, _r14, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_k12, _r15, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_k20, _r23, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_k21, _r24, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_k22, _r25, _sum3); _mm256_storeu_ps(outptr0 + 24, _sum3); @@ -288,15 +288,15 @@ static void convdw3x3s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const M __m256 _r21 = _mm256_loadu_ps(r2 + 8); __m256 _r22 = _mm256_loadu_ps(r2 + 16); - _sum0 = _mm256_fmadd_ps(_k00, _r00, _sum0); - _sum0 = _mm256_fmadd_ps(_k01, _r01, _sum0); - _sum0 = _mm256_fmadd_ps(_k02, _r02, _sum0); - _sum0 = _mm256_fmadd_ps(_k10, _r10, _sum0); - _sum0 = _mm256_fmadd_ps(_k11, _r11, _sum0); - _sum0 = _mm256_fmadd_ps(_k12, _r12, _sum0); - _sum0 = _mm256_fmadd_ps(_k20, _r20, _sum0); - _sum0 = _mm256_fmadd_ps(_k21, _r21, _sum0); - _sum0 = _mm256_fmadd_ps(_k22, _r22, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k00, _r00, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k01, _r01, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k02, _r02, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k10, _r10, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k11, _r11, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k12, _r12, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k20, _r20, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k21, _r21, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k22, _r22, _sum0); __m256 _sum1 = _bias0; __m256 _r03 = _mm256_loadu_ps(r0 + 24); @@ -304,15 +304,15 @@ static void convdw3x3s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const M __m256 _r23 = _mm256_loadu_ps(r2 + 24); _mm256_storeu_ps(outptr0, _sum0); - _sum1 = _mm256_fmadd_ps(_k00, _r01, _sum1); - _sum1 = _mm256_fmadd_ps(_k01, _r02, _sum1); - _sum1 = _mm256_fmadd_ps(_k02, _r03, _sum1); - _sum1 = _mm256_fmadd_ps(_k10, _r11, _sum1); - _sum1 = _mm256_fmadd_ps(_k11, _r12, _sum1); - _sum1 = _mm256_fmadd_ps(_k12, _r13, _sum1); - _sum1 = _mm256_fmadd_ps(_k20, _r21, _sum1); - _sum1 = _mm256_fmadd_ps(_k21, _r22, _sum1); - _sum1 = _mm256_fmadd_ps(_k22, _r23, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k00, _r01, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k01, _r02, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k02, _r03, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k10, _r11, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k11, _r12, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k12, _r13, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k20, _r21, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k21, _r22, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k22, _r23, _sum1); _mm256_storeu_ps(outptr0 + 8, _sum1); @@ -335,15 +335,15 @@ static void convdw3x3s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const M __m256 _r21 = _mm256_loadu_ps(r2 + 8); __m256 _r22 = _mm256_loadu_ps(r2 + 16); - _sum0 = _mm256_fmadd_ps(_k00, _r00, _sum0); - _sum0 = _mm256_fmadd_ps(_k01, _r01, _sum0); - _sum0 = _mm256_fmadd_ps(_k02, _r02, _sum0); - _sum0 = _mm256_fmadd_ps(_k10, _r10, _sum0); - _sum0 = _mm256_fmadd_ps(_k11, _r11, _sum0); - _sum0 = _mm256_fmadd_ps(_k12, _r12, _sum0); - _sum0 = _mm256_fmadd_ps(_k20, _r20, _sum0); - _sum0 = _mm256_fmadd_ps(_k21, _r21, _sum0); - _sum0 = _mm256_fmadd_ps(_k22, _r22, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k00, _r00, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k01, _r01, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k02, _r02, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k10, _r10, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k11, _r11, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k12, _r12, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k20, _r20, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k21, _r21, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k22, _r22, _sum0); _mm256_storeu_ps(outptr0, _sum0); @@ -419,15 +419,15 @@ static void convdw3x3s2_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const M __m256 _r21 = _mm256_loadu_ps(r2 + 8); __m256 _r22 = _mm256_loadu_ps(r2 + 16); - _sum0 = _mm256_fmadd_ps(_k00, _r00, _sum0); - _sum0 = _mm256_fmadd_ps(_k01, _r01, _sum0); - _sum0 = _mm256_fmadd_ps(_k02, _r02, _sum0); - _sum0 = _mm256_fmadd_ps(_k10, _r10, _sum0); - _sum0 = _mm256_fmadd_ps(_k11, _r11, _sum0); - _sum0 = _mm256_fmadd_ps(_k12, _r12, _sum0); - _sum0 = _mm256_fmadd_ps(_k20, _r20, _sum0); - _sum0 = _mm256_fmadd_ps(_k21, _r21, _sum0); - _sum0 = _mm256_fmadd_ps(_k22, _r22, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k00, _r00, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k01, _r01, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k02, _r02, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k10, _r10, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k11, _r11, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k12, _r12, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k20, _r20, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k21, _r21, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k22, _r22, _sum0); __m256 _sum1 = _bias0; __m256 _r03 = _mm256_loadu_ps(r0 + 24); @@ -438,15 +438,15 @@ static void convdw3x3s2_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const M __m256 _r24 = _mm256_loadu_ps(r2 + 32); _mm256_storeu_ps(outptr0, _sum0); - _sum1 = _mm256_fmadd_ps(_k00, _r02, _sum1); - _sum1 = _mm256_fmadd_ps(_k01, _r03, _sum1); - _sum1 = _mm256_fmadd_ps(_k02, _r04, _sum1); - _sum1 = _mm256_fmadd_ps(_k10, _r12, _sum1); - _sum1 = _mm256_fmadd_ps(_k11, _r13, _sum1); - _sum1 = _mm256_fmadd_ps(_k12, _r14, _sum1); - _sum1 = _mm256_fmadd_ps(_k20, _r22, _sum1); - _sum1 = _mm256_fmadd_ps(_k21, _r23, _sum1); - _sum1 = _mm256_fmadd_ps(_k22, _r24, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k00, _r02, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k01, _r03, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k02, _r04, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k10, _r12, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k11, _r13, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k12, _r14, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k20, _r22, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k21, _r23, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k22, _r24, _sum1); __m256 _sum2 = _bias0; __m256 _r05 = _mm256_loadu_ps(r0 + 40); @@ -457,15 +457,15 @@ static void convdw3x3s2_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const M __m256 _r26 = _mm256_loadu_ps(r2 + 48); _mm256_storeu_ps(outptr0 + 8, _sum1); - _sum2 = _mm256_fmadd_ps(_k00, _r04, _sum2); - _sum2 = _mm256_fmadd_ps(_k01, _r05, _sum2); - _sum2 = _mm256_fmadd_ps(_k02, _r06, _sum2); - _sum2 = _mm256_fmadd_ps(_k10, _r14, _sum2); - _sum2 = _mm256_fmadd_ps(_k11, _r15, _sum2); - _sum2 = _mm256_fmadd_ps(_k12, _r16, _sum2); - _sum2 = _mm256_fmadd_ps(_k20, _r24, _sum2); - _sum2 = _mm256_fmadd_ps(_k21, _r25, _sum2); - _sum2 = _mm256_fmadd_ps(_k22, _r26, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_k00, _r04, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_k01, _r05, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_k02, _r06, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_k10, _r14, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_k11, _r15, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_k12, _r16, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_k20, _r24, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_k21, _r25, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_k22, _r26, _sum2); __m256 _sum3 = _bias0; __m256 _r07 = _mm256_loadu_ps(r0 + 56); @@ -476,15 +476,15 @@ static void convdw3x3s2_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const M __m256 _r28 = _mm256_loadu_ps(r2 + 64); _mm256_storeu_ps(outptr0 + 16, _sum2); - _sum3 = _mm256_fmadd_ps(_k00, _r06, _sum3); - _sum3 = _mm256_fmadd_ps(_k01, _r07, _sum3); - _sum3 = _mm256_fmadd_ps(_k02, _r08, _sum3); - _sum3 = _mm256_fmadd_ps(_k10, _r16, _sum3); - _sum3 = _mm256_fmadd_ps(_k11, _r17, _sum3); - _sum3 = _mm256_fmadd_ps(_k12, _r18, _sum3); - _sum3 = _mm256_fmadd_ps(_k20, _r26, _sum3); - _sum3 = _mm256_fmadd_ps(_k21, _r27, _sum3); - _sum3 = _mm256_fmadd_ps(_k22, _r28, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_k00, _r06, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_k01, _r07, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_k02, _r08, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_k10, _r16, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_k11, _r17, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_k12, _r18, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_k20, _r26, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_k21, _r27, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_k22, _r28, _sum3); _mm256_storeu_ps(outptr0 + 24, _sum3); @@ -507,15 +507,15 @@ static void convdw3x3s2_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const M __m256 _r21 = _mm256_loadu_ps(r2 + 8); __m256 _r22 = _mm256_loadu_ps(r2 + 16); - _sum0 = _mm256_fmadd_ps(_k00, _r00, _sum0); - _sum0 = _mm256_fmadd_ps(_k01, _r01, _sum0); - _sum0 = _mm256_fmadd_ps(_k02, _r02, _sum0); - _sum0 = _mm256_fmadd_ps(_k10, _r10, _sum0); - _sum0 = _mm256_fmadd_ps(_k11, _r11, _sum0); - _sum0 = _mm256_fmadd_ps(_k12, _r12, _sum0); - _sum0 = _mm256_fmadd_ps(_k20, _r20, _sum0); - _sum0 = _mm256_fmadd_ps(_k21, _r21, _sum0); - _sum0 = _mm256_fmadd_ps(_k22, _r22, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k00, _r00, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k01, _r01, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k02, _r02, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k10, _r10, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k11, _r11, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k12, _r12, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k20, _r20, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k21, _r21, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k22, _r22, _sum0); __m256 _sum1 = _bias0; __m256 _r03 = _mm256_loadu_ps(r0 + 24); @@ -526,15 +526,15 @@ static void convdw3x3s2_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const M __m256 _r24 = _mm256_loadu_ps(r2 + 32); _mm256_storeu_ps(outptr0, _sum0); - _sum1 = _mm256_fmadd_ps(_k00, _r02, _sum1); - _sum1 = _mm256_fmadd_ps(_k01, _r03, _sum1); - _sum1 = _mm256_fmadd_ps(_k02, _r04, _sum1); - _sum1 = _mm256_fmadd_ps(_k10, _r12, _sum1); - _sum1 = _mm256_fmadd_ps(_k11, _r13, _sum1); - _sum1 = _mm256_fmadd_ps(_k12, _r14, _sum1); - _sum1 = _mm256_fmadd_ps(_k20, _r22, _sum1); - _sum1 = _mm256_fmadd_ps(_k21, _r23, _sum1); - _sum1 = _mm256_fmadd_ps(_k22, _r24, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k00, _r02, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k01, _r03, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k02, _r04, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k10, _r12, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k11, _r13, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k12, _r14, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k20, _r22, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k21, _r23, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k22, _r24, _sum1); _mm256_storeu_ps(outptr0 + 8, _sum1); @@ -557,15 +557,15 @@ static void convdw3x3s2_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const M __m256 _r21 = _mm256_loadu_ps(r2 + 8); __m256 _r22 = _mm256_loadu_ps(r2 + 16); - _sum0 = _mm256_fmadd_ps(_k00, _r00, _sum0); - _sum0 = _mm256_fmadd_ps(_k01, _r01, _sum0); - _sum0 = _mm256_fmadd_ps(_k02, _r02, _sum0); - _sum0 = _mm256_fmadd_ps(_k10, _r10, _sum0); - _sum0 = _mm256_fmadd_ps(_k11, _r11, _sum0); - _sum0 = _mm256_fmadd_ps(_k12, _r12, _sum0); - _sum0 = _mm256_fmadd_ps(_k20, _r20, _sum0); - _sum0 = _mm256_fmadd_ps(_k21, _r21, _sum0); - _sum0 = _mm256_fmadd_ps(_k22, _r22, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k00, _r00, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k01, _r01, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k02, _r02, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k10, _r10, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k11, _r11, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k12, _r12, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k20, _r20, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k21, _r21, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k22, _r22, _sum0); _mm256_storeu_ps(outptr0, _sum0); r0 += 2 * 8; r1 += 2 * 8; diff --git a/src/layer/x86/convolutiondepthwise_3x3_pack8_fp16.h b/src/layer/x86/convolutiondepthwise_3x3_pack8_fp16.h index b75e51c79..d1389694f 100644 --- a/src/layer/x86/convolutiondepthwise_3x3_pack8_fp16.h +++ b/src/layer/x86/convolutiondepthwise_3x3_pack8_fp16.h @@ -67,15 +67,15 @@ static void convdw3x3s1_fp16_pack8_avx(const Mat& bottom_blob, Mat& top_blob, co __m256 _r21 = _mm256_loadu_ps(r2 + 8); __m256 _r22 = _mm256_loadu_ps(r2 + 16); - _sum0 = _mm256_fmadd_ps(_k00, _r00, _sum0); - _sum0 = _mm256_fmadd_ps(_k01, _r01, _sum0); - _sum0 = _mm256_fmadd_ps(_k02, _r02, _sum0); - _sum0 = _mm256_fmadd_ps(_k10, _r10, _sum0); - _sum0 = _mm256_fmadd_ps(_k11, _r11, _sum0); - _sum0 = _mm256_fmadd_ps(_k12, _r12, _sum0); - _sum0 = _mm256_fmadd_ps(_k20, _r20, _sum0); - _sum0 = _mm256_fmadd_ps(_k21, _r21, _sum0); - _sum0 = _mm256_fmadd_ps(_k22, _r22, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k00, _r00, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k01, _r01, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k02, _r02, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k10, _r10, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k11, _r11, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k12, _r12, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k20, _r20, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k21, _r21, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k22, _r22, _sum0); __m256 _sum1 = _bias0; __m256 _r03 = _mm256_loadu_ps(r0 + 24); @@ -83,15 +83,15 @@ static void convdw3x3s1_fp16_pack8_avx(const Mat& bottom_blob, Mat& top_blob, co __m256 _r23 = _mm256_loadu_ps(r2 + 24); _mm256_storeu_ps(outptr0, _sum0); - _sum1 = _mm256_fmadd_ps(_k00, _r01, _sum1); - _sum1 = _mm256_fmadd_ps(_k01, _r02, _sum1); - _sum1 = _mm256_fmadd_ps(_k02, _r03, _sum1); - _sum1 = _mm256_fmadd_ps(_k10, _r11, _sum1); - _sum1 = _mm256_fmadd_ps(_k11, _r12, _sum1); - _sum1 = _mm256_fmadd_ps(_k12, _r13, _sum1); - _sum1 = _mm256_fmadd_ps(_k20, _r21, _sum1); - _sum1 = _mm256_fmadd_ps(_k21, _r22, _sum1); - _sum1 = _mm256_fmadd_ps(_k22, _r23, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k00, _r01, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k01, _r02, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k02, _r03, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k10, _r11, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k11, _r12, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k12, _r13, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k20, _r21, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k21, _r22, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k22, _r23, _sum1); __m256 _sum2 = _bias0; __m256 _r04 = _mm256_loadu_ps(r0 + 32); @@ -99,15 +99,15 @@ static void convdw3x3s1_fp16_pack8_avx(const Mat& bottom_blob, Mat& top_blob, co __m256 _r24 = _mm256_loadu_ps(r2 + 32); _mm256_storeu_ps(outptr0 + 8, _sum1); - _sum2 = _mm256_fmadd_ps(_k00, _r02, _sum2); - _sum2 = _mm256_fmadd_ps(_k01, _r03, _sum2); - _sum2 = _mm256_fmadd_ps(_k02, _r04, _sum2); - _sum2 = _mm256_fmadd_ps(_k10, _r12, _sum2); - _sum2 = _mm256_fmadd_ps(_k11, _r13, _sum2); - _sum2 = _mm256_fmadd_ps(_k12, _r14, _sum2); - _sum2 = _mm256_fmadd_ps(_k20, _r22, _sum2); - _sum2 = _mm256_fmadd_ps(_k21, _r23, _sum2); - _sum2 = _mm256_fmadd_ps(_k22, _r24, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_k00, _r02, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_k01, _r03, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_k02, _r04, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_k10, _r12, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_k11, _r13, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_k12, _r14, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_k20, _r22, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_k21, _r23, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_k22, _r24, _sum2); __m256 _sum3 = _bias0; __m256 _r05 = _mm256_loadu_ps(r0 + 40); @@ -115,15 +115,15 @@ static void convdw3x3s1_fp16_pack8_avx(const Mat& bottom_blob, Mat& top_blob, co __m256 _r25 = _mm256_loadu_ps(r2 + 40); _mm256_storeu_ps(outptr0 + 16, _sum2); - _sum3 = _mm256_fmadd_ps(_k00, _r03, _sum3); - _sum3 = _mm256_fmadd_ps(_k01, _r04, _sum3); - _sum3 = _mm256_fmadd_ps(_k02, _r05, _sum3); - _sum3 = _mm256_fmadd_ps(_k10, _r13, _sum3); - _sum3 = _mm256_fmadd_ps(_k11, _r14, _sum3); - _sum3 = _mm256_fmadd_ps(_k12, _r15, _sum3); - _sum3 = _mm256_fmadd_ps(_k20, _r23, _sum3); - _sum3 = _mm256_fmadd_ps(_k21, _r24, _sum3); - _sum3 = _mm256_fmadd_ps(_k22, _r25, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_k00, _r03, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_k01, _r04, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_k02, _r05, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_k10, _r13, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_k11, _r14, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_k12, _r15, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_k20, _r23, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_k21, _r24, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_k22, _r25, _sum3); __m256 _sum4 = _bias0; __m256 _r06 = _mm256_loadu_ps(r0 + 48); @@ -131,15 +131,15 @@ static void convdw3x3s1_fp16_pack8_avx(const Mat& bottom_blob, Mat& top_blob, co __m256 _r26 = _mm256_loadu_ps(r2 + 48); _mm256_storeu_ps(outptr0 + 24, _sum3); - _sum4 = _mm256_fmadd_ps(_k00, _r04, _sum4); - _sum4 = _mm256_fmadd_ps(_k01, _r05, _sum4); - _sum4 = _mm256_fmadd_ps(_k02, _r06, _sum4); - _sum4 = _mm256_fmadd_ps(_k10, _r14, _sum4); - _sum4 = _mm256_fmadd_ps(_k11, _r15, _sum4); - _sum4 = _mm256_fmadd_ps(_k12, _r16, _sum4); - _sum4 = _mm256_fmadd_ps(_k20, _r24, _sum4); - _sum4 = _mm256_fmadd_ps(_k21, _r25, _sum4); - _sum4 = _mm256_fmadd_ps(_k22, _r26, _sum4); + _sum4 = _mm256_comp_fmadd_ps(_k00, _r04, _sum4); + _sum4 = _mm256_comp_fmadd_ps(_k01, _r05, _sum4); + _sum4 = _mm256_comp_fmadd_ps(_k02, _r06, _sum4); + _sum4 = _mm256_comp_fmadd_ps(_k10, _r14, _sum4); + _sum4 = _mm256_comp_fmadd_ps(_k11, _r15, _sum4); + _sum4 = _mm256_comp_fmadd_ps(_k12, _r16, _sum4); + _sum4 = _mm256_comp_fmadd_ps(_k20, _r24, _sum4); + _sum4 = _mm256_comp_fmadd_ps(_k21, _r25, _sum4); + _sum4 = _mm256_comp_fmadd_ps(_k22, _r26, _sum4); __m256 _sum5 = _bias0; __m256 _r07 = _mm256_loadu_ps(r0 + 56); @@ -147,15 +147,15 @@ static void convdw3x3s1_fp16_pack8_avx(const Mat& bottom_blob, Mat& top_blob, co __m256 _r27 = _mm256_loadu_ps(r2 + 56); _mm256_storeu_ps(outptr0 + 32, _sum4); - _sum5 = _mm256_fmadd_ps(_k00, _r05, _sum5); - _sum5 = _mm256_fmadd_ps(_k01, _r06, _sum5); - _sum5 = _mm256_fmadd_ps(_k02, _r07, _sum5); - _sum5 = _mm256_fmadd_ps(_k10, _r15, _sum5); - _sum5 = _mm256_fmadd_ps(_k11, _r16, _sum5); - _sum5 = _mm256_fmadd_ps(_k12, _r17, _sum5); - _sum5 = _mm256_fmadd_ps(_k20, _r25, _sum5); - _sum5 = _mm256_fmadd_ps(_k21, _r26, _sum5); - _sum5 = _mm256_fmadd_ps(_k22, _r27, _sum5); + _sum5 = _mm256_comp_fmadd_ps(_k00, _r05, _sum5); + _sum5 = _mm256_comp_fmadd_ps(_k01, _r06, _sum5); + _sum5 = _mm256_comp_fmadd_ps(_k02, _r07, _sum5); + _sum5 = _mm256_comp_fmadd_ps(_k10, _r15, _sum5); + _sum5 = _mm256_comp_fmadd_ps(_k11, _r16, _sum5); + _sum5 = _mm256_comp_fmadd_ps(_k12, _r17, _sum5); + _sum5 = _mm256_comp_fmadd_ps(_k20, _r25, _sum5); + _sum5 = _mm256_comp_fmadd_ps(_k21, _r26, _sum5); + _sum5 = _mm256_comp_fmadd_ps(_k22, _r27, _sum5); __m256 _sum6 = _bias0; __m256 _r08 = _mm256_loadu_ps(r0 + 64); @@ -163,15 +163,15 @@ static void convdw3x3s1_fp16_pack8_avx(const Mat& bottom_blob, Mat& top_blob, co __m256 _r28 = _mm256_loadu_ps(r2 + 64); _mm256_storeu_ps(outptr0 + 40, _sum5); - _sum6 = _mm256_fmadd_ps(_k00, _r06, _sum6); - _sum6 = _mm256_fmadd_ps(_k01, _r07, _sum6); - _sum6 = _mm256_fmadd_ps(_k02, _r08, _sum6); - _sum6 = _mm256_fmadd_ps(_k10, _r16, _sum6); - _sum6 = _mm256_fmadd_ps(_k11, _r17, _sum6); - _sum6 = _mm256_fmadd_ps(_k12, _r18, _sum6); - _sum6 = _mm256_fmadd_ps(_k20, _r26, _sum6); - _sum6 = _mm256_fmadd_ps(_k21, _r27, _sum6); - _sum6 = _mm256_fmadd_ps(_k22, _r28, _sum6); + _sum6 = _mm256_comp_fmadd_ps(_k00, _r06, _sum6); + _sum6 = _mm256_comp_fmadd_ps(_k01, _r07, _sum6); + _sum6 = _mm256_comp_fmadd_ps(_k02, _r08, _sum6); + _sum6 = _mm256_comp_fmadd_ps(_k10, _r16, _sum6); + _sum6 = _mm256_comp_fmadd_ps(_k11, _r17, _sum6); + _sum6 = _mm256_comp_fmadd_ps(_k12, _r18, _sum6); + _sum6 = _mm256_comp_fmadd_ps(_k20, _r26, _sum6); + _sum6 = _mm256_comp_fmadd_ps(_k21, _r27, _sum6); + _sum6 = _mm256_comp_fmadd_ps(_k22, _r28, _sum6); __m256 _sum7 = _bias0; __m256 _r09 = _mm256_loadu_ps(r0 + 72); @@ -179,15 +179,15 @@ static void convdw3x3s1_fp16_pack8_avx(const Mat& bottom_blob, Mat& top_blob, co __m256 _r29 = _mm256_loadu_ps(r2 + 72); _mm256_storeu_ps(outptr0 + 48, _sum6); - _sum7 = _mm256_fmadd_ps(_k00, _r07, _sum7); - _sum7 = _mm256_fmadd_ps(_k01, _r08, _sum7); - _sum7 = _mm256_fmadd_ps(_k02, _r09, _sum7); - _sum7 = _mm256_fmadd_ps(_k10, _r17, _sum7); - _sum7 = _mm256_fmadd_ps(_k11, _r18, _sum7); - _sum7 = _mm256_fmadd_ps(_k12, _r19, _sum7); - _sum7 = _mm256_fmadd_ps(_k20, _r27, _sum7); - _sum7 = _mm256_fmadd_ps(_k21, _r28, _sum7); - _sum7 = _mm256_fmadd_ps(_k22, _r29, _sum7); + _sum7 = _mm256_comp_fmadd_ps(_k00, _r07, _sum7); + _sum7 = _mm256_comp_fmadd_ps(_k01, _r08, _sum7); + _sum7 = _mm256_comp_fmadd_ps(_k02, _r09, _sum7); + _sum7 = _mm256_comp_fmadd_ps(_k10, _r17, _sum7); + _sum7 = _mm256_comp_fmadd_ps(_k11, _r18, _sum7); + _sum7 = _mm256_comp_fmadd_ps(_k12, _r19, _sum7); + _sum7 = _mm256_comp_fmadd_ps(_k20, _r27, _sum7); + _sum7 = _mm256_comp_fmadd_ps(_k21, _r28, _sum7); + _sum7 = _mm256_comp_fmadd_ps(_k22, _r29, _sum7); _mm256_storeu_ps(outptr0 + 56, _sum7); r0 += 64; @@ -209,15 +209,15 @@ static void convdw3x3s1_fp16_pack8_avx(const Mat& bottom_blob, Mat& top_blob, co __m256 _r21 = _mm256_loadu_ps(r2 + 8); __m256 _r22 = _mm256_loadu_ps(r2 + 16); - _sum0 = _mm256_fmadd_ps(_k00, _r00, _sum0); - _sum0 = _mm256_fmadd_ps(_k01, _r01, _sum0); - _sum0 = _mm256_fmadd_ps(_k02, _r02, _sum0); - _sum0 = _mm256_fmadd_ps(_k10, _r10, _sum0); - _sum0 = _mm256_fmadd_ps(_k11, _r11, _sum0); - _sum0 = _mm256_fmadd_ps(_k12, _r12, _sum0); - _sum0 = _mm256_fmadd_ps(_k20, _r20, _sum0); - _sum0 = _mm256_fmadd_ps(_k21, _r21, _sum0); - _sum0 = _mm256_fmadd_ps(_k22, _r22, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k00, _r00, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k01, _r01, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k02, _r02, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k10, _r10, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k11, _r11, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k12, _r12, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k20, _r20, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k21, _r21, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k22, _r22, _sum0); __m256 _sum1 = _bias0; __m256 _r03 = _mm256_loadu_ps(r0 + 24); @@ -225,15 +225,15 @@ static void convdw3x3s1_fp16_pack8_avx(const Mat& bottom_blob, Mat& top_blob, co __m256 _r23 = _mm256_loadu_ps(r2 + 24); _mm256_storeu_ps(outptr0, _sum0); - _sum1 = _mm256_fmadd_ps(_k00, _r01, _sum1); - _sum1 = _mm256_fmadd_ps(_k01, _r02, _sum1); - _sum1 = _mm256_fmadd_ps(_k02, _r03, _sum1); - _sum1 = _mm256_fmadd_ps(_k10, _r11, _sum1); - _sum1 = _mm256_fmadd_ps(_k11, _r12, _sum1); - _sum1 = _mm256_fmadd_ps(_k12, _r13, _sum1); - _sum1 = _mm256_fmadd_ps(_k20, _r21, _sum1); - _sum1 = _mm256_fmadd_ps(_k21, _r22, _sum1); - _sum1 = _mm256_fmadd_ps(_k22, _r23, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k00, _r01, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k01, _r02, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k02, _r03, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k10, _r11, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k11, _r12, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k12, _r13, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k20, _r21, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k21, _r22, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k22, _r23, _sum1); __m256 _sum2 = _bias0; __m256 _r04 = _mm256_loadu_ps(r0 + 32); @@ -241,15 +241,15 @@ static void convdw3x3s1_fp16_pack8_avx(const Mat& bottom_blob, Mat& top_blob, co __m256 _r24 = _mm256_loadu_ps(r2 + 32); _mm256_storeu_ps(outptr0 + 8, _sum1); - _sum2 = _mm256_fmadd_ps(_k00, _r02, _sum2); - _sum2 = _mm256_fmadd_ps(_k01, _r03, _sum2); - _sum2 = _mm256_fmadd_ps(_k02, _r04, _sum2); - _sum2 = _mm256_fmadd_ps(_k10, _r12, _sum2); - _sum2 = _mm256_fmadd_ps(_k11, _r13, _sum2); - _sum2 = _mm256_fmadd_ps(_k12, _r14, _sum2); - _sum2 = _mm256_fmadd_ps(_k20, _r22, _sum2); - _sum2 = _mm256_fmadd_ps(_k21, _r23, _sum2); - _sum2 = _mm256_fmadd_ps(_k22, _r24, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_k00, _r02, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_k01, _r03, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_k02, _r04, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_k10, _r12, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_k11, _r13, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_k12, _r14, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_k20, _r22, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_k21, _r23, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_k22, _r24, _sum2); __m256 _sum3 = _bias0; __m256 _r05 = _mm256_loadu_ps(r0 + 40); @@ -257,15 +257,15 @@ static void convdw3x3s1_fp16_pack8_avx(const Mat& bottom_blob, Mat& top_blob, co __m256 _r25 = _mm256_loadu_ps(r2 + 40); _mm256_storeu_ps(outptr0 + 16, _sum2); - _sum3 = _mm256_fmadd_ps(_k00, _r03, _sum3); - _sum3 = _mm256_fmadd_ps(_k01, _r04, _sum3); - _sum3 = _mm256_fmadd_ps(_k02, _r05, _sum3); - _sum3 = _mm256_fmadd_ps(_k10, _r13, _sum3); - _sum3 = _mm256_fmadd_ps(_k11, _r14, _sum3); - _sum3 = _mm256_fmadd_ps(_k12, _r15, _sum3); - _sum3 = _mm256_fmadd_ps(_k20, _r23, _sum3); - _sum3 = _mm256_fmadd_ps(_k21, _r24, _sum3); - _sum3 = _mm256_fmadd_ps(_k22, _r25, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_k00, _r03, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_k01, _r04, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_k02, _r05, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_k10, _r13, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_k11, _r14, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_k12, _r15, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_k20, _r23, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_k21, _r24, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_k22, _r25, _sum3); _mm256_storeu_ps(outptr0 + 24, _sum3); @@ -288,15 +288,15 @@ static void convdw3x3s1_fp16_pack8_avx(const Mat& bottom_blob, Mat& top_blob, co __m256 _r21 = _mm256_loadu_ps(r2 + 8); __m256 _r22 = _mm256_loadu_ps(r2 + 16); - _sum0 = _mm256_fmadd_ps(_k00, _r00, _sum0); - _sum0 = _mm256_fmadd_ps(_k01, _r01, _sum0); - _sum0 = _mm256_fmadd_ps(_k02, _r02, _sum0); - _sum0 = _mm256_fmadd_ps(_k10, _r10, _sum0); - _sum0 = _mm256_fmadd_ps(_k11, _r11, _sum0); - _sum0 = _mm256_fmadd_ps(_k12, _r12, _sum0); - _sum0 = _mm256_fmadd_ps(_k20, _r20, _sum0); - _sum0 = _mm256_fmadd_ps(_k21, _r21, _sum0); - _sum0 = _mm256_fmadd_ps(_k22, _r22, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k00, _r00, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k01, _r01, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k02, _r02, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k10, _r10, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k11, _r11, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k12, _r12, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k20, _r20, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k21, _r21, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k22, _r22, _sum0); __m256 _sum1 = _bias0; __m256 _r03 = _mm256_loadu_ps(r0 + 24); @@ -304,15 +304,15 @@ static void convdw3x3s1_fp16_pack8_avx(const Mat& bottom_blob, Mat& top_blob, co __m256 _r23 = _mm256_loadu_ps(r2 + 24); _mm256_storeu_ps(outptr0, _sum0); - _sum1 = _mm256_fmadd_ps(_k00, _r01, _sum1); - _sum1 = _mm256_fmadd_ps(_k01, _r02, _sum1); - _sum1 = _mm256_fmadd_ps(_k02, _r03, _sum1); - _sum1 = _mm256_fmadd_ps(_k10, _r11, _sum1); - _sum1 = _mm256_fmadd_ps(_k11, _r12, _sum1); - _sum1 = _mm256_fmadd_ps(_k12, _r13, _sum1); - _sum1 = _mm256_fmadd_ps(_k20, _r21, _sum1); - _sum1 = _mm256_fmadd_ps(_k21, _r22, _sum1); - _sum1 = _mm256_fmadd_ps(_k22, _r23, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k00, _r01, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k01, _r02, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k02, _r03, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k10, _r11, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k11, _r12, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k12, _r13, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k20, _r21, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k21, _r22, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k22, _r23, _sum1); _mm256_storeu_ps(outptr0 + 8, _sum1); @@ -335,15 +335,15 @@ static void convdw3x3s1_fp16_pack8_avx(const Mat& bottom_blob, Mat& top_blob, co __m256 _r21 = _mm256_loadu_ps(r2 + 8); __m256 _r22 = _mm256_loadu_ps(r2 + 16); - _sum0 = _mm256_fmadd_ps(_k00, _r00, _sum0); - _sum0 = _mm256_fmadd_ps(_k01, _r01, _sum0); - _sum0 = _mm256_fmadd_ps(_k02, _r02, _sum0); - _sum0 = _mm256_fmadd_ps(_k10, _r10, _sum0); - _sum0 = _mm256_fmadd_ps(_k11, _r11, _sum0); - _sum0 = _mm256_fmadd_ps(_k12, _r12, _sum0); - _sum0 = _mm256_fmadd_ps(_k20, _r20, _sum0); - _sum0 = _mm256_fmadd_ps(_k21, _r21, _sum0); - _sum0 = _mm256_fmadd_ps(_k22, _r22, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k00, _r00, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k01, _r01, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k02, _r02, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k10, _r10, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k11, _r11, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k12, _r12, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k20, _r20, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k21, _r21, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k22, _r22, _sum0); _mm256_storeu_ps(outptr0, _sum0); @@ -419,15 +419,15 @@ static void convdw3x3s2_fp16_pack8_avx(const Mat& bottom_blob, Mat& top_blob, co __m256 _r21 = _mm256_loadu_ps(r2 + 8); __m256 _r22 = _mm256_loadu_ps(r2 + 16); - _sum0 = _mm256_fmadd_ps(_k00, _r00, _sum0); - _sum0 = _mm256_fmadd_ps(_k01, _r01, _sum0); - _sum0 = _mm256_fmadd_ps(_k02, _r02, _sum0); - _sum0 = _mm256_fmadd_ps(_k10, _r10, _sum0); - _sum0 = _mm256_fmadd_ps(_k11, _r11, _sum0); - _sum0 = _mm256_fmadd_ps(_k12, _r12, _sum0); - _sum0 = _mm256_fmadd_ps(_k20, _r20, _sum0); - _sum0 = _mm256_fmadd_ps(_k21, _r21, _sum0); - _sum0 = _mm256_fmadd_ps(_k22, _r22, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k00, _r00, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k01, _r01, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k02, _r02, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k10, _r10, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k11, _r11, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k12, _r12, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k20, _r20, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k21, _r21, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k22, _r22, _sum0); __m256 _sum1 = _bias0; __m256 _r03 = _mm256_loadu_ps(r0 + 24); @@ -438,15 +438,15 @@ static void convdw3x3s2_fp16_pack8_avx(const Mat& bottom_blob, Mat& top_blob, co __m256 _r24 = _mm256_loadu_ps(r2 + 32); _mm256_storeu_ps(outptr0, _sum0); - _sum1 = _mm256_fmadd_ps(_k00, _r02, _sum1); - _sum1 = _mm256_fmadd_ps(_k01, _r03, _sum1); - _sum1 = _mm256_fmadd_ps(_k02, _r04, _sum1); - _sum1 = _mm256_fmadd_ps(_k10, _r12, _sum1); - _sum1 = _mm256_fmadd_ps(_k11, _r13, _sum1); - _sum1 = _mm256_fmadd_ps(_k12, _r14, _sum1); - _sum1 = _mm256_fmadd_ps(_k20, _r22, _sum1); - _sum1 = _mm256_fmadd_ps(_k21, _r23, _sum1); - _sum1 = _mm256_fmadd_ps(_k22, _r24, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k00, _r02, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k01, _r03, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k02, _r04, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k10, _r12, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k11, _r13, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k12, _r14, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k20, _r22, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k21, _r23, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k22, _r24, _sum1); __m256 _sum2 = _bias0; __m256 _r05 = _mm256_loadu_ps(r0 + 40); @@ -457,15 +457,15 @@ static void convdw3x3s2_fp16_pack8_avx(const Mat& bottom_blob, Mat& top_blob, co __m256 _r26 = _mm256_loadu_ps(r2 + 48); _mm256_storeu_ps(outptr0 + 8, _sum1); - _sum2 = _mm256_fmadd_ps(_k00, _r04, _sum2); - _sum2 = _mm256_fmadd_ps(_k01, _r05, _sum2); - _sum2 = _mm256_fmadd_ps(_k02, _r06, _sum2); - _sum2 = _mm256_fmadd_ps(_k10, _r14, _sum2); - _sum2 = _mm256_fmadd_ps(_k11, _r15, _sum2); - _sum2 = _mm256_fmadd_ps(_k12, _r16, _sum2); - _sum2 = _mm256_fmadd_ps(_k20, _r24, _sum2); - _sum2 = _mm256_fmadd_ps(_k21, _r25, _sum2); - _sum2 = _mm256_fmadd_ps(_k22, _r26, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_k00, _r04, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_k01, _r05, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_k02, _r06, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_k10, _r14, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_k11, _r15, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_k12, _r16, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_k20, _r24, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_k21, _r25, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_k22, _r26, _sum2); __m256 _sum3 = _bias0; __m256 _r07 = _mm256_loadu_ps(r0 + 56); @@ -476,15 +476,15 @@ static void convdw3x3s2_fp16_pack8_avx(const Mat& bottom_blob, Mat& top_blob, co __m256 _r28 = _mm256_loadu_ps(r2 + 64); _mm256_storeu_ps(outptr0 + 16, _sum2); - _sum3 = _mm256_fmadd_ps(_k00, _r06, _sum3); - _sum3 = _mm256_fmadd_ps(_k01, _r07, _sum3); - _sum3 = _mm256_fmadd_ps(_k02, _r08, _sum3); - _sum3 = _mm256_fmadd_ps(_k10, _r16, _sum3); - _sum3 = _mm256_fmadd_ps(_k11, _r17, _sum3); - _sum3 = _mm256_fmadd_ps(_k12, _r18, _sum3); - _sum3 = _mm256_fmadd_ps(_k20, _r26, _sum3); - _sum3 = _mm256_fmadd_ps(_k21, _r27, _sum3); - _sum3 = _mm256_fmadd_ps(_k22, _r28, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_k00, _r06, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_k01, _r07, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_k02, _r08, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_k10, _r16, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_k11, _r17, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_k12, _r18, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_k20, _r26, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_k21, _r27, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_k22, _r28, _sum3); _mm256_storeu_ps(outptr0 + 24, _sum3); @@ -507,15 +507,15 @@ static void convdw3x3s2_fp16_pack8_avx(const Mat& bottom_blob, Mat& top_blob, co __m256 _r21 = _mm256_loadu_ps(r2 + 8); __m256 _r22 = _mm256_loadu_ps(r2 + 16); - _sum0 = _mm256_fmadd_ps(_k00, _r00, _sum0); - _sum0 = _mm256_fmadd_ps(_k01, _r01, _sum0); - _sum0 = _mm256_fmadd_ps(_k02, _r02, _sum0); - _sum0 = _mm256_fmadd_ps(_k10, _r10, _sum0); - _sum0 = _mm256_fmadd_ps(_k11, _r11, _sum0); - _sum0 = _mm256_fmadd_ps(_k12, _r12, _sum0); - _sum0 = _mm256_fmadd_ps(_k20, _r20, _sum0); - _sum0 = _mm256_fmadd_ps(_k21, _r21, _sum0); - _sum0 = _mm256_fmadd_ps(_k22, _r22, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k00, _r00, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k01, _r01, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k02, _r02, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k10, _r10, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k11, _r11, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k12, _r12, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k20, _r20, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k21, _r21, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k22, _r22, _sum0); __m256 _sum1 = _bias0; __m256 _r03 = _mm256_loadu_ps(r0 + 24); @@ -526,15 +526,15 @@ static void convdw3x3s2_fp16_pack8_avx(const Mat& bottom_blob, Mat& top_blob, co __m256 _r24 = _mm256_loadu_ps(r2 + 32); _mm256_storeu_ps(outptr0, _sum0); - _sum1 = _mm256_fmadd_ps(_k00, _r02, _sum1); - _sum1 = _mm256_fmadd_ps(_k01, _r03, _sum1); - _sum1 = _mm256_fmadd_ps(_k02, _r04, _sum1); - _sum1 = _mm256_fmadd_ps(_k10, _r12, _sum1); - _sum1 = _mm256_fmadd_ps(_k11, _r13, _sum1); - _sum1 = _mm256_fmadd_ps(_k12, _r14, _sum1); - _sum1 = _mm256_fmadd_ps(_k20, _r22, _sum1); - _sum1 = _mm256_fmadd_ps(_k21, _r23, _sum1); - _sum1 = _mm256_fmadd_ps(_k22, _r24, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k00, _r02, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k01, _r03, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k02, _r04, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k10, _r12, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k11, _r13, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k12, _r14, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k20, _r22, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k21, _r23, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_k22, _r24, _sum1); _mm256_storeu_ps(outptr0 + 8, _sum1); @@ -557,15 +557,15 @@ static void convdw3x3s2_fp16_pack8_avx(const Mat& bottom_blob, Mat& top_blob, co __m256 _r21 = _mm256_loadu_ps(r2 + 8); __m256 _r22 = _mm256_loadu_ps(r2 + 16); - _sum0 = _mm256_fmadd_ps(_k00, _r00, _sum0); - _sum0 = _mm256_fmadd_ps(_k01, _r01, _sum0); - _sum0 = _mm256_fmadd_ps(_k02, _r02, _sum0); - _sum0 = _mm256_fmadd_ps(_k10, _r10, _sum0); - _sum0 = _mm256_fmadd_ps(_k11, _r11, _sum0); - _sum0 = _mm256_fmadd_ps(_k12, _r12, _sum0); - _sum0 = _mm256_fmadd_ps(_k20, _r20, _sum0); - _sum0 = _mm256_fmadd_ps(_k21, _r21, _sum0); - _sum0 = _mm256_fmadd_ps(_k22, _r22, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k00, _r00, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k01, _r01, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k02, _r02, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k10, _r10, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k11, _r11, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k12, _r12, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k20, _r20, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k21, _r21, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k22, _r22, _sum0); _mm256_storeu_ps(outptr0, _sum0); r0 += 2 * 8; r1 += 2 * 8; diff --git a/src/layer/x86/convolutiondepthwise_5x5_pack8.h b/src/layer/x86/convolutiondepthwise_5x5_pack8.h index bd6f373e6..36efa3267 100644 --- a/src/layer/x86/convolutiondepthwise_5x5_pack8.h +++ b/src/layer/x86/convolutiondepthwise_5x5_pack8.h @@ -61,11 +61,11 @@ static void convdw5x5s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const M __m256 _k04 = _mm256_loadu_ps(k0 + 32); k0 += 40; - _sum0 = _mm256_fmadd_ps(_k00, _r00, _sum0); - _sum0 = _mm256_fmadd_ps(_k01, _r01, _sum0); - _sum0 = _mm256_fmadd_ps(_k02, _r02, _sum0); - _sum0 = _mm256_fmadd_ps(_k03, _r03, _sum0); - _sum0 = _mm256_fmadd_ps(_k04, _r04, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k00, _r00, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k01, _r01, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k02, _r02, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k03, _r03, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k04, _r04, _sum0); __m256 _r10 = _mm256_loadu_ps(r1); __m256 _r11 = _mm256_loadu_ps(r1 + 8); @@ -80,11 +80,11 @@ static void convdw5x5s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const M __m256 _k14 = _mm256_loadu_ps(k0 + 32); k0 += 40; - _sum0 = _mm256_fmadd_ps(_k10, _r10, _sum0); - _sum0 = _mm256_fmadd_ps(_k11, _r11, _sum0); - _sum0 = _mm256_fmadd_ps(_k12, _r12, _sum0); - _sum0 = _mm256_fmadd_ps(_k13, _r13, _sum0); - _sum0 = _mm256_fmadd_ps(_k14, _r14, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k10, _r10, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k11, _r11, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k12, _r12, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k13, _r13, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k14, _r14, _sum0); __m256 _r20 = _mm256_loadu_ps(r2); __m256 _r21 = _mm256_loadu_ps(r2 + 8); @@ -99,11 +99,11 @@ static void convdw5x5s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const M __m256 _k24 = _mm256_loadu_ps(k0 + 32); k0 += 40; - _sum0 = _mm256_fmadd_ps(_k20, _r20, _sum0); - _sum0 = _mm256_fmadd_ps(_k21, _r21, _sum0); - _sum0 = _mm256_fmadd_ps(_k22, _r22, _sum0); - _sum0 = _mm256_fmadd_ps(_k23, _r23, _sum0); - _sum0 = _mm256_fmadd_ps(_k24, _r24, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k20, _r20, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k21, _r21, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k22, _r22, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k23, _r23, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k24, _r24, _sum0); __m256 _r30 = _mm256_loadu_ps(r3); __m256 _r31 = _mm256_loadu_ps(r3 + 8); @@ -118,11 +118,11 @@ static void convdw5x5s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const M __m256 _k34 = _mm256_loadu_ps(k0 + 32); k0 += 40; - _sum0 = _mm256_fmadd_ps(_k30, _r30, _sum0); - _sum0 = _mm256_fmadd_ps(_k31, _r31, _sum0); - _sum0 = _mm256_fmadd_ps(_k32, _r32, _sum0); - _sum0 = _mm256_fmadd_ps(_k33, _r33, _sum0); - _sum0 = _mm256_fmadd_ps(_k34, _r34, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k30, _r30, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k31, _r31, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k32, _r32, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k33, _r33, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k34, _r34, _sum0); __m256 _r40 = _mm256_loadu_ps(r4); __m256 _r41 = _mm256_loadu_ps(r4 + 8); @@ -137,11 +137,11 @@ static void convdw5x5s1_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const M __m256 _k44 = _mm256_loadu_ps(k0 + 32); k0 -= 160; - _sum0 = _mm256_fmadd_ps(_k40, _r40, _sum0); - _sum0 = _mm256_fmadd_ps(_k41, _r41, _sum0); - _sum0 = _mm256_fmadd_ps(_k42, _r42, _sum0); - _sum0 = _mm256_fmadd_ps(_k43, _r43, _sum0); - _sum0 = _mm256_fmadd_ps(_k44, _r44, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k40, _r40, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k41, _r41, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k42, _r42, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k43, _r43, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k44, _r44, _sum0); _mm256_storeu_ps(outptr0, _sum0); @@ -215,11 +215,11 @@ static void convdw5x5s2_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const M __m256 _k04 = _mm256_loadu_ps(k0 + 32); k0 += 40; - _sum0 = _mm256_fmadd_ps(_k00, _r00, _sum0); - _sum0 = _mm256_fmadd_ps(_k01, _r01, _sum0); - _sum0 = _mm256_fmadd_ps(_k02, _r02, _sum0); - _sum0 = _mm256_fmadd_ps(_k03, _r03, _sum0); - _sum0 = _mm256_fmadd_ps(_k04, _r04, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k00, _r00, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k01, _r01, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k02, _r02, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k03, _r03, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k04, _r04, _sum0); __m256 _r10 = _mm256_loadu_ps(r1); __m256 _r11 = _mm256_loadu_ps(r1 + 8); @@ -234,11 +234,11 @@ static void convdw5x5s2_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const M __m256 _k14 = _mm256_loadu_ps(k0 + 32); k0 += 40; - _sum0 = _mm256_fmadd_ps(_k10, _r10, _sum0); - _sum0 = _mm256_fmadd_ps(_k11, _r11, _sum0); - _sum0 = _mm256_fmadd_ps(_k12, _r12, _sum0); - _sum0 = _mm256_fmadd_ps(_k13, _r13, _sum0); - _sum0 = _mm256_fmadd_ps(_k14, _r14, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k10, _r10, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k11, _r11, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k12, _r12, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k13, _r13, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k14, _r14, _sum0); __m256 _r20 = _mm256_loadu_ps(r2); __m256 _r21 = _mm256_loadu_ps(r2 + 8); @@ -253,11 +253,11 @@ static void convdw5x5s2_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const M __m256 _k24 = _mm256_loadu_ps(k0 + 32); k0 += 40; - _sum0 = _mm256_fmadd_ps(_k20, _r20, _sum0); - _sum0 = _mm256_fmadd_ps(_k21, _r21, _sum0); - _sum0 = _mm256_fmadd_ps(_k22, _r22, _sum0); - _sum0 = _mm256_fmadd_ps(_k23, _r23, _sum0); - _sum0 = _mm256_fmadd_ps(_k24, _r24, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k20, _r20, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k21, _r21, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k22, _r22, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k23, _r23, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k24, _r24, _sum0); __m256 _r30 = _mm256_loadu_ps(r3); __m256 _r31 = _mm256_loadu_ps(r3 + 8); @@ -272,11 +272,11 @@ static void convdw5x5s2_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const M __m256 _k34 = _mm256_loadu_ps(k0 + 32); k0 += 40; - _sum0 = _mm256_fmadd_ps(_k30, _r30, _sum0); - _sum0 = _mm256_fmadd_ps(_k31, _r31, _sum0); - _sum0 = _mm256_fmadd_ps(_k32, _r32, _sum0); - _sum0 = _mm256_fmadd_ps(_k33, _r33, _sum0); - _sum0 = _mm256_fmadd_ps(_k34, _r34, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k30, _r30, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k31, _r31, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k32, _r32, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k33, _r33, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k34, _r34, _sum0); __m256 _r40 = _mm256_loadu_ps(r4); __m256 _r41 = _mm256_loadu_ps(r4 + 8); @@ -291,11 +291,11 @@ static void convdw5x5s2_pack8_avx(const Mat& bottom_blob, Mat& top_blob, const M __m256 _k44 = _mm256_loadu_ps(k0 + 32); k0 -= 160; - _sum0 = _mm256_fmadd_ps(_k40, _r40, _sum0); - _sum0 = _mm256_fmadd_ps(_k41, _r41, _sum0); - _sum0 = _mm256_fmadd_ps(_k42, _r42, _sum0); - _sum0 = _mm256_fmadd_ps(_k43, _r43, _sum0); - _sum0 = _mm256_fmadd_ps(_k44, _r44, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k40, _r40, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k41, _r41, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k42, _r42, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k43, _r43, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_k44, _r44, _sum0); _mm256_storeu_ps(outptr0, _sum0); diff --git a/src/layer/x86/convolutiondepthwise_x86.cpp b/src/layer/x86/convolutiondepthwise_x86.cpp index 4222763e3..733d51d1d 100644 --- a/src/layer/x86/convolutiondepthwise_x86.cpp +++ b/src/layer/x86/convolutiondepthwise_x86.cpp @@ -30,7 +30,9 @@ namespace ncnn { #if __SSE2__ #if __AVX__ +#if __AVX2__ #include "convolutiondepthwise_3x3_pack8_fp16.h" +#endif #include "convolutiondepthwise_3x3_pack8.h" #include "convolutiondepthwise_5x5_pack8.h" #endif @@ -44,7 +46,7 @@ ConvolutionDepthWise_x86::ConvolutionDepthWise_x86() { #if __SSE2__ support_packing = true; -#if __AVX__ +#if __AVX2__ support_weight_fp16_storage = true; #endif #endif // __SSE2__ @@ -134,6 +136,7 @@ int ConvolutionDepthWise_x86::create_pipeline(const Option& opt) // pack8 if (elempack == 8) { +#if __AVX2__ if (opt.use_weight_fp16_storage && kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { Mat weight_data_r2 = weight_data.reshape(maxk, group); @@ -150,7 +153,7 @@ int ConvolutionDepthWise_x86::create_pipeline(const Option& opt) ncnn::cast_float32_to_float16(weight_data_tmp, weight_data_packed, opt); return 0; } - +#endif Mat weight_data_r2 = weight_data.reshape(maxk, group); convert_packing(weight_data_r2, weight_data_packed, 8); @@ -360,11 +363,14 @@ int ConvolutionDepthWise_x86::forward(const Mat& bottom_blob, Mat& top_blob, con { if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 1 && stride_h == 1) { +#if __AVX2__ + if (opt.use_weight_fp16_storage) { convdw3x3s1_fp16_pack8_avx(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt); } - else +#endif + if (!opt.use_weight_fp16_storage) { convdw3x3s1_pack8_avx(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt); } @@ -378,11 +384,13 @@ int ConvolutionDepthWise_x86::forward(const Mat& bottom_blob, Mat& top_blob, con } if (kernel_w == 3 && kernel_h == 3 && dilation_w == 1 && dilation_h == 1 && stride_w == 2 && stride_h == 2) { +#if __AVX2__ if (opt.use_weight_fp16_storage) { convdw3x3s2_fp16_pack8_avx(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt); } - else +#endif + if (!opt.use_weight_fp16_storage) { convdw3x3s2_pack8_avx(bottom_blob_bordered, top_blob, weight_data_packed, bias_data, opt); } @@ -463,7 +471,7 @@ int ConvolutionDepthWise_x86::forward(const Mat& bottom_blob, Mat& top_blob, con { __m256 _val = _mm256_loadu_ps(sptr + space_ofs[k] * 8); __m256 _w = _mm256_loadu_ps(kptr + k * 8); - _sum = _mm256_fmadd_ps(_val, _w, _sum); + _sum = _mm256_comp_fmadd_ps(_val, _w, _sum); } _sum = activation_avx(_sum, activation_type, activation_params); diff --git a/src/layer/x86/dequantize_x86.cpp b/src/layer/x86/dequantize_x86.cpp index 0d31dddd5..16906481b 100644 --- a/src/layer/x86/dequantize_x86.cpp +++ b/src/layer/x86/dequantize_x86.cpp @@ -21,6 +21,8 @@ #endif // __AVX__ #endif // __SSE2__ +#include "x86_usability.h" + namespace ncnn { Dequantize_x86::Dequantize_x86() @@ -75,7 +77,7 @@ int Dequantize_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& float* ptr = (float*)top_blob + i * 8; __m256 _v = _mm256_cvtepi32_ps(_mm256_loadu_si256((const __m256i*)intptr)); - _v = _mm256_fmadd_ps(_v, _scale, _bias); + _v = _mm256_comp_fmadd_ps(_v, _scale, _bias); _mm256_storeu_ps(ptr, _v); } } @@ -89,7 +91,7 @@ int Dequantize_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& __m256 _bias = _mm256_loadu_ps((const float*)bias_data + i * 8); __m256 _v = _mm256_cvtepi32_ps(_mm256_loadu_si256((const __m256i*)intptr)); - _v = _mm256_fmadd_ps(_v, _scale, _bias); + _v = _mm256_comp_fmadd_ps(_v, _scale, _bias); _mm256_storeu_ps(ptr, _v); } } @@ -122,7 +124,7 @@ int Dequantize_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& __m256 _scale = _mm256_loadu_ps((const float*)scale_data + i * 8); __m256 _v = _mm256_cvtepi32_ps(_mm256_loadu_si256((const __m256i*)intptr)); - _v = _mm256_fmadd_ps(_v, _scale, _bias); + _v = _mm256_comp_fmadd_ps(_v, _scale, _bias); _mm256_storeu_ps(ptr, _v); } } @@ -137,7 +139,7 @@ int Dequantize_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& __m256 _scale = _mm256_loadu_ps((const float*)scale_data + i * 8); __m256 _bias = _mm256_loadu_ps((const float*)bias_data + i * 8); __m256 _v = _mm256_cvtepi32_ps(_mm256_loadu_si256((const __m256i*)intptr)); - _v = _mm256_fmadd_ps(_v, _scale, _bias); + _v = _mm256_comp_fmadd_ps(_v, _scale, _bias); _mm256_storeu_ps(ptr, _v); } } @@ -188,7 +190,7 @@ int Dequantize_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& for (int j = 0; j < w; j++) { __m256 _v = _mm256_cvtepi32_ps(_mm256_loadu_si256((const __m256i*)intptr)); - _v = _mm256_fmadd_ps(_v, _scale, _bias); + _v = _mm256_comp_fmadd_ps(_v, _scale, _bias); _mm256_storeu_ps(ptr, _v); intptr += 8; @@ -244,7 +246,7 @@ int Dequantize_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& for (int i = 0; i < size; i++) { __m256 _v = _mm256_cvtepi32_ps(_mm256_loadu_si256((const __m256i*)intptr)); - _v = _mm256_fmadd_ps(_v, _scale, _bias); + _v = _mm256_comp_fmadd_ps(_v, _scale, _bias); _mm256_storeu_ps(ptr, _v); intptr += 8; diff --git a/src/layer/x86/eltwise_x86.cpp b/src/layer/x86/eltwise_x86.cpp index c8df8d435..5f6e6d6c5 100644 --- a/src/layer/x86/eltwise_x86.cpp +++ b/src/layer/x86/eltwise_x86.cpp @@ -20,6 +20,7 @@ #include #endif // __AVX__ #endif // __SSE2__ +#include "x86_usability.h" namespace ncnn { @@ -160,7 +161,7 @@ int Eltwise_x86::forward(const std::vector& bottom_blobs, std::vector& __m256 _p = _mm256_loadu_ps(ptr); __m256 _p1 = _mm256_loadu_ps(ptr1); _p = _mm256_mul_ps(_p, _coeff0); - _p = _mm256_fmadd_ps(_p1, _coeff1, _p); + _p = _mm256_comp_fmadd_ps(_p1, _coeff1, _p); _mm256_storeu_ps(outptr, _p); ptr += 8; @@ -183,7 +184,7 @@ int Eltwise_x86::forward(const std::vector& bottom_blobs, std::vector& { __m256 _p = _mm256_loadu_ps(outptr); __m256 _p1 = _mm256_loadu_ps(ptr); - _p = _mm256_fmadd_ps(_p1, _coeff, _p); + _p = _mm256_comp_fmadd_ps(_p1, _coeff, _p); _mm256_storeu_ps(outptr, _p); ptr += 8; diff --git a/src/layer/x86/hardsigmoid_x86.cpp b/src/layer/x86/hardsigmoid_x86.cpp index d4fc5732e..95e6e8a2e 100644 --- a/src/layer/x86/hardsigmoid_x86.cpp +++ b/src/layer/x86/hardsigmoid_x86.cpp @@ -21,6 +21,8 @@ #endif // __AVX__ #endif // __SSE2__ +#include "x86_usability.h" + namespace ncnn { HardSigmoid_x86::HardSigmoid_x86() @@ -53,7 +55,7 @@ int HardSigmoid_x86::forward_inplace(Mat& bottom_top_blob, const Option& opt) co { __m256 _p = _mm256_loadu_ps(ptr); __m256 _ans = _mm256_set1_ps(beta); - _ans = _mm256_fmadd_ps(_p, _mm256_set1_ps(alpha), _ans); + _ans = _mm256_comp_fmadd_ps(_p, _mm256_set1_ps(alpha), _ans); _ans = _mm256_max_ps(_ans, _zero); _ans = _mm256_min_ps(_ans, _one); _mm256_storeu_ps(ptr, _ans); @@ -105,7 +107,7 @@ int HardSigmoid_x86::forward_inplace(Mat& bottom_top_blob, const Option& opt) co { __m256 _p = _mm256_loadu_ps(ptr); __m256 _ans = _mm256_set1_ps(beta); - _ans = _mm256_fmadd_ps(_p, _mm256_set1_ps(alpha), _ans); + _ans = _mm256_comp_fmadd_ps(_p, _mm256_set1_ps(alpha), _ans); _ans = _mm256_max_ps(_ans, _zero); _ans = _mm256_min_ps(_ans, _one); _mm256_storeu_ps(ptr, _ans); diff --git a/src/layer/x86/hardswish_x86.cpp b/src/layer/x86/hardswish_x86.cpp index ff4e0f5a1..c1bfba27e 100644 --- a/src/layer/x86/hardswish_x86.cpp +++ b/src/layer/x86/hardswish_x86.cpp @@ -21,6 +21,8 @@ #endif // __AVX__ #endif // __SSE2__ +#include "x86_usability.h" + namespace ncnn { HardSwish_x86::HardSwish_x86() @@ -53,7 +55,7 @@ int HardSwish_x86::forward_inplace(Mat& bottom_top_blob, const Option& opt) cons { __m256 _p = _mm256_loadu_ps(ptr); __m256 _ans = _mm256_set1_ps(beta); - _ans = _mm256_fmadd_ps(_p, _mm256_set1_ps(alpha), _ans); + _ans = _mm256_comp_fmadd_ps(_p, _mm256_set1_ps(alpha), _ans); _ans = _mm256_max_ps(_ans, _zero); _ans = _mm256_min_ps(_ans, _one); _ans = _mm256_mul_ps(_ans, _p); @@ -107,7 +109,7 @@ int HardSwish_x86::forward_inplace(Mat& bottom_top_blob, const Option& opt) cons { __m256 _p = _mm256_loadu_ps(ptr); __m256 _ans = _mm256_set1_ps(beta); - _ans = _mm256_fmadd_ps(_p, _mm256_set1_ps(alpha), _ans); + _ans = _mm256_comp_fmadd_ps(_p, _mm256_set1_ps(alpha), _ans); _ans = _mm256_max_ps(_ans, _zero); _ans = _mm256_min_ps(_ans, _one); _ans = _mm256_mul_ps(_ans, _p); diff --git a/src/layer/x86/innerproduct_x86.cpp b/src/layer/x86/innerproduct_x86.cpp index afb353c13..aae92a3cd 100644 --- a/src/layer/x86/innerproduct_x86.cpp +++ b/src/layer/x86/innerproduct_x86.cpp @@ -32,7 +32,7 @@ InnerProduct_x86::InnerProduct_x86() { #if __SSE2__ support_packing = true; -#if __AVX__ +#if __AVX2__ support_weight_fp16_storage = true; #endif #endif // __SSE2__ @@ -100,7 +100,7 @@ int InnerProduct_x86::create_pipeline(const Option& opt) } } -#if __AVX__ +#if __AVX2__ if (opt.use_weight_fp16_storage && weight_data.elemsize == 4u) { ncnn::cast_float32_to_float16(weight_data, weight_data_fp16, opt); @@ -204,14 +204,14 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio __m256 _k5 = _mm256_set1_ps(kptr[5]); __m256 _k6 = _mm256_set1_ps(kptr[6]); __m256 _k7 = _mm256_set1_ps(kptr[7]); - _sum0 = _mm256_fmadd_ps(_val, _k0, _sum0); - _sum1 = _mm256_fmadd_ps(_val, _k1, _sum1); - _sum2 = _mm256_fmadd_ps(_val, _k2, _sum2); - _sum3 = _mm256_fmadd_ps(_val, _k3, _sum3); - _sum4 = _mm256_fmadd_ps(_val, _k4, _sum4); - _sum5 = _mm256_fmadd_ps(_val, _k5, _sum5); - _sum6 = _mm256_fmadd_ps(_val, _k6, _sum6); - _sum7 = _mm256_fmadd_ps(_val, _k7, _sum7); + _sum0 = _mm256_comp_fmadd_ps(_val, _k0, _sum0); + _sum1 = _mm256_comp_fmadd_ps(_val, _k1, _sum1); + _sum2 = _mm256_comp_fmadd_ps(_val, _k2, _sum2); + _sum3 = _mm256_comp_fmadd_ps(_val, _k3, _sum3); + _sum4 = _mm256_comp_fmadd_ps(_val, _k4, _sum4); + _sum5 = _mm256_comp_fmadd_ps(_val, _k5, _sum5); + _sum6 = _mm256_comp_fmadd_ps(_val, _k6, _sum6); + _sum7 = _mm256_comp_fmadd_ps(_val, _k7, _sum7); m += 8; kptr += 8; @@ -267,21 +267,21 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio __m256 _val7 = _mm256_broadcast_ss(m + 7); __m256 _w0 = _mm256_loadu_ps(kptr); - _sum = _mm256_fmadd_ps(_val0, _w0, _sum); + _sum = _mm256_comp_fmadd_ps(_val0, _w0, _sum); __m256 _w1 = _mm256_loadu_ps(kptr + 8); - _sum = _mm256_fmadd_ps(_val1, _w1, _sum); + _sum = _mm256_comp_fmadd_ps(_val1, _w1, _sum); __m256 _w2 = _mm256_loadu_ps(kptr + 16); - _sum = _mm256_fmadd_ps(_val2, _w2, _sum); + _sum = _mm256_comp_fmadd_ps(_val2, _w2, _sum); __m256 _w3 = _mm256_loadu_ps(kptr + 24); - _sum = _mm256_fmadd_ps(_val3, _w3, _sum); + _sum = _mm256_comp_fmadd_ps(_val3, _w3, _sum); __m256 _w4 = _mm256_loadu_ps(kptr + 32); - _sum = _mm256_fmadd_ps(_val4, _w4, _sum); + _sum = _mm256_comp_fmadd_ps(_val4, _w4, _sum); __m256 _w5 = _mm256_loadu_ps(kptr + 40); - _sum = _mm256_fmadd_ps(_val5, _w5, _sum); + _sum = _mm256_comp_fmadd_ps(_val5, _w5, _sum); __m256 _w6 = _mm256_loadu_ps(kptr + 48); - _sum = _mm256_fmadd_ps(_val6, _w6, _sum); + _sum = _mm256_comp_fmadd_ps(_val6, _w6, _sum); __m256 _w7 = _mm256_loadu_ps(kptr + 56); - _sum = _mm256_fmadd_ps(_val7, _w7, _sum); + _sum = _mm256_comp_fmadd_ps(_val7, _w7, _sum); m += 8; kptr += 64; @@ -294,13 +294,13 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio __m256 _val3 = _mm256_broadcast_ss(m + 3); __m256 _w0 = _mm256_loadu_ps(kptr); - _sum = _mm256_fmadd_ps(_val0, _w0, _sum); + _sum = _mm256_comp_fmadd_ps(_val0, _w0, _sum); __m256 _w1 = _mm256_loadu_ps(kptr + 8); - _sum = _mm256_fmadd_ps(_val1, _w1, _sum); + _sum = _mm256_comp_fmadd_ps(_val1, _w1, _sum); __m256 _w2 = _mm256_loadu_ps(kptr + 16); - _sum = _mm256_fmadd_ps(_val2, _w2, _sum); + _sum = _mm256_comp_fmadd_ps(_val2, _w2, _sum); __m256 _w3 = _mm256_loadu_ps(kptr + 24); - _sum = _mm256_fmadd_ps(_val3, _w3, _sum); + _sum = _mm256_comp_fmadd_ps(_val3, _w3, _sum); m += 4; kptr += 32; @@ -309,7 +309,7 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio { __m256 _val = _mm256_set1_ps(m[0]); __m256 _w = _mm256_loadu_ps(kptr); - _sum = _mm256_fmadd_ps(_val, _w, _sum); + _sum = _mm256_comp_fmadd_ps(_val, _w, _sum); m += 1; kptr += 8; @@ -356,14 +356,14 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio for (; i < num_input; i++) { __m128 _val = _mm_loadu_ps(m); - _sum0 = _mm_fmadd_ps(_val, _mm_set1_ps(kptr[0]), _sum0); - _sum1 = _mm_fmadd_ps(_val, _mm_set1_ps(kptr[1]), _sum1); - _sum2 = _mm_fmadd_ps(_val, _mm_set1_ps(kptr[2]), _sum2); - _sum3 = _mm_fmadd_ps(_val, _mm_set1_ps(kptr[3]), _sum3); - _sum4 = _mm_fmadd_ps(_val, _mm_set1_ps(kptr[4]), _sum4); - _sum5 = _mm_fmadd_ps(_val, _mm_set1_ps(kptr[5]), _sum5); - _sum6 = _mm_fmadd_ps(_val, _mm_set1_ps(kptr[6]), _sum6); - _sum7 = _mm_fmadd_ps(_val, _mm_set1_ps(kptr[7]), _sum7); + _sum0 = _mm_comp_fmadd_ps(_val, _mm_set1_ps(kptr[0]), _sum0); + _sum1 = _mm_comp_fmadd_ps(_val, _mm_set1_ps(kptr[1]), _sum1); + _sum2 = _mm_comp_fmadd_ps(_val, _mm_set1_ps(kptr[2]), _sum2); + _sum3 = _mm_comp_fmadd_ps(_val, _mm_set1_ps(kptr[3]), _sum3); + _sum4 = _mm_comp_fmadd_ps(_val, _mm_set1_ps(kptr[4]), _sum4); + _sum5 = _mm_comp_fmadd_ps(_val, _mm_set1_ps(kptr[5]), _sum5); + _sum6 = _mm_comp_fmadd_ps(_val, _mm_set1_ps(kptr[6]), _sum6); + _sum7 = _mm_comp_fmadd_ps(_val, _mm_set1_ps(kptr[7]), _sum7); m += 4; kptr += 8; @@ -420,14 +420,14 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio __m256 _val5 = _mm256_loadu_ps(m + 40); __m256 _val6 = _mm256_loadu_ps(m + 48); __m256 _val7 = _mm256_loadu_ps(m + 56); - _sum0 = _mm256_fmadd_ps(_val0, _mm256_set1_ps(kptr[0]), _sum0); - _sum1 = _mm256_fmadd_ps(_val1, _mm256_set1_ps(kptr[1]), _sum1); - _sum2 = _mm256_fmadd_ps(_val2, _mm256_set1_ps(kptr[2]), _sum2); - _sum3 = _mm256_fmadd_ps(_val3, _mm256_set1_ps(kptr[3]), _sum3); - _sum0 = _mm256_fmadd_ps(_val4, _mm256_set1_ps(kptr[4]), _sum0); - _sum1 = _mm256_fmadd_ps(_val5, _mm256_set1_ps(kptr[5]), _sum1); - _sum2 = _mm256_fmadd_ps(_val6, _mm256_set1_ps(kptr[6]), _sum2); - _sum3 = _mm256_fmadd_ps(_val7, _mm256_set1_ps(kptr[7]), _sum3); + _sum0 = _mm256_comp_fmadd_ps(_val0, _mm256_set1_ps(kptr[0]), _sum0); + _sum1 = _mm256_comp_fmadd_ps(_val1, _mm256_set1_ps(kptr[1]), _sum1); + _sum2 = _mm256_comp_fmadd_ps(_val2, _mm256_set1_ps(kptr[2]), _sum2); + _sum3 = _mm256_comp_fmadd_ps(_val3, _mm256_set1_ps(kptr[3]), _sum3); + _sum0 = _mm256_comp_fmadd_ps(_val4, _mm256_set1_ps(kptr[4]), _sum0); + _sum1 = _mm256_comp_fmadd_ps(_val5, _mm256_set1_ps(kptr[5]), _sum1); + _sum2 = _mm256_comp_fmadd_ps(_val6, _mm256_set1_ps(kptr[6]), _sum2); + _sum3 = _mm256_comp_fmadd_ps(_val7, _mm256_set1_ps(kptr[7]), _sum3); m += 64; kptr += 8; @@ -438,10 +438,10 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio __m256 _val1 = _mm256_loadu_ps(m + 8); __m256 _val2 = _mm256_loadu_ps(m + 16); __m256 _val3 = _mm256_loadu_ps(m + 24); - _sum0 = _mm256_fmadd_ps(_val0, _mm256_set1_ps(kptr[0]), _sum0); - _sum1 = _mm256_fmadd_ps(_val1, _mm256_set1_ps(kptr[1]), _sum1); - _sum2 = _mm256_fmadd_ps(_val2, _mm256_set1_ps(kptr[2]), _sum2); - _sum3 = _mm256_fmadd_ps(_val3, _mm256_set1_ps(kptr[3]), _sum3); + _sum0 = _mm256_comp_fmadd_ps(_val0, _mm256_set1_ps(kptr[0]), _sum0); + _sum1 = _mm256_comp_fmadd_ps(_val1, _mm256_set1_ps(kptr[1]), _sum1); + _sum2 = _mm256_comp_fmadd_ps(_val2, _mm256_set1_ps(kptr[2]), _sum2); + _sum3 = _mm256_comp_fmadd_ps(_val3, _mm256_set1_ps(kptr[3]), _sum3); m += 32; kptr += 4; @@ -450,7 +450,7 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio { __m256 _val = _mm256_loadu_ps(m); __m256 _k = _mm256_set1_ps(kptr[0]); - _sum0 = _mm256_fmadd_ps(_val, _k, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_val, _k, _sum0); m += 8; kptr += 1; @@ -496,24 +496,24 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio __m256 _val1 = _mm256_loadu_ps(m + 8); __m256 _val2 = _mm256_loadu_ps(m + 16); __m256 _val3 = _mm256_loadu_ps(m + 24); - _sum0 = _mm256_fmadd_ps(_val0, _mm256_set1_ps(kptr[0]), _sum0); - _sum1 = _mm256_fmadd_ps(_val0, _mm256_set1_ps(kptr[1]), _sum1); - _sum2 = _mm256_fmadd_ps(_val0, _mm256_set1_ps(kptr[2]), _sum2); - _sum3 = _mm256_fmadd_ps(_val0, _mm256_set1_ps(kptr[3]), _sum3); - _sum0 = _mm256_fmadd_ps(_val1, _mm256_set1_ps(kptr[4]), _sum0); - _sum1 = _mm256_fmadd_ps(_val1, _mm256_set1_ps(kptr[5]), _sum1); - _sum2 = _mm256_fmadd_ps(_val1, _mm256_set1_ps(kptr[6]), _sum2); - _sum3 = _mm256_fmadd_ps(_val1, _mm256_set1_ps(kptr[7]), _sum3); + _sum0 = _mm256_comp_fmadd_ps(_val0, _mm256_set1_ps(kptr[0]), _sum0); + _sum1 = _mm256_comp_fmadd_ps(_val0, _mm256_set1_ps(kptr[1]), _sum1); + _sum2 = _mm256_comp_fmadd_ps(_val0, _mm256_set1_ps(kptr[2]), _sum2); + _sum3 = _mm256_comp_fmadd_ps(_val0, _mm256_set1_ps(kptr[3]), _sum3); + _sum0 = _mm256_comp_fmadd_ps(_val1, _mm256_set1_ps(kptr[4]), _sum0); + _sum1 = _mm256_comp_fmadd_ps(_val1, _mm256_set1_ps(kptr[5]), _sum1); + _sum2 = _mm256_comp_fmadd_ps(_val1, _mm256_set1_ps(kptr[6]), _sum2); + _sum3 = _mm256_comp_fmadd_ps(_val1, _mm256_set1_ps(kptr[7]), _sum3); kptr += 8; - _sum0 = _mm256_fmadd_ps(_val2, _mm256_set1_ps(kptr[0]), _sum0); - _sum1 = _mm256_fmadd_ps(_val2, _mm256_set1_ps(kptr[1]), _sum1); - _sum2 = _mm256_fmadd_ps(_val2, _mm256_set1_ps(kptr[2]), _sum2); - _sum3 = _mm256_fmadd_ps(_val2, _mm256_set1_ps(kptr[3]), _sum3); - _sum0 = _mm256_fmadd_ps(_val3, _mm256_set1_ps(kptr[4]), _sum0); - _sum1 = _mm256_fmadd_ps(_val3, _mm256_set1_ps(kptr[5]), _sum1); - _sum2 = _mm256_fmadd_ps(_val3, _mm256_set1_ps(kptr[6]), _sum2); - _sum3 = _mm256_fmadd_ps(_val3, _mm256_set1_ps(kptr[7]), _sum3); + _sum0 = _mm256_comp_fmadd_ps(_val2, _mm256_set1_ps(kptr[0]), _sum0); + _sum1 = _mm256_comp_fmadd_ps(_val2, _mm256_set1_ps(kptr[1]), _sum1); + _sum2 = _mm256_comp_fmadd_ps(_val2, _mm256_set1_ps(kptr[2]), _sum2); + _sum3 = _mm256_comp_fmadd_ps(_val2, _mm256_set1_ps(kptr[3]), _sum3); + _sum0 = _mm256_comp_fmadd_ps(_val3, _mm256_set1_ps(kptr[4]), _sum0); + _sum1 = _mm256_comp_fmadd_ps(_val3, _mm256_set1_ps(kptr[5]), _sum1); + _sum2 = _mm256_comp_fmadd_ps(_val3, _mm256_set1_ps(kptr[6]), _sum2); + _sum3 = _mm256_comp_fmadd_ps(_val3, _mm256_set1_ps(kptr[7]), _sum3); m += 32; kptr += 8; @@ -521,10 +521,10 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio for (; i < num_input; i++) { __m256 _val = _mm256_loadu_ps(m); - _sum0 = _mm256_fmadd_ps(_val, _mm256_set1_ps(kptr[0]), _sum0); - _sum1 = _mm256_fmadd_ps(_val, _mm256_set1_ps(kptr[1]), _sum1); - _sum2 = _mm256_fmadd_ps(_val, _mm256_set1_ps(kptr[2]), _sum2); - _sum3 = _mm256_fmadd_ps(_val, _mm256_set1_ps(kptr[3]), _sum3); + _sum0 = _mm256_comp_fmadd_ps(_val, _mm256_set1_ps(kptr[0]), _sum0); + _sum1 = _mm256_comp_fmadd_ps(_val, _mm256_set1_ps(kptr[1]), _sum1); + _sum2 = _mm256_comp_fmadd_ps(_val, _mm256_set1_ps(kptr[2]), _sum2); + _sum3 = _mm256_comp_fmadd_ps(_val, _mm256_set1_ps(kptr[3]), _sum3); m += 8; kptr += 4; @@ -648,21 +648,21 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio __m128 _val7 = _mm_broadcast_ss(m + 7); __m128 _w0 = _mm_loadu_ps(kptr); - _sum = _mm_fmadd_ps(_val0, _w0, _sum); + _sum = _mm_comp_fmadd_ps(_val0, _w0, _sum); __m128 _w1 = _mm_loadu_ps(kptr + 4); - _sum = _mm_fmadd_ps(_val1, _w1, _sum); + _sum = _mm_comp_fmadd_ps(_val1, _w1, _sum); __m128 _w2 = _mm_loadu_ps(kptr + 8); - _sum = _mm_fmadd_ps(_val2, _w2, _sum); + _sum = _mm_comp_fmadd_ps(_val2, _w2, _sum); __m128 _w3 = _mm_loadu_ps(kptr + 12); - _sum = _mm_fmadd_ps(_val3, _w3, _sum); + _sum = _mm_comp_fmadd_ps(_val3, _w3, _sum); __m128 _w4 = _mm_loadu_ps(kptr + 16); - _sum = _mm_fmadd_ps(_val4, _w4, _sum); + _sum = _mm_comp_fmadd_ps(_val4, _w4, _sum); __m128 _w5 = _mm_loadu_ps(kptr + 20); - _sum = _mm_fmadd_ps(_val5, _w5, _sum); + _sum = _mm_comp_fmadd_ps(_val5, _w5, _sum); __m128 _w6 = _mm_loadu_ps(kptr + 24); - _sum = _mm_fmadd_ps(_val6, _w6, _sum); + _sum = _mm_comp_fmadd_ps(_val6, _w6, _sum); __m128 _w7 = _mm_loadu_ps(kptr + 28); - _sum = _mm_fmadd_ps(_val7, _w7, _sum); + _sum = _mm_comp_fmadd_ps(_val7, _w7, _sum); m += 8; kptr += 32; @@ -806,7 +806,7 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio { __m256 _m = _mm256_loadu_ps(m); __m256 _w = _mm256_loadu_ps(kptr); - _sum = _mm256_fmadd_ps(_m, _w, _sum); + _sum = _mm256_comp_fmadd_ps(_m, _w, _sum); m += 8; kptr += 8; @@ -846,12 +846,12 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio return 0; } -#if __AVX__ +#if __AVX2__ if (opt.use_weight_fp16_storage) { return forward_fp16(bottom_blob, top_blob, opt); } -#endif // __AVX__ +#endif // __AVX2__ // flatten Mat bottom_blob_flattened = bottom_blob; @@ -922,21 +922,21 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio __m256 _val7 = _mm256_broadcast_ss(sptr + 7); __m256 _w0 = _mm256_loadu_ps(kptr); - _sum0 = _mm256_fmadd_ps(_val0, _w0, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_val0, _w0, _sum0); __m256 _w1 = _mm256_loadu_ps(kptr + 8); - _sum1 = _mm256_fmadd_ps(_val1, _w1, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_val1, _w1, _sum1); __m256 _w2 = _mm256_loadu_ps(kptr + 16); - _sum2 = _mm256_fmadd_ps(_val2, _w2, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_val2, _w2, _sum2); __m256 _w3 = _mm256_loadu_ps(kptr + 24); - _sum3 = _mm256_fmadd_ps(_val3, _w3, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_val3, _w3, _sum3); __m256 _w4 = _mm256_loadu_ps(kptr + 32); - _sum4 = _mm256_fmadd_ps(_val4, _w4, _sum4); + _sum4 = _mm256_comp_fmadd_ps(_val4, _w4, _sum4); __m256 _w5 = _mm256_loadu_ps(kptr + 40); - _sum5 = _mm256_fmadd_ps(_val5, _w5, _sum5); + _sum5 = _mm256_comp_fmadd_ps(_val5, _w5, _sum5); __m256 _w6 = _mm256_loadu_ps(kptr + 48); - _sum6 = _mm256_fmadd_ps(_val6, _w6, _sum6); + _sum6 = _mm256_comp_fmadd_ps(_val6, _w6, _sum6); __m256 _w7 = _mm256_loadu_ps(kptr + 56); - _sum7 = _mm256_fmadd_ps(_val7, _w7, _sum7); + _sum7 = _mm256_comp_fmadd_ps(_val7, _w7, _sum7); sptr += 8; kptr += 64; @@ -949,13 +949,13 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio __m256 _val3 = _mm256_broadcast_ss(sptr + 3); __m256 _w0 = _mm256_loadu_ps(kptr); - _sum0 = _mm256_fmadd_ps(_val0, _w0, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_val0, _w0, _sum0); __m256 _w1 = _mm256_loadu_ps(kptr + 8); - _sum1 = _mm256_fmadd_ps(_val1, _w1, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_val1, _w1, _sum1); __m256 _w2 = _mm256_loadu_ps(kptr + 16); - _sum2 = _mm256_fmadd_ps(_val2, _w2, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_val2, _w2, _sum2); __m256 _w3 = _mm256_loadu_ps(kptr + 24); - _sum3 = _mm256_fmadd_ps(_val3, _w3, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_val3, _w3, _sum3); sptr += 4; kptr += 32; @@ -964,7 +964,7 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio { __m256 _val = _mm256_set1_ps(sptr[0]); __m256 _w = _mm256_loadu_ps(kptr); - _sum0 = _mm256_fmadd_ps(_val, _w, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_val, _w, _sum0); sptr += 1; kptr += 8; @@ -1026,21 +1026,21 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio __m128 _val7 = _mm_broadcast_ss(sptr + 7); __m128 _w0 = _mm_loadu_ps(kptr); - _sum0 = _mm_fmadd_ps(_val0, _w0, _sum0); + _sum0 = _mm_comp_fmadd_ps(_val0, _w0, _sum0); __m128 _w1 = _mm_loadu_ps(kptr + 4); - _sum1 = _mm_fmadd_ps(_val1, _w1, _sum1); + _sum1 = _mm_comp_fmadd_ps(_val1, _w1, _sum1); __m128 _w2 = _mm_loadu_ps(kptr + 8); - _sum2 = _mm_fmadd_ps(_val2, _w2, _sum2); + _sum2 = _mm_comp_fmadd_ps(_val2, _w2, _sum2); __m128 _w3 = _mm_loadu_ps(kptr + 12); - _sum3 = _mm_fmadd_ps(_val3, _w3, _sum3); + _sum3 = _mm_comp_fmadd_ps(_val3, _w3, _sum3); __m128 _w4 = _mm_loadu_ps(kptr + 16); - _sum4 = _mm_fmadd_ps(_val4, _w4, _sum4); + _sum4 = _mm_comp_fmadd_ps(_val4, _w4, _sum4); __m128 _w5 = _mm_loadu_ps(kptr + 20); - _sum5 = _mm_fmadd_ps(_val5, _w5, _sum5); + _sum5 = _mm_comp_fmadd_ps(_val5, _w5, _sum5); __m128 _w6 = _mm_loadu_ps(kptr + 24); - _sum6 = _mm_fmadd_ps(_val6, _w6, _sum6); + _sum6 = _mm_comp_fmadd_ps(_val6, _w6, _sum6); __m128 _w7 = _mm_loadu_ps(kptr + 28); - _sum7 = _mm_fmadd_ps(_val7, _w7, _sum7); + _sum7 = _mm_comp_fmadd_ps(_val7, _w7, _sum7); sptr += 8; kptr += 32; @@ -1146,21 +1146,21 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio __m256 _m = _mm256_loadu_ps(m); __m256 _w0 = _mm256_loadu_ps(w0); - _sum0 = _mm256_fmadd_ps(_m, _w0, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_m, _w0, _sum0); __m256 _w1 = _mm256_loadu_ps(w1); - _sum1 = _mm256_fmadd_ps(_m, _w1, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_m, _w1, _sum1); __m256 _w2 = _mm256_loadu_ps(w2); - _sum2 = _mm256_fmadd_ps(_m, _w2, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_m, _w2, _sum2); __m256 _w3 = _mm256_loadu_ps(w3); - _sum3 = _mm256_fmadd_ps(_m, _w3, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_m, _w3, _sum3); __m256 _w4 = _mm256_loadu_ps(w4); - _sum4 = _mm256_fmadd_ps(_m, _w4, _sum4); + _sum4 = _mm256_comp_fmadd_ps(_m, _w4, _sum4); __m256 _w5 = _mm256_loadu_ps(w5); - _sum5 = _mm256_fmadd_ps(_m, _w5, _sum5); + _sum5 = _mm256_comp_fmadd_ps(_m, _w5, _sum5); __m256 _w6 = _mm256_loadu_ps(w6); - _sum6 = _mm256_fmadd_ps(_m, _w6, _sum6); + _sum6 = _mm256_comp_fmadd_ps(_m, _w6, _sum6); __m256 _w7 = _mm256_loadu_ps(w7); - _sum7 = _mm256_fmadd_ps(_m, _w7, _sum7); + _sum7 = _mm256_comp_fmadd_ps(_m, _w7, _sum7); m += 8; w0 += 8; @@ -1242,13 +1242,13 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio __m256 _m = _mm256_loadu_ps(m); __m256 _w0 = _mm256_loadu_ps(w0); - _sum0 = _mm256_fmadd_ps(_m, _w0, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_m, _w0, _sum0); __m256 _w1 = _mm256_loadu_ps(w1); - _sum1 = _mm256_fmadd_ps(_m, _w1, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_m, _w1, _sum1); __m256 _w2 = _mm256_loadu_ps(w2); - _sum2 = _mm256_fmadd_ps(_m, _w2, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_m, _w2, _sum2); __m256 _w3 = _mm256_loadu_ps(w3); - _sum3 = _mm256_fmadd_ps(_m, _w3, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_m, _w3, _sum3); m += 8; w0 += 8; @@ -1336,7 +1336,7 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio __m256 _m = _mm256_loadu_ps(m); __m256 _w = _mm256_loadu_ps(w); - _sum = _mm256_fmadd_ps(_m, _w, _sum); + _sum = _mm256_comp_fmadd_ps(_m, _w, _sum); m += 8; w += 8; @@ -1377,7 +1377,7 @@ int InnerProduct_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Optio return 0; } -#if __AVX__ +#if __AVX2__ int InnerProduct_x86::forward_fp16(const Mat& bottom_blob, Mat& top_blob, const Option& opt) const { @@ -1456,28 +1456,28 @@ int InnerProduct_x86::forward_fp16(const Mat& bottom_blob, Mat& top_blob, const __m256 _m = _mm256_loadu_ps(m); __m256 _w0 = loadfp16(w0); - _sum0 = _mm256_fmadd_ps(_m, _w0, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_m, _w0, _sum0); __m256 _w1 = loadfp16(w1); - _sum1 = _mm256_fmadd_ps(_m, _w1, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_m, _w1, _sum1); __m256 _w2 = loadfp16(w2); - _sum2 = _mm256_fmadd_ps(_m, _w2, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_m, _w2, _sum2); __m256 _w3 = loadfp16(w3); - _sum3 = _mm256_fmadd_ps(_m, _w3, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_m, _w3, _sum3); __m256 _w4 = loadfp16(w4); - _sum4 = _mm256_fmadd_ps(_m, _w4, _sum4); + _sum4 = _mm256_comp_fmadd_ps(_m, _w4, _sum4); __m256 _w5 = loadfp16(w5); - _sum5 = _mm256_fmadd_ps(_m, _w5, _sum5); + _sum5 = _mm256_comp_fmadd_ps(_m, _w5, _sum5); __m256 _w6 = loadfp16(w6); - _sum6 = _mm256_fmadd_ps(_m, _w6, _sum6); + _sum6 = _mm256_comp_fmadd_ps(_m, _w6, _sum6); __m256 _w7 = loadfp16(w7); - _sum7 = _mm256_fmadd_ps(_m, _w7, _sum7); + _sum7 = _mm256_comp_fmadd_ps(_m, _w7, _sum7); m += 8; w0 += 8; @@ -1521,28 +1521,28 @@ int InnerProduct_x86::forward_fp16(const Mat& bottom_blob, Mat& top_blob, const __m256 _m = _mm256_loadu_ps(_m_f); __m256 _w0 = loadfp16(fp16_weights[0]); - _sum0 = _mm256_fmadd_ps(_m, _w0, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_m, _w0, _sum0); __m256 _w1 = loadfp16(fp16_weights[1]); - _sum1 = _mm256_fmadd_ps(_m, _w1, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_m, _w1, _sum1); __m256 _w2 = loadfp16(fp16_weights[2]); - _sum2 = _mm256_fmadd_ps(_m, _w2, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_m, _w2, _sum2); __m256 _w3 = loadfp16(fp16_weights[3]); - _sum3 = _mm256_fmadd_ps(_m, _w3, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_m, _w3, _sum3); __m256 _w4 = loadfp16(fp16_weights[4]); - _sum4 = _mm256_fmadd_ps(_m, _w4, _sum4); + _sum4 = _mm256_comp_fmadd_ps(_m, _w4, _sum4); __m256 _w5 = loadfp16(fp16_weights[5]); - _sum5 = _mm256_fmadd_ps(_m, _w5, _sum5); + _sum5 = _mm256_comp_fmadd_ps(_m, _w5, _sum5); __m256 _w6 = loadfp16(fp16_weights[6]); - _sum6 = _mm256_fmadd_ps(_m, _w6, _sum6); + _sum6 = _mm256_comp_fmadd_ps(_m, _w6, _sum6); __m256 _w7 = loadfp16(fp16_weights[7]); - _sum7 = _mm256_fmadd_ps(_m, _w7, _sum7); + _sum7 = _mm256_comp_fmadd_ps(_m, _w7, _sum7); } __m256 _sums = HorizontalSums(_sum0, _sum1, _sum2, _sum3, _sum4, _sum5, _sum6, _sum7); @@ -1587,16 +1587,16 @@ int InnerProduct_x86::forward_fp16(const Mat& bottom_blob, Mat& top_blob, const __m256 _m = _mm256_loadu_ps(m); __m256 _w0 = loadfp16(w0); - _sum0 = _mm256_fmadd_ps(_m, _w0, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_m, _w0, _sum0); __m256 _w1 = loadfp16(w1); - _sum1 = _mm256_fmadd_ps(_m, _w1, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_m, _w1, _sum1); __m256 _w2 = loadfp16(w2); - _sum2 = _mm256_fmadd_ps(_m, _w2, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_m, _w2, _sum2); __m256 _w3 = loadfp16(w3); - _sum3 = _mm256_fmadd_ps(_m, _w3, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_m, _w3, _sum3); m += 8; w0 += 8; @@ -1626,16 +1626,16 @@ int InnerProduct_x86::forward_fp16(const Mat& bottom_blob, Mat& top_blob, const __m256 _m = _mm256_loadu_ps(_m_f); __m256 _w0 = loadfp16(fp16_weights[0]); - _sum0 = _mm256_fmadd_ps(_m, _w0, _sum0); + _sum0 = _mm256_comp_fmadd_ps(_m, _w0, _sum0); __m256 _w1 = loadfp16(fp16_weights[1]); - _sum1 = _mm256_fmadd_ps(_m, _w1, _sum1); + _sum1 = _mm256_comp_fmadd_ps(_m, _w1, _sum1); __m256 _w2 = loadfp16(fp16_weights[2]); - _sum2 = _mm256_fmadd_ps(_m, _w2, _sum2); + _sum2 = _mm256_comp_fmadd_ps(_m, _w2, _sum2); __m256 _w3 = loadfp16(fp16_weights[3]); - _sum3 = _mm256_fmadd_ps(_m, _w3, _sum3); + _sum3 = _mm256_comp_fmadd_ps(_m, _w3, _sum3); } __m128 _sums = HorizontalSums(_sum0, _sum1, _sum2, _sum3); @@ -1665,7 +1665,7 @@ int InnerProduct_x86::forward_fp16(const Mat& bottom_blob, Mat& top_blob, const __m256 _m = _mm256_loadu_ps(m); __m256 _w = loadfp16(w); - _sum = _mm256_fmadd_ps(_m, _w, _sum); + _sum = _mm256_comp_fmadd_ps(_m, _w, _sum); m += 8; w += 8; @@ -1686,7 +1686,7 @@ int InnerProduct_x86::forward_fp16(const Mat& bottom_blob, Mat& top_blob, const __m256 _m = _mm256_loadu_ps(_m_f); __m256 _w = loadfp16(fp16_weights); - _sum = _mm256_fmadd_ps(_m, _w, _sum); + _sum = _mm256_comp_fmadd_ps(_m, _w, _sum); } sum += _mm256_reduce_add_ps(_sum); @@ -1696,7 +1696,7 @@ int InnerProduct_x86::forward_fp16(const Mat& bottom_blob, Mat& top_blob, const } return 0; } -#endif // __AVX__ +#endif // __AVX2__ #if NCNN_INT8 int InnerProduct_x86::create_pipeline_int8_x86(const Option& opt) diff --git a/src/layer/x86/lstm_x86.cpp b/src/layer/x86/lstm_x86.cpp index d42195ef2..e7e1dfef7 100644 --- a/src/layer/x86/lstm_x86.cpp +++ b/src/layer/x86/lstm_x86.cpp @@ -46,6 +46,7 @@ int LSTM_x86::create_pipeline(const Option& opt) return 0; } #ifdef __AVX__ +#ifdef __AVX2__ static int lstm_fp16(const Mat& bottom_blob, Mat& top_blob, int reverse, const Mat& weight_xc, const Mat& bias_c, const Mat& weight_hc, Mat& hidden_state, Mat& cell_state, const Option& opt) { @@ -118,14 +119,14 @@ static int lstm_fp16(const Mat& bottom_blob, Mat& top_blob, int reverse, const M for (; nn_num_size > 0; nn_num_size--) { __m256 xi = _mm256_loadu_ps(x); - _sumI_0 = _mm256_fmadd_ps(loadfp16(weight_xc_I_0), xi, _sumI_0); - _sumF_0 = _mm256_fmadd_ps(loadfp16(weight_xc_F_0), xi, _sumF_0); - _sumO_0 = _mm256_fmadd_ps(loadfp16(weight_xc_O_0), xi, _sumO_0); - _sumG_0 = _mm256_fmadd_ps(loadfp16(weight_xc_G_0), xi, _sumG_0); - _sumI_1 = _mm256_fmadd_ps(loadfp16(weight_xc_I_1), xi, _sumI_1); - _sumF_1 = _mm256_fmadd_ps(loadfp16(weight_xc_F_1), xi, _sumF_1); - _sumO_1 = _mm256_fmadd_ps(loadfp16(weight_xc_O_1), xi, _sumO_1); - _sumG_1 = _mm256_fmadd_ps(loadfp16(weight_xc_G_1), xi, _sumG_1); + _sumI_0 = _mm256_comp_fmadd_ps(loadfp16(weight_xc_I_0), xi, _sumI_0); + _sumF_0 = _mm256_comp_fmadd_ps(loadfp16(weight_xc_F_0), xi, _sumF_0); + _sumO_0 = _mm256_comp_fmadd_ps(loadfp16(weight_xc_O_0), xi, _sumO_0); + _sumG_0 = _mm256_comp_fmadd_ps(loadfp16(weight_xc_G_0), xi, _sumG_0); + _sumI_1 = _mm256_comp_fmadd_ps(loadfp16(weight_xc_I_1), xi, _sumI_1); + _sumF_1 = _mm256_comp_fmadd_ps(loadfp16(weight_xc_F_1), xi, _sumF_1); + _sumO_1 = _mm256_comp_fmadd_ps(loadfp16(weight_xc_O_1), xi, _sumO_1); + _sumG_1 = _mm256_comp_fmadd_ps(loadfp16(weight_xc_G_1), xi, _sumG_1); x += 8; weight_xc_I_0 += 8; weight_xc_F_0 += 8; @@ -142,14 +143,14 @@ static int lstm_fp16(const Mat& bottom_blob, Mat& top_blob, int reverse, const M { __m256 h_cont = _mm256_loadu_ps(hidden_ptr_r); - _sumI_0 = _mm256_fmadd_ps(loadfp16(weight_hc_I_0), h_cont, _sumI_0); - _sumF_0 = _mm256_fmadd_ps(loadfp16(weight_hc_F_0), h_cont, _sumF_0); - _sumO_0 = _mm256_fmadd_ps(loadfp16(weight_hc_O_0), h_cont, _sumO_0); - _sumG_0 = _mm256_fmadd_ps(loadfp16(weight_hc_G_0), h_cont, _sumG_0); - _sumI_1 = _mm256_fmadd_ps(loadfp16(weight_hc_I_1), h_cont, _sumI_1); - _sumF_1 = _mm256_fmadd_ps(loadfp16(weight_hc_F_1), h_cont, _sumF_1); - _sumO_1 = _mm256_fmadd_ps(loadfp16(weight_hc_O_1), h_cont, _sumO_1); - _sumG_1 = _mm256_fmadd_ps(loadfp16(weight_hc_G_1), h_cont, _sumG_1); + _sumI_0 = _mm256_comp_fmadd_ps(loadfp16(weight_hc_I_0), h_cont, _sumI_0); + _sumF_0 = _mm256_comp_fmadd_ps(loadfp16(weight_hc_F_0), h_cont, _sumF_0); + _sumO_0 = _mm256_comp_fmadd_ps(loadfp16(weight_hc_O_0), h_cont, _sumO_0); + _sumG_0 = _mm256_comp_fmadd_ps(loadfp16(weight_hc_G_0), h_cont, _sumG_0); + _sumI_1 = _mm256_comp_fmadd_ps(loadfp16(weight_hc_I_1), h_cont, _sumI_1); + _sumF_1 = _mm256_comp_fmadd_ps(loadfp16(weight_hc_F_1), h_cont, _sumF_1); + _sumO_1 = _mm256_comp_fmadd_ps(loadfp16(weight_hc_O_1), h_cont, _sumO_1); + _sumG_1 = _mm256_comp_fmadd_ps(loadfp16(weight_hc_G_1), h_cont, _sumG_1); hidden_ptr_r += 8; weight_hc_I_0 += 8; weight_hc_F_0 += 8; @@ -188,14 +189,14 @@ static int lstm_fp16(const Mat& bottom_blob, Mat& top_blob, int reverse, const M weight_xc_G_1++; } __m256 xi = _mm256_loadu_ps(_xi_f); - _sumI_0 = _mm256_fmadd_ps(loadfp16(fp16_weights[0]), xi, _sumI_0); - _sumF_0 = _mm256_fmadd_ps(loadfp16(fp16_weights[1]), xi, _sumF_0); - _sumO_0 = _mm256_fmadd_ps(loadfp16(fp16_weights[2]), xi, _sumO_0); - _sumG_0 = _mm256_fmadd_ps(loadfp16(fp16_weights[3]), xi, _sumG_0); - _sumI_1 = _mm256_fmadd_ps(loadfp16(fp16_weights[4]), xi, _sumI_1); - _sumF_1 = _mm256_fmadd_ps(loadfp16(fp16_weights[5]), xi, _sumF_1); - _sumO_1 = _mm256_fmadd_ps(loadfp16(fp16_weights[6]), xi, _sumO_1); - _sumG_1 = _mm256_fmadd_ps(loadfp16(fp16_weights[7]), xi, _sumG_1); + _sumI_0 = _mm256_comp_fmadd_ps(loadfp16(fp16_weights[0]), xi, _sumI_0); + _sumF_0 = _mm256_comp_fmadd_ps(loadfp16(fp16_weights[1]), xi, _sumF_0); + _sumO_0 = _mm256_comp_fmadd_ps(loadfp16(fp16_weights[2]), xi, _sumO_0); + _sumG_0 = _mm256_comp_fmadd_ps(loadfp16(fp16_weights[3]), xi, _sumG_0); + _sumI_1 = _mm256_comp_fmadd_ps(loadfp16(fp16_weights[4]), xi, _sumI_1); + _sumF_1 = _mm256_comp_fmadd_ps(loadfp16(fp16_weights[5]), xi, _sumF_1); + _sumO_1 = _mm256_comp_fmadd_ps(loadfp16(fp16_weights[6]), xi, _sumO_1); + _sumG_1 = _mm256_comp_fmadd_ps(loadfp16(fp16_weights[7]), xi, _sumG_1); } if (remain_num_output != 0) { @@ -225,14 +226,14 @@ static int lstm_fp16(const Mat& bottom_blob, Mat& top_blob, int reverse, const M weight_hc_G_1++; } __m256 h_cont = _mm256_loadu_ps(_hcont_f); - _sumI_0 = _mm256_fmadd_ps(loadfp16(fp16_weights[0]), h_cont, _sumI_0); - _sumF_0 = _mm256_fmadd_ps(loadfp16(fp16_weights[1]), h_cont, _sumF_0); - _sumO_0 = _mm256_fmadd_ps(loadfp16(fp16_weights[2]), h_cont, _sumO_0); - _sumG_0 = _mm256_fmadd_ps(loadfp16(fp16_weights[3]), h_cont, _sumG_0); - _sumI_1 = _mm256_fmadd_ps(loadfp16(fp16_weights[4]), h_cont, _sumI_1); - _sumF_1 = _mm256_fmadd_ps(loadfp16(fp16_weights[5]), h_cont, _sumF_1); - _sumO_1 = _mm256_fmadd_ps(loadfp16(fp16_weights[6]), h_cont, _sumO_1); - _sumG_1 = _mm256_fmadd_ps(loadfp16(fp16_weights[7]), h_cont, _sumG_1); + _sumI_0 = _mm256_comp_fmadd_ps(loadfp16(fp16_weights[0]), h_cont, _sumI_0); + _sumF_0 = _mm256_comp_fmadd_ps(loadfp16(fp16_weights[1]), h_cont, _sumF_0); + _sumO_0 = _mm256_comp_fmadd_ps(loadfp16(fp16_weights[2]), h_cont, _sumO_0); + _sumG_0 = _mm256_comp_fmadd_ps(loadfp16(fp16_weights[3]), h_cont, _sumG_0); + _sumI_1 = _mm256_comp_fmadd_ps(loadfp16(fp16_weights[4]), h_cont, _sumI_1); + _sumF_1 = _mm256_comp_fmadd_ps(loadfp16(fp16_weights[5]), h_cont, _sumF_1); + _sumO_1 = _mm256_comp_fmadd_ps(loadfp16(fp16_weights[6]), h_cont, _sumO_1); + _sumG_1 = _mm256_comp_fmadd_ps(loadfp16(fp16_weights[7]), h_cont, _sumG_1); } float sums[8]; _mm256_storeu_ps(sums, HorizontalSums(_sumI_0, _sumF_0, _sumO_0, _sumG_0, _sumI_1, _sumF_1, _sumO_1, _sumG_1)); @@ -291,10 +292,10 @@ static int lstm_fp16(const Mat& bottom_blob, Mat& top_blob, int reverse, const M for (; nn_num_size > 0; nn_num_size--) { __m256 xi = _mm256_loadu_ps(x); - _sumI = _mm256_fmadd_ps(loadfp16(weight_xc_I), xi, _sumI); - _sumF = _mm256_fmadd_ps(loadfp16(weight_xc_F), xi, _sumF); - _sumO = _mm256_fmadd_ps(loadfp16(weight_xc_O), xi, _sumO); - _sumG = _mm256_fmadd_ps(loadfp16(weight_xc_G), xi, _sumG); + _sumI = _mm256_comp_fmadd_ps(loadfp16(weight_xc_I), xi, _sumI); + _sumF = _mm256_comp_fmadd_ps(loadfp16(weight_xc_F), xi, _sumF); + _sumO = _mm256_comp_fmadd_ps(loadfp16(weight_xc_O), xi, _sumO); + _sumG = _mm256_comp_fmadd_ps(loadfp16(weight_xc_G), xi, _sumG); x += 8; weight_xc_I += 8; weight_xc_F += 8; @@ -307,10 +308,10 @@ static int lstm_fp16(const Mat& bottom_blob, Mat& top_blob, int reverse, const M { __m256 h_cont = _mm256_loadu_ps(hidden_ptr_r); - _sumI = _mm256_fmadd_ps(loadfp16(weight_hc_I), h_cont, _sumI); - _sumF = _mm256_fmadd_ps(loadfp16(weight_hc_F), h_cont, _sumF); - _sumO = _mm256_fmadd_ps(loadfp16(weight_hc_O), h_cont, _sumO); - _sumG = _mm256_fmadd_ps(loadfp16(weight_hc_G), h_cont, _sumG); + _sumI = _mm256_comp_fmadd_ps(loadfp16(weight_hc_I), h_cont, _sumI); + _sumF = _mm256_comp_fmadd_ps(loadfp16(weight_hc_F), h_cont, _sumF); + _sumO = _mm256_comp_fmadd_ps(loadfp16(weight_hc_O), h_cont, _sumO); + _sumG = _mm256_comp_fmadd_ps(loadfp16(weight_hc_G), h_cont, _sumG); hidden_ptr_r += 8; weight_hc_I += 8; weight_hc_F += 8; @@ -337,10 +338,10 @@ static int lstm_fp16(const Mat& bottom_blob, Mat& top_blob, int reverse, const M weight_xc_G++; } __m256 xi = _mm256_loadu_ps(_xi_f); - _sumI = _mm256_fmadd_ps(loadfp16(fp16_weights[0]), xi, _sumI); - _sumF = _mm256_fmadd_ps(loadfp16(fp16_weights[1]), xi, _sumF); - _sumO = _mm256_fmadd_ps(loadfp16(fp16_weights[2]), xi, _sumO); - _sumG = _mm256_fmadd_ps(loadfp16(fp16_weights[3]), xi, _sumG); + _sumI = _mm256_comp_fmadd_ps(loadfp16(fp16_weights[0]), xi, _sumI); + _sumF = _mm256_comp_fmadd_ps(loadfp16(fp16_weights[1]), xi, _sumF); + _sumO = _mm256_comp_fmadd_ps(loadfp16(fp16_weights[2]), xi, _sumO); + _sumG = _mm256_comp_fmadd_ps(loadfp16(fp16_weights[3]), xi, _sumG); } if (remain_num_output != 0) { @@ -362,10 +363,10 @@ static int lstm_fp16(const Mat& bottom_blob, Mat& top_blob, int reverse, const M weight_hc_G++; } __m256 h_cont = _mm256_loadu_ps(_hcont_f); - _sumI = _mm256_fmadd_ps(loadfp16(fp16_weights[0]), h_cont, _sumI); - _sumF = _mm256_fmadd_ps(loadfp16(fp16_weights[1]), h_cont, _sumF); - _sumO = _mm256_fmadd_ps(loadfp16(fp16_weights[2]), h_cont, _sumO); - _sumG = _mm256_fmadd_ps(loadfp16(fp16_weights[3]), h_cont, _sumG); + _sumI = _mm256_comp_fmadd_ps(loadfp16(fp16_weights[0]), h_cont, _sumI); + _sumF = _mm256_comp_fmadd_ps(loadfp16(fp16_weights[1]), h_cont, _sumF); + _sumO = _mm256_comp_fmadd_ps(loadfp16(fp16_weights[2]), h_cont, _sumO); + _sumG = _mm256_comp_fmadd_ps(loadfp16(fp16_weights[3]), h_cont, _sumG); } float sums[4]; @@ -445,7 +446,7 @@ static int lstm_fp16(const Mat& bottom_blob, Mat& top_blob, int reverse, const M return 0; } - +#endif static int lstm(const Mat& bottom_blob, Mat& top_blob, int reverse, const Mat& weight_xc, const Mat& bias_c, const Mat& weight_hc, Mat& hidden_state, Mat& cell_state, const Option& opt) { int size = bottom_blob.w; @@ -519,14 +520,14 @@ static int lstm(const Mat& bottom_blob, Mat& top_blob, int reverse, const Mat& w for (; nn_num_size > 0; nn_num_size--) { __m256 xi = _mm256_loadu_ps(x); - _sumI_0 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_xc_I_0), xi, _sumI_0); - _sumF_0 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_xc_F_0), xi, _sumF_0); - _sumO_0 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_xc_O_0), xi, _sumO_0); - _sumG_0 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_xc_G_0), xi, _sumG_0); - _sumI_1 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_xc_I_1), xi, _sumI_1); - _sumF_1 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_xc_F_1), xi, _sumF_1); - _sumO_1 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_xc_O_1), xi, _sumO_1); - _sumG_1 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_xc_G_1), xi, _sumG_1); + _sumI_0 = _mm256_comp_fmadd_ps(_mm256_loadu_ps(weight_xc_I_0), xi, _sumI_0); + _sumF_0 = _mm256_comp_fmadd_ps(_mm256_loadu_ps(weight_xc_F_0), xi, _sumF_0); + _sumO_0 = _mm256_comp_fmadd_ps(_mm256_loadu_ps(weight_xc_O_0), xi, _sumO_0); + _sumG_0 = _mm256_comp_fmadd_ps(_mm256_loadu_ps(weight_xc_G_0), xi, _sumG_0); + _sumI_1 = _mm256_comp_fmadd_ps(_mm256_loadu_ps(weight_xc_I_1), xi, _sumI_1); + _sumF_1 = _mm256_comp_fmadd_ps(_mm256_loadu_ps(weight_xc_F_1), xi, _sumF_1); + _sumO_1 = _mm256_comp_fmadd_ps(_mm256_loadu_ps(weight_xc_O_1), xi, _sumO_1); + _sumG_1 = _mm256_comp_fmadd_ps(_mm256_loadu_ps(weight_xc_G_1), xi, _sumG_1); x += 8; weight_xc_I_0 += 8; weight_xc_F_0 += 8; @@ -543,14 +544,14 @@ static int lstm(const Mat& bottom_blob, Mat& top_blob, int reverse, const Mat& w { __m256 h_cont = _mm256_loadu_ps(hidden_ptr_r); - _sumI_0 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_hc_I_0), h_cont, _sumI_0); - _sumF_0 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_hc_F_0), h_cont, _sumF_0); - _sumO_0 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_hc_O_0), h_cont, _sumO_0); - _sumG_0 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_hc_G_0), h_cont, _sumG_0); - _sumI_1 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_hc_I_1), h_cont, _sumI_1); - _sumF_1 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_hc_F_1), h_cont, _sumF_1); - _sumO_1 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_hc_O_1), h_cont, _sumO_1); - _sumG_1 = _mm256_fmadd_ps(_mm256_loadu_ps(weight_hc_G_1), h_cont, _sumG_1); + _sumI_0 = _mm256_comp_fmadd_ps(_mm256_loadu_ps(weight_hc_I_0), h_cont, _sumI_0); + _sumF_0 = _mm256_comp_fmadd_ps(_mm256_loadu_ps(weight_hc_F_0), h_cont, _sumF_0); + _sumO_0 = _mm256_comp_fmadd_ps(_mm256_loadu_ps(weight_hc_O_0), h_cont, _sumO_0); + _sumG_0 = _mm256_comp_fmadd_ps(_mm256_loadu_ps(weight_hc_G_0), h_cont, _sumG_0); + _sumI_1 = _mm256_comp_fmadd_ps(_mm256_loadu_ps(weight_hc_I_1), h_cont, _sumI_1); + _sumF_1 = _mm256_comp_fmadd_ps(_mm256_loadu_ps(weight_hc_F_1), h_cont, _sumF_1); + _sumO_1 = _mm256_comp_fmadd_ps(_mm256_loadu_ps(weight_hc_O_1), h_cont, _sumO_1); + _sumG_1 = _mm256_comp_fmadd_ps(_mm256_loadu_ps(weight_hc_G_1), h_cont, _sumG_1); hidden_ptr_r += 8; weight_hc_I_0 += 8; weight_hc_F_0 += 8; @@ -662,10 +663,10 @@ static int lstm(const Mat& bottom_blob, Mat& top_blob, int reverse, const Mat& w for (; nn_num_size > 0; nn_num_size--) { __m256 xi = _mm256_loadu_ps(x); - _sumI = _mm256_fmadd_ps(_mm256_loadu_ps(weight_xc_I), xi, _sumI); - _sumF = _mm256_fmadd_ps(_mm256_loadu_ps(weight_xc_F), xi, _sumF); - _sumO = _mm256_fmadd_ps(_mm256_loadu_ps(weight_xc_O), xi, _sumO); - _sumG = _mm256_fmadd_ps(_mm256_loadu_ps(weight_xc_G), xi, _sumG); + _sumI = _mm256_comp_fmadd_ps(_mm256_loadu_ps(weight_xc_I), xi, _sumI); + _sumF = _mm256_comp_fmadd_ps(_mm256_loadu_ps(weight_xc_F), xi, _sumF); + _sumO = _mm256_comp_fmadd_ps(_mm256_loadu_ps(weight_xc_O), xi, _sumO); + _sumG = _mm256_comp_fmadd_ps(_mm256_loadu_ps(weight_xc_G), xi, _sumG); x += 8; weight_xc_I += 8; weight_xc_F += 8; @@ -678,10 +679,10 @@ static int lstm(const Mat& bottom_blob, Mat& top_blob, int reverse, const Mat& w { __m256 h_cont = _mm256_loadu_ps(hidden_ptr_r); - _sumI = _mm256_fmadd_ps(_mm256_loadu_ps(weight_hc_I), h_cont, _sumI); - _sumF = _mm256_fmadd_ps(_mm256_loadu_ps(weight_hc_F), h_cont, _sumF); - _sumO = _mm256_fmadd_ps(_mm256_loadu_ps(weight_hc_O), h_cont, _sumO); - _sumG = _mm256_fmadd_ps(_mm256_loadu_ps(weight_hc_G), h_cont, _sumG); + _sumI = _mm256_comp_fmadd_ps(_mm256_loadu_ps(weight_hc_I), h_cont, _sumI); + _sumF = _mm256_comp_fmadd_ps(_mm256_loadu_ps(weight_hc_F), h_cont, _sumF); + _sumO = _mm256_comp_fmadd_ps(_mm256_loadu_ps(weight_hc_O), h_cont, _sumO); + _sumG = _mm256_comp_fmadd_ps(_mm256_loadu_ps(weight_hc_G), h_cont, _sumG); hidden_ptr_r += 8; weight_hc_I += 8; weight_hc_F += 8; @@ -819,6 +820,7 @@ int LSTM_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) // Uni directional if (direction == 0 || direction == 1) { +#if __AVX2__ if (opt.use_weight_fp16_storage) { // Uni directional @@ -828,11 +830,14 @@ int LSTM_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) } else { +#endif // Uni directional int ret = lstm(bottom_blob, top_blob, direction, weight_xc_data.channel(0), bias_c_data.channel(0), weight_hc_data.channel(0), hidden, cell, opt); if (ret != 0) return ret; +#if __AVX2__ } +#endif } if (direction == 2) @@ -844,7 +849,7 @@ int LSTM_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) Mat top_blob_reverse(num_output, T, 4u, opt.workspace_allocator); if (top_blob_reverse.empty()) return -100; - +#if __AVX2__ if (opt.use_weight_fp16_storage) { // Uni directional @@ -854,14 +859,18 @@ int LSTM_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) } else { +#endif + // Uni directional int ret0 = lstm(bottom_blob, top_blob_forward, 0, weight_xc_data.channel(0), bias_c_data.channel(0), weight_hc_data.channel(0), hidden, cell, opt); if (ret0 != 0) return ret0; +#if __AVX2__ } - +#endif hidden.fill(0.0f); cell.fill(0.0f); +#if __AVX2__ if (opt.use_weight_fp16_storage) { // Uni directional @@ -871,11 +880,14 @@ int LSTM_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& opt) } else { +#endif // Uni directional int ret1 = lstm(bottom_blob, top_blob_reverse, 1, weight_xc_data.channel(1), bias_c_data.channel(1), weight_hc_data.channel(1), hidden, cell, opt); if (ret1 != 0) return ret1; +#if __AVX2__ } +#endif // concat w for (int i = 0; i < T; i++) @@ -916,7 +928,7 @@ int LSTM_x86::forward(const std::vector& bottom_blobs, std::vector& to top_blob.create(num_output, T, 4u, opt.blob_allocator); if (top_blob.empty()) return -100; - +#if __AVX2__ if (opt.use_weight_fp16_storage) { // Uni directional @@ -926,11 +938,14 @@ int LSTM_x86::forward(const std::vector& bottom_blobs, std::vector& to } else { +#endif // Uni directional int ret = lstm(bottom_blob, top_blob, direction, weight_xc_data.channel(0), bias_c_data.channel(0), weight_hc_data.channel(0), hidden_state, cell_state, opt); if (ret != 0) return ret; +#if __AVX2__ } +#endif return 0; #else return LSTM::forward(bottom_blobs, top_blobs, opt); diff --git a/src/layer/x86/quantize_x86.cpp b/src/layer/x86/quantize_x86.cpp index 174eb1278..119fc1987 100644 --- a/src/layer/x86/quantize_x86.cpp +++ b/src/layer/x86/quantize_x86.cpp @@ -63,7 +63,11 @@ int Quantize_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& o __m256 _v = _mm256_loadu_ps(ptr); _v = _mm256_mul_ps(_v, _scale); +#if __AVX2__ *(int64_t*)outptr = float2int8_avx(_v); +#else + float2int8_loop(_v, outptr); +#endif } } else @@ -77,7 +81,11 @@ int Quantize_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& o __m256 _v = _mm256_loadu_ps(ptr); __m256 _scale = _mm256_loadu_ps((const float*)scale_data + i * 8); _v = _mm256_mul_ps(_v, _scale); +#if __AVX2__ *(int64_t*)outptr = float2int8_avx(_v); +#else + float2int8_loop(_v, outptr); +#endif } } } @@ -102,6 +110,7 @@ int Quantize_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& o signed char* outptr = top_blob.row(i); int j = 0; +#if __AVX2__ for (; j + 1 < w; j += 2) { __m256 _v0 = _mm256_loadu_ps(ptr); @@ -123,6 +132,16 @@ int Quantize_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& o ptr += 8; outptr += 8; } +#else + for (; j < w; j++) + { + __m256 _v = _mm256_loadu_ps(ptr); + _v = _mm256_mul_ps(_v, _scale); + float2int8_loop(_v, outptr); + ptr += 8; + outptr += 8; + } +#endif } } else @@ -136,6 +155,7 @@ int Quantize_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& o __m256 _scale = _mm256_loadu_ps((const float*)scale_data + i * 8); int j = 0; +#if __AVX2__ for (; j + 1 < w; j += 2) { __m256 _v0 = _mm256_loadu_ps(ptr); @@ -157,6 +177,16 @@ int Quantize_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& o ptr += 8; outptr += 8; } +#else + for (; j < w; j++) + { + __m256 _v = _mm256_loadu_ps(ptr); + _v = _mm256_mul_ps(_v, _scale); + float2int8_loop(_v, outptr); + ptr += 8; + outptr += 8; + } +#endif } } } @@ -183,6 +213,7 @@ int Quantize_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& o signed char* outptr = top_blob.channel(q); int i = 0; +#if __AVX2__ for (; i + 1 < size; i += 2) { __m256 _v0 = _mm256_loadu_ps(ptr); @@ -204,6 +235,16 @@ int Quantize_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& o ptr += 8; outptr += 8; } +#else + for (; i < size; i++) + { + __m256 _v = _mm256_loadu_ps(ptr); + _v = _mm256_mul_ps(_v, _scale); + float2int8_loop(_v, outptr); + ptr += 8; + outptr += 8; + } +#endif } } else @@ -217,6 +258,8 @@ int Quantize_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& o __m256 _scale = _mm256_loadu_ps((const float*)scale_data + q * 8); int i = 0; +#if __AVX2__ + for (; i + 1 < size; i += 2) { __m256 _v0 = _mm256_loadu_ps(ptr); @@ -238,6 +281,16 @@ int Quantize_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& o ptr += 8; outptr += 8; } +#else + for (; i < size; i++) + { + __m256 _v = _mm256_loadu_ps(ptr); + _v = _mm256_mul_ps(_v, _scale); + float2int8_loop(_v, outptr); + ptr += 8; + outptr += 8; + } +#endif } } } diff --git a/src/layer/x86/requantize_x86.cpp b/src/layer/x86/requantize_x86.cpp index 8c223f636..1f9d99a7f 100644 --- a/src/layer/x86/requantize_x86.cpp +++ b/src/layer/x86/requantize_x86.cpp @@ -16,9 +16,9 @@ #if __SSE2__ #include -#if __AVX__ +#if __AVX2__ #include -#endif // __AVX__ +#endif // __AVX2__ #endif // __SSE2__ #include "x86_activation.h" @@ -51,7 +51,7 @@ int Requantize_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& if (scale_in_data_size == 1 && scale_out_data_size == 1) { -#if __AVX__ +#if __AVX2__ __m256 _scale_in = _mm256_set1_ps(scale_in_data[0]); __m256 _scale_out = _mm256_set1_ps(scale_out_data[0]); #else @@ -67,7 +67,7 @@ int Requantize_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& const int* intptr = (const int*)bottom_blob + i * 8; signed char* ptr = (signed char*)top_blob + i * 8; -#if __AVX__ +#if __AVX2__ __m256 _v = _mm256_cvtepi32_ps(_mm256_loadu_si256((const __m256i*)intptr)); _v = _mm256_mul_ps(_v, _scale_in); _v = activation_avx(_v, activation_type, activation_params); @@ -88,7 +88,7 @@ int Requantize_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& } else if (bias_data_size == 1) { -#if __AVX__ +#if __AVX2__ __m256 _bias = _mm256_set1_ps(bias_data[0]); #else __m128 _bias = _mm_set1_ps(bias_data[0]); @@ -100,9 +100,9 @@ int Requantize_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& const int* intptr = (const int*)bottom_blob + i * 8; signed char* ptr = (signed char*)top_blob + i * 8; -#if __AVX__ +#if __AVX2__ __m256 _v = _mm256_cvtepi32_ps(_mm256_loadu_si256((const __m256i*)intptr)); - _v = _mm256_fmadd_ps(_v, _scale_in, _bias); + _v = _mm256_comp_fmadd_ps(_v, _scale_in, _bias); _v = activation_avx(_v, activation_type, activation_params); _v = _mm256_mul_ps(_v, _scale_out); *(int64_t*)ptr = float2int8_avx(_v); @@ -127,10 +127,10 @@ int Requantize_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& const int* intptr = (const int*)bottom_blob + i * 8; signed char* ptr = (signed char*)top_blob + i * 8; -#if __AVX__ +#if __AVX2__ __m256 _bias = _mm256_loadu_ps((const float*)bias_data + i * 8); __m256 _v = _mm256_cvtepi32_ps(_mm256_loadu_si256((const __m256i*)intptr)); - _v = _mm256_fmadd_ps(_v, _scale_in, _bias); + _v = _mm256_comp_fmadd_ps(_v, _scale_in, _bias); _v = activation_avx(_v, activation_type, activation_params); _v = _mm256_mul_ps(_v, _scale_out); *(int64_t*)ptr = float2int8_avx(_v); @@ -152,7 +152,7 @@ int Requantize_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& } else if (scale_in_data_size == 1 && scale_out_data_size > 1) { -#if __AVX__ +#if __AVX2__ __m256 _scale_in = _mm256_set1_ps(scale_in_data[0]); #else __m128 _scale_in = _mm_set1_ps(scale_in_data[0]); @@ -166,7 +166,7 @@ int Requantize_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& const int* intptr = (const int*)bottom_blob + i * 8; signed char* ptr = (signed char*)top_blob + i * 8; -#if __AVX__ +#if __AVX2__ __m256 _scale_out = _mm256_loadu_ps((const float*)scale_out_data + i * 8); __m256 _v = _mm256_cvtepi32_ps(_mm256_loadu_si256((const __m256i*)intptr)); _v = _mm256_mul_ps(_v, _scale_in); @@ -190,7 +190,7 @@ int Requantize_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& } else if (bias_data_size == 1) { -#if __AVX__ +#if __AVX2__ __m256 _bias = _mm256_set1_ps(bias_data[0]); #else __m128 _bias = _mm_set1_ps(bias_data[0]); @@ -202,10 +202,10 @@ int Requantize_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& const int* intptr = (const int*)bottom_blob + i * 8; signed char* ptr = (signed char*)top_blob + i * 8; -#if __AVX__ +#if __AVX2__ __m256 _scale_out = _mm256_loadu_ps((const float*)scale_out_data + i * 8); __m256 _v = _mm256_cvtepi32_ps(_mm256_loadu_si256((const __m256i*)intptr)); - _v = _mm256_fmadd_ps(_v, _scale_in, _bias); + _v = _mm256_comp_fmadd_ps(_v, _scale_in, _bias); _v = activation_avx(_v, activation_type, activation_params); _v = _mm256_mul_ps(_v, _scale_out); *(int64_t*)ptr = float2int8_avx(_v); @@ -232,11 +232,11 @@ int Requantize_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& const int* intptr = (const int*)bottom_blob + i * 8; signed char* ptr = (signed char*)top_blob + i * 8; -#if __AVX__ +#if __AVX2__ __m256 _scale_out = _mm256_loadu_ps((const float*)scale_out_data + i * 8); __m256 _bias = _mm256_loadu_ps((const float*)bias_data + i * 8); __m256 _v = _mm256_cvtepi32_ps(_mm256_loadu_si256((const __m256i*)intptr)); - _v = _mm256_fmadd_ps(_v, _scale_in, _bias); + _v = _mm256_comp_fmadd_ps(_v, _scale_in, _bias); _v = activation_avx(_v, activation_type, activation_params); _v = _mm256_mul_ps(_v, _scale_out); *(int64_t*)ptr = float2int8_avx(_v); @@ -260,7 +260,7 @@ int Requantize_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& } else if (scale_in_data_size > 1 && scale_out_data_size == 1) { -#if __AVX__ +#if __AVX2__ __m256 _scale_out = _mm256_set1_ps(scale_out_data[0]); #else __m128 _scale_out = _mm_set1_ps(scale_out_data[0]); @@ -274,7 +274,7 @@ int Requantize_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& const int* intptr = (const int*)bottom_blob + i * 8; signed char* ptr = (signed char*)top_blob + i * 8; -#if __AVX__ +#if __AVX2__ __m256 _scale_in = _mm256_loadu_ps((const float*)scale_in_data + i * 8); __m256 _v = _mm256_cvtepi32_ps(_mm256_loadu_si256((const __m256i*)intptr)); _v = _mm256_mul_ps(_v, _scale_in); @@ -298,7 +298,7 @@ int Requantize_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& } else if (bias_data_size == 1) { -#if __AVX__ +#if __AVX2__ __m256 _bias = _mm256_set1_ps(bias_data[0]); #else __m128 _bias = _mm_set1_ps(bias_data[0]); @@ -310,10 +310,10 @@ int Requantize_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& const int* intptr = (const int*)bottom_blob + i * 8; signed char* ptr = (signed char*)top_blob + i * 8; -#if __AVX__ +#if __AVX2__ __m256 _scale_in = _mm256_loadu_ps((const float*)scale_in_data + i * 8); __m256 _v = _mm256_cvtepi32_ps(_mm256_loadu_si256((const __m256i*)intptr)); - _v = _mm256_fmadd_ps(_v, _scale_in, _bias); + _v = _mm256_comp_fmadd_ps(_v, _scale_in, _bias); _v = activation_avx(_v, activation_type, activation_params); _v = _mm256_mul_ps(_v, _scale_out); *(int64_t*)ptr = float2int8_avx(_v); @@ -340,11 +340,11 @@ int Requantize_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& const int* intptr = (const int*)bottom_blob + i * 8; signed char* ptr = (signed char*)top_blob + i * 8; -#if __AVX__ +#if __AVX2__ __m256 _scale_in = _mm256_loadu_ps((const float*)scale_in_data + i * 8); __m256 _bias = _mm256_loadu_ps((const float*)bias_data + i * 8); __m256 _v = _mm256_cvtepi32_ps(_mm256_loadu_si256((const __m256i*)intptr)); - _v = _mm256_fmadd_ps(_v, _scale_in, _bias); + _v = _mm256_comp_fmadd_ps(_v, _scale_in, _bias); _v = activation_avx(_v, activation_type, activation_params); _v = _mm256_mul_ps(_v, _scale_out); *(int64_t*)ptr = float2int8_avx(_v); @@ -376,7 +376,7 @@ int Requantize_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& const int* intptr = (const int*)bottom_blob + i * 8; signed char* ptr = (signed char*)top_blob + i * 8; -#if __AVX__ +#if __AVX2__ __m256 _scale_in = _mm256_loadu_ps((const float*)scale_in_data + i * 8); __m256 _scale_out = _mm256_loadu_ps((const float*)scale_out_data + i * 8); __m256 _v = _mm256_cvtepi32_ps(_mm256_loadu_si256((const __m256i*)intptr)); @@ -403,7 +403,7 @@ int Requantize_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& } else if (bias_data_size == 1) { -#if __AVX__ +#if __AVX2__ __m256 _bias = _mm256_set1_ps(bias_data[0]); #else __m128 _bias = _mm_set1_ps(bias_data[0]); @@ -415,11 +415,11 @@ int Requantize_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& const int* intptr = (const int*)bottom_blob + i * 8; signed char* ptr = (signed char*)top_blob + i * 8; -#if __AVX__ +#if __AVX2__ __m256 _scale_in = _mm256_loadu_ps((const float*)scale_in_data + i * 8); __m256 _scale_out = _mm256_loadu_ps((const float*)scale_out_data + i * 8); __m256 _v = _mm256_cvtepi32_ps(_mm256_loadu_si256((const __m256i*)intptr)); - _v = _mm256_fmadd_ps(_v, _scale_in, _bias); + _v = _mm256_comp_fmadd_ps(_v, _scale_in, _bias); _v = activation_avx(_v, activation_type, activation_params); _v = _mm256_mul_ps(_v, _scale_out); *(int64_t*)ptr = float2int8_avx(_v); @@ -448,12 +448,12 @@ int Requantize_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& const int* intptr = (const int*)bottom_blob + i * 8; signed char* ptr = (signed char*)top_blob + i * 8; -#if __AVX__ +#if __AVX2__ __m256 _scale_in = _mm256_loadu_ps((const float*)scale_in_data + i * 8); __m256 _scale_out = _mm256_loadu_ps((const float*)scale_out_data + i * 8); __m256 _bias = _mm256_loadu_ps((const float*)bias_data + i * 8); __m256 _v = _mm256_cvtepi32_ps(_mm256_loadu_si256((const __m256i*)intptr)); - _v = _mm256_fmadd_ps(_v, _scale_in, _bias); + _v = _mm256_comp_fmadd_ps(_v, _scale_in, _bias); _v = activation_avx(_v, activation_type, activation_params); _v = _mm256_mul_ps(_v, _scale_out); *(int64_t*)ptr = float2int8_avx(_v); @@ -496,7 +496,7 @@ int Requantize_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& const int* intptr = bottom_blob.row(i); signed char* ptr = top_blob.row(i); -#if __AVX__ +#if __AVX2__ __m256 _scale_in = scale_in_data_size == 1 ? _mm256_set1_ps(scale_in_data[0]) : _mm256_loadu_ps((const float*)scale_in_data + i * 8); __m256 _scale_out = scale_out_data_size == 1 ? _mm256_set1_ps(scale_out_data[0]) : _mm256_loadu_ps((const float*)scale_out_data + i * 8); #else @@ -508,7 +508,7 @@ int Requantize_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& for (int j = 0; j < w; j++) { -#if __AVX__ +#if __AVX2__ __m256 _v = _mm256_cvtepi32_ps(_mm256_loadu_si256((const __m256i*)intptr)); _v = _mm256_mul_ps(_v, _scale_in); _v = activation_avx(_v, activation_type, activation_params); @@ -539,7 +539,7 @@ int Requantize_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& const int* intptr = bottom_blob.row(i); signed char* ptr = top_blob.row(i); -#if __AVX__ +#if __AVX2__ __m256 _scale_in = scale_in_data_size == 1 ? _mm256_set1_ps(scale_in_data[0]) : _mm256_loadu_ps((const float*)scale_in_data + i * 8); __m256 _scale_out = scale_out_data_size == 1 ? _mm256_set1_ps(scale_out_data[0]) : _mm256_loadu_ps((const float*)scale_out_data + i * 8); __m256 _bias = bias_data_size == 1 ? _mm256_set1_ps(bias_data[0]) : _mm256_loadu_ps((const float*)bias_data + i * 8); @@ -554,9 +554,9 @@ int Requantize_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& for (int j = 0; j < w; j++) { -#if __AVX__ +#if __AVX2__ __m256 _v = _mm256_cvtepi32_ps(_mm256_loadu_si256((const __m256i*)intptr)); - _v = _mm256_fmadd_ps(_v, _scale_in, _bias); + _v = _mm256_comp_fmadd_ps(_v, _scale_in, _bias); _v = activation_avx(_v, activation_type, activation_params); _v = _mm256_mul_ps(_v, _scale_out); *(int64_t*)ptr = float2int8_avx(_v); @@ -598,7 +598,7 @@ int Requantize_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& const int* intptr = bottom_blob.channel(q); signed char* ptr = top_blob.channel(q); -#if __AVX__ +#if __AVX2__ __m256 _scale_in = scale_in_data_size == 1 ? _mm256_set1_ps(scale_in_data[0]) : _mm256_loadu_ps((const float*)scale_in_data + q * 8); __m256 _scale_out = scale_out_data_size == 1 ? _mm256_set1_ps(scale_out_data[0]) : _mm256_loadu_ps((const float*)scale_out_data + q * 8); #else @@ -610,7 +610,7 @@ int Requantize_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& for (int i = 0; i < size; i++) { -#if __AVX__ +#if __AVX2__ __m256 _v = _mm256_cvtepi32_ps(_mm256_loadu_si256((const __m256i*)intptr)); _v = _mm256_mul_ps(_v, _scale_in); _v = activation_avx(_v, activation_type, activation_params); @@ -641,7 +641,7 @@ int Requantize_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& const int* intptr = bottom_blob.channel(q); signed char* ptr = top_blob.channel(q); -#if __AVX__ +#if __AVX2__ __m256 _scale_in = scale_in_data_size == 1 ? _mm256_set1_ps(scale_in_data[0]) : _mm256_loadu_ps((const float*)scale_in_data + q * 8); __m256 _scale_out = scale_out_data_size == 1 ? _mm256_set1_ps(scale_out_data[0]) : _mm256_loadu_ps((const float*)scale_out_data + q * 8); __m256 _bias = bias_data_size == 1 ? _mm256_set1_ps(bias_data[0]) : _mm256_loadu_ps((const float*)bias_data + q * 8); @@ -656,9 +656,9 @@ int Requantize_x86::forward(const Mat& bottom_blob, Mat& top_blob, const Option& for (int i = 0; i < size; i++) { -#if __AVX__ +#if __AVX2__ __m256 _v = _mm256_cvtepi32_ps(_mm256_loadu_si256((const __m256i*)intptr)); - _v = _mm256_fmadd_ps(_v, _scale_in, _bias); + _v = _mm256_comp_fmadd_ps(_v, _scale_in, _bias); _v = activation_avx(_v, activation_type, activation_params); _v = _mm256_mul_ps(_v, _scale_out); *(int64_t*)ptr = float2int8_avx(_v); diff --git a/src/layer/x86/scale_x86.cpp b/src/layer/x86/scale_x86.cpp index c3cd70328..280da0bea 100644 --- a/src/layer/x86/scale_x86.cpp +++ b/src/layer/x86/scale_x86.cpp @@ -20,7 +20,7 @@ #include #endif // __AVX__ #endif // __SSE2__ - +#include "x86_usability.h" namespace ncnn { Scale_x86::Scale_x86() @@ -58,7 +58,7 @@ int Scale_x86::forward_inplace(std::vector& bottom_top_blobs, const Option& __m256 _p = _mm256_loadu_ps(ptr); __m256 _s = _mm256_loadu_ps(scale + i * 8); __m256 _bias = _mm256_loadu_ps(bias + i * 8); - _p = _mm256_fmadd_ps(_p, _s, _bias); + _p = _mm256_comp_fmadd_ps(_p, _s, _bias); _mm256_storeu_ps(ptr, _p); } } @@ -94,7 +94,7 @@ int Scale_x86::forward_inplace(std::vector& bottom_top_blobs, const Option& for (int j = 0; j < w; j++) { __m256 _p = _mm256_loadu_ps(ptr); - _p = _mm256_fmadd_ps(_p, _s, _bias); + _p = _mm256_comp_fmadd_ps(_p, _s, _bias); _mm256_storeu_ps(ptr, _p); ptr += 8; @@ -140,7 +140,7 @@ int Scale_x86::forward_inplace(std::vector& bottom_top_blobs, const Option& for (int i = 0; i < size; i++) { __m256 _p = _mm256_loadu_ps(ptr); - _p = _mm256_fmadd_ps(_p, _s, _bias); + _p = _mm256_comp_fmadd_ps(_p, _s, _bias); _mm256_storeu_ps(ptr, _p); ptr += 8; @@ -335,7 +335,7 @@ int Scale_x86::forward_inplace(std::vector& bottom_top_blobs, const Option& for (; nn > 0; nn--) { __m256 _p = _mm256_loadu_ps(ptr); - _p = _mm256_fmadd_ps(_p, _s, _bias); + _p = _mm256_comp_fmadd_ps(_p, _s, _bias); _mm256_storeu_ps(ptr, _p); ptr += 8; diff --git a/src/layer/x86/x86_activation.h b/src/layer/x86/x86_activation.h index af35b679e..d0944c6ca 100644 --- a/src/layer/x86/x86_activation.h +++ b/src/layer/x86/x86_activation.h @@ -138,7 +138,11 @@ static inline __m256 tanh_avx(__m256 inputs) { const __m256 one = _mm256_set1_ps(1.0f); const __m256 two = _mm256_set1_ps(2.0f); +#if __AVX2__ return _mm256_fmsub_ps(sigmoid_avx(_mm256_mul_ps(inputs, two)), two, one); +#else + return _mm256_sub_ps(_mm256_mul_ps(sigmoid_avx(_mm256_mul_ps(inputs, two)), two), one); +#endif } static inline __m256 mish_avx(__m256 inputs) diff --git a/src/layer/x86/x86_usability.h b/src/layer/x86/x86_usability.h index 5a6682eb4..012ace3cc 100644 --- a/src/layer/x86/x86_usability.h +++ b/src/layer/x86/x86_usability.h @@ -119,14 +119,35 @@ static inline __m128i float2int8_sse(const __m128& _v0, const __m128& _v1, const #if __AVX__ #include +#ifndef __AVX2__ +static inline __m256 _mm256_comp_fmadd_ps(__m256 _a, const __m256 _b, const __m256 _c) +{ + return _mm256_add_ps(_mm256_mul_ps(_a, _b), _c); +} +static inline __m128 _mm_comp_fmadd_ps(__m128 _a, const __m128 _b, const __m128 _c) +{ + return _mm_add_ps(_mm_mul_ps(_a, _b), _c); +} +#else +static inline __m128 _mm_comp_fmadd_ps(__m128 _a, const __m128 _b, const __m128 _c) +{ + return _mm_fmadd_ps(_a, _b, _c); +} +static inline __m256 _mm256_comp_fmadd_ps(__m256 _a, const __m256 _b, const __m256 _c) +{ + return _mm256_fmadd_ps(_a, _b, _c); +} +#endif +#if __AVX2__ static inline __m256 loadfp16(const unsigned short* ptr) { return _mm256_cvtph_ps(_mm_lddqu_si128((__m128i*)(ptr))); } +#endif static inline __m256 _mm256_fmadd_1_ps(__m256 a, __m256 b, float c) { - return _mm256_fmadd_ps(b, _mm256_set1_ps(c), a); + return _mm256_comp_fmadd_ps(b, _mm256_set1_ps(c), a); } static inline __m256 _mm256_fmrsub_1_ps(__m256 a, __m256 b, float c) @@ -212,7 +233,7 @@ static inline float _mm256_reduce_add_ps(__m256 x) /* Conversion to float is a no-op on x86-64 */ return _mm_cvtss_f32(x32); } - +#if __AVX2__ static inline int64_t float2int8_avx(const __m256& _v0) { __m256i _v0_i = _mm256_cvtps_epi32(_mm256_round_ps(_v0, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC)); @@ -249,10 +270,21 @@ static inline __m128i float2int8_avx(const __m256& _v0, const __m256& _v1) return _mm256_extractf128_si256(_v8, 0); } +#else +static inline void float2int8_loop(const __m256& _v0, signed char* output) +{ + float data_0[8]; + _mm256_storeu_ps(data_0, _v0); + for (int i = 0; i < 8; i++) + { + output[i] = float2int8(data_0[i]); + } +} +#endif -static inline void _mm256_fmadd_ps4(__m256& _sum, - const __m256& _w0, const __m256& _w1, const __m256& _w2, const __m256& _w3, - const __m256& _v0, const __m256& _v1, const __m256& _v2, const __m256& _v3) +static inline void _mm256_comp_fmadd_ps4(__m256& _sum, + const __m256& _w0, const __m256& _w1, const __m256& _w2, const __m256& _w3, + const __m256& _v0, const __m256& _v1, const __m256& _v2, const __m256& _v3) { __m256 _mul0 = _mm256_mul_ps(_w0, _v0); __m256 _mul1 = _mm256_mul_ps(_w1, _v1); @@ -264,14 +296,15 @@ static inline void _mm256_fmadd_ps4(__m256& _sum, _sum = _mm256_add_ps(_sum, _sum0123); } -static inline void _mm256_fmadd_ps8(__m256& _sum, - const __m256& _w0, const __m256& _w1, const __m256& _w2, const __m256& _w3, const __m256& _w4, const __m256& _w5, const __m256& _w6, const __m256& _w7, - const __m256& _v0, const __m256& _v1, const __m256& _v2, const __m256& _v3, const __m256& _v4, const __m256& _v5, const __m256& _v6, const __m256& _v7) +static inline void _mm256_comp_fmadd_ps8(__m256& _sum, + const __m256& _w0, const __m256& _w1, const __m256& _w2, const __m256& _w3, const __m256& _w4, const __m256& _w5, const __m256& _w6, const __m256& _w7, + const __m256& _v0, const __m256& _v1, const __m256& _v2, const __m256& _v3, const __m256& _v4, const __m256& _v5, const __m256& _v6, const __m256& _v7) { - _mm256_fmadd_ps4(_sum, _w0, _w1, _w2, _w3, _v0, _v1, _v2, _v3); + _mm256_comp_fmadd_ps4(_sum, _w0, _w1, _w2, _w3, _v0, _v1, _v2, _v3); - _mm256_fmadd_ps4(_sum, _w4, _w5, _w6, _w7, _v4, _v5, _v6, _v7); + _mm256_comp_fmadd_ps4(_sum, _w4, _w5, _w6, _w7, _v4, _v5, _v6, _v7); } + #endif // __AVX__ #endif // __SSE2__ diff --git a/src/layer/x86/yolov3detectionoutput_x86.cpp b/src/layer/x86/yolov3detectionoutput_x86.cpp index 2d9ca2212..6e61e3c43 100644 --- a/src/layer/x86/yolov3detectionoutput_x86.cpp +++ b/src/layer/x86/yolov3detectionoutput_x86.cpp @@ -54,9 +54,9 @@ int Yolov3DetectionOutput_x86::forward(const std::vector& bottom_blobs, std size_t mask_offset = b * num_box; int net_w = (int)(anchors_scale[b] * w); int net_h = (int)(anchors_scale[b] * h); - //printf("%d %d\n", net_w, net_h); +//printf("%d %d\n", net_w, net_h); - //printf("%d %d %d\n", w, h, channels); +//printf("%d %d %d\n", w, h, channels); #pragma omp parallel for num_threads(opt.num_threads) for (int pp = 0; pp < num_box; pp++) { @@ -107,7 +107,7 @@ int Yolov3DetectionOutput_x86::forward(const std::vector& bottom_blobs, std float* ptr = ((float*)scores.data) + i * w + j; float* end = ptr + num_class * cs; int q = 0; -#if __AVX__ +#if __AVX2__ float* end8 = ptr + (num_class & -8) * cs; unsigned long index; diff --git a/src/layer_registry.h.in b/src/layer_registry.h.in index f544dd43d..9ca32384f 100644 --- a/src/layer_registry.h.in +++ b/src/layer_registry.h.in @@ -11,6 +11,11 @@ static const layer_registry_entry layer_registry_avx2[] = { @layer_registry_avx2@ }; #endif // NCNN_RUNTIME_CPU && NCNN_AVX2 +#if NCNN_RUNTIME_CPU && NCNN_AVX +static const layer_registry_entry layer_registry_avx[] = { +@layer_registry_avx@ +}; +#endif // NCNN_RUNTIME_CPU && NCNN_AVX #if NCNN_RUNTIME_CPU && NCNN_ARM82 && !__APPLE__ static const layer_registry_entry layer_registry_arm82[] = { diff --git a/src/net.cpp b/src/net.cpp index 3ff124243..927411f0b 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -800,8 +800,8 @@ int NetPrivate::convert_layout(Mat& bottom_blob, const Layer* layer, const Optio { if (elembits == 32) { -#if NCNN_AVX2 - if (elemcount % 8 == 0 && ncnn::cpu_support_x86_avx2()) +#if (NCNN_AVX2 || NCNN_AVX) + if (elemcount % 8 == 0 && (ncnn::cpu_support_x86_avx2() || ncnn::cpu_support_x86_avx())) dst_elempack = 8; else if (elemcount % 4 == 0) dst_elempack = 4; diff --git a/src/platform.h.in b/src/platform.h.in index 2c5eedddc..6f8723344 100644 --- a/src/platform.h.in +++ b/src/platform.h.in @@ -30,6 +30,7 @@ #cmakedefine01 NCNN_VULKAN #cmakedefine01 NCNN_RUNTIME_CPU #cmakedefine01 NCNN_AVX2 +#cmakedefine01 NCNN_AVX #cmakedefine01 NCNN_ARM82 #cmakedefine01 NCNN_ARM82DOT #cmakedefine01 NCNN_MSA diff --git a/tests/testutil.h b/tests/testutil.h index 58cca68c6..d0efdaec7 100644 --- a/tests/testutil.h +++ b/tests/testutil.h @@ -421,8 +421,8 @@ int test_layer_cpu(int typeindex, const ncnn::ParamDict& pd, const std::vector