diff --git a/src/gpu.cpp b/src/gpu.cpp index 00a711d09..c2f489544 100644 --- a/src/gpu.cpp +++ b/src/gpu.cpp @@ -98,6 +98,7 @@ struct layer_shader_registry_entry static const layer_shader_registry_entry layer_shader_registry[] = { #include "layer_shader_registry.h" + }; static const int layer_shader_registry_entry_count = sizeof(layer_shader_registry) / sizeof(layer_shader_registry_entry); @@ -3734,7 +3735,7 @@ int VulkanDevice::create_pipeline_layout(int push_constant_count, VkDescriptorSe return 0; } -int VulkanDevice::create_pipeline(VkShaderModule shader_module, VkPipelineLayout pipeline_layout, const std::vector& specializations, uint32_t subgroup_size, VkPipeline* pipeline) const +int VulkanDevice::create_pipeline(VkShaderModule shader_module, VkPipelineLayout pipeline_layout, const std::vector& specializations, uint32_t subgroup_size, VkPipeline* pipeline, VkPipelineCache pipeline_cache) const { const int specialization_count = specializations.size(); @@ -3792,7 +3793,7 @@ int VulkanDevice::create_pipeline(VkShaderModule shader_module, VkPipelineLayout computePipelineCreateInfo.basePipelineHandle = 0; computePipelineCreateInfo.basePipelineIndex = 0; - VkResult ret = vkCreateComputePipelines(d->device, 0, 1, &computePipelineCreateInfo, 0, pipeline); + VkResult ret = vkCreateComputePipelines(d->device, pipeline_cache, 1, &computePipelineCreateInfo, 0, pipeline); if (ret != VK_SUCCESS) { NCNN_LOGE("vkCreateComputePipelines failed %d", ret); @@ -3871,6 +3872,18 @@ int VulkanDevice::create_descriptor_update_template(int binding_count, const int return 0; } +int VulkanDevice::create_pipeline_cache(const VkPipelineCacheCreateInfo* pCreateInfo, const VkAllocationCallbacks* pAllocator, VkPipelineCache* pPipelineCache) const +{ + VkResult ret = vkCreatePipelineCache(d->device, pCreateInfo, pAllocator, pPipelineCache); + if (ret != VK_SUCCESS) + { + NCNN_LOGE("vkCreatePipelineCache failed %d", ret); + return -1; + } + + return 0; +} + uint32_t VulkanDevice::find_memory_index(uint32_t memory_type_bits, VkFlags required, VkFlags preferred, VkFlags preferred_not) const { const VkPhysicalDeviceMemoryProperties& memory_properties = info.physicalDeviceMemoryProperties(); diff --git a/src/gpu.h b/src/gpu.h index 7863b2e21..c3256339b 100644 --- a/src/gpu.h +++ b/src/gpu.h @@ -419,8 +419,9 @@ public: // helper for creating pipeline int create_descriptorset_layout(int binding_count, const int* binding_types, VkDescriptorSetLayout* descriptorset_layout) const; int create_pipeline_layout(int push_constant_count, VkDescriptorSetLayout descriptorset_layout, VkPipelineLayout* pipeline_layout) const; - int create_pipeline(VkShaderModule shader_module, VkPipelineLayout pipeline_layout, const std::vector& specializations, uint32_t subgroup_size, VkPipeline* pipeline) const; + int create_pipeline(VkShaderModule shader_module, VkPipelineLayout pipeline_layout, const std::vector& specializations, uint32_t subgroup_size, VkPipeline* pipeline, VkPipelineCache pipeline_cache = 0) const; int create_descriptor_update_template(int binding_count, const int* binding_types, VkDescriptorSetLayout descriptorset_layout, VkPipelineLayout pipeline_layout, VkDescriptorUpdateTemplateKHR* descriptor_update_template) const; + int create_pipeline_cache(const VkPipelineCacheCreateInfo* pCreateInfo, const VkAllocationCallbacks* pAllocator, VkPipelineCache* pPipelineCache) const; uint32_t find_memory_index(uint32_t memory_type_bits, VkFlags required, VkFlags preferred, VkFlags preferred_not) const; bool is_mappable(uint32_t memory_type_index) const; diff --git a/src/pipelinecache.cpp b/src/pipelinecache.cpp index 1bd274514..95438682f 100644 --- a/src/pipelinecache.cpp +++ b/src/pipelinecache.cpp @@ -110,8 +110,37 @@ public: ShaderInfo shader_info; // TODO use pointer ? }; + struct spv_param + { + union + { + struct + { + int32_t shader_type_index; + uint32_t opt_bits; + }; + uint64_t d0; + }; + }; + + struct pipeline_cache_header + { + uint32_t magic = 0x5a545546; + uint32_t vendorID; // VkPhysicalDeviceProperties::vendorID + uint32_t deviceID; // VkPhysicalDeviceProperties::deviceID + uint32_t driverVersion; // VkPhysicalDeviceProperties::driverVersion + uint8_t uuid[VK_UUID_SIZE]; // VkPhysicalDeviceProperties::pipelineCacheUUID + + uint32_t spv_size; // size of spirv data + uint32_t pipeline_cache_size; + }; + mutable std::vector cache_digests; mutable std::vector cache_artifacts; + + VkPipelineCache vk_pipeline_cache = 0; // VK_NULL_HANDLE + mutable std::vector > > cache_spirv_module; // digest(index,opt) -> spirv data + mutable Mutex cache_lock; }; @@ -160,6 +189,18 @@ PipelineCachePrivate::pipeline_cache_digest::pipeline_cache_digest(int _shader_t PipelineCache::PipelineCache(const VulkanDevice* _vkdev) : vkdev(_vkdev), d(new PipelineCachePrivate) { + VkPipelineCacheCreateInfo pipelineCacheCreateInfo{}; + pipelineCacheCreateInfo.sType = VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO; + pipelineCacheCreateInfo.initialDataSize = 0; // zeros for empty cache + pipelineCacheCreateInfo.pInitialData = nullptr; + + int ret = 0; + ret = _vkdev->create_pipeline_cache(&pipelineCacheCreateInfo, 0, &d->vk_pipeline_cache); + if (ret != 0) + { + NCNN_LOGE("create_pipeline_cache failed %d", ret); + d->vk_pipeline_cache = 0; + } } PipelineCache::~PipelineCache() @@ -218,6 +259,12 @@ void PipelineCache::clear() d->cache_digests.clear(); d->cache_artifacts.clear(); + + if (d->vk_pipeline_cache) + { + vkDestroyPipelineCache(vkdev->vkdevice(), d->vk_pipeline_cache, 0); + d->vk_pipeline_cache = 0; + } } int PipelineCache::get_pipeline(const uint32_t* spv_data, size_t spv_data_size, const std::vector& specializations, @@ -381,18 +428,289 @@ int PipelineCache::get_pipeline(int shader_type_index, const Option& opt, const return 0; } +int PipelineCache::save_cache(std::vector& buf) const +{ + if (!vkdev) + { + NCNN_LOGE("vkdev is null"); + return -1; + } + MutexLockGuard lock(d->cache_lock); + + PipelineCachePrivate::pipeline_cache_header header; + + // Platform information + header.vendorID = vkdev->info.vendor_id(); + header.deviceID = vkdev->info.device_id(); + header.driverVersion = vkdev->info.driver_version(); + memcpy(header.uuid, vkdev->info.pipeline_cache_uuid(), VK_UUID_SIZE); + + header.spv_size = d->cache_spirv_module.size(); + + size_t buf_size = 0; + if (vkGetPipelineCacheData(vkdev->vkdevice(), d->vk_pipeline_cache, &buf_size, nullptr) != VK_SUCCESS) + { + NCNN_LOGE("vkGetPipelineCacheData failed"); + return -1; + } + header.pipeline_cache_size = (uint32_t)buf_size; + + std::vector pipe_data(header.pipeline_cache_size); + if (vkGetPipelineCacheData(vkdev->vkdevice(), d->vk_pipeline_cache, &buf_size, pipe_data.data()) != VK_SUCCESS) + { + NCNN_LOGE("vkGetPipelineCacheData failed"); + return -1; + } + + buf.resize(sizeof(header)); + memcpy(buf.data(), &header, sizeof(header)); + + // spv_digest and spv_data + for (size_t i = 0; i < d->cache_spirv_module.size(); i++) + { + const PipelineCachePrivate::spv_param& sd = d->cache_spirv_module[i].first; + const std::vector& spv_data = d->cache_spirv_module[i].second; + uint32_t size = (uint32_t)spv_data.size(); + + size_t current_buf_size = buf.size(); + buf.resize(current_buf_size + sizeof(sd) + sizeof(size) + spv_data.size() * sizeof(uint32_t)); + + memcpy(buf.data() + current_buf_size, &sd, sizeof(sd)); + current_buf_size += sizeof(sd); + memcpy(buf.data() + current_buf_size, &size, sizeof(size)); + current_buf_size += sizeof(size); + + memcpy(buf.data() + current_buf_size, spv_data.data(), spv_data.size() * sizeof(uint32_t)); + } + + buf.insert(buf.end(), pipe_data.begin(), pipe_data.end()); + return 0; +} + +int PipelineCache::load_cache(const std::vector& buf) const +{ + if (!vkdev) + { + NCNN_LOGE("vkdev is null"); + return -1; + } + MutexLockGuard lock(d->cache_lock); + + // Corrected struct name to pipeline_cache_header (lowercase h) + if (buf.size() < sizeof(PipelineCachePrivate::pipeline_cache_header)) + { + NCNN_LOGE("Invalid cache buffer size: too small for header"); + return -1; + } + + PipelineCachePrivate::pipeline_cache_header header; + memcpy(&header, buf.data(), sizeof(header)); + + // Validate magic number + if (header.magic != 0x5a545546) + { + NCNN_LOGE("Invalid cache magic number"); + return -1; + } + + // Validate platform information for compatibility + if (header.vendorID != vkdev->info.vendor_id() || header.deviceID != vkdev->info.device_id() || header.driverVersion != vkdev->info.driver_version() || memcmp(header.uuid, vkdev->info.pipeline_cache_uuid(), VK_UUID_SIZE) != 0) + { + NCNN_LOGE("Cache platform mismatch, might be incompatible."); + return -1; + } + + size_t current_offset = sizeof(header); + + // Load SPIR-V data and associated spv_param + d->cache_spirv_module.reserve(header.spv_size); + + for (uint32_t i = 0; i < header.spv_size; ++i) + { + if (current_offset + sizeof(PipelineCachePrivate::spv_param) + sizeof(uint32_t) > buf.size()) + { + NCNN_LOGE("Invalid cache buffer size: incomplete spv_param or size for entry %u", i); + return -1; + } + + PipelineCachePrivate::spv_param sd; + memcpy(&sd, buf.data() + current_offset, sizeof(sd)); + current_offset += sizeof(sd); + + uint32_t spv_vec_size_uint32; // Size in uint32_t units + memcpy(&spv_vec_size_uint32, buf.data() + current_offset, sizeof(spv_vec_size_uint32)); + current_offset += sizeof(spv_vec_size_uint32); + + size_t spv_data_byte_size = spv_vec_size_uint32 * sizeof(uint32_t); + + if (current_offset + spv_data_byte_size > buf.size()) + { + NCNN_LOGE("Invalid cache buffer size: incomplete spv_data for entry %u", i); + return -1; + } + + std::vector spirv_data(spv_vec_size_uint32); + memcpy(spirv_data.data(), buf.data() + current_offset, spv_data_byte_size); + current_offset += spv_data_byte_size; + + d->cache_spirv_module.push_back({sd, spirv_data}); + } + + // Load Vulkan Pipeline Cache Data + if (current_offset + header.pipeline_cache_size > buf.size()) + { + NCNN_LOGE("Invalid cache buffer size: incomplete pipeline cache data"); + return -1; + } + + if (d->vk_pipeline_cache) + { + vkDestroyPipelineCache(vkdev->vkdevice(), d->vk_pipeline_cache, 0); + d->vk_pipeline_cache = 0; + } + + VkPipelineCacheCreateInfo pipelineCacheCreateInfo{}; + pipelineCacheCreateInfo.sType = VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO; + pipelineCacheCreateInfo.initialDataSize = header.pipeline_cache_size; + pipelineCacheCreateInfo.pInitialData = buf.data() + current_offset; + + int ret = vkdev->create_pipeline_cache(&pipelineCacheCreateInfo, 0, &d->vk_pipeline_cache); + if (ret != 0) + { + NCNN_LOGE("create_pipeline_cache with initial data failed %d", ret); + d->vk_pipeline_cache = 0; + return -1; + } + + return 0; +} + +int PipelineCache::save_cache(FILE* fp) const +{ + if (!fp) + { + NCNN_LOGE("Invalid FILE pointer for saving cache."); + return -1; + } + + std::vector buf; + int ret = save_cache(buf); + if (ret != 0) + { + NCNN_LOGE("Failed to get cache data into buffer for saving to file."); + return ret; + } + + if (fwrite(buf.data(), 1, buf.size(), fp) != buf.size()) + { + NCNN_LOGE("Failed to write cache data to file."); + return -1; + } + + return 0; +} + +int PipelineCache::load_cache(FILE* fp) const +{ + if (!fp) + { + NCNN_LOGE("Invalid FILE pointer for loading cache."); + return -1; + } + + fseek(fp, 0, SEEK_END); + long file_size = ftell(fp); + fseek(fp, 0, SEEK_SET); + + if (file_size < 0) + { + NCNN_LOGE("Failed to determine file size for loading cache."); + return -1; + } + + std::vector buf(file_size); + if (fread(buf.data(), 1, file_size, fp) != (size_t)file_size) + { + NCNN_LOGE("Failed to read cache data from file."); + return -1; + } + + return load_cache(buf); +} + +int PipelineCache::save_cache(const char* filename) const +{ + if (!filename) + { + NCNN_LOGE("Invalid filename for saving cache."); + return -1; + } + + FILE* fp = fopen(filename, "wb"); + if (!fp) + { + NCNN_LOGE("Failed to open file %s for writing cache.", filename); + return -1; + } + + int ret = save_cache(fp); + fclose(fp); + + return ret; +} + +int PipelineCache::load_cache(const char* filename) const +{ + if (!filename) + { + NCNN_LOGE("Invalid filename for loading cache."); + return -1; + } + + FILE* fp = fopen(filename, "rb"); + if (!fp) + { + NCNN_LOGE("Failed to open file %s for reading cache.", filename); + return -1; + } + + int ret = load_cache(fp); + fclose(fp); + + return ret; +} + int PipelineCache::create_shader_module(int shader_type_index, const Option& opt, uint32_t local_size_x, uint32_t local_size_y, uint32_t local_size_z, VkShaderModule* _shader_module, ShaderInfo& si) const { + uint32_t opt_bits = 0 << 7 + | opt.use_fp16_packed << 6 + | opt.use_fp16_storage << 5 + | opt.use_fp16_arithmetic << 4 + | opt.use_int8_storage << 3 + | opt.use_int8_arithmetic << 2; + std::vector spirv; - int retc = compile_spirv_module(shader_type_index, opt, spirv); + int retc = 0; + + for (int i = 0; i < d->cache_spirv_module.size(); i++) + { + if (d->cache_spirv_module[i].first.d0 == PipelineCachePrivate::spv_param({shader_type_index, opt_bits}).d0) // hit cache + { + spirv = d->cache_spirv_module[i].second; + goto hit_cache; + } + } + + retc = compile_spirv_module(shader_type_index, opt, spirv); if (retc != 0) { NCNN_LOGE("compile_spirv_module failed %d", retc); return -1; } - + d->cache_spirv_module.push_back({{shader_type_index, opt_bits}, spirv}); +hit_cache: const uint32_t* spv_data = spirv.data(); size_t spv_data_size = spirv.size() * 4; @@ -445,7 +763,7 @@ int PipelineCache::new_pipeline(VkShaderModule shader_module, const ShaderInfo& if (ret != 0) goto ERROR_PipelineCache; - ret = vkdev->create_pipeline(shader_module, pipeline_layout, specializations, subgroup_size, &pipeline); + ret = vkdev->create_pipeline(shader_module, pipeline_layout, specializations, subgroup_size, &pipeline, d->vk_pipeline_cache); if (ret != 0) goto ERROR_PipelineCache; diff --git a/src/pipelinecache.h b/src/pipelinecache.h index b93c0cfd8..9962b645f 100644 --- a/src/pipelinecache.h +++ b/src/pipelinecache.h @@ -42,6 +42,16 @@ public: VkDescriptorUpdateTemplateKHR* descriptor_update_template, ShaderInfo& shader_info) const; + int save_cache(std::vector& buf) const; + int load_cache(const std::vector& buf) const; + +#ifdef NCNN_STDIO + int save_cache(FILE* fp) const; + int load_cache(FILE* fp) const; + int save_cache(const char* fp) const; + int load_cache(const char* fp) const; +#endif + protected: int create_shader_module(int shader_type_index, const Option& opt, uint32_t local_size_x, uint32_t local_size_y, uint32_t local_size_z, diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 5a0940e88..d442c7735 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -65,6 +65,7 @@ ncnn_add_test(paramdict) if(NCNN_VULKAN) ncnn_add_test(command) + ncnn_add_test(pipecache) endif() if(CMAKE_SYSTEM_NAME STREQUAL "Emscripten") diff --git a/tests/test_pipecache.cpp b/tests/test_pipecache.cpp new file mode 100644 index 000000000..8148b07da --- /dev/null +++ b/tests/test_pipecache.cpp @@ -0,0 +1,235 @@ +// Copyright 2021 Tencent +// SPDX-License-Identifier: BSD-3-Clause + +#include "datareader.h" +#include "gpu.h" +#include "mat.h" +#include "net.h" +#include "pipelinecache.h" +#include "testutil.h" +#include "benchmark.h" + +#include +#include +#include + +class DataReaderFromEmpty : public ncnn::DataReader +{ +public: + virtual int scan(const char* format, void* p) const + { + (void)format; // unused + (void)p; // unused + return 0; + } + virtual size_t read(void* buf, size_t size) const + { + memset(buf, 0, size); + return size; + } +}; + +static int warmup_gpu_pipecache() +{ + printf("==================================================\n"); + printf(" Warmup: Testing Basic Cache IO \n"); + printf("==================================================\n"); + + ncnn::Net net; + net.opt.use_vulkan_compute = true; + + net.load_param_mem("7767517\n2 2\nInput input0 0 1 input0\nSigmoid sigmoid0 1 1 input0 output0"); + net.load_model((unsigned char*)""); + + ncnn::Mat input0 = RandomMat(224, 224); + ncnn::Mat output0; + { + ncnn::Extractor ex = net.create_extractor(); + ex.input("input0", input0); + ex.extract("output0", output0); + } + + if (output0.empty()) + { + fprintf(stderr, "Warmup failed: initial extraction failed.\n"); + return -1; + } + + const char* cache_path = "./sigmoid_pipecache.bin"; + if (net.opt.pipeline_cache->save_cache(cache_path) != 0) + { + fprintf(stderr, "Warmup failed: could not save pipeline cache to %s\n", cache_path); + return -1; + } + printf("Warmup: Pipeline cache saved successfully.\n"); + + ncnn::Net net2; + net2.opt.use_vulkan_compute = true; + + net2.load_param_mem("7767517\n2 2\nInput input0 0 1 input0\nSigmoid sigmoid0 1 1 input0 output0"); + net2.opt.pipeline_cache = new ncnn::PipelineCache(net2.vulkan_device()); + if (net2.opt.pipeline_cache->load_cache(cache_path) != 0) + { + fprintf(stderr, "Warmup failed: could not load pipeline cache from %s\n", cache_path); + return -1; + } + printf("Warmup: Pipeline cache loaded successfully.\n"); + net2.load_model((unsigned char*)""); + + ncnn::Mat output0_2; + { + ncnn::Extractor ex2 = net2.create_extractor(); + ex2.input("input0", input0); + ex2.extract("output0", output0_2); + } + + if (output0_2.empty()) + { + fprintf(stderr, "Warmup failed: extraction after loading cache failed.\n"); + return -1; + } + + if (CompareMat(output0, output0_2, 0.001) != 0) + { + fprintf(stderr, "Warmup failed: output mismatch after loading cache.\n"); + return -1; + } + + printf("Warmup PASSED: Outputs are identical.\n"); + return 0; +} + +static int test_gpu_pipecache_performance() +{ + ncnn::Mat output_no_cache; + double time_no_cache = 0; + + const char* cache_path = "./mobilenet_pipecache.bin"; + DataReaderFromEmpty dr; + ncnn::Mat input = RandomMat(224, 224, 3); + +#ifdef __EMSCRIPTEN__ +#define MODEL_DIR "/working" +#else +#define MODEL_DIR "../../benchmark" +#endif + + // ------------------------------------------------- + // 1. Without cache + // ------------------------------------------------- + printf("\n==================================================\n"); + printf(" Performance Test: Without Pipeline Cache \n"); + printf("==================================================\n"); + { + ncnn::Net net_no_cache; + net_no_cache.opt.use_vulkan_compute = true; + + auto start = ncnn::get_current_time(); + + net_no_cache.load_param(MODEL_DIR "/mobilenet_v3.param"); + net_no_cache.load_model(dr); + + auto end = ncnn::get_current_time(); + time_no_cache = end - start; + printf("Model loading time without cache: %lf ms\n", time_no_cache); + + ncnn::Extractor ex = net_no_cache.create_extractor(); + ex.input("data", input); + ex.extract("output", output_no_cache); + + if (output_no_cache.empty()) + { + fprintf(stderr, "Test failed: extraction without cache failed.\n"); + return -1; + } + + // save cache + if (net_no_cache.opt.pipeline_cache->save_cache(cache_path) != 0) + { + fprintf(stderr, "Test failed: could not save pipeline cache to %s\n", cache_path); + return -1; + } + printf("Pipeline cache generated and saved to %s\n", cache_path); + } + + // ------------------------------------------------- + // 2. With Cache + // ------------------------------------------------- + ncnn::Mat output_with_cache; + double time_with_cache = 0; + printf("\n==================================================\n"); + printf(" Performance Test: With Pipeline Cache \n"); + printf("==================================================\n"); + { + ncnn::Net net_with_cache; + net_with_cache.opt.use_vulkan_compute = true; + + auto start = ncnn::get_current_time(); + + net_with_cache.load_param(MODEL_DIR "/mobilenet_v3.param"); // after load param vkdev will be create + net_with_cache.opt.pipeline_cache = new ncnn::PipelineCache(net_with_cache.vulkan_device()); + // load from cache + if (net_with_cache.opt.pipeline_cache->load_cache(cache_path) != 0) + { + fprintf(stderr, "Test failed: could not load pipeline cache from %s\n", cache_path); + return -1; + } + net_with_cache.load_model(dr); + + auto end = ncnn::get_current_time(); + time_with_cache = end - start; + printf("Model loading time with cache: %lf ms\n", time_with_cache); + + ncnn::Extractor ex2 = net_with_cache.create_extractor(); + ex2.input("data", input); + ex2.extract("output", output_with_cache); + + if (output_with_cache.empty()) + { + fprintf(stderr, "Test failed: extraction with cache failed.\n"); + return -1; + } + delete net_with_cache.opt.pipeline_cache; // clean up pipeline cache by hand + } + + // ------------------------------------------------- + // 3. Verification + // ------------------------------------------------- + printf("\n==================================================\n"); + printf(" Verification and Summary \n"); + printf("==================================================\n"); + + bool is_output_same = (CompareMat(output_no_cache, output_with_cache, 0.001) == 0); + + printf("Output verification: %s\n", (is_output_same ? "SUCCESS" : "FAILURE")); + printf("--------------------------------------------------\n"); + printf("Performance Summary:\n"); + printf(" - Without Cache: %f ms\n", time_no_cache); + printf(" - With Cache: %f ms\n", time_with_cache); + + if (time_no_cache > 0) + { + double speedup = (time_no_cache - time_with_cache) / time_no_cache * 100; + printf(" - Speedup: %f%%\n", speedup); + } + + if (!is_output_same) + { + fprintf(stderr, "\nTest FAILED due to output mismatch.\n"); + return -1; + } + + printf("\nTest PASSED.\n"); + return 0; +} + +int main() +{ + // warming up + if (warmup_gpu_pipecache() != 0) + { + return -1; + } + + return test_gpu_pipecache_performance(); +}