Fix DumpParameters issue when only need dump input data

Fix code warnings in debug

Fix code warnings2

mindspore/ccsrc/debug/data_dump/cpu_e2e_dump.cc

@@ -179,6 +179,10 @@ void CPUE2eDump::DumpSingleAnfNode(const AnfNodePtr &anf_node, const size_t outp
 
 void CPUE2eDump::DumpParameters(const session::KernelGraph *graph, uint32_t graph_id) {
   MS_EXCEPTION_IF_NULL(graph);
+  auto &dump_json_parser = DumpJsonParser::GetInstance();
+  if (!dump_json_parser.OutputNeedDump()) {
+    return;
+  }
   MS_LOG(INFO) << "Start e2e dump parameters.";
   const std::string &dump_path = GenerateDumpPath(graph_id);
 
@@ -198,6 +202,10 @@ void CPUE2eDump::DumpParametersData() {
 
 void CPUE2eDump::DumpConstants(const session::KernelGraph *graph, uint32_t graph_id) {
   MS_EXCEPTION_IF_NULL(graph);
+  auto &dump_json_parser = DumpJsonParser::GetInstance();
+  if (!dump_json_parser.OutputNeedDump()) {
+    return;
+  }
   MS_LOG(INFO) << "Start e2e dump constant.";
   uint32_t cur_iteration = DumpJsonParser::GetInstance().cur_dump_iter();
   if (cur_iteration != 0) {

mindspore/ccsrc/debug/data_dump/e2e_dump.cc

@@ -186,7 +186,7 @@ void E2eDump::DumpOutputImpl(const CNodePtr &node, bool trans_flag, const std::s
     if (DumpJsonParser::GetInstance().IsStatisticDump() &&
         (IsDeviceTargetGPU() || Debugger::GetInstance()->GetAscendKernelByKernelFlag())) {
       TensorStatDump stat_dump(op_type, op_name, task_id, stream_id, timestamp, false, j, j);
-      stat_dump.DumpTensorStatsToFile(GetKernelNodeName(node), dump_path, debugger);
+      (void)stat_dump.DumpTensorStatsToFile(GetKernelNodeName(node), dump_path, debugger);
     }
     if (DumpJsonParser::GetInstance().IsTensorDump()) {
       if (IsDeviceTargetGPU()) {
@@ -287,7 +287,7 @@ void E2eDump::DumpInputImpl(const CNodePtr &node, bool trans_flag, const std::st
     if (DumpJsonParser::GetInstance().IsStatisticDump() &&
         (IsDeviceTargetGPU() || Debugger::GetInstance()->GetAscendKernelByKernelFlag())) {
       TensorStatDump stat_dump(op_type, op_name, task_id, stream_id, timestamp, true, j, slot);
-      stat_dump.DumpTensorStatsToFile(tensor_name, dump_path, debugger);
+      (void)stat_dump.DumpTensorStatsToFile(tensor_name, dump_path, debugger);
     }
     if (DumpJsonParser::GetInstance().IsTensorDump()) {
       if (IsDeviceTargetGPU()) {
@@ -345,7 +345,7 @@ void E2eDump::DumpSingleAnfNode(const AnfNodePtr &anf_node, const size_t output_
   if (IsDeviceTargetGPU()) {
     if (dump_json_parser.IsStatisticDump()) {
       TensorStatDump stat_dump("Parameter", dump_name, task_id, stream_id, timestamp, false, 0, 0);
-      stat_dump.DumpTensorStatsToFile(node_name, dump_path, debugger);
+      (void)stat_dump.DumpTensorStatsToFile(node_name, dump_path, debugger);
     }
     if (dump_json_parser.IsTensorDump()) {
       DumpGPUMemToFile(file_path, node_name, *addr, int_shapes, type, device_type, trans_flag, 0, debugger);
@@ -367,7 +367,7 @@ void E2eDump::DumpSingleParameterNode(const AnfNodePtr &anf_node, const std::str
   MS_EXCEPTION_IF_NULL(anf_node);
   auto &dump_json_parser = DumpJsonParser::GetInstance();
   std::string node_name = GetKernelNodeName(anf_node);
-  if (!anf_node->isa<Parameter>() || !dump_json_parser.NeedDump(node_name)) {
+  if (!anf_node->isa<Parameter>() || !dump_json_parser.NeedDump(node_name) || !dump_json_parser.OutputNeedDump()) {
     return;
   }
   DumpJsonParser::GetInstance().MatchKernel(node_name);
@@ -388,7 +388,7 @@ void E2eDump::DumpSingleParameterNode(const AnfNodePtr &anf_node, const std::str
   if (IsDeviceTargetGPU()) {
     if (dump_json_parser.IsStatisticDump()) {
       TensorStatDump stat_dump("Parameter", node_name, task_id, stream_id, timestamp, false, 0, 0);
-      stat_dump.DumpTensorStatsToFile(node_name, dump_path, debugger);
+      (void)stat_dump.DumpTensorStatsToFile(node_name, dump_path, debugger);
     }
     if (dump_json_parser.IsTensorDump()) {
       DumpGPUMemToFile(file_path, node_name, *addr, int_shapes, type, device_type, trans_flag, 0, debugger);
@@ -534,8 +534,8 @@ void E2eDump::DumpRunIter(const KernelGraphPtr &graph, uint32_t rank_id) {
     // dump history for all iterations in the epoch
     Debugger::GetInstance()->UpdateGraphIterMap(graph->graph_id(), iter_num);
     auto graph_iter_map = Debugger::GetInstance()->GetGraphIterMap();
-    auto step_per_epoch = graph_iter_map[graph->graph_id()];
-    for (int i = 0; i < step_per_epoch; i++) {
+    auto step_per_epoch = IntToSize(graph_iter_map[graph->graph_id()]);
+    for (size_t i = 0; i < step_per_epoch; i++) {
       auto step = (json_parser.cur_dump_iter() * step_per_epoch) + i;
       fout << (std::to_string(step) + "\n");
     }
@@ -567,7 +567,7 @@ void E2eDump::DumpData(const session::KernelGraph *graph, uint32_t rank_id, cons
     MS_LOG(INFO) << "Current graph id is " << graph_id;
     std::string dump_path = GenerateDumpPath(graph_id, rank_id);
     if (dump_json_parser.IsStatisticDump()) {
-      TensorStatDump::OpenStatisticsFile(dump_path);
+      (void)TensorStatDump::OpenStatisticsFile(dump_path);
     }
     DumpInput(graph, dump_path, debugger);
     DumpOutput(graph, dump_path, debugger);
@@ -799,7 +799,7 @@ uint64_t UnpackUint64Value(char *ptr) {
 #if defined(__APPLE__)
   return *reinterpret_cast<const uint64_t *>(ptr);
 #else
-  return le16toh(*reinterpret_cast<const uint64_t *>(ptr));
+  return le64toh(*reinterpret_cast<const uint64_t *>(ptr));
 #endif
 }
 
@@ -811,7 +811,7 @@ std::string IntToHexString(const uint64_t value) {
 
 nlohmann::json E2eDump::ParseOverflowInfo(char *data_ptr) {
   uint32_t index = 0;
-  uint64_t model_id = UnpackUint64Value(data_ptr + index);
+  uint64_t model_id = UnpackUint64Value(data_ptr);
   index += kUint64Size;
   uint64_t stream_id = UnpackUint64Value(data_ptr + index);
   index += kUint64Size;

mindspore/ccsrc/debug/data_dump/tensor_stat_dump.cc

@@ -17,6 +17,7 @@
 #include "debug/data_dump/tensor_stat_dump.h"
 
 #include <memory>
+#include <map>
 #include "utils/file_utils.h"
 #include "debug/common.h"
 #include "debug/debug_services.h"
@@ -65,14 +66,14 @@ bool CsvWriter::OpenFile(const std::string &path, const std::string &header) {
   }
   if (first_time_opening) {
     file_ << header;
-    file_.flush();
+    (void)file_.flush();
     file_path_str_ = path;
   }
   MS_LOG(INFO) << "Opened file: " << file_path_value;
   return true;
 }
 
-void CsvWriter::CloseFile() {
+void CsvWriter::CloseFile() noexcept {
   if (file_.is_open()) {
     file_.close();
     ChangeFileMode(file_path_str_, S_IRUSR);
@@ -85,7 +86,7 @@ void CsvWriter::WriteToCsv(const T &val, bool end_line) {
   file_ << val;
   if (end_line) {
     file_ << kEndLine;
-    file_.flush();
+    (void)file_.flush();
   } else {
     file_ << kSeparator;
   }
@@ -156,7 +157,7 @@ bool TensorStatDump::DumpTensorStatsToFile(const std::string &original_kernel_na
   return DumpTensorStatsToFile(dump_path, data);
 }
 
-bool TensorStatDump::DumpTensorStatsToFile(const std::string &dump_path, std::shared_ptr<TensorData> data) {
+bool TensorStatDump::DumpTensorStatsToFile(const std::string &dump_path, const std::shared_ptr<TensorData> data) {
   if (data == nullptr) {
     MS_LOG(INFO) << "Tensor data is empty, skipping current statistics";
     return false;

mindspore/ccsrc/debug/data_dump/tensor_stat_dump.h

@@ -37,7 +37,7 @@ class CsvWriter {
   ~CsvWriter();
   DISABLE_COPY_AND_ASSIGN(CsvWriter)
   bool OpenFile(const std::string &path, const std::string &header = "");
-  void CloseFile();
+  void CloseFile() noexcept;
   template <typename T>
   void WriteToCsv(const T &val, bool end_line = false);
 
@@ -57,7 +57,7 @@ class TensorStatDump {
   TensorStatDump(const std::string &op_type, const std::string &op_name, const std::string &task_id,
                  const std::string &stream_id, const std::string &timestamp, const std::string &io, size_t slot,
                  size_t tensor_loader_slot);
-  bool DumpTensorStatsToFile(const std::string &dump_path, std::shared_ptr<TensorData> data);
+  bool DumpTensorStatsToFile(const std::string &dump_path, const std::shared_ptr<TensorData> data);
   bool DumpTensorStatsToFile(const std::string &original_kernel_name, const std::string &dump_path,
                              const Debugger *debugger);
 

mindspore/ccsrc/debug/debug_services.cc

@@ -897,7 +897,7 @@ void DebugServices::ProcessConvertToHostFormat(const std::vector<std::string> &f
         if (candidate.find(file_n + ".") != std::string::npos && candidate.rfind(kNpyExt) != std::string::npos) {
           // we found a converted file for this op
           std::string found_file = dump_key + "/" + candidate;
-          result_list->insert(found_file);
+          (void)result_list->insert(found_file);
         }
       }
     }
@@ -1008,13 +1008,13 @@ void DebugServices::ProcessConvertList(const std::string &prefix_dump_file_name,
     }
     if (file_name.rfind(kNpyExt) == std::string::npos) {
       std::size_t second_dot = file_name.find(".", file_name.find(prefix_dump_file_name + ".", type_pos + 1));
-      file_name_w_o_perfix.replace(type_pos + 1, second_dot - type_pos - 1, prefix_dump_file_name);
+      (void)file_name_w_o_perfix.replace(type_pos + 1, second_dot - type_pos - 1, prefix_dump_file_name);
       // if file matches prefix and is in device format add to candidate files to convert.
       (*dir_to_files_map)[specific_dump_dir].push_back(std::make_pair(file_name, file_name_w_o_perfix));
     } else {
       // otherwise, if file matches prefix and already has been converted to host format
       // add to result of converted files.
-      result_list->insert(file_path);
+      (void)result_list->insert(file_path);
     }
   }
   (void)closedir(d);
@@ -1803,7 +1803,7 @@ uint32_t DebugServices::GetPrevIteration(const std::shared_ptr<TensorData> &tens
                   << " is the first run iteration for tensor: " << tensor->GetName();
     return UINT32_MAX;
   }
-  it--;
+  (void)it--;
   prev_iter = *it;
   tensor->SetPrevIteration(prev_iter);
   return prev_iter;

mindspore/ccsrc/debug/debugger/debugger.cc

@@ -490,7 +490,7 @@ void Debugger::DumpParamsAndConstAndHistory() {
     return;
   }
   LoadParametersAllGraphs();
-  (void)E2eDump::DumpParametersData(GetRankID(), debugger_.get());
+  E2eDump::DumpParametersData(GetRankID(), debugger_.get());
   // Whether constant data was already dumped for the current root graph.
   bool cur_root_graph_checked = std::find(visited_root_graph_ids_.begin(), visited_root_graph_ids_.end(),
                                           cur_root_graph_id_) != visited_root_graph_ids_.end();
@@ -1131,13 +1131,13 @@ void AddTensorStatInfo(const DebugServices::TensorStat &tensor_stat,
   tensor_statistics->set_max_value(static_cast<float>(tensor_stat.max_value));
   tensor_statistics->set_min_value(static_cast<float>(tensor_stat.min_value));
   tensor_statistics->set_avg_value(static_cast<float>(tensor_stat.avg_value));
-  tensor_statistics->set_count(tensor_stat.count);
-  tensor_statistics->set_neg_zero_count(tensor_stat.neg_zero_count);
-  tensor_statistics->set_pos_zero_count(tensor_stat.pos_zero_count);
-  tensor_statistics->set_nan_count(tensor_stat.nan_count);
-  tensor_statistics->set_neg_inf_count(tensor_stat.neg_inf_count);
-  tensor_statistics->set_pos_inf_count(tensor_stat.pos_inf_count);
-  tensor_statistics->set_zero_count(tensor_stat.zero_count);
+  tensor_statistics->set_count(SizeToInt(tensor_stat.count));
+  tensor_statistics->set_neg_zero_count(SizeToInt(tensor_stat.neg_zero_count));
+  tensor_statistics->set_pos_zero_count(SizeToInt(tensor_stat.pos_zero_count));
+  tensor_statistics->set_nan_count(SizeToInt(tensor_stat.nan_count));
+  tensor_statistics->set_neg_inf_count(SizeToInt(tensor_stat.neg_inf_count));
+  tensor_statistics->set_pos_inf_count(SizeToInt(tensor_stat.pos_inf_count));
+  tensor_statistics->set_zero_count(SizeToInt(tensor_stat.zero_count));
 
   tensor_summary_list->push_back(tensor_summary_item);
 }
@@ -1780,7 +1780,7 @@ std::shared_ptr<DumpDataBuilder> Debugger::LoadDumpDataBuilder(const std::string
   return dump_data_construct_map_[node_name];
 }
 
-void Debugger::ClearDumpDataBuilder(const std::string &node_name) { dump_data_construct_map_.erase(node_name); }
+void Debugger::ClearDumpDataBuilder(const std::string &node_name) { (void)dump_data_construct_map_.erase(node_name); }
 #endif
 
 }  // namespace mindspore

mindspore/ccsrc/debug/debugger/debugger_utils.cc

@@ -88,8 +88,8 @@ void LoadInputs(const CNodePtr &cnode, const KernelLaunchInfo *launch_info, uint
     auto device_addr = device_context->CreateDeviceAddress(addr->addr, addr->size, format, type);
     string input_tensor_name = input_kernel_name + ':' + "0";
     ShapeVector int_shapes = trans::GetRuntimePaddingShape(input_kernel, PARAMETER_OUTPUT_INDEX);
-    auto ret =
-      device_addr->LoadMemToHost(input_tensor_name, exec_order, format, int_shapes, type, 0, true, root_graph_id);
+    auto ret = device_addr->LoadMemToHost(input_tensor_name, UintToInt(exec_order), format, int_shapes, type, 0, true,
+                                          root_graph_id);
     if (!ret) {
       MS_LOG(ERROR) << "LoadMemToHost:"
                     << ", tensor_name:" << input_tensor_name << ", host_format:" << format << ".!";
@@ -124,7 +124,8 @@ void LoadOutputs(const CNodePtr &cnode, const KernelLaunchInfo *launch_info, uin
     auto device_addr = device_context->CreateDeviceAddress(addr->addr, addr->size, format, type);
     string tensor_name = kernel_name + ':' + std::to_string(j);
     ShapeVector int_shapes = trans::GetRuntimePaddingShape(cnode, j);
-    auto ret = device_addr->LoadMemToHost(tensor_name, exec_order, format, int_shapes, type, j, false, root_graph_id);
+    auto ret =
+      device_addr->LoadMemToHost(tensor_name, UintToInt(exec_order), format, int_shapes, type, j, false, root_graph_id);
     if (!ret) {
       MS_LOG(ERROR) << "LoadMemToHost:"
                     << ", tensor_name:" << tensor_name << ", host_format:" << format << ".!";
@@ -299,6 +300,7 @@ void SuperKernelE2eDump(const KernelGraphPtr &graph) {
  */
 int32_t DumpDataCallBack(const DumpChunk *dump_chunk, int32_t size) {
   MS_LOG(DEBUG) << "ADX DumpDataCallBack is called";
+  MS_LOG(DEBUG) << "The dump_chunk size is: " << size;
   string file_name = dump_chunk->fileName;
   uint32_t isLastChunk = dump_chunk->isLastChunk;
 

mindspore/ccsrc/debug/debugger/tensor_summary.cc

@@ -174,31 +174,31 @@ void TensorSummary<T>::TensorStatistics(DbgDataType dtype_value) {
   if (dtype_value == DT_BOOL) {
     is_bool_ = true;
   }
-  const int default_threads = 32;
-  const int default_elements_per_thread = 10000;
+  const uint64_t default_threads = 32;
+  const uint64_t default_elements_per_thread = 10000;
 
   if (num_elements_ <= default_elements_per_thread) {
     return TensorStatisticsSingleThread();
   }
-  int desired_threads = num_elements_ / default_elements_per_thread;
-  int actual_threads = std::min(desired_threads, default_threads);
-  int actual_elements_per_thread = num_elements_ / actual_threads;
+  uint64_t desired_threads = num_elements_ / default_elements_per_thread;
+  uint64_t actual_threads = std::min(desired_threads, default_threads);
+  uint64_t actual_elements_per_thread = num_elements_ / actual_threads;
 
   // Use multithread to calculate statistic on chunks of data
   void *previous_tensor_ptr = nullptr;
   size_t offset = 0;
   std::vector<std::unique_ptr<TensorSummary<T>>> summary_vec;
   std::vector<std::future<void>> summary_future_vec;
-  for (int i = 0; i < actual_threads; i++) {
-    int num_elements_for_thread;
+  for (uint64_t i = 0; i < actual_threads; i++) {
+    uint64_t num_elements_for_thread;
     if (i == actual_threads - 1) {
       num_elements_for_thread = num_elements_ - offset;
     } else {
       num_elements_for_thread = actual_elements_per_thread;
     }
-    summary_vec.emplace_back(std::make_unique<TensorSummary<T>>(current_tensor_ptr_ + offset, previous_tensor_ptr,
-                                                                num_elements_for_thread, 0));
-    summary_future_vec.emplace_back(
+    (void)summary_vec.emplace_back(std::make_unique<TensorSummary<T>>(current_tensor_ptr_ + offset, previous_tensor_ptr,
+                                                                      num_elements_for_thread, 0));
+    (void)summary_future_vec.emplace_back(
       std::async(std::launch::async, &TensorSummary<T>::TensorStatisticsSingleThread, summary_vec[i].get()));
     offset += num_elements_for_thread;
   }

tests/st/debugger/test_read_tensors.py

@@ -19,9 +19,9 @@ Read tensor test script for offline debugger APIs.
 import os
 import json
 import tempfile
-import mindspore.offline_debug.dbg_services as d
-import numpy as np
 import pytest
+import numpy as np
+import mindspore.offline_debug.dbg_services as d
 from tests.security_utils import security_off_wrap
 from dump_test_utils import build_dump_structure, write_tensor_to_json