From 2facd80c8d24c603b7545a05e6985d3d56cd5d8f Mon Sep 17 00:00:00 2001 From: Jamie Nisbet Date: Tue, 28 Apr 2020 14:59:32 -0400 Subject: [PATCH] updated print functions for info diagnostics fix some ci failures more ci fixes more ci fails yet another ci fail more ci stuff still ci more ci more ci more ci --- .../dataset/engine/datasetops/barrier_op.cc | 18 +++++++--- .../dataset/engine/datasetops/batch_op.cc | 18 +++++++--- .../dataset/engine/datasetops/dataset_op.cc | 24 ++++++++----- .../engine/datasetops/device_queue_op.cc | 18 +++++++--- .../dataset/engine/datasetops/filter_op.cc | 25 ++++++++----- .../ccsrc/dataset/engine/datasetops/map_op.cc | 34 +++++++++++------- .../dataset/engine/datasetops/parallel_op.cc | 16 +++++---- .../dataset/engine/datasetops/pipeline_op.cc | 26 +++++++++++--- .../dataset/engine/datasetops/project_op.cc | 22 +++++++++--- .../dataset/engine/datasetops/rename_op.cc | 26 ++++++++++---- .../dataset/engine/datasetops/repeat_op.cc | 35 ++++++++++-------- .../dataset/engine/datasetops/shuffle_op.cc | 22 ++++++++---- .../dataset/engine/datasetops/skip_op.cc | 20 +++++++---- .../engine/datasetops/source/celeba_op.cc | 17 +++++++-- .../engine/datasetops/source/cifar_op.cc | 17 +++++++-- .../engine/datasetops/source/generator_op.cc | 23 +++++++++++- .../engine/datasetops/source/generator_op.h | 5 +++ .../datasetops/source/image_folder_op.cc | 19 +++++++--- .../engine/datasetops/source/manifest_op.cc | 17 +++++++-- .../engine/datasetops/source/mindrecord_op.cc | 28 ++++++++------- .../engine/datasetops/source/mnist_op.cc | 18 +++++++--- .../engine/datasetops/source/storage_op.cc | 36 +++++++------------ .../engine/datasetops/source/text_file_op.cc | 25 +++++++++++++ .../engine/datasetops/source/text_file_op.h | 5 +++ .../engine/datasetops/source/tf_reader_op.cc | 31 ++++++++++++++++ .../engine/datasetops/source/tf_reader_op.h | 5 +++ .../engine/datasetops/source/voc_op.cc | 18 +++++++--- .../dataset/engine/datasetops/take_op.cc | 21 +++++++---- .../ccsrc/dataset/engine/datasetops/zip_op.cc | 18 +++++++--- .../ccsrc/dataset/engine/execution_tree.cc | 33 +++++++++++++---- .../ccsrc/dataset/engine/execution_tree.h | 14 ++++++-- 31 files changed, 483 insertions(+), 171 deletions(-) diff --git a/mindspore/ccsrc/dataset/engine/datasetops/barrier_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/barrier_op.cc index b50a7788da..3d5b682155 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/barrier_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/barrier_op.cc @@ -14,6 +14,7 @@ * limitations under the License. */ #include "dataset/engine/datasetops/barrier_op.h" +#include #include #include "dataset/core/constants.h" #include "dataset/engine/data_buffer.h" @@ -214,10 +215,19 @@ Status BarrierOp::getNextTensorRow(TensorRow *new_row) { // A function that prints info about the Operator void BarrierOp::Print(std::ostream &out, bool show_all) const { - // Call base class printer first - PipelineOp::Print(out, show_all); - out << "\nBarrierOp:\n" - << "\nCondition " << condition_name_ << "\n\n"; + // Always show the id and name as first line regardless if this summary or detailed print + out << "(" << std::setw(2) << operator_id_ << ") :"; + if (!show_all) { + // Call the super class for displaying any common 1-liner info + PipelineOp::Print(out, show_all); + // Then show any custom derived-internal 1-liner info for this op + out << "\n"; + } else { + // Call the super class for displaying any common detailed info + PipelineOp::Print(out, show_all); + // Then show any custom derived-internal stuff + out << "\nCondition: " << condition_name_ << "\n\n"; + } } // overwrite function and handle eof diff --git a/mindspore/ccsrc/dataset/engine/datasetops/batch_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/batch_op.cc index c80078cb44..ad8b95b625 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/batch_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/batch_op.cc @@ -15,6 +15,7 @@ */ #include "dataset/engine/datasetops/batch_op.h" #include +#include #include "common/utils.h" #include "dataset/engine/data_buffer.h" #include "dataset/engine/db_connector.h" @@ -102,10 +103,19 @@ Status BatchOp::operator()() { } void BatchOp::Print(std::ostream &out, bool show_all) const { - ParallelOp::Print(out, show_all); - out << "\nBatchOp:\n" - << "number of parallel workers: " << num_workers_ << "\nBatch size: " << start_batch_size_ - << "\nDrop remainder: " << (drop_ ? "yes" : "no") << "\n\n"; + // Always show the id and name as first line regardless if this summary or detailed print + out << "(" << std::setw(2) << operator_id_ << ") :"; + if (!show_all) { + // Call the super class for displaying any common 1-liner info + ParallelOp::Print(out, show_all); + // Then show any custom derived-internal 1-liner info for this op + out << " [batch size: " << start_batch_size_ << "]\n"; + } else { + // Call the super class for displaying any common detailed info + ParallelOp::Print(out, show_all); + // Then show any custom derived-internal stuff + out << "\nStart batch size: " << start_batch_size_ << "\nDrop remainder: " << (drop_ ? "yes" : "no") << "\n\n"; + } } Status BatchOp::BatchRows(const std::unique_ptr *source_table, diff --git a/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.cc index 5e3ea3dc44..adbf42487e 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.cc @@ -92,18 +92,24 @@ void DatasetOp::CreateConnector(int32_t num_producers, int32_t num_consumers) { // A print method typically used for debugging. showAll of true will recursively descend to child prints void DatasetOp::Print(std::ostream &out, bool show_all) const { + // When show_all is false, we display a 1 liner piece of text for the op. + // When show_all is true, we display more detailed output for the op. + // Derived printers should show their own header info, then call base class printer, followed by + // derived-specific items. + // For now, the base class doesn't have any summary info to show so it's a no-op in that case. if (show_all) { + // The detailed display will show common base class info of the op. Allow the derived class to print + // it's own id and name though as the first line. + out << "\nNumber of children : " << child_.size(); for (size_t i = 0; i < child_.size(); i++) { - child_[i]->Print(out, show_all); + out << "\n Child[" << i << "] id: " << child_[i]->id(); } - } - out << "\n-------------------------" - << "\nOperator # : " << operator_id_ << "\nNumber of children : " << child_.size() - << "\nNumber of parents : " << parent_.size() << "\nConnector queue size : " << oc_queue_size_ - << "\nOperator control flags : 0x" << std::hex << std::setw(8) << std::setfill('0') << op_ctrl_flags_ << std::dec - << std::setfill(' ') << "\nHas parents:\n"; - for (size_t i = 0; i < parent_.size(); i++) { - out << "Parent[" << i << "] id: " << parent_[i]->id() << "\n"; + out << "\nNumber of parents : " << parent_.size(); + for (size_t i = 0; i < parent_.size(); i++) { + out << "\n Parent[" << i << "] id: " << parent_[i]->id(); + } + out << "\nConnector queue size : " << oc_queue_size_ << "\nOperator control flags : 0x" << std::hex + << std::setw(8) << std::setfill('0') << op_ctrl_flags_ << std::dec << std::setfill(' '); } } diff --git a/mindspore/ccsrc/dataset/engine/datasetops/device_queue_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/device_queue_op.cc index 71e4ce64a4..2c91d36259 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/device_queue_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/device_queue_op.cc @@ -14,7 +14,7 @@ * limitations under the License. */ #include "dataset/engine/datasetops/device_queue_op.h" - +#include #include #include @@ -246,9 +246,19 @@ Status DeviceQueueOp::SendDataToCPU() { } void DeviceQueueOp::Print(std::ostream &out, bool show_all) const { - PipelineOp::Print(out, show_all); - - out << "DeviceQueueOp: channelName: " << channel_name_ << ", prefetchSize: " << prefetch_size_ << '\n'; + // Always show the id and name as first line regardless if this summary or detailed print + out << "(" << std::setw(2) << operator_id_ << ") :"; + if (!show_all) { + // Call the super class for displaying any common 1-liner info + PipelineOp::Print(out, show_all); + // Then show any custom derived-internal 1-liner info for this op + out << "\n"; + } else { + // Call the super class for displaying any common detailed info + PipelineOp::Print(out, show_all); + // Then show any custom derived-internal stuff + out << "\nChannel name: " << channel_name_ << "\nPrefetch size: " << prefetch_size_ << "\n\n"; + } } } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/filter_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/filter_op.cc index ce312ce3d9..5ede8ad6f4 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/filter_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/filter_op.cc @@ -16,6 +16,7 @@ #include "dataset/engine/datasetops/filter_op.h" #include #include +#include #include #include #include @@ -88,14 +89,22 @@ Status FilterOp::ValidateInColumns(const std::unordered_map:"; + if (!show_all) { + // Call the super class for displaying any common 1-liner info + ParallelOp::Print(out, show_all); + // Then show any custom derived-internal 1-liner info for this op + out << "\n"; + } else { + // Call the super class for displaying any common detailed info + ParallelOp::Print(out, show_all); + // Then show any custom derived-internal stuff + out << "\nInput column names:"; + for (size_t i = 0; i < in_columns_.size(); i++) { + out << " " << in_columns_[i]; + } + out << "\n\n"; } } diff --git a/mindspore/ccsrc/dataset/engine/datasetops/map_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/map_op.cc index b6d603bac9..4cbe2ac603 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/map_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/map_op.cc @@ -15,6 +15,7 @@ */ #include "dataset/engine/datasetops/map_op.h" #include +#include #include #include #include @@ -81,20 +82,27 @@ int32_t MapOp::num_consumers() const { // A print method typically used for debugging void MapOp::Print(std::ostream &out, bool show_all) const { - // Call base class printer first - ParallelOp::Print(out, show_all); - - // Then display our own stuff - out << "\nMapOp:"; - out << "\n Input column names:"; - for (size_t i = 0; i < in_columns_.size(); i++) { - out << " " << in_columns_[i]; - } - out << "\n TensorOps:"; - for (size_t i = 0; i < tfuncs_.size(); i++) { - out << " " << tfuncs_[i]; + // Always show the id and name as first line regardless if this summary or detailed print + out << "(" << std::setw(2) << operator_id_ << ") :"; + if (!show_all) { + // Call the super class for displaying any common 1-liner info + ParallelOp::Print(out, show_all); + // Then show any custom derived-internal 1-liner info for this op + out << "\n"; + } else { + // Call the super class for displaying any common detailed info + ParallelOp::Print(out, show_all); + // Then show any custom derived-internal stuff + out << "\nInput column names:"; + for (size_t i = 0; i < in_columns_.size(); i++) { + out << " " << in_columns_[i]; + } + out << "\n TensorOps:"; + for (size_t i = 0; i < tfuncs_.size(); i++) { + out << " " << tfuncs_[i]; + } + out << "\n\n"; } - out << "\n"; } // This class functor will provide the master loop that drives the logic for performing the work diff --git a/mindspore/ccsrc/dataset/engine/datasetops/parallel_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/parallel_op.cc index 4b2af2250a..2eeb931554 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/parallel_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/parallel_op.cc @@ -55,12 +55,16 @@ Status ParallelOp::CreateWorkerConnector(int32_t worker_connector_size) { // A print method typically used for debugging void ParallelOp::Print(std::ostream &out, bool show_all) const { - // Call base class printer first - DatasetOp::Print(out, show_all); - - // Then show our own stuff - out << "ParallelOp:"; - out << "\n Num workers : " << num_workers_ << "\n"; + // Summary 1-liner print + if (!show_all) { + out << " [workers: " << num_workers_ << "]"; + // Call super class printer + DatasetOp::Print(out, show_all); + } else { + // Detailed print + DatasetOp::Print(out, show_all); + out << "\nNum workers: " << num_workers_; + } } // Override base class reset to provide reset actions specific to the ParallelOp class. diff --git a/mindspore/ccsrc/dataset/engine/datasetops/pipeline_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/pipeline_op.cc index 56fc24883a..69ace1ed9a 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/pipeline_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/pipeline_op.cc @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +#include #include #include "dataset/engine/datasetops/pipeline_op.h" @@ -23,11 +24,26 @@ PipelineOp::PipelineOp(int32_t op_connector_size) : DatasetOp(op_connector_size) // A print method typically used for debugging void PipelineOp::Print(std::ostream &out, bool show_all) const { - // Call base class printer first - DatasetOp::Print(out, show_all); - - // Then display our own stuff for the pipeline op - // out << "This is a pipeline op print. nothing to display here at the moment.\n"; + // Summary 1-liner print + if (!show_all) { + out << " [workers: "; + if (this->inlined()) { + out << "0 (inlined)]"; + } else { + out << "1]"; // Pipeline ops only have 1 worker + } + // Call super class printer + DatasetOp::Print(out, show_all); + } else { + // Detailed print + DatasetOp::Print(out, show_all); + out << "\nNum workers: "; + if (this->inlined()) { + out << "0 (inlined)"; + } else { + out << "1"; // Pipeline ops only have 1 worker + } + } } } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/project_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/project_op.cc index b87967dde8..128d3e68e5 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/project_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/project_op.cc @@ -16,6 +16,7 @@ #include "dataset/engine/datasetops/project_op.h" #include +#include #include #include #include @@ -49,12 +50,23 @@ ProjectOp::ProjectOp(const std::vector &columns_to_project) : PipelineOp(0), columns_to_project_(columns_to_project) {} void ProjectOp::Print(std::ostream &out, bool show_all) const { - PipelineOp::Print(out, show_all); - out << "ProjectOp: columns that are projected: "; - for (size_t i = 0; i < columns_to_project_.size(); i++) { - out << columns_to_project_[i] << " "; + // Always show the id and name as first line regardless if this summary or detailed print + out << "(" << std::setw(2) << operator_id_ << ") :"; + if (!show_all) { + // Call the super class for displaying any common 1-liner info + PipelineOp::Print(out, show_all); + // Then show any custom derived-internal 1-liner info for this op + out << "\n"; + } else { + // Call the super class for displaying any common detailed info + PipelineOp::Print(out, show_all); + // Then show any custom derived-internal stuff + out << "\nColumns that are projected:"; + for (size_t i = 0; i < columns_to_project_.size(); i++) { + out << "\n" << columns_to_project_[i]; + } + out << "\n\n"; } - out << '\n'; } // Gets a buffer from the child operator and projects the buffer. diff --git a/mindspore/ccsrc/dataset/engine/datasetops/rename_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/rename_op.cc index 725476bf91..5f354abb04 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/rename_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/rename_op.cc @@ -14,7 +14,7 @@ * limitations under the License. */ #include "dataset/engine/datasetops/rename_op.h" - +#include #include #include #include @@ -138,11 +138,25 @@ Status RenameOp::RenameBuffer(std::unique_ptr *input_buffer) { // prints rename void RenameOp::Print(std::ostream &out, // In: The output stream to print to bool show_all) const { // In: T/F if it should print everything - // Call base class printer first - PipelineOp::Print(out, show_all); - out << "\nRenameOp:\n"; - for (size_t i = 0; i < in_columns_.size(); ++i) { - out << "\nin Columns: " << in_columns_[i] << "\nOut Columns: " << out_columns_[i] << "\n\n"; + // Always show the id and name as first line regardless if this summary or detailed print + out << "(" << std::setw(2) << operator_id_ << ") :"; + if (!show_all) { + // Call the super class for displaying any common 1-liner info + PipelineOp::Print(out, show_all); + // Then show any custom derived-internal 1-liner info for this op + out << "\n"; + } else { + // Call the super class for displaying any common detailed info + PipelineOp::Print(out, show_all); + // Then show any custom derived-internal stuff + out << "\nIn columns:"; + for (size_t i = 0; i < in_columns_.size(); ++i) { + out << "\n " << in_columns_[i]; + } + for (size_t i = 0; i < out_columns_.size(); ++i) { + out << "\n " << out_columns_[i]; + } + out << "\n\n"; } } diff --git a/mindspore/ccsrc/dataset/engine/datasetops/repeat_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/repeat_op.cc index 33c731c400..065631eb31 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/repeat_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/repeat_op.cc @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +#include #include #include @@ -51,22 +52,28 @@ RepeatOp::~RepeatOp() {} // A print method typically used for debugging void RepeatOp::Print(std::ostream &out, bool show_all) const { - // Call base class printer first - PipelineOp::Print(out, show_all); - - // Then display our own stuff - out << "RepeatOp:" - << "\nCurrent repeat count: " << repeat_count_ << "\nMax repeat count: " << max_repeats_ - << "\nLeaf Nodes in my execution path:"; - if (!eoe_ops_.empty()) { - out << "\n"; - for (size_t i = 0; i < eoe_ops_.size(); i++) { - out << " Operator: " << eoe_ops_[i]->id() << "\n"; - } + // Always show the id and name as first line regardless if this summary or detailed print + out << "(" << std::setw(2) << operator_id_ << ") :"; + if (!show_all) { + // Call the super class for displaying any common 1-liner info + PipelineOp::Print(out, show_all); + // Then show any custom derived-internal 1-liner info for this op + out << " [repeats: " << max_repeats_ << "]\n"; } else { - out << " kNone."; + // Call the super class for displaying any common detailed info + PipelineOp::Print(out, show_all); + // Then show any custom derived-internal stuff + out << "\nCurrent repeat count: " << repeat_count_ << "\nMax repeat count: " << max_repeats_ + << "\nLeaf Nodes in execution path:"; + if (!eoe_ops_.empty()) { + for (size_t i = 0; i < eoe_ops_.size(); i++) { + out << "\n Operator: " << eoe_ops_[i]->id(); + } + } else { + out << " None."; + } + out << "\n\n"; } - out << "\n-------------------------\n\n"; // End the display with this line } // Base-class override for executing specific RepeatOp configurations. This code will be called diff --git a/mindspore/ccsrc/dataset/engine/datasetops/shuffle_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/shuffle_op.cc index 422c38f2f2..7b09bcef4d 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/shuffle_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/shuffle_op.cc @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -108,13 +109,20 @@ Status ShuffleOp::SelfReset() { // A print method typically used for debugging void ShuffleOp::Print(std::ostream &out, bool show_all) const { - // Call base class printer first - PipelineOp::Print(out, show_all); - - // Then display our own stuff - out << "ShuffleOp:\n Shuffle size: " << shuffle_size_ << "\n rows_per_buffer_: " << rows_per_buffer_ - << "\n shuffle_buffer_state_: " << shuffle_buffer_state_ << "\n shuffle_seed_: " << shuffle_seed_; - out << "\n-------------------------\n\n"; // End the display with this line + // Always show the id and name as first line regardless if this summary or detailed print + out << "(" << std::setw(2) << operator_id_ << ") :"; + if (!show_all) { + // Call the super class for displaying any common 1-liner info + PipelineOp::Print(out, show_all); + // Then show any custom derived-internal 1-liner info for this op + out << " [shuffle size: " << shuffle_size_ << "]\n"; + } else { + // Call the super class for displaying any common detailed info + PipelineOp::Print(out, show_all); + // Then show any custom derived-internal stuff + out << "\nShuffle size: " << shuffle_size_ << "\nRows per buffer: " << rows_per_buffer_ + << "\nShuffle buffer state: " << shuffle_buffer_state_ << "\nShuffle seed: " << shuffle_seed_ << "\n\n"; + } } // Private function to add a new row to the shuffle buffer. diff --git a/mindspore/ccsrc/dataset/engine/datasetops/skip_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/skip_op.cc index d851f2c699..ec45d5d25e 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/skip_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/skip_op.cc @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +#include #include #include @@ -51,12 +52,19 @@ SkipOp::~SkipOp() {} // A print method typically used for debugging void SkipOp::Print(std::ostream &out, bool show_all) const { - // Call base class printer first - PipelineOp::Print(out, show_all); - - // Then display our own stuff - out << "SkipOp:" - << "\nCurrent skip count: " << skip_count_ << "\nMax skip count: " << max_skips_; + // Always show the id and name as first line regardless if this summary or detailed print + out << "(" << std::setw(2) << operator_id_ << ") :"; + if (!show_all) { + // Call the super class for displaying any common 1-liner info + PipelineOp::Print(out, show_all); + // Then show any custom derived-internal 1-liner info for this op + out << " [skips: " << max_skips_ << "]\n"; + } else { + // Call the super class for displaying any common detailed info + PipelineOp::Print(out, show_all); + // Then show any custom derived-internal stuff + out << "\nSkip count: " << skip_count_ << "\nMax skips: " << max_skips_ << "\n\n"; + } } // Since the buffer may contain multi rows, this function will drop the rows diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/celeba_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/celeba_op.cc index 2394380ea4..998672bca5 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/celeba_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/celeba_op.cc @@ -16,6 +16,7 @@ #include "dataset/engine/datasetops/source/celeba_op.h" #include +#include #include "dataset/core/config_manager.h" #include "dataset/util/path.h" #include "dataset/engine/datasetops/source/sampler/sequential_sampler.h" @@ -434,9 +435,19 @@ Status CelebAOp::LoadTensorRow(const std::pair } void CelebAOp::Print(std::ostream &out, bool show_all) const { - DatasetOp::Print(out, show_all); - out << "\nnumber of parallel workers:" << num_workers_ << "\nNumber of rows:" << num_rows_exact_ - << "\nceleba dir: " << folder_path_ << "\n-------------------------\n"; + // Always show the id and name as first line regardless if this summary or detailed print + out << "(" << std::setw(2) << operator_id_ << ") :"; + if (!show_all) { + // Call the super class for displaying any common 1-liner info + ParallelOp::Print(out, show_all); + // Then show any custom derived-internal 1-liner info for this op + out << "\n"; + } else { + // Call the super class for displaying any common detailed info + ParallelOp::Print(out, show_all); + // Then show any custom derived-internal stuff + out << "\nNumber of rows:" << num_rows_exact_ << "\nceleba dir: " << folder_path_ << "\n\n"; + } } // Reset Sampler and wakeup Master thread (functor) diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/cifar_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/cifar_op.cc index 0c2d57ff42..4f69938a31 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/cifar_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/cifar_op.cc @@ -17,6 +17,7 @@ #include #include +#include #include #include "common/utils.h" @@ -225,9 +226,19 @@ Status CifarOp::LoadBuffer(const std::vector &keys, std::unique_ptr:"; + if (!show_all) { + // Call the super class for displaying any common 1-liner info + ParallelOp::Print(out, show_all); + // Then show any custom derived-internal 1-liner info for this op + out << "\n"; + } else { + // Call the super class for displaying any common detailed info + ParallelOp::Print(out, show_all); + // Then show any custom derived-internal stuff + out << "\nNumber of rows:" << num_rows_ << "\nCifar directory: " << folder_path_ << "\n\n"; + } } // Reset Sampler and wakeup Master thread (functor) diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/generator_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/generator_op.cc index 37a74f019a..b0c8b8af35 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/generator_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/generator_op.cc @@ -13,8 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "dataset/core/global_context.h" #include "dataset/engine/datasetops/source/generator_op.h" +#include +#include "dataset/core/global_context.h" #include "dataset/engine/db_connector.h" #include "dataset/engine/data_buffer.h" #include "dataset/engine/execution_tree.h" @@ -58,6 +59,26 @@ GeneratorOp::GeneratorOp(py::function generator_function, std::vectorDealloc(); } +void GeneratorOp::Print(std::ostream &out, bool show_all) const { + // Always show the id and name as first line regardless if this summary or detailed print + out << "(" << std::setw(2) << operator_id_ << ") :"; + if (!show_all) { + // Call the super class for displaying any common 1-liner info + PipelineOp::Print(out, show_all); + // Then show any custom derived-internal 1-liner info for this op + out << "\n"; + } else { + // Call the super class for displaying any common detailed info + PipelineOp::Print(out, show_all); + // Then show any custom derived-internal stuff + out << "\nColumn names:\n"; + for (int i = 0; i < column_names_.size(); ++i) { + out << "\n " << column_names_[i]; + } + out << "\n\n"; + } +} + void GeneratorOp::Dealloc() noexcept { // Setup GIL state PyGILState_STATE gstate; diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/generator_op.h b/mindspore/ccsrc/dataset/engine/datasetops/source/generator_op.h index a5407a9b09..8165fed970 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/generator_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/generator_op.h @@ -95,6 +95,11 @@ class GeneratorOp : public PipelineOp { ~GeneratorOp(); + // A print method typically used for debugging + // @param out - The output stream to write output to + // @param show_all - A bool to control if you want to show all info or just a summary + void Print(std::ostream &out, bool show_all) const override; + // << Stream output operator overload // @notes This allows you to write the debug print info using stream operators // @param out - reference to the output stream being overloaded diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/image_folder_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/image_folder_op.cc index 32d7171c8f..b52404ce69 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/image_folder_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/image_folder_op.cc @@ -14,9 +14,8 @@ * limitations under the License. */ #include "dataset/engine/datasetops/source/image_folder_op.h" - #include - +#include #include "common/utils.h" #include "dataset/core/config_manager.h" #include "dataset/core/tensor_shape.h" @@ -243,9 +242,19 @@ Status ImageFolderOp::LoadBuffer(const std::vector &keys, std::unique_p } void ImageFolderOp::Print(std::ostream &out, bool show_all) const { - DatasetOp::Print(out, show_all); - out << "\nnumber of parallel workers:" << num_workers_ << "\nNumber of rows:" << num_rows_ - << "\nImageFolder Directory: " << folder_path_ << "\n-------------------------\n"; + // Always show the id and name as first line regardless if this summary or detailed print + out << "(" << std::setw(2) << operator_id_ << ") :"; + if (!show_all) { + // Call the super class for displaying any common 1-liner info + ParallelOp::Print(out, show_all); + // Then show any custom derived-internal 1-liner info for this op + out << "\n"; + } else { + // Call the super class for displaying any common detailed info + ParallelOp::Print(out, show_all); + // Then show any custom derived-internal stuff + out << "\nNumber of rows:" << num_rows_ << "\nImageFolder directory: " << folder_path_ << "\n\n"; + } } // Reset Sampler and wakeup Master thread (functor) diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/manifest_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/manifest_op.cc index ab0c012416..fb47274aae 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/manifest_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/manifest_op.cc @@ -17,6 +17,7 @@ #include #include +#include #include #include "common/utils.h" @@ -239,9 +240,19 @@ Status ManifestOp::LoadBuffer(const std::vector &keys, std::unique_ptr< } void ManifestOp::Print(std::ostream &out, bool show_all) const { - DatasetOp::Print(out, show_all); - out << "\nnumber of parallel workers:" << num_workers_ << "\nNumber of rows:" << num_rows_ - << "\nManifest file: " << file_ << "\n-------------------------\n"; + // Always show the id and name as first line regardless if this summary or detailed print + out << "(" << std::setw(2) << operator_id_ << ") :"; + if (!show_all) { + // Call the super class for displaying any common 1-liner info + ParallelOp::Print(out, show_all); + // Then show any custom derived-internal 1-liner info for this op + out << "\n"; + } else { + // Call the super class for displaying any common detailed info + ParallelOp::Print(out, show_all); + // Then show any custom derived-internal stuff + out << "\nNumber of rows:" << num_rows_ << "\nManifest file: " << file_ << "\n\n"; + } } // Reset Sampler and wakeup Master thread (functor) diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/mindrecord_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/mindrecord_op.cc index 72dee6f2e6..c8bbf0e3c4 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/mindrecord_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/mindrecord_op.cc @@ -17,6 +17,7 @@ #include #include +#include #include #include @@ -179,18 +180,21 @@ MindRecordOp::~MindRecordOp() {} // A print method typically used for debugging void MindRecordOp::Print(std::ostream &out, bool show_all) const { - // Call base class printer first - ParallelOp::Print(out, show_all); - - // Then display our own stuff - out << "\nMindRecordOp:"; - out << "\n 1 Dataset file : " << dataset_file_; - out << "\n Number of rows : " << num_rows_; - out << "\n Rows per buffer : " << rows_per_buffer_; - out << "\n Number of buffers : " << buffers_needed_; - out << "\n Number of ShardReader workers : " << num_mind_record_workers_; - - out << "\n\n"; + // Always show the id and name as first line regardless if this summary or detailed print + out << "(" << std::setw(2) << operator_id_ << ") :"; + if (!show_all) { + // Call the super class for displaying any common 1-liner info + ParallelOp::Print(out, show_all); + // Then show any custom derived-internal 1-liner info for this op + out << "\n"; + } else { + // Call the super class for displaying any common detailed info + ParallelOp::Print(out, show_all); + // Then show any custom derived-internal stuff + out << "\n1 Dataset file : " << dataset_file_ << "\nNumber of rows : " << num_rows_ + << "\nRows per buffer : " << rows_per_buffer_ << "\nNumber of buffers : " << buffers_needed_ + << "\nNumber of ShardReader workers : " << num_mind_record_workers_ << "\n\n"; + } } template diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/mnist_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/mnist_op.cc index fbf041e985..33aca001e9 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/mnist_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/mnist_op.cc @@ -16,7 +16,7 @@ #include "dataset/engine/datasetops/source/mnist_op.h" #include - +#include #include "common/utils.h" #include "dataset/core/config_manager.h" #include "dataset/core/tensor_shape.h" @@ -190,9 +190,19 @@ Status MnistOp::LoadBuffer(const std::vector &keys, std::unique_ptr:"; + if (!show_all) { + // Call the super class for displaying any common 1-liner info + ParallelOp::Print(out, show_all); + // Then show any custom derived-internal 1-liner info for this op + out << "\n"; + } else { + // Call the super class for displaying any common detailed info + ParallelOp::Print(out, show_all); + // Then show any custom derived-internal stuff + out << "\nNumber of rows:" << num_rows_ << "\nMNIST Directory: " << folder_path_ << "\n\n"; + } } // Reset Sampler and wakeup Master thread (functor) diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/storage_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/storage_op.cc index 2ca957ae6d..f310a097ee 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/storage_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/storage_op.cc @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -319,31 +320,18 @@ StorageOp::~StorageOp() {} // A print method typically used for debugging void StorageOp::Print(std::ostream &out, bool show_all) const { - // Call base class printer first - ParallelOp::Print(out, show_all); - - // Then display our own stuff - out << "\nStorageOp:"; - out << "\n Dataset files dir : " << dataset_files_dir_ << "\n Dataset schema file : " << schema_file_; - if (!dataset_file_list_.empty()) { - out << "\n Dataset Files List:\n"; - for (auto filename : dataset_file_list_) { - out << " " << filename << "\n"; - } - } - out << "\n\n"; - if (!data_buffers_.empty()) { - out << std::boolalpha << " Number of DataBuffers inside StorageOp: " << data_buffers_.size() - << "\n Number of rows: " << num_rows_ << "\n Rows per buffer: " << rows_per_buffer_ << "\n\n DataBuffers:\n"; - - // Iterate over each DataBuffer and display the buffer id and the buffer - int32_t i = 0; - for (i = 0; i < data_buffers_.size(); i++) { - out << " " << i << ")\n"; - data_buffers_[i]->Print(out, show_all); - } + // Always show the id and name as first line regardless if this summary or detailed print + out << "(" << std::setw(2) << operator_id_ << ") :"; + if (!show_all) { + // Call the super class for displaying any common 1-liner info + ParallelOp::Print(out, show_all); + // Then show any custom derived-internal 1-liner info for this op + out << "\n"; } else { - out << "DataCache is empty!\n"; + // Call the super class for displaying any common detailed info + ParallelOp::Print(out, show_all); + // Then show any custom derived-internal stuff + out << "\nDetailed operator printing has not been implemented for this op.\n\n"; } } diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/text_file_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/text_file_op.cc index 8ab186761e..ac14f3261e 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/text_file_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/text_file_op.cc @@ -16,6 +16,7 @@ #include #include +#include #include #include #include @@ -90,6 +91,30 @@ TextFileOp::TextFileOp(int32_t num_workers, int64_t rows_per_buffer, int64_t num worker_connector_size_ = worker_connector_size; } +// A print method typically used for debugging +void TextFileOp::Print(std::ostream &out, bool show_all) const { + // Always show the id and name as first line regardless if this summary or detailed print + out << "(" << std::setw(2) << operator_id_ << ") :"; + if (!show_all) { + // Call the super class for displaying any common 1-liner info + ParallelOp::Print(out, show_all); + // Then show any custom derived-internal 1-liner info for this op + out << "\n"; + } else { + // Call the super class for displaying any common detailed info + ParallelOp::Print(out, show_all); + // Then show any custom derived-internal stuff + out << "\nRows per buffer: " << rows_per_buffer_ << "\nSample count: " << num_samples_ + << "\nDevice id: " << device_id_ << "\nNumber of devices: " << num_devices_ + << "\nShuffle files: " << ((shuffle_files_) ? "yes" : "no") << "\nText files list:\n"; + for (int i = 0; i < text_files_list_.size(); ++i) { + out << " " << text_files_list_[i]; + } + out << "\nData Schema:\n"; + out << *data_schema_ << "\n\n"; + } +} + Status TextFileOp::Init() { RETURN_IF_NOT_OK(filename_index_->insert(text_files_list_)); diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/text_file_op.h b/mindspore/ccsrc/dataset/engine/datasetops/source/text_file_op.h index 49f224ffc3..305b2596fa 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/text_file_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/text_file_op.h @@ -144,6 +144,11 @@ class TextFileOp : public ParallelOp { // Default destructor ~TextFileOp() = default; + // A print method typically used for debugging + // @param out - The output stream to write output to + // @param show_all - A bool to control if you want to show all info or just a summary + void Print(std::ostream &out, bool show_all) const override; + // Instantiates the internal queues and connectors // @return Status - the error code returned Status Init(); diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.cc index 50c60caa86..e4121bf8f8 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.cc @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -155,6 +156,36 @@ TFReaderOp::TFReaderOp(int32_t num_workers, int32_t worker_connector_size, int64 worker_connector_size_ = worker_connector_size; } +// A print method typically used for debugging +void TFReaderOp::Print(std::ostream &out, bool show_all) const { + // Always show the id and name as first line regardless if this summary or detailed print + out << "(" << std::setw(2) << operator_id_ << ") :"; + if (!show_all) { + // Call the super class for displaying any common 1-liner info + ParallelOp::Print(out, show_all); + // Then show any custom derived-internal 1-liner info for this op + out << "\n"; + } else { + // Call the super class for displaying any common detailed info + ParallelOp::Print(out, show_all); + // Then show any custom derived-internal stuff + out << "\nRows per buffer: " << rows_per_buffer_ << "\nTotal rows: " << total_rows_ << "\nDevice id: " << device_id_ + << "\nNumber of devices: " << num_devices_ << "\nShuffle files: " << ((shuffle_files_) ? "yes" : "no") + << "\nDataset files list:\n"; + for (int i = 0; i < dataset_files_list_.size(); ++i) { + out << " " << dataset_files_list_[i]; + } + if (!columns_to_load_.empty()) { + out << "\nColumns to load:\n"; + for (int i = 0; i < columns_to_load_.size(); ++i) { + out << " " << columns_to_load_[i]; + } + } + out << "\nData Schema:\n"; + out << *data_schema_ << "\n\n"; + } +} + Status TFReaderOp::Init() { if (data_schema_->Empty()) { RETURN_IF_NOT_OK(CreateSchema(dataset_files_list_[0], columns_to_load_)); diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.h b/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.h index 560cff114f..f0f08c7971 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.h @@ -188,6 +188,11 @@ class TFReaderOp : public ParallelOp { // Default destructor ~TFReaderOp() = default; + // A print method typically used for debugging + // @param out - The output stream to write output to + // @param show_all - A bool to control if you want to show all info or just a summary + void Print(std::ostream &out, bool show_all) const override; + // Instantiates the internal queues and connectors. // @return Status - the error code returned. Status Init(); diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/voc_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/voc_op.cc index e523aa84d6..2befa1a3be 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/voc_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/voc_op.cc @@ -16,7 +16,7 @@ #include "dataset/engine/datasetops/source/voc_op.h" #include - +#include #include "common/utils.h" #include "dataset/core/config_manager.h" #include "dataset/core/tensor_shape.h" @@ -133,9 +133,19 @@ Status VOCOp::operator()() { } void VOCOp::Print(std::ostream &out, bool show_all) const { - DatasetOp::Print(out, show_all); - out << "\nnumber of parallel workers:" << num_workers_ << "\nNumber of rows:" << num_rows_ - << "\nVOC Directory: " << folder_path_ << "\n-------------------\n"; + // Always show the id and name as first line regardless if this summary or detailed print + out << "(" << std::setw(2) << operator_id_ << ") :"; + if (!show_all) { + // Call the super class for displaying any common 1-liner info + ParallelOp::Print(out, show_all); + // Then show any custom derived-internal 1-liner info for this op + out << "\n"; + } else { + // Call the super class for displaying any common detailed info + ParallelOp::Print(out, show_all); + // Then show any custom derived-internal stuff + out << "\nNumber of rows: " << num_rows_ << "\nVOC Directory: " << folder_path_ << "\n\n"; + } } Status VOCOp::Reset() { diff --git a/mindspore/ccsrc/dataset/engine/datasetops/take_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/take_op.cc index 5d7df58153..872c4c27c5 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/take_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/take_op.cc @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - +#include #include #include "common/utils.h" @@ -47,12 +47,19 @@ TakeOp::TakeOp(int32_t count) : PipelineOp(0), max_takes_(count), take_count_(0) // A print method typically used for debugging void TakeOp::Print(std::ostream &out, bool show_all) const { - // Call base class printer first - PipelineOp::Print(out, show_all); - - // Then display our own stuff - out << "TakeOp:" - << "\nCurrent take count: " << take_count_ << "\nMax take count: " << max_takes_; + // Always show the id and name as first line regardless if this summary or detailed print + out << "(" << std::setw(2) << operator_id_ << ") :"; + if (!show_all) { + // Call the super class for displaying any common 1-liner info + PipelineOp::Print(out, show_all); + // Then show any custom derived-internal 1-liner info for this op + out << " [takes: " << max_takes_ << "]\n"; + } else { + // Call the super class for displaying any common detailed info + PipelineOp::Print(out, show_all); + // Then show any custom derived-internal stuff + out << "\nTake count: " << take_count_ << "\nMax takes: " << max_takes_ << "\n\n"; + } } // This function will be call muti times to returns the buffer, when meet required max take count or meet diff --git a/mindspore/ccsrc/dataset/engine/datasetops/zip_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/zip_op.cc index ec771740c1..bb8bddcc09 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/zip_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/zip_op.cc @@ -15,6 +15,7 @@ */ #include "dataset/engine/datasetops/zip_op.h" #include +#include #include "dataset/core/constants.h" #include "dataset/engine/data_buffer.h" #include "dataset/engine/db_connector.h" @@ -224,10 +225,19 @@ Status ZipOp::drainPipeline() { // A function that prints info about the Operator void ZipOp::Print(std::ostream &out, // In: The output stream to print to bool show_all) const { // In: T/F if it should print everything - // Call base class printer first - PipelineOp::Print(out, show_all); - out << "\nZipOp:\n" - << "\nDatasets: " << children_num_ << "\n\n"; + // Always show the id and name as first line regardless if this is summary or detailed print + out << "(" << std::setw(2) << operator_id_ << ") :"; + if (!show_all) { + // Call the super class for displaying any common 1-liner info + PipelineOp::Print(out, show_all); + // Then show any custom derived-internal 1-liner info for this op + out << "\n"; + } else { + // Call the super class for displaying any common detailed info + PipelineOp::Print(out, show_all); + // Then show any custom derived-internal stuff + out << "\nDatasets: " << children_num_ << "\n\n"; + } } // overwrite function and handle eof diff --git a/mindspore/ccsrc/dataset/engine/execution_tree.cc b/mindspore/ccsrc/dataset/engine/execution_tree.cc index ebfa532195..dbcc201d48 100644 --- a/mindspore/ccsrc/dataset/engine/execution_tree.cc +++ b/mindspore/ccsrc/dataset/engine/execution_tree.cc @@ -81,13 +81,29 @@ Status ExecutionTree::AssignRoot(const std::shared_ptr &op) { } // A print method typically used for debugging -void ExecutionTree::Print(std::ostream &out, bool show_all) const { - out << "Total number of nodes in the ExecutionTree (may or may not be connected nodes): " << id_count_ - << "\nTree state: " << static_cast(tree_state_) << "\n"; - if (root_ != nullptr) { - // Just call the printer on the root node. Each node descends to it's children to print them if - // showAll is true. - root_->Print(out, show_all); +void ExecutionTree::Print(std::ostream &out) const { + out << "Execution tree summary:\n" + << "-----------------------\n"; + this->PrintNode(out, root_, "", true, false); + out << "\nExecution tree operator details:\n" + << "--------------------------------\n"; + this->PrintNode(out, root_, "", true, true); +} + +// A helper functions for doing the recursive printing +void ExecutionTree::PrintNode(std::ostream &out, const std::shared_ptr &dataset_op, std::string indent, + bool last, bool detailed) const { + // Decide which printer to use based on detailed arg. + if (!detailed) { + out << indent << "+- " << *dataset_op; + indent += (last ? " " : "| "); + } else { + dataset_op->Print(out, detailed); + } + + // Descend to children + for (int32_t i = 0; i < dataset_op->child_.size(); ++i) { + this->PrintNode(out, dataset_op->child_[i], indent, (i == (dataset_op->child_.size() - 1)), detailed); } } @@ -100,6 +116,9 @@ Status ExecutionTree::Launch() { " Expected state: " + std::to_string(static_cast(kDeTStateReady)); RETURN_STATUS_UNEXPECTED(err_msg); } + std::ostringstream ss; + ss << *this; + MS_LOG(INFO) << "Printing the tree before launch tasks:\n" << ss.str(); for (auto itr = this->begin(); itr != this->end(); ++itr) { // An inlined operator is one that has an output connector size of 0, and it does not // require a thread to execute. Instead, the work of this operator is executed inlined diff --git a/mindspore/ccsrc/dataset/engine/execution_tree.h b/mindspore/ccsrc/dataset/engine/execution_tree.h index 0f6cdfc165..838eb3a014 100644 --- a/mindspore/ccsrc/dataset/engine/execution_tree.h +++ b/mindspore/ccsrc/dataset/engine/execution_tree.h @@ -19,6 +19,7 @@ #include #include #include +#include #include #include "dataset/engine/datasetops/dataset_op.h" #include "dataset/util/status.h" @@ -114,8 +115,7 @@ class ExecutionTree { // A print method typically used for debugging // @param out - The output stream to write output to - // @param show_all - A bool to control if you want to show all info or just a summary - void Print(std::ostream &out, bool show_all) const; + void Print(std::ostream &out) const; // Returns an iterator positioned at the start // @return Iterator - The iterator @@ -133,7 +133,7 @@ class ExecutionTree { // @param exe_tree - reference to the execution tree to display // @return - the output stream must be returned friend std::ostream &operator<<(std::ostream &out, ExecutionTree &exe_tree) { - exe_tree.Print(out, false); + exe_tree.Print(out); return out; } @@ -178,6 +178,14 @@ class ExecutionTree { TaskGroup *AllTasks() const { return tg_.get(); } private: + // A helper functions for doing the recursive printing + // @param dataset_op - The dataset op to print + // @param indent - an indent string for aligning child levels in output + // @param last - an indicator if it's the last child or not + // @param detailed - should it display the detailed node output or the summary line + void PrintNode(std::ostream &out, const std::shared_ptr &dataset_op, std::string indent, bool last, + bool detailed) const; + std::unique_ptr tg_; // Class for worker management std::shared_ptr root_; // The root node of the tree int32_t id_count_; // Counter for generating operator id's