diff --git a/mindspore/ccsrc/dataset/engine/datasetops/barrier_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/barrier_op.cc index b50a7788da..3d5b682155 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/barrier_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/barrier_op.cc @@ -14,6 +14,7 @@ * limitations under the License. */ #include "dataset/engine/datasetops/barrier_op.h" +#include #include #include "dataset/core/constants.h" #include "dataset/engine/data_buffer.h" @@ -214,10 +215,19 @@ Status BarrierOp::getNextTensorRow(TensorRow *new_row) { // A function that prints info about the Operator void BarrierOp::Print(std::ostream &out, bool show_all) const { - // Call base class printer first - PipelineOp::Print(out, show_all); - out << "\nBarrierOp:\n" - << "\nCondition " << condition_name_ << "\n\n"; + // Always show the id and name as first line regardless if this summary or detailed print + out << "(" << std::setw(2) << operator_id_ << ") :"; + if (!show_all) { + // Call the super class for displaying any common 1-liner info + PipelineOp::Print(out, show_all); + // Then show any custom derived-internal 1-liner info for this op + out << "\n"; + } else { + // Call the super class for displaying any common detailed info + PipelineOp::Print(out, show_all); + // Then show any custom derived-internal stuff + out << "\nCondition: " << condition_name_ << "\n\n"; + } } // overwrite function and handle eof diff --git a/mindspore/ccsrc/dataset/engine/datasetops/batch_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/batch_op.cc index c80078cb44..ad8b95b625 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/batch_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/batch_op.cc @@ -15,6 +15,7 @@ */ #include "dataset/engine/datasetops/batch_op.h" #include +#include #include "common/utils.h" #include "dataset/engine/data_buffer.h" #include "dataset/engine/db_connector.h" @@ -102,10 +103,19 @@ Status BatchOp::operator()() { } void BatchOp::Print(std::ostream &out, bool show_all) const { - ParallelOp::Print(out, show_all); - out << "\nBatchOp:\n" - << "number of parallel workers: " << num_workers_ << "\nBatch size: " << start_batch_size_ - << "\nDrop remainder: " << (drop_ ? "yes" : "no") << "\n\n"; + // Always show the id and name as first line regardless if this summary or detailed print + out << "(" << std::setw(2) << operator_id_ << ") :"; + if (!show_all) { + // Call the super class for displaying any common 1-liner info + ParallelOp::Print(out, show_all); + // Then show any custom derived-internal 1-liner info for this op + out << " [batch size: " << start_batch_size_ << "]\n"; + } else { + // Call the super class for displaying any common detailed info + ParallelOp::Print(out, show_all); + // Then show any custom derived-internal stuff + out << "\nStart batch size: " << start_batch_size_ << "\nDrop remainder: " << (drop_ ? "yes" : "no") << "\n\n"; + } } Status BatchOp::BatchRows(const std::unique_ptr *source_table, diff --git a/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.cc index 5e3ea3dc44..adbf42487e 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.cc @@ -92,18 +92,24 @@ void DatasetOp::CreateConnector(int32_t num_producers, int32_t num_consumers) { // A print method typically used for debugging. showAll of true will recursively descend to child prints void DatasetOp::Print(std::ostream &out, bool show_all) const { + // When show_all is false, we display a 1 liner piece of text for the op. + // When show_all is true, we display more detailed output for the op. + // Derived printers should show their own header info, then call base class printer, followed by + // derived-specific items. + // For now, the base class doesn't have any summary info to show so it's a no-op in that case. if (show_all) { + // The detailed display will show common base class info of the op. Allow the derived class to print + // it's own id and name though as the first line. + out << "\nNumber of children : " << child_.size(); for (size_t i = 0; i < child_.size(); i++) { - child_[i]->Print(out, show_all); + out << "\n Child[" << i << "] id: " << child_[i]->id(); } - } - out << "\n-------------------------" - << "\nOperator # : " << operator_id_ << "\nNumber of children : " << child_.size() - << "\nNumber of parents : " << parent_.size() << "\nConnector queue size : " << oc_queue_size_ - << "\nOperator control flags : 0x" << std::hex << std::setw(8) << std::setfill('0') << op_ctrl_flags_ << std::dec - << std::setfill(' ') << "\nHas parents:\n"; - for (size_t i = 0; i < parent_.size(); i++) { - out << "Parent[" << i << "] id: " << parent_[i]->id() << "\n"; + out << "\nNumber of parents : " << parent_.size(); + for (size_t i = 0; i < parent_.size(); i++) { + out << "\n Parent[" << i << "] id: " << parent_[i]->id(); + } + out << "\nConnector queue size : " << oc_queue_size_ << "\nOperator control flags : 0x" << std::hex + << std::setw(8) << std::setfill('0') << op_ctrl_flags_ << std::dec << std::setfill(' '); } } diff --git a/mindspore/ccsrc/dataset/engine/datasetops/device_queue_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/device_queue_op.cc index 71e4ce64a4..2c91d36259 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/device_queue_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/device_queue_op.cc @@ -14,7 +14,7 @@ * limitations under the License. */ #include "dataset/engine/datasetops/device_queue_op.h" - +#include #include #include @@ -246,9 +246,19 @@ Status DeviceQueueOp::SendDataToCPU() { } void DeviceQueueOp::Print(std::ostream &out, bool show_all) const { - PipelineOp::Print(out, show_all); - - out << "DeviceQueueOp: channelName: " << channel_name_ << ", prefetchSize: " << prefetch_size_ << '\n'; + // Always show the id and name as first line regardless if this summary or detailed print + out << "(" << std::setw(2) << operator_id_ << ") :"; + if (!show_all) { + // Call the super class for displaying any common 1-liner info + PipelineOp::Print(out, show_all); + // Then show any custom derived-internal 1-liner info for this op + out << "\n"; + } else { + // Call the super class for displaying any common detailed info + PipelineOp::Print(out, show_all); + // Then show any custom derived-internal stuff + out << "\nChannel name: " << channel_name_ << "\nPrefetch size: " << prefetch_size_ << "\n\n"; + } } } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/filter_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/filter_op.cc index ce312ce3d9..5ede8ad6f4 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/filter_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/filter_op.cc @@ -16,6 +16,7 @@ #include "dataset/engine/datasetops/filter_op.h" #include #include +#include #include #include #include @@ -88,14 +89,22 @@ Status FilterOp::ValidateInColumns(const std::unordered_map:"; + if (!show_all) { + // Call the super class for displaying any common 1-liner info + ParallelOp::Print(out, show_all); + // Then show any custom derived-internal 1-liner info for this op + out << "\n"; + } else { + // Call the super class for displaying any common detailed info + ParallelOp::Print(out, show_all); + // Then show any custom derived-internal stuff + out << "\nInput column names:"; + for (size_t i = 0; i < in_columns_.size(); i++) { + out << " " << in_columns_[i]; + } + out << "\n\n"; } } diff --git a/mindspore/ccsrc/dataset/engine/datasetops/map_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/map_op.cc index b6d603bac9..4cbe2ac603 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/map_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/map_op.cc @@ -15,6 +15,7 @@ */ #include "dataset/engine/datasetops/map_op.h" #include +#include #include #include #include @@ -81,20 +82,27 @@ int32_t MapOp::num_consumers() const { // A print method typically used for debugging void MapOp::Print(std::ostream &out, bool show_all) const { - // Call base class printer first - ParallelOp::Print(out, show_all); - - // Then display our own stuff - out << "\nMapOp:"; - out << "\n Input column names:"; - for (size_t i = 0; i < in_columns_.size(); i++) { - out << " " << in_columns_[i]; - } - out << "\n TensorOps:"; - for (size_t i = 0; i < tfuncs_.size(); i++) { - out << " " << tfuncs_[i]; + // Always show the id and name as first line regardless if this summary or detailed print + out << "(" << std::setw(2) << operator_id_ << ") :"; + if (!show_all) { + // Call the super class for displaying any common 1-liner info + ParallelOp::Print(out, show_all); + // Then show any custom derived-internal 1-liner info for this op + out << "\n"; + } else { + // Call the super class for displaying any common detailed info + ParallelOp::Print(out, show_all); + // Then show any custom derived-internal stuff + out << "\nInput column names:"; + for (size_t i = 0; i < in_columns_.size(); i++) { + out << " " << in_columns_[i]; + } + out << "\n TensorOps:"; + for (size_t i = 0; i < tfuncs_.size(); i++) { + out << " " << tfuncs_[i]; + } + out << "\n\n"; } - out << "\n"; } // This class functor will provide the master loop that drives the logic for performing the work diff --git a/mindspore/ccsrc/dataset/engine/datasetops/parallel_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/parallel_op.cc index 4b2af2250a..2eeb931554 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/parallel_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/parallel_op.cc @@ -55,12 +55,16 @@ Status ParallelOp::CreateWorkerConnector(int32_t worker_connector_size) { // A print method typically used for debugging void ParallelOp::Print(std::ostream &out, bool show_all) const { - // Call base class printer first - DatasetOp::Print(out, show_all); - - // Then show our own stuff - out << "ParallelOp:"; - out << "\n Num workers : " << num_workers_ << "\n"; + // Summary 1-liner print + if (!show_all) { + out << " [workers: " << num_workers_ << "]"; + // Call super class printer + DatasetOp::Print(out, show_all); + } else { + // Detailed print + DatasetOp::Print(out, show_all); + out << "\nNum workers: " << num_workers_; + } } // Override base class reset to provide reset actions specific to the ParallelOp class. diff --git a/mindspore/ccsrc/dataset/engine/datasetops/pipeline_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/pipeline_op.cc index 56fc24883a..69ace1ed9a 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/pipeline_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/pipeline_op.cc @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +#include #include #include "dataset/engine/datasetops/pipeline_op.h" @@ -23,11 +24,26 @@ PipelineOp::PipelineOp(int32_t op_connector_size) : DatasetOp(op_connector_size) // A print method typically used for debugging void PipelineOp::Print(std::ostream &out, bool show_all) const { - // Call base class printer first - DatasetOp::Print(out, show_all); - - // Then display our own stuff for the pipeline op - // out << "This is a pipeline op print. nothing to display here at the moment.\n"; + // Summary 1-liner print + if (!show_all) { + out << " [workers: "; + if (this->inlined()) { + out << "0 (inlined)]"; + } else { + out << "1]"; // Pipeline ops only have 1 worker + } + // Call super class printer + DatasetOp::Print(out, show_all); + } else { + // Detailed print + DatasetOp::Print(out, show_all); + out << "\nNum workers: "; + if (this->inlined()) { + out << "0 (inlined)"; + } else { + out << "1"; // Pipeline ops only have 1 worker + } + } } } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/project_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/project_op.cc index b87967dde8..128d3e68e5 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/project_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/project_op.cc @@ -16,6 +16,7 @@ #include "dataset/engine/datasetops/project_op.h" #include +#include #include #include #include @@ -49,12 +50,23 @@ ProjectOp::ProjectOp(const std::vector &columns_to_project) : PipelineOp(0), columns_to_project_(columns_to_project) {} void ProjectOp::Print(std::ostream &out, bool show_all) const { - PipelineOp::Print(out, show_all); - out << "ProjectOp: columns that are projected: "; - for (size_t i = 0; i < columns_to_project_.size(); i++) { - out << columns_to_project_[i] << " "; + // Always show the id and name as first line regardless if this summary or detailed print + out << "(" << std::setw(2) << operator_id_ << ") :"; + if (!show_all) { + // Call the super class for displaying any common 1-liner info + PipelineOp::Print(out, show_all); + // Then show any custom derived-internal 1-liner info for this op + out << "\n"; + } else { + // Call the super class for displaying any common detailed info + PipelineOp::Print(out, show_all); + // Then show any custom derived-internal stuff + out << "\nColumns that are projected:"; + for (size_t i = 0; i < columns_to_project_.size(); i++) { + out << "\n" << columns_to_project_[i]; + } + out << "\n\n"; } - out << '\n'; } // Gets a buffer from the child operator and projects the buffer. diff --git a/mindspore/ccsrc/dataset/engine/datasetops/rename_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/rename_op.cc index 725476bf91..5f354abb04 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/rename_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/rename_op.cc @@ -14,7 +14,7 @@ * limitations under the License. */ #include "dataset/engine/datasetops/rename_op.h" - +#include #include #include #include @@ -138,11 +138,25 @@ Status RenameOp::RenameBuffer(std::unique_ptr *input_buffer) { // prints rename void RenameOp::Print(std::ostream &out, // In: The output stream to print to bool show_all) const { // In: T/F if it should print everything - // Call base class printer first - PipelineOp::Print(out, show_all); - out << "\nRenameOp:\n"; - for (size_t i = 0; i < in_columns_.size(); ++i) { - out << "\nin Columns: " << in_columns_[i] << "\nOut Columns: " << out_columns_[i] << "\n\n"; + // Always show the id and name as first line regardless if this summary or detailed print + out << "(" << std::setw(2) << operator_id_ << ") :"; + if (!show_all) { + // Call the super class for displaying any common 1-liner info + PipelineOp::Print(out, show_all); + // Then show any custom derived-internal 1-liner info for this op + out << "\n"; + } else { + // Call the super class for displaying any common detailed info + PipelineOp::Print(out, show_all); + // Then show any custom derived-internal stuff + out << "\nIn columns:"; + for (size_t i = 0; i < in_columns_.size(); ++i) { + out << "\n " << in_columns_[i]; + } + for (size_t i = 0; i < out_columns_.size(); ++i) { + out << "\n " << out_columns_[i]; + } + out << "\n\n"; } } diff --git a/mindspore/ccsrc/dataset/engine/datasetops/repeat_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/repeat_op.cc index 33c731c400..065631eb31 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/repeat_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/repeat_op.cc @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +#include #include #include @@ -51,22 +52,28 @@ RepeatOp::~RepeatOp() {} // A print method typically used for debugging void RepeatOp::Print(std::ostream &out, bool show_all) const { - // Call base class printer first - PipelineOp::Print(out, show_all); - - // Then display our own stuff - out << "RepeatOp:" - << "\nCurrent repeat count: " << repeat_count_ << "\nMax repeat count: " << max_repeats_ - << "\nLeaf Nodes in my execution path:"; - if (!eoe_ops_.empty()) { - out << "\n"; - for (size_t i = 0; i < eoe_ops_.size(); i++) { - out << " Operator: " << eoe_ops_[i]->id() << "\n"; - } + // Always show the id and name as first line regardless if this summary or detailed print + out << "(" << std::setw(2) << operator_id_ << ") :"; + if (!show_all) { + // Call the super class for displaying any common 1-liner info + PipelineOp::Print(out, show_all); + // Then show any custom derived-internal 1-liner info for this op + out << " [repeats: " << max_repeats_ << "]\n"; } else { - out << " kNone."; + // Call the super class for displaying any common detailed info + PipelineOp::Print(out, show_all); + // Then show any custom derived-internal stuff + out << "\nCurrent repeat count: " << repeat_count_ << "\nMax repeat count: " << max_repeats_ + << "\nLeaf Nodes in execution path:"; + if (!eoe_ops_.empty()) { + for (size_t i = 0; i < eoe_ops_.size(); i++) { + out << "\n Operator: " << eoe_ops_[i]->id(); + } + } else { + out << " None."; + } + out << "\n\n"; } - out << "\n-------------------------\n\n"; // End the display with this line } // Base-class override for executing specific RepeatOp configurations. This code will be called diff --git a/mindspore/ccsrc/dataset/engine/datasetops/shuffle_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/shuffle_op.cc index 422c38f2f2..7b09bcef4d 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/shuffle_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/shuffle_op.cc @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -108,13 +109,20 @@ Status ShuffleOp::SelfReset() { // A print method typically used for debugging void ShuffleOp::Print(std::ostream &out, bool show_all) const { - // Call base class printer first - PipelineOp::Print(out, show_all); - - // Then display our own stuff - out << "ShuffleOp:\n Shuffle size: " << shuffle_size_ << "\n rows_per_buffer_: " << rows_per_buffer_ - << "\n shuffle_buffer_state_: " << shuffle_buffer_state_ << "\n shuffle_seed_: " << shuffle_seed_; - out << "\n-------------------------\n\n"; // End the display with this line + // Always show the id and name as first line regardless if this summary or detailed print + out << "(" << std::setw(2) << operator_id_ << ") :"; + if (!show_all) { + // Call the super class for displaying any common 1-liner info + PipelineOp::Print(out, show_all); + // Then show any custom derived-internal 1-liner info for this op + out << " [shuffle size: " << shuffle_size_ << "]\n"; + } else { + // Call the super class for displaying any common detailed info + PipelineOp::Print(out, show_all); + // Then show any custom derived-internal stuff + out << "\nShuffle size: " << shuffle_size_ << "\nRows per buffer: " << rows_per_buffer_ + << "\nShuffle buffer state: " << shuffle_buffer_state_ << "\nShuffle seed: " << shuffle_seed_ << "\n\n"; + } } // Private function to add a new row to the shuffle buffer. diff --git a/mindspore/ccsrc/dataset/engine/datasetops/skip_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/skip_op.cc index d851f2c699..ec45d5d25e 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/skip_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/skip_op.cc @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +#include #include #include @@ -51,12 +52,19 @@ SkipOp::~SkipOp() {} // A print method typically used for debugging void SkipOp::Print(std::ostream &out, bool show_all) const { - // Call base class printer first - PipelineOp::Print(out, show_all); - - // Then display our own stuff - out << "SkipOp:" - << "\nCurrent skip count: " << skip_count_ << "\nMax skip count: " << max_skips_; + // Always show the id and name as first line regardless if this summary or detailed print + out << "(" << std::setw(2) << operator_id_ << ") :"; + if (!show_all) { + // Call the super class for displaying any common 1-liner info + PipelineOp::Print(out, show_all); + // Then show any custom derived-internal 1-liner info for this op + out << " [skips: " << max_skips_ << "]\n"; + } else { + // Call the super class for displaying any common detailed info + PipelineOp::Print(out, show_all); + // Then show any custom derived-internal stuff + out << "\nSkip count: " << skip_count_ << "\nMax skips: " << max_skips_ << "\n\n"; + } } // Since the buffer may contain multi rows, this function will drop the rows diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/celeba_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/celeba_op.cc index 2394380ea4..998672bca5 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/celeba_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/celeba_op.cc @@ -16,6 +16,7 @@ #include "dataset/engine/datasetops/source/celeba_op.h" #include +#include #include "dataset/core/config_manager.h" #include "dataset/util/path.h" #include "dataset/engine/datasetops/source/sampler/sequential_sampler.h" @@ -434,9 +435,19 @@ Status CelebAOp::LoadTensorRow(const std::pair } void CelebAOp::Print(std::ostream &out, bool show_all) const { - DatasetOp::Print(out, show_all); - out << "\nnumber of parallel workers:" << num_workers_ << "\nNumber of rows:" << num_rows_exact_ - << "\nceleba dir: " << folder_path_ << "\n-------------------------\n"; + // Always show the id and name as first line regardless if this summary or detailed print + out << "(" << std::setw(2) << operator_id_ << ") :"; + if (!show_all) { + // Call the super class for displaying any common 1-liner info + ParallelOp::Print(out, show_all); + // Then show any custom derived-internal 1-liner info for this op + out << "\n"; + } else { + // Call the super class for displaying any common detailed info + ParallelOp::Print(out, show_all); + // Then show any custom derived-internal stuff + out << "\nNumber of rows:" << num_rows_exact_ << "\nceleba dir: " << folder_path_ << "\n\n"; + } } // Reset Sampler and wakeup Master thread (functor) diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/cifar_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/cifar_op.cc index 0c2d57ff42..4f69938a31 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/cifar_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/cifar_op.cc @@ -17,6 +17,7 @@ #include #include +#include #include #include "common/utils.h" @@ -225,9 +226,19 @@ Status CifarOp::LoadBuffer(const std::vector &keys, std::unique_ptr:"; + if (!show_all) { + // Call the super class for displaying any common 1-liner info + ParallelOp::Print(out, show_all); + // Then show any custom derived-internal 1-liner info for this op + out << "\n"; + } else { + // Call the super class for displaying any common detailed info + ParallelOp::Print(out, show_all); + // Then show any custom derived-internal stuff + out << "\nNumber of rows:" << num_rows_ << "\nCifar directory: " << folder_path_ << "\n\n"; + } } // Reset Sampler and wakeup Master thread (functor) diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/generator_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/generator_op.cc index 37a74f019a..b0c8b8af35 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/generator_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/generator_op.cc @@ -13,8 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "dataset/core/global_context.h" #include "dataset/engine/datasetops/source/generator_op.h" +#include +#include "dataset/core/global_context.h" #include "dataset/engine/db_connector.h" #include "dataset/engine/data_buffer.h" #include "dataset/engine/execution_tree.h" @@ -58,6 +59,26 @@ GeneratorOp::GeneratorOp(py::function generator_function, std::vectorDealloc(); } +void GeneratorOp::Print(std::ostream &out, bool show_all) const { + // Always show the id and name as first line regardless if this summary or detailed print + out << "(" << std::setw(2) << operator_id_ << ") :"; + if (!show_all) { + // Call the super class for displaying any common 1-liner info + PipelineOp::Print(out, show_all); + // Then show any custom derived-internal 1-liner info for this op + out << "\n"; + } else { + // Call the super class for displaying any common detailed info + PipelineOp::Print(out, show_all); + // Then show any custom derived-internal stuff + out << "\nColumn names:\n"; + for (int i = 0; i < column_names_.size(); ++i) { + out << "\n " << column_names_[i]; + } + out << "\n\n"; + } +} + void GeneratorOp::Dealloc() noexcept { // Setup GIL state PyGILState_STATE gstate; diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/generator_op.h b/mindspore/ccsrc/dataset/engine/datasetops/source/generator_op.h index a5407a9b09..8165fed970 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/generator_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/generator_op.h @@ -95,6 +95,11 @@ class GeneratorOp : public PipelineOp { ~GeneratorOp(); + // A print method typically used for debugging + // @param out - The output stream to write output to + // @param show_all - A bool to control if you want to show all info or just a summary + void Print(std::ostream &out, bool show_all) const override; + // << Stream output operator overload // @notes This allows you to write the debug print info using stream operators // @param out - reference to the output stream being overloaded diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/image_folder_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/image_folder_op.cc index 32d7171c8f..b52404ce69 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/image_folder_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/image_folder_op.cc @@ -14,9 +14,8 @@ * limitations under the License. */ #include "dataset/engine/datasetops/source/image_folder_op.h" - #include - +#include #include "common/utils.h" #include "dataset/core/config_manager.h" #include "dataset/core/tensor_shape.h" @@ -243,9 +242,19 @@ Status ImageFolderOp::LoadBuffer(const std::vector &keys, std::unique_p } void ImageFolderOp::Print(std::ostream &out, bool show_all) const { - DatasetOp::Print(out, show_all); - out << "\nnumber of parallel workers:" << num_workers_ << "\nNumber of rows:" << num_rows_ - << "\nImageFolder Directory: " << folder_path_ << "\n-------------------------\n"; + // Always show the id and name as first line regardless if this summary or detailed print + out << "(" << std::setw(2) << operator_id_ << ") :"; + if (!show_all) { + // Call the super class for displaying any common 1-liner info + ParallelOp::Print(out, show_all); + // Then show any custom derived-internal 1-liner info for this op + out << "\n"; + } else { + // Call the super class for displaying any common detailed info + ParallelOp::Print(out, show_all); + // Then show any custom derived-internal stuff + out << "\nNumber of rows:" << num_rows_ << "\nImageFolder directory: " << folder_path_ << "\n\n"; + } } // Reset Sampler and wakeup Master thread (functor) diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/manifest_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/manifest_op.cc index ab0c012416..fb47274aae 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/manifest_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/manifest_op.cc @@ -17,6 +17,7 @@ #include #include +#include #include #include "common/utils.h" @@ -239,9 +240,19 @@ Status ManifestOp::LoadBuffer(const std::vector &keys, std::unique_ptr< } void ManifestOp::Print(std::ostream &out, bool show_all) const { - DatasetOp::Print(out, show_all); - out << "\nnumber of parallel workers:" << num_workers_ << "\nNumber of rows:" << num_rows_ - << "\nManifest file: " << file_ << "\n-------------------------\n"; + // Always show the id and name as first line regardless if this summary or detailed print + out << "(" << std::setw(2) << operator_id_ << ") :"; + if (!show_all) { + // Call the super class for displaying any common 1-liner info + ParallelOp::Print(out, show_all); + // Then show any custom derived-internal 1-liner info for this op + out << "\n"; + } else { + // Call the super class for displaying any common detailed info + ParallelOp::Print(out, show_all); + // Then show any custom derived-internal stuff + out << "\nNumber of rows:" << num_rows_ << "\nManifest file: " << file_ << "\n\n"; + } } // Reset Sampler and wakeup Master thread (functor) diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/mindrecord_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/mindrecord_op.cc index 72dee6f2e6..c8bbf0e3c4 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/mindrecord_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/mindrecord_op.cc @@ -17,6 +17,7 @@ #include #include +#include #include #include @@ -179,18 +180,21 @@ MindRecordOp::~MindRecordOp() {} // A print method typically used for debugging void MindRecordOp::Print(std::ostream &out, bool show_all) const { - // Call base class printer first - ParallelOp::Print(out, show_all); - - // Then display our own stuff - out << "\nMindRecordOp:"; - out << "\n 1 Dataset file : " << dataset_file_; - out << "\n Number of rows : " << num_rows_; - out << "\n Rows per buffer : " << rows_per_buffer_; - out << "\n Number of buffers : " << buffers_needed_; - out << "\n Number of ShardReader workers : " << num_mind_record_workers_; - - out << "\n\n"; + // Always show the id and name as first line regardless if this summary or detailed print + out << "(" << std::setw(2) << operator_id_ << ") :"; + if (!show_all) { + // Call the super class for displaying any common 1-liner info + ParallelOp::Print(out, show_all); + // Then show any custom derived-internal 1-liner info for this op + out << "\n"; + } else { + // Call the super class for displaying any common detailed info + ParallelOp::Print(out, show_all); + // Then show any custom derived-internal stuff + out << "\n1 Dataset file : " << dataset_file_ << "\nNumber of rows : " << num_rows_ + << "\nRows per buffer : " << rows_per_buffer_ << "\nNumber of buffers : " << buffers_needed_ + << "\nNumber of ShardReader workers : " << num_mind_record_workers_ << "\n\n"; + } } template diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/mnist_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/mnist_op.cc index fbf041e985..33aca001e9 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/mnist_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/mnist_op.cc @@ -16,7 +16,7 @@ #include "dataset/engine/datasetops/source/mnist_op.h" #include - +#include #include "common/utils.h" #include "dataset/core/config_manager.h" #include "dataset/core/tensor_shape.h" @@ -190,9 +190,19 @@ Status MnistOp::LoadBuffer(const std::vector &keys, std::unique_ptr:"; + if (!show_all) { + // Call the super class for displaying any common 1-liner info + ParallelOp::Print(out, show_all); + // Then show any custom derived-internal 1-liner info for this op + out << "\n"; + } else { + // Call the super class for displaying any common detailed info + ParallelOp::Print(out, show_all); + // Then show any custom derived-internal stuff + out << "\nNumber of rows:" << num_rows_ << "\nMNIST Directory: " << folder_path_ << "\n\n"; + } } // Reset Sampler and wakeup Master thread (functor) diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/storage_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/storage_op.cc index 2ca957ae6d..f310a097ee 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/storage_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/storage_op.cc @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -319,31 +320,18 @@ StorageOp::~StorageOp() {} // A print method typically used for debugging void StorageOp::Print(std::ostream &out, bool show_all) const { - // Call base class printer first - ParallelOp::Print(out, show_all); - - // Then display our own stuff - out << "\nStorageOp:"; - out << "\n Dataset files dir : " << dataset_files_dir_ << "\n Dataset schema file : " << schema_file_; - if (!dataset_file_list_.empty()) { - out << "\n Dataset Files List:\n"; - for (auto filename : dataset_file_list_) { - out << " " << filename << "\n"; - } - } - out << "\n\n"; - if (!data_buffers_.empty()) { - out << std::boolalpha << " Number of DataBuffers inside StorageOp: " << data_buffers_.size() - << "\n Number of rows: " << num_rows_ << "\n Rows per buffer: " << rows_per_buffer_ << "\n\n DataBuffers:\n"; - - // Iterate over each DataBuffer and display the buffer id and the buffer - int32_t i = 0; - for (i = 0; i < data_buffers_.size(); i++) { - out << " " << i << ")\n"; - data_buffers_[i]->Print(out, show_all); - } + // Always show the id and name as first line regardless if this summary or detailed print + out << "(" << std::setw(2) << operator_id_ << ") :"; + if (!show_all) { + // Call the super class for displaying any common 1-liner info + ParallelOp::Print(out, show_all); + // Then show any custom derived-internal 1-liner info for this op + out << "\n"; } else { - out << "DataCache is empty!\n"; + // Call the super class for displaying any common detailed info + ParallelOp::Print(out, show_all); + // Then show any custom derived-internal stuff + out << "\nDetailed operator printing has not been implemented for this op.\n\n"; } } diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/text_file_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/text_file_op.cc index 8ab186761e..ac14f3261e 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/text_file_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/text_file_op.cc @@ -16,6 +16,7 @@ #include #include +#include #include #include #include @@ -90,6 +91,30 @@ TextFileOp::TextFileOp(int32_t num_workers, int64_t rows_per_buffer, int64_t num worker_connector_size_ = worker_connector_size; } +// A print method typically used for debugging +void TextFileOp::Print(std::ostream &out, bool show_all) const { + // Always show the id and name as first line regardless if this summary or detailed print + out << "(" << std::setw(2) << operator_id_ << ") :"; + if (!show_all) { + // Call the super class for displaying any common 1-liner info + ParallelOp::Print(out, show_all); + // Then show any custom derived-internal 1-liner info for this op + out << "\n"; + } else { + // Call the super class for displaying any common detailed info + ParallelOp::Print(out, show_all); + // Then show any custom derived-internal stuff + out << "\nRows per buffer: " << rows_per_buffer_ << "\nSample count: " << num_samples_ + << "\nDevice id: " << device_id_ << "\nNumber of devices: " << num_devices_ + << "\nShuffle files: " << ((shuffle_files_) ? "yes" : "no") << "\nText files list:\n"; + for (int i = 0; i < text_files_list_.size(); ++i) { + out << " " << text_files_list_[i]; + } + out << "\nData Schema:\n"; + out << *data_schema_ << "\n\n"; + } +} + Status TextFileOp::Init() { RETURN_IF_NOT_OK(filename_index_->insert(text_files_list_)); diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/text_file_op.h b/mindspore/ccsrc/dataset/engine/datasetops/source/text_file_op.h index 49f224ffc3..305b2596fa 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/text_file_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/text_file_op.h @@ -144,6 +144,11 @@ class TextFileOp : public ParallelOp { // Default destructor ~TextFileOp() = default; + // A print method typically used for debugging + // @param out - The output stream to write output to + // @param show_all - A bool to control if you want to show all info or just a summary + void Print(std::ostream &out, bool show_all) const override; + // Instantiates the internal queues and connectors // @return Status - the error code returned Status Init(); diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.cc index 50c60caa86..e4121bf8f8 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.cc @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -155,6 +156,36 @@ TFReaderOp::TFReaderOp(int32_t num_workers, int32_t worker_connector_size, int64 worker_connector_size_ = worker_connector_size; } +// A print method typically used for debugging +void TFReaderOp::Print(std::ostream &out, bool show_all) const { + // Always show the id and name as first line regardless if this summary or detailed print + out << "(" << std::setw(2) << operator_id_ << ") :"; + if (!show_all) { + // Call the super class for displaying any common 1-liner info + ParallelOp::Print(out, show_all); + // Then show any custom derived-internal 1-liner info for this op + out << "\n"; + } else { + // Call the super class for displaying any common detailed info + ParallelOp::Print(out, show_all); + // Then show any custom derived-internal stuff + out << "\nRows per buffer: " << rows_per_buffer_ << "\nTotal rows: " << total_rows_ << "\nDevice id: " << device_id_ + << "\nNumber of devices: " << num_devices_ << "\nShuffle files: " << ((shuffle_files_) ? "yes" : "no") + << "\nDataset files list:\n"; + for (int i = 0; i < dataset_files_list_.size(); ++i) { + out << " " << dataset_files_list_[i]; + } + if (!columns_to_load_.empty()) { + out << "\nColumns to load:\n"; + for (int i = 0; i < columns_to_load_.size(); ++i) { + out << " " << columns_to_load_[i]; + } + } + out << "\nData Schema:\n"; + out << *data_schema_ << "\n\n"; + } +} + Status TFReaderOp::Init() { if (data_schema_->Empty()) { RETURN_IF_NOT_OK(CreateSchema(dataset_files_list_[0], columns_to_load_)); diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.h b/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.h index 560cff114f..f0f08c7971 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.h @@ -188,6 +188,11 @@ class TFReaderOp : public ParallelOp { // Default destructor ~TFReaderOp() = default; + // A print method typically used for debugging + // @param out - The output stream to write output to + // @param show_all - A bool to control if you want to show all info or just a summary + void Print(std::ostream &out, bool show_all) const override; + // Instantiates the internal queues and connectors. // @return Status - the error code returned. Status Init(); diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/voc_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/voc_op.cc index e523aa84d6..2befa1a3be 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/voc_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/voc_op.cc @@ -16,7 +16,7 @@ #include "dataset/engine/datasetops/source/voc_op.h" #include - +#include #include "common/utils.h" #include "dataset/core/config_manager.h" #include "dataset/core/tensor_shape.h" @@ -133,9 +133,19 @@ Status VOCOp::operator()() { } void VOCOp::Print(std::ostream &out, bool show_all) const { - DatasetOp::Print(out, show_all); - out << "\nnumber of parallel workers:" << num_workers_ << "\nNumber of rows:" << num_rows_ - << "\nVOC Directory: " << folder_path_ << "\n-------------------\n"; + // Always show the id and name as first line regardless if this summary or detailed print + out << "(" << std::setw(2) << operator_id_ << ") :"; + if (!show_all) { + // Call the super class for displaying any common 1-liner info + ParallelOp::Print(out, show_all); + // Then show any custom derived-internal 1-liner info for this op + out << "\n"; + } else { + // Call the super class for displaying any common detailed info + ParallelOp::Print(out, show_all); + // Then show any custom derived-internal stuff + out << "\nNumber of rows: " << num_rows_ << "\nVOC Directory: " << folder_path_ << "\n\n"; + } } Status VOCOp::Reset() { diff --git a/mindspore/ccsrc/dataset/engine/datasetops/take_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/take_op.cc index 5d7df58153..872c4c27c5 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/take_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/take_op.cc @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - +#include #include #include "common/utils.h" @@ -47,12 +47,19 @@ TakeOp::TakeOp(int32_t count) : PipelineOp(0), max_takes_(count), take_count_(0) // A print method typically used for debugging void TakeOp::Print(std::ostream &out, bool show_all) const { - // Call base class printer first - PipelineOp::Print(out, show_all); - - // Then display our own stuff - out << "TakeOp:" - << "\nCurrent take count: " << take_count_ << "\nMax take count: " << max_takes_; + // Always show the id and name as first line regardless if this summary or detailed print + out << "(" << std::setw(2) << operator_id_ << ") :"; + if (!show_all) { + // Call the super class for displaying any common 1-liner info + PipelineOp::Print(out, show_all); + // Then show any custom derived-internal 1-liner info for this op + out << " [takes: " << max_takes_ << "]\n"; + } else { + // Call the super class for displaying any common detailed info + PipelineOp::Print(out, show_all); + // Then show any custom derived-internal stuff + out << "\nTake count: " << take_count_ << "\nMax takes: " << max_takes_ << "\n\n"; + } } // This function will be call muti times to returns the buffer, when meet required max take count or meet diff --git a/mindspore/ccsrc/dataset/engine/datasetops/zip_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/zip_op.cc index ec771740c1..bb8bddcc09 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/zip_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/zip_op.cc @@ -15,6 +15,7 @@ */ #include "dataset/engine/datasetops/zip_op.h" #include +#include #include "dataset/core/constants.h" #include "dataset/engine/data_buffer.h" #include "dataset/engine/db_connector.h" @@ -224,10 +225,19 @@ Status ZipOp::drainPipeline() { // A function that prints info about the Operator void ZipOp::Print(std::ostream &out, // In: The output stream to print to bool show_all) const { // In: T/F if it should print everything - // Call base class printer first - PipelineOp::Print(out, show_all); - out << "\nZipOp:\n" - << "\nDatasets: " << children_num_ << "\n\n"; + // Always show the id and name as first line regardless if this is summary or detailed print + out << "(" << std::setw(2) << operator_id_ << ") :"; + if (!show_all) { + // Call the super class for displaying any common 1-liner info + PipelineOp::Print(out, show_all); + // Then show any custom derived-internal 1-liner info for this op + out << "\n"; + } else { + // Call the super class for displaying any common detailed info + PipelineOp::Print(out, show_all); + // Then show any custom derived-internal stuff + out << "\nDatasets: " << children_num_ << "\n\n"; + } } // overwrite function and handle eof diff --git a/mindspore/ccsrc/dataset/engine/execution_tree.cc b/mindspore/ccsrc/dataset/engine/execution_tree.cc index ebfa532195..dbcc201d48 100644 --- a/mindspore/ccsrc/dataset/engine/execution_tree.cc +++ b/mindspore/ccsrc/dataset/engine/execution_tree.cc @@ -81,13 +81,29 @@ Status ExecutionTree::AssignRoot(const std::shared_ptr &op) { } // A print method typically used for debugging -void ExecutionTree::Print(std::ostream &out, bool show_all) const { - out << "Total number of nodes in the ExecutionTree (may or may not be connected nodes): " << id_count_ - << "\nTree state: " << static_cast(tree_state_) << "\n"; - if (root_ != nullptr) { - // Just call the printer on the root node. Each node descends to it's children to print them if - // showAll is true. - root_->Print(out, show_all); +void ExecutionTree::Print(std::ostream &out) const { + out << "Execution tree summary:\n" + << "-----------------------\n"; + this->PrintNode(out, root_, "", true, false); + out << "\nExecution tree operator details:\n" + << "--------------------------------\n"; + this->PrintNode(out, root_, "", true, true); +} + +// A helper functions for doing the recursive printing +void ExecutionTree::PrintNode(std::ostream &out, const std::shared_ptr &dataset_op, std::string indent, + bool last, bool detailed) const { + // Decide which printer to use based on detailed arg. + if (!detailed) { + out << indent << "+- " << *dataset_op; + indent += (last ? " " : "| "); + } else { + dataset_op->Print(out, detailed); + } + + // Descend to children + for (int32_t i = 0; i < dataset_op->child_.size(); ++i) { + this->PrintNode(out, dataset_op->child_[i], indent, (i == (dataset_op->child_.size() - 1)), detailed); } } @@ -100,6 +116,9 @@ Status ExecutionTree::Launch() { " Expected state: " + std::to_string(static_cast(kDeTStateReady)); RETURN_STATUS_UNEXPECTED(err_msg); } + std::ostringstream ss; + ss << *this; + MS_LOG(INFO) << "Printing the tree before launch tasks:\n" << ss.str(); for (auto itr = this->begin(); itr != this->end(); ++itr) { // An inlined operator is one that has an output connector size of 0, and it does not // require a thread to execute. Instead, the work of this operator is executed inlined diff --git a/mindspore/ccsrc/dataset/engine/execution_tree.h b/mindspore/ccsrc/dataset/engine/execution_tree.h index 0f6cdfc165..838eb3a014 100644 --- a/mindspore/ccsrc/dataset/engine/execution_tree.h +++ b/mindspore/ccsrc/dataset/engine/execution_tree.h @@ -19,6 +19,7 @@ #include #include #include +#include #include #include "dataset/engine/datasetops/dataset_op.h" #include "dataset/util/status.h" @@ -114,8 +115,7 @@ class ExecutionTree { // A print method typically used for debugging // @param out - The output stream to write output to - // @param show_all - A bool to control if you want to show all info or just a summary - void Print(std::ostream &out, bool show_all) const; + void Print(std::ostream &out) const; // Returns an iterator positioned at the start // @return Iterator - The iterator @@ -133,7 +133,7 @@ class ExecutionTree { // @param exe_tree - reference to the execution tree to display // @return - the output stream must be returned friend std::ostream &operator<<(std::ostream &out, ExecutionTree &exe_tree) { - exe_tree.Print(out, false); + exe_tree.Print(out); return out; } @@ -178,6 +178,14 @@ class ExecutionTree { TaskGroup *AllTasks() const { return tg_.get(); } private: + // A helper functions for doing the recursive printing + // @param dataset_op - The dataset op to print + // @param indent - an indent string for aligning child levels in output + // @param last - an indicator if it's the last child or not + // @param detailed - should it display the detailed node output or the summary line + void PrintNode(std::ostream &out, const std::shared_ptr &dataset_op, std::string indent, bool last, + bool detailed) const; + std::unique_ptr tg_; // Class for worker management std::shared_ptr root_; // The root node of the tree int32_t id_count_; // Counter for generating operator id's