Completely remove unused ExecTree optimizer code, rewrite ExecutionTree::Prepare()

mindspore/ccsrc/minddata/dataset/engine/datasetops/batch_op.cc

@@ -1,5 +1,5 @@
 /**
- * Copyright 2019 Huawei Technologies Co., Ltd
+ * Copyright 2019-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -16,7 +16,6 @@
 #include "minddata/dataset/engine/datasetops/batch_op.h"
 
 #include <utility>
-#include <iomanip>
 
 #include "utils/ms_utils.h"
 #ifdef ENABLE_PYTHON
@@ -24,7 +23,6 @@
 #endif
 #include "minddata/dataset/engine/data_buffer.h"
 #include "minddata/dataset/engine/db_connector.h"
-#include "minddata/dataset/engine/opt/pass.h"
 #include "minddata/dataset/kernels/data/data_utils.h"
 #include "minddata/dataset/util/status.h"
 
@@ -498,18 +496,6 @@ Status BatchOp::UnpackPadInfo(const PadInfo &pad_info,
   return Status::OK();
 }
 
-// Visitor accept method for NodePass
-Status BatchOp::Accept(NodePass *p, bool *const modified) {
-  // Downcast shared pointer then call visitor
-  return p->RunOnNode(shared_from_base<BatchOp>(), modified);
-}
-
-// Visitor pre-accept method for NodePass
-Status BatchOp::PreAccept(NodePass *p, bool *const modified) {
-  // Downcast shared pointer then call visitor
-  return p->PreRunOnNode(shared_from_base<BatchOp>(), modified);
-}
-
 Status BatchOp::ComputeColMap() {
   CHECK_FAIL_RETURN_UNEXPECTED(child_.size() == 1,
                                "Batch has " + std::to_string(child_.size()) + " child/children, expects only 1 child.");

mindspore/ccsrc/minddata/dataset/engine/datasetops/batch_op.h

@@ -1,5 +1,5 @@
 /**
- * Copyright 2019 Huawei Technologies Co., Ltd
+ * Copyright 2019-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -193,18 +193,6 @@ class BatchOp : public ParallelOp {
   // @return Status The status code returned
   Status operator()() override;
 
-  // Base-class override for NodePass visitor acceptor.
-  // @param p - Pointer to the NodePass to be accepted.
-  // @param modified - Whether this node visit modified the pipeline.
-  // @return - Status of the node visit.
-  Status Accept(NodePass *p, bool *const modified) override;
-
-  // Base-class override for NodePass visitor acceptor.
-  // @param p - Pointer to the NodePass to be accepted.
-  // @param modified - Whether this node visit modified the pipeline.
-  // @return - Status of the node visit.
-  Status PreAccept(NodePass *p, bool *const modified) override;
-
   // Op name getter
   // @return Name of the current Op
   std::string Name() const override { return kBatchOp; }

mindspore/ccsrc/minddata/dataset/engine/datasetops/bucket_batch_by_length_op.cc

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Huawei Technologies Co., Ltd
+ * Copyright 2020-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -15,22 +15,15 @@
  */
 #include "minddata/dataset/engine/datasetops/bucket_batch_by_length_op.h"
 
-#include <map>
 #include <memory>
 #include <string>
 #include <utility>
 #include <vector>
 
-#include "pybind11/numpy.h"
-#include "pybind11/pybind11.h"
-#include "pybind11/stl.h"
-#include "minddata/dataset/core/pybind_support.h"
 #include "minddata/dataset/core/config_manager.h"
-#include "minddata/dataset/core/tensor.h"
 #include "minddata/dataset/core/tensor_shape.h"
 #include "minddata/dataset/engine/dataset_iterator.h"
 #include "minddata/dataset/engine/datasetops/parallel_op.h"
-#include "minddata/dataset/engine/opt/pass.h"
 #include "minddata/dataset/util/status.h"
 
 namespace py = pybind11;

mindspore/ccsrc/minddata/dataset/engine/datasetops/build_sentence_piece_vocab_op.cc

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Huawei Technologies Co., Ltd
+ * Copyright 2020-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -18,7 +18,6 @@
 
 #include <iomanip>
 #include "minddata/dataset/core/config_manager.h"
-#include "minddata/dataset/engine/opt/pass.h"
 
 namespace mindspore {
 namespace dataset {
@@ -160,12 +159,6 @@ void BuildSentencePieceVocabOp::Next(std::string *sentence) {
   ret_status_ = Status::OK();
 }
 
-// Pre-Visitor accept method for NodePass
-Status BuildSentencePieceVocabOp::PreAccept(NodePass *p, bool *const modified) {
-  // Downcast shared pointer then call the pre-visitation
-  return p->PreRunOnNode(shared_from_base<BuildSentencePieceVocabOp>(), modified);
-}
-
 Status BuildSentencePieceVocabOp::Builder::Build(std::shared_ptr<BuildSentencePieceVocabOp> *op) {
   (*op) = std::make_shared<BuildSentencePieceVocabOp>(builder_vocab_, builder_col_names_, builder_vocab_size_,
                                                       builder_character_coverage_, builder_model_type_, builder_params_,

mindspore/ccsrc/minddata/dataset/engine/datasetops/build_sentence_piece_vocab_op.h

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Huawei Technologies Co., Ltd
+ * Copyright 2020-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -63,7 +63,7 @@ class BuildSentencePieceVocabOp : public PipelineOp {
     }
 
     // Setter method
-    // @param float charactor corverage - to determine the minimum symbols
+    // @param float character corverage - to determine the minimum symbols
     // @return Builder & reference to builder class object
     Builder &SetCharacterCoverage(float character_coverage) {
       builder_character_coverage_ = character_coverage;
@@ -166,11 +166,6 @@ class BuildSentencePieceVocabOp : public PipelineOp {
   bool Done();
   void Next(std::string *sentence);
 
-  /// \param[in] p The node to visit
-  /// \param[out] modified Indicator if the node was modified
-  /// \return Status of the node visit
-  Status PreAccept(NodePass *p, bool *const modified) override;
-
  private:
   bool read_done_;
   Status ret_status_;

mindspore/ccsrc/minddata/dataset/engine/datasetops/build_vocab_op.cc

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Huawei Technologies Co., Ltd
+ * Copyright 2020-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -23,7 +23,6 @@
 #include <unordered_map>
 #include <utility>
 #include "minddata/dataset/core/config_manager.h"
-#include "minddata/dataset/engine/opt/pass.h"
 
 namespace mindspore {
 namespace dataset {
@@ -230,11 +229,5 @@ void BuildVocabOp::Print(std::ostream &out, bool show_all) const {
         << "\n\n";
   }
 }
-
-// Pre-Visitor accept method for NodePass
-Status BuildVocabOp::PreAccept(NodePass *p, bool *const modified) {
-  // Downcast shared pointer then call the pre-visitation
-  return p->PreRunOnNode(shared_from_base<BuildVocabOp>(), modified);
-}
 }  // namespace dataset
 }  // namespace mindspore

mindspore/ccsrc/minddata/dataset/engine/datasetops/build_vocab_op.h

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Huawei Technologies Co., Ltd
+ * Copyright 2020-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -168,12 +168,6 @@ class BuildVocabOp : public ParallelOp {
 
   Status Reset() override { RETURN_STATUS_UNEXPECTED("Reset shouldn't be called in BuildVocabOp"); }
 
-  /// \brief Base-class override for NodePass pre-visit acceptor
-  /// \param[in] p The node to visit
-  /// \param[out] modified Indicator if the node was modified
-  /// \return Status of the node visit
-  Status PreAccept(NodePass *p, bool *const modified) override;
-
  private:
   const int32_t interval_;
   bool special_first_;

mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_lookup_op.cc

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Huawei Technologies Co., Ltd
+ * Copyright 2020-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,7 +14,6 @@
  * limitations under the License.
  */
 #include "minddata/dataset/engine/datasetops/cache_lookup_op.h"
-#include "minddata/dataset/engine/opt/pass.h"
 #include "minddata/dataset/core/config_manager.h"
 #include "minddata/dataset/core/constants.h"
 #include "minddata/dataset/core/global_context.h"
@@ -133,11 +132,5 @@ Status CacheLookupOp::ComputeColMap() {
   // So we will just return OK and let MergeOp (our parent) to handle it.
   return Status::OK();
 }
-
-// Visitor accept method for NodePass
-Status CacheLookupOp::Accept(NodePass *p, bool *const modified) {
-  // Downcast shared pointer then call visitor
-  return p->RunOnNode(shared_from_base<CacheLookupOp>(), modified);
-}
 }  // namespace dataset
 }  // namespace mindspore

mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_lookup_op.h

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Huawei Technologies Co., Ltd
+ * Copyright 2020-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -103,12 +103,6 @@ class CacheLookupOp : public CacheBase, public SamplerRT {
   bool AllowCacheMiss() override { return true; }
   std::string Name() const override { return kCacheLookupOp; }
 
-  /// \brief Base-class override for NodePass visitor acceptor
-  /// \param[in] p The node to visit
-  /// \param[out] modified Indicator if the node was modified
-  /// \return Status of the node visit
-  Status Accept(NodePass *p, bool *const modified) override;
-
  protected:
   Status ComputeColMap() override;
 

mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_merge_op.cc

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Huawei Technologies Co., Ltd
+ * Copyright 2020-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -23,7 +23,6 @@
 #include "minddata/dataset/core/config_manager.h"
 #include "minddata/dataset/core/constants.h"
 #include "minddata/dataset/core/global_context.h"
-#include "minddata/dataset/engine/opt/pass.h"
 #include "minddata/dataset/engine/execution_tree.h"
 #include "minddata/dataset/util/system_pool.h"
 #include "minddata/dataset/util/task_manager.h"
@@ -205,10 +204,10 @@ Status CacheMergeOp::Cleaner() {
   return Status::OK();
 }
 
-Status CacheMergeOp::PrepareNodePostAction() {  // Run any common code from super class first before adding our own
-                                                // specific logic
+Status CacheMergeOp::PrepareOperator() {  // Run any common code from super class first before adding our own
+                                          // specific logic
   CHECK_FAIL_RETURN_UNEXPECTED(child_.size() == 2, "Incorrect number of children");
-  RETURN_IF_NOT_OK(ParallelOp::PrepareNodePostAction());
+  RETURN_IF_NOT_OK(DatasetOp::PrepareOperator());
   // Get the computed check sum from all ops in the cache miss class
   uint32_t cache_crc = DatasetOp::GenerateCRC(child_[kCacheMissChildIdx]);
   // This is a mappable cache op so the id's need to be generated.
@@ -263,18 +262,6 @@ Status CacheMergeOp::Builder::Build(std::shared_ptr<CacheMergeOp> *ptr) {
   return Status::OK();
 }
 
-// Pre-Visitor accept method for NodePass
-Status CacheMergeOp::PreAccept(NodePass *p, bool *const modified) {
-  // Downcast shared pointer then call the pre-visitation
-  return p->PreRunOnNode(shared_from_base<CacheMergeOp>(), modified);
-}
-
-// Visitor accept method for NodePass
-Status CacheMergeOp::Accept(NodePass *p, bool *const modified) {
-  // Downcast shared pointer then call visitor
-  return p->RunOnNode(shared_from_base<CacheMergeOp>(), modified);
-}
-
 Status CacheMergeOp::EoeReceived(int32_t worker_id) {
   // Send the eoe up.
   MS_LOG(DEBUG) << "Cache merge sending eoe";

mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_merge_op.h

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Huawei Technologies Co., Ltd
+ * Copyright 2020-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -144,7 +144,7 @@ class CacheMergeOp : public ParallelOp {
   /// \param numWorkers Number of parallel workers as a derived class of ParallelOp
   /// \param opConnector Size Connector size as a derived class of ParallelOp
   /// \param numCleaners Number of cleaners to move cache miss rows into the cache server
-  /// \param cache_client CacheClient to commmunicate with the Cache server
+  /// \param cache_client CacheClient to communicate with the Cache server
   /// \param sampler as a derived class of ParallelOp
   CacheMergeOp(int32_t numWorkers, int32_t opConnectorSize, int32_t numCleaners,
                std::shared_ptr<CacheClient> cache_client, const std::shared_ptr<SamplerRT> &sampler);
@@ -157,31 +157,24 @@ class CacheMergeOp : public ParallelOp {
     return out;
   }
   /// \brief Master thread responsible to spawn all the necessary worker threads for the two streams and
-  /// the threads for the cleaners.
+  ///     the threads for the cleaners.
   /// \return
   Status operator()() override;
+
   /// \brief Entry function for worker thread that fetch rows from CacheLookupOp
   /// \param workerId
   /// \return Status object
   Status WorkerEntry(int32_t workerId) override;
-  Status PrepareNodePostAction() override;
+
+  /// \brief Perform specific post-operations on CacheOp
+  /// \return Status The status code returned
+  Status PrepareOperator() override;
+
   /// \brief Entry function for worker thread that fetch rows from the cache miss stream
   /// \param workerId
   /// \return Status object
   Status CacheMissWorkerEntry(int32_t workerId);
 
-  /// \brief Base-class override for NodePass pre-visit acceptor
-  /// \param[in] p The node to visit
-  /// \param[out] modified Indicator if the node was modified
-  /// \return Status of the node visit
-  Status PreAccept(NodePass *p, bool *const modified) override;
-
-  /// \brief Base-class override for NodePass visitor acceptor
-  /// \param[in] p The node to visit
-  /// \param[out] modified Indicator if the node was modified
-  /// \return Status of the node visit
-  Status Accept(NodePass *p, bool *const modified) override;
-
   /// \brief Base-class override for eoe handling
   /// \param worker_id
   /// \return Status object

mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_op.cc

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Huawei Technologies Co., Ltd
+ * Copyright 2020-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,14 +17,12 @@
 
 #include <memory>
 #include <utility>
-#include <vector>
 #include "minddata/dataset/core/config_manager.h"
 #include "minddata/dataset/core/constants.h"
 #include "minddata/dataset/core/global_context.h"
 #include "minddata/dataset/engine/datasetops/repeat_op.h"
 #include "minddata/dataset/engine/data_buffer.h"
 #include "minddata/dataset/engine/execution_tree.h"
-#include "minddata/dataset/engine/opt/pass.h"
 #include "minddata/dataset/util/log_adapter.h"
 #include "minddata/dataset/util/task_manager.h"
 
@@ -207,9 +205,6 @@ Status CacheOp::RegisterResources() {
   return Status::OK();
 }
 
-// Base-class override for setting specific CacheOp configurations. This code will be called
-// during the execution tree prepare phase BEFORE traversing down to child operators.
-uint32_t CacheOp::PrepareFlags() const { return ExecutionTree::kDePrepCache; }
 // Base-class override for special eoe handler.
 // CacheOp must override this because it shall not perform default handling of eoe. Instead
 // the CacheOp manages actions related to the end of the epoch.
@@ -225,21 +220,9 @@ Status CacheOp::EofReceived(int32_t worker_id) {
   return Status::OK();
 }
 
-// Pre-Visitor accept method for NodePass
-Status CacheOp::PreAccept(NodePass *p, bool *const modified) {
-  // Downcast shared pointer then call the pre-visitation
-  return p->PreRunOnNode(shared_from_base<CacheOp>(), modified);
-}
-
-// Visitor accept method for NodePass
-Status CacheOp::Accept(NodePass *p, bool *const modified) {
-  // Downcast shared pointer then call visitor
-  return p->RunOnNode(shared_from_base<CacheOp>(), modified);
-}
-
-Status CacheOp::PrepareNodePostAction() {
+Status CacheOp::PrepareOperator() {
   // Run any common code from super class first before adding our own
-  RETURN_IF_NOT_OK(ParallelOp::PrepareNodePostAction());
+  RETURN_IF_NOT_OK(DatasetOp::PrepareOperator());
   // Get the computed check sum from all ops in our cache path below us and ask the cache op to create it's cache
   uint32_t cache_crc = DatasetOp::GenerateCRC(shared_from_this());
   // This is a non-mappable cache op so the id's need to be generated.

mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_op.h

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Huawei Technologies Co., Ltd
+ * Copyright 2020-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -113,35 +113,35 @@ class CacheOp : public CacheBase, public RandomAccessOp {
   // Destructor
   ~CacheOp();
 
-  /// \brief Base-class override for setting specific CacheOp configurations. This code will be called
-  /// during the execution tree prepare phase BEFORE traversing down to child operators.
-  uint32_t PrepareFlags() const override;
   /// \brief Base-class override for special eoe handler.
-  /// CacheOp must override this because it shall not perform default handling of eoe. Instead
-  /// the CacheOp manages actions related to the end of the epoch.
+  /// \notes CacheOp must override this because it shall not perform default handling of eoe. Instead
+  ///     the CacheOp manages actions related to the end of the epoch.
   /// \return Status The status code returned
   Status EoeReceived(int32_t worker_id) override;
-  /// \brief Base-class override for NodePass pre-visit acceptor
-  /// \param[in] p The node to visit
-  /// \param[out] modified Indicator if the node was modified
-  /// \return Status of the node visit
-  Status PreAccept(NodePass *p, bool *const modified) override;
-  /// \brief Base-class override for NodePass visitor acceptor
-  /// \param[in] p The node to visit
-  /// \param[out] modified Indicator if the node was modified
-  /// \return Status of the node visit
-  Status Accept(NodePass *p, bool *const modified) override;
+
   /// \brief Base-class override for handling cases when an eof is received.
   /// \param worker_id - The worker id
   /// \return Status The status code returned
   Status EofReceived(int32_t worker_id) override;
+
+  // \brief Class functor operator ().
+  /// \return Status The status code returned
   Status operator()() override;
+
+  /// \brief Entry function for worker thread that fetch rows from CacheLookupOp
+  /// \param workerId
+  /// \return Status The status code returned
   Status WorkerEntry(int32_t worker_id) override;
-  /// \brief Base-class override for handling cases if we allow cache miss
+
+  /// \brief Base-class override for handling cases if we allow cache miss.
   bool AllowCacheMiss() override { return false; }
-  /// \brief Base-class override for the name of this operator
+
+  /// \brief Base-class override for the name of this operator.
   std::string Name() const override { return kCacheOp; }
-  Status PrepareNodePostAction() override;
+
+  /// \brief Perform specific post-operations on CacheOp
+  /// \return Status The status code returned
+  Status PrepareOperator() override;
 
  private:
   WaitPost rows_cache_done_;

mindspore/ccsrc/minddata/dataset/engine/datasetops/concat_op.cc

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Huawei Technologies Co., Ltd
+ * Copyright 2020-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -21,7 +21,6 @@
 #include "minddata/dataset/core/config_manager.h"
 #include "minddata/dataset/engine/data_buffer.h"
 #include "minddata/dataset/engine/db_connector.h"
-#include "minddata/dataset/engine/opt/pass.h"
 #include "utils/ms_utils.h"
 
 namespace mindspore {
@@ -70,6 +69,13 @@ void ConcatOp::Print(std::ostream &out, bool show_all) const {
   }
 }
 
+// This definition is added to pass the cyclomatic complexity rule of <= 20 units
+// The NOLINT directive is to disable cpplint check.
+// Clang format and cpplint give conflicting recommendations on this line below.
+#define f(fv, sv, shard_index)                                                                     \
+  (((fv) == -1 && (sv) == -1) || ((fv) < (sv) && (shard_index) >= (fv) && (shard_index) < (sv)) || \
+   ((fv) > (sv) && ((shard_index) >= (fv) || (shard_index) < (sv))))  // NOLINT
+
 // Main entry point for Concat
 Status ConcatOp::operator()() {
   children_num_ = static_cast<int32_t>(child_.size());
@@ -78,6 +84,7 @@ Status ConcatOp::operator()() {
   int eof_count = 0;
   int sample_number = 0;
   bool is_not_mappable = true;
+  bool is_not_mappable_or_second_ne_zero = true;
   int num_shard = 1;
   int shard_index = 0;
   std::shared_ptr<DistributedSamplerRT> distribute_sampler = std::dynamic_pointer_cast<DistributedSamplerRT>(sampler_);
@@ -99,26 +106,28 @@ Status ConcatOp::operator()() {
         RETURN_IF_NOT_OK(Verify(i, buf));
       }
       // 3. Put the data into output_connector
-      if (!children_flag_and_nums_.empty()) is_not_mappable = children_flag_and_nums_[i].first;
+      if (!children_flag_and_nums_.empty()) {
+        is_not_mappable = children_flag_and_nums_[i].first;
+        is_not_mappable_or_second_ne_zero = is_not_mappable || (!children_flag_and_nums_[i].second);
+      }
       while (!buf->eoe() && !buf->eof()) {
         // if dataset is not mappable or generator dataset which source is yield, cannot get the number of samples in
         // python layer), we use filtering to get data
-        if (sample_number % num_shard == shard_index && (is_not_mappable || !children_flag_and_nums_[i].second)) {
+        if (sample_number % num_shard == shard_index && is_not_mappable_or_second_ne_zero) {
           RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(buf)));
-        } else if (!is_not_mappable && children_flag_and_nums_[i].second) {  // if dataset is mappable or generator
-                                                                             // dataset which source is not yield
+        } else if (!is_not_mappable_or_second_ne_zero) {
+          // if dataset is mappable or generator dataset which source is not yield,
           // get the start and end subscripts of valid values
           int fv = children_start_end_index_[i].first, sv = children_start_end_index_[i].second;
 
           // determine whether the data allocated to the current shard id is false data
-          if ((fv == -1 && sv == -1) || (fv < sv && shard_index >= fv && shard_index < sv) ||
-              (fv > sv && (shard_index >= fv || shard_index < sv))) {
+          if (f(fv, sv, shard_index)) {
             RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(buf)));
           }
         }
 
-        // if dataSet is no mappable or generator dataset` which source is yeild, sample_number+=1
-        if (is_not_mappable || !children_flag_and_nums_[i].second) {
+        // if dataset is not mappable or generator dataset which source is yield, sample_number+=1
+        if (is_not_mappable_or_second_ne_zero) {
           sample_number++;
         }
 
@@ -127,7 +136,7 @@ Status ConcatOp::operator()() {
 
       // if dataset is mappable,We don't use filtering to pick data.
       // so sample_number plus the length of the entire dataset
-      if (!is_not_mappable && children_flag_and_nums_[i].second) {
+      if (!is_not_mappable_or_second_ne_zero) {
         sample_number += children_flag_and_nums_[i].second;
       }
     }
@@ -190,12 +199,6 @@ Status ConcatOp::ComputeColMap() {
   return Status::OK();
 }
 
-// Visitor pre-accept method for NodePass
-Status ConcatOp::PreAccept(NodePass *p, bool *const modified) {
-  // Downcast shared pointer then call visitor
-  return p->PreRunOnNode(shared_from_base<ConcatOp>(), modified);
-}
-
 // Gets the number of classes
 Status ConcatOp::GetNumClasses(int64_t *num_classes) {
   int64_t max_num_classes = -1;

mindspore/ccsrc/minddata/dataset/engine/datasetops/concat_op.h

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Huawei Technologies Co., Ltd
+ * Copyright 2020-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -105,12 +105,6 @@ class ConcatOp : public PipelineOp {
   // @return - Status
   Status ComputeColMap() override;
 
-  /// \brief Base-class override for NodePass pre-visit acceptor
-  /// \param[in] p The node to visit
-  /// \param[out] modified Indicator if the node was modified
-  /// \return Status of the node visit
-  Status PreAccept(NodePass *p, bool *const modified) override;
-
   /// \brief Gets the number of classes
   /// \param[out] num_classes the number of classes
   /// \return Status - The status code return

mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.cc

@@ -1,5 +1,5 @@
 /**
- * Copyright 2019 Huawei Technologies Co., Ltd
+ * Copyright 2019-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -26,10 +26,8 @@
 #include "minddata/dataset/engine/execution_tree.h"
 #include "minddata/dataset/engine/datasetops/device_queue_op.h"
 #include "minddata/dataset/engine/datasetops/source/sampler/sampler.h"
-#include "minddata/dataset/engine/datasetops/epoch_ctrl_op.h"
 #include "minddata/dataset/engine/data_buffer.h"
 #include "minddata/dataset/engine/db_connector.h"
-#include "minddata/dataset/engine/opt/pass.h"
 #ifndef ENABLE_ANDROID
 #include "utils/system/crc32c.h"
 #include "utils/log_adapter.h"
@@ -338,13 +336,8 @@ Status DatasetOp::EofReceived(int32_t worker_id) {
   return (out_connector_->Add(static_cast<int>(worker_id), std::move(eof_buffer)));
 }
 
-// During tree prepare phase, operators may have specific pre-operations to perform depending on
-// their role.
-Status DatasetOp::PrepareNodePreAction() { return Status::OK(); }
-
-// During tree prepare phase, operators may have specific post-operations to perform depending on
-// their role.
-Status DatasetOp::PrepareNodePostAction() {
+// During tree prepare phase, operators may have specific post-operations to perform depending on their role.
+Status DatasetOp::PrepareOperator() {
   // Creating Connector object for each op.
   // The consumer of the root node is assumed to be one thread.
   // If multiple threads are consuming from the root node, they will get the ordered data in round robin fashion.
@@ -364,9 +357,6 @@ Status DatasetOp::PrepareNodePostAction() {
   return Status::OK();
 }
 
-// Getter function.  Base class does not have any special flags setting.
-uint32_t DatasetOp::PrepareFlags() const { return ExecutionTree::kDePrepNone; }
-
 // Derived classes may implement the reset function if the operator is stateful and needs
 // specific reset handling that is not contained in this common code version of the reset.
 Status DatasetOp::Reset() {
@@ -402,18 +392,6 @@ Status DatasetOp::ComputeColMap() {
   return Status::OK();
 }
 
-Status DatasetOp::PreAccept(NodePass *p, bool *const modified) {
-  // DatasetOp is the base class of visitor target pre-visit.
-  // This method will only be called if its derived class does not implement one.
-  return p->PreRunOnNode(shared_from_this(), modified);
-}
-
-Status DatasetOp::Accept(NodePass *p, bool *const modified) {
-  // DatasetOp is the base class of visitor target.
-  // This method will only be called if its derived class does not implement one.
-  return p->RunOnNode(shared_from_this(), modified);
-}
-
 // Getter for the sampler, and it also removes the sampler from the op
 Status DatasetOp::FetchRemoveSampler(std::shared_ptr<SamplerRT> *sampler) {
   *sampler = sampler_;  // It's okay if it sampler_ points to nullptr

mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.h

@@ -78,19 +78,19 @@ class DatasetOp : public std::enable_shared_from_this<DatasetOp> {
   // Flags that control operator runtime behaviours
   enum OpState { kDeOpRunning = 0, kDeOpIdle = 1, kDeOpTerminated };
 
-  /// Constructor
+  /// \brief Constructor
   /// \param op_connector_size - The size for the output connector of this operator.
   /// \param sampler - The sampler for the op
   DatasetOp(int32_t op_connector_size, std::shared_ptr<SamplerRT> sampler);
 
-  /// Destructor
+  /// \brief Destructor
   virtual ~DatasetOp() { tree_ = nullptr; }
 
-  /// Adds a operator to become our child.
+  /// \brief Adds a operator to become our child.
   /// \param child - shared pointer to the child to add.
   Status AddChild(std::shared_ptr<DatasetOp> child);
 
-  /// Remove a operator from our children.
+  /// \brief Remove a operator from our children.
   /// \param child - shared pointer to the child to remove.
   Status RemoveChild(std::shared_ptr<DatasetOp> child);
 
@@ -114,9 +114,9 @@ class DatasetOp : public std::enable_shared_from_this<DatasetOp> {
   // Getter function to get all of our parents.
   std::vector<DatasetOp *> parents() const;
 
-  // Inserts a operator as the parent current op.
-  // Inserted op will become the sole parent of the current op.
-  // The existing parent of the current op will be transferred to the inserted op.
+  // \brief Inserts a operator as the parent current op.
+  // \notes Inserted op will become the sole parent of the current op.
+  //     The existing parent of the current op will be transferred to the inserted op.
   Status InsertAsParent(std::shared_ptr<DatasetOp> to_add);
 
   /// \brief Creates the connector within this operator
@@ -140,9 +140,9 @@ class DatasetOp : public std::enable_shared_from_this<DatasetOp> {
   }
 
   /// \brief Class functor operator ().
-  /// DatasetOps operate by launching a thread (see ExecutionTree).
-  /// This pure virtual version makes the requirement that derived classes must provide a functor
-  /// that will execute their main runtime loop code.
+  /// \notes DatasetOps operate by launching a thread (see ExecutionTree).
+  ///     This pure virtual version makes the requirement that derived classes must provide a functor
+  ///     that will execute their main runtime loop code.
   /// \return Status The status code returned
   virtual Status operator()() = 0;
 
@@ -170,8 +170,8 @@ class DatasetOp : public std::enable_shared_from_this<DatasetOp> {
   virtual Status GetNextBuffer(std::unique_ptr<DataBuffer> *p_buffer, int32_t worker_id, bool retry_if_eoe);
 
   /// \brief Gets the next buffer from the given child .  This function also has built-in eoe and eof
-  /// message handling so that child classes don't have to manually code pass-through logic when
-  /// those messages are received.
+  ///     message handling so that child classes don't have to manually code pass-through logic when
+  ///     those messages are received.
   /// \param p_buffer - The shared pointer for the fetched buffer to return (by reference)
   /// \param worker_id - The worker id
   /// \return Status The status code returned
@@ -194,44 +194,34 @@ class DatasetOp : public std::enable_shared_from_this<DatasetOp> {
   virtual Status GetClassIndexing(std::vector<std::pair<std::string, std::vector<int32_t>>> *output_class_indexing);
 
   /// \brief Performs handling for when an eoe message is received.
-  /// The base class implementation simply flows the eoe message to output. Derived classes
-  /// may override if they need to perform special eoe handling.
+  ///     The base class implementation simply flows the eoe message to output. Derived classes
+  ///     may override if they need to perform special eoe handling.
   /// \param worker_id - The worker id
   /// \return Status The status code returned
   virtual Status EoeReceived(int32_t worker_id);
 
   /// \brief Performs handling for when an eof message is received.
-  /// The base class implementation simply flows the eof message to output. Derived classes
-  /// may override if they need to perform special eof handling.
+  ///     The base class implementation simply flows the eof message to output. Derived classes
+  ///     may override if they need to perform special eof handling.
   /// \param worker_id - The worker id
   /// \return Status The status code returned
   virtual Status EofReceived(int32_t worker_id);
 
   /// \brief Derived classes may implement the reset function if the operator is stateful and needs
-  /// specific reset handling that is not contained in this common code version of the reset
+  ///     specific reset handling that is not contained in this common code version of the reset
   /// \return Status The status code returned
   virtual Status Reset();
 
-  /// \brief During tree prepare phase, operators may have specific pre-operations to perform depending on
-  /// their role.
-  /// \notes Derived versions of this function should always call it's superclass version first
-  /// before providing their own implementations.
-  virtual Status PrepareNodePreAction();
-
   /// \brief During tree prepare phase, operators may have specific post-operations to perform depending on
-  /// their role.
+  ///     their role.
   /// \notes Derived versions of this function should always call it's superclass version first
-  /// before providing their own implementations.
-  virtual Status PrepareNodePostAction();
+  ///     before providing their own implementations.
+  virtual Status PrepareOperator();
 
   /// \brief Getter function
   /// \return The operator id
   int32_t id() const { return operator_id_; }
 
-  /// \brief Getter function
-  /// \return The prepare flags
-  virtual uint32_t PrepareFlags() const;
-
   /// \brief Getter function
   /// \return The number of workers in this op
   virtual int32_t num_workers() const = 0;
@@ -315,22 +305,6 @@ class DatasetOp : public std::enable_shared_from_this<DatasetOp> {
   /// \return Vector of Children
   std::vector<std::shared_ptr<DatasetOp>> Children() const { return child_; }
 
-  /// \brief Base method for NodePass pre-visit.  A tree walk consists of walking down the tree and also walking back up
-  ///     in a depth-first order.  PreAccept is the node visit on the way down, whereas the regular Accept is the main
-  ///     visit on the way back up the tree during a post-order traversal. Subclass needs to override this if it
-  ///     requires special node visit access. Check "dataset/engine/opt/pass.h" for more details.
-  /// \param[in] p The node to visit
-  /// \param[out] modified Indicator if the node was modified
-  /// \return Status of the node visit
-  virtual Status PreAccept(NodePass *p, bool *const modified);
-
-  /// \brief Base method for NodePass visit. Subclass needs to override this if it requires special node visit access.
-  ///     Check "dataset/engine/opt/pass.h" for more details.
-  /// \param[in] p The node to visit
-  /// \param[out] modified Indicator if the node was modified
-  /// \return Status of the node visit
-  virtual Status Accept(NodePass *p, bool *const modified);
-
   /// Op name getter
   /// \return Name of the current Op
   virtual std::string Name() const = 0;
@@ -401,7 +375,7 @@ class DatasetOp : public std::enable_shared_from_this<DatasetOp> {
   void AddParent(DatasetOp *parent);
 
   /// Compute the current op's column map using its child's column map.
-  /// Get called during the tree post-prepare phase in PrepareNodePostAction.
+  /// Get called during the tree post-prepare phase in PrepareOperator.
   /// This base implementation just inherits the map from child 0, and can only be used if the number of children is 1.
   /// Operations changing the column map it inherits from the child must overwrite this function.
   /// \return - Status

mindspore/ccsrc/minddata/dataset/engine/datasetops/device_queue_op.cc

@@ -1,5 +1,5 @@
 /**
- * Copyright 2019 Huawei Technologies Co., Ltd
+ * Copyright 2019-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -16,17 +16,10 @@
 
 #include "minddata/dataset/engine/datasetops/device_queue_op.h"
 
-#include <iomanip>
 #include <iostream>
 #include <memory>
-#include <utility>
-#include "minddata/dataset/core/config_manager.h"
-#include "minddata/dataset/core/global_context.h"
 #include "minddata/dataset/engine/data_buffer.h"
 #include "minddata/dataset/engine/dataset_iterator.h"
-#include "minddata/dataset/engine/datasetops/epoch_ctrl_op.h"
-#include "minddata/dataset/engine/opt/pass.h"
-#include "minddata/dataset/engine/perf/profiling.h"
 #include "minddata/dataset/util/status.h"
 #include "minddata/dataset/util/task_manager.h"
 
@@ -490,12 +483,6 @@ void DeviceQueueOp::Print(std::ostream &out, bool show_all) const {
   }
 }
 
-// Visitor accept method for NodePass
-Status DeviceQueueOp::Accept(NodePass *p, bool *const modified) {
-  // Downcast shared pointer then call visitor
-  return p->RunOnNode(shared_from_base<DeviceQueueOp>(), modified);
-}
-
 void DeviceQueueOp::ProfilingRecorder(bool isProfilingEnable, std::shared_ptr<DeviceQueueTracing> profiling_node,
                                       int64_t send_batch, int32_t tdt_cost, uint64_t *batch_start_time,
                                       uint64_t *end_time, int32_t connector_capacity, int32_t connector_size) {

mindspore/ccsrc/minddata/dataset/engine/datasetops/device_queue_op.h

@@ -1,5 +1,5 @@
 /**
- * Copyright 2019 Huawei Technologies Co., Ltd
+ * Copyright 2019-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -168,12 +168,6 @@ class DeviceQueueOp : public PipelineOp {
 
   Status operator()() override;
 
-  // Base-class override for NodePass visitor acceptor.
-  // @param p - Pointer to the NodePass to be accepted.
-  // @param modified - Whether this node visit modified the pipeline.
-  // @return - Status of the node visit.
-  Status Accept(NodePass *p, bool *const modified) override;
-
   // Record the pipeline profiling info
   void ProfilingRecorder(bool isProfilingEnable, std::shared_ptr<DeviceQueueTracing> profiling_node, int64_t send_batch,
                          int32_t tdt_cost, uint64_t *batch_start_time, uint64_t *end_time, int32_t connector_capacity,

mindspore/ccsrc/minddata/dataset/engine/datasetops/epoch_ctrl_op.cc

@@ -19,7 +19,6 @@
 
 #include "minddata/dataset/engine/datasetops/epoch_ctrl_op.h"
 #include "minddata/dataset/engine/data_buffer.h"
-#include "minddata/dataset/engine/opt/pass.h"
 #include "minddata/dataset/util/log_adapter.h"
 
 namespace mindspore {
@@ -102,17 +101,6 @@ Status EpochCtrlOp::EoeReceived(int32_t worker_id) {
   return Status::OK();
 }
 
-// Pre-Visitor accept method for NodePass
-Status EpochCtrlOp::PreAccept(NodePass *p, bool *const modified) {
-  // Downcast shared pointer then call the pre-visitation
-  return p->PreRunOnNode(shared_from_base<EpochCtrlOp>(), modified);
-}
-
-// Visitor accept method for NodePass
-Status EpochCtrlOp::Accept(NodePass *p, bool *const modified) {
-  // Downcast shared pointer then call the pre-visitation
-  return p->RunOnNode(shared_from_base<EpochCtrlOp>(), modified);
-}
 int64_t EpochCtrlOp::GetTreeRepeatCount() { return child_[0]->GetTreeRepeatCount(); }
 }  // namespace dataset
 }  // namespace mindspore

mindspore/ccsrc/minddata/dataset/engine/datasetops/epoch_ctrl_op.h

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Huawei Technologies Co., Ltd
+ * Copyright 2020-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -42,7 +42,7 @@ class EpochCtrlOp : public RepeatOp {
     Status Build(std::shared_ptr<EpochCtrlOp> *);
   };
 
-  // Contructor
+  // Constructor
   explicit EpochCtrlOp(int32_t num_epoch);
 
   // Destructor
@@ -65,18 +65,6 @@ class EpochCtrlOp : public RepeatOp {
   // @param worker_id - The worker id
   Status EoeReceived(int32_t worker_id) override;
 
-  /// \brief Base-class override for NodePass pre-visit acceptor
-  /// \param[in] p The node to visit
-  /// \param[out] modified Indicator if the node was modified
-  /// \return Status of the node visit
-  Status PreAccept(NodePass *p, bool *const modified) override;
-
-  /// \brief Base-class override for NodePass visitor acceptor
-  /// \param[in] p The node to visit
-  /// \param[out] modified Indicator if the node was modified
-  /// \return Status of the node visit
-  Status Accept(NodePass *p, bool *const modified) override;
-
   int64_t GetTreeRepeatCount() override;
 };
 }  // namespace dataset

mindspore/ccsrc/minddata/dataset/engine/datasetops/filter_op.cc

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Huawei Technologies Co., Ltd
+ * Copyright 2020-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -16,18 +16,14 @@
 #include "minddata/dataset/engine/datasetops/filter_op.h"
 #include <algorithm>
 #include <cstring>
-#include <iomanip>
 #include <iostream>
 #include <memory>
 #include <vector>
 #include "minddata/dataset/core/config_manager.h"
-#include "minddata/dataset/core/constants.h"
 #include "minddata/dataset/core/global_context.h"
 #include "minddata/dataset/core/tensor.h"
 #include "minddata/dataset/engine/data_buffer.h"
-#include "minddata/dataset/engine/db_connector.h"
 #include "minddata/dataset/engine/execution_tree.h"
-#include "minddata/dataset/engine/opt/pass.h"
 #include "minddata/dataset/kernels/tensor_op.h"
 #include "minddata/dataset/util/log_adapter.h"
 #include "minddata/dataset/util/task_manager.h"
@@ -247,17 +243,5 @@ Status FilterOp::InvokePredicateFunc(const TensorRow &input, bool *out_predicate
   return Status(StatusCode::kSuccess, "FilterOp predicate func call succeed");
 }
 
-// Visitor accept method for NodePass
-Status FilterOp::Accept(NodePass *p, bool *const modified) {
-  // Downcast shared pointer then call visitor
-  return p->RunOnNode(shared_from_base<FilterOp>(), modified);
-}
-
-// Visitor pre-accept method for NodePass
-Status FilterOp::PreAccept(NodePass *p, bool *const modified) {
-  // Downcast shared pointer then call visitor
-  return p->PreRunOnNode(shared_from_base<FilterOp>(), modified);
-}
-
 }  // namespace dataset
 }  // namespace mindspore

mindspore/ccsrc/minddata/dataset/engine/datasetops/filter_op.h

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Huawei Technologies Co., Ltd
+ * Copyright 2020-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -121,18 +121,6 @@ class FilterOp : public ParallelOp {
   // @param show_all A bool to control if you want to show all info or just a summary.
   void Print(std::ostream &out, bool show_all) const override;
 
-  /// \brief Base-class override for NodePass pre-visit acceptor
-  /// \param[in] p The node to visit
-  /// \param[out] modified Indicator if the node was modified
-  /// \return Status of the node visit
-  Status PreAccept(NodePass *p, bool *const modified) override;
-
-  // Base-class override for NodePass visitor acceptor.
-  // @param p - Pointer to the NodePass to be accepted.
-  // @param modified - Whether this node visit modified the pipeline.
-  // @return - Status of the node visit.
-  Status Accept(NodePass *p, bool *const modified) override;
-
   // Op name getter
   // @return Name of the current Op
   std::string Name() const override { return kFilterOp; }

mindspore/ccsrc/minddata/dataset/engine/datasetops/map_op/map_op.cc

@@ -1,5 +1,5 @@
 /**
- * Copyright 2019 Huawei Technologies Co., Ltd
+ * Copyright 2019-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -26,7 +26,7 @@
 #include "minddata/dataset/engine/data_buffer.h"
 #include "minddata/dataset/engine/datasetops/map_op/cpu_map_job.h"
 #include "minddata/dataset/engine/datasetops/map_op/gpu_map_job.h"
-#include "minddata/dataset/engine/opt/pass.h"
+#include "minddata/dataset/engine/execution_tree.h"
 #include "minddata/dataset/kernels/tensor_op.h"
 #include "minddata/dataset/util/log_adapter.h"
 #include "minddata/dataset/util/task_manager.h"
@@ -445,18 +445,6 @@ void MapOp::CreateFinalColMap(std::unordered_map<std::string, int32_t> *col_name
   }
 }
 
-// Visitor accept method for NodePass
-Status MapOp::Accept(NodePass *p, bool *const modified) {
-  // Downcast shared pointer then call visitor
-  return p->RunOnNode(shared_from_base<MapOp>(), modified);
-}
-
-// Visitor pre-accept method for NodePass
-Status MapOp::PreAccept(NodePass *p, bool *const modified) {
-  // Downcast shared pointer then call visitor
-  return p->PreRunOnNode(shared_from_base<MapOp>(), modified);
-}
-
 Status MapOp::WaitForWorkers() {
   // reset num_paused workers to 0
   num_workers_paused_ = 0;

mindspore/ccsrc/minddata/dataset/engine/datasetops/map_op/map_op.h

@@ -1,5 +1,5 @@
 /**
- * Copyright 2019 Huawei Technologies Co., Ltd
+ * Copyright 2019-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -177,18 +177,6 @@ class MapOp : public ParallelOp {
   // @return the number of threads consuming data from previous op's output Connector.
   int32_t num_consumers() const override;
 
-  /// \brief Base-class override for NodePass pre-visit acceptor
-  /// \param[in] p The node to visit
-  /// \param[out] modified Indicator if the node was modified
-  /// \return Status of the node visit
-  Status PreAccept(NodePass *p, bool *const modified) override;
-
-  /// \brief Base-class override for NodePass visitor acceptor.
-  /// \param[in] p Pointer to the NodePass to be accepted.
-  /// \param[out] modified Whether this node visit modified the pipeline.
-  /// \return - Status of the node visit.
-  Status Accept(NodePass *p, bool *const modified) override;
-
   // Op name getter
   // @return Name of the current Op
   std::string Name() const override { return kMapOp; }

mindspore/ccsrc/minddata/dataset/engine/datasetops/parallel_op.h

@@ -1,5 +1,5 @@
 /**
- * Copyright 2019 Huawei Technologies Co., Ltd
+ * Copyright 2019-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -68,26 +68,6 @@ class ParallelOp : public DatasetOp {
     return out;
   }
 
-  // During tree prepare phase, operators may have specific pre-operations to perform depending on
-  // their role.
-  // @notes Derived versions of this function should always call it's superclass version first
-  // before providing their own implementations.
-  // @return Status - The error return code
-  Status PrepareNodePreAction() override {
-    // Run common code from super class before adding ParallelOp specific logic
-    return (DatasetOp::PrepareNodePreAction());
-  }
-
-  // During tree prepare phase, operators may have specific post-operations to perform depending on
-  // their role.
-  // @notes Derived versions of this function should always call it's superclass version first
-  // before providing their own implementations.
-  // @return Status - The error return code
-  Status PrepareNodePostAction() override {
-    // Run common code from super class before adding ParallelOp specific logic
-    return (DatasetOp::PrepareNodePostAction());
-  }
-
   // Override base class reset to provide reset actions specific to the ParallelOp class.
   // @return Status The status code returned
   Status Reset() override;

mindspore/ccsrc/minddata/dataset/engine/datasetops/pipeline_op.h

@@ -1,5 +1,5 @@
 /**
- * Copyright 2019 Huawei Technologies Co., Ltd
+ * Copyright 2019-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -67,24 +67,6 @@ class PipelineOp : public DatasetOp {
   // @return The number of threads that push data to the output connector
   int32_t num_producers() const override { return 1; }
 
-  // During tree prepare phase, operators may have specific pre-operations to perform depending on
-  // their role.
-  // @notes Derived versions of this function should always call it's superclass version first
-  // before providing their own implementations.
-  Status PrepareNodePreAction() override {
-    // Run common code from super class before adding PipelineOp specific logic
-    return (DatasetOp::PrepareNodePreAction());
-  }
-
-  // During tree prepare phase, operators may have specific post-operations to perform depending on
-  // their role.
-  // @notes Derived versions of this function should always call it's superclass version first
-  // before providing their own implementations.
-  Status PrepareNodePostAction() override {
-    // Run common code from super class before adding PipelineOp specific logic
-    return (DatasetOp::PrepareNodePostAction());
-  }
-
  protected:
   // *******************************************************************************
   // I'm predicting there will be common arguments or functionality for pipeline ops,

mindspore/ccsrc/minddata/dataset/engine/datasetops/project_op.cc

@@ -1,5 +1,5 @@
 /**
- * Copyright 2019 Huawei Technologies Co., Ltd
+ * Copyright 2019-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -23,9 +23,7 @@
 #include <utility>
 #include <vector>
 #include "minddata/dataset/engine/data_buffer.h"
-#include "minddata/dataset/engine/db_connector.h"
 #include "minddata/dataset/engine/execution_tree.h"
-#include "minddata/dataset/engine/opt/pass.h"
 #include "minddata/dataset/util/log_adapter.h"
 
 namespace mindspore {
@@ -132,12 +130,6 @@ Status ProjectOp::EoeReceived(int32_t worker_id) {
 
 Status ProjectOp::EofReceived(int32_t worker_id) { return Status::OK(); }
 
-// Visitor accept method for NodePass
-Status ProjectOp::Accept(NodePass *p, bool *const modified) {
-  // Downcast shared pointer then call visitor
-  return p->RunOnNode(shared_from_base<ProjectOp>(), modified);
-}
-
 // Compute the column map and save it into our own column name map
 // We cannot use the super class ComputeColMap here because we're making a modification of the
 // map from the child map.

mindspore/ccsrc/minddata/dataset/engine/datasetops/project_op.h

@@ -1,5 +1,5 @@
 /**
- * Copyright 2019 Huawei Technologies Co., Ltd
+ * Copyright 2019-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -101,12 +101,6 @@ class ProjectOp : public PipelineOp {
   // @return Status The status code returned
   Status EofReceived(int32_t worker_id) override;
 
-  // Base-class override for NodePass visitor acceptor.
-  // @param p - Pointer to the NodePass to be accepted.
-  // @param modified - Whether this node visit modified the pipeline.
-  // @return - Status of the node visit.
-  Status Accept(NodePass *p, bool *const modified) override;
-
   // Op name getter
   // @return Name of the current Op
   std::string Name() const override { return kProjectOp; }

mindspore/ccsrc/minddata/dataset/engine/datasetops/rename_op.cc

@@ -1,5 +1,5 @@
 /**
- * Copyright 2019 Huawei Technologies Co., Ltd
+ * Copyright 2019-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -24,7 +24,6 @@
 #include "minddata/dataset/core/global_context.h"
 #include "minddata/dataset/engine/data_buffer.h"
 #include "minddata/dataset/engine/db_connector.h"
-#include "minddata/dataset/engine/opt/pass.h"
 #include "minddata/dataset/util/log_adapter.h"
 
 namespace mindspore {
@@ -171,10 +170,5 @@ Status RenameOp::EoeReceived(int32_t) {
   return Status::OK();
 }
 
-// Visitor accept method for NodePass
-Status RenameOp::Accept(NodePass *p, bool *const modified) {
-  // Downcast shared pointer then call visitor
-  return p->RunOnNode(shared_from_base<RenameOp>(), modified);
-}
 }  // namespace dataset
 }  // namespace mindspore

mindspore/ccsrc/minddata/dataset/engine/datasetops/rename_op.h

@@ -1,5 +1,5 @@
 /**
- * Copyright 2019 Huawei Technologies Co., Ltd
+ * Copyright 2019-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -110,12 +110,6 @@ class RenameOp : public PipelineOp {
   // @return Status The status code returned
   Status operator()() override;
 
-  // Base-class override for NodePass visitor acceptor.
-  // @param p - Pointer to the NodePass to be accepted.
-  // @param modified - Whether this node visit modified the pipeline.
-  // @return - Status of the node visit.
-  Status Accept(NodePass *p, bool *const modified) override;
-
   // Op name getter
   // @return Name of the current Op
   std::string Name() const override { return kRenameOp; }

mindspore/ccsrc/minddata/dataset/engine/datasetops/repeat_op.cc

@@ -1,5 +1,5 @@
 /**
- * Copyright 2019 Huawei Technologies Co., Ltd
+ * Copyright 2019-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -20,8 +20,6 @@
 #include "minddata/dataset/engine/execution_tree.h"
 #include "minddata/dataset/engine/datasetops/repeat_op.h"
 #include "minddata/dataset/engine/data_buffer.h"
-#include "minddata/dataset/engine/db_connector.h"
-#include "minddata/dataset/engine/opt/pass.h"
 #include "minddata/dataset/util/log_adapter.h"
 
 namespace mindspore {
@@ -175,18 +173,6 @@ int32_t RepeatOp::num_producers() const {
   }
 }
 
-// Pre-Visitor accept method for NodePass
-Status RepeatOp::PreAccept(NodePass *p, bool *const modified) {
-  // Downcast shared pointer then call the pre-visitation
-  return p->PreRunOnNode(shared_from_base<RepeatOp>(), modified);
-}
-
-// Visitor accept method for NodePass
-Status RepeatOp::Accept(NodePass *p, bool *const modified) {
-  // Downcast shared pointer then call visitor
-  return p->RunOnNode(shared_from_base<RepeatOp>(), modified);
-}
-
 int64_t RepeatOp::GetTreeRepeatCount() { return num_repeats_; }
 }  // namespace dataset
 }  // namespace mindspore

mindspore/ccsrc/minddata/dataset/engine/datasetops/repeat_op.h

@@ -1,5 +1,5 @@
 /**
- * Copyright 2019 Huawei Technologies Co., Ltd
+ * Copyright 2019-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -109,18 +109,6 @@ class RepeatOp : public PipelineOp {
   // @param workerId - The worker id
   int32_t num_producers() const override;
 
-  /// \brief Base-class override for NodePass pre-visit acceptor
-  /// \param[in] p The node to visit
-  /// \param[out] modified Indicator if the node was modified
-  /// \return Status of the node visit
-  Status PreAccept(NodePass *p, bool *const modified) override;
-
-  /// \brief Base-class override for NodePass visitor acceptor
-  /// \param[in] p The node to visit
-  /// \param[out] modified Indicator if the node was modified
-  /// \return Status of the node visit
-  Status Accept(NodePass *p, bool *const modified) override;
-
   // Op name getter
   // @return Name of the current Op
   std::string Name() const override { return kRepeatOp; }

mindspore/ccsrc/minddata/dataset/engine/datasetops/shuffle_op.cc

@@ -1,5 +1,5 @@
 /**
- * Copyright 2019 Huawei Technologies Co., Ltd
+ * Copyright 2019-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -16,12 +16,9 @@
 #if defined(_WIN32) || defined(_WIN64)
 #include <stdlib.h>
 #endif
-#include <securec.h>
-#include <algorithm>
 #include <chrono>
 #include <iomanip>
 #include <iostream>
-#include <limits>
 #include <random>
 #include <utility>
 
@@ -30,7 +27,6 @@
 #include "minddata/dataset/engine/dataset_iterator.h"
 #include "minddata/dataset/engine/data_buffer.h"
 #include "minddata/dataset/engine/db_connector.h"
-#include "minddata/dataset/engine/opt/pass.h"
 #include "minddata/dataset/util/log_adapter.h"
 #include "minddata/dataset/util/random.h"
 #include "minddata/dataset/util/status.h"
@@ -291,11 +287,5 @@ Status ShuffleOp::EoeReceived(int32_t worker_id) {
   state_ = OpState::kDeOpIdle;
   return Status::OK();
 }
-
-// Visitor accept method for NodePass
-Status ShuffleOp::Accept(NodePass *p, bool *const modified) {
-  // Downcast shared pointer then call visitor
-  return p->RunOnNode(shared_from_base<ShuffleOp>(), modified);
-}
 }  // namespace dataset
 }  // namespace mindspore

mindspore/ccsrc/minddata/dataset/engine/datasetops/shuffle_op.h

@@ -1,5 +1,5 @@
 /**
- * Copyright 2019 Huawei Technologies Co., Ltd
+ * Copyright 2019-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -155,12 +155,6 @@ class ShuffleOp : public PipelineOp {
   // @return Status The status code returned
   Status EoeReceived(int32_t worker_id) override;
 
-  // Base-class override for NodePass visitor acceptor.
-  // @param p - Pointer to the NodePass to be accepted.
-  // @param modified - Whether this node visit modified the pipeline.
-  // @return - Status of the node visit.
-  Status Accept(NodePass *p, bool *const modified) override;
-
   // Op name getter
   // @return Name of the current Op
   std::string Name() const override { return kShuffleOp; }

mindspore/ccsrc/minddata/dataset/engine/datasetops/skip_op.cc

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Huawei Technologies Co., Ltd
+ * Copyright 2020-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -22,7 +22,6 @@
 #include "minddata/dataset/engine/datasetops/skip_op.h"
 #include "minddata/dataset/engine/db_connector.h"
 #include "minddata/dataset/engine/execution_tree.h"
-#include "minddata/dataset/engine/opt/pass.h"
 #include "minddata/dataset/util/log_adapter.h"
 
 namespace mindspore {
@@ -124,17 +123,5 @@ Status SkipOp::EofReceived(int32_t worker_id) {
   return Status::OK();
 }
 
-// Visitor accept method for NodePass
-Status SkipOp::Accept(NodePass *p, bool *const modified) {
-  // Downcast shared pointer then call visitor
-  return p->RunOnNode(shared_from_base<SkipOp>(), modified);
-}
-
-// Visitor pre-accept method for NodePass
-Status SkipOp::PreAccept(NodePass *p, bool *const modified) {
-  // Downcast shared pointer then call visitor
-  return p->PreRunOnNode(shared_from_base<SkipOp>(), modified);
-}
-
 }  // namespace dataset
 }  // namespace mindspore

mindspore/ccsrc/minddata/dataset/engine/datasetops/skip_op.h

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Huawei Technologies Co., Ltd
+ * Copyright 2020-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -74,18 +74,6 @@ class SkipOp : public PipelineOp {
   // @param worker_id - The worker id
   Status EofReceived(int32_t worker_id) override;
 
-  // Base-class override for NodePass visitor acceptor.
-  // @param p - Pointer to the NodePass to be accepted.
-  // @param modified - Whether this node visit modified the pipeline.
-  // @return - Status of the node visit.
-  Status Accept(NodePass *p, bool *const modified) override;
-
-  /// \brief Base-class override for NodePass pre-visit acceptor
-  /// \param[in] p The node to visit
-  /// \param[out] modified Indicator if the node was modified
-  /// \return Status of the node visit
-  Status PreAccept(NodePass *p, bool *const modified) override;
-
   // Op name getter
   // @return Name of the current Op
   std::string Name() const override { return kSkipOp; }

mindspore/ccsrc/minddata/dataset/engine/datasetops/source/album_op.cc

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Huawei Technologies Co., Ltd
+ * Copyright 2020-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -21,7 +21,6 @@
 #include "minddata/dataset/engine/datasetops/source/sampler/sequential_sampler.h"
 #include "minddata/dataset/engine/db_connector.h"
 #include "minddata/dataset/engine/execution_tree.h"
-#include "minddata/dataset/engine/opt/pass.h"
 #ifndef ENABLE_ANDROID
 #include "minddata/dataset/kernels/image/image_utils.h"
 #else
@@ -590,12 +589,6 @@ Status AlbumOp::LaunchThreadsAndInitOp() {
   return Status::OK();
 }
 
-// Visitor accept method for NodePass
-Status AlbumOp::Accept(NodePass *p, bool *const modified) {
-  // Downcast shared pointer then call visitor
-  return p->RunOnNode(shared_from_base<AlbumOp>(), modified);
-}
-
 Status AlbumOp::ComputeColMap() {
   // Set the column name map (base class field)
   if (column_name_id_map_.empty()) {

mindspore/ccsrc/minddata/dataset/engine/datasetops/source/album_op.h

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Huawei Technologies Co., Ltd
+ * Copyright 2020-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -192,12 +192,6 @@ class AlbumOp : public ParallelOp, public RandomAccessOp {
   /// \return bool - if file is bad then return false
   bool CheckImageType(const std::string &file_name, bool *valid);
 
-  // Base-class override for NodePass visitor acceptor.
-  // @param p - Pointer to the NodePass to be accepted.
-  // @param modified - Whether this node visit modified the pipeline.
-  // @return - Status of the node visit.
-  Status Accept(NodePass *p, bool *const modified) override;
-
   // Op name getter
   // @return Name of the current Op
   std::string Name() const override { return "AlbumOp"; }

mindspore/ccsrc/minddata/dataset/engine/datasetops/source/celeba_op.cc

@@ -1,6 +1,6 @@
 /**
- * Copyright 2019 Huawei Technologies Co., Ltd
-
+ * Copyright 2019-2021 Huawei Technologies Co., Ltd
+ *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
@@ -23,7 +23,6 @@
 #include "minddata/dataset/engine/datasetops/source/sampler/sequential_sampler.h"
 #include "minddata/dataset/engine/data_schema.h"
 #include "minddata/dataset/engine/execution_tree.h"
-#include "minddata/dataset/engine/opt/pass.h"
 #ifndef ENABLE_ANDROID
 #include "minddata/dataset/kernels/image/image_utils.h"
 #else
@@ -440,12 +439,6 @@ Status CelebAOp::Reset() {
   return Status::OK();
 }
 
-// Visitor accept method for NodePass
-Status CelebAOp::Accept(NodePass *p, bool *const modified) {
-  // Downcast shared pointer then call visitor
-  return p->RunOnNode(shared_from_base<CelebAOp>(), modified);
-}
-
 Status CelebAOp::ComputeColMap() {
   // Set the column name map (base class field)
   if (column_name_id_map_.empty()) {

mindspore/ccsrc/minddata/dataset/engine/datasetops/source/celeba_op.h

@@ -1,6 +1,6 @@
 /**
- * Copyright 2019 Huawei Technologies Co., Ltd
-
+ * Copyright 2019-2021 Huawei Technologies Co., Ltd
+ *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
@@ -169,12 +169,6 @@ class CelebAOp : public ParallelOp, RandomAccessOp {
   // @return Status The status code returned
   Status AddIOBlock(std::unique_ptr<DataBuffer> *data_buffer);
 
-  /// \brief Base-class override for NodePass visitor acceptor
-  /// \param[in] p Pointer to the NodePass to be accepted
-  /// \param[out] modified Indicator if the node was changed at all
-  /// \return Status of the node visit
-  Status Accept(NodePass *p, bool *const modified) override;
-
   // Op name getter
   // @return Name of the current Op
   std::string Name() const override { return "CelebAOp"; }

mindspore/ccsrc/minddata/dataset/engine/datasetops/source/cifar_op.cc

@@ -1,5 +1,5 @@
 /**
- * Copyright 2019 Huawei Technologies Co., Ltd
+ * Copyright 2019-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -26,7 +26,6 @@
 #include "minddata/dataset/engine/datasetops/source/sampler/sequential_sampler.h"
 #include "minddata/dataset/engine/db_connector.h"
 #include "minddata/dataset/engine/execution_tree.h"
-#include "minddata/dataset/engine/opt/pass.h"
 #include "utils/ms_utils.h"
 
 namespace mindspore {
@@ -500,12 +499,6 @@ Status CifarOp::CountTotalRows(const std::string &dir, const std::string &usage,
   }
 }
 
-// Visitor accept method for NodePass
-Status CifarOp::Accept(NodePass *p, bool *const modified) {
-  // Downcast shared pointer then call visitor
-  return p->RunOnNode(shared_from_base<CifarOp>(), modified);
-}
-
 Status CifarOp::ComputeColMap() {
   // set the column name map (base class field)
   if (column_name_id_map_.empty()) {

mindspore/ccsrc/minddata/dataset/engine/datasetops/source/cifar_op.h

@@ -1,5 +1,5 @@
 /**
- * Copyright 2019 Huawei Technologies Co., Ltd
+ * Copyright 2019-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -165,12 +165,6 @@ class CifarOp : public ParallelOp, public RandomAccessOp {
   // @return
   static Status CountTotalRows(const std::string &dir, const std::string &usage, bool isCIFAR10, int64_t *count);
 
-  /// \brief Base-class override for NodePass visitor acceptor
-  /// \param[in] p Pointer to the NodePass to be accepted
-  /// \param[out] modified Indicator if the node was changed at all
-  /// \return Status of the node visit
-  Status Accept(NodePass *p, bool *const modified) override;
-
   // Op name getter
   // @return Name of the current Op
   std::string Name() const override { return "CifarOp"; }

mindspore/ccsrc/minddata/dataset/engine/datasetops/source/clue_op.cc

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Huawei Technologies Co., Ltd
+ * Copyright 2020-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -26,7 +26,6 @@
 #include "minddata/dataset/util/task_manager.h"
 #include "minddata/dataset/engine/jagged_connector.h"
 #include "minddata/dataset/engine/execution_tree.h"
-#include "minddata/dataset/engine/opt/pass.h"
 #include "minddata/dataset/engine/datasetops/source/io_block.h"
 #include "minddata/dataset/util/random.h"
 
@@ -549,11 +548,5 @@ Status ClueOp::ComputeColMap() {
   return Status::OK();
 }
 
-// Visitor accept method for NodePass
-Status ClueOp::Accept(NodePass *p, bool *const modified) {
-  // Downcast shared pointer then call visitor
-  return p->RunOnNode(shared_from_base<ClueOp>(), modified);
-}
-
 }  // namespace dataset
 }  // namespace mindspore

mindspore/ccsrc/minddata/dataset/engine/datasetops/source/clue_op.h

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Huawei Technologies Co., Ltd
+ * Copyright 2020-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -177,12 +177,6 @@ class ClueOp : public ParallelOp {
   // @return Name of the current Op
   std::string Name() const override { return "ClueOp"; }
 
-  // Base-class override for NodePass visitor acceptor.
-  // @param p - Pointer to the NodePass to be accepted.
-  // @param modified - Whether this node visit modified the pipeline.
-  // @return - Status of the node visit.
-  Status Accept(NodePass *p, bool *const modified) override;
-
  private:
   // The entry point for when workers are launched.
   // @param worker_id - the id of the worker that is executing this function.

mindspore/ccsrc/minddata/dataset/engine/datasetops/source/coco_op.cc

@@ -1,5 +1,5 @@
 /**
- * Copyright 2019 Huawei Technologies Co., Ltd
+ * Copyright 2019-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -22,7 +22,7 @@
 #include "minddata/dataset/core/tensor_shape.h"
 #include "minddata/dataset/engine/datasetops/source/sampler/sequential_sampler.h"
 #include "minddata/dataset/engine/db_connector.h"
-#include "minddata/dataset/engine/opt/pass.h"
+#include "minddata/dataset/engine/execution_tree.h"
 
 namespace mindspore {
 namespace dataset {
@@ -674,12 +674,6 @@ Status CocoOp::GetClassIndexing(const std::string &dir, const std::string &file,
   return Status::OK();
 }
 
-// Visitor accept method for NodePass
-Status CocoOp::Accept(NodePass *p, bool *const modified) {
-  // Downcast shared pointer then call visitor
-  return p->RunOnNode(shared_from_base<CocoOp>(), modified);
-}
-
 Status CocoOp::ComputeColMap() {
   // Set the column name map (base class field)
   if (column_name_id_map_.empty()) {

mindspore/ccsrc/minddata/dataset/engine/datasetops/source/coco_op.h

@@ -1,5 +1,5 @@
 /**
- * Copyright 2019 Huawei Technologies Co., Ltd
+ * Copyright 2019-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -203,12 +203,6 @@ class CocoOp : public ParallelOp, public RandomAccessOp {
   static Status GetClassIndexing(const std::string &dir, const std::string &task_type, const std::string &task_mode,
                                  std::vector<std::pair<std::string, std::vector<int32_t>>> *output_class_indexing);
 
-  /// \brief Base-class override for NodePass visitor acceptor
-  /// \param[in] p Pointer to the NodePass to be accepted
-  /// \param[out] modified Indicator if the node was changed at all
-  /// \return Status of the node visit
-  Status Accept(NodePass *p, bool *const modified) override;
-
   // Op name getter
   // @return Name of the current Op
   std::string Name() const override { return "CocoOp"; }

mindspore/ccsrc/minddata/dataset/engine/datasetops/source/csv_op.cc

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Huawei Technologies Co., Ltd
+ * Copyright 2020-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -23,7 +23,6 @@
 #include "minddata/dataset/core/config_manager.h"
 #include "minddata/dataset/engine/jagged_connector.h"
 #include "minddata/dataset/engine/execution_tree.h"
-#include "minddata/dataset/engine/opt/pass.h"
 #include "minddata/dataset/util/random.h"
 
 namespace mindspore {
@@ -916,11 +915,5 @@ Status CsvOp::ComputeColMap() {
   return Status::OK();
 }
 
-// Visitor accept method for NodePass
-Status CsvOp::Accept(NodePass *p, bool *const modified) {
-  // Downcast shared pointer then call visitor
-  return p->RunOnNode(shared_from_base<CsvOp>(), modified);
-}
-
 }  // namespace dataset
 }  // namespace mindspore

mindspore/ccsrc/minddata/dataset/engine/datasetops/source/csv_op.h

@@ -298,12 +298,6 @@ class CsvOp : public ParallelOp {
   // @return Vector of the input file names
   std::vector<std::string> FileNames() { return csv_files_list_; }
 
-  // Base-class override for NodePass visitor acceptor.
-  // @param p - Pointer to the NodePass to be accepted.
-  // @param modified - Whether this node visit modified the pipeline.
-  // @return - Status of the node visit.
-  Status Accept(NodePass *p, bool *const modified) override;
-
   // Op name getter
   // @return Name of the current Op
   std::string Name() const override { return "CsvOp"; }

mindspore/ccsrc/minddata/dataset/engine/datasetops/source/generator_op.cc

@@ -1,5 +1,5 @@
 /**
- * Copyright 2019 Huawei Technologies Co., Ltd
+ * Copyright 2019-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -16,11 +16,9 @@
 #include "minddata/dataset/engine/datasetops/source/generator_op.h"
 #include <iomanip>
 #include "minddata/dataset/core/global_context.h"
-#include "minddata/dataset/engine/db_connector.h"
 #include "minddata/dataset/engine/data_buffer.h"
 #include "minddata/dataset/engine/execution_tree.h"
 #include "minddata/dataset/util/task_manager.h"
-#include "minddata/dataset/engine/opt/pass.h"
 
 namespace mindspore {
 namespace dataset {
@@ -278,12 +276,6 @@ Status GeneratorOp::Reset() {
   return Status(StatusCode::kSuccess, "GeneratorOp Reset Succeed");
 }
 
-// Visitor accept method for NodePass
-Status GeneratorOp::Accept(NodePass *p, bool *const modified) {
-  // Downcast shared pointer then call visitor
-  return p->RunOnNode(shared_from_base<GeneratorOp>(), modified);
-}
-
 Status GeneratorOp::ComputeColMap() {
   // Setup column names map (base class field)
   if (column_name_id_map_.empty()) {

mindspore/ccsrc/minddata/dataset/engine/datasetops/source/generator_op.h

@@ -1,5 +1,5 @@
 /**
- * Copyright 2019 Huawei Technologies Co., Ltd
+ * Copyright 2019-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -126,12 +126,6 @@ class GeneratorOp : public PipelineOp, public RandomAccessOp {
   /// \return Status The status code returned
   Status Reset() override;
 
-  /// Base-class override for NodePass visitor acceptor.
-  /// \param p - Pointer to the NodePass to be accepted.
-  /// \param modified - Whether this node visit modified the pipeline.
-  /// \return - Status of the node visit.
-  Status Accept(NodePass *p, bool *const modified) override;
-
   /// Op name getter
   /// \return Name of the current Op
   std::string Name() const override { return "GeneratorOp"; }

mindspore/ccsrc/minddata/dataset/engine/datasetops/source/image_folder_op.cc

@@ -1,5 +1,5 @@
 /**
- * Copyright 2019 Huawei Technologies Co., Ltd
+ * Copyright 2019-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -22,7 +22,6 @@
 #include "minddata/dataset/engine/datasetops/source/sampler/sequential_sampler.h"
 #include "minddata/dataset/engine/db_connector.h"
 #include "minddata/dataset/engine/execution_tree.h"
-#include "minddata/dataset/engine/opt/pass.h"
 
 namespace mindspore {
 namespace dataset {
@@ -450,12 +449,6 @@ Status ImageFolderOp::CountRowsAndClasses(const std::string &path, const std::se
   return Status::OK();
 }
 
-// Visitor accept method for NodePass
-Status ImageFolderOp::Accept(NodePass *p, bool *const modified) {
-  // Downcast shared pointer then call visitor
-  return p->RunOnNode(shared_from_base<ImageFolderOp>(), modified);
-}
-
 Status ImageFolderOp::ComputeColMap() {
   // Set the column name map (base class field)
   if (column_name_id_map_.empty()) {

mindspore/ccsrc/minddata/dataset/engine/datasetops/source/image_folder_op.h

@@ -1,5 +1,5 @@
 /**
- * Copyright 2019 Huawei Technologies Co., Ltd
+ * Copyright 2019-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -207,12 +207,6 @@ class ImageFolderOp : public ParallelOp, public RandomAccessOp {
   static Status CountRowsAndClasses(const std::string &path, const std::set<std::string> &exts, int64_t *num_rows,
                                     int64_t *num_classes, std::map<std::string, int32_t> class_index);
 
-  // Base-class override for NodePass visitor acceptor.
-  // @param p - Pointer to the NodePass to be accepted.
-  // @param modified - Whether this node visit modified the pipeline.
-  // @return - Status of the node visit.
-  Status Accept(NodePass *p, bool *const modified) override;
-
   // Op name getter
   // @return Name of the current Op
   std::string Name() const override { return "ImageFolderOp"; }

mindspore/ccsrc/minddata/dataset/engine/datasetops/source/manifest_op.cc

@@ -1,5 +1,5 @@
 /**
- * Copyright 2019 Huawei Technologies Co., Ltd
+ * Copyright 2019-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -27,7 +27,6 @@
 #include "minddata/dataset/engine/datasetops/source/sampler/sequential_sampler.h"
 #include "minddata/dataset/engine/db_connector.h"
 #include "minddata/dataset/engine/execution_tree.h"
-#include "minddata/dataset/engine/opt/pass.h"
 
 namespace mindspore {
 namespace dataset {
@@ -436,12 +435,6 @@ Status ManifestOp::GetClassIndexing(const std::string &file, const py::dict &dic
 }
 #endif
 
-// Visitor accept method for NodePass
-Status ManifestOp::Accept(NodePass *p, bool *const modified) {
-  // Downcast shared pointer then call visitor
-  return p->RunOnNode(shared_from_base<ManifestOp>(), modified);
-}
-
 Status ManifestOp::ComputeColMap() {
   // Set the column name map (base class field)
   if (column_name_id_map_.empty()) {

mindspore/ccsrc/minddata/dataset/engine/datasetops/source/manifest_op.h

@@ -1,5 +1,5 @@
 /**
- * Copyright 2019 Huawei Technologies Co., Ltd
+ * Copyright 2019-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -180,12 +180,6 @@ class ManifestOp : public ParallelOp, public RandomAccessOp {
                                  std::map<std::string, int32_t> *output_class_indexing);
 #endif
 
-  /// \brief Base-class override for NodePass visitor acceptor
-  /// \param[in] p Pointer to the NodePass to be accepted
-  /// \param[out] modified Indicator if the node was changed at all
-  /// \return Status of the node visit
-  Status Accept(NodePass *p, bool *const modified) override;
-
   // Op name getter
   // @return Name of the current Op
   std::string Name() const override { return "ManifestOp"; }

mindspore/ccsrc/minddata/dataset/engine/datasetops/source/mindrecord_op.cc

@@ -1,5 +1,5 @@
 /**
- * Copyright 2019 Huawei Technologies Co., Ltd
+ * Copyright 2019-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -29,7 +29,6 @@
 #include "minddata/dataset/engine/datasetops/dataset_op.h"
 #include "minddata/dataset/engine/db_connector.h"
 #include "minddata/dataset/engine/execution_tree.h"
-#include "minddata/dataset/engine/opt/pass.h"
 #include "minddata/dataset/util/log_adapter.h"
 
 namespace mindspore {
@@ -461,12 +460,6 @@ Status MindRecordOp::CountTotalRows(const std::vector<std::string> dataset_path,
   return Status::OK();
 }
 
-// Visitor accept method for NodePass
-Status MindRecordOp::Accept(NodePass *p, bool *const modified) {
-  // Downcast shared pointer then call visitor
-  return p->RunOnNode(shared_from_base<MindRecordOp>(), modified);
-}
-
 Status MindRecordOp::ComputeColMap() {
   if (column_name_id_map_.empty()) {
     for (int i = 0; i < static_cast<int>(columns_to_load_.size()); i++) {

mindspore/ccsrc/minddata/dataset/engine/datasetops/source/mindrecord_op.h

@@ -1,5 +1,5 @@
 /**
- * Copyright 2019 Huawei Technologies Co., Ltd
+ * Copyright 2019-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -202,12 +202,6 @@ class MindRecordOp : public ParallelOp {
 
   Status Init();
 
-  // Base-class override for NodePass visitor acceptor.
-  // @param p - Pointer to the NodePass to be accepted.
-  // @param modified - Whether this node visit modified the pipeline.
-  // @return - Status of the node visit.
-  Status Accept(NodePass *p, bool *const modified) override;
-
   // Op name getter
   // @return Name of the current Op
   std::string Name() const override { return "MindRecordOp"; }

mindspore/ccsrc/minddata/dataset/engine/datasetops/source/mnist_op.cc

@@ -1,5 +1,5 @@
 /**
- * Copyright 2019 Huawei Technologies Co., Ltd
+ * Copyright 2019-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -24,7 +24,6 @@
 #include "minddata/dataset/engine/datasetops/source/sampler/sequential_sampler.h"
 #include "minddata/dataset/engine/db_connector.h"
 #include "minddata/dataset/engine/execution_tree.h"
-#include "minddata/dataset/engine/opt/pass.h"
 
 namespace mindspore {
 namespace dataset {
@@ -459,12 +458,6 @@ Status MnistOp::CountTotalRows(const std::string &dir, const std::string &usage,
   return Status::OK();
 }
 
-// Visitor accept method for NodePass
-Status MnistOp::Accept(NodePass *p, bool *const modified) {
-  // Downcast shared pointer then call visitor
-  return p->RunOnNode(shared_from_base<MnistOp>(), modified);
-}
-
 Status MnistOp::ComputeColMap() {
   // set the column name map (base class field)
   if (column_name_id_map_.empty()) {

mindspore/ccsrc/minddata/dataset/engine/datasetops/source/mnist_op.h

@@ -1,5 +1,5 @@
 /**
- * Copyright 2019 Huawei Technologies Co., Ltd
+ * Copyright 2019-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -158,12 +158,6 @@ class MnistOp : public ParallelOp, public RandomAccessOp {
   // @return
   static Status CountTotalRows(const std::string &dir, const std::string &usage, int64_t *count);
 
-  /// \brief Base-class override for NodePass visitor acceptor
-  /// \param[in] p Pointer to the NodePass to be accepted
-  /// \param[out] modified Indicator if the node was changed at all
-  /// \return Status of the node visit
-  Status Accept(NodePass *p, bool *const modified) override;
-
   // Op name getter
   // @return Name of the current Op
   std::string Name() const override { return "MnistOp"; }

mindspore/ccsrc/minddata/dataset/engine/datasetops/source/random_data_op.cc

@@ -1,5 +1,5 @@
 /**
- * Copyright 2019 Huawei Technologies Co., Ltd
+ * Copyright 2019-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -24,7 +24,6 @@
 #include "minddata/dataset/util/random.h"
 #include "minddata/dataset/util/wait_post.h"
 #include "minddata/dataset/engine/datasetops/source/sampler/sequential_sampler.h"
-#include "minddata/dataset/engine/opt/pass.h"
 
 namespace mindspore {
 namespace dataset {
@@ -406,12 +405,6 @@ Status RandomDataOp::Reset() {
   return Status::OK();
 }
 
-// Visitor accept method for NodePass
-Status RandomDataOp::Accept(NodePass *p, bool *const modified) {
-  // Downcast shared pointer then call visitor
-  return p->RunOnNode(shared_from_base<RandomDataOp>(), modified);
-}
-
 Status RandomDataOp::ComputeColMap() {
   // Extract the column name mapping from the schema and save it in the class.
   if (column_name_id_map_.empty()) {

mindspore/ccsrc/minddata/dataset/engine/datasetops/source/random_data_op.h

@@ -1,5 +1,5 @@
 /**
- * Copyright 2019 Huawei Technologies Co., Ltd
+ * Copyright 2019-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -249,12 +249,6 @@ class RandomDataOp : public ParallelOp {
     return ++buffer_id_;
   }
 
-  // Base-class override for NodePass visitor acceptor.
-  // @param p - Pointer to the NodePass to be accepted.
-  // @param modified - Whether this node visit modified the pipeline.
-  // @return - Status of the node visit.
-  Status Accept(NodePass *p, bool *const modified) override;
-
   // Private function for computing the assignment of the column name map.
   // @return - Status
   Status ComputeColMap() override;

mindspore/ccsrc/minddata/dataset/engine/datasetops/source/text_file_op.cc

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Huawei Technologies Co., Ltd
+ * Copyright 2020-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -21,7 +21,6 @@
 #include <string>
 #include <utility>
 
-#include "utils/ms_utils.h"
 #include "minddata/dataset/engine/datasetops/source/text_file_op.h"
 #include "minddata/dataset/core/config_manager.h"
 #include "minddata/dataset/util/task_manager.h"
@@ -29,7 +28,6 @@
 #include "minddata/dataset/util/random.h"
 #include "minddata/dataset/engine/datasetops/source/io_block.h"
 #include "minddata/dataset/engine/execution_tree.h"
-#include "minddata/dataset/engine/opt/pass.h"
 
 namespace mindspore {
 namespace dataset {
@@ -499,11 +497,5 @@ Status TextFileOp::ComputeColMap() {
   return Status::OK();
 }
 
-// Visitor accept method for NodePass
-Status TextFileOp::Accept(NodePass *p, bool *const modified) {
-  // Downcast shared pointer then call visitor
-  return p->RunOnNode(shared_from_base<TextFileOp>(), modified);
-}
-
 }  // namespace dataset
 }  // namespace mindspore

mindspore/ccsrc/minddata/dataset/engine/datasetops/source/text_file_op.h

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Huawei Technologies Co., Ltd
+ * Copyright 2020-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -177,12 +177,6 @@ class TextFileOp : public ParallelOp {
   // @return Vector of the input file names
   std::vector<std::string> FileNames() { return text_files_list_; }
 
-  // Base-class override for NodePass visitor acceptor.
-  // @param p - Pointer to the NodePass to be accepted.
-  // @param modified - Whether this node visit modified the pipeline.
-  // @return - Status of the node visit.
-  Status Accept(NodePass *p, bool *const modified) override;
-
  private:
   // The entry point for when workers are launched.
   // @param worker_id - the id of the worker that is executing this function.

mindspore/ccsrc/minddata/dataset/engine/datasetops/source/tf_reader_op.cc

@@ -1,5 +1,5 @@
 /**
- * Copyright 2019 Huawei Technologies Co., Ltd
+ * Copyright 2019-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -26,7 +26,6 @@
 #include <vector>
 
 #include "proto/example.pb.h"
-#include "./securec.h"
 #include "minddata/dataset/core/config_manager.h"
 #include "minddata/dataset/core/global_context.h"
 #include "minddata/dataset/engine/data_schema.h"
@@ -34,7 +33,6 @@
 #include "minddata/dataset/engine/db_connector.h"
 #include "minddata/dataset/engine/execution_tree.h"
 #include "minddata/dataset/engine/jagged_connector.h"
-#include "minddata/dataset/engine/opt/pass.h"
 #include "minddata/dataset/util/random.h"
 #include "minddata/dataset/util/status.h"
 #include "minddata/dataset/util/task_manager.h"
@@ -1020,12 +1018,6 @@ int64_t TFReaderOp::CountTotalRowsSectioned(const std::vector<std::string> &file
   return rows_read;
 }
 
-// Visitor accept method for NodePass
-Status TFReaderOp::Accept(NodePass *p, bool *const modified) {
-  // Downcast shared pointer then call visitor
-  return p->RunOnNode(shared_from_base<TFReaderOp>(), modified);
-}
-
 Status TFReaderOp::ComputeColMap() {
   // Construct the column name map for this operator (base class field)
   if (column_name_id_map_.empty()) {
@@ -1038,26 +1030,5 @@ Status TFReaderOp::ComputeColMap() {
   return Status::OK();
 }
 
-// During tree prepare phase, operators may have specific post-operations to perform depending on
-// their role.
-Status TFReaderOp::PrepareNodePostAction() {
-  // Run common code from super class before adding TFReaderOp specific handling
-  RETURN_IF_NOT_OK(ParallelOp::PrepareNodePostAction());
-
-  // Now that the sampler has been saved for the cache, we need to adjust the TFReaderOp to turn it into
-  // a simpler producer of all data (no shuffling or sharding or anything)
-  if (!BitTest(tree_->PrepareFlags(), ExecutionTree::kDePrepCache)) {
-    // This sanity check had been delayed until now in the prepare loop.
-    // If we are not in a cache path, then we can validate the file-based sharding config.
-    // If we are in a cache path, there is no file-based sharding so the check is not correct in that
-    // situation.
-    if (!equal_rows_per_shard_ && dataset_files_list_.size() < static_cast<uint32_t>(num_devices_)) {
-      RETURN_STATUS_UNEXPECTED("Invalid file, not enough tfrecord files provided.\n");
-    }
-  }
-
-  return Status::OK();
-}
-
 }  // namespace dataset
 }  // namespace mindspore

mindspore/ccsrc/minddata/dataset/engine/datasetops/source/tf_reader_op.h

@@ -1,5 +1,5 @@
 /**
- * Copyright 2019 Huawei Technologies Co., Ltd
+ * Copyright 2019-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -222,12 +222,6 @@ class TFReaderOp : public ParallelOp {
   static Status CountTotalRows(int64_t *out_total_rows, const std::vector<std::string> &filenames, int64_t threads = 1,
                                bool estimate = false);
 
-  // Base-class override for NodePass visitor acceptor.
-  // @param p - Pointer to the NodePass to be accepted.
-  // @param modified - Whether this node visit modified the pipeline.
-  // @return - Status of the node visit.
-  Status Accept(NodePass *p, bool *const modified) override;
-
   // Op name getter
   // @return Name of the current Op
   std::string Name() const override { return "TFReaderOp"; }
@@ -236,12 +230,6 @@ class TFReaderOp : public ParallelOp {
   // @return Vector of the input file names
   std::vector<std::string> FileNames() { return dataset_files_list_; }
 
-  // During tree prepare phase, operators may have specific post-operations to perform depending on
-  // their role.
-  // @notes Derived versions of this function should always call it's superclass version first
-  // before providing their own implementations.
-  Status PrepareNodePostAction() override;
-
   static bool ValidateFirstRowCrc(const std::string &filename);
 
  private:

mindspore/ccsrc/minddata/dataset/engine/datasetops/source/voc_op.cc

@@ -1,5 +1,5 @@
 /**
- * Copyright 2019 Huawei Technologies Co., Ltd
+ * Copyright 2019-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -24,7 +24,6 @@
 #include "minddata/dataset/engine/datasetops/source/sampler/sequential_sampler.h"
 #include "minddata/dataset/engine/db_connector.h"
 #include "minddata/dataset/engine/execution_tree.h"
-#include "minddata/dataset/engine/opt/pass.h"
 #include "utils/ms_utils.h"
 
 namespace mindspore {
@@ -494,12 +493,6 @@ Status VOCOp::GetClassIndexing(const std::string &dir, const std::string &task_t
 }
 #endif
 
-// Visitor accept method for NodePass
-Status VOCOp::Accept(NodePass *p, bool *const modified) {
-  // Downcast shared pointer then call visitor
-  return p->RunOnNode(shared_from_base<VOCOp>(), modified);
-}
-
 Status VOCOp::ComputeColMap() {
   // Set the column name map (base class field)
   if (column_name_id_map_.empty()) {

mindspore/ccsrc/minddata/dataset/engine/datasetops/source/voc_op.h

@@ -1,5 +1,5 @@
 /**
- * Copyright 2019 Huawei Technologies Co., Ltd
+ * Copyright 2019-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -210,12 +210,6 @@ class VOCOp : public ParallelOp, public RandomAccessOp {
                                  const py::dict &dict, std::map<std::string, int32_t> *output_class_indexing);
 #endif
 
-  /// \brief Base-class override for NodePass visitor acceptor
-  /// \param[in] p Pointer to the NodePass to be accepted
-  /// \param[out] modified Indicator if the node was changed at all
-  /// \return Status of the node visit
-  Status Accept(NodePass *p, bool *const modified) override;
-
   // Op name getter
   // @return Name of the current Op
   std::string Name() const override { return "VOCOp"; }

mindspore/ccsrc/minddata/dataset/engine/datasetops/take_op.cc

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Huawei Technologies Co., Ltd
+ * Copyright 2020-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -23,7 +23,6 @@
 #include "minddata/dataset/engine/datasetops/take_op.h"
 #include "minddata/dataset/engine/db_connector.h"
 #include "minddata/dataset/engine/execution_tree.h"
-#include "minddata/dataset/engine/opt/pass.h"
 
 namespace mindspore {
 namespace dataset {
@@ -127,17 +126,5 @@ Status TakeOp::FillBuffer(std::unique_ptr<DataBuffer> *buffer, std::unique_ptr<D
   return Status::OK();
 }
 
-// Visitor accept method for NodePass
-Status TakeOp::Accept(NodePass *p, bool *const modified) {
-  // Downcast shared pointer then call visitor
-  return p->RunOnNode(shared_from_base<TakeOp>(), modified);
-}
-
-// Visitor pre-accept method for NodePass
-Status TakeOp::PreAccept(NodePass *p, bool *const modified) {
-  // Downcast shared pointer then call visitor
-  return p->PreRunOnNode(shared_from_base<TakeOp>(), modified);
-}
-
 }  // namespace dataset
 }  // namespace mindspore

mindspore/ccsrc/minddata/dataset/engine/datasetops/take_op.h

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Huawei Technologies Co., Ltd
+ * Copyright 2020-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -78,18 +78,6 @@ class TakeOp : public PipelineOp {
   // @return Status The status code returned
   Status operator()() override;
 
-  // Base-class override for NodePass visitor acceptor.
-  // @param p - Pointer to the NodePass to be accepted.
-  // @param modified - Whether this node visit modified the pipeline.
-  // @return - Status of the node visit.
-  Status Accept(NodePass *p, bool *const modified) override;
-
-  /// \brief Base-class override for NodePass pre-visit acceptor
-  /// \param[in] p The node to visit
-  /// \param[out] modified Indicator if the node was modified
-  /// \return Status of the node visit
-  Status PreAccept(NodePass *p, bool *const modified) override;
-
   // Op name getter
   // @return Name of the current Op
   std::string Name() const override { return kTakeOp; }

mindspore/ccsrc/minddata/dataset/engine/datasetops/zip_op.cc

@@ -1,5 +1,5 @@
 /**
- * Copyright 2019 Huawei Technologies Co., Ltd
+ * Copyright 2019-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -20,7 +20,6 @@
 #include "minddata/dataset/core/constants.h"
 #include "minddata/dataset/engine/data_buffer.h"
 #include "minddata/dataset/engine/db_connector.h"
-#include "minddata/dataset/engine/opt/pass.h"
 #include "minddata/dataset/core/config_manager.h"
 #include "minddata/dataset/core/global_context.h"
 #include "minddata/dataset/util/log_adapter.h"
@@ -238,18 +237,6 @@ Status ZipOp::EoeReceived(int32_t) {
   return Status::OK();
 }
 
-// Visitor pre-accept method for NodePass
-Status ZipOp::PreAccept(NodePass *p, bool *const modified) {
-  // Downcast shared pointer then call visitor
-  return p->PreRunOnNode(shared_from_base<ZipOp>(), modified);
-}
-
-// Visitor accept method for NodePass
-Status ZipOp::Accept(NodePass *p, bool *const modified) {
-  // Downcast shared pointer then call visitor
-  return p->RunOnNode(shared_from_base<ZipOp>(), modified);
-}
-
 Status ZipOp::ComputeColMap() {
   if (column_name_id_map_.empty()) {
     column_name_id_map_ = {};

mindspore/ccsrc/minddata/dataset/engine/datasetops/zip_op.h

@@ -1,5 +1,5 @@
 /**
- * Copyright 2019 Huawei Technologies Co., Ltd
+ * Copyright 2019-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -104,18 +104,6 @@ class ZipOp : public PipelineOp {
   // @return Status The status code returned
   Status operator()() override;
 
-  /// \brief Base-class override for NodePass pre-visit acceptor
-  /// \param[in] p The node to visit
-  /// \param[out] modified Indicator if the node was modified
-  /// \return Status of the node visit
-  Status PreAccept(NodePass *p, bool *const modified) override;
-
-  /// \brief Base-class override for NodePass visitor acceptor.
-  /// \param[in] p Pointer to the NodePass to be accepted.
-  /// \param[out] modified Whether this node visit modified the pipeline.
-  /// \return - Status of the node visit.
-  Status Accept(NodePass *p, bool *const modified) override;
-
   // Op name getter
   // @return Name of the current Op
   std::string Name() const override { return kZipOp; }

mindspore/ccsrc/minddata/dataset/engine/execution_tree.cc

@@ -29,7 +29,7 @@
 namespace mindspore {
 namespace dataset {
 // Constructor
-ExecutionTree::ExecutionTree() : id_count_(0), tree_state_(kDeTStateInit), prepare_flags_(kDePrepNone) {
+ExecutionTree::ExecutionTree() : id_count_(0), tree_state_(kDeTStateInit) {
   tg_ = std::make_unique<TaskGroup>();
   profiling_manager_ = std::make_unique<ProfilingManager>(this);
 #if defined(ENABLE_GPUQUE) || defined(ENABLE_TDTQUE)
@@ -66,11 +66,11 @@ Status ExecutionTree::AssociateNode(const std::shared_ptr<DatasetOp> &op) {
   if (op->tree_ == this) {
     return Status::OK();
   }
-  if (tree_state_ != kDeTStateInit && tree_state_ != kDeTStateBuilding && tree_state_ != kDeTStatePrepare) {
+  if (tree_state_ != kDeTStateInit && tree_state_ != kDeTStateBuilding) {
     std::string err_msg =
       "Invalid tree state for adding a node. Current state: " + std::to_string(static_cast<int>(tree_state_)) +
       " Expected states: " + std::to_string(static_cast<int>(kDeTStateInit)) + " or " +
-      std::to_string(static_cast<int>(kDeTStateBuilding)) + " or " + std::to_string(static_cast<int>(kDeTStatePrepare));
+      std::to_string(static_cast<int>(kDeTStateBuilding));
     RETURN_STATUS_UNEXPECTED(err_msg);
   }
 
@@ -172,10 +172,10 @@ Status ExecutionTree::Launch() {
 #endif
 
   // Tree must be built and prepared before it can be launched!
-  if (tree_state_ != kDeTStateReady) {
+  if (tree_state_ != kDeTStatePrepared) {
     std::string err_msg =
       "Invalid tree state for launching tree. Current state: " + std::to_string(static_cast<int>(tree_state_)) +
-      " Expected state: " + std::to_string(static_cast<int>(kDeTStateReady));
+      " Expected state: " + std::to_string(static_cast<int>(kDeTStatePrepared));
     RETURN_STATUS_UNEXPECTED(err_msg);
   }
 
@@ -244,43 +244,31 @@ Status ExecutionTree::LaunchWorkers(int32_t num_workers, std::function<Status(ui
   return Status::OK();
 }
 
-// walk the tree to perform modifications to the tree or specific nodes within the tree to get
-// it ready for execution.
-//
-// This driver is deprecated.
+// Walks the tree to perform modifications to the tree in post-order to get it ready for execution.
 Status ExecutionTree::Prepare() {
-  // The tree is ready to be prepared.
-  tree_state_ = kDeTStatePrepare;
-
   if (root_ == nullptr) {
     RETURN_STATUS_UNEXPECTED("Please assign one operator as the root of this tree.");
   }
 
-  // Start the recursive prepare
-  RETURN_IF_NOT_OK(this->PrepareNode(root_));
-  tree_state_ = kDeTStateReady;
-  return Status::OK();
-}
+  std::vector<std::shared_ptr<DatasetOp>> fifo;
+  std::shared_ptr<DatasetOp> op = root_;
+  size_t index = 0;
 
-// Recursive function used during prepare phase to visit a node and drive any pre- and post-
-// node actions during a tree walk.
-Status ExecutionTree::PrepareNode(const std::shared_ptr<DatasetOp> &dataset_op) {
-  // Before going down into children, make any prepare flags updates based on this operator.
-  uint32_t op_prep_flags = dataset_op->PrepareFlags();
-  BitSet(&prepare_flags_, op_prep_flags);
+  // Build a FIFO queue with the root at the beginning and continue adding its descendants to the queue.
+  fifo.push_back(op);
+  do {
+    op = fifo[index];
+    fifo.insert(fifo.end(), op->child_.begin(), op->child_.end());
+    ++index;
+  } while (index < fifo.size());
 
-  // Now, descend to children
-  for (const auto &i : dataset_op->child_) {
-    RETURN_IF_NOT_OK(this->PrepareNode(i));
+  // By iterating from the end of the FIFO queue, we simulate the post-order walk.
+  for (auto rit = fifo.crbegin(); rit != fifo.crend(); ++rit) {
+    RETURN_IF_NOT_OK((*rit)->PrepareOperator());
   }
 
-  // No more children, now we execute any prepare actions before going back up the
-  // the tree on recursive function
-  RETURN_IF_NOT_OK(dataset_op->PrepareNodePostAction());
-
-  // Then clear the flags from this op now that we have prepared it.
-  BitClear(&prepare_flags_, op_prep_flags);
-
+  // The tree is prepared.
+  tree_state_ = kDeTStatePrepared;
   return Status::OK();
 }
 }  // namespace dataset

mindspore/ccsrc/minddata/dataset/engine/execution_tree.h

@@ -39,19 +39,11 @@ class Pass;
 using OptPass = std::vector<std::unique_ptr<Pass>>;
 class ExecutionTree {
  public:
-  // Prepare flags used during tree prepare phase
-  enum PrepareFlags {
-    kDePrepNone = 0,
-    kDePrepRepeat = 1,  //  Processing a repeat operation
-    kDePrepCache = 2    //  Processing a cache operation
-  };
-
   // State flags for the lifecycle of the tree
   enum TreeState {
     kDeTStateInit = 0,   // The freshly initialized state after construction
     kDeTStateBuilding,   // The tree is being built, nodes are being added
-    kDeTStatePrepare,    // The tree has been assigned a root node and is pending prepare
-    kDeTStateReady,      // The tree has been prepared and is ready to be launched
+    kDeTStatePrepared,   // The tree has been prepared and is ready to be launched
     kDeTStateExecuting,  // The tree has been launched and is executing
     kDeTStateEpochEnd,   // The tree has been received end of epoch signal, just for profiling
     kDeTStateFinished    // The tree has been drained, dataset iterator received EOF
@@ -111,122 +103,101 @@ class ExecutionTree {
   // Destructor
   ~ExecutionTree();
 
-  // Associates a DatasetOp with this tree. This assigns a valid node id to the operator and
-  // provides it with a link to the tree. A node cannot form any relationships (parent/child) with
-  // other nodes unless they are associated with the same tree.
-  // @param op - The operator to associate
-  // @return Status The status code returned
+  /// \brief Associates a DatasetOp with this tree. This assigns a valid node id to the operator and
+  ///     provides it with a link to the tree. A node cannot form any relationships (parent/child) with
+  ///     other nodes unless they are associated with the same tree.
+  /// \param op - The operator to associate
+  /// \return Status The status code returned
   Status AssociateNode(const std::shared_ptr<DatasetOp> &op);
 
-  // Sets the root node of the tree
-  // @param op - The operator to assign as root
-  // @return Status The status code returned
+  /// \brief Set the root node of the tree
+  /// \param op - The operator to assign as root
+  /// \return Status The status code returned
   Status AssignRoot(const std::shared_ptr<DatasetOp> &op);
 
-  // Start the execution of the tree
-  // @return Status The status code returned
+  /// \brief Start the execution of the tree
+  /// \return Status The status code returned
   Status Launch();
 
-  /// A print method typically used for debugging
+  /// /brief A print method typically used for debugging
   /// \param out - The output stream to write output to
   void Print(std::ostream &out, const std::shared_ptr<DatasetOp> &op = nullptr) const;
 
-  // Returns an iterator positioned at the start
-  // @return Iterator - The iterator
+  /// \brief Return an iterator positioned at the start
+  /// \return Iterator - The iterator
   ExecutionTree::Iterator begin(const std::shared_ptr<DatasetOp> &root = nullptr) const {
     return Iterator(root == nullptr ? root_ : root);
   }
 
-  // Returns an iterator positioned at the end
-  // @return Iterator - The iterator
+  /// \brief Return an iterator positioned at the end
+  /// \return Iterator - The iterator
   ExecutionTree::Iterator end() const { return Iterator(nullptr); }
 
-  // << Stream output operator overload
-  // @notes This allows you to write the debug print info using stream operators
-  // @param out - reference to the output stream being overloaded
-  // @param exe_tree - reference to the execution tree to display
-  // @return - the output stream must be returned
+  /// \brief << Stream output operator overload
+  /// \notes This allows you to write the debug print info using stream operators
+  /// \param out - reference to the output stream being overloaded
+  /// \param exe_tree - reference to the execution tree to display
+  /// \return - the output stream must be returned
   friend std::ostream &operator<<(std::ostream &out, ExecutionTree &exe_tree) {
     exe_tree.Print(out);
     return out;
   }
 
-  // Given the number of workers, launches the worker entry function for each. Essentially a
-  // wrapper for the TaskGroup handling that is stored inside the execution tree.
-  // @param num_workers - The number of workers to launch
-  // @param func - The function entry point that workers will execute
-  // @param name - The description of worker to launch
-  // @param op_id - The id of corresponding operator, if not inherit from dataset op then it is -1.
-  // @return Status The status code returned
+  /// \brief Given the number of workers, launches the worker entry function for each. Essentially a
+  ///     wrapper for the TaskGroup handling that is stored inside the execution tree.
+  /// \param num_workers - The number of workers to launch
+  /// \param func - The function entry point that workers will execute
+  /// \param name - The description of worker to launch
+  /// \param op_id - The id of corresponding operator, if not inherit from dataset op then it is -1.
+  /// \return Status The status code returned
   Status LaunchWorkers(int32_t num_workers, std::function<Status(uint32_t)> func, std::string name = "",
                        int32_t operator_id = -1);
 
-  // Getter method
-  // @return shared_ptr to the root operator
+  /// \brief Getter method
+  /// \return shared_ptr to the root operator
   std::shared_ptr<DatasetOp> root() const { return root_; }
 
-  // Getter method
-  // @return the prepare flags
-  uint32_t PrepareFlags() const { return prepare_flags_; }
-
-  // Compulsory transformation/action pre optimization.
-  // @return Status The status code returned
-  Status PreAction();
-
-  // Compulsory transformation/action post optimization.
-  // @return Status The status code returned
-  Status PostAction();
-
-  // The DEPRECATED driver of the prepare phase of the execution tree. The prepare phase will recursively
-  // walk the tree to perform modifications to the tree or specific nodes within the tree to get
-  // it ready for execution.
-  // @param Total number of epochs that will be run on this tree
-  // @return Status The status code returned
+  /// \brief The prepare phase walks the tree in post-order to perform modifications to get it ready for execution.
+  /// \return Status The status code returned
   Status Prepare();
 
-  // Recursive function used during prepare phase to visit a node and drive any pre- and post-
-  // node actions during a tree walk.
-  // @param op - The dataset op to work on
-  // @return Status The status code returned
-  Status PrepareNode(const std::shared_ptr<DatasetOp> &dataset_op);
-
-  // Return the pointer to the TaskGroup
-  // @return raw pointer to the TaskGroup
+  /// \brief Return the pointer to the TaskGroup
+  /// \return raw pointer to the TaskGroup
   TaskGroup *const AllTasks() const { return tg_.get(); }
 
-  // Return if the ExecutionTree is at end of epoch status
-  // @return bool - true is ExecutionTree is end of epoch status
+  /// \brief Return if the ExecutionTree is at end of epoch status
+  /// \return bool - true is ExecutionTree is end of epoch status
   bool IsEpochEnd() const { return tree_state_ == TreeState::kDeTStateEpochEnd; }
 
-  // Set the ExecutionTree to EOE state
+  /// \brief Set the ExecutionTree to EOE state
   void SetEpochEnd() { tree_state_ = TreeState::kDeTStateEpochEnd; }
 
-  // Set the ExecutionTree to executing state
+  /// \brief Set the ExecutionTree to executing state
   void SetExecuting() { tree_state_ = TreeState::kDeTStateExecuting; }
 
-  // Return if the ExecutionTree is finished (iterator receives EOF).
-  // @return Bool - true is ExecutionTree is finished
+  /// \brief Set the ExecutionTree to Finished state.
+  void SetFinished() { tree_state_ = TreeState::kDeTStateFinished; }
+
+  /// \brief Return if the ExecutionTree is finished (iterator receives EOF).
+  /// \return Bool - true is ExecutionTree is finished
   bool isFinished() const { return tree_state_ == TreeState::kDeTStateFinished; }
 
-  // Return if the ExecutionTree is ready.
-  // @return Bool - true is ExecutionTree is ready
+  /// \brief Return if the ExecutionTree is ready.
+  /// \return Bool - true is ExecutionTree is ready
   bool isPrepared() const {
-    return tree_state_ == TreeState::kDeTStateReady || tree_state_ == kDeTStateExecuting ||
-           tree_state_ == kDeTStateFinished;
+    return tree_state_ == TreeState::kDeTStatePrepared || tree_state_ == TreeState::kDeTStateExecuting ||
+           tree_state_ == TreeState::kDeTStateFinished;
   }
 
-  // Set the ExecutionTree to Finished state.
-  void SetFinished() { tree_state_ = TreeState::kDeTStateFinished; }
-
-  // Getter for profiling manager, no ownership
+  /// \brief Getter for profiling manager, no ownership
   ProfilingManager *GetProfilingManager() { return profiling_manager_.get(); }
 
  private:
-  // A helper functions for doing the recursive printing
-  // @param dataset_op - The dataset op to print
-  // @param indent - an indent string for aligning child levels in output
-  // @param last - an indicator if it's the last child or not
-  // @param detailed - should it display the detailed node output or the summary line
+  /// \brief A helper functions for doing the recursive printing
+  /// \param dataset_op - The dataset op to print
+  /// \param indent - an indent string for aligning child levels in output
+  /// \param last - an indicator if it's the last child or not
+  /// \param detailed - should it display the detailed node output or the summary line
   void PrintNode(std::ostream &out, const std::shared_ptr<DatasetOp> &dataset_op, std::string indent, bool last,
                  bool detailed) const;
 

mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/tf_record_node.cc

@@ -24,6 +24,7 @@
 
 #include "minddata/dataset/engine/datasetops/source/tf_reader_op.h"
 #include "minddata/dataset/engine/jagged_connector.h"
+#include "minddata/dataset/engine/opt/pass.h"
 #include "minddata/dataset/util/status.h"
 #include "utils/system/crc32c.h"
 
@@ -246,5 +247,16 @@ Status TFRecordNode::MakeSimpleProducer() {
   return Status::OK();
 }
 
+// Visitor accepting method for IRNodePass
+Status TFRecordNode::Accept(IRNodePass *p, bool *const modified) {
+  // Downcast shared pointer then call visitor
+  return p->Visit(shared_from_base<TFRecordNode>(), modified);
+}
+
+// Visitor accepting method for IRNodePass
+Status TFRecordNode::AcceptAfter(IRNodePass *p, bool *const modified) {
+  // Downcast shared pointer then call visitor
+  return p->VisitAfter(shared_from_base<TFRecordNode>(), modified);
+}
 }  // namespace dataset
 }  // namespace mindspore

mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/tf_record_node.h

@@ -30,6 +30,8 @@ namespace dataset {
 /// \class TFRecordNode
 /// \brief A Dataset derived class to represent TFRecord dataset
 class TFRecordNode : public NonMappableSourceNode {
+  friend class CacheValidationPass;
+
  public:
   /// \brief Constructor
   /// \note Parameter 'schema' is the path to the schema file
@@ -140,6 +142,18 @@ class TFRecordNode : public NonMappableSourceNode {
   /// \return Status of the function
   Status MakeSimpleProducer() override;
 
+  /// \brief Base-class override for accepting IRNodePass visitor
+  /// \param[in] p The node to visit
+  /// \param[out] modified Indicator if the node was modified
+  /// \return Status of the node visit
+  Status Accept(IRNodePass *p, bool *const modified) override;
+
+  /// \brief Base-class override for accepting IRNodePass visitor
+  /// \param[in] p The node to visit
+  /// \param[out] modified Indicator if the node was modified
+  /// \return Status of the node visit
+  Status AcceptAfter(IRNodePass *p, bool *const modified) override;
+
  private:
   std::vector<std::string> dataset_files_;
   std::string schema_path_;  // schema_path_ path to schema file. It is set when type of schema parameter is string

mindspore/ccsrc/minddata/dataset/engine/opt/pass.cc

@@ -43,6 +43,7 @@
 #include "minddata/dataset/engine/ir/datasetops/source/generator_node.h"
 #endif
 #include "minddata/dataset/engine/ir/datasetops/source/random_node.h"
+#include "minddata/dataset/engine/ir/datasetops/source/tf_record_node.h"
 #ifdef ENABLE_PYTHON
 #include "minddata/dataset/engine/ir/datasetops/sync_wait_node.h"
 #endif
@@ -50,53 +51,6 @@
 #include "minddata/dataset/engine/ir/datasetops/transfer_node.h"
 #include "minddata/dataset/engine/ir/datasetops/zip_node.h"
 
-//////////////////////////////////
-// This section of code will be removed once the migration of optimizer from DatasetOp to DatasetNode is done.
-#include "minddata/dataset/engine/datasetops/batch_op.h"
-#include "minddata/dataset/engine/datasetops/build_vocab_op.h"
-#ifndef ENABLE_ANDROID
-#include "minddata/dataset/engine/datasetops/build_sentence_piece_vocab_op.h"
-#include "minddata/dataset/engine/datasetops/cache_op.h"
-#include "minddata/dataset/engine/datasetops/cache_merge_op.h"
-#include "minddata/dataset/engine/datasetops/cache_lookup_op.h"
-#endif
-#include "minddata/dataset/engine/datasetops/concat_op.h"
-#include "minddata/dataset/engine/datasetops/dataset_op.h"
-#include "minddata/dataset/engine/datasetops/device_queue_op.h"
-#include "minddata/dataset/engine/datasetops/epoch_ctrl_op.h"
-#include "minddata/dataset/engine/datasetops/map_op/map_op.h"
-#include "minddata/dataset/engine/datasetops/project_op.h"
-#include "minddata/dataset/engine/datasetops/rename_op.h"
-#include "minddata/dataset/engine/datasetops/repeat_op.h"
-#include "minddata/dataset/engine/datasetops/skip_op.h"
-#include "minddata/dataset/engine/datasetops/shuffle_op.h"
-#include "minddata/dataset/engine/datasetops/source/album_op.h"
-#include "minddata/dataset/engine/datasetops/source/celeba_op.h"
-#include "minddata/dataset/engine/datasetops/source/cifar_op.h"
-#include "minddata/dataset/engine/datasetops/source/coco_op.h"
-#include "minddata/dataset/engine/datasetops/source/manifest_op.h"
-#ifndef ENABLE_ANDROID
-#include "minddata/dataset/engine/datasetops/source/mindrecord_op.h"
-#endif
-#include "minddata/dataset/engine/datasetops/source/mnist_op.h"
-#include "minddata/dataset/engine/datasetops/source/random_data_op.h"
-#ifndef ENABLE_ANDROID
-#include "minddata/dataset/engine/datasetops/source/tf_reader_op.h"
-#include "minddata/dataset/engine/datasetops/source/clue_op.h"
-#include "minddata/dataset/engine/datasetops/source/csv_op.h"
-#include "minddata/dataset/engine/datasetops/source/text_file_op.h"
-#endif
-
-#ifdef ENABLE_PYTHON
-#include "minddata/dataset/engine/datasetops/source/voc_op.h"
-#include "minddata/dataset/engine/datasetops/filter_op.h"
-#include "minddata/dataset/engine/datasetops/source/generator_op.h"
-#endif
-#include "minddata/dataset/engine/datasetops/source/image_folder_op.h"
-#include "minddata/dataset/engine/datasetops/take_op.h"
-#include "minddata/dataset/engine/datasetops/zip_op.h"
-//////////////////////////////////
-
 namespace mindspore {
 namespace dataset {
 
@@ -295,6 +249,12 @@ Status IRNodePass::Visit(std::shared_ptr<TakeNode> node, bool *const modified) {
 Status IRNodePass::VisitAfter(std::shared_ptr<TakeNode> node, bool *const modified) {
   return VisitAfter(std::static_pointer_cast<DatasetNode>(node), modified);
 }
+Status IRNodePass::Visit(std::shared_ptr<TFRecordNode> node, bool *const modified) {
+  return Visit(std::static_pointer_cast<NonMappableSourceNode>(node), modified);
+}
+Status IRNodePass::VisitAfter(std::shared_ptr<TFRecordNode> node, bool *const modified) {
+  return VisitAfter(std::static_pointer_cast<NonMappableSourceNode>(node), modified);
+}
 Status IRNodePass::Visit(std::shared_ptr<TransferNode> node, bool *const modified) {
   return Visit(std::static_pointer_cast<DatasetNode>(node), modified);
 }
@@ -332,287 +292,5 @@ Status IRNodePass::Visit(std::shared_ptr<MappableSourceNode> node, bool *const m
 Status IRNodePass::Visit(std::shared_ptr<NonMappableSourceNode> node, bool *const modified) {
   return Visit(std::static_pointer_cast<DatasetNode>(node), modified);
 }
-
-//////////////////////////////////
-// This section of code will be removed once the migration of optimizer from DatasetOp to DatasetNode is done.
-// Driver method for TreePass
-Status TreePass::Run(ExecutionTree *tree, bool *const modified) {
-  if (tree == nullptr || modified == nullptr) {
-    return Status(StatusCode::kMDUnexpectedError, "Null pointer passed to TreePass");
-  }
-  // Initialize modified flag
-  *modified = false;
-  return this->RunOnTree(tree, modified);
-}
-
-// Driver method for NodePass
-Status NodePass::Run(ExecutionTree *tree, bool *const modified) {
-  if (tree == nullptr || modified == nullptr) {
-    return Status(StatusCode::kMDUnexpectedError, "Null pointer passed to NodePass");
-  }
-  // Initialize modified flag
-  *modified = false;
-  std::shared_ptr<DatasetOp> root = tree->root();
-  if (traversalOrder_ == Order::DFS) {
-    // DFS
-    return DFSNodeVisit(root, modified);
-  } else if (traversalOrder_ == Order::BFS) {
-    // BFS
-    return BFSNodeVisit(root, modified);
-  }
-  return Status::OK();
-}
-
-// Helper function to perform DFS visit
-Status NodePass::DFSNodeVisit(std::shared_ptr<DatasetOp> node, bool *const modified) {
-  RETURN_IF_NOT_OK(node->PreAccept(this, modified));
-  for (const auto &c : node->Children()) {
-    RETURN_IF_NOT_OK(this->DFSNodeVisit(c, modified));
-  }
-  return node->Accept(this, modified);
-}
-
-// Helper function to perform BFS visit
-Status NodePass::BFSNodeVisit(std::shared_ptr<DatasetOp> root, bool *const modified) {
-  // Initialize bfs queue with root
-  std::queue<std::shared_ptr<DatasetOp>> bfsQueue;
-  bfsQueue.push(root);
-
-  // BFS loop
-  while (!bfsQueue.empty()) {
-    // Pop the front of the bfs queue
-    auto curNode = bfsQueue.front();
-    bfsQueue.pop();
-
-    // Run node pass
-    RETURN_IF_NOT_OK(curNode->Accept(this, modified));
-
-    // Push children into bfs queue
-    for (const auto &c : curNode->Children()) {
-      bfsQueue.push(c);
-    }
-  }
-  return Status::OK();
-}
-
-Status NodePass::RunOnNode(std::shared_ptr<BatchOp> node, bool *const modified) {
-  // Fallback to base class visitor by default
-  return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
-}
-
-Status NodePass::RunOnNode(std::shared_ptr<MapOp> node, bool *const modified) {
-  // Fallback to base class visitor by default
-  return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
-}
-
-Status NodePass::RunOnNode(std::shared_ptr<ProjectOp> node, bool *const modified) {
-  // Fallback to base class visitor by default
-  return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
-}
-
-Status NodePass::RunOnNode(std::shared_ptr<RenameOp> node, bool *const modified) {
-  // Fallback to base class visitor by default
-  return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
-}
-
-Status NodePass::RunOnNode(std::shared_ptr<SkipOp> node, bool *const modified) {
-  // Fallback to base class visitor by default
-  return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
-}
-
-Status NodePass::RunOnNode(std::shared_ptr<ShuffleOp> node, bool *const modified) {
-  // Fallback to base class visitor by default
-  return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
-}
-
-Status NodePass::RunOnNode(std::shared_ptr<RandomDataOp> node, bool *const modified) {
-  // Fallback to base class visitor by default
-  return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
-}
-
-Status NodePass::RunOnNode(std::shared_ptr<TakeOp> node, bool *const modified) {
-  // Fallback to base class visitor by default
-  return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
-}
-
-Status NodePass::RunOnNode(std::shared_ptr<ZipOp> node, bool *const modified) {
-  // Fallback to base class visitor by default
-  return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
-}
-
-Status NodePass::RunOnNode(std::shared_ptr<DeviceQueueOp> node, bool *const modified) {
-  // Fallback to base class visitor by default
-  return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
-}
-
-Status NodePass::RunOnNode(std::shared_ptr<ImageFolderOp> node, bool *const modified) {
-  // Fallback to base class visitor by default
-  return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
-}
-
-Status NodePass::RunOnNode(std::shared_ptr<AlbumOp> node, bool *const modified) {
-  // Fallback to base class visitor by default
-  return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
-}
-
-Status NodePass::RunOnNode(std::shared_ptr<MnistOp> node, bool *const modified) {
-  // Fallback to base class visitor by default
-  return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
-}
-
-Status NodePass::RunOnNode(std::shared_ptr<CifarOp> node, bool *const modified) {
-  // Fallback to base class visitor by default
-  return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
-}
-
-Status NodePass::RunOnNode(std::shared_ptr<CelebAOp> node, bool *const modified) {
-  // Fallback to base class visitor by default
-  return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
-}
-
-Status NodePass::RunOnNode(std::shared_ptr<CocoOp> node, bool *const modified) {
-  // Fallback to base class visitor by default
-  return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
-}
-
-Status NodePass::RunOnNode(std::shared_ptr<RepeatOp> node, bool *const modified) {
-  // Fallback to base class visitor by default
-  return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
-}
-
-Status NodePass::RunOnNode(std::shared_ptr<EpochCtrlOp> node, bool *const modified) {
-  // Fallback to base class visitor by default
-  return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
-}
-
-Status NodePass::PreRunOnNode(std::shared_ptr<RepeatOp> node, bool *const modified) {
-  // Fallback to base class visitor by default
-  return PreRunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
-}
-
-Status NodePass::PreRunOnNode(std::shared_ptr<EpochCtrlOp> node, bool *const modified) {
-  // Fallback to base class visitor by default
-  return PreRunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
-}
-
-Status NodePass::PreRunOnNode(std::shared_ptr<BuildVocabOp> node, bool *const modified) {
-  // Fallback to base class visitor by default
-  return PreRunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
-}
-
-Status NodePass::PreRunOnNode(std::shared_ptr<ZipOp> node, bool *const modified) {
-  // Fallback to base class visitor by default
-  return PreRunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
-}
-
-Status NodePass::PreRunOnNode(std::shared_ptr<MapOp> node, bool *const modified) {
-  // Fallback to base class visitor by default
-  return PreRunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
-}
-
-Status NodePass::PreRunOnNode(std::shared_ptr<ConcatOp> node, bool *const modified) {
-  // Fallback to base class visitor by default
-  return PreRunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
-}
-
-Status NodePass::PreRunOnNode(std::shared_ptr<TakeOp> node, bool *const modified) {
-  // Fallback to base class visitor by default
-  return PreRunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
-}
-
-Status NodePass::PreRunOnNode(std::shared_ptr<SkipOp> node, bool *const modified) {
-  // Fallback to base class visitor by default
-  return PreRunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
-}
-
-Status NodePass::PreRunOnNode(std::shared_ptr<BatchOp> node, bool *const modified) {
-  // Fallback to base class visitor by default
-  return PreRunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
-}
-
-#ifndef ENABLE_ANDROID
-Status NodePass::RunOnNode(std::shared_ptr<MindRecordOp> node, bool *const modified) {
-  // Fallback to base class visitor by default
-  return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
-}
-
-Status NodePass::RunOnNode(std::shared_ptr<TFReaderOp> node, bool *const modified) {
-  // Fallback to base class visitor by default
-  return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
-}
-
-Status NodePass::RunOnNode(std::shared_ptr<CacheOp> node, bool *const modified) {
-  // Fallback to base class visitor by default
-  return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
-}
-
-Status NodePass::RunOnNode(std::shared_ptr<CacheMergeOp> node, bool *const modified) {
-  // Fallback to base class visitor by default
-  return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
-}
-
-Status NodePass::RunOnNode(std::shared_ptr<CacheLookupOp> node, bool *const modified) {
-  // Fallback to base class visitor by default
-  return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
-}
-
-Status NodePass::RunOnNode(std::shared_ptr<ClueOp> node, bool *const modified) {
-  // Fallback to base class visitor by default
-  return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
-}
-
-Status NodePass::RunOnNode(std::shared_ptr<CsvOp> node, bool *const modified) {
-  // Fallback to base class visitor by default
-  return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
-}
-
-Status NodePass::RunOnNode(std::shared_ptr<TextFileOp> node, bool *const modified) {
-  // Fallback to base class visitor by default
-  return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
-}
-
-Status NodePass::PreRunOnNode(std::shared_ptr<CacheOp> node, bool *const modified) {
-  // Fallback to base class visitor by default
-  return PreRunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
-}
-
-Status NodePass::PreRunOnNode(std::shared_ptr<CacheMergeOp> node, bool *const modified) {
-  // Fallback to base class visitor by default
-  return PreRunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
-}
-
-Status NodePass::PreRunOnNode(std::shared_ptr<BuildSentencePieceVocabOp> node, bool *const modified) {
-  // Fallback to base class visitor by default
-  return PreRunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
-}
-#endif
-
-#ifdef ENABLE_PYTHON
-Status NodePass::RunOnNode(std::shared_ptr<FilterOp> node, bool *const modified) {
-  // Fallback to base class visitor by default
-  return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
-}
-
-Status NodePass::RunOnNode(std::shared_ptr<GeneratorOp> node, bool *const modified) {
-  // Fallback to base class visitor by default
-  return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
-}
-
-Status NodePass::RunOnNode(std::shared_ptr<ManifestOp> node, bool *const modified) {
-  // Fallback to base class visitor by default
-  return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
-}
-
-Status NodePass::RunOnNode(std::shared_ptr<VOCOp> node, bool *const modified) {
-  // Fallback to base class visitor by default
-  return RunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
-}
-
-Status NodePass::PreRunOnNode(std::shared_ptr<FilterOp> node, bool *const modified) {
-  // Fallback to base class visitor by default
-  return PreRunOnNode(std::static_pointer_cast<DatasetOp>(node), modified);
-}
-#endif
-//////////////////////////////////
 }  // namespace dataset
 }  // namespace mindspore

mindspore/ccsrc/minddata/dataset/engine/opt/pass.h

@@ -46,6 +46,7 @@ class RootNode;
 class ShuffleNode;
 class SkipNode;
 class TakeNode;
+class TFRecordNode;
 class TransferNode;
 class ZipNode;
 #ifdef ENABLE_PYTHON
@@ -76,47 +77,6 @@ class TextFileNode;
 class TFRecordNode;
 #endif
 
-//////////////////////////////////
-// This section of code will be removed once the migration of optimizer from DatasetOp to DatasetNode is done.
-class BatchOp;
-class MapOp;
-class ProjectOp;
-class RenameOp;
-class SkipOp;
-class ShuffleOp;
-class AlbumOp;
-class RandomDataOp;
-class RepeatOp;
-class TakeOp;
-class ZipOp;
-class DeviceQueueOp;
-class ImageFolderOp;
-class MnistOp;
-class ManifestOp;
-class CifarOp;
-class VOCOp;
-class CocoOp;
-class CelebAOp;
-class EpochCtrlOp;
-class BuildVocabOp;
-class ConcatOp;
-#ifndef ENABLE_ANDROID
-class MindRecordOp;
-class TFReaderOp;
-class CacheOp;
-class CacheMergeOp;
-class CacheLookupOp;
-class BuildSentencePieceVocabOp;
-class ClueOp;
-class CsvOp;
-class TextFileOp;
-#endif
-#ifdef ENABLE_PYTHON
-class FilterOp;
-class GeneratorOp;
-#endif
-//////////////////////////////////
-
 // The base class Pass is the basic unit of tree transformation.
 // The actual implementation of the passes will be derived from here.
 class IRPass : public std::enable_shared_from_this<IRPass> {
@@ -200,6 +160,10 @@ class IRNodePass : public IRPass {
   virtual Status VisitAfter(std::shared_ptr<BatchNode> node, bool *const modified);
   virtual Status Visit(std::shared_ptr<BucketBatchByLengthNode> node, bool *const modified);
   virtual Status VisitAfter(std::shared_ptr<BucketBatchByLengthNode> node, bool *const modified);
+#ifndef ENABLE_ANDROID
+  virtual Status Visit(std::shared_ptr<BuildSentenceVocabNode> node, bool *const modified);
+  virtual Status VisitAfter(std::shared_ptr<BuildSentenceVocabNode> node, bool *const modified);
+#endif
   virtual Status Visit(std::shared_ptr<BuildVocabNode> node, bool *const modified);
   virtual Status VisitAfter(std::shared_ptr<BuildVocabNode> node, bool *const modified);
   virtual Status Visit(std::shared_ptr<ConcatNode> node, bool *const modified);
@@ -240,20 +204,18 @@ class IRNodePass : public IRPass {
   virtual Status VisitAfter(std::shared_ptr<ShuffleNode> node, bool *const modified);
   virtual Status Visit(std::shared_ptr<SkipNode> node, bool *const modified);
   virtual Status VisitAfter(std::shared_ptr<SkipNode> node, bool *const modified);
+#ifdef ENABLE_PYTHON
+  virtual Status Visit(std::shared_ptr<SyncWaitNode> node, bool *const modified);
+  virtual Status VisitAfter(std::shared_ptr<SyncWaitNode> node, bool *const modified);
+#endif
   virtual Status Visit(std::shared_ptr<TakeNode> node, bool *const modified);
   virtual Status VisitAfter(std::shared_ptr<TakeNode> node, bool *const modified);
+  virtual Status Visit(std::shared_ptr<TFRecordNode> node, bool *const modified);
+  virtual Status VisitAfter(std::shared_ptr<TFRecordNode> node, bool *const modified);
   virtual Status Visit(std::shared_ptr<TransferNode> node, bool *const modified);
   virtual Status VisitAfter(std::shared_ptr<TransferNode> node, bool *const modified);
   virtual Status Visit(std::shared_ptr<ZipNode> node, bool *const modified);
   virtual Status VisitAfter(std::shared_ptr<ZipNode> node, bool *const modified);
-#ifdef ENABLE_PYTHON
-  virtual Status Visit(std::shared_ptr<SyncWaitNode> node, bool *const modified);
-  virtual Status VisitAfter(std::shared_ptr<SyncWaitNode> node, bool *const modified);
-#endif
-#ifndef ENABLE_ANDROID
-  virtual Status Visit(std::shared_ptr<BuildSentenceVocabNode> node, bool *const modified);
-  virtual Status VisitAfter(std::shared_ptr<BuildSentenceVocabNode> node, bool *const modified);
-#endif
 
   // leaf-IR Node
   virtual Status Visit(std::shared_ptr<MappableSourceNode> node, bool *const modified);
@@ -269,141 +231,6 @@ class IRNodePass : public IRPass {
   // Tree traversal order of the NodePass
   Order traversalOrder_;
 };
-
-//////////////////////////////////
-// This section of code will be removed once the migration of optimizer from DatasetOp to DatasetNode is done.
-// The base class Pass is the basic unit of tree transformation.
-// The actual implementation of the passes will be derived from here.
-class Pass : public std::enable_shared_from_this<Pass> {
- public:
-  // Run the transformation pass against the execution tree.
-  // @param tree - Pointer to the execution tree to be transformed.
-  // @param modified - Pointer to the modified flag,
-  virtual Status Run(ExecutionTree *tree, bool *const modified) = 0;
-
-  virtual ~Pass() = default;
-};
-
-// TreePass is a basic Pass class which performs transformation on ExecutionTree directly.
-class TreePass : public Pass {
- public:
-  /// \brief Run the transformation pass against the execution tree.
-  /// \param[in,out] tree Pointer to the execution tree to be transformed.
-  /// \param[in,out] modified Indicate if the tree was modified
-  Status Run(ExecutionTree *tree, bool *const modified) final;
-
-  /// \brief Derived classes may implement the runOnTree function to implement tree transformation.
-  ///     "modified" flag needs to be set to true if tree is modified during the pass execution.
-  /// \param[in,out] tree The tree to operate on.
-  /// \param[in,out] Indicate of the tree was modified.
-  /// \return Status The status code returned
-  virtual Status RunOnTree(ExecutionTree *tree, bool *const modified) { return Status::OK(); }
-};
-
-// NodePass is a base Pass class which performs transformation on node visiting.
-// NodePass implements Visitor design pattern.
-// The visiting happens twice for each node in the DFS traversal, one on the way down of the traversal,
-// and the other when all the descending nodes are visited.
-// Actual transformation is done by implementing a new derived class of NodePass.
-// The derived class will implement the method Visit()/VisitAfter() passing specified node types
-// it wants to action on them, overriding the ones defined in NodePass.
-// If the derived class wants to perform the same action on all node types,
-// it can simply implement the method Visit()/VisitAfter() passing the base class DatasetNode.
-// This is made possible by overloading the method Visit()/VisitAfter() on each node type to fall back
-// to call the Visit()/VisitAfter() in this parent NodePass class.
-class NodePass : public Pass {
- public:
-  // Tree traversal order
-  enum Order { DFS, BFS };
-
-  // Constructor
-  // Default DFS traversal
-  explicit NodePass(Order order = Order::DFS) { traversalOrder_ = order; }
-
-  ~NodePass() = default;
-
-  /// \brief Run the transformation pass against the execution tree
-  /// \param[in,out] tree Pointer to the execution tree to be transformed
-  /// \param[in,out] modified Indicator if the tree was changed
-  Status Run(ExecutionTree *tree, bool *const modified) final;
-
-  /// \brief Derived classes may implement the PreRunOnNode function to implement any initial visit work on the way down
-  ///     a tree traversal.  "modified" flag needs to be set to true if tree is modified during the pass execution
-  /// \param[in] node The node being visited
-  /// \param[out] modified Indicator if the node was changed at all
-  /// \return Status The status code returned
-  virtual Status PreRunOnNode(std::shared_ptr<DatasetOp> node, bool *const modified) { return Status::OK(); }
-
-  /// \brief Derived classes may implement the RunOnNode function to implement node level tree transformation
-  ///     "modified" flag needs to be set to true if tree is modified during the pass execution
-  /// \param[in] node The node being visited
-  /// \param[out] modified Indicator if the node was changed at all.
-  /// \return Status The status code returned
-  virtual Status RunOnNode(std::shared_ptr<DatasetOp> node, bool *const modified) { return Status::OK(); }
-
-  // Visit methods to be overridden.
-  // Note that member template can not be virtual, any op which wants to work with NodePass should declare RunOnNode
-  // of its own type and override "Accept" from DatasetOp.
-  virtual Status RunOnNode(std::shared_ptr<BatchOp> node, bool *const modified);
-  virtual Status RunOnNode(std::shared_ptr<MapOp> node, bool *const modified);
-  virtual Status RunOnNode(std::shared_ptr<ProjectOp> node, bool *const modified);
-  virtual Status RunOnNode(std::shared_ptr<RenameOp> node, bool *const modified);
-  virtual Status RunOnNode(std::shared_ptr<SkipOp> node, bool *const modified);
-  virtual Status RunOnNode(std::shared_ptr<ShuffleOp> node, bool *const modified);
-  virtual Status RunOnNode(std::shared_ptr<RandomDataOp> node, bool *const modified);
-  virtual Status RunOnNode(std::shared_ptr<AlbumOp> node, bool *const modified);
-  virtual Status RunOnNode(std::shared_ptr<TakeOp> node, bool *const modified);
-  virtual Status RunOnNode(std::shared_ptr<ZipOp> node, bool *const modified);
-  virtual Status RunOnNode(std::shared_ptr<DeviceQueueOp> node, bool *const modified);
-  virtual Status RunOnNode(std::shared_ptr<ImageFolderOp> node, bool *const modified);
-  virtual Status RunOnNode(std::shared_ptr<MnistOp> node, bool *const modified);
-  virtual Status RunOnNode(std::shared_ptr<CifarOp> node, bool *const modified);
-  virtual Status RunOnNode(std::shared_ptr<CocoOp> node, bool *const modified);
-  virtual Status RunOnNode(std::shared_ptr<CelebAOp> node, bool *const modified);
-  virtual Status RunOnNode(std::shared_ptr<RepeatOp> node, bool *const modified);
-  virtual Status RunOnNode(std::shared_ptr<EpochCtrlOp> node, bool *const modified);
-  virtual Status PreRunOnNode(std::shared_ptr<RepeatOp> node, bool *const modified);
-  virtual Status PreRunOnNode(std::shared_ptr<EpochCtrlOp> node, bool *const modified);
-  virtual Status PreRunOnNode(std::shared_ptr<BuildVocabOp> node, bool *const modified);
-  virtual Status PreRunOnNode(std::shared_ptr<ZipOp> node, bool *const modified);
-  virtual Status PreRunOnNode(std::shared_ptr<MapOp> node, bool *const modified);
-  virtual Status PreRunOnNode(std::shared_ptr<ConcatOp> node, bool *const modified);
-  virtual Status PreRunOnNode(std::shared_ptr<TakeOp> node, bool *const modified);
-  virtual Status PreRunOnNode(std::shared_ptr<SkipOp> node, bool *const modified);
-  virtual Status PreRunOnNode(std::shared_ptr<BatchOp> node, bool *const modified);
-#ifndef ENABLE_ANDROID
-  virtual Status RunOnNode(std::shared_ptr<MindRecordOp> node, bool *const modified);
-  virtual Status RunOnNode(std::shared_ptr<TFReaderOp> node, bool *const modified);
-  virtual Status RunOnNode(std::shared_ptr<CacheMergeOp> node, bool *const modified);
-  virtual Status RunOnNode(std::shared_ptr<CacheLookupOp> node, bool *const modified);
-  virtual Status RunOnNode(std::shared_ptr<CacheOp> node, bool *const modified);
-  virtual Status RunOnNode(std::shared_ptr<ClueOp> node, bool *const modified);
-  virtual Status RunOnNode(std::shared_ptr<CsvOp> node, bool *const modified);
-  virtual Status RunOnNode(std::shared_ptr<TextFileOp> node, bool *const modified);
-  virtual Status PreRunOnNode(std::shared_ptr<CacheOp> node, bool *const modified);
-  virtual Status PreRunOnNode(std::shared_ptr<CacheMergeOp> node, bool *const modified);
-  virtual Status PreRunOnNode(std::shared_ptr<BuildSentencePieceVocabOp> node, bool *const modified);
-#endif
-#ifdef ENABLE_PYTHON
-  virtual Status RunOnNode(std::shared_ptr<FilterOp> node, bool *const modified);
-  virtual Status RunOnNode(std::shared_ptr<ManifestOp> node, bool *const modified);
-  virtual Status RunOnNode(std::shared_ptr<GeneratorOp> node, bool *const modified);
-  virtual Status RunOnNode(std::shared_ptr<VOCOp> node, bool *const modified);
-  virtual Status PreRunOnNode(std::shared_ptr<FilterOp> node, bool *const modified);
-#endif
-
- private:
-  // Helper function to perform DFS visit
-  Status DFSNodeVisit(std::shared_ptr<DatasetOp> node, bool *const modified);
-
-  // Helper function to perform BFS visit
-  Status BFSNodeVisit(std::shared_ptr<DatasetOp> root, bool *const modified);
-
-  // Tree traversal order of the NodePass
-  Order traversalOrder_;
-};
-//////////////////////////////////
-
 }  // namespace dataset
 }  // namespace mindspore
 

mindspore/ccsrc/minddata/dataset/engine/opt/pre/cache_validation_pass.cc

@@ -23,6 +23,7 @@
 #include "minddata/dataset/engine/ir/datasetops/map_node.h"
 #include "minddata/dataset/engine/ir/datasetops/repeat_node.h"
 #include "minddata/dataset/engine/ir/datasetops/skip_node.h"
+#include "minddata/dataset/engine/ir/datasetops/source/tf_record_node.h"
 #include "minddata/dataset/engine/ir/datasetops/take_node.h"
 #include "minddata/dataset/engine/ir/datasetops/zip_node.h"
 
@@ -157,6 +158,22 @@ Status CacheValidationPass::VisitAfter(std::shared_ptr<RepeatNode> node, bool *c
   return Status::OK();
 }
 
+Status CacheValidationPass::VisitAfter(std::shared_ptr<TFRecordNode> node, bool *const modified) {
+  MS_LOG(DEBUG) << "CacheValidationPass::VisitAfter(<TFRecordNode>): visiting " << node->Name() << ".";
+  if (!is_cached_) {
+    // If we are not in a cache path, then we must validate the file-based sharding config.
+    // If we are in a cache path, there is no file-based sharding so the check is not required.
+    if (!node->shard_equal_rows_ && node->dataset_files_.size() < static_cast<uint32_t>(node->num_shards_)) {
+      RETURN_STATUS_UNEXPECTED("Invalid file, not enough tfrecord files provided.\n");
+    }
+  }
+  // Reset the flag when this node is cached and is already visited
+  if (node->IsCached()) {
+    is_cached_ = false;
+  }
+  return Status::OK();
+}
+
 Status CacheValidationPass::VisitAfter(std::shared_ptr<DatasetNode> node, bool *const modified) {
   MS_LOG(DEBUG) << "CacheValidationPass::VisitAfter(<DatasetNode>): visiting " << node->Name() << ".";
   // Reset the flag when all descendants are visited

mindspore/ccsrc/minddata/dataset/engine/opt/pre/cache_validation_pass.h

@@ -37,61 +37,67 @@ class CacheValidationPass : public IRNodePass {
 
   /// \brief Returns an error if BatchNode exists under a cache
   /// \param[in] node The node being visited
-  /// \param[inout] modified Indicator if the node was changed at all
+  /// \param[in,out] modified Indicator if the node was changed at all
   /// \return Status The status code returned
   Status Visit(std::shared_ptr<BatchNode> node, bool *const modified) override;
 
   /// \brief Returns an error if ConcatNode exists under a cache
   /// \param[in] node The node being visited
-  /// \param[inout] modified Indicator if the node was changed at all
+  /// \param[in,out] modified Indicator if the node was changed at all
   /// \return Status The status code returned
   Status Visit(std::shared_ptr<ConcatNode> node, bool *const modified) override;
 
   /// \brief Returns an error if FilterNode exists under a cache
   /// \param[in] node The node being visited
-  /// \param[inout] modified Indicator if the node was changed at all
+  /// \param[in,out] modified Indicator if the node was changed at all
   /// \return Status The status code returned
   Status Visit(std::shared_ptr<FilterNode> node, bool *const modified) override;
 
   /// \brief Returns an error if SkipNode exists under a cache
   /// \param[in] node The node being visited
-  /// \param[inout] modified Indicator if the node was changed at all
+  /// \param[in,out] modified Indicator if the node was changed at all
   /// \return Status The status code returned
   Status Visit(std::shared_ptr<SkipNode> node, bool *const modified) override;
 
   /// \brief Returns an error if TakeNode exists under a cache
   /// \param[in] node The node being visited
-  /// \param[inout] modified Indicator if the node was changed at all
+  /// \param[in,out] modified Indicator if the node was changed at all
   /// \return Status The status code returned
   Status Visit(std::shared_ptr<TakeNode> node, bool *const modified) override;
 
   /// \brief Returns an error if ZipNode exists under a cache
   /// \param[in] node The node being visited
-  /// \param[inout] modified Indicator if the node was changed at all
+  /// \param[in,out] modified Indicator if the node was changed at all
   /// \return Status The status code returned
   Status Visit(std::shared_ptr<ZipNode> node, bool *const modified) override;
 
   /// \brief Returns an error if MapNode with non-deterministic tensor operations exists under a cache
   /// \param[in] node The node being visited
-  /// \param[inout] modified Indicator if the node was changed at all
+  /// \param[in,out] modified Indicator if the node was changed at all
   /// \return Status The status code returned
   Status Visit(std::shared_ptr<MapNode> node, bool *const modified) override;
 
   /// \brief Returns an error if there is a cache over another cache
   /// \param[in] node The node being visited
-  /// \param[inout] modified Indicator if the node was changed at all
+  /// \param[in,out] modified Indicator if the node was changed at all
   /// \return Status The status code returned
   Status Visit(std::shared_ptr<DatasetNode> node, bool *const modified) override;
 
   /// \brief Identifies and block repeat under cache scenarios
   /// \param[in] node The node being visited
-  /// \param[inout] modified Indicator if the node was changed at all
+  /// \param[in,out] modified Indicator if the node was changed at all
   /// \return Status The status code returned
   Status VisitAfter(std::shared_ptr<RepeatNode> node, bool *const modified) override;
 
   /// \brief Identifies the subtree above this node as not being cached
   /// \param[in] node The node being visited
-  /// \param[inout] modified Indicator if the node was changed at all
+  /// \param[in,out] modified Indicator if the node was changed at all
+  /// \return Status The status code returned
+  Status VisitAfter(std::shared_ptr<TFRecordNode> node, bool *const modified) override;
+
+  /// \brief Identifies the subtree above this node as not being cached
+  /// \param[in] node The node being visited
+  /// \param[in,out] modified Indicator if the node was changed at all
   /// \return Status The status code returned
   Status VisitAfter(std::shared_ptr<DatasetNode> node, bool *const modified) override;