zzy34407230
/
mindspore2022
Not watched
1
0
Fork 0
Code
Releases 22 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 693b4cfdc8
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '693b4cfdc8'
${ noResults }
mindspore2022/mindspore/ccsrc/minddata/dataset/engine/opt/pass.cc

297 lines
13 kB
Raw Blame History 

  1. /**

  2.  * Copyright 2020-2021 Huawei Technologies Co., Ltd

  3.  *

  4.  * Licensed under the Apache License, Version 2.0 (the "License");

  5.  * you may not use this file except in compliance with the License.

  6.  * You may obtain a copy of the License at

  7.  *

  8.  * http://www.apache.org/licenses/LICENSE-2.0

  9.  *

  10.  * Unless required by applicable law or agreed to in writing, software

  11.  * distributed under the License is distributed on an "AS IS" BASIS,

  12.  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

  13.  * See the License for the specific language governing permissions and

  14.  * limitations under the License.

  15.  */

  16. 

  17. #include "minddata/dataset/engine/opt/pass.h"

  18. #include "minddata/dataset/engine/ir/datasetops/batch_node.h"

  19. #include "minddata/dataset/engine/ir/datasetops/bucket_batch_by_length_node.h"

  20. #ifndef ENABLE_ANDROID

  21. #include "minddata/dataset/engine/ir/datasetops/build_sentence_piece_vocab_node.h"

  22. #endif

  23. #include "minddata/dataset/engine/ir/datasetops/build_vocab_node.h"

  24. #ifndef ENABLE_ANDROID

  25. #include "minddata/dataset/engine/ir/datasetops/cache_node.h"

  26. #include "minddata/dataset/engine/ir/datasetops/cache_merge_node.h"

  27. #include "minddata/dataset/engine/ir/datasetops/cache_lookup_node.h"

  28. #endif

  29. #include "minddata/dataset/engine/ir/datasetops/concat_node.h"

  30. #include "minddata/dataset/engine/ir/datasetops/epoch_ctrl_node.h"

  31. #include "minddata/dataset/engine/ir/datasetops/filter_node.h"

  32. #include "minddata/dataset/engine/ir/datasetops/map_node.h"

  33. #include "minddata/dataset/engine/ir/datasetops/project_node.h"

  34. #include "minddata/dataset/engine/ir/datasetops/rename_node.h"

  35. #include "minddata/dataset/engine/ir/datasetops/repeat_node.h"

  36. #include "minddata/dataset/engine/ir/datasetops/root_node.h"

  37. #include "minddata/dataset/engine/ir/datasetops/shuffle_node.h"

  38. #include "minddata/dataset/engine/ir/datasetops/skip_node.h"

  39. #ifndef ENABLE_ANDROID

  40. #include "minddata/dataset/engine/ir/datasetops/source/minddata_node.h"

  41. #endif

  42. #ifdef ENABLE_PYTHON

  43. #include "minddata/dataset/engine/ir/datasetops/source/generator_node.h"

  44. #endif

  45. #include "minddata/dataset/engine/ir/datasetops/source/random_node.h"

  46. #include "minddata/dataset/engine/ir/datasetops/source/tf_record_node.h"

  47. #ifdef ENABLE_PYTHON

  48. #include "minddata/dataset/engine/ir/datasetops/sync_wait_node.h"

  49. #endif

  50. #include "minddata/dataset/engine/ir/datasetops/take_node.h"

  51. #include "minddata/dataset/engine/ir/datasetops/transfer_node.h"

  52. #include "minddata/dataset/engine/ir/datasetops/zip_node.h"

  53. 

  54. namespace mindspore {

  55. namespace dataset {

  56. 

  57. // Driver method for TreePass

  58. Status IRTreePass::Run(std::shared_ptr<DatasetNode> root_ir, bool *const modified) {

  59.   if (root_ir == nullptr || modified == nullptr) {

  60.     return Status(StatusCode::kMDUnexpectedError, "Null pointer passed to TreePass");

  61.   }

  62.   // Initialize modified flag

  63.   *modified = false;

  64.   return this->RunOnTree(root_ir, modified);

  65. }

  66. 

  67. // Driver method for NodePass

  68. Status IRNodePass::Run(std::shared_ptr<DatasetNode> root_ir, bool *const modified) {

  69.   if (root_ir == nullptr || modified == nullptr) {

  70.     return Status(StatusCode::kMDUnexpectedError, "Null pointer passed to NodePass");

  71.   }

  72.   // Initialize modified flag

  73.   *modified = false;

  74.   if (traversalOrder_ == Order::DFS) {

  75.     // DFS

  76.     return DFSNodeVisit(root_ir, modified);

  77.   } else if (traversalOrder_ == Order::BFS) {

  78.     // BFS

  79.     return BFSNodeVisit(root_ir, modified);

  80.   }

  81.   return Status::OK();

  82. }

  83. 

  84. // Helper function to perform DFS visit

  85. Status IRNodePass::DFSNodeVisit(std::shared_ptr<DatasetNode> node_ir, bool *const modified) {

  86.   bool m = false;

  87. 

  88.   RETURN_IF_NOT_OK(node_ir->Accept(this, &m));

  89.   *modified = *modified || m;

  90.   for (const auto &c : node_ir->Children()) {

  91.     RETURN_IF_NOT_OK(this->DFSNodeVisit(c, &m));

  92.     *modified = *modified || m;

  93.   }

  94.   RETURN_IF_NOT_OK(node_ir->AcceptAfter(this, &m));

  95.   *modified = *modified || m;

  96.   return Status::OK();

  97. }

  98. 

  99. // Helper function to perform BFS visit

  100. Status IRNodePass::BFSNodeVisit(std::shared_ptr<DatasetNode> node_ir, bool *const modified) {

  101.   bool m = false;

  102. 

  103.   // Initialize bfs queue with root

  104.   std::queue<std::shared_ptr<DatasetNode>> bfsQueue;

  105.   bfsQueue.push(node_ir);

  106. 

  107.   // BFS loop

  108.   while (!bfsQueue.empty()) {

  109.     // Pop the front of the bfs queue

  110.     auto curNode = bfsQueue.front();

  111.     bfsQueue.pop();

  112. 

  113.     // Run node pass

  114.     RETURN_IF_NOT_OK(curNode->Accept(this, &m));

  115.     *modified = *modified || m;

  116. 

  117.     // Push children into bfs queue

  118.     for (const auto &c : curNode->Children()) {

  119.       bfsQueue.push(c);

  120.     }

  121.   }

  122.   return Status::OK();

  123. }

  124. 

  125. // For non-leaf IR node

  126. Status IRNodePass::Visit(std::shared_ptr<BatchNode> node, bool *const modified) {

  127.   return Visit(std::static_pointer_cast<DatasetNode>(node), modified);

  128. }

  129. Status IRNodePass::VisitAfter(std::shared_ptr<BatchNode> node, bool *const modified) {

  130.   return VisitAfter(std::static_pointer_cast<DatasetNode>(node), modified);

  131. }

  132. Status IRNodePass::Visit(std::shared_ptr<BucketBatchByLengthNode> node, bool *const modified) {

  133.   return Visit(std::static_pointer_cast<DatasetNode>(node), modified);

  134. }

  135. Status IRNodePass::VisitAfter(std::shared_ptr<BucketBatchByLengthNode> node, bool *const modified) {

  136.   return VisitAfter(std::static_pointer_cast<DatasetNode>(node), modified);

  137. }

  138. Status IRNodePass::Visit(std::shared_ptr<BuildVocabNode> node, bool *const modified) {

  139.   return Visit(std::static_pointer_cast<DatasetNode>(node), modified);

  140. }

  141. Status IRNodePass::VisitAfter(std::shared_ptr<BuildVocabNode> node, bool *const modified) {

  142.   return VisitAfter(std::static_pointer_cast<DatasetNode>(node), modified);

  143. }

  144. Status IRNodePass::Visit(std::shared_ptr<ConcatNode> node, bool *const modified) {

  145.   return Visit(std::static_pointer_cast<DatasetNode>(node), modified);

  146. }

  147. Status IRNodePass::VisitAfter(std::shared_ptr<ConcatNode> node, bool *const modified) {

  148.   return VisitAfter(std::static_pointer_cast<DatasetNode>(node), modified);

  149. }

  150. #ifndef ENABLE_ANDROID

  151. Status IRNodePass::Visit(std::shared_ptr<CacheLookupNode> node, bool *const modified) {

  152.   return Visit(std::static_pointer_cast<DatasetNode>(node), modified);

  153. }

  154. Status IRNodePass::VisitAfter(std::shared_ptr<CacheLookupNode> node, bool *const modified) {

  155.   return VisitAfter(std::static_pointer_cast<DatasetNode>(node), modified);

  156. }

  157. Status IRNodePass::Visit(std::shared_ptr<CacheMergeNode> node, bool *const modified) {

  158.   return Visit(std::static_pointer_cast<DatasetNode>(node), modified);

  159. }

  160. Status IRNodePass::VisitAfter(std::shared_ptr<CacheMergeNode> node, bool *const modified) {

  161.   return VisitAfter(std::static_pointer_cast<DatasetNode>(node), modified);

  162. }

  163. Status IRNodePass::Visit(std::shared_ptr<CacheNode> node, bool *const modified) {

  164.   return Visit(std::static_pointer_cast<DatasetNode>(node), modified);

  165. }

  166. Status IRNodePass::VisitAfter(std::shared_ptr<CacheNode> node, bool *const modified) {

  167.   return VisitAfter(std::static_pointer_cast<DatasetNode>(node), modified);

  168. }

  169. #endif

  170. Status IRNodePass::Visit(std::shared_ptr<EpochCtrlNode> node, bool *const modified) {

  171.   return Visit(std::static_pointer_cast<DatasetNode>(node), modified);

  172. }

  173. Status IRNodePass::VisitAfter(std::shared_ptr<EpochCtrlNode> node, bool *const modified) {

  174.   return VisitAfter(std::static_pointer_cast<DatasetNode>(node), modified);

  175. }

  176. Status IRNodePass::Visit(std::shared_ptr<FilterNode> node, bool *const modified) {

  177.   return Visit(std::static_pointer_cast<DatasetNode>(node), modified);

  178. }

  179. Status IRNodePass::VisitAfter(std::shared_ptr<FilterNode> node, bool *const modified) {

  180.   return VisitAfter(std::static_pointer_cast<DatasetNode>(node), modified);

  181. }

  182. #ifdef ENABLE_PYTHON

  183. Status IRNodePass::Visit(std::shared_ptr<GeneratorNode> node, bool *const modified) {

  184.   return Visit(std::static_pointer_cast<MappableSourceNode>(node), modified);

  185. }

  186. Status IRNodePass::VisitAfter(std::shared_ptr<GeneratorNode> node, bool *const modified) {

  187.   return VisitAfter(std::static_pointer_cast<MappableSourceNode>(node), modified);

  188. }

  189. #endif

  190. Status IRNodePass::Visit(std::shared_ptr<MapNode> node, bool *const modified) {

  191.   return Visit(std::static_pointer_cast<DatasetNode>(node), modified);

  192. }

  193. Status IRNodePass::VisitAfter(std::shared_ptr<MapNode> node, bool *const modified) {

  194.   return VisitAfter(std::static_pointer_cast<DatasetNode>(node), modified);

  195. }

  196. #ifndef ENABLE_ANDROID

  197. Status IRNodePass::Visit(std::shared_ptr<MindDataNode> node, bool *const modified) {

  198.   return Visit(std::static_pointer_cast<MappableSourceNode>(node), modified);

  199. }

  200. Status IRNodePass::VisitAfter(std::shared_ptr<MindDataNode> node, bool *const modified) {

  201.   return VisitAfter(std::static_pointer_cast<MappableSourceNode>(node), modified);

  202. }

  203. #endif

  204. Status IRNodePass::Visit(std::shared_ptr<ProjectNode> node, bool *const modified) {

  205.   return Visit(std::static_pointer_cast<DatasetNode>(node), modified);

  206. }

  207. Status IRNodePass::VisitAfter(std::shared_ptr<ProjectNode> node, bool *const modified) {

  208.   return VisitAfter(std::static_pointer_cast<DatasetNode>(node), modified);

  209. }

  210. Status IRNodePass::Visit(std::shared_ptr<RandomNode> node, bool *const modified) {

  211.   return Visit(std::static_pointer_cast<NonMappableSourceNode>(node), modified);

  212. }

  213. Status IRNodePass::VisitAfter(std::shared_ptr<RandomNode> node, bool *const modified) {

  214.   return VisitAfter(std::static_pointer_cast<NonMappableSourceNode>(node), modified);

  215. }

  216. Status IRNodePass::Visit(std::shared_ptr<RenameNode> node, bool *const modified) {

  217.   return Visit(std::static_pointer_cast<DatasetNode>(node), modified);

  218. }

  219. Status IRNodePass::VisitAfter(std::shared_ptr<RenameNode> node, bool *const modified) {

  220.   return VisitAfter(std::static_pointer_cast<DatasetNode>(node), modified);

  221. }

  222. Status IRNodePass::Visit(std::shared_ptr<RepeatNode> node, bool *const modified) {

  223.   return Visit(std::static_pointer_cast<DatasetNode>(node), modified);

  224. }

  225. Status IRNodePass::VisitAfter(std::shared_ptr<RepeatNode> node, bool *const modified) {

  226.   return VisitAfter(std::static_pointer_cast<DatasetNode>(node), modified);

  227. }

  228. Status IRNodePass::Visit(std::shared_ptr<RootNode> node, bool *const modified) {

  229.   return Visit(std::static_pointer_cast<DatasetNode>(node), modified);

  230. }

  231. Status IRNodePass::VisitAfter(std::shared_ptr<RootNode> node, bool *const modified) {

  232.   return VisitAfter(std::static_pointer_cast<DatasetNode>(node), modified);

  233. }

  234. Status IRNodePass::Visit(std::shared_ptr<ShuffleNode> node, bool *const modified) {

  235.   return Visit(std::static_pointer_cast<DatasetNode>(node), modified);

  236. }

  237. Status IRNodePass::VisitAfter(std::shared_ptr<ShuffleNode> node, bool *const modified) {

  238.   return VisitAfter(std::static_pointer_cast<DatasetNode>(node), modified);

  239. }

  240. Status IRNodePass::Visit(std::shared_ptr<SkipNode> node, bool *const modified) {

  241.   return Visit(std::static_pointer_cast<DatasetNode>(node), modified);

  242. }

  243. Status IRNodePass::VisitAfter(std::shared_ptr<SkipNode> node, bool *const modified) {

  244.   return VisitAfter(std::static_pointer_cast<DatasetNode>(node), modified);

  245. }

  246. Status IRNodePass::Visit(std::shared_ptr<TakeNode> node, bool *const modified) {

  247.   return Visit(std::static_pointer_cast<DatasetNode>(node), modified);

  248. }

  249. Status IRNodePass::VisitAfter(std::shared_ptr<TakeNode> node, bool *const modified) {

  250.   return VisitAfter(std::static_pointer_cast<DatasetNode>(node), modified);

  251. }

  252. Status IRNodePass::Visit(std::shared_ptr<TFRecordNode> node, bool *const modified) {

  253.   return Visit(std::static_pointer_cast<NonMappableSourceNode>(node), modified);

  254. }

  255. Status IRNodePass::VisitAfter(std::shared_ptr<TFRecordNode> node, bool *const modified) {

  256.   return VisitAfter(std::static_pointer_cast<NonMappableSourceNode>(node), modified);

  257. }

  258. Status IRNodePass::Visit(std::shared_ptr<TransferNode> node, bool *const modified) {

  259.   return Visit(std::static_pointer_cast<DatasetNode>(node), modified);

  260. }

  261. Status IRNodePass::VisitAfter(std::shared_ptr<TransferNode> node, bool *const modified) {

  262.   return VisitAfter(std::static_pointer_cast<DatasetNode>(node), modified);

  263. }

  264. Status IRNodePass::Visit(std::shared_ptr<ZipNode> node, bool *const modified) {

  265.   return Visit(std::static_pointer_cast<DatasetNode>(node), modified);

  266. }

  267. Status IRNodePass::VisitAfter(std::shared_ptr<ZipNode> node, bool *const modified) {

  268.   return VisitAfter(std::static_pointer_cast<DatasetNode>(node), modified);

  269. }

  270. #ifdef ENABLE_PYTHON

  271. Status IRNodePass::Visit(std::shared_ptr<SyncWaitNode> node, bool *const modified) {

  272.   return Visit(std::static_pointer_cast<DatasetNode>(node), modified);

  273. }

  274. Status IRNodePass::VisitAfter(std::shared_ptr<SyncWaitNode> node, bool *const modified) {

  275.   return VisitAfter(std::static_pointer_cast<DatasetNode>(node), modified);

  276. }

  277. #endif

  278. #ifndef ENABLE_ANDROID

  279. Status IRNodePass::Visit(std::shared_ptr<BuildSentenceVocabNode> node, bool *const modified) {

  280.   return Visit(std::static_pointer_cast<DatasetNode>(node), modified);

  281. }

  282. Status IRNodePass::VisitAfter(std::shared_ptr<BuildSentenceVocabNode> node, bool *const modified) {

  283.   return VisitAfter(std::static_pointer_cast<DatasetNode>(node), modified);

  284. }

  285. #endif

  286. 

  287. // leaf-IR Node

  288. Status IRNodePass::Visit(std::shared_ptr<MappableSourceNode> node, bool *const modified) {

  289.   return Visit(std::static_pointer_cast<DatasetNode>(node), modified);

  290. }

  291. 

  292. Status IRNodePass::Visit(std::shared_ptr<NonMappableSourceNode> node, bool *const modified) {

  293.   return Visit(std::static_pointer_cast<DatasetNode>(node), modified);

  294. }

  295. }  // namespace dataset

  296. }  // namespace mindspore