Browse Source

fix cache op core dump

tags/v0.7.0-beta
qianlong 5 years ago
parent
commit
bf87b1d1fb
2 changed files with 36 additions and 6 deletions
  1. +3
    -0
      mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.cc
  2. +33
    -6
      tests/ut/python/dataset/test_cache_map.py

+ 3
- 0
mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.cc View File

@@ -392,6 +392,9 @@ uint32_t DatasetOp::GenerateCRC(const std::shared_ptr<DatasetOp> &op) {
ss_str = std::regex_replace(ss_str, std::regex("Num workers.*\n"), "");
ss_str = std::regex_replace(ss_str, std::regex("\\[workers.*\\]"), "");

// Filter out Number of rows when generating the check sum
ss_str = std::regex_replace(ss_str, std::regex("Number of rows.*\n"), "");

// Filter out the Operator control flags field when generating the check sum
ss_str = std::regex_replace(ss_str, std::regex("Operator control flags.*\n"), "");



+ 33
- 6
tests/ut/python/dataset/test_cache_map.py View File

@@ -104,11 +104,11 @@ def test_cache_map_basic3():
decode_op = c_vision.Decode()
ds1 = ds1.repeat(4)
ds1 = ds1.map(input_columns=["image"], operations=decode_op, cache=some_cache)
print("ds1.dataset_size is ", ds1.get_dataset_size())
logger.info("ds1.dataset_size is ", ds1.get_dataset_size())

num_iter = 0
for _ in ds1.create_dict_iterator():
print("get data from dataset")
logger.info("get data from dataset")
num_iter += 1

logger.info("Number of data in ds1: {} ".format(num_iter))
@@ -116,6 +116,31 @@ def test_cache_map_basic3():
logger.info('test_cache_basic3 Ended.\n')


def test_cache_map_basic4():
"""
Test different rows result in core dump
"""
logger.info("Test cache basic 4")
some_cache = ds.DatasetCache(session_id=1, size=0, spilling=True)

# This DATA_DIR only has 2 images in it
ds1 = ds.ImageFolderDatasetV2(dataset_dir=DATA_DIR, cache=some_cache)
decode_op = c_vision.Decode()
ds1 = ds1.repeat(4)
ds1 = ds1.map(input_columns=["image"], operations=decode_op)
logger.info("ds1.dataset_size is ", ds1.get_dataset_size())
shape = ds1.output_shapes()

num_iter = 0
for _ in ds1.create_dict_iterator():
logger.info("get data from dataset")
num_iter += 1

logger.info("Number of data in ds1: {} ".format(num_iter))
assert num_iter == 8
logger.info('test_cache_basic3 Ended.\n')


def test_cache_map_failure1():
"""
Test nested cache (failure)
@@ -154,10 +179,12 @@ def test_cache_map_failure1():

if __name__ == '__main__':
test_cache_map_basic1()
print("test_cache_map_basic1 success.")
logger.info("test_cache_map_basic1 success.")
test_cache_map_basic2()
print("test_cache_map_basic2 success.")
logger.info("test_cache_map_basic2 success.")
test_cache_map_basic3()
print("test_cache_map_basic3 success.")
logger.info("test_cache_map_basic3 success.")
test_cache_map_basic4()
logger.info("test_cache_map_basic3 success.")
test_cache_map_failure1()
print("test_cache_map_failure1 success.")
logger.info("test_cache_map_failure1 success.")

Loading…
Cancel
Save