Browse Source

[MD] Tidy up UT and device queue info messages

feature/build-system-rewrite
Cathy Wong 4 years ago
parent
commit
fb469f1ad2
4 changed files with 15 additions and 12 deletions
  1. +5
    -5
      mindspore/ccsrc/minddata/dataset/engine/datasetops/device_queue_op.cc
  2. +5
    -3
      tests/ut/python/dataset/test_batch.py
  3. +2
    -1
      tests/ut/python/dataset/test_datasets_textfileop.py
  4. +3
    -3
      tests/ut/python/dataset/test_sync_wait.py

+ 5
- 5
mindspore/ccsrc/minddata/dataset/engine/datasetops/device_queue_op.cc View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2021 Huawei Technologies Co., Ltd
* Copyright 2019-2022 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -285,7 +285,7 @@ Status DeviceQueueOp::SendDataToAscend() {
send_finished_ = true;
}
tree_->SetFinished();
MS_LOG(INFO) << "Device queue send " << send_batch << " batch.";
MS_LOG(INFO) << "ExecutionTree finished. Device queue sent number of batches: " << send_batch;

return Status::OK();
}
@@ -505,7 +505,7 @@ Status DeviceQueueOp::PushDataToGPU() {
send_finished_ = true;
}
tree_->SetFinished();
MS_LOG(INFO) << "Device queue send " << send_batch << " batch.";
MS_LOG(INFO) << "ExecutionTree finished. Device queue pushed number of batches: " << send_batch;

GpuBufferMgr::GetInstance().Close(handle);
GpuBufferMgr::GetInstance().CloseConfirm();
@@ -581,7 +581,7 @@ Status DeviceQueueOp::WorkerEntry(int32_t worker_id) {
RETURN_IF_NOT_OK(receive_queues_[worker_id]->PopFront(&current_row));
}

MS_LOG(INFO) << "Device queue worker id " << worker_id << "proc " << batch_num << "batch.";
MS_LOG(INFO) << "Device queue worker id " << worker_id << " processed number of batches: " << batch_num;
// Add empty data_item vector with eoe_flag=false as quit flag.
GpuConnectorItem connector_item = {{}, false};
RETURN_IF_NOT_OK(gpu_connector_->Add(worker_id, std::move(connector_item)));
@@ -641,7 +641,7 @@ Status DeviceQueueOp::SendDataToGPU() {
RETURN_IF_NOT_OK(receive_queues_[num_buf++ % num_workers_]->Add(std::move(quit_flag)));
}

MS_LOG(INFO) << "Device queue receive " << num_buf - num_workers_ << " batch.";
MS_LOG(INFO) << "Device queue received number of batches and EOEs: " << (num_buf - num_workers_);
return Status::OK();
}



+ 5
- 3
tests/ut/python/dataset/test_batch.py View File

@@ -1,4 +1,4 @@
# Copyright 2019 Huawei Technologies Co., Ltd
# Copyright 2019-2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -240,9 +240,11 @@ def test_batch_12():

def test_batch_13():
"""
Test batch: python_multiprocessing is True and does not work for per_batch_map is None
Feature: Batch op
Description: Test python_multiprocessing is True with per_batch_map is None
Expectation: python_multiprocessing is True is ignored when per_batch_map is None
"""
logger.info("test_batch_12")
logger.info("test_batch_13")
# define parameters
batch_size = True



+ 2
- 1
tests/ut/python/dataset/test_datasets_textfileop.py View File

@@ -1,4 +1,4 @@
# Copyright 2020 Huawei Technologies Co., Ltd
# Copyright 2020-2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -268,6 +268,7 @@ if __name__ == "__main__":
test_textline_dataset_num_samples()
test_textline_dataset_distribution()
test_textline_dataset_repeat()
test_textline_dataset_output_tensor()
test_textline_dataset_get_datasetsize()
test_textline_dataset_to_device()
test_textline_dataset_exceptions()

+ 3
- 3
tests/ut/python/dataset/test_sync_wait.py View File

@@ -1,4 +1,5 @@
# Copyright 2019 Huawei Technologies Co., Ltd
# Copyright 2019i-2022 Huawei Technologies Co., Ltd

#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -232,14 +233,13 @@ def test_sync_exception_04():

def test_sync_exception_05():
"""
Test sync: with wrong batch size in update
Test sync: with wrong condition name in update
"""
logger.info("test_sync_exception_05")

dataset = ds.GeneratorDataset(gen, column_names=["input"])
count = 0
aug = Augment(0)
# try to create dataset with batch_size < 0
dataset = dataset.sync_wait(condition_name="every batch", callback=aug.update)
dataset = dataset.map(operations=[aug.preprocess], input_columns=["input"])
with pytest.raises(RuntimeError) as e:


Loading…
Cancel
Save