From 7d79376bece088cf2dd7f8e541c39f60d8009e36 Mon Sep 17 00:00:00 2001 From: jiangshuqiang <962978787@qq.com> Date: Thu, 14 Jan 2021 14:52:14 +0800 Subject: [PATCH] remove max_file_size limitation for export files --- mindspore/train/callback/_summary_collector.py | 1 + mindspore/train/summary/_summary_adapter.py | 7 ++----- mindspore/train/summary/summary_record.py | 11 ++++++----- mindspore/train/summary/writer.py | 4 ---- 4 files changed, 9 insertions(+), 14 deletions(-) diff --git a/mindspore/train/callback/_summary_collector.py b/mindspore/train/callback/_summary_collector.py index be8e4ca04d..1f5b82ff44 100644 --- a/mindspore/train/callback/_summary_collector.py +++ b/mindspore/train/callback/_summary_collector.py @@ -129,6 +129,7 @@ class SummaryCollector(Callback): specify `max_file_size=4 * 1024**3`. export_options (Union[None, dict]): Perform custom operations on the export data. Default: None, it means there is no export data. + Note that the size of export files is not limited by the max_file_size. You can customize the export data with a dictionary. For example, you can set {'tensor_format': 'npy'} to export tensor as npy file. The data that supports control is shown below. diff --git a/mindspore/train/summary/_summary_adapter.py b/mindspore/train/summary/_summary_adapter.py index ff19d94551..498d2d825c 100644 --- a/mindspore/train/summary/_summary_adapter.py +++ b/mindspore/train/summary/_summary_adapter.py @@ -36,14 +36,14 @@ EVENT_FILE_INIT_VERSION = 1 F32_MIN, F32_MAX = np.finfo(np.float32).min, np.finfo(np.float32).max -def get_event_file_name(prefix, suffix, seconds=None): +def get_event_file_name(prefix, suffix, time_second): """ Create file name: file_prefix + EVENT_FILE_NAME_MARK + time(seconds) + "." + Hostname + file_suffix. Args: prefix (str): The prefix of file name. suffix (str): The suffix of file name. - seconds (str): The time stamp of file name. + time_second (str): The time stamp of file name. Returns: String, the name of event log file. @@ -51,9 +51,6 @@ def get_event_file_name(prefix, suffix, seconds=None): Validator.check_str_by_regular(prefix) Validator.check_str_by_regular(suffix) file_name = "" - time_second = str(int(time.time())) - if seconds is not None: - time_second = seconds hostname = platform.node() if prefix is not None: diff --git a/mindspore/train/summary/summary_record.py b/mindspore/train/summary/summary_record.py index 01ccb5e3e0..8a914e1099 100644 --- a/mindspore/train/summary/summary_record.py +++ b/mindspore/train/summary/summary_record.py @@ -108,6 +108,7 @@ class SummaryRecord: in recording data. Default: False, this means that error logs are printed and no exception is thrown. export_options (Union[None, dict]): Perform custom operations on the export data. Default: None, it means there is no export data. + Note that the size of export files is not limited by the max_file_size. You can customize the export data with a dictionary. For example, you can set {'tensor_format': 'npy'} to export tensor as npy file. The data that supports control is shown below. @@ -158,19 +159,19 @@ class SummaryRecord: self.network = network self.has_graph = False - seconds = str(int(time.time())) + time_second = str(int(time.time())) # create the summary writer file - self.event_file_name = get_event_file_name(self.prefix, self.suffix, seconds) + self.event_file_name = get_event_file_name(self.prefix, self.suffix, time_second) self.full_file_name = os.path.join(self.log_path, self.event_file_name) self._export_options = process_export_options(export_options) export_dir = '' if self._export_options is not None: - export_dir = "export_{}".format(seconds) + export_dir = "export_{}".format(time_second) filename_dict = dict(summary=self.full_file_name, - lineage=get_event_file_name(self.prefix, '_lineage'), - explainer=get_event_file_name(self.prefix, '_explain'), + lineage=get_event_file_name(self.prefix, '_lineage', time_second), + explainer=get_event_file_name(self.prefix, '_explain', time_second), exporter=export_dir) self._event_writer = WriterPool(log_dir, max_file_size, diff --git a/mindspore/train/summary/writer.py b/mindspore/train/summary/writer.py index 8dff209f3a..a54ea962fc 100644 --- a/mindspore/train/summary/writer.py +++ b/mindspore/train/summary/writer.py @@ -163,10 +163,6 @@ class ExportWriter(BaseWriter): if disk_usage(path).free < required_length * FREE_DISK_SPACE_TIMES: raise RuntimeError(f"The disk space may be soon exhausted by the '{path}'.") - if max_file_size is not None and max_file_size < required_length: - raise RuntimeWarning(f"'max_file_size' reached: There are {max_file_size} bytes remaining, " - f"but the '{path}' requires to write {required_length} bytes.") - np_path = "{}/{}_{}.npy".format(path, tag, step) np.save(np_path, np_value) os.chmod(np_path, FILE_MODE)