From e410462a4b2dabbed764fd389fe7a9e95cc2cbaa Mon Sep 17 00:00:00 2001 From: luopengting Date: Wed, 28 Oct 2020 17:00:40 +0800 Subject: [PATCH] ignore the value in dataset graph if its type is an object, log message about image only if the dataset_sink_mode is False --- mindspore/train/callback/_dataset_graph.py | 15 +++++++++------ mindspore/train/callback/_summary_collector.py | 7 ++++--- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/mindspore/train/callback/_dataset_graph.py b/mindspore/train/callback/_dataset_graph.py index e8c8dcb2ba..3630c389e4 100644 --- a/mindspore/train/callback/_dataset_graph.py +++ b/mindspore/train/callback/_dataset_graph.py @@ -13,9 +13,9 @@ # limitations under the License. # ============================================================================ """Define dataset graph related operations.""" -import json from importlib import import_module +from mindspore import log as logger from mindspore.train import lineage_pb2 @@ -33,9 +33,11 @@ class DatasetGraph: """ dataset_package = import_module('mindspore.dataset') dataset_dict = dataset_package.serialize(dataset) - json_str = json.dumps(dataset_dict, indent=2) - dataset_dict = json.loads(json_str) dataset_graph_proto = lineage_pb2.DatasetGraph() + if not isinstance(dataset_dict, dict): + logger.warning("The dataset graph serialized from dataset object is not a dict. " + "Its type is %r.", type(dataset_dict).__name__) + return dataset_graph_proto if "children" in dataset_dict: children = dataset_dict.pop("children") if children: @@ -92,7 +94,7 @@ class DatasetGraph: message (Operation): Enhancement operation proto message. """ for key, value in operation.items(): - if isinstance(value, list): + if isinstance(value, (list, tuple)): if all(isinstance(ele, int) for ele in value): message.size.extend(value) else: @@ -118,11 +120,12 @@ class DatasetGraph: message.mapInt[key] = value elif isinstance(value, float): message.mapDouble[key] = value - elif isinstance(value, list) and key != "operations": + elif isinstance(value, (list, tuple)) and key != "operations": if value: replace_value_list = list(map(lambda x: "" if x is None else x, value)) message.mapStrList[key].strValue.extend(replace_value_list) elif value is None: message.mapStr[key] = "None" else: - raise ValueError(f"Parameter {key} is not supported in event package.") + logger.warning("The parameter %r is not recorded, because its type is not supported in event package. " + "Its type is %r.", key, type(value).__name__) diff --git a/mindspore/train/callback/_summary_collector.py b/mindspore/train/callback/_summary_collector.py index 9c87fca475..630c047fcc 100644 --- a/mindspore/train/callback/_summary_collector.py +++ b/mindspore/train/callback/_summary_collector.py @@ -464,7 +464,8 @@ class SummaryCollector(Callback): try: self._record.add_value(PluginEnum.IMAGE.value, 'input_data/auto', input_data) except (TypeError, ValueError): - logger.warning('The input data of network are not image, so will not collect by SummaryCollector.') + if not self._dataset_sink_mode: + logger.warning('The input data of network are not image, so will not collect by SummaryCollector.') self._collect_specified_data['collect_input_data'] = False return @@ -636,8 +637,8 @@ class SummaryCollector(Callback): """ learning_rate = optimizer.learning_rate if not isinstance(learning_rate, Parameter): - logger.warning("The learning rate detected in the optimizer " - "is not a Parameter type, so it is not recorded.") + logger.warning("The learning rate detected in the optimizer is not a Parameter type, " + "so it is not recorded. Its type is %r.", type(learning_rate).__name__) return None return learning_rate.data