support for histogram API

mindinsight/backend/datavisual/train_visual_api.py

@@ -24,6 +24,7 @@ from flask import jsonify
 from mindinsight.conf import settings
 from mindinsight.datavisual.utils.tools import get_train_id
 from mindinsight.datavisual.utils.tools import if_nan_inf_to_none
+from mindinsight.datavisual.processors.histogram_processor import HistogramProcessor
 from mindinsight.datavisual.processors.images_processor import ImageProcessor
 from mindinsight.datavisual.processors.scalars_processor import ScalarsProcessor
 from mindinsight.datavisual.processors.graph_processor import GraphProcessor
@@ -146,6 +147,22 @@ def graph_search_single_node():
     return jsonify(resp)
 
 
+@BLUEPRINT.route("/datavisual/histograms", methods=["GET"])
+def histogram():
+    """
+    Interface to obtain histogram data.
+
+    Returns:
+        Response, which contains a JSON object.
+    """
+    tag = request.args.get("tag", default=None)
+    train_id = get_train_id(request)
+
+    processor = HistogramProcessor(DATA_MANAGER)
+    response = processor.get_histograms(train_id, tag)
+    return jsonify(response)
+
+
 def init_module(app):
     """
     Init module entry.

mindinsight/conf/constants.py

@@ -56,3 +56,4 @@ MAX_GRAPH_TAG_SIZE = 10
 MAX_IMAGE_STEP_SIZE_PER_TAG = 10
 MAX_SCALAR_STEP_SIZE_PER_TAG = 1000
 MAX_GRAPH_STEP_SIZE_PER_TAG = 1
+MAX_HISTOGRAM_STEP_SIZE_PER_TAG = 50

mindinsight/datavisual/common/enums.py

@@ -37,3 +37,4 @@ class PluginNameEnum(BaseEnum):
     IMAGE = 'image'
     SCALAR = 'scalar'
     GRAPH = 'graph'
+    HISTOGRAM = 'histogram'

mindinsight/datavisual/common/exceptions.py

@@ -138,3 +138,12 @@ class ScalarNotExistError(MindInsightException):
         super(ScalarNotExistError, self).__init__(DataVisualErrors.SCALAR_NOT_EXIST,
                                                   error_msg,
                                                   http_code=400)
+
+
+class HistogramNotExistError(MindInsightException):
+    """Unable to get histogram values based on a given condition."""
+    def __init__(self, error_detail):
+        error_msg = f'Histogram value is not exist. Detail: {error_detail}'
+        super(HistogramNotExistError, self).__init__(DataVisualErrors.HISTOGRAM_NOT_EXIST,
+                                                     error_msg,
+                                                     http_code=400)

mindinsight/datavisual/data_transform/events_data.py

@@ -41,6 +41,7 @@ CONFIG = {
             PluginNameEnum.SCALAR.value: settings.MAX_SCALAR_STEP_SIZE_PER_TAG,
             PluginNameEnum.IMAGE.value: settings.MAX_IMAGE_STEP_SIZE_PER_TAG,
             PluginNameEnum.GRAPH.value: settings.MAX_GRAPH_STEP_SIZE_PER_TAG,
+            PluginNameEnum.HISTOGRAM.value: settings.MAX_HISTOGRAM_STEP_SIZE_PER_TAG
         }
 }
 

mindinsight/datavisual/data_transform/ms_data_loader.py

@@ -234,6 +234,16 @@ class MSDataLoader:
                                                value=value.image)
                     self._events_data.add_tensor_event(tensor_event)
 
+                if value.HasField('histogram'):
+                    histogram_msg = value.histogram
+                    tag = '{}/{}'.format(value.tag, PluginNameEnum.HISTOGRAM.value)
+                    tensor_event = TensorEvent(wall_time=event.wall_time,
+                                                step=event.step,
+                                                tag=tag,
+                                                plugin_name=PluginNameEnum.HISTOGRAM.value,
+                                                value=histogram_msg)
+                    self._events_data.add_tensor_event(tensor_event)
+
         if event.HasField('graph_def'):
             graph_proto = event.graph_def
             graph = MSGraph()

mindinsight/datavisual/processors/histogram_processor.py

@@ -0,0 +1,71 @@
+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Histogram Processor APIs."""
+from mindinsight.utils.exceptions import ParamValueError
+from mindinsight.datavisual.common.log import logger
+from mindinsight.datavisual.common.validation import Validation
+from mindinsight.datavisual.common.exceptions import HistogramNotExistError
+from mindinsight.datavisual.processors.base_processor import BaseProcessor
+
+
+class HistogramProcessor(BaseProcessor):
+    """Histogram Processor."""
+    def get_histograms(self, train_id, tag):
+        """
+        Builds a JSON-serializable object with information about histogram data.
+
+        Args:
+            train_id (str): The ID of the events data.
+            tag (str): The name of the tag the histogram data all belong to.
+
+        Returns:
+            dict, a dict including the `train_id`, `tag`, and `histograms'.
+                    {
+                        "train_id": ****,
+                        "tag": ****,
+                        "histograms": [{
+                            "wall_time": ****,
+                            "step": ****,
+                            "bucket": [[**, **, **]],
+                            },
+                            {...}
+                        ]
+                    }
+        """
+        Validation.check_param_empty(train_id=train_id, tag=tag)
+        logger.info("Start to process histogram data...")
+        try:
+            tensors = self._data_manager.list_tensors(train_id, tag)
+        except ParamValueError as err:
+            raise HistogramNotExistError(err.message)
+
+        histograms = []
+        for tensor in tensors:
+            buckets = []
+            for bucket in tensor.value.buckets:
+                buckets.append([bucket.left, bucket.width, bucket.count])
+            histograms.append({
+                "wall_time": tensor.wall_time,
+                "step": tensor.step,
+                "buckets": buckets
+            })
+
+        logger.info("Histogram data processing is finished!")
+        response = {
+            "train_id": train_id,
+            "tag": tag,
+            "histograms": histograms
+        }
+        return response

mindinsight/datavisual/proto_files/mindinsight_summary.proto

@@ -61,6 +61,30 @@ message Summary {
         required bytes encoded_image = 4;
     }
 
+    message Histogram {
+      message bucket{
+          // Counting number of values fallen in [left, left + width).
+          // For the rightmost bucket, the range is [left, left + width].
+          required double left = 1;
+          required double width = 2;
+          required int64 count = 3;
+      }
+
+      repeated bucket buckets = 1;
+      optional int64 nan_count = 2;
+      optional int64 pos_inf_count = 3;
+      optional int64 neg_inf_count = 4;
+
+      // max, min, sum will not take nan and inf into account.
+      // If there is no valid value in tensor, max and min will be nan, sum will be 0.
+      optional double max = 5;
+      optional double min = 6;
+      optional double sum = 7;
+
+      // total number of values. including nan and inf.
+      optional int64 count = 8;
+    }
+
     message Value {
         // Tag name for the data.
         required string tag = 1;
@@ -70,6 +94,7 @@ message Summary {
             float scalar_value = 3;
             Image image = 4;
             TensorProto tensor = 8;
+            Histogram histogram = 9;
         }
     }
 

mindinsight/datavisual/proto_files/mindinsight_summary_pb2.py

@@ -1,13 +1,11 @@
+# -*- coding: utf-8 -*-
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # source: mindinsight_summary.proto
 
-import sys
-_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import message as _message
 from google.protobuf import reflection as _reflection
 from google.protobuf import symbol_database as _symbol_database
-from google.protobuf import descriptor_pb2
 # @@protoc_insertion_point(imports)
 
 _sym_db = _symbol_database.Default()
@@ -20,10 +18,10 @@ DESCRIPTOR = _descriptor.FileDescriptor(
   name='mindinsight_summary.proto',
   package='mindinsight',
   syntax='proto2',
-  serialized_pb=_b('\n\x19mindinsight_summary.proto\x12\x0bmindinsight\x1a\x18mindinsight_anf_ir.proto\"\x9a\x01\n\x05\x45vent\x12\x11\n\twall_time\x18\x01 \x02(\x01\x12\x0c\n\x04step\x18\x02 \x01(\x03\x12\x11\n\x07version\x18\x03 \x01(\tH\x00\x12,\n\tgraph_def\x18\x04 \x01(\x0b\x32\x17.mindinsight.GraphProtoH\x00\x12\'\n\x07summary\x18\x05 \x01(\x0b\x32\x14.mindinsight.SummaryH\x00\x42\x06\n\x04what\"\x98\x02\n\x07Summary\x12)\n\x05value\x18\x01 \x03(\x0b\x32\x1a.mindinsight.Summary.Value\x1aQ\n\x05Image\x12\x0e\n\x06height\x18\x01 \x02(\x05\x12\r\n\x05width\x18\x02 \x02(\x05\x12\x12\n\ncolorspace\x18\x03 \x02(\x05\x12\x15\n\rencoded_image\x18\x04 \x02(\x0c\x1a\x8e\x01\n\x05Value\x12\x0b\n\x03tag\x18\x01 \x02(\t\x12\x16\n\x0cscalar_value\x18\x03 \x01(\x02H\x00\x12+\n\x05image\x18\x04 \x01(\x0b\x32\x1a.mindinsight.Summary.ImageH\x00\x12*\n\x06tensor\x18\x08 \x01(\x0b\x32\x18.mindinsight.TensorProtoH\x00\x42\x07\n\x05valueB\x03\xf8\x01\x01')
+  serialized_options=b'\370\001\001',
+  serialized_pb=b'\n\x19mindinsight_summary.proto\x12\x0bmindinsight\x1a\x18mindinsight_anf_ir.proto\"\x9a\x01\n\x05\x45vent\x12\x11\n\twall_time\x18\x01 \x02(\x01\x12\x0c\n\x04step\x18\x02 \x01(\x03\x12\x11\n\x07version\x18\x03 \x01(\tH\x00\x12,\n\tgraph_def\x18\x04 \x01(\x0b\x32\x17.mindinsight.GraphProtoH\x00\x12\'\n\x07summary\x18\x05 \x01(\x0b\x32\x14.mindinsight.SummaryH\x00\x42\x06\n\x04what\"\xc0\x04\n\x07Summary\x12)\n\x05value\x18\x01 \x03(\x0b\x32\x1a.mindinsight.Summary.Value\x1aQ\n\x05Image\x12\x0e\n\x06height\x18\x01 \x02(\x05\x12\r\n\x05width\x18\x02 \x02(\x05\x12\x12\n\ncolorspace\x18\x03 \x02(\x05\x12\x15\n\rencoded_image\x18\x04 \x02(\x0c\x1a\xf0\x01\n\tHistogram\x12\x36\n\x07\x62uckets\x18\x01 \x03(\x0b\x32%.mindinsight.Summary.Histogram.bucket\x12\x11\n\tnan_count\x18\x02 \x01(\x03\x12\x15\n\rpos_inf_count\x18\x03 \x01(\x03\x12\x15\n\rneg_inf_count\x18\x04 \x01(\x03\x12\x0b\n\x03max\x18\x05 \x01(\x01\x12\x0b\n\x03min\x18\x06 \x01(\x01\x12\x0b\n\x03sum\x18\x07 \x01(\x01\x12\r\n\x05\x63ount\x18\x08 \x01(\x03\x1a\x34\n\x06\x62ucket\x12\x0c\n\x04left\x18\x01 \x02(\x01\x12\r\n\x05width\x18\x02 \x02(\x01\x12\r\n\x05\x63ount\x18\x03 \x02(\x03\x1a\xc3\x01\n\x05Value\x12\x0b\n\x03tag\x18\x01 \x02(\t\x12\x16\n\x0cscalar_value\x18\x03 \x01(\x02H\x00\x12+\n\x05image\x18\x04 \x01(\x0b\x32\x1a.mindinsight.Summary.ImageH\x00\x12*\n\x06tensor\x18\x08 \x01(\x0b\x32\x18.mindinsight.TensorProtoH\x00\x12\x33\n\thistogram\x18\t \x01(\x0b\x32\x1e.mindinsight.Summary.HistogramH\x00\x42\x07\n\x05valueB\x03\xf8\x01\x01'
   ,
   dependencies=[mindinsight__anf__ir__pb2.DESCRIPTOR,])
-_sym_db.RegisterFileDescriptor(DESCRIPTOR)
 
 
 
@@ -41,42 +39,42 @@ _EVENT = _descriptor.Descriptor(
       has_default_value=False, default_value=float(0),
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      options=None),
+      serialized_options=None, file=DESCRIPTOR),
     _descriptor.FieldDescriptor(
       name='step', full_name='mindinsight.Event.step', index=1,
       number=2, type=3, cpp_type=2, label=1,
       has_default_value=False, default_value=0,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      options=None),
+      serialized_options=None, file=DESCRIPTOR),
     _descriptor.FieldDescriptor(
       name='version', full_name='mindinsight.Event.version', index=2,
       number=3, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
+      has_default_value=False, default_value=b"".decode('utf-8'),
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      options=None),
+      serialized_options=None, file=DESCRIPTOR),
     _descriptor.FieldDescriptor(
       name='graph_def', full_name='mindinsight.Event.graph_def', index=3,
       number=4, type=11, cpp_type=10, label=1,
       has_default_value=False, default_value=None,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      options=None),
+      serialized_options=None, file=DESCRIPTOR),
     _descriptor.FieldDescriptor(
       name='summary', full_name='mindinsight.Event.summary', index=4,
       number=5, type=11, cpp_type=10, label=1,
       has_default_value=False, default_value=None,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      options=None),
+      serialized_options=None, file=DESCRIPTOR),
   ],
   extensions=[
   ],
   nested_types=[],
   enum_types=[
   ],
-  options=None,
+  serialized_options=None,
   is_extendable=False,
   syntax='proto2',
   extension_ranges=[],
@@ -103,35 +101,35 @@ _SUMMARY_IMAGE = _descriptor.Descriptor(
       has_default_value=False, default_value=0,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      options=None),
+      serialized_options=None, file=DESCRIPTOR),
     _descriptor.FieldDescriptor(
       name='width', full_name='mindinsight.Summary.Image.width', index=1,
       number=2, type=5, cpp_type=1, label=2,
       has_default_value=False, default_value=0,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      options=None),
+      serialized_options=None, file=DESCRIPTOR),
     _descriptor.FieldDescriptor(
       name='colorspace', full_name='mindinsight.Summary.Image.colorspace', index=2,
       number=3, type=5, cpp_type=1, label=2,
       has_default_value=False, default_value=0,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      options=None),
+      serialized_options=None, file=DESCRIPTOR),
     _descriptor.FieldDescriptor(
       name='encoded_image', full_name='mindinsight.Summary.Image.encoded_image', index=3,
       number=4, type=12, cpp_type=9, label=2,
-      has_default_value=False, default_value=_b(""),
+      has_default_value=False, default_value=b"",
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      options=None),
+      serialized_options=None, file=DESCRIPTOR),
   ],
   extensions=[
   ],
   nested_types=[],
   enum_types=[
   ],
-  options=None,
+  serialized_options=None,
   is_extendable=False,
   syntax='proto2',
   extension_ranges=[],
@@ -141,6 +139,129 @@ _SUMMARY_IMAGE = _descriptor.Descriptor(
   serialized_end=361,
 )
 
+_SUMMARY_HISTOGRAM_BUCKET = _descriptor.Descriptor(
+  name='bucket',
+  full_name='mindinsight.Summary.Histogram.bucket',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='left', full_name='mindinsight.Summary.Histogram.bucket.left', index=0,
+      number=1, type=1, cpp_type=5, label=2,
+      has_default_value=False, default_value=float(0),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='width', full_name='mindinsight.Summary.Histogram.bucket.width', index=1,
+      number=2, type=1, cpp_type=5, label=2,
+      has_default_value=False, default_value=float(0),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='count', full_name='mindinsight.Summary.Histogram.bucket.count', index=2,
+      number=3, type=3, cpp_type=2, label=2,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto2',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=552,
+  serialized_end=604,
+)
+
+_SUMMARY_HISTOGRAM = _descriptor.Descriptor(
+  name='Histogram',
+  full_name='mindinsight.Summary.Histogram',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='buckets', full_name='mindinsight.Summary.Histogram.buckets', index=0,
+      number=1, type=11, cpp_type=10, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='nan_count', full_name='mindinsight.Summary.Histogram.nan_count', index=1,
+      number=2, type=3, cpp_type=2, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='pos_inf_count', full_name='mindinsight.Summary.Histogram.pos_inf_count', index=2,
+      number=3, type=3, cpp_type=2, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='neg_inf_count', full_name='mindinsight.Summary.Histogram.neg_inf_count', index=3,
+      number=4, type=3, cpp_type=2, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='max', full_name='mindinsight.Summary.Histogram.max', index=4,
+      number=5, type=1, cpp_type=5, label=1,
+      has_default_value=False, default_value=float(0),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='min', full_name='mindinsight.Summary.Histogram.min', index=5,
+      number=6, type=1, cpp_type=5, label=1,
+      has_default_value=False, default_value=float(0),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='sum', full_name='mindinsight.Summary.Histogram.sum', index=6,
+      number=7, type=1, cpp_type=5, label=1,
+      has_default_value=False, default_value=float(0),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='count', full_name='mindinsight.Summary.Histogram.count', index=7,
+      number=8, type=3, cpp_type=2, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+  ],
+  extensions=[
+  ],
+  nested_types=[_SUMMARY_HISTOGRAM_BUCKET, ],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto2',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=364,
+  serialized_end=604,
+)
+
 _SUMMARY_VALUE = _descriptor.Descriptor(
   name='Value',
   full_name='mindinsight.Summary.Value',
@@ -151,38 +272,45 @@ _SUMMARY_VALUE = _descriptor.Descriptor(
     _descriptor.FieldDescriptor(
       name='tag', full_name='mindinsight.Summary.Value.tag', index=0,
       number=1, type=9, cpp_type=9, label=2,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
+      has_default_value=False, default_value=b"".decode('utf-8'),
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      options=None),
+      serialized_options=None, file=DESCRIPTOR),
     _descriptor.FieldDescriptor(
       name='scalar_value', full_name='mindinsight.Summary.Value.scalar_value', index=1,
       number=3, type=2, cpp_type=6, label=1,
       has_default_value=False, default_value=float(0),
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      options=None),
+      serialized_options=None, file=DESCRIPTOR),
     _descriptor.FieldDescriptor(
       name='image', full_name='mindinsight.Summary.Value.image', index=2,
       number=4, type=11, cpp_type=10, label=1,
       has_default_value=False, default_value=None,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      options=None),
+      serialized_options=None, file=DESCRIPTOR),
     _descriptor.FieldDescriptor(
       name='tensor', full_name='mindinsight.Summary.Value.tensor', index=3,
       number=8, type=11, cpp_type=10, label=1,
       has_default_value=False, default_value=None,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      options=None),
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='histogram', full_name='mindinsight.Summary.Value.histogram', index=4,
+      number=9, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
   ],
   extensions=[
   ],
   nested_types=[],
   enum_types=[
   ],
-  options=None,
+  serialized_options=None,
   is_extendable=False,
   syntax='proto2',
   extension_ranges=[],
@@ -191,8 +319,8 @@ _SUMMARY_VALUE = _descriptor.Descriptor(
       name='value', full_name='mindinsight.Summary.Value.value',
       index=0, containing_type=None, fields=[]),
   ],
-  serialized_start=364,
-  serialized_end=506,
+  serialized_start=607,
+  serialized_end=802,
 )
 
 _SUMMARY = _descriptor.Descriptor(
@@ -208,21 +336,21 @@ _SUMMARY = _descriptor.Descriptor(
       has_default_value=False, default_value=[],
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      options=None),
+      serialized_options=None, file=DESCRIPTOR),
   ],
   extensions=[
   ],
-  nested_types=[_SUMMARY_IMAGE, _SUMMARY_VALUE, ],
+  nested_types=[_SUMMARY_IMAGE, _SUMMARY_HISTOGRAM, _SUMMARY_VALUE, ],
   enum_types=[
   ],
-  options=None,
+  serialized_options=None,
   is_extendable=False,
   syntax='proto2',
   extension_ranges=[],
   oneofs=[
   ],
   serialized_start=226,
-  serialized_end=506,
+  serialized_end=802,
 )
 
 _EVENT.fields_by_name['graph_def'].message_type = mindinsight__anf__ir__pb2._GRAPHPROTO
@@ -237,8 +365,12 @@ _EVENT.oneofs_by_name['what'].fields.append(
   _EVENT.fields_by_name['summary'])
 _EVENT.fields_by_name['summary'].containing_oneof = _EVENT.oneofs_by_name['what']
 _SUMMARY_IMAGE.containing_type = _SUMMARY
+_SUMMARY_HISTOGRAM_BUCKET.containing_type = _SUMMARY_HISTOGRAM
+_SUMMARY_HISTOGRAM.fields_by_name['buckets'].message_type = _SUMMARY_HISTOGRAM_BUCKET
+_SUMMARY_HISTOGRAM.containing_type = _SUMMARY
 _SUMMARY_VALUE.fields_by_name['image'].message_type = _SUMMARY_IMAGE
 _SUMMARY_VALUE.fields_by_name['tensor'].message_type = mindinsight__anf__ir__pb2._TENSORPROTO
+_SUMMARY_VALUE.fields_by_name['histogram'].message_type = _SUMMARY_HISTOGRAM
 _SUMMARY_VALUE.containing_type = _SUMMARY
 _SUMMARY_VALUE.oneofs_by_name['value'].fields.append(
   _SUMMARY_VALUE.fields_by_name['scalar_value'])
@@ -249,41 +381,60 @@ _SUMMARY_VALUE.fields_by_name['image'].containing_oneof = _SUMMARY_VALUE.oneofs_
 _SUMMARY_VALUE.oneofs_by_name['value'].fields.append(
   _SUMMARY_VALUE.fields_by_name['tensor'])
 _SUMMARY_VALUE.fields_by_name['tensor'].containing_oneof = _SUMMARY_VALUE.oneofs_by_name['value']
+_SUMMARY_VALUE.oneofs_by_name['value'].fields.append(
+  _SUMMARY_VALUE.fields_by_name['histogram'])
+_SUMMARY_VALUE.fields_by_name['histogram'].containing_oneof = _SUMMARY_VALUE.oneofs_by_name['value']
 _SUMMARY.fields_by_name['value'].message_type = _SUMMARY_VALUE
 DESCRIPTOR.message_types_by_name['Event'] = _EVENT
 DESCRIPTOR.message_types_by_name['Summary'] = _SUMMARY
+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
 
-Event = _reflection.GeneratedProtocolMessageType('Event', (_message.Message,), dict(
-  DESCRIPTOR = _EVENT,
-  __module__ = 'mindinsight_summary_pb2'
+Event = _reflection.GeneratedProtocolMessageType('Event', (_message.Message,), {
+  'DESCRIPTOR' : _EVENT,
+  '__module__' : 'mindinsight_summary_pb2'
   # @@protoc_insertion_point(class_scope:mindinsight.Event)
-  ))
+  })
 _sym_db.RegisterMessage(Event)
 
-Summary = _reflection.GeneratedProtocolMessageType('Summary', (_message.Message,), dict(
+Summary = _reflection.GeneratedProtocolMessageType('Summary', (_message.Message,), {
 
-  Image = _reflection.GeneratedProtocolMessageType('Image', (_message.Message,), dict(
-    DESCRIPTOR = _SUMMARY_IMAGE,
-    __module__ = 'mindinsight_summary_pb2'
+  'Image' : _reflection.GeneratedProtocolMessageType('Image', (_message.Message,), {
+    'DESCRIPTOR' : _SUMMARY_IMAGE,
+    '__module__' : 'mindinsight_summary_pb2'
     # @@protoc_insertion_point(class_scope:mindinsight.Summary.Image)
-    ))
+    })
+  ,
+
+  'Histogram' : _reflection.GeneratedProtocolMessageType('Histogram', (_message.Message,), {
+
+    'bucket' : _reflection.GeneratedProtocolMessageType('bucket', (_message.Message,), {
+      'DESCRIPTOR' : _SUMMARY_HISTOGRAM_BUCKET,
+      '__module__' : 'mindinsight_summary_pb2'
+      # @@protoc_insertion_point(class_scope:mindinsight.Summary.Histogram.bucket)
+      })
+    ,
+    'DESCRIPTOR' : _SUMMARY_HISTOGRAM,
+    '__module__' : 'mindinsight_summary_pb2'
+    # @@protoc_insertion_point(class_scope:mindinsight.Summary.Histogram)
+    })
   ,
 
-  Value = _reflection.GeneratedProtocolMessageType('Value', (_message.Message,), dict(
-    DESCRIPTOR = _SUMMARY_VALUE,
-    __module__ = 'mindinsight_summary_pb2'
+  'Value' : _reflection.GeneratedProtocolMessageType('Value', (_message.Message,), {
+    'DESCRIPTOR' : _SUMMARY_VALUE,
+    '__module__' : 'mindinsight_summary_pb2'
     # @@protoc_insertion_point(class_scope:mindinsight.Summary.Value)
-    ))
+    })
   ,
-  DESCRIPTOR = _SUMMARY,
-  __module__ = 'mindinsight_summary_pb2'
+  'DESCRIPTOR' : _SUMMARY,
+  '__module__' : 'mindinsight_summary_pb2'
   # @@protoc_insertion_point(class_scope:mindinsight.Summary)
-  ))
+  })
 _sym_db.RegisterMessage(Summary)
 _sym_db.RegisterMessage(Summary.Image)
+_sym_db.RegisterMessage(Summary.Histogram)
+_sym_db.RegisterMessage(Summary.Histogram.bucket)
 _sym_db.RegisterMessage(Summary.Value)
 
 
-DESCRIPTOR.has_options = True
-DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\370\001\001'))
+DESCRIPTOR._options = None
 # @@protoc_insertion_point(module_scope)

mindinsight/utils/constant.py

@@ -62,3 +62,4 @@ class DataVisualErrors(Enum):
     GRAPH_NOT_EXIST = 12
     IMAGE_NOT_EXIST = 13
     SCALAR_NOT_EXIST = 14
+    HISTOGRAM_NOT_EXIST = 15

tests/st/func/datavisual/conftest.py

@@ -65,6 +65,7 @@ def init_summary_logs():
 
         summaries_metadata.update(
             log_operations.create_summary_logs(summary_base_dir, constants.SUMMARY_DIR_NUM_SECOND,
+                                               constants.SUMMARY_DIR_PREFIX,
                                                constants.SUMMARY_DIR_NUM_FIRST))
         summaries_metadata.update(
             log_operations.create_multiple_logs(summary_base_dir, constants.MULTIPLE_DIR_NAME,

tests/ut/datavisual/processors/test_train_task_manager.py

@@ -70,7 +70,7 @@ class TestTrainTaskManager:
     def load_data(self):
         """Load data."""
         log_operation = LogOperations()
-        self._plugins_id_map = {'image': [], 'scalar': [], 'graph': []}
+        self._plugins_id_map = {'image': [], 'scalar': [], 'graph': [], 'histogram': []}
         self._events_names = []
         self._train_id_list = []
 

tests/utils/log_generators/histogram_log_generator.py

@@ -0,0 +1,114 @@
+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Log generator for histogram data."""
+import time
+
+import numpy as np
+
+from mindinsight.datavisual.proto_files import mindinsight_summary_pb2 as summary_pb2
+
+from .log_generator import LogGenerator
+
+
+class HistogramLogGenerator(LogGenerator):
+    """
+    Log generator for histogram data.
+
+    This is a log generator writing histogram data. User can use it to generate fake
+    summary logs about histogram.
+    """
+
+    def generate_event(self, values):
+        """
+        Method for generating histogram event.
+
+        Args:
+            values (dict): A dict contains:
+                {
+                    wall_time (float): Timestamp.
+                    step (int): Train step.
+                    value (float): Histogram value.
+                    tag (str): Tag name.
+                }
+
+       Returns:
+            summary_pb2.Event.
+
+        """
+        histogram_event = summary_pb2.Event()
+        histogram_event.wall_time = values.get('wall_time')
+        histogram_event.step = values.get('step')
+
+        value = histogram_event.summary.value.add()
+        value.tag = values.get('tag')
+
+        buckets = values.get('buckets')
+        for bucket in buckets:
+            left, width, count = bucket
+            bucket = value.histogram.buckets.add()
+            bucket.left = left
+            bucket.width = width
+            bucket.count = count
+
+        return histogram_event
+
+    def generate_log(self, file_path, steps_list, tag_name):
+        """
+        Generate log for external calls.
+
+        Args:
+            file_path (str): Path to write logs.
+            steps_list (list): A list consists of step.
+            tag_name (str): Tag name.
+
+        Returns:
+            list[dict], generated histogram metadata.
+            None, to be consistent with return value of HistogramGenerator.
+
+        """
+        histogram_metadata = []
+        for step in steps_list:
+            histogram = dict()
+
+            wall_time = time.time()
+            histogram.update({'wall_time': wall_time})
+            histogram.update({'step': step})
+            histogram.update({'tag': tag_name})
+
+            # Construct buckets
+            buckets = []
+            leftmost = list(np.random.randn(11))
+            leftmost.sort()
+            for i in range(10):
+                left = leftmost[i]
+                width = leftmost[i+1] - left
+                count = np.random.randint(20)
+                bucket = [left, width, count]
+                buckets.append(bucket)
+
+            histogram.update({'buckets': buckets})
+            histogram_metadata.append(histogram)
+
+            self._write_log_one_step(file_path, histogram)
+
+        return histogram_metadata, None
+
+
+if __name__ == "__main__":
+    histogram_log_generator = HistogramLogGenerator()
+    test_file_name = '%s.%s.%s' % ('histogram', 'summary', str(time.time()))
+    test_steps = [1, 3, 5]
+    test_tag = "test_histogram_tag_name"
+    histogram_log_generator.generate_log(test_file_name, test_steps, test_tag)

tests/utils/log_operations.py

@@ -24,11 +24,13 @@ from mindinsight.datavisual.common.enums import PluginNameEnum
 from .log_generators.graph_log_generator import GraphLogGenerator
 from .log_generators.images_log_generator import ImagesLogGenerator
 from .log_generators.scalars_log_generator import ScalarsLogGenerator
+from .log_generators.histogram_log_generator import HistogramLogGenerator
 
 log_generators = {
     PluginNameEnum.GRAPH.value: GraphLogGenerator(),
     PluginNameEnum.IMAGE.value: ImagesLogGenerator(),
-    PluginNameEnum.SCALAR.value: ScalarsLogGenerator()
+    PluginNameEnum.SCALAR.value: ScalarsLogGenerator(),
+    PluginNameEnum.HISTOGRAM.value: HistogramLogGenerator()
 }