add explainer json api and data encap for job list, job meta, saliency maps and evaluations

json return null if ExplainManager returns None

use fromtimestamp

use fromtimestamp

wrong name

wrong json key

use offset as page offset

enhance comments

rm mock_explainer_api.py

update copyright year 2020

remove unused import

mindinsight/backend/explainer/__init__.py

@@ -0,0 +1,31 @@
+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""
+module init file.
+"""
+from mindinsight.backend.explainer.explainer_api import init_module as init_query_module
+
+
+def init_module(app):
+    """
+    Init module entry.
+
+    Args:
+        app (Flask): A Flask instance.
+
+    Returns:
+
+    """
+    init_query_module(app)

mindinsight/backend/explainer/explainer_api.py

@@ -0,0 +1,180 @@
+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Explainer restful api."""
+
+import os
+import urllib.parse
+
+from flask import Blueprint
+from flask import jsonify
+from flask import request
+
+from mindinsight.conf import settings
+from mindinsight.utils.exceptions import ParamMissError
+from mindinsight.utils.exceptions import ParamValueError
+from mindinsight.datavisual.common.validation import Validation
+from mindinsight.datavisual.data_transform.summary_watcher import SummaryWatcher
+from mindinsight.datavisual.utils.tools import get_train_id
+from mindinsight.explainer.manager.explain_manager import ExplainManager
+from mindinsight.explainer.encapsulator.explain_job_encap import ExplainJobEncap
+from mindinsight.explainer.encapsulator.saliency_encap import SaliencyEncap
+from mindinsight.explainer.encapsulator.evaluation_encap import EvaluationEncap
+
+
+URL_PREFIX = settings.URL_PATH_PREFIX+settings.API_PREFIX
+BLUEPRINT = Blueprint("explainer", __name__, url_prefix=URL_PREFIX)
+
+STATIC_EXPLAIN_MGR = True
+
+
+class ExplainManagerHolder:
+    """ExplainManger instance holder."""
+
+    static_instance = None
+
+    @classmethod
+    def get_instance(cls):
+        if cls.static_instance:
+            return cls.static_instance
+        instance = ExplainManager(settings.SUMMARY_BASE_DIR)
+        instance.start_load_data()
+        return instance
+
+    @classmethod
+    def initialize(cls):
+        if STATIC_EXPLAIN_MGR:
+            cls.static_instance = ExplainManager(settings.SUMMARY_BASE_DIR)
+            cls.static_instance.start_load_data()
+
+
+def _image_url_formatter(train_id, image_id, image_type):
+    """returns image url."""
+    train_id = urllib.parse.quote(str(train_id))
+    image_id = urllib.parse.quote(str(image_id))
+    image_type = urllib.parse.quote(str(image_type))
+    return f"{URL_PREFIX}/explainer/image?train_id={train_id}&image_id={image_id}&type={image_type}"
+
+
+@BLUEPRINT.route("/explainer/explain-jobs", methods=["GET"])
+def query_explain_jobs():
+    """Query explain jobs."""
+    offset = request.args.get("offset", default=0)
+    limit = request.args.get("limit", default=10)
+    train_id = get_train_id(request)
+    offset = Validation.check_offset(offset=offset)
+    limit = Validation.check_limit(limit, min_value=1, max_value=SummaryWatcher.MAX_SUMMARY_DIR_COUNT)
+
+    encapsulator = ExplainJobEncap(ExplainManagerHolder.get_instance())
+    total, jobs = encapsulator.query_explain_jobs(offset, limit, train_id)
+
+    return jsonify({
+        'name': os.path.basename(os.path.realpath(settings.SUMMARY_BASE_DIR)),
+        'total': total,
+        'explain_jobs': jobs,
+    })
+
+
+@BLUEPRINT.route("/explainer/explain-job", methods=["GET"])
+def query_explain_job():
+    """Query explain job meta-data."""
+    train_id = get_train_id(request)
+    if train_id is None:
+        raise ParamMissError("train_id")
+    encapsulator = ExplainJobEncap(ExplainManagerHolder.get_instance())
+    metadata = encapsulator.query_meta(train_id)
+
+    return jsonify(metadata)
+
+
+@BLUEPRINT.route("/explainer/saliency", methods=["POST"])
+def query_saliency():
+    """Query saliency map related results."""
+
+    data = request.get_json(silent=True)
+
+    train_id = data.get("train_id")
+    if train_id is None:
+        raise ParamMissError('train_id')
+
+    labels = data.get("labels")
+    explainers = data.get("explainers")
+    limit = data.get("limit", 10)
+    limit = Validation.check_limit(limit, min_value=1, max_value=100)
+    offset = data.get("offset", 0)
+    offset = Validation.check_offset(offset=offset)
+    sorted_name = data.get("sorted_name")
+    sorted_type = data.get("sorted_type", "descending")
+
+    encapsulator = SaliencyEncap(
+        _image_url_formatter,
+        ExplainManagerHolder.get_instance())
+    count, samples = encapsulator.query_saliency_maps(train_id=train_id,
+                                                      labels=labels,
+                                                      explainers=explainers,
+                                                      limit=limit,
+                                                      offset=offset,
+                                                      sorted_name=sorted_name,
+                                                      sorted_type=sorted_type)
+
+    return jsonify({
+        "count": count,
+        "samples": samples
+    })
+
+
+@BLUEPRINT.route("/explainer/evaluation", methods=["GET"])
+def query_evaluation():
+    """Query saliency explainer evaluation scores."""
+    train_id = get_train_id(request)
+    if train_id is None:
+        raise ParamMissError("train_id")
+    encapsulator = EvaluationEncap(ExplainManagerHolder.get_instance())
+    scores = encapsulator.query_explainer_scores(train_id)
+    return jsonify({
+        "explainer_scores": scores,
+    })
+
+
+@BLUEPRINT.route("/explainer/image", methods=["GET"])
+def query_image():
+    """Query image"""
+    train_id = get_train_id(request)
+    if train_id is None:
+        raise ParamMissError("train_id")
+    image_id = request.args.get("image_id")
+    if image_id is None:
+        raise ParamMissError("image_id")
+    image_type = request.args.get("type")
+    if image_type is None:
+        raise ParamMissError("type")
+    if image_type not in ("original", "overlay"):
+        raise ParamValueError(f"type:{image_type}")
+
+    encapsulator = ExplainJobEncap(ExplainManagerHolder.get_instance())
+    image = encapsulator.query_image_binary(train_id, image_id, image_type)
+
+    return image
+
+
+def init_module(app):
+    """
+    Init module entry.
+
+    Args:
+        app: the application obj.
+
+    """
+    ExplainManagerHolder.initialize()
+    app.register_blueprint(BLUEPRINT)

mindinsight/explainer/encapsulator/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Explainer data encapsulator, provides query interfaces."""

mindinsight/explainer/encapsulator/evaluation_encap.py

@@ -0,0 +1,30 @@
+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Explainer evaluation encapsulator."""
+
+import copy
+
+from mindinsight.explainer.encapsulator.explain_data_encap import ExplainDataEncap
+
+
+class EvaluationEncap(ExplainDataEncap):
+    """Explainer evaluation encapsulator."""
+
+    def query_explainer_scores(self, train_id):
+        """Query evaluation scores."""
+        job = self.job_manager.get_job(train_id)
+        if job is None:
+            return None
+        return copy.deepcopy(job.explainer_scores)

mindinsight/explainer/encapsulator/explain_data_encap.py

@@ -0,0 +1,26 @@
+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Explain data encapsulator base class."""
+
+
+class ExplainDataEncap:
+    """Explain data encapsulator base class."""
+
+    def __init__(self, job_manager):
+        self._job_manager = job_manager
+
+    @property
+    def job_manager(self):
+        return self._job_manager

mindinsight/explainer/encapsulator/explain_job_encap.py

@@ -0,0 +1,126 @@
+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Explain job list encapsulator."""
+
+import copy
+from datetime import datetime
+
+from mindinsight.utils.exceptions import ParamValueError
+from mindinsight.datavisual.data_transform.summary_watcher import SummaryWatcher
+from mindinsight.explainer.encapsulator.explain_data_encap import ExplainDataEncap
+
+
+class ExplainJobEncap(ExplainDataEncap):
+    """Explain job list encapsulator."""
+
+    DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S"
+
+    def query_explain_jobs(self, offset, limit, train_id):
+        """
+        Query explain job list.
+        Args:
+            offset (int): offset
+            limit (int): max. no. of items to be returned
+            train_id (str): job id
+        Returns:
+            Tuple[int, List[Dict]], total no. of jobs and job list
+        """
+        watcher = SummaryWatcher()
+        total, dir_infos = \
+            watcher.list_explain_directories(self.job_manager.summary_base_dir,
+                                             offset=offset, limit=limit)
+        obj_offset = offset * limit
+        job_infos = []
+
+        if train_id is None:
+            end = total
+            if obj_offset + limit < end:
+                end = obj_offset + limit
+
+            for i in range(obj_offset, end):
+                job_id = dir_infos[i]["relative_path"]
+                job = self.job_manager.get_job(job_id)
+                if job is not None:
+                    job_infos.append(self._job_2_info(job))
+        else:
+            job = self.job_manager.get_job(train_id)
+            if job is not None:
+                job_infos.append(self._job_2_info(job))
+
+        return total, job_infos
+
+    def query_meta(self, train_id):
+        """
+        Query explain job meta-data
+        Args:
+            train_id (str): job id
+        Returns:
+            Dict, the metadata
+        """
+        job = self.job_manager.get_job(train_id)
+        if job is None:
+            return None
+        return self._job_2_meta(job)
+
+    def query_image_binary(self, train_id, image_id, image_type):
+        """
+        Query image binary content.
+        Args:
+            train_id (str): job id
+            image_id (str): image id
+            image_type (str) 'original' or 'overlay'
+        Returns:
+            bytes, image binary
+        """
+        job = self.job_manager.get_job(train_id)
+
+        if job is None:
+            return None
+        if image_type == "original":
+            binary = job.retrieve_image(image_id)
+        elif image_type == "overlay":
+            binary = job.retrieve_overlay(image_id)
+        else:
+            raise ParamValueError(f"image_type:{image_type}")
+
+        return binary
+
+    @classmethod
+    def _job_2_info(cls, job):
+        """Convert ExplainJob object to jsonable info object"""
+        info = dict()
+        info["train_id"] = job.train_id
+        info["create_time"] = datetime.fromtimestamp(job.create_time)\
+            .strftime(cls.DATETIME_FORMAT)
+        info["update_time"] = datetime.fromtimestamp(job.latest_update_time)\
+            .strftime(cls.DATETIME_FORMAT)
+        return info
+
+    @classmethod
+    def _job_2_meta(cls, job):
+        """Convert ExplainJob's meta-data to jsonable info object"""
+        info = cls._job_2_info(job)
+        info["sample_count"] = job.sample_count
+        info["classes"] = copy.deepcopy(job.all_classes)
+        saliency_info = dict()
+        if job.min_confidence is None:
+            saliency_info["min_confidence"] = 0.5
+        else:
+            saliency_info["min_confidence"] = job.min_confidence
+        saliency_info["explainers"] = list(job.explainers)
+        saliency_info["metrics"] = list(job.metrics)
+        info["saliency"] = saliency_info
+        info["uncertainty"] = {"enabled": False}
+        return info

mindinsight/explainer/encapsulator/saliency_encap.py

@@ -0,0 +1,122 @@
+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Saliency map encapsulator."""
+
+import copy
+
+from mindinsight.explainer.encapsulator.explain_data_encap import ExplainDataEncap
+
+
+def _sort_key_confid(sample):
+    """Samples sort key by the max. confidence"""
+    max_confid = None
+    for inference in sample["inferences"]:
+        if max_confid is None or inference["confidence"] > max_confid:
+            max_confid = inference["confidence"]
+    return max_confid
+
+
+class SaliencyEncap(ExplainDataEncap):
+    """Saliency map encapsulator."""
+
+    def __init__(self, image_url_formatter, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._image_url_formatter = image_url_formatter
+
+    def query_saliency_maps(self,
+                            train_id,
+                            labels,
+                            explainers,
+                            limit,
+                            offset,
+                            sorted_name,
+                            sorted_type):
+        """
+        Query saliency maps
+        Args:
+            train_id (str): job id
+            labels (List[str]): labels filter
+            explainers (List[str]): explainers of saliency maps to be shown
+            limit (int): max. no. of items to be returned
+            offset (int): item offset
+            sorted_name (str): field to be sorted
+            sorted_type (str): 'ascending' or 'descending' order
+
+        Returns:
+            Tuple[int, List[dict]], total no. of samples after filtering and
+                list of sample result
+        """
+        job = self.job_manager.get_job(train_id)
+        if job is None:
+            return None
+
+        samples = copy.deepcopy(job.get_all_samples())
+        if labels:
+            filtered = []
+            for sample in samples:
+                has_label = False
+                for label in sample["labels"]:
+                    if label in labels:
+                        has_label = True
+                        break
+                if has_label:
+                    filtered.append(sample)
+            samples = filtered
+
+        reverse = sorted_type == "descending"
+        if sorted_name == "confidence":
+            samples.sort(key=_sort_key_confid, reverse=reverse)
+
+        sample_infos = []
+        obj_offset = offset*limit
+        count = len(samples)
+        end = count
+        if obj_offset + limit < end:
+            end = obj_offset + limit
+        for i in range(obj_offset, end):
+            sample = samples[i]
+            sample_infos.append(self._touch_sample(sample, job, explainers))
+
+        return count, sample_infos
+
+    def _touch_sample(self, sample, job, explainers):
+        """
+        Final editing the sample info
+        Args:
+            sample (dict): sample info
+            job (ExplainJob): job
+            explainers (List[str]): explainer names
+        Returns:
+            Dict, edited sample info
+        """
+        sample["image"] = self._get_image_url(job.train_id, sample["id"], "original")
+        for inference in sample["inferences"]:
+
+            new_list = []
+            for saliency_map in inference["saliency_maps"]:
+                if explainers and saliency_map["explainer"] not in explainers:
+                    continue
+                saliency_map["overlay"] = self._get_image_url(job.train_id,
+                                                              saliency_map["overlay"],
+                                                              "overlay")
+                new_list.append(saliency_map)
+            inference["saliency_maps"] = new_list
+        return sample
+
+    def _get_image_url(self, train_id, image_id, image_type):
+        """Returns image's url"""
+        if self._image_url_formatter is None:
+            return image_id
+        return self._image_url_formatter(train_id, image_id, image_type)