Huawei_Technology
/
mindspore-mindinsight
Not watched
1
0
Fork 0
Code
Releases 11 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
Browse Source

add st for debugger module

tags/v1.1.0
yelihua 5 years ago
parent
0ec9e08112
commit
28a6f8405c
25 changed files with 2008 additions and 7 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +3
    -0
      mindinsight/debugger/debugger_server.py
  2. +2
    -2
      mindinsight/debugger/stream_handler/tensor_handler.py
  3. +6
    -0
      tests/st/func/debugger/__init__.py
  4. +28
    -4
      tests/st/func/debugger/conftest.py
  5. +1
    -0
      tests/st/func/debugger/expect_results/restful_results/before_train_begin.json
  6. +58
    -0
      tests/st/func/debugger/expect_results/restful_results/compare_tensors.json
  7. +1
    -0
      tests/st/func/debugger/expect_results/restful_results/create_and_delete_watchpoint.json
  8. +672
    -0
      tests/st/func/debugger/expect_results/restful_results/retrieve_aggregation_scope_node.json
  9. +542
    -0
      tests/st/func/debugger/expect_results/restful_results/retrieve_all.json
  10. +23
    -0
      tests/st/func/debugger/expect_results/restful_results/retrieve_empty_tensor_history.json
  11. +1
    -0
      tests/st/func/debugger/expect_results/restful_results/retrieve_empty_watchpoint_hit_list
  12. +39
    -0
      tests/st/func/debugger/expect_results/restful_results/retrieve_full_tensor_history.json
  13. +1
    -0
      tests/st/func/debugger/expect_results/restful_results/retrieve_next_node_on_gpu.json
  14. +1
    -0
      tests/st/func/debugger/expect_results/restful_results/retrieve_node_by_bfs.json
  15. +1
    -0
      tests/st/func/debugger/expect_results/restful_results/retrieve_node_by_bfs_ascend.json
  16. +1
    -0
      tests/st/func/debugger/expect_results/restful_results/retrieve_scope_node.json
  17. +1
    -0
      tests/st/func/debugger/expect_results/restful_results/retrieve_single_node.json
  18. +1
    -0
      tests/st/func/debugger/expect_results/restful_results/retrieve_single_watchpoint_hit.json
  19. +1
    -0
      tests/st/func/debugger/expect_results/restful_results/retrieve_tensor_value.json
  20. +1
    -0
      tests/st/func/debugger/expect_results/restful_results/retrieve_watchpoint_hit.json
  21. +1
    -0
      tests/st/func/debugger/expect_results/restful_results/search_unwatched_leaf_node.json
  22. +206
    -0
      tests/st/func/debugger/mock_ms_client.py
  23. +4
    -1
      tests/st/func/debugger/test_graph_handler.py
  24. +337
    -0
      tests/st/func/debugger/test_restful_api.py
  25. +76
    -0
      tests/st/func/debugger/utils.py

+ 3
- 0
mindinsight/debugger/debugger_server.py View File

@@ -72,6 +72,8 @@ class DebuggerServer:

def stop(self):
"""Stop debugger server."""
log.info("Send terminate info to client.")
self.control({'mode': 'terminate'})
self.grpc_server_manager.stop(grace=None)
self.back_server.join()
log.info("Stop debugger server.")
@@ -698,6 +700,7 @@ class DebuggerServer:
metadata_stream (MetadataHandler): The metadata stream handler.
"""
metadata_stream.state = 'pending'
self.cache_store.clean_data()
event = get_ack_reply()
event.exit = True
self.cache_store.put_command(event)


+ 2
- 2
mindinsight/debugger/stream_handler/tensor_handler.py View File

@@ -227,9 +227,9 @@ class TensorHandler(StreamHandlerBase):
"""Update has_prev_step field in tensor info."""
flag = None
cur_tensor_value = bool(tensor_info and tensor_info.get('value') is not None)
if node_type == NodeTypeEnum.PARAMETER.value and cur_tensor_value:
if node_type == NodeTypeEnum.PARAMETER.value:
flag = self._get_prev_tensor_value_status(tensor_name)
if flag:
if flag and cur_tensor_value:
tensor_info['has_prev_step'] = True
return flag



+ 6
- 0
tests/st/func/debugger/__init__.py View File

@@ -12,3 +12,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
"""
Function:
Test debugger system test.
Usage:
pytest tests/st/func/debugger
"""

+ 28
- 4
tests/st/func/debugger/conftest.py View File

@@ -16,20 +16,28 @@
Description: This file is used for some common util.
"""
import os
from unittest.mock import Mock

import pytest
from flask import Response

from mindinsight.conf import settings
from mindinsight.datavisual.utils import tools
from mindinsight.debugger.proto import ms_graph_pb2
from mindinsight.debugger.stream_handler.graph_handler import GraphHandler


graph_proto_file = os.path.join(
GRAPH_PROTO_FILE = os.path.join(
os.path.dirname(__file__), '../../../utils/resource/graph_pb/lenet.pb'
)

DEBUGGER_BASE_URL = '/v1/mindinsight/debugger'
DEBUGGER_EXPECTED_RESULTS = os.path.join(os.path.dirname(__file__), 'expect_results')


def init_graph_handler():
"""Init graph proto."""
with open(graph_proto_file, 'rb') as f:
content = f.read()
with open(GRAPH_PROTO_FILE, 'rb') as file_handler:
content = file_handler.read()

graph = ms_graph_pb2.GraphProto()
graph.ParseFromString(content)
@@ -38,3 +46,19 @@ def init_graph_handler():
graph_handler.put(graph)

return graph_handler


@pytest.fixture(scope='session')
def app_client():
"""This fixture is flask server."""
packages = ["mindinsight.backend.debugger"]
settings.ENABLE_DEBUGGER = True
mock_obj = Mock(return_value=packages)
tools.find_app_package = mock_obj

from mindinsight.backend.application import APP
from mindinsight.backend.debugger.debugger_api import BACKEND_SERVER
APP.response_class = Response
client = APP.test_client()
yield client
BACKEND_SERVER.stop()

+ 1
- 0
tests/st/func/debugger/expect_results/restful_results/before_train_begin.json View File

@@ -0,0 +1 @@
{"metadata": {"state": "pending", "step": 0, "device_name": "", "ip": "", "node_name": "", "backend": ""}}

+ 58
- 0
tests/st/func/debugger/expect_results/restful_results/compare_tensors.json View File

@@ -0,0 +1,58 @@
{
"tensor_value": {
"full_name": "Default/args0:0",
"step": 3,
"dtype": "DT_FLOAT32",
"shape": [
2,
3
],
"diff": [
[
[
1.0,
1.0,
0.0
],
[
2.0,
2.0,
0.0
],
[
3.0,
3.0,
0.0
]
],
[
[
4.0,
4.0,
0.0
],
[
5.0,
5.0,
0.0
],
[
6.0,
6.0,
0.0
]
]
],
"statistics": {
"max": 0.0,
"min": 0.0,
"avg": 0.0,
"count": 6,
"nan_count": 0,
"neg_inf_count": 0,
"pos_inf_count": 0,
"overall_max": 0.0,
"overall_min": 0.0
}
}
}

+ 1
- 0
tests/st/func/debugger/expect_results/restful_results/create_and_delete_watchpoint.json View File

@@ -0,0 +1 @@
{"watch_points": [{"id": 1, "watch_condition": {"condition": "MAX_GT", "param": 1.0}}, {"id": 2, "watch_condition": {"condition": "MAX_LT", "param": -1.0}}, {"id": 3, "watch_condition": {"condition": "MIN_GT", "param": 1e+32}}, {"id": 5, "watch_condition": {"condition": "MAX_MIN_GT", "param": 0}}, {"id": 6, "watch_condition": {"condition": "MAX_MIN_LT", "param": 0}}, {"id": 7, "watch_condition": {"condition": "MEAN_GT", "param": 0}}, {"id": 8, "watch_condition": {"condition": "MEAN_LT", "param": 0}}, {"id": 9, "watch_condition": {"condition": "INF"}}, {"id": 10, "watch_condition": {"condition": "OVERFLOW"}}]}

+ 672
- 0
tests/st/func/debugger/expect_results/restful_results/retrieve_aggregation_scope_node.json View File

@@ -0,0 +1,672 @@
{
"graph": {
"nodes": [
{
"name": "Default/optimizer-Momentum/Parameter[18]_7/fc3.bias",
"type": "Parameter",
"attr": {
"type": "DT_TENSOR[DT_FLOAT32]",
"shape": "[[10]]"
},
"input": {},
"output": {
"Default/optimizer-Momentum/ApplyMomentum[8]_1/ApplyMomentum-op22": {
"shape": [
[
10
]
],
"edge_type": "data",
"independent_layout": true,
"data_type": "DT_TENSOR[DT_FLOAT32]"
}
},
"output_i": 0,
"proxy_input": {},
"proxy_output": {
"Default/optimizer-Momentum/ApplyMomentum[8]_1": {
"edge_type": "data"
}
},
"subnode_count": 0,
"independent_layout": true
},
{
"name": "Default/optimizer-Momentum/Parameter[18]_7/moments.fc3.bias",
"type": "Parameter",
"attr": {
"type": "DT_TENSOR[DT_FLOAT32]",
"shape": "[[10]]"
},
"input": {},
"output": {
"Default/optimizer-Momentum/ApplyMomentum[8]_1/ApplyMomentum-op22": {
"shape": [
[
10
]
],
"edge_type": "data",
"independent_layout": true,
"data_type": "DT_TENSOR[DT_FLOAT32]"
}
},
"output_i": 0,
"proxy_input": {},
"proxy_output": {
"Default/optimizer-Momentum/ApplyMomentum[8]_1": {
"edge_type": "data"
}
},
"subnode_count": 0,
"independent_layout": true
},
{
"name": "Default/optimizer-Momentum/Parameter[18]_7/learning_rate",
"type": "Parameter",
"attr": {
"type": "DT_TENSOR[DT_FLOAT32]",
"shape": "[[]]"
},
"input": {},
"output": {
"Default/optimizer-Momentum/ApplyMomentum[8]_1/ApplyMomentum-op22": {
"shape": [
[]
],
"edge_type": "data",
"independent_layout": true,
"data_type": "DT_TENSOR[DT_FLOAT32]"
},
"Default/optimizer-Momentum/ApplyMomentum[8]_1/ApplyMomentum-op25": {
"shape": [
[]
],
"edge_type": "data",
"independent_layout": true,
"data_type": "DT_TENSOR[DT_FLOAT32]"
},
"Default/optimizer-Momentum/ApplyMomentum[8]_1/ApplyMomentum-op30": {
"shape": [
[]
],
"edge_type": "data",
"independent_layout": true,
"data_type": "DT_TENSOR[DT_FLOAT32]"
},
"Default/optimizer-Momentum/ApplyMomentum[8]_1/ApplyMomentum-op33": {
"shape": [
[]
],
"edge_type": "data",
"independent_layout": true,
"data_type": "DT_TENSOR[DT_FLOAT32]"
},
"Default/optimizer-Momentum/ApplyMomentum[8]_1/ApplyMomentum-op38": {
"shape": [
[]
],
"edge_type": "data",
"independent_layout": true,
"data_type": "DT_TENSOR[DT_FLOAT32]"
},
"Default/optimizer-Momentum/ApplyMomentum[8]_1/ApplyMomentum-op41": {
"shape": [
[]
],
"edge_type": "data",
"independent_layout": true,
"data_type": "DT_TENSOR[DT_FLOAT32]"
},
"Default/optimizer-Momentum/ApplyMomentum[8]_1/ApplyMomentum-op49": {
"shape": [
[]
],
"edge_type": "data",
"independent_layout": true,
"data_type": "DT_TENSOR[DT_FLOAT32]"
},
"Default/optimizer-Momentum/ApplyMomentum[8]_1/ApplyMomentum-op56": {
"shape": [
[]
],
"edge_type": "data",
"independent_layout": true,
"data_type": "DT_TENSOR[DT_FLOAT32]"
}
},
"output_i": 0,
"proxy_input": {},
"proxy_output": {
"Default/optimizer-Momentum/ApplyMomentum[8]_1": {
"edge_type": "data"
}
},
"subnode_count": 0,
"independent_layout": true
},
{
"name": "Default/optimizer-Momentum/Parameter[18]_7/momentum",
"type": "Parameter",
"attr": {
"type": "DT_TENSOR[DT_FLOAT32]",
"shape": "[[]]"
},
"input": {},
"output": {
"Default/optimizer-Momentum/ApplyMomentum[8]_1/ApplyMomentum-op22": {
"shape": [
[]
],
"edge_type": "data",
"independent_layout": true,
"data_type": "DT_TENSOR[DT_FLOAT32]"
},
"Default/optimizer-Momentum/ApplyMomentum[8]_1/ApplyMomentum-op25": {
"shape": [
[]
],
"edge_type": "data",
"independent_layout": true,
"data_type": "DT_TENSOR[DT_FLOAT32]"
},
"Default/optimizer-Momentum/ApplyMomentum[8]_1/ApplyMomentum-op30": {
"shape": [
[]
],
"edge_type": "data",
"independent_layout": true,
"data_type": "DT_TENSOR[DT_FLOAT32]"
},
"Default/optimizer-Momentum/ApplyMomentum[8]_1/ApplyMomentum-op33": {
"shape": [
[]
],
"edge_type": "data",
"independent_layout": true,
"data_type": "DT_TENSOR[DT_FLOAT32]"
},
"Default/optimizer-Momentum/ApplyMomentum[8]_1/ApplyMomentum-op38": {
"shape": [
[]
],
"edge_type": "data",
"independent_layout": true,
"data_type": "DT_TENSOR[DT_FLOAT32]"
},
"Default/optimizer-Momentum/ApplyMomentum[8]_1/ApplyMomentum-op41": {
"shape": [
[]
],
"edge_type": "data",
"independent_layout": true,
"data_type": "DT_TENSOR[DT_FLOAT32]"
},
"Default/optimizer-Momentum/ApplyMomentum[8]_1/ApplyMomentum-op49": {
"shape": [
[]
],
"edge_type": "data",
"independent_layout": true,
"data_type": "DT_TENSOR[DT_FLOAT32]"
},
"Default/optimizer-Momentum/ApplyMomentum[8]_1/ApplyMomentum-op56": {
"shape": [
[]
],
"edge_type": "data",
"independent_layout": true,
"data_type": "DT_TENSOR[DT_FLOAT32]"
}
},
"output_i": 0,
"proxy_input": {},
"proxy_output": {
"Default/optimizer-Momentum/ApplyMomentum[8]_1": {
"edge_type": "data"
}
},
"subnode_count": 0,
"independent_layout": true
},
{
"name": "Default/optimizer-Momentum/Parameter[18]_7/fc3.weight",
"type": "Parameter",
"attr": {
"type": "DT_TENSOR[DT_FLOAT32]",
"shape": "[[10, 84]]"
},
"input": {},
"output": {
"Default/optimizer-Momentum/ApplyMomentum[8]_1/ApplyMomentum-op25": {
"shape": [
[
10,
84
]
],
"edge_type": "data",
"independent_layout": true,
"data_type": "DT_TENSOR[DT_FLOAT32]"
}
},
"output_i": 0,
"proxy_input": {},
"proxy_output": {
"Default/optimizer-Momentum/ApplyMomentum[8]_1": {
"edge_type": "data"
}
},
"subnode_count": 0,
"independent_layout": true
},
{
"name": "Default/optimizer-Momentum/Parameter[18]_7/moments.fc3.weight",
"type": "Parameter",
"attr": {
"type": "DT_TENSOR[DT_FLOAT32]",
"shape": "[[10, 84]]"
},
"input": {},
"output": {
"Default/optimizer-Momentum/ApplyMomentum[8]_1/ApplyMomentum-op25": {
"shape": [
[
10,
84
]
],
"edge_type": "data",
"independent_layout": true,
"data_type": "DT_TENSOR[DT_FLOAT32]"
}
},
"output_i": 0,
"proxy_input": {},
"proxy_output": {
"Default/optimizer-Momentum/ApplyMomentum[8]_1": {
"edge_type": "data"
}
},
"subnode_count": 0,
"independent_layout": true
},
{
"name": "Default/optimizer-Momentum/Parameter[18]_7/fc2.bias",
"type": "Parameter",
"attr": {
"type": "DT_TENSOR[DT_FLOAT32]",
"shape": "[[84]]"
},
"input": {},
"output": {
"Default/optimizer-Momentum/ApplyMomentum[8]_1/ApplyMomentum-op30": {
"shape": [
[
84
]
],
"edge_type": "data",
"independent_layout": true,
"data_type": "DT_TENSOR[DT_FLOAT32]"
}
},
"output_i": 0,
"proxy_input": {},
"proxy_output": {
"Default/optimizer-Momentum/ApplyMomentum[8]_1": {
"edge_type": "data"
}
},
"subnode_count": 0,
"independent_layout": true
},
{
"name": "Default/optimizer-Momentum/Parameter[18]_7/moments.fc2.bias",
"type": "Parameter",
"attr": {
"type": "DT_TENSOR[DT_FLOAT32]",
"shape": "[[84]]"
},
"input": {},
"output": {
"Default/optimizer-Momentum/ApplyMomentum[8]_1/ApplyMomentum-op30": {
"shape": [
[
84
]
],
"edge_type": "data",
"independent_layout": true,
"data_type": "DT_TENSOR[DT_FLOAT32]"
}
},
"output_i": 0,
"proxy_input": {},
"proxy_output": {
"Default/optimizer-Momentum/ApplyMomentum[8]_1": {
"edge_type": "data"
}
},
"subnode_count": 0,
"independent_layout": true
},
{
"name": "Default/optimizer-Momentum/Parameter[18]_7/fc2.weight",
"type": "Parameter",
"attr": {
"type": "DT_TENSOR[DT_FLOAT32]",
"shape": "[[84, 120]]"
},
"input": {},
"output": {
"Default/optimizer-Momentum/ApplyMomentum[8]_1/ApplyMomentum-op33": {
"shape": [
[
84,
120
]
],
"edge_type": "data",
"independent_layout": true,
"data_type": "DT_TENSOR[DT_FLOAT32]"
}
},
"output_i": 0,
"proxy_input": {},
"proxy_output": {
"Default/optimizer-Momentum/ApplyMomentum[8]_1": {
"edge_type": "data"
}
},
"subnode_count": 0,
"independent_layout": true
},
{
"name": "Default/optimizer-Momentum/Parameter[18]_7/moments.fc2.weight",
"type": "Parameter",
"attr": {
"type": "DT_TENSOR[DT_FLOAT32]",
"shape": "[[84, 120]]"
},
"input": {},
"output": {
"Default/optimizer-Momentum/ApplyMomentum[8]_1/ApplyMomentum-op33": {
"shape": [
[
84,
120
]
],
"edge_type": "data",
"independent_layout": true,
"data_type": "DT_TENSOR[DT_FLOAT32]"
}
},
"output_i": 0,
"proxy_input": {},
"proxy_output": {
"Default/optimizer-Momentum/ApplyMomentum[8]_1": {
"edge_type": "data"
}
},
"subnode_count": 0,
"independent_layout": true
},
{
"name": "Default/optimizer-Momentum/Parameter[18]_7/fc1.bias",
"type": "Parameter",
"attr": {
"type": "DT_TENSOR[DT_FLOAT32]",
"shape": "[[120]]"
},
"input": {},
"output": {
"Default/optimizer-Momentum/ApplyMomentum[8]_1/ApplyMomentum-op38": {
"shape": [
[
120
]
],
"edge_type": "data",
"independent_layout": true,
"data_type": "DT_TENSOR[DT_FLOAT32]"
}
},
"output_i": 0,
"proxy_input": {},
"proxy_output": {
"Default/optimizer-Momentum/ApplyMomentum[8]_1": {
"edge_type": "data"
}
},
"subnode_count": 0,
"independent_layout": true
},
{
"name": "Default/optimizer-Momentum/Parameter[18]_7/moments.fc1.bias",
"type": "Parameter",
"attr": {
"type": "DT_TENSOR[DT_FLOAT32]",
"shape": "[[120]]"
},
"input": {},
"output": {
"Default/optimizer-Momentum/ApplyMomentum[8]_1/ApplyMomentum-op38": {
"shape": [
[
120
]
],
"edge_type": "data",
"independent_layout": true,
"data_type": "DT_TENSOR[DT_FLOAT32]"
}
},
"output_i": 0,
"proxy_input": {},
"proxy_output": {
"Default/optimizer-Momentum/ApplyMomentum[8]_1": {
"edge_type": "data"
}
},
"subnode_count": 0,
"independent_layout": true
},
{
"name": "Default/optimizer-Momentum/Parameter[18]_7/fc1.weight",
"type": "Parameter",
"attr": {
"type": "DT_TENSOR[DT_FLOAT32]",
"shape": "[[120, 400]]"
},
"input": {},
"output": {
"Default/optimizer-Momentum/ApplyMomentum[8]_1/ApplyMomentum-op41": {
"shape": [
[
120,
400
]
],
"edge_type": "data",
"independent_layout": true,
"data_type": "DT_TENSOR[DT_FLOAT32]"
}
},
"output_i": 0,
"proxy_input": {},
"proxy_output": {
"Default/optimizer-Momentum/ApplyMomentum[8]_1": {
"edge_type": "data"
}
},
"subnode_count": 0,
"independent_layout": true
},
{
"name": "Default/optimizer-Momentum/Parameter[18]_7/moments.fc1.weight",
"type": "Parameter",
"attr": {
"type": "DT_TENSOR[DT_FLOAT32]",
"shape": "[[120, 400]]"
},
"input": {},
"output": {
"Default/optimizer-Momentum/ApplyMomentum[8]_1/ApplyMomentum-op41": {
"shape": [
[
120,
400
]
],
"edge_type": "data",
"independent_layout": true,
"data_type": "DT_TENSOR[DT_FLOAT32]"
}
},
"output_i": 0,
"proxy_input": {},
"proxy_output": {
"Default/optimizer-Momentum/ApplyMomentum[8]_1": {
"edge_type": "data"
}
},
"subnode_count": 0,
"independent_layout": true
},
{
"name": "Default/optimizer-Momentum/Parameter[18]_7/conv2.weight",
"type": "Parameter",
"attr": {
"type": "DT_TENSOR[DT_FLOAT32]",
"shape": "[[16, 6, 5, 5]]"
},
"input": {},
"output": {
"Default/optimizer-Momentum/ApplyMomentum[8]_1/ApplyMomentum-op49": {
"shape": [
[
16,
6,
5,
5
]
],
"edge_type": "data",
"independent_layout": true,
"data_type": "DT_TENSOR[DT_FLOAT32]"
}
},
"output_i": 0,
"proxy_input": {},
"proxy_output": {
"Default/optimizer-Momentum/ApplyMomentum[8]_1": {
"edge_type": "data"
}
},
"subnode_count": 0,
"independent_layout": true
},
{
"name": "Default/optimizer-Momentum/Parameter[18]_7/moments.conv2.weight",
"type": "Parameter",
"attr": {
"type": "DT_TENSOR[DT_FLOAT32]",
"shape": "[[16, 6, 5, 5]]"
},
"input": {},
"output": {
"Default/optimizer-Momentum/ApplyMomentum[8]_1/ApplyMomentum-op49": {
"shape": [
[
16,
6,
5,
5
]
],
"edge_type": "data",
"independent_layout": true,
"data_type": "DT_TENSOR[DT_FLOAT32]"
}
},
"output_i": 0,
"proxy_input": {},
"proxy_output": {
"Default/optimizer-Momentum/ApplyMomentum[8]_1": {
"edge_type": "data"
}
},
"subnode_count": 0,
"independent_layout": true
},
{
"name": "Default/optimizer-Momentum/Parameter[18]_7/conv1.weight",
"type": "Parameter",
"attr": {
"type": "DT_TENSOR[DT_FLOAT32]",
"shape": "[[6, 1, 5, 5]]"
},
"input": {},
"output": {
"Default/optimizer-Momentum/ApplyMomentum[8]_1/ApplyMomentum-op56": {
"shape": [
[
6,
1,
5,
5
]
],
"edge_type": "data",
"independent_layout": true,
"data_type": "DT_TENSOR[DT_FLOAT32]"
}
},
"output_i": 0,
"proxy_input": {},
"proxy_output": {
"Default/optimizer-Momentum/ApplyMomentum[8]_1": {
"edge_type": "data"
}
},
"subnode_count": 0,
"independent_layout": true
},
{
"name": "Default/optimizer-Momentum/Parameter[18]_7/moments.conv1.weight",
"type": "Parameter",
"attr": {
"type": "DT_TENSOR[DT_FLOAT32]",
"shape": "[[6, 1, 5, 5]]"
},
"input": {},
"output": {
"Default/optimizer-Momentum/ApplyMomentum[8]_1/ApplyMomentum-op56": {
"shape": [
[
6,
1,
5,
5
]
],
"edge_type": "data",
"independent_layout": true,
"data_type": "DT_TENSOR[DT_FLOAT32]"
}
},
"output_i": 0,
"proxy_input": {},
"proxy_output": {
"Default/optimizer-Momentum/ApplyMomentum[8]_1": {
"edge_type": "data"
}
},
"subnode_count": 0,
"independent_layout": true
}
]
}
}

+ 542
- 0
tests/st/func/debugger/expect_results/restful_results/retrieve_all.json View File

@@ -0,0 +1,542 @@
{
"metadata": {
"state": "waiting",
"step": 1,
"device_name": "0",
"node_name": "",
"backend": "Ascend"
},
"graph": {
"nodes": [
{
"name": "Default",
"type": "name_scope",
"attr": {},
"input": {
"Gradients/Default/network-WithLossCell/_backbone-LeNet5/fc3-Dense/gradBiasAdd/BiasAddGrad-op21": {
"shape": [
[
10
]
],
"edge_type": "data",
"independent_layout": false,
"data_type": "DT_TENSOR[DT_FLOAT32]"
},
"Gradients/Default/network-WithLossCell/_backbone-LeNet5/fc3-Dense/gradMatMul/MatMul[6]_5/MatMul-op24": {
"shape": [
[
10,
84
]
],
"edge_type": "data",
"independent_layout": false,
"data_type": "DT_TENSOR[DT_FLOAT32]"
},
"Gradients/Default/network-WithLossCell/_backbone-LeNet5/fc3-Dense/gradBiasAdd/BiasAddGrad-op29": {
"shape": [
[
84
]
],
"edge_type": "data",
"independent_layout": false,
"data_type": "DT_TENSOR[DT_FLOAT32]"
},
"Gradients/Default/network-WithLossCell/_backbone-LeNet5/fc3-Dense/gradMatMul/MatMul[6]_5/MatMul-op32": {
"shape": [
[
84,
120
]
],
"edge_type": "data",
"independent_layout": false,
"data_type": "DT_TENSOR[DT_FLOAT32]"
},
"Gradients/Default/network-WithLossCell/_backbone-LeNet5/fc3-Dense/gradBiasAdd/BiasAddGrad-op37": {
"shape": [
[
120
]
],
"edge_type": "data",
"independent_layout": false,
"data_type": "DT_TENSOR[DT_FLOAT32]"
},
"Gradients/Default/network-WithLossCell/_backbone-LeNet5/fc3-Dense/gradMatMul/MatMul[6]_5/MatMul-op40": {
"shape": [
[
120,
400
]
],
"edge_type": "data",
"independent_layout": false,
"data_type": "DT_TENSOR[DT_FLOAT32]"
},
"Gradients/Default/network-WithLossCell/_backbone-LeNet5/conv2-Conv2d/gradConv2D/Conv2DBackpropFilter-op48": {
"shape": [
[
16,
6,
5,
5
]
],
"edge_type": "data",
"independent_layout": false,
"data_type": "DT_TENSOR[DT_FLOAT32]"
},
"Gradients/Default/network-WithLossCell/_backbone-LeNet5/conv1-Conv2d/gradConv2D/Conv2DBackpropFilter-op55": {
"shape": [
[
6,
1,
5,
5
]
],
"edge_type": "data",
"independent_layout": false,
"data_type": "DT_TENSOR[DT_FLOAT32]"
}
},
"output": {
"Gradients/Default/network-WithLossCell/_backbone-LeNet5/conv2-Conv2d/gradConv2D/Conv2DBackpropInput-op52": {
"shape": [
[
16,
6,
5,
5
]
],
"edge_type": "data",
"independent_layout": false,
"data_type": "DT_TENSOR[DT_FLOAT32]"
},
"Gradients/Default/network-WithLossCell/_backbone-LeNet5/conv1-Conv2d/gradConv2D/Conv2DBackpropFilter-op55": {
"shape": [
[
32,
1,
32,
32
]
],
"edge_type": "data",
"independent_layout": false,
"data_type": "DT_TENSOR[DT_FLOAT32]"
},
"Gradients/Default/network-WithLossCell/_backbone-LeNet5/max_pool2d-MaxPool2d/gradMaxPoolWithArgmax/MaxPoolGradWithArgmax-op53": {
"shape": [
[
32,
6,
4,
14
]
],
"edge_type": "data",
"independent_layout": false,
"data_type": "DT_TENSOR[DT_UINT16]"
},
"Gradients/Default/network-WithLossCell/_backbone-LeNet5/max_pool2d-MaxPool2d/gradMaxPoolWithArgmax/MaxPoolGradWithArgmax-op46": {
"shape": [
[
32,
16,
4,
3
]
],
"edge_type": "data",
"independent_layout": false,
"data_type": "DT_TENSOR[DT_UINT16]"
},
"Gradients/Default/network-WithLossCell/_backbone-LeNet5/fc3-Dense/gradMatMul/MatMul[6]_5/MatMul-op40": {
"shape": [
[
32,
400
]
],
"edge_type": "data",
"independent_layout": false,
"data_type": "DT_TENSOR[DT_FLOAT32]"
},
"Gradients/Default/network-WithLossCell/_backbone-LeNet5/relu-ReLU/gradReLU/ReluGrad-op36": {
"shape": [
[
32,
120
]
],
"edge_type": "data",
"independent_layout": false,
"data_type": "DT_TENSOR[DT_FLOAT32]"
},
"Gradients/Default/network-WithLossCell/_backbone-LeNet5/fc3-Dense/gradMatMul/MatMul[6]_5/MatMul-op32": {
"shape": [
[
32,
120
]
],
"edge_type": "data",
"independent_layout": false,
"data_type": "DT_TENSOR[DT_FLOAT32]"
},
"Gradients/Default/network-WithLossCell/_backbone-LeNet5/relu-ReLU/gradReLU/ReluGrad-op28": {
"shape": [
[
32,
84
]
],
"edge_type": "data",
"independent_layout": false,
"data_type": "DT_TENSOR[DT_FLOAT32]"
},
"Gradients/Default/network-WithLossCell/_backbone-LeNet5/fc3-Dense/gradMatMul/MatMul[6]_5/MatMul-op24": {
"shape": [
[
32,
84
]
],
"edge_type": "data",
"independent_layout": false,
"data_type": "DT_TENSOR[DT_FLOAT32]"
},
"Gradients/Default/network-WithLossCell/_backbone-LeNet5/conv2-Conv2d/gradConv2D/Conv2DBackpropFilter-op48": {
"shape": [
[
32,
6,
14,
14
]
],
"edge_type": "data",
"independent_layout": false,
"data_type": "DT_TENSOR[DT_FLOAT32]"
},
"Gradients/Default/network-WithLossCell/_loss_fn-SoftmaxCrossEntropyWithLogits/gradSoftmaxCrossEntropyWithLogits/Mul-op20": {
"shape": [
[
32,
10
]
],
"edge_type": "data",
"independent_layout": false,
"data_type": "DT_TENSOR[DT_FLOAT32]"
},
"Gradients/Default/network-WithLossCell/_backbone-LeNet5/relu-ReLU/gradReLU/ReluGradV2-op92": {
"shape": [
[
32,
1,
10,
10,
2
]
],
"edge_type": "data",
"independent_layout": false,
"data_type": "DT_TENSOR[DT_UINT8]"
},
"Gradients/Default/network-WithLossCell/_backbone-LeNet5/relu-ReLU/gradReLU/ReluGradV2-op94": {
"shape": [
[
32,
1,
28,
28,
2
]
],
"edge_type": "data",
"independent_layout": false,
"data_type": "DT_TENSOR[DT_UINT8]"
}
},
"output_i": 0,
"proxy_input": {},
"proxy_output": {},
"subnode_count": 7,
"independent_layout": false
},
{
"name": "Gradients",
"type": "name_scope",
"attr": {},
"input": {
"Default/tuple_getitem[10]_0/tuple_getitem-op210": {
"shape": [
[
32,
10
]
],
"edge_type": "data",
"independent_layout": false,
"data_type": "DT_TENSOR[DT_FLOAT32]"
},
"Default/network-WithLossCell/_backbone-LeNet5/relu-ReLU/ReLU-op15": {
"shape": [
[
32,
84
]
],
"edge_type": "data",
"independent_layout": false,
"data_type": "DT_TENSOR[DT_FLOAT32]"
},
"Default/network-WithLossCell/_backbone-LeNet5/relu-ReLU/ReLU-op12": {
"shape": [
[
32,
120
]
],
"edge_type": "data",
"independent_layout": false,
"data_type": "DT_TENSOR[DT_FLOAT32]"
},
"Default/network-WithLossCell/_backbone-LeNet5/max_pool2d-MaxPool2d/Cast-op205": {
"shape": [
[
32,
16,
10,
10
]
],
"edge_type": "data",
"independent_layout": false,
"data_type": "DT_TENSOR[DT_FLOAT32]"
},
"Default/tuple_getitem[10]_0/tuple_getitem-op206": {
"shape": [
[
32,
16,
4,
3
]
],
"edge_type": "data",
"independent_layout": false,
"data_type": "DT_TENSOR[DT_UINT16]"
},
"Default/tuple_getitem[10]_0/tuple_getitem-op202": {
"shape": [
[
32,
1,
10,
10,
2
]
],
"edge_type": "data",
"independent_layout": false,
"data_type": "DT_TENSOR[DT_UINT8]"
},
"Default/tuple_getitem[10]_0/tuple_getitem-op197": {
"shape": [
[
32,
6,
14,
14
]
],
"edge_type": "data",
"independent_layout": false,
"data_type": "DT_TENSOR[DT_FLOAT32]"
},
"Default/network-WithLossCell/_backbone-LeNet5/conv2-Conv2d/Cast-op188": {
"shape": [
[
16,
6,
5,
5
]
],
"edge_type": "data",
"independent_layout": false,
"data_type": "DT_TENSOR[DT_FLOAT32]"
},
"Default/network-WithLossCell/_backbone-LeNet5/max_pool2d-MaxPool2d/Cast-op195": {
"shape": [
[
32,
6,
28,
28
]
],
"edge_type": "data",
"independent_layout": false,
"data_type": "DT_TENSOR[DT_FLOAT32]"
},
"Default/tuple_getitem[10]_0/tuple_getitem-op196": {
"shape": [
[
32,
6,
4,
14
]
],
"edge_type": "data",
"independent_layout": false,
"data_type": "DT_TENSOR[DT_UINT16]"
},
"Default/tuple_getitem[10]_0/tuple_getitem-op192": {
"shape": [
[
32,
1,
28,
28,
2
]
],
"edge_type": "data",
"independent_layout": false,
"data_type": "DT_TENSOR[DT_UINT8]"
},
"Default/network-WithLossCell/_backbone-LeNet5/conv1-Conv2d/Cast-op190": {
"shape": [
[
32,
1,
32,
32
]
],
"edge_type": "data",
"independent_layout": false,
"data_type": "DT_TENSOR[DT_FLOAT32]"
},
"Default/network-WithLossCell/_backbone-LeNet5/flatten-Flatten/Reshape-op9": {
"shape": [
[
32,
400
]
],
"edge_type": "data",
"independent_layout": false,
"data_type": "DT_TENSOR[DT_FLOAT32]"
}
},
"output": {
"Default/optimizer-Momentum/ApplyMomentum[8]_1/ApplyMomentum-op22": {
"shape": [
[
10
]
],
"edge_type": "data",
"independent_layout": false,
"data_type": "DT_TENSOR[DT_FLOAT32]"
},
"Default/optimizer-Momentum/ApplyMomentum[8]_1/ApplyMomentum-op30": {
"shape": [
[
84
]
],
"edge_type": "data",
"independent_layout": false,
"data_type": "DT_TENSOR[DT_FLOAT32]"
},
"Default/optimizer-Momentum/ApplyMomentum[8]_1/ApplyMomentum-op38": {
"shape": [
[
120
]
],
"edge_type": "data",
"independent_layout": false,
"data_type": "DT_TENSOR[DT_FLOAT32]"
},
"Default/optimizer-Momentum/ApplyMomentum[8]_1/ApplyMomentum-op49": {
"shape": [
[
16,
6,
5,
5
]
],
"edge_type": "data",
"independent_layout": false,
"data_type": "DT_TENSOR[DT_FLOAT32]"
},
"Default/optimizer-Momentum/ApplyMomentum[8]_1/ApplyMomentum-op56": {
"shape": [
[
6,
1,
5,
5
]
],
"edge_type": "data",
"independent_layout": false,
"data_type": "DT_TENSOR[DT_FLOAT32]"
},
"Default/optimizer-Momentum/ApplyMomentum[8]_1/ApplyMomentum-op25": {
"shape": [
[
10,
84
]
],
"edge_type": "data",
"independent_layout": false,
"data_type": "DT_TENSOR[DT_FLOAT32]"
},
"Default/optimizer-Momentum/ApplyMomentum[8]_1/ApplyMomentum-op33": {
"shape": [
[
84,
120
]
],
"edge_type": "data",
"independent_layout": false,
"data_type": "DT_TENSOR[DT_FLOAT32]"
},
"Default/optimizer-Momentum/ApplyMomentum[8]_1/ApplyMomentum-op41": {
"shape": [
[
120,
400
]
],
"edge_type": "data",
"independent_layout": false,
"data_type": "DT_TENSOR[DT_FLOAT32]"
}
},
"output_i": 0,
"proxy_input": {},
"proxy_output": {},
"subnode_count": 1,
"independent_layout": false
}
]
},
"watch_points": []
}

+ 23
- 0
tests/st/func/debugger/expect_results/restful_results/retrieve_empty_tensor_history.json View File

@@ -0,0 +1,23 @@
{
"tensor_history": [
{
"name": "Default/TransData-op99:0",
"full_name": "Default/TransData-op99:0",
"node_type": "TransData",
"type": "output"
},
{
"name": "Default/args0:0",
"full_name": "Default/args0:0",
"node_type": "Parameter",
"type": "input"
}
],
"metadata": {
"state": "waiting",
"step": 1,
"device_name": "0",
"node_name": "",
"backend": "Ascend"
}
}

+ 1
- 0
tests/st/func/debugger/expect_results/restful_results/retrieve_empty_watchpoint_hit_list View File

@@ -0,0 +1 @@
{"watch_point_hits": []}

+ 39
- 0
tests/st/func/debugger/expect_results/restful_results/retrieve_full_tensor_history.json View File

@@ -0,0 +1,39 @@
{
"tensor_history": [
{
"name": "Default/TransData-op99:0",
"full_name": "Default/TransData-op99:0",
"node_type": "TransData",
"type": "output",
"step": 1,
"dtype": "DT_FLOAT32",
"shape": [
2,
3
],
"has_prev_step": false,
"value": "click to view"
},
{
"name": "Default/args0:0",
"full_name": "Default/args0:0",
"node_type": "Parameter",
"type": "input",
"step": 1,
"dtype": "DT_FLOAT32",
"shape": [
2,
3
],
"has_prev_step": true,
"value": "click to view"
}
],
"metadata": {
"state": "waiting",
"step": 1,
"device_name": "0",
"node_name": "",
"backend": "Ascend"
}
}

+ 1
- 0
tests/st/func/debugger/expect_results/restful_results/retrieve_next_node_on_gpu.json
File diff suppressed because it is too large
View File


+ 1
- 0
tests/st/func/debugger/expect_results/restful_results/retrieve_node_by_bfs.json
File diff suppressed because it is too large
View File


+ 1
- 0
tests/st/func/debugger/expect_results/restful_results/retrieve_node_by_bfs_ascend.json View File

@@ -0,0 +1 @@
{}

+ 1
- 0
tests/st/func/debugger/expect_results/restful_results/retrieve_scope_node.json
File diff suppressed because it is too large
View File


+ 1
- 0
tests/st/func/debugger/expect_results/restful_results/retrieve_single_node.json
File diff suppressed because it is too large
View File


+ 1
- 0
tests/st/func/debugger/expect_results/restful_results/retrieve_single_watchpoint_hit.json
File diff suppressed because it is too large
View File


+ 1
- 0
tests/st/func/debugger/expect_results/restful_results/retrieve_tensor_value.json View File

@@ -0,0 +1 @@
{"tensor_value": {"full_name": "Default/TransData-op99:0", "step": 1, "dtype": "DT_FLOAT32", "shape": [2, 3], "has_prev_step": false, "statistics": {"max": 6.0, "min": 5.0, "avg": 5.5, "count": 2, "nan_count": 0, "neg_inf_count": 0, "pos_inf_count": 0, "overall_max": 6.0, "overall_min": 1.0}, "value": [5.0, 6.0], "name": "Default/TransData-op99:0"}}

+ 1
- 0
tests/st/func/debugger/expect_results/restful_results/retrieve_watchpoint_hit.json View File

@@ -0,0 +1 @@
{"watch_point_hits": [{"node_name": "Default/TransData-op99", "watch_points": [{"id": 1, "watch_condition": {"condition": "INF"}}]}, {"node_name": "Default/optimizer-Momentum/ApplyMomentum[8]_1/ApplyMomentum-op25", "watch_points": [{"id": 1, "watch_condition": {"condition": "INF"}}]}]}

+ 1
- 0
tests/st/func/debugger/expect_results/restful_results/search_unwatched_leaf_node.json View File

@@ -0,0 +1 @@
{"nodes": [{"name": "Default", "type": null, "nodes": [{"name": "Default/optimizer-Momentum", "type": null, "nodes": [{"name": "Default/optimizer-Momentum/Parameter[18]_7", "type": null, "nodes": [{"name": "Default/optimizer-Momentum/Parameter[18]_7/moments.fc3.bias", "type": "Parameter", "nodes": [], "watched": 0}], "watched": 1}], "watched": 1}], "watched": 1}]}

+ 206
- 0
tests/st/func/debugger/mock_ms_client.py View File

@@ -0,0 +1,206 @@
# Copyright 2020 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
"""Mocked MindSpore debugger client."""
from threading import Thread

import grpc
import numpy as np

from mindinsight.debugger.proto import ms_graph_pb2
from mindinsight.debugger.proto.debug_grpc_pb2 import Metadata, WatchpointHit, Chunk, EventReply
from mindinsight.debugger.proto.debug_grpc_pb2_grpc import EventListenerStub
from mindinsight.debugger.proto.ms_graph_pb2 import TensorProto, DataType
from tests.st.func.debugger.conftest import GRAPH_PROTO_FILE


class MockDebuggerClient:
"""Mocked Debugger client."""

def __init__(self, hostname='localhost:50051', backend='Ascend'):
channel = grpc.insecure_channel(hostname)
self.stub = EventListenerStub(channel)
self.flag = True
self._step = 0
self._watchpoint_id = 0
self._leaf_node = []
self._cur_node = ''
self._backend = backend

def _clean(self):
"""Clean cache."""
self._step = 0
self._watchpoint_id = 0
self._leaf_node = []
self._cur_node = ''

def get_thread_instance(self):
"""Get debugger client thread."""
return MockDebuggerClientThread(self)

def next_node(self, name=None):
"""Update the current node to next node."""
if not self._cur_node:
self._cur_node = self._leaf_node[0]
return
cur_index = self._leaf_node.index(self._cur_node)
# if name is not None, go to the specified node.
if not name:
next_index = cur_index + 1
else:
next_index = self._leaf_node.index(name)
# update step
if next_index <= cur_index or next_index == len(self._leaf_node):
self._step += 1
# update current node
if next_index == len(self._leaf_node):
self._cur_node = self._leaf_node[0]
else:
self._cur_node = self._leaf_node[next_index]

def command_loop(self):
"""Wait for the command."""
total_steps = 100
wait_flag = True
while self.flag and wait_flag:
if self._step > total_steps:
self.send_metadata_cmd(training_done=True)
return
wait_flag = self._wait_cmd()

def _wait_cmd(self):
"""Wait for command and deal with command."""
metadata = self.get_metadata_cmd()
response = self.stub.WaitCMD(metadata)
assert response.status == EventReply.Status.OK
if response.HasField('run_cmd'):
self._deal_with_run_cmd(response)
elif response.HasField('view_cmd'):
for tensor in response.view_cmd.tensors:
self.send_tensor_cmd(in_tensor=tensor)
elif response.HasField('set_cmd'):
self._watchpoint_id += 1
elif response.HasField('exit'):
self._watchpoint_id = 0
self._step = 0
return False
return True

def _deal_with_run_cmd(self, response):
self._step += response.run_cmd.run_steps
if response.run_cmd.run_level == 'node':
self.next_node(response.run_cmd.node_name)
if self._watchpoint_id > 0:
self.send_watchpoint_hit()

def get_metadata_cmd(self, training_done=False):
"""Construct metadata message."""
metadata = Metadata()
metadata.device_name = '0'
metadata.cur_step = self._step
metadata.cur_node = self._cur_node
metadata.backend = self._backend
metadata.training_done = training_done
return metadata

def send_metadata_cmd(self, training_done=False):
"""Send metadata command."""
self._clean()
metadata = self.get_metadata_cmd(training_done)
response = self.stub.SendMetadata(metadata)
assert response.status == EventReply.Status.OK
if training_done is False:
self.send_graph_cmd()

def send_graph_cmd(self):
"""Send graph to debugger server."""
self._step = 1
with open(GRAPH_PROTO_FILE, 'rb') as file_handle:
content = file_handle.read()
size = len(content)
graph = ms_graph_pb2.GraphProto()
graph.ParseFromString(content)
graph.name = 'graph_name'
self._leaf_node = [node.full_name for node in graph.node]
# the max limit of grpc data size is 4kb
# split graph into 3kb per chunk
chunk_size = 1024 * 1024 * 3
chunks = []
for index in range(0, size, chunk_size):
sub_size = min(chunk_size, size - index)
sub_chunk = Chunk(buffer=content[index: index + sub_size])
chunks.append(sub_chunk)
response = self.stub.SendGraph(self._generate_graph(chunks))
assert response.status == EventReply.Status.OK
# go to command loop
self.command_loop()

@staticmethod
def _generate_graph(chunks):
"""Construct graph generator."""
for buffer in chunks:
yield buffer

def send_tensor_cmd(self, in_tensor=None):
"""Send tensor info with value."""
response = self.stub.SendTensors(self.generate_tensor(in_tensor))
assert response.status == EventReply.Status.OK

@staticmethod
def generate_tensor(in_tensor=None):
"""Generate tensor message."""
tensor_content = np.asarray([1, 2, 3, 4, 5, 6]).astype(np.float32).tobytes()
tensors = [TensorProto(), TensorProto()]
tensors[0].CopyFrom(in_tensor)
tensors[0].data_type = DataType.DT_FLOAT32
tensors[0].dims.extend([2, 3])
tensors[1].CopyFrom(tensors[0])
tensors[0].tensor_content = tensor_content[:12]
tensors[1].tensor_content = tensor_content[12:]
tensors[0].finished = 0
tensors[1].finished = 1
for sub_tensor in tensors:
yield sub_tensor

def send_watchpoint_hit(self):
"""Send watchpoint hit value."""
tensors = [TensorProto(node_name='Default/TransData-op99', slot='0'),
TensorProto(node_name='Default/optimizer-Momentum/ApplyMomentum-op25', slot='0')]
response = self.stub.SendWatchpointHits(self._generate_hits(tensors))
assert response.status == EventReply.Status.OK

@staticmethod
def _generate_hits(tensors):
"""Construct watchpoint hits."""
for tensor in tensors:
hit = WatchpointHit()
hit.id = 1
hit.tensor.CopyFrom(tensor)
yield hit


class MockDebuggerClientThread:
"""Mocked debugger client thread."""
def __init__(self, debugger_client):
self._debugger_client = debugger_client
self._debugger_client_thread = Thread(target=debugger_client.send_metadata_cmd)

def __enter__(self, backend='Ascend'):
self._debugger_client.flag = True
self._debugger_client_thread.start()
return self._debugger_client_thread

def __exit__(self, exc_type, exc_val, exc_tb):
self._debugger_client_thread.join(timeout=5)
self._debugger_client.flag = False

+ 4
- 1
tests/st/func/debugger/test_graph_handler.py View File

@@ -13,7 +13,10 @@
# limitations under the License.
# ============================================================================
"""
Description: This file is used for testing graph handler.
Function:
Test query debugger graph handler.
Usage:
pytest tests/st/func/debugger/test_graph_handler.py
"""
import os



+ 337
- 0
tests/st/func/debugger/test_restful_api.py View File

@@ -0,0 +1,337 @@
# Copyright 2020 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
"""
Function:
Test query debugger restful api.
Usage:
pytest tests/st/func/debugger/test_restful_api.py
"""
import os

import pytest

from tests.st.func.debugger.conftest import DEBUGGER_BASE_URL
from tests.st.func.debugger.mock_ms_client import MockDebuggerClient
from tests.st.func.debugger.utils import check_waiting_state, get_request_result, \
send_and_compare_result


class TestAscendDebugger:
"""Test debugger on Ascend backend."""

@classmethod
def setup_class(cls):
"""Setup class."""
cls._debugger_client = MockDebuggerClient(backend='Ascend')

@staticmethod
def _send_terminate_cmd(app_client):
"""Send terminate command to debugger client."""
url = os.path.join(DEBUGGER_BASE_URL, 'control')
body_data = {'mode': 'terminate'}
send_and_compare_result(app_client, url, body_data)

@staticmethod
def _create_watchpoint(app_client, condition, expect_id):
"""Create watchpoint."""
url = 'create_watchpoint'
body_data = {'condition': condition,
'watch_nodes': ['Default/optimizer-Momentum/Parameter[18]_7',
'Default/TransData-op99']}
res = get_request_result(app_client, url, body_data)
assert res.get('id') == expect_id

@pytest.mark.level0
@pytest.mark.env_single
@pytest.mark.platform_x86_cpu
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_gpu_training
@pytest.mark.platform_x86_ascend_training
def test_before_train_begin(self, app_client):
"""Test retrieve all."""
url = 'retrieve'
body_data = {'mode': 'all'}
expect_file = 'before_train_begin.json'
send_and_compare_result(app_client, url, body_data, expect_file)

@pytest.mark.level0
@pytest.mark.env_single
@pytest.mark.platform_x86_cpu
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_gpu_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.parametrize("body_data, expect_file", [
({'mode': 'all'}, 'retrieve_all.json'),
({'mode': 'node', 'params': {'name': 'Default'}}, 'retrieve_scope_node.json'),
({'mode': 'node', 'params': {'name': 'Default/optimizer-Momentum/Parameter[18]_7'}},
'retrieve_aggregation_scope_node.json'),
({'mode': 'node', 'params': {
'name': 'Default/TransData-op99',
'single_node': True}}, 'retrieve_single_node.json'),
({'mode': 'watchpoint_hit'}, 'retrieve_empty_watchpoint_hit_list')
])
def test_retrieve_when_train_begin(self, app_client, body_data, expect_file):
"""Test retrieve when train_begin."""
url = 'retrieve'
with self._debugger_client.get_thread_instance():
flag = check_waiting_state(app_client)
assert flag is True
send_and_compare_result(app_client, url, body_data, expect_file)
self._send_terminate_cmd(app_client)

@pytest.mark.level0
@pytest.mark.env_single
@pytest.mark.platform_x86_cpu
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_gpu_training
@pytest.mark.platform_x86_ascend_training
def test_create_and_delete_watchpoint(self, app_client):
"""Test create and delete watchpoint."""
with self._debugger_client.get_thread_instance():
flag = check_waiting_state(app_client)
assert flag is True
conditions = [
{'condition': 'MAX_GT', 'param': 1.0},
{'condition': 'MAX_LT', 'param': -1.0},
{'condition': 'MIN_GT', 'param': 1e+32},
{'condition': 'MIN_LT', 'param': -1e+32},
{'condition': 'MAX_MIN_GT', 'param': 0},
{'condition': 'MAX_MIN_LT', 'param': 0},
{'condition': 'MEAN_GT', 'param': 0},
{'condition': 'MEAN_LT', 'param': 0},
{'condition': 'INF'},
{'condition': 'OVERFLOW'},
]
for idx, condition in enumerate(conditions):
self._create_watchpoint(app_client, condition, idx + 1)
# delete 4-th watchpoint
url = 'delete_watchpoint'
body_data = {'watch_point_id': 4}
get_request_result(app_client, url, body_data)
# test watchpoint list
url = 'retrieve'
body_data = {'mode': 'watchpoint'}
expect_file = 'create_and_delete_watchpoint.json'
send_and_compare_result(app_client, url, body_data, expect_file)
self._send_terminate_cmd(app_client)

@pytest.mark.level0
@pytest.mark.env_single
@pytest.mark.platform_x86_cpu
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_gpu_training
@pytest.mark.platform_x86_ascend_training
def test_update_watchpoint(self, app_client):
"""Test retrieve when train_begin."""
watch_point_id = 1
leaf_node_name = 'Default/optimizer-Momentum/Parameter[18]_7/moments.fc3.bias'
with self._debugger_client.get_thread_instance():
flag = check_waiting_state(app_client)
assert flag is True
condition = {'condition': 'INF'}
self._create_watchpoint(app_client, condition, watch_point_id)
# update watchpoint watchpoint list
url = 'update_watchpoint'
body_data = {'watch_point_id': watch_point_id,
'watch_nodes': [leaf_node_name],
'mode': 0}
get_request_result(app_client, url, body_data)
# get updated nodes
url = 'search'
body_data = {'name': leaf_node_name, 'watch_point_id': watch_point_id}
expect_file = 'search_unwatched_leaf_node.json'
send_and_compare_result(app_client, url, body_data, expect_file, method='get')
self._send_terminate_cmd(app_client)

@pytest.mark.level0
@pytest.mark.env_single
@pytest.mark.platform_x86_cpu
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_gpu_training
@pytest.mark.platform_x86_ascend_training
def test_watchpoint_hit(self, app_client):
"""Test retrieve watchpoint hit."""
with self._debugger_client.get_thread_instance():
flag = check_waiting_state(app_client)
assert flag is True
self._create_watchpoint(app_client, condition={'condition': 'INF'}, expect_id=1)
# send run command to get watchpoint hit
url = 'control'
body_data = {'mode': 'continue',
'steps': 2}
res = get_request_result(app_client, url, body_data)
assert res == {'metadata': {'state': 'running'}}
# wait for server has received watchpoint hit
flag = check_waiting_state(app_client)
assert flag is True
# check watchpoint hit list
url = 'retrieve'
body_data = {'mode': 'watchpoint_hit'}
expect_file = 'retrieve_watchpoint_hit.json'
send_and_compare_result(app_client, url, body_data, expect_file)
# check single watchpoint hit
body_data = {
'mode': 'watchpoint_hit',
'params': {
'name': 'Default/TransData-op99',
'single_node': True,
'watch_point_id': 1
}
}
expect_file = 'retrieve_single_watchpoint_hit.json'
send_and_compare_result(app_client, url, body_data, expect_file)
self._send_terminate_cmd(app_client)

@pytest.mark.level0
@pytest.mark.env_single
@pytest.mark.platform_x86_cpu
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_gpu_training
@pytest.mark.platform_x86_ascend_training
def test_retrieve_tensor_value(self, app_client):
"""Test retrieve tensor value."""
node_name = 'Default/TransData-op99'
with self._debugger_client.get_thread_instance():
flag = check_waiting_state(app_client)
assert flag is True
# prepare tensor value
url = 'retrieve_tensor_history'
body_data = {'name': node_name}
expect_file = 'retrieve_empty_tensor_history.json'
send_and_compare_result(app_client, url, body_data, expect_file)
# check full tensor history from poll data
res = get_request_result(
app_client=app_client, url='poll_data', body_data={'pos': 0}, method='get')
assert res.get('receive_tensor', {}).get('node_name') == node_name
expect_file = 'retrieve_full_tensor_history.json'
send_and_compare_result(app_client, url, body_data, expect_file)
# check tensor value
url = 'tensors'
body_data = {
'name': node_name + ':0',
'detail': 'data',
'shape': '[1, 1:3]'
}
expect_file = 'retrieve_tensor_value.json'
send_and_compare_result(app_client, url, body_data, expect_file, method='get')
self._send_terminate_cmd(app_client)

@pytest.mark.level0
@pytest.mark.env_single
@pytest.mark.platform_x86_cpu
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_gpu_training
@pytest.mark.platform_x86_ascend_training
def test_compare_tensor_value(self, app_client):
"""Test compare tensor value."""
node_name = 'Default/args0'
with self._debugger_client.get_thread_instance():
flag = check_waiting_state(app_client)
assert flag is True
# prepare tensor values
url = 'control'
body_data = {'mode': 'continue',
'steps': 2}
get_request_result(app_client, url, body_data)
flag = check_waiting_state(app_client)
assert flag is True
get_request_result(
app_client=app_client, url='retrieve_tensor_history', body_data={'name': node_name})
res = get_request_result(
app_client=app_client, url='poll_data', body_data={'pos': 0}, method='get')
assert res.get('receive_tensor', {}).get('node_name') == node_name
# get compare results
url = 'tensor-comparisons'
body_data = {
'name': node_name + ':0',
'detail': 'data',
'shape': '[:, :]',
'tolerance': 1
}
expect_file = 'compare_tensors.json'
send_and_compare_result(app_client, url, body_data, expect_file, method='get')
self._send_terminate_cmd(app_client)

@pytest.mark.level0
@pytest.mark.env_single
@pytest.mark.platform_x86_cpu
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_gpu_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.parametrize("body_data, expect_file", [
({'ascend': True}, 'retrieve_node_by_bfs_ascend.json'),
({'name': 'Default/args0', 'ascend': False}, 'retrieve_node_by_bfs.json')
])
def test_retrieve_bfs_node(self, app_client, body_data, expect_file):
"""Test retrieve bfs node."""
with self._debugger_client.get_thread_instance():
flag = check_waiting_state(app_client)
assert flag is True
# prepare tensor values
url = 'retrieve_node_by_bfs'
send_and_compare_result(app_client, url, body_data, expect_file, method='get')
self._send_terminate_cmd(app_client)

@pytest.mark.level0
@pytest.mark.env_single
@pytest.mark.platform_x86_cpu
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_gpu_training
@pytest.mark.platform_x86_ascend_training
def test_next_node_on_gpu(self, app_client):
"""Test get next node on GPU."""
gpu_debugger_client = MockDebuggerClient(backend='GPU')
with gpu_debugger_client.get_thread_instance():
flag = check_waiting_state(app_client)
assert flag is True
# send run command to get watchpoint hit
url = 'control'
body_data = {'mode': 'continue',
'level': 'node',
'name': 'Default/TransData-op99'}
res = get_request_result(app_client, url, body_data)
assert res == {'metadata': {'state': 'running'}}
# get metadata
flag = check_waiting_state(app_client)
assert flag is True
url = 'retrieve'
body_data = {'mode': 'all'}
expect_file = 'retrieve_next_node_on_gpu.json'
send_and_compare_result(app_client, url, body_data, expect_file)
self._send_terminate_cmd(app_client)

@pytest.mark.level0
@pytest.mark.env_single
@pytest.mark.platform_x86_cpu
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_gpu_training
@pytest.mark.platform_x86_ascend_training
def test_pause(self, app_client):
"""Test pause the training."""
with self._debugger_client.get_thread_instance():
flag = check_waiting_state(app_client)
assert flag is True
# send run command to execute to next node
url = 'control'
body_data = {'mode': 'continue',
'steps': -1}
res = get_request_result(app_client, url, body_data)
assert res == {'metadata': {'state': 'running'}}
# send pause command
url = 'control'
body_data = {'mode': 'pause'}
res = get_request_result(app_client, url, body_data)
assert res == {'metadata': {'state': 'waiting'}}
self._send_terminate_cmd(app_client)

+ 76
- 0
tests/st/func/debugger/utils.py View File

@@ -0,0 +1,76 @@
# Copyright 2020 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
"""Test debugger server utils."""
import json
import os
import time

from tests.st.func.debugger.conftest import DEBUGGER_EXPECTED_RESULTS, DEBUGGER_BASE_URL
from tests.utils.tools import compare_result_with_file, get_url


def check_waiting_state(app_client):
"""Wait for waiting state."""
url = 'retrieve'
body_data = {'mode': 'all'}
max_try_times = 30
count = 0
while count < max_try_times:
res = get_request_result(app_client, url, body_data)
state = res.get('metadata', {}).get('state')
if state == 'waiting':
return True
count += 1
time.sleep(0.1)
return False


def get_request_result(app_client, url, body_data, method='post', expect_code=200):
"""Get request result."""
real_url = os.path.join(DEBUGGER_BASE_URL, url)
if method == 'post':
response = app_client.post(real_url, data=json.dumps(body_data))
else:
real_url = get_url(real_url, body_data)
response = app_client.get(real_url)
assert response.status_code == expect_code
res = response.get_json()
return res


def send_and_compare_result(app_client, url, body_data, expect_file=None, method='post'):
"""Send and compare result."""
res = get_request_result(app_client, url, body_data, method=method)
delete_random_items(res)
if expect_file:
real_path = os.path.join(DEBUGGER_EXPECTED_RESULTS, 'restful_results', expect_file)
compare_result_with_file(res, real_path)


def send_and_save_result(app_client, url, body_data, file_path, method='post'):
"""Send and save result."""
res = get_request_result(app_client, url, body_data, method=method)
delete_random_items(res)
real_path = os.path.join(DEBUGGER_EXPECTED_RESULTS, 'restful_results', file_path)
json.dump(res, open(real_path, 'w'))


def delete_random_items(res):
"""delete the random items in metadata."""
if res.get('metadata'):
if res['metadata'].get('ip'):
res['metadata'].pop('ip')
if res['metadata'].get('pos'):
res['metadata'].pop('pos')

Write Preview
Loading…
Cancel
Save