|
|
@@ -34,7 +34,6 @@ public class ExperimentInstanceStatusTask { |
|
|
// 首先查到所有非终止态的实验实例 |
|
|
// 首先查到所有非终止态的实验实例 |
|
|
List<ExperimentIns> experimentInsList = experimentInsService.queryByExperimentIsNotTerminated(); |
|
|
List<ExperimentIns> experimentInsList = experimentInsService.queryByExperimentIsNotTerminated(); |
|
|
// 去argo查询状态 |
|
|
// 去argo查询状态 |
|
|
List<ExperimentIns> updateList = new ArrayList<>(); |
|
|
|
|
|
if (experimentInsList != null && experimentInsList.size() > 0) { |
|
|
if (experimentInsList != null && experimentInsList.size() > 0) { |
|
|
for (ExperimentIns experimentIns : experimentInsList) { |
|
|
for (ExperimentIns experimentIns : experimentInsList) { |
|
|
//当原本状态为null或非终止态时才调用argo接口 |
|
|
//当原本状态为null或非终止态时才调用argo接口 |
|
|
@@ -49,48 +48,6 @@ public class ExperimentInstanceStatusTask { |
|
|
synchronized (experimentIds) { |
|
|
synchronized (experimentIds) { |
|
|
experimentIds.add(experimentIns.getExperimentId()); |
|
|
experimentIds.add(experimentIns.getExperimentId()); |
|
|
} |
|
|
} |
|
|
updateList.add(experimentIns); |
|
|
|
|
|
} |
|
|
|
|
|
} |
|
|
|
|
|
if (updateList.size() > 0) { |
|
|
|
|
|
experimentInsDao.insertOrUpdateBatch(updateList); |
|
|
|
|
|
|
|
|
|
|
|
//遍历模型关系表,找到 |
|
|
|
|
|
List<ModelDependency> modelDependencyList = new ArrayList<ModelDependency>(); |
|
|
|
|
|
for (ExperimentIns experimentIns : updateList) { |
|
|
|
|
|
ModelDependency modelDependencyquery = new ModelDependency(); |
|
|
|
|
|
modelDependencyquery.setExpInsId(experimentIns.getId()); |
|
|
|
|
|
modelDependencyquery.setState(2); |
|
|
|
|
|
|
|
|
|
|
|
List<ModelDependency> modelDependencyListquery = modelDependencyDao.queryByModelDependency(modelDependencyquery); |
|
|
|
|
|
if (modelDependencyListquery == null || modelDependencyListquery.size() == 0) { |
|
|
|
|
|
continue; |
|
|
|
|
|
} |
|
|
|
|
|
ModelDependency modelDependency = modelDependencyListquery.get(0); |
|
|
|
|
|
//查看状态, |
|
|
|
|
|
if (StringUtils.equals("Failed", experimentIns.getStatus())) { |
|
|
|
|
|
//取出节点状态 |
|
|
|
|
|
String trainTask = modelDependency.getTrainTask(); |
|
|
|
|
|
Map<String, Object> trainMap = JacksonUtil.parseJSONStr2Map(trainTask); |
|
|
|
|
|
String task_id = (String) trainMap.get("task_id"); |
|
|
|
|
|
if (StringUtils.isEmpty(task_id)) { |
|
|
|
|
|
continue; |
|
|
|
|
|
} |
|
|
|
|
|
String nodesStatus = experimentIns.getNodesStatus(); |
|
|
|
|
|
Map<String, Object> nodeMaps = JacksonUtil.parseJSONStr2Map(nodesStatus); |
|
|
|
|
|
Map<String, Object> nodeMap = JacksonUtil.parseJSONStr2Map(JacksonUtil.toJSONString(nodeMaps.get(task_id))); |
|
|
|
|
|
|
|
|
|
|
|
if (nodeMap == null) { |
|
|
|
|
|
continue; |
|
|
|
|
|
} |
|
|
|
|
|
if (!StringUtils.equals("Succeeded", (String) nodeMap.get("phase"))) { |
|
|
|
|
|
modelDependency.setState(0); |
|
|
|
|
|
modelDependencyList.add(modelDependency); |
|
|
|
|
|
} |
|
|
|
|
|
} |
|
|
|
|
|
} |
|
|
|
|
|
if (modelDependencyList.size() > 0) { |
|
|
|
|
|
modelDependencyDao.insertOrUpdateBatch(modelDependencyList); |
|
|
|
|
|
} |
|
|
} |
|
|
} |
|
|
} |
|
|
} |
|
|
} |
|
|
@@ -105,7 +62,7 @@ public class ExperimentInstanceStatusTask { |
|
|
for (Integer experimentId : experimentIds) { |
|
|
for (Integer experimentId : experimentIds) { |
|
|
// 获取当前实验的所有实例列表 |
|
|
// 获取当前实验的所有实例列表 |
|
|
List<ExperimentIns> insList = experimentInsService.getByExperimentId(experimentId); |
|
|
List<ExperimentIns> insList = experimentInsService.getByExperimentId(experimentId); |
|
|
List<String> statusList = new ArrayList<String>(); |
|
|
|
|
|
|
|
|
List<String> statusList = new ArrayList<>(); |
|
|
// 更新实验状态列表 |
|
|
// 更新实验状态列表 |
|
|
for (int i = 0; i < insList.size(); i++) { |
|
|
for (int i = 0; i < insList.size(); i++) { |
|
|
statusList.add(insList.get(i).getStatus()); |
|
|
statusList.add(insList.get(i).getStatus()); |
|
|
|