From e71b3f8e848734586905b680ddd74afb19bb6bad Mon Sep 17 00:00:00 2001 From: fanshuai <1141904845@qq.com> Date: Fri, 31 May 2024 17:58:54 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=B0=E5=A4=84=E4=BE=9D=E8=B5=96=E5=85=B3?= =?UTF-8?q?=E7=B3=BB=E5=9B=BE=E8=B0=B1~?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../model/ModelDependencyController.java | 7 ++ .../dependencydomain/ProjectDepency.java | 3 + .../dependencydomain/TrainTaskDepency.java | 3 + .../platform/mapper/ModelDependencyDao.java | 2 + .../ExperimentInstanceStatusTask.java | 59 +++++++++++--- .../service/ModelDependencyService.java | 11 ++- .../service/impl/ExperimentServiceImpl.java | 39 ++++++--- .../impl/ModelDependencyServiceImpl.java | 80 +++++++++++++++++++ .../platform/vo/ModelDependcyTreeVo.java | 59 ++++++++++++++ .../ModelDependencyDaoMapper.xml | 10 ++- 10 files changed, 248 insertions(+), 25 deletions(-) create mode 100644 ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelDependcyTreeVo.java diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/model/ModelDependencyController.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/model/ModelDependencyController.java index e2382a69..3f8b5cd5 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/model/ModelDependencyController.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/model/ModelDependencyController.java @@ -5,6 +5,7 @@ import com.ruoyi.common.core.web.domain.GenericsAjaxResult; import com.ruoyi.platform.domain.ModelDependency; import com.ruoyi.platform.domain.ModelsVersion; import com.ruoyi.platform.service.ModelDependencyService; +import com.ruoyi.platform.vo.ModelDependcyTreeVo; import io.swagger.annotations.ApiOperation; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageRequest; @@ -102,5 +103,11 @@ public class ModelDependencyController extends BaseController { return genericsSuccess(this.modelDependencyService.removeById(id)); } + + @PostMapping("/queryModelAtlas") + @ApiOperation("根据对象查询") + public GenericsAjaxResult queryModelAtlas(@RequestBody ModelDependency modelDependency) throws Exception { + return genericsSuccess(this.modelDependencyService.getModelDependencyTree(modelDependency)); + } } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/dependencydomain/ProjectDepency.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/dependencydomain/ProjectDepency.java index 0fdce081..98502afb 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/dependencydomain/ProjectDepency.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/dependencydomain/ProjectDepency.java @@ -1,9 +1,12 @@ package com.ruoyi.platform.domain.dependencydomain; +import com.fasterxml.jackson.databind.PropertyNamingStrategy; +import com.fasterxml.jackson.databind.annotation.JsonNaming; import lombok.Data; import java.io.Serializable; @Data +@JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy.class) public class ProjectDepency implements Serializable { private String url; private String name; diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/dependencydomain/TrainTaskDepency.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/dependencydomain/TrainTaskDepency.java index 536894d9..857306c8 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/dependencydomain/TrainTaskDepency.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/dependencydomain/TrainTaskDepency.java @@ -1,9 +1,12 @@ package com.ruoyi.platform.domain.dependencydomain; +import com.fasterxml.jackson.databind.PropertyNamingStrategy; +import com.fasterxml.jackson.databind.annotation.JsonNaming; import lombok.Data; import java.io.Serializable; @Data +@JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy.class) public class TrainTaskDepency implements Serializable { //训练任务名 private String name; diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/ModelDependencyDao.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/ModelDependencyDao.java index 725dc678..35c1bfc5 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/ModelDependencyDao.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/ModelDependencyDao.java @@ -82,5 +82,7 @@ public interface ModelDependencyDao { int deleteById(Integer id); List queryByModelDependency(@Param("modelDependency") ModelDependency modelDependency); + + List queryChildrenByVersionId(@Param("model_id")String modelId, @Param("version")String version); } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/scheduling/ExperimentInstanceStatusTask.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/scheduling/ExperimentInstanceStatusTask.java index b7ea8648..1764b262 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/scheduling/ExperimentInstanceStatusTask.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/scheduling/ExperimentInstanceStatusTask.java @@ -1,24 +1,26 @@ package com.ruoyi.platform.scheduling; -import com.ruoyi.common.security.utils.SecurityUtils; import com.ruoyi.platform.domain.Experiment; import com.ruoyi.platform.domain.ExperimentIns; +import com.ruoyi.platform.domain.ModelDependency; import com.ruoyi.platform.mapper.ExperimentDao; import com.ruoyi.platform.mapper.ExperimentInsDao; +import com.ruoyi.platform.mapper.ModelDependencyDao; import com.ruoyi.platform.service.ExperimentInsService; -import com.ruoyi.platform.service.ExperimentService; -import com.ruoyi.platform.utils.JsonUtils; -import com.ruoyi.system.api.model.LoginUser; -import io.swagger.models.auth.In; +import com.ruoyi.platform.service.ModelDependencyService; +import com.ruoyi.platform.utils.JacksonUtil; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Page; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; import javax.annotation.Resource; import java.io.IOException; -import java.util.*; -import java.util.stream.Collectors; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Map; @Component() public class ExperimentInstanceStatusTask { @@ -28,7 +30,8 @@ public class ExperimentInstanceStatusTask { private ExperimentDao experimentDao; @Resource private ExperimentInsDao experimentInsDao; - + @Resource + private ModelDependencyDao modelDependencyDao; private List experimentIds = new ArrayList<>(); @Scheduled(cron = "0/30 * * * * ?") // 每30S执行一次 @@ -55,12 +58,50 @@ public class ExperimentInstanceStatusTask { updateList.add(experimentIns); } - experimentInsDao.update(experimentIns); +// experimentInsDao.update(experimentIns); } } if (updateList.size() > 0){ experimentInsDao.insertOrUpdateBatch(updateList); + + //遍历模型关系表，找到 + List modelDependencyList = new ArrayList(); + for (ExperimentIns experimentIns : updateList){ + ModelDependency modelDependencyquery = new ModelDependency(); + modelDependencyquery.setExpInsId(experimentIns.getId()); + modelDependencyquery.setState(2); + + List modelDependencyListquery = modelDependencyDao.queryByModelDependency(modelDependencyquery); + if (modelDependencyListquery==null||modelDependencyListquery.size()==0){ + continue; + } + ModelDependency modelDependency = modelDependencyListquery.get(0); + //查看状态， + if (StringUtils.equals("Succeeded",experimentIns.getStatus())){ + modelDependency.setState(1); + modelDependencyList.add(modelDependency); + }else{ + //取出节点状态 + String trainTask = modelDependency.getTrainTask(); + Map trainMap = JacksonUtil.parseJSONStr2Map(trainTask); + String task_id = (String) trainMap.get("task_id"); + if (StringUtils.isEmpty(task_id)){ + continue; + } + String nodesStatus = experimentIns.getNodesStatus(); + Map nodeMaps = JacksonUtil.parseJSONStr2Map(nodesStatus); + Map nodeMap = JacksonUtil.parseJSONStr2Map((String) nodeMaps.get(task_id)); + if (nodeMap==null){ + continue; + } + if (StringUtils.equals("Succeeded",(String)nodeMap.get("phase"))){ + modelDependency.setState(1); + modelDependencyList.add(modelDependency); + } + } + } + modelDependencyDao.insertOrUpdateBatch(modelDependencyList); } } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ModelDependencyService.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ModelDependencyService.java index 6f629ac8..3883a7b7 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ModelDependencyService.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ModelDependencyService.java @@ -1,6 +1,7 @@ package com.ruoyi.platform.service; import com.ruoyi.platform.domain.ModelDependency; +import com.ruoyi.platform.vo.ModelDependcyTreeVo; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageRequest; @@ -32,20 +33,20 @@ public interface ModelDependencyService { Page queryByPage(ModelDependency modelDependency, PageRequest pageRequest); /** - * 新增数据 + * 修改数据 * * @param modelDependency 实例对象 * @return 实例对象 */ - ModelDependency insert(ModelDependency modelDependency); + ModelDependency update(ModelDependency modelDependency); /** - * 修改数据 + * 新增数据 * * @param modelDependency 实例对象 * @return 实例对象 */ - ModelDependency update(ModelDependency modelDependency); + ModelDependency insert(ModelDependency modelDependency); /** * 通过主键删除数据 @@ -58,4 +59,6 @@ public interface ModelDependencyService { String removeById(Integer id); List queryByModelDependency(ModelDependency modelDependency); + + ModelDependcyTreeVo getModelDependencyTree(ModelDependency modelDependency) throws Exception; } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentServiceImpl.java index c0c02dd5..89b2da4a 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentServiceImpl.java @@ -269,7 +269,7 @@ public class ExperimentServiceImpl implements ExperimentService { ExperimentIns insert = experimentInsService.insert(experimentIns); //插入到模型依赖关系表 - insertModelDependendcy(dependendcy,insert.getId(),experiment.getName()); + insertModelDependendcy(dependendcy,insert.getId(),experiment.getName(),params); }catch (Exception e){ throw new RuntimeException(e); @@ -279,7 +279,7 @@ public class ExperimentServiceImpl implements ExperimentService { return experiment; } - private void insertModelDependendcy(Map dependendcy,Integer experimentInsId,String expermentName)throws Exception { + private void insertModelDependendcy(Map dependendcy,Integer experimentInsId,String expermentName,List> params)throws Exception { Iterator> dependendcyIterator = dependendcy.entrySet().iterator(); while (dependendcyIterator.hasNext()) { ModelDependency modelDependency = new ModelDependency(); @@ -328,29 +328,46 @@ public class ExperimentServiceImpl implements ExperimentService { trainTaskDepency.setName(expermentName); //训练任务 modelDependency.setTrainTask(JsonUtils.objectToJson(trainTaskDepency)); - + modelDependency.setExpInsId(experimentInsId); //处理test数据 List> testDatasetsList = (List>) modelDel.get("test"); + List> resultTestDatasets = new ArrayList>(); for(int i=0;i datasets = testDatasetsList.get(i); - Dataset dataset = datasetService.queryById((Integer) datasets.get("dataset_id")); - if (dataset == null){ - throw new Exception("源数据集不存在"); + List> realDataSetList = (List>) datasets.get("datasets"); + for(int j=0;j realDataSet = realDataSetList.get(j); + Dataset dataset = datasetService.queryById((Integer) realDataSet.get("dataset_id")); + if (dataset == null){ + throw new Exception("源数据集不存在"); + } + realDataSet.put("dataset_name", dataset.getName()); + resultTestDatasets.add(realDataSet); } - datasets.put("dataset_name", dataset.getName()); + } //测试数据集 - modelDependency.setTestDataset(JsonUtils.objectToJson(testDatasetsList)); + modelDependency.setTestDataset(JsonUtils.objectToJson(resultTestDatasets)); //检查是否存在target，如果存在说明在流水线用了节点导入，如果没有说明没有导入，等待手动push List> modelTargetList = (List>) modelDel.get("target"); if (modelTargetList==null||modelTargetList.size()==0){ - modelDependency.setState(0); - }else { modelDependency.setState(1); + modelDependencyService.insert(modelDependency); + }else { + modelDependency.setState(2); for(int i=0;i model = modelTargetList.get(i); - modelDependency.setVersion((String) model.get("model_version")); + String version = null; + //可能是参数，必须从实验参数读取 + if (params != null) { + for (Map param : params) { + if (param.containsKey("param_name") && StringUtils.equals("model_version",(String)param.get("param_name"))) { + version = param.get("param_value").toString(); + } + } + } + modelDependency.setVersion(StringUtils.isEmpty(version)?(String)model.get("model_version"):version); modelDependency.setCurrentModelId((Integer) model.get("model_id")); //因为可能有多成果模型，多次插入 modelDependencyService.insert(modelDependency); diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelDependencyServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelDependencyServiceImpl.java index c4cb93b9..d3f62d35 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelDependencyServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelDependencyServiceImpl.java @@ -5,6 +5,8 @@ import com.ruoyi.platform.domain.ModelDependency; import com.ruoyi.platform.domain.ModelsVersion; import com.ruoyi.platform.mapper.ModelDependencyDao; import com.ruoyi.platform.service.ModelDependencyService; +import com.ruoyi.platform.utils.JacksonUtil; +import com.ruoyi.platform.vo.ModelDependcyTreeVo; import com.ruoyi.system.api.model.LoginUser; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Service; @@ -13,8 +15,10 @@ import org.springframework.data.domain.PageImpl; import org.springframework.data.domain.PageRequest; import javax.annotation.Resource; +import java.util.ArrayList; import java.util.Date; import java.util.List; +import java.util.Map; /** * (ModelDependency)表服务实现类 @@ -27,6 +31,7 @@ public class ModelDependencyServiceImpl implements ModelDependencyService { @Resource private ModelDependencyDao modelDependencyDao; + /** * 通过ID查询单条数据 * @@ -62,6 +67,81 @@ public class ModelDependencyServiceImpl implements ModelDependencyService { return this.modelDependencyDao.queryByModelDependency(modelDependency); } + @Override + public ModelDependcyTreeVo getModelDependencyTree(ModelDependency modelDependencyQuery) throws Exception { + //查询当前模型 + List modelDependencyList = modelDependencyDao.queryByModelDependency(modelDependencyQuery); + if (modelDependencyList==null||modelDependencyList.size()==0){ + throw new Exception("当前模型依赖关系不存在"); + } + ModelDependency modelDependency = modelDependencyList.get(0); + ModelDependcyTreeVo modelDependcyTreeVo = ModelDependencyConvertToTree(modelDependency); + //递归父模型 + processParentModel(modelDependcyTreeVo); + //递归子模型 + processChildrenModel(modelDependcyTreeVo); + return modelDependcyTreeVo; + } + + /** + * 递归父模型 + * @param modelDependcyTreeVo + */ + private void processParentModel(ModelDependcyTreeVo modelDependcyTreeVo){ + if (modelDependcyTreeVo.getParentModelsMap() != null) { + List> parentMaps = modelDependcyTreeVo.getParentModelsMap(); + List ps = new ArrayList(); + for (Map parent:parentMaps) { + Integer model_id = (Integer) parent.get("model_id"); + String version = (String) parent.get("model_version"); + ModelDependency modelDependencyQuery = new ModelDependency(); + modelDependencyQuery.setVersion(version); + modelDependencyQuery.setCurrentModelId(model_id); + List modelDependencyList = modelDependencyDao.queryByModelDependency(modelDependencyQuery); + if (modelDependencyList!=null&&modelDependencyList.size()>=0){ + for (ModelDependency modelDependency:modelDependencyList){ + ModelDependcyTreeVo modelDependcyTreeVoIn = ModelDependencyConvertToTree(modelDependency); + processParentModel(modelDependcyTreeVoIn); + ps.add(modelDependcyTreeVoIn); + } + } + + } + modelDependcyTreeVo.setParentModels(ps); + } + } + + + private void processChildrenModel(ModelDependcyTreeVo modelDependcyTreeVo){ + String version = modelDependcyTreeVo.getVersion(); + Integer modelId = modelDependcyTreeVo.getCurrentModelId(); + List cs = new ArrayList(); + //查儿子们 + + List modelDependencyList = modelDependencyDao.queryChildrenByVersionId("\"model_id\":"+modelId, "\"model_version\":\""+version+"\""); + if (modelDependencyList!=null&&modelDependencyList.size()>=0){ + for (ModelDependency modelDependency:modelDependencyList){ + ModelDependcyTreeVo modelDependcyTreeVoIn = ModelDependencyConvertToTree(modelDependency); + processChildrenModel(modelDependcyTreeVoIn); + cs.add(modelDependcyTreeVoIn); + } + } + modelDependcyTreeVo.setChildrenModels(cs); + } + private ModelDependcyTreeVo ModelDependencyConvertToTree(ModelDependency modelDependency){ + ModelDependcyTreeVo modelDependcyTreeVo = new ModelDependcyTreeVo(); + modelDependcyTreeVo.setCurrentModelId(modelDependency.getCurrentModelId()); + modelDependcyTreeVo.setExpInsId(modelDependency.getExpInsId()); + modelDependcyTreeVo.setVersion(modelDependency.getVersion()); + modelDependcyTreeVo.setRefItem(modelDependency.getRefItem()); + modelDependcyTreeVo.setTrainTask(JacksonUtil.parseJSONStr2Map(modelDependency.getTrainTask())); + modelDependcyTreeVo.setTrainDataset(JacksonUtil.parseJSONStr2MapList(modelDependency.getTrainDataset())); + modelDependcyTreeVo.setTestDataset(JacksonUtil.parseJSONStr2MapList(modelDependency.getTestDataset())); + modelDependcyTreeVo.setProjectDependency(JacksonUtil.parseJSONStr2Map(modelDependency.getProjectDependency())); + modelDependcyTreeVo.setParentModelsMap(JacksonUtil.parseJSONStr2MapList(modelDependency.getParentModels())); + return modelDependcyTreeVo; + } + /** * 新增数据 * diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelDependcyTreeVo.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelDependcyTreeVo.java new file mode 100644 index 00000000..65a3cc24 --- /dev/null +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelDependcyTreeVo.java @@ -0,0 +1,59 @@ +package com.ruoyi.platform.vo; + +import com.fasterxml.jackson.databind.PropertyNamingStrategy; +import com.fasterxml.jackson.databind.annotation.JsonNaming; +import lombok.Data; + +import java.util.List; +import java.util.Map; + +@Data +@JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy.class) +public class ModelDependcyTreeVo { + /** + * 当前模型id + */ + private Integer currentModelId; + /** + * 实验实例id + */ + private Integer expInsId; + + /** + * 版本 + */ + private String version; + + + /** + * 引用项目 + */ + private String refItem; + /** + * 训练任务 + */ + private Map trainTask; + /** + * 训练数据集 + */ + private List> trainDataset; + /** + * 测试数据集 + */ + private List> testDataset; + /** + * 依赖项目 + */ + private Map projectDependency; + + private List> parentModelsMap; + /** + * 父模型 + */ + private List parentModels; + + /** + * 子模型 + */ + private List childrenModels; +} diff --git a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ModelDependencyDaoMapper.xml b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ModelDependencyDaoMapper.xml index 9dc78d81..e991008e 100644 --- a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ModelDependencyDaoMapper.xml +++ b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ModelDependencyDaoMapper.xml @@ -83,10 +83,18 @@ limit #{pageable.offset}, #{pageable.pageSize} +