到处依赖关系图谱~

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/model/ModelDependencyController.java

@@ -5,6 +5,7 @@ import com.ruoyi.common.core.web.domain.GenericsAjaxResult;
 import com.ruoyi.platform.domain.ModelDependency;
 import com.ruoyi.platform.domain.ModelsVersion;
 import com.ruoyi.platform.service.ModelDependencyService;
+import com.ruoyi.platform.vo.ModelDependcyTreeVo;
 import io.swagger.annotations.ApiOperation;
 import org.springframework.data.domain.Page;
 import org.springframework.data.domain.PageRequest;
@@ -102,5 +103,11 @@ public class ModelDependencyController extends BaseController {
         return genericsSuccess(this.modelDependencyService.removeById(id));
     }
 
+
+    @PostMapping("/queryModelAtlas")
+    @ApiOperation("根据对象查询")
+    public GenericsAjaxResult<ModelDependcyTreeVo> queryModelAtlas(@RequestBody ModelDependency modelDependency) throws Exception {
+        return genericsSuccess(this.modelDependencyService.getModelDependencyTree(modelDependency));
+    }
 }
 

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/dependencydomain/ProjectDepency.java

@@ -1,9 +1,12 @@
 package com.ruoyi.platform.domain.dependencydomain;
 
+import com.fasterxml.jackson.databind.PropertyNamingStrategy;
+import com.fasterxml.jackson.databind.annotation.JsonNaming;
 import lombok.Data;
 
 import java.io.Serializable;
 @Data
+@JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy.class)
 public class ProjectDepency implements Serializable {
     private String url;
     private String name;

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/dependencydomain/TrainTaskDepency.java

@@ -1,9 +1,12 @@
 package com.ruoyi.platform.domain.dependencydomain;
 
+import com.fasterxml.jackson.databind.PropertyNamingStrategy;
+import com.fasterxml.jackson.databind.annotation.JsonNaming;
 import lombok.Data;
 
 import java.io.Serializable;
 @Data
+@JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy.class)
 public class TrainTaskDepency  implements Serializable {
     //训练任务名
     private String name;

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/ModelDependencyDao.java

@@ -82,5 +82,7 @@ public interface ModelDependencyDao {
     int deleteById(Integer id);
 
     List<ModelDependency> queryByModelDependency(@Param("modelDependency") ModelDependency modelDependency);
+
+    List<ModelDependency> queryChildrenByVersionId(@Param("model_id")String modelId, @Param("version")String version);
 }
 

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/scheduling/ExperimentInstanceStatusTask.java

@@ -1,24 +1,26 @@
 package com.ruoyi.platform.scheduling;
 
-import com.ruoyi.common.security.utils.SecurityUtils;
 import com.ruoyi.platform.domain.Experiment;
 import com.ruoyi.platform.domain.ExperimentIns;
+import com.ruoyi.platform.domain.ModelDependency;
 import com.ruoyi.platform.mapper.ExperimentDao;
 import com.ruoyi.platform.mapper.ExperimentInsDao;
+import com.ruoyi.platform.mapper.ModelDependencyDao;
 import com.ruoyi.platform.service.ExperimentInsService;
-import com.ruoyi.platform.service.ExperimentService;
-import com.ruoyi.platform.utils.JsonUtils;
-import com.ruoyi.system.api.model.LoginUser;
-import io.swagger.models.auth.In;
+import com.ruoyi.platform.service.ModelDependencyService;
+import com.ruoyi.platform.utils.JacksonUtil;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.data.domain.Page;
 import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Component;
 
 import javax.annotation.Resource;
 import java.io.IOException;
-import java.util.*;
-import java.util.stream.Collectors;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
 
 @Component()
 public class ExperimentInstanceStatusTask {
@@ -28,7 +30,8 @@ public class ExperimentInstanceStatusTask {
     private ExperimentDao experimentDao;
     @Resource
     private ExperimentInsDao experimentInsDao;
-
+    @Resource
+    private ModelDependencyDao modelDependencyDao;
     private List<Integer> experimentIds = new ArrayList<>();
 
     @Scheduled(cron = "0/30 * * * * ?") // 每30S执行一次
@@ -55,12 +58,50 @@ public class ExperimentInstanceStatusTask {
                     updateList.add(experimentIns);
 
                 }
-                experimentInsDao.update(experimentIns);
+//                experimentInsDao.update(experimentIns);
             }
 
         }
         if (updateList.size() > 0){
             experimentInsDao.insertOrUpdateBatch(updateList);
+
+            //遍历模型关系表，找到
+            List<ModelDependency> modelDependencyList = new ArrayList<ModelDependency>();
+            for (ExperimentIns experimentIns : updateList){
+                ModelDependency modelDependencyquery = new ModelDependency();
+                modelDependencyquery.setExpInsId(experimentIns.getId());
+                modelDependencyquery.setState(2);
+
+                List<ModelDependency> modelDependencyListquery = modelDependencyDao.queryByModelDependency(modelDependencyquery);
+                if (modelDependencyListquery==null||modelDependencyListquery.size()==0){
+                    continue;
+                }
+                ModelDependency modelDependency = modelDependencyListquery.get(0);
+                //查看状态，
+                if (StringUtils.equals("Succeeded",experimentIns.getStatus())){
+                    modelDependency.setState(1);
+                    modelDependencyList.add(modelDependency);
+                }else{
+                    //取出节点状态
+                    String trainTask = modelDependency.getTrainTask();
+                    Map<String, Object> trainMap = JacksonUtil.parseJSONStr2Map(trainTask);
+                    String task_id = (String) trainMap.get("task_id");
+                    if (StringUtils.isEmpty(task_id)){
+                        continue;
+                    }
+                    String nodesStatus = experimentIns.getNodesStatus();
+                    Map<String, Object> nodeMaps = JacksonUtil.parseJSONStr2Map(nodesStatus);
+                    Map<String, Object> nodeMap = JacksonUtil.parseJSONStr2Map((String) nodeMaps.get(task_id));
+                    if (nodeMap==null){
+                        continue;
+                    }
+                    if (StringUtils.equals("Succeeded",(String)nodeMap.get("phase"))){
+                        modelDependency.setState(1);
+                        modelDependencyList.add(modelDependency);
+                    }
+                }
+            }
+            modelDependencyDao.insertOrUpdateBatch(modelDependencyList);
         }
 
     }

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ModelDependencyService.java

@@ -1,6 +1,7 @@
 package com.ruoyi.platform.service;
 
 import com.ruoyi.platform.domain.ModelDependency;
+import com.ruoyi.platform.vo.ModelDependcyTreeVo;
 import org.springframework.data.domain.Page;
 import org.springframework.data.domain.PageRequest;
 
@@ -32,20 +33,20 @@ public interface ModelDependencyService {
     Page<ModelDependency> queryByPage(ModelDependency modelDependency, PageRequest pageRequest);
 
     /**
-     * 新增数据
+     * 修改数据
      *
      * @param modelDependency 实例对象
      * @return 实例对象
      */
-    ModelDependency insert(ModelDependency modelDependency);
+    ModelDependency update(ModelDependency modelDependency);
 
     /**
-     * 修改数据
+     * 新增数据
      *
      * @param modelDependency 实例对象
      * @return 实例对象
      */
-    ModelDependency update(ModelDependency modelDependency);
+    ModelDependency insert(ModelDependency modelDependency);
 
     /**
      * 通过主键删除数据
@@ -58,4 +59,6 @@ public interface ModelDependencyService {
     String removeById(Integer id);
 
     List<ModelDependency> queryByModelDependency(ModelDependency modelDependency);
+
+    ModelDependcyTreeVo getModelDependencyTree(ModelDependency modelDependency) throws Exception;
 }

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentServiceImpl.java

@@ -269,7 +269,7 @@ public class ExperimentServiceImpl implements ExperimentService {
             ExperimentIns insert = experimentInsService.insert(experimentIns);
 
             //插入到模型依赖关系表
-            insertModelDependendcy(dependendcy,insert.getId(),experiment.getName());
+            insertModelDependendcy(dependendcy,insert.getId(),experiment.getName(),params);
 
         }catch (Exception e){
             throw new RuntimeException(e);
@@ -279,7 +279,7 @@ public class ExperimentServiceImpl implements ExperimentService {
         return experiment;
     }
 
-    private void insertModelDependendcy(Map<String ,Object> dependendcy,Integer experimentInsId,String expermentName)throws Exception {
+    private void insertModelDependendcy(Map<String ,Object> dependendcy,Integer experimentInsId,String expermentName,List<Map<String, Object>> params)throws Exception {
         Iterator<Map.Entry<String, Object>> dependendcyIterator = dependendcy.entrySet().iterator();
         while (dependendcyIterator.hasNext()) {
             ModelDependency modelDependency = new ModelDependency();
@@ -328,29 +328,46 @@ public class ExperimentServiceImpl implements ExperimentService {
             trainTaskDepency.setName(expermentName);
             //训练任务
             modelDependency.setTrainTask(JsonUtils.objectToJson(trainTaskDepency));
-
+            modelDependency.setExpInsId(experimentInsId);
             //处理test数据
             List<Map<String, Object>> testDatasetsList = (List<Map<String, Object>>) modelDel.get("test");
+            List<Map<String, Object>> resultTestDatasets = new ArrayList<Map<String, Object>>();
             for(int i=0;i<testDatasetsList.size();i++){
                 Map<String, Object> datasets = testDatasetsList.get(i);
-                Dataset dataset = datasetService.queryById((Integer) datasets.get("dataset_id"));
-                if (dataset == null){
-                    throw new  Exception("源数据集不存在");
+                List<Map<String, Object>>   realDataSetList = (List<Map<String, Object>>)  datasets.get("datasets");
+                for(int j=0;j<realDataSetList.size();j++){
+                    Map<String, Object> realDataSet = realDataSetList.get(j);
+                    Dataset dataset = datasetService.queryById((Integer) realDataSet.get("dataset_id"));
+                    if (dataset == null){
+                        throw new  Exception("源数据集不存在");
+                    }
+                    realDataSet.put("dataset_name", dataset.getName());
+                    resultTestDatasets.add(realDataSet);
                 }
-                datasets.put("dataset_name", dataset.getName());
+
             }
             //测试数据集
-            modelDependency.setTestDataset(JsonUtils.objectToJson(testDatasetsList));
+            modelDependency.setTestDataset(JsonUtils.objectToJson(resultTestDatasets));
 
             //检查是否存在target，如果存在说明在流水线用了节点导入，如果没有说明没有导入，等待手动push
             List<Map<String, Object>> modelTargetList = (List<Map<String, Object>>) modelDel.get("target");
             if (modelTargetList==null||modelTargetList.size()==0){
-                modelDependency.setState(0);
-            }else {
                 modelDependency.setState(1);
+                modelDependencyService.insert(modelDependency);
+            }else {
+                modelDependency.setState(2);
                 for(int i=0;i<modelTargetList.size();i++){
                     Map<String, Object> model = modelTargetList.get(i);
-                    modelDependency.setVersion((String) model.get("model_version"));
+                    String version = null;
+                    //可能是参数，必须从实验参数读取
+                    if (params != null) {
+                        for (Map<String, Object> param : params) {
+                            if (param.containsKey("param_name") && StringUtils.equals("model_version",(String)param.get("param_name"))) {
+                                version = param.get("param_value").toString();
+                            }
+                        }
+                    }
+                    modelDependency.setVersion(StringUtils.isEmpty(version)?(String)model.get("model_version"):version);
                     modelDependency.setCurrentModelId((Integer) model.get("model_id"));
                     //因为可能有多成果模型，多次插入
                     modelDependencyService.insert(modelDependency);

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelDependencyServiceImpl.java

@@ -5,6 +5,8 @@ import com.ruoyi.platform.domain.ModelDependency;
 import com.ruoyi.platform.domain.ModelsVersion;
 import com.ruoyi.platform.mapper.ModelDependencyDao;
 import com.ruoyi.platform.service.ModelDependencyService;
+import com.ruoyi.platform.utils.JacksonUtil;
+import com.ruoyi.platform.vo.ModelDependcyTreeVo;
 import com.ruoyi.system.api.model.LoginUser;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.stereotype.Service;
@@ -13,8 +15,10 @@ import org.springframework.data.domain.PageImpl;
 import org.springframework.data.domain.PageRequest;
 
 import javax.annotation.Resource;
+import java.util.ArrayList;
 import java.util.Date;
 import java.util.List;
+import java.util.Map;
 
 /**
  * (ModelDependency)表服务实现类
@@ -27,6 +31,7 @@ public class ModelDependencyServiceImpl implements ModelDependencyService {
     @Resource
     private ModelDependencyDao modelDependencyDao;
 
+
     /**
      * 通过ID查询单条数据
      *
@@ -62,6 +67,81 @@ public class ModelDependencyServiceImpl implements ModelDependencyService {
         return this.modelDependencyDao.queryByModelDependency(modelDependency);
     }
 
+    @Override
+    public ModelDependcyTreeVo getModelDependencyTree(ModelDependency modelDependencyQuery) throws Exception {
+        //查询当前模型
+        List<ModelDependency> modelDependencyList = modelDependencyDao.queryByModelDependency(modelDependencyQuery);
+        if (modelDependencyList==null||modelDependencyList.size()==0){
+            throw new Exception("当前模型依赖关系不存在");
+        }
+        ModelDependency modelDependency = modelDependencyList.get(0);
+        ModelDependcyTreeVo modelDependcyTreeVo = ModelDependencyConvertToTree(modelDependency);
+        //递归父模型
+        processParentModel(modelDependcyTreeVo);
+        //递归子模型
+        processChildrenModel(modelDependcyTreeVo);
+        return modelDependcyTreeVo;
+    }
+
+    /**
+     * 递归父模型
+     * @param modelDependcyTreeVo
+     */
+    private void processParentModel(ModelDependcyTreeVo modelDependcyTreeVo){
+        if (modelDependcyTreeVo.getParentModelsMap() != null) {
+            List<Map<String, Object>> parentMaps = modelDependcyTreeVo.getParentModelsMap();
+            List<ModelDependcyTreeVo> ps = new ArrayList<ModelDependcyTreeVo>();
+            for (Map<String, Object> parent:parentMaps) {
+                Integer model_id = (Integer) parent.get("model_id");
+                String version = (String) parent.get("model_version");
+                ModelDependency modelDependencyQuery = new ModelDependency();
+                modelDependencyQuery.setVersion(version);
+                modelDependencyQuery.setCurrentModelId(model_id);
+                List<ModelDependency> modelDependencyList = modelDependencyDao.queryByModelDependency(modelDependencyQuery);
+                if (modelDependencyList!=null&&modelDependencyList.size()>=0){
+                    for (ModelDependency modelDependency:modelDependencyList){
+                        ModelDependcyTreeVo modelDependcyTreeVoIn = ModelDependencyConvertToTree(modelDependency);
+                        processParentModel(modelDependcyTreeVoIn);
+                        ps.add(modelDependcyTreeVoIn);
+                    }
+                }
+
+            }
+            modelDependcyTreeVo.setParentModels(ps);
+        }
+    }
+
+
+    private void processChildrenModel(ModelDependcyTreeVo modelDependcyTreeVo){
+        String version = modelDependcyTreeVo.getVersion();
+        Integer modelId = modelDependcyTreeVo.getCurrentModelId();
+        List<ModelDependcyTreeVo> cs = new ArrayList<ModelDependcyTreeVo>();
+        //查儿子们
+
+        List<ModelDependency> modelDependencyList = modelDependencyDao.queryChildrenByVersionId("\"model_id\":"+modelId, "\"model_version\":\""+version+"\"");
+        if (modelDependencyList!=null&&modelDependencyList.size()>=0){
+            for (ModelDependency modelDependency:modelDependencyList){
+                ModelDependcyTreeVo modelDependcyTreeVoIn = ModelDependencyConvertToTree(modelDependency);
+                processChildrenModel(modelDependcyTreeVoIn);
+                cs.add(modelDependcyTreeVoIn);
+            }
+        }
+        modelDependcyTreeVo.setChildrenModels(cs);
+    }
+    private ModelDependcyTreeVo ModelDependencyConvertToTree(ModelDependency modelDependency){
+        ModelDependcyTreeVo modelDependcyTreeVo = new ModelDependcyTreeVo();
+        modelDependcyTreeVo.setCurrentModelId(modelDependency.getCurrentModelId());
+        modelDependcyTreeVo.setExpInsId(modelDependency.getExpInsId());
+        modelDependcyTreeVo.setVersion(modelDependency.getVersion());
+        modelDependcyTreeVo.setRefItem(modelDependency.getRefItem());
+        modelDependcyTreeVo.setTrainTask(JacksonUtil.parseJSONStr2Map(modelDependency.getTrainTask()));
+        modelDependcyTreeVo.setTrainDataset(JacksonUtil.parseJSONStr2MapList(modelDependency.getTrainDataset()));
+        modelDependcyTreeVo.setTestDataset(JacksonUtil.parseJSONStr2MapList(modelDependency.getTestDataset()));
+        modelDependcyTreeVo.setProjectDependency(JacksonUtil.parseJSONStr2Map(modelDependency.getProjectDependency()));
+        modelDependcyTreeVo.setParentModelsMap(JacksonUtil.parseJSONStr2MapList(modelDependency.getParentModels()));
+        return modelDependcyTreeVo;
+    }
+
     /**
      * 新增数据
      *

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelDependcyTreeVo.java

@@ -0,0 +1,59 @@
+package com.ruoyi.platform.vo;
+
+import com.fasterxml.jackson.databind.PropertyNamingStrategy;
+import com.fasterxml.jackson.databind.annotation.JsonNaming;
+import lombok.Data;
+
+import java.util.List;
+import java.util.Map;
+
+@Data
+@JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy.class)
+public class ModelDependcyTreeVo {
+    /**
+     * 当前模型id
+     */
+    private Integer currentModelId;
+    /**
+     * 实验实例id
+     */
+    private Integer expInsId;
+
+    /**
+     * 版本
+     */
+    private String version;
+
+
+    /**
+     * 引用项目
+     */
+    private String refItem;
+    /**
+     * 训练任务
+     */
+    private Map trainTask;
+    /**
+     * 训练数据集
+     */
+    private List<Map<String, Object>> trainDataset;
+    /**
+     * 测试数据集
+     */
+    private List<Map<String, Object>> testDataset;
+    /**
+     * 依赖项目
+     */
+    private Map projectDependency;
+
+    private List<Map<String, Object>> parentModelsMap;
+    /**
+     * 父模型
+     */
+    private List<ModelDependcyTreeVo> parentModels;
+
+    /**
+     * 子模型
+     */
+    private List<ModelDependcyTreeVo> childrenModels;
+}

ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ModelDependencyDaoMapper.xml

@@ -83,10 +83,18 @@
         limit #{pageable.offset}, #{pageable.pageSize}
     </select>
 
+    <select id="queryChildrenByVersionId" resultMap="ModelDependencyMap">
+        select
+        id,current_model_id,exp_ins_id,parent_models,ref_item,train_task,train_dataset,test_dataset,project_dependency,version,create_by,create_time,update_by,update_time,state
+        from model_dependency
+        <where>
+            parent_models like concat('%', #{model_id}, '%') AND   parent_models like concat('%', #{version}, '%')
+        </where>
+    </select>
     <!--查询指定行数据-->
     <select id="queryByModelDependency" resultMap="ModelDependencyMap">
         select
-        id,current_model_id,expInsId,parent_models,ref_item,train_task,train_dataset,test_dataset,project_dependency,version,create_by,create_time,update_by,update_time,state
+        id,current_model_id,exp_ins_id,parent_models,ref_item,train_task,train_dataset,test_dataset,project_dependency,version,create_by,create_time,update_by,update_time,state
         from model_dependency
         <where>
             <if test="modelDependency.id != null">