| @@ -54,6 +54,7 @@ public class ModelDependencyController extends BaseController { | |||
| @GetMapping("/queryModelDependency") | |||
| @ApiOperation("根据对象查询") | |||
| public GenericsAjaxResult<List<ModelDependency>> queryByModelDependency(@RequestBody ModelDependency modelDependency) { | |||
| modelDependency.setState(1); | |||
| return genericsSuccess(this.modelDependencyService.queryByModelDependency(modelDependency)); | |||
| } | |||
| @@ -44,6 +44,14 @@ public class ModelDependency implements Serializable { | |||
| * 训练数据集 | |||
| */ | |||
| private String trainDataset; | |||
| /** | |||
| * 训练参数 | |||
| */ | |||
| private String trainParams; | |||
| /** | |||
| * 训练镜像 | |||
| */ | |||
| private String trainImage; | |||
| /** | |||
| * 测试数据集 | |||
| */ | |||
| @@ -80,7 +88,6 @@ public class ModelDependency implements Serializable { | |||
| */ | |||
| private Integer state; | |||
| public Integer getId() { | |||
| return id; | |||
| } | |||
| @@ -137,6 +144,22 @@ public class ModelDependency implements Serializable { | |||
| this.trainDataset = trainDataset; | |||
| } | |||
| public String getTrainParams() { | |||
| return trainParams; | |||
| } | |||
| public void setTrainParams(String trainParams) { | |||
| this.trainParams = trainParams; | |||
| } | |||
| public String getTrainImage(String name) { | |||
| return trainImage; | |||
| } | |||
| public void setTrainImage(String trainImage) { | |||
| this.trainImage = trainImage; | |||
| } | |||
| public String getTestDataset() { | |||
| return testDataset; | |||
| } | |||
| @@ -161,7 +184,6 @@ public class ModelDependency implements Serializable { | |||
| this.version = version; | |||
| } | |||
| public String getCreateBy() { | |||
| return createBy; | |||
| } | |||
| @@ -201,6 +223,5 @@ public class ModelDependency implements Serializable { | |||
| public void setState(Integer state) { | |||
| this.state = state; | |||
| } | |||
| } | |||
| @@ -28,7 +28,7 @@ public interface ModelDependencyDao { | |||
| * @param pageable 分页对象 | |||
| * @return 对象列表 | |||
| */ | |||
| List<ModelDependency> queryAllByLimit(@Param("modelDependency") ModelDependency modelDependency, @Param("pageable") Pageable pageable); | |||
| List<ModelDependency> queryAllByLimit(ModelDependency modelDependency, @Param("pageable") Pageable pageable); | |||
| @@ -38,7 +38,7 @@ public interface ModelDependencyDao { | |||
| * @param modelDependency 查询条件 | |||
| * @return 总行数 | |||
| */ | |||
| long count(@Param("modelDependency") ModelDependency modelDependency); | |||
| long count(ModelDependency modelDependency); | |||
| /** | |||
| * 新增数据 | |||
| @@ -46,7 +46,7 @@ public interface ModelDependencyDao { | |||
| * @param modelDependency 实例对象 | |||
| * @return 影响行数 | |||
| */ | |||
| int insert(@Param("modelDependency") ModelDependency modelDependency); | |||
| int insert(ModelDependency modelDependency); | |||
| /** | |||
| * 批量新增数据(MyBatis原生foreach方法) | |||
| @@ -71,7 +71,7 @@ public interface ModelDependencyDao { | |||
| * @param modelDependency 实例对象 | |||
| * @return 影响行数 | |||
| */ | |||
| int update(@Param("modelDependency") ModelDependency modelDependency); | |||
| int update(ModelDependency modelDependency); | |||
| /** | |||
| * 通过主键删除数据 | |||
| @@ -78,10 +78,7 @@ public class ExperimentInstanceStatusTask { | |||
| } | |||
| ModelDependency modelDependency = modelDependencyListquery.get(0); | |||
| //查看状态, | |||
| if (StringUtils.equals("Succeeded",experimentIns.getStatus())){ | |||
| modelDependency.setState(1); | |||
| modelDependencyList.add(modelDependency); | |||
| }else{ | |||
| if (StringUtils.equals("Failed",experimentIns.getStatus())){ | |||
| //取出节点状态 | |||
| String trainTask = modelDependency.getTrainTask(); | |||
| Map<String, Object> trainMap = JacksonUtil.parseJSONStr2Map(trainTask); | |||
| @@ -91,17 +88,20 @@ public class ExperimentInstanceStatusTask { | |||
| } | |||
| String nodesStatus = experimentIns.getNodesStatus(); | |||
| Map<String, Object> nodeMaps = JacksonUtil.parseJSONStr2Map(nodesStatus); | |||
| Map<String, Object> nodeMap = JacksonUtil.parseJSONStr2Map((String) nodeMaps.get(task_id)); | |||
| Map<String, Object> nodeMap = JacksonUtil.parseJSONStr2Map(JacksonUtil.toJSONString(nodeMaps.get(task_id))); | |||
| if (nodeMap==null){ | |||
| continue; | |||
| } | |||
| if (StringUtils.equals("Succeeded",(String)nodeMap.get("phase"))){ | |||
| modelDependency.setState(1); | |||
| if (!StringUtils.equals("Succeeded",(String)nodeMap.get("phase"))){ | |||
| modelDependency.setState(0); | |||
| modelDependencyList.add(modelDependency); | |||
| } | |||
| } | |||
| } | |||
| modelDependencyDao.insertOrUpdateBatch(modelDependencyList); | |||
| if (modelDependencyList.size()>0) { | |||
| modelDependencyDao.insertOrUpdateBatch(modelDependencyList); | |||
| } | |||
| } | |||
| } | |||
| @@ -226,8 +226,7 @@ public class ExperimentServiceImpl implements ExperimentService { | |||
| throw new RuntimeException("转换流水线失败"); | |||
| } | |||
| Map<String, Object> converMap = JsonUtils.jsonToMap(convertRes); | |||
| //得到dependendcy | |||
| Map<String ,Object> dependendcy = (Map<String, Object>)converMap.get("model_dependency"); | |||
| // 组装运行接口json | |||
| Map<String, Object> runReqMap = new HashMap<>(); | |||
| runReqMap.put("data", converMap.get("data")); | |||
| @@ -269,7 +268,11 @@ public class ExperimentServiceImpl implements ExperimentService { | |||
| ExperimentIns insert = experimentInsService.insert(experimentIns); | |||
| //插入到模型依赖关系表 | |||
| insertModelDependency(dependendcy,insert.getId(),experiment.getName(),params); | |||
| //得到dependendcy | |||
| Map<String, Object> converMap2 = JsonUtils.jsonToMap(JacksonUtil.replaceInAarry(convertRes, params)); | |||
| Map<String ,Object> dependendcy = (Map<String, Object>)converMap2.get("model_dependency"); | |||
| Map<String ,Object> trainInfo = (Map<String, Object>)converMap2.get("component_info"); | |||
| insertModelDependency(dependendcy,trainInfo,insert.getId(),experiment.getName()); | |||
| }catch (Exception e){ | |||
| throw new RuntimeException(e); | |||
| @@ -278,17 +281,23 @@ public class ExperimentServiceImpl implements ExperimentService { | |||
| experiment.setExperimentInsList(updatedExperimentInsList); | |||
| return experiment; | |||
| } | |||
| private void insertModelDependency(Map<String ,Object> dependendcy, Integer experimentInsId, String experimentName, List<Map<String, Object>> params) throws Exception { | |||
| private void insertModelDependency(Map<String ,Object> dependendcy,Map<String ,Object> trainInfo, Integer experimentInsId, String experimentName) throws Exception { | |||
| Iterator<Map.Entry<String, Object>> dependendcyIterator = dependendcy.entrySet().iterator(); | |||
| Map<String, Object> modelTrain = (Map<String, Object>) trainInfo.get("model_train"); | |||
| Map<String, Object> modelEvaluate = (Map<String, Object>) trainInfo.get("model_evaluate"); | |||
| Map<String, Object> modelExport = (Map<String, Object>) trainInfo.get("model_export"); | |||
| while (dependendcyIterator.hasNext()) { | |||
| ModelDependency modelDependency = new ModelDependency(); | |||
| Map.Entry<String, Object> entry = dependendcyIterator.next(); | |||
| String key = entry.getKey(); | |||
| Map<String, Object> modelDel = (Map<String, Object>) entry.getValue(); | |||
| Map<String, Object> source = (Map<String, Object>) modelDel.get("source"); | |||
| List<Map<String, Object>> test = (List<Map<String, Object>>) modelDel.get("test"); | |||
| List<Map<String, Object>> target = (List<Map<String, Object>>) modelDel.get("target"); | |||
| String sourceTaskId = (String) source.get("task_id"); | |||
| Map<String, Object> modelTrainMap = (Map<String, Object>)modelTrain.get(sourceTaskId); | |||
| //处理project数据 | |||
| Map<String, String> projectMap = (Map<String, String>) modelDel.get("project"); | |||
| Map<String, String> projectMap = (Map<String, String>) modelTrainMap.get("project"); | |||
| ProjectDepency projectDepency = new ProjectDepency(); | |||
| projectDepency.setBranch(projectMap.get("branch")); | |||
| String projectUrl = projectMap.get("url"); | |||
| @@ -296,86 +305,193 @@ public class ExperimentServiceImpl implements ExperimentService { | |||
| projectDepency.setName(projectUrl.substring(projectUrl.lastIndexOf('/') + 1, projectUrl.length() - 4)); | |||
| //依赖项目 | |||
| modelDependency.setProjectDependency(JsonUtils.objectToJson(projectDepency)); | |||
| //处理镜像 | |||
| Map<String, String> imagesMap = (Map<String, String>) modelTrainMap.get("image"); | |||
| modelDependency.setTrainImage(imagesMap.get("name")); | |||
| List<Map<String, Object>> trainParamList = (List<Map<String, Object>>) modelTrainMap.get("params"); | |||
| modelDependency.setTrainParams(JsonUtils.objectToJson(trainParamList)); | |||
| //处理source数据 | |||
| Map<String, Object> sourceMap = (Map<String, Object>) modelDel.get("source"); | |||
| List<Map<String, Object>> modelsList = (List<Map<String, Object>>) sourceMap.get("models"); | |||
| for(int i=0;i<modelsList.size();i++){ | |||
| Map<String, Object> model = modelsList.get(i); | |||
| Models models = modelsService.queryById((Integer) model.get("model_id")); | |||
| if (models == null){ | |||
| throw new Exception("源模型不存在"); | |||
| List<Map<String, Object>> modelsList = (List<Map<String, Object>>) modelTrainMap.get("models"); | |||
| if (modelsList != null) { | |||
| for (int i = 0; i < modelsList.size(); i++) { | |||
| Map<String, Object> model = modelsList.get(i); | |||
| Models models = modelsService.queryById((Integer) model.get("model_id")); | |||
| if (models == null) { | |||
| throw new Exception("源模型不存在"); | |||
| } | |||
| model.put("model_name", models.getName()); | |||
| } | |||
| model.put("model_name", models.getName()); | |||
| //父模型 | |||
| modelDependency.setParentModels(JsonUtils.objectToJson(modelsList)); | |||
| } | |||
| //父模型 | |||
| modelDependency.setParentModels(JsonUtils.objectToJson(modelsList)); | |||
| List<Map<String, Object>> datasetsList = (List<Map<String, Object>>) sourceMap.get("datasets"); | |||
| for(int i=0;i<datasetsList.size();i++){ | |||
| Map<String, Object> datasets = datasetsList.get(i); | |||
| Dataset dataset = datasetService.queryById((Integer) datasets.get("dataset_id")); | |||
| if (dataset == null){ | |||
| throw new Exception("源数据集不存在"); | |||
| List<Map<String, Object>> datasetsList = (List<Map<String, Object>>) modelTrainMap.get("datasets"); | |||
| if (datasetsList != null) { | |||
| for (int i = 0; i < datasetsList.size(); i++) { | |||
| Map<String, Object> datasets = datasetsList.get(i); | |||
| Dataset dataset = datasetService.queryById((Integer) datasets.get("dataset_id")); | |||
| if (dataset == null) { | |||
| throw new Exception("源数据集不存在"); | |||
| } | |||
| datasets.put("dataset_name", dataset.getName()); | |||
| } | |||
| datasets.put("dataset_name", dataset.getName()); | |||
| } | |||
| //训练数据集 | |||
| modelDependency.setTrainDataset(JsonUtils.objectToJson(datasetsList)); | |||
| TrainTaskDepency trainTaskDepency = new TrainTaskDepency(); | |||
| trainTaskDepency.setTaskId(key); | |||
| trainTaskDepency.setTaskId(sourceTaskId); | |||
| trainTaskDepency.setInsId(experimentInsId); | |||
| trainTaskDepency.setName(experimentName); | |||
| //训练任务 | |||
| modelDependency.setTrainTask(JsonUtils.objectToJson(trainTaskDepency)); | |||
| modelDependency.setExpInsId(experimentInsId); | |||
| //处理test数据 | |||
| List<Map<String, Object>> testDatasetsList = (List<Map<String, Object>>) modelDel.get("test"); | |||
| List<Map<String, Object>> resultTestDatasets = new ArrayList<Map<String, Object>>(); | |||
| for(int i=0;i<testDatasetsList.size();i++){ | |||
| Map<String, Object> datasets = testDatasetsList.get(i); | |||
| List<Map<String, Object>> realDataSetList = (List<Map<String, Object>>) datasets.get("datasets"); | |||
| for(int j=0;j<realDataSetList.size();j++){ | |||
| Map<String, Object> realDataSet = realDataSetList.get(j); | |||
| Dataset dataset = datasetService.queryById((Integer) realDataSet.get("dataset_id")); | |||
| if (dataset == null){ | |||
| throw new Exception("源数据集不存在"); | |||
| //处理test数据 | |||
| if (test != null) { | |||
| for(int i=0;i<test.size();i++){ | |||
| Map<String, Object> testMap = test.get(i); | |||
| String testTaskId = (String) testMap.get("task_id"); | |||
| Map<String, Object> evaluateMap = (Map<String, Object>) modelEvaluate.get(testTaskId); | |||
| List<Map<String, Object>> realDataSetList = (List<Map<String, Object>>) evaluateMap.get("datasets"); | |||
| for(int j=0;j<realDataSetList.size();j++){ | |||
| Map<String, Object> realDataSet = realDataSetList.get(j); | |||
| Dataset dataset = datasetService.queryById((Integer) realDataSet.get("dataset_id")); | |||
| if (dataset == null){ | |||
| throw new Exception("源数据集不存在"); | |||
| } | |||
| realDataSet.put("dataset_name", dataset.getName()); | |||
| resultTestDatasets.add(realDataSet); | |||
| } | |||
| realDataSet.put("dataset_name", dataset.getName()); | |||
| resultTestDatasets.add(realDataSet); | |||
| } | |||
| } | |||
| } | |||
| //测试数据集 | |||
| modelDependency.setTestDataset(JsonUtils.objectToJson(resultTestDatasets)); | |||
| //检查是否存在target,如果存在说明在流水线用了节点导入,如果没有说明没有导入,等待手动push | |||
| List<Map<String, Object>> modelTargetList = (List<Map<String, Object>>) modelDel.get("target"); | |||
| if (modelTargetList==null||modelTargetList.size()==0){ | |||
| modelDependency.setState(1); | |||
| modelDependencyService.insert(modelDependency); | |||
| }else { | |||
| modelDependency.setState(2); | |||
| for(int i=0;i<modelTargetList.size();i++){ | |||
| Map<String, Object> model = modelTargetList.get(i); | |||
| String version = null; | |||
| //可能是参数,必须从实验参数读取 | |||
| if (params != null) { | |||
| for (Map<String, Object> param : params) { | |||
| if (param.containsKey("param_name") && StringUtils.equals("model_version",(String) param.get("param_name"))) { | |||
| version = param.get("param_value").toString(); | |||
| } | |||
| } | |||
| modelDependency.setTestDataset(JsonUtils.objectToJson(resultTestDatasets)); | |||
| } | |||
| //处理target数据 | |||
| if (target != null) { | |||
| for (int i = 0; i < target.size(); i++) { | |||
| Map<String, Object> targetMap = target.get(i); | |||
| String targetaskId = (String) targetMap.get("task_id"); | |||
| Map<String, Object> exportMap = (Map<String, Object>) modelExport.get(targetaskId); | |||
| List<Map<String, Object>> modelTargetList = (List<Map<String, Object>>) exportMap.get("models"); | |||
| modelDependency.setState(2); | |||
| for (int j = 0; j < modelTargetList.size(); j++) { | |||
| Map<String, Object> model = modelTargetList.get(i); | |||
| modelDependency.setVersion((String) model.get("model_version")); | |||
| modelDependency.setCurrentModelId((Integer) model.get("model_id")); | |||
| //因为可能有多成果模型,多次插入 | |||
| modelDependencyService.insert(modelDependency); | |||
| } | |||
| modelDependency.setVersion(StringUtils.isEmpty(version)?(String)model.get("model_version"):version); | |||
| modelDependency.setCurrentModelId((Integer) model.get("model_id")); | |||
| //因为可能有多成果模型,多次插入 | |||
| modelDependencyService.insert(modelDependency); | |||
| } | |||
| }else { | |||
| modelDependency.setState(2); | |||
| modelDependencyService.insert(modelDependency); | |||
| } | |||
| } | |||
| } | |||
| /** | |||
| * 被废弃的旧JSON | |||
| * @param experiment | |||
| * @return | |||
| * @throws Exception | |||
| */ | |||
| // private void insertModelDependency(Map<String ,Object> dependendcy,Map<String ,Object> trainInfo, Integer experimentInsId, String experimentName, List<Map<String, Object>> params) throws Exception { | |||
| // Iterator<Map.Entry<String, Object>> dependendcyIterator = dependendcy.entrySet().iterator(); | |||
| // while (dependendcyIterator.hasNext()) { | |||
| // ModelDependency modelDependency = new ModelDependency(); | |||
| // Map.Entry<String, Object> entry = dependendcyIterator.next(); | |||
| // String key = entry.getKey(); | |||
| // Map<String, Object> modelDel = (Map<String, Object>) entry.getValue(); | |||
| // //处理project数据 | |||
| // Map<String, String> projectMap = (Map<String, String>) modelDel.get("project"); | |||
| // ProjectDepency projectDepency = new ProjectDepency(); | |||
| // projectDepency.setBranch(projectMap.get("branch")); | |||
| // String projectUrl = projectMap.get("url"); | |||
| // projectDepency.setUrl(projectUrl); | |||
| // projectDepency.setName(projectUrl.substring(projectUrl.lastIndexOf('/') + 1, projectUrl.length() - 4)); | |||
| // //依赖项目 | |||
| // modelDependency.setProjectDependency(JsonUtils.objectToJson(projectDepency)); | |||
| // //处理source数据 | |||
| // Map<String, Object> sourceMap = (Map<String, Object>) modelDel.get("source"); | |||
| // List<Map<String, Object>> modelsList = (List<Map<String, Object>>) sourceMap.get("models"); | |||
| // for(int i=0;i<modelsList.size();i++){ | |||
| // Map<String, Object> model = modelsList.get(i); | |||
| // Models models = modelsService.queryById((Integer) model.get("model_id")); | |||
| // if (models == null){ | |||
| // throw new Exception("源模型不存在"); | |||
| // } | |||
| // model.put("model_name", models.getName()); | |||
| // } | |||
| // //父模型 | |||
| // modelDependency.setParentModels(JsonUtils.objectToJson(modelsList)); | |||
| // | |||
| // List<Map<String, Object>> datasetsList = (List<Map<String, Object>>) sourceMap.get("datasets"); | |||
| // for(int i=0;i<datasetsList.size();i++){ | |||
| // Map<String, Object> datasets = datasetsList.get(i); | |||
| // Dataset dataset = datasetService.queryById((Integer) datasets.get("dataset_id")); | |||
| // if (dataset == null){ | |||
| // throw new Exception("源数据集不存在"); | |||
| // } | |||
| // datasets.put("dataset_name", dataset.getName()); | |||
| // } | |||
| // //训练数据集 | |||
| // modelDependency.setTrainDataset(JsonUtils.objectToJson(datasetsList)); | |||
| // | |||
| // TrainTaskDepency trainTaskDepency = new TrainTaskDepency(); | |||
| // trainTaskDepency.setTaskId(key); | |||
| // trainTaskDepency.setInsId(experimentInsId); | |||
| // trainTaskDepency.setName(experimentName); | |||
| // //训练任务 | |||
| // modelDependency.setTrainTask(JsonUtils.objectToJson(trainTaskDepency)); | |||
| // modelDependency.setExpInsId(experimentInsId); | |||
| // //处理test数据 | |||
| // List<Map<String, Object>> testDatasetsList = (List<Map<String, Object>>) modelDel.get("test"); | |||
| // List<Map<String, Object>> resultTestDatasets = new ArrayList<Map<String, Object>>(); | |||
| // for(int i=0;i<testDatasetsList.size();i++){ | |||
| // Map<String, Object> datasets = testDatasetsList.get(i); | |||
| // List<Map<String, Object>> realDataSetList = (List<Map<String, Object>>) datasets.get("datasets"); | |||
| // for(int j=0;j<realDataSetList.size();j++){ | |||
| // Map<String, Object> realDataSet = realDataSetList.get(j); | |||
| // Dataset dataset = datasetService.queryById((Integer) realDataSet.get("dataset_id")); | |||
| // if (dataset == null){ | |||
| // throw new Exception("源数据集不存在"); | |||
| // } | |||
| // realDataSet.put("dataset_name", dataset.getName()); | |||
| // resultTestDatasets.add(realDataSet); | |||
| // } | |||
| // | |||
| // } | |||
| // //测试数据集 | |||
| // modelDependency.setTestDataset(JsonUtils.objectToJson(resultTestDatasets)); | |||
| // | |||
| // //检查是否存在target,如果存在说明在流水线用了节点导入,如果没有说明没有导入,等待手动push | |||
| // List<Map<String, Object>> modelTargetList = (List<Map<String, Object>>) modelDel.get("target"); | |||
| // if (modelTargetList==null||modelTargetList.size()==0){ | |||
| // modelDependency.setState(1); | |||
| // modelDependencyService.insert(modelDependency); | |||
| // }else { | |||
| // modelDependency.setState(2); | |||
| // for(int i=0;i<modelTargetList.size();i++){ | |||
| // Map<String, Object> model = modelTargetList.get(i); | |||
| // String version = null; | |||
| // //可能是参数,必须从实验参数读取 | |||
| // if (params != null) { | |||
| // for (Map<String, Object> param : params) { | |||
| // if (param.containsKey("param_name") && StringUtils.equals("model_version",(String) param.get("param_name"))) { | |||
| // version = param.get("param_value").toString(); | |||
| // } | |||
| // } | |||
| // } | |||
| // modelDependency.setVersion(StringUtils.isEmpty(version)?(String)model.get("model_version"):version); | |||
| // modelDependency.setCurrentModelId((Integer) model.get("model_id")); | |||
| // //因为可能有多成果模型,多次插入 | |||
| // modelDependencyService.insert(modelDependency); | |||
| // } | |||
| // } | |||
| // } | |||
| // } | |||
| @Override | |||
| public Experiment addAndRunExperiment(Experiment experiment) throws Exception { | |||
| // 第一步: 调用add方法插入实验记录到数据库 | |||
| @@ -217,5 +217,4 @@ public class ModelDependencyServiceImpl implements ModelDependencyService { | |||
| return this.modelDependencyDao.update(modelDependency)>0?"删除成功":"删除失败"; | |||
| } | |||
| } | |||
| @@ -206,6 +206,16 @@ public class ModelsVersionServiceImpl implements ModelsVersionService { | |||
| String result = this.removeById(modelsVersion.getId()); | |||
| results.put(modelsVersion.getId(), result); | |||
| } | |||
| // 删除依赖关系 | |||
| ModelDependency modelDependency = new ModelDependency(); | |||
| modelDependency.setCurrentModelId(modelsId); | |||
| modelDependency.setVersion(version); | |||
| modelDependency.setState(1); | |||
| List<ModelDependency> modelDependencyList = modelDependencyService.queryByModelDependency(modelDependency); | |||
| if (modelDependencyList.size()>0){ | |||
| ModelDependency modelDependency1 = modelDependencyList.get(0); | |||
| modelDependencyService.removeById(modelDependency1.getId()); | |||
| } | |||
| return results; | |||
| } | |||
| @@ -6,6 +6,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; | |||
| import com.fasterxml.jackson.databind.SerializationFeature; | |||
| import com.fasterxml.jackson.databind.type.CollectionType; | |||
| import com.ruoyi.common.core.utils.StringUtils; | |||
| import org.apache.xmlbeans.impl.xb.xsdschema.Public; | |||
| import java.text.SimpleDateFormat; | |||
| import java.util.ArrayList; | |||
| @@ -58,6 +59,21 @@ public class JacksonUtil { | |||
| } | |||
| } | |||
| public static String replaceInAarry(String res, List<Map<String, Object>> params) { | |||
| // 解析 JSON 数组 | |||
| if (params == null) { | |||
| return res; | |||
| } | |||
| for (int i = 0; i < params.size(); i++) { | |||
| Map<String, Object> stringObjectMap = params.get(i); | |||
| if (res.contains("${" + stringObjectMap.get("param_name") + "}")) { | |||
| res = res.replace("${" + stringObjectMap.get("param_name") + "}", String.valueOf( stringObjectMap.get("param_value"))); | |||
| } | |||
| } | |||
| return res; | |||
| } | |||
| /** | |||
| * 获取ObjectMapper。其本身是线程安全的,可以作为成员变量,但传入的参数不可能每次都一样,所以不使用成员变量。 | |||
| * | |||
| @@ -10,6 +10,8 @@ | |||
| <result property="refItem" column="ref_item" jdbcType="VARCHAR"/> | |||
| <result property="trainTask" column="train_task" jdbcType="VARCHAR"/> | |||
| <result property="trainDataset" column="train_dataset" jdbcType="VARCHAR"/> | |||
| <result property="trainParams" column="train_params" jdbcType="VARCHAR"/> | |||
| <result property="trainImage" column="train_image" jdbcType="VARCHAR"/> | |||
| <result property="testDataset" column="test_dataset" jdbcType="VARCHAR"/> | |||
| <result property="projectDependency" column="project_dependency" jdbcType="VARCHAR"/> | |||
| <result property="version" column="version" jdbcType="VARCHAR"/> | |||
| @@ -20,18 +22,18 @@ | |||
| <result property="state" column="state" jdbcType="INTEGER"/> | |||
| </resultMap> | |||
| <!--查询单个--> | |||
| <select id="queryById" resultMap="ModelDependencyMap"> | |||
| <select id="queryChildrenByVersionId" resultMap="ModelDependencyMap"> | |||
| select | |||
| id,current_model_id,exp_ins_id,parent_models,ref_item,train_task,train_dataset,test_dataset,project_dependency,version,create_by,create_time,update_by,update_time,state | |||
| id,current_model_id,exp_ins_id,parent_models,ref_item,train_task,train_dataset,train_params,train_image,test_dataset,project_dependency,version,create_by,create_time,update_by,update_time,state | |||
| from model_dependency | |||
| where id = #{id} | |||
| <where> | |||
| parent_models like concat('%', #{model_id}, '%') AND parent_models like concat('%', #{version}, '%') | |||
| </where> | |||
| </select> | |||
| <!--查询指定行数据--> | |||
| <select id="queryAllByLimit" resultMap="ModelDependencyMap"> | |||
| <select id="queryByModelDependency" resultMap="ModelDependencyMap"> | |||
| select | |||
| id,current_model_id,exp_ins_id,parent_models,ref_item,train_task,train_dataset,test_dataset,project_dependency,version,create_by,create_time,update_by,update_time,state | |||
| id,current_model_id,exp_ins_id,parent_models,ref_item,train_task,train_dataset,train_params,train_image,test_dataset,project_dependency,version,create_by,create_time,update_by,update_time,state | |||
| from model_dependency | |||
| <where> | |||
| <if test="modelDependency.id != null"> | |||
| @@ -55,6 +57,12 @@ | |||
| <if test="modelDependency.trainDataset != null and modelDependency.trainDataset != ''"> | |||
| and train_dataset = #{modelDependency.trainDataset} | |||
| </if> | |||
| <if test="modelDependency.trainParams != null and modelDependency.trainParams != ''"> | |||
| and train_params = #{modelDependency.trainParams} | |||
| </if> | |||
| <if test="modelDependency.trainImage != null and modelDependency.trainImage != ''"> | |||
| and train_image = #{modelDependency.trainImage} | |||
| </if> | |||
| <if test="modelDependency.testDataset != null and modelDependency.testDataset != ''"> | |||
| and test_dataset = #{modelDependency.testDataset} | |||
| </if> | |||
| @@ -80,69 +88,76 @@ | |||
| and state = #{modelDependency.state} | |||
| </if> | |||
| </where> | |||
| limit #{pageable.offset}, #{pageable.pageSize} | |||
| </select> | |||
| <select id="queryChildrenByVersionId" resultMap="ModelDependencyMap"> | |||
| <!--查询单个--> | |||
| <select id="queryById" resultMap="ModelDependencyMap"> | |||
| select | |||
| id,current_model_id,exp_ins_id,parent_models,ref_item,train_task,train_dataset,test_dataset,project_dependency,version,create_by,create_time,update_by,update_time,state | |||
| id, current_model_id, exp_ins_id, parent_models, ref_item, train_task, train_dataset, train_params, train_image, test_dataset, project_dependency, version, create_by, create_time, update_by, update_time, state | |||
| from model_dependency | |||
| <where> | |||
| parent_models like concat('%', #{model_id}, '%') AND parent_models like concat('%', #{version}, '%') | |||
| </where> | |||
| where id = #{id} | |||
| </select> | |||
| <!--查询指定行数据--> | |||
| <select id="queryByModelDependency" resultMap="ModelDependencyMap"> | |||
| <select id="queryAllByLimit" resultMap="ModelDependencyMap"> | |||
| select | |||
| id,current_model_id,exp_ins_id,parent_models,ref_item,train_task,train_dataset,test_dataset,project_dependency,version,create_by,create_time,update_by,update_time,state | |||
| id, current_model_id, exp_ins_id, parent_models, ref_item, train_task, train_dataset, train_params, train_image, | |||
| test_dataset, project_dependency, version, create_by, create_time, update_by, update_time, state | |||
| from model_dependency | |||
| <where> | |||
| <if test="modelDependency.id != null"> | |||
| and id = #{modelDependency.id} | |||
| <if test="id != null"> | |||
| and id = #{id} | |||
| </if> | |||
| <if test="modelDependency.currentModelId != null"> | |||
| and current_model_id = #{modelDependency.currentModelId} | |||
| <if test="currentModelId != null"> | |||
| and current_model_id = #{currentModelId} | |||
| </if> | |||
| <if test="modelDependency.expInsId != null"> | |||
| and exp_ins_id = #{modelDependency.expInsId} | |||
| <if test="expInsId != null"> | |||
| and exp_ins_id = #{expInsId} | |||
| </if> | |||
| <if test="modelDependency.parentModels != null"> | |||
| and parent_models = #{modelDependency.parentModels} | |||
| <if test="parentModels != null and parentModels != ''"> | |||
| and parent_models = #{parentModels} | |||
| </if> | |||
| <if test="modelDependency.refItem != null and modelDependency.refItem != ''"> | |||
| and ref_item = #{modelDependency.refItem} | |||
| <if test="refItem != null and refItem != ''"> | |||
| and ref_item = #{refItem} | |||
| </if> | |||
| <if test="modelDependency.trainTask != null and modelDependency.trainTask != ''"> | |||
| and train_task = #{modelDependency.trainTask} | |||
| <if test="trainTask != null and trainTask != ''"> | |||
| and train_task = #{trainTask} | |||
| </if> | |||
| <if test="modelDependency.trainDataset != null and modelDependency.trainDataset != ''"> | |||
| and train_dataset = #{modelDependency.trainDataset} | |||
| <if test="trainDataset != null and trainDataset != ''"> | |||
| and train_dataset = #{trainDataset} | |||
| </if> | |||
| <if test="modelDependency.testDataset != null and modelDependency.testDataset != ''"> | |||
| and test_dataset = #{modelDependency.testDataset} | |||
| <if test="trainParams != null and trainParams != ''"> | |||
| and train_params = #{trainParams} | |||
| </if> | |||
| <if test="modelDependency.projectDependency != null and modelDependency.projectDependency != ''"> | |||
| and project_dependency = #{modelDependency.projectDependency} | |||
| <if test="trainImage != null and trainImage != ''"> | |||
| and train_image = #{trainImage} | |||
| </if> | |||
| <if test="modelDependency.version != null and modelDependency.version != ''"> | |||
| and version = #{modelDependency.version} | |||
| <if test="testDataset != null and testDataset != ''"> | |||
| and test_dataset = #{testDataset} | |||
| </if> | |||
| <if test="modelDependency.createBy != null and modelDependency.createBy != ''"> | |||
| and create_by = #{modelDependency.createBy} | |||
| <if test="projectDependency != null and projectDependency != ''"> | |||
| and project_dependency = #{projectDependency} | |||
| </if> | |||
| <if test="modelDependency.createTime != null"> | |||
| and create_time = #{modelDependency.createTime} | |||
| <if test="version != null and version != ''"> | |||
| and version = #{version} | |||
| </if> | |||
| <if test="modelDependency.updateBy != null and modelDependency.updateBy != ''"> | |||
| and update_by = #{modelDependency.updateBy} | |||
| <if test="createBy != null and createBy != ''"> | |||
| and create_by = #{createBy} | |||
| </if> | |||
| <if test="modelDependency.updateTime != null"> | |||
| and update_time = #{modelDependency.updateTime} | |||
| <if test="createTime != null"> | |||
| and create_time = #{createTime} | |||
| </if> | |||
| <if test="modelDependency.state != null"> | |||
| and state = #{modelDependency.state} | |||
| <if test="updateBy != null and updateBy != ''"> | |||
| and update_by = #{updateBy} | |||
| </if> | |||
| <if test="updateTime != null"> | |||
| and update_time = #{updateTime} | |||
| </if> | |||
| <if test="state != null"> | |||
| and state = #{state} | |||
| </if> | |||
| </where> | |||
| limit #{pageable.offset}, #{pageable.pageSize} | |||
| </select> | |||
| <!--统计总行数--> | |||
| @@ -150,158 +165,163 @@ | |||
| select count(1) | |||
| from model_dependency | |||
| <where> | |||
| <if test="modelDependency.id != null"> | |||
| and id = #{modelDependency.id} | |||
| <if test="id != null"> | |||
| and id = #{id} | |||
| </if> | |||
| <if test="modelDependency.currentModelId != null"> | |||
| and current_model_id = #{modelDependency.currentModelId} | |||
| <if test="currentModelId != null"> | |||
| and current_model_id = #{currentModelId} | |||
| </if> | |||
| <if test="modelDependency.expInsId != null"> | |||
| and exp_ins_id = #{modelDependency.expInsId} | |||
| <if test="expInsId != null"> | |||
| and exp_ins_id = #{expInsId} | |||
| </if> | |||
| <if test="modelDependency.parentModels != null"> | |||
| and parent_models = #{modelDependency.parentModels} | |||
| <if test="parentModels != null and parentModels != ''"> | |||
| and parent_models = #{parentModels} | |||
| </if> | |||
| <if test="modelDependency.refItem != null and modelDependency.refItem != ''"> | |||
| and ref_item = #{modelDependency.refItem} | |||
| <if test="refItem != null and refItem != ''"> | |||
| and ref_item = #{refItem} | |||
| </if> | |||
| <if test="modelDependency.trainTask != null and modelDependency.trainTask != ''"> | |||
| and train_task = #{modelDependency.trainTask} | |||
| <if test="trainTask != null and trainTask != ''"> | |||
| and train_task = #{trainTask} | |||
| </if> | |||
| <if test="modelDependency.trainDataset != null and modelDependency.trainDataset != ''"> | |||
| and train_dataset = #{modelDependency.trainDataset} | |||
| <if test="trainDataset != null and trainDataset != ''"> | |||
| and train_dataset = #{trainDataset} | |||
| </if> | |||
| <if test="modelDependency.testDataset != null and modelDependency.testDataset != ''"> | |||
| and test_dataset = #{modelDependency.testDataset} | |||
| <if test="trainParams != null and trainParams != ''"> | |||
| and train_params = #{trainParams} | |||
| </if> | |||
| <if test="modelDependency.projectDependency != null and modelDependency.projectDependency != ''"> | |||
| and project_dependency = #{modelDependency.projectDependency} | |||
| <if test="trainImage != null and trainImage != ''"> | |||
| and train_image = #{trainImage} | |||
| </if> | |||
| <if test="modelDependency.version != null and modelDependency.version != ''"> | |||
| and version = #{modelDependency.version} | |||
| <if test="testDataset != null and testDataset != ''"> | |||
| and test_dataset = #{testDataset} | |||
| </if> | |||
| <if test="modelDependency.createBy != null and modelDependency.createBy != ''"> | |||
| and create_by = #{modelDependency.createBy} | |||
| <if test="projectDependency != null and projectDependency != ''"> | |||
| and project_dependency = #{projectDependency} | |||
| </if> | |||
| <if test="modelDependency.createTime != null"> | |||
| and create_time = #{modelDependency.createTime} | |||
| <if test="version != null and version != ''"> | |||
| and version = #{version} | |||
| </if> | |||
| <if test="modelDependency.updateBy != null and modelDependency.updateBy != ''"> | |||
| and update_by = #{modelDependency.updateBy} | |||
| <if test="createBy != null and createBy != ''"> | |||
| and create_by = #{createBy} | |||
| </if> | |||
| <if test="modelDependency.updateTime != null"> | |||
| and update_time = #{modelDependency.updateTime} | |||
| <if test="createTime != null"> | |||
| and create_time = #{createTime} | |||
| </if> | |||
| <if test="modelDependency.state != null"> | |||
| and state = #{modelDependency.state} | |||
| <if test="updateBy != null and updateBy != ''"> | |||
| and update_by = #{updateBy} | |||
| </if> | |||
| <if test="updateTime != null"> | |||
| and update_time = #{updateTime} | |||
| </if> | |||
| <if test="state != null"> | |||
| and state = #{state} | |||
| </if> | |||
| </where> | |||
| </select> | |||
| <!--新增所有列--> | |||
| <insert id="insert" keyProperty="id" useGeneratedKeys="true"> | |||
| insert into model_dependency (current_model_id, exp_ins_id, parent_models, ref_item, train_task, train_dataset, test_dataset, project_dependency,version, create_by, create_time, update_by, update_time, state) | |||
| values (#{modelDependency.currentModelId}, #{modelDependency.expInsId},#{modelDependency.parentModels}, #{modelDependency.refItem}, #{modelDependency.trainTask}, #{modelDependency.trainDataset}, #{modelDependency.testDataset}, #{modelDependency.projectDependency},#{modelDependency.version}, #{modelDependency.createBy}, #{modelDependency.createTime}, #{modelDependency.updateBy}, #{modelDependency.updateTime}, #{modelDependency.state}) | |||
| insert into model_dependency(current_model_id, exp_ins_id, parent_models, ref_item, train_task, train_dataset, train_params, train_image, test_dataset, project_dependency, version, create_by, create_time, update_by, update_time, state) | |||
| values (#{currentModelId}, #{expInsId}, #{parentModels}, #{refItem}, #{trainTask}, #{trainDataset}, #{trainParams}, #{trainImage}, #{testDataset}, #{projectDependency}, #{version}, #{createBy}, #{createTime}, #{updateBy}, #{updateTime}, #{state}) | |||
| </insert> | |||
| <insert id="insertBatch" keyProperty="id" useGeneratedKeys="true"> | |||
| insert into model_dependency ( | |||
| current_model_id, | |||
| exp_ins_id, | |||
| parent_models, | |||
| ref_item, | |||
| train_task, | |||
| train_dataset, | |||
| test_dataset, | |||
| project_dependency, | |||
| version, | |||
| create_by, | |||
| create_time, | |||
| update_by, | |||
| update_time, | |||
| state | |||
| ) | |||
| insert into model_dependency(current_model_id, exp_ins_id, parent_models, ref_item, train_task, train_dataset, | |||
| train_params, train_image, test_dataset, project_dependency, version, create_by, create_time, update_by, | |||
| update_time, state) | |||
| values | |||
| <foreach collection="entities" item="entity" separator=","> | |||
| ( | |||
| #{entity.currentModelId}, | |||
| #{entity.expInsId}, | |||
| #{entity.parentModels}, | |||
| #{entity.refItem}, | |||
| #{entity.trainTask}, | |||
| #{entity.trainDataset}, | |||
| #{entity.testDataset}, | |||
| #{entity.projectDependency}, | |||
| #{entity.version}, | |||
| #{entity.createBy}, | |||
| #{entity.createTime}, | |||
| #{entity.updateBy}, | |||
| #{entity.updateTime}, | |||
| #{entity.state} | |||
| ) | |||
| (#{entity.currentModelId}, #{entity.expInsId}, #{entity.parentModels}, #{entity.refItem}, | |||
| #{entity.trainTask}, #{entity.trainDataset}, #{entity.trainParams}, #{entity.trainImage}, | |||
| #{entity.testDataset}, #{entity.projectDependency}, #{entity.version}, #{entity.createBy}, | |||
| #{entity.createTime}, #{entity.updateBy}, #{entity.updateTime}, #{entity.state}) | |||
| </foreach> | |||
| </insert> | |||
| <insert id="insertOrUpdateBatch" keyProperty="id" useGeneratedKeys="true"> | |||
| insert into model_dependency(current_model_idparent_modelsref_itemtrain_tasktrain_datasettest_datasetcreate_bycreate_timeupdate_byupdate_timestate) | |||
| insert into model_dependency(current_model_id, exp_ins_id, parent_models, ref_item, train_task, train_dataset, | |||
| train_params, train_image, test_dataset, project_dependency, version, create_by, create_time, update_by, | |||
| update_time, state) | |||
| values | |||
| <foreach collection="entities" item="entity" separator=","> | |||
| (#{entity.currentModelId}#{entity.parentModels}#{entity.refItem}#{entity.trainTask}#{entity.trainDataset}#{entity.testDataset}#{entity.createBy}#{entity.createTime}#{entity.updateBy}#{entity.updateTime}#{entity.state}) | |||
| (#{entity.currentModelId}, #{entity.expInsId}, #{entity.parentModels}, #{entity.refItem}, | |||
| #{entity.trainTask}, #{entity.trainDataset}, #{entity.trainParams}, #{entity.trainImage}, | |||
| #{entity.testDataset}, #{entity.projectDependency}, #{entity.version}, #{entity.createBy}, | |||
| #{entity.createTime}, #{entity.updateBy}, #{entity.updateTime}, #{entity.state}) | |||
| </foreach> | |||
| on duplicate key update | |||
| current_model_id = values(current_model_id)parent_models = values(parent_models)ref_item = values(ref_item)train_task = values(train_task)train_dataset = values(train_dataset)test_dataset = values(test_dataset)create_by = values(create_by)create_time = values(create_time)update_by = values(update_by)update_time = values(update_time)state = values(state) | |||
| current_model_id = values(current_model_id), | |||
| exp_ins_id = values(exp_ins_id), | |||
| parent_models = values(parent_models), | |||
| ref_item = values(ref_item), | |||
| train_task = values(train_task), | |||
| train_dataset = values(train_dataset), | |||
| train_params = values(train_params), | |||
| train_image = values(train_image), | |||
| test_dataset = values(test_dataset), | |||
| project_dependency = values(project_dependency), | |||
| version = values(version), | |||
| create_by = values(create_by), | |||
| create_time = values(create_time), | |||
| update_by = values(update_by), | |||
| update_time = values(update_time), | |||
| state = values(state) | |||
| </insert> | |||
| <!--通过主键修改数据--> | |||
| <update id="update"> | |||
| update model_dependency | |||
| <set> | |||
| <if test="modelDependency.currentModelId != null"> | |||
| current_model_id = #{modelDependency.currentModelId}, | |||
| <if test="currentModelId != null"> | |||
| current_model_id = #{currentModelId}, | |||
| </if> | |||
| <if test="modelDependency.expInsId != null"> | |||
| exp_ins_id = #{modelDependency.expInsId}, | |||
| <if test="expInsId != null"> | |||
| exp_ins_id = #{expInsId}, | |||
| </if> | |||
| <if test="modelDependency.parentModels != null"> | |||
| parent_models = #{modelDependency.parentModels}, | |||
| <if test="parentModels != null and parentModels != ''"> | |||
| parent_models = #{parentModels}, | |||
| </if> | |||
| <if test="modelDependency.refItem != null and modelDependency.refItem != ''"> | |||
| ref_item = #{modelDependency.refItem}, | |||
| <if test="refItem != null and refItem != ''"> | |||
| ref_item = #{refItem}, | |||
| </if> | |||
| <if test="modelDependency.trainTask != null and modelDependency.trainTask != ''"> | |||
| train_task = #{modelDependency.trainTask}, | |||
| <if test="trainTask != null and trainTask != ''"> | |||
| train_task = #{trainTask}, | |||
| </if> | |||
| <if test="modelDependency.trainDataset != null and modelDependency.trainDataset != ''"> | |||
| train_dataset = #{modelDependency.trainDataset}, | |||
| <if test="trainDataset != null and trainDataset != ''"> | |||
| train_dataset = #{trainDataset}, | |||
| </if> | |||
| <if test="modelDependency.testDataset != null and modelDependency.testDataset != ''"> | |||
| test_dataset = #{modelDependency.testDataset}, | |||
| <if test="trainParams != null and trainParams != ''"> | |||
| train_params = #{trainParams}, | |||
| </if> | |||
| <if test="modelDependency.projectDependency != null and modelDependency.projectDependency != ''"> | |||
| project_dependency = #{modelDependency.projectDependency}, | |||
| <if test="trainImage != null and trainImage != ''"> | |||
| train_image = #{trainImage}, | |||
| </if> | |||
| <if test="modelDependency.version != null and modelDependency.version != ''"> | |||
| version = #{modelDependency.version}, | |||
| <if test="testDataset != null and testDataset != ''"> | |||
| test_dataset = #{testDataset}, | |||
| </if> | |||
| <if test="modelDependency.createBy != null and modelDependency.createBy != ''"> | |||
| create_by = #{modelDependency.createBy}, | |||
| <if test="projectDependency != null and projectDependency != ''"> | |||
| project_dependency = #{projectDependency}, | |||
| </if> | |||
| <if test="modelDependency.createTime != null"> | |||
| create_time = #{modelDependency.createTime}, | |||
| <if test="version != null and version != ''"> | |||
| version = #{version}, | |||
| </if> | |||
| <if test="modelDependency.updateBy != null and modelDependency.updateBy != ''"> | |||
| update_by = #{modelDependency.updateBy}, | |||
| <if test="createBy != null and createBy != ''"> | |||
| create_by = #{createBy}, | |||
| </if> | |||
| <if test="modelDependency.updateTime != null"> | |||
| update_time = #{modelDependency.updateTime}, | |||
| <if test="createTime != null"> | |||
| create_time = #{createTime}, | |||
| </if> | |||
| <if test="modelDependency.state != null"> | |||
| state = #{modelDependency.state}, | |||
| <if test="updateBy != null and updateBy != ''"> | |||
| update_by = #{updateBy}, | |||
| </if> | |||
| <if test="updateTime != null"> | |||
| update_time = #{updateTime}, | |||
| </if> | |||
| <if test="state != null"> | |||
| state = #{state}, | |||
| </if> | |||
| </set> | |||
| where id = #{modelDependency.id} | |||
| where id = #{id} and state = 1 | |||
| </update> | |||
| <!--通过主键删除--> | |||