From 59eaa5f78751e1a240690b627c3fa6fcff323e9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=A5=BF=E5=A4=A7=E9=94=90?= <1070211640@qq.com> Date: Mon, 20 May 2024 13:56:53 +0800 Subject: [PATCH 01/31] =?UTF-8?q?=E7=BB=84=E4=BB=B6=E6=96=B0=E5=A2=9E?= =?UTF-8?q?=E5=AD=97=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/ruoyi/platform/domain/Component.java | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/Component.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/Component.java index 2e1d5de2..5f6f9cdc 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/Component.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/Component.java @@ -1,6 +1,7 @@ package com.ruoyi.platform.domain; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonRawValue; import com.fasterxml.jackson.databind.PropertyNamingStrategy; import com.fasterxml.jackson.databind.annotation.JsonNaming; import io.swagger.annotations.ApiModelProperty; @@ -58,6 +59,7 @@ public class Component implements Serializable { */ @JsonProperty("env_variables") @ApiModelProperty(name = "env_variables", value = "环境变量") + @JsonRawValue private String envVirables; /** * 资源规格 @@ -78,12 +80,20 @@ public class Component implements Serializable { * 输入参数 */ @ApiModelProperty(name = "in_parameters" ,value = "输入参数") + @JsonRawValue private String inParameters; /** * 输出参数 */ @ApiModelProperty(name = "out_parameters" ,value = "输出参数") + @JsonRawValue private String outParameters; + + /** + * 可见范围 + */ + @ApiModelProperty(name = "available_range" ,value = "1公开0私有") + private int availableRange; /** * 描述 */ @@ -124,6 +134,8 @@ public class Component implements Serializable { @ApiModelProperty(name = "state" , value = "状态") private Integer state; + + public Integer getId() { return id; } @@ -228,6 +240,14 @@ public class Component implements Serializable { this.outParameters = outParameters; } + public int getAvailableRange() { + return availableRange; + } + + public void setAvailableRange(int availableRange) { + this.availableRange = availableRange; + } + public String getDescription() { return description; } From 63ddf436f4709078e5c66966d9e742d7ffa4f3b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=A5=BF=E5=A4=A7=E9=94=90?= <1070211640@qq.com> Date: Mon, 20 May 2024 14:07:25 +0800 Subject: [PATCH 02/31] =?UTF-8?q?=E7=BB=84=E4=BB=B6=E6=96=B0=E5=A2=9E?= =?UTF-8?q?=E5=AD=97=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../managementPlatform/ComponentDaoMapper.xml | 26 +++++++++++++------ 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ComponentDaoMapper.xml b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ComponentDaoMapper.xml index d4af39d0..84b369ff 100644 --- a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ComponentDaoMapper.xml +++ b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ComponentDaoMapper.xml @@ -16,6 +16,7 @@ + @@ -28,7 +29,7 @@ @@ -42,7 +43,7 @@ + select + id,current_model_id,parent_model_id,ref_item,train_task,train_dataset,test_dataset,create_by,create_time,update_by,update_time,state + from model_dependency + where id = #{id} + + + + + + + + + + + insert into model_dependency ( + current_model_id, + parent_model_id, + ref_item, + train_task, + train_dataset, + test_dataset, + create_by, + create_time, + update_by, + update_time, + state + ) + values ( + #{currentModelId}, + #{parentModelId}, + #{refItem}, + #{trainTask}, + #{trainDataset}, + #{testDataset}, + #{createBy}, + #{createTime}, + #{updateBy}, + #{updateTime}, + #{state} + ) + + + + + insert into model_dependency ( + current_model_id, + parent_model_id, + ref_item, + train_task, + train_dataset, + test_dataset, + create_by, + create_time, + update_by, + update_time, + state + ) + values + + ( + #{entity.currentModelId}, + #{entity.parentModelId}, + #{entity.refItem}, + #{entity.trainTask}, + #{entity.trainDataset}, + #{entity.testDataset}, + #{entity.createBy}, + #{entity.createTime}, + #{entity.updateBy}, + #{entity.updateTime}, + #{entity.state} + ) + + + + + + insert into model_dependency(current_model_idparent_model_idref_itemtrain_tasktrain_datasettest_datasetcreate_bycreate_timeupdate_byupdate_timestate) + values + + (#{entity.currentModelId}#{entity.parentModelId}#{entity.refItem}#{entity.trainTask}#{entity.trainDataset}#{entity.testDataset}#{entity.createBy}#{entity.createTime}#{entity.updateBy}#{entity.updateTime}#{entity.state}) + + on duplicate key update +current_model_id = values(current_model_id)parent_model_id = values(parent_model_id)ref_item = values(ref_item)train_task = values(train_task)train_dataset = values(train_dataset)test_dataset = values(test_dataset)create_by = values(create_by)create_time = values(create_time)update_by = values(update_by)update_time = values(update_time)state = values(state) + + + + + update model_dependency + + + current_model_id = #{modelDependency.currentModelId}, + + + parent_model_id = #{modelDependency.parentModelId}, + + + ref_item = #{modelDependency.refItem}, + + + train_task = #{modelDependency.trainTask}, + + + train_dataset = #{modelDependency.trainDataset}, + + + test_dataset = #{modelDependency.testDataset}, + + + create_by = #{modelDependency.createBy}, + + + create_time = #{modelDependency.createTime}, + + + update_by = #{modelDependency.updateBy}, + + + update_time = #{modelDependency.updateTime}, + + + state = #{modelDependency.state}, + + + where id = #{id} + + + + + delete from model_dependency where id = #{id} + + + + From e27e26e3d71856e373b47172b97470abb585e8e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=A5=BF=E5=A4=A7=E9=94=90?= <1070211640@qq.com> Date: Thu, 30 May 2024 10:44:20 +0800 Subject: [PATCH 10/31] =?UTF-8?q?=E6=A8=A1=E5=9E=8B=E4=BE=9D=E8=B5=96?= =?UTF-8?q?=E6=96=B0=E5=A2=9E=E9=A1=B9=E7=9B=AE=E4=BE=9D=E8=B5=96=E5=AD=97?= =?UTF-8?q?=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../platform/domain/ModelDependency.java | 21 ++++++++++++++++++- .../ModelDependencyDaoMapper.xml | 18 ++++++++++++++-- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/ModelDependency.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/ModelDependency.java index 6cb082cb..45c28ca5 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/ModelDependency.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/ModelDependency.java @@ -1,5 +1,8 @@ package com.ruoyi.platform.domain; +import com.fasterxml.jackson.databind.PropertyNamingStrategy; +import com.fasterxml.jackson.databind.annotation.JsonNaming; + import java.util.Date; import java.io.Serializable; @@ -9,6 +12,7 @@ import java.io.Serializable; * @author Xidaray * @since 2024-05-29 13:51:23 */ +@JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy.class) public class ModelDependency implements Serializable { private static final long serialVersionUID = -86753423714028539L; /** @@ -39,7 +43,14 @@ public class ModelDependency implements Serializable { * 测试数据集 */ private String testDataset; -/** + /** + * 依赖项目 + */ + private String projectDependency; + + + + /** * 创建者 */ private String createBy; @@ -117,6 +128,14 @@ public class ModelDependency implements Serializable { this.testDataset = testDataset; } + public String getProjectDependency() { + return projectDependency; + } + + public void setProjectDependency(String projectDependency) { + this.projectDependency = projectDependency; + } + public String getCreateBy() { return createBy; } diff --git a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ModelDependencyDaoMapper.xml b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ModelDependencyDaoMapper.xml index c34b8a0d..b3b3b53b 100644 --- a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ModelDependencyDaoMapper.xml +++ b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ModelDependencyDaoMapper.xml @@ -10,6 +10,7 @@ + @@ -20,7 +21,7 @@ @@ -28,7 +29,7 @@ select - id,current_model_id,parent_model_id,ref_item,train_task,train_dataset,test_dataset,project_dependency,create_by,create_time,update_by,update_time,state + id,current_model_id,parent_models,ref_item,train_task,train_dataset,test_dataset,project_dependency,create_by,create_time,update_by,update_time,state from model_dependency where id = #{id} @@ -29,7 +29,7 @@ - + - insert into model_dependency ( - current_model_id, - parent_models, - ref_item, - train_task, - train_dataset, - test_dataset, - project_dependency, - create_by, - create_time, - update_by, - update_time, - state - ) - values ( - #{modelDependency.currentModelId}, - #{modelDependency.parentModels}, - #{modelDependency.refItem}, - #{modelDependency.trainTask}, - #{modelDependency.trainDataset}, - #{modelDependency.testDataset}, - #{modelDependency.projectDependency}, - #{modelDependency.createBy}, - #{modelDependency.createTime}, - #{modelDependency.updateBy}, - #{modelDependency.updateTime}, - #{modelDependency.state} - ) + insert into model_dependency (current_model_id, parent_models, ref_item, train_task, train_dataset, test_dataset, project_dependency, create_by, create_time, update_by, update_time, state) + values (#{modelDependency.currentModelId}, #{modelDependency.parentModels}, #{modelDependency.refItem}, #{modelDependency.trainTask}, #{modelDependency.trainDataset}, #{modelDependency.testDataset}, #{modelDependency.projectDependency}, #{modelDependency.createBy}, #{modelDependency.createTime}, #{modelDependency.updateBy}, #{modelDependency.updateTime}, #{modelDependency.state}) - - insert into model_dependency ( current_model_id, @@ -242,7 +214,7 @@ current_model_id = values(current_model_id)parent_models = values(parent_models) state = #{modelDependency.state}, - where id = #{id} + where id = #{modelDependency.id} From 12effaf3549170e276e9866fa015e1819762482d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=A5=BF=E5=A4=A7=E9=94=90?= <1070211640@qq.com> Date: Fri, 31 May 2024 09:48:02 +0800 Subject: [PATCH 16/31] =?UTF-8?q?=E6=A8=A1=E5=9E=8B=E4=BE=9D=E8=B5=96?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E5=AD=97=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../platform/domain/ModelDependency.java | 30 +++++++++++++++- .../ModelDependencyDaoMapper.xml | 34 ++++++++++++++++--- 2 files changed, 58 insertions(+), 6 deletions(-) diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/ModelDependency.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/ModelDependency.java index 94798fcb..8c75e2a7 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/ModelDependency.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/ModelDependency.java @@ -23,7 +23,14 @@ public class ModelDependency implements Serializable { * 当前模型id */ private Integer currentModelId; -/** + /** + * 实验实例id + */ + private Integer expInsId; + + + + /** * 父模型 */ private String parentModels; @@ -48,6 +55,10 @@ public class ModelDependency implements Serializable { */ private String projectDependency; + /** + * 版本 + */ + private String version; /** @@ -88,6 +99,14 @@ public class ModelDependency implements Serializable { this.currentModelId = currentModelId; } + public Integer getExpInsId() { + return expInsId; + } + + public void setExpInsId(Integer expInsId) { + this.expInsId = expInsId; + } + public String getParentModels() { return parentModels; } @@ -136,6 +155,15 @@ public class ModelDependency implements Serializable { this.projectDependency = projectDependency; } + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public String getCreateBy() { return createBy; } diff --git a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ModelDependencyDaoMapper.xml b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ModelDependencyDaoMapper.xml index f31a75a0..62582e93 100644 --- a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ModelDependencyDaoMapper.xml +++ b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ModelDependencyDaoMapper.xml @@ -5,12 +5,14 @@ + + @@ -21,7 +23,7 @@ @@ -29,7 +31,7 @@ + + + + insert into model_dependency (current_model_id,expInsId, parent_models, ref_item, train_task, train_dataset, test_dataset, project_dependency,version, create_by, create_time, update_by, update_time, state) From ada8b8f4476207c0e454a09af37f33ce9fdae0ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=A5=BF=E5=A4=A7=E9=94=90?= <1070211640@qq.com> Date: Fri, 31 May 2024 17:18:02 +0800 Subject: [PATCH 18/31] =?UTF-8?q?fix:=E4=BF=AE=E5=A4=8Dbug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/ruoyi/platform/domain/ModelDependency.java | 2 -- .../managementPlatform/ModelDependencyDaoMapper.xml | 10 +++++----- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/ModelDependency.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/ModelDependency.java index 8c75e2a7..dd2913bc 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/ModelDependency.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/ModelDependency.java @@ -28,8 +28,6 @@ public class ModelDependency implements Serializable { */ private Integer expInsId; - - /** * 父模型 */ diff --git a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ModelDependencyDaoMapper.xml b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ModelDependencyDaoMapper.xml index 9dc78d81..8a095b6f 100644 --- a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ModelDependencyDaoMapper.xml +++ b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ModelDependencyDaoMapper.xml @@ -23,7 +23,7 @@ @@ -31,7 +31,7 @@ select - id,current_model_id,expInsId,parent_models,ref_item,train_task,train_dataset,test_dataset,project_dependency,version,create_by,create_time,update_by,update_time,state + id,current_model_id,exp_ins_id,parent_models,ref_item,train_task,train_dataset,test_dataset,project_dependency,version,create_by,create_time,update_by,update_time,state from model_dependency @@ -193,14 +193,14 @@ - insert into model_dependency (current_model_id,expInsId, parent_models, ref_item, train_task, train_dataset, test_dataset, project_dependency,version, create_by, create_time, update_by, update_time, state) + insert into model_dependency (current_model_id, exp_ins_id, parent_models, ref_item, train_task, train_dataset, test_dataset, project_dependency,version, create_by, create_time, update_by, update_time, state) values (#{modelDependency.currentModelId}, #{modelDependency.expInsId},#{modelDependency.parentModels}, #{modelDependency.refItem}, #{modelDependency.trainTask}, #{modelDependency.trainDataset}, #{modelDependency.testDataset}, #{modelDependency.projectDependency},#{modelDependency.version}, #{modelDependency.createBy}, #{modelDependency.createTime}, #{modelDependency.updateBy}, #{modelDependency.updateTime}, #{modelDependency.state}) insert into model_dependency ( current_model_id, - expInsId, + exp_ins_id, parent_models, ref_item, train_task, From e71b3f8e848734586905b680ddd74afb19bb6bad Mon Sep 17 00:00:00 2001 From: fanshuai <1141904845@qq.com> Date: Fri, 31 May 2024 17:58:54 +0800 Subject: [PATCH 19/31] =?UTF-8?q?=E5=88=B0=E5=A4=84=E4=BE=9D=E8=B5=96?= =?UTF-8?q?=E5=85=B3=E7=B3=BB=E5=9B=BE=E8=B0=B1~?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../model/ModelDependencyController.java | 7 ++ .../dependencydomain/ProjectDepency.java | 3 + .../dependencydomain/TrainTaskDepency.java | 3 + .../platform/mapper/ModelDependencyDao.java | 2 + .../ExperimentInstanceStatusTask.java | 59 +++++++++++--- .../service/ModelDependencyService.java | 11 ++- .../service/impl/ExperimentServiceImpl.java | 39 ++++++--- .../impl/ModelDependencyServiceImpl.java | 80 +++++++++++++++++++ .../platform/vo/ModelDependcyTreeVo.java | 59 ++++++++++++++ .../ModelDependencyDaoMapper.xml | 10 ++- 10 files changed, 248 insertions(+), 25 deletions(-) create mode 100644 ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelDependcyTreeVo.java diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/model/ModelDependencyController.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/model/ModelDependencyController.java index e2382a69..3f8b5cd5 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/model/ModelDependencyController.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/model/ModelDependencyController.java @@ -5,6 +5,7 @@ import com.ruoyi.common.core.web.domain.GenericsAjaxResult; import com.ruoyi.platform.domain.ModelDependency; import com.ruoyi.platform.domain.ModelsVersion; import com.ruoyi.platform.service.ModelDependencyService; +import com.ruoyi.platform.vo.ModelDependcyTreeVo; import io.swagger.annotations.ApiOperation; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageRequest; @@ -102,5 +103,11 @@ public class ModelDependencyController extends BaseController { return genericsSuccess(this.modelDependencyService.removeById(id)); } + + @PostMapping("/queryModelAtlas") + @ApiOperation("根据对象查询") + public GenericsAjaxResult queryModelAtlas(@RequestBody ModelDependency modelDependency) throws Exception { + return genericsSuccess(this.modelDependencyService.getModelDependencyTree(modelDependency)); + } } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/dependencydomain/ProjectDepency.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/dependencydomain/ProjectDepency.java index 0fdce081..98502afb 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/dependencydomain/ProjectDepency.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/dependencydomain/ProjectDepency.java @@ -1,9 +1,12 @@ package com.ruoyi.platform.domain.dependencydomain; +import com.fasterxml.jackson.databind.PropertyNamingStrategy; +import com.fasterxml.jackson.databind.annotation.JsonNaming; import lombok.Data; import java.io.Serializable; @Data +@JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy.class) public class ProjectDepency implements Serializable { private String url; private String name; diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/dependencydomain/TrainTaskDepency.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/dependencydomain/TrainTaskDepency.java index 536894d9..857306c8 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/dependencydomain/TrainTaskDepency.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/dependencydomain/TrainTaskDepency.java @@ -1,9 +1,12 @@ package com.ruoyi.platform.domain.dependencydomain; +import com.fasterxml.jackson.databind.PropertyNamingStrategy; +import com.fasterxml.jackson.databind.annotation.JsonNaming; import lombok.Data; import java.io.Serializable; @Data +@JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy.class) public class TrainTaskDepency implements Serializable { //训练任务名 private String name; diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/ModelDependencyDao.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/ModelDependencyDao.java index 725dc678..35c1bfc5 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/ModelDependencyDao.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/ModelDependencyDao.java @@ -82,5 +82,7 @@ public interface ModelDependencyDao { int deleteById(Integer id); List queryByModelDependency(@Param("modelDependency") ModelDependency modelDependency); + + List queryChildrenByVersionId(@Param("model_id")String modelId, @Param("version")String version); } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/scheduling/ExperimentInstanceStatusTask.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/scheduling/ExperimentInstanceStatusTask.java index b7ea8648..1764b262 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/scheduling/ExperimentInstanceStatusTask.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/scheduling/ExperimentInstanceStatusTask.java @@ -1,24 +1,26 @@ package com.ruoyi.platform.scheduling; -import com.ruoyi.common.security.utils.SecurityUtils; import com.ruoyi.platform.domain.Experiment; import com.ruoyi.platform.domain.ExperimentIns; +import com.ruoyi.platform.domain.ModelDependency; import com.ruoyi.platform.mapper.ExperimentDao; import com.ruoyi.platform.mapper.ExperimentInsDao; +import com.ruoyi.platform.mapper.ModelDependencyDao; import com.ruoyi.platform.service.ExperimentInsService; -import com.ruoyi.platform.service.ExperimentService; -import com.ruoyi.platform.utils.JsonUtils; -import com.ruoyi.system.api.model.LoginUser; -import io.swagger.models.auth.In; +import com.ruoyi.platform.service.ModelDependencyService; +import com.ruoyi.platform.utils.JacksonUtil; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Page; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; import javax.annotation.Resource; import java.io.IOException; -import java.util.*; -import java.util.stream.Collectors; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Map; @Component() public class ExperimentInstanceStatusTask { @@ -28,7 +30,8 @@ public class ExperimentInstanceStatusTask { private ExperimentDao experimentDao; @Resource private ExperimentInsDao experimentInsDao; - + @Resource + private ModelDependencyDao modelDependencyDao; private List experimentIds = new ArrayList<>(); @Scheduled(cron = "0/30 * * * * ?") // 每30S执行一次 @@ -55,12 +58,50 @@ public class ExperimentInstanceStatusTask { updateList.add(experimentIns); } - experimentInsDao.update(experimentIns); +// experimentInsDao.update(experimentIns); } } if (updateList.size() > 0){ experimentInsDao.insertOrUpdateBatch(updateList); + + //遍历模型关系表，找到 + List modelDependencyList = new ArrayList(); + for (ExperimentIns experimentIns : updateList){ + ModelDependency modelDependencyquery = new ModelDependency(); + modelDependencyquery.setExpInsId(experimentIns.getId()); + modelDependencyquery.setState(2); + + List modelDependencyListquery = modelDependencyDao.queryByModelDependency(modelDependencyquery); + if (modelDependencyListquery==null||modelDependencyListquery.size()==0){ + continue; + } + ModelDependency modelDependency = modelDependencyListquery.get(0); + //查看状态， + if (StringUtils.equals("Succeeded",experimentIns.getStatus())){ + modelDependency.setState(1); + modelDependencyList.add(modelDependency); + }else{ + //取出节点状态 + String trainTask = modelDependency.getTrainTask(); + Map trainMap = JacksonUtil.parseJSONStr2Map(trainTask); + String task_id = (String) trainMap.get("task_id"); + if (StringUtils.isEmpty(task_id)){ + continue; + } + String nodesStatus = experimentIns.getNodesStatus(); + Map nodeMaps = JacksonUtil.parseJSONStr2Map(nodesStatus); + Map nodeMap = JacksonUtil.parseJSONStr2Map((String) nodeMaps.get(task_id)); + if (nodeMap==null){ + continue; + } + if (StringUtils.equals("Succeeded",(String)nodeMap.get("phase"))){ + modelDependency.setState(1); + modelDependencyList.add(modelDependency); + } + } + } + modelDependencyDao.insertOrUpdateBatch(modelDependencyList); } } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ModelDependencyService.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ModelDependencyService.java index 6f629ac8..3883a7b7 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ModelDependencyService.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ModelDependencyService.java @@ -1,6 +1,7 @@ package com.ruoyi.platform.service; import com.ruoyi.platform.domain.ModelDependency; +import com.ruoyi.platform.vo.ModelDependcyTreeVo; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageRequest; @@ -32,20 +33,20 @@ public interface ModelDependencyService { Page queryByPage(ModelDependency modelDependency, PageRequest pageRequest); /** - * 新增数据 + * 修改数据 * * @param modelDependency 实例对象 * @return 实例对象 */ - ModelDependency insert(ModelDependency modelDependency); + ModelDependency update(ModelDependency modelDependency); /** - * 修改数据 + * 新增数据 * * @param modelDependency 实例对象 * @return 实例对象 */ - ModelDependency update(ModelDependency modelDependency); + ModelDependency insert(ModelDependency modelDependency); /** * 通过主键删除数据 @@ -58,4 +59,6 @@ public interface ModelDependencyService { String removeById(Integer id); List queryByModelDependency(ModelDependency modelDependency); + + ModelDependcyTreeVo getModelDependencyTree(ModelDependency modelDependency) throws Exception; } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentServiceImpl.java index c0c02dd5..89b2da4a 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentServiceImpl.java @@ -269,7 +269,7 @@ public class ExperimentServiceImpl implements ExperimentService { ExperimentIns insert = experimentInsService.insert(experimentIns); //插入到模型依赖关系表 - insertModelDependendcy(dependendcy,insert.getId(),experiment.getName()); + insertModelDependendcy(dependendcy,insert.getId(),experiment.getName(),params); }catch (Exception e){ throw new RuntimeException(e); @@ -279,7 +279,7 @@ public class ExperimentServiceImpl implements ExperimentService { return experiment; } - private void insertModelDependendcy(Map dependendcy,Integer experimentInsId,String expermentName)throws Exception { + private void insertModelDependendcy(Map dependendcy,Integer experimentInsId,String expermentName,List> params)throws Exception { Iterator> dependendcyIterator = dependendcy.entrySet().iterator(); while (dependendcyIterator.hasNext()) { ModelDependency modelDependency = new ModelDependency(); @@ -328,29 +328,46 @@ public class ExperimentServiceImpl implements ExperimentService { trainTaskDepency.setName(expermentName); //训练任务 modelDependency.setTrainTask(JsonUtils.objectToJson(trainTaskDepency)); - + modelDependency.setExpInsId(experimentInsId); //处理test数据 List> testDatasetsList = (List>) modelDel.get("test"); + List> resultTestDatasets = new ArrayList>(); for(int i=0;i datasets = testDatasetsList.get(i); - Dataset dataset = datasetService.queryById((Integer) datasets.get("dataset_id")); - if (dataset == null){ - throw new Exception("源数据集不存在"); + List> realDataSetList = (List>) datasets.get("datasets"); + for(int j=0;j realDataSet = realDataSetList.get(j); + Dataset dataset = datasetService.queryById((Integer) realDataSet.get("dataset_id")); + if (dataset == null){ + throw new Exception("源数据集不存在"); + } + realDataSet.put("dataset_name", dataset.getName()); + resultTestDatasets.add(realDataSet); } - datasets.put("dataset_name", dataset.getName()); + } //测试数据集 - modelDependency.setTestDataset(JsonUtils.objectToJson(testDatasetsList)); + modelDependency.setTestDataset(JsonUtils.objectToJson(resultTestDatasets)); //检查是否存在target，如果存在说明在流水线用了节点导入，如果没有说明没有导入，等待手动push List> modelTargetList = (List>) modelDel.get("target"); if (modelTargetList==null||modelTargetList.size()==0){ - modelDependency.setState(0); - }else { modelDependency.setState(1); + modelDependencyService.insert(modelDependency); + }else { + modelDependency.setState(2); for(int i=0;i model = modelTargetList.get(i); - modelDependency.setVersion((String) model.get("model_version")); + String version = null; + //可能是参数，必须从实验参数读取 + if (params != null) { + for (Map param : params) { + if (param.containsKey("param_name") && StringUtils.equals("model_version",(String)param.get("param_name"))) { + version = param.get("param_value").toString(); + } + } + } + modelDependency.setVersion(StringUtils.isEmpty(version)?(String)model.get("model_version"):version); modelDependency.setCurrentModelId((Integer) model.get("model_id")); //因为可能有多成果模型，多次插入 modelDependencyService.insert(modelDependency); diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelDependencyServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelDependencyServiceImpl.java index c4cb93b9..d3f62d35 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelDependencyServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelDependencyServiceImpl.java @@ -5,6 +5,8 @@ import com.ruoyi.platform.domain.ModelDependency; import com.ruoyi.platform.domain.ModelsVersion; import com.ruoyi.platform.mapper.ModelDependencyDao; import com.ruoyi.platform.service.ModelDependencyService; +import com.ruoyi.platform.utils.JacksonUtil; +import com.ruoyi.platform.vo.ModelDependcyTreeVo; import com.ruoyi.system.api.model.LoginUser; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Service; @@ -13,8 +15,10 @@ import org.springframework.data.domain.PageImpl; import org.springframework.data.domain.PageRequest; import javax.annotation.Resource; +import java.util.ArrayList; import java.util.Date; import java.util.List; +import java.util.Map; /** * (ModelDependency)表服务实现类 @@ -27,6 +31,7 @@ public class ModelDependencyServiceImpl implements ModelDependencyService { @Resource private ModelDependencyDao modelDependencyDao; + /** * 通过ID查询单条数据 * @@ -62,6 +67,81 @@ public class ModelDependencyServiceImpl implements ModelDependencyService { return this.modelDependencyDao.queryByModelDependency(modelDependency); } + @Override + public ModelDependcyTreeVo getModelDependencyTree(ModelDependency modelDependencyQuery) throws Exception { + //查询当前模型 + List modelDependencyList = modelDependencyDao.queryByModelDependency(modelDependencyQuery); + if (modelDependencyList==null||modelDependencyList.size()==0){ + throw new Exception("当前模型依赖关系不存在"); + } + ModelDependency modelDependency = modelDependencyList.get(0); + ModelDependcyTreeVo modelDependcyTreeVo = ModelDependencyConvertToTree(modelDependency); + //递归父模型 + processParentModel(modelDependcyTreeVo); + //递归子模型 + processChildrenModel(modelDependcyTreeVo); + return modelDependcyTreeVo; + } + + /** + * 递归父模型 + * @param modelDependcyTreeVo + */ + private void processParentModel(ModelDependcyTreeVo modelDependcyTreeVo){ + if (modelDependcyTreeVo.getParentModelsMap() != null) { + List> parentMaps = modelDependcyTreeVo.getParentModelsMap(); + List ps = new ArrayList(); + for (Map parent:parentMaps) { + Integer model_id = (Integer) parent.get("model_id"); + String version = (String) parent.get("model_version"); + ModelDependency modelDependencyQuery = new ModelDependency(); + modelDependencyQuery.setVersion(version); + modelDependencyQuery.setCurrentModelId(model_id); + List modelDependencyList = modelDependencyDao.queryByModelDependency(modelDependencyQuery); + if (modelDependencyList!=null&&modelDependencyList.size()>=0){ + for (ModelDependency modelDependency:modelDependencyList){ + ModelDependcyTreeVo modelDependcyTreeVoIn = ModelDependencyConvertToTree(modelDependency); + processParentModel(modelDependcyTreeVoIn); + ps.add(modelDependcyTreeVoIn); + } + } + + } + modelDependcyTreeVo.setParentModels(ps); + } + } + + + private void processChildrenModel(ModelDependcyTreeVo modelDependcyTreeVo){ + String version = modelDependcyTreeVo.getVersion(); + Integer modelId = modelDependcyTreeVo.getCurrentModelId(); + List cs = new ArrayList(); + //查儿子们 + + List modelDependencyList = modelDependencyDao.queryChildrenByVersionId("\"model_id\":"+modelId, "\"model_version\":\""+version+"\""); + if (modelDependencyList!=null&&modelDependencyList.size()>=0){ + for (ModelDependency modelDependency:modelDependencyList){ + ModelDependcyTreeVo modelDependcyTreeVoIn = ModelDependencyConvertToTree(modelDependency); + processChildrenModel(modelDependcyTreeVoIn); + cs.add(modelDependcyTreeVoIn); + } + } + modelDependcyTreeVo.setChildrenModels(cs); + } + private ModelDependcyTreeVo ModelDependencyConvertToTree(ModelDependency modelDependency){ + ModelDependcyTreeVo modelDependcyTreeVo = new ModelDependcyTreeVo(); + modelDependcyTreeVo.setCurrentModelId(modelDependency.getCurrentModelId()); + modelDependcyTreeVo.setExpInsId(modelDependency.getExpInsId()); + modelDependcyTreeVo.setVersion(modelDependency.getVersion()); + modelDependcyTreeVo.setRefItem(modelDependency.getRefItem()); + modelDependcyTreeVo.setTrainTask(JacksonUtil.parseJSONStr2Map(modelDependency.getTrainTask())); + modelDependcyTreeVo.setTrainDataset(JacksonUtil.parseJSONStr2MapList(modelDependency.getTrainDataset())); + modelDependcyTreeVo.setTestDataset(JacksonUtil.parseJSONStr2MapList(modelDependency.getTestDataset())); + modelDependcyTreeVo.setProjectDependency(JacksonUtil.parseJSONStr2Map(modelDependency.getProjectDependency())); + modelDependcyTreeVo.setParentModelsMap(JacksonUtil.parseJSONStr2MapList(modelDependency.getParentModels())); + return modelDependcyTreeVo; + } + /** * 新增数据 * diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelDependcyTreeVo.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelDependcyTreeVo.java new file mode 100644 index 00000000..65a3cc24 --- /dev/null +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelDependcyTreeVo.java @@ -0,0 +1,59 @@ +package com.ruoyi.platform.vo; + +import com.fasterxml.jackson.databind.PropertyNamingStrategy; +import com.fasterxml.jackson.databind.annotation.JsonNaming; +import lombok.Data; + +import java.util.List; +import java.util.Map; + +@Data +@JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy.class) +public class ModelDependcyTreeVo { + /** + * 当前模型id + */ + private Integer currentModelId; + /** + * 实验实例id + */ + private Integer expInsId; + + /** + * 版本 + */ + private String version; + + + /** + * 引用项目 + */ + private String refItem; + /** + * 训练任务 + */ + private Map trainTask; + /** + * 训练数据集 + */ + private List> trainDataset; + /** + * 测试数据集 + */ + private List> testDataset; + /** + * 依赖项目 + */ + private Map projectDependency; + + private List> parentModelsMap; + /** + * 父模型 + */ + private List parentModels; + + /** + * 子模型 + */ + private List childrenModels; +} diff --git a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ModelDependencyDaoMapper.xml b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ModelDependencyDaoMapper.xml index 9dc78d81..e991008e 100644 --- a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ModelDependencyDaoMapper.xml +++ b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ModelDependencyDaoMapper.xml @@ -83,10 +83,18 @@ limit #{pageable.offset}, #{pageable.pageSize} + + select + id,name,status,computing_resource,standard,env_variable,image,dataset,model,alt_field1,alt_field2,create_by,create_time,update_by,update_time,state + from dev_environment + where id = #{id} + + + + + + + + + + + + insert into dev_environment(name,status,computing_resource,standard,env_variable,image,dataset,model,alt_field1,alt_field2,create_by,create_time,update_by,update_time,state) + values (#{name},#{status},#{computingResource},#{standard},#{envVariable},#{image},#{dataset},#{model},#{altField1},#{altField2},#{createBy},#{createTime},#{updateBy},#{updateTime},#{state}) + + + + insert into dev_environment(name,status,computing_resource,standard,env_variable,image,dataset,model,alt_field1,alt_field2,create_by,create_time,update_by,update_time,state ) + values + + (#{entity.name},#{entity.status},#{entity.computingResource},#{entity.standard},#{entity.envVariable},#{entity.image},#{entity.dataset},#{entity.model},#{entity.altField1},#{entity.altField2},#{entity.createBy},#{entity.createTime},#{entity.updateBy},#{entity.updateTime},#{entity.state}) + + + + + insert into dev_environment(name,status,computing_resource,standard,env_variable,image,dataset,model,alt_field1,alt_field2,create_by,create_time,update_by,update_time,state) + values + + (#{entity.name}#{entity.status}#{entity.computingResource}#{entity.standard}#{entity.envVariable}#{entity.image}#{entity.dataset}#{entity.model}#{entity.altField1}#{entity.altField2}#{entity.createBy}#{entity.createTime}#{entity.updateBy}#{entity.updateTime}#{entity.state}) + + on duplicate key update +name = values(name)status = values(status)computing_resource = values(computing_resource)standard = values(standard)env_variable = values(env_variable)image = values(image)dataset = values(dataset)model = values(model)alt_field1 = values(alt_field1)alt_field2 = values(alt_field2)create_by = values(create_by)create_time = values(create_time)update_by = values(update_by)update_time = values(update_time)state = values(state) + + + + + update dev_environment + + + name = #{devEnvironment.name}, + + + status = #{devEnvironment.status}, + + + computing_resource = #{devEnvironment.computingResource}, + + + standard = #{devEnvironment.standard}, + + + env_variable = #{devEnvironment.envVariable}, + + + image = #{devEnvironment.image}, + + + dataset = #{devEnvironment.dataset}, + + + model = #{devEnvironment.model}, + + + alt_field1 = #{devEnvironment.altField1}, + + + alt_field2 = #{devEnvironment.altField2}, + + + create_by = #{devEnvironment.createBy}, + + + create_time = #{devEnvironment.createTime}, + + + update_by = #{devEnvironment.updateBy}, + + + update_time = #{devEnvironment.updateTime}, + + + state = #{devEnvironment.state}, + + + where id = #{devEnvironment.id} + + + + + + delete from dev_environment where id = #{id} + + + + From f65f8849a4c18362f56d5353e8952000188d03a8 Mon Sep 17 00:00:00 2001 From: fanshuai <1141904845@qq.com> Date: Tue, 4 Jun 2024 08:50:37 +0800 Subject: [PATCH 25/31] =?UTF-8?q?=E6=96=B0=E5=A2=9Ename?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/impl/ModelDependencyServiceImpl.java | 11 +++++++++++ .../service/impl/ModelsVersionServiceImpl.java | 2 ++ .../com/ruoyi/platform/vo/ModelDependcyTreeVo.java | 2 ++ 3 files changed, 15 insertions(+) diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelDependencyServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelDependencyServiceImpl.java index 1ba0995e..bb486072 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelDependencyServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelDependencyServiceImpl.java @@ -2,9 +2,11 @@ package com.ruoyi.platform.service.impl; import com.ruoyi.common.security.utils.SecurityUtils; import com.ruoyi.platform.domain.ModelDependency; +import com.ruoyi.platform.domain.Models; import com.ruoyi.platform.domain.ModelsVersion; import com.ruoyi.platform.mapper.ModelDependencyDao; import com.ruoyi.platform.service.ModelDependencyService; +import com.ruoyi.platform.service.ModelsService; import com.ruoyi.platform.utils.JacksonUtil; import com.ruoyi.platform.vo.ModelDependcyTreeVo; import com.ruoyi.system.api.model.LoginUser; @@ -19,6 +21,7 @@ import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; /** * (ModelDependency)表服务实现类 @@ -31,6 +34,8 @@ public class ModelDependencyServiceImpl implements ModelDependencyService { @Resource private ModelDependencyDao modelDependencyDao; + @Resource + private ModelsService modelsService; /** * 通过ID查询单条数据 @@ -98,10 +103,12 @@ public class ModelDependencyServiceImpl implements ModelDependencyService { ModelDependency modelDependencyQuery = new ModelDependency(); modelDependencyQuery.setVersion(version); modelDependencyQuery.setCurrentModelId(model_id); + List modelDependencyList = modelDependencyDao.queryByModelDependency(modelDependencyQuery); if (modelDependencyList!=null&&modelDependencyList.size()>=0){ for (ModelDependency modelDependency:modelDependencyList){ ModelDependcyTreeVo modelDependcyTreeVoIn = ModelDependencyConvertToTree(modelDependency); + modelDependcyTreeVoIn.setCurrentModelName((String) parent.get("model_name")); processParentModel(modelDependcyTreeVoIn); ps.add(modelDependcyTreeVoIn); } @@ -140,6 +147,10 @@ public class ModelDependencyServiceImpl implements ModelDependencyService { modelDependcyTreeVo.setTestDataset(JacksonUtil.parseJSONStr2MapList(modelDependency.getTestDataset())); modelDependcyTreeVo.setProjectDependency(JacksonUtil.parseJSONStr2Map(modelDependency.getProjectDependency())); modelDependcyTreeVo.setParentModelsMap(JacksonUtil.parseJSONStr2MapList(modelDependency.getParentModels())); + Models models = modelsService.queryById(modelDependency.getCurrentModelId()); + if (models!=null) { + modelDependcyTreeVo.setCurrentModelName(models.getName()); + } return modelDependcyTreeVo; } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsVersionServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsVersionServiceImpl.java index 07ce624f..cfa793c8 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsVersionServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsVersionServiceImpl.java @@ -13,6 +13,7 @@ import com.ruoyi.platform.service.ModelsVersionService; import com.ruoyi.system.api.model.LoginUser; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageImpl; import org.springframework.data.domain.PageRequest; @@ -39,6 +40,7 @@ public class ModelsVersionServiceImpl implements ModelsVersionService { @Resource private ModelsDao modelsDao; @Resource + @Lazy private ModelDependencyService modelDependencyService; // 固定存储桶名 @Value("${minio.dataReleaseBucketName}") diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelDependcyTreeVo.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelDependcyTreeVo.java index 65a3cc24..5814ffca 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelDependcyTreeVo.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelDependcyTreeVo.java @@ -56,4 +56,6 @@ public class ModelDependcyTreeVo { * 子模型 */ private List childrenModels; + + private String currentModelName; } From 79101d4587b4dd141ecad1240dc544abc5a30ecb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=A5=BF=E5=A4=A7=E9=94=90?= <1070211640@qq.com> Date: Tue, 4 Jun 2024 14:11:55 +0800 Subject: [PATCH 26/31] =?UTF-8?q?feat:=E5=BC=80=E5=8F=91=E7=8E=AF=E5=A2=83?= =?UTF-8?q?=E5=90=8E=E7=AB=AF=E6=A1=86=E6=9E=B6=E6=90=AD=E5=BB=BA=EF=BC=8C?= =?UTF-8?q?=E5=A2=9E=E5=88=A0=E6=94=B9=E6=9F=A5=E6=8E=A5=E5=8F=A3=E5=AE=9E?= =?UTF-8?q?=E7=8E=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../DevEnvironmentController.java | 8 ++++---- .../platform/mapper/DevEnvironmentDao.java | 1 + .../service/impl/ExperimentServiceImpl.java | 12 ++++++------ .../impl/ModelDependencyServiceImpl.java | 12 ++++++------ .../DevEnvironmentDaoMapper.xml | 17 ++++++++++++++++- 5 files changed, 33 insertions(+), 17 deletions(-) diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/devEnvironment/DevEnvironmentController.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/devEnvironment/DevEnvironmentController.java index a845eae2..3935d50f 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/devEnvironment/DevEnvironmentController.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/devEnvironment/DevEnvironmentController.java @@ -60,7 +60,7 @@ public class DevEnvironmentController extends BaseController { * @return 新增结果 */ @PostMapping - public ResponseEntity add(DevEnvironment devEnvironment) { + public ResponseEntity add(@RequestBody DevEnvironment devEnvironment) { return ResponseEntity.ok(this.devEnvironmentService.insert(devEnvironment)); } @@ -71,7 +71,7 @@ public class DevEnvironmentController extends BaseController { * @return 编辑结果 */ @PutMapping - public ResponseEntity edit(DevEnvironment devEnvironment) { + public ResponseEntity edit(@RequestBody DevEnvironment devEnvironment) { return ResponseEntity.ok(this.devEnvironmentService.update(devEnvironment)); } @@ -82,8 +82,8 @@ public class DevEnvironmentController extends BaseController { * @return 删除是否成功 */ @DeleteMapping("{id}") - public ResponseEntity deleteById(@PathVariable("id") Integer id) { - return ResponseEntity.ok(this.devEnvironmentService.deleteById(id)); + public ResponseEntity deleteById(@PathVariable("id") Integer id) { + return ResponseEntity.ok(this.devEnvironmentService.removeById(id)); } } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/DevEnvironmentDao.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/DevEnvironmentDao.java index 0cb652ff..e4f4bbe7 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/DevEnvironmentDao.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/DevEnvironmentDao.java @@ -58,6 +58,7 @@ public interface DevEnvironmentDao { * 批量新增或按主键更新数据（MyBatis原生foreach方法） * * @param entities List 实例对象列表 + * * @return 影响行数 * @throws org.springframework.jdbc.BadSqlGrammarException 入参是空List的时候会抛SQL语句错误的异常，请自行校验入参 */ diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentServiceImpl.java index 89b2da4a..b53d488b 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentServiceImpl.java @@ -269,7 +269,7 @@ public class ExperimentServiceImpl implements ExperimentService { ExperimentIns insert = experimentInsService.insert(experimentIns); //插入到模型依赖关系表 - insertModelDependendcy(dependendcy,insert.getId(),experiment.getName(),params); + insertModelDependency(dependendcy,insert.getId(),experiment.getName(),params); }catch (Exception e){ throw new RuntimeException(e); @@ -279,7 +279,7 @@ public class ExperimentServiceImpl implements ExperimentService { return experiment; } - private void insertModelDependendcy(Map dependendcy,Integer experimentInsId,String expermentName,List> params)throws Exception { + private void insertModelDependency(Map dependendcy, Integer experimentInsId, String experimentName, List> params) throws Exception { Iterator> dependendcyIterator = dependendcy.entrySet().iterator(); while (dependendcyIterator.hasNext()) { ModelDependency modelDependency = new ModelDependency(); @@ -325,7 +325,7 @@ public class ExperimentServiceImpl implements ExperimentService { TrainTaskDepency trainTaskDepency = new TrainTaskDepency(); trainTaskDepency.setTaskId(key); trainTaskDepency.setInsId(experimentInsId); - trainTaskDepency.setName(expermentName); + trainTaskDepency.setName(experimentName); //训练任务 modelDependency.setTrainTask(JsonUtils.objectToJson(trainTaskDepency)); modelDependency.setExpInsId(experimentInsId); @@ -334,12 +334,12 @@ public class ExperimentServiceImpl implements ExperimentService { List> resultTestDatasets = new ArrayList>(); for(int i=0;i datasets = testDatasetsList.get(i); - List> realDataSetList = (List>) datasets.get("datasets"); + List> realDataSetList = (List>) datasets.get("datasets"); for(int j=0;j realDataSet = realDataSetList.get(j); Dataset dataset = datasetService.queryById((Integer) realDataSet.get("dataset_id")); if (dataset == null){ - throw new Exception("源数据集不存在"); + throw new Exception("源数据集不存在"); } realDataSet.put("dataset_name", dataset.getName()); resultTestDatasets.add(realDataSet); @@ -362,7 +362,7 @@ public class ExperimentServiceImpl implements ExperimentService { //可能是参数，必须从实验参数读取 if (params != null) { for (Map param : params) { - if (param.containsKey("param_name") && StringUtils.equals("model_version",(String)param.get("param_name"))) { + if (param.containsKey("param_name") && StringUtils.equals("model_version",(String) param.get("param_name"))) { version = param.get("param_value").toString(); } } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelDependencyServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelDependencyServiceImpl.java index fcc0d7f7..4d9747e3 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelDependencyServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelDependencyServiceImpl.java @@ -101,9 +101,9 @@ public class ModelDependencyServiceImpl implements ModelDependencyService { List modelDependencyList = modelDependencyDao.queryByModelDependency(modelDependencyQuery); if (modelDependencyList!=null&&modelDependencyList.size()>=0){ for (ModelDependency modelDependency:modelDependencyList){ - ModelDependcyTreeVo modelDependcyTreeVoIn = ModelDependencyConvertToTree(modelDependency); - processParentModel(modelDependcyTreeVoIn); - ps.add(modelDependcyTreeVoIn); + ModelDependcyTreeVo modelDependencyTreeVoIn = ModelDependencyConvertToTree(modelDependency); + processParentModel(modelDependencyTreeVoIn); + ps.add(modelDependencyTreeVoIn); } } @@ -122,9 +122,9 @@ public class ModelDependencyServiceImpl implements ModelDependencyService { List modelDependencyList = modelDependencyDao.queryChildrenByVersionId("\"model_id\":"+modelId, "\"model_version\":\""+version+"\""); if (modelDependencyList!=null&&modelDependencyList.size()>=0){ for (ModelDependency modelDependency:modelDependencyList){ - ModelDependcyTreeVo modelDependcyTreeVoIn = ModelDependencyConvertToTree(modelDependency); - processChildrenModel(modelDependcyTreeVoIn); - cs.add(modelDependcyTreeVoIn); + ModelDependcyTreeVo modelDependencyTreeVoIn = ModelDependencyConvertToTree(modelDependency); + processChildrenModel(modelDependencyTreeVoIn); + cs.add(modelDependencyTreeVoIn); } } modelDependcyTreeVo.setChildrenModels(cs); diff --git a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/DevEnvironmentDaoMapper.xml b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/DevEnvironmentDaoMapper.xml index 60d43637..432e9dba 100644 --- a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/DevEnvironmentDaoMapper.xml +++ b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/DevEnvironmentDaoMapper.xml @@ -147,7 +147,22 @@ insert into dev_environment(name,status,computing_resource,standard,env_variable,image,dataset,model,alt_field1,alt_field2,create_by,create_time,update_by,update_time,state) - values (#{name},#{status},#{computingResource},#{standard},#{envVariable},#{image},#{dataset},#{model},#{altField1},#{altField2},#{createBy},#{createTime},#{updateBy},#{updateTime},#{state}) + values (#{devEnvironment.name}, + #{devEnvironment.status}, + #{devEnvironment.computingResource}, + #{devEnvironment.standard}, + #{devEnvironment.envVariable}, + #{devEnvironment.image}, + #{devEnvironment.dataset}, + #{devEnvironment.model}, + #{devEnvironment.altField1}, + #{devEnvironment.altField2}, + #{devEnvironment.createBy}, + #{devEnvironment.createTime}, + #{devEnvironment.updateBy}, + #{devEnvironment.updateTime}, + #{devEnvironment.state} + ) From 8e03aab445252f3edea4339939731d4a3a4e9ca6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=A5=BF=E5=A4=A7=E9=94=90?= <1070211640@qq.com> Date: Tue, 4 Jun 2024 14:15:29 +0800 Subject: [PATCH 27/31] =?UTF-8?q?fix:=E6=A8=A1=E5=9E=8B=E4=BE=9D=E8=B5=96?= =?UTF-8?q?=E6=94=B9=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../platform/service/impl/ModelDependencyServiceImpl.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelDependencyServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelDependencyServiceImpl.java index b3a07000..776247c2 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelDependencyServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelDependencyServiceImpl.java @@ -106,10 +106,10 @@ public class ModelDependencyServiceImpl implements ModelDependencyService { List modelDependencyList = modelDependencyDao.queryByModelDependency(modelDependencyQuery); if (modelDependencyList!=null&&modelDependencyList.size()>=0){ for (ModelDependency modelDependency:modelDependencyList){ - ModelDependcyTreeVo modelDependcyTreeVoIn = ModelDependencyConvertToTree(modelDependency); - modelDependcyTreeVoIn.setCurrentModelName((String) parent.get("model_name")); - processParentModel(modelDependcyTreeVoIn); - ps.add(modelDependcyTreeVoIn); + ModelDependcyTreeVo modelDependencyTreeVoIn = ModelDependencyConvertToTree(modelDependency); + modelDependencyTreeVoIn.setCurrentModelName((String) parent.get("model_name")); + processParentModel(modelDependencyTreeVoIn); + ps.add(modelDependencyTreeVoIn); } } From ea6400bb4bd36b0319f135732b688b43f5eee89b Mon Sep 17 00:00:00 2001 From: fanshuai <1141904845@qq.com> Date: Wed, 5 Jun 2024 15:07:12 +0800 Subject: [PATCH 28/31] =?UTF-8?q?=E6=A8=A1=E5=9E=8B=E6=BC=94=E5=8C=96?= =?UTF-8?q?=E6=94=B9=E9=80=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../model/ModelDependencyController.java | 1 + .../platform/domain/ModelDependency.java | 27 +- .../platform/mapper/ModelDependencyDao.java | 8 +- .../ExperimentInstanceStatusTask.java | 16 +- .../service/impl/ExperimentServiceImpl.java | 246 ++++++++++---- .../impl/ModelDependencyServiceImpl.java | 1 - .../impl/ModelsVersionServiceImpl.java | 10 + .../com/ruoyi/platform/utils/JacksonUtil.java | 16 + .../ModelDependencyDaoMapper.xml | 306 ++++++++++-------- 9 files changed, 407 insertions(+), 224 deletions(-) diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/model/ModelDependencyController.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/model/ModelDependencyController.java index a5febcd4..9afd7399 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/model/ModelDependencyController.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/model/ModelDependencyController.java @@ -54,6 +54,7 @@ public class ModelDependencyController extends BaseController { @GetMapping("/queryModelDependency") @ApiOperation("根据对象查询") public GenericsAjaxResult> queryByModelDependency(@RequestBody ModelDependency modelDependency) { + modelDependency.setState(1); return genericsSuccess(this.modelDependencyService.queryByModelDependency(modelDependency)); } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/ModelDependency.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/ModelDependency.java index dd2913bc..1809d7e5 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/ModelDependency.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/ModelDependency.java @@ -44,6 +44,14 @@ public class ModelDependency implements Serializable { * 训练数据集 */ private String trainDataset; + /** + * 训练参数 + */ + private String trainParams; + /** + * 训练镜像 + */ + private String trainImage; /** * 测试数据集 */ @@ -80,7 +88,6 @@ public class ModelDependency implements Serializable { */ private Integer state; - public Integer getId() { return id; } @@ -137,6 +144,22 @@ public class ModelDependency implements Serializable { this.trainDataset = trainDataset; } + public String getTrainParams() { + return trainParams; + } + + public void setTrainParams(String trainParams) { + this.trainParams = trainParams; + } + + public String getTrainImage(String name) { + return trainImage; + } + + public void setTrainImage(String trainImage) { + this.trainImage = trainImage; + } + public String getTestDataset() { return testDataset; } @@ -161,7 +184,6 @@ public class ModelDependency implements Serializable { this.version = version; } - public String getCreateBy() { return createBy; } @@ -201,6 +223,5 @@ public class ModelDependency implements Serializable { public void setState(Integer state) { this.state = state; } - } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/ModelDependencyDao.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/ModelDependencyDao.java index 35c1bfc5..3a999886 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/ModelDependencyDao.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/ModelDependencyDao.java @@ -28,7 +28,7 @@ public interface ModelDependencyDao { * @param pageable 分页对象 * @return 对象列表 */ - List queryAllByLimit(@Param("modelDependency") ModelDependency modelDependency, @Param("pageable") Pageable pageable); + List queryAllByLimit(ModelDependency modelDependency, @Param("pageable") Pageable pageable); @@ -38,7 +38,7 @@ public interface ModelDependencyDao { * @param modelDependency 查询条件 * @return 总行数 */ - long count(@Param("modelDependency") ModelDependency modelDependency); + long count(ModelDependency modelDependency); /** * 新增数据 @@ -46,7 +46,7 @@ public interface ModelDependencyDao { * @param modelDependency 实例对象 * @return 影响行数 */ - int insert(@Param("modelDependency") ModelDependency modelDependency); + int insert(ModelDependency modelDependency); /** * 批量新增数据（MyBatis原生foreach方法） @@ -71,7 +71,7 @@ public interface ModelDependencyDao { * @param modelDependency 实例对象 * @return 影响行数 */ - int update(@Param("modelDependency") ModelDependency modelDependency); + int update(ModelDependency modelDependency); /** * 通过主键删除数据 diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/scheduling/ExperimentInstanceStatusTask.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/scheduling/ExperimentInstanceStatusTask.java index 1764b262..91ed6847 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/scheduling/ExperimentInstanceStatusTask.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/scheduling/ExperimentInstanceStatusTask.java @@ -78,10 +78,7 @@ public class ExperimentInstanceStatusTask { } ModelDependency modelDependency = modelDependencyListquery.get(0); //查看状态， - if (StringUtils.equals("Succeeded",experimentIns.getStatus())){ - modelDependency.setState(1); - modelDependencyList.add(modelDependency); - }else{ + if (StringUtils.equals("Failed",experimentIns.getStatus())){ //取出节点状态 String trainTask = modelDependency.getTrainTask(); Map trainMap = JacksonUtil.parseJSONStr2Map(trainTask); @@ -91,17 +88,20 @@ public class ExperimentInstanceStatusTask { } String nodesStatus = experimentIns.getNodesStatus(); Map nodeMaps = JacksonUtil.parseJSONStr2Map(nodesStatus); - Map nodeMap = JacksonUtil.parseJSONStr2Map((String) nodeMaps.get(task_id)); + Map nodeMap = JacksonUtil.parseJSONStr2Map(JacksonUtil.toJSONString(nodeMaps.get(task_id))); + if (nodeMap==null){ continue; } - if (StringUtils.equals("Succeeded",(String)nodeMap.get("phase"))){ - modelDependency.setState(1); + if (!StringUtils.equals("Succeeded",(String)nodeMap.get("phase"))){ + modelDependency.setState(0); modelDependencyList.add(modelDependency); } } } - modelDependencyDao.insertOrUpdateBatch(modelDependencyList); + if (modelDependencyList.size()>0) { + modelDependencyDao.insertOrUpdateBatch(modelDependencyList); + } } } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentServiceImpl.java index b53d488b..407ff6ae 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentServiceImpl.java @@ -226,8 +226,7 @@ public class ExperimentServiceImpl implements ExperimentService { throw new RuntimeException("转换流水线失败"); } Map converMap = JsonUtils.jsonToMap(convertRes); - //得到dependendcy - Map dependendcy = (Map)converMap.get("model_dependency"); + // 组装运行接口json Map runReqMap = new HashMap<>(); runReqMap.put("data", converMap.get("data")); @@ -269,7 +268,11 @@ public class ExperimentServiceImpl implements ExperimentService { ExperimentIns insert = experimentInsService.insert(experimentIns); //插入到模型依赖关系表 - insertModelDependency(dependendcy,insert.getId(),experiment.getName(),params); + //得到dependendcy + Map converMap2 = JsonUtils.jsonToMap(JacksonUtil.replaceInAarry(convertRes, params)); + Map dependendcy = (Map)converMap2.get("model_dependency"); + Map trainInfo = (Map)converMap2.get("component_info"); + insertModelDependency(dependendcy,trainInfo,insert.getId(),experiment.getName()); }catch (Exception e){ throw new RuntimeException(e); @@ -278,17 +281,23 @@ public class ExperimentServiceImpl implements ExperimentService { experiment.setExperimentInsList(updatedExperimentInsList); return experiment; } - - private void insertModelDependency(Map dependendcy, Integer experimentInsId, String experimentName, List> params) throws Exception { + private void insertModelDependency(Map dependendcy,Map trainInfo, Integer experimentInsId, String experimentName) throws Exception { Iterator> dependendcyIterator = dependendcy.entrySet().iterator(); + Map modelTrain = (Map) trainInfo.get("model_train"); + Map modelEvaluate = (Map) trainInfo.get("model_evaluate"); + Map modelExport = (Map) trainInfo.get("model_export"); while (dependendcyIterator.hasNext()) { ModelDependency modelDependency = new ModelDependency(); - Map.Entry entry = dependendcyIterator.next(); - String key = entry.getKey(); Map modelDel = (Map) entry.getValue(); + Map source = (Map) modelDel.get("source"); + List> test = (List>) modelDel.get("test"); + List> target = (List>) modelDel.get("target"); + String sourceTaskId = (String) source.get("task_id"); + + Map modelTrainMap = (Map)modelTrain.get(sourceTaskId); //处理project数据 - Map projectMap = (Map) modelDel.get("project"); + Map projectMap = (Map) modelTrainMap.get("project"); ProjectDepency projectDepency = new ProjectDepency(); projectDepency.setBranch(projectMap.get("branch")); String projectUrl = projectMap.get("url"); @@ -296,86 +305,193 @@ public class ExperimentServiceImpl implements ExperimentService { projectDepency.setName(projectUrl.substring(projectUrl.lastIndexOf('/') + 1, projectUrl.length() - 4)); //依赖项目 modelDependency.setProjectDependency(JsonUtils.objectToJson(projectDepency)); + //处理镜像 + Map imagesMap = (Map) modelTrainMap.get("image"); + modelDependency.setTrainImage(imagesMap.get("name")); + List> trainParamList = (List>) modelTrainMap.get("params"); + modelDependency.setTrainParams(JsonUtils.objectToJson(trainParamList)); //处理source数据 - Map sourceMap = (Map) modelDel.get("source"); - List> modelsList = (List>) sourceMap.get("models"); - for(int i=0;i model = modelsList.get(i); - Models models = modelsService.queryById((Integer) model.get("model_id")); - if (models == null){ - throw new Exception("源模型不存在"); + List> modelsList = (List>) modelTrainMap.get("models"); + if (modelsList != null) { + for (int i = 0; i < modelsList.size(); i++) { + Map model = modelsList.get(i); + Models models = modelsService.queryById((Integer) model.get("model_id")); + if (models == null) { + throw new Exception("源模型不存在"); + } + model.put("model_name", models.getName()); } - model.put("model_name", models.getName()); + //父模型 + modelDependency.setParentModels(JsonUtils.objectToJson(modelsList)); } - //父模型 - modelDependency.setParentModels(JsonUtils.objectToJson(modelsList)); - - List> datasetsList = (List>) sourceMap.get("datasets"); - for(int i=0;i datasets = datasetsList.get(i); - Dataset dataset = datasetService.queryById((Integer) datasets.get("dataset_id")); - if (dataset == null){ - throw new Exception("源数据集不存在"); + List> datasetsList = (List>) modelTrainMap.get("datasets"); + if (datasetsList != null) { + for (int i = 0; i < datasetsList.size(); i++) { + Map datasets = datasetsList.get(i); + Dataset dataset = datasetService.queryById((Integer) datasets.get("dataset_id")); + if (dataset == null) { + throw new Exception("源数据集不存在"); + } + datasets.put("dataset_name", dataset.getName()); } - datasets.put("dataset_name", dataset.getName()); } //训练数据集 modelDependency.setTrainDataset(JsonUtils.objectToJson(datasetsList)); - TrainTaskDepency trainTaskDepency = new TrainTaskDepency(); - trainTaskDepency.setTaskId(key); + trainTaskDepency.setTaskId(sourceTaskId); trainTaskDepency.setInsId(experimentInsId); trainTaskDepency.setName(experimentName); //训练任务 modelDependency.setTrainTask(JsonUtils.objectToJson(trainTaskDepency)); modelDependency.setExpInsId(experimentInsId); - //处理test数据 - List> testDatasetsList = (List>) modelDel.get("test"); List> resultTestDatasets = new ArrayList>(); - for(int i=0;i datasets = testDatasetsList.get(i); - List> realDataSetList = (List>) datasets.get("datasets"); - for(int j=0;j realDataSet = realDataSetList.get(j); - Dataset dataset = datasetService.queryById((Integer) realDataSet.get("dataset_id")); - if (dataset == null){ - throw new Exception("源数据集不存在"); + //处理test数据 + if (test != null) { + for(int i=0;i testMap = test.get(i); + String testTaskId = (String) testMap.get("task_id"); + Map evaluateMap = (Map) modelEvaluate.get(testTaskId); + List> realDataSetList = (List>) evaluateMap.get("datasets"); + for(int j=0;j realDataSet = realDataSetList.get(j); + Dataset dataset = datasetService.queryById((Integer) realDataSet.get("dataset_id")); + if (dataset == null){ + throw new Exception("源数据集不存在"); + } + realDataSet.put("dataset_name", dataset.getName()); + resultTestDatasets.add(realDataSet); } - realDataSet.put("dataset_name", dataset.getName()); - resultTestDatasets.add(realDataSet); - } + } - } //测试数据集 - modelDependency.setTestDataset(JsonUtils.objectToJson(resultTestDatasets)); - - //检查是否存在target，如果存在说明在流水线用了节点导入，如果没有说明没有导入，等待手动push - List> modelTargetList = (List>) modelDel.get("target"); - if (modelTargetList==null||modelTargetList.size()==0){ - modelDependency.setState(1); - modelDependencyService.insert(modelDependency); - }else { - modelDependency.setState(2); - for(int i=0;i model = modelTargetList.get(i); - String version = null; - //可能是参数，必须从实验参数读取 - if (params != null) { - for (Map param : params) { - if (param.containsKey("param_name") && StringUtils.equals("model_version",(String) param.get("param_name"))) { - version = param.get("param_value").toString(); - } - } + modelDependency.setTestDataset(JsonUtils.objectToJson(resultTestDatasets)); + } + //处理target数据 + if (target != null) { + for (int i = 0; i < target.size(); i++) { + Map targetMap = target.get(i); + String targetaskId = (String) targetMap.get("task_id"); + Map exportMap = (Map) modelExport.get(targetaskId); + List> modelTargetList = (List>) exportMap.get("models"); + modelDependency.setState(2); + for (int j = 0; j < modelTargetList.size(); j++) { + Map model = modelTargetList.get(i); + modelDependency.setVersion((String) model.get("model_version")); + modelDependency.setCurrentModelId((Integer) model.get("model_id")); + //因为可能有多成果模型，多次插入 + modelDependencyService.insert(modelDependency); } - modelDependency.setVersion(StringUtils.isEmpty(version)?(String)model.get("model_version"):version); - modelDependency.setCurrentModelId((Integer) model.get("model_id")); - //因为可能有多成果模型，多次插入 - modelDependencyService.insert(modelDependency); } + }else { + modelDependency.setState(2); + modelDependencyService.insert(modelDependency); } + } } + /** + * 被废弃的旧JSON + * @param experiment + * @return + * @throws Exception + */ +// private void insertModelDependency(Map dependendcy,Map trainInfo, Integer experimentInsId, String experimentName, List> params) throws Exception { +// Iterator> dependendcyIterator = dependendcy.entrySet().iterator(); +// while (dependendcyIterator.hasNext()) { +// ModelDependency modelDependency = new ModelDependency(); +// Map.Entry entry = dependendcyIterator.next(); +// String key = entry.getKey(); +// Map modelDel = (Map) entry.getValue(); +// //处理project数据 +// Map projectMap = (Map) modelDel.get("project"); +// ProjectDepency projectDepency = new ProjectDepency(); +// projectDepency.setBranch(projectMap.get("branch")); +// String projectUrl = projectMap.get("url"); +// projectDepency.setUrl(projectUrl); +// projectDepency.setName(projectUrl.substring(projectUrl.lastIndexOf('/') + 1, projectUrl.length() - 4)); +// //依赖项目 +// modelDependency.setProjectDependency(JsonUtils.objectToJson(projectDepency)); +// //处理source数据 +// Map sourceMap = (Map) modelDel.get("source"); +// List> modelsList = (List>) sourceMap.get("models"); +// for(int i=0;i model = modelsList.get(i); +// Models models = modelsService.queryById((Integer) model.get("model_id")); +// if (models == null){ +// throw new Exception("源模型不存在"); +// } +// model.put("model_name", models.getName()); +// } +// //父模型 +// modelDependency.setParentModels(JsonUtils.objectToJson(modelsList)); +// +// List> datasetsList = (List>) sourceMap.get("datasets"); +// for(int i=0;i datasets = datasetsList.get(i); +// Dataset dataset = datasetService.queryById((Integer) datasets.get("dataset_id")); +// if (dataset == null){ +// throw new Exception("源数据集不存在"); +// } +// datasets.put("dataset_name", dataset.getName()); +// } +// //训练数据集 +// modelDependency.setTrainDataset(JsonUtils.objectToJson(datasetsList)); +// +// TrainTaskDepency trainTaskDepency = new TrainTaskDepency(); +// trainTaskDepency.setTaskId(key); +// trainTaskDepency.setInsId(experimentInsId); +// trainTaskDepency.setName(experimentName); +// //训练任务 +// modelDependency.setTrainTask(JsonUtils.objectToJson(trainTaskDepency)); +// modelDependency.setExpInsId(experimentInsId); +// //处理test数据 +// List> testDatasetsList = (List>) modelDel.get("test"); +// List> resultTestDatasets = new ArrayList>(); +// for(int i=0;i datasets = testDatasetsList.get(i); +// List> realDataSetList = (List>) datasets.get("datasets"); +// for(int j=0;j realDataSet = realDataSetList.get(j); +// Dataset dataset = datasetService.queryById((Integer) realDataSet.get("dataset_id")); +// if (dataset == null){ +// throw new Exception("源数据集不存在"); +// } +// realDataSet.put("dataset_name", dataset.getName()); +// resultTestDatasets.add(realDataSet); +// } +// +// } +// //测试数据集 +// modelDependency.setTestDataset(JsonUtils.objectToJson(resultTestDatasets)); +// +// //检查是否存在target，如果存在说明在流水线用了节点导入，如果没有说明没有导入，等待手动push +// List> modelTargetList = (List>) modelDel.get("target"); +// if (modelTargetList==null||modelTargetList.size()==0){ +// modelDependency.setState(1); +// modelDependencyService.insert(modelDependency); +// }else { +// modelDependency.setState(2); +// for(int i=0;i model = modelTargetList.get(i); +// String version = null; +// //可能是参数，必须从实验参数读取 +// if (params != null) { +// for (Map param : params) { +// if (param.containsKey("param_name") && StringUtils.equals("model_version",(String) param.get("param_name"))) { +// version = param.get("param_value").toString(); +// } +// } +// } +// modelDependency.setVersion(StringUtils.isEmpty(version)?(String)model.get("model_version"):version); +// modelDependency.setCurrentModelId((Integer) model.get("model_id")); +// //因为可能有多成果模型，多次插入 +// modelDependencyService.insert(modelDependency); +// } +// } +// } +// } + @Override public Experiment addAndRunExperiment(Experiment experiment) throws Exception { // 第一步: 调用add方法插入实验记录到数据库 diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelDependencyServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelDependencyServiceImpl.java index 776247c2..93b6113b 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelDependencyServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelDependencyServiceImpl.java @@ -217,5 +217,4 @@ public class ModelDependencyServiceImpl implements ModelDependencyService { return this.modelDependencyDao.update(modelDependency)>0?"删除成功":"删除失败"; } - } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsVersionServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsVersionServiceImpl.java index cfa793c8..e0b1207f 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsVersionServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsVersionServiceImpl.java @@ -206,6 +206,16 @@ public class ModelsVersionServiceImpl implements ModelsVersionService { String result = this.removeById(modelsVersion.getId()); results.put(modelsVersion.getId(), result); } + // 删除依赖关系 + ModelDependency modelDependency = new ModelDependency(); + modelDependency.setCurrentModelId(modelsId); + modelDependency.setVersion(version); + modelDependency.setState(1); + List modelDependencyList = modelDependencyService.queryByModelDependency(modelDependency); + if (modelDependencyList.size()>0){ + ModelDependency modelDependency1 = modelDependencyList.get(0); + modelDependencyService.removeById(modelDependency1.getId()); + } return results; } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/JacksonUtil.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/JacksonUtil.java index 5de227d5..6fd59389 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/JacksonUtil.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/JacksonUtil.java @@ -6,6 +6,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.databind.type.CollectionType; import com.ruoyi.common.core.utils.StringUtils; +import org.apache.xmlbeans.impl.xb.xsdschema.Public; import java.text.SimpleDateFormat; import java.util.ArrayList; @@ -58,6 +59,21 @@ public class JacksonUtil { } } + + + public static String replaceInAarry(String res, List> params) { + // 解析 JSON 数组 + if (params == null) { + return res; + } + for (int i = 0; i < params.size(); i++) { + Map stringObjectMap = params.get(i); + if (res.contains("${" + stringObjectMap.get("param_name") + "}")) { + res = res.replace("${" + stringObjectMap.get("param_name") + "}", String.valueOf( stringObjectMap.get("param_value"))); + } + } + return res; + } /** * 获取ObjectMapper。其本身是线程安全的，可以作为成员变量，但传入的参数不可能每次都一样，所以不使用成员变量。 * diff --git a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ModelDependencyDaoMapper.xml b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ModelDependencyDaoMapper.xml index 07e1cc7a..939df96e 100644 --- a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ModelDependencyDaoMapper.xml +++ b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ModelDependencyDaoMapper.xml @@ -10,6 +10,8 @@ + + @@ -20,18 +22,18 @@ - - select - id,current_model_id,exp_ins_id,parent_models,ref_item,train_task,train_dataset,test_dataset,project_dependency,version,create_by,create_time,update_by,update_time,state + id,current_model_id,exp_ins_id,parent_models,ref_item,train_task,train_dataset,train_params,train_image,test_dataset,project_dependency,version,create_by,create_time,update_by,update_time,state from model_dependency - where id = #{id} + + parent_models like concat('%', #{model_id}, '%') AND parent_models like concat('%', #{version}, '%') + - - select - id,current_model_id,exp_ins_id,parent_models,ref_item,train_task,train_dataset,test_dataset,project_dependency,version,create_by,create_time,update_by,update_time,state + id,current_model_id,exp_ins_id,parent_models,ref_item,train_task,train_dataset,train_params,train_image,test_dataset,project_dependency,version,create_by,create_time,update_by,update_time,state from model_dependency @@ -55,6 +57,12 @@ and train_dataset = #{modelDependency.trainDataset} + + and train_params = #{modelDependency.trainParams} + + + and train_image = #{modelDependency.trainImage} + and test_dataset = #{modelDependency.testDataset} @@ -80,69 +88,76 @@ and state = #{modelDependency.state} - limit #{pageable.offset}, #{pageable.pageSize} - select - id,current_model_id,exp_ins_id,parent_models,ref_item,train_task,train_dataset,test_dataset,project_dependency,version,create_by,create_time,update_by,update_time,state + id, current_model_id, exp_ins_id, parent_models, ref_item, train_task, train_dataset, train_params, train_image, test_dataset, project_dependency, version, create_by, create_time, update_by, update_time, state from model_dependency - - parent_models like concat('%', #{model_id}, '%') AND parent_models like concat('%', #{version}, '%') - + where id = #{id} + - select - id,current_model_id,exp_ins_id,parent_models,ref_item,train_task,train_dataset,test_dataset,project_dependency,version,create_by,create_time,update_by,update_time,state + id, current_model_id, exp_ins_id, parent_models, ref_item, train_task, train_dataset, train_params, train_image, + test_dataset, project_dependency, version, create_by, create_time, update_by, update_time, state from model_dependency - - and id = #{modelDependency.id} + + and id = #{id} - - and current_model_id = #{modelDependency.currentModelId} + + and current_model_id = #{currentModelId} - - and exp_ins_id = #{modelDependency.expInsId} + + and exp_ins_id = #{expInsId} - - and parent_models = #{modelDependency.parentModels} + + and parent_models = #{parentModels} - - and ref_item = #{modelDependency.refItem} + + and ref_item = #{refItem} - - and train_task = #{modelDependency.trainTask} + + and train_task = #{trainTask} - - and train_dataset = #{modelDependency.trainDataset} + + and train_dataset = #{trainDataset} - - and test_dataset = #{modelDependency.testDataset} + + and train_params = #{trainParams} - - and project_dependency = #{modelDependency.projectDependency} + + and train_image = #{trainImage} - - and version = #{modelDependency.version} + + and test_dataset = #{testDataset} - - and create_by = #{modelDependency.createBy} + + and project_dependency = #{projectDependency} - - and create_time = #{modelDependency.createTime} + + and version = #{version} - - and update_by = #{modelDependency.updateBy} + + and create_by = #{createBy} - - and update_time = #{modelDependency.updateTime} + + and create_time = #{createTime} - - and state = #{modelDependency.state} + + and update_by = #{updateBy} + + + and update_time = #{updateTime} + + + and state = #{state} + limit #{pageable.offset}, #{pageable.pageSize} @@ -150,158 +165,163 @@ select count(1) from model_dependency - - and id = #{modelDependency.id} + + and id = #{id} - - and current_model_id = #{modelDependency.currentModelId} + + and current_model_id = #{currentModelId} - - and exp_ins_id = #{modelDependency.expInsId} + + and exp_ins_id = #{expInsId} - - and parent_models = #{modelDependency.parentModels} + + and parent_models = #{parentModels} - - and ref_item = #{modelDependency.refItem} + + and ref_item = #{refItem} - - and train_task = #{modelDependency.trainTask} + + and train_task = #{trainTask} - - and train_dataset = #{modelDependency.trainDataset} + + and train_dataset = #{trainDataset} - - and test_dataset = #{modelDependency.testDataset} + + and train_params = #{trainParams} - - and project_dependency = #{modelDependency.projectDependency} + + and train_image = #{trainImage} - - and version = #{modelDependency.version} + + and test_dataset = #{testDataset} - - and create_by = #{modelDependency.createBy} + + and project_dependency = #{projectDependency} - - and create_time = #{modelDependency.createTime} + + and version = #{version} - - and update_by = #{modelDependency.updateBy} + + and create_by = #{createBy} - - and update_time = #{modelDependency.updateTime} + + and create_time = #{createTime} - - and state = #{modelDependency.state} + + and update_by = #{updateBy} + + + and update_time = #{updateTime} + + + and state = #{state} - - insert into model_dependency (current_model_id, exp_ins_id, parent_models, ref_item, train_task, train_dataset, test_dataset, project_dependency,version, create_by, create_time, update_by, update_time, state) - values (#{modelDependency.currentModelId}, #{modelDependency.expInsId},#{modelDependency.parentModels}, #{modelDependency.refItem}, #{modelDependency.trainTask}, #{modelDependency.trainDataset}, #{modelDependency.testDataset}, #{modelDependency.projectDependency},#{modelDependency.version}, #{modelDependency.createBy}, #{modelDependency.createTime}, #{modelDependency.updateBy}, #{modelDependency.updateTime}, #{modelDependency.state}) + insert into model_dependency(current_model_id, exp_ins_id, parent_models, ref_item, train_task, train_dataset, train_params, train_image, test_dataset, project_dependency, version, create_by, create_time, update_by, update_time, state) + values (#{currentModelId}, #{expInsId}, #{parentModels}, #{refItem}, #{trainTask}, #{trainDataset}, #{trainParams}, #{trainImage}, #{testDataset}, #{projectDependency}, #{version}, #{createBy}, #{createTime}, #{updateBy}, #{updateTime}, #{state}) - insert into model_dependency ( - current_model_id, - exp_ins_id, - parent_models, - ref_item, - train_task, - train_dataset, - test_dataset, - project_dependency, - version, - create_by, - create_time, - update_by, - update_time, - state - ) + insert into model_dependency(current_model_id, exp_ins_id, parent_models, ref_item, train_task, train_dataset, + train_params, train_image, test_dataset, project_dependency, version, create_by, create_time, update_by, + update_time, state) values - ( - #{entity.currentModelId}, - #{entity.expInsId}, - #{entity.parentModels}, - #{entity.refItem}, - #{entity.trainTask}, - #{entity.trainDataset}, - #{entity.testDataset}, - #{entity.projectDependency}, - #{entity.version}, - #{entity.createBy}, - #{entity.createTime}, - #{entity.updateBy}, - #{entity.updateTime}, - #{entity.state} - ) + (#{entity.currentModelId}, #{entity.expInsId}, #{entity.parentModels}, #{entity.refItem}, + #{entity.trainTask}, #{entity.trainDataset}, #{entity.trainParams}, #{entity.trainImage}, + #{entity.testDataset}, #{entity.projectDependency}, #{entity.version}, #{entity.createBy}, + #{entity.createTime}, #{entity.updateBy}, #{entity.updateTime}, #{entity.state}) - - insert into model_dependency(current_model_idparent_modelsref_itemtrain_tasktrain_datasettest_datasetcreate_bycreate_timeupdate_byupdate_timestate) + insert into model_dependency(current_model_id, exp_ins_id, parent_models, ref_item, train_task, train_dataset, + train_params, train_image, test_dataset, project_dependency, version, create_by, create_time, update_by, + update_time, state) values - (#{entity.currentModelId}#{entity.parentModels}#{entity.refItem}#{entity.trainTask}#{entity.trainDataset}#{entity.testDataset}#{entity.createBy}#{entity.createTime}#{entity.updateBy}#{entity.updateTime}#{entity.state}) + (#{entity.currentModelId}, #{entity.expInsId}, #{entity.parentModels}, #{entity.refItem}, + #{entity.trainTask}, #{entity.trainDataset}, #{entity.trainParams}, #{entity.trainImage}, + #{entity.testDataset}, #{entity.projectDependency}, #{entity.version}, #{entity.createBy}, + #{entity.createTime}, #{entity.updateBy}, #{entity.updateTime}, #{entity.state}) on duplicate key update -current_model_id = values(current_model_id)parent_models = values(parent_models)ref_item = values(ref_item)train_task = values(train_task)train_dataset = values(train_dataset)test_dataset = values(test_dataset)create_by = values(create_by)create_time = values(create_time)update_by = values(update_by)update_time = values(update_time)state = values(state) + current_model_id = values(current_model_id), + exp_ins_id = values(exp_ins_id), + parent_models = values(parent_models), + ref_item = values(ref_item), + train_task = values(train_task), + train_dataset = values(train_dataset), + train_params = values(train_params), + train_image = values(train_image), + test_dataset = values(test_dataset), + project_dependency = values(project_dependency), + version = values(version), + create_by = values(create_by), + create_time = values(create_time), + update_by = values(update_by), + update_time = values(update_time), + state = values(state) update model_dependency - - current_model_id = #{modelDependency.currentModelId}, + + current_model_id = #{currentModelId}, - - exp_ins_id = #{modelDependency.expInsId}, + + exp_ins_id = #{expInsId}, - - parent_models = #{modelDependency.parentModels}, + + parent_models = #{parentModels}, - - ref_item = #{modelDependency.refItem}, + + ref_item = #{refItem}, - - train_task = #{modelDependency.trainTask}, + + train_task = #{trainTask}, - - train_dataset = #{modelDependency.trainDataset}, + + train_dataset = #{trainDataset}, - - test_dataset = #{modelDependency.testDataset}, + + train_params = #{trainParams}, - - project_dependency = #{modelDependency.projectDependency}, + + train_image = #{trainImage}, - - version = #{modelDependency.version}, + + test_dataset = #{testDataset}, - - create_by = #{modelDependency.createBy}, + + project_dependency = #{projectDependency}, - - create_time = #{modelDependency.createTime}, + + version = #{version}, - - update_by = #{modelDependency.updateBy}, + + create_by = #{createBy}, - - update_time = #{modelDependency.updateTime}, + + create_time = #{createTime}, - - state = #{modelDependency.state}, + + update_by = #{updateBy}, + + + update_time = #{updateTime}, + + + state = #{state}, - where id = #{modelDependency.id} + where id = #{id} and state = 1 From 257c4d704c549b14b7f8afaedfe33f1b0811636a Mon Sep 17 00:00:00 2001 From: fanshuai <1141904845@qq.com> Date: Thu, 6 Jun 2024 16:04:27 +0800 Subject: [PATCH 29/31] =?UTF-8?q?=E6=A8=A1=E5=9E=8B=E6=BC=94=E5=8C=96?= =?UTF-8?q?=E6=94=B9=E9=80=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../model/ModelDependencyController.java | 3 +- .../model/ModelsVersionController.java | 3 +- .../platform/domain/ModelDependency.java | 22 ++++++++- .../service/ModelDependencyService.java | 3 +- .../service/ModelsVersionService.java | 3 +- .../impl/ModelDependencyServiceImpl.java | 45 +++++++++++++++---- .../service/impl/ModelsServiceImpl.java | 1 + .../impl/ModelsVersionServiceImpl.java | 3 +- .../platform/vo/ModelDependcyTreeVo.java | 12 +++++ 9 files changed, 80 insertions(+), 15 deletions(-) diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/model/ModelDependencyController.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/model/ModelDependencyController.java index 9afd7399..a62e8c61 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/model/ModelDependencyController.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/model/ModelDependencyController.java @@ -13,6 +13,7 @@ import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.*; import javax.annotation.Resource; +import java.io.IOException; import java.util.List; /** @@ -53,7 +54,7 @@ public class ModelDependencyController extends BaseController { */ @GetMapping("/queryModelDependency") @ApiOperation("根据对象查询") - public GenericsAjaxResult> queryByModelDependency(@RequestBody ModelDependency modelDependency) { + public GenericsAjaxResult> queryByModelDependency(@RequestBody ModelDependency modelDependency) throws IOException { modelDependency.setState(1); return genericsSuccess(this.modelDependencyService.queryByModelDependency(modelDependency)); } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/model/ModelsVersionController.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/model/ModelsVersionController.java index c966e92c..74a4ebf0 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/model/ModelsVersionController.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/model/ModelsVersionController.java @@ -11,6 +11,7 @@ import org.springframework.data.domain.PageRequest; import org.springframework.web.bind.annotation.*; import javax.annotation.Resource; +import java.io.IOException; import java.util.List; import java.util.Map; @@ -129,7 +130,7 @@ public class ModelsVersionController extends BaseController { @DeleteMapping("/deleteVersion") @ApiOperation(value = "逻辑删除模型版本", notes = "根据模型ID和版本逻辑删除模型版本记录。") public GenericsAjaxResult> deleteModelsVersion(@RequestParam("models_id") Integer modelsId, - @RequestParam("version") String version) { + @RequestParam("version") String version) throws IOException { return genericsSuccess(this.modelsVersionService.deleteModelsVersion(modelsId, version)); } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/ModelDependency.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/ModelDependency.java index 1809d7e5..6f53e4d0 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/ModelDependency.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/ModelDependency.java @@ -2,6 +2,7 @@ package com.ruoyi.platform.domain; import com.fasterxml.jackson.databind.PropertyNamingStrategy; import com.fasterxml.jackson.databind.annotation.JsonNaming; +import io.swagger.annotations.ApiModelProperty; import java.util.Date; import java.io.Serializable; @@ -88,6 +89,9 @@ public class ModelDependency implements Serializable { */ private Integer state; + private Long workflowId; + private int availableRange; + public Integer getId() { return id; } @@ -152,7 +156,7 @@ public class ModelDependency implements Serializable { this.trainParams = trainParams; } - public String getTrainImage(String name) { + public String getTrainImage() { return trainImage; } @@ -223,5 +227,21 @@ public class ModelDependency implements Serializable { public void setState(Integer state) { this.state = state; } + + public Long getWorkflowId() { + return workflowId; + } + + public void setWorkflowId(Long workflowId) { + this.workflowId = workflowId; + } + + public int getAvailableRange() { + return availableRange; + } + + public void setAvailableRange(int availableRange) { + this.availableRange = availableRange; + } } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ModelDependencyService.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ModelDependencyService.java index 3883a7b7..5c8b9d1d 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ModelDependencyService.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ModelDependencyService.java @@ -5,6 +5,7 @@ import com.ruoyi.platform.vo.ModelDependcyTreeVo; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageRequest; +import java.io.IOException; import java.util.List; /** @@ -58,7 +59,7 @@ public interface ModelDependencyService { String removeById(Integer id); - List queryByModelDependency(ModelDependency modelDependency); + List queryByModelDependency(ModelDependency modelDependency) throws IOException; ModelDependcyTreeVo getModelDependencyTree(ModelDependency modelDependency) throws Exception; } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ModelsVersionService.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ModelsVersionService.java index 6938da95..723bc745 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ModelsVersionService.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ModelsVersionService.java @@ -6,6 +6,7 @@ import com.ruoyi.platform.domain.ModelsVersion; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageRequest; +import java.io.IOException; import java.util.List; import java.util.Map; @@ -68,7 +69,7 @@ public interface ModelsVersionService { Map queryByModelsIdAndVersion(Integer modelsId, String version); - Map deleteModelsVersion(Integer modelsId, String version); + Map deleteModelsVersion(Integer modelsId, String version) throws IOException; String addModelVersions(List modelsVersions) throws Exception; diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelDependencyServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelDependencyServiceImpl.java index 93b6113b..99b21bae 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelDependencyServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelDependencyServiceImpl.java @@ -1,22 +1,24 @@ package com.ruoyi.platform.service.impl; import com.ruoyi.common.security.utils.SecurityUtils; -import com.ruoyi.platform.domain.ModelDependency; -import com.ruoyi.platform.domain.Models; -import com.ruoyi.platform.domain.ModelsVersion; +import com.ruoyi.platform.domain.*; import com.ruoyi.platform.mapper.ModelDependencyDao; +import com.ruoyi.platform.service.ExperimentInsService; +import com.ruoyi.platform.service.ExperimentService; import com.ruoyi.platform.service.ModelDependencyService; import com.ruoyi.platform.service.ModelsService; import com.ruoyi.platform.utils.JacksonUtil; import com.ruoyi.platform.vo.ModelDependcyTreeVo; import com.ruoyi.system.api.model.LoginUser; import org.apache.commons.lang3.StringUtils; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageImpl; import org.springframework.data.domain.PageRequest; import javax.annotation.Resource; +import java.io.IOException; import java.util.ArrayList; import java.util.Date; import java.util.List; @@ -36,7 +38,12 @@ public class ModelDependencyServiceImpl implements ModelDependencyService { @Resource private ModelsService modelsService; - + @Lazy + @Resource + private ExperimentService experimentService; + @Lazy + @Resource + private ExperimentInsService experimentInsService; /** * 通过ID查询单条数据 * @@ -68,15 +75,31 @@ public class ModelDependencyServiceImpl implements ModelDependencyService { * @return 查询结果 */ @Override - public List queryByModelDependency(ModelDependency modelDependency) { - return this.modelDependencyDao.queryByModelDependency(modelDependency); + public List queryByModelDependency(ModelDependency modelDependency) throws IOException { + List modelDependencyList = this.modelDependencyDao.queryByModelDependency(modelDependency); + /** + * 补充workFlow_id + 是否共有 + */ + if (modelDependencyList == null || modelDependencyList.size()==0){ + return new ArrayList<>(); + } + for (ModelDependency md : modelDependencyList){ + Integer currentModelId = md.getCurrentModelId(); + Integer expInsId = md.getExpInsId(); + Models models = modelsService.queryById(currentModelId); + ExperimentIns experimentIns = experimentInsService.queryById(expInsId); + Experiment experiment = experimentService.queryById(experimentIns.getExperimentId()); + md.setWorkflowId(experiment.getWorkflowId()); + md.setAvailableRange(models.getAvailableRange()); + } + return modelDependencyList; } @Override public ModelDependcyTreeVo getModelDependencyTree(ModelDependency modelDependencyQuery) throws Exception { //查询当前模型 modelDependencyQuery.setState(1); - List modelDependencyList = modelDependencyDao.queryByModelDependency(modelDependencyQuery); + List modelDependencyList = this.queryByModelDependency(modelDependencyQuery); if (modelDependencyList==null || modelDependencyList.size()==0){ throw new Exception("当前模型依赖关系不存在"); } @@ -93,7 +116,7 @@ public class ModelDependencyServiceImpl implements ModelDependencyService { * 递归父模型 * @param modelDependcyTreeVo */ - private void processParentModel(ModelDependcyTreeVo modelDependcyTreeVo){ + private void processParentModel(ModelDependcyTreeVo modelDependcyTreeVo) throws IOException { if (modelDependcyTreeVo.getParentModelsMap() != null) { List> parentMaps = modelDependcyTreeVo.getParentModelsMap(); List ps = new ArrayList(); @@ -103,7 +126,7 @@ public class ModelDependencyServiceImpl implements ModelDependencyService { ModelDependency modelDependencyQuery = new ModelDependency(); modelDependencyQuery.setVersion(version); modelDependencyQuery.setCurrentModelId(model_id); - List modelDependencyList = modelDependencyDao.queryByModelDependency(modelDependencyQuery); + List modelDependencyList = this.queryByModelDependency(modelDependencyQuery); if (modelDependencyList!=null&&modelDependencyList.size()>=0){ for (ModelDependency modelDependency:modelDependencyList){ ModelDependcyTreeVo modelDependencyTreeVoIn = ModelDependencyConvertToTree(modelDependency); @@ -144,9 +167,13 @@ public class ModelDependencyServiceImpl implements ModelDependencyService { modelDependcyTreeVo.setRefItem(modelDependency.getRefItem()); modelDependcyTreeVo.setTrainTask(JacksonUtil.parseJSONStr2Map(modelDependency.getTrainTask())); modelDependcyTreeVo.setTrainDataset(JacksonUtil.parseJSONStr2MapList(modelDependency.getTrainDataset())); + modelDependcyTreeVo.setTrainImage(modelDependency.getTrainImage()); + modelDependcyTreeVo.setTrainParams(JacksonUtil.parseJSONStr2TList(modelDependency.getTrainParams(),Object.class,null)); modelDependcyTreeVo.setTestDataset(JacksonUtil.parseJSONStr2MapList(modelDependency.getTestDataset())); modelDependcyTreeVo.setProjectDependency(JacksonUtil.parseJSONStr2Map(modelDependency.getProjectDependency())); modelDependcyTreeVo.setParentModelsMap(JacksonUtil.parseJSONStr2MapList(modelDependency.getParentModels())); + modelDependcyTreeVo.setWorkflowId(modelDependency.getWorkflowId()); + modelDependcyTreeVo.setAvailableRange(modelDependcyTreeVo.getAvailableRange()); Models models = modelsService.queryById(modelDependency.getCurrentModelId()); if (models!=null) { modelDependcyTreeVo.setCurrentModelName(models.getName()); diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java index d0a504ee..b3603484 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java @@ -192,6 +192,7 @@ public class ModelsServiceImpl implements ModelsService { * * @param id models_version表的主键 * @return 文件内容 + * */ @Override diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsVersionServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsVersionServiceImpl.java index e0b1207f..08d94324 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsVersionServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsVersionServiceImpl.java @@ -20,6 +20,7 @@ import org.springframework.data.domain.PageRequest; import org.springframework.stereotype.Service; import javax.annotation.Resource; +import java.io.IOException; import java.lang.reflect.Field; import java.util.Date; import java.util.HashMap; @@ -197,7 +198,7 @@ public class ModelsVersionServiceImpl implements ModelsVersionService { } @Override - public Map deleteModelsVersion(Integer modelsId, String version) { + public Map deleteModelsVersion(Integer modelsId, String version) throws IOException { Map results = new HashMap(); // 根据模型ID和版本查询所有模型版本 List versions = this.modelsVersionDao.queryAllByModelsVersion(modelsId, version); diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelDependcyTreeVo.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelDependcyTreeVo.java index 5814ffca..54aef8e8 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelDependcyTreeVo.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelDependcyTreeVo.java @@ -37,6 +37,15 @@ public class ModelDependcyTreeVo { * 训练数据集 */ private List> trainDataset; + + /** + * 训练参数 + */ + private List trainParams; + /** + * 训练镜像 + */ + private String trainImage; /** * 测试数据集 */ @@ -58,4 +67,7 @@ public class ModelDependcyTreeVo { private List childrenModels; private String currentModelName; + + private Long workflowId; + private int availableRange; } From 3d3113506b95f54789353454a966737ecd985619 Mon Sep 17 00:00:00 2001 From: fanshuai <1141904845@qq.com> Date: Thu, 6 Jun 2024 16:34:59 +0800 Subject: [PATCH 30/31] =?UTF-8?q?=E6=A8=A1=E5=9E=8B=E6=BC=94=E5=8C=96?= =?UTF-8?q?=E6=94=B9=E9=80=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../platform/domain/ModelDependency.java | 10 +-- .../impl/ModelDependencyServiceImpl.java | 61 ++++++++++--------- .../platform/vo/ModelDependcyTreeVo.java | 5 +- .../platform/vo/ModelVersionDependcyVo.java | 48 +++++++++++++++ 4 files changed, 88 insertions(+), 36 deletions(-) create mode 100644 ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelVersionDependcyVo.java diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/ModelDependency.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/ModelDependency.java index 6f53e4d0..18f9f872 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/ModelDependency.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/ModelDependency.java @@ -90,7 +90,7 @@ public class ModelDependency implements Serializable { private Integer state; private Long workflowId; - private int availableRange; + private Models models; public Integer getId() { return id; @@ -236,12 +236,12 @@ public class ModelDependency implements Serializable { this.workflowId = workflowId; } - public int getAvailableRange() { - return availableRange; + public Models getModels() { + return models; } - public void setAvailableRange(int availableRange) { - this.availableRange = availableRange; + public void setModels(Models models) { + this.models = models; } } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelDependencyServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelDependencyServiceImpl.java index 99b21bae..91b071ca 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelDependencyServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelDependencyServiceImpl.java @@ -3,12 +3,10 @@ package com.ruoyi.platform.service.impl; import com.ruoyi.common.security.utils.SecurityUtils; import com.ruoyi.platform.domain.*; import com.ruoyi.platform.mapper.ModelDependencyDao; -import com.ruoyi.platform.service.ExperimentInsService; -import com.ruoyi.platform.service.ExperimentService; -import com.ruoyi.platform.service.ModelDependencyService; -import com.ruoyi.platform.service.ModelsService; +import com.ruoyi.platform.service.*; import com.ruoyi.platform.utils.JacksonUtil; import com.ruoyi.platform.vo.ModelDependcyTreeVo; +import com.ruoyi.platform.vo.ModelVersionDependcyVo; import com.ruoyi.system.api.model.LoginUser; import org.apache.commons.lang3.StringUtils; import org.springframework.context.annotation.Lazy; @@ -38,6 +36,8 @@ public class ModelDependencyServiceImpl implements ModelDependencyService { @Resource private ModelsService modelsService; + @Resource + private ModelsVersionService modelsVersionService; @Lazy @Resource private ExperimentService experimentService; @@ -77,21 +77,6 @@ public class ModelDependencyServiceImpl implements ModelDependencyService { @Override public List queryByModelDependency(ModelDependency modelDependency) throws IOException { List modelDependencyList = this.modelDependencyDao.queryByModelDependency(modelDependency); - /** - * 补充workFlow_id + 是否共有 - */ - if (modelDependencyList == null || modelDependencyList.size()==0){ - return new ArrayList<>(); - } - for (ModelDependency md : modelDependencyList){ - Integer currentModelId = md.getCurrentModelId(); - Integer expInsId = md.getExpInsId(); - Models models = modelsService.queryById(currentModelId); - ExperimentIns experimentIns = experimentInsService.queryById(expInsId); - Experiment experiment = experimentService.queryById(experimentIns.getExperimentId()); - md.setWorkflowId(experiment.getWorkflowId()); - md.setAvailableRange(models.getAvailableRange()); - } return modelDependencyList; } @@ -130,7 +115,6 @@ public class ModelDependencyServiceImpl implements ModelDependencyService { if (modelDependencyList!=null&&modelDependencyList.size()>=0){ for (ModelDependency modelDependency:modelDependencyList){ ModelDependcyTreeVo modelDependencyTreeVoIn = ModelDependencyConvertToTree(modelDependency); - modelDependencyTreeVoIn.setCurrentModelName((String) parent.get("model_name")); processParentModel(modelDependencyTreeVoIn); ps.add(modelDependencyTreeVoIn); } @@ -142,7 +126,7 @@ public class ModelDependencyServiceImpl implements ModelDependencyService { } - private void processChildrenModel(ModelDependcyTreeVo modelDependcyTreeVo){ + private void processChildrenModel(ModelDependcyTreeVo modelDependcyTreeVo) throws IOException { String version = modelDependcyTreeVo.getVersion(); Integer modelId = modelDependcyTreeVo.getCurrentModelId(); List cs = new ArrayList(); @@ -159,7 +143,7 @@ public class ModelDependencyServiceImpl implements ModelDependencyService { modelDependcyTreeVo.setChildrenModels(cs); } - private ModelDependcyTreeVo ModelDependencyConvertToTree(ModelDependency modelDependency){ + private ModelDependcyTreeVo ModelDependencyConvertToTree(ModelDependency modelDependency) throws IOException { ModelDependcyTreeVo modelDependcyTreeVo = new ModelDependcyTreeVo(); modelDependcyTreeVo.setCurrentModelId(modelDependency.getCurrentModelId()); modelDependcyTreeVo.setExpInsId(modelDependency.getExpInsId()); @@ -172,12 +156,33 @@ public class ModelDependencyServiceImpl implements ModelDependencyService { modelDependcyTreeVo.setTestDataset(JacksonUtil.parseJSONStr2MapList(modelDependency.getTestDataset())); modelDependcyTreeVo.setProjectDependency(JacksonUtil.parseJSONStr2Map(modelDependency.getProjectDependency())); modelDependcyTreeVo.setParentModelsMap(JacksonUtil.parseJSONStr2MapList(modelDependency.getParentModels())); - modelDependcyTreeVo.setWorkflowId(modelDependency.getWorkflowId()); - modelDependcyTreeVo.setAvailableRange(modelDependcyTreeVo.getAvailableRange()); - Models models = modelsService.queryById(modelDependency.getCurrentModelId()); - if (models!=null) { - modelDependcyTreeVo.setCurrentModelName(models.getName()); - } + + /** + * 补充workFlow_id + 是否共有 + */ + Integer currentModelId = modelDependency.getCurrentModelId(); + Integer expInsId = modelDependency.getExpInsId(); + Models models = modelsService.queryById(currentModelId); + ModelsVersion modelsVersionquery = new ModelsVersion(); + modelsVersionquery.setModelsId(currentModelId); + modelsVersionquery.setVersion(modelDependency.getVersion()); + ModelsVersion modelsVersion = modelsVersionService.queryByModelsVersion(modelsVersionquery); + ExperimentIns experimentIns = experimentInsService.queryById(expInsId); + Experiment experiment = experimentService.queryById(experimentIns.getExperimentId()); + ModelVersionDependcyVo modelVersionDependcyVo = new ModelVersionDependcyVo(); + modelVersionDependcyVo.setName(models.getName()); + modelVersionDependcyVo.setAvailableRange(models.getAvailableRange()); + modelVersionDependcyVo.setDescription(models.getDescription()); + modelVersionDependcyVo.setModelTag(models.getModelTag()); + modelVersionDependcyVo.setModelType(models.getModelType()); + modelVersionDependcyVo.setModelTagName(models.getModelTagName()); + modelVersionDependcyVo.setModelTypeName(models.getModelTypeName()); + modelVersionDependcyVo.setFileName(modelsVersion.getFileName()); + modelVersionDependcyVo.setFileSize(modelsVersion.getFileSize()); + modelVersionDependcyVo.setUrl(modelsVersion.getUrl()); + modelDependcyTreeVo.setWorkflowId(experiment.getWorkflowId()); + modelDependcyTreeVo.setModelVersionDependcyVo(modelVersionDependcyVo); + return modelDependcyTreeVo; } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelDependcyTreeVo.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelDependcyTreeVo.java index 54aef8e8..8abe4033 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelDependcyTreeVo.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelDependcyTreeVo.java @@ -2,6 +2,7 @@ package com.ruoyi.platform.vo; import com.fasterxml.jackson.databind.PropertyNamingStrategy; import com.fasterxml.jackson.databind.annotation.JsonNaming; +import com.ruoyi.platform.domain.Models; import lombok.Data; import java.util.List; @@ -66,8 +67,6 @@ public class ModelDependcyTreeVo { */ private List childrenModels; - private String currentModelName; - private Long workflowId; - private int availableRange; + private ModelVersionDependcyVo modelVersionDependcyVo; } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelVersionDependcyVo.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelVersionDependcyVo.java new file mode 100644 index 00000000..9d835a41 --- /dev/null +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelVersionDependcyVo.java @@ -0,0 +1,48 @@ +package com.ruoyi.platform.vo; + +import com.fasterxml.jackson.databind.PropertyNamingStrategy; +import com.fasterxml.jackson.databind.annotation.JsonNaming; +import com.ruoyi.platform.annotations.CheckDuplicate; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import java.io.Serializable; + +@Data +@JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy.class) +public class ModelVersionDependcyVo implements Serializable { + + + @ApiModelProperty(name = "name") + private String name; + + // private String version; + @ApiModelProperty(name = "description") + private String description; + + @ApiModelProperty(value = "模型可见范围，1表示公开，0表示私有") + private int availableRange; + + @ApiModelProperty(value = "模型类型(模型框架)") + private String modelType; + + @ApiModelProperty(value = "模型标签") + private String modelTag; + + + @ApiModelProperty(value = "模型类型名") + private String modelTypeName; + + @ApiModelProperty(value = "模型tag名") + private String modelTagName; + + @ApiModelProperty(value = "模型存储地址") + private String url; + + @ApiModelProperty(value = "文件名") + private String fileName; + + @ApiModelProperty(value = "文件大小") + private String fileSize; + +} From c25e8113768a7d735d6e2b235e6acc723465f473 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=A5=BF=E5=A4=A7=E9=94=90?= <1070211640@qq.com> Date: Thu, 6 Jun 2024 17:12:57 +0800 Subject: [PATCH 31/31] =?UTF-8?q?feat:=E5=BC=80=E5=8F=91=E7=8E=AF=E5=A2=83?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E9=9B=86=E6=A8=A1=E5=9E=8B=E5=8F=AF=E9=85=8D?= =?UTF-8?q?=E7=BD=AE=E4=B8=80=E9=98=B6=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../controller/jupyter/JupyterController.java | 35 ++++- .../platform/service/JupyterService.java | 6 + .../impl/DevEnvironmentServiceImpl.java | 11 ++ .../service/impl/JupyterServiceImpl.java | 65 ++++++++++ .../ruoyi/platform/utils/K8sClientUtil.java | 121 +++++++++++++++++- 5 files changed, 230 insertions(+), 8 deletions(-) diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/jupyter/JupyterController.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/jupyter/JupyterController.java index a533193d..19a3476c 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/jupyter/JupyterController.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/jupyter/JupyterController.java @@ -4,11 +4,11 @@ import com.ruoyi.common.core.web.controller.BaseController; import com.ruoyi.common.core.web.domain.AjaxResult; import com.ruoyi.common.core.web.domain.GenericsAjaxResult; import com.ruoyi.platform.service.JupyterService; +import com.ruoyi.platform.vo.FrameLogPathVo; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import org.springframework.web.bind.annotation.*; import javax.annotation.Resource; import java.io.File; @@ -28,6 +28,35 @@ public class JupyterController extends BaseController { return genericsSuccess(jupyterService.getJupyterServiceUrl()); } + + /** + * 启动jupyter容器接口 + * + * @param id 开发环境配置id + * @return url + */ + @PostMapping("/run/{id}") + @ApiOperation("根据开发环境id启动jupyter pod") + @ApiResponse + public GenericsAjaxResult runJupyter(@PathVariable("id") Integer id) throws Exception { + return genericsSuccess(this.jupyterService.runJupyterService(id)); + } + + + /** + * 停止jupyter容器接口 + * + * @param id 开发环境配置id + * @return 操作结果 + */ + @DeleteMapping("/stop/{id}") + @ApiOperation("根据开发环境id停止jupyter pod") + @ApiResponse + public GenericsAjaxResult stopJupyter(@PathVariable("id") Integer id) throws Exception { + return genericsSuccess(this.jupyterService.stopJupyterService(id)); + } + + @GetMapping(value = "/upload") public AjaxResult upload() throws Exception { File file = new File("D://nexus-deploy.yaml"); diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/JupyterService.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/JupyterService.java index d41c6a31..af62a509 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/JupyterService.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/JupyterService.java @@ -1,5 +1,7 @@ package com.ruoyi.platform.service; +import com.ruoyi.platform.vo.FrameLogPathVo; + import java.io.InputStream; public interface JupyterService { @@ -8,4 +10,8 @@ public interface JupyterService { void upload(InputStream inputStream); void mlflow(); + + String runJupyterService(Integer id); + + String stopJupyterService(Integer id) throws Exception; } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/DevEnvironmentServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/DevEnvironmentServiceImpl.java index a23a44bd..d5afec8b 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/DevEnvironmentServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/DevEnvironmentServiceImpl.java @@ -4,7 +4,10 @@ import com.ruoyi.common.security.utils.SecurityUtils; import com.ruoyi.platform.domain.DevEnvironment; import com.ruoyi.platform.mapper.DevEnvironmentDao; import com.ruoyi.platform.service.DevEnvironmentService; +import com.ruoyi.platform.service.JupyterService; +import com.ruoyi.platform.utils.JacksonUtil; import com.ruoyi.system.api.model.LoginUser; +import io.kubernetes.client.openapi.models.V1PersistentVolumeClaim; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Service; import org.springframework.data.domain.Page; @@ -13,6 +16,7 @@ import org.springframework.data.domain.PageRequest; import javax.annotation.Resource; import java.util.Date; +import java.util.Map; /** * (DevEnvironment)表服务实现类 @@ -25,6 +29,11 @@ public class DevEnvironmentServiceImpl implements DevEnvironmentService { @Resource private DevEnvironmentDao devEnvironmentDao; + + @Resource + private JupyterService jupyterService; + + /** * 通过ID查询单条数据 * @@ -111,4 +120,6 @@ public class DevEnvironmentServiceImpl implements DevEnvironmentService { devEnvironment.setState(0); return this.devEnvironmentDao.update(devEnvironment)>0?"删除成功":"删除失败"; } + + } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/JupyterServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/JupyterServiceImpl.java index 5b94a350..01a41535 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/JupyterServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/JupyterServiceImpl.java @@ -1,7 +1,12 @@ package com.ruoyi.platform.service.impl; +import com.ruoyi.common.redis.service.RedisService; import com.ruoyi.common.security.utils.SecurityUtils; +import com.ruoyi.platform.domain.DevEnvironment; +import com.ruoyi.platform.mapper.DevEnvironmentDao; +import com.ruoyi.platform.service.DevEnvironmentService; import com.ruoyi.platform.service.JupyterService; +import com.ruoyi.platform.utils.JacksonUtil; import com.ruoyi.platform.utils.K8sClientUtil; import com.ruoyi.platform.utils.MinioUtil; import com.ruoyi.platform.utils.MlflowUtil; @@ -13,6 +18,7 @@ import org.springframework.stereotype.Service; import javax.annotation.Resource; import java.io.InputStream; import java.util.List; +import java.util.Map; @Service public class JupyterServiceImpl implements JupyterService { @@ -39,6 +45,15 @@ public class JupyterServiceImpl implements JupyterService { @Resource private MlflowUtil mlflowUtil; + @Resource + private DevEnvironmentDao devEnvironmentDao; + + @Resource + private DevEnvironmentService devEnvironmentService; + + @Resource + private RedisService redisService; + public JupyterServiceImpl(MinioUtil minioUtil) { this.minioUtil = minioUtil; } @@ -53,6 +68,54 @@ public class JupyterServiceImpl implements JupyterService { return masterIp + ":" + podPort; } + @Override + public String runJupyterService(Integer id) { + DevEnvironment devEnvironment = this.devEnvironmentDao.queryById(id); + String envName = devEnvironment.getName(); + //TODO 设置环境变量 + + // 提取数据集,模型信息，得到数据集模型的path + Map dataset = JacksonUtil.parseJSONStr2Map(devEnvironment.getDataset()); + String datasetPath = (String) dataset.get("path"); + Map model = JacksonUtil.parseJSONStr2Map(devEnvironment.getModel()); + String modelPath = (String) model.get("path"); + + LoginUser loginUser = SecurityUtils.getLoginUser(); + String podName = loginUser.getUsername().toLowerCase() + "-editor-pod"; + String pvcName = loginUser.getUsername().toLowerCase() + "-editor-pvc"; + V1PersistentVolumeClaim pvc = k8sClientUtil.createPvc(namespace, pvcName, storage, storageClassName); + + //TODO 设置镜像可配置，这里先用默认镜像启动pod + + // 调用修改后的 createPod 方法，传入额外的参数 + Integer podPort = k8sClientUtil.createConfiguredPod(podName, namespace, port, mountPath, pvc, image, datasetPath, modelPath); + return masterIp + ":" + podPort; + + + } + + @Override + public String stopJupyterService(Integer id) throws Exception { + DevEnvironment devEnvironment = this.devEnvironmentDao.queryById(id); + if (devEnvironment==null){ + throw new Exception("开发环境配置不存在"); + } + + LoginUser loginUser = SecurityUtils.getLoginUser(); + String podName = loginUser.getUsername().toLowerCase() + "-editor-pod"; + + // 使用 Kubernetes API 删除 Pod + String deleteResult = k8sClientUtil.deletePod(podName, namespace); + + // 检查 Pod 是否存在 + boolean exists = k8sClientUtil.checkPodExists(podName, namespace); + if (exists) { + throw new Exception("Pod " + podName + " 删除失败"); + } + return deleteResult + "，编辑器已停止"; + } + + @Override public void upload(InputStream inputStream) { try { @@ -71,4 +134,6 @@ public class JupyterServiceImpl implements JupyterService { } + + } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/K8sClientUtil.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/K8sClientUtil.java index 5f146711..9a5a2ab4 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/K8sClientUtil.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/K8sClientUtil.java @@ -22,9 +22,7 @@ import org.springframework.stereotype.Component; import javax.annotation.PostConstruct; import java.io.BufferedReader; import java.io.InputStreamReader; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.Map; +import java.util.*; /** * k8s客户端 @@ -282,13 +280,13 @@ public class K8sClientUtil { .endSpec() .build(); - try { pod = api.createNamespacedPod(namespace, pod, null, null, null); } catch (ApiException e) { log.error("创建pod异常:" + e.getResponseBody(), e); } catch (Exception e) { log.error("创建pod系统异常:", e); + } V1Service service = createService(namespace, podName + "-svc", port, selector); @@ -324,7 +322,6 @@ public class K8sClientUtil { for (V1Pod pod1 : v1PodList.getItems()) { if (StringUtils.equals(pod1.getMetadata().getName(), podName)) { // PVC 已存在 - V1Service service = createService(namespace, podName + "-svc", port, selector); if (service != null) { return service.getSpec().getPorts().get(0).getNodePort(); @@ -378,6 +375,73 @@ public class K8sClientUtil { return service.getSpec().getPorts().get(0).getNodePort(); } + + public Integer createConfiguredPod(String podName, String namespace, Integer port, String mountPath, V1PersistentVolumeClaim pvc, String image, String datasetPath, String modelPath) { + Map selector = new LinkedHashMap<>(); + selector.put("k8s-jupyter", podName); + + CoreV1Api api = new CoreV1Api(apiClient); + V1PodList v1PodList = null; + try { + v1PodList = api.listNamespacedPod(namespace, null, null, null, null, null, null, null, null, null, null); + } catch (ApiException e) { + log.error("获取 POD 异常:", e); + } + if (v1PodList != null) { + for (V1Pod pod1 : v1PodList.getItems()) { + // PVC 已存在 + if (StringUtils.equals(pod1.getMetadata().getName(), podName)) { + V1Service service = createService(namespace, podName + "-svc", port, selector); + if (service != null) { + return service.getSpec().getPorts().get(0).getNodePort(); + } + } + } + } + + // 配置卷和卷挂载 + List volumeMounts = new ArrayList<>(); + volumeMounts.add(new V1VolumeMount().name("workspace").mountPath(mountPath)); + volumeMounts.add(new V1VolumeMount().name("dataset").mountPath("/datasets").subPath(datasetPath).readOnly(true)); + volumeMounts.add(new V1VolumeMount().name("model").mountPath("/model").subPath(modelPath).readOnly(true)); + + List volumes = new ArrayList<>(); + volumes.add(new V1Volume().name("workspace").persistentVolumeClaim(new V1PersistentVolumeClaimVolumeSource().claimName(pvc.getMetadata().getName()))); + volumes.add(new V1Volume().name("dataset").persistentVolumeClaim(new V1PersistentVolumeClaimVolumeSource().claimName(pvc.getMetadata().getName()))); + volumes.add(new V1Volume().name("model").persistentVolumeClaim(new V1PersistentVolumeClaimVolumeSource().claimName(pvc.getMetadata().getName()))); + + V1Pod pod = new V1PodBuilder() + .withNewMetadata() + .withName(podName) + .withLabels(selector) + .endMetadata() + .withNewSpec() + .addNewContainer() + .withName(podName) + .withImage(image) + .withPorts(new V1ContainerPort().containerPort(port).protocol("TCP")) + .withVolumeMounts(volumeMounts) + .endContainer() + .withVolumes(volumes) + .withTerminationGracePeriodSeconds(14400L) + .endSpec() + .build(); + + try { + pod = api.createNamespacedPod(namespace, pod, null, null, null); + } catch (ApiException e) { + log.error("创建pod异常:" + e.getResponseBody(), e); + } catch (Exception e) { + log.error("创建pod系统异常:", e); + } + + V1Service service = createService(namespace, podName + "-svc", port, selector); + return service.getSpec().getPorts().get(0).getNodePort(); + } + + + + /** * 根据获取namespace,deploymentName的Pod Name * @@ -495,4 +559,51 @@ public class K8sClientUtil { } return pod; } + + /** + * 删除 Pod + * + * @param podName Pod 名称 + * @param namespace 命名空间 + * @throws ApiException 异常 + */ + public String deletePod(String podName, String namespace) throws ApiException { + CoreV1Api api = new CoreV1Api(apiClient); + try { + V1Pod pod = api.deleteNamespacedPod(podName, namespace, null, null, null, null, null, null); + return "Pod " + podName + " 删除请求已发送"; + } catch (ApiException e) { + if (e.getCode() == 404) { + return "Pod " + podName + " 不存在"; + } else { + log.error("删除pod异常:" + e.getResponseBody(), e); + throw e; + } + } + } + + /** + * 检查 Pod 是否存在 + * + * @param podName Pod 名称 + * @param namespace 命名空间 + * @return 是否存在 + * @throws ApiException 异常 + */ + public boolean checkPodExists(String podName, String namespace) throws ApiException { + CoreV1Api api = new CoreV1Api(apiClient); + try { + api.readNamespacedPod(podName, namespace, null,false,false); + return true; + } catch (ApiException e) { + if (e.getCode() == 404) { + return false; + } else { + log.error("检查pod存在性异常:" + e.getResponseBody(), e); + throw e; + } + } + } + + }