From ea3c20e3bb023e48c136b8a62106876a9f717543 Mon Sep 17 00:00:00 2001 From: chenzhihang <709011834@qq.com> Date: Tue, 29 Oct 2024 13:40:49 +0800 Subject: [PATCH 01/18] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E8=87=AA=E5=8A=A8?= =?UTF-8?q?=E6=9C=BA=E5=99=A8=E5=AD=A6=E4=B9=A0=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../controller/autoML/AutoMLController.java | 35 +++++++++++++++++++ .../codeConfig/CodeConfigController.java | 1 - .../com/ruoyi/platform/domain/AutoMl.java | 31 ++++++++++++++++ .../com/ruoyi/platform/mapper/AutoMLDao.java | 13 +++++++ .../ruoyi/platform/service/AutoMLService.java | 9 +++++ .../service/impl/AutoMLServiceImpl.java | 25 +++++++++++++ .../managementPlatform/AutoMLDaoMapper.xml | 23 ++++++++++++ 7 files changed, 136 insertions(+), 1 deletion(-) create mode 100644 ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/autoML/AutoMLController.java create mode 100644 ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/AutoMl.java create mode 100644 ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/AutoMLDao.java create mode 100644 ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AutoMLService.java create mode 100644 ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMLServiceImpl.java create mode 100644 ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMLDaoMapper.xml diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/autoML/AutoMLController.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/autoML/AutoMLController.java new file mode 100644 index 00000000..48e666fd --- /dev/null +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/autoML/AutoMLController.java @@ -0,0 +1,35 @@ +package com.ruoyi.platform.controller.autoML; + +import com.ruoyi.common.core.web.controller.BaseController; +import com.ruoyi.common.core.web.domain.GenericsAjaxResult; +import com.ruoyi.platform.domain.AutoMl; +import com.ruoyi.platform.service.AutoMLService; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageRequest; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import javax.annotation.Resource; + +@RestController +@RequestMapping("autoML") +@Api("自动机器学习") +public class AutoMLController extends BaseController { + + @Resource + private AutoMLService autoMLService; + + @GetMapping + @ApiOperation("分页查询") + public GenericsAjaxResult> queryByPage(@RequestParam("page") int page, + @RequestParam("size") int size, + @RequestParam(value = "mlName", required = false) String mlName) { + PageRequest pageRequest = PageRequest.of(page, size); + return genericsSuccess(this.autoMLService.queryByPage(mlName, pageRequest)); + + } +} diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/codeConfig/CodeConfigController.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/codeConfig/CodeConfigController.java index 3aa6c37f..b445a232 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/codeConfig/CodeConfigController.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/codeConfig/CodeConfigController.java @@ -7,7 +7,6 @@ import com.ruoyi.platform.service.CodeConfigService; import io.swagger.annotations.Api; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageRequest; -import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.*; import javax.annotation.Resource; diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/AutoMl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/AutoMl.java new file mode 100644 index 00000000..409e5726 --- /dev/null +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/AutoMl.java @@ -0,0 +1,31 @@ +package com.ruoyi.platform.domain; + +import com.fasterxml.jackson.databind.PropertyNamingStrategy; +import com.fasterxml.jackson.databind.annotation.JsonNaming; +import lombok.Data; + +import java.util.Date; + +@Data +@JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy.class) +public class AutoMl { + private Long id; + + private String mlName; + + private String mlDescription; + + private Integer state; + + private String runState; + + private Double progress; + + private String createBy; + + private Date createTime; + + private String updateBy; + + private Date updateTime; +} diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/AutoMLDao.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/AutoMLDao.java new file mode 100644 index 00000000..3334999b --- /dev/null +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/AutoMLDao.java @@ -0,0 +1,13 @@ +package com.ruoyi.platform.mapper; + +import com.ruoyi.platform.domain.AutoMl; +import org.apache.ibatis.annotations.Param; +import org.springframework.data.domain.Pageable; + +import java.util.List; + +public interface AutoMLDao { + long count(@Param("mlName") String mlName); + + List queryByPage(@Param("mlName") String mlName, @Param("pageable") Pageable pageable); +} diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AutoMLService.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AutoMLService.java new file mode 100644 index 00000000..37de7921 --- /dev/null +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AutoMLService.java @@ -0,0 +1,9 @@ +package com.ruoyi.platform.service; + +import com.ruoyi.platform.domain.AutoMl; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageRequest; + +public interface AutoMLService { + Page queryByPage(String mlName, PageRequest pageRequest); +} diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMLServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMLServiceImpl.java new file mode 100644 index 00000000..4fe834ff --- /dev/null +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMLServiceImpl.java @@ -0,0 +1,25 @@ +package com.ruoyi.platform.service.impl; + +import com.ruoyi.platform.domain.AutoMl; +import com.ruoyi.platform.mapper.AutoMLDao; +import com.ruoyi.platform.service.AutoMLService; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageImpl; +import org.springframework.data.domain.PageRequest; +import org.springframework.stereotype.Service; + +import javax.annotation.Resource; +import java.util.List; + +@Service("autoMLService") +public class AutoMLServiceImpl implements AutoMLService { + @Resource + private AutoMLDao autoMLDao; + + @Override + public Page queryByPage(String mlName, PageRequest pageRequest) { + long total = autoMLDao.count(mlName); + List autoMls = autoMLDao.queryByPage(mlName, pageRequest); + return new PageImpl<>(autoMls, pageRequest, total); + } +} diff --git a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMLDaoMapper.xml b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMLDaoMapper.xml new file mode 100644 index 00000000..9d39b807 --- /dev/null +++ b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMLDaoMapper.xml @@ -0,0 +1,23 @@ + + + + + + + + + + + state = 1 + + and ml_name like concat('%', #{mlName}, '%') + + + + \ No newline at end of file From abd5c094740817fbaecd0e8aafd2fe1fcbec9837 Mon Sep 17 00:00:00 2001 From: chenzhihang <709011834@qq.com> Date: Wed, 30 Oct 2024 10:15:29 +0800 Subject: [PATCH 02/18] =?UTF-8?q?=E8=87=AA=E5=8A=A8=E6=9C=BA=E5=99=A8?= =?UTF-8?q?=E5=AD=A6=E4=B9=A0=E6=8E=A5=E5=8F=A3=E5=BC=80=E5=8F=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../controller/autoML/AutoMLController.java | 23 ++++++++++--- .../com/ruoyi/platform/mapper/AutoMLDao.java | 6 ++++ .../ruoyi/platform/service/AutoMLService.java | 6 ++++ .../service/impl/AutoMLServiceImpl.java | 34 +++++++++++++++++++ .../src/main/resources/bootstrap.yml | 10 +++--- .../managementPlatform/AutoMLDaoMapper.xml | 30 ++++++++++++++++ 6 files changed, 100 insertions(+), 9 deletions(-) diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/autoML/AutoMLController.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/autoML/AutoMLController.java index 48e666fd..8b0c9a74 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/autoML/AutoMLController.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/autoML/AutoMLController.java @@ -8,10 +8,7 @@ import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageRequest; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.bind.annotation.*; import javax.annotation.Resource; @@ -30,6 +27,24 @@ public class AutoMLController extends BaseController { @RequestParam(value = "mlName", required = false) String mlName) { PageRequest pageRequest = PageRequest.of(page, size); return genericsSuccess(this.autoMLService.queryByPage(mlName, pageRequest)); + } + + @PostMapping + @ApiOperation("新增自动机器学习") + public GenericsAjaxResult addAutoMl(@RequestBody AutoMl autoMl) { + return genericsSuccess(this.autoMLService.save(autoMl)); + } + @PutMapping + @ApiOperation("编辑自动机器学习") + public GenericsAjaxResult editAutoMl(@RequestBody AutoMl autoMl){ + return genericsSuccess(this.autoMLService.edit(autoMl)); } + + @DeleteMapping("{id}") + @ApiOperation("删除自动机器学习") + public GenericsAjaxResult deleteAutoMl(@PathVariable("id") Long id){ + return genericsSuccess(this.autoMLService.delete(id)); + } + } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/AutoMLDao.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/AutoMLDao.java index 3334999b..ed68d8a9 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/AutoMLDao.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/AutoMLDao.java @@ -10,4 +10,10 @@ public interface AutoMLDao { long count(@Param("mlName") String mlName); List queryByPage(@Param("mlName") String mlName, @Param("pageable") Pageable pageable); + + AutoMl getAutoById(@Param("id") Long id); + + int save(@Param("autoMl") AutoMl autoMl); + + int edit(@Param("autoMl") AutoMl autoMl); } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AutoMLService.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AutoMLService.java index 37de7921..ce06b9c4 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AutoMLService.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AutoMLService.java @@ -6,4 +6,10 @@ import org.springframework.data.domain.PageRequest; public interface AutoMLService { Page queryByPage(String mlName, PageRequest pageRequest); + + AutoMl save(AutoMl autoMl); + + String edit(AutoMl autoMl); + + String delete(Long id); } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMLServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMLServiceImpl.java index 4fe834ff..4504c4f7 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMLServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMLServiceImpl.java @@ -1,8 +1,12 @@ package com.ruoyi.platform.service.impl; +import com.ruoyi.common.security.utils.SecurityUtils; +import com.ruoyi.platform.constant.Constant; import com.ruoyi.platform.domain.AutoMl; import com.ruoyi.platform.mapper.AutoMLDao; import com.ruoyi.platform.service.AutoMLService; +import com.ruoyi.system.api.model.LoginUser; +import org.apache.commons.lang3.StringUtils; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageImpl; import org.springframework.data.domain.PageRequest; @@ -22,4 +26,34 @@ public class AutoMLServiceImpl implements AutoMLService { List autoMls = autoMLDao.queryByPage(mlName, pageRequest); return new PageImpl<>(autoMls, pageRequest, total); } + + @Override + public AutoMl save(AutoMl autoMl) { + autoMLDao.save(autoMl); + return autoMl; + } + + @Override + public String edit(AutoMl autoMl) { + autoMLDao.edit(autoMl); + return "修改成功"; + } + + @Override + public String delete(Long id) { + AutoMl autoMl = autoMLDao.getAutoById(id); + if (autoMl == null) { + throw new RuntimeException("服务不存在"); + } + + LoginUser loginUser = SecurityUtils.getLoginUser(); + String username = loginUser.getUsername(); + String createBy = autoMl.getCreateBy(); + if (!(StringUtils.equals(username, "admin") || StringUtils.equals(username, createBy))) { + throw new RuntimeException("无权限删除该服务"); + } + + autoMl.setState(Constant.State_invalid); + return autoMLDao.edit(autoMl) > 0 ? "删除成功" : "删除失败"; + } } diff --git a/ruoyi-modules/management-platform/src/main/resources/bootstrap.yml b/ruoyi-modules/management-platform/src/main/resources/bootstrap.yml index 22eaf828..29dd7cd4 100644 --- a/ruoyi-modules/management-platform/src/main/resources/bootstrap.yml +++ b/ruoyi-modules/management-platform/src/main/resources/bootstrap.yml @@ -14,16 +14,16 @@ spring: nacos: discovery: # 服务注册地址 - server-addr: nacos-ci4s.argo.svc:8848 + server-addr: 172.20.32.181:18848 username: nacos - password: h1n2x3j4y5@ + password: nacos retry: enabled: true +# namespace: 6caf5d79-c4ce-4e3b-a357-141b74e52a01 config: - username: nacos - password: h1n2x3j4y5@ +# namespace: 6caf5d79-c4ce-4e3b-a357-141b74e52a01 # 配置中心地址 - server-addr: nacos-ci4s.argo.svc:8848 + server-addr: 172.20.32.181:18848 # 配置文件格式 file-extension: yml # 共享配置 diff --git a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMLDaoMapper.xml b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMLDaoMapper.xml index 9d39b807..788c2dec 100644 --- a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMLDaoMapper.xml +++ b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMLDaoMapper.xml @@ -1,6 +1,32 @@ + + insert into auto_ml(ml_name, ml_description) + values (#{autoMl.mlName}, #{autoMl.mlDescription}) + + + + update auto_ml + + + ml_name = #{autoMl.mlName}, + + + ml_description = #{autoMl.mlDescription}, + + + run_state = #{autoMl.runState}, + + + progress = #{autoMl.progress}, + + + state = #{autoMl.state}, + + + where id = #{autoMl.id} + + + state = 1 From fe9e9ed2caadd59434e6ce72d33ca42b4b625fcf Mon Sep 17 00:00:00 2001 From: chenzhihang <709011834@qq.com> Date: Sat, 16 Nov 2024 16:21:55 +0800 Subject: [PATCH 03/18] =?UTF-8?q?=E8=87=AA=E5=8A=A8=E6=9C=BA=E5=99=A8?= =?UTF-8?q?=E5=AD=A6=E4=B9=A0=E5=BC=80=E5=8F=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../controller/autoML/AutoMLController.java | 48 +++- .../com/ruoyi/platform/domain/AutoMlIns.java | 77 ++++++ .../com/ruoyi/platform/mapper/AutoMLDao.java | 2 +- .../ruoyi/platform/mapper/AutoMLInsDao.java | 21 ++ .../ruoyi/platform/service/AutoMLService.java | 19 +- .../service/impl/AutoMLServiceImpl.java | 223 +++++++++++++++++- .../service/impl/JupyterServiceImpl.java | 2 - .../managementPlatform/AutoMLDaoMapper.xml | 2 +- .../managementPlatform/AutoMLInsDaoMapper.xml | 132 +++++++++++ 9 files changed, 514 insertions(+), 12 deletions(-) create mode 100644 ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/AutoMlIns.java create mode 100644 ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/AutoMLInsDao.java create mode 100644 ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMLInsDaoMapper.xml diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/autoML/AutoMLController.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/autoML/AutoMLController.java index 8b0c9a74..47e291ee 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/autoML/AutoMLController.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/autoML/AutoMLController.java @@ -1,14 +1,17 @@ package com.ruoyi.platform.controller.autoML; import com.ruoyi.common.core.web.controller.BaseController; +import com.ruoyi.common.core.web.domain.AjaxResult; import com.ruoyi.common.core.web.domain.GenericsAjaxResult; import com.ruoyi.platform.domain.AutoMl; +import com.ruoyi.platform.domain.AutoMlIns; import com.ruoyi.platform.service.AutoMLService; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageRequest; import org.springframework.web.bind.annotation.*; +import org.springframework.web.multipart.MultipartFile; import javax.annotation.Resource; @@ -24,7 +27,7 @@ public class AutoMLController extends BaseController { @ApiOperation("分页查询") public GenericsAjaxResult> queryByPage(@RequestParam("page") int page, @RequestParam("size") int size, - @RequestParam(value = "mlName", required = false) String mlName) { + @RequestParam(value = "ml_name", required = false) String mlName) { PageRequest pageRequest = PageRequest.of(page, size); return genericsSuccess(this.autoMLService.queryByPage(mlName, pageRequest)); } @@ -37,14 +40,53 @@ public class AutoMLController extends BaseController { @PutMapping @ApiOperation("编辑自动机器学习") - public GenericsAjaxResult editAutoMl(@RequestBody AutoMl autoMl){ + public GenericsAjaxResult editAutoMl(@RequestBody AutoMl autoMl) { return genericsSuccess(this.autoMLService.edit(autoMl)); } @DeleteMapping("{id}") @ApiOperation("删除自动机器学习") - public GenericsAjaxResult deleteAutoMl(@PathVariable("id") Long id){ + public GenericsAjaxResult deleteAutoMl(@PathVariable("id") Long id) { return genericsSuccess(this.autoMLService.delete(id)); } + @GetMapping("/autoMlIns") + @ApiOperation("分页查询自动机器学习实例") + public GenericsAjaxResult> queryByPage(@RequestParam("page") int page, + @RequestParam("size") int size, + @RequestParam("auto_ml_id") Long autoMlId) { + PageRequest pageRequest = PageRequest.of(page, size); + return genericsSuccess(this.autoMLService.queryAutoMlInsByPage(autoMlId, pageRequest)); + } + + @PostMapping("/autoMlIns") + @ApiOperation("新增自动机器学习实例") + public GenericsAjaxResult addAutoMlIns(@RequestBody AutoMlIns autoMlIns) throws Exception { + return genericsSuccess(this.autoMLService.saveAutoMlIns(autoMlIns)); + } + + @PutMapping("/autoMlIns") + @ApiOperation("编辑自动机器学习实例") + public GenericsAjaxResult editAutoMlIns(@RequestBody AutoMlIns autoMlIns) throws Exception { + return genericsSuccess(this.autoMLService.editAutoMlIns(autoMlIns)); + } + + @DeleteMapping("/autoMlIns/{id}") + @ApiOperation("删除自动机器学习实例") + public GenericsAjaxResult deleteAutoMlIns(@PathVariable("id") Long id) { + return genericsSuccess(this.autoMLService.deleteAutoMlIns(id)); + } + + @CrossOrigin(origins = "*", allowedHeaders = "*") + @PostMapping("/upload") + @ApiOperation(value = "上传数据文件csv", notes = "上传数据文件csv，并将信息存入数据库。") + public AjaxResult upload(@RequestParam("file") MultipartFile file, @RequestParam("uuid") String uuid) throws Exception { + return AjaxResult.success(this.autoMLService.upload(file, uuid)); + } + + @PostMapping("{id}") + @ApiOperation("运行自动机器学习实验") + public GenericsAjaxResult runAutoML(@PathVariable("id") Long id) throws Exception { + return genericsSuccess(this.autoMLService.runAutoMlIns(id)); + } } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/AutoMlIns.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/AutoMlIns.java new file mode 100644 index 00000000..653c212d --- /dev/null +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/AutoMlIns.java @@ -0,0 +1,77 @@ +package com.ruoyi.platform.domain; + +import com.baomidou.mybatisplus.annotation.TableField; +import com.fasterxml.jackson.databind.PropertyNamingStrategy; +import com.fasterxml.jackson.databind.annotation.JsonNaming; +import com.ruoyi.platform.vo.VersionVo; +import lombok.Data; + +import java.util.Date; + +@Data +@JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy.class) +public class AutoMlIns { + private Long id; + + private Long autoMlId; + + private String taskType; + + private String datasetName; + + private Integer timeLeftForThisTask; + + private Integer perRunTimeLimit; + + private Integer ensembleSize; + + private String ensembleClass; + + private Integer ensembleNbest; + + private Integer maxModelsOnDisc; + + private Integer seed; + + private Integer memoryLimit; + + private String includeClassifier; + + private String includeFeaturePreprocessor; + + private String includeRegressor; + + private String excludeClassifier; + + private String excludeRegressor; + + private String excludeFeaturePreprocessor; + + private String resamplingStrategy; + + private Float trainSize; + + private Boolean shuffle; + + private Integer folds; + + private Boolean deleteTmpFolderAfterTerminate; + + private String dataCsv; + + private String targetColumns; + + private Integer state; + + private String createBy; + + private String updateBy; + + private Date createTime; + + private Date updateTime; + + @TableField(exist = false) + private VersionVo versionVo; + +} diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/AutoMLDao.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/AutoMLDao.java index ed68d8a9..b10f5235 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/AutoMLDao.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/AutoMLDao.java @@ -11,7 +11,7 @@ public interface AutoMLDao { List queryByPage(@Param("mlName") String mlName, @Param("pageable") Pageable pageable); - AutoMl getAutoById(@Param("id") Long id); + AutoMl getAutoMlById(@Param("id") Long id); int save(@Param("autoMl") AutoMl autoMl); diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/AutoMLInsDao.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/AutoMLInsDao.java new file mode 100644 index 00000000..a541fc53 --- /dev/null +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/AutoMLInsDao.java @@ -0,0 +1,21 @@ +package com.ruoyi.platform.mapper; + +import com.ruoyi.platform.domain.AutoMlIns; +import org.apache.ibatis.annotations.Param; +import org.springframework.data.domain.Pageable; + +import java.util.List; + +public interface AutoMLInsDao { + long count(@Param("autoMlId") Long autoMlId); + + List queryByPage(@Param("autoMlId") Long autoMlId, @Param("pageable") Pageable pageable); + + int save(@Param("autoMlIns") AutoMlIns autoMlIns); + + int edit(@Param("autoMlIns") AutoMlIns autoMlIns); + + AutoMlIns getById(@Param("id") Long id); + + AutoMlIns findByDatasetName(@Param("datasetName") String datasetName, @Param("autoMlId") Long autoMlId); +} diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AutoMLService.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AutoMLService.java index ce06b9c4..d475d5a3 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AutoMLService.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AutoMLService.java @@ -1,15 +1,32 @@ package com.ruoyi.platform.service; import com.ruoyi.platform.domain.AutoMl; +import com.ruoyi.platform.domain.AutoMlIns; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageRequest; +import org.springframework.web.multipart.MultipartFile; + +import java.util.List; +import java.util.Map; public interface AutoMLService { - Page queryByPage(String mlName, PageRequest pageRequest); + Page queryByPage(String mlName, PageRequest pageRequest); AutoMl save(AutoMl autoMl); String edit(AutoMl autoMl); String delete(Long id); + + Page queryAutoMlInsByPage(Long autoMlId, PageRequest pageRequest); + + AutoMlIns saveAutoMlIns(AutoMlIns autoMlIns) throws Exception; + + String editAutoMlIns(AutoMlIns autoMlIns) throws Exception; + + String deleteAutoMlIns(Long id); + + Map upload(MultipartFile file, String uuid) throws Exception; + + String runAutoMlIns(Long id) throws Exception; } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMLServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMLServiceImpl.java index 4504c4f7..19bb878b 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMLServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMLServiceImpl.java @@ -3,22 +3,57 @@ package com.ruoyi.platform.service.impl; import com.ruoyi.common.security.utils.SecurityUtils; import com.ruoyi.platform.constant.Constant; import com.ruoyi.platform.domain.AutoMl; +import com.ruoyi.platform.domain.AutoMlIns; import com.ruoyi.platform.mapper.AutoMLDao; +import com.ruoyi.platform.mapper.AutoMLInsDao; import com.ruoyi.platform.service.AutoMLService; -import com.ruoyi.system.api.model.LoginUser; +import com.ruoyi.platform.utils.DVCUtils; +import com.ruoyi.platform.utils.FileUtil; +import com.ruoyi.platform.utils.K8sClientUtil; +import io.kubernetes.client.openapi.models.V1Pod; +import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageImpl; import org.springframework.data.domain.PageRequest; import org.springframework.stereotype.Service; +import org.springframework.web.multipart.MultipartFile; import javax.annotation.Resource; +import java.io.File; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; @Service("autoMLService") public class AutoMLServiceImpl implements AutoMLService { + @Value("${harbor.serviceNS}") + private String serviceNS; + @Value("${dockerpush.proxyUrl}") + private String proxyUrl; + @Value("${dockerpush.mountPath}") + private String mountPath; + @Value("${minio.pvcName}") + private String pvcName; + @Value("${automl.image}") + private String image; + @Value("${git.localPath}") + String localPath; + + private static final Logger logger = LoggerFactory.getLogger(ModelsServiceImpl.class); + @Resource private AutoMLDao autoMLDao; + @Resource + private AutoMLInsDao autoMLInsDao; + @Resource + private K8sClientUtil k8sClientUtil; + @Resource + private DVCUtils dvcUtils; @Override public Page queryByPage(String mlName, PageRequest pageRequest) { @@ -29,25 +64,29 @@ public class AutoMLServiceImpl implements AutoMLService { @Override public AutoMl save(AutoMl autoMl) { + String username = SecurityUtils.getLoginUser().getUsername(); + autoMl.setCreateBy(username); + autoMl.setUpdateBy(username); autoMLDao.save(autoMl); return autoMl; } @Override public String edit(AutoMl autoMl) { + String username = SecurityUtils.getLoginUser().getUsername(); + autoMl.setUpdateBy(username); autoMLDao.edit(autoMl); return "修改成功"; } @Override public String delete(Long id) { - AutoMl autoMl = autoMLDao.getAutoById(id); + AutoMl autoMl = autoMLDao.getAutoMlById(id); if (autoMl == null) { throw new RuntimeException("服务不存在"); } - LoginUser loginUser = SecurityUtils.getLoginUser(); - String username = loginUser.getUsername(); + String username = SecurityUtils.getLoginUser().getUsername(); String createBy = autoMl.getCreateBy(); if (!(StringUtils.equals(username, "admin") || StringUtils.equals(username, createBy))) { throw new RuntimeException("无权限删除该服务"); @@ -56,4 +95,180 @@ public class AutoMLServiceImpl implements AutoMLService { autoMl.setState(Constant.State_invalid); return autoMLDao.edit(autoMl) > 0 ? "删除成功" : "删除失败"; } + + @Override + public Page queryAutoMlInsByPage(Long autoMlId, PageRequest pageRequest) { + long total = autoMLInsDao.count(autoMlId); + List autoMlIns = autoMLInsDao.queryByPage(autoMlId, pageRequest); + return new PageImpl<>(autoMlIns, pageRequest, total); + } + + @Override + public AutoMlIns saveAutoMlIns(AutoMlIns autoMlIns) throws Exception { + String ci4sUsername = SecurityUtils.getLoginUser().getUsername(); + AutoMlIns oldAutoMlIns = autoMLInsDao.findByDatasetName(autoMlIns.getDatasetName(), autoMlIns.getAutoMlId()); + if (oldAutoMlIns != null) { + throw new RuntimeException("数据集名称已存在"); + } + + String sourcePath = autoMlIns.getVersionVo().getUrl(); + String rootPath = localPath + ci4sUsername + "/automl/" + autoMlIns.getAutoMlId() + "/" + autoMlIns.getDatasetName(); + dvcUtils.moveFiles(sourcePath, rootPath); + autoMlIns.setDataCsv(rootPath + "/" + autoMlIns.getVersionVo().getFileName()); + autoMlIns.setCreateBy(ci4sUsername); + autoMlIns.setUpdateBy(ci4sUsername); + autoMLInsDao.save(autoMlIns); + return autoMlIns; + } + + @Override + public String editAutoMlIns(AutoMlIns autoMlIns) throws Exception { + AutoMlIns oldAutoMlIns = autoMLInsDao.findByDatasetName(autoMlIns.getDatasetName(), autoMlIns.getAutoMlId()); + if (oldAutoMlIns != null && !oldAutoMlIns.getId().equals(autoMlIns.getId())) { + throw new RuntimeException("数据集名称已存在"); + } + String ci4sUsername = SecurityUtils.getLoginUser().getUsername(); + autoMlIns.setUpdateBy(ci4sUsername); + if (autoMlIns.getVersionVo() != null && StringUtils.isNotEmpty(autoMlIns.getVersionVo().getUrl())) { + String sourcePath = autoMlIns.getVersionVo().getUrl(); + String rootPath = localPath + ci4sUsername + "/automl/" + autoMlIns.getAutoMlId() + "/" + autoMlIns.getDatasetName(); + dvcUtils.moveFiles(sourcePath, rootPath); + autoMlIns.setDataCsv(rootPath + "/" + autoMlIns.getVersionVo().getFileName()); + } + autoMLInsDao.edit(autoMlIns); + return "修改成功"; + } + + @Override + public String deleteAutoMlIns(Long id) { + AutoMlIns autoMlIns = autoMLInsDao.getById(id); + if (autoMlIns == null) { + throw new RuntimeException("实例不存在"); + } + String username = SecurityUtils.getLoginUser().getUsername(); + String createBy = autoMlIns.getCreateBy(); + if (!(StringUtils.equals(username, "admin") || StringUtils.equals(username, createBy))) { + throw new RuntimeException("无权限删除该实例"); + } + autoMlIns.setState(Constant.State_invalid); + return autoMLInsDao.edit(autoMlIns) > 0 ? "删除成功" : "删除失败"; + } + + @Override + public Map upload(MultipartFile file, String uuid) throws Exception { + Map result = new HashMap<>(); + + String username = SecurityUtils.getLoginUser().getUsername(); + String fileName = file.getOriginalFilename(); + String path = localPath + "temp/" + username + "/automl_data/" + uuid; + long sizeInBytes = file.getSize(); + String formattedSize = FileUtil.formatFileSize(sizeInBytes); + File targetFile = new File(path, file.getOriginalFilename()); + // 确保目录存在 + targetFile.getParentFile().mkdirs(); + // 保存文件到目标路径 + FileUtils.copyInputStreamToFile(file.getInputStream(), targetFile); + // 返回上传文件的路径 + result.put("fileName", fileName); + result.put("url", path); // objectName根据实际情况定义 + result.put("fileSize", formattedSize); + return result; + } + + @Override + public String runAutoMlIns(Long id) throws Exception { + AutoMlIns autoMlIns = autoMLInsDao.getById(id); + if (autoMlIns == null) { + throw new Exception("开发环境配置不存在"); + } + + StringBuffer command = new StringBuffer(); + command.append("nohup python /opt/automl.py --task_type " + autoMlIns.getTaskType()); + if (StringUtils.isNotEmpty(autoMlIns.getDataCsv())) { + command.append(" --data_csv " + autoMlIns.getDataCsv()); + } else { + throw new Exception("训练数据为空"); + } + if (StringUtils.isNotEmpty(autoMlIns.getTargetColumns())) { + command.append(" --target_columns " + autoMlIns.getTargetColumns()); + } else { + throw new Exception("目标列为空"); + } + if (StringUtils.isNotEmpty(autoMlIns.getDatasetName())) { + command.append(" --dataset_name " + autoMlIns.getDatasetName()); + } else { + throw new Exception("数据集名称为空"); + } + +// String username = SecurityUtils.getLoginUser().getUsername().toLowerCase(); + String username = "admin"; + //构造pod名称 + String podName = username + "-autoMlIns-pod-" + id; + V1Pod pod = k8sClientUtil.createPodWithEnv(podName, serviceNS, proxyUrl, mountPath, pvcName, image); + + if (autoMlIns.getTimeLeftForThisTask() != null) { + command.append(" --time_left_for_this_task " + autoMlIns.getTimeLeftForThisTask()); + } + if (autoMlIns.getPerRunTimeLimit() != null) { + command.append(" --per_run_time_limit " + autoMlIns.getPerRunTimeLimit()); + } + if (autoMlIns.getEnsembleSize() != null) { + command.append(" --ensemble_size " + autoMlIns.getEnsembleSize()); + } + if (StringUtils.isNotEmpty(autoMlIns.getEnsembleClass())) { + command.append(" --ensemble_class " + autoMlIns.getEnsembleClass()); + } + if (autoMlIns.getEnsembleNbest() != null) { + command.append(" --ensemble_nbest " + autoMlIns.getEnsembleNbest()); + } + if (autoMlIns.getMaxModelsOnDisc() != null) { + command.append(" --max_models_on_disc " + autoMlIns.getMaxModelsOnDisc()); + } + if (autoMlIns.getSeed() != null) { + command.append(" --seed " + autoMlIns.getSeed()); + } + if (autoMlIns.getMemoryLimit() != null) { + command.append(" --memory_limit " + autoMlIns.getMemoryLimit()); + } + if (StringUtils.isNotEmpty(autoMlIns.getIncludeClassifier())) { + command.append(" --include_classifier " + autoMlIns.getIncludeClassifier()); + } + if (StringUtils.isNotEmpty(autoMlIns.getIncludeRegressor())) { + command.append(" --include_regressor " + autoMlIns.getIncludeRegressor()); + } + if (StringUtils.isNotEmpty(autoMlIns.getIncludeFeaturePreprocessor())) { + command.append(" --include_feature_preprocessor " + autoMlIns.getIncludeFeaturePreprocessor()); + } + if (StringUtils.isNotEmpty(autoMlIns.getExcludeClassifier())) { + command.append(" --exclude_classifier " + autoMlIns.getExcludeClassifier()); + } + if (StringUtils.isNotEmpty(autoMlIns.getExcludeRegressor())) { + command.append(" --exclude_regressor " + autoMlIns.getExcludeRegressor()); + } + if (StringUtils.isNotEmpty(autoMlIns.getExcludeFeaturePreprocessor())) { + command.append(" --exclude_feature_preprocessor " + autoMlIns.getExcludeFeaturePreprocessor()); + } + if (StringUtils.isNotEmpty(autoMlIns.getResamplingStrategy())) { + command.append(" --resampling_strategy " + autoMlIns.getResamplingStrategy()); + } + if (autoMlIns.getTrainSize() != null) { + command.append(" --train_size " + autoMlIns.getTrainSize()); + } + if (autoMlIns.getShuffle() != null) { + command.append(" --shuffle " + autoMlIns.getShuffle()); + } + if (autoMlIns.getFolds() != null) { + command.append(" --folds " + autoMlIns.getFolds()); + } + command.append(" &"); + CompletableFuture.supplyAsync(() -> { + try { + String log = k8sClientUtil.executeCommand(pod, String.valueOf(command)); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + return null; + }); + return "执行成功"; + } } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/JupyterServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/JupyterServiceImpl.java index 87111365..02cb59a2 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/JupyterServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/JupyterServiceImpl.java @@ -104,8 +104,6 @@ public class JupyterServiceImpl implements JupyterService { // String pvcName = loginUser.getUsername().toLowerCase() + "-editor-pvc"; // V1PersistentVolumeClaim pvc = k8sClientUtil.createPvc(namespace, pvcName, storage, storageClassName); - //TODO 设置镜像可配置，这里先用默认镜像启动pod - // 调用修改后的 createPod 方法，传入额外的参数 // Integer podPort = k8sClientUtil.createConfiguredPod(podName, namespace, port, mountPath, pvc, devEnvironment, minioPvcName, datasetPath, modelPath); Integer podPort = k8sClientUtil.createConfiguredPod(podName, namespace, port, mountPath, null, devEnvironment, minioPvcName, datasetPath, modelPath); diff --git a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMLDaoMapper.xml b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMLDaoMapper.xml index 788c2dec..8f4bc742 100644 --- a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMLDaoMapper.xml +++ b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMLDaoMapper.xml @@ -38,7 +38,7 @@ - select * from auto_ml where id = #{id} diff --git a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMLInsDaoMapper.xml b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMLInsDaoMapper.xml new file mode 100644 index 00000000..7f85a9d5 --- /dev/null +++ b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMLInsDaoMapper.xml @@ -0,0 +1,132 @@ + + + + + insert into auto_ml_ins(auto_ml_id, task_type, dataset_name, time_left_for_this_task, per_run_time_limit, + ensemble_size, ensemble_class, ensemble_nbest, max_models_on_disc, seed, memory_limit, + include_classifier, include_feature_preprocessor, include_regressor, exclude_classifier, + exclude_regressor, exclude_feature_preprocessor, resampling_strategy, train_size, + shuffle, folds, delete_tmp_folder_after_terminate, data_csv, target_columns, create_by, + update_by) + values (#{autoMlIns.autoMlId}, #{autoMlIns.taskType}, #{autoMlIns.datasetName}, + #{autoMlIns.timeLeftForThisTask}, #{autoMlIns.perRunTimeLimit}, + #{autoMlIns.ensembleSize}, #{autoMlIns.ensembleClass}, #{autoMlIns.ensembleNbest}, + #{autoMlIns.maxModelsOnDisc}, #{autoMlIns.seed}, + #{autoMlIns.memoryLimit}, #{autoMlIns.includeClassifier}, #{autoMlIns.includeFeaturePreprocessor}, + #{autoMlIns.includeRegressor}, #{autoMlIns.excludeClassifier}, + #{autoMlIns.excludeRegressor}, #{autoMlIns.excludeFeaturePreprocessor}, #{autoMlIns.resamplingStrategy}, + #{autoMlIns.trainSize}, #{autoMlIns.shuffle}, + #{autoMlIns.folds}, #{autoMlIns.deleteTmpFolderAfterTerminate}, #{autoMlIns.dataCsv}, + #{autoMlIns.targetColumns}, #{autoMlIns.createBy}, #{autoMlIns.updateBy}) + + + + update auto_ml_ins + + + task_type = #{autoMlIns.taskType}, + + + dataset_name = #{autoMlIns.datasetName}, + + + time_left_for_this_task = #{autoMlIns.timeLeftForThisTask}, + + + per_run_time_limit = #{autoMlIns.perRunTimeLimit}, + + + ensemble_size = #{autoMlIns.ensembleSize}, + + + ensemble_class = #{autoMlIns.ensembleClass}, + + + ensemble_nbest = #{autoMlIns.ensembleNbest}, + + + max_models_on_disc = #{autoMlIns.maxModelsOnDisc}, + + + seed = #{autoMlIns.seed}, + + + memory_limit = #{autoMlIns.memoryLimit}, + + + include_classifier = #{autoMlIns.includeClassifier}, + + + include_feature_preprocessor = #{autoMlIns.includeFeaturePreprocessor}, + + + include_regressor = #{autoMlIns.includeRegressor}, + + + exclude_classifier = #{autoMlIns.excludeClassifier}, + + + exclude_regressor = #{autoMlIns.excludeRegressor}, + + + exclude_feature_preprocessor = #{autoMlIns.excludeFeaturePreprocessor}, + + + resampling_strategy = #{autoMlIns.resamplingStrategy}, + + + train_size = #{autoMlIns.trainSize}, + + + shuffle = #{autoMlIns.shuffle}, + + + folds = #{autoMlIns.folds}, + + + delete_tmp_folder_after_terminate = #{autoMlIns.deleteTmpFolderAfterTerminate}, + + + data_csv = #{autoMlIns.dataCsv}, + + + target_columns = #{autoMlIns.targetColumns}, + + + state = #{autoMlIns.state}, + + + update_by = #{autoMlIns.updateBy}, + + + where id = #{autoMlIns.id} + + + + + + + + + + + + + state = 1 and auto_ml_id = #{autoMlId} + + + \ No newline at end of file From ce32d520e7259b7b68c0f989f1319a7c847167b6 Mon Sep 17 00:00:00 2001 From: chenzhihang <709011834@qq.com> Date: Tue, 19 Nov 2024 17:30:16 +0800 Subject: [PATCH 04/18] =?UTF-8?q?=E8=87=AA=E5=8A=A8=E6=9C=BA=E5=99=A8?= =?UTF-8?q?=E5=AD=A6=E4=B9=A0=E5=BC=80=E5=8F=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../controller/autoML/AutoMLController.java | 32 +-- .../com/ruoyi/platform/domain/AutoMl.java | 57 ++++++ .../com/ruoyi/platform/domain/AutoMlIns.java | 77 -------- .../com/ruoyi/platform/mapper/AutoMLDao.java | 2 + .../ruoyi/platform/mapper/AutoMLInsDao.java | 21 -- .../ruoyi/platform/service/AutoMLService.java | 16 +- .../service/impl/AutoMLServiceImpl.java | 184 +++++++----------- .../managementPlatform/AutoMLDaoMapper.xml | 104 +++++++++- .../managementPlatform/AutoMLInsDaoMapper.xml | 132 ------------- 9 files changed, 234 insertions(+), 391 deletions(-) delete mode 100644 ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/AutoMlIns.java delete mode 100644 ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/AutoMLInsDao.java delete mode 100644 ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMLInsDaoMapper.xml diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/autoML/AutoMLController.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/autoML/AutoMLController.java index 47e291ee..4bd1a36e 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/autoML/AutoMLController.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/autoML/AutoMLController.java @@ -4,7 +4,6 @@ import com.ruoyi.common.core.web.controller.BaseController; import com.ruoyi.common.core.web.domain.AjaxResult; import com.ruoyi.common.core.web.domain.GenericsAjaxResult; import com.ruoyi.platform.domain.AutoMl; -import com.ruoyi.platform.domain.AutoMlIns; import com.ruoyi.platform.service.AutoMLService; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; @@ -34,13 +33,13 @@ public class AutoMLController extends BaseController { @PostMapping @ApiOperation("新增自动机器学习") - public GenericsAjaxResult addAutoMl(@RequestBody AutoMl autoMl) { + public GenericsAjaxResult addAutoMl(@RequestBody AutoMl autoMl) throws Exception { return genericsSuccess(this.autoMLService.save(autoMl)); } @PutMapping @ApiOperation("编辑自动机器学习") - public GenericsAjaxResult editAutoMl(@RequestBody AutoMl autoMl) { + public GenericsAjaxResult editAutoMl(@RequestBody AutoMl autoMl) throws Exception { return genericsSuccess(this.autoMLService.edit(autoMl)); } @@ -50,33 +49,6 @@ public class AutoMLController extends BaseController { return genericsSuccess(this.autoMLService.delete(id)); } - @GetMapping("/autoMlIns") - @ApiOperation("分页查询自动机器学习实例") - public GenericsAjaxResult> queryByPage(@RequestParam("page") int page, - @RequestParam("size") int size, - @RequestParam("auto_ml_id") Long autoMlId) { - PageRequest pageRequest = PageRequest.of(page, size); - return genericsSuccess(this.autoMLService.queryAutoMlInsByPage(autoMlId, pageRequest)); - } - - @PostMapping("/autoMlIns") - @ApiOperation("新增自动机器学习实例") - public GenericsAjaxResult addAutoMlIns(@RequestBody AutoMlIns autoMlIns) throws Exception { - return genericsSuccess(this.autoMLService.saveAutoMlIns(autoMlIns)); - } - - @PutMapping("/autoMlIns") - @ApiOperation("编辑自动机器学习实例") - public GenericsAjaxResult editAutoMlIns(@RequestBody AutoMlIns autoMlIns) throws Exception { - return genericsSuccess(this.autoMLService.editAutoMlIns(autoMlIns)); - } - - @DeleteMapping("/autoMlIns/{id}") - @ApiOperation("删除自动机器学习实例") - public GenericsAjaxResult deleteAutoMlIns(@PathVariable("id") Long id) { - return genericsSuccess(this.autoMLService.deleteAutoMlIns(id)); - } - @CrossOrigin(origins = "*", allowedHeaders = "*") @PostMapping("/upload") @ApiOperation(value = "上传数据文件csv", notes = "上传数据文件csv，并将信息存入数据库。") diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/AutoMl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/AutoMl.java index 409e5726..43b8255f 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/AutoMl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/AutoMl.java @@ -1,20 +1,74 @@ package com.ruoyi.platform.domain; +import com.baomidou.mybatisplus.annotation.TableField; import com.fasterxml.jackson.databind.PropertyNamingStrategy; import com.fasterxml.jackson.databind.annotation.JsonNaming; +import com.ruoyi.platform.vo.VersionVo; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; import lombok.Data; import java.util.Date; @Data @JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy.class) +@ApiModel(description = "自动机器学习") public class AutoMl { private Long id; + @ApiModelProperty(value = "实验名称") private String mlName; + @ApiModelProperty(value = "实验描述") private String mlDescription; + @ApiModelProperty(value = "任务类型") + private String taskType; + + private String datasetName; + + private Integer timeLeftForThisTask; + + private Integer perRunTimeLimit; + + private Integer ensembleSize; + + private String ensembleClass; + + private Integer ensembleNbest; + + private Integer maxModelsOnDisc; + + private Integer seed; + + private Integer memoryLimit; + + private String includeClassifier; + + private String includeFeaturePreprocessor; + + private String includeRegressor; + + private String excludeClassifier; + + private String excludeRegressor; + + private String excludeFeaturePreprocessor; + + private String resamplingStrategy; + + private Float trainSize; + + private Boolean shuffle; + + private Integer folds; + + private Boolean deleteTmpFolderAfterTerminate; + + private String dataCsv; + + private String targetColumns; + private Integer state; private String runState; @@ -28,4 +82,7 @@ public class AutoMl { private String updateBy; private Date updateTime; + + @TableField(exist = false) + private VersionVo versionVo; } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/AutoMlIns.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/AutoMlIns.java deleted file mode 100644 index 653c212d..00000000 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/AutoMlIns.java +++ /dev/null @@ -1,77 +0,0 @@ -package com.ruoyi.platform.domain; - -import com.baomidou.mybatisplus.annotation.TableField; -import com.fasterxml.jackson.databind.PropertyNamingStrategy; -import com.fasterxml.jackson.databind.annotation.JsonNaming; -import com.ruoyi.platform.vo.VersionVo; -import lombok.Data; - -import java.util.Date; - -@Data -@JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy.class) -public class AutoMlIns { - private Long id; - - private Long autoMlId; - - private String taskType; - - private String datasetName; - - private Integer timeLeftForThisTask; - - private Integer perRunTimeLimit; - - private Integer ensembleSize; - - private String ensembleClass; - - private Integer ensembleNbest; - - private Integer maxModelsOnDisc; - - private Integer seed; - - private Integer memoryLimit; - - private String includeClassifier; - - private String includeFeaturePreprocessor; - - private String includeRegressor; - - private String excludeClassifier; - - private String excludeRegressor; - - private String excludeFeaturePreprocessor; - - private String resamplingStrategy; - - private Float trainSize; - - private Boolean shuffle; - - private Integer folds; - - private Boolean deleteTmpFolderAfterTerminate; - - private String dataCsv; - - private String targetColumns; - - private Integer state; - - private String createBy; - - private String updateBy; - - private Date createTime; - - private Date updateTime; - - @TableField(exist = false) - private VersionVo versionVo; - -} diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/AutoMLDao.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/AutoMLDao.java index b10f5235..f99f2156 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/AutoMLDao.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/AutoMLDao.java @@ -13,6 +13,8 @@ public interface AutoMLDao { AutoMl getAutoMlById(@Param("id") Long id); + AutoMl getAutoMlByName(@Param("mlName") String mlName); + int save(@Param("autoMl") AutoMl autoMl); int edit(@Param("autoMl") AutoMl autoMl); diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/AutoMLInsDao.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/AutoMLInsDao.java deleted file mode 100644 index a541fc53..00000000 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/AutoMLInsDao.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.ruoyi.platform.mapper; - -import com.ruoyi.platform.domain.AutoMlIns; -import org.apache.ibatis.annotations.Param; -import org.springframework.data.domain.Pageable; - -import java.util.List; - -public interface AutoMLInsDao { - long count(@Param("autoMlId") Long autoMlId); - - List queryByPage(@Param("autoMlId") Long autoMlId, @Param("pageable") Pageable pageable); - - int save(@Param("autoMlIns") AutoMlIns autoMlIns); - - int edit(@Param("autoMlIns") AutoMlIns autoMlIns); - - AutoMlIns getById(@Param("id") Long id); - - AutoMlIns findByDatasetName(@Param("datasetName") String datasetName, @Param("autoMlId") Long autoMlId); -} diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AutoMLService.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AutoMLService.java index d475d5a3..6d5fd355 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AutoMLService.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AutoMLService.java @@ -1,32 +1,22 @@ package com.ruoyi.platform.service; import com.ruoyi.platform.domain.AutoMl; -import com.ruoyi.platform.domain.AutoMlIns; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageRequest; import org.springframework.web.multipart.MultipartFile; -import java.util.List; import java.util.Map; public interface AutoMLService { Page queryByPage(String mlName, PageRequest pageRequest); - AutoMl save(AutoMl autoMl); + AutoMl save(AutoMl autoMl) throws Exception; - String edit(AutoMl autoMl); + String edit(AutoMl autoMl) throws Exception; String delete(Long id); - Page queryAutoMlInsByPage(Long autoMlId, PageRequest pageRequest); - - AutoMlIns saveAutoMlIns(AutoMlIns autoMlIns) throws Exception; - - String editAutoMlIns(AutoMlIns autoMlIns) throws Exception; - - String deleteAutoMlIns(Long id); - - Map upload(MultipartFile file, String uuid) throws Exception; + Map upload(MultipartFile file, String uuid) throws Exception; String runAutoMlIns(Long id) throws Exception; } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMLServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMLServiceImpl.java index 19bb878b..9cdd3b40 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMLServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMLServiceImpl.java @@ -3,9 +3,7 @@ package com.ruoyi.platform.service.impl; import com.ruoyi.common.security.utils.SecurityUtils; import com.ruoyi.platform.constant.Constant; import com.ruoyi.platform.domain.AutoMl; -import com.ruoyi.platform.domain.AutoMlIns; import com.ruoyi.platform.mapper.AutoMLDao; -import com.ruoyi.platform.mapper.AutoMLInsDao; import com.ruoyi.platform.service.AutoMLService; import com.ruoyi.platform.utils.DVCUtils; import com.ruoyi.platform.utils.FileUtil; @@ -49,8 +47,6 @@ public class AutoMLServiceImpl implements AutoMLService { @Resource private AutoMLDao autoMLDao; @Resource - private AutoMLInsDao autoMLInsDao; - @Resource private K8sClientUtil k8sClientUtil; @Resource private DVCUtils dvcUtils; @@ -63,18 +59,40 @@ public class AutoMLServiceImpl implements AutoMLService { } @Override - public AutoMl save(AutoMl autoMl) { + public AutoMl save(AutoMl autoMl) throws Exception { + AutoMl autoMlByName = autoMLDao.getAutoMlByName(autoMl.getMlName()); + if (autoMlByName != null) { + throw new RuntimeException("实验名称已存在"); + } + String username = SecurityUtils.getLoginUser().getUsername(); autoMl.setCreateBy(username); autoMl.setUpdateBy(username); + + String sourcePath = autoMl.getVersionVo().getUrl(); + String rootPath = localPath + username + "/automl/" + autoMl.getMlName() + "/" + autoMl.getDatasetName(); + dvcUtils.moveFiles(sourcePath, rootPath); + autoMl.setDataCsv(rootPath + "/" + autoMl.getVersionVo().getFileName()); autoMLDao.save(autoMl); return autoMl; } @Override - public String edit(AutoMl autoMl) { + public String edit(AutoMl autoMl) throws Exception { + AutoMl oldAutoMl = autoMLDao.getAutoMlByName(autoMl.getMlName()); + if (oldAutoMl != null && !oldAutoMl.getId().equals(autoMl.getId())) { + throw new RuntimeException("实验名称已存在"); + } + String username = SecurityUtils.getLoginUser().getUsername(); autoMl.setUpdateBy(username); + if (autoMl.getVersionVo() != null && StringUtils.isNotEmpty(autoMl.getVersionVo().getUrl())) { + String sourcePath = autoMl.getVersionVo().getUrl(); + String rootPath = localPath + username + "/automl/" + autoMl.getMlName() + "/" + autoMl.getDatasetName(); + dvcUtils.moveFiles(sourcePath, rootPath); + autoMl.setDataCsv(rootPath + "/" + autoMl.getVersionVo().getFileName()); + } + autoMLDao.edit(autoMl); return "修改成功"; } @@ -86,7 +104,8 @@ public class AutoMLServiceImpl implements AutoMLService { throw new RuntimeException("服务不存在"); } - String username = SecurityUtils.getLoginUser().getUsername(); +// String username = SecurityUtils.getLoginUser().getUsername(); + String username = "admin"; String createBy = autoMl.getCreateBy(); if (!(StringUtils.equals(username, "admin") || StringUtils.equals(username, createBy))) { throw new RuntimeException("无权限删除该服务"); @@ -96,69 +115,12 @@ public class AutoMLServiceImpl implements AutoMLService { return autoMLDao.edit(autoMl) > 0 ? "删除成功" : "删除失败"; } - @Override - public Page queryAutoMlInsByPage(Long autoMlId, PageRequest pageRequest) { - long total = autoMLInsDao.count(autoMlId); - List autoMlIns = autoMLInsDao.queryByPage(autoMlId, pageRequest); - return new PageImpl<>(autoMlIns, pageRequest, total); - } - - @Override - public AutoMlIns saveAutoMlIns(AutoMlIns autoMlIns) throws Exception { - String ci4sUsername = SecurityUtils.getLoginUser().getUsername(); - AutoMlIns oldAutoMlIns = autoMLInsDao.findByDatasetName(autoMlIns.getDatasetName(), autoMlIns.getAutoMlId()); - if (oldAutoMlIns != null) { - throw new RuntimeException("数据集名称已存在"); - } - - String sourcePath = autoMlIns.getVersionVo().getUrl(); - String rootPath = localPath + ci4sUsername + "/automl/" + autoMlIns.getAutoMlId() + "/" + autoMlIns.getDatasetName(); - dvcUtils.moveFiles(sourcePath, rootPath); - autoMlIns.setDataCsv(rootPath + "/" + autoMlIns.getVersionVo().getFileName()); - autoMlIns.setCreateBy(ci4sUsername); - autoMlIns.setUpdateBy(ci4sUsername); - autoMLInsDao.save(autoMlIns); - return autoMlIns; - } - - @Override - public String editAutoMlIns(AutoMlIns autoMlIns) throws Exception { - AutoMlIns oldAutoMlIns = autoMLInsDao.findByDatasetName(autoMlIns.getDatasetName(), autoMlIns.getAutoMlId()); - if (oldAutoMlIns != null && !oldAutoMlIns.getId().equals(autoMlIns.getId())) { - throw new RuntimeException("数据集名称已存在"); - } - String ci4sUsername = SecurityUtils.getLoginUser().getUsername(); - autoMlIns.setUpdateBy(ci4sUsername); - if (autoMlIns.getVersionVo() != null && StringUtils.isNotEmpty(autoMlIns.getVersionVo().getUrl())) { - String sourcePath = autoMlIns.getVersionVo().getUrl(); - String rootPath = localPath + ci4sUsername + "/automl/" + autoMlIns.getAutoMlId() + "/" + autoMlIns.getDatasetName(); - dvcUtils.moveFiles(sourcePath, rootPath); - autoMlIns.setDataCsv(rootPath + "/" + autoMlIns.getVersionVo().getFileName()); - } - autoMLInsDao.edit(autoMlIns); - return "修改成功"; - } - - @Override - public String deleteAutoMlIns(Long id) { - AutoMlIns autoMlIns = autoMLInsDao.getById(id); - if (autoMlIns == null) { - throw new RuntimeException("实例不存在"); - } - String username = SecurityUtils.getLoginUser().getUsername(); - String createBy = autoMlIns.getCreateBy(); - if (!(StringUtils.equals(username, "admin") || StringUtils.equals(username, createBy))) { - throw new RuntimeException("无权限删除该实例"); - } - autoMlIns.setState(Constant.State_invalid); - return autoMLInsDao.edit(autoMlIns) > 0 ? "删除成功" : "删除失败"; - } - @Override public Map upload(MultipartFile file, String uuid) throws Exception { Map result = new HashMap<>(); - String username = SecurityUtils.getLoginUser().getUsername(); +// String username = SecurityUtils.getLoginUser().getUsername(); + String username = "admin"; String fileName = file.getOriginalFilename(); String path = localPath + "temp/" + username + "/automl_data/" + uuid; long sizeInBytes = file.getSize(); @@ -177,25 +139,25 @@ public class AutoMLServiceImpl implements AutoMLService { @Override public String runAutoMlIns(Long id) throws Exception { - AutoMlIns autoMlIns = autoMLInsDao.getById(id); - if (autoMlIns == null) { + AutoMl autoMl = autoMLDao.getAutoMlById(id); + if (autoMl == null) { throw new Exception("开发环境配置不存在"); } StringBuffer command = new StringBuffer(); - command.append("nohup python /opt/automl.py --task_type " + autoMlIns.getTaskType()); - if (StringUtils.isNotEmpty(autoMlIns.getDataCsv())) { - command.append(" --data_csv " + autoMlIns.getDataCsv()); + command.append("nohup python /opt/automl.py --task_type " + autoMl.getTaskType()); + if (StringUtils.isNotEmpty(autoMl.getDataCsv())) { + command.append(" --data_csv " + autoMl.getDataCsv()); } else { throw new Exception("训练数据为空"); } - if (StringUtils.isNotEmpty(autoMlIns.getTargetColumns())) { - command.append(" --target_columns " + autoMlIns.getTargetColumns()); + if (StringUtils.isNotEmpty(autoMl.getTargetColumns())) { + command.append(" --target_columns " + autoMl.getTargetColumns()); } else { throw new Exception("目标列为空"); } - if (StringUtils.isNotEmpty(autoMlIns.getDatasetName())) { - command.append(" --dataset_name " + autoMlIns.getDatasetName()); + if (StringUtils.isNotEmpty(autoMl.getDatasetName())) { + command.append(" --dataset_name " + autoMl.getDatasetName()); } else { throw new Exception("数据集名称为空"); } @@ -206,59 +168,59 @@ public class AutoMLServiceImpl implements AutoMLService { String podName = username + "-autoMlIns-pod-" + id; V1Pod pod = k8sClientUtil.createPodWithEnv(podName, serviceNS, proxyUrl, mountPath, pvcName, image); - if (autoMlIns.getTimeLeftForThisTask() != null) { - command.append(" --time_left_for_this_task " + autoMlIns.getTimeLeftForThisTask()); + if (autoMl.getTimeLeftForThisTask() != null) { + command.append(" --time_left_for_this_task " + autoMl.getTimeLeftForThisTask()); } - if (autoMlIns.getPerRunTimeLimit() != null) { - command.append(" --per_run_time_limit " + autoMlIns.getPerRunTimeLimit()); + if (autoMl.getPerRunTimeLimit() != null) { + command.append(" --per_run_time_limit " + autoMl.getPerRunTimeLimit()); } - if (autoMlIns.getEnsembleSize() != null) { - command.append(" --ensemble_size " + autoMlIns.getEnsembleSize()); + if (autoMl.getEnsembleSize() != null) { + command.append(" --ensemble_size " + autoMl.getEnsembleSize()); } - if (StringUtils.isNotEmpty(autoMlIns.getEnsembleClass())) { - command.append(" --ensemble_class " + autoMlIns.getEnsembleClass()); + if (StringUtils.isNotEmpty(autoMl.getEnsembleClass())) { + command.append(" --ensemble_class " + autoMl.getEnsembleClass()); } - if (autoMlIns.getEnsembleNbest() != null) { - command.append(" --ensemble_nbest " + autoMlIns.getEnsembleNbest()); + if (autoMl.getEnsembleNbest() != null) { + command.append(" --ensemble_nbest " + autoMl.getEnsembleNbest()); } - if (autoMlIns.getMaxModelsOnDisc() != null) { - command.append(" --max_models_on_disc " + autoMlIns.getMaxModelsOnDisc()); + if (autoMl.getMaxModelsOnDisc() != null) { + command.append(" --max_models_on_disc " + autoMl.getMaxModelsOnDisc()); } - if (autoMlIns.getSeed() != null) { - command.append(" --seed " + autoMlIns.getSeed()); + if (autoMl.getSeed() != null) { + command.append(" --seed " + autoMl.getSeed()); } - if (autoMlIns.getMemoryLimit() != null) { - command.append(" --memory_limit " + autoMlIns.getMemoryLimit()); + if (autoMl.getMemoryLimit() != null) { + command.append(" --memory_limit " + autoMl.getMemoryLimit()); } - if (StringUtils.isNotEmpty(autoMlIns.getIncludeClassifier())) { - command.append(" --include_classifier " + autoMlIns.getIncludeClassifier()); + if (StringUtils.isNotEmpty(autoMl.getIncludeClassifier())) { + command.append(" --include_classifier " + autoMl.getIncludeClassifier()); } - if (StringUtils.isNotEmpty(autoMlIns.getIncludeRegressor())) { - command.append(" --include_regressor " + autoMlIns.getIncludeRegressor()); + if (StringUtils.isNotEmpty(autoMl.getIncludeRegressor())) { + command.append(" --include_regressor " + autoMl.getIncludeRegressor()); } - if (StringUtils.isNotEmpty(autoMlIns.getIncludeFeaturePreprocessor())) { - command.append(" --include_feature_preprocessor " + autoMlIns.getIncludeFeaturePreprocessor()); + if (StringUtils.isNotEmpty(autoMl.getIncludeFeaturePreprocessor())) { + command.append(" --include_feature_preprocessor " + autoMl.getIncludeFeaturePreprocessor()); } - if (StringUtils.isNotEmpty(autoMlIns.getExcludeClassifier())) { - command.append(" --exclude_classifier " + autoMlIns.getExcludeClassifier()); + if (StringUtils.isNotEmpty(autoMl.getExcludeClassifier())) { + command.append(" --exclude_classifier " + autoMl.getExcludeClassifier()); } - if (StringUtils.isNotEmpty(autoMlIns.getExcludeRegressor())) { - command.append(" --exclude_regressor " + autoMlIns.getExcludeRegressor()); + if (StringUtils.isNotEmpty(autoMl.getExcludeRegressor())) { + command.append(" --exclude_regressor " + autoMl.getExcludeRegressor()); } - if (StringUtils.isNotEmpty(autoMlIns.getExcludeFeaturePreprocessor())) { - command.append(" --exclude_feature_preprocessor " + autoMlIns.getExcludeFeaturePreprocessor()); + if (StringUtils.isNotEmpty(autoMl.getExcludeFeaturePreprocessor())) { + command.append(" --exclude_feature_preprocessor " + autoMl.getExcludeFeaturePreprocessor()); } - if (StringUtils.isNotEmpty(autoMlIns.getResamplingStrategy())) { - command.append(" --resampling_strategy " + autoMlIns.getResamplingStrategy()); + if (StringUtils.isNotEmpty(autoMl.getResamplingStrategy())) { + command.append(" --resampling_strategy " + autoMl.getResamplingStrategy()); } - if (autoMlIns.getTrainSize() != null) { - command.append(" --train_size " + autoMlIns.getTrainSize()); + if (autoMl.getTrainSize() != null) { + command.append(" --train_size " + autoMl.getTrainSize()); } - if (autoMlIns.getShuffle() != null) { - command.append(" --shuffle " + autoMlIns.getShuffle()); + if (autoMl.getShuffle() != null) { + command.append(" --shuffle " + autoMl.getShuffle()); } - if (autoMlIns.getFolds() != null) { - command.append(" --folds " + autoMlIns.getFolds()); + if (autoMl.getFolds() != null) { + command.append(" --folds " + autoMl.getFolds()); } command.append(" &"); CompletableFuture.supplyAsync(() -> { diff --git a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMLDaoMapper.xml b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMLDaoMapper.xml index 8f4bc742..0af3beb7 100644 --- a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMLDaoMapper.xml +++ b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMLDaoMapper.xml @@ -2,8 +2,23 @@ - insert into auto_ml(ml_name, ml_description) - values (#{autoMl.mlName}, #{autoMl.mlDescription}) + insert into auto_ml(ml_name, ml_description, task_type, dataset_name, time_left_for_this_task, + per_run_time_limit, ensemble_size, ensemble_class, ensemble_nbest, max_models_on_disc, seed, + memory_limit, + include_classifier, include_feature_preprocessor, include_regressor, exclude_classifier, + exclude_regressor, exclude_feature_preprocessor, resampling_strategy, train_size, + shuffle, folds, data_csv, target_columns, create_by, + update_by) + values (#{autoMl.mlName}, #{autoMl.mlDescription}, #{autoMl.taskType}, #{autoMl.datasetName}, + #{autoMl.timeLeftForThisTask}, #{autoMl.perRunTimeLimit}, + #{autoMl.ensembleSize}, #{autoMl.ensembleClass}, #{autoMl.ensembleNbest}, + #{autoMl.maxModelsOnDisc}, #{autoMl.seed}, + #{autoMl.memoryLimit}, #{autoMl.includeClassifier}, #{autoMl.includeFeaturePreprocessor}, + #{autoMl.includeRegressor}, #{autoMl.excludeClassifier}, + #{autoMl.excludeRegressor}, #{autoMl.excludeFeaturePreprocessor}, #{autoMl.resamplingStrategy}, + #{autoMl.trainSize}, #{autoMl.shuffle}, + #{autoMl.folds}, #{autoMl.dataCsv}, + #{autoMl.targetColumns}, #{autoMl.createBy}, #{autoMl.updateBy}) @@ -15,11 +30,77 @@ ml_description = #{autoMl.mlDescription}, - - run_state = #{autoMl.runState}, + + + + + + + + task_type = #{autoMl.taskType}, - - progress = #{autoMl.progress}, + + dataset_name = #{autoMl.datasetName}, + + + time_left_for_this_task = #{autoMl.timeLeftForThisTask}, + + + per_run_time_limit = #{autoMl.perRunTimeLimit}, + + + ensemble_size = #{autoMl.ensembleSize}, + + + ensemble_class = #{autoMl.ensembleClass}, + + + ensemble_nbest = #{autoMl.ensembleNbest}, + + + max_models_on_disc = #{autoMl.maxModelsOnDisc}, + + + seed = #{autoMl.seed}, + + + memory_limit = #{autoMl.memoryLimit}, + + + include_classifier = #{autoMl.includeClassifier}, + + + include_feature_preprocessor = #{autoMl.includeFeaturePreprocessor}, + + + include_regressor = #{autoMl.includeRegressor}, + + + exclude_classifier = #{autoMl.excludeClassifier}, + + + exclude_regressor = #{autoMl.excludeRegressor}, + + + exclude_feature_preprocessor = #{autoMl.excludeFeaturePreprocessor}, + + + resampling_strategy = #{autoMl.resamplingStrategy}, + + + train_size = #{autoMl.trainSize}, + + + shuffle = #{autoMl.shuffle}, + + + folds = #{autoMl.folds}, + + + data_csv = #{autoMl.dataCsv}, + + + target_columns = #{autoMl.targetColumns}, state = #{autoMl.state}, @@ -39,7 +120,16 @@ + + diff --git a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMLInsDaoMapper.xml b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMLInsDaoMapper.xml deleted file mode 100644 index 7f85a9d5..00000000 --- a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMLInsDaoMapper.xml +++ /dev/null @@ -1,132 +0,0 @@ - - - - - insert into auto_ml_ins(auto_ml_id, task_type, dataset_name, time_left_for_this_task, per_run_time_limit, - ensemble_size, ensemble_class, ensemble_nbest, max_models_on_disc, seed, memory_limit, - include_classifier, include_feature_preprocessor, include_regressor, exclude_classifier, - exclude_regressor, exclude_feature_preprocessor, resampling_strategy, train_size, - shuffle, folds, delete_tmp_folder_after_terminate, data_csv, target_columns, create_by, - update_by) - values (#{autoMlIns.autoMlId}, #{autoMlIns.taskType}, #{autoMlIns.datasetName}, - #{autoMlIns.timeLeftForThisTask}, #{autoMlIns.perRunTimeLimit}, - #{autoMlIns.ensembleSize}, #{autoMlIns.ensembleClass}, #{autoMlIns.ensembleNbest}, - #{autoMlIns.maxModelsOnDisc}, #{autoMlIns.seed}, - #{autoMlIns.memoryLimit}, #{autoMlIns.includeClassifier}, #{autoMlIns.includeFeaturePreprocessor}, - #{autoMlIns.includeRegressor}, #{autoMlIns.excludeClassifier}, - #{autoMlIns.excludeRegressor}, #{autoMlIns.excludeFeaturePreprocessor}, #{autoMlIns.resamplingStrategy}, - #{autoMlIns.trainSize}, #{autoMlIns.shuffle}, - #{autoMlIns.folds}, #{autoMlIns.deleteTmpFolderAfterTerminate}, #{autoMlIns.dataCsv}, - #{autoMlIns.targetColumns}, #{autoMlIns.createBy}, #{autoMlIns.updateBy}) - - - - update auto_ml_ins - - - task_type = #{autoMlIns.taskType}, - - - dataset_name = #{autoMlIns.datasetName}, - - - time_left_for_this_task = #{autoMlIns.timeLeftForThisTask}, - - - per_run_time_limit = #{autoMlIns.perRunTimeLimit}, - - - ensemble_size = #{autoMlIns.ensembleSize}, - - - ensemble_class = #{autoMlIns.ensembleClass}, - - - ensemble_nbest = #{autoMlIns.ensembleNbest}, - - - max_models_on_disc = #{autoMlIns.maxModelsOnDisc}, - - - seed = #{autoMlIns.seed}, - - - memory_limit = #{autoMlIns.memoryLimit}, - - - include_classifier = #{autoMlIns.includeClassifier}, - - - include_feature_preprocessor = #{autoMlIns.includeFeaturePreprocessor}, - - - include_regressor = #{autoMlIns.includeRegressor}, - - - exclude_classifier = #{autoMlIns.excludeClassifier}, - - - exclude_regressor = #{autoMlIns.excludeRegressor}, - - - exclude_feature_preprocessor = #{autoMlIns.excludeFeaturePreprocessor}, - - - resampling_strategy = #{autoMlIns.resamplingStrategy}, - - - train_size = #{autoMlIns.trainSize}, - - - shuffle = #{autoMlIns.shuffle}, - - - folds = #{autoMlIns.folds}, - - - delete_tmp_folder_after_terminate = #{autoMlIns.deleteTmpFolderAfterTerminate}, - - - data_csv = #{autoMlIns.dataCsv}, - - - target_columns = #{autoMlIns.targetColumns}, - - - state = #{autoMlIns.state}, - - - update_by = #{autoMlIns.updateBy}, - - - where id = #{autoMlIns.id} - - - - - - - - - - - - - state = 1 and auto_ml_id = #{autoMlId} - - - \ No newline at end of file From 0ede04ec79c6fab3c7fa22fbc91ce7ab8e24bbf7 Mon Sep 17 00:00:00 2001 From: chenzhihang <709011834@qq.com> Date: Fri, 22 Nov 2024 08:49:46 +0800 Subject: [PATCH 05/18] =?UTF-8?q?=E8=87=AA=E5=8A=A8=E6=9C=BA=E5=99=A8?= =?UTF-8?q?=E5=AD=A6=E4=B9=A0=E5=BC=80=E5=8F=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/ruoyi/platform/domain/AutoMl.java | 100 +++++++++++++++++- .../managementPlatform/AutoMLDaoMapper.xml | 22 +++- 2 files changed, 117 insertions(+), 5 deletions(-) diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/AutoMl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/AutoMl.java index 43b8255f..e1aa5506 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/AutoMl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/AutoMl.java @@ -22,31 +22,86 @@ public class AutoMl { @ApiModelProperty(value = "实验描述") private String mlDescription; - @ApiModelProperty(value = "任务类型") + @ApiModelProperty(value = "任务类型：classification或regression") private String taskType; + @ApiModelProperty(value = "数据集名称") private String datasetName; + @ApiModelProperty(value = "搜索合适模型的时间限制（以秒为单位）。通过增加这个值，auto-sklearn有更高的机会找到更好的模型。默认3600，非必传。") private Integer timeLeftForThisTask; + @ApiModelProperty(value = "单次调用机器学习模型的时间限制（以秒为单位）。如果机器学习算法运行超过时间限制，将终止模型拟合。将这个值设置得足够高，这样典型的机器学习算法就可以适用于训练数据。默认600，非必传。") private Integer perRunTimeLimit; + @ApiModelProperty(value = "集成模型数量，如果设置为0，则没有集成。默认50，非必传。") private Integer ensembleSize; + @ApiModelProperty(value = "设置为None将禁用集成构建，设置为SingleBest仅使用单个最佳模型而不是集成，设置为default，它将对单目标问题使用EnsembleSelection，对多目标问题使用MultiObjectiveDummyEnsemble。默认default，非必传。") private String ensembleClass; + @ApiModelProperty(value = "在构建集成时只考虑ensemble_nbest模型。这是受到了“最大限度地利用集成选择”中引入的库修剪概念的启发。这是独立于ensemble_class参数的，并且这个修剪步骤是在构造集成之前完成的。默认50，非必传。") private Integer ensembleNbest; + @ApiModelProperty(value = "定义在磁盘中保存的模型的最大数量。额外的模型数量将被永久删除。由于这个变量的性质，它设置了一个集成可以使用多少个模型的上限。必须是大于等于1的整数。如果设置为None，则所有模型都保留在磁盘上。默认50，非必传。") private Integer maxModelsOnDisc; + @ApiModelProperty(value = "随机种子，将决定输出文件名。默认1，非必传。") private Integer seed; + @ApiModelProperty(value = "机器学习算法的内存限制（MB）。如果auto-sklearn试图分配超过memory_limit MB，它将停止拟合机器学习算法。默认3072，非必传。") private Integer memoryLimit; + @ApiModelProperty(value = "如果为None，则使用所有可能的分类算法。否则，指定搜索中包含的步骤和组件。有关可用组件，请参见/pipeline/components//*。与参数exclude不兼容。多选，逗号分隔。包含：adaboost\n" + + "bernoulli_nb\n" + + "decision_tree\n" + + "extra_trees\n" + + "gaussian_nb\n" + + "gradient_boosting\n" + + "k_nearest_neighbors\n" + + "lda\n" + + "liblinear_svc\n" + + "libsvm_svc\n" + + "mlp\n" + + "multinomial_nb\n" + + "passive_aggressive\n" + + "qda\n" + + "random_forest\n" + + "sgd") private String includeClassifier; + @ApiModelProperty(value = "如果为None，则使用所有可能的特征预处理算法。否则，指定搜索中包含的步骤和组件。有关可用组件，请参见/pipeline/components//*。与参数exclude不兼容。多选，逗号分隔。包含：densifier\n" + + "extra_trees_preproc_for_classification\n" + + "extra_trees_preproc_for_regression\n" + + "fast_ica\n" + + "feature_agglomeration\n" + + "kernel_pca\n" + + "kitchen_sinks\n" + + "liblinear_svc_preprocessor\n" + + "no_preprocessing\n" + + "nystroem_sampler\n" + + "pca\n" + + "polynomial\n" + + "random_trees_embedding\n" + + "select_percentile_classification\n" + + "select_percentile_regression\n" + + "select_rates_classification\n" + + "select_rates_regression\n" + + "truncatedSVD") private String includeFeaturePreprocessor; + @ApiModelProperty(value = "如果为None，则使用所有可能的回归算法。否则，指定搜索中包含的步骤和组件。有关可用组件，请参见/pipeline/components//*。与参数exclude不兼容。多选，逗号分隔。包含：adaboost,\n" + + "ard_regression,\n" + + "decision_tree,\n" + + "extra_trees,\n" + + "gaussian_process,\n" + + "gradient_boosting,\n" + + "k_nearest_neighbors,\n" + + "liblinear_svr,\n" + + "libsvm_svr,\n" + + "mlp,\n" + + "random_forest,\n" + + "sgd") private String includeRegressor; private String excludeClassifier; @@ -55,20 +110,61 @@ public class AutoMl { private String excludeFeaturePreprocessor; + @ApiModelProperty(value = "如何处理过拟合，如果使用基于“cv”的方法或Splitter对象，可能需要使用resampling_strategy_arguments。holdout或crossValid") private String resamplingStrategy; + @ApiModelProperty(value = "训练集的比率，0到1之间") private Float trainSize; + @ApiModelProperty(value = "拆分数据前是否进行shuffle") private Boolean shuffle; + @ApiModelProperty(value = "当resamplingStrategy为crossValid时，此项必填。为整数") private Integer folds; - private Boolean deleteTmpFolderAfterTerminate; + @ApiModelProperty(value = "文件夹存放配置输出和日志文件，默认/tmp/automl") + private String tmpFolder; + @ApiModelProperty(value = "数据集csv文件路径") private String dataCsv; + @ApiModelProperty(value = "数据集csv文件中哪几列是预测目标列，逗号分隔") private String targetColumns; + @ApiModelProperty(value = "自定义指标名称") + private String metricName; + + @ApiModelProperty(value = "模型优化目标指标及权重，json格式。分类的指标包含：accuracy\n" + + "balanced_accuracy\n" + + "roc_auc\n" + + "average_precision\n" + + "log_loss\n" + + "precision_macro\n" + + "precision_micro\n" + + "precision_samples\n" + + "precision_weighted\n" + + "recall_macro\n" + + "recall_micro\n" + + "recall_samples\n" + + "recall_weighted\n" + + "f1_macro\n" + + "f1_micro\n" + + "f1_samples\n" + + "f1_weighted\n" + + "回归的指标包含：mean_absolute_error\n" + + "mean_squared_error\n" + + "root_mean_squared_error\n" + + "mean_squared_log_error\n" + + "median_absolute_error\n" + + "r2") + private String metrics; + + @ApiModelProperty(value = "指标优化方向，是越大越好还是越小越好") + private Boolean greaterIsBetter; + + @ApiModelProperty(value = "模型计算并打印指标") + private String scoringFunctions; + private Integer state; private String runState; diff --git a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMLDaoMapper.xml b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMLDaoMapper.xml index 0af3beb7..55f8e8f5 100644 --- a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMLDaoMapper.xml +++ b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMLDaoMapper.xml @@ -7,8 +7,8 @@ memory_limit, include_classifier, include_feature_preprocessor, include_regressor, exclude_classifier, exclude_regressor, exclude_feature_preprocessor, resampling_strategy, train_size, - shuffle, folds, data_csv, target_columns, create_by, - update_by) + shuffle, folds, data_csv, target_columns, metric_name, metrics,greater_is_better,scoring_functions,tmp_folder, + create_by,update_by) values (#{autoMl.mlName}, #{autoMl.mlDescription}, #{autoMl.taskType}, #{autoMl.datasetName}, #{autoMl.timeLeftForThisTask}, #{autoMl.perRunTimeLimit}, #{autoMl.ensembleSize}, #{autoMl.ensembleClass}, #{autoMl.ensembleNbest}, @@ -18,7 +18,8 @@ #{autoMl.excludeRegressor}, #{autoMl.excludeFeaturePreprocessor}, #{autoMl.resamplingStrategy}, #{autoMl.trainSize}, #{autoMl.shuffle}, #{autoMl.folds}, #{autoMl.dataCsv}, - #{autoMl.targetColumns}, #{autoMl.createBy}, #{autoMl.updateBy}) + #{autoMl.targetColumns}, #{autoMl.metricName}, #{autoMl.metrics},#{autoMl.greaterIsBetter},#{autoMl.scoringFunctions},#{autoMl.tmpFolder}, + #{autoMl.createBy}, #{autoMl.updateBy}) @@ -99,6 +100,21 @@ data_csv = #{autoMl.dataCsv}, + + tmp_folder = #{autoMl.tmpFolder}, + + + metric_name = #{autoMl.metricName}, + + + metrics = #{autoMl.metrics}, + + + greater_is_better = #{autoMl.greaterIsBetter}, + + + scoring_functions = #{autoMl.scoringFunctions}, + target_columns = #{autoMl.targetColumns}, From 468b812f24598f168dbba51b722cf1895a48aae0 Mon Sep 17 00:00:00 2001 From: chenzhihang <709011834@qq.com> Date: Fri, 22 Nov 2024 11:22:53 +0800 Subject: [PATCH 06/18] =?UTF-8?q?=E8=87=AA=E5=8A=A8=E6=9C=BA=E5=99=A8?= =?UTF-8?q?=E5=AD=A6=E4=B9=A0=E5=BC=80=E5=8F=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/java/com/ruoyi/platform/domain/AutoMl.java | 7 +++++-- .../mapper/managementPlatform/AutoMLDaoMapper.xml | 7 +++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/AutoMl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/AutoMl.java index e1aa5506..ec9b5931 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/AutoMl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/AutoMl.java @@ -110,16 +110,19 @@ public class AutoMl { private String excludeFeaturePreprocessor; + @ApiModelProperty(value = "训练集的比率，0到1之间") + private Float testSize; + @ApiModelProperty(value = "如何处理过拟合，如果使用基于“cv”的方法或Splitter对象，可能需要使用resampling_strategy_arguments。holdout或crossValid") private String resamplingStrategy; - @ApiModelProperty(value = "训练集的比率，0到1之间") + @ApiModelProperty(value = "重采样划分训练集和验证集，训练集的比率，0到1之间") private Float trainSize; @ApiModelProperty(value = "拆分数据前是否进行shuffle") private Boolean shuffle; - @ApiModelProperty(value = "当resamplingStrategy为crossValid时，此项必填。为整数") + @ApiModelProperty(value = "交叉验证的折数，当resamplingStrategy为crossValid时，此项必填，为整数") private Integer folds; @ApiModelProperty(value = "文件夹存放配置输出和日志文件，默认/tmp/automl") diff --git a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMLDaoMapper.xml b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMLDaoMapper.xml index 55f8e8f5..e5fecd5a 100644 --- a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMLDaoMapper.xml +++ b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMLDaoMapper.xml @@ -6,7 +6,7 @@ per_run_time_limit, ensemble_size, ensemble_class, ensemble_nbest, max_models_on_disc, seed, memory_limit, include_classifier, include_feature_preprocessor, include_regressor, exclude_classifier, - exclude_regressor, exclude_feature_preprocessor, resampling_strategy, train_size, + exclude_regressor, exclude_feature_preprocessor, test_size, resampling_strategy, train_size, shuffle, folds, data_csv, target_columns, metric_name, metrics,greater_is_better,scoring_functions,tmp_folder, create_by,update_by) values (#{autoMl.mlName}, #{autoMl.mlDescription}, #{autoMl.taskType}, #{autoMl.datasetName}, @@ -15,7 +15,7 @@ #{autoMl.maxModelsOnDisc}, #{autoMl.seed}, #{autoMl.memoryLimit}, #{autoMl.includeClassifier}, #{autoMl.includeFeaturePreprocessor}, #{autoMl.includeRegressor}, #{autoMl.excludeClassifier}, - #{autoMl.excludeRegressor}, #{autoMl.excludeFeaturePreprocessor}, #{autoMl.resamplingStrategy}, + #{autoMl.excludeRegressor}, #{autoMl.excludeFeaturePreprocessor}, #{autoMl.testSize}, #{autoMl.resamplingStrategy}, #{autoMl.trainSize}, #{autoMl.shuffle}, #{autoMl.folds}, #{autoMl.dataCsv}, #{autoMl.targetColumns}, #{autoMl.metricName}, #{autoMl.metrics},#{autoMl.greaterIsBetter},#{autoMl.scoringFunctions},#{autoMl.tmpFolder}, @@ -85,6 +85,9 @@ exclude_feature_preprocessor = #{autoMl.excludeFeaturePreprocessor}, + + test_size = #{autoMl.testSize}, + resampling_strategy = #{autoMl.resamplingStrategy}, From 72744f287db4c92e5119654577b962c17960d4d9 Mon Sep 17 00:00:00 2001 From: chenzhihang <709011834@qq.com> Date: Fri, 22 Nov 2024 11:23:35 +0800 Subject: [PATCH 07/18] =?UTF-8?q?=E8=87=AA=E5=8A=A8=E6=9C=BA=E5=99=A8?= =?UTF-8?q?=E5=AD=A6=E4=B9=A0=E5=BC=80=E5=8F=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/java/com/ruoyi/platform/domain/AutoMl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/AutoMl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/AutoMl.java index ec9b5931..e37f6bbb 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/AutoMl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/AutoMl.java @@ -110,7 +110,7 @@ public class AutoMl { private String excludeFeaturePreprocessor; - @ApiModelProperty(value = "训练集的比率，0到1之间") + @ApiModelProperty(value = "测试集的比率，0到1之间") private Float testSize; @ApiModelProperty(value = "如何处理过拟合，如果使用基于“cv”的方法或Splitter对象，可能需要使用resampling_strategy_arguments。holdout或crossValid") From 9976c8e972bd80179979d22960d3818fc883fdfd Mon Sep 17 00:00:00 2001 From: chenzhihang <709011834@qq.com> Date: Mon, 25 Nov 2024 10:06:13 +0800 Subject: [PATCH 08/18] =?UTF-8?q?=E8=87=AA=E5=8A=A8=E6=9C=BA=E5=99=A8?= =?UTF-8?q?=E5=AD=A6=E4=B9=A0=E5=BC=80=E5=8F=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ruoyi/platform/controller/autoML/AutoMLController.java | 5 +++++ .../main/java/com/ruoyi/platform/service/AutoMLService.java | 2 ++ .../com/ruoyi/platform/service/impl/AutoMLServiceImpl.java | 5 +++++ 3 files changed, 12 insertions(+) diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/autoML/AutoMLController.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/autoML/AutoMLController.java index 4bd1a36e..10b5a217 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/autoML/AutoMLController.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/autoML/AutoMLController.java @@ -42,6 +42,11 @@ public class AutoMLController extends BaseController { public GenericsAjaxResult editAutoMl(@RequestBody AutoMl autoMl) throws Exception { return genericsSuccess(this.autoMLService.edit(autoMl)); } + @GetMapping("/getAutoMlDetail") + @ApiOperation("获取自动机器学习详细信息") + public GenericsAjaxResult getAutoMlDetail(@RequestParam("id") Long id){ + return genericsSuccess(this.autoMLService.getAutoMlDetail(id)); + } @DeleteMapping("{id}") @ApiOperation("删除自动机器学习") diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AutoMLService.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AutoMLService.java index 6d5fd355..2b678382 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AutoMLService.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AutoMLService.java @@ -16,6 +16,8 @@ public interface AutoMLService { String delete(Long id); + AutoMl getAutoMlDetail(Long id); + Map upload(MultipartFile file, String uuid) throws Exception; String runAutoMlIns(Long id) throws Exception; diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMLServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMLServiceImpl.java index 9cdd3b40..f052f32e 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMLServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMLServiceImpl.java @@ -115,6 +115,11 @@ public class AutoMLServiceImpl implements AutoMLService { return autoMLDao.edit(autoMl) > 0 ? "删除成功" : "删除失败"; } + @Override + public AutoMl getAutoMlDetail(Long id) { + return autoMLDao.getAutoMlById(id); + } + @Override public Map upload(MultipartFile file, String uuid) throws Exception { Map result = new HashMap<>(); From 93057f735895a9831f453fc7066ed39b0436304a Mon Sep 17 00:00:00 2001 From: chenzhihang <709011834@qq.com> Date: Mon, 25 Nov 2024 16:01:11 +0800 Subject: [PATCH 09/18] =?UTF-8?q?=E8=87=AA=E5=8A=A8=E6=9C=BA=E5=99=A8?= =?UTF-8?q?=E5=AD=A6=E4=B9=A0=E5=BC=80=E5=8F=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../controller/autoML/AutoMLController.java | 12 +- .../com/ruoyi/platform/domain/AutoMl.java | 11 +- .../ruoyi/platform/service/AutoMLService.java | 8 +- .../service/impl/AutoMLServiceImpl.java | 64 +++--- .../java/com/ruoyi/platform/vo/AutoMlVo.java | 183 ++++++++++++++++++ .../managementPlatform/AutoMLDaoMapper.xml | 54 ++---- 6 files changed, 246 insertions(+), 86 deletions(-) create mode 100644 ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/AutoMlVo.java diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/autoML/AutoMLController.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/autoML/AutoMLController.java index 10b5a217..d859e491 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/autoML/AutoMLController.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/autoML/AutoMLController.java @@ -5,6 +5,7 @@ import com.ruoyi.common.core.web.domain.AjaxResult; import com.ruoyi.common.core.web.domain.GenericsAjaxResult; import com.ruoyi.platform.domain.AutoMl; import com.ruoyi.platform.service.AutoMLService; +import com.ruoyi.platform.vo.AutoMlVo; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import org.springframework.data.domain.Page; @@ -13,6 +14,7 @@ import org.springframework.web.bind.annotation.*; import org.springframework.web.multipart.MultipartFile; import javax.annotation.Resource; +import java.io.IOException; @RestController @RequestMapping("autoML") @@ -33,18 +35,18 @@ public class AutoMLController extends BaseController { @PostMapping @ApiOperation("新增自动机器学习") - public GenericsAjaxResult addAutoMl(@RequestBody AutoMl autoMl) throws Exception { - return genericsSuccess(this.autoMLService.save(autoMl)); + public GenericsAjaxResult addAutoMl(@RequestBody AutoMlVo autoMlVo) throws Exception { + return genericsSuccess(this.autoMLService.save(autoMlVo)); } @PutMapping @ApiOperation("编辑自动机器学习") - public GenericsAjaxResult editAutoMl(@RequestBody AutoMl autoMl) throws Exception { - return genericsSuccess(this.autoMLService.edit(autoMl)); + public GenericsAjaxResult editAutoMl(@RequestBody AutoMlVo autoMlVo) throws Exception { + return genericsSuccess(this.autoMLService.edit(autoMlVo)); } @GetMapping("/getAutoMlDetail") @ApiOperation("获取自动机器学习详细信息") - public GenericsAjaxResult getAutoMlDetail(@RequestParam("id") Long id){ + public GenericsAjaxResult getAutoMlDetail(@RequestParam("id") Long id) throws IOException { return genericsSuccess(this.autoMLService.getAutoMlDetail(id)); } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/AutoMl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/AutoMl.java index e37f6bbb..811168da 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/AutoMl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/AutoMl.java @@ -9,6 +9,7 @@ import io.swagger.annotations.ApiModelProperty; import lombok.Data; import java.util.Date; +import java.util.Map; @Data @JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy.class) @@ -25,9 +26,6 @@ public class AutoMl { @ApiModelProperty(value = "任务类型：classification或regression") private String taskType; - @ApiModelProperty(value = "数据集名称") - private String datasetName; - @ApiModelProperty(value = "搜索合适模型的时间限制（以秒为单位）。通过增加这个值，auto-sklearn有更高的机会找到更好的模型。默认3600，非必传。") private Integer timeLeftForThisTask; @@ -128,9 +126,6 @@ public class AutoMl { @ApiModelProperty(value = "文件夹存放配置输出和日志文件，默认/tmp/automl") private String tmpFolder; - @ApiModelProperty(value = "数据集csv文件路径") - private String dataCsv; - @ApiModelProperty(value = "数据集csv文件中哪几列是预测目标列，逗号分隔") private String targetColumns; @@ -182,6 +177,6 @@ public class AutoMl { private Date updateTime; - @TableField(exist = false) - private VersionVo versionVo; + private String dataset; + } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AutoMLService.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AutoMLService.java index 2b678382..78e25480 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AutoMLService.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AutoMLService.java @@ -1,22 +1,24 @@ package com.ruoyi.platform.service; import com.ruoyi.platform.domain.AutoMl; +import com.ruoyi.platform.vo.AutoMlVo; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageRequest; import org.springframework.web.multipart.MultipartFile; +import java.io.IOException; import java.util.Map; public interface AutoMLService { Page queryByPage(String mlName, PageRequest pageRequest); - AutoMl save(AutoMl autoMl) throws Exception; + AutoMl save(AutoMlVo autoMlVo) throws Exception; - String edit(AutoMl autoMl) throws Exception; + String edit(AutoMlVo autoMlVo) throws Exception; String delete(Long id); - AutoMl getAutoMlDetail(Long id); + AutoMlVo getAutoMlDetail(Long id) throws IOException; Map upload(MultipartFile file, String uuid) throws Exception; diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMLServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMLServiceImpl.java index f052f32e..40fd4ce2 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMLServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMLServiceImpl.java @@ -5,14 +5,14 @@ import com.ruoyi.platform.constant.Constant; import com.ruoyi.platform.domain.AutoMl; import com.ruoyi.platform.mapper.AutoMLDao; import com.ruoyi.platform.service.AutoMLService; -import com.ruoyi.platform.utils.DVCUtils; -import com.ruoyi.platform.utils.FileUtil; -import com.ruoyi.platform.utils.K8sClientUtil; +import com.ruoyi.platform.utils.*; +import com.ruoyi.platform.vo.AutoMlVo; import io.kubernetes.client.openapi.models.V1Pod; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageImpl; @@ -22,6 +22,7 @@ import org.springframework.web.multipart.MultipartFile; import javax.annotation.Resource; import java.io.File; +import java.io.IOException; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -59,39 +60,35 @@ public class AutoMLServiceImpl implements AutoMLService { } @Override - public AutoMl save(AutoMl autoMl) throws Exception { - AutoMl autoMlByName = autoMLDao.getAutoMlByName(autoMl.getMlName()); + public AutoMl save(AutoMlVo autoMlVo) throws Exception { + AutoMl autoMlByName = autoMLDao.getAutoMlByName(autoMlVo.getMlName()); if (autoMlByName != null) { throw new RuntimeException("实验名称已存在"); } - + AutoMl autoMl = new AutoMl(); + BeanUtils.copyProperties(autoMlVo, autoMl); String username = SecurityUtils.getLoginUser().getUsername(); autoMl.setCreateBy(username); autoMl.setUpdateBy(username); - - String sourcePath = autoMl.getVersionVo().getUrl(); - String rootPath = localPath + username + "/automl/" + autoMl.getMlName() + "/" + autoMl.getDatasetName(); - dvcUtils.moveFiles(sourcePath, rootPath); - autoMl.setDataCsv(rootPath + "/" + autoMl.getVersionVo().getFileName()); + String datasetJson = JacksonUtil.toJSONString(autoMlVo.getDataset()); + autoMl.setDataset(datasetJson); autoMLDao.save(autoMl); return autoMl; } @Override - public String edit(AutoMl autoMl) throws Exception { - AutoMl oldAutoMl = autoMLDao.getAutoMlByName(autoMl.getMlName()); - if (oldAutoMl != null && !oldAutoMl.getId().equals(autoMl.getId())) { + public String edit(AutoMlVo autoMlVo) throws Exception { + AutoMl oldAutoMl = autoMLDao.getAutoMlByName(autoMlVo.getMlName()); + if (oldAutoMl != null && !oldAutoMl.getId().equals(autoMlVo.getId())) { throw new RuntimeException("实验名称已存在"); } - - String username = SecurityUtils.getLoginUser().getUsername(); + AutoMl autoMl = new AutoMl(); + BeanUtils.copyProperties(autoMlVo, autoMl); +// String username = SecurityUtils.getLoginUser().getUsername(); + String username = "admin"; autoMl.setUpdateBy(username); - if (autoMl.getVersionVo() != null && StringUtils.isNotEmpty(autoMl.getVersionVo().getUrl())) { - String sourcePath = autoMl.getVersionVo().getUrl(); - String rootPath = localPath + username + "/automl/" + autoMl.getMlName() + "/" + autoMl.getDatasetName(); - dvcUtils.moveFiles(sourcePath, rootPath); - autoMl.setDataCsv(rootPath + "/" + autoMl.getVersionVo().getFileName()); - } + String datasetJson = JacksonUtil.toJSONString(autoMlVo.getDataset()); + autoMl.setDataset(datasetJson); autoMLDao.edit(autoMl); return "修改成功"; @@ -104,8 +101,7 @@ public class AutoMLServiceImpl implements AutoMLService { throw new RuntimeException("服务不存在"); } -// String username = SecurityUtils.getLoginUser().getUsername(); - String username = "admin"; + String username = SecurityUtils.getLoginUser().getUsername(); String createBy = autoMl.getCreateBy(); if (!(StringUtils.equals(username, "admin") || StringUtils.equals(username, createBy))) { throw new RuntimeException("无权限删除该服务"); @@ -116,8 +112,14 @@ public class AutoMLServiceImpl implements AutoMLService { } @Override - public AutoMl getAutoMlDetail(Long id) { - return autoMLDao.getAutoMlById(id); + public AutoMlVo getAutoMlDetail(Long id) throws IOException { + AutoMl autoMl = autoMLDao.getAutoMlById(id); + AutoMlVo autoMlVo = new AutoMlVo(); + BeanUtils.copyProperties(autoMl, autoMlVo); + if (StringUtils.isNotEmpty(autoMl.getDataset())) { + autoMlVo.setDataset(JsonUtils.jsonToMap(autoMl.getDataset())); + } + return autoMlVo; } @Override @@ -151,21 +153,11 @@ public class AutoMLServiceImpl implements AutoMLService { StringBuffer command = new StringBuffer(); command.append("nohup python /opt/automl.py --task_type " + autoMl.getTaskType()); - if (StringUtils.isNotEmpty(autoMl.getDataCsv())) { - command.append(" --data_csv " + autoMl.getDataCsv()); - } else { - throw new Exception("训练数据为空"); - } if (StringUtils.isNotEmpty(autoMl.getTargetColumns())) { command.append(" --target_columns " + autoMl.getTargetColumns()); } else { throw new Exception("目标列为空"); } - if (StringUtils.isNotEmpty(autoMl.getDatasetName())) { - command.append(" --dataset_name " + autoMl.getDatasetName()); - } else { - throw new Exception("数据集名称为空"); - } // String username = SecurityUtils.getLoginUser().getUsername().toLowerCase(); String username = "admin"; diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/AutoMlVo.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/AutoMlVo.java new file mode 100644 index 00000000..3fbe5cc4 --- /dev/null +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/AutoMlVo.java @@ -0,0 +1,183 @@ +package com.ruoyi.platform.vo; + +import com.baomidou.mybatisplus.annotation.TableField; +import com.fasterxml.jackson.databind.PropertyNamingStrategy; +import com.fasterxml.jackson.databind.annotation.JsonNaming; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import java.util.Date; +import java.util.Map; + +@Data +@JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy.class) +@ApiModel(description = "自动机器学习") +public class AutoMlVo { + private Long id; + + @ApiModelProperty(value = "实验名称") + private String mlName; + + @ApiModelProperty(value = "实验描述") + private String mlDescription; + + @ApiModelProperty(value = "任务类型：classification或regression") + private String taskType; + + @ApiModelProperty(value = "搜索合适模型的时间限制（以秒为单位）。通过增加这个值，auto-sklearn有更高的机会找到更好的模型。默认3600，非必传。") + private Integer timeLeftForThisTask; + + @ApiModelProperty(value = "单次调用机器学习模型的时间限制（以秒为单位）。如果机器学习算法运行超过时间限制，将终止模型拟合。将这个值设置得足够高，这样典型的机器学习算法就可以适用于训练数据。默认600，非必传。") + private Integer perRunTimeLimit; + + @ApiModelProperty(value = "集成模型数量，如果设置为0，则没有集成。默认50，非必传。") + private Integer ensembleSize; + + @ApiModelProperty(value = "设置为None将禁用集成构建，设置为SingleBest仅使用单个最佳模型而不是集成，设置为default，它将对单目标问题使用EnsembleSelection，对多目标问题使用MultiObjectiveDummyEnsemble。默认default，非必传。") + private String ensembleClass; + + @ApiModelProperty(value = "在构建集成时只考虑ensemble_nbest模型。这是受到了“最大限度地利用集成选择”中引入的库修剪概念的启发。这是独立于ensemble_class参数的，并且这个修剪步骤是在构造集成之前完成的。默认50，非必传。") + private Integer ensembleNbest; + + @ApiModelProperty(value = "定义在磁盘中保存的模型的最大数量。额外的模型数量将被永久删除。由于这个变量的性质，它设置了一个集成可以使用多少个模型的上限。必须是大于等于1的整数。如果设置为None，则所有模型都保留在磁盘上。默认50，非必传。") + private Integer maxModelsOnDisc; + + @ApiModelProperty(value = "随机种子，将决定输出文件名。默认1，非必传。") + private Integer seed; + + @ApiModelProperty(value = "机器学习算法的内存限制（MB）。如果auto-sklearn试图分配超过memory_limit MB，它将停止拟合机器学习算法。默认3072，非必传。") + private Integer memoryLimit; + + @ApiModelProperty(value = "如果为None，则使用所有可能的分类算法。否则，指定搜索中包含的步骤和组件。有关可用组件，请参见/pipeline/components//*。与参数exclude不兼容。多选，逗号分隔。包含：adaboost\n" + + "bernoulli_nb\n" + + "decision_tree\n" + + "extra_trees\n" + + "gaussian_nb\n" + + "gradient_boosting\n" + + "k_nearest_neighbors\n" + + "lda\n" + + "liblinear_svc\n" + + "libsvm_svc\n" + + "mlp\n" + + "multinomial_nb\n" + + "passive_aggressive\n" + + "qda\n" + + "random_forest\n" + + "sgd") + private String includeClassifier; + + @ApiModelProperty(value = "如果为None，则使用所有可能的特征预处理算法。否则，指定搜索中包含的步骤和组件。有关可用组件，请参见/pipeline/components//*。与参数exclude不兼容。多选，逗号分隔。包含：densifier\n" + + "extra_trees_preproc_for_classification\n" + + "extra_trees_preproc_for_regression\n" + + "fast_ica\n" + + "feature_agglomeration\n" + + "kernel_pca\n" + + "kitchen_sinks\n" + + "liblinear_svc_preprocessor\n" + + "no_preprocessing\n" + + "nystroem_sampler\n" + + "pca\n" + + "polynomial\n" + + "random_trees_embedding\n" + + "select_percentile_classification\n" + + "select_percentile_regression\n" + + "select_rates_classification\n" + + "select_rates_regression\n" + + "truncatedSVD") + private String includeFeaturePreprocessor; + + @ApiModelProperty(value = "如果为None，则使用所有可能的回归算法。否则，指定搜索中包含的步骤和组件。有关可用组件，请参见/pipeline/components//*。与参数exclude不兼容。多选，逗号分隔。包含：adaboost,\n" + + "ard_regression,\n" + + "decision_tree,\n" + + "extra_trees,\n" + + "gaussian_process,\n" + + "gradient_boosting,\n" + + "k_nearest_neighbors,\n" + + "liblinear_svr,\n" + + "libsvm_svr,\n" + + "mlp,\n" + + "random_forest,\n" + + "sgd") + private String includeRegressor; + + private String excludeClassifier; + + private String excludeRegressor; + + private String excludeFeaturePreprocessor; + + @ApiModelProperty(value = "测试集的比率，0到1之间") + private Float testSize; + + @ApiModelProperty(value = "如何处理过拟合，如果使用基于“cv”的方法或Splitter对象，可能需要使用resampling_strategy_arguments。holdout或crossValid") + private String resamplingStrategy; + + @ApiModelProperty(value = "重采样划分训练集和验证集，训练集的比率，0到1之间") + private Float trainSize; + + @ApiModelProperty(value = "拆分数据前是否进行shuffle") + private Boolean shuffle; + + @ApiModelProperty(value = "交叉验证的折数，当resamplingStrategy为crossValid时，此项必填，为整数") + private Integer folds; + + @ApiModelProperty(value = "文件夹存放配置输出和日志文件，默认/tmp/automl") + private String tmpFolder; + + @ApiModelProperty(value = "数据集csv文件中哪几列是预测目标列，逗号分隔") + private String targetColumns; + + @ApiModelProperty(value = "自定义指标名称") + private String metricName; + + @ApiModelProperty(value = "模型优化目标指标及权重，json格式。分类的指标包含：accuracy\n" + + "balanced_accuracy\n" + + "roc_auc\n" + + "average_precision\n" + + "log_loss\n" + + "precision_macro\n" + + "precision_micro\n" + + "precision_samples\n" + + "precision_weighted\n" + + "recall_macro\n" + + "recall_micro\n" + + "recall_samples\n" + + "recall_weighted\n" + + "f1_macro\n" + + "f1_micro\n" + + "f1_samples\n" + + "f1_weighted\n" + + "回归的指标包含：mean_absolute_error\n" + + "mean_squared_error\n" + + "root_mean_squared_error\n" + + "mean_squared_log_error\n" + + "median_absolute_error\n" + + "r2") + private String metrics; + + @ApiModelProperty(value = "指标优化方向，是越大越好还是越小越好") + private Boolean greaterIsBetter; + + @ApiModelProperty(value = "模型计算并打印指标") + private String scoringFunctions; + + private Integer state; + + private String runState; + + private Double progress; + + private String createBy; + + private Date createTime; + + private String updateBy; + + private Date updateTime; + + /** + * 对应数据集 + */ + private Map dataset; +} diff --git a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMLDaoMapper.xml b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMLDaoMapper.xml index e5fecd5a..73a37ff0 100644 --- a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMLDaoMapper.xml +++ b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMLDaoMapper.xml @@ -2,23 +2,26 @@ - insert into auto_ml(ml_name, ml_description, task_type, dataset_name, time_left_for_this_task, + insert into auto_ml(ml_name, ml_description, task_type, dataset, time_left_for_this_task, per_run_time_limit, ensemble_size, ensemble_class, ensemble_nbest, max_models_on_disc, seed, memory_limit, include_classifier, include_feature_preprocessor, include_regressor, exclude_classifier, exclude_regressor, exclude_feature_preprocessor, test_size, resampling_strategy, train_size, - shuffle, folds, data_csv, target_columns, metric_name, metrics,greater_is_better,scoring_functions,tmp_folder, - create_by,update_by) - values (#{autoMl.mlName}, #{autoMl.mlDescription}, #{autoMl.taskType}, #{autoMl.datasetName}, + shuffle, folds, target_columns, metric_name, metrics, greater_is_better, scoring_functions, + tmp_folder, + create_by, update_by) + values (#{autoMl.mlName}, #{autoMl.mlDescription}, #{autoMl.taskType}, #{autoMl.dataset}, #{autoMl.timeLeftForThisTask}, #{autoMl.perRunTimeLimit}, #{autoMl.ensembleSize}, #{autoMl.ensembleClass}, #{autoMl.ensembleNbest}, #{autoMl.maxModelsOnDisc}, #{autoMl.seed}, #{autoMl.memoryLimit}, #{autoMl.includeClassifier}, #{autoMl.includeFeaturePreprocessor}, #{autoMl.includeRegressor}, #{autoMl.excludeClassifier}, - #{autoMl.excludeRegressor}, #{autoMl.excludeFeaturePreprocessor}, #{autoMl.testSize}, #{autoMl.resamplingStrategy}, + #{autoMl.excludeRegressor}, #{autoMl.excludeFeaturePreprocessor}, #{autoMl.testSize}, + #{autoMl.resamplingStrategy}, #{autoMl.trainSize}, #{autoMl.shuffle}, - #{autoMl.folds}, #{autoMl.dataCsv}, - #{autoMl.targetColumns}, #{autoMl.metricName}, #{autoMl.metrics},#{autoMl.greaterIsBetter},#{autoMl.scoringFunctions},#{autoMl.tmpFolder}, + #{autoMl.folds}, + #{autoMl.targetColumns}, #{autoMl.metricName}, #{autoMl.metrics}, #{autoMl.greaterIsBetter}, + #{autoMl.scoringFunctions}, #{autoMl.tmpFolder}, #{autoMl.createBy}, #{autoMl.updateBy}) @@ -40,8 +43,8 @@ task_type = #{autoMl.taskType}, - - dataset_name = #{autoMl.datasetName}, + + dataset = #{autoMl.dataset}, time_left_for_this_task = #{autoMl.timeLeftForThisTask}, @@ -67,24 +70,13 @@ memory_limit = #{autoMl.memoryLimit}, - - include_classifier = #{autoMl.includeClassifier}, - - - include_feature_preprocessor = #{autoMl.includeFeaturePreprocessor}, - - - include_regressor = #{autoMl.includeRegressor}, - - - exclude_classifier = #{autoMl.excludeClassifier}, - - - exclude_regressor = #{autoMl.excludeRegressor}, - - - exclude_feature_preprocessor = #{autoMl.excludeFeaturePreprocessor}, - + include_classifier = #{autoMl.includeClassifier}, + include_feature_preprocessor = #{autoMl.includeFeaturePreprocessor}, + include_regressor = #{autoMl.includeRegressor}, + exclude_classifier = #{autoMl.excludeClassifier}, + exclude_regressor = #{autoMl.excludeRegressor}, + exclude_feature_preprocessor = #{autoMl.excludeFeaturePreprocessor}, + scoring_functions = #{autoMl.scoringFunctions}, test_size = #{autoMl.testSize}, @@ -100,9 +92,6 @@ folds = #{autoMl.folds}, - - data_csv = #{autoMl.dataCsv}, - tmp_folder = #{autoMl.tmpFolder}, @@ -112,12 +101,9 @@ metrics = #{autoMl.metrics}, - + greater_is_better = #{autoMl.greaterIsBetter}, - - scoring_functions = #{autoMl.scoringFunctions}, - target_columns = #{autoMl.targetColumns}, From a3fa7d4f1b5a12510f108c16fdbd55e96873f801 Mon Sep 17 00:00:00 2001 From: chenzhihang <709011834@qq.com> Date: Wed, 27 Nov 2024 10:17:51 +0800 Subject: [PATCH 10/18] =?UTF-8?q?=E8=87=AA=E5=8A=A8=E6=9C=BA=E5=99=A8?= =?UTF-8?q?=E5=AD=A6=E4=B9=A0=E5=BC=80=E5=8F=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...LController.java => AutoMlController.java} | 20 +++--- .../autoML/AutoMlInsController.java | 49 ++++++++++++++ .../com/ruoyi/platform/domain/AutoMlIns.java | 35 ++++++++++ .../mapper/{AutoMLDao.java => AutoMlDao.java} | 2 +- .../ruoyi/platform/mapper/AutoMlInsDao.java | 19 ++++++ .../platform/service/AutoMlInsService.java | 18 ++++++ ...{AutoMLService.java => AutoMlService.java} | 2 +- .../service/impl/AutoMlInsServiceImpl.java | 60 +++++++++++++++++ ...erviceImpl.java => AutoMlServiceImpl.java} | 30 ++++----- .../{AutoMLDaoMapper.xml => AutoMlDao.xml} | 2 +- .../managementPlatform/AutoMlInsDao.xml | 64 +++++++++++++++++++ 11 files changed, 272 insertions(+), 29 deletions(-) rename ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/autoML/{AutoMLController.java => AutoMlController.java} (80%) create mode 100644 ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/autoML/AutoMlInsController.java create mode 100644 ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/AutoMlIns.java rename ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/{AutoMLDao.java => AutoMlDao.java} (94%) create mode 100644 ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/AutoMlInsDao.java create mode 100644 ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AutoMlInsService.java rename ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/{AutoMLService.java => AutoMlService.java} (95%) create mode 100644 ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMlInsServiceImpl.java rename ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/{AutoMLServiceImpl.java => AutoMlServiceImpl.java} (92%) rename ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/{AutoMLDaoMapper.xml => AutoMlDao.xml} (99%) create mode 100644 ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMlInsDao.xml diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/autoML/AutoMLController.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/autoML/AutoMlController.java similarity index 80% rename from ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/autoML/AutoMLController.java rename to ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/autoML/AutoMlController.java index d859e491..981b9fb1 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/autoML/AutoMLController.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/autoML/AutoMlController.java @@ -4,7 +4,7 @@ import com.ruoyi.common.core.web.controller.BaseController; import com.ruoyi.common.core.web.domain.AjaxResult; import com.ruoyi.common.core.web.domain.GenericsAjaxResult; import com.ruoyi.platform.domain.AutoMl; -import com.ruoyi.platform.service.AutoMLService; +import com.ruoyi.platform.service.AutoMlService; import com.ruoyi.platform.vo.AutoMlVo; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; @@ -19,10 +19,10 @@ import java.io.IOException; @RestController @RequestMapping("autoML") @Api("自动机器学习") -public class AutoMLController extends BaseController { +public class AutoMlController extends BaseController { @Resource - private AutoMLService autoMLService; + private AutoMlService autoMlService; @GetMapping @ApiOperation("分页查询") @@ -30,42 +30,42 @@ public class AutoMLController extends BaseController { @RequestParam("size") int size, @RequestParam(value = "ml_name", required = false) String mlName) { PageRequest pageRequest = PageRequest.of(page, size); - return genericsSuccess(this.autoMLService.queryByPage(mlName, pageRequest)); + return genericsSuccess(this.autoMlService.queryByPage(mlName, pageRequest)); } @PostMapping @ApiOperation("新增自动机器学习") public GenericsAjaxResult addAutoMl(@RequestBody AutoMlVo autoMlVo) throws Exception { - return genericsSuccess(this.autoMLService.save(autoMlVo)); + return genericsSuccess(this.autoMlService.save(autoMlVo)); } @PutMapping @ApiOperation("编辑自动机器学习") public GenericsAjaxResult editAutoMl(@RequestBody AutoMlVo autoMlVo) throws Exception { - return genericsSuccess(this.autoMLService.edit(autoMlVo)); + return genericsSuccess(this.autoMlService.edit(autoMlVo)); } @GetMapping("/getAutoMlDetail") @ApiOperation("获取自动机器学习详细信息") public GenericsAjaxResult getAutoMlDetail(@RequestParam("id") Long id) throws IOException { - return genericsSuccess(this.autoMLService.getAutoMlDetail(id)); + return genericsSuccess(this.autoMlService.getAutoMlDetail(id)); } @DeleteMapping("{id}") @ApiOperation("删除自动机器学习") public GenericsAjaxResult deleteAutoMl(@PathVariable("id") Long id) { - return genericsSuccess(this.autoMLService.delete(id)); + return genericsSuccess(this.autoMlService.delete(id)); } @CrossOrigin(origins = "*", allowedHeaders = "*") @PostMapping("/upload") @ApiOperation(value = "上传数据文件csv", notes = "上传数据文件csv，并将信息存入数据库。") public AjaxResult upload(@RequestParam("file") MultipartFile file, @RequestParam("uuid") String uuid) throws Exception { - return AjaxResult.success(this.autoMLService.upload(file, uuid)); + return AjaxResult.success(this.autoMlService.upload(file, uuid)); } @PostMapping("{id}") @ApiOperation("运行自动机器学习实验") public GenericsAjaxResult runAutoML(@PathVariable("id") Long id) throws Exception { - return genericsSuccess(this.autoMLService.runAutoMlIns(id)); + return genericsSuccess(this.autoMlService.runAutoMlIns(id)); } } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/autoML/AutoMlInsController.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/autoML/AutoMlInsController.java new file mode 100644 index 00000000..3aaba610 --- /dev/null +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/autoML/AutoMlInsController.java @@ -0,0 +1,49 @@ +package com.ruoyi.platform.controller.autoML; + +import com.ruoyi.common.core.web.controller.BaseController; +import com.ruoyi.common.core.web.domain.GenericsAjaxResult; +import com.ruoyi.platform.domain.AutoMlIns; +import com.ruoyi.platform.service.AutoMlInsService; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageRequest; +import org.springframework.web.bind.annotation.*; + +import javax.annotation.Resource; +import java.io.IOException; +import java.util.List; + +@RestController +@RequestMapping("autoMLIns") +@Api("自动机器学习实验实例") +public class AutoMlInsController extends BaseController { + + @Resource + private AutoMlInsService autoMLInsService; + + @GetMapping + @ApiOperation("分页查询") + public GenericsAjaxResult> queryByPage(AutoMlIns autoMlIns, int page, int size) throws IOException { + PageRequest pageRequest = PageRequest.of(page, size); + return genericsSuccess(this.autoMLInsService.queryByPage(autoMlIns, pageRequest)); + } + + @PostMapping + @ApiOperation("新增实验实例") + public GenericsAjaxResult add(@RequestBody AutoMlIns autoMlIns) { + return genericsSuccess(this.autoMLInsService.insert(autoMlIns)); + } + + @DeleteMapping("{id}") + @ApiOperation("删除实验实例") + public GenericsAjaxResult deleteById(@PathVariable("id") Long id) { + return genericsSuccess(this.autoMLInsService.removeById(id)); + } + + @DeleteMapping("batchDelete") + @ApiOperation("批量删除实验实例") + public GenericsAjaxResult batchDelete(@RequestBody List ids) throws Exception { + return genericsSuccess(this.autoMLInsService.batchDelete(ids)); + } +} diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/AutoMlIns.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/AutoMlIns.java new file mode 100644 index 00000000..b5d023ae --- /dev/null +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/AutoMlIns.java @@ -0,0 +1,35 @@ +package com.ruoyi.platform.domain; + +import com.fasterxml.jackson.databind.PropertyNamingStrategy; +import com.fasterxml.jackson.databind.annotation.JsonNaming; +import io.swagger.annotations.ApiModel; +import lombok.Data; + +import java.util.Date; + +@Data +@JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy.class) +@ApiModel(description = "自动机器学习实验实例") +public class AutoMlIns { + private Long id; + + private Long autoMlId; + + private String modelPath; + + private String imgPath; + + private Integer state; + + private String status; + + private String nodeStatus; + + private String nodeResult; + + private String param; + + private Date createTime; + + private Date updateTime; +} diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/AutoMLDao.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/AutoMlDao.java similarity index 94% rename from ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/AutoMLDao.java rename to ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/AutoMlDao.java index f99f2156..eeffd683 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/AutoMLDao.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/AutoMlDao.java @@ -6,7 +6,7 @@ import org.springframework.data.domain.Pageable; import java.util.List; -public interface AutoMLDao { +public interface AutoMlDao { long count(@Param("mlName") String mlName); List queryByPage(@Param("mlName") String mlName, @Param("pageable") Pageable pageable); diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/AutoMlInsDao.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/AutoMlInsDao.java new file mode 100644 index 00000000..7468c8f3 --- /dev/null +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/AutoMlInsDao.java @@ -0,0 +1,19 @@ +package com.ruoyi.platform.mapper; + +import com.ruoyi.platform.domain.AutoMlIns; +import org.apache.ibatis.annotations.Param; +import org.springframework.data.domain.Pageable; + +import java.util.List; + +public interface AutoMlInsDao { + long count(@Param("autoMlIns") AutoMlIns autoMlIns); + + List queryAllByLimit(@Param("autoMlIns") AutoMlIns autoMlIns, @Param("pageable") Pageable pageable); + + int insert(@Param("autoMlIns") AutoMlIns autoMlIns); + + int update(@Param("autoMlIns") AutoMlIns autoMlIns); + + AutoMlIns queryById(@Param("id") Long id); +} diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AutoMlInsService.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AutoMlInsService.java new file mode 100644 index 00000000..4b7a8601 --- /dev/null +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AutoMlInsService.java @@ -0,0 +1,18 @@ +package com.ruoyi.platform.service; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageRequest; +import com.ruoyi.platform.domain.AutoMlIns; + +import java.io.IOException; +import java.util.List; + +public interface AutoMlInsService { + + Page queryByPage(AutoMlIns autoMlIns, PageRequest pageRequest) throws IOException; + + AutoMlIns insert(AutoMlIns autoMlIns); + + String removeById(Long id); + + String batchDelete(List ids); +} diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AutoMLService.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AutoMlService.java similarity index 95% rename from ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AutoMLService.java rename to ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AutoMlService.java index 78e25480..a2bad796 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AutoMLService.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AutoMlService.java @@ -9,7 +9,7 @@ import org.springframework.web.multipart.MultipartFile; import java.io.IOException; import java.util.Map; -public interface AutoMLService { +public interface AutoMlService { Page queryByPage(String mlName, PageRequest pageRequest); AutoMl save(AutoMlVo autoMlVo) throws Exception; diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMlInsServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMlInsServiceImpl.java new file mode 100644 index 00000000..5b190226 --- /dev/null +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMlInsServiceImpl.java @@ -0,0 +1,60 @@ +package com.ruoyi.platform.service.impl; + +import com.ruoyi.platform.constant.Constant; +import com.ruoyi.platform.domain.AutoMlIns; +import com.ruoyi.platform.mapper.AutoMlInsDao; +import com.ruoyi.platform.service.AutoMlInsService; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageImpl; +import org.springframework.data.domain.PageRequest; +import org.springframework.stereotype.Service; + +import javax.annotation.Resource; +import java.io.IOException; +import java.util.List; + +@Service +public class AutoMlInsServiceImpl implements AutoMlInsService { + @Resource + private AutoMlInsDao autoMlInsDao; + + + @Override + public Page queryByPage(AutoMlIns autoMlIns, PageRequest pageRequest) throws IOException { + long total = this.autoMlInsDao.count(autoMlIns); + List autoMlInsList = this.autoMlInsDao.queryAllByLimit(autoMlIns, pageRequest); + return new PageImpl<>(autoMlInsList, pageRequest, total); + } + + @Override + public AutoMlIns insert(AutoMlIns autoMlIns) { + this.autoMlInsDao.insert(autoMlIns); + return autoMlIns; + } + + @Override + public String removeById(Long id) { + AutoMlIns autoMlIns = autoMlInsDao.queryById(id); + if (autoMlIns == null) { + return "实验实例不存在"; + } + autoMlIns.setState(Constant.State_invalid); + int update = autoMlInsDao.update(autoMlIns); + if (update > 0) { + return "删除成功"; + } else { + return "删除失败"; + } + } + + @Override + public String batchDelete(List ids) { + for (Long id : ids) { + String result = removeById(id); + if (!"删除成功".equals(result)) { + return result; + } + } + return "删除成功"; + } +} diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMLServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMlServiceImpl.java similarity index 92% rename from ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMLServiceImpl.java rename to ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMlServiceImpl.java index 40fd4ce2..7669232a 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMLServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMlServiceImpl.java @@ -3,8 +3,8 @@ package com.ruoyi.platform.service.impl; import com.ruoyi.common.security.utils.SecurityUtils; import com.ruoyi.platform.constant.Constant; import com.ruoyi.platform.domain.AutoMl; -import com.ruoyi.platform.mapper.AutoMLDao; -import com.ruoyi.platform.service.AutoMLService; +import com.ruoyi.platform.mapper.AutoMlDao; +import com.ruoyi.platform.service.AutoMlService; import com.ruoyi.platform.utils.*; import com.ruoyi.platform.vo.AutoMlVo; import io.kubernetes.client.openapi.models.V1Pod; @@ -29,7 +29,7 @@ import java.util.Map; import java.util.concurrent.CompletableFuture; @Service("autoMLService") -public class AutoMLServiceImpl implements AutoMLService { +public class AutoMlServiceImpl implements AutoMlService { @Value("${harbor.serviceNS}") private String serviceNS; @Value("${dockerpush.proxyUrl}") @@ -46,22 +46,20 @@ public class AutoMLServiceImpl implements AutoMLService { private static final Logger logger = LoggerFactory.getLogger(ModelsServiceImpl.class); @Resource - private AutoMLDao autoMLDao; + private AutoMlDao autoMlDao; @Resource private K8sClientUtil k8sClientUtil; - @Resource - private DVCUtils dvcUtils; @Override public Page queryByPage(String mlName, PageRequest pageRequest) { - long total = autoMLDao.count(mlName); - List autoMls = autoMLDao.queryByPage(mlName, pageRequest); + long total = autoMlDao.count(mlName); + List autoMls = autoMlDao.queryByPage(mlName, pageRequest); return new PageImpl<>(autoMls, pageRequest, total); } @Override public AutoMl save(AutoMlVo autoMlVo) throws Exception { - AutoMl autoMlByName = autoMLDao.getAutoMlByName(autoMlVo.getMlName()); + AutoMl autoMlByName = autoMlDao.getAutoMlByName(autoMlVo.getMlName()); if (autoMlByName != null) { throw new RuntimeException("实验名称已存在"); } @@ -72,13 +70,13 @@ public class AutoMLServiceImpl implements AutoMLService { autoMl.setUpdateBy(username); String datasetJson = JacksonUtil.toJSONString(autoMlVo.getDataset()); autoMl.setDataset(datasetJson); - autoMLDao.save(autoMl); + autoMlDao.save(autoMl); return autoMl; } @Override public String edit(AutoMlVo autoMlVo) throws Exception { - AutoMl oldAutoMl = autoMLDao.getAutoMlByName(autoMlVo.getMlName()); + AutoMl oldAutoMl = autoMlDao.getAutoMlByName(autoMlVo.getMlName()); if (oldAutoMl != null && !oldAutoMl.getId().equals(autoMlVo.getId())) { throw new RuntimeException("实验名称已存在"); } @@ -90,13 +88,13 @@ public class AutoMLServiceImpl implements AutoMLService { String datasetJson = JacksonUtil.toJSONString(autoMlVo.getDataset()); autoMl.setDataset(datasetJson); - autoMLDao.edit(autoMl); + autoMlDao.edit(autoMl); return "修改成功"; } @Override public String delete(Long id) { - AutoMl autoMl = autoMLDao.getAutoMlById(id); + AutoMl autoMl = autoMlDao.getAutoMlById(id); if (autoMl == null) { throw new RuntimeException("服务不存在"); } @@ -108,12 +106,12 @@ public class AutoMLServiceImpl implements AutoMLService { } autoMl.setState(Constant.State_invalid); - return autoMLDao.edit(autoMl) > 0 ? "删除成功" : "删除失败"; + return autoMlDao.edit(autoMl) > 0 ? "删除成功" : "删除失败"; } @Override public AutoMlVo getAutoMlDetail(Long id) throws IOException { - AutoMl autoMl = autoMLDao.getAutoMlById(id); + AutoMl autoMl = autoMlDao.getAutoMlById(id); AutoMlVo autoMlVo = new AutoMlVo(); BeanUtils.copyProperties(autoMl, autoMlVo); if (StringUtils.isNotEmpty(autoMl.getDataset())) { @@ -146,7 +144,7 @@ public class AutoMLServiceImpl implements AutoMLService { @Override public String runAutoMlIns(Long id) throws Exception { - AutoMl autoMl = autoMLDao.getAutoMlById(id); + AutoMl autoMl = autoMlDao.getAutoMlById(id); if (autoMl == null) { throw new Exception("开发环境配置不存在"); } diff --git a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMLDaoMapper.xml b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMlDao.xml similarity index 99% rename from ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMLDaoMapper.xml rename to ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMlDao.xml index 73a37ff0..b6a3c4e7 100644 --- a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMLDaoMapper.xml +++ b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMlDao.xml @@ -1,6 +1,6 @@ - + insert into auto_ml(ml_name, ml_description, task_type, dataset, time_left_for_this_task, per_run_time_limit, ensemble_size, ensemble_class, ensemble_nbest, max_models_on_disc, seed, diff --git a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMlInsDao.xml b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMlInsDao.xml new file mode 100644 index 00000000..7aa36bdf --- /dev/null +++ b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMlInsDao.xml @@ -0,0 +1,64 @@ + + + + + + insert into auto_ml_ins(auto_ml_id, model_path, img_path, node_status, node_result, param) + values (#{autoMlIns.autoMlId}, #{autoMlIns.modelPath}, #{autoMlIns.imgPath}, #{autoMlIns.nodeStatus}, + #{autoMlIns.nodeResult}, #{autoMlIns.param}) + + + + update auto_ml_ins + + + model_path = #{autoMlIns.modelPath}, + + + img_path = #{autoMlIns.imgPath}, + + + status = #{autoMlIns.status}, + + + node_status = #{autoMlIns.nodeStatus}, + + + node_result = #{autoMlIns.nodeResult}, + + + state = #{autoMlIns.state}, + + + + + + + + + + \ No newline at end of file From c75fa47ce0b1ed259b241a20c29e317b602786bc Mon Sep 17 00:00:00 2001 From: chenzhihang <709011834@qq.com> Date: Wed, 27 Nov 2024 11:15:13 +0800 Subject: [PATCH 11/18] =?UTF-8?q?=E8=87=AA=E5=8A=A8=E6=9C=BA=E5=99=A8?= =?UTF-8?q?=E5=AD=A6=E4=B9=A0=E5=BC=80=E5=8F=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ruoyi/platform/controller/autoML/AutoMlInsController.java | 2 +- .../src/main/java/com/ruoyi/platform/domain/AutoMlIns.java | 2 ++ .../main/resources/mapper/managementPlatform/AutoMlInsDao.xml | 4 ++-- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/autoML/AutoMlInsController.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/autoML/AutoMlInsController.java index 3aaba610..6d7b68d3 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/autoML/AutoMlInsController.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/autoML/AutoMlInsController.java @@ -43,7 +43,7 @@ public class AutoMlInsController extends BaseController { @DeleteMapping("batchDelete") @ApiOperation("批量删除实验实例") - public GenericsAjaxResult batchDelete(@RequestBody List ids) throws Exception { + public GenericsAjaxResult batchDelete(@RequestBody List ids) { return genericsSuccess(this.autoMLInsService.batchDelete(ids)); } } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/AutoMlIns.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/AutoMlIns.java index b5d023ae..d31711bd 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/AutoMlIns.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/AutoMlIns.java @@ -29,6 +29,8 @@ public class AutoMlIns { private String param; + private String source; + private Date createTime; private Date updateTime; diff --git a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMlInsDao.xml b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMlInsDao.xml index 7aa36bdf..05168691 100644 --- a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMlInsDao.xml +++ b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMlInsDao.xml @@ -3,9 +3,9 @@ - insert into auto_ml_ins(auto_ml_id, model_path, img_path, node_status, node_result, param) + insert into auto_ml_ins(auto_ml_id, model_path, img_path, node_status, node_result, param, source) values (#{autoMlIns.autoMlId}, #{autoMlIns.modelPath}, #{autoMlIns.imgPath}, #{autoMlIns.nodeStatus}, - #{autoMlIns.nodeResult}, #{autoMlIns.param}) + #{autoMlIns.nodeResult}, #{autoMlIns.param}, #{autoMlIns.source}) From 2f5c3d447645426a8284b80898a5ccf8aa849092 Mon Sep 17 00:00:00 2001 From: chenzhihang <709011834@qq.com> Date: Thu, 28 Nov 2024 15:01:43 +0800 Subject: [PATCH 12/18] =?UTF-8?q?=E8=87=AA=E5=8A=A8=E6=9C=BA=E5=99=A8?= =?UTF-8?q?=E5=AD=A6=E4=B9=A0=E5=BC=80=E5=8F=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../controller/autoML/AutoMlController.java | 2 +- .../autoML/AutoMlInsController.java | 6 + .../com/ruoyi/platform/domain/AutoMl.java | 2 + .../com/ruoyi/platform/domain/AutoMlIns.java | 7 + .../ruoyi/platform/mapper/AutoMlInsDao.java | 4 + .../scheduling/AutoMlInsStatusTask.java | 97 ++++++++++ .../platform/service/AutoMlInsService.java | 6 + .../service/impl/AutoMlInsServiceImpl.java | 168 +++++++++++++++++- .../service/impl/AutoMlServiceImpl.java | 157 +++++++--------- .../com/ruoyi/platform/vo/AutoMlParamVo.java | 155 ++++++++++++++++ .../mapper/managementPlatform/AutoMlDao.xml | 6 +- .../managementPlatform/AutoMlInsDao.xml | 24 ++- 12 files changed, 531 insertions(+), 103 deletions(-) create mode 100644 ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/scheduling/AutoMlInsStatusTask.java create mode 100644 ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/AutoMlParamVo.java diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/autoML/AutoMlController.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/autoML/AutoMlController.java index 981b9fb1..9c1028bb 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/autoML/AutoMlController.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/autoML/AutoMlController.java @@ -63,7 +63,7 @@ public class AutoMlController extends BaseController { return AjaxResult.success(this.autoMlService.upload(file, uuid)); } - @PostMapping("{id}") + @PostMapping("/run/{id}") @ApiOperation("运行自动机器学习实验") public GenericsAjaxResult runAutoML(@PathVariable("id") Long id) throws Exception { return genericsSuccess(this.autoMlService.runAutoMlIns(id)); diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/autoML/AutoMlInsController.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/autoML/AutoMlInsController.java index 6d7b68d3..125fd668 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/autoML/AutoMlInsController.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/autoML/AutoMlInsController.java @@ -46,4 +46,10 @@ public class AutoMlInsController extends BaseController { public GenericsAjaxResult batchDelete(@RequestBody List ids) { return genericsSuccess(this.autoMLInsService.batchDelete(ids)); } + + @PutMapping("{id}") + @ApiOperation("终止实验实例") + public GenericsAjaxResult terminateAutoMlIns(@PathVariable("id") Long id) { + return genericsSuccess(this.autoMLInsService.terminateAutoMlIns(id)); + } } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/AutoMl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/AutoMl.java index 811168da..a0f130cc 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/AutoMl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/AutoMl.java @@ -179,4 +179,6 @@ public class AutoMl { private String dataset; + @ApiModelProperty(value = "状态列表") + private String statusList; } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/AutoMlIns.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/AutoMlIns.java index d31711bd..ba425dcf 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/AutoMlIns.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/AutoMlIns.java @@ -3,6 +3,7 @@ package com.ruoyi.platform.domain; import com.fasterxml.jackson.databind.PropertyNamingStrategy; import com.fasterxml.jackson.databind.annotation.JsonNaming; import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; import lombok.Data; import java.util.Date; @@ -31,6 +32,12 @@ public class AutoMlIns { private String source; + @ApiModelProperty(value = "Argo实例名称") + private String argoInsName; + + @ApiModelProperty(value = "Argo命名空间") + private String argoInsNs; + private Date createTime; private Date updateTime; diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/AutoMlInsDao.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/AutoMlInsDao.java index 7468c8f3..7fcc3ca9 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/AutoMlInsDao.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/AutoMlInsDao.java @@ -11,9 +11,13 @@ public interface AutoMlInsDao { List queryAllByLimit(@Param("autoMlIns") AutoMlIns autoMlIns, @Param("pageable") Pageable pageable); + List getByAutoMlId(@Param("autoMlId") Long AutoMlId); + int insert(@Param("autoMlIns") AutoMlIns autoMlIns); int update(@Param("autoMlIns") AutoMlIns autoMlIns); AutoMlIns queryById(@Param("id") Long id); + + List queryByAutoMlInsIsNotTerminated(); } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/scheduling/AutoMlInsStatusTask.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/scheduling/AutoMlInsStatusTask.java new file mode 100644 index 00000000..5c94284f --- /dev/null +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/scheduling/AutoMlInsStatusTask.java @@ -0,0 +1,97 @@ +package com.ruoyi.platform.scheduling; + +import com.ruoyi.platform.domain.AutoMl; +import com.ruoyi.platform.domain.AutoMlIns; +import com.ruoyi.platform.mapper.AutoMlDao; +import com.ruoyi.platform.mapper.AutoMlInsDao; +import com.ruoyi.platform.service.AutoMlInsService; +import org.apache.commons.lang3.StringUtils; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import javax.annotation.Resource; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +@Component() +public class AutoMlInsStatusTask { + + @Resource + private AutoMlInsService autoMlInsService; + + @Resource + private AutoMlInsDao autoMlInsDao; + + @Resource + private AutoMlDao autoMlDao; + + private List autoMlIds = new ArrayList<>(); + + @Scheduled(cron = "0/30 * * * * ?") // 每30S执行一次 + public void executeAutoMlInsStatus() throws Exception { + // 首先查到所有非终止态的实验实例 + List autoMlInsList = autoMlInsService.queryByAutoMlInsIsNotTerminated(); + + // 去argo查询状态 + List updateList = new ArrayList<>(); + if (autoMlInsList != null && autoMlInsList.size() > 0) { + for (AutoMlIns autoMlIns : autoMlInsList) { + //当原本状态为null或非终止态时才调用argo接口 + try { + autoMlIns = autoMlInsService.queryStatusFromArgo(autoMlIns); + } catch (Exception e) { + autoMlIns.setStatus("Failed"); + } + // 线程安全的添加操作 + synchronized (autoMlIds) { + autoMlIds.add(autoMlIns.getAutoMlId()); + } + updateList.add(autoMlIns); + } + if (updateList.size() > 0) { + for (AutoMlIns autoMlIns : updateList) { + autoMlInsDao.update(autoMlIns); + } + } + } + } + + @Scheduled(cron = "0/30 * * * * ?") // / 每30S执行一次 + public void executeAutoMlStatus() throws Exception { + if (autoMlIds.size() == 0) { + return; + } + // 存储需要更新的实验对象列表 + List updateAutoMls = new ArrayList<>(); + for (Long autoMlId : autoMlIds) { + // 获取当前实验的所有实例列表 + List insList = autoMlInsDao.getByAutoMlId(autoMlId); + List statusList = new ArrayList<>(); + // 更新实验状态列表 + for (int i = 0; i < insList.size(); i++) { + statusList.add(insList.get(i).getStatus()); + } + String subStatus = statusList.toString().substring(1, statusList.toString().length() - 1); + AutoMl autoMl = autoMlDao.getAutoMlById(autoMlId); + if (!StringUtils.equals(autoMl.getStatusList(), subStatus)) { + autoMl.setStatusList(subStatus); + updateAutoMls.add(autoMl); + autoMlDao.edit(autoMl); + } + } + + if (!updateAutoMls.isEmpty()) { + // 使用Iterator进行安全的删除操作 + Iterator iterator = autoMlIds.iterator(); + while (iterator.hasNext()) { + Long autoMlId = iterator.next(); + for (AutoMl autoMl : updateAutoMls) { + if (autoMl.getId().equals(autoMlId)) { + iterator.remove(); + } + } + } + } + } +} diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AutoMlInsService.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AutoMlInsService.java index 4b7a8601..d4956a22 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AutoMlInsService.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AutoMlInsService.java @@ -15,4 +15,10 @@ public interface AutoMlInsService { String removeById(Long id); String batchDelete(List ids); + + List queryByAutoMlInsIsNotTerminated(); + + AutoMlIns queryStatusFromArgo(AutoMlIns autoMlIns); + + boolean terminateAutoMlIns(Long id); } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMlInsServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMlInsServiceImpl.java index 5b190226..3d11123e 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMlInsServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMlInsServiceImpl.java @@ -4,6 +4,11 @@ import com.ruoyi.platform.constant.Constant; import com.ruoyi.platform.domain.AutoMlIns; import com.ruoyi.platform.mapper.AutoMlInsDao; import com.ruoyi.platform.service.AutoMlInsService; +import com.ruoyi.platform.utils.DateUtils; +import com.ruoyi.platform.utils.HttpUtils; +import com.ruoyi.platform.utils.JsonUtils; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Value; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageImpl; import org.springframework.data.domain.PageRequest; @@ -11,10 +16,17 @@ import org.springframework.stereotype.Service; import javax.annotation.Resource; import java.io.IOException; -import java.util.List; +import java.util.*; @Service public class AutoMlInsServiceImpl implements AutoMlInsService { + @Value("${argo.url}") + private String argoUrl; + @Value("${argo.workflowStatus}") + private String argoWorkflowStatus; + @Value("${argo.workflowTermination}") + private String argoWorkflowTermination; + @Resource private AutoMlInsDao autoMlInsDao; @@ -38,6 +50,14 @@ public class AutoMlInsServiceImpl implements AutoMlInsService { if (autoMlIns == null) { return "实验实例不存在"; } + + if (StringUtils.isEmpty(autoMlIns.getStatus())) { + autoMlIns = queryStatusFromArgo(autoMlIns); + } + if (StringUtils.equals(autoMlIns.getStatus(), "Running")) { + return "实验实例正在运行，不可删除"; + } + autoMlIns.setState(Constant.State_invalid); int update = autoMlInsDao.update(autoMlIns); if (update > 0) { @@ -57,4 +77,150 @@ public class AutoMlInsServiceImpl implements AutoMlInsService { } return "删除成功"; } + + @Override + public List queryByAutoMlInsIsNotTerminated() { + return autoMlInsDao.queryByAutoMlInsIsNotTerminated(); + } + + @Override + public AutoMlIns queryStatusFromArgo(AutoMlIns ins) { + String namespace = ins.getArgoInsNs(); + String name = ins.getArgoInsName(); + Long id = ins.getId(); + + // 创建请求数据map + Map requestData = new HashMap<>(); + requestData.put("namespace", namespace); + requestData.put("name", name); + + // 创建发送数据map，将请求数据作为"data"键的值 + Map res = new HashMap<>(); + res.put("data", requestData); + + try { + // 发送POST请求到Argo工作流状态查询接口，并将请求数据转换为JSON + String req = HttpUtils.sendPost(argoUrl + argoWorkflowStatus, null, JsonUtils.mapToJson(res)); + // 检查响应是否为空或无内容 + if (req == null || StringUtils.isEmpty(req)) { + throw new RuntimeException("工作流状态响应为空。"); + } + // 将响应的JSON字符串转换为Map对象 + Map runResMap = JsonUtils.jsonToMap(req); + // 从响应Map中获取"data"部分 + Map data = (Map) runResMap.get("data"); + if (data == null || data.isEmpty()) { + throw new RuntimeException("工作流数据为空."); + } + // 从"data"中获取"status"部分，并返回"phase"的值 + Map status = (Map) data.get("status"); + if (status == null || status.isEmpty()) { + throw new RuntimeException("工作流状态为空。"); + } + + //解析流水线结束时间 + String finishedAtString = (String) status.get("finishedAt"); + if (finishedAtString != null && !finishedAtString.isEmpty()) { + Date finishTime = DateUtils.convertUTCtoShanghaiDate(finishedAtString); + ins.setUpdateTime(finishTime); + } + + // 解析nodes字段,提取节点状态并转换为JSON字符串 + Map nodes = (Map) status.get("nodes"); + Map modifiedNodes = new LinkedHashMap<>(); + if (nodes != null) { + for (Map.Entry nodeEntry : nodes.entrySet()) { + Map nodeDetails = (Map) nodeEntry.getValue(); + String templateName = (String) nodeDetails.get("displayName"); + modifiedNodes.put(templateName, nodeDetails); + } + } + + String nodeStatusJson = JsonUtils.mapToJson(modifiedNodes); + ins.setNodeStatus(nodeStatusJson); + + //终止态为终止不改 + if (!StringUtils.equals(ins.getStatus(), "Terminated")) { + ins.setStatus(StringUtils.isNotEmpty((String) status.get("phase")) ? (String) status.get("phase") : "Pending"); + } + if (StringUtils.equals(ins.getStatus(), "Error")) { + ins.setStatus("Failed"); + } + return ins; + } catch (Exception e) { + throw new RuntimeException("查询状态失败: " + e.getMessage(), e); + } + } + + @Override + public boolean terminateAutoMlIns(Long id) { + AutoMlIns autoMlIns = autoMlInsDao.queryById(id); + if (autoMlIns == null) { + throw new IllegalStateException("实验实例未查询到，id: " + id); + } + + String currentStatus = autoMlIns.getStatus(); + String name = autoMlIns.getArgoInsName(); + String namespace = autoMlIns.getArgoInsNs(); + + // 获取当前状态，如果为空，则从Argo查询 + if (StringUtils.isEmpty(currentStatus)) { + currentStatus = queryStatusFromArgo(autoMlIns).getStatus(); + } + // 只有状态是"Running"时才能终止实例 + if (!currentStatus.equalsIgnoreCase(Constant.Running)) { + return false; // 如果不是"Running"状态，则不执行终止操作 + } + + // 创建请求数据map + Map requestData = new HashMap<>(); + requestData.put("namespace", namespace); + requestData.put("name", name); + // 创建发送数据map，将请求数据作为"data"键的值 + Map res = new HashMap<>(); + res.put("data", requestData); + + try { + // 发送POST请求到Argo工作流状态查询接口，并将请求数据转换为JSON + String req = HttpUtils.sendPost(argoUrl + argoWorkflowTermination, null, JsonUtils.mapToJson(res)); + // 检查响应是否为空或无内容 + if (StringUtils.isEmpty(req)) { + throw new RuntimeException("终止响应内容为空。"); + + } + // 将响应的JSON字符串转换为Map对象 + Map runResMap = JsonUtils.jsonToMap(req); + // 从响应Map中直接获取"errCode"的值 + Integer errCode = (Integer) runResMap.get("errCode"); + if (errCode != null && errCode == 0) { + //更新autoMlIns,确保状态更新被保存到数据库 + AutoMlIns ins = queryStatusFromArgo(autoMlIns); + String nodeStatus = ins.getNodeStatus(); + Map nodeMap = JsonUtils.jsonToMap(nodeStatus); + + // 遍历 map + for (Map.Entry entry : nodeMap.entrySet()) { + // 获取每个 Map 中的值并强制转换为 Map + Map innerMap = (Map) entry.getValue(); + // 检查 phase 的值 + if (innerMap.containsKey("phase")) { + String phaseValue = (String) innerMap.get("phase"); + // 如果值不等于 Succeeded，则赋值为 Failed + if (!StringUtils.equals("Succeeded", phaseValue)) { + innerMap.put("phase", "Failed"); + } + } + } + ins.setNodeStatus(JsonUtils.mapToJson(nodeMap)); + ins.setStatus(Constant.Terminated); + ins.setUpdateTime(new Date()); + this.autoMlInsDao.update(ins); + return true; + } else { + return false; + } + } catch (Exception e) { + throw new RuntimeException("终止实例错误: " + e.getMessage(), e); + } + } } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMlServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMlServiceImpl.java index 7669232a..4b7e58e9 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMlServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMlServiceImpl.java @@ -3,15 +3,19 @@ package com.ruoyi.platform.service.impl; import com.ruoyi.common.security.utils.SecurityUtils; import com.ruoyi.platform.constant.Constant; import com.ruoyi.platform.domain.AutoMl; +import com.ruoyi.platform.domain.AutoMlIns; import com.ruoyi.platform.mapper.AutoMlDao; +import com.ruoyi.platform.mapper.AutoMlInsDao; import com.ruoyi.platform.service.AutoMlService; -import com.ruoyi.platform.utils.*; +import com.ruoyi.platform.utils.FileUtil; +import com.ruoyi.platform.utils.HttpUtils; +import com.ruoyi.platform.utils.JacksonUtil; +import com.ruoyi.platform.utils.JsonUtils; +import com.ruoyi.platform.vo.AutoMlParamVo; import com.ruoyi.platform.vo.AutoMlVo; -import io.kubernetes.client.openapi.models.V1Pod; +import org.apache.commons.collections4.MapUtils; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.data.domain.Page; @@ -26,29 +30,24 @@ import java.io.IOException; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.CompletableFuture; @Service("autoMLService") public class AutoMlServiceImpl implements AutoMlService { - @Value("${harbor.serviceNS}") - private String serviceNS; - @Value("${dockerpush.proxyUrl}") - private String proxyUrl; - @Value("${dockerpush.mountPath}") - private String mountPath; - @Value("${minio.pvcName}") - private String pvcName; - @Value("${automl.image}") - private String image; @Value("${git.localPath}") String localPath; - private static final Logger logger = LoggerFactory.getLogger(ModelsServiceImpl.class); + @Value("${argo.url}") + private String argoUrl; + @Value("${argo.convertAutoML}") + String convertAutoML; + @Value("${argo.workflowRun}") + private String argoWorkflowRun; @Resource private AutoMlDao autoMlDao; + @Resource - private K8sClientUtil k8sClientUtil; + private AutoMlInsDao autoMlInsDao; @Override public Page queryByPage(String mlName, PageRequest pageRequest) { @@ -146,86 +145,52 @@ public class AutoMlServiceImpl implements AutoMlService { public String runAutoMlIns(Long id) throws Exception { AutoMl autoMl = autoMlDao.getAutoMlById(id); if (autoMl == null) { - throw new Exception("开发环境配置不存在"); - } - - StringBuffer command = new StringBuffer(); - command.append("nohup python /opt/automl.py --task_type " + autoMl.getTaskType()); - if (StringUtils.isNotEmpty(autoMl.getTargetColumns())) { - command.append(" --target_columns " + autoMl.getTargetColumns()); - } else { - throw new Exception("目标列为空"); - } - -// String username = SecurityUtils.getLoginUser().getUsername().toLowerCase(); - String username = "admin"; - //构造pod名称 - String podName = username + "-autoMlIns-pod-" + id; - V1Pod pod = k8sClientUtil.createPodWithEnv(podName, serviceNS, proxyUrl, mountPath, pvcName, image); - - if (autoMl.getTimeLeftForThisTask() != null) { - command.append(" --time_left_for_this_task " + autoMl.getTimeLeftForThisTask()); - } - if (autoMl.getPerRunTimeLimit() != null) { - command.append(" --per_run_time_limit " + autoMl.getPerRunTimeLimit()); - } - if (autoMl.getEnsembleSize() != null) { - command.append(" --ensemble_size " + autoMl.getEnsembleSize()); - } - if (StringUtils.isNotEmpty(autoMl.getEnsembleClass())) { - command.append(" --ensemble_class " + autoMl.getEnsembleClass()); - } - if (autoMl.getEnsembleNbest() != null) { - command.append(" --ensemble_nbest " + autoMl.getEnsembleNbest()); - } - if (autoMl.getMaxModelsOnDisc() != null) { - command.append(" --max_models_on_disc " + autoMl.getMaxModelsOnDisc()); - } - if (autoMl.getSeed() != null) { - command.append(" --seed " + autoMl.getSeed()); - } - if (autoMl.getMemoryLimit() != null) { - command.append(" --memory_limit " + autoMl.getMemoryLimit()); - } - if (StringUtils.isNotEmpty(autoMl.getIncludeClassifier())) { - command.append(" --include_classifier " + autoMl.getIncludeClassifier()); - } - if (StringUtils.isNotEmpty(autoMl.getIncludeRegressor())) { - command.append(" --include_regressor " + autoMl.getIncludeRegressor()); - } - if (StringUtils.isNotEmpty(autoMl.getIncludeFeaturePreprocessor())) { - command.append(" --include_feature_preprocessor " + autoMl.getIncludeFeaturePreprocessor()); - } - if (StringUtils.isNotEmpty(autoMl.getExcludeClassifier())) { - command.append(" --exclude_classifier " + autoMl.getExcludeClassifier()); - } - if (StringUtils.isNotEmpty(autoMl.getExcludeRegressor())) { - command.append(" --exclude_regressor " + autoMl.getExcludeRegressor()); - } - if (StringUtils.isNotEmpty(autoMl.getExcludeFeaturePreprocessor())) { - command.append(" --exclude_feature_preprocessor " + autoMl.getExcludeFeaturePreprocessor()); - } - if (StringUtils.isNotEmpty(autoMl.getResamplingStrategy())) { - command.append(" --resampling_strategy " + autoMl.getResamplingStrategy()); - } - if (autoMl.getTrainSize() != null) { - command.append(" --train_size " + autoMl.getTrainSize()); - } - if (autoMl.getShuffle() != null) { - command.append(" --shuffle " + autoMl.getShuffle()); - } - if (autoMl.getFolds() != null) { - command.append(" --folds " + autoMl.getFolds()); - } - command.append(" &"); - CompletableFuture.supplyAsync(() -> { - try { - String log = k8sClientUtil.executeCommand(pod, String.valueOf(command)); - } catch (Exception e) { - logger.error(e.getMessage(), e); + throw new Exception("自动机器学习配置不存在"); + } + + AutoMlParamVo autoMlParam = new AutoMlParamVo(); + BeanUtils.copyProperties(autoMl, autoMlParam); + autoMlParam.setDataset(JsonUtils.jsonToMap(autoMl.getDataset())); + String param = JsonUtils.objectToJson(autoMlParam); + // 调argo转换接口 + try { + String convertRes = HttpUtils.sendPost(argoUrl + convertAutoML, param); + if (convertRes == null || StringUtils.isEmpty(convertRes)) { + throw new RuntimeException("转换流水线失败"); } - return null; - }); + Map converMap = JsonUtils.jsonToMap(convertRes); + // 组装运行接口json + Map output = (Map) converMap.get("output"); + Map runReqMap = new HashMap<>(); + runReqMap.put("data", converMap.get("data")); + // 调argo运行接口 + String runRes = HttpUtils.sendPost(argoUrl + argoWorkflowRun, JsonUtils.mapToJson(runReqMap)); + + if (runRes == null || StringUtils.isEmpty(runRes)) { + throw new RuntimeException("Failed to run workflow."); + } + Map runResMap = JsonUtils.jsonToMap(runRes); + Map data = (Map) runResMap.get("data"); + //判断data为空 + if (data == null || MapUtils.isEmpty(data)) { + throw new RuntimeException("Failed to run workflow."); + } + Map metadata = (Map) data.get("metadata"); + // 插入记录到实验实例表 + AutoMlIns autoMlIns = new AutoMlIns(); + autoMlIns.setAutoMlId(autoMl.getId()); + autoMlIns.setArgoInsNs((String) metadata.get("namespace")); + autoMlIns.setArgoInsName((String) metadata.get("name")); + autoMlIns.setParam(param); + autoMlIns.setStatus(Constant.Pending); + //替换argoInsName + String outputString = JsonUtils.mapToJson(output); + autoMlIns.setNodeResult(outputString.replace("{{workflow.name}}", (String) metadata.get("name"))); + autoMlInsDao.insert(autoMlIns); + + } catch (Exception e) { + throw new RuntimeException(e); + } return "执行成功"; } } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/AutoMlParamVo.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/AutoMlParamVo.java new file mode 100644 index 00000000..540631af --- /dev/null +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/AutoMlParamVo.java @@ -0,0 +1,155 @@ +package com.ruoyi.platform.vo; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.PropertyNamingStrategy; +import com.fasterxml.jackson.databind.annotation.JsonNaming; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import java.util.Map; + +@Data +@JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy.class) +@JsonInclude(JsonInclude.Include.NON_NULL) +@ApiModel(description = "自动机器学习参数") +public class AutoMlParamVo { + @ApiModelProperty(value = "任务类型：classification或regression") + private String taskType; + + @ApiModelProperty(value = "搜索合适模型的时间限制（以秒为单位）。通过增加这个值，auto-sklearn有更高的机会找到更好的模型。默认3600，非必传。") + private Integer timeLeftForThisTask; + + @ApiModelProperty(value = "单次调用机器学习模型的时间限制（以秒为单位）。如果机器学习算法运行超过时间限制，将终止模型拟合。将这个值设置得足够高，这样典型的机器学习算法就可以适用于训练数据。默认600，非必传。") + private Integer perRunTimeLimit; + + @ApiModelProperty(value = "集成模型数量，如果设置为0，则没有集成。默认50，非必传。") + private Integer ensembleSize; + + @ApiModelProperty(value = "设置为None将禁用集成构建，设置为SingleBest仅使用单个最佳模型而不是集成，设置为default，它将对单目标问题使用EnsembleSelection，对多目标问题使用MultiObjectiveDummyEnsemble。默认default，非必传。") + private String ensembleClass; + + @ApiModelProperty(value = "在构建集成时只考虑ensemble_nbest模型。这是受到了“最大限度地利用集成选择”中引入的库修剪概念的启发。这是独立于ensemble_class参数的，并且这个修剪步骤是在构造集成之前完成的。默认50，非必传。") + private Integer ensembleNbest; + + @ApiModelProperty(value = "定义在磁盘中保存的模型的最大数量。额外的模型数量将被永久删除。由于这个变量的性质，它设置了一个集成可以使用多少个模型的上限。必须是大于等于1的整数。如果设置为None，则所有模型都保留在磁盘上。默认50，非必传。") + private Integer maxModelsOnDisc; + + @ApiModelProperty(value = "随机种子，将决定输出文件名。默认1，非必传。") + private Integer seed; + + @ApiModelProperty(value = "机器学习算法的内存限制（MB）。如果auto-sklearn试图分配超过memory_limit MB，它将停止拟合机器学习算法。默认3072，非必传。") + private Integer memoryLimit; + + @ApiModelProperty(value = "如果为None，则使用所有可能的分类算法。否则，指定搜索中包含的步骤和组件。有关可用组件，请参见/pipeline/components//*。与参数exclude不兼容。多选，逗号分隔。包含：adaboost\n" + + "bernoulli_nb\n" + + "decision_tree\n" + + "extra_trees\n" + + "gaussian_nb\n" + + "gradient_boosting\n" + + "k_nearest_neighbors\n" + + "lda\n" + + "liblinear_svc\n" + + "libsvm_svc\n" + + "mlp\n" + + "multinomial_nb\n" + + "passive_aggressive\n" + + "qda\n" + + "random_forest\n" + + "sgd") + private String includeClassifier; + + @ApiModelProperty(value = "如果为None，则使用所有可能的特征预处理算法。否则，指定搜索中包含的步骤和组件。有关可用组件，请参见/pipeline/components//*。与参数exclude不兼容。多选，逗号分隔。包含：densifier\n" + + "extra_trees_preproc_for_classification\n" + + "extra_trees_preproc_for_regression\n" + + "fast_ica\n" + + "feature_agglomeration\n" + + "kernel_pca\n" + + "kitchen_sinks\n" + + "liblinear_svc_preprocessor\n" + + "no_preprocessing\n" + + "nystroem_sampler\n" + + "pca\n" + + "polynomial\n" + + "random_trees_embedding\n" + + "select_percentile_classification\n" + + "select_percentile_regression\n" + + "select_rates_classification\n" + + "select_rates_regression\n" + + "truncatedSVD") + private String includeFeaturePreprocessor; + + @ApiModelProperty(value = "如果为None，则使用所有可能的回归算法。否则，指定搜索中包含的步骤和组件。有关可用组件，请参见/pipeline/components//*。与参数exclude不兼容。多选，逗号分隔。包含：adaboost,\n" + + "ard_regression,\n" + + "decision_tree,\n" + + "extra_trees,\n" + + "gaussian_process,\n" + + "gradient_boosting,\n" + + "k_nearest_neighbors,\n" + + "liblinear_svr,\n" + + "libsvm_svr,\n" + + "mlp,\n" + + "random_forest,\n" + + "sgd") + private String includeRegressor; + + private String excludeClassifier; + + private String excludeRegressor; + + private String excludeFeaturePreprocessor; + + @ApiModelProperty(value = "测试集的比率，0到1之间") + private Float testSize; + + @ApiModelProperty(value = "如何处理过拟合，如果使用基于“cv”的方法或Splitter对象，可能需要使用resampling_strategy_arguments。holdout或crossValid") + private String resamplingStrategy; + + @ApiModelProperty(value = "重采样划分训练集和验证集，训练集的比率，0到1之间") + private Float trainSize; + + @ApiModelProperty(value = "拆分数据前是否进行shuffle") + private Boolean shuffle; + + @ApiModelProperty(value = "交叉验证的折数，当resamplingStrategy为crossValid时，此项必填，为整数") + private Integer folds; + + @ApiModelProperty(value = "数据集csv文件中哪几列是预测目标列，逗号分隔") + private String targetColumns; + + @ApiModelProperty(value = "自定义指标名称") + private String metricName; + + @ApiModelProperty(value = "模型优化目标指标及权重，json格式。分类的指标包含：accuracy\n" + + "balanced_accuracy\n" + + "roc_auc\n" + + "average_precision\n" + + "log_loss\n" + + "precision_macro\n" + + "precision_micro\n" + + "precision_samples\n" + + "precision_weighted\n" + + "recall_macro\n" + + "recall_micro\n" + + "recall_samples\n" + + "recall_weighted\n" + + "f1_macro\n" + + "f1_micro\n" + + "f1_samples\n" + + "f1_weighted\n" + + "回归的指标包含：mean_absolute_error\n" + + "mean_squared_error\n" + + "root_mean_squared_error\n" + + "mean_squared_log_error\n" + + "median_absolute_error\n" + + "r2") + private String metrics; + + @ApiModelProperty(value = "指标优化方向，是越大越好还是越小越好") + private Boolean greaterIsBetter; + + @ApiModelProperty(value = "模型计算并打印指标") + private String scoringFunctions; + + private Map dataset; +} diff --git a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMlDao.xml b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMlDao.xml index b6a3c4e7..610fe143 100644 --- a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMlDao.xml +++ b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMlDao.xml @@ -34,9 +34,9 @@ ml_description = #{autoMl.mlDescription}, - - - + + status_list = #{autoMl.statusList}, + diff --git a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMlInsDao.xml b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMlInsDao.xml index 05168691..ca1300d3 100644 --- a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMlInsDao.xml +++ b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMlInsDao.xml @@ -3,9 +3,9 @@ - insert into auto_ml_ins(auto_ml_id, model_path, img_path, node_status, node_result, param, source) + insert into auto_ml_ins(auto_ml_id, model_path, img_path, node_status, node_result, param, source, argo_ins_name, argo_ins_ns) values (#{autoMlIns.autoMlId}, #{autoMlIns.modelPath}, #{autoMlIns.imgPath}, #{autoMlIns.nodeStatus}, - #{autoMlIns.nodeResult}, #{autoMlIns.param}, #{autoMlIns.source}) + #{autoMlIns.nodeResult}, #{autoMlIns.param}, #{autoMlIns.source}, #{autoMlIns.argoInsName}, #{autoMlIns.argoInsNs}) @@ -29,7 +29,11 @@ state = #{autoMlIns.state}, + + update_time = #{autoMlIns.updateTime}, + + where id = #{autoMlIns.id} + + + + \ No newline at end of file From 9af40754e78dea138c3d90b1408df6a32e00041c Mon Sep 17 00:00:00 2001 From: chenzhihang <709011834@qq.com> Date: Thu, 28 Nov 2024 17:10:19 +0800 Subject: [PATCH 13/18] =?UTF-8?q?=E8=87=AA=E5=8A=A8=E6=9C=BA=E5=99=A8?= =?UTF-8?q?=E5=AD=A6=E4=B9=A0=E5=BC=80=E5=8F=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../autoML/AutoMlInsController.java | 6 +++++ .../platform/service/AutoMlInsService.java | 2 ++ .../service/impl/AutoMlInsServiceImpl.java | 12 ++++++---- .../service/impl/AutoMlServiceImpl.java | 22 +++++++++++++++---- 4 files changed, 34 insertions(+), 8 deletions(-) diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/autoML/AutoMlInsController.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/autoML/AutoMlInsController.java index 125fd668..874ec59c 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/autoML/AutoMlInsController.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/autoML/AutoMlInsController.java @@ -52,4 +52,10 @@ public class AutoMlInsController extends BaseController { public GenericsAjaxResult terminateAutoMlIns(@PathVariable("id") Long id) { return genericsSuccess(this.autoMLInsService.terminateAutoMlIns(id)); } + + @GetMapping("{id}") + @ApiOperation("查看实验实例详情") + public GenericsAjaxResult getDetailById(@PathVariable("id") Long id) { + return genericsSuccess(this.autoMLInsService.getDetailById(id)); + } } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AutoMlInsService.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AutoMlInsService.java index d4956a22..bb417ed2 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AutoMlInsService.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AutoMlInsService.java @@ -21,4 +21,6 @@ public interface AutoMlInsService { AutoMlIns queryStatusFromArgo(AutoMlIns autoMlIns); boolean terminateAutoMlIns(Long id); + + AutoMlIns getDetailById(Long id); } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMlInsServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMlInsServiceImpl.java index 3d11123e..cd01b672 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMlInsServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMlInsServiceImpl.java @@ -87,7 +87,6 @@ public class AutoMlInsServiceImpl implements AutoMlInsService { public AutoMlIns queryStatusFromArgo(AutoMlIns ins) { String namespace = ins.getArgoInsNs(); String name = ins.getArgoInsName(); - Long id = ins.getId(); // 创建请求数据map Map requestData = new HashMap<>(); @@ -140,11 +139,11 @@ public class AutoMlInsServiceImpl implements AutoMlInsService { ins.setNodeStatus(nodeStatusJson); //终止态为终止不改 - if (!StringUtils.equals(ins.getStatus(), "Terminated")) { - ins.setStatus(StringUtils.isNotEmpty((String) status.get("phase")) ? (String) status.get("phase") : "Pending"); + if (!StringUtils.equals(ins.getStatus(), Constant.Terminated)) { + ins.setStatus(StringUtils.isNotEmpty((String) status.get("phase")) ? (String) status.get("phase") : Constant.Pending); } if (StringUtils.equals(ins.getStatus(), "Error")) { - ins.setStatus("Failed"); + ins.setStatus(Constant.Failed); } return ins; } catch (Exception e) { @@ -223,4 +222,9 @@ public class AutoMlInsServiceImpl implements AutoMlInsService { throw new RuntimeException("终止实例错误: " + e.getMessage(), e); } } + + @Override + public AutoMlIns getDetailById(Long id) { + return null; + } } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMlServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMlServiceImpl.java index 4b7e58e9..97aea010 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMlServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMlServiceImpl.java @@ -27,6 +27,7 @@ import org.springframework.web.multipart.MultipartFile; import javax.annotation.Resource; import java.io.File; import java.io.IOException; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -43,6 +44,9 @@ public class AutoMlServiceImpl implements AutoMlService { @Value("${argo.workflowRun}") private String argoWorkflowRun; + @Value("${minio.endpoint}") + private String minioEndpoint; + @Resource private AutoMlDao autoMlDao; @@ -81,8 +85,7 @@ public class AutoMlServiceImpl implements AutoMlService { } AutoMl autoMl = new AutoMl(); BeanUtils.copyProperties(autoMlVo, autoMl); -// String username = SecurityUtils.getLoginUser().getUsername(); - String username = "admin"; + String username = SecurityUtils.getLoginUser().getUsername(); autoMl.setUpdateBy(username); String datasetJson = JacksonUtil.toJSONString(autoMlVo.getDataset()); autoMl.setDataset(datasetJson); @@ -123,8 +126,7 @@ public class AutoMlServiceImpl implements AutoMlService { public Map upload(MultipartFile file, String uuid) throws Exception { Map result = new HashMap<>(); -// String username = SecurityUtils.getLoginUser().getUsername(); - String username = "admin"; + String username = SecurityUtils.getLoginUser().getUsername(); String fileName = file.getOriginalFilename(); String path = localPath + "temp/" + username + "/automl_data/" + uuid; long sizeInBytes = file.getSize(); @@ -186,6 +188,18 @@ public class AutoMlServiceImpl implements AutoMlService { //替换argoInsName String outputString = JsonUtils.mapToJson(output); autoMlIns.setNodeResult(outputString.replace("{{workflow.name}}", (String) metadata.get("name"))); + + Map param_output = (Map) output.get("param_output"); + List output1 = (ArrayList) param_output.values().toArray()[0]; + Map output2 = (Map) output1.get(0); + String outputPath = minioEndpoint + "/" + output2.get("path").replace("{{workflow.name}}", (String) metadata.get("name")) + "/"; + autoMlIns.setModelPath(outputPath + "save_model.joblib"); + if (Constant.AutoMl_Classification.equals(autoMl.getTaskType())) { + autoMlIns.setImgPath(outputPath + "Auto-sklearn_accuracy_over_time.png" + "," + outputPath + "Train_Confusion_Matrix.png" + "," + outputPath + "Test_Confusion_Matrix.png"); + } else { + autoMlIns.setImgPath(outputPath + "Auto-sklearn_accuracy_over_time.png" + "," + outputPath + "regression.png"); + } + autoMlInsDao.insert(autoMlIns); } catch (Exception e) { From 48bd1f9e89454b6525b35d360485400022057550 Mon Sep 17 00:00:00 2001 From: chenzhihang <709011834@qq.com> Date: Thu, 28 Nov 2024 17:11:28 +0800 Subject: [PATCH 14/18] =?UTF-8?q?=E8=87=AA=E5=8A=A8=E6=9C=BA=E5=99=A8?= =?UTF-8?q?=E5=AD=A6=E4=B9=A0=E5=BC=80=E5=8F=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/ruoyi/platform/service/impl/AutoMlInsServiceImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMlInsServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMlInsServiceImpl.java index cd01b672..a672da1f 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMlInsServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMlInsServiceImpl.java @@ -225,6 +225,6 @@ public class AutoMlInsServiceImpl implements AutoMlInsService { @Override public AutoMlIns getDetailById(Long id) { - return null; + return this.autoMlInsDao.queryById(id); } } From cce148fd3914ac39049abf887a0fca0fab63b59c Mon Sep 17 00:00:00 2001 From: chenzhihang <709011834@qq.com> Date: Fri, 29 Nov 2024 16:19:43 +0800 Subject: [PATCH 15/18] =?UTF-8?q?=E8=87=AA=E5=8A=A8=E6=9C=BA=E5=99=A8?= =?UTF-8?q?=E5=AD=A6=E4=B9=A0=E5=BC=80=E5=8F=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/ruoyi/platform/constant/Constant.java | 3 +++ .../com/ruoyi/platform/domain/AutoMlIns.java | 6 +++++ .../service/impl/AutoMlInsServiceImpl.java | 23 +++++++++++++++++-- .../service/impl/AutoMlServiceImpl.java | 4 +++- .../managementPlatform/AutoMlInsDao.xml | 20 ++++++++-------- 5 files changed, 44 insertions(+), 12 deletions(-) diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/constant/Constant.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/constant/Constant.java index 834f63e4..2f72576e 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/constant/Constant.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/constant/Constant.java @@ -29,10 +29,13 @@ public class Constant { public final static String Running = "Running"; public final static String Failed = "Failed"; public final static String Pending = "Pending"; + public final static String Terminated = "Terminated"; public final static String Init = "Init"; public final static String Stopped = "Stopped"; public final static String Succeeded = "Succeeded"; public final static String Type_Train = "train"; public final static String Type_Evaluate = "evaluate"; + + public final static String AutoMl_Classification = "classification"; } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/AutoMlIns.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/AutoMlIns.java index ba425dcf..98df8125 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/AutoMlIns.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/AutoMlIns.java @@ -16,10 +16,14 @@ public class AutoMlIns { private Long autoMlId; + private String resultPath; + private String modelPath; private String imgPath; + private String runHistoryPath; + private Integer state; private String status; @@ -41,4 +45,6 @@ public class AutoMlIns { private Date createTime; private Date updateTime; + + private Date finishTime; } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMlInsServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMlInsServiceImpl.java index a672da1f..7b945e76 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMlInsServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMlInsServiceImpl.java @@ -1,7 +1,9 @@ package com.ruoyi.platform.service.impl; import com.ruoyi.platform.constant.Constant; +import com.ruoyi.platform.domain.AutoMl; import com.ruoyi.platform.domain.AutoMlIns; +import com.ruoyi.platform.mapper.AutoMlDao; import com.ruoyi.platform.mapper.AutoMlInsDao; import com.ruoyi.platform.service.AutoMlInsService; import com.ruoyi.platform.utils.DateUtils; @@ -29,7 +31,8 @@ public class AutoMlInsServiceImpl implements AutoMlInsService { @Resource private AutoMlInsDao autoMlInsDao; - + @Resource + private AutoMlDao autoMlDao; @Override public Page queryByPage(AutoMlIns autoMlIns, PageRequest pageRequest) throws IOException { @@ -61,6 +64,7 @@ public class AutoMlInsServiceImpl implements AutoMlInsService { autoMlIns.setState(Constant.State_invalid); int update = autoMlInsDao.update(autoMlIns); if (update > 0) { + updateAutoMlStatus(autoMlIns.getAutoMlId()); return "删除成功"; } else { return "删除失败"; @@ -121,7 +125,7 @@ public class AutoMlInsServiceImpl implements AutoMlInsService { String finishedAtString = (String) status.get("finishedAt"); if (finishedAtString != null && !finishedAtString.isEmpty()) { Date finishTime = DateUtils.convertUTCtoShanghaiDate(finishedAtString); - ins.setUpdateTime(finishTime); + ins.setFinishTime(finishTime); } // 解析nodes字段,提取节点状态并转换为JSON字符串 @@ -227,4 +231,19 @@ public class AutoMlInsServiceImpl implements AutoMlInsService { public AutoMlIns getDetailById(Long id) { return this.autoMlInsDao.queryById(id); } + + public void updateAutoMlStatus(Long autoMlId) { + List insList = autoMlInsDao.getByAutoMlId(autoMlId); + List statusList = new ArrayList<>(); + // 更新实验状态列表 + for (int i = 0; i < insList.size(); i++) { + statusList.add(insList.get(i).getStatus()); + } + String subStatus = statusList.toString().substring(1, statusList.toString().length() - 1); + AutoMl autoMl = autoMlDao.getAutoMlById(autoMlId); + if (!StringUtils.equals(autoMl.getStatusList(), subStatus)) { + autoMl.setStatusList(subStatus); + autoMlDao.edit(autoMl); + } + } } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMlServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMlServiceImpl.java index 97aea010..7b842299 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMlServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMlServiceImpl.java @@ -199,7 +199,9 @@ public class AutoMlServiceImpl implements AutoMlService { } else { autoMlIns.setImgPath(outputPath + "Auto-sklearn_accuracy_over_time.png" + "," + outputPath + "regression.png"); } - + autoMlIns.setResultPath(outputPath + "result.txt"); + String seed = autoMl.getSeed() != null ? String.valueOf(autoMl.getSeed()) : "1"; + autoMlIns.setRunHistoryPath(outputPath + "smac3-output/run_" + seed + "/runhistory.json"); autoMlInsDao.insert(autoMlIns); } catch (Exception e) { diff --git a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMlInsDao.xml b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMlInsDao.xml index ca1300d3..26f9c20d 100644 --- a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMlInsDao.xml +++ b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMlInsDao.xml @@ -3,9 +3,12 @@ - insert into auto_ml_ins(auto_ml_id, model_path, img_path, node_status, node_result, param, source, argo_ins_name, argo_ins_ns) - values (#{autoMlIns.autoMlId}, #{autoMlIns.modelPath}, #{autoMlIns.imgPath}, #{autoMlIns.nodeStatus}, - #{autoMlIns.nodeResult}, #{autoMlIns.param}, #{autoMlIns.source}, #{autoMlIns.argoInsName}, #{autoMlIns.argoInsNs}) + insert into auto_ml_ins(auto_ml_id, result_path, model_path, img_path, run_history_path, node_status, + node_result, param, source, argo_ins_name, argo_ins_ns, status) + values (#{autoMlIns.autoMlId}, #{autoMlIns.resultPath}, #{autoMlIns.modelPath}, #{autoMlIns.imgPath}, + #{autoMlIns.runHistoryPath}, #{autoMlIns.nodeStatus}, + #{autoMlIns.nodeResult}, #{autoMlIns.param}, #{autoMlIns.source}, #{autoMlIns.argoInsName}, + #{autoMlIns.argoInsNs}, #{autoMlIns.status}) @@ -32,6 +35,9 @@ update_time = #{autoMlIns.updateTime}, + + finish_time = #{autoMlIns.finishTime}, + where id = #{autoMlIns.id} @@ -41,9 +47,7 @@ from auto_ml_ins state = 1 - - and auto_ml_id = #{autoMlIns.autoMlId} - + and auto_ml_id = #{autoMlIns.autoMlId} @@ -51,9 +55,7 @@ select * from auto_ml_ins state = 1 - - and auto_ml_id = #{autoMlIns.autoMlId} - + and auto_ml_id = #{autoMlIns.autoMlId} order by update_time DESC limit #{pageable.offset}, #{pageable.pageSize} From e624cd117029eb649575066217a8f2250876ece4 Mon Sep 17 00:00:00 2001 From: chenzhihang <709011834@qq.com> Date: Sat, 30 Nov 2024 09:26:40 +0800 Subject: [PATCH 16/18] =?UTF-8?q?=E8=87=AA=E5=8A=A8=E6=9C=BA=E5=99=A8?= =?UTF-8?q?=E5=AD=A6=E4=B9=A0=E5=BC=80=E5=8F=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/ruoyi/platform/service/AutoMlInsService.java | 5 ++++- .../com/ruoyi/platform/service/impl/AutoMlServiceImpl.java | 6 +++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AutoMlInsService.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AutoMlInsService.java index bb417ed2..2131fc44 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AutoMlInsService.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AutoMlInsService.java @@ -1,7 +1,8 @@ package com.ruoyi.platform.service; + +import com.ruoyi.platform.domain.AutoMlIns; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageRequest; -import com.ruoyi.platform.domain.AutoMlIns; import java.io.IOException; import java.util.List; @@ -23,4 +24,6 @@ public interface AutoMlInsService { boolean terminateAutoMlIns(Long id); AutoMlIns getDetailById(Long id); + + void updateAutoMlStatus(Long autoMlId); } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMlServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMlServiceImpl.java index 7b842299..d5e436e8 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMlServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMlServiceImpl.java @@ -6,6 +6,7 @@ import com.ruoyi.platform.domain.AutoMl; import com.ruoyi.platform.domain.AutoMlIns; import com.ruoyi.platform.mapper.AutoMlDao; import com.ruoyi.platform.mapper.AutoMlInsDao; +import com.ruoyi.platform.service.AutoMlInsService; import com.ruoyi.platform.service.AutoMlService; import com.ruoyi.platform.utils.FileUtil; import com.ruoyi.platform.utils.HttpUtils; @@ -53,6 +54,9 @@ public class AutoMlServiceImpl implements AutoMlService { @Resource private AutoMlInsDao autoMlInsDao; + @Resource + private AutoMlInsService autoMlInsService; + @Override public Page queryByPage(String mlName, PageRequest pageRequest) { long total = autoMlDao.count(mlName); @@ -203,7 +207,7 @@ public class AutoMlServiceImpl implements AutoMlService { String seed = autoMl.getSeed() != null ? String.valueOf(autoMl.getSeed()) : "1"; autoMlIns.setRunHistoryPath(outputPath + "smac3-output/run_" + seed + "/runhistory.json"); autoMlInsDao.insert(autoMlIns); - + autoMlInsService.updateAutoMlStatus(id); } catch (Exception e) { throw new RuntimeException(e); } From fb6c8e1a4df4f1eb482c63e4f4d015d27dc7f7b0 Mon Sep 17 00:00:00 2001 From: chenzhihang <709011834@qq.com> Date: Sat, 30 Nov 2024 09:29:13 +0800 Subject: [PATCH 17/18] =?UTF-8?q?=E8=87=AA=E5=8A=A8=E6=9C=BA=E5=99=A8?= =?UTF-8?q?=E5=AD=A6=E4=B9=A0=E5=BC=80=E5=8F=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/ruoyi/platform/service/impl/AutoMlInsServiceImpl.java | 1 + 1 file changed, 1 insertion(+) diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMlInsServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMlInsServiceImpl.java index 7b945e76..4fee6d4b 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMlInsServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AutoMlInsServiceImpl.java @@ -218,6 +218,7 @@ public class AutoMlInsServiceImpl implements AutoMlInsService { ins.setStatus(Constant.Terminated); ins.setUpdateTime(new Date()); this.autoMlInsDao.update(ins); + updateAutoMlStatus(autoMlIns.getAutoMlId()); return true; } else { return false; From 8a0b086d721f70989cdfcef2c306d86f19530322 Mon Sep 17 00:00:00 2001 From: chenzhihang <709011834@qq.com> Date: Mon, 2 Dec 2024 14:24:23 +0800 Subject: [PATCH 18/18] =?UTF-8?q?=E8=87=AA=E5=8A=A8=E6=9C=BA=E5=99=A8?= =?UTF-8?q?=E5=AD=A6=E4=B9=A0=E5=BC=80=E5=8F=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/resources/mapper/managementPlatform/AutoMlDao.xml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMlDao.xml b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMlDao.xml index 610fe143..c4453c5c 100644 --- a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMlDao.xml +++ b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AutoMlDao.xml @@ -77,6 +77,7 @@ exclude_regressor = #{autoMl.excludeRegressor}, exclude_feature_preprocessor = #{autoMl.excludeFeaturePreprocessor}, scoring_functions = #{autoMl.scoringFunctions}, + metrics = #{autoMl.metrics}, test_size = #{autoMl.testSize}, @@ -98,9 +99,6 @@ metric_name = #{autoMl.metricName}, - - metrics = #{autoMl.metrics}, - greater_is_better = #{autoMl.greaterIsBetter},