Browse Source

自动机器学习优化

pull/225/head
chenzhihang 11 months ago
parent
commit
0cc24f50a2
17 changed files with 1089 additions and 22 deletions
  1. +1
    -0
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/autoML/AutoMlController.java
  2. +61
    -0
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/machineLearn/MachineLearnController.java
  3. +60
    -0
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/machineLearn/MachineLearnInsController.java
  4. +42
    -0
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/MachineLearn.java
  5. +51
    -0
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/MachineLearnIns.java
  6. +23
    -0
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/MachineLearnDao.java
  7. +23
    -0
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/MachineLearnInsDao.java
  8. +119
    -0
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/scheduling/MLStatusTask.java
  9. +28
    -0
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/MachineLearnInsService.java
  10. +19
    -0
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/MachineLearnService.java
  11. +259
    -0
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/MachineLearnInsServiceImpl.java
  12. +220
    -0
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/MachineLearnServiceImpl.java
  13. +15
    -21
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java
  14. +1
    -1
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/TextClassificationInsServiceImpl.java
  15. +2
    -0
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/TextClassificationParamVo.java
  16. +82
    -0
      ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/MachineLearnDaoMapper.xml
  17. +83
    -0
      ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/MachineLearnInsDaoMapper.xml

+ 1
- 0
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/autoML/AutoMlController.java View File

@@ -44,6 +44,7 @@ public class AutoMlController extends BaseController {
public GenericsAjaxResult<String> editAutoMl(@RequestBody AutoMlVo autoMlVo) throws Exception {
return genericsSuccess(this.autoMlService.edit(autoMlVo));
}

@GetMapping("/getAutoMlDetail")
@ApiOperation("获取自动机器学习详细信息")
public GenericsAjaxResult<AutoMlVo> getAutoMlDetail(@RequestParam("id") Long id) throws IOException {


+ 61
- 0
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/machineLearn/MachineLearnController.java View File

@@ -0,0 +1,61 @@
package com.ruoyi.platform.controller.machineLearn;

import com.ruoyi.common.core.web.controller.BaseController;
import com.ruoyi.common.core.web.domain.GenericsAjaxResult;
import com.ruoyi.platform.domain.MachineLearn;
import com.ruoyi.platform.service.MachineLearnService;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.web.bind.annotation.*;

import javax.annotation.Resource;

@RestController
@RequestMapping("machineLearn")
@Api("自动机器学习")
public class MachineLearnController extends BaseController {
@Resource
private MachineLearnService machineLearnService;

@GetMapping
@ApiOperation("分页查询")
public GenericsAjaxResult<Page<MachineLearn>> queryByPage(@RequestParam("page") int page,
@RequestParam("size") int size,
@RequestParam(value = "name", required = false) String name,
@RequestParam(value = "type", required = false) String type) {
PageRequest pageRequest = PageRequest.of(page, size);
return genericsSuccess(this.machineLearnService.queryByPage(name, type, pageRequest));
}

@PostMapping
@ApiOperation("新增自动机器学习")
public GenericsAjaxResult<MachineLearn> add(@RequestBody MachineLearn machineLearn) {
return genericsSuccess(this.machineLearnService.add(machineLearn));
}

@PutMapping
@ApiOperation("编辑自动机器学习")
public GenericsAjaxResult<String> edit(@RequestBody MachineLearn machineLearn) throws Exception {
return genericsSuccess(this.machineLearnService.edit(machineLearn));
}

@GetMapping("/getMLDetail")
@ApiOperation("获取自动机器学习详细信息")
public GenericsAjaxResult<MachineLearn> getMLDetail(@RequestParam("id") Long id) {
return genericsSuccess(this.machineLearnService.getMLDetail(id));
}

@DeleteMapping("{id}")
@ApiOperation("删除自动机器学习")
public GenericsAjaxResult<String> delete(@PathVariable("id") Long id) {
return genericsSuccess(this.machineLearnService.delete(id));
}

@PostMapping("/run/{id}")
@ApiOperation("运行自动机器学习")
public GenericsAjaxResult<String> runMachineLearn(@PathVariable("id") Long id) throws Exception {
return genericsSuccess(this.machineLearnService.runMachineLearn(id));
}
}

+ 60
- 0
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/machineLearn/MachineLearnInsController.java View File

@@ -0,0 +1,60 @@
package com.ruoyi.platform.controller.machineLearn;

import com.ruoyi.common.core.web.controller.BaseController;
import com.ruoyi.common.core.web.domain.GenericsAjaxResult;
import com.ruoyi.platform.domain.MachineLearnIns;
import com.ruoyi.platform.service.MachineLearnInsService;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import org.springframework.web.bind.annotation.*;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;

import javax.annotation.Resource;
import java.util.List;

@RestController
@RequestMapping("machineLearnIns")
@Api("自动机器学习实验实例")
public class MachineLearnInsController extends BaseController {

@Resource
private MachineLearnInsService machineLearnInsService;

@GetMapping
@ApiOperation("分页查询")
public GenericsAjaxResult<Page<MachineLearnIns>> queryByPage(Long machineLearnId, int page, int size) {
PageRequest pageRequest = PageRequest.of(page, size);
return genericsSuccess(this.machineLearnInsService.queryByPage(machineLearnId, pageRequest));
}

@PostMapping
@ApiOperation("新增实验实例")
public GenericsAjaxResult<MachineLearnIns> add(@RequestBody MachineLearnIns machineLearnIns) {
return genericsSuccess(this.machineLearnInsService.insert(machineLearnIns));
}

@DeleteMapping("{id}")
@ApiOperation("删除实验实例")
public GenericsAjaxResult<String> deleteById(@PathVariable("id") Long id) {
return genericsSuccess(this.machineLearnInsService.removeById(id));
}

@DeleteMapping("batchDelete")
@ApiOperation("批量删除实验实例")
public GenericsAjaxResult<String> batchDelete(@RequestBody List<Long> ids) {
return genericsSuccess(this.machineLearnInsService.batchDelete(ids));
}

@PutMapping("{id}")
@ApiOperation("终止实验实例")
public GenericsAjaxResult<Boolean> terminateMLIns(@PathVariable("id") Long id) throws Exception {
return genericsSuccess(this.machineLearnInsService.terminateMLIns(id));
}

@GetMapping("{id}")
@ApiOperation("查看实验实例详情")
public GenericsAjaxResult<MachineLearnIns> getDetailById(@PathVariable("id") Long id) {
return genericsSuccess(this.machineLearnInsService.getDetailById(id));
}
}

+ 42
- 0
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/MachineLearn.java View File

@@ -0,0 +1,42 @@
package com.ruoyi.platform.domain;

import com.fasterxml.jackson.databind.PropertyNamingStrategy;
import com.fasterxml.jackson.databind.annotation.JsonNaming;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;

import java.util.Date;

@Data
@JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy.class)
@ApiModel(description = "机器学习")
public class MachineLearn {
private Long id;

private Long experimentId;

@ApiModelProperty(value = "类型")
private String type;

@ApiModelProperty(value = "实验名称")
private String name;

@ApiModelProperty(value = "实验描述")
private String description;

private String param;

private String createBy;

private Date createTime;

private String updateBy;

private Date updateTime;

private Integer state;

@ApiModelProperty(value = "状态列表")
private String statusList;
}

+ 51
- 0
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/MachineLearnIns.java View File

@@ -0,0 +1,51 @@
package com.ruoyi.platform.domain;

import com.fasterxml.jackson.databind.PropertyNamingStrategy;
import com.fasterxml.jackson.databind.annotation.JsonNaming;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;

import java.util.Date;

@Data
@JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy.class)
@ApiModel(description = "自动机器学习实验实例")
public class MachineLearnIns {

private Long id;

private Long machineLearnId;

private String type;

private String resultPath;

private String modelPath;

private String imgPath;

private String runHistoryPath;

private Integer state;

private String status;

private String nodeStatus;

private String nodeResult;

private String param;

@ApiModelProperty(value = "Argo实例名称")
private String argoInsName;

@ApiModelProperty(value = "Argo命名空间")
private String argoInsNs;

private Date createTime;

private Date updateTime;

private Date finishTime;
}

+ 23
- 0
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/MachineLearnDao.java View File

@@ -0,0 +1,23 @@
package com.ruoyi.platform.mapper;

import com.ruoyi.platform.domain.MachineLearn;
import org.apache.ibatis.annotations.Param;
import org.springframework.data.domain.Pageable;

import java.util.List;

public interface MachineLearnDao {
long count(@Param("name") String name, @Param("type") String type);

List<MachineLearn> queryByPage(@Param("name") String name, @Param("type") String type, @Param("pageable") Pageable pageable);

int save(@Param("machineLearn") MachineLearn machineLearn);

int edit(@Param("machineLearn") MachineLearn machineLearn);

MachineLearn getMachineLearnById(@Param("id") Long id);

MachineLearn getMachineLearnByName(@Param("name") String name);

List<MachineLearn> queryByDatasetId(@Param("datasetId") String datasetId);
}

+ 23
- 0
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/MachineLearnInsDao.java View File

@@ -0,0 +1,23 @@
package com.ruoyi.platform.mapper;

import com.ruoyi.platform.domain.MachineLearnIns;
import org.apache.ibatis.annotations.Param;
import org.springframework.data.domain.Pageable;

import java.util.List;

public interface MachineLearnInsDao {
long count(@Param("machineLearnId") Long machineLearnId);

List<MachineLearnIns> queryAllByLimit(@Param("machineLearnId") Long machineLearnId, @Param("pageable") Pageable pageable);

List<MachineLearnIns> getByMachineLearnId(@Param("machineLearnId") Long machineLearnId);

int insert(@Param("machineLearnIns") MachineLearnIns machineLearnIns);

int update(@Param("machineLearnIns") MachineLearnIns machineLearnIns);

MachineLearnIns queryById(@Param("id") Long id);

List<MachineLearnIns> queryNotTerminated();
}

+ 119
- 0
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/scheduling/MLStatusTask.java View File

@@ -0,0 +1,119 @@
package com.ruoyi.platform.scheduling;

import com.ruoyi.platform.domain.*;
import com.ruoyi.platform.mapper.MachineLearnDao;
import com.ruoyi.platform.mapper.MachineLearnInsDao;
import com.ruoyi.platform.mapper.ResourceOccupyDao;
import com.ruoyi.platform.service.MachineLearnInsService;
import com.ruoyi.platform.service.ResourceOccupyService;
import com.ruoyi.system.api.constant.Constant;
import org.apache.commons.lang3.StringUtils;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;

import javax.annotation.Resource;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;

@Component()
public class MLStatusTask {
@Resource
private MachineLearnInsService machineLearnInsService;
@Resource
private MachineLearnInsDao machineLearnInsDao;
@Resource
private MachineLearnDao machineLearnDao;
@Resource
private ResourceOccupyDao resourceOccupyDao;
@Resource
private ResourceOccupyService resourceOccupyService;

private List<Long> machineLearnIds = new ArrayList<>();

@Scheduled(cron = "0/30 * * * * ?") // 每30S执行一次
public void executeMachineLearnInsStatus() {
// 首先查到所有非终止态的实验实例
List<MachineLearnIns> insList = machineLearnInsService.queryNotTerminated();

// 去argo查询状态
List<MachineLearnIns> updateList = new ArrayList<>();
if (insList != null && insList.size() > 0) {
for (MachineLearnIns ins : insList) {
//当原本状态为null或非终止态时才调用argo接口
try {
Long userId = resourceOccupyDao.getResourceOccupyByTask(Constant.TaskType_ML, ins.getMachineLearnId(), ins.getId(), null).get(0).getUserId();
if (userId != null) {
if (resourceOccupyDao.getUserCredit(userId) <= 0) {
ins.setStatus(Constant.Failed);
machineLearnInsService.terminateMLIns(ins.getId());
} else {
ins = machineLearnInsService.queryStatusFromArgo(ins);
// 扣除积分
if (Constant.Running.equals(ins.getStatus())) {
resourceOccupyService.deducing(Constant.TaskType_ML, null, ins.getId(), null, null);
} else if (Constant.Failed.equals(ins.getStatus()) || Constant.Terminated.equals(ins.getStatus())
|| Constant.Succeeded.equals(ins.getStatus())) {
resourceOccupyService.endDeduce(Constant.TaskType_ML, null, ins.getId(), null, null);
}
}
} else {
ins = machineLearnInsService.queryStatusFromArgo(ins);
}
} catch (Exception e) {
ins.setStatus(Constant.Failed);
// 结束扣除积分
resourceOccupyService.endDeduce(Constant.TaskType_ML, null, ins.getId(), null, null);
}
// 线程安全的添加操作
synchronized (machineLearnIds) {
machineLearnIds.add(ins.getMachineLearnId());
}
updateList.add(ins);
if (updateList.size() > 0) {
for (MachineLearnIns machineLearnIns : updateList) {
machineLearnInsDao.update(machineLearnIns);
}
}
}
}
}

@Scheduled(cron = "0/30 * * * * ?") // / 每30S执行一次
public void executeMachineLearn() {
if (machineLearnIds.size() == 0) {
return;
}
// 存储需要更新的实验对象列表
List<MachineLearn> updateMLs = new ArrayList<>();
for (Long machineLearnId : machineLearnIds) {
// 获取当前实验的所有实例列表
List<MachineLearnIns> insList = machineLearnInsDao.getByMachineLearnId(machineLearnId);
List<String> statusList = new ArrayList<>();
// 更新实验状态列表
for (int i = 0; i < insList.size(); i++) {
statusList.add(insList.get(i).getStatus());
}
String subStatus = statusList.toString().substring(1, statusList.toString().length() - 1);
MachineLearn machineLearn = machineLearnDao.getMachineLearnById(machineLearnId);
if (!StringUtils.equals(machineLearn.getStatusList(), subStatus)) {
machineLearn.setStatusList(subStatus);
updateMLs.add(machineLearn);
machineLearnDao.edit(machineLearn);
}
}

if (!updateMLs.isEmpty()) {
// 使用Iterator进行安全的删除操作
Iterator<Long> iterator = machineLearnIds.iterator();
while (iterator.hasNext()) {
Long machineLearnId = iterator.next();
for (MachineLearn machineLearn : updateMLs) {
if (machineLearn.getId().equals(machineLearnId)) {
iterator.remove();
}
}
}
}
}
}

+ 28
- 0
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/MachineLearnInsService.java View File

@@ -0,0 +1,28 @@
package com.ruoyi.platform.service;

import com.ruoyi.platform.domain.MachineLearnIns;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;

import java.util.List;

public interface MachineLearnInsService {

Page<MachineLearnIns> queryByPage(Long machineLearnId, PageRequest pageRequest);

MachineLearnIns insert(MachineLearnIns machineLearnIns);

String removeById(Long id);

String batchDelete(List<Long> ids);

List<MachineLearnIns> queryNotTerminated();

MachineLearnIns queryStatusFromArgo(MachineLearnIns machineLearnIns);

boolean terminateMLIns(Long id) throws Exception;

MachineLearnIns getDetailById(Long id);

void updateMLStatus(Long machineLearnId);
}

+ 19
- 0
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/MachineLearnService.java View File

@@ -0,0 +1,19 @@
package com.ruoyi.platform.service;

import com.ruoyi.platform.domain.MachineLearn;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;

public interface MachineLearnService {
Page<MachineLearn> queryByPage(String name, String type, PageRequest pageRequest);

MachineLearn add(MachineLearn machineLearn);

String edit(MachineLearn machineLearn);

MachineLearn getMLDetail(Long id);

String delete(Long id);

String runMachineLearn(Long id) throws Exception;
}

+ 259
- 0
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/MachineLearnInsServiceImpl.java View File

@@ -0,0 +1,259 @@
package com.ruoyi.platform.service.impl;

import com.ruoyi.platform.domain.MachineLearn;
import com.ruoyi.platform.domain.MachineLearnIns;
import com.ruoyi.platform.mapper.MachineLearnDao;
import com.ruoyi.platform.mapper.MachineLearnInsDao;
import com.ruoyi.platform.service.MachineLearnInsService;
import com.ruoyi.platform.service.ResourceOccupyService;
import com.ruoyi.platform.utils.DateUtils;
import com.ruoyi.platform.utils.HttpUtils;
import com.ruoyi.platform.utils.JsonUtils;
import com.ruoyi.system.api.constant.Constant;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageImpl;
import org.springframework.data.domain.PageRequest;
import org.springframework.transaction.annotation.Transactional;

import javax.annotation.Resource;
import java.util.*;

@Service
public class MachineLearnInsServiceImpl implements MachineLearnInsService {
@Value("${argo.url}")
private String argoUrl;
@Value("${argo.workflowStatus}")
private String argoWorkflowStatus;
@Value("${argo.workflowTermination}")
private String argoWorkflowTermination;

@Resource
private MachineLearnInsDao machineLearnInsDao;
@Resource
private MachineLearnDao machineLearnDao;
@Resource
private ResourceOccupyService resourceOccupyService;

@Override
public Page<MachineLearnIns> queryByPage(Long machineLearnId, PageRequest pageRequest) {
long count = machineLearnInsDao.count(machineLearnId);
List<MachineLearnIns> machineLearnInsList = machineLearnInsDao.queryAllByLimit(machineLearnId, pageRequest);
return new PageImpl<>(machineLearnInsList, pageRequest, count);
}

@Override
public MachineLearnIns insert(MachineLearnIns machineLearnIns) {
machineLearnInsDao.insert(machineLearnIns);
return machineLearnIns;
}

@Override
@Transactional
public String removeById(Long id) {
MachineLearnIns machineLearnIns = machineLearnInsDao.queryById(id);
if (machineLearnIns == null) {
return "实验实例不存在";
}
if (StringUtils.isEmpty(machineLearnIns.getStatus())) {
machineLearnIns = queryStatusFromArgo(machineLearnIns);
}
if (StringUtils.equals(machineLearnIns.getStatus(), Constant.Running)) {
return "实验实例正在运行,不可删除";
}
machineLearnIns.setState(Constant.State_invalid);
int update = machineLearnInsDao.update(machineLearnIns);
if (update > 0) {
resourceOccupyService.deleteTaskState(Constant.TaskType_ML, machineLearnIns.getMachineLearnId(), id);
updateMLStatus(machineLearnIns.getMachineLearnId());
return "删除成功";
} else {
return "删除失败";
}
}

@Override
public String batchDelete(List<Long> ids) {
for (Long id : ids) {
String result = removeById(id);
if (!"删除成功".equals(result)) {
return result;
}
}
return "删除成功";
}

@Override
public List<MachineLearnIns> queryNotTerminated() {
return machineLearnInsDao.queryNotTerminated();
}

@Override
public MachineLearnIns queryStatusFromArgo(MachineLearnIns ins) {
String namespace = ins.getArgoInsNs();
String name = ins.getArgoInsName();

// 创建请求数据map
Map<String, Object> requestData = new HashMap<>();
requestData.put("namespace", namespace);
requestData.put("name", name);

// 创建发送数据map,将请求数据作为"data"键的值
Map<String, Object> res = new HashMap<>();
res.put("data", requestData);

try {
// 发送POST请求到Argo工作流状态查询接口,并将请求数据转换为JSON
String req = HttpUtils.sendPost(argoUrl + argoWorkflowStatus, null, JsonUtils.mapToJson(res));
// 检查响应是否为空或无内容
if (req == null || StringUtils.isEmpty(req)) {
throw new RuntimeException("工作流状态响应为空");
}
// 将响应的JSON字符串转换为Map对象
Map<String, Object> runResMap = JsonUtils.jsonToMap(req);
// 从响应Map中获取"data"部分
Map<String, Object> data = (Map<String, Object>) runResMap.get("data");
if (data == null || data.isEmpty()) {
throw new RuntimeException("工作流数据为空");
}
// 从"data"中获取"status"部分,并返回"phase"的值
Map<String, Object> status = (Map<String, Object>) data.get("status");
if (status == null || status.isEmpty()) {
throw new RuntimeException("工作流状态为空");
}
//解析流水线结束时间
String finishedAtString = (String) status.get("finishedAt");
if (finishedAtString != null && !finishedAtString.isEmpty()) {
Date finishTime = DateUtils.convertUTCtoShanghaiDate(finishedAtString);
ins.setFinishTime(finishTime);
}
// 解析nodes字段,提取节点状态并转换为JSON字符串
Map<String, Object> nodes = (Map<String, Object>) status.get("nodes");
Map<String, Object> modifiedNodes = new LinkedHashMap<>();
if (nodes != null) {
for (Map.Entry<String, Object> nodeEntry : nodes.entrySet()) {
Map<String, Object> nodeDetails = (Map<String, Object>) nodeEntry.getValue();
String templateName = (String) nodeDetails.get("displayName");
modifiedNodes.put(templateName, nodeDetails);
}
}

String nodeStatusJson = JsonUtils.mapToJson(modifiedNodes);
ins.setNodeStatus(nodeStatusJson);

//终止态为终止不改
if (!StringUtils.equals(ins.getStatus(), Constant.Terminated)) {
ins.setStatus(StringUtils.isNotEmpty((String) status.get("phase")) ? (String) status.get("phase") : Constant.Pending);
}
if (StringUtils.equals(ins.getStatus(), Constant.Error)) {
ins.setStatus(Constant.Failed);
}
return ins;
} catch (Exception e) {
throw new RuntimeException("查询状态失败: " + e.getMessage(), e);
}
}


@Override
public boolean terminateMLIns(Long id) throws Exception {
MachineLearnIns machineLearnIns = machineLearnInsDao.queryById(id);
if (machineLearnIns == null) {
throw new IllegalStateException("实验实例未查询到");
}
String currentStatus = machineLearnIns.getStatus();
String name = machineLearnIns.getArgoInsName();
String namespace = machineLearnIns.getArgoInsNs();

// 获取当前状态,如果为空,则从Argo查询
if (StringUtils.isEmpty(currentStatus)) {
currentStatus = queryStatusFromArgo(machineLearnIns).getStatus();
}
// 只有状态是"Running"时才能终止实例
if (!currentStatus.equalsIgnoreCase(Constant.Running)) {
throw new Exception("终止错误,只有运行状态的实例才能终止"); // 如果不是"Running"状态,则不执行终止操作
}

// 创建请求数据map
Map<String, Object> requestData = new HashMap<>();
requestData.put("namespace", namespace);
requestData.put("name", name);
// 创建发送数据map,将请求数据作为"data"键的值
Map<String, Object> res = new HashMap<>();
res.put("data", requestData);

try {
// 发送POST请求到Argo工作流状态查询接口,并将请求数据转换为JSON
String req = HttpUtils.sendPost(argoUrl + argoWorkflowTermination, null, JsonUtils.mapToJson(res));
// 检查响应是否为空或无内容
if (StringUtils.isEmpty(req)) {
throw new RuntimeException("终止响应内容为空");

}
// 将响应的JSON字符串转换为Map对象
Map<String, Object> runResMap = JsonUtils.jsonToMap(req);
// 从响应Map中直接获取"errCode"的值
Integer errCode = (Integer) runResMap.get("errCode");
if (errCode != null && errCode == 0) {
MachineLearnIns ins = queryStatusFromArgo(machineLearnIns);
String nodeStatus = ins.getNodeStatus();
Map<String, Object> nodeMap = JsonUtils.jsonToMap(nodeStatus);

// 遍历 map
for (Map.Entry<String, Object> entry : nodeMap.entrySet()) {
// 获取每个 Map 中的值并强制转换为 Map
Map<String, Object> innerMap = (Map<String, Object>) entry.getValue();
// 检查 phase 的值
if (innerMap.containsKey("phase")) {
String phaseValue = (String) innerMap.get("phase");
// 如果值不等于 Succeeded,则赋值为 Failed
if (!StringUtils.equals(Constant.Succeeded, phaseValue)) {
innerMap.put("phase", Constant.Failed);
}
}
}
ins.setNodeStatus(JsonUtils.mapToJson(nodeMap));
ins.setStatus(Constant.Terminated);
ins.setUpdateTime(new Date());
ins.setFinishTime(new Date());
machineLearnInsDao.update(ins);
updateMLStatus(ins.getMachineLearnId());
// 结束扣积分
resourceOccupyService.endDeduce(Constant.TaskType_ML, null, id, null, null);
return true;
} else {
return false;
}
} catch (Exception e) {
throw new RuntimeException("终止实例错误: " + e.getMessage(), e);
}
}

@Override
public MachineLearnIns getDetailById(Long id) {
MachineLearnIns machineLearnIns = machineLearnInsDao.queryById(id);
if (Constant.Running.equals(machineLearnIns.getStatus()) || Constant.Pending.equals(machineLearnIns.getStatus())) {
machineLearnIns = queryStatusFromArgo(machineLearnIns);
}
return machineLearnIns;
}

@Override
public void updateMLStatus(Long machineLearnId) {
List<MachineLearnIns> insList = machineLearnInsDao.getByMachineLearnId(machineLearnId);
List<String> statusList = new ArrayList<>();
// 更新实验状态列表
for (int i = 0; i < insList.size(); i++) {
statusList.add(insList.get(i).getStatus());
}
String subStatus = statusList.toString().substring(1, statusList.toString().length() - 1);
MachineLearn machineLearn = machineLearnDao.getMachineLearnById(machineLearnId);

if (!StringUtils.equals(machineLearn.getStatusList(), subStatus)) {
machineLearn.setStatusList(subStatus);
machineLearnDao.edit(machineLearn);
}
}
}

+ 220
- 0
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/MachineLearnServiceImpl.java View File

@@ -0,0 +1,220 @@
package com.ruoyi.platform.service.impl;

import com.ruoyi.common.security.utils.SecurityUtils;
import com.ruoyi.platform.domain.MachineLearn;
import com.ruoyi.platform.domain.MachineLearnIns;
import com.ruoyi.platform.mapper.MachineLearnDao;
import com.ruoyi.platform.mapper.MachineLearnInsDao;
import com.ruoyi.platform.service.MachineLearnInsService;
import com.ruoyi.platform.service.MachineLearnService;
import com.ruoyi.platform.service.ResourceOccupyService;
import com.ruoyi.platform.utils.HttpUtils;
import com.ruoyi.platform.utils.JsonUtils;
import com.ruoyi.platform.vo.AutoMlParamVo;
import com.ruoyi.platform.vo.TextClassificationParamVo;
import com.ruoyi.system.api.constant.Constant;
import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageImpl;
import org.springframework.data.domain.PageRequest;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;

import javax.annotation.Resource;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

@Service
public class MachineLearnServiceImpl implements MachineLearnService {
@Value("${argo.url}")
private String argoUrl;
@Value("${argo.convertAutoML}")
String convertAutoML;
@Value("${argo.convertTextClassification}")
String convertTextClassification;
@Value("${argo.workflowRun}")
private String argoWorkflowRun;
@Value("${minio.endpointIp}")
private String minioEndpoint;

@Resource
private MachineLearnDao machineLearnDao;
@Resource
private MachineLearnInsDao machineLearnInsDao;
@Resource
private ResourceOccupyService resourceOccupyService;
@Autowired
private MachineLearnInsService machineLearnInsService;

@Override
public Page<MachineLearn> queryByPage(String name, String type, PageRequest pageRequest) {
long total = machineLearnDao.count(name, type);
List<MachineLearn> autoMls = machineLearnDao.queryByPage(name, type, pageRequest);
return new PageImpl<>(autoMls, pageRequest, total);
}

@Override
public MachineLearn add(MachineLearn machineLearn) {
if (machineLearn.getName().length() >= 64) {
throw new RuntimeException("实验名称大于最大长度");
}
MachineLearn machineLearnByName = machineLearnDao.getMachineLearnByName(machineLearn.getName());
if (machineLearnByName != null) {
throw new RuntimeException("实验名称已存在");
}
String username = SecurityUtils.getLoginUser().getUsername();
machineLearn.setCreateBy(username);
machineLearn.setUpdateBy(username);
machineLearnDao.save(machineLearn);
return machineLearn;
}

@Override
public String edit(MachineLearn machineLearn) {
MachineLearn machineLearnByName = machineLearnDao.getMachineLearnByName(machineLearn.getName());
if (machineLearnByName != null && !machineLearnByName.getId().equals(machineLearn.getId())) {
throw new RuntimeException("实验名称已存在");
}
machineLearn.setUpdateBy(SecurityUtils.getLoginUser().getUsername());
machineLearnDao.edit(machineLearn);
return "修改成功";
}

@Override
public MachineLearn getMLDetail(Long id) {
return machineLearnDao.getMachineLearnById(id);
}

@Override
@Transactional
public String delete(Long id) {
MachineLearn machineLearn = machineLearnDao.getMachineLearnById(id);
if (machineLearn == null) {
throw new RuntimeException("实验不存在");
}
String username = SecurityUtils.getLoginUser().getUsername();
String createBy = machineLearn.getCreateBy();
if (!(StringUtils.equals(username, "admin") || StringUtils.equals(username, createBy))) {
throw new RuntimeException("无权限删除该实验");
}
machineLearn.setUpdateBy(SecurityUtils.getLoginUser().getUsername());
machineLearn.setState(Constant.State_invalid);
resourceOccupyService.deleteTaskState(Constant.TaskType_ML, id, null);
return machineLearnDao.edit(machineLearn) > 0 ? "删除成功" : "删除失败";
}

@Override
@Transactional
public String runMachineLearn(Long id) throws Exception {
MachineLearn machineLearn = machineLearnDao.getMachineLearnById(id);
if (machineLearn == null) {
throw new Exception("自动机器学习配置不存在");
}
// 调argo转换接口
try {
String convertRes = null;
String param = null;
String modelType = null;
String taskType = null;
Integer seed = null;
Integer computingResourceId = null;
switch (machineLearn.getType()) {
case Constant.ML_CSV: {
AutoMlParamVo paramVo = JsonUtils.jsonToObject(machineLearn.getParam(), AutoMlParamVo.class);
taskType = paramVo.getTaskType();
seed = paramVo.getSeed();
param = JsonUtils.objectToJson(paramVo);
convertRes = HttpUtils.sendPost(argoUrl + convertAutoML, param);
break;
}
case Constant.ML_TextClassification: {
TextClassificationParamVo paramVo = JsonUtils.jsonToObject(machineLearn.getParam(), TextClassificationParamVo.class);
modelType = paramVo.getModelType();
computingResourceId = paramVo.getComputingResourceId();
if (resourceOccupyService.haveResource(computingResourceId, 1)) {
param = JsonUtils.getConvertParam(paramVo);
convertRes = HttpUtils.sendPost(argoUrl + convertTextClassification, param);
}
break;
}
case Constant.ML_VideoClassification: {
// todo
break;
}
}

if (convertRes == null || StringUtils.isEmpty(convertRes)) {
throw new RuntimeException("转换流水线失败");
}
Map<String, Object> converMap = JsonUtils.jsonToMap(convertRes);
// 组装运行接口json
Map<String, Object> output = (Map<String, Object>) converMap.get("output");
Map<String, Object> runReqMap = new HashMap<>();
runReqMap.put("data", converMap.get("data"));
// 调argo运行接口
String runRes = HttpUtils.sendPost(argoUrl + argoWorkflowRun, JsonUtils.mapToJson(runReqMap));

if (runRes == null || StringUtils.isEmpty(runRes)) {
throw new RuntimeException("运行流水线失败");
}
Map<String, Object> runResMap = JsonUtils.jsonToMap(runRes);
Map<String, Object> data = (Map<String, Object>) runResMap.get("data");
//判断data为空
if (data == null || MapUtils.isEmpty(data)) {
throw new RuntimeException("运行流水线失败");
}
Map<String, Object> metadata = (Map<String, Object>) data.get("metadata");
// 插入记录到实验实例表
MachineLearnIns machineLearnIns = new MachineLearnIns();
machineLearnIns.setMachineLearnId(id);
machineLearnIns.setType(machineLearn.getType());
machineLearnIns.setArgoInsNs((String) metadata.get("namespace"));
machineLearnIns.setArgoInsName((String) metadata.get("name"));
machineLearnIns.setParam(param);
machineLearnIns.setStatus(Constant.Pending);
//替换argoInsName
String outputString = JsonUtils.mapToJson(output);
machineLearnIns.setNodeResult(outputString.replace("{{workflow.name}}", (String) metadata.get("name")));
Map<String, Object> param_output = (Map<String, Object>) output.get("param_output");
List output1 = (ArrayList) param_output.values().toArray()[0];
Map<String, String> output2 = (Map<String, String>) output1.get(0);
String outputPath = minioEndpoint + "/" + output2.get("path").replace("{{workflow.name}}", (String) metadata.get("name")) + "/";

switch (machineLearn.getType()) {
case Constant.ML_CSV: {
machineLearnIns.setModelPath(outputPath + "save_model.joblib");
if (Constant.AutoMl_Classification.equals(taskType)) {
machineLearnIns.setImgPath(outputPath + "Auto-sklearn_metric_over_time.png" + "," + outputPath + "Train_Confusion_Matrix.png" + "," + outputPath + "Test_Confusion_Matrix.png");
} else {
machineLearnIns.setImgPath(outputPath + "Auto-sklearn_metric_over_time.png" + "," + outputPath + "regression.png");
}
machineLearnIns.setResultPath(outputPath + "result.txt");
String seedStr = seed != null ? String.valueOf(seed) : "1";
machineLearnIns.setRunHistoryPath(outputPath + "smac3-output/run_" + seedStr + "/runhistory.json");
break;
}
case Constant.ML_TextClassification: {
machineLearnIns.setModelPath(outputPath + "/saved_dict/" + modelType + ".ckpt");
break;
}
case Constant.ML_VideoClassification: {
break;
}
}
machineLearnInsDao.insert(machineLearnIns);
machineLearnInsService.updateMLStatus(id);
if (computingResourceId != null) {
// 记录开始扣除积分
resourceOccupyService.startDeduce(computingResourceId, 1, Constant.TaskType_ML, id, machineLearnIns.getId(), null, machineLearn.getName(), null, null);
}
} catch (Exception e) {
throw new RuntimeException(e);
}
return "执行成功";
}
}

+ 15
- 21
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java View File

@@ -51,9 +51,7 @@ public class NewDatasetServiceImpl implements NewDatasetService {
@Resource
private AssetWorkflowDao assetWorkflowDao;
@Resource
private AutoMlDao autoMlDao;
@Resource
private TextClassificationDao textClassificationDao;
private MachineLearnDao machineLearnDao;
@Resource
private RayDao rayDao;
@Resource
@@ -422,16 +420,14 @@ public class NewDatasetServiceImpl implements NewDatasetService {

HashMap<String, String> map = new HashMap<>();
map.put("id", String.valueOf(repoId));
List<AutoMl> autoMlList = autoMlDao.queryByDatasetId(JSON.toJSONString(map));
if (autoMlList != null && !autoMlList.isEmpty()) {
String autoMls = String.join(",", autoMlList.stream().map(AutoMl::getMlName).collect(Collectors.toSet()));
throw new Exception("该数据集被自动机器学习:" + autoMls + "使用,不能删除,请先删除自动机器学习");
}

List<TextClassification> textClassificationList = textClassificationDao.queryByDatasetId(JSON.toJSONString(map));
if (textClassificationList != null && !textClassificationList.isEmpty()) {
String textClassifications = String.join(",", textClassificationList.stream().map(TextClassification::getName).collect(Collectors.toSet()));
throw new Exception("该数据集被自动机器学习文本分类:" + textClassifications + "使用,不能删除,请先删除自动机器学习文本分类");
HashMap<Object, Object> mLQueryMap = new HashMap<>();
mLQueryMap.put("dataset", map);
List<MachineLearn> machineLearnList = machineLearnDao.queryByDatasetId(JSON.toJSONString(mLQueryMap));

if (machineLearnList != null && !machineLearnList.isEmpty()) {
String autoMls = String.join(",", machineLearnList.stream().map(MachineLearn::getName).collect(Collectors.toSet()));
throw new Exception("该数据集被自动机器学习:" + autoMls + "使用,不能删除,请先删除自动机器学习");
}

List<Ray> rayList = rayDao.queryByDatasetId(JSON.toJSONString(map));
@@ -471,16 +467,14 @@ public class NewDatasetServiceImpl implements NewDatasetService {
HashMap<String, String> map = new HashMap<>();
map.put("id", String.valueOf(repoId));
map.put("version", version);
List<AutoMl> autoMlList = autoMlDao.queryByDatasetId(JSON.toJSONString(map));
if (autoMlList != null && !autoMlList.isEmpty()) {
String autoMls = String.join(",", autoMlList.stream().map(AutoMl::getMlName).collect(Collectors.toSet()));
throw new Exception("该数据集版本被自动机器学习:" + autoMls + "使用,不能删除,请先删除自动机器学习");
}

List<TextClassification> textClassificationList = textClassificationDao.queryByDatasetId(JSON.toJSONString(map));
if (textClassificationList != null && !textClassificationList.isEmpty()) {
String textClassifications = String.join(",", textClassificationList.stream().map(TextClassification::getName).collect(Collectors.toSet()));
throw new Exception("该数据集版本被自动机器学习文本分类:" + textClassifications + "使用,不能删除,请先删除自动机器学习文本分类");
HashMap<Object, Object> mLQueryMap = new HashMap<>();
mLQueryMap.put("dataset", map);
List<MachineLearn> machineLearnList = machineLearnDao.queryByDatasetId(JSON.toJSONString(mLQueryMap));

if (machineLearnList != null && !machineLearnList.isEmpty()) {
String autoMls = String.join(",", machineLearnList.stream().map(MachineLearn::getName).collect(Collectors.toSet()));
throw new Exception("该数据集版本被自动机器学习:" + autoMls + "使用,不能删除,请先删除自动机器学习");
}

List<Ray> rayList = rayDao.queryByDatasetId(JSON.toJSONString(map));


+ 1
- 1
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/TextClassificationInsServiceImpl.java View File

@@ -249,7 +249,7 @@ public class TextClassificationInsServiceImpl implements TextClassificationInsSe
statusList.add(insList.get(i).getStatus());
}
String subStatus = statusList.toString().substring(1, statusList.toString().length() - 1);
TextClassification textClassification = textClassificationDao.getById(new Long(textClassificationId));
TextClassification textClassification = textClassificationDao.getById(textClassificationId);

if (!StringUtils.equals(textClassification.getStatusList(), subStatus)) {
textClassification.setStatusList(subStatus);


+ 2
- 0
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/TextClassificationParamVo.java View File

@@ -28,4 +28,6 @@ public class TextClassificationParamVo {

@ApiModelProperty(value = "学习率")
private Float lr;

private Integer computingResourceId;
}

+ 82
- 0
ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/MachineLearnDaoMapper.xml View File

@@ -0,0 +1,82 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.ruoyi.platform.mapper.MachineLearnDao">

<select id="count" resultType="java.lang.Long">
select count(1) from machine_learn
<include refid="common_condition"></include>
</select>

<select id="queryByPage" resultType="com.ruoyi.platform.domain.MachineLearn">
select * from machine_learn
<include refid="common_condition"></include>
order by create_time desc limit #{pageable.offset}, #{pageable.pageSize}
</select>

<insert id="save" keyProperty="id" useGeneratedKeys="true">
insert into machine_learn(name, description, type, param, create_by, update_by)
values (#{machineLearn.name}, #{machineLearn.description}, #{machineLearn.type},
#{machineLearn.param}, #{machineLearn.createBy}, #{machineLearn.updateBy})
</insert>

<update id="edit">
update machine_learn
<set>
<if test="machineLearn.name != null and machineLearn.name !=''">
name = #{machineLearn.name},
</if>
<if test="machineLearn.description != null and machineLearn.description !=''">
description = #{machineLearn.description},
</if>
<if test="machineLearn.type != null and machineLearn.type !=''">
type = #{machineLearn.type},
</if>
<if test="machineLearn.param != null and machineLearn.param !=''">
param = #{machineLearn.param},
</if>
<if test="machineLearn.statusList != null and machineLearn.statusList !=''">
status_list = #{machineLearn.statusList},
</if>
<if test="machineLearn.state != null">
state = #{machineLearn.state},
</if>
<if test="machineLearn.updateBy != null and machineLearn.updateBy !=''">
update_by = #{machineLearn.updateBy},
</if>
</set>
where id = #{machineLearn.id}
</update>


<select id="getMachineLearnById" resultType="com.ruoyi.platform.domain.MachineLearn">
select *
from machine_learn
where id = #{id}
</select>

<select id="getMachineLearnByName" resultType="com.ruoyi.platform.domain.MachineLearn">
select *
from machine_learn
where name = #{name}
and state = 1
</select>

<select id="queryByDatasetId" resultType="com.ruoyi.platform.domain.MachineLearn">
select *
from machine_learn
where JSON_CONTAINS(param, #{datasetId})
and state = 1
</select>

<sql id="common_condition">
<where>
<if test="name != null and name != ''">
and name like concat('%', #{name}, '%')
</if>
<if test="type != null and type != ''">
and type = #{type}
</if>
and state = 1
</where>
</sql>
</mapper>

+ 83
- 0
ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/MachineLearnInsDaoMapper.xml View File

@@ -0,0 +1,83 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.ruoyi.platform.mapper.MachineLearnInsDao">
<insert id="insert" keyProperty="id" useGeneratedKeys="true">
insert into machine_learn_ins(machine_learn_id, type, result_path, model_path, img_path, run_history_path, node_status,
node_result, param, argo_ins_name, argo_ins_ns, status)
values (#{machineLearnIns.machineLearnId}, #{machineLearnIns.type}, #{machineLearnIns.resultPath}, #{machineLearnIns.modelPath}, #{machineLearnIns.imgPath},
#{machineLearnIns.runHistoryPath}, #{machineLearnIns.nodeStatus},
#{machineLearnIns.nodeResult}, #{machineLearnIns.param},#{machineLearnIns.argoInsName},
#{machineLearnIns.argoInsNs}, #{machineLearnIns.status}
)
</insert>

<update id="update">
update machine_learn_ins
<set>
<if test="machineLearnIns.type != null and machineLearnIns.type != ''">
type = #{machineLearnIns.type},
</if>
<if test="machineLearnIns.status != null and machineLearnIns.status != ''">
status = #{machineLearnIns.status},
</if>
<if test="machineLearnIns.state != null">
state = #{machineLearnIns.state},
</if>
<if test="machineLearnIns.updateTime != null">
update_time = #{machineLearnIns.updateTime},
</if>
<if test="machineLearnIns.finishTime != null">
finish_time = #{machineLearnIns.finishTime},
</if>
<if test="machineLearnIns.nodeStatus != null and machineLearnIns.nodeStatus != ''">
node_status = #{machineLearnIns.nodeStatus},
</if>
<if test="machineLearnIns.nodeResult != null and machineLearnIns.nodeResult != ''">
node_result = #{machineLearnIns.nodeResult},
</if>
</set>
where id = #{machineLearnIns.id}
</update>

<select id="count" resultType="java.lang.Long">
select count(1)
from machine_learn_ins
<where>
state = 1
and machine_learn_id = #{machineLearnId}
</where>
</select>

<select id="queryAllByLimit" resultType="com.ruoyi.platform.domain.MachineLearnIns">
select * from machine_learn_ins
<where>
state = 1
and machine_learn_id = #{machineLearnId}
</where>
order by update_time DESC
limit #{pageable.offset}, #{pageable.pageSize}
</select>

<select id="queryById" resultType="com.ruoyi.platform.domain.MachineLearnIns">
select * from machine_learn_ins
<where>
state = 1 and id = #{id}
</where>
</select>

<select id="queryNotTerminated" resultType="com.ruoyi.platform.domain.MachineLearnIns">
select *
from machine_learn_ins
where (status NOT IN ('Terminated', 'Succeeded', 'Failed')
OR status IS NULL)
and state = 1
</select>

<select id="getByMachineLearnId" resultType="com.ruoyi.platform.domain.MachineLearnIns">
select *
from machine_learn_ins
where machine_learn_id = #{machineLearnId}
and state = 1
order by update_time DESC limit 5
</select>
</mapper>

Loading…
Cancel
Save