Browse Source

积分功能开发

dev-credits
chenzhihang 10 months ago
parent
commit
64b62830af
27 changed files with 254 additions and 302 deletions
  1. +8
    -0
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/constant/Constant.java
  2. +1
    -1
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/service/ServiceController.java
  3. +5
    -129
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/DevEnvironment.java
  4. +2
    -0
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/Ray.java
  5. +1
    -1
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/ResourceOccupy.java
  6. +2
    -0
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/ServiceVersion.java
  7. +1
    -18
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/DevEnvironmentDao.java
  8. +3
    -1
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/ResourceOccupyDao.java
  9. +2
    -0
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/ServiceDao.java
  10. +7
    -0
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/scheduling/RayInsStatusTask.java
  11. +44
    -0
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/scheduling/ResourceOccupyTask.java
  12. +3
    -3
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ResourceOccupyService.java
  13. +2
    -2
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ServiceService.java
  14. +10
    -8
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/DevEnvironmentServiceImpl.java
  15. +8
    -8
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/JupyterServiceImpl.java
  16. +6
    -0
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/RayInsServiceImpl.java
  17. +58
    -53
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/RayServiceImpl.java
  18. +12
    -5
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ResourceOccupyServiceImpl.java
  19. +27
    -15
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ServiceServiceImpl.java
  20. +8
    -8
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/K8sClientUtil.java
  21. +3
    -0
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/DevEnvironmentVo.java
  22. +2
    -0
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/serviceVos/ServiceVersionVo.java
  23. +13
    -43
      ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/DevEnvironmentDaoMapper.xml
  24. +9
    -4
      ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/RayDaoMapper.xml
  25. +1
    -1
      ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/RayInsDaoMapper.xml
  26. +11
    -2
      ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ResourceOccupy.xml
  27. +5
    -0
      ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ServiceDaoMapper.xml

+ 8
- 0
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/constant/Constant.java View File

@@ -49,4 +49,12 @@ public class Constant {
public final static String Asset_Type_Image = "image";
public final static String Asset_Type_Code = "code";
public final static String Asset_Type_Service = "service";

// 任务类型
public final static String TaskType_Dev = "dev_environment";
public final static String TaskType_Workflow = "workflow";
public final static String TaskType_AutoMl = "auto_ml";
public final static String TaskType_Ray = "ray";
public final static String TaskType_ActiveLearn = "active_learn";
public final static String TaskType_Service = "service";
}

+ 1
- 1
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/service/ServiceController.java View File

@@ -61,7 +61,7 @@ public class ServiceController extends BaseController {

@PostMapping("/serviceVersion")
@ApiOperation("新增服务版本")
public GenericsAjaxResult<ServiceVersion> addServiceVersion(@RequestBody ServiceVersionVo serviceVersionVo) {
public GenericsAjaxResult<ServiceVersion> addServiceVersion(@RequestBody ServiceVersionVo serviceVersionVo) throws Exception {
return genericsSuccess(serviceService.addServiceVersion(serviceVersionVo));
}



+ 5
- 129
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/DevEnvironment.java View File

@@ -2,6 +2,7 @@ package com.ruoyi.platform.domain;

import com.fasterxml.jackson.databind.PropertyNamingStrategy;
import com.fasterxml.jackson.databind.annotation.JsonNaming;
import lombok.Data;

import java.util.Date;
import java.io.Serializable;
@@ -13,6 +14,7 @@ import java.io.Serializable;
* @since 2024-06-03 15:17:37
*/
@JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy.class)
@Data
public class DevEnvironment implements Serializable {
private static final long serialVersionUID = 936999018935545992L;
/**
@@ -31,6 +33,9 @@ public class DevEnvironment implements Serializable {
* 计算资源
*/
private String computingResource;

private Integer computingResourceId;

/**
* 资源规格
*/
@@ -80,134 +85,5 @@ public class DevEnvironment implements Serializable {
*/
private Integer state;


public Integer getId() {
return id;
}

public void setId(Integer id) {
this.id = id;
}

public String getName() {
return name;
}

public void setName(String name) {
this.name = name;
}

public String getStatus() {
return status;
}

public void setStatus(String status) {
this.status = status;
}

public String getComputingResource() {
return computingResource;
}

public void setComputingResource(String computingResource) {
this.computingResource = computingResource;
}

public String getStandard() {
return standard;
}

public void setStandard(String standard) {
this.standard = standard;
}

public String getEnvVariable() {
return envVariable;
}

public void setEnvVariable(String envVariable) {
this.envVariable = envVariable;
}

public String getImage() {
return image;
}

public void setImage(String image) {
this.image = image;
}

public String getDataset() {
return dataset;
}

public void setDataset(String dataset) {
this.dataset = dataset;
}

public String getModel() {
return model;
}

public void setModel(String model) {
this.model = model;
}

public String getUrl() {
return url;
}

public void setUrl(String url) {
this.url = url;
}

public String getAltField2() {
return altField2;
}

public void setAltField2(String altField2) {
this.altField2 = altField2;
}

public String getCreateBy() {
return createBy;
}

public void setCreateBy(String createBy) {
this.createBy = createBy;
}

public Date getCreateTime() {
return createTime;
}

public void setCreateTime(Date createTime) {
this.createTime = createTime;
}

public String getUpdateBy() {
return updateBy;
}

public void setUpdateBy(String updateBy) {
this.updateBy = updateBy;
}

public Date getUpdateTime() {
return updateTime;
}

public void setUpdateTime(Date updateTime) {
this.updateTime = updateTime;
}

public Integer getState() {
return state;
}

public void setState(Integer state) {
this.state = state;
}

}


+ 2
- 0
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/Ray.java View File

@@ -67,6 +67,8 @@ public class Ray {

private String resource;

private Integer computingResourceId;

private Integer state;

private String createBy;


+ 1
- 1
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/ResourceOccupy.java View File

@@ -35,5 +35,5 @@ public class ResourceOccupy {
private String taskType;

@ApiModelProperty("类型id")
private Integer taskId;
private Long taskId;
}

+ 2
- 0
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/ServiceVersion.java View File

@@ -26,6 +26,8 @@ public class ServiceVersion implements Serializable {

private String resource;

private Integer computingResourceId;

private Integer replicas;

private String mountPath;


+ 1
- 18
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/DevEnvironmentDao.java View File

@@ -46,24 +46,6 @@ public interface DevEnvironmentDao {
*/
int insert(@Param("devEnvironment") DevEnvironment devEnvironment);

/**
* 批量新增数据(MyBatis原生foreach方法)
*
* @param entities List<DevEnvironment> 实例对象列表
* @return 影响行数
*/
int insertBatch(@Param("entities") List<DevEnvironment> entities);

/**
* 批量新增或按主键更新数据(MyBatis原生foreach方法)
*
* @param entities List<DevEnvironment> 实例对象列表
*
* @return 影响行数
* @throws org.springframework.jdbc.BadSqlGrammarException 入参是空List的时候会抛SQL语句错误的异常,请自行校验入参
*/
int insertOrUpdateBatch(@Param("entities") List<DevEnvironment> entities);

/**
* 修改数据
*
@@ -80,5 +62,6 @@ public interface DevEnvironmentDao {
*/
int deleteById(Integer id);

List<DevEnvironment> getRunning();
}


+ 3
- 1
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/ResourceOccupyDao.java View File

@@ -11,5 +11,7 @@ public interface ResourceOccupyDao {

int edit(@Param("resourceOccupy") ResourceOccupy resourceOccupy);

ResourceOccupy getResourceOccupyById(@Param("id") Integer id);
ResourceOccupy getResourceOccupyByTask(@Param("taskType") String taskType, @Param("taskId") Long taskId);

int deduceCredit(@Param("credit") Float credit, @Param("userId") Long userId);
}

+ 2
- 0
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/ServiceDao.java View File

@@ -36,4 +36,6 @@ public interface ServiceDao {
Service getServiceByName(@Param("serviceName") String serviceName);

ServiceVersion getSvByVersion(@Param("version") String version, @Param("serviceId") Long serviceId);

List<ServiceVersion> getRunning();
}

+ 7
- 0
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/scheduling/RayInsStatusTask.java View File

@@ -6,6 +6,7 @@ import com.ruoyi.platform.domain.RayIns;
import com.ruoyi.platform.mapper.RayDao;
import com.ruoyi.platform.mapper.RayInsDao;
import com.ruoyi.platform.service.RayInsService;
import com.ruoyi.platform.service.ResourceOccupyService;
import org.apache.commons.lang3.StringUtils;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
@@ -25,6 +26,9 @@ public class RayInsStatusTask {
@Resource
private RayDao rayDao;

@Resource
private ResourceOccupyService resourceOccupyService;

private List<Long> rayIds = new ArrayList<>();

@Scheduled(cron = "0/30 * * * * ?") // 每30S执行一次
@@ -38,6 +42,9 @@ public class RayInsStatusTask {
//当原本状态为null或非终止态时才调用argo接口
try {
rayIns = rayInsService.queryStatusFromArgo(rayIns);
if (Constant.Running.equals(rayIns.getStatus())) {
resourceOccupyService.deducing(Constant.TaskType_Ray, rayIns.getId());
}
} catch (Exception e) {
rayIns.setStatus(Constant.Failed);
}


+ 44
- 0
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/scheduling/ResourceOccupyTask.java View File

@@ -0,0 +1,44 @@
package com.ruoyi.platform.scheduling;

import com.ruoyi.platform.constant.Constant;
import com.ruoyi.platform.domain.DevEnvironment;
import com.ruoyi.platform.domain.ServiceVersion;
import com.ruoyi.platform.mapper.DevEnvironmentDao;
import com.ruoyi.platform.mapper.ServiceDao;
import com.ruoyi.platform.service.ResourceOccupyService;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;

import javax.annotation.Resource;
import java.util.List;

@Component()
public class ResourceOccupyTask {

@Resource
private ResourceOccupyService resourceOccupyService;

@Resource
private DevEnvironmentDao devEnvironmentDao;

@Resource
private ServiceDao serviceDao;

// 开发环境功能扣除积分
@Scheduled(cron = "0 0/10 * * * ?") // 每10分钟执行一次
public void devDeduceCredit() {
List<DevEnvironment> devEnvironments = devEnvironmentDao.getRunning();
for (DevEnvironment devEnvironment : devEnvironments) {
resourceOccupyService.deducing(Constant.TaskType_Dev, Long.valueOf(devEnvironment.getId()));
}
}

// 服务功能扣除积分
@Scheduled(cron = "0 0/10 * * * ?") // 每10分钟执行一次
public void serviceDeduceCredit() {
List<ServiceVersion> serviceVersions = serviceDao.getRunning();
for (ServiceVersion serviceVersion : serviceVersions) {
resourceOccupyService.deducing(Constant.TaskType_Service, serviceVersion.getId());
}
}
}

+ 3
- 3
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ResourceOccupyService.java View File

@@ -4,10 +4,10 @@ public interface ResourceOccupyService {

Boolean haveResource(Integer computingResourceId) throws Exception;

void startDeduce(Integer computingResourceId);
void startDeduce(Integer computingResourceId, String taskType, Long taskId);

void endDeduce(Integer id);
void endDeduce(String taskType, Long taskId);

void deducing();
void deducing(String taskType, Long taskId);

}

+ 2
- 2
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ServiceService.java View File

@@ -18,7 +18,7 @@ public interface ServiceService {

Service addService(Service service);

ServiceVersion addServiceVersion(ServiceVersionVo serviceVersionVo);
ServiceVersion addServiceVersion(ServiceVersionVo serviceVersionVo) throws Exception;

Service editService(Service service);

@@ -34,7 +34,7 @@ public interface ServiceService {

String deleteServiceVersion(Long id);

String runServiceVersion(Long id);
String runServiceVersion(Long id) throws Exception;

String stopServiceVersion(Long id);



+ 10
- 8
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/DevEnvironmentServiceImpl.java View File

@@ -10,18 +10,16 @@ import com.ruoyi.platform.utils.JacksonUtil;
import com.ruoyi.platform.vo.DevEnvironmentVo;
import com.ruoyi.platform.vo.PodStatusVo;
import com.ruoyi.system.api.model.LoginUser;
import io.kubernetes.client.openapi.models.V1PersistentVolumeClaim;
import org.apache.commons.lang3.StringUtils;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageImpl;
import org.springframework.data.domain.PageRequest;
import org.springframework.stereotype.Service;

import javax.annotation.Resource;
import java.util.Date;
import java.util.List;
import java.util.Map;

/**
* (DevEnvironment)表服务实现类
@@ -54,7 +52,7 @@ public class DevEnvironmentServiceImpl implements DevEnvironmentService {
* 分页查询
*
* @param devEnvironment 筛选条件
* @param pageRequest 分页对象
* @param pageRequest 分页对象
* @return 查询结果
*/
@Override
@@ -64,12 +62,15 @@ public class DevEnvironmentServiceImpl implements DevEnvironmentService {

//查询每个开发环境的pod状态,注意:只有pod为非终止态时才去调状态接口
devEnvironmentList.forEach(devEnv -> {
try{
try {
if (!devEnv.getStatus().equals(PodStatus.Terminated.getName()) &&
!devEnv.getStatus().equals(PodStatus.Failed.getName())) {
PodStatusVo podStatusVo = this.jupyterService.getJupyterStatus(devEnv);
devEnv.setStatus(podStatusVo.getStatus());
devEnv.setUrl(podStatusVo.getUrl());
if(!devEnv.getStatus().equals(podStatusVo.getStatus())){
this.devEnvironmentDao.update(devEnv);
}
}
} catch (Exception e) {
devEnv.setStatus(PodStatus.Unknown.getName());
@@ -94,6 +95,7 @@ public class DevEnvironmentServiceImpl implements DevEnvironmentService {
//状态先设为未知
devEnvironment.setStatus("Unknown");
devEnvironment.setComputingResource(devEnvironmentVo.getComputingResource());
devEnvironment.setComputingResourceId(devEnvironmentVo.getComputingResourceId());
devEnvironment.setStandard(devEnvironmentVo.getStandard());
devEnvironment.setEnvVariable(devEnvironmentVo.getEnvVariable());
devEnvironment.setImage(devEnvironmentVo.getImage());
@@ -140,7 +142,7 @@ public class DevEnvironmentServiceImpl implements DevEnvironmentService {
@Override
public String removeById(Integer id) throws Exception {
DevEnvironment devEnvironment = this.devEnvironmentDao.queryById(id);
if (devEnvironment == null){
if (devEnvironment == null) {
return "开发环境信息不存在";
}

@@ -148,13 +150,13 @@ public class DevEnvironmentServiceImpl implements DevEnvironmentService {
LoginUser loginUser = SecurityUtils.getLoginUser();
String username = loginUser.getUsername();
String createdBy = devEnvironment.getCreateBy();
if (!(StringUtils.equals(username,"admin") || StringUtils.equals(username,createdBy))){
if (!(StringUtils.equals(username, "admin") || StringUtils.equals(username, createdBy))) {
return "无权限删除该开发环境";
}

jupyterService.stopJupyterService(id);
devEnvironment.setState(0);
return this.devEnvironmentDao.update(devEnvironment)>0?"删除成功":"删除失败";
return this.devEnvironmentDao.update(devEnvironment) > 0 ? "删除成功" : "删除失败";
}




+ 8
- 8
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/JupyterServiceImpl.java View File

@@ -9,6 +9,7 @@ import com.ruoyi.platform.mapper.ComputingResourceDao;
import com.ruoyi.platform.mapper.DevEnvironmentDao;
import com.ruoyi.platform.service.DevEnvironmentService;
import com.ruoyi.platform.service.JupyterService;
import com.ruoyi.platform.service.ResourceOccupyService;
import com.ruoyi.platform.utils.JacksonUtil;
import com.ruoyi.platform.utils.K8sClientUtil;
import com.ruoyi.platform.utils.MinioUtil;
@@ -56,9 +57,6 @@ public class JupyterServiceImpl implements JupyterService {
@Resource
private DevEnvironmentDao devEnvironmentDao;

@Resource
private ComputingResourceDao computingResourceDao;

@Resource
@Lazy
private DevEnvironmentService devEnvironmentService;
@@ -66,6 +64,9 @@ public class JupyterServiceImpl implements JupyterService {
@Resource
private RedisService redisService;

@Resource
private ResourceOccupyService resourceOccupyService;

public JupyterServiceImpl(MinioUtil minioUtil) {
this.minioUtil = minioUtil;
}
@@ -109,7 +110,7 @@ public class JupyterServiceImpl implements JupyterService {
Integer podPort = k8sClientUtil.createConfiguredPod(podName, namespace, port, mountPath, null, devEnvironment, minioPvcName, datasetPath, modelPath);
String url = masterIp + ":" + podPort;
redisService.setCacheObject(podName, masterIp + ":" + podPort);
devEnvironment.setStatus("Pending");
devEnvironment.setStatus(Constant.Pending);
devEnvironment.setUrl(url);
this.devEnvironmentService.update(devEnvironment);
return url;
@@ -133,16 +134,15 @@ public class JupyterServiceImpl implements JupyterService {
return "pod不存在!";
}

if (Constant.Computing_Resource_GPU.equals(devEnvironment.getComputingResource())) {
computingResourceDao.updateUsedStateByNode(pod.getSpec().getNodeName(), Constant.Used_State_unused);
}
// 结束扣积分
resourceOccupyService.endDeduce(Constant.TaskType_Dev, Long.valueOf(id));

// 使用 Kubernetes API 删除 Pod
String deleteResult = k8sClientUtil.deletePod(podName, namespace);
// 删除service
k8sClientUtil.deleteService(svcName, namespace);

devEnvironment.setStatus("Terminated");
devEnvironment.setStatus(Constant.Terminated);
this.devEnvironmentService.update(devEnvironment);
return deleteResult + ",编辑器已停止";



+ 6
- 0
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/RayInsServiceImpl.java View File

@@ -6,6 +6,7 @@ import com.ruoyi.platform.domain.RayIns;
import com.ruoyi.platform.mapper.RayDao;
import com.ruoyi.platform.mapper.RayInsDao;
import com.ruoyi.platform.service.RayInsService;
import com.ruoyi.platform.service.ResourceOccupyService;
import com.ruoyi.platform.utils.*;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
@@ -48,6 +49,9 @@ public class RayInsServiceImpl implements RayInsService {
@Resource
private RayDao rayDao;

@Resource
private ResourceOccupyService resourceOccupyService;

@Resource
private MinioUtil minioUtil;

@@ -163,6 +167,8 @@ public class RayInsServiceImpl implements RayInsService {
ins.setUpdateTime(new Date());
rayInsDao.update(ins);
updateRayStatus(rayIns.getRayId());
// 结束扣积分
resourceOccupyService.endDeduce(Constant.TaskType_Ray, id);
return true;
} else {
return false;


+ 58
- 53
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/RayServiceImpl.java View File

@@ -10,6 +10,7 @@ import com.ruoyi.platform.mapper.RayDao;
import com.ruoyi.platform.mapper.RayInsDao;
import com.ruoyi.platform.service.RayInsService;
import com.ruoyi.platform.service.RayService;
import com.ruoyi.platform.service.ResourceOccupyService;
import com.ruoyi.platform.utils.HttpUtils;
import com.ruoyi.platform.utils.JacksonUtil;
import com.ruoyi.platform.utils.JsonUtils;
@@ -42,8 +43,8 @@ public class RayServiceImpl implements RayService {
@Value("${argo.workflowRun}")
private String argoWorkflowRun;

@Value("${minio.endpoint}")
private String minioEndpoint;
@Resource
private ResourceOccupyService resourceOccupyService;

@Resource
private RayDao rayDao;
@@ -151,58 +152,62 @@ public class RayServiceImpl implements RayService {
throw new Exception("自动超参数寻优配置不存在");
}

RayParamVo rayParamVo = new RayParamVo();
BeanUtils.copyProperties(ray, rayParamVo);
rayParamVo.setCodeConfig(JsonUtils.jsonToMap(ray.getCodeConfig()));
rayParamVo.setDataset(JsonUtils.jsonToMap(ray.getDataset()));
rayParamVo.setModel(JsonUtils.jsonToMap(ray.getModel()));
rayParamVo.setImage(JsonUtils.jsonToMap(ray.getImage()));
String param = JsonUtils.objectToJson(rayParamVo);

// 调argo转换接口
try {
String convertRes = HttpUtils.sendPost(argoUrl + convertRay, param);
if (convertRes == null || StringUtils.isEmpty(convertRes)) {
throw new RuntimeException("转换流水线失败");
}
Map<String, Object> converMap = JsonUtils.jsonToMap(convertRes);
// 组装运行接口json
Map<String, Object> output = (Map<String, Object>) converMap.get("output");
Map<String, Object> runReqMap = new HashMap<>();
runReqMap.put("data", converMap.get("data"));
// 调argo运行接口
String runRes = HttpUtils.sendPost(argoUrl + argoWorkflowRun, JsonUtils.mapToJson(runReqMap));
if (runRes == null || StringUtils.isEmpty(runRes)) {
throw new RuntimeException("Failed to run workflow.");
}
Map<String, Object> runResMap = JsonUtils.jsonToMap(runRes);
Map<String, Object> data = (Map<String, Object>) runResMap.get("data");
//判断data为空
if (data == null || MapUtils.isEmpty(data)) {
throw new RuntimeException("Failed to run workflow.");
// 记录开始扣积分
if (resourceOccupyService.haveResource(ray.getComputingResourceId())) {
RayParamVo rayParamVo = new RayParamVo();
BeanUtils.copyProperties(ray, rayParamVo);
rayParamVo.setCodeConfig(JsonUtils.jsonToMap(ray.getCodeConfig()));
rayParamVo.setDataset(JsonUtils.jsonToMap(ray.getDataset()));
rayParamVo.setModel(JsonUtils.jsonToMap(ray.getModel()));
rayParamVo.setImage(JsonUtils.jsonToMap(ray.getImage()));
String param = JsonUtils.objectToJson(rayParamVo);

// 调argo转换接口
try {
String convertRes = HttpUtils.sendPost(argoUrl + convertRay, param);
if (convertRes == null || StringUtils.isEmpty(convertRes)) {
throw new RuntimeException("转换流水线失败");
}
Map<String, Object> converMap = JsonUtils.jsonToMap(convertRes);
// 组装运行接口json
Map<String, Object> output = (Map<String, Object>) converMap.get("output");
Map<String, Object> runReqMap = new HashMap<>();
runReqMap.put("data", converMap.get("data"));
// 调argo运行接口
String runRes = HttpUtils.sendPost(argoUrl + argoWorkflowRun, JsonUtils.mapToJson(runReqMap));
if (runRes == null || StringUtils.isEmpty(runRes)) {
throw new RuntimeException("运行失败");
}
Map<String, Object> runResMap = JsonUtils.jsonToMap(runRes);
Map<String, Object> data = (Map<String, Object>) runResMap.get("data");
//判断data为空
if (data == null || MapUtils.isEmpty(data)) {
throw new RuntimeException("运行失败");
}
Map<String, Object> metadata = (Map<String, Object>) data.get("metadata");

// 插入记录到实验实例表
RayIns rayIns = new RayIns();
rayIns.setRayId(ray.getId());
rayIns.setArgoInsNs((String) metadata.get("namespace"));
rayIns.setArgoInsName((String) metadata.get("name"));
rayIns.setParam(param);
rayIns.setStatus(Constant.Pending);
//替换argoInsName
String outputString = JsonUtils.mapToJson(output);
rayIns.setNodeResult(outputString.replace("{{workflow.name}}", (String) metadata.get("name")));

Map<String, Object> param_output = (Map<String, Object>) output.get("param_output");
List output1 = (ArrayList) param_output.values().toArray()[0];
Map<String, String> output2 = (Map<String, String>) output1.get(0);
String outputPath = output2.get("path").replace("{{workflow.name}}", (String) metadata.get("name")) + "/hpo";
rayIns.setResultPath(outputPath);
rayInsDao.insert(rayIns);
rayInsService.updateRayStatus(id);
resourceOccupyService.startDeduce(ray.getComputingResourceId(), Constant.TaskType_Ray, rayIns.getId());
} catch (Exception e) {
throw new RuntimeException(e);
}
Map<String, Object> metadata = (Map<String, Object>) data.get("metadata");

// 插入记录到实验实例表
RayIns rayIns = new RayIns();
rayIns.setRayId(ray.getId());
rayIns.setArgoInsNs((String) metadata.get("namespace"));
rayIns.setArgoInsName((String) metadata.get("name"));
rayIns.setParam(param);
rayIns.setStatus(Constant.Pending);
//替换argoInsName
String outputString = JsonUtils.mapToJson(output);
rayIns.setNodeResult(outputString.replace("{{workflow.name}}", (String) metadata.get("name")));

Map<String, Object> param_output = (Map<String, Object>) output.get("param_output");
List output1 = (ArrayList) param_output.values().toArray()[0];
Map<String, String> output2 = (Map<String, String>) output1.get(0);
String outputPath = output2.get("path").replace("{{workflow.name}}", (String) metadata.get("name")) + "/hpo";
rayIns.setResultPath(outputPath);
rayInsDao.insert(rayIns);
rayInsService.updateRayStatus(id);
} catch (Exception e) {
throw new RuntimeException(e);
}
return "执行成功";
}


+ 12
- 5
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ResourceOccupyServiceImpl.java View File

@@ -45,27 +45,34 @@ public class ResourceOccupyServiceImpl implements ResourceOccupyService {
}

@Override
public void startDeduce(Integer computingResourceId) {
public void startDeduce(Integer computingResourceId, String taskType, Long taskId) {
ResourceOccupy resourceOccupy = new ResourceOccupy();
ComputingResource computingResource = computingResourceDao.queryById(computingResourceId);
resourceOccupy.setComputingResourceId(computingResourceId);
LoginUser loginUser = SecurityUtils.getLoginUser();
resourceOccupy.setUserId(loginUser.getUserid());
resourceOccupy.setCreditPerHour(computingResource.getCreditPerHour());
resourceOccupy.setTaskType(taskType);
resourceOccupy.setTaskId(taskId);
resourceOccupyDao.save(resourceOccupy);
}

@Override
public void endDeduce(Integer id) {
ResourceOccupy resourceOccupy = resourceOccupyDao.getResourceOccupyById(id);
public void endDeduce(String taskType, Long taskId) {
ResourceOccupy resourceOccupy = resourceOccupyDao.getResourceOccupyByTask(taskType, taskId);
deducing(taskType, taskId);
resourceOccupy.setState(Constant.State_invalid);
deducing();
resourceOccupy.setDeduceLastTime(new Date());
resourceOccupyDao.edit(resourceOccupy);
}

@Override
public void deducing() {
public void deducing(String taskType, Long taskId) {
ResourceOccupy resourceOccupy = resourceOccupyDao.getResourceOccupyByTask(taskType, taskId);
long timeDifferenceMillis = new Date().getTime() - resourceOccupy.getDeduceLastTime().getTime();
Float hours = (float) (timeDifferenceMillis / (1000 * 60 * 60));

float deduceCredit = resourceOccupy.getCreditPerHour() * hours;
resourceOccupyDao.deduceCredit(deduceCredit, resourceOccupy.getUserId());
}
}

+ 27
- 15
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ServiceServiceImpl.java View File

@@ -8,6 +8,7 @@ import com.ruoyi.platform.domain.AssetWorkflow;
import com.ruoyi.platform.domain.ServiceVersion;
import com.ruoyi.platform.mapper.AssetWorkflowDao;
import com.ruoyi.platform.mapper.ServiceDao;
import com.ruoyi.platform.service.ResourceOccupyService;
import com.ruoyi.platform.service.ServiceService;
import com.ruoyi.platform.utils.ConvertUtil;
import com.ruoyi.platform.utils.HttpUtils;
@@ -45,6 +46,9 @@ public class ServiceServiceImpl implements ServiceService {
@Resource
private AssetWorkflowDao assetWorkflowDao;

@Resource
private ResourceOccupyService resourceOccupyService;

@Override
public Page<com.ruoyi.platform.domain.Service> queryByPageService(com.ruoyi.platform.domain.Service service, PageRequest pageRequest) {
long total = serviceDao.countService(service);
@@ -110,7 +114,7 @@ public class ServiceServiceImpl implements ServiceService {
}

@Override
public ServiceVersion addServiceVersion(ServiceVersionVo serviceVersionVo) {
public ServiceVersion addServiceVersion(ServiceVersionVo serviceVersionVo) throws Exception {
ServiceVersion svByVersion = serviceDao.getSvByVersion(serviceVersionVo.getVersion(), serviceVersionVo.getServiceId());
if (svByVersion != null) {
throw new RuntimeException("服务版本已存在,无法新增");
@@ -234,7 +238,7 @@ public class ServiceServiceImpl implements ServiceService {
}

@Override
public String runServiceVersion(Long id) {
public String runServiceVersion(Long id) throws Exception {
ServiceVersion serviceVersion = serviceDao.getServiceVersionById(id);
com.ruoyi.platform.domain.Service service = serviceDao.getServiceById(serviceVersion.getServiceId());
HashMap<String, Object> paramMap = new HashMap<>();
@@ -249,23 +253,29 @@ public class ServiceServiceImpl implements ServiceService {
paramMap.put("model", JSONObject.parseObject(serviceVersion.getModel()));
paramMap.put("service_type", service.getServiceType());
paramMap.put("deploy_type", serviceVersion.getDeployType());
String req = HttpUtils.sendPost(argoUrl + modelService + "/create", JSON.toJSONString(paramMap));
if (StringUtils.isNotEmpty(req)) {
Map<String, Object> reqMap = JacksonUtil.parseJSONStr2Map(req);
if ((Integer) reqMap.get("code") == 200) {
Map<String, String> data = (Map<String, String>) reqMap.get("data");
serviceVersion.setUrl(data.get("url"));
serviceVersion.setDeploymentName(data.get("deployment_name"));
serviceVersion.setSvcName(data.get("svc_name"));
serviceVersion.setRunState(Constant.Pending);
serviceDao.updateServiceVersion(serviceVersion);
return "启动成功";

// 记录开始扣积分
if (resourceOccupyService.haveResource(serviceVersion.getComputingResourceId())) {
String req = HttpUtils.sendPost(argoUrl + modelService + "/create", JSON.toJSONString(paramMap));
if (StringUtils.isNotEmpty(req)) {
Map<String, Object> reqMap = JacksonUtil.parseJSONStr2Map(req);
if ((Integer) reqMap.get("code") == 200) {
resourceOccupyService.startDeduce(serviceVersion.getComputingResourceId(), Constant.TaskType_Service, serviceVersion.getId());
Map<String, String> data = (Map<String, String>) reqMap.get("data");
serviceVersion.setUrl(data.get("url"));
serviceVersion.setDeploymentName(data.get("deployment_name"));
serviceVersion.setSvcName(data.get("svc_name"));
serviceVersion.setRunState(Constant.Pending);
serviceDao.updateServiceVersion(serviceVersion);
return "启动成功";
} else {
throw new RuntimeException("启动失败");
}
} else {
throw new RuntimeException("启动失败");
}
} else {
throw new RuntimeException("启动失败");
}
return "启动失败";
}

@Override
@@ -277,6 +287,8 @@ public class ServiceServiceImpl implements ServiceService {
if (StringUtils.isNotEmpty(req)) {
serviceVersion.setRunState(Constant.Stopped);
serviceDao.updateServiceVersion(serviceVersion);
// 结束扣积分
resourceOccupyService.endDeduce(Constant.TaskType_Service, id);
return "停止成功";
} else {
throw new RuntimeException("停止失败");


+ 8
- 8
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/K8sClientUtil.java View File

@@ -3,7 +3,7 @@ package com.ruoyi.platform.utils;
import com.alibaba.fastjson2.JSON;
import com.ruoyi.platform.constant.Constant;
import com.ruoyi.platform.domain.DevEnvironment;
import com.ruoyi.platform.mapper.ComputingResourceDao;
import com.ruoyi.platform.service.ResourceOccupyService;
import io.kubernetes.client.Exec;
import io.kubernetes.client.custom.IntOrString;
import io.kubernetes.client.custom.Quantity;
@@ -12,9 +12,7 @@ import io.kubernetes.client.openapi.ApiException;
import io.kubernetes.client.openapi.apis.AppsV1Api;
import io.kubernetes.client.openapi.apis.CoreV1Api;
import io.kubernetes.client.openapi.models.*;
import io.kubernetes.client.util.ClientBuilder;
import io.kubernetes.client.util.Config;
import io.kubernetes.client.util.credentials.AccessTokenAuthentication;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang.StringUtils;
import org.json.JSONObject;
@@ -50,7 +48,7 @@ public class K8sClientUtil {
private static ApiClient apiClient;

@Resource
private ComputingResourceDao computingResourceDao;
private ResourceOccupyService resourceOccupyService;

/**
* 构建集群POD内通过SA访问的客户端
@@ -509,10 +507,12 @@ public class K8sClientUtil {
.build();

try {
pod = api.createNamespacedPod(namespace, pod, null, null, null);
String nodeName = getNodeName(podName, namespace);
if (Constant.Computing_Resource_GPU.equals(devEnvironment.getComputingResource())) {
computingResourceDao.updateUsedStateByNode(nodeName, Constant.Used_State_used);
// 记录开始扣积分
if (resourceOccupyService.haveResource(devEnvironment.getComputingResourceId())) {
pod = api.createNamespacedPod(namespace, pod, null, null, null);
String nodeName = getNodeName(podName, namespace);

resourceOccupyService.startDeduce(devEnvironment.getComputingResourceId(), Constant.TaskType_Dev, Long.valueOf(devEnvironment.getId()));
}
} catch (ApiException e) {
throw new RuntimeException("创建pod异常:" + e.getResponseBody());


+ 3
- 0
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/DevEnvironmentVo.java View File

@@ -19,6 +19,9 @@ public class DevEnvironmentVo implements Serializable {
* 计算资源
*/
private String computingResource;

private Integer computingResourceId;

/**
* 资源规格
*/


+ 2
- 0
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/serviceVos/ServiceVersionVo.java View File

@@ -28,6 +28,8 @@ public class ServiceVersionVo {

private String resource;

private Integer computingResourceId;

private Integer replicas;

private String mountPath;


+ 13
- 43
ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/DevEnvironmentDaoMapper.xml View File

@@ -1,38 +1,18 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.ruoyi.platform.mapper.DevEnvironmentDao">

<resultMap type="com.ruoyi.platform.domain.DevEnvironment" id="DevEnvironmentMap">
<result property="id" column="id" jdbcType="INTEGER"/>
<result property="name" column="name" jdbcType="VARCHAR"/>
<result property="status" column="status" jdbcType="VARCHAR"/>
<result property="computingResource" column="computing_resource" jdbcType="VARCHAR"/>
<result property="standard" column="standard" jdbcType="VARCHAR"/>
<result property="envVariable" column="env_variable" jdbcType="VARCHAR"/>
<result property="image" column="image" jdbcType="VARCHAR"/>
<result property="dataset" column="dataset" jdbcType="VARCHAR"/>
<result property="model" column="model" jdbcType="VARCHAR"/>
<result property="url" column="url" jdbcType="VARCHAR"/>
<result property="altField2" column="alt_field2" jdbcType="VARCHAR"/>
<result property="createBy" column="create_by" jdbcType="VARCHAR"/>
<result property="createTime" column="create_time" jdbcType="TIMESTAMP"/>
<result property="updateBy" column="update_by" jdbcType="VARCHAR"/>
<result property="updateTime" column="update_time" jdbcType="TIMESTAMP"/>
<result property="state" column="state" jdbcType="INTEGER"/>
</resultMap>

<!--查询单个-->
<select id="queryById" resultMap="DevEnvironmentMap">
<select id="queryById" resultType="com.ruoyi.platform.domain.DevEnvironment">
select
id,name,status,computing_resource,standard,env_variable,image,dataset,model,url,alt_field2,create_by,create_time,update_by,update_time,state
id,name,status,computing_resource,computing_resource_id, standard,env_variable,image,dataset,model,url,alt_field2,create_by,create_time,update_by,update_time,state
from dev_environment
where id = #{id} and state = 1
</select>

<!--查询指定行数据-->
<select id="queryAllByLimit" resultMap="DevEnvironmentMap">
<select id="queryAllByLimit" resultType="com.ruoyi.platform.domain.DevEnvironment">
select
id,name,status,computing_resource,standard,env_variable,image,dataset,model,url,alt_field2,create_by,create_time,update_by,update_time,state
id,name,status,computing_resource,computing_resource_id,standard,env_variable,image,dataset,model,url,alt_field2,create_by,create_time,update_by,update_time,state
from dev_environment
<where>
state = 1
@@ -146,12 +126,17 @@
</where>
</select>

<select id="getRunning" resultType="com.ruoyi.platform.domain.DevEnvironment">
select * from dev_environment where state = 1 and status = 'Running'
</select>

<!--新增所有列-->
<insert id="insert" keyProperty="id" useGeneratedKeys="true">
insert into dev_environment(name,status,computing_resource,standard,env_variable,image,dataset,model,url,alt_field2,create_by,create_time,update_by,update_time,state)
insert into dev_environment(name,status,computing_resource,computing_resource_id,standard,env_variable,image,dataset,model,url,alt_field2,create_by,create_time,update_by,update_time,state)
values (#{devEnvironment.name},
#{devEnvironment.status},
#{devEnvironment.computingResource},
#{devEnvironment.computingResourceId},
#{devEnvironment.standard},
#{devEnvironment.envVariable},
#{devEnvironment.image},
@@ -167,24 +152,6 @@
)
</insert>

<insert id="insertBatch" keyProperty="id" useGeneratedKeys="true">
insert into dev_environment(name,status,computing_resource,standard,env_variable,image,dataset,model,url,alt_field2,create_by,create_time,update_by,update_time,state )
values
<foreach collection="entities" item="entity" separator=",">
(#{entity.name},#{entity.status},#{entity.computingResource},#{entity.standard},#{entity.envVariable},#{entity.image},#{entity.dataset},#{entity.model},#{entity.url},#{entity.altField2},#{entity.createBy},#{entity.createTime},#{entity.updateBy},#{entity.updateTime},#{entity.state})
</foreach>
</insert>

<insert id="insertOrUpdateBatch" keyProperty="id" useGeneratedKeys="true">
insert into dev_environment(name,status,computing_resource,standard,env_variable,image,dataset,model,url,alt_field2,create_by,create_time,update_by,update_time,state)
values
<foreach collection="entities" item="entity" separator=",">
(#{entity.name}#{entity.status}#{entity.computingResource}#{entity.standard}#{entity.envVariable}#{entity.image}#{entity.dataset}#{entity.model}#{entity.url}#{entity.altField2}#{entity.createBy}#{entity.createTime}#{entity.updateBy}#{entity.updateTime}#{entity.state})
</foreach>
on duplicate key update
name = values(name)status = values(status)computing_resource = values(computing_resource)standard = values(standard)env_variable = values(env_variable)image = values(image)dataset = values(dataset)model = values(model)url = values(url)alt_field2 = values(alt_field2)create_by = values(create_by)create_time = values(create_time)update_by = values(update_by)update_time = values(update_time)state = values(state)
</insert>

<!--通过主键修改数据-->
<update id="update">
update dev_environment
@@ -198,6 +165,9 @@ name = values(name)status = values(status)computing_resource = values(computing_
<if test="devEnvironment.computingResource != null and devEnvironment.computingResource != ''">
computing_resource = #{devEnvironment.computingResource},
</if>
<if test="devEnvironment.computingResourceId != null and devEnvironment.computingResourceId != ''">
computing_resource_id = #{devEnvironment.computingResourceId},
</if>
<if test="devEnvironment.standard != null and devEnvironment.standard != ''">
standard = #{devEnvironment.standard},
</if>


+ 9
- 4
ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/RayDaoMapper.xml View File

@@ -2,13 +2,15 @@
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.ruoyi.platform.mapper.RayDao">
<insert id="save">
insert into ray(name, description, dataset, model, code_config, main_py, num_samples, parameters, points_to_evaluate, storage_path,
insert into ray(name, description, dataset, model, code_config, main_py, num_samples, parameters,
points_to_evaluate, storage_path,
search_alg, scheduler, metric, mode, max_t,
min_samples_required, resource, image, create_by, update_by)
values (#{ray.name}, #{ray.description}, #{ray.dataset}, #{ray.model}, #{ray.codeConfig}, #{ray.mainPy}, #{ray.numSamples}, #{ray.parameters},
min_samples_required, resource, computing_resource_id, image, create_by, update_by)
values (#{ray.name}, #{ray.description}, #{ray.dataset}, #{ray.model}, #{ray.codeConfig}, #{ray.mainPy},
#{ray.numSamples}, #{ray.parameters},
#{ray.pointsToEvaluate}, #{ray.storagePath},
#{ray.searchAlg}, #{ray.scheduler}, #{ray.metric}, #{ray.mode}, #{ray.maxT}, #{ray.minSamplesRequired},
#{ray.resource}, #{ray.image}, #{ray.createBy}, #{ray.updateBy})
#{ray.resource}, #{ray.computingResourceId}, #{ray.image}, #{ray.createBy}, #{ray.updateBy})
</insert>

<update id="edit">
@@ -68,6 +70,9 @@
<if test="ray.resource != null">
resource = #{ray.resource},
</if>
<if test="ray.computingResourceId != null">
computing_resource_id = #{ray.computingResourceId},
</if>
<if test="ray.updateBy != null and ray.updateBy !=''">
update_by = #{ray.updateBy},
</if>


+ 1
- 1
ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/RayInsDaoMapper.xml View File

@@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.ruoyi.platform.mapper.RayInsDao">
<insert id="insert">
<insert id="insert" keyProperty="id" useGeneratedKeys="true">
insert into ray_ins(ray_id, result_path, argo_ins_name, argo_ins_ns, node_status, node_result, param, source,
status)
values (#{rayIns.rayId}, #{rayIns.resultPath}, #{rayIns.argoInsName}, #{rayIns.argoInsNs},


+ 11
- 2
ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ResourceOccupy.xml View File

@@ -19,13 +19,22 @@
where id = #{resourceOccupy.id}
</update>

<update id="deduceCredit">
update sys_user
set credit = credit - #{credit}
where user_id = #{userId}
</update>

<select id="haveResource" resultType="java.lang.Boolean">
select case when used + #{need} &lt;= total then TRUE else FALSE end
from resource
where id = #{id}
</select>

<select id="getResourceOccupyById" resultType="com.ruoyi.platform.domain.ResourceOccupy">
select * from resource_occupy where id = #{id}
<select id="getResourceOccupyByTask" resultType="com.ruoyi.platform.domain.ResourceOccupy">
select *
from resource_occupy
where task_type = #{task_type}
and task_id = #{task_id}
</select>
</mapper>

+ 5
- 0
ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ServiceDaoMapper.xml View File

@@ -84,6 +84,11 @@
where service_id = #{serviceId} and version = #{version} and state = 1
</select>

<select id="getRunning" resultType="com.ruoyi.platform.domain.ServiceVersion">
select *
from service_version where state = 1 and status = 'Running'
</select>

<insert id="insertService" keyProperty="id" useGeneratedKeys="true">
insert into service(service_name, service_type, description, create_by, update_by)
values (#{service.serviceName}, #{service.serviceType}, #{service.description}, #{service.createBy}, #{service.updateBy})


Loading…
Cancel
Save