Browse Source

多形态资源库删除时,判断是否有流水线使用

dev-complex-computation
chenzhihang 1 year ago
parent
commit
5fa86c7ee5
26 changed files with 351 additions and 64 deletions
  1. +7
    -1
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/constant/Constant.java
  2. +1
    -1
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/codeConfig/CodeConfigController.java
  3. +2
    -2
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/dataset/NewDatasetFromGitController.java
  4. +1
    -1
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/image/ImageVersionController.java
  5. +1
    -1
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/service/ServiceController.java
  6. +26
    -0
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/AssetWorkflow.java
  7. +15
    -0
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/AssetWorkflowDao.java
  8. +1
    -0
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/ImageVersionDao.java
  9. +0
    -8
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/WorkflowDao.java
  10. +1
    -1
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/CodeConfigService.java
  11. +0
    -4
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ImageService.java
  12. +1
    -1
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ImageVersionService.java
  13. +1
    -1
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/NewDatasetService.java
  14. +1
    -1
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ServiceService.java
  15. +0
    -1
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/WorkflowService.java
  16. +13
    -2
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/CodeConfigServiceImpl.java
  17. +1
    -1
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentInsServiceImpl.java
  18. +13
    -4
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ImageServiceImpl.java
  19. +12
    -2
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ImageVersionServiceImpl.java
  20. +17
    -4
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java
  21. +19
    -3
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java
  22. +11
    -2
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ServiceServiceImpl.java
  23. +176
    -18
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/WorkflowServiceImpl.java
  24. +27
    -0
      ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AssetWorkflowDaoMapper.xml
  25. +4
    -0
      ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ImageVersionDaoMapper.xml
  26. +0
    -5
      ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/WorkflowDaoMapper.xml

+ 7
- 1
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/constant/Constant.java View File

@@ -34,7 +34,7 @@ public class Constant {
public final static String Running = "Running";
public final static String Failed = "Failed";
public final static String Pending = "Pending";
public final static String Terminated = "Terminated";
public final static String Terminated = "Terminated";
public final static String Init = "Init";
public final static String Stopped = "Stopped";
public final static String Succeeded = "Succeeded";
@@ -43,4 +43,10 @@ public class Constant {
public final static String Type_Evaluate = "evaluate";

public final static String AutoMl_Classification = "classification";

public final static String Asset_Type_Dataset = "dataset";
public final static String Asset_Type_Model = "model";
public final static String Asset_Type_Image = "image";
public final static String Asset_Type_Code = "code";
public final static String Asset_Type_Service = "service";
}

+ 1
- 1
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/codeConfig/CodeConfigController.java View File

@@ -62,7 +62,7 @@ public class CodeConfigController extends BaseController {
}

@DeleteMapping("{id}")
public GenericsAjaxResult<String> delete(@PathVariable("id") Long id) {
public GenericsAjaxResult<String> delete(@PathVariable("id") Long id) throws Exception {
return genericsSuccess(this.codeConfigService.removeById(id));
}
}

+ 2
- 2
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/dataset/NewDatasetFromGitController.java View File

@@ -99,14 +99,14 @@ public class NewDatasetFromGitController {

@DeleteMapping("/deleteDatasetVersion")
@ApiOperation(value = "删除数据集版本")
public AjaxResult deleteDatasetVersion(@RequestParam("identifier") String repo, @RequestParam("owner") String owner, @RequestParam("version") String version,
public AjaxResult deleteDatasetVersion(@RequestParam("id") Integer id, @RequestParam("identifier") String repo, @RequestParam("owner") String owner, @RequestParam("version") String version,
@RequestParam("relative_paths") String relativePaths) throws Exception {
// 查询版本,如果是最后一个版本,则不能删除
List<Map<String, Object>> versionList = this.newDatasetService.getVersionList(repo, owner);
if (versionList.size() == 1) {
return AjaxResult.error("当前数据集只有一个版本,不能删除该版本");
}
this.newDatasetService.deleteDatasetVersionNew(repo, owner, version, relativePaths);
this.newDatasetService.deleteDatasetVersionNew(id, repo, owner, version, relativePaths);
return AjaxResult.success();
}



+ 1
- 1
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/image/ImageVersionController.java View File

@@ -89,7 +89,7 @@ public class ImageVersionController extends BaseController {
* @return 删除是否成功
*/
@DeleteMapping("{id}")
public GenericsAjaxResult<String> deleteById(@PathVariable("id") Integer id) {
public GenericsAjaxResult<String> deleteById(@PathVariable("id") Integer id) throws Exception {
return genericsSuccess(this.imageVersionService.removeById(id));
}



+ 1
- 1
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/service/ServiceController.java View File

@@ -103,7 +103,7 @@ public class ServiceController extends BaseController {

@DeleteMapping("{id}")
@ApiOperation("删除服务")
public GenericsAjaxResult<String> deleteService(@PathVariable("id") Long id) {
public GenericsAjaxResult<String> deleteService(@PathVariable("id") Long id) throws Exception {
return genericsSuccess(serviceService.deleteService(id));
}



+ 26
- 0
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/AssetWorkflow.java View File

@@ -0,0 +1,26 @@
package com.ruoyi.platform.domain;

import com.fasterxml.jackson.databind.PropertyNamingStrategy;
import com.fasterxml.jackson.databind.annotation.JsonNaming;
import io.swagger.annotations.ApiModel;
import lombok.Data;

@Data
@JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy.class)
@ApiModel(description = "AI资产-流水线引用关系")
public class AssetWorkflow {
private Long id;

private Long workflowId;

private String workflowName;

private Long assetId;

private String assetVersion;

private String type;

private Integer state;
}


+ 15
- 0
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/AssetWorkflowDao.java View File

@@ -0,0 +1,15 @@
package com.ruoyi.platform.mapper;

import com.ruoyi.platform.domain.AssetWorkflow;
import org.apache.ibatis.annotations.Param;

import java.util.List;

public interface AssetWorkflowDao {

List<AssetWorkflow> getAssetWorkflow(@Param("assetId") Long assetId, @Param("type") String type, @Param("assetVersion") String assetVersion);

int insert(@Param("assetWorkflow") AssetWorkflow assetWorkflow);

int deleteByWorkFlowId(@Param("workFlowId") Long workFlowId);
}

+ 1
- 0
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/ImageVersionDao.java View File

@@ -81,5 +81,6 @@ public interface ImageVersionDao {

List<ImageVersion> queryByImageId(Integer imageId);

ImageVersion queryByUrl(String url);
}


+ 0
- 8
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/WorkflowDao.java View File

@@ -85,14 +85,6 @@ public interface WorkflowDao {
*/
int update(@Param("workflow") Workflow workflow);

/**
* 通过主键删除数据
*
* @param id 主键
* @return 影响行数
*/
int deleteById(Long id);

/**
* 通过名字模糊查询
*


+ 1
- 1
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/CodeConfigService.java View File

@@ -14,6 +14,6 @@ public interface CodeConfigService {

CodeConfig update(CodeConfig codeConfig);

String removeById(Long id);
String removeById(Long id) throws Exception;

}

+ 0
- 4
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ImageService.java View File

@@ -75,10 +75,6 @@ public interface ImageService {

String removeById(Integer id) throws Exception;





String insertImageAndVersion(ImageVo imageVo) throws Exception;

/**


+ 1
- 1
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ImageVersionService.java View File

@@ -62,5 +62,5 @@ public interface ImageVersionService {
List<ImageVersion> queryByImageId(Integer id);


String removeById(Integer id);
String removeById(Integer id) throws Exception;
}

+ 1
- 1
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/NewDatasetService.java View File

@@ -35,5 +35,5 @@ public interface NewDatasetService {

void deleteDatasetNew(Integer repoId, String repo, String owner, Boolean isPublic) throws Exception;

void deleteDatasetVersionNew(String repo, String owner, String version, String relativePath) throws Exception;
void deleteDatasetVersionNew(Integer repoId, String repo, String owner, String version, String relativePath) throws Exception;
}

+ 1
- 1
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ServiceService.java View File

@@ -30,7 +30,7 @@ public interface ServiceService {

Map<String, Object> serviceVersionCompare(Long id1, Long id2) throws IllegalAccessException;

String deleteService(Long id);
String deleteService(Long id) throws Exception;

String deleteServiceVersion(Long id);



+ 0
- 1
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/WorkflowService.java View File

@@ -53,7 +53,6 @@ public interface WorkflowService {
* @param id 主键
* @return 是否成功
*/
boolean deleteById(Long id);
String removeById(Long id) throws Exception;
/**
* 按流水线名字模糊分页查询


+ 13
- 2
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/CodeConfigServiceImpl.java View File

@@ -2,7 +2,9 @@ package com.ruoyi.platform.service.impl;

import com.ruoyi.common.security.utils.SecurityUtils;
import com.ruoyi.platform.constant.Constant;
import com.ruoyi.platform.domain.AssetWorkflow;
import com.ruoyi.platform.domain.CodeConfig;
import com.ruoyi.platform.mapper.AssetWorkflowDao;
import com.ruoyi.platform.mapper.CodeConfigDao;
import com.ruoyi.platform.service.CodeConfigService;
import com.ruoyi.system.api.model.LoginUser;
@@ -15,13 +17,15 @@ import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.util.Date;
import java.util.List;
import java.util.stream.Collectors;

@Service("codeConfigService")
public class CodeConfigServiceImpl implements CodeConfigService {

@Resource
private CodeConfigDao codeConfigDao;

@Resource
private AssetWorkflowDao assetWorkflowDao;

@Override
public Page<CodeConfig> queryByPage(CodeConfig codeConfig, PageRequest pageRequest) {
@@ -72,11 +76,18 @@ public class CodeConfigServiceImpl implements CodeConfigService {
}

@Override
public String removeById(Long id) {
public String removeById(Long id) throws Exception {
CodeConfig codeConfig = this.codeConfigDao.queryById(id);
if (codeConfig == null) {
return "代码配置不存在";
}

List<AssetWorkflow> assetWorkflow = assetWorkflowDao.getAssetWorkflow(id, Constant.Asset_Type_Code, null);
if (assetWorkflow != null && !assetWorkflow.isEmpty()) {
String workflows = String.join(",", assetWorkflow.stream().map(AssetWorkflow::getWorkflowName).collect(Collectors.toSet()));
throw new Exception("该代码配置被流水线:" + workflows + "使用,不能删除,请先删除流水线。");
}

LoginUser loginUser = SecurityUtils.getLoginUser();
String username = loginUser.getUsername();
String createBy = codeConfig.getCreateBy();


+ 1
- 1
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentInsServiceImpl.java View File

@@ -447,7 +447,7 @@ public class ExperimentInsServiceImpl implements ExperimentInsService {
DatasetTempStorage datasetTempStorage = datasetTempStorageDao.queryByInsId(JSON.toJSONString(queryMap));
Map<String, Object> source = JsonUtils.jsonToMap(datasetTempStorage.getSource());
String relativePath = ci4sUsername + "/datasets/" + source.get("repo_id") + "/" + source.get("identifier") + "/" + source.get("version") + "/dataset";
newDatasetService.deleteDatasetVersionNew((String) source.get("identifier"), (String) source.get("owner"), (String) source.get("version"), relativePath);
newDatasetService.deleteDatasetVersionNew((Integer) source.get("repo_id"), (String) source.get("identifier"), (String) source.get("owner"), (String) source.get("version"), relativePath);
break;
}
}


+ 13
- 4
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ImageServiceImpl.java View File

@@ -3,9 +3,11 @@ package com.ruoyi.platform.service.impl;
import com.alibaba.fastjson2.util.DateUtils;
import com.ruoyi.common.security.utils.SecurityUtils;
import com.ruoyi.platform.constant.Constant;
import com.ruoyi.platform.domain.AssetWorkflow;
import com.ruoyi.platform.domain.DevEnvironment;
import com.ruoyi.platform.domain.Image;
import com.ruoyi.platform.domain.ImageVersion;
import com.ruoyi.platform.mapper.AssetWorkflowDao;
import com.ruoyi.platform.mapper.DevEnvironmentDao;
import com.ruoyi.platform.mapper.ImageDao;
import com.ruoyi.platform.mapper.ImageVersionDao;
@@ -35,6 +37,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;

/**
* (Image)表服务实现类
@@ -51,6 +54,9 @@ public class ImageServiceImpl implements ImageService {
@Resource
private DevEnvironmentDao devEnvironmentDao;

@Resource
private AssetWorkflowDao assetWorkflowDao;

@Resource
private ImageVersionService imageVersionService;

@@ -170,18 +176,21 @@ public class ImageServiceImpl implements ImageService {
throw new Exception("镜像不存在");
}

List<AssetWorkflow> assetWorkflow = assetWorkflowDao.getAssetWorkflow(Long.valueOf(id), Constant.Asset_Type_Image, null);
if (assetWorkflow != null && !assetWorkflow.isEmpty()) {
String workflows = String.join(",", assetWorkflow.stream().map(AssetWorkflow::getWorkflowName).collect(Collectors.toSet()));
throw new Exception("该镜像被流水线:" + workflows + "使用,不能删除,请先删除流水线。");
}

//判断权限,只有admin和创建者本身可以删除该数据集
LoginUser loginUser = SecurityUtils.getLoginUser();
String username = loginUser.getUsername();


String createdBy = image.getCreateBy();
if (!(StringUtils.equals(username, "admin") || !StringUtils.equals(username, createdBy))) {
throw new Exception("无权限删除该镜像");
}
// if (!imageVersionService.queryByImageId(id).isEmpty()){
// throw new Exception("请先删除该镜像下的版本文件");
// }

List<ImageVersion> imageVersions = imageVersionService.queryByImageId(id);

for (ImageVersion imageVersion : imageVersions) {


+ 12
- 2
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ImageVersionServiceImpl.java View File

@@ -1,7 +1,10 @@
package com.ruoyi.platform.service.impl;

import com.ruoyi.common.security.utils.SecurityUtils;
import com.ruoyi.platform.constant.Constant;
import com.ruoyi.platform.domain.AssetWorkflow;
import com.ruoyi.platform.domain.ImageVersion;
import com.ruoyi.platform.mapper.AssetWorkflowDao;
import com.ruoyi.platform.mapper.ImageDao;
import com.ruoyi.platform.mapper.ImageVersionDao;
import com.ruoyi.platform.domain.ModelsVersion;
@@ -16,6 +19,7 @@ import org.springframework.data.domain.PageRequest;
import javax.annotation.Resource;
import java.util.Date;
import java.util.List;
import java.util.stream.Collectors;

/**
* (ImageVersion)表服务实现类
@@ -29,7 +33,7 @@ public class ImageVersionServiceImpl implements ImageVersionService {
private ImageVersionDao imageVersionDao;

@Resource
private ImageDao imageDao;
private AssetWorkflowDao assetWorkflowDao;

/**
* 通过ID查询单条数据
@@ -61,12 +65,18 @@ public class ImageVersionServiceImpl implements ImageVersionService {
}

@Override
public String removeById(Integer id) {
public String removeById(Integer id) throws Exception {
ImageVersion imageVersion = this.imageVersionDao.queryById(id);
if (imageVersion == null){
return "该版本下模型文件信息不存在";
}

List<AssetWorkflow> assetWorkflow = assetWorkflowDao.getAssetWorkflow(Long.valueOf(imageVersion.getImageId()), Constant.Asset_Type_Image, imageVersion.getTagName());
if (assetWorkflow != null && !assetWorkflow.isEmpty()) {
String workflows = String.join(",", assetWorkflow.stream().map(AssetWorkflow::getWorkflowName).collect(Collectors.toSet()));
throw new Exception("该镜像版本被流水线:" + workflows + "使用,不能删除,请先删除流水线。");
}

//判断权限,只有admin和创建者本身可以删除该数据集
LoginUser loginUser = SecurityUtils.getLoginUser();
String username = loginUser.getUsername();


+ 17
- 4
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java View File

@@ -7,11 +7,9 @@ import com.ruoyi.common.core.utils.DateUtils;
import com.ruoyi.common.security.utils.SecurityUtils;
import com.ruoyi.platform.annotations.CheckDuplicate;
import com.ruoyi.platform.constant.Constant;
import com.ruoyi.platform.domain.AssetIcon;
import com.ruoyi.platform.domain.ModelDependency1;
import com.ruoyi.platform.domain.Models;
import com.ruoyi.platform.domain.ModelsVersion;
import com.ruoyi.platform.domain.*;
import com.ruoyi.platform.domain.dependencydomain.TrainTaskDepency;
import com.ruoyi.platform.mapper.AssetWorkflowDao;
import com.ruoyi.platform.mapper.ModelDependency1Dao;
import com.ruoyi.platform.mapper.ModelsDao;
import com.ruoyi.platform.mapper.ModelsVersionDao;
@@ -92,6 +90,9 @@ public class ModelsServiceImpl implements ModelsService {
@Resource
private DVCUtils dvcUtils;

@Resource
private AssetWorkflowDao assetWorkflowDao;

// 固定存储桶名
@Value("${minio.dataReleaseBucketName}")
private String bucketName;
@@ -1145,6 +1146,12 @@ public class ModelsServiceImpl implements ModelsService {

@Override
public void deleteModel(Integer repoId, String identifier, String owner, Boolean isPublic) throws Exception {
List<AssetWorkflow> assetWorkflow = assetWorkflowDao.getAssetWorkflow(Long.valueOf(repoId), Constant.Asset_Type_Model, null);
if (assetWorkflow != null && !assetWorkflow.isEmpty()) {
String workflows = String.join(",", assetWorkflow.stream().map(AssetWorkflow::getWorkflowName).collect(Collectors.toSet()));
throw new Exception("该模型被流水线:" + workflows + "使用,不能删除,请先删除流水线。");
}

String token = gitService.checkoutToken();
gitService.deleteProject(token, owner, identifier);
//删除模型依赖
@@ -1162,6 +1169,12 @@ public class ModelsServiceImpl implements ModelsService {

@Override
public void deleteVersion(Integer repoId, String identifier, String owner, String version, String relativePath) throws Exception {
List<AssetWorkflow> assetWorkflow = assetWorkflowDao.getAssetWorkflow(Long.valueOf(repoId), Constant.Asset_Type_Model, version);
if (assetWorkflow != null && !assetWorkflow.isEmpty()) {
String workflows = String.join(",", assetWorkflow.stream().map(AssetWorkflow::getWorkflowName).collect(Collectors.toSet()));
throw new Exception("该模型版本被流水线:" + workflows + "使用,不能删除,请先删除流水线。");
}

String token = gitService.checkoutToken();
String rootPath = Paths.get(localPath + "/" + relativePath).getParent().toString();
gitService.deleteBranch(token, owner, identifier, version, rootPath);


+ 19
- 3
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java View File

@@ -3,8 +3,10 @@ package com.ruoyi.platform.service.impl;
import com.ruoyi.common.core.utils.DateUtils;
import com.ruoyi.common.security.utils.SecurityUtils;
import com.ruoyi.platform.constant.Constant;
import com.ruoyi.platform.domain.AssetWorkflow;
import com.ruoyi.platform.domain.Dataset;
import com.ruoyi.platform.domain.DatasetTempStorage;
import com.ruoyi.platform.mapper.AssetWorkflowDao;
import com.ruoyi.platform.service.DatasetTempStorageService;
import com.ruoyi.platform.service.GitService;
import com.ruoyi.platform.service.NewDatasetService;
@@ -50,6 +52,8 @@ public class NewDatasetServiceImpl implements NewDatasetService {

@Resource
private GitService gitService;
@Resource
private AssetWorkflowDao assetWorkflowDao;

@Value("${spring.redis.host}")
private String redisHost;
@@ -398,6 +402,12 @@ public class NewDatasetServiceImpl implements NewDatasetService {

@Override
public void deleteDatasetNew(Integer repoId, String repo, String owner, Boolean isPublic) throws Exception {
List<AssetWorkflow> assetWorkflow = assetWorkflowDao.getAssetWorkflow(Long.valueOf(repoId), Constant.Asset_Type_Dataset, null);
if (assetWorkflow != null && !assetWorkflow.isEmpty()) {
String workflows = String.join(",", assetWorkflow.stream().map(AssetWorkflow::getWorkflowName).collect(Collectors.toSet()));
throw new Exception("该数据集被流水线:" + workflows + "使用,不能删除,请先删除流水线。");
}

String token = gitService.checkoutToken();
gitService.deleteProject(token, owner, repo);

@@ -407,7 +417,13 @@ public class NewDatasetServiceImpl implements NewDatasetService {
}

@Override
public void deleteDatasetVersionNew(String repo, String owner, String version, String relativePath) throws Exception {
public void deleteDatasetVersionNew(Integer repoId, String repo, String owner, String version, String relativePath) throws Exception {
List<AssetWorkflow> assetWorkflow = assetWorkflowDao.getAssetWorkflow(Long.valueOf(repoId), Constant.Asset_Type_Dataset, version);
if (assetWorkflow != null && !assetWorkflow.isEmpty()) {
String workflows = String.join(",", assetWorkflow.stream().map(AssetWorkflow::getWorkflowName).collect(Collectors.toSet()));
throw new Exception("该数据集版本被流水线:" + workflows + "使用,不能删除,请先删除流水线。");
}

String token = gitService.checkoutToken();
String rootPath = Paths.get(localPathlocal + "/" + relativePath).getParent().toString();
gitService.deleteBranch(token, owner, repo, version, rootPath);
@@ -545,8 +561,8 @@ public class NewDatasetServiceImpl implements NewDatasetService {
Map<String, Object> author = (Map<String, Object>) map.get("author");
newDatasetVo.setCreateBy((String) author.get("name"));
newDatasetVo.setOwner((String) author.get("login"));
String name = (String)map.get("name");
newDatasetVo.setName(name.substring((datasetTopic +"-").length()));
String name = (String) map.get("name");
newDatasetVo.setName(name.substring((datasetTopic + "-").length()));
result.add(newDatasetVo);
}
}


+ 11
- 2
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ServiceServiceImpl.java View File

@@ -4,7 +4,9 @@ import com.alibaba.fastjson2.JSON;
import com.alibaba.fastjson2.JSONObject;
import com.ruoyi.common.security.utils.SecurityUtils;
import com.ruoyi.platform.constant.Constant;
import com.ruoyi.platform.domain.AssetWorkflow;
import com.ruoyi.platform.domain.ServiceVersion;
import com.ruoyi.platform.mapper.AssetWorkflowDao;
import com.ruoyi.platform.mapper.ServiceDao;
import com.ruoyi.platform.service.ServiceService;
import com.ruoyi.platform.utils.ConvertUtil;
@@ -40,7 +42,8 @@ public class ServiceServiceImpl implements ServiceService {

@Resource
private ServiceDao serviceDao;

@Resource
private AssetWorkflowDao assetWorkflowDao;
@Override
public Page<com.ruoyi.platform.domain.Service> queryByPageService(com.ruoyi.platform.domain.Service service, PageRequest pageRequest) {
long total = serviceDao.countService(service);
@@ -182,12 +185,18 @@ public class ServiceServiceImpl implements ServiceService {
}

@Override
public String deleteService(Long id) {
public String deleteService(Long id) throws Exception {
com.ruoyi.platform.domain.Service service = serviceDao.getServiceById(id);
if (service == null) {
throw new RuntimeException("服务不存在");
}

List<AssetWorkflow> assetWorkflow = assetWorkflowDao.getAssetWorkflow(Long.valueOf(id), Constant.Asset_Type_Service, null);
if (assetWorkflow != null && !assetWorkflow.isEmpty()) {
String workflows = String.join(",", assetWorkflow.stream().map(AssetWorkflow::getWorkflowName).collect(Collectors.toSet()));
throw new Exception("该服务被流水线:" + workflows + "使用,不能删除,请先删除流水线。");
}

LoginUser loginUser = SecurityUtils.getLoginUser();
String username = loginUser.getUsername();
String createBy = service.getCreateBy();


+ 176
- 18
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/WorkflowServiceImpl.java View File

@@ -2,15 +2,19 @@ package com.ruoyi.platform.service.impl;

import com.ruoyi.common.security.utils.SecurityUtils;
import com.ruoyi.platform.annotations.CheckDuplicate;
import com.ruoyi.platform.constant.Constant;
import com.ruoyi.platform.domain.AssetWorkflow;
import com.ruoyi.platform.domain.Experiment;
import com.ruoyi.platform.domain.ImageVersion;
import com.ruoyi.platform.domain.Workflow;
import com.ruoyi.platform.mapper.AssetWorkflowDao;
import com.ruoyi.platform.mapper.ImageVersionDao;
import com.ruoyi.platform.mapper.WorkflowDao;
import com.ruoyi.platform.service.ExperimentService;
import com.ruoyi.platform.service.WorkflowService;
import com.ruoyi.platform.utils.HttpUtils;
import com.ruoyi.platform.utils.JsonUtils;
import com.ruoyi.platform.utils.MinioUtil;
import com.ruoyi.platform.utils.NewHttpUtils;
import com.ruoyi.system.api.model.LoginUser;
import org.apache.commons.lang.StringUtils;
import org.springframework.beans.factory.annotation.Value;
@@ -18,8 +22,10 @@ import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageImpl;
import org.springframework.data.domain.PageRequest;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;

import javax.annotation.Resource;
import java.io.IOException;
import java.lang.reflect.Field;
import java.util.Date;
import java.util.HashMap;
@@ -36,6 +42,10 @@ import java.util.Map;
public class WorkflowServiceImpl implements WorkflowService {
@Resource
private WorkflowDao workflowDao;
@Resource
private ImageVersionDao imageVersionDao;
@Resource
private AssetWorkflowDao assetWorkflowDao;

@Resource
private ExperimentService experimentService;
@@ -45,8 +55,7 @@ public class WorkflowServiceImpl implements WorkflowService {

@Value("${argo.workflowCopy}")
private String argoWorkflowCopy;
@Resource
private NewHttpUtils httpUtils;

private final MinioUtil minioUtil;

public WorkflowServiceImpl(MinioUtil minioUtil) {
@@ -101,6 +110,7 @@ public class WorkflowServiceImpl implements WorkflowService {
workflow.setCreateTime(new Date());
workflow.setState(1);
this.workflowDao.insert(workflow);
saveAssetWorkFlow(workflow);
return workflow;
}

@@ -117,6 +127,8 @@ public class WorkflowServiceImpl implements WorkflowService {
workflow.setUpdateBy(loginUser.getUsername());
workflow.setUpdateTime(new Date());
this.workflowDao.update(workflow);
assetWorkflowDao.deleteByWorkFlowId(workflow.getId());
saveAssetWorkFlow(workflow);
return this.queryById(workflow.getId());
}

@@ -127,53 +139,51 @@ public class WorkflowServiceImpl implements WorkflowService {
* @return 是否成功
*/
@Override
public boolean deleteById(Long id) {
return this.workflowDao.deleteById(id) > 0;
}
@Override
@Transactional
public String removeById(Long id) throws Exception {
//先根据id提取出对应的流水线
Workflow workflow = workflowDao.queryById(id);
if (workflow == null){
if (workflow == null) {
throw new Exception("流水线不存在");
}
//判断权限,只有admin和创建者本身可以删除该流水线
LoginUser loginUser = SecurityUtils.getLoginUser();
String username = loginUser.getUsername();
String createdBy = workflow.getCreateBy();
if (!(StringUtils.equals(username,"admin") || StringUtils.equals(username,createdBy))){
if (!(StringUtils.equals(username, "admin") || StringUtils.equals(username, createdBy))) {
throw new Exception("无权限删除该流水线");
}

//判断这个流水线是否有相关实验存在
List<Experiment> experimentList = experimentService.queryByWorkflowId(id);
if (experimentList!=null&&experimentList.size()>0){
if (experimentList != null && experimentList.size() > 0) {
throw new Exception("该流水线存在实验,无法删除");
}
workflow.setState(0);
return this.workflowDao.update(workflow)>0?"删除成功":"删除失败";
assetWorkflowDao.deleteByWorkFlowId(id);
return this.workflowDao.update(workflow) > 0 ? "删除成功" : "删除失败";
}

@Override
public Page<Workflow> queryByName(String name) {
return new PageImpl<>(this.workflowDao.queryByName(name));
}



@Override
public Workflow duplicateWorkflow(Long id) {
//先去查找数据库中存在的数据
Workflow workflow = this.queryById(id);
if (workflow!= null) {
if (workflow != null) {
try {
Workflow duplicateWorkflow = new Workflow();
duplicateWorkflow.setName(workflow.getName()+"-copy");
duplicateWorkflow.setName(workflow.getName() + "-copy");
String oldDag = workflow.getDag();
// 创建请求数据的Json(map)
Map<String,Object> requestData = new HashMap<>();
Map<String, Object> requestData = new HashMap<>();
requestData.put("data", oldDag);
// 发送POST请求到Argo工作流复制接口,并将请求数据转换为JSON
String req = HttpUtils.sendPost(argoUrl + argoWorkflowCopy ,null, JsonUtils.mapToJson(requestData));
String req = HttpUtils.sendPost(argoUrl + argoWorkflowCopy, null, JsonUtils.mapToJson(requestData));
// 检查响应是否为空或无内容
if (StringUtils.isEmpty(req)) {
throw new RuntimeException("工作流复制接口响应内容为空");
@@ -182,7 +192,7 @@ public class WorkflowServiceImpl implements WorkflowService {
Map<String, Object> runResMap = JsonUtils.jsonToMap(req);
// 从响应Map中获取"data"的值,也就是日志的值
String newDag = (String) runResMap.get("data");
if (newDag == null){
if (newDag == null) {
throw new RuntimeException("响应内容为空。");
}
duplicateWorkflow.setDag(newDag);
@@ -197,7 +207,6 @@ public class WorkflowServiceImpl implements WorkflowService {
return null;



}

public void checkDeclaredName(Workflow insert) throws Exception {
@@ -220,7 +229,156 @@ public class WorkflowServiceImpl implements WorkflowService {
}
}

public void saveAssetWorkFlow(Workflow workflow) throws IOException {
if (StringUtils.isNotEmpty(workflow.getDag())) {
Map<String, Object> dagMap = JsonUtils.jsonToMap(workflow.getDag());
List<Map<String, Object>> nodes = (List<Map<String, Object>>) dagMap.get("nodes");
for (Map<String, Object> node : nodes) {
String componentName = (String) node.get("component_name");
// 自定义执行
if ("self-command-execute".equals(componentName)) {
//保存镜像-流水线关系
saveImageWorkFlow((String) node.get("image"), workflow.getId(), workflow.getName());
}
// 代码拉取组件
else if ("git-clone".equals(componentName)) {
//保存镜像-流水线关系
saveImageWorkFlow((String) node.get("image"), workflow.getId(), workflow.getName());
//保存代码-流水线关系
saveCodeWorkFlow((String) node.get("in_parameters"), workflow.getId(), workflow.getName());
}
// 模型训练
else if ("model-train".equals(componentName)) {
//保存镜像-流水线关系
saveImageWorkFlow((String) node.get("image"), workflow.getId(), workflow.getName());
//保存数据集-流水线关系
saveDatasetWorkFlow((String) node.get("in_parameters"), workflow.getId(), workflow.getName());
//保存模型-流水线关系
saveModelWorkFlow((String) node.get("in_parameters"), workflow.getId(), workflow.getName());
}
// 分布式训练
else if ("distributed-model-train".equals(componentName)) {
//保存镜像-流水线关系
saveImageWorkFlow((String) node.get("image"), workflow.getId(), workflow.getName());
//保存数据集-流水线关系
saveDatasetWorkFlow((String) node.get("in_parameters"), workflow.getId(), workflow.getName());
//保存模型-流水线关系
saveModelWorkFlow((String) node.get("in_parameters"), workflow.getId(), workflow.getName());
}
// 模型测试
else if ("model-evaluate".equals(componentName)) {
//保存镜像-流水线关系
saveImageWorkFlow((String) node.get("image"), workflow.getId(), workflow.getName());
//保存数据集-流水线关系
saveDatasetWorkFlow((String) node.get("in_parameters"), workflow.getId(), workflow.getName());
//保存模型-流水线关系
saveModelWorkFlow((String) node.get("in_parameters"), workflow.getId(), workflow.getName());
}
// 数据预处理
else if ("general-data-process".equals(componentName)) {
//保存镜像-流水线关系
saveImageWorkFlow((String) node.get("image"), workflow.getId(), workflow.getName());
//保存数据集-流水线关系
saveDatasetWorkFlow((String) node.get("in_parameters"), workflow.getId(), workflow.getName());
}
// 模型导出
else if ("model-export".equals(componentName)) {
//保存镜像-流水线关系
saveImageWorkFlow((String) node.get("image"), workflow.getId(), workflow.getName());
//保存模型-流水线关系
saveModelWorkFlow1((String) node.get("in_parameters"), workflow.getId(), workflow.getName());
}
// 数据集导出
else if ("dataset-export".equals(componentName)) {
//保存镜像-流水线关系
saveImageWorkFlow((String) node.get("image"), workflow.getId(), workflow.getName());
//保存数据集-流水线关系
saveDatasetWorkFlow1((String) node.get("in_parameters"), workflow.getId(), workflow.getName());
}
// 模型部署
else if ("model-deploy".equals(componentName)) {
//保存镜像-流水线关系
saveImageWorkFlow((String) node.get("image"), workflow.getId(), workflow.getName());
//保存模型-流水线关系
saveModelWorkFlow((String) node.get("in_parameters"), workflow.getId(), workflow.getName());
//保存服务-流水线关系
saveServiceWorkFlow((String) node.get("in_parameters"), workflow.getId(), workflow.getName());
//保存镜像-流水线关系
saveImageWorkFlow1((String) node.get("in_parameters"), workflow.getId(), workflow.getName());
}
}
}
}

public void saveImageWorkFlow(String image, Long workflowId, String workflowName) {
String[] imageSplit = image.split(":");
String tagName = imageSplit[1];
ImageVersion imageVersion = imageVersionDao.queryByUrl(image);
insertAssetWorkFlow(workflowId, workflowName, Long.valueOf(imageVersion.getImageId()), tagName, Constant.Asset_Type_Image);
}

public void saveImageWorkFlow1(String inParameters, Long workflowId, String workflowName) throws IOException {
Map<String, Object> paramMap = JsonUtils.jsonToMap(inParameters);
Map<String, Object> image = (Map<String, Object>) paramMap.get("--image");
saveImageWorkFlow((String) image.get("value"), workflowId, workflowName);
}

public void saveCodeWorkFlow(String inParameters, Long workflowId, String workflowName) throws IOException {
Map<String, Object> paramMap = JsonUtils.jsonToMap(inParameters);
Map<String, Object> code_config = (Map<String, Object>) paramMap.get("--code_config");
String value = (String) code_config.get("value");
Map<String, Object> valueMap = JsonUtils.jsonToMap(value);
insertAssetWorkFlow(workflowId, workflowName, Long.valueOf((Integer) valueMap.get("id")), null, Constant.Asset_Type_Code);
}

public void saveDatasetWorkFlow(String inParameters, Long workflowId, String workflowName) throws IOException {
Map<String, Object> paramMap = JsonUtils.jsonToMap(inParameters);
Map<String, Object> dataset = (Map<String, Object>) paramMap.get("--dataset");
String value = (String) dataset.get("value");
Map<String, Object> valueMap = JsonUtils.jsonToMap(value);
insertAssetWorkFlow(workflowId, workflowName, Long.valueOf((String) valueMap.get("id")), (String) valueMap.get("version"), Constant.Asset_Type_Dataset);
}

public void saveDatasetWorkFlow1(String inParameters, Long workflowId, String workflowName) throws IOException {
Map<String, Object> paramMap = JsonUtils.jsonToMap(inParameters);
Map<String, Object> dataset = (Map<String, Object>) paramMap.get("--dataset_id");
String value = (String) dataset.get("value");
Map<String, Object> valueMap = JsonUtils.jsonToMap(value);
insertAssetWorkFlow(workflowId, workflowName, Long.valueOf((String) valueMap.get("id")), null, Constant.Asset_Type_Dataset);
}

public void saveModelWorkFlow(String inParameters, Long workflowId, String workflowName) throws IOException {
Map<String, Object> paramMap = JsonUtils.jsonToMap(inParameters);
Map<String, Object> model = (Map<String, Object>) paramMap.get("--model_name");
String value = (String) model.get("value");
Map<String, Object> valueMap = JsonUtils.jsonToMap(value);
insertAssetWorkFlow(workflowId, workflowName, Long.valueOf((String) valueMap.get("id")), (String) valueMap.get("version"), Constant.Asset_Type_Model);
}

public void saveModelWorkFlow1(String inParameters, Long workflowId, String workflowName) throws IOException {
Map<String, Object> paramMap = JsonUtils.jsonToMap(inParameters);
Map<String, Object> model = (Map<String, Object>) paramMap.get("--model_id");
String value = (String) model.get("value");
Map<String, Object> valueMap = JsonUtils.jsonToMap(value);
insertAssetWorkFlow(workflowId, workflowName, Long.valueOf((String) valueMap.get("id")), null, Constant.Asset_Type_Model);
}

public void saveServiceWorkFlow(String inParameters, Long workflowId, String workflowName) throws IOException {
Map<String, Object> paramMap = JsonUtils.jsonToMap(inParameters);
Map<String, Object> service = (Map<String, Object>) paramMap.get("--service_name");
String value = (String) service.get("value");
Map<String, Object> valueMap = JsonUtils.jsonToMap(value);
insertAssetWorkFlow(workflowId, workflowName, Long.valueOf((Integer) valueMap.get("id")), null, Constant.Asset_Type_Service);
}

public void insertAssetWorkFlow(Long workflowId, String workflowName, Long assetId, String version, String type) {
AssetWorkflow assetWorkflow = new AssetWorkflow();
assetWorkflow.setWorkflowId(workflowId);
assetWorkflow.setWorkflowName(workflowName);
assetWorkflow.setAssetId(assetId);
assetWorkflow.setAssetVersion(version);
assetWorkflow.setType(type);
assetWorkflowDao.insert(assetWorkflow);
}

}

+ 27
- 0
ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/AssetWorkflowDaoMapper.xml View File

@@ -0,0 +1,27 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.ruoyi.platform.mapper.AssetWorkflowDao">
<insert id="insert">
insert into asset_workflow(work_flow_id, workflow_name, asset_id, asset_version, type)
values (#{assetWorkflow.workflowId}, #{assetWorkflow.workflowName}, #{assetWorkflow.assetId},
#{assetWorkflow.assetVersion},
#{assetWorkflow.type})
</insert>

<update id="deleteByWorkFlowId">
update asset_workflow
set state = 0
where work_flow_id = #{workFlowId}
</update>

<select id="getAssetWorkflow" resultType="com.ruoyi.platform.domain.AssetWorkflow">
select * from asset_workflow
where asset_id = #{assetId}
and state = 1
and type = #{type}
<if test="assetVersion != null and assetVersion != ''">
and asset_version = #{assetVersion}
</if>
</select>
</mapper>


+ 4
- 0
ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ImageVersionDaoMapper.xml View File

@@ -123,6 +123,10 @@
</where>
</select>

<select id="queryByUrl" resultType="com.ruoyi.platform.domain.ImageVersion">
select * from image_version where url = #{url} and state = 1
</select>

<!--新增所有列-->
<insert id="insert" keyProperty="id" useGeneratedKeys="true">
insert into image_version(image_id, version, url, tag_name, file_size, status, create_by, create_time, update_by, update_time, state)


+ 0
- 5
ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/WorkflowDaoMapper.xml View File

@@ -175,11 +175,6 @@
where id = #{workflow.id}
</update>

<!--通过主键删除-->
<delete id="deleteById">
delete from workflow where id = #{id}
</delete>

<!--通过流水线名字进行模糊查询-->
<select id="queryByName" resultMap="WorkflowMap">
select


Loading…
Cancel
Save