Browse Source

Merge branch 'dev' of code.gitlink.org.cn:ci4s/ci4sManagement-cloud into dev

pull/14/head
cp3hnu 1 year ago
parent
commit
5b98773386
18 changed files with 107 additions and 66 deletions
  1. +4
    -2
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/experiment/ExperimentInsController.java
  2. +3
    -1
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/model/ModelsVersionController.java
  3. +1
    -0
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/DatasetVersion.java
  4. +2
    -0
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/ModelsVersion.java
  5. +1
    -1
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/DatasetVersionService.java
  6. +1
    -0
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ModelsService.java
  7. +5
    -3
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ModelsVersionService.java
  8. +2
    -4
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/DatasetServiceImpl.java
  9. +4
    -3
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/DatasetVersionServiceImpl.java
  10. +3
    -2
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentInsServiceImpl.java
  11. +4
    -2
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentServiceImpl.java
  12. +15
    -16
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ImageServiceImpl.java
  13. +4
    -2
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/JupyterServiceImpl.java
  14. +1
    -0
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java
  15. +22
    -6
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsVersionServiceImpl.java
  16. +5
    -3
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/TensorBoardServiceImpl.java
  17. +27
    -18
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/K8sClientUtil.java
  18. +3
    -3
      ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ModelsVersionDaoMapper.xml

+ 4
- 2
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/experiment/ExperimentInsController.java View File

@@ -173,9 +173,11 @@ public class ExperimentInsController extends BaseController {
* *
* @return 运行日志 * @return 运行日志
*/ */
@GetMapping("/nodeResult")
@PostMapping("/nodeResult")
@ApiOperation("查询实例节点结果") @ApiOperation("查询实例节点结果")
public GenericsAjaxResult<List> getNodeResult(@PathVariable("id") Integer id, @PathVariable("node_id") String nodeId) throws Exception {
public GenericsAjaxResult<List> getNodeResult(@RequestBody Map map) throws Exception {
Integer id = Integer.parseInt((String) map.get("id"));
String nodeId = (String) map.get("node_id");
return genericsSuccess(this.experimentInsService.getNodeResult(id,nodeId)); return genericsSuccess(this.experimentInsService.getNodeResult(id,nodeId));
} }




+ 3
- 1
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/model/ModelsVersionController.java View File

@@ -77,7 +77,7 @@ public class ModelsVersionController extends BaseController {
* @return 新增结果 * @return 新增结果
*/ */
@PostMapping @PostMapping
public GenericsAjaxResult<ModelsVersion> add(@RequestBody ModelsVersion modelsVersion) {
public GenericsAjaxResult<ModelsVersion> add(@RequestBody ModelsVersion modelsVersion) throws Exception {
return genericsSuccess(this.modelsVersionService.insert(modelsVersion)); return genericsSuccess(this.modelsVersionService.insert(modelsVersion));
} }


@@ -100,6 +100,7 @@ public class ModelsVersionController extends BaseController {
* @return 编辑结果 * @return 编辑结果
*/ */
@PutMapping @PutMapping
@ApiOperation("编辑模型")
public GenericsAjaxResult<ModelsVersion> edit(@RequestBody ModelsVersion modelsVersion) { public GenericsAjaxResult<ModelsVersion> edit(@RequestBody ModelsVersion modelsVersion) {
return genericsSuccess(this.modelsVersionService.update(modelsVersion)); return genericsSuccess(this.modelsVersionService.update(modelsVersion));
} }
@@ -111,6 +112,7 @@ public class ModelsVersionController extends BaseController {
* @return 删除是否成功 * @return 删除是否成功
*/ */
@DeleteMapping("{id}") @DeleteMapping("{id}")
@ApiOperation("删除模型")
public GenericsAjaxResult<String> deleteById(@PathVariable("id") Integer id) { public GenericsAjaxResult<String> deleteById(@PathVariable("id") Integer id) {
return genericsSuccess(this.modelsVersionService.removeById(id)); return genericsSuccess(this.modelsVersionService.removeById(id));
} }


+ 1
- 0
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/DatasetVersion.java View File

@@ -27,6 +27,7 @@ public class DatasetVersion implements Serializable {
private Integer datasetId; private Integer datasetId;


@ApiModelProperty(value = "版本") @ApiModelProperty(value = "版本")
@CheckDuplicate
private String version; private String version;


@ApiModelProperty(value = "数据集存储地址") @ApiModelProperty(value = "数据集存储地址")


+ 2
- 0
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/ModelsVersion.java View File

@@ -2,6 +2,7 @@ package com.ruoyi.platform.domain;


import com.fasterxml.jackson.databind.PropertyNamingStrategy; import com.fasterxml.jackson.databind.PropertyNamingStrategy;
import com.fasterxml.jackson.databind.annotation.JsonNaming; import com.fasterxml.jackson.databind.annotation.JsonNaming;
import com.ruoyi.platform.annotations.CheckDuplicate;
import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty; import io.swagger.annotations.ApiModelProperty;


@@ -28,6 +29,7 @@ private Integer id;
private Integer modelsId; private Integer modelsId;


@ApiModelProperty(value = "版本") @ApiModelProperty(value = "版本")
@CheckDuplicate
private String version; private String version;


@ApiModelProperty(value = "模型存储地址") @ApiModelProperty(value = "模型存储地址")


+ 1
- 1
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/DatasetVersionService.java View File

@@ -72,5 +72,5 @@ public interface DatasetVersionService {


void checkDeclaredVersion(DatasetVersion insert) throws Exception; void checkDeclaredVersion(DatasetVersion insert) throws Exception;


String addDatasetVersions(List<DatasetVersion> datasetVersions);
String addDatasetVersions(List<DatasetVersion> datasetVersions) throws Exception;
} }

+ 1
- 0
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ModelsService.java View File

@@ -80,4 +80,5 @@ public interface ModelsService {
String insertModelAndVersion(ModelsVo modelsVo) throws Exception; String insertModelAndVersion(ModelsVo modelsVo) throws Exception;


String readFileContent(Integer modelsId, String version) throws Exception; String readFileContent(Integer modelsId, String version) throws Exception;

} }

+ 5
- 3
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ModelsVersionService.java View File

@@ -40,7 +40,7 @@ public interface ModelsVersionService {
* @param modelsVersion 实例对象 * @param modelsVersion 实例对象
* @return 实例对象 * @return 实例对象
*/ */
ModelsVersion insert(ModelsVersion modelsVersion);
ModelsVersion insert(ModelsVersion modelsVersion) throws Exception;


/** /**
* 修改数据 * 修改数据
@@ -62,7 +62,7 @@ public interface ModelsVersionService {


List<ModelsVersion> queryByModelsId(Integer modelsId); List<ModelsVersion> queryByModelsId(Integer modelsId);


ModelsVersion duplicateModelsVersion(ModelsVersion oldModelsVersion);
ModelsVersion duplicateModelsVersion(ModelsVersion oldModelsVersion) throws Exception;


ModelsVersion queryByModelsVersion(ModelsVersion modelsVersion); ModelsVersion queryByModelsVersion(ModelsVersion modelsVersion);


@@ -70,5 +70,7 @@ public interface ModelsVersionService {


Map<Integer, String> deleteModelsVersion(Integer modelsId, String version); Map<Integer, String> deleteModelsVersion(Integer modelsId, String version);


String addModelVersions(List<ModelsVersion> modelsVersions);
String addModelVersions(List<ModelsVersion> modelsVersions) throws Exception;

void checkDeclaredVersion(ModelsVersion insert) throws Exception;
} }

+ 2
- 4
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/DatasetServiceImpl.java View File

@@ -5,8 +5,6 @@ import com.ruoyi.common.security.utils.SecurityUtils;
import com.ruoyi.platform.annotations.CheckDuplicate; import com.ruoyi.platform.annotations.CheckDuplicate;
import com.ruoyi.platform.domain.Dataset; import com.ruoyi.platform.domain.Dataset;
import com.ruoyi.platform.domain.DatasetVersion; import com.ruoyi.platform.domain.DatasetVersion;
import com.ruoyi.platform.domain.Models;
import com.ruoyi.platform.domain.ModelsVersion;
import com.ruoyi.platform.mapper.DatasetDao; import com.ruoyi.platform.mapper.DatasetDao;
import com.ruoyi.platform.mapper.DatasetVersionDao; import com.ruoyi.platform.mapper.DatasetVersionDao;
import com.ruoyi.platform.service.DatasetService; import com.ruoyi.platform.service.DatasetService;
@@ -17,9 +15,7 @@ import com.ruoyi.platform.utils.MinioUtil;
import com.ruoyi.platform.vo.DatasetVersionVo; import com.ruoyi.platform.vo.DatasetVersionVo;
import com.ruoyi.platform.vo.DatasetVo; import com.ruoyi.platform.vo.DatasetVo;
import com.ruoyi.system.api.model.LoginUser; import com.ruoyi.system.api.model.LoginUser;
import io.minio.MinioClient;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.io.InputStreamResource; import org.springframework.core.io.InputStreamResource;
import org.springframework.data.domain.Page; import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageImpl; import org.springframework.data.domain.PageImpl;
@@ -257,6 +253,7 @@ public class DatasetServiceImpl implements DatasetService {


} }



@Override @Override
public Map uploadDatasetPipeline(DatasetVersion datasetVersion) throws Exception { public Map uploadDatasetPipeline(DatasetVersion datasetVersion) throws Exception {
String username = SecurityUtils.getLoginUser().getUsername(); String username = SecurityUtils.getLoginUser().getUsername();
@@ -306,6 +303,7 @@ public class DatasetServiceImpl implements DatasetService {
// 使用Stream API提取version字段,并去重 // 使用Stream API提取version字段,并去重
return versions.stream() return versions.stream()
.map(DatasetVersion::getVersion) // 提取每个DatasetVersion对象的version属性 .map(DatasetVersion::getVersion) // 提取每个DatasetVersion对象的version属性
.filter(version -> version != null && !version.isEmpty()) //忽略null或空字符串的version
.distinct() // 去重 .distinct() // 去重
.collect(Collectors.toList()); // 收集到List中 .collect(Collectors.toList()); // 收集到List中
} }


+ 4
- 3
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/DatasetVersionServiceImpl.java View File

@@ -164,7 +164,7 @@ public class DatasetVersionServiceImpl implements DatasetVersionService {
} }


@Override @Override
public String addDatasetVersions(List<DatasetVersion> datasetVersions) {
public String addDatasetVersions(List<DatasetVersion> datasetVersions) throws Exception {
try { try {
// 遍历传入的 datasetVersions 列表 // 遍历传入的 datasetVersions 列表
for(DatasetVersion datasetVersion : datasetVersions) { for(DatasetVersion datasetVersion : datasetVersions) {
@@ -174,14 +174,15 @@ public class DatasetVersionServiceImpl implements DatasetVersionService {
return "新增数据集版本成功"; return "新增数据集版本成功";
} catch (Exception e) { } catch (Exception e) {
// 如果出现异常,返回异常消息 // 如果出现异常,返回异常消息
return "新增数据集版本失败: " + e.getMessage();
throw new Exception("新增数据集版本失败: " + e.getMessage());

} }




} }


private void insertPrepare(DatasetVersion datasetVersion) throws Exception { private void insertPrepare(DatasetVersion datasetVersion) throws Exception {
//checkDeclaredVersion(datasetVersion);
checkDeclaredVersion(datasetVersion);
LoginUser loginUser = SecurityUtils.getLoginUser(); LoginUser loginUser = SecurityUtils.getLoginUser();
datasetVersion.setCreateBy(loginUser.getUsername()); datasetVersion.setCreateBy(loginUser.getUsername());
datasetVersion.setUpdateBy(loginUser.getUsername()); datasetVersion.setUpdateBy(loginUser.getUsername());


+ 3
- 2
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentInsServiceImpl.java View File

@@ -54,7 +54,8 @@ public class ExperimentInsServiceImpl implements ExperimentInsService {
private String argoWorkflowPodLog; private String argoWorkflowPodLog;
@Value("${argo.ins.logsLines}") @Value("${argo.ins.logsLines}")
private int logsLines; private int logsLines;

@Resource
private K8sClientUtil k8sClientUtil;
private final MinioUtil minioUtil; private final MinioUtil minioUtil;


public ExperimentInsServiceImpl(MinioUtil minioUtil) { public ExperimentInsServiceImpl(MinioUtil minioUtil) {
@@ -520,7 +521,7 @@ public class ExperimentInsServiceImpl implements ExperimentInsService {


@Override @Override
public String getRealtimePodLogFromPod(PodLogVo podLogVo) { public String getRealtimePodLogFromPod(PodLogVo podLogVo) {
return K8sClientUtil.getPodLogs(podLogVo.getPodName(), podLogVo.getNamespace(),podLogVo.getContainerName(), logsLines);
return k8sClientUtil.getPodLogs(podLogVo.getPodName(), podLogVo.getNamespace(),podLogVo.getContainerName(), logsLines);
} }


private boolean isTerminatedState(ExperimentIns ins) throws IOException { private boolean isTerminatedState(ExperimentIns ins) throws IOException {


+ 4
- 2
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentServiceImpl.java View File

@@ -229,8 +229,10 @@ public class ExperimentServiceImpl implements ExperimentService {
//这里全局参数是一个json数组,需要转换成一个list<Map> //这里全局参数是一个json数组,需要转换成一个list<Map>
List<Map<String, Object>> params = JacksonUtil.parseJSONStr2MapList(StringUtils.isEmpty(experiment.getGlobalParam()) ? "[]" : experiment.getGlobalParam()); List<Map<String, Object>> params = JacksonUtil.parseJSONStr2MapList(StringUtils.isEmpty(experiment.getGlobalParam()) ? "[]" : experiment.getGlobalParam());
runReqMap.put("params", params); runReqMap.put("params", params);

runReqMap.put("experiment", new HashMap<String, Object>().put("name", "experiment-"+experiment.getId()));
//// 实验字段的Map,不要写成一行!
Map<String, Object> experimentMap = new HashMap<>();
experimentMap.put("name", "experiment-"+experiment.getId());
runReqMap.put("experiment", experimentMap);


Map<String ,Object> output = (Map<String, Object>) converMap.get("output"); Map<String ,Object> output = (Map<String, Object>) converMap.get("output");
// 调argo运行接口 // 调argo运行接口


+ 15
- 16
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ImageServiceImpl.java View File

@@ -1,30 +1,28 @@
package com.ruoyi.platform.service.impl; package com.ruoyi.platform.service.impl;


import com.ruoyi.common.security.utils.SecurityUtils; import com.ruoyi.common.security.utils.SecurityUtils;
import com.ruoyi.platform.domain.*;
import com.ruoyi.platform.domain.Image;
import com.ruoyi.platform.domain.ImageVersion;
import com.ruoyi.platform.mapper.ImageDao; import com.ruoyi.platform.mapper.ImageDao;
import com.ruoyi.platform.mapper.ImageVersionDao; import com.ruoyi.platform.mapper.ImageVersionDao;
import com.ruoyi.platform.service.ImageService; import com.ruoyi.platform.service.ImageService;
import com.ruoyi.platform.service.ImageVersionService; import com.ruoyi.platform.service.ImageVersionService;
import com.ruoyi.platform.service.MinioService; import com.ruoyi.platform.service.MinioService;
import com.ruoyi.platform.utils.FileUtil;
import com.ruoyi.platform.utils.K8sClientUtil; import com.ruoyi.platform.utils.K8sClientUtil;
import com.ruoyi.platform.vo.ImageVo; import com.ruoyi.platform.vo.ImageVo;
import com.ruoyi.system.api.model.LoginUser; import com.ruoyi.system.api.model.LoginUser;
import io.kubernetes.client.openapi.models.V1Pod; import io.kubernetes.client.openapi.models.V1Pod;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.data.domain.Page; import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageImpl; import org.springframework.data.domain.PageImpl;
import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.PageRequest;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.multipart.MultipartFile; import org.springframework.web.multipart.MultipartFile;


import javax.annotation.Resource; import javax.annotation.Resource;
import java.io.InputStream;
import java.util.Date; import java.util.Date;
import java.util.HashMap;
import java.util.Map; import java.util.Map;


/** /**
@@ -45,7 +43,8 @@ public class ImageServiceImpl implements ImageService {


@Resource @Resource
private ImageVersionDao imageVersionDao; private ImageVersionDao imageVersionDao;

@Resource
private K8sClientUtil k8sClientUtil;
@Resource @Resource
private MinioService minioService; private MinioService minioService;
@Value("${harbor.bucketName}") @Value("${harbor.bucketName}")
@@ -194,17 +193,17 @@ public class ImageServiceImpl implements ImageService {
@Override @Override
public String createImageFromNet(String imageName, String imageTag, String netPath) throws Exception { public String createImageFromNet(String imageName, String imageTag, String netPath) throws Exception {
// 得到容器 // 得到容器
V1Pod pod = K8sClientUtil.getNSPodList(serviceNS, deploymentName);
V1Pod pod = k8sClientUtil.getNSPodList(serviceNS, deploymentName);
if (pod == null) { if (pod == null) {
throw new Exception("镜像推送服务不存在"); throw new Exception("镜像推送服务不存在");
} }
String loginCmd = "docker login -u " + harborUser +" -p "+harborpassword+" "+harborUrl; String loginCmd = "docker login -u " + harborUser +" -p "+harborpassword+" "+harborUrl;
// 执行命令 docker login -u admin -p Harbor12345 172.20.32.187 // 执行命令 docker login -u admin -p Harbor12345 172.20.32.187
String loginlog = K8sClientUtil.executeCommand(pod,loginCmd);
String loginlog = k8sClientUtil.executeCommand(pod,loginCmd);
// 在这个容器的/data/admin 目录下执行命令 docker load -i fileName 得到返回的镜像名字name:tag // 在这个容器的/data/admin 目录下执行命令 docker load -i fileName 得到返回的镜像名字name:tag
String username = SecurityUtils.getLoginUser().getUsername(); String username = SecurityUtils.getLoginUser().getUsername();
// //
String logs2 = K8sClientUtil.executeCommand(pod,"docker pull "+netPath);
String logs2 = k8sClientUtil.executeCommand(pod,"docker pull "+netPath);
// 在容器里执行 docker tag name:tag nexus3.kube-system.svc:8083/imageName:imageTag // 在容器里执行 docker tag name:tag nexus3.kube-system.svc:8083/imageName:imageTag
if (StringUtils.isNoneBlank(logs2)){ if (StringUtils.isNoneBlank(logs2)){
String substring = logs2.substring(logs2.indexOf(harborUrl), logs2.length()); String substring = logs2.substring(logs2.indexOf(harborUrl), logs2.length());
@@ -212,8 +211,8 @@ public class ImageServiceImpl implements ImageService {
String cmd1 = "docker tag " + cleanedString+ " " + harborUrl+"/"+repository+"/"+username+"/" + imageName + imageTag; String cmd1 = "docker tag " + cleanedString+ " " + harborUrl+"/"+repository+"/"+username+"/" + imageName + imageTag;
String imageUrl = harborUrl+"/"+repository+"/"+username+"/" + imageName + imageTag; String imageUrl = harborUrl+"/"+repository+"/"+username+"/" + imageName + imageTag;
String cmd2 = "docker push " + imageUrl; String cmd2 = "docker push " + imageUrl;
String s = K8sClientUtil.executeCommand(pod, cmd1);
if (StringUtils.isNotEmpty(K8sClientUtil.executeCommand(pod, cmd2))){
String s = k8sClientUtil.executeCommand(pod, cmd1);
if (StringUtils.isNotEmpty(k8sClientUtil.executeCommand(pod, cmd2))){
return imageUrl; return imageUrl;
}else { }else {
throw new Exception("拉取公网镜像失败,请检查网络或者镜像地址"); throw new Exception("拉取公网镜像失败,请检查网络或者镜像地址");
@@ -226,18 +225,18 @@ public class ImageServiceImpl implements ImageService {
@Override @Override
public String createImageFromLocal(String imageName, String imageTag, String path) throws Exception { public String createImageFromLocal(String imageName, String imageTag, String path) throws Exception {
// 得到容器 // 得到容器
V1Pod pod = K8sClientUtil.getNSPodList(serviceNS, deploymentName);
V1Pod pod = k8sClientUtil.getNSPodList(serviceNS, deploymentName);
if (pod == null) { if (pod == null) {
throw new Exception("镜像推送服务不存在"); throw new Exception("镜像推送服务不存在");
} }
String loginCmd = "docker login -u " + harborUser +" -p "+harborpassword+" "+harborUrl; String loginCmd = "docker login -u " + harborUser +" -p "+harborpassword+" "+harborUrl;
// 执行命令 docker login -u admin -p Harbor12345 172.20.32.187 // 执行命令 docker login -u admin -p Harbor12345 172.20.32.187
String loginlog = K8sClientUtil.executeCommand(pod,loginCmd);
String loginlog = k8sClientUtil.executeCommand(pod,loginCmd);
// 在这个容器的/data/admin 目录下执行命令 docker load -i fileName 得到返回的镜像名字name:tag // 在这个容器的/data/admin 目录下执行命令 docker load -i fileName 得到返回的镜像名字name:tag
String username = SecurityUtils.getLoginUser().getUsername(); String username = SecurityUtils.getLoginUser().getUsername();
// //
String filePath = "/data/argo-workflow/" + bucketName + "/" +path; String filePath = "/data/argo-workflow/" + bucketName + "/" +path;
String logs2 = K8sClientUtil.executeCommand(pod,"docker load -i "+filePath);
String logs2 = k8sClientUtil.executeCommand(pod,"docker load -i "+filePath);
// 在容器里执行 docker tag name:tag nexus3.kube-system.svc:8083/imageName:imageTag // 在容器里执行 docker tag name:tag nexus3.kube-system.svc:8083/imageName:imageTag
if (StringUtils.isNoneBlank(logs2)){ if (StringUtils.isNoneBlank(logs2)){
String substring = logs2.substring(logs2.indexOf(harborUrl), logs2.length()); String substring = logs2.substring(logs2.indexOf(harborUrl), logs2.length());
@@ -245,8 +244,8 @@ public class ImageServiceImpl implements ImageService {
String cmd1 = "docker tag " + cleanedString+ " " + harborUrl+"/"+repository+"/"+username+"/" + imageName + imageTag; String cmd1 = "docker tag " + cleanedString+ " " + harborUrl+"/"+repository+"/"+username+"/" + imageName + imageTag;
String imageUrl = harborUrl+"/"+repository+"/"+username+"/" + imageName + imageTag; String imageUrl = harborUrl+"/"+repository+"/"+username+"/" + imageName + imageTag;
String cmd2 = "docker push " + imageUrl; String cmd2 = "docker push " + imageUrl;
String s = K8sClientUtil.executeCommand(pod, cmd1);
if (StringUtils.isNotEmpty(K8sClientUtil.executeCommand(pod, cmd2))){
String s = k8sClientUtil.executeCommand(pod, cmd1);
if (StringUtils.isNotEmpty(k8sClientUtil.executeCommand(pod, cmd2))){
return imageUrl; return imageUrl;
}else { }else {
throw new Exception("解析镜像压缩包失败,请检查镜像文件"); throw new Exception("解析镜像压缩包失败,请检查镜像文件");


+ 4
- 2
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/JupyterServiceImpl.java View File

@@ -34,6 +34,8 @@ public class JupyterServiceImpl implements JupyterService {


private final MinioUtil minioUtil; private final MinioUtil minioUtil;


@Resource
private K8sClientUtil k8sClientUtil;
@Resource @Resource
private MlflowUtil mlflowUtil; private MlflowUtil mlflowUtil;


@@ -46,8 +48,8 @@ public class JupyterServiceImpl implements JupyterService {
LoginUser loginUser = SecurityUtils.getLoginUser(); LoginUser loginUser = SecurityUtils.getLoginUser();
String podName = loginUser.getUsername().toLowerCase() + "-editor-pod"; String podName = loginUser.getUsername().toLowerCase() + "-editor-pod";
String pvcName = loginUser.getUsername().toLowerCase() + "-editor-pvc"; String pvcName = loginUser.getUsername().toLowerCase() + "-editor-pvc";
V1PersistentVolumeClaim pvc = K8sClientUtil.createPvc(namespace, pvcName, storage,storageClassName);
Integer podPort = K8sClientUtil.createPod(podName, namespace, port, mountPath, pvc, image);
V1PersistentVolumeClaim pvc = k8sClientUtil.createPvc(namespace, pvcName, storage,storageClassName);
Integer podPort = k8sClientUtil.createPod(podName, namespace, port, mountPath, pvc, image);
return masterIp + ":" + podPort; return masterIp + ":" + podPort;
} }




+ 1
- 0
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java View File

@@ -354,6 +354,7 @@ public class ModelsServiceImpl implements ModelsService {
// 使用Stream API提取version字段,并去重 // 使用Stream API提取version字段,并去重
return versions.stream() return versions.stream()
.map(ModelsVersion::getVersion) // 提取每个ModelVersion对象的version属性 .map(ModelsVersion::getVersion) // 提取每个ModelVersion对象的version属性
.filter(version -> version != null && !version.isEmpty()) //忽略null或空字符串的version
.distinct() // 去重 .distinct() // 去重
.collect(Collectors.toList()); // 收集到List中 .collect(Collectors.toList()); // 收集到List中
} }


+ 22
- 6
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsVersionServiceImpl.java View File

@@ -1,7 +1,7 @@
package com.ruoyi.platform.service.impl; package com.ruoyi.platform.service.impl;


import com.ruoyi.common.security.utils.SecurityUtils; import com.ruoyi.common.security.utils.SecurityUtils;
import com.ruoyi.platform.domain.DatasetVersion;
import com.ruoyi.platform.annotations.CheckDuplicate;
import com.ruoyi.platform.domain.ModelsVersion; import com.ruoyi.platform.domain.ModelsVersion;
import com.ruoyi.platform.mapper.ModelsDao; import com.ruoyi.platform.mapper.ModelsDao;
import com.ruoyi.platform.mapper.ModelsVersionDao; import com.ruoyi.platform.mapper.ModelsVersionDao;
@@ -14,6 +14,7 @@ import org.springframework.data.domain.PageRequest;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;


import javax.annotation.Resource; import javax.annotation.Resource;
import java.lang.reflect.Field;
import java.util.Date; import java.util.Date;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
@@ -64,7 +65,7 @@ public class ModelsVersionServiceImpl implements ModelsVersionService {
* @return 实例对象 * @return 实例对象
*/ */
@Override @Override
public ModelsVersion insert(ModelsVersion modelsVersion) {
public ModelsVersion insert(ModelsVersion modelsVersion) throws Exception {
//插入预备 //插入预备
insertPrepare(modelsVersion); insertPrepare(modelsVersion);
this.modelsVersionDao.insert(modelsVersion); this.modelsVersionDao.insert(modelsVersion);
@@ -135,7 +136,7 @@ public class ModelsVersionServiceImpl implements ModelsVersionService {
* @param oldModelsVersion 待更新的模型版本 * @param oldModelsVersion 待更新的模型版本
* @return 新的模型版本记录 * @return 新的模型版本记录
*/ */
public ModelsVersion duplicateModelsVersion(ModelsVersion oldModelsVersion) {
public ModelsVersion duplicateModelsVersion(ModelsVersion oldModelsVersion) throws Exception {
ModelsVersion newModelsVersion = new ModelsVersion(); ModelsVersion newModelsVersion = new ModelsVersion();
newModelsVersion.setModelsId(oldModelsVersion.getModelsId()); newModelsVersion.setModelsId(oldModelsVersion.getModelsId());
newModelsVersion.setVersion(oldModelsVersion.getVersion()); newModelsVersion.setVersion(oldModelsVersion.getVersion());
@@ -177,7 +178,7 @@ public class ModelsVersionServiceImpl implements ModelsVersionService {
} }


@Override @Override
public String addModelVersions(List<ModelsVersion> modelsVersions) {
public String addModelVersions(List<ModelsVersion> modelsVersions) throws Exception {
try { try {
// 遍历传入的 datasetVersions 列表 // 遍历传入的 datasetVersions 列表
for(ModelsVersion modelsVersion : modelsVersions) { for(ModelsVersion modelsVersion : modelsVersions) {
@@ -187,11 +188,26 @@ public class ModelsVersionServiceImpl implements ModelsVersionService {
return "新增模型版本成功"; return "新增模型版本成功";
} catch (Exception e) { } catch (Exception e) {
// 如果出现异常,返回异常消息 // 如果出现异常,返回异常消息
return "新增模型版本失败: " + e.getMessage();
throw new Exception("新增模型版本失败: " + e.getMessage()) ;
} }
} }


private void insertPrepare(ModelsVersion modelsVersion) {
@Override
public void checkDeclaredVersion(ModelsVersion insert) throws Exception {
ModelsVersion modelsVersion = modelsVersionDao.queryByModelsVersion(insert);
if (modelsVersion != null) {
Field[] fields = ModelsVersion.class.getDeclaredFields();
for (Field field : fields) {
if (field.isAnnotationPresent(CheckDuplicate.class)) {
CheckDuplicate annotation = field.getAnnotation(CheckDuplicate.class);
throw new Exception(field.getName() + annotation.message());
}
}
}
}

private void insertPrepare(ModelsVersion modelsVersion) throws Exception {
checkDeclaredVersion(modelsVersion);
LoginUser loginUser = SecurityUtils.getLoginUser(); LoginUser loginUser = SecurityUtils.getLoginUser();
modelsVersion.setCreateBy(loginUser.getUsername()); modelsVersion.setCreateBy(loginUser.getUsername());
modelsVersion.setUpdateBy(loginUser.getUsername()); modelsVersion.setUpdateBy(loginUser.getUsername());


+ 5
- 3
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/TensorBoardServiceImpl.java View File

@@ -12,6 +12,7 @@ import net.sf.jsqlparser.schema.Database;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;


import javax.annotation.Resource;
import java.text.SimpleDateFormat; import java.text.SimpleDateFormat;
import java.util.Date; import java.util.Date;


@@ -25,7 +26,8 @@ public class TensorBoardServiceImpl implements TensorBoardService {
private String mountPath; private String mountPath;
@Value("${tensorBoard.masterIp}") @Value("${tensorBoard.masterIp}")
private String masterIp; private String masterIp;

@Resource
private K8sClientUtil k8sClientUtil;
@Override @Override
public String getTensorBoardStatus(FrameLogPathVo frameLogPathVo){ public String getTensorBoardStatus(FrameLogPathVo frameLogPathVo){
String status = TensorBoardStatus.Terminated.getName(); String status = TensorBoardStatus.Terminated.getName();
@@ -36,7 +38,7 @@ public class TensorBoardServiceImpl implements TensorBoardService {
String podName = loginUser.getUsername().toLowerCase()+"-"+frameLogPathVo.getPath().split("/")[2]+ "-tensorboard-pod"; String podName = loginUser.getUsername().toLowerCase()+"-"+frameLogPathVo.getPath().split("/")[2]+ "-tensorboard-pod";


try { try {
String podStatus = K8sClientUtil.getPodStatus(podName, StringUtils.isEmpty(frameLogPathVo.getNamespace()) ? "default" : frameLogPathVo.getNamespace());
String podStatus = k8sClientUtil.getPodStatus(podName, StringUtils.isEmpty(frameLogPathVo.getNamespace()) ? "default" : frameLogPathVo.getNamespace());
System.out.println(podStatus); System.out.println(podStatus);
} catch (Exception e) { } catch (Exception e) {
return TensorBoardStatus.Terminated.getName(); return TensorBoardStatus.Terminated.getName();
@@ -51,7 +53,7 @@ public class TensorBoardServiceImpl implements TensorBoardService {
} }
LoginUser loginUser = SecurityUtils.getLoginUser(); LoginUser loginUser = SecurityUtils.getLoginUser();
String podName = loginUser.getUsername().toLowerCase()+"-"+frameLogPathVo.getPath().split("/")[2]+ "-tensorboard-pod"; String podName = loginUser.getUsername().toLowerCase()+"-"+frameLogPathVo.getPath().split("/")[2]+ "-tensorboard-pod";
Integer podPort = K8sClientUtil.createPodWithSubPath(podName, StringUtils.isEmpty(frameLogPathVo.getNamespace())?"default":frameLogPathVo.getNamespace(), port, mountPath,frameLogPathVo.getPath(), frameLogPathVo.getPvcName(), image);
Integer podPort = k8sClientUtil.createPodWithSubPath(podName, StringUtils.isEmpty(frameLogPathVo.getNamespace())?"default":frameLogPathVo.getNamespace(), port, mountPath,frameLogPathVo.getPath(), frameLogPathVo.getPvcName(), image);
return masterIp + ":" + podPort; return masterIp + ":" + podPort;
} }
} }

+ 27
- 18
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/K8sClientUtil.java View File

@@ -35,12 +35,8 @@ import java.util.Map;
@Slf4j @Slf4j
@Component @Component
public class K8sClientUtil { public class K8sClientUtil {
@Value("${k8s.http}")
private String http; private String http;

@Value("${k8s.token}")
private String token; private String token;

/** /**
* k8s-api客户端 * k8s-api客户端
*/ */
@@ -54,6 +50,19 @@ public class K8sClientUtil {
* 3. service-account namespace * 3. service-account namespace
* 4. master endpoints(ip, port) from pre-set environment variables * 4. master endpoints(ip, port) from pre-set environment variables
*/ */
@Autowired
public K8sClientUtil(@Value("${k8s.http}") String http, @Value("${k8s.token}") String token) {
this.http = http;
this.token = token;
try {
this.apiClient = new ClientBuilder().
setBasePath(http).setVerifyingSsl(false).
setAuthentication(new AccessTokenAuthentication(token)).build();
} catch (Exception e) {
log.error("构建K8s-Client异常", e);
throw new RuntimeException("构建K8s-Client异常");
}
}
// @Autowired // @Autowired
// public K8sClientUtil() { // public K8sClientUtil() {
// try { // try {
@@ -65,12 +74,12 @@ public class K8sClientUtil {
// throw new RuntimeException("构建K8s-Client异常"); // throw new RuntimeException("构建K8s-Client异常");
// } // }
// } // }
@PostConstruct
public void init() {
this.apiClient = new ClientBuilder().
setBasePath(http).setVerifyingSsl(false).
setAuthentication(new AccessTokenAuthentication(token)).build();
}
// @PostConstruct
// public void init() {
// this.apiClient = new ClientBuilder().
// setBasePath(http).setVerifyingSsl(false).
// setAuthentication(new AccessTokenAuthentication(token)).build();
// }


/** /**
* 构建集群外通过UA访问的客户端 * 构建集群外通过UA访问的客户端
@@ -78,7 +87,7 @@ public class K8sClientUtil {
* *
* @param kubeConfigPath kube连接配置文件 * @param kubeConfigPath kube连接配置文件
*/ */
// public K8sClientUtil(String kubeConfigPath) {
// public K8sClientUtil() {
// try { // try {
// this.apiClient = new ClientBuilder(). // this.apiClient = new ClientBuilder().
// setBasePath("https://172.20.32.181:6443").setVerifyingSsl(false). // setBasePath("https://172.20.32.181:6443").setVerifyingSsl(false).
@@ -174,7 +183,7 @@ public class K8sClientUtil {
* @param pvcName 服务名称 * @param pvcName 服务名称
* @return 创建成功的service对象 * @return 创建成功的service对象
*/ */
public static V1PersistentVolumeClaim createPvc(String namespace, String pvcName ,String storage, String storageClassName){
public V1PersistentVolumeClaim createPvc(String namespace, String pvcName ,String storage, String storageClassName){
CoreV1Api api = new CoreV1Api(apiClient); CoreV1Api api = new CoreV1Api(apiClient);


V1PersistentVolumeClaimList pvcList = null; V1PersistentVolumeClaimList pvcList = null;
@@ -229,7 +238,7 @@ public class K8sClientUtil {
* @return 创建成功的pod,的nodePort端口 * @return 创建成功的pod,的nodePort端口
*/ */


public static Integer createPod(String podName, String namespace, Integer port ,String mountPath, V1PersistentVolumeClaim pvc, String image){
public Integer createPod(String podName, String namespace, Integer port ,String mountPath, V1PersistentVolumeClaim pvc, String image){


Map<String, String> selector = new LinkedHashMap<String, String>(); Map<String, String> selector = new LinkedHashMap<String, String>();
selector.put("k8s-jupyter", podName); selector.put("k8s-jupyter", podName);
@@ -299,7 +308,7 @@ public class K8sClientUtil {
* @return 创建成功的pod,的nodePort端口 * @return 创建成功的pod,的nodePort端口
*/ */


public static Integer createPodWithSubPath(String podName, String namespace, Integer port ,String mountPath,String subPath,String pvcName, String image){
public Integer createPodWithSubPath(String podName, String namespace, Integer port ,String mountPath,String subPath,String pvcName, String image){


Map<String, String> selector = new LinkedHashMap<String, String>(); Map<String, String> selector = new LinkedHashMap<String, String>();
selector.put("k8s-jupyter", podName); selector.put("k8s-jupyter", podName);
@@ -361,7 +370,7 @@ public class K8sClientUtil {
* *
* @return podList * @return podList
*/ */
public static V1Pod getNSPodList(String namespace,String deploymentName) throws Exception {
public V1Pod getNSPodList(String namespace,String deploymentName) throws Exception {
// new a CoreV1Api // new a CoreV1Api
CoreV1Api api = new CoreV1Api(apiClient); CoreV1Api api = new CoreV1Api(apiClient);
V1PodList v1PodList = null; V1PodList v1PodList = null;
@@ -381,7 +390,7 @@ public class K8sClientUtil {
return null; return null;
} }


public static String executeCommand(V1Pod item, String command) {
public String executeCommand(V1Pod item, String command) {
try { try {
// 创建API实例 // 创建API实例
// 创建Exec实例 // 创建Exec实例
@@ -408,13 +417,13 @@ public class K8sClientUtil {
* @param podName Pod的名称 * @param podName Pod的名称
* @param namespace Pod所在的Namespace * @param namespace Pod所在的Namespace
*/ */
public static String getPodStatus(String podName, String namespace) throws Exception {
public String getPodStatus(String podName, String namespace) throws Exception {
CoreV1Api api = new CoreV1Api(apiClient); CoreV1Api api = new CoreV1Api(apiClient);
V1Pod pod = api.readNamespacedPod(podName, namespace, null, null, null); V1Pod pod = api.readNamespacedPod(podName, namespace, null, null, null);
return pod.getStatus().getPhase(); return pod.getStatus().getPhase();
} }


public static String getPodLogs(String podName,String namespace,String container,int line) {
public String getPodLogs(String podName,String namespace,String container,int line) {
CoreV1Api api = new CoreV1Api(apiClient); CoreV1Api api = new CoreV1Api(apiClient);
try { try {
String log = api.readNamespacedPodLog(podName, namespace, StringUtils.isEmpty(container)?null:container, null, null, null, null,null, null, line, null); String log = api.readNamespacedPodLog(podName, namespace, StringUtils.isEmpty(container)?null:container, null, null, null, null,null, null, line, null);


+ 3
- 3
ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ModelsVersionDaoMapper.xml View File

@@ -169,7 +169,7 @@
<!--新增所有列--> <!--新增所有列-->
<insert id="insert" keyProperty="id" useGeneratedKeys="true"> <insert id="insert" keyProperty="id" useGeneratedKeys="true">
insert into models_version(models_id, version, url, file_name, file_size, status, description, create_by, create_time, update_by, update_time, state) insert into models_version(models_id, version, url, file_name, file_size, status, description, create_by, create_time, update_by, update_time, state)
values (#{modelsVersion.modelsId}, #{modelsVersion.version}, #{modelsVersion.url}, #{modelsVersion.fileName}, #{modelsVersion.fileSize}, #{modelsVersion.status}, #{modelsVersion.createBy}, #{modelsVersion.createTime}, #{modelsVersion.updateBy}, #{modelsVersion.updateTime}, #{modelsVersion.state})
values (#{modelsVersion.modelsId}, #{modelsVersion.version}, #{modelsVersion.url}, #{modelsVersion.fileName}, #{modelsVersion.fileSize}, #{modelsVersion.status}, #{modelsVersion.description}, #{modelsVersion.createBy}, #{modelsVersion.createTime}, #{modelsVersion.updateBy}, #{modelsVersion.updateTime}, #{modelsVersion.state})
</insert> </insert>




@@ -178,7 +178,7 @@
(models_id, version, url, file_name, file_size, status, description, create_by, create_time, update_by, update_time, state) (models_id, version, url, file_name, file_size, status, description, create_by, create_time, update_by, update_time, state)
values values
<foreach collection="entities" item="entity" separator=","> <foreach collection="entities" item="entity" separator=",">
(#{entity.modelsId}, #{entity.version}, #{entity.url}, #{entity.fileName}, #{entity.fileSize}, #{entity.status}, #{entity.createBy}, #{entity.createTime}, #{entity.updateBy}, #{entity.updateTime}, #{entity.state})
(#{entity.modelsId}, #{entity.version}, #{entity.url}, #{entity.fileName}, #{entity.fileSize}, #{entity.status}, #{entity.description},#{entity.createBy}, #{entity.createTime}, #{entity.updateBy}, #{entity.updateTime}, #{entity.state})
</foreach> </foreach>
</insert> </insert>


@@ -188,7 +188,7 @@
(models_id, version, url, file_name, file_size, status, description, create_by, create_time, update_by, update_time, state) (models_id, version, url, file_name, file_size, status, description, create_by, create_time, update_by, update_time, state)
values values
<foreach collection="entities" item="entity" separator=","> <foreach collection="entities" item="entity" separator=",">
(#{entity.modelsId}, #{entity.version}, #{entity.url}, #{entity.fileName}, #{entity.fileSize}, #{entity.status}, #{entity.createBy}, #{entity.createTime}, #{entity.updateBy}, #{entity.updateTime}, #{entity.state})
(#{entity.modelsId}, #{entity.version}, #{entity.url}, #{entity.fileName}, #{entity.fileSize}, #{entity.status},#{entity.description}, #{entity.createBy}, #{entity.createTime}, #{entity.updateBy}, #{entity.updateTime}, #{entity.state})
</foreach> </foreach>
on duplicate key update on duplicate key update
models_id = values(models_id), models_id = values(models_id),


Loading…
Cancel
Save