diff --git a/k8s/template-yaml/k8s-7management.yaml b/k8s/template-yaml/k8s-7management.yaml index 4fcddb15..edc1c621 100644 --- a/k8s/template-yaml/k8s-7management.yaml +++ b/k8s/template-yaml/k8s-7management.yaml @@ -26,10 +26,11 @@ spec: volumeMounts: - name: resource-volume mountPath: /home/resource/ + subPath: mini-model-platform-data volumes: - name: resource-volume - persistentVolumeClaim: - claimName: platform-data-pvc-nfs + hostPath: + path: /platform-data --- apiVersion: v1 kind: Service diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/aim/AimController.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/aim/AimController.java index 31a6cba1..28632dcb 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/aim/AimController.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/aim/AimController.java @@ -24,19 +24,19 @@ public class AimController extends BaseController { @GetMapping("/getExpTrainInfos/{experiment_id}") @ApiOperation("获取当前实验的模型训练指标信息") @ApiResponse - public GenericsAjaxResult> getExpTrainInfos(@RequestParam(value = "offset", required = false) String offset, - @RequestParam(value = "limit") int limit, - @PathVariable("experiment_id") Integer experimentId) throws Exception { - return genericsSuccess(aimService.getExpTrainInfos(experimentId, offset, limit)); + public GenericsAjaxResult> getExpTrainInfos(@RequestParam(value = "page") int page, + @RequestParam(value = "size") int size, + @PathVariable("experiment_id") Integer experimentId) { + return genericsSuccess(aimService.getExpInfos(true, experimentId, page, size)); } @GetMapping("/getExpEvaluateInfos/{experiment_id}") @ApiOperation("获取当前实验的模型推理指标信息") @ApiResponse - public GenericsAjaxResult> getExpEvaluateInfos(@RequestParam(value = "offset", required = false) String offset, - @RequestParam(value = "limit") int limit, - @PathVariable("experiment_id") Integer experimentId) throws Exception { - return genericsSuccess(aimService.getExpEvaluateInfos(experimentId, offset, limit)); + public GenericsAjaxResult> getExpEvaluateInfos(@RequestParam(value = "page") int page, + @RequestParam(value = "size") int size, + @PathVariable("experiment_id") Integer experimentId) { + return genericsSuccess(aimService.getExpInfos(false, experimentId, page, size)); } @PostMapping("/getExpMetrics") diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/service/ServiceController.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/service/ServiceController.java index 7d973bae..3eb28e4c 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/service/ServiceController.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/service/ServiceController.java @@ -89,6 +89,12 @@ public class ServiceController extends BaseController { return genericsSuccess(serviceService.getServiceVersion(id)); } + @GetMapping("serviceVersionCompare") + @ApiOperation("服务版本版本对比") + public GenericsAjaxResult> serviceVersionCompare(@RequestParam("id1") Long id1, @RequestParam("id2") Long id2) throws IllegalAccessException { + return genericsSuccess(serviceService.serviceVersionCompare(id1, id2)); + } + @GetMapping("/serviceVersionList/{id}") @ApiOperation("查询服务版本列表") public GenericsAjaxResult> serviceVersionList(@PathVariable("id") Long id) { diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/scheduling/ExperimentInstanceStatusTask.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/scheduling/ExperimentInstanceStatusTask.java index eaf6ee12..59d8bf47 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/scheduling/ExperimentInstanceStatusTask.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/scheduling/ExperimentInstanceStatusTask.java @@ -49,32 +49,48 @@ public class ExperimentInstanceStatusTask { //运行成功的实验实例记录指标数值 if (Constant.Succeeded.equals(experimentIns.getStatus())) { Map metricRecord = JacksonUtil.parseJSONStr2Map(experimentIns.getMetricRecord()); - List> trainMetricRecord = (List>) metricRecord.get("train"); - List> evaluateMetricRecord = (List>) metricRecord.get("evaluate"); + List> trainMetricRecords = (List>) metricRecord.get("train"); + List> evaluateMetricRecords = (List>) metricRecord.get("evaluate"); HashMap metricValue = new HashMap<>(); - HashMap trainMetricValue = new HashMap<>(); - HashMap evaluateMetricValue = new HashMap<>(); + HashMap trainMetricValues = new HashMap<>(); + HashMap evaluateMetricValues = new HashMap<>(); - if (trainMetricRecord != null && !trainMetricRecord.isEmpty()) { - String runId = (String) trainMetricRecord.get(0).get("run_id"); - List expTrainInfos = aimService.getExpTrainInfos1(true, experimentIns.getExperimentId(), runId); - for (InsMetricInfoVo expTrainInfo : expTrainInfos) { - Map metrics = expTrainInfo.getMetrics(); - trainMetricValue.putAll(metrics); + if (trainMetricRecords != null && !trainMetricRecords.isEmpty()) { + for (Map trainMetricRecord : trainMetricRecords) { + HashMap trainMetricValue = new HashMap<>(); + String taskId = (String) trainMetricRecord.get("task_id"); + if (taskId.startsWith("model-train")) { + String runId = (String) trainMetricRecord.get("run_id"); + List expTrainInfos = aimService.getExpInfos1(true, experimentIns.getExperimentId(), runId); + for (InsMetricInfoVo expTrainInfo : expTrainInfos) { + Map metrics = expTrainInfo.getMetrics(); + trainMetricValue.putAll(metrics); + trainMetricValue.put("run_hash", expTrainInfo.getRunId()); + } + } + trainMetricValues.put(taskId, trainMetricValue); } } - if (evaluateMetricRecord != null && !evaluateMetricRecord.isEmpty()) { - String runId = (String) evaluateMetricRecord.get(0).get("run_id"); - List expTrainInfos = aimService.getExpTrainInfos1(false, experimentIns.getExperimentId(), runId); - for (InsMetricInfoVo expTrainInfo : expTrainInfos) { - Map metrics = expTrainInfo.getMetrics(); - evaluateMetricValue.putAll(metrics); + if (evaluateMetricRecords != null && !evaluateMetricRecords.isEmpty()) { + for (Map evaluateMetricRecord : evaluateMetricRecords) { + HashMap evaluateMetricValue = new HashMap<>(); + String taskId = (String) evaluateMetricRecord.get("task_id"); + if (taskId.startsWith("model-evaluate")) { + String runId = (String) evaluateMetricRecord.get("run_id"); + List expTrainInfos = aimService.getExpInfos1(false, experimentIns.getExperimentId(), runId); + for (InsMetricInfoVo expTrainInfo : expTrainInfos) { + Map metrics = expTrainInfo.getMetrics(); + evaluateMetricValue.putAll(metrics); + evaluateMetricValue.put("run_hash", expTrainInfo.getRunId()); + } + } + evaluateMetricValues.put(taskId, evaluateMetricValue); } } - metricValue.put("train", trainMetricValue); - metricValue.put("evaluate", evaluateMetricValue); + metricValue.put("train", trainMetricValues); + metricValue.put("evaluate", evaluateMetricValues); experimentIns.setMetricValue(JsonUtils.mapToJson(metricValue)); } experimentIns.setUpdateTime(new Date()); diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AimService.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AimService.java index 3e2ccb77..abcbc645 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AimService.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AimService.java @@ -10,10 +10,12 @@ public interface AimService { List getExpTrainInfos(Integer experimentId, String offset, int limit) throws Exception; - List getExpTrainInfos1(boolean isTrain, Integer experimentId, String runId) throws Exception; - List getExpEvaluateInfos(Integer experimentId, String offset, int limit) throws Exception; + List getExpInfos(boolean isTrain, Integer experimentId, int page, int size); + + List getExpInfos1(boolean isTrain, Integer experimentId, String runId) throws Exception; + String getExpMetrics(List runIds) throws Exception; HashMap queryMetricsParams(String runId) throws Exception; diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ServiceService.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ServiceService.java index 6ce6bc6a..5c69f257 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ServiceService.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ServiceService.java @@ -28,6 +28,8 @@ public interface ServiceService { ServiceVersionVo getServiceVersion(Long id); + Map serviceVersionCompare(Long id1, Long id2) throws IllegalAccessException; + String deleteService(Long id); String deleteServiceVersion(Long id); diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AimServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AimServiceImpl.java index a2b21202..1bfdb77d 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AimServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AimServiceImpl.java @@ -1,7 +1,10 @@ package com.ruoyi.platform.service.impl; import com.alibaba.fastjson2.JSON; +import com.alibaba.fastjson2.JSONArray; +import com.alibaba.fastjson2.JSONObject; import com.ruoyi.platform.domain.ExperimentIns; +import com.ruoyi.platform.mapper.ExperimentInsDao; import com.ruoyi.platform.service.AimService; import com.ruoyi.platform.service.ExperimentInsService; import com.ruoyi.platform.utils.AIM64EncoderUtil; @@ -11,6 +14,7 @@ import com.ruoyi.platform.utils.JsonUtils; import com.ruoyi.platform.vo.InsMetricInfoVo; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Value; +import org.springframework.data.domain.PageRequest; import org.springframework.stereotype.Service; import javax.annotation.Resource; @@ -23,6 +27,8 @@ import java.util.stream.Collectors; public class AimServiceImpl implements AimService { @Resource private ExperimentInsService experimentInsService; + @Resource + private ExperimentInsDao experimentInsDao; @Value("${aim.url}") private String aimUrl; @@ -145,9 +151,59 @@ public class AimServiceImpl implements AimService { return aimRunInfoList; } + public List getExpInfos(boolean isTrain, Integer experimentId, int page, int size) { + PageRequest pageRequest = PageRequest.of(page, size); + ExperimentIns query = new ExperimentIns(); + query.setExperimentId(experimentId); + List experimentInsList = experimentInsDao.queryAllByLimit(query, pageRequest); + if (experimentInsList == null || experimentInsList.size() == 0) { + return new ArrayList<>(); + } + List aimRunInfoList = new ArrayList<>(); + for (ExperimentIns experimentIns : experimentInsList) { + InsMetricInfoVo aimRunInfo = new InsMetricInfoVo(); + aimRunInfo.setExperimentInsId(experimentIns.getId()); + aimRunInfo.setStartTime(experimentIns.getCreateTime()); + aimRunInfo.setStatus(experimentIns.getStatus()); + + //解析参数 + JSONArray params = JSON.parseArray(experimentIns.getGlobalParam()); + HashMap paramsMap = new HashMap<>(); + List paramsNames = new ArrayList<>(); + + for (int i = 0; i < params.size(); i++) { + JSONObject jsonObject = params.getJSONObject(i); + String paramName = jsonObject.getString("param_name"); + String paramValue = jsonObject.getString("param_value"); + paramsMap.put(paramName, paramValue); + paramsNames.add(paramName); + } + aimRunInfo.setParams(paramsMap); + aimRunInfo.setParamsNames(paramsNames); + + //解析数据集 + Map metricRecord = JacksonUtil.parseJSONStr2Map(experimentIns.getMetricRecord()); + if (isTrain) { + aimRunInfo.setDataset(getDataset("train", metricRecord)); + } else { + aimRunInfo.setDataset(getDataset("evaluate", metricRecord)); + } + + //解析指标 + Map metricValue = JacksonUtil.parseJSONStr2Map(experimentIns.getMetricValue()); + if (isTrain) { + setMetricValue("train",metricValue,aimRunInfo); + } else { + setMetricValue("evaluate",metricValue,aimRunInfo); + } + aimRunInfoList.add(aimRunInfo); + } + return aimRunInfoList; + } + @Override - public List getExpTrainInfos1(boolean isTrain, Integer experimentId, String runId) throws Exception { + public List getExpInfos1(boolean isTrain, Integer experimentId, String runId) throws Exception { String encodedUrlString = URLEncoder.encode("run.id==\"" + runId + "\"", "UTF-8"); String url = aimProxyUrl + "/api/runs/search/run?query=" + encodedUrlString; String s = HttpUtils.sendGet(url, null); @@ -235,6 +291,40 @@ public class AimServiceImpl implements AimService { return aimRunInfoList; } + private List getDataset(String isTrain, Map metricRecord) { + List datasetList = new ArrayList<>(); + List> trainMetricRecords = (List>) metricRecord.get(isTrain); + for (Map trainMetricRecord : trainMetricRecords) { + String taskId = (String) trainMetricRecord.get("task_id"); + if (taskId.startsWith("model-" + isTrain)) { + List> datasets = (List>) trainMetricRecord.get("datasets"); + for (Map dataset : datasets) { + String datasetName = dataset.get("dataset_name") + ":" + dataset.get("dataset_version"); + datasetList.add(datasetName); + } + } + } + return datasetList; + } + + private void setMetricValue(String isTrain, Map metricValue, InsMetricInfoVo aimRunInfo){ + Map metricValues = (Map) metricValue.get(isTrain); + + HashMap metrics = new HashMap<>(); + List metricsNames = new ArrayList<>(); + for (String key : metricValues.keySet()) { + Map valueMap = (Map) metricValues.get(key); + aimRunInfo.setRunId((String) valueMap.get("run_hash")); + + metrics.putAll(valueMap.entrySet().stream().filter(entry -> !entry.getKey().equals("run_hash")).collect(Collectors.toMap( + Map.Entry::getKey, + Map.Entry::getValue + ))); + metricsNames.addAll(valueMap.keySet().stream().filter(entry -> !entry.equals("run_hash")).collect(Collectors.toList())); + } + aimRunInfo.setMetrics(metrics); + aimRunInfo.setMetricsNames(metricsNames); + } private List getTrainDateSet(List> records, String aimrunId) { List datasetList = new ArrayList<>(); diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/DatasetVersionServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/DatasetVersionServiceImpl.java index 441ea786..4a00ddb4 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/DatasetVersionServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/DatasetVersionServiceImpl.java @@ -165,9 +165,9 @@ public class DatasetVersionServiceImpl implements DatasetVersionService { .findFirst() .ifPresent(datasetVersion -> { String url = datasetVersion.getUrl(); -// response.put("path", url); - String path = bucketName + '/' + url.substring(0, url.lastIndexOf('/')); - response.put("path", path); + response.put("path", url); +// String path = bucketName + '/' + url.substring(0, url.lastIndexOf('/')); +// response.put("path", path); }); response.put("content", datasetVersionList); diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentInsServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentInsServiceImpl.java index 1319f0a9..b7ff69b4 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentInsServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentInsServiceImpl.java @@ -172,15 +172,6 @@ public class ExperimentInsServiceImpl implements ExperimentInsService { public Page queryByPage(ExperimentIns experimentIns, PageRequest pageRequest) throws IOException { long total = this.experimentInsDao.count(experimentIns); List experimentInsList = this.experimentInsDao.queryAllByLimit(experimentIns, pageRequest); -// if (experimentInsList!=null && experimentInsList.size()>0) { -// for (ExperimentIns ins : experimentInsList) { -// //如果实验实例不为空或者 -// if (ins != null && StringUtils.isEmpty(ins.getStatus())) { -// ins = this.queryStatusFromArgo(ins); -// this.update(ins); -// } -// } -// } return new PageImpl<>(experimentInsList, pageRequest, total); } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ImageServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ImageServiceImpl.java index faacf8dd..92d2bb45 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ImageServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ImageServiceImpl.java @@ -61,10 +61,10 @@ public class ImageServiceImpl implements ImageService { @Resource private MinioService minioService; -// @Value("${minio.dataReleaseBucketName}") -// private String bucketName; - @Value("${harbor.bucketName}") + @Value("${minio.dataReleaseBucketName}") private String bucketName; +// @Value("${harbor.bucketName}") +// private String bucketName; @Value("${harbor.repository}") private String repository; @Value("${harbor.harborUrl}") diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/JupyterServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/JupyterServiceImpl.java index 1c861949..87111365 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/JupyterServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/JupyterServiceImpl.java @@ -101,14 +101,14 @@ public class JupyterServiceImpl implements JupyterService { //构造pod名称 String podName = loginUser.getUsername().toLowerCase() + "-editor-pod" + "-" + id; //新建编辑器的pvc - String pvcName = loginUser.getUsername().toLowerCase() + "-editor-pvc"; - V1PersistentVolumeClaim pvc = k8sClientUtil.createPvc(namespace, pvcName, storage, storageClassName); +// String pvcName = loginUser.getUsername().toLowerCase() + "-editor-pvc"; +// V1PersistentVolumeClaim pvc = k8sClientUtil.createPvc(namespace, pvcName, storage, storageClassName); //TODO 设置镜像可配置,这里先用默认镜像启动pod // 调用修改后的 createPod 方法,传入额外的参数 - Integer podPort = k8sClientUtil.createConfiguredPod(podName, namespace, port, mountPath, pvc, devEnvironment, minioPvcName, datasetPath, modelPath); -// Integer podPort = k8sClientUtil.createConfiguredPod(podName, namespace, port, mountPath, null, devEnvironment, minioPvcName, datasetPath, modelPath); +// Integer podPort = k8sClientUtil.createConfiguredPod(podName, namespace, port, mountPath, pvc, devEnvironment, minioPvcName, datasetPath, modelPath); + Integer podPort = k8sClientUtil.createConfiguredPod(podName, namespace, port, mountPath, null, devEnvironment, minioPvcName, datasetPath, modelPath); String url = masterIp + ":" + podPort; redisService.setCacheObject(podName, masterIp + ":" + podPort); devEnvironment.setStatus("Pending"); diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java index f69b7359..ff053e78 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java @@ -1300,7 +1300,7 @@ public class ModelsServiceImpl implements ModelsService { for (int i = 0; i < trainMetrics.size(); i++) { JSONObject jsonObject = trainMetrics.getJSONObject(i); String runId = jsonObject.getString("run_id"); - List expTrainInfos = aimsService.getExpTrainInfos1(true, modelMetaVo.getTrainTask().getExperimentId(), runId); + List expTrainInfos = aimsService.getExpInfos1(true, modelMetaVo.getTrainTask().getExperimentId(), runId); for (InsMetricInfoVo expTrainInfo : expTrainInfos) { Map metrics1 = expTrainInfo.getMetrics(); train.putAll(metrics1); @@ -1322,7 +1322,7 @@ public class ModelsServiceImpl implements ModelsService { for (int i = 0; i < testMetrics.size(); i++) { JSONObject jsonObject = testMetrics.getJSONObject(i); String runId = jsonObject.getString("run_id"); - List expTestInfos = aimsService.getExpTrainInfos1(false, modelMetaVo.getTrainTask().getExperimentId(), runId); + List expTestInfos = aimsService.getExpInfos1(false, modelMetaVo.getTrainTask().getExperimentId(), runId); for (InsMetricInfoVo expTestInfo : expTestInfos) { Map metrics1 = expTestInfo.getMetrics(); evaluate.putAll(metrics1); diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsVersionServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsVersionServiceImpl.java index c4dafa96..274b3c1d 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsVersionServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsVersionServiceImpl.java @@ -43,9 +43,7 @@ public class ModelsVersionServiceImpl implements ModelsVersionService { @Resource @Lazy private ModelDependencyService modelDependencyService; - // 固定存储桶名 - @Value("${minio.dataReleaseBucketName}") - private String bucketName; + /** * 通过ID查询单条数据 diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ServiceServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ServiceServiceImpl.java index 11b30f45..5cd35fb7 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ServiceServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ServiceServiceImpl.java @@ -7,6 +7,7 @@ import com.ruoyi.platform.constant.Constant; import com.ruoyi.platform.domain.ServiceVersion; import com.ruoyi.platform.mapper.ServiceDao; import com.ruoyi.platform.service.ServiceService; +import com.ruoyi.platform.utils.ConvertUtil; import com.ruoyi.platform.utils.HttpUtils; import com.ruoyi.platform.utils.JacksonUtil; import com.ruoyi.platform.vo.serviceVos.ServiceCodeConfigVo; @@ -157,6 +158,28 @@ public class ServiceServiceImpl implements ServiceService { return serviceVersionVo; } + @Override + public Map serviceVersionCompare(Long id1, Long id2) throws IllegalAccessException { + HashMap result = new HashMap<>(); + + ServiceVersion serviceVersion1 = serviceDao.getServiceVersionById(id1); + ServiceVersion serviceVersion2 = serviceDao.getServiceVersionById(id2); + com.ruoyi.platform.domain.Service service = serviceDao.getServiceById(serviceVersion1.getServiceId()); + + ServiceVersionVo serviceVersionVo1 = getServiceVersionVo(serviceVersion1); + ServiceVersionVo serviceVersionVo2 = getServiceVersionVo(serviceVersion2); + serviceVersionVo1.setServiceName(service.getServiceName()); + serviceVersionVo2.setServiceName(service.getServiceName()); + + Map compareMap = ConvertUtil.compareObjects(serviceVersion1, serviceVersion2); + + result.put("version1", serviceVersionVo1); + result.put("version2", serviceVersionVo2); + result.put("differences", compareMap); + + return result; + } + @Override public String deleteService(Long id) { com.ruoyi.platform.domain.Service service = serviceDao.getServiceById(id); diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/TensorBoardServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/TensorBoardServiceImpl.java index dca3651d..83b87d82 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/TensorBoardServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/TensorBoardServiceImpl.java @@ -59,17 +59,17 @@ public class TensorBoardServiceImpl implements TensorBoardService { @Override public String runTensorBoard(FrameLogPathVo frameLogPathVo) throws Exception { - if (StringUtils.isEmpty(frameLogPathVo.getPath())||StringUtils.isEmpty(frameLogPathVo.getPvcName())){ - throw new Exception("存储路径或存储为空"); - } -// if (StringUtils.isEmpty(frameLogPathVo.getPath())) { -// throw new Exception("存储路径为空"); +// if (StringUtils.isEmpty(frameLogPathVo.getPath())||StringUtils.isEmpty(frameLogPathVo.getPvcName())){ +// throw new Exception("存储路径或存储为空"); // } + if (StringUtils.isEmpty(frameLogPathVo.getPath())) { + throw new Exception("存储路径为空"); + } LoginUser loginUser = SecurityUtils.getLoginUser(); String podName = loginUser.getUsername().toLowerCase()+"-"+frameLogPathVo.getPath().split("/")[2]+ "-tensorboard-pod"; - Integer podPort = k8sClientUtil.createPodWithSubPath(podName, StringUtils.isEmpty(frameLogPathVo.getNamespace()) ? "default" : frameLogPathVo.getNamespace(), port, mountPath, frameLogPathVo.getPath(), frameLogPathVo.getPvcName(), image); -// Integer podPort = k8sClientUtil.createPodWithSubPath(podName, StringUtils.isEmpty(frameLogPathVo.getNamespace()) ? "default" : frameLogPathVo.getNamespace(), port, mountPath, frameLogPathVo.getPath(), image); +// Integer podPort = k8sClientUtil.createPodWithSubPath(podName, StringUtils.isEmpty(frameLogPathVo.getNamespace()) ? "default" : frameLogPathVo.getNamespace(), port, mountPath, frameLogPathVo.getPath(), frameLogPathVo.getPvcName(), image); + Integer podPort = k8sClientUtil.createPodWithSubPath(podName, StringUtils.isEmpty(frameLogPathVo.getNamespace()) ? "default" : frameLogPathVo.getNamespace(), port, mountPath, frameLogPathVo.getPath(), image); redisService.setCacheObject(podName, masterIp + ":" + podPort); return masterIp + ":" + podPort; } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/ConvertUtil.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/ConvertUtil.java index a4195a17..95bb18cd 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/ConvertUtil.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/ConvertUtil.java @@ -158,5 +158,40 @@ public class ConvertUtil { } return sb.toString(); } + + private static String toSnakeCase(String camelCase) { + return camelCase.replaceAll( + String.format("%s|%s|%s", + "(?<=[a-z])(?=[A-Z])", // 小写字母后接大写字母 + "(?<=[^A-Z])(?=[A-Z][a-z])", // 非大写字母后接大写字母开头的小写字母 + "(?<=[A-Z])(?=[A-Z][a-z][0-9])" // 大写字母后接另一个大写字母开头的小写字母和数字(可选,根据需求调整) + ), + "_" + ).toLowerCase(); + } + + public static Map compareObjects(Object obj1, Object obj2) throws IllegalAccessException { + Map differences = new HashMap<>(); + + // Get the class of the first object + Class clazz = obj1.getClass(); + + // Iterate over all fields of the class + for (Field field : clazz.getDeclaredFields()) { + // Set the fields to be accessible + field.setAccessible(true); + + // Get the field value from both objects + Object value1 = field.get(obj1); + Object value2 = field.get(obj2); + + // Compare the field values + if ((value1 !=null && !value1.equals(value2)) || (value2 !=null && !value2.equals(value1))) { + differences.put(toSnakeCase(field.getName()), "Field " + field.getName() + " differs: " + value1 + " vs " + value2); + } + } + + return differences; + } } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/K8sClientUtil.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/K8sClientUtil.java index ce0b2817..14d4c795 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/K8sClientUtil.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/K8sClientUtil.java @@ -36,8 +36,8 @@ import java.util.*; @Component public class K8sClientUtil { -// @Value("${jupyter.hostPath}") -// private String hostPath; + @Value("${jupyter.hostPath}") + private String hostPath; private String http; private String token; @@ -316,8 +316,8 @@ public class K8sClientUtil { * @param image 镜像 * @return 创建成功的pod,的nodePort端口 */ - public Integer createPodWithSubPath(String podName, String namespace, Integer port, String mountPath, String subPath, String pvcName, String image) { -// public Integer createPodWithSubPath(String podName, String namespace, Integer port, String mountPath, String subPath, String image) { +// public Integer createPodWithSubPath(String podName, String namespace, Integer port, String mountPath, String subPath, String pvcName, String image) { + public Integer createPodWithSubPath(String podName, String namespace, Integer port, String mountPath, String subPath, String image) { Map selector = new LinkedHashMap<>(); selector.put("k8s-jupyter", podName); @@ -341,6 +341,9 @@ public class K8sClientUtil { } } + int lastIndex = hostPath.lastIndexOf('/'); + String newPath = hostPath.substring(0, lastIndex); + V1Pod pod = new V1PodBuilder() .withNewMetadata() .withName(podName) @@ -368,8 +371,8 @@ public class K8sClientUtil { .endContainer() .addNewVolume() .withName("workspace") -// .withHostPath(new V1HostPathVolumeSource().path(hostPath).type("DirectoryOrCreate")) - .withPersistentVolumeClaim(new V1PersistentVolumeClaimVolumeSource().claimName(pvcName)) + .withHostPath(new V1HostPathVolumeSource().path(newPath).type("DirectoryOrCreate")) +// .withPersistentVolumeClaim(new V1PersistentVolumeClaimVolumeSource().claimName(pvcName)) .endVolume() .withTerminationGracePeriodSeconds(14400L) .endSpec() @@ -451,25 +454,25 @@ public class K8sClientUtil { } } - // 配置卷和卷挂载 -// List volumeMounts = new ArrayList<>(); -// volumeMounts.add(new V1VolumeMount().name("workspace").mountPath("/opt/notebooks")); -// volumeMounts.add(new V1VolumeMount().name("data").mountPath("/opt/dataset").subPath(datasetPath).readOnly(true)); -// volumeMounts.add(new V1VolumeMount().name("data").mountPath("/opt/model").subPath(modelPath).readOnly(true)); -// -// List volumes = new ArrayList<>(); -// volumes.add(new V1Volume().name("workspace").hostPath(new V1HostPathVolumeSource().path(hostPath + "/" + podName + "/notebooks").type("DirectoryOrCreate"))); -// volumes.add(new V1Volume().name("data").hostPath(new V1HostPathVolumeSource().path(hostPath).type("DirectoryOrCreate"))); - // 配置卷和卷挂载 List volumeMounts = new ArrayList<>(); - volumeMounts.add(new V1VolumeMount().name("workspace").mountPath(mountPath)); - volumeMounts.add(new V1VolumeMount().name("minio-pvc").mountPath("/opt/data").subPath(datasetPath).readOnly(true)); - volumeMounts.add(new V1VolumeMount().name("minio-pvc").mountPath("/opt/model").subPath(modelPath).readOnly(true)); + volumeMounts.add(new V1VolumeMount().name("workspace").mountPath("/opt/notebooks")); + volumeMounts.add(new V1VolumeMount().name("data").mountPath("/opt/dataset").subPath(datasetPath).readOnly(true)); + volumeMounts.add(new V1VolumeMount().name("data").mountPath("/opt/model").subPath(modelPath).readOnly(true)); List volumes = new ArrayList<>(); - volumes.add(new V1Volume().name("workspace").persistentVolumeClaim(new V1PersistentVolumeClaimVolumeSource().claimName(pvc.getMetadata().getName()))); - volumes.add(new V1Volume().name("minio-pvc").persistentVolumeClaim(new V1PersistentVolumeClaimVolumeSource().claimName(dataPvcName))); + volumes.add(new V1Volume().name("workspace").hostPath(new V1HostPathVolumeSource().path(hostPath + "/" + podName + "/notebooks").type("DirectoryOrCreate"))); + volumes.add(new V1Volume().name("data").hostPath(new V1HostPathVolumeSource().path(hostPath).type("DirectoryOrCreate"))); + + // 配置卷和卷挂载 +// List volumeMounts = new ArrayList<>(); +// volumeMounts.add(new V1VolumeMount().name("workspace").mountPath(mountPath)); +// volumeMounts.add(new V1VolumeMount().name("minio-pvc").mountPath("/opt/data").subPath(datasetPath).readOnly(true)); +// volumeMounts.add(new V1VolumeMount().name("minio-pvc").mountPath("/opt/model").subPath(modelPath).readOnly(true)); +// +// List volumes = new ArrayList<>(); +// volumes.add(new V1Volume().name("workspace").persistentVolumeClaim(new V1PersistentVolumeClaimVolumeSource().claimName(pvc.getMetadata().getName()))); +// volumes.add(new V1Volume().name("minio-pvc").persistentVolumeClaim(new V1PersistentVolumeClaimVolumeSource().claimName(dataPvcName))); //配置资源 diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/MinioUtil.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/MinioUtil.java index 7bd5951b..f311065a 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/MinioUtil.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/MinioUtil.java @@ -394,21 +394,24 @@ public class MinioUtil { for (Result result : results) { Item item = result.get(); String objectName = item.objectName(); - InputStream objectStream = minioClient.getObject( - GetObjectArgs.builder().bucket(bucketName).object(objectName).build()); - - // Create a zip entry for each object - ZipEntry zipEntry = new ZipEntry(objectName); - zip.putNextEntry(zipEntry); - - // Write object data to zip stream - byte[] buffer = new byte[1024]; - int bytesRead; - while ((bytesRead = objectStream.read(buffer)) != -1) { - zip.write(buffer, 0, bytesRead); + if (!objectName.contains(".git")) { + InputStream objectStream = minioClient.getObject( + GetObjectArgs.builder().bucket(bucketName).object(objectName).build()); + + + // Create a zip entry for each object + ZipEntry zipEntry = new ZipEntry(objectName); + zip.putNextEntry(zipEntry); + + // Write object data to zip stream + byte[] buffer = new byte[1024]; + int bytesRead; + while ((bytesRead = objectStream.read(buffer)) != -1) { + zip.write(buffer, 0, bytesRead); + } + zip.closeEntry(); + objectStream.close(); } - zip.closeEntry(); - objectStream.close(); } zip.finish(); diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/FrameLogPathVo.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/FrameLogPathVo.java index e4c8bd26..da32b84d 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/FrameLogPathVo.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/FrameLogPathVo.java @@ -10,7 +10,7 @@ public class FrameLogPathVo implements Serializable { String path; String namespace; - String pvcName; +// String pvcName; public String getPath() { return path; } @@ -27,11 +27,11 @@ public class FrameLogPathVo implements Serializable { this.namespace = namespace; } - public String getPvcName() { - return pvcName; - } - - public void setPvcName(String pvcName) { - this.pvcName = pvcName; - } +// public String getPvcName() { +// return pvcName; +// } +// +// public void setPvcName(String pvcName) { +// this.pvcName = pvcName; +// } } diff --git a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ExperimentInsDaoMapper.xml b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ExperimentInsDaoMapper.xml index 8d22dc41..3ee4bd28 100644 --- a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ExperimentInsDaoMapper.xml +++ b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ExperimentInsDaoMapper.xml @@ -275,10 +275,10 @@ - insert into experiment_ins (id, experiment_id, argo_ins_name, argo_ins_ns, status, nodes_status, nodes_result, nodes_logs, global_param,metric_record, start_time, finish_time, create_by, create_time, update_by, update_time, state) + insert into experiment_ins (id, experiment_id, argo_ins_name, argo_ins_ns, status, nodes_status, nodes_result, nodes_logs, global_param,metric_record, start_time, finish_time, create_by, create_time, update_by, update_time, state, metric_value) values - (#{item.id}, #{item.experimentId}, #{item.argoInsName}, #{item.argoInsNs}, #{item.status}, #{item.nodesStatus}, #{item.nodesResult}, #{item.nodesLogs}, #{item.globalParam},#{item.metricRecord}, #{item.startTime}, #{item.finishTime}, #{item.createBy}, #{item.createTime}, #{item.updateBy}, #{item.updateTime}, #{item.state}) + (#{item.id}, #{item.experimentId}, #{item.argoInsName}, #{item.argoInsNs}, #{item.status}, #{item.nodesStatus}, #{item.nodesResult}, #{item.nodesLogs}, #{item.globalParam},#{item.metricRecord}, #{item.startTime}, #{item.finishTime}, #{item.createBy}, #{item.createTime}, #{item.updateBy}, #{item.updateTime}, #{item.state}, #{item.metricValue}) ON DUPLICATE KEY UPDATE experiment_id = VALUES(experiment_id), @@ -296,7 +296,8 @@ create_time = VALUES(create_time), update_by = VALUES(update_by), update_time = VALUES(update_time), - state = VALUES(state); + state = VALUES(state), + metric_value = VALUES(metric_value);