From d1c35e4bee525224c69ee2925d967e44ab123a7c Mon Sep 17 00:00:00 2001 From: chenzhihang <709011834@qq.com> Date: Mon, 21 Oct 2024 16:21:29 +0800 Subject: [PATCH 01/16] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E6=9C=8D=E5=8A=A1?= =?UTF-8?q?=E7=89=88=E6=9C=AC=E5=AF=B9=E6=AF=94=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../controller/service/ServiceController.java | 6 +++++ .../service/impl/ServiceServiceImpl.java | 23 ++++++++++++++++++ .../com/ruoyi/platform/utils/ConvertUtil.java | 24 +++++++++++++++++++ 3 files changed, 53 insertions(+) diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/service/ServiceController.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/service/ServiceController.java index 7d973bae..3eb28e4c 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/service/ServiceController.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/service/ServiceController.java @@ -89,6 +89,12 @@ public class ServiceController extends BaseController { return genericsSuccess(serviceService.getServiceVersion(id)); } + @GetMapping("serviceVersionCompare") + @ApiOperation("服务版本版本对比") + public GenericsAjaxResult> serviceVersionCompare(@RequestParam("id1") Long id1, @RequestParam("id2") Long id2) throws IllegalAccessException { + return genericsSuccess(serviceService.serviceVersionCompare(id1, id2)); + } + @GetMapping("/serviceVersionList/{id}") @ApiOperation("查询服务版本列表") public GenericsAjaxResult> serviceVersionList(@PathVariable("id") Long id) { diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ServiceServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ServiceServiceImpl.java index 11b30f45..36e0ef08 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ServiceServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ServiceServiceImpl.java @@ -7,6 +7,7 @@ import com.ruoyi.platform.constant.Constant; import com.ruoyi.platform.domain.ServiceVersion; import com.ruoyi.platform.mapper.ServiceDao; import com.ruoyi.platform.service.ServiceService; +import com.ruoyi.platform.utils.ConvertUtil; import com.ruoyi.platform.utils.HttpUtils; import com.ruoyi.platform.utils.JacksonUtil; import com.ruoyi.platform.vo.serviceVos.ServiceCodeConfigVo; @@ -157,6 +158,28 @@ public class ServiceServiceImpl implements ServiceService { return serviceVersionVo; } + @Override + public Map serviceVersionCompare(Long id1, Long id2) throws IllegalAccessException { + HashMap result = new HashMap<>(); + + ServiceVersion serviceVersion1 = serviceDao.getServiceVersionById(id1); + ServiceVersion serviceVersion2 = serviceDao.getServiceVersionById(id2); + com.ruoyi.platform.domain.Service service = serviceDao.getServiceById(serviceVersion1.getServiceId()); + + ServiceVersionVo serviceVersionVo1 = getServiceVersionVo(serviceVersion1); + ServiceVersionVo serviceVersionVo2 = getServiceVersionVo(serviceVersion1); + serviceVersionVo1.setServiceName(service.getServiceName()); + serviceVersionVo2.setServiceName(service.getServiceName()); + + Map compareMap = ConvertUtil.compareObjects(serviceVersion1, serviceVersion2); + + result.put("version1", serviceVersionVo1); + result.put("version2", serviceVersionVo2); + result.put("differences", compareMap); + + return result; + } + @Override public String deleteService(Long id) { com.ruoyi.platform.domain.Service service = serviceDao.getServiceById(id); diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/ConvertUtil.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/ConvertUtil.java index a4195a17..52c89dd3 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/ConvertUtil.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/ConvertUtil.java @@ -158,5 +158,29 @@ public class ConvertUtil { } return sb.toString(); } + + public static Map compareObjects(Object obj1, Object obj2) throws IllegalAccessException { + Map differences = new HashMap<>(); + + // Get the class of the first object + Class clazz = obj1.getClass(); + + // Iterate over all fields of the class + for (Field field : clazz.getDeclaredFields()) { + // Set the fields to be accessible + field.setAccessible(true); + + // Get the field value from both objects + Object value1 = field.get(obj1); + Object value2 = field.get(obj2); + + // Compare the field values + if ((value1 !=null && !value1.equals(value2)) || (value2 !=null && !value2.equals(value1))) { + differences.put(field.getName(), "Field " + field.getName() + " differs: " + value1 + " vs " + value2); + } + } + + return differences; + } } From 957c38228a3509e7acf334f8608dcab4f0d6d7fd Mon Sep 17 00:00:00 2001 From: chenzhihang <709011834@qq.com> Date: Mon, 21 Oct 2024 16:21:36 +0800 Subject: [PATCH 02/16] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E6=9C=8D=E5=8A=A1?= =?UTF-8?q?=E7=89=88=E6=9C=AC=E5=AF=B9=E6=AF=94=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/java/com/ruoyi/platform/service/ServiceService.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ServiceService.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ServiceService.java index 6ce6bc6a..5c69f257 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ServiceService.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ServiceService.java @@ -28,6 +28,8 @@ public interface ServiceService { ServiceVersionVo getServiceVersion(Long id); + Map serviceVersionCompare(Long id1, Long id2) throws IllegalAccessException; + String deleteService(Long id); String deleteServiceVersion(Long id); From 0b5756e1bb2aec14c61bd2a7607f223afa674832 Mon Sep 17 00:00:00 2001 From: chenzhihang <709011834@qq.com> Date: Mon, 21 Oct 2024 16:59:40 +0800 Subject: [PATCH 03/16] =?UTF-8?q?=E5=AE=9E=E9=AA=8C=E5=AE=9E=E4=BE=8B?= =?UTF-8?q?=E6=88=90=E5=8A=9F=E8=BF=90=E8=A1=8C=E7=BB=93=E6=9D=9F=E5=90=8E?= =?UTF-8?q?=E8=AE=B0=E5=BD=95=E6=8C=87=E6=A0=87=E6=95=B0=E6=8D=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../mapper/managementPlatform/ExperimentInsDaoMapper.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ExperimentInsDaoMapper.xml b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ExperimentInsDaoMapper.xml index 8d22dc41..d23de89a 100644 --- a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ExperimentInsDaoMapper.xml +++ b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ExperimentInsDaoMapper.xml @@ -296,7 +296,8 @@ create_time = VALUES(create_time), update_by = VALUES(update_by), update_time = VALUES(update_time), - state = VALUES(state); + state = VALUES(state), + metric_value = VALUES(metric_value); From 1ec6bec29d4ef45cfed087610d6086612ce2063c Mon Sep 17 00:00:00 2001 From: chenzhihang <709011834@qq.com> Date: Mon, 21 Oct 2024 17:14:58 +0800 Subject: [PATCH 04/16] =?UTF-8?q?=E5=AE=9E=E9=AA=8C=E5=AE=9E=E4=BE=8B?= =?UTF-8?q?=E6=88=90=E5=8A=9F=E8=BF=90=E8=A1=8C=E7=BB=93=E6=9D=9F=E5=90=8E?= =?UTF-8?q?=E8=AE=B0=E5=BD=95=E6=8C=87=E6=A0=87=E6=95=B0=E6=8D=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ExperimentInstanceStatusTask.java | 38 ++++++++++++------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/scheduling/ExperimentInstanceStatusTask.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/scheduling/ExperimentInstanceStatusTask.java index eaf6ee12..64f43582 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/scheduling/ExperimentInstanceStatusTask.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/scheduling/ExperimentInstanceStatusTask.java @@ -49,28 +49,38 @@ public class ExperimentInstanceStatusTask { //运行成功的实验实例记录指标数值 if (Constant.Succeeded.equals(experimentIns.getStatus())) { Map metricRecord = JacksonUtil.parseJSONStr2Map(experimentIns.getMetricRecord()); - List> trainMetricRecord = (List>) metricRecord.get("train"); - List> evaluateMetricRecord = (List>) metricRecord.get("evaluate"); + List> trainMetricRecords = (List>) metricRecord.get("train"); + List> evaluateMetricRecords = (List>) metricRecord.get("evaluate"); HashMap metricValue = new HashMap<>(); HashMap trainMetricValue = new HashMap<>(); HashMap evaluateMetricValue = new HashMap<>(); - if (trainMetricRecord != null && !trainMetricRecord.isEmpty()) { - String runId = (String) trainMetricRecord.get(0).get("run_id"); - List expTrainInfos = aimService.getExpTrainInfos1(true, experimentIns.getExperimentId(), runId); - for (InsMetricInfoVo expTrainInfo : expTrainInfos) { - Map metrics = expTrainInfo.getMetrics(); - trainMetricValue.putAll(metrics); + if (trainMetricRecords != null && !trainMetricRecords.isEmpty()) { + for (Map trainMetricRecord : trainMetricRecords) { + String taskId = (String) trainMetricRecord.get("task_id"); + if (taskId.startsWith("model-export")) { + String runId = (String) trainMetricRecord.get("run_id"); + List expTrainInfos = aimService.getExpTrainInfos1(true, experimentIns.getExperimentId(), runId); + for (InsMetricInfoVo expTrainInfo : expTrainInfos) { + Map metrics = expTrainInfo.getMetrics(); + trainMetricValue.putAll(metrics); + } + } } } - if (evaluateMetricRecord != null && !evaluateMetricRecord.isEmpty()) { - String runId = (String) evaluateMetricRecord.get(0).get("run_id"); - List expTrainInfos = aimService.getExpTrainInfos1(false, experimentIns.getExperimentId(), runId); - for (InsMetricInfoVo expTrainInfo : expTrainInfos) { - Map metrics = expTrainInfo.getMetrics(); - evaluateMetricValue.putAll(metrics); + if (evaluateMetricRecords != null && !evaluateMetricRecords.isEmpty()) { + for (Map evaluateMetricRecord : evaluateMetricRecords) { + String taskId = (String) evaluateMetricRecord.get("task_id"); + if (taskId.startsWith("model-export")) { + String runId = (String) evaluateMetricRecord.get("run_id"); + List expTrainInfos = aimService.getExpTrainInfos1(false, experimentIns.getExperimentId(), runId); + for (InsMetricInfoVo expTrainInfo : expTrainInfos) { + Map metrics = expTrainInfo.getMetrics(); + evaluateMetricValue.putAll(metrics); + } + } } } metricValue.put("train", trainMetricValue); From 803eea2054f7e9552c39c63fdd6721fe8c0886c4 Mon Sep 17 00:00:00 2001 From: chenzhihang <709011834@qq.com> Date: Mon, 21 Oct 2024 17:39:39 +0800 Subject: [PATCH 05/16] =?UTF-8?q?=E5=AE=9E=E9=AA=8C=E5=AE=9E=E4=BE=8B?= =?UTF-8?q?=E6=88=90=E5=8A=9F=E8=BF=90=E8=A1=8C=E7=BB=93=E6=9D=9F=E5=90=8E?= =?UTF-8?q?=E8=AE=B0=E5=BD=95=E6=8C=87=E6=A0=87=E6=95=B0=E6=8D=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../platform/scheduling/ExperimentInstanceStatusTask.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/scheduling/ExperimentInstanceStatusTask.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/scheduling/ExperimentInstanceStatusTask.java index 64f43582..dad643c6 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/scheduling/ExperimentInstanceStatusTask.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/scheduling/ExperimentInstanceStatusTask.java @@ -59,7 +59,7 @@ public class ExperimentInstanceStatusTask { if (trainMetricRecords != null && !trainMetricRecords.isEmpty()) { for (Map trainMetricRecord : trainMetricRecords) { String taskId = (String) trainMetricRecord.get("task_id"); - if (taskId.startsWith("model-export")) { + if (taskId.startsWith("model-train")) { String runId = (String) trainMetricRecord.get("run_id"); List expTrainInfos = aimService.getExpTrainInfos1(true, experimentIns.getExperimentId(), runId); for (InsMetricInfoVo expTrainInfo : expTrainInfos) { @@ -73,7 +73,7 @@ public class ExperimentInstanceStatusTask { if (evaluateMetricRecords != null && !evaluateMetricRecords.isEmpty()) { for (Map evaluateMetricRecord : evaluateMetricRecords) { String taskId = (String) evaluateMetricRecord.get("task_id"); - if (taskId.startsWith("model-export")) { + if (taskId.startsWith("model-evaluate")) { String runId = (String) evaluateMetricRecord.get("run_id"); List expTrainInfos = aimService.getExpTrainInfos1(false, experimentIns.getExperimentId(), runId); for (InsMetricInfoVo expTrainInfo : expTrainInfos) { From 2568d651e12ac3bc28109e18f7c772e12684acbc Mon Sep 17 00:00:00 2001 From: chenzhihang <709011834@qq.com> Date: Mon, 21 Oct 2024 18:04:04 +0800 Subject: [PATCH 06/16] =?UTF-8?q?=E5=AE=9E=E9=AA=8C=E5=AE=9E=E4=BE=8B?= =?UTF-8?q?=E6=88=90=E5=8A=9F=E8=BF=90=E8=A1=8C=E7=BB=93=E6=9D=9F=E5=90=8E?= =?UTF-8?q?=E8=AE=B0=E5=BD=95=E6=8C=87=E6=A0=87=E6=95=B0=E6=8D=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../mapper/managementPlatform/ExperimentInsDaoMapper.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ExperimentInsDaoMapper.xml b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ExperimentInsDaoMapper.xml index d23de89a..50d7bbc7 100644 --- a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ExperimentInsDaoMapper.xml +++ b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ExperimentInsDaoMapper.xml @@ -275,10 +275,10 @@ - insert into experiment_ins (id, experiment_id, argo_ins_name, argo_ins_ns, status, nodes_status, nodes_result, nodes_logs, global_param,metric_record, start_time, finish_time, create_by, create_time, update_by, update_time, state) + insert into experiment_ins (id, experiment_id, argo_ins_name, argo_ins_ns, status, nodes_status, nodes_result, nodes_logs, global_param,metric_record, start_time, finish_time, create_by, create_time, update_by, update_time, state, metric_value) values - (#{item.id}, #{item.experimentId}, #{item.argoInsName}, #{item.argoInsNs}, #{item.status}, #{item.nodesStatus}, #{item.nodesResult}, #{item.nodesLogs}, #{item.globalParam},#{item.metricRecord}, #{item.startTime}, #{item.finishTime}, #{item.createBy}, #{item.createTime}, #{item.updateBy}, #{item.updateTime}, #{item.state}) + (#{item.id}, #{item.experimentId}, #{item.argoInsName}, #{item.argoInsNs}, #{item.status}, #{item.nodesStatus}, #{item.nodesResult}, #{item.nodesLogs}, #{item.globalParam},#{item.metricRecord}, #{item.startTime}, #{item.finishTime}, #{item.createBy}, #{item.createTime}, #{item.updateBy}, #{item.updateTime}, #{item.state}, #{item.metric_value}) ON DUPLICATE KEY UPDATE experiment_id = VALUES(experiment_id), From 7f9cb9f3abb6e82c39aad0a2ea6bdef7af0636c3 Mon Sep 17 00:00:00 2001 From: chenzhihang <709011834@qq.com> Date: Mon, 21 Oct 2024 18:11:14 +0800 Subject: [PATCH 07/16] =?UTF-8?q?=E5=AE=9E=E9=AA=8C=E5=AE=9E=E4=BE=8B?= =?UTF-8?q?=E6=88=90=E5=8A=9F=E8=BF=90=E8=A1=8C=E7=BB=93=E6=9D=9F=E5=90=8E?= =?UTF-8?q?=E8=AE=B0=E5=BD=95=E6=8C=87=E6=A0=87=E6=95=B0=E6=8D=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../mapper/managementPlatform/ExperimentInsDaoMapper.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ExperimentInsDaoMapper.xml b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ExperimentInsDaoMapper.xml index 50d7bbc7..3ee4bd28 100644 --- a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ExperimentInsDaoMapper.xml +++ b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ExperimentInsDaoMapper.xml @@ -278,7 +278,7 @@ insert into experiment_ins (id, experiment_id, argo_ins_name, argo_ins_ns, status, nodes_status, nodes_result, nodes_logs, global_param,metric_record, start_time, finish_time, create_by, create_time, update_by, update_time, state, metric_value) values - (#{item.id}, #{item.experimentId}, #{item.argoInsName}, #{item.argoInsNs}, #{item.status}, #{item.nodesStatus}, #{item.nodesResult}, #{item.nodesLogs}, #{item.globalParam},#{item.metricRecord}, #{item.startTime}, #{item.finishTime}, #{item.createBy}, #{item.createTime}, #{item.updateBy}, #{item.updateTime}, #{item.state}, #{item.metric_value}) + (#{item.id}, #{item.experimentId}, #{item.argoInsName}, #{item.argoInsNs}, #{item.status}, #{item.nodesStatus}, #{item.nodesResult}, #{item.nodesLogs}, #{item.globalParam},#{item.metricRecord}, #{item.startTime}, #{item.finishTime}, #{item.createBy}, #{item.createTime}, #{item.updateBy}, #{item.updateTime}, #{item.state}, #{item.metricValue}) ON DUPLICATE KEY UPDATE experiment_id = VALUES(experiment_id), From 3b75b5f3879861f84fe1bc41cfd0ee5baa9b999d Mon Sep 17 00:00:00 2001 From: chenzhihang <709011834@qq.com> Date: Tue, 22 Oct 2024 08:38:01 +0800 Subject: [PATCH 08/16] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=AE=9E=E9=AA=8C?= =?UTF-8?q?=E5=AF=B9=E6=AF=94=E5=88=86=E9=A1=B5=E6=9F=A5=E8=AF=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../scheduling/ExperimentInstanceStatusTask.java | 4 ++-- .../main/java/com/ruoyi/platform/service/AimService.java | 4 ++-- .../com/ruoyi/platform/service/impl/AimServiceImpl.java | 7 ++++++- .../platform/service/impl/ExperimentInsServiceImpl.java | 9 --------- .../ruoyi/platform/service/impl/ModelsServiceImpl.java | 4 ++-- 5 files changed, 12 insertions(+), 16 deletions(-) diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/scheduling/ExperimentInstanceStatusTask.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/scheduling/ExperimentInstanceStatusTask.java index dad643c6..a5298747 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/scheduling/ExperimentInstanceStatusTask.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/scheduling/ExperimentInstanceStatusTask.java @@ -61,7 +61,7 @@ public class ExperimentInstanceStatusTask { String taskId = (String) trainMetricRecord.get("task_id"); if (taskId.startsWith("model-train")) { String runId = (String) trainMetricRecord.get("run_id"); - List expTrainInfos = aimService.getExpTrainInfos1(true, experimentIns.getExperimentId(), runId); + List expTrainInfos = aimService.getExpInfos1(true, experimentIns.getExperimentId(), runId); for (InsMetricInfoVo expTrainInfo : expTrainInfos) { Map metrics = expTrainInfo.getMetrics(); trainMetricValue.putAll(metrics); @@ -75,7 +75,7 @@ public class ExperimentInstanceStatusTask { String taskId = (String) evaluateMetricRecord.get("task_id"); if (taskId.startsWith("model-evaluate")) { String runId = (String) evaluateMetricRecord.get("run_id"); - List expTrainInfos = aimService.getExpTrainInfos1(false, experimentIns.getExperimentId(), runId); + List expTrainInfos = aimService.getExpInfos1(false, experimentIns.getExperimentId(), runId); for (InsMetricInfoVo expTrainInfo : expTrainInfos) { Map metrics = expTrainInfo.getMetrics(); evaluateMetricValue.putAll(metrics); diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AimService.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AimService.java index 3e2ccb77..699d0036 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AimService.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AimService.java @@ -10,10 +10,10 @@ public interface AimService { List getExpTrainInfos(Integer experimentId, String offset, int limit) throws Exception; - List getExpTrainInfos1(boolean isTrain, Integer experimentId, String runId) throws Exception; - List getExpEvaluateInfos(Integer experimentId, String offset, int limit) throws Exception; + List getExpInfos1(boolean isTrain, Integer experimentId, String runId) throws Exception; + String getExpMetrics(List runIds) throws Exception; HashMap queryMetricsParams(String runId) throws Exception; diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AimServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AimServiceImpl.java index a2b21202..3bae116b 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AimServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AimServiceImpl.java @@ -11,6 +11,7 @@ import com.ruoyi.platform.utils.JsonUtils; import com.ruoyi.platform.vo.InsMetricInfoVo; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Value; +import org.springframework.data.domain.PageRequest; import org.springframework.stereotype.Service; import javax.annotation.Resource; @@ -145,9 +146,13 @@ public class AimServiceImpl implements AimService { return aimRunInfoList; } +// private List getExpInfos(Integer experimentId, int page, int size){ +// PageRequest pageRequest = PageRequest.of(page,size); +// +// } @Override - public List getExpTrainInfos1(boolean isTrain, Integer experimentId, String runId) throws Exception { + public List getExpInfos1(boolean isTrain, Integer experimentId, String runId) throws Exception { String encodedUrlString = URLEncoder.encode("run.id==\"" + runId + "\"", "UTF-8"); String url = aimProxyUrl + "/api/runs/search/run?query=" + encodedUrlString; String s = HttpUtils.sendGet(url, null); diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentInsServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentInsServiceImpl.java index 1319f0a9..b7ff69b4 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentInsServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentInsServiceImpl.java @@ -172,15 +172,6 @@ public class ExperimentInsServiceImpl implements ExperimentInsService { public Page queryByPage(ExperimentIns experimentIns, PageRequest pageRequest) throws IOException { long total = this.experimentInsDao.count(experimentIns); List experimentInsList = this.experimentInsDao.queryAllByLimit(experimentIns, pageRequest); -// if (experimentInsList!=null && experimentInsList.size()>0) { -// for (ExperimentIns ins : experimentInsList) { -// //如果实验实例不为空或者 -// if (ins != null && StringUtils.isEmpty(ins.getStatus())) { -// ins = this.queryStatusFromArgo(ins); -// this.update(ins); -// } -// } -// } return new PageImpl<>(experimentInsList, pageRequest, total); } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java index f69b7359..ff053e78 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java @@ -1300,7 +1300,7 @@ public class ModelsServiceImpl implements ModelsService { for (int i = 0; i < trainMetrics.size(); i++) { JSONObject jsonObject = trainMetrics.getJSONObject(i); String runId = jsonObject.getString("run_id"); - List expTrainInfos = aimsService.getExpTrainInfos1(true, modelMetaVo.getTrainTask().getExperimentId(), runId); + List expTrainInfos = aimsService.getExpInfos1(true, modelMetaVo.getTrainTask().getExperimentId(), runId); for (InsMetricInfoVo expTrainInfo : expTrainInfos) { Map metrics1 = expTrainInfo.getMetrics(); train.putAll(metrics1); @@ -1322,7 +1322,7 @@ public class ModelsServiceImpl implements ModelsService { for (int i = 0; i < testMetrics.size(); i++) { JSONObject jsonObject = testMetrics.getJSONObject(i); String runId = jsonObject.getString("run_id"); - List expTestInfos = aimsService.getExpTrainInfos1(false, modelMetaVo.getTrainTask().getExperimentId(), runId); + List expTestInfos = aimsService.getExpInfos1(false, modelMetaVo.getTrainTask().getExperimentId(), runId); for (InsMetricInfoVo expTestInfo : expTestInfos) { Map metrics1 = expTestInfo.getMetrics(); evaluate.putAll(metrics1); From a6f54528d9ba1f606e0fe3b540e25084753075de Mon Sep 17 00:00:00 2001 From: chenzhihang <709011834@qq.com> Date: Tue, 22 Oct 2024 08:57:44 +0800 Subject: [PATCH 09/16] =?UTF-8?q?=E5=BC=80=E5=8F=91=E6=94=AF=E6=8C=81juice?= =?UTF-8?q?fs=E5=AD=98=E5=82=A8=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../impl/DatasetVersionServiceImpl.java | 6 +-- .../service/impl/ImageServiceImpl.java | 6 +-- .../service/impl/JupyterServiceImpl.java | 16 +++---- .../service/impl/TensorBoardServiceImpl.java | 14 +++---- .../ruoyi/platform/utils/K8sClientUtil.java | 42 +++++++++---------- .../com/ruoyi/platform/vo/FrameLogPathVo.java | 16 +++---- 6 files changed, 50 insertions(+), 50 deletions(-) diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/DatasetVersionServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/DatasetVersionServiceImpl.java index 441ea786..4a00ddb4 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/DatasetVersionServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/DatasetVersionServiceImpl.java @@ -165,9 +165,9 @@ public class DatasetVersionServiceImpl implements DatasetVersionService { .findFirst() .ifPresent(datasetVersion -> { String url = datasetVersion.getUrl(); -// response.put("path", url); - String path = bucketName + '/' + url.substring(0, url.lastIndexOf('/')); - response.put("path", path); + response.put("path", url); +// String path = bucketName + '/' + url.substring(0, url.lastIndexOf('/')); +// response.put("path", path); }); response.put("content", datasetVersionList); diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ImageServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ImageServiceImpl.java index faacf8dd..92d2bb45 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ImageServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ImageServiceImpl.java @@ -61,10 +61,10 @@ public class ImageServiceImpl implements ImageService { @Resource private MinioService minioService; -// @Value("${minio.dataReleaseBucketName}") -// private String bucketName; - @Value("${harbor.bucketName}") + @Value("${minio.dataReleaseBucketName}") private String bucketName; +// @Value("${harbor.bucketName}") +// private String bucketName; @Value("${harbor.repository}") private String repository; @Value("${harbor.harborUrl}") diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/JupyterServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/JupyterServiceImpl.java index 3f2f7c9f..20e1e916 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/JupyterServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/JupyterServiceImpl.java @@ -90,25 +90,25 @@ public class JupyterServiceImpl implements JupyterService { // 提取数据集,模型信息，得到数据集模型的path Map dataset = JacksonUtil.parseJSONStr2Map(devEnvironment.getDataset()); - String datasetPath = "argo-workflow" + "/" + dataset.get("path"); -// String datasetPath = (String) dataset.get("path"); +// String datasetPath = "argo-workflow" + "/" + dataset.get("path"); + String datasetPath = (String) dataset.get("path"); Map model = JacksonUtil.parseJSONStr2Map(devEnvironment.getModel()); - String modelPath = "argo-workflow" + "/" + model.get("path"); -// String modelPath = (String) model.get("path"); +// String modelPath = "argo-workflow" + "/" + model.get("path"); + String modelPath = (String) model.get("path"); LoginUser loginUser = SecurityUtils.getLoginUser(); //构造pod名称 String podName = loginUser.getUsername().toLowerCase() + "-editor-pod" + "-" + id; //新建编辑器的pvc - String pvcName = loginUser.getUsername().toLowerCase() + "-editor-pvc"; - V1PersistentVolumeClaim pvc = k8sClientUtil.createPvc(namespace, pvcName, storage, storageClassName); +// String pvcName = loginUser.getUsername().toLowerCase() + "-editor-pvc"; +// V1PersistentVolumeClaim pvc = k8sClientUtil.createPvc(namespace, pvcName, storage, storageClassName); //TODO 设置镜像可配置，这里先用默认镜像启动pod // 调用修改后的 createPod 方法，传入额外的参数 - Integer podPort = k8sClientUtil.createConfiguredPod(podName, namespace, port, mountPath, pvc, devEnvironment, minioPvcName, datasetPath, modelPath); -// Integer podPort = k8sClientUtil.createConfiguredPod(podName, namespace, port, mountPath, null, devEnvironment, minioPvcName, datasetPath, modelPath); +// Integer podPort = k8sClientUtil.createConfiguredPod(podName, namespace, port, mountPath, pvc, devEnvironment, minioPvcName, datasetPath, modelPath); + Integer podPort = k8sClientUtil.createConfiguredPod(podName, namespace, port, mountPath, null, devEnvironment, minioPvcName, datasetPath, modelPath); String url = masterIp + ":" + podPort; redisService.setCacheObject(podName, masterIp + ":" + podPort); devEnvironment.setStatus("Pending"); diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/TensorBoardServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/TensorBoardServiceImpl.java index dca3651d..83b87d82 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/TensorBoardServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/TensorBoardServiceImpl.java @@ -59,17 +59,17 @@ public class TensorBoardServiceImpl implements TensorBoardService { @Override public String runTensorBoard(FrameLogPathVo frameLogPathVo) throws Exception { - if (StringUtils.isEmpty(frameLogPathVo.getPath())||StringUtils.isEmpty(frameLogPathVo.getPvcName())){ - throw new Exception("存储路径或存储为空"); - } -// if (StringUtils.isEmpty(frameLogPathVo.getPath())) { -// throw new Exception("存储路径为空"); +// if (StringUtils.isEmpty(frameLogPathVo.getPath())||StringUtils.isEmpty(frameLogPathVo.getPvcName())){ +// throw new Exception("存储路径或存储为空"); // } + if (StringUtils.isEmpty(frameLogPathVo.getPath())) { + throw new Exception("存储路径为空"); + } LoginUser loginUser = SecurityUtils.getLoginUser(); String podName = loginUser.getUsername().toLowerCase()+"-"+frameLogPathVo.getPath().split("/")[2]+ "-tensorboard-pod"; - Integer podPort = k8sClientUtil.createPodWithSubPath(podName, StringUtils.isEmpty(frameLogPathVo.getNamespace()) ? "default" : frameLogPathVo.getNamespace(), port, mountPath, frameLogPathVo.getPath(), frameLogPathVo.getPvcName(), image); -// Integer podPort = k8sClientUtil.createPodWithSubPath(podName, StringUtils.isEmpty(frameLogPathVo.getNamespace()) ? "default" : frameLogPathVo.getNamespace(), port, mountPath, frameLogPathVo.getPath(), image); +// Integer podPort = k8sClientUtil.createPodWithSubPath(podName, StringUtils.isEmpty(frameLogPathVo.getNamespace()) ? "default" : frameLogPathVo.getNamespace(), port, mountPath, frameLogPathVo.getPath(), frameLogPathVo.getPvcName(), image); + Integer podPort = k8sClientUtil.createPodWithSubPath(podName, StringUtils.isEmpty(frameLogPathVo.getNamespace()) ? "default" : frameLogPathVo.getNamespace(), port, mountPath, frameLogPathVo.getPath(), image); redisService.setCacheObject(podName, masterIp + ":" + podPort); return masterIp + ":" + podPort; } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/K8sClientUtil.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/K8sClientUtil.java index 49d56853..024298d8 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/K8sClientUtil.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/K8sClientUtil.java @@ -34,8 +34,8 @@ import java.util.*; @Component public class K8sClientUtil { -// @Value("${jupyter.hostPath}") -// private String hostPath; + @Value("${jupyter.hostPath}") + private String hostPath; private String http; private String token; @@ -314,8 +314,8 @@ public class K8sClientUtil { * @param image 镜像 * @return 创建成功的pod，的nodePort端口 */ - public Integer createPodWithSubPath(String podName, String namespace, Integer port, String mountPath, String subPath, String pvcName, String image) { -// public Integer createPodWithSubPath(String podName, String namespace, Integer port, String mountPath, String subPath, String image) { +// public Integer createPodWithSubPath(String podName, String namespace, Integer port, String mountPath, String subPath, String pvcName, String image) { + public Integer createPodWithSubPath(String podName, String namespace, Integer port, String mountPath, String subPath, String image) { Map selector = new LinkedHashMap<>(); selector.put("k8s-jupyter", podName); @@ -366,8 +366,8 @@ public class K8sClientUtil { .endContainer() .addNewVolume() .withName("workspace") -// .withHostPath(new V1HostPathVolumeSource().path(hostPath).type("DirectoryOrCreate")) - .withPersistentVolumeClaim(new V1PersistentVolumeClaimVolumeSource().claimName(pvcName)) + .withHostPath(new V1HostPathVolumeSource().path(hostPath).type("DirectoryOrCreate")) +// .withPersistentVolumeClaim(new V1PersistentVolumeClaimVolumeSource().claimName(pvcName)) .endVolume() .withTerminationGracePeriodSeconds(14400L) .endSpec() @@ -449,25 +449,25 @@ public class K8sClientUtil { } } - // 配置卷和卷挂载 -// List volumeMounts = new ArrayList<>(); -// volumeMounts.add(new V1VolumeMount().name("workspace").mountPath("/opt/notebooks")); -// volumeMounts.add(new V1VolumeMount().name("data").mountPath("/opt/dataset").subPath(datasetPath).readOnly(true)); -// volumeMounts.add(new V1VolumeMount().name("data").mountPath("/opt/model").subPath(modelPath).readOnly(true)); -// -// List volumes = new ArrayList<>(); -// volumes.add(new V1Volume().name("workspace").hostPath(new V1HostPathVolumeSource().path(hostPath + "/" + podName + "/notebooks").type("DirectoryOrCreate"))); -// volumes.add(new V1Volume().name("data").hostPath(new V1HostPathVolumeSource().path(hostPath).type("DirectoryOrCreate"))); - // 配置卷和卷挂载 List volumeMounts = new ArrayList<>(); - volumeMounts.add(new V1VolumeMount().name("workspace").mountPath(mountPath)); - volumeMounts.add(new V1VolumeMount().name("minio-pvc").mountPath("/opt/data").subPath(datasetPath).readOnly(true)); - volumeMounts.add(new V1VolumeMount().name("minio-pvc").mountPath("/opt/model").subPath(modelPath).readOnly(true)); + volumeMounts.add(new V1VolumeMount().name("workspace").mountPath("/opt/notebooks")); + volumeMounts.add(new V1VolumeMount().name("data").mountPath("/opt/dataset").subPath(datasetPath).readOnly(true)); + volumeMounts.add(new V1VolumeMount().name("data").mountPath("/opt/model").subPath(modelPath).readOnly(true)); List volumes = new ArrayList<>(); - volumes.add(new V1Volume().name("workspace").persistentVolumeClaim(new V1PersistentVolumeClaimVolumeSource().claimName(pvc.getMetadata().getName()))); - volumes.add(new V1Volume().name("minio-pvc").persistentVolumeClaim(new V1PersistentVolumeClaimVolumeSource().claimName(dataPvcName))); + volumes.add(new V1Volume().name("workspace").hostPath(new V1HostPathVolumeSource().path(hostPath + "/" + podName + "/notebooks").type("DirectoryOrCreate"))); + volumes.add(new V1Volume().name("data").hostPath(new V1HostPathVolumeSource().path(hostPath).type("DirectoryOrCreate"))); + + // 配置卷和卷挂载 +// List volumeMounts = new ArrayList<>(); +// volumeMounts.add(new V1VolumeMount().name("workspace").mountPath(mountPath)); +// volumeMounts.add(new V1VolumeMount().name("minio-pvc").mountPath("/opt/data").subPath(datasetPath).readOnly(true)); +// volumeMounts.add(new V1VolumeMount().name("minio-pvc").mountPath("/opt/model").subPath(modelPath).readOnly(true)); +// +// List volumes = new ArrayList<>(); +// volumes.add(new V1Volume().name("workspace").persistentVolumeClaim(new V1PersistentVolumeClaimVolumeSource().claimName(pvc.getMetadata().getName()))); +// volumes.add(new V1Volume().name("minio-pvc").persistentVolumeClaim(new V1PersistentVolumeClaimVolumeSource().claimName(dataPvcName))); //配置资源 diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/FrameLogPathVo.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/FrameLogPathVo.java index e4c8bd26..da32b84d 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/FrameLogPathVo.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/FrameLogPathVo.java @@ -10,7 +10,7 @@ public class FrameLogPathVo implements Serializable { String path; String namespace; - String pvcName; +// String pvcName; public String getPath() { return path; } @@ -27,11 +27,11 @@ public class FrameLogPathVo implements Serializable { this.namespace = namespace; } - public String getPvcName() { - return pvcName; - } - - public void setPvcName(String pvcName) { - this.pvcName = pvcName; - } +// public String getPvcName() { +// return pvcName; +// } +// +// public void setPvcName(String pvcName) { +// this.pvcName = pvcName; +// } } From 6266261b22cc7d1317274c58cbbcba12aa371c31 Mon Sep 17 00:00:00 2001 From: chenzhihang <709011834@qq.com> Date: Tue, 22 Oct 2024 10:07:10 +0800 Subject: [PATCH 10/16] =?UTF-8?q?=E5=AE=9E=E9=AA=8C=E5=AE=9E=E4=BE=8B?= =?UTF-8?q?=E6=88=90=E5=8A=9F=E8=BF=90=E8=A1=8C=E7=BB=93=E6=9D=9F=E5=90=8E?= =?UTF-8?q?=E8=AE=B0=E5=BD=95=E6=8C=87=E6=A0=87=E6=95=B0=E6=8D=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ruoyi/platform/scheduling/ExperimentInstanceStatusTask.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/scheduling/ExperimentInstanceStatusTask.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/scheduling/ExperimentInstanceStatusTask.java index a5298747..283a0288 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/scheduling/ExperimentInstanceStatusTask.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/scheduling/ExperimentInstanceStatusTask.java @@ -65,6 +65,7 @@ public class ExperimentInstanceStatusTask { for (InsMetricInfoVo expTrainInfo : expTrainInfos) { Map metrics = expTrainInfo.getMetrics(); trainMetricValue.putAll(metrics); + trainMetricValue.put("run_hash",expTrainInfo.getRunId()); } } } @@ -79,6 +80,7 @@ public class ExperimentInstanceStatusTask { for (InsMetricInfoVo expTrainInfo : expTrainInfos) { Map metrics = expTrainInfo.getMetrics(); evaluateMetricValue.putAll(metrics); + evaluateMetricValue.put("run_hash",expTrainInfo.getRunId()); } } } From c02a0f01500022ae20e288329c5fe52ec08350b2 Mon Sep 17 00:00:00 2001 From: chenzhihang <709011834@qq.com> Date: Tue, 22 Oct 2024 14:18:18 +0800 Subject: [PATCH 11/16] =?UTF-8?q?=E5=BC=80=E5=8F=91=E6=94=AF=E6=8C=81juice?= =?UTF-8?q?fs=E5=AD=98=E5=82=A8=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- k8s/template-yaml/k8s-7management.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/k8s/template-yaml/k8s-7management.yaml b/k8s/template-yaml/k8s-7management.yaml index 4fcddb15..edc1c621 100644 --- a/k8s/template-yaml/k8s-7management.yaml +++ b/k8s/template-yaml/k8s-7management.yaml @@ -26,10 +26,11 @@ spec: volumeMounts: - name: resource-volume mountPath: /home/resource/ + subPath: mini-model-platform-data volumes: - name: resource-volume - persistentVolumeClaim: - claimName: platform-data-pvc-nfs + hostPath: + path: /platform-data --- apiVersion: v1 kind: Service From 3002e7b0c59a6e5a54b7561a3710aba55e991010 Mon Sep 17 00:00:00 2001 From: chenzhihang <709011834@qq.com> Date: Tue, 22 Oct 2024 15:37:39 +0800 Subject: [PATCH 12/16] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=AE=9E=E9=AA=8C?= =?UTF-8?q?=E5=AF=B9=E6=AF=94=E5=88=86=E9=A1=B5=E6=9F=A5=E8=AF=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../controller/aim/AimController.java | 16 ++-- .../ExperimentInstanceStatusTask.java | 16 ++-- .../ruoyi/platform/service/AimService.java | 2 + .../platform/service/impl/AimServiceImpl.java | 93 ++++++++++++++++++- 4 files changed, 109 insertions(+), 18 deletions(-) diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/aim/AimController.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/aim/AimController.java index 31a6cba1..28632dcb 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/aim/AimController.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/aim/AimController.java @@ -24,19 +24,19 @@ public class AimController extends BaseController { @GetMapping("/getExpTrainInfos/{experiment_id}") @ApiOperation("获取当前实验的模型训练指标信息") @ApiResponse - public GenericsAjaxResult> getExpTrainInfos(@RequestParam(value = "offset", required = false) String offset, - @RequestParam(value = "limit") int limit, - @PathVariable("experiment_id") Integer experimentId) throws Exception { - return genericsSuccess(aimService.getExpTrainInfos(experimentId, offset, limit)); + public GenericsAjaxResult> getExpTrainInfos(@RequestParam(value = "page") int page, + @RequestParam(value = "size") int size, + @PathVariable("experiment_id") Integer experimentId) { + return genericsSuccess(aimService.getExpInfos(true, experimentId, page, size)); } @GetMapping("/getExpEvaluateInfos/{experiment_id}") @ApiOperation("获取当前实验的模型推理指标信息") @ApiResponse - public GenericsAjaxResult> getExpEvaluateInfos(@RequestParam(value = "offset", required = false) String offset, - @RequestParam(value = "limit") int limit, - @PathVariable("experiment_id") Integer experimentId) throws Exception { - return genericsSuccess(aimService.getExpEvaluateInfos(experimentId, offset, limit)); + public GenericsAjaxResult> getExpEvaluateInfos(@RequestParam(value = "page") int page, + @RequestParam(value = "size") int size, + @PathVariable("experiment_id") Integer experimentId) { + return genericsSuccess(aimService.getExpInfos(false, experimentId, page, size)); } @PostMapping("/getExpMetrics") diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/scheduling/ExperimentInstanceStatusTask.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/scheduling/ExperimentInstanceStatusTask.java index 283a0288..59d8bf47 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/scheduling/ExperimentInstanceStatusTask.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/scheduling/ExperimentInstanceStatusTask.java @@ -53,11 +53,12 @@ public class ExperimentInstanceStatusTask { List> evaluateMetricRecords = (List>) metricRecord.get("evaluate"); HashMap metricValue = new HashMap<>(); - HashMap trainMetricValue = new HashMap<>(); - HashMap evaluateMetricValue = new HashMap<>(); + HashMap trainMetricValues = new HashMap<>(); + HashMap evaluateMetricValues = new HashMap<>(); if (trainMetricRecords != null && !trainMetricRecords.isEmpty()) { for (Map trainMetricRecord : trainMetricRecords) { + HashMap trainMetricValue = new HashMap<>(); String taskId = (String) trainMetricRecord.get("task_id"); if (taskId.startsWith("model-train")) { String runId = (String) trainMetricRecord.get("run_id"); @@ -65,14 +66,16 @@ public class ExperimentInstanceStatusTask { for (InsMetricInfoVo expTrainInfo : expTrainInfos) { Map metrics = expTrainInfo.getMetrics(); trainMetricValue.putAll(metrics); - trainMetricValue.put("run_hash",expTrainInfo.getRunId()); + trainMetricValue.put("run_hash", expTrainInfo.getRunId()); } } + trainMetricValues.put(taskId, trainMetricValue); } } if (evaluateMetricRecords != null && !evaluateMetricRecords.isEmpty()) { for (Map evaluateMetricRecord : evaluateMetricRecords) { + HashMap evaluateMetricValue = new HashMap<>(); String taskId = (String) evaluateMetricRecord.get("task_id"); if (taskId.startsWith("model-evaluate")) { String runId = (String) evaluateMetricRecord.get("run_id"); @@ -80,13 +83,14 @@ public class ExperimentInstanceStatusTask { for (InsMetricInfoVo expTrainInfo : expTrainInfos) { Map metrics = expTrainInfo.getMetrics(); evaluateMetricValue.putAll(metrics); - evaluateMetricValue.put("run_hash",expTrainInfo.getRunId()); + evaluateMetricValue.put("run_hash", expTrainInfo.getRunId()); } } + evaluateMetricValues.put(taskId, evaluateMetricValue); } } - metricValue.put("train", trainMetricValue); - metricValue.put("evaluate", evaluateMetricValue); + metricValue.put("train", trainMetricValues); + metricValue.put("evaluate", evaluateMetricValues); experimentIns.setMetricValue(JsonUtils.mapToJson(metricValue)); } experimentIns.setUpdateTime(new Date()); diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AimService.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AimService.java index 699d0036..abcbc645 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AimService.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AimService.java @@ -12,6 +12,8 @@ public interface AimService { List getExpEvaluateInfos(Integer experimentId, String offset, int limit) throws Exception; + List getExpInfos(boolean isTrain, Integer experimentId, int page, int size); + List getExpInfos1(boolean isTrain, Integer experimentId, String runId) throws Exception; String getExpMetrics(List runIds) throws Exception; diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AimServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AimServiceImpl.java index 3bae116b..1bfdb77d 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AimServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AimServiceImpl.java @@ -1,7 +1,10 @@ package com.ruoyi.platform.service.impl; import com.alibaba.fastjson2.JSON; +import com.alibaba.fastjson2.JSONArray; +import com.alibaba.fastjson2.JSONObject; import com.ruoyi.platform.domain.ExperimentIns; +import com.ruoyi.platform.mapper.ExperimentInsDao; import com.ruoyi.platform.service.AimService; import com.ruoyi.platform.service.ExperimentInsService; import com.ruoyi.platform.utils.AIM64EncoderUtil; @@ -24,6 +27,8 @@ import java.util.stream.Collectors; public class AimServiceImpl implements AimService { @Resource private ExperimentInsService experimentInsService; + @Resource + private ExperimentInsDao experimentInsDao; @Value("${aim.url}") private String aimUrl; @@ -146,10 +151,56 @@ public class AimServiceImpl implements AimService { return aimRunInfoList; } -// private List getExpInfos(Integer experimentId, int page, int size){ -// PageRequest pageRequest = PageRequest.of(page,size); -// -// } + public List getExpInfos(boolean isTrain, Integer experimentId, int page, int size) { + PageRequest pageRequest = PageRequest.of(page, size); + ExperimentIns query = new ExperimentIns(); + query.setExperimentId(experimentId); + List experimentInsList = experimentInsDao.queryAllByLimit(query, pageRequest); + if (experimentInsList == null || experimentInsList.size() == 0) { + return new ArrayList<>(); + } + List aimRunInfoList = new ArrayList<>(); + for (ExperimentIns experimentIns : experimentInsList) { + InsMetricInfoVo aimRunInfo = new InsMetricInfoVo(); + aimRunInfo.setExperimentInsId(experimentIns.getId()); + aimRunInfo.setStartTime(experimentIns.getCreateTime()); + aimRunInfo.setStatus(experimentIns.getStatus()); + + //解析参数 + JSONArray params = JSON.parseArray(experimentIns.getGlobalParam()); + HashMap paramsMap = new HashMap<>(); + List paramsNames = new ArrayList<>(); + + for (int i = 0; i < params.size(); i++) { + JSONObject jsonObject = params.getJSONObject(i); + String paramName = jsonObject.getString("param_name"); + String paramValue = jsonObject.getString("param_value"); + paramsMap.put(paramName, paramValue); + paramsNames.add(paramName); + } + aimRunInfo.setParams(paramsMap); + aimRunInfo.setParamsNames(paramsNames); + + //解析数据集 + Map metricRecord = JacksonUtil.parseJSONStr2Map(experimentIns.getMetricRecord()); + if (isTrain) { + aimRunInfo.setDataset(getDataset("train", metricRecord)); + } else { + aimRunInfo.setDataset(getDataset("evaluate", metricRecord)); + } + + //解析指标 + Map metricValue = JacksonUtil.parseJSONStr2Map(experimentIns.getMetricValue()); + if (isTrain) { + setMetricValue("train",metricValue,aimRunInfo); + } else { + setMetricValue("evaluate",metricValue,aimRunInfo); + } + aimRunInfoList.add(aimRunInfo); + } + return aimRunInfoList; + } + @Override public List getExpInfos1(boolean isTrain, Integer experimentId, String runId) throws Exception { @@ -240,6 +291,40 @@ public class AimServiceImpl implements AimService { return aimRunInfoList; } + private List getDataset(String isTrain, Map metricRecord) { + List datasetList = new ArrayList<>(); + List> trainMetricRecords = (List>) metricRecord.get(isTrain); + for (Map trainMetricRecord : trainMetricRecords) { + String taskId = (String) trainMetricRecord.get("task_id"); + if (taskId.startsWith("model-" + isTrain)) { + List> datasets = (List>) trainMetricRecord.get("datasets"); + for (Map dataset : datasets) { + String datasetName = dataset.get("dataset_name") + ":" + dataset.get("dataset_version"); + datasetList.add(datasetName); + } + } + } + return datasetList; + } + + private void setMetricValue(String isTrain, Map metricValue, InsMetricInfoVo aimRunInfo){ + Map metricValues = (Map) metricValue.get(isTrain); + + HashMap metrics = new HashMap<>(); + List metricsNames = new ArrayList<>(); + for (String key : metricValues.keySet()) { + Map valueMap = (Map) metricValues.get(key); + aimRunInfo.setRunId((String) valueMap.get("run_hash")); + + metrics.putAll(valueMap.entrySet().stream().filter(entry -> !entry.getKey().equals("run_hash")).collect(Collectors.toMap( + Map.Entry::getKey, + Map.Entry::getValue + ))); + metricsNames.addAll(valueMap.keySet().stream().filter(entry -> !entry.equals("run_hash")).collect(Collectors.toList())); + } + aimRunInfo.setMetrics(metrics); + aimRunInfo.setMetricsNames(metricsNames); + } private List getTrainDateSet(List> records, String aimrunId) { List datasetList = new ArrayList<>(); From 9fe35a2979496888cf7303167023fa289c098a21 Mon Sep 17 00:00:00 2001 From: chenzhihang <709011834@qq.com> Date: Tue, 22 Oct 2024 16:37:25 +0800 Subject: [PATCH 13/16] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=9C=8D=E5=8A=A1?= =?UTF-8?q?=E5=AF=B9=E6=AF=94bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ruoyi/platform/service/impl/ModelsVersionServiceImpl.java | 4 +--- .../com/ruoyi/platform/service/impl/ServiceServiceImpl.java | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsVersionServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsVersionServiceImpl.java index c4dafa96..274b3c1d 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsVersionServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsVersionServiceImpl.java @@ -43,9 +43,7 @@ public class ModelsVersionServiceImpl implements ModelsVersionService { @Resource @Lazy private ModelDependencyService modelDependencyService; - // 固定存储桶名 - @Value("${minio.dataReleaseBucketName}") - private String bucketName; + /** * 通过ID查询单条数据 diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ServiceServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ServiceServiceImpl.java index 36e0ef08..5cd35fb7 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ServiceServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ServiceServiceImpl.java @@ -167,7 +167,7 @@ public class ServiceServiceImpl implements ServiceService { com.ruoyi.platform.domain.Service service = serviceDao.getServiceById(serviceVersion1.getServiceId()); ServiceVersionVo serviceVersionVo1 = getServiceVersionVo(serviceVersion1); - ServiceVersionVo serviceVersionVo2 = getServiceVersionVo(serviceVersion1); + ServiceVersionVo serviceVersionVo2 = getServiceVersionVo(serviceVersion2); serviceVersionVo1.setServiceName(service.getServiceName()); serviceVersionVo2.setServiceName(service.getServiceName()); From 3a7eda377baa8b228b00bfbe838c581719dc96e8 Mon Sep 17 00:00:00 2001 From: chenzhihang <709011834@qq.com> Date: Tue, 22 Oct 2024 17:00:02 +0800 Subject: [PATCH 14/16] =?UTF-8?q?=E5=BC=80=E5=8F=91=E6=94=AF=E6=8C=81juice?= =?UTF-8?q?fs=E5=AD=98=E5=82=A8=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/java/com/ruoyi/platform/utils/K8sClientUtil.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/K8sClientUtil.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/K8sClientUtil.java index ca3fe694..14d4c795 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/K8sClientUtil.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/K8sClientUtil.java @@ -341,6 +341,9 @@ public class K8sClientUtil { } } + int lastIndex = hostPath.lastIndexOf('/'); + String newPath = hostPath.substring(0, lastIndex); + V1Pod pod = new V1PodBuilder() .withNewMetadata() .withName(podName) @@ -368,7 +371,7 @@ public class K8sClientUtil { .endContainer() .addNewVolume() .withName("workspace") - .withHostPath(new V1HostPathVolumeSource().path(hostPath).type("DirectoryOrCreate")) + .withHostPath(new V1HostPathVolumeSource().path(newPath).type("DirectoryOrCreate")) // .withPersistentVolumeClaim(new V1PersistentVolumeClaimVolumeSource().claimName(pvcName)) .endVolume() .withTerminationGracePeriodSeconds(14400L) From 7a59ed2ff571e3d5657079f3c1b45f206604b3d6 Mon Sep 17 00:00:00 2001 From: chenzhihang <709011834@qq.com> Date: Tue, 22 Oct 2024 17:30:45 +0800 Subject: [PATCH 15/16] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=9C=8D=E5=8A=A1?= =?UTF-8?q?=E5=AF=B9=E6=AF=94bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/ruoyi/platform/utils/ConvertUtil.java | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/ConvertUtil.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/ConvertUtil.java index 52c89dd3..95bb18cd 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/ConvertUtil.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/ConvertUtil.java @@ -159,6 +159,17 @@ public class ConvertUtil { return sb.toString(); } + private static String toSnakeCase(String camelCase) { + return camelCase.replaceAll( + String.format("%s|%s|%s", + "(?<=[a-z])(?=[A-Z])", // 小写字母后接大写字母 + "(?<=[^A-Z])(?=[A-Z][a-z])", // 非大写字母后接大写字母开头的小写字母 + "(?<=[A-Z])(?=[A-Z][a-z][0-9])" // 大写字母后接另一个大写字母开头的小写字母和数字（可选，根据需求调整） + ), + "_" + ).toLowerCase(); + } + public static Map compareObjects(Object obj1, Object obj2) throws IllegalAccessException { Map differences = new HashMap<>(); @@ -176,7 +187,7 @@ public class ConvertUtil { // Compare the field values if ((value1 !=null && !value1.equals(value2)) || (value2 !=null && !value2.equals(value1))) { - differences.put(field.getName(), "Field " + field.getName() + " differs: " + value1 + " vs " + value2); + differences.put(toSnakeCase(field.getName()), "Field " + field.getName() + " differs: " + value1 + " vs " + value2); } } From fc1c3fe2b438f49bde7c4e96ea8aa3ab1aeed47e Mon Sep 17 00:00:00 2001 From: chenzhihang <709011834@qq.com> Date: Wed, 23 Oct 2024 08:52:06 +0800 Subject: [PATCH 16/16] =?UTF-8?q?=E6=96=87=E4=BB=B6=E6=89=93=E5=8C=85?= =?UTF-8?q?=E4=B8=8B=E8=BD=BD=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/ruoyi/platform/utils/MinioUtil.java | 31 ++++++++++--------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/MinioUtil.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/MinioUtil.java index 7bd5951b..f311065a 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/MinioUtil.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/MinioUtil.java @@ -394,21 +394,24 @@ public class MinioUtil { for (Result result : results) { Item item = result.get(); String objectName = item.objectName(); - InputStream objectStream = minioClient.getObject( - GetObjectArgs.builder().bucket(bucketName).object(objectName).build()); - - // Create a zip entry for each object - ZipEntry zipEntry = new ZipEntry(objectName); - zip.putNextEntry(zipEntry); - - // Write object data to zip stream - byte[] buffer = new byte[1024]; - int bytesRead; - while ((bytesRead = objectStream.read(buffer)) != -1) { - zip.write(buffer, 0, bytesRead); + if (!objectName.contains(".git")) { + InputStream objectStream = minioClient.getObject( + GetObjectArgs.builder().bucket(bucketName).object(objectName).build()); + + + // Create a zip entry for each object + ZipEntry zipEntry = new ZipEntry(objectName); + zip.putNextEntry(zipEntry); + + // Write object data to zip stream + byte[] buffer = new byte[1024]; + int bytesRead; + while ((bytesRead = objectStream.read(buffer)) != -1) { + zip.write(buffer, 0, bytesRead); + } + zip.closeEntry(); + objectStream.close(); } - zip.closeEntry(); - objectStream.close(); } zip.finish();