Merge remote-tracking branch 'origin/dev' into test

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/aim/AimController.java

@@ -1,6 +1,7 @@
 package com.ruoyi.platform.controller.aim;
 
 import com.ruoyi.common.core.web.controller.BaseController;
+import com.ruoyi.common.core.web.domain.AjaxResult;
 import com.ruoyi.common.core.web.domain.GenericsAjaxResult;
 import com.ruoyi.platform.service.AimService;
 import com.ruoyi.platform.vo.InsMetricInfoVo;
@@ -24,19 +25,19 @@ public class AimController extends BaseController {
     @GetMapping("/getExpTrainInfos/{experiment_id}")
     @ApiOperation("获取当前实验的模型训练指标信息")
     @ApiResponse
-    public GenericsAjaxResult<List<InsMetricInfoVo>> getExpTrainInfos(@RequestParam(value = "page") int page,
-                                                                      @RequestParam(value = "size") int size,
-                                                                      @PathVariable("experiment_id") Integer experimentId) {
-        return genericsSuccess(aimService.getExpInfos(true, experimentId, page, size));
+    public AjaxResult getExpTrainInfos(@RequestParam(value = "page") int page,
+                                       @RequestParam(value = "size") int size,
+                                       @PathVariable("experiment_id") Integer experimentId) {
+        return AjaxResult.success(aimService.getExpInfos(true, experimentId, page, size));
     }
 
     @GetMapping("/getExpEvaluateInfos/{experiment_id}")
     @ApiOperation("获取当前实验的模型推理指标信息")
     @ApiResponse
-    public GenericsAjaxResult<List<InsMetricInfoVo>> getExpEvaluateInfos(@RequestParam(value = "page") int page,
+    public AjaxResult getExpEvaluateInfos(@RequestParam(value = "page") int page,
                                                                          @RequestParam(value = "size") int size,
                                                                          @PathVariable("experiment_id") Integer experimentId) {
-        return genericsSuccess(aimService.getExpInfos(false, experimentId, page, size));
+        return AjaxResult.success(aimService.getExpInfos(false, experimentId, page, size));
     }
 
     @PostMapping("/getExpMetrics")

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AimService.java

@@ -1,6 +1,7 @@
 package com.ruoyi.platform.service;
 
 import com.ruoyi.platform.vo.InsMetricInfoVo;
+import org.springframework.data.domain.Page;
 
 import java.util.HashMap;
 import java.util.List;
@@ -12,7 +13,7 @@ public interface AimService {
 
     List<InsMetricInfoVo> getExpEvaluateInfos(Integer experimentId, String offset, int limit) throws Exception;
 
-    List<InsMetricInfoVo> getExpInfos(boolean isTrain, Integer experimentId, int page, int size);
+    Page<InsMetricInfoVo> getExpInfos(boolean isTrain, Integer experimentId, int page, int size);
 
     List<InsMetricInfoVo> getExpInfos1(boolean isTrain, Integer experimentId, String runId) throws Exception;
 

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AimServiceImpl.java

@@ -14,6 +14,8 @@ import com.ruoyi.platform.utils.JsonUtils;
 import com.ruoyi.platform.vo.InsMetricInfoVo;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Value;
+import org.springframework.data.domain.Page;
+import org.springframework.data.domain.PageImpl;
 import org.springframework.data.domain.PageRequest;
 import org.springframework.stereotype.Service;
 
@@ -151,14 +153,12 @@ public class AimServiceImpl implements AimService {
         return aimRunInfoList;
     }
 
-    public List<InsMetricInfoVo> getExpInfos(boolean isTrain, Integer experimentId, int page, int size) {
+    public Page<InsMetricInfoVo> getExpInfos(boolean isTrain, Integer experimentId, int page, int size) {
         PageRequest pageRequest = PageRequest.of(page, size);
         ExperimentIns query = new ExperimentIns();
         query.setExperimentId(experimentId);
+        long count = experimentInsDao.count(query);
         List<ExperimentIns> experimentInsList = experimentInsDao.queryAllByLimit(query, pageRequest);
-        if (experimentInsList == null || experimentInsList.size() == 0) {
-            return new ArrayList<>();
-        }
         List<InsMetricInfoVo> aimRunInfoList = new ArrayList<>();
         for (ExperimentIns experimentIns : experimentInsList) {
             InsMetricInfoVo aimRunInfo = new InsMetricInfoVo();
@@ -190,15 +190,17 @@ public class AimServiceImpl implements AimService {
             }
 
             //解析指标
-            Map<String, Object> metricValue = JacksonUtil.parseJSONStr2Map(experimentIns.getMetricValue());
-            if (isTrain) {
-                setMetricValue("train",metricValue,aimRunInfo);
-            } else {
-                setMetricValue("evaluate",metricValue,aimRunInfo);
+            if (StringUtils.isNotEmpty(experimentIns.getMetricValue())) {
+                Map<String, Object> metricValue = JacksonUtil.parseJSONStr2Map(experimentIns.getMetricValue());
+                if (isTrain) {
+                    setMetricValue("train", metricValue, aimRunInfo);
+                } else {
+                    setMetricValue("evaluate", metricValue, aimRunInfo);
+                }
             }
             aimRunInfoList.add(aimRunInfo);
         }
-        return aimRunInfoList;
+        return new PageImpl<>(aimRunInfoList, pageRequest, count);
     }
 
 
@@ -307,20 +309,23 @@ public class AimServiceImpl implements AimService {
         return datasetList;
     }
 
-    private void setMetricValue(String isTrain, Map<String, Object> metricValue, InsMetricInfoVo aimRunInfo){
+    private void setMetricValue(String isTrain, Map<String, Object> metricValue, InsMetricInfoVo aimRunInfo) {
         Map<String, Object> metricValues = (Map<String, Object>) metricValue.get(isTrain);
 
         HashMap<String, Object> metrics = new HashMap<>();
         List<String> metricsNames = new ArrayList<>();
-        for (String key : metricValues.keySet()) {
-            Map<String, Object> valueMap = (Map<String, Object>) metricValues.get(key);
-            aimRunInfo.setRunId((String) valueMap.get("run_hash"));
-
-            metrics.putAll(valueMap.entrySet().stream().filter(entry -> !entry.getKey().equals("run_hash")).collect(Collectors.toMap(
-                    Map.Entry::getKey,
-                    Map.Entry::getValue
-            )));
-            metricsNames.addAll(valueMap.keySet().stream().filter(entry -> !entry.equals("run_hash")).collect(Collectors.toList()));
+
+        if(metricValues != null){
+            for (String key : metricValues.keySet()) {
+                Map<String, Object> valueMap = (Map<String, Object>) metricValues.get(key);
+                aimRunInfo.setRunId((String) valueMap.get("run_hash"));
+
+                metrics.putAll(valueMap.entrySet().stream().filter(entry -> !entry.getKey().equals("run_hash")).collect(Collectors.toMap(
+                        Map.Entry::getKey,
+                        Map.Entry::getValue
+                )));
+                metricsNames.addAll(valueMap.keySet().stream().filter(entry -> !entry.equals("run_hash")).collect(Collectors.toList()));
+            }
         }
         aimRunInfo.setMetrics(metrics);
         aimRunInfo.setMetricsNames(metricsNames);

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java

@@ -654,7 +654,7 @@ public class ModelsServiceImpl implements ModelsService {
                 try {
                     dvcUtils.dvcInit(rootPath);
                     // 配置远程S3地址
-                    String s3Path = "management-platform-files/" + ci4sUsername + "/model/" + gitlinIid + "/" + repositoryName + "/" + branchName;
+                    String s3Path = bucketName + "/mini-model-management-platform-files/" + ci4sUsername + "/model/" + gitlinIid + "/" + repositoryName + "/" + branchName;
                     dvcUtils.dvcRemoteAdd(rootPath, s3Path);
                     dvcUtils.dvcConfigS3Credentials(rootPath, endpoint);
                     dvcUtils.dvcConfigS3Credentials2(rootPath, accessKeyId);
@@ -826,7 +826,7 @@ public class ModelsServiceImpl implements ModelsService {
             }
 
             // 配置远程S3地址
-            String s3Path = "management-platform-files/" + ci4sUsername + "/model/" + modelsVo.getId() + "/" + repositoryName + "/" + branchName;
+            String s3Path = bucketName + "/mini-model-management-platform-files/" + ci4sUsername + "/model/" + modelsVo.getId() + "/" + repositoryName + "/" + branchName;
             dvcUtils.dvcRemoteAdd(rootPath, s3Path);
             dvcUtils.dvcConfigS3Credentials(rootPath, endpoint);
             dvcUtils.dvcConfigS3Credentials2(rootPath, accessKeyId);

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java

@@ -63,6 +63,8 @@ public class NewDatasetServiceImpl implements NewDatasetService {
     String endpoint;
     @Value("${git.endpoint}")
     String gitendpoint;
+    @Value("${minio.dataReleaseBucketName}")
+    private String bucketName;
     @Value("${git.localPath}")
     String localPathlocal;
     @Resource
@@ -123,7 +125,7 @@ public class NewDatasetServiceImpl implements NewDatasetService {
 
         // 命令行操作 git clone 项目地址
         dvcUtils.gitClone(localPath, projectUrl, branchName, gitLinkUsername, gitLinkPassword);
-        String s3Path = "management-platform-files" + "/" + relatePath + "/" + branchName;
+        String s3Path = bucketName + "/mini-model-management-platform-files" + "/" + relatePath + "/" + branchName;
         //干掉目标文件夹
         dvcUtils.deleteDirectory(datasetPath);
         dvcUtils.moveFiles(sourcePath, datasetPath);
@@ -216,7 +218,7 @@ public class NewDatasetServiceImpl implements NewDatasetService {
         dvcUtils.dvcCheckout(localPath);
 
         // 准备数据
-        String s3Path = "management-platform-files" + "/" + relatePath + "/" + branchName;
+        String s3Path = bucketName + "/mini-model-management-platform-files" + "/" + relatePath + "/" + branchName;
         //干掉目标文件夹
         dvcUtils.deleteDirectory(datasetPath);
 

ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ExperimentInsDaoMapper.xml

@@ -13,6 +13,7 @@
         <result property="nodesLogs" column="nodes_logs" jdbcType="VARCHAR"/>
         <result property="globalParam" column="global_param" jdbcType="VARCHAR"/>
         <result property="metricRecord" column="metric_record" jdbcType="VARCHAR"/>
+        <result property="metricValue" column="metric_value" jdbcType="VARCHAR"/>
         <result property="startTime" column="start_time" jdbcType="TIMESTAMP"/>
         <result property="finishTime" column="finish_time" jdbcType="TIMESTAMP"/>
         <result property="createBy" column="create_by" jdbcType="VARCHAR"/>
@@ -25,7 +26,7 @@
 
     <!--查询非终止态的实例-->
     <select id="queryByExperimentIsNotTerminated" resultMap="ExperimentInsMap">
-        select id, experiment_id, argo_ins_name, argo_ins_ns, status, nodes_status,nodes_result, nodes_logs,global_param,metric_record, start_time, finish_time, create_by, create_time, update_by, update_time, state
+        select id, experiment_id, argo_ins_name, argo_ins_ns, status, nodes_status,nodes_result, nodes_logs,global_param,metric_record,metric_value, start_time, finish_time, create_by, create_time, update_by, update_time, state
         from experiment_ins
         where (status NOT IN ('Terminated', 'Succeeded', 'Failed')
     OR status IS NULL) and state = 1
@@ -60,7 +61,7 @@
 
     <!--查询列表-->
     <select id="getByExperimentId" resultMap="ExperimentInsMap">
-        select id, experiment_id, argo_ins_name, argo_ins_ns, status, nodes_status,nodes_result, nodes_logs, global_param,metric_record, start_time, finish_time, create_by, create_time, update_by, update_time, state
+        select id, experiment_id, argo_ins_name, argo_ins_ns, status, nodes_status,nodes_result, nodes_logs, global_param,metric_record,metric_value, start_time, finish_time, create_by, create_time, update_by, update_time, state
         from experiment_ins
         where experiment_id = #{experiment_id} and state = 1
         order by update_time DESC
@@ -69,7 +70,7 @@
 
     <!--查询最近的3个实例列表-->
     <select id="getLatestInsList" resultMap="ExperimentInsMap">
-        select id, experiment_id, argo_ins_name, argo_ins_ns, status, nodes_status,nodes_result, nodes_logs, global_param,metric_record, start_time, finish_time, create_by, create_time, update_by, update_time, state
+        select id, experiment_id, argo_ins_name, argo_ins_ns, status, nodes_status,nodes_result, nodes_logs, global_param,metric_record,metric_value, start_time, finish_time, create_by, create_time, update_by, update_time, state
         from experiment_ins
         where state = 1
         order by create_time DESC
@@ -80,7 +81,7 @@
 
     <select id="queryByExperiment" resultMap="ExperimentInsMap">
         select
-        id, experiment_id, argo_ins_name, argo_ins_ns, status, nodes_status,nodes_result, nodes_logs,global_param,metric_record, start_time, finish_time, create_by, create_time, update_by, update_time, state
+        id, experiment_id, argo_ins_name, argo_ins_ns, status, nodes_status,nodes_result, nodes_logs,global_param,metric_record,metric_value, start_time, finish_time, create_by, create_time, update_by, update_time, state
         from experiment_ins
         <where>
             state = 1
@@ -135,7 +136,7 @@
     <!--查询指定行数据-->
     <select id="queryAllByLimit" resultMap="ExperimentInsMap">
         select
-        id, experiment_id, argo_ins_name, argo_ins_ns, status, nodes_status,nodes_result, nodes_logs,global_param,metric_record, start_time, finish_time, create_by, create_time, update_by, update_time, state
+        id, experiment_id, argo_ins_name, argo_ins_ns, status, nodes_status,nodes_result, nodes_logs,global_param,metric_record, metric_value, start_time, finish_time, create_by, create_time, update_by, update_time, state
         from experiment_ins
         <where>
             state = 1
@@ -244,23 +245,23 @@
     </select>
 
     <select id="queryByExperimentId" resultMap="ExperimentInsMap">
-        select id, experiment_id, argo_ins_name, argo_ins_ns, status, nodes_status,nodes_result, nodes_logs, global_param,metric_record, start_time, finish_time, create_by, create_time, update_by, update_time, state
+        select id, experiment_id, argo_ins_name, argo_ins_ns, status, nodes_status,nodes_result, nodes_logs, global_param,metric_record,metric_value, start_time, finish_time, create_by, create_time, update_by, update_time, state
         from experiment_ins
         where experiment_id = #{id} and state = 1
     </select>
 
     <!--新增所有列-->
     <insert id="insert" keyProperty="id" useGeneratedKeys="true">
-        insert into experiment_ins(experiment_id,argo_ins_name,argo_ins_ns,status,nodes_status,nodes_result,nodes_logs,global_param,metric_record,start_time,finish_time,create_by,create_time,update_by,update_time,state)
-        values (#{experimentIns.experimentId},#{experimentIns.argoInsName},#{experimentIns.argoInsNs},#{experimentIns.status},#{experimentIns.nodesStatus},#{experimentIns.nodesResult},#{experimentIns.nodesLogs},#{experimentIns.globalParam},#{experimentIns.metricRecord},#{experimentIns.startTime},#{experimentIns.finishTime},#{experimentIns.createBy},#{experimentIns.createTime},#{experimentIns.updateBy},#{experimentIns.updateTime},#{experimentIns.state})
+        insert into experiment_ins(experiment_id,argo_ins_name,argo_ins_ns,status,nodes_status,nodes_result,nodes_logs,global_param,metric_record,metric_value,start_time,finish_time,create_by,create_time,update_by,update_time,state)
+        values (#{experimentIns.experimentId},#{experimentIns.argoInsName},#{experimentIns.argoInsNs},#{experimentIns.status},#{experimentIns.nodesStatus},#{experimentIns.nodesResult},#{experimentIns.nodesLogs},#{experimentIns.globalParam},#{experimentIns.metricRecord},#{experimentIns.metricValue},#{experimentIns.startTime},#{experimentIns.finishTime},#{experimentIns.createBy},#{experimentIns.createTime},#{experimentIns.updateBy},#{experimentIns.updateTime},#{experimentIns.state})
     </insert>
 
     <insert id="insertBatch" keyProperty="id" useGeneratedKeys="true">
         insert into
-        experiment_ins(experiment_id,argo_ins_name,argo_ins_ns,status,nodes_status,nodes_result,nodes_logs,global_param,metric_record,start_time,finish_time,create_by,create_time,update_by,update_time,state)
+        experiment_ins(experiment_id,argo_ins_name,argo_ins_ns,status,nodes_status,nodes_result,nodes_logs,global_param,metric_record,metric_value,start_time,finish_time,create_by,create_time,update_by,update_time,state)
         values
         <foreach collection="entities" item="entity" separator=",">
-            (#{entity.experimentId},#{entity.argoInsName},#{entity.argoInsNs},#{entity.status},#{entity.nodesStatus},#{entity.nodesResult},#{entity.nodesLogs},#{entity.globalParam},#{entity.metricRecord},#{entity.startTime},#{entity.finishTime},#{entity.createBy},#{entity.createTime},#{entity.updateBy},#{entity.updateTime},#{entity.state})
+            (#{entity.experimentId},#{entity.argoInsName},#{entity.argoInsNs},#{entity.status},#{entity.nodesStatus},#{entity.nodesResult},#{entity.nodesLogs},#{entity.globalParam},#{entity.metricRecord},#{entity.metricValue},#{entity.startTime},#{entity.finishTime},#{entity.createBy},#{entity.createTime},#{entity.updateBy},#{entity.updateTime},#{entity.state})
         </foreach>
     </insert>