From 952f469fbc5a79c76c25b5020aa3e12bf3813074 Mon Sep 17 00:00:00 2001 From: chenzhihang <709011834@qq.com> Date: Wed, 16 Oct 2024 14:35:55 +0800 Subject: [PATCH] =?UTF-8?q?=E6=9F=A5=E8=AF=A2=E6=A8=A1=E5=9E=8B=E7=89=88?= =?UTF-8?q?=E6=9C=AC=E6=8C=87=E6=A0=87=E5=AF=B9=E6=AF=94?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../RuoYiManagementPlatformApplication.java | 2 + .../model/NewModelFromGitController.java | 7 ++++ .../platform/mapper/ModelDependency1Dao.java | 2 +- .../ruoyi/platform/service/AimService.java | 6 +++ .../ruoyi/platform/service/ModelsService.java | 2 + .../platform/service/impl/AimServiceImpl.java | 42 +++++++++++++++++++ .../service/impl/ExperimentServiceImpl.java | 2 +- .../service/impl/ModelsServiceImpl.java | 22 ++++++++-- .../com/ruoyi/platform/vo/ModelMetaVo.java | 3 ++ .../ModelDependency1DaoMapper.xml | 3 +- 10 files changed, 84 insertions(+), 7 deletions(-) diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/RuoYiManagementPlatformApplication.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/RuoYiManagementPlatformApplication.java index b3b22ac9..a950808a 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/RuoYiManagementPlatformApplication.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/RuoYiManagementPlatformApplication.java @@ -5,6 +5,7 @@ import com.ruoyi.common.security.annotation.EnableRyFeignClients; import com.ruoyi.common.swagger.annotation.EnableCustomSwagger2; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.scheduling.annotation.EnableAsync; import org.springframework.scheduling.annotation.EnableScheduling; /** @@ -17,6 +18,7 @@ import org.springframework.scheduling.annotation.EnableScheduling; @EnableRyFeignClients @SpringBootApplication @EnableScheduling +@EnableAsync public class RuoYiManagementPlatformApplication { public static void main(String[] args) { SpringApplication.run(RuoYiManagementPlatformApplication.class, args); diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/model/NewModelFromGitController.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/model/NewModelFromGitController.java index e7eb47ef..ab4c5925 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/model/NewModelFromGitController.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/model/NewModelFromGitController.java @@ -84,6 +84,13 @@ public class NewModelFromGitController { return AjaxResult.success(this.modelsService.queryVersions(pageRequest, identifier, owner)); } + @GetMapping("/queryVersionsMetrics") + @ApiOperation("查询版本指标") + public AjaxResult queryVersionsMetrics(@RequestParam("runIds") List runIds) throws Exception { + return AjaxResult.success(this.modelsService.queryVersionsMetrics(runIds)); + } + + @GetMapping("/getVersionList") @ApiOperation(value = "获取模型分支列表") public AjaxResult getVersionList(@RequestParam("identifier") String identifier, @RequestParam("owner") String owner) throws Exception { diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/ModelDependency1Dao.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/ModelDependency1Dao.java index c44bd8bd..721496ea 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/ModelDependency1Dao.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/ModelDependency1Dao.java @@ -9,7 +9,7 @@ public interface ModelDependency1Dao { int insert(ModelDependency1 modelDependency1); - int updateState(@Param("repoId") Integer repoId, @Param("identifier") String identifier, @Param("version") String version, @Param("state") Integer state); + int updateState(@Param("repoId") Integer repoId, @Param("identifier") String identifier, @Param("version") String version, @Param("meta") String meta, @Param("state") Integer state); List queryModelDependency(@Param("modelName") String modelName, @Param("repoId") Integer repoId, @Param("owner") String owner); diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AimService.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AimService.java index 9f3868f5..b6c03292 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AimService.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AimService.java @@ -2,7 +2,9 @@ package com.ruoyi.platform.service; import com.ruoyi.platform.vo.InsMetricInfoVo; +import java.util.HashMap; import java.util.List; +import java.util.Map; public interface AimService { @@ -13,4 +15,8 @@ public interface AimService { List getExpEvaluateInfos(Integer experimentId) throws Exception; String getExpMetrics(List runIds) throws Exception; + + HashMap queryMetricsParams(String runId) throws Exception; + + List> getBatchMetric(String runHash, String params); } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ModelsService.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ModelsService.java index 8b51d696..6c5f6a58 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ModelsService.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ModelsService.java @@ -101,6 +101,8 @@ public interface ModelsService { Page> queryVersions(PageRequest pageRequest, String identifier, String owner) throws Exception; + List>> queryVersionsMetrics(List runIds) throws Exception; + List> getVersionList(String identifier, String owner) throws Exception; ModelsVo getModelDetail(Integer id, String identifier, String owner, String version) throws Exception; diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AimServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AimServiceImpl.java index 1b754404..663158ba 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AimServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AimServiceImpl.java @@ -1,5 +1,6 @@ package com.ruoyi.platform.service.impl; +import com.alibaba.fastjson2.JSON; import com.ruoyi.platform.domain.ExperimentIns; import com.ruoyi.platform.service.AimService; import com.ruoyi.platform.service.ExperimentInsService; @@ -13,6 +14,7 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import javax.annotation.Resource; +import java.io.UnsupportedEncodingException; import java.net.URLEncoder; import java.util.*; import java.util.stream.Collectors; @@ -245,4 +247,44 @@ public class AimServiceImpl implements AimService { } return datasetList; } + + @Override + public HashMap queryMetricsParams(String runId) throws UnsupportedEncodingException { + String encodedUrlString = URLEncoder.encode("run.id==\"" + runId + "\"", "UTF-8"); + String url = aimProxyUrl + "/api/runs/search/run?query=" + encodedUrlString; + String s = httpUtils.sendGet(url, null); + List> response = JacksonUtil.parseJSONStr2MapList(s); + if (response == null || response.size() == 0) { + return new HashMap<>(); + } + + HashMap resultMap = new HashMap<>(); + List> paramList = new ArrayList<>(); + + Map map = response.get(0); + LinkedHashMap traces = (LinkedHashMap) map.get("traces"); + if (traces != null) { + List> metrics = traces.get("metric"); + for (Map metric : metrics) { + Map metricParam = new HashMap<>(); + metricParam.put("context", metric.get("context")); + metricParam.put("name", metric.get("name")); + paramList.add(metricParam); + } + resultMap.put("params", JSON.toJSONString(paramList)); + } + resultMap.put("run_hash", map.get("run_hash")); + return resultMap; + } + + @Override + public List> getBatchMetric(String runHash, String params) { + String url = aimUrl + "/api/runs/" + runHash + "/metric/get-batch"; + String response = httpUtils.sendPost(url, null, params); + if (StringUtils.isNotEmpty(response)) { + return JacksonUtil.parseJSONStr2MapList(response); + } + return null; + } + } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentServiceImpl.java index dfc8b5d4..e57296ca 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentServiceImpl.java @@ -560,7 +560,7 @@ public class ExperimentServiceImpl implements ExperimentService { //处理指标 HashMap metricMap = JSON.parseObject(metricRecord, HashMap.class); - modelMetaVo.setMetrics(metricMap); + modelMetaVo.setMetricsParams(metricMap); //训练数据集 List> trainDatasetList = (List>) modelTrainMap.get("datasets"); diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java index 8000a953..17a6093a 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java @@ -747,7 +747,7 @@ public class ModelsServiceImpl implements ModelsService { if (buildingModel != null) { modelMetaVo = JSON.parseObject(buildingModel.getMeta(), ModelMetaVo.class); //获取指标 - getMetrics(modelMetaVo); + transMetrics(modelMetaVo); } //拼接生产的元数据后写入yaml文件 @@ -806,7 +806,7 @@ public class ModelsServiceImpl implements ModelsService { modelDependency1Dao.insert(modelDependency); } else { //更新模型依赖 - modelDependency1Dao.updateState(modelsVo.getId(), modelsVo.getIdentifier(), modelsVo.getVersion(), Constant.State_valid); + modelDependency1Dao.updateState(modelsVo.getId(), modelsVo.getIdentifier(), modelsVo.getVersion(), meta, Constant.State_valid); } } else { //保存模型依赖 @@ -991,10 +991,24 @@ public class ModelsServiceImpl implements ModelsService { if (modelMetaVo.getMetrics() != null) { branch.putAll(modelMetaVo.getMetrics()); } + if (modelMetaVo.getMetricsParams() != null) { + branch.putAll(modelMetaVo.getMetricsParams()); + } } return new PageImpl<>(result, pageRequest, collect.size()); } + @Override + public List>> queryVersionsMetrics(List runIds) throws Exception { + List>> batchMetrics = new ArrayList<>(); + for (String runId : runIds) { + HashMap map = aimsService.queryMetricsParams(runId); + List> batchMetric = aimsService.getBatchMetric((String) map.get("run_hash"), (String) map.get("params")); + batchMetrics.add(batchMetric); + } + return batchMetrics; + } + @Override public List> getVersionList(String identifier, String owner) throws Exception { String token = gitService.checkoutToken(); @@ -1201,12 +1215,12 @@ public class ModelsServiceImpl implements ModelsService { return userInfo; } - void getMetrics(ModelMetaVo modelMetaVo) throws Exception { + void transMetrics(ModelMetaVo modelMetaVo) throws Exception { HashMap result = new HashMap<>(); HashMap train = new HashMap<>(); HashMap evaluate = new HashMap<>(); - HashMap metrics = modelMetaVo.getMetrics(); + HashMap metrics = modelMetaVo.getMetricsParams(); JSONArray trainMetrics = (JSONArray) metrics.get("train"); if (trainMetrics != null) { for (int i = 0; i < trainMetrics.size(); i++) { diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelMetaVo.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelMetaVo.java index e1b84905..2a5a538e 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelMetaVo.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelMetaVo.java @@ -59,6 +59,9 @@ public class ModelMetaVo implements Serializable { @ApiModelProperty(value = "指标") private HashMap metrics; + @ApiModelProperty(value = "指标查询参数") + private HashMap metricsParams; + @ApiModelProperty(value = "训练任务") private TrainTaskDepency trainTask; diff --git a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ModelDependency1DaoMapper.xml b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ModelDependency1DaoMapper.xml index bc2a58fb..4e77a29b 100644 --- a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ModelDependency1DaoMapper.xml +++ b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ModelDependency1DaoMapper.xml @@ -86,7 +86,8 @@ update model_dependency1 - set state = 1 + set state = 1, + meta = #{meta} where repo_id = #{repoId} and identifier = #{identifier} and version = #{version}