diff --git a/k8s/deploy.sh b/k8s/deploy.sh index 36036fe0..5b3f7887 100755 --- a/k8s/deploy.sh +++ b/k8s/deploy.sh @@ -143,7 +143,7 @@ fi if [ "$service" == "all" ]; then #部署前端 -# build_and_deploy "nginx-dockerfile" "172.20.32.187/ci4s/ci4s-front:${tag}" "k8s-12front.yaml" + build_and_deploy "nginx-dockerfile" "172.20.32.187/ci4s/ci4s-front:${tag}" "k8s-12front.yaml" #部署管理平台 build_and_deploy "managent-dockerfile" "172.20.32.187/ci4s/ci4s-managent:${tag}" "k8s-7management.yaml" #部署认证中心 diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/RuoYiManagementPlatformApplication.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/RuoYiManagementPlatformApplication.java index b3b22ac9..a950808a 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/RuoYiManagementPlatformApplication.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/RuoYiManagementPlatformApplication.java @@ -5,6 +5,7 @@ import com.ruoyi.common.security.annotation.EnableRyFeignClients; import com.ruoyi.common.swagger.annotation.EnableCustomSwagger2; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.scheduling.annotation.EnableAsync; import org.springframework.scheduling.annotation.EnableScheduling; /** @@ -17,6 +18,7 @@ import org.springframework.scheduling.annotation.EnableScheduling; @EnableRyFeignClients @SpringBootApplication @EnableScheduling +@EnableAsync public class RuoYiManagementPlatformApplication { public static void main(String[] args) { SpringApplication.run(RuoYiManagementPlatformApplication.class, args); diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/experiment/ExperimentInsController.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/experiment/ExperimentInsController.java index 1c36c1d2..8d18c409 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/experiment/ExperimentInsController.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/experiment/ExperimentInsController.java @@ -106,6 +106,12 @@ public class ExperimentInsController extends BaseController { return genericsSuccess(this.experimentInsService.removeById(id)); } + @DeleteMapping("batchDelete") + @ApiOperation("批量删除实验实例") + public GenericsAjaxResult batchDelete(@RequestBody List ids) throws Exception{ + return genericsSuccess(this.experimentInsService.batchDelete(ids)); + } + /** * 终止实验实例 * diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/model/NewModelFromGitController.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/model/NewModelFromGitController.java index 426c3de4..ab4c5925 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/model/NewModelFromGitController.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/model/NewModelFromGitController.java @@ -2,11 +2,9 @@ package com.ruoyi.platform.controller.model; import com.ruoyi.common.core.web.domain.AjaxResult; import com.ruoyi.platform.service.ModelsService; -import com.ruoyi.platform.utils.DVCUtils; import com.ruoyi.platform.vo.ModelsVo; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; -import org.eclipse.jgit.api.errors.GitAPIException; import org.springframework.core.io.InputStreamResource; import org.springframework.data.domain.PageRequest; import org.springframework.http.ResponseEntity; @@ -14,7 +12,6 @@ import org.springframework.web.bind.annotation.*; import org.springframework.web.multipart.MultipartFile; import javax.annotation.Resource; -import java.io.IOException; import java.util.List; import java.util.Map; @@ -77,6 +74,23 @@ public class NewModelFromGitController { } } + @GetMapping("/queryVersions") + @ApiOperation("分页查询模型版本") + public AjaxResult queryVersions(@RequestParam(value = "page") int page, + @RequestParam(value = "size") int size, + @RequestParam("identifier") String identifier, + @RequestParam("owner") String owner) throws Exception { + PageRequest pageRequest = PageRequest.of(page, size); + return AjaxResult.success(this.modelsService.queryVersions(pageRequest, identifier, owner)); + } + + @GetMapping("/queryVersionsMetrics") + @ApiOperation("查询版本指标") + public AjaxResult queryVersionsMetrics(@RequestParam("runIds") List runIds) throws Exception { + return AjaxResult.success(this.modelsService.queryVersionsMetrics(runIds)); + } + + @GetMapping("/getVersionList") @ApiOperation(value = "获取模型分支列表") public AjaxResult getVersionList(@RequestParam("identifier") String identifier, @RequestParam("owner") String owner) throws Exception { diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/ModelDependency1Dao.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/ModelDependency1Dao.java index 28baa541..721496ea 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/ModelDependency1Dao.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/ModelDependency1Dao.java @@ -9,7 +9,7 @@ public interface ModelDependency1Dao { int insert(ModelDependency1 modelDependency1); - int updateState(@Param("repoId") Integer repoId, @Param("identifier") String identifier, @Param("version") String version, @Param("state") Integer state); + int updateState(@Param("repoId") Integer repoId, @Param("identifier") String identifier, @Param("version") String version, @Param("meta") String meta, @Param("state") Integer state); List queryModelDependency(@Param("modelName") String modelName, @Param("repoId") Integer repoId, @Param("owner") String owner); @@ -24,4 +24,6 @@ public interface ModelDependency1Dao { int deleteModel(@Param("repoId") Integer repoId, @Param("identifier") String identifier, @Param("owner") String owner, @Param("version") String version); int deleteModelDependency(@Param("parentModel") String parentModel); + + String getMeta(@Param("identifier") String identifier, @Param("owner") String owner, @Param("version") String version); } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AimService.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AimService.java index 9f3868f5..b6c03292 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AimService.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AimService.java @@ -2,7 +2,9 @@ package com.ruoyi.platform.service; import com.ruoyi.platform.vo.InsMetricInfoVo; +import java.util.HashMap; import java.util.List; +import java.util.Map; public interface AimService { @@ -13,4 +15,8 @@ public interface AimService { List getExpEvaluateInfos(Integer experimentId) throws Exception; String getExpMetrics(List runIds) throws Exception; + + HashMap queryMetricsParams(String runId) throws Exception; + + List> getBatchMetric(String runHash, String params); } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ExperimentInsService.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ExperimentInsService.java index 046e0186..3953a01e 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ExperimentInsService.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ExperimentInsService.java @@ -73,6 +73,9 @@ public interface ExperimentInsService { * @param id 主键 * @return 是否成功 */ + + String batchDelete(List ids); + boolean deleteById(Integer id); diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ModelsService.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ModelsService.java index 067f90a7..6c5f6a58 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ModelsService.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ModelsService.java @@ -99,6 +99,10 @@ public interface ModelsService { Page newPersonalQueryByPage(ModelsVo modelsVo, PageRequest pageRequest) throws Exception; + Page> queryVersions(PageRequest pageRequest, String identifier, String owner) throws Exception; + + List>> queryVersionsMetrics(List runIds) throws Exception; + List> getVersionList(String identifier, String owner) throws Exception; ModelsVo getModelDetail(Integer id, String identifier, String owner, String version) throws Exception; diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AimServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AimServiceImpl.java index 1b754404..7fd7c706 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AimServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AimServiceImpl.java @@ -1,18 +1,17 @@ package com.ruoyi.platform.service.impl; +import com.alibaba.fastjson2.JSON; import com.ruoyi.platform.domain.ExperimentIns; import com.ruoyi.platform.service.AimService; import com.ruoyi.platform.service.ExperimentInsService; -import com.ruoyi.platform.utils.AIM64EncoderUtil; -import com.ruoyi.platform.utils.JacksonUtil; -import com.ruoyi.platform.utils.JsonUtils; -import com.ruoyi.platform.utils.NewHttpUtils; +import com.ruoyi.platform.utils.*; import com.ruoyi.platform.vo.InsMetricInfoVo; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import javax.annotation.Resource; +import java.io.UnsupportedEncodingException; import java.net.URLEncoder; import java.util.*; import java.util.stream.Collectors; @@ -26,8 +25,6 @@ public class AimServiceImpl implements AimService { private String aimUrl; @Value("${aim.proxyUrl}") private String aimProxyUrl; - @Resource - private NewHttpUtils httpUtils; @Override public List getExpTrainInfos(Integer experimentId) throws Exception { @@ -52,7 +49,7 @@ public class AimServiceImpl implements AimService { } String encodedUrlString = URLEncoder.encode("run.experiment==\"" + experimentName + "\"", "UTF-8"); String url = aimProxyUrl + "/api/runs/search/run?query=" + encodedUrlString; - String s = httpUtils.sendGet(url, null); + String s = HttpUtils.sendGet(url, null); List> response = JacksonUtil.parseJSONStr2MapList(s); System.out.println("response: " + JacksonUtil.toJSONString(response)); if (response == null || response.size() == 0) { @@ -142,7 +139,7 @@ public class AimServiceImpl implements AimService { public List getExpTrainInfos1(boolean isTrain, Integer experimentId, String runId) throws Exception { String encodedUrlString = URLEncoder.encode("run.id==\"" + runId + "\"", "UTF-8"); String url = aimProxyUrl + "/api/runs/search/run?query=" + encodedUrlString; - String s = httpUtils.sendGet(url, null); + String s = HttpUtils.sendGet(url, null); List> response = JacksonUtil.parseJSONStr2MapList(s); System.out.println("response: " + JacksonUtil.toJSONString(response)); if (response == null || response.size() == 0) { @@ -245,4 +242,44 @@ public class AimServiceImpl implements AimService { } return datasetList; } + + @Override + public HashMap queryMetricsParams(String runId) throws UnsupportedEncodingException { + String encodedUrlString = URLEncoder.encode("run.id==\"" + runId + "\"", "UTF-8"); + String url = aimProxyUrl + "/api/runs/search/run?query=" + encodedUrlString; + String s = HttpUtils.sendGet(url, null); + List> response = JacksonUtil.parseJSONStr2MapList(s); + if (response == null || response.size() == 0) { + return new HashMap<>(); + } + + HashMap resultMap = new HashMap<>(); + List> paramList = new ArrayList<>(); + + Map map = response.get(0); + LinkedHashMap traces = (LinkedHashMap) map.get("traces"); + if (traces != null) { + List> metrics = traces.get("metric"); + for (Map metric : metrics) { + Map metricParam = new HashMap<>(); + metricParam.put("context", metric.get("context")); + metricParam.put("name", metric.get("name")); + paramList.add(metricParam); + } + resultMap.put("params", JSON.toJSONString(paramList)); + } + resultMap.put("run_hash", map.get("run_hash")); + return resultMap; + } + + @Override + public List> getBatchMetric(String runHash, String params) { + String url = aimUrl + "/api/runs/" + runHash + "/metric/get-batch"; + String response = HttpUtils.sendPost(url, params); + if (StringUtils.isNotEmpty(response)) { + return JacksonUtil.parseJSONStr2MapList(response); + } + return null; + } + } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentInsServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentInsServiceImpl.java index 7b9c57f7..4b73d671 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentInsServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentInsServiceImpl.java @@ -36,13 +36,13 @@ public class ExperimentInsServiceImpl implements ExperimentInsService { @Resource private WorkflowService workflowService; @Value("${argo.url}") - private String argoUrl; + private String argoUrl; @Value("${argo.convert}") - private String argoConvert; + private String argoConvert; @Value("${argo.workflowRun}") - private String argoWorkflowRun; + private String argoWorkflowRun; @Value("${argo.workflowStatus}") - private String argoWorkflowStatus; + private String argoWorkflowStatus; @Value("${argo.workflowTermination}") private String argoWorkflowTermination; @Value("${argo.workflowLog}") @@ -60,6 +60,7 @@ public class ExperimentInsServiceImpl implements ExperimentInsService { @Resource private K8sClientUtil k8sClientUtil; private final MinioUtil minioUtil; + public ExperimentInsServiceImpl(MinioUtil minioUtil) { this.minioUtil = minioUtil; } @@ -97,8 +98,6 @@ public class ExperimentInsServiceImpl implements ExperimentInsService { } /** - * - * * 根据实验ID查找所有具有相同ID的实例，并将它们添加到实验列表中 * * @param experimentId 实验ID @@ -187,8 +186,6 @@ public class ExperimentInsServiceImpl implements ExperimentInsService { } - - /** * 修改数据 * @@ -207,7 +204,7 @@ public class ExperimentInsServiceImpl implements ExperimentInsService { @Override public String removeById(Integer id) { ExperimentIns experimentIns = experimentInsDao.queryById(id); - if (experimentIns == null){ + if (experimentIns == null) { return "实验实例不存在"; } @@ -215,21 +212,31 @@ public class ExperimentInsServiceImpl implements ExperimentInsService { LoginUser loginUser = SecurityUtils.getLoginUser(); String username = loginUser.getUsername(); String createdBy = experimentIns.getCreateBy(); - if (!(StringUtils.equals(username,"admin") || StringUtils.equals(username,createdBy))){ + if (!(StringUtils.equals(username, "admin") || StringUtils.equals(username, createdBy))) { return "无权限删除该流水线"; } - if (StringUtils.isEmpty(experimentIns.getStatus())){ + if (StringUtils.isEmpty(experimentIns.getStatus())) { experimentIns = queryStatusFromArgo(experimentIns); } - if (StringUtils.equals(experimentIns.getStatus(),"Running")){ + if (StringUtils.equals(experimentIns.getStatus(), "Running")) { return "实验实例正在运行，不可删除"; } experimentIns.setState(0); - return this.experimentInsDao.update(experimentIns)>0?"删除成功":"删除失败"; + return this.experimentInsDao.update(experimentIns) > 0 ? "删除成功" : "删除失败"; } + @Override + public String batchDelete(List ids) { + for (Integer id : ids) { + String result = removeById(id); + if (!"删除成功".equals(result)) { + return result; + } + } + return "删除成功"; + } /** @@ -256,16 +263,16 @@ public class ExperimentInsServiceImpl implements ExperimentInsService { Integer id = ins.getId(); // 创建请求数据map ExperimentIns experimentIns = this.experimentInsDao.queryById(id); - Map requestData = new HashMap<>(); + Map requestData = new HashMap<>(); requestData.put("namespace", namespace); requestData.put("name", name); // 创建发送数据map，将请求数据作为"data"键的值 - Map res = new HashMap<>(); + Map res = new HashMap<>(); res.put("data", requestData); try { // 发送POST请求到Argo工作流状态查询接口，并将请求数据转换为JSON - String req = httpUtils.sendPost(argoUrl + argoWorkflowStatus,null, JsonUtils.mapToJson(res)); + String req = httpUtils.sendPost(argoUrl + argoWorkflowStatus, null, JsonUtils.mapToJson(res)); // 检查响应是否为空或无内容 if (req == null || StringUtils.isEmpty(req)) { throw new RuntimeException("工作流状态响应为空。"); @@ -293,10 +300,10 @@ public class ExperimentInsServiceImpl implements ExperimentInsService { // 解析nodes字段,提取节点状态并转换为JSON字符串 Map nodes = (Map) status.get("nodes"); - Map modifiedNodes = new LinkedHashMap<>(); - if (nodes != null ) { + Map modifiedNodes = new LinkedHashMap<>(); + if (nodes != null) { for (Map.Entry nodeEntry : nodes.entrySet()) { - Map nodeDetails = (Map) nodeEntry.getValue(); + Map nodeDetails = (Map) nodeEntry.getValue(); String templateName = (String) nodeDetails.get("displayName"); modifiedNodes.put(templateName, nodeDetails); } @@ -306,10 +313,10 @@ public class ExperimentInsServiceImpl implements ExperimentInsService { experimentIns.setNodesStatus(nodeStatusJson); //终止态为终止不改 - if (!StringUtils.equals(experimentIns.getStatus(),"Terminated")) { - experimentIns.setStatus(StringUtils.isNotEmpty((String) status.get("phase"))?(String) status.get("phase"):"Pending"); + if (!StringUtils.equals(experimentIns.getStatus(), "Terminated")) { + experimentIns.setStatus(StringUtils.isNotEmpty((String) status.get("phase")) ? (String) status.get("phase") : "Pending"); } - if (StringUtils.equals(experimentIns.getStatus(),"Error")) { + if (StringUtils.equals(experimentIns.getStatus(), "Error")) { experimentIns.setStatus("Failed"); } @@ -350,16 +357,16 @@ public class ExperimentInsServiceImpl implements ExperimentInsService { } // 创建请求数据map - Map requestData = new HashMap<>(); + Map requestData = new HashMap<>(); requestData.put("namespace", namespace); requestData.put("name", name); // 创建发送数据map，将请求数据作为"data"键的值 - Map res = new HashMap<>(); + Map res = new HashMap<>(); res.put("data", requestData); try { // 发送POST请求到Argo工作流状态查询接口，并将请求数据转换为JSON - String req = httpUtils.sendPost(argoUrl + argoWorkflowTermination,null, JsonUtils.mapToJson(res)); + String req = httpUtils.sendPost(argoUrl + argoWorkflowTermination, null, JsonUtils.mapToJson(res)); // 检查响应是否为空或无内容 if (StringUtils.isEmpty(req)) { throw new RuntimeException("终止响应内容为空。"); @@ -423,20 +430,20 @@ public class ExperimentInsServiceImpl implements ExperimentInsService { String namespace = experimentIns.getArgoInsNs(); // 创建请求数据的Json(map) - Map requestData = new HashMap<>(); + Map requestData = new HashMap<>(); requestData.put("namespace", namespace); requestData.put("name", name); //先写死这两个数据项 - requestData.put("workflow-type","workflows"); - requestData.put("artifact-name","main-logs"); - requestData.put("component-id",componentId); + requestData.put("workflow-type", "workflows"); + requestData.put("artifact-name", "main-logs"); + requestData.put("component-id", componentId); // 创建发送数据map，将请求数据作为"data"键的值 - Map res = new HashMap<>(); + Map res = new HashMap<>(); res.put("data", requestData); try { // 发送POST请求到Argo工作流日志查询接口，并将请求数据转换为JSON - String req = httpUtils.sendPost(argoUrl + argoWorkflowLog, null,JsonUtils.mapToJson(res)); + String req = httpUtils.sendPost(argoUrl + argoWorkflowLog, null, JsonUtils.mapToJson(res)); // 检查响应是否为空或无内容 if (StringUtils.isEmpty(req)) { throw new RuntimeException("响应内容为空"); @@ -445,7 +452,7 @@ public class ExperimentInsServiceImpl implements ExperimentInsService { Map runResMap = JsonUtils.jsonToMap(req); // 从响应Map中获取"data"的值,也就是日志的值 String experimentInsLog = (String) runResMap.get("data"); - if (experimentInsLog == null){ + if (experimentInsLog == null) { throw new RuntimeException("日志为空。"); } //返回日志内容 @@ -463,28 +470,28 @@ public class ExperimentInsServiceImpl implements ExperimentInsService { //查询 实例 ExperimentIns experimentIns = this.experimentInsDao.queryById(id); if (experimentIns == null) { - throw new Exception("实验实例未查询到，id:" + id); + throw new Exception("实验实例未查询到，id:" + id); } //找到 节点 节点结果 String nodesResultString = experimentIns.getNodesResult(); if (StringUtils.isEmpty(nodesResultString)) { - throw new Exception("实验实例未查询到节点结果，id:" + id); + throw new Exception("实验实例未查询到节点结果，id:" + id); } Map nodesResult = JsonUtils.jsonToMap(nodesResultString); Map paramOutput = (Map) nodesResult.get("param_output"); - List> nodeList = (List>) paramOutput.get(nodeId); + List> nodeList = (List>) paramOutput.get(nodeId); //遍历 查询minio for (int i = 0; i < nodeList.size(); i++) { Map map = nodeList.get(i); String path = (String) map.get("path"); String bucketName = path.substring(0, path.indexOf("/")); - String prefix = path.substring(path.indexOf("/")+1,path.length())+"/"; - if (StringUtils.equals( (String)map.get("type"),"file")){ + String prefix = path.substring(path.indexOf("/") + 1, path.length()) + "/"; + if (StringUtils.equals((String) map.get("type"), "file")) { List fileInfo = minioUtil.listFilesInDirectory(bucketName, prefix); - map.put("value",fileInfo); - }else if (StringUtils.equals( (String)map.get("type"),"string")){ + map.put("value", fileInfo); + } else if (StringUtils.equals((String) map.get("type"), "string")) { String resultInfo = minioUtil.readObjectAsString(bucketName, prefix); - map.put("value",resultInfo); + map.put("value", resultInfo); } results.add(map); //组装 @@ -501,14 +508,14 @@ public class ExperimentInsServiceImpl implements ExperimentInsService { String taskId = logRequest.getTaskId(); String startTime = logRequest.getStartTime(); // 创建请求数据的Json(map) - Map requestData = new HashMap<>(); + Map requestData = new HashMap<>(); requestData.put("component_id", componentId); requestData.put("namespace", nameSpace); requestData.put("name", name); - requestData.put("task_id",taskId); - requestData.put("start_time",startTime); + requestData.put("task_id", taskId); + requestData.put("start_time", startTime); // 创建发送数据map，将请求数据作为"data"键的值 - Map res = new HashMap<>(); + Map res = new HashMap<>(); res.put("data", requestData); try { @@ -531,9 +538,9 @@ public class ExperimentInsServiceImpl implements ExperimentInsService { @Override public Map getRealtimePodLog(String podName, String startTime) { - Map requestData = new HashMap<>(); - requestData.put("pod_name",podName); - requestData.put("start_time",startTime); + Map requestData = new HashMap<>(); + requestData.put("pod_name", podName); + requestData.put("start_time", startTime); try { // 将Map转换为JSON字符串 String req = HttpUtils.sendPost(argoUrl + argoWorkflowPodLog, JsonUtils.mapToJson(requestData)); @@ -554,7 +561,7 @@ public class ExperimentInsServiceImpl implements ExperimentInsService { @Override public String getRealtimePodLogFromPod(PodLogVo podLogVo) { - return k8sClientUtil.getPodLogs(podLogVo.getPodName(), podLogVo.getNamespace(),podLogVo.getContainerName(), logsLines); + return k8sClientUtil.getPodLogs(podLogVo.getPodName(), podLogVo.getNamespace(), podLogVo.getContainerName(), logsLines); } @Override @@ -573,7 +580,7 @@ public class ExperimentInsServiceImpl implements ExperimentInsService { experimentIns.setStatus(status.toString()); // 设置实例的状态为枚举中的状态 // 查询具有相同状态的实例数量 - Long count = experimentInsDao.count(experimentIns); + Long count = experimentInsDao.count(experimentIns); // 将状态及其对应的实例数量放入map中 statusCountMap.put(status.toString(), count); @@ -588,7 +595,7 @@ public class ExperimentInsServiceImpl implements ExperimentInsService { boolean flag = true; List terminatedStates = Arrays.asList("Succeeded", "Failed"); flag = terminatedStates.contains(status); - if (StringUtils.equals(status, "Terminated")){ + if (StringUtils.equals(status, "Terminated")) { //如果跟node_status里面不一样，就要去更新node_status的信息 String nodesStatus = ins.getNodesStatus(); Map nodeMap = JsonUtils.jsonToMap(nodesStatus); @@ -597,7 +604,7 @@ public class ExperimentInsServiceImpl implements ExperimentInsService { .findFirst() .orElse(null); Map workflowMap = (Map) nodeMap.get(keyStartsWithWorkflow); - if (workflowMap != null){ + if (workflowMap != null) { flag = StringUtils.equals("Terminated", (String) workflowMap.get("phase")); } } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentServiceImpl.java index dfc8b5d4..e57296ca 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentServiceImpl.java @@ -560,7 +560,7 @@ public class ExperimentServiceImpl implements ExperimentService { //处理指标 HashMap metricMap = JSON.parseObject(metricRecord, HashMap.class); - modelMetaVo.setMetrics(metricMap); + modelMetaVo.setMetricsParams(metricMap); //训练数据集 List> trainDatasetList = (List>) modelTrainMap.get("datasets"); diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java index 9a09ab84..f4d06f2f 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java @@ -747,7 +747,7 @@ public class ModelsServiceImpl implements ModelsService { if (buildingModel != null) { modelMetaVo = JSON.parseObject(buildingModel.getMeta(), ModelMetaVo.class); //获取指标 - getMetrics(modelMetaVo); + transMetrics(modelMetaVo); } //拼接生产的元数据后写入yaml文件 @@ -806,7 +806,7 @@ public class ModelsServiceImpl implements ModelsService { modelDependency1Dao.insert(modelDependency); } else { //更新模型依赖 - modelDependency1Dao.updateState(modelsVo.getId(), modelsVo.getIdentifier(), modelsVo.getVersion(), Constant.State_valid); + modelDependency1Dao.updateState(modelsVo.getId(), modelsVo.getIdentifier(), modelsVo.getVersion(), meta, Constant.State_valid); } } else { //保存模型依赖 @@ -973,11 +973,47 @@ public class ModelsServiceImpl implements ModelsService { return new PageImpl<>(result, pageRequest, collect.size()); } + @Override + public Page> queryVersions(PageRequest pageRequest, String identifier, String owner) throws Exception { + String token = gitService.checkoutToken(); + List> collect = gitService.getBrancheList(token, owner, identifier); + List> result = collect.stream() + .filter(branch -> !"master".equals(branch.get("name"))) + .skip((pageRequest.getPageNumber()) * pageRequest.getPageSize()).limit(pageRequest.getPageSize()) + .collect(Collectors.toList()); + + for (Map branch : result) { + String meta = modelDependency1Dao.getMeta(identifier, owner, (String) branch.get("name")); + ModelMetaVo modelMetaVo = JSON.parseObject(meta, ModelMetaVo.class); + if (modelMetaVo.getParams() != null) { + branch.putAll(modelMetaVo.getParams()); + } + if (modelMetaVo.getMetrics() != null) { + branch.putAll(modelMetaVo.getMetrics()); + } + if (modelMetaVo.getMetricsParams() != null) { + branch.putAll(modelMetaVo.getMetricsParams()); + } + } + return new PageImpl<>(result, pageRequest, collect.size()); + } + + @Override + public List>> queryVersionsMetrics(List runIds) throws Exception { + List>> batchMetrics = new ArrayList<>(); + for (String runId : runIds) { + HashMap map = aimsService.queryMetricsParams(runId); + List> batchMetric = aimsService.getBatchMetric((String) map.get("run_hash"), (String) map.get("params")); + batchMetrics.add(batchMetric); + } + return batchMetrics; + } + @Override public List> getVersionList(String identifier, String owner) throws Exception { String token = gitService.checkoutToken(); - List> brancheList = gitService.getBrancheList(token, owner, identifier); - return brancheList.stream() + List> branchList = gitService.getBrancheList(token, owner, identifier); + return branchList.stream() .filter(branch -> !"master".equals(branch.get("name"))) .collect(Collectors.toList()); } @@ -1179,12 +1215,12 @@ public class ModelsServiceImpl implements ModelsService { return userInfo; } - void getMetrics(ModelMetaVo modelMetaVo) throws Exception { + void transMetrics(ModelMetaVo modelMetaVo) throws Exception { HashMap result = new HashMap<>(); HashMap train = new HashMap<>(); HashMap evaluate = new HashMap<>(); - HashMap metrics = modelMetaVo.getMetrics(); + HashMap metrics = modelMetaVo.getMetricsParams(); JSONArray trainMetrics = (JSONArray) metrics.get("train"); if (trainMetrics != null) { for (int i = 0; i < trainMetrics.size(); i++) { diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/HttpUtils.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/HttpUtils.java index 1de5d84e..6a130f0f 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/HttpUtils.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/HttpUtils.java @@ -1,5 +1,6 @@ package com.ruoyi.platform.utils; +import com.ruoyi.common.core.utils.StringUtils; import lombok.extern.slf4j.Slf4j; import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; @@ -101,12 +102,13 @@ public class HttpUtils { // public static String sendGetWithToken(String url, String param,String token) { // return sendGet(url, param, "UTF-8",token); // } + /** * 向指定 URL 发送带 token 的 GET 方法的请求，使用 Apache HttpClient * - * @param url 发送请求的 URL - * @param param 请求参数，请求参数应该是 name1=value1&name2=value2 的形式。 - * @param token 认证 token + * @param url 发送请求的 URL + * @param param 请求参数，请求参数应该是 name1=value1&name2=value2 的形式。 + * @param token 认证 token * @return 所代表远程资源的响应结果 */ public static String sendGetWithToken(String url, String param, String token) { @@ -148,6 +150,7 @@ public class HttpUtils { } return result; } + /** * 向指定 URL 发送带token的GET方法的请求 * @@ -156,7 +159,7 @@ public class HttpUtils { * @param contentType 编码类型 * @return 所代表远程资源的响应结果 */ - public static String sendGet(String url, String param, String contentType,String token) { + public static String sendGet(String url, String param, String contentType, String token) { StringBuilder result = new StringBuilder(); BufferedReader in = null; try { @@ -196,7 +199,6 @@ public class HttpUtils { } - /** * 向指定 URL 发送GET方法的请求 * @@ -209,7 +211,10 @@ public class HttpUtils { StringBuilder result = new StringBuilder(); BufferedReader in = null; try { - String urlNameString = url + "?" + param; + String urlNameString = url; + if (StringUtils.isNotEmpty(param)) { + urlNameString = url + "?" + param; + } log.info("sendGet - {}", urlNameString); URL realUrl = new URL(urlNameString); URLConnection connection = realUrl.openConnection(); @@ -245,11 +250,11 @@ public class HttpUtils { /** * 向指定 URL 发送带token的POST方法的请求 * - * @param url 发送请求的 URL + * @param url 发送请求的 URL * @param * @return 所代表远程资源的响应结果 */ - public static String sendPostWithToken(String url, Object params, String token) throws Exception { + public static String sendPostWithToken(String url, Object params, String token) throws Exception { String resultStr = null; HttpPost httpPost = new HttpPost(url); if (params != null) { @@ -518,11 +523,10 @@ public class HttpUtils { } - /** * 发送 HTTP 请求并返回二进制数据流（OutputStream）。 * - * @param url 请求的 URL 地址。 + * @param url 请求的 URL 地址。 * @param headers 头节点。 * @return 服务器响应的二进制数据流（OutputStream）。 * @throws IOException 如果请求失败或发生其他 I/O 错误。 @@ -626,7 +630,6 @@ public class HttpUtils { } - // 其他方法保持不变 /** diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelMetaVo.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelMetaVo.java index e1b84905..2a5a538e 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelMetaVo.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelMetaVo.java @@ -59,6 +59,9 @@ public class ModelMetaVo implements Serializable { @ApiModelProperty(value = "指标") private HashMap metrics; + @ApiModelProperty(value = "指标查询参数") + private HashMap metricsParams; + @ApiModelProperty(value = "训练任务") private TrainTaskDepency trainTask; diff --git a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ModelDependency1DaoMapper.xml b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ModelDependency1DaoMapper.xml index 5266f05f..4e77a29b 100644 --- a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ModelDependency1DaoMapper.xml +++ b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ModelDependency1DaoMapper.xml @@ -60,6 +60,15 @@ order by create_time desc limit 1 + + update model_dependency1 set state = 0 @@ -77,7 +86,8 @@ update model_dependency1 - set state = 1 + set state = 1, + meta = #{meta} where repo_id = #{repoId} and identifier = #{identifier} and version = #{version}