| @@ -143,7 +143,7 @@ fi | |||
| if [ "$service" == "all" ]; then | |||
| #部署前端 | |||
| # build_and_deploy "nginx-dockerfile" "172.20.32.187/ci4s/ci4s-front:${tag}" "k8s-12front.yaml" | |||
| build_and_deploy "nginx-dockerfile" "172.20.32.187/ci4s/ci4s-front:${tag}" "k8s-12front.yaml" | |||
| #部署管理平台 | |||
| build_and_deploy "managent-dockerfile" "172.20.32.187/ci4s/ci4s-managent:${tag}" "k8s-7management.yaml" | |||
| #部署认证中心 | |||
| @@ -5,6 +5,7 @@ import com.ruoyi.common.security.annotation.EnableRyFeignClients; | |||
| import com.ruoyi.common.swagger.annotation.EnableCustomSwagger2; | |||
| import org.springframework.boot.SpringApplication; | |||
| import org.springframework.boot.autoconfigure.SpringBootApplication; | |||
| import org.springframework.scheduling.annotation.EnableAsync; | |||
| import org.springframework.scheduling.annotation.EnableScheduling; | |||
| /** | |||
| @@ -17,6 +18,7 @@ import org.springframework.scheduling.annotation.EnableScheduling; | |||
| @EnableRyFeignClients | |||
| @SpringBootApplication | |||
| @EnableScheduling | |||
| @EnableAsync | |||
| public class RuoYiManagementPlatformApplication { | |||
| public static void main(String[] args) { | |||
| SpringApplication.run(RuoYiManagementPlatformApplication.class, args); | |||
| @@ -106,6 +106,12 @@ public class ExperimentInsController extends BaseController { | |||
| return genericsSuccess(this.experimentInsService.removeById(id)); | |||
| } | |||
| @DeleteMapping("batchDelete") | |||
| @ApiOperation("批量删除实验实例") | |||
| public GenericsAjaxResult<String> batchDelete(@RequestBody List<Integer> ids) throws Exception{ | |||
| return genericsSuccess(this.experimentInsService.batchDelete(ids)); | |||
| } | |||
| /** | |||
| * 终止实验实例 | |||
| * | |||
| @@ -2,11 +2,9 @@ package com.ruoyi.platform.controller.model; | |||
| import com.ruoyi.common.core.web.domain.AjaxResult; | |||
| import com.ruoyi.platform.service.ModelsService; | |||
| import com.ruoyi.platform.utils.DVCUtils; | |||
| import com.ruoyi.platform.vo.ModelsVo; | |||
| import io.swagger.annotations.Api; | |||
| import io.swagger.annotations.ApiOperation; | |||
| import org.eclipse.jgit.api.errors.GitAPIException; | |||
| import org.springframework.core.io.InputStreamResource; | |||
| import org.springframework.data.domain.PageRequest; | |||
| import org.springframework.http.ResponseEntity; | |||
| @@ -14,7 +12,6 @@ import org.springframework.web.bind.annotation.*; | |||
| import org.springframework.web.multipart.MultipartFile; | |||
| import javax.annotation.Resource; | |||
| import java.io.IOException; | |||
| import java.util.List; | |||
| import java.util.Map; | |||
| @@ -77,6 +74,23 @@ public class NewModelFromGitController { | |||
| } | |||
| } | |||
| @GetMapping("/queryVersions") | |||
| @ApiOperation("分页查询模型版本") | |||
| public AjaxResult queryVersions(@RequestParam(value = "page") int page, | |||
| @RequestParam(value = "size") int size, | |||
| @RequestParam("identifier") String identifier, | |||
| @RequestParam("owner") String owner) throws Exception { | |||
| PageRequest pageRequest = PageRequest.of(page, size); | |||
| return AjaxResult.success(this.modelsService.queryVersions(pageRequest, identifier, owner)); | |||
| } | |||
| @GetMapping("/queryVersionsMetrics") | |||
| @ApiOperation("查询版本指标") | |||
| public AjaxResult queryVersionsMetrics(@RequestParam("runIds") List<String> runIds) throws Exception { | |||
| return AjaxResult.success(this.modelsService.queryVersionsMetrics(runIds)); | |||
| } | |||
| @GetMapping("/getVersionList") | |||
| @ApiOperation(value = "获取模型分支列表") | |||
| public AjaxResult getVersionList(@RequestParam("identifier") String identifier, @RequestParam("owner") String owner) throws Exception { | |||
| @@ -9,7 +9,7 @@ public interface ModelDependency1Dao { | |||
| int insert(ModelDependency1 modelDependency1); | |||
| int updateState(@Param("repoId") Integer repoId, @Param("identifier") String identifier, @Param("version") String version, @Param("state") Integer state); | |||
| int updateState(@Param("repoId") Integer repoId, @Param("identifier") String identifier, @Param("version") String version, @Param("meta") String meta, @Param("state") Integer state); | |||
| List<ModelDependency1> queryModelDependency(@Param("modelName") String modelName, @Param("repoId") Integer repoId, @Param("owner") String owner); | |||
| @@ -24,4 +24,6 @@ public interface ModelDependency1Dao { | |||
| int deleteModel(@Param("repoId") Integer repoId, @Param("identifier") String identifier, @Param("owner") String owner, @Param("version") String version); | |||
| int deleteModelDependency(@Param("parentModel") String parentModel); | |||
| String getMeta(@Param("identifier") String identifier, @Param("owner") String owner, @Param("version") String version); | |||
| } | |||
| @@ -2,7 +2,9 @@ package com.ruoyi.platform.service; | |||
| import com.ruoyi.platform.vo.InsMetricInfoVo; | |||
| import java.util.HashMap; | |||
| import java.util.List; | |||
| import java.util.Map; | |||
| public interface AimService { | |||
| @@ -13,4 +15,8 @@ public interface AimService { | |||
| List<InsMetricInfoVo> getExpEvaluateInfos(Integer experimentId) throws Exception; | |||
| String getExpMetrics(List<String> runIds) throws Exception; | |||
| HashMap<String, Object> queryMetricsParams(String runId) throws Exception; | |||
| List<Map<String, Object>> getBatchMetric(String runHash, String params); | |||
| } | |||
| @@ -73,6 +73,9 @@ public interface ExperimentInsService { | |||
| * @param id 主键 | |||
| * @return 是否成功 | |||
| */ | |||
| String batchDelete(List<Integer> ids); | |||
| boolean deleteById(Integer id); | |||
| @@ -99,6 +99,10 @@ public interface ModelsService { | |||
| Page<ModelsVo> newPersonalQueryByPage(ModelsVo modelsVo, PageRequest pageRequest) throws Exception; | |||
| Page<Map<String, Object>> queryVersions(PageRequest pageRequest, String identifier, String owner) throws Exception; | |||
| List<List<Map<String, Object>>> queryVersionsMetrics(List<String> runIds) throws Exception; | |||
| List<Map<String, Object>> getVersionList(String identifier, String owner) throws Exception; | |||
| ModelsVo getModelDetail(Integer id, String identifier, String owner, String version) throws Exception; | |||
| @@ -1,18 +1,17 @@ | |||
| package com.ruoyi.platform.service.impl; | |||
| import com.alibaba.fastjson2.JSON; | |||
| import com.ruoyi.platform.domain.ExperimentIns; | |||
| import com.ruoyi.platform.service.AimService; | |||
| import com.ruoyi.platform.service.ExperimentInsService; | |||
| import com.ruoyi.platform.utils.AIM64EncoderUtil; | |||
| import com.ruoyi.platform.utils.JacksonUtil; | |||
| import com.ruoyi.platform.utils.JsonUtils; | |||
| import com.ruoyi.platform.utils.NewHttpUtils; | |||
| import com.ruoyi.platform.utils.*; | |||
| import com.ruoyi.platform.vo.InsMetricInfoVo; | |||
| import org.apache.commons.lang3.StringUtils; | |||
| import org.springframework.beans.factory.annotation.Value; | |||
| import org.springframework.stereotype.Service; | |||
| import javax.annotation.Resource; | |||
| import java.io.UnsupportedEncodingException; | |||
| import java.net.URLEncoder; | |||
| import java.util.*; | |||
| import java.util.stream.Collectors; | |||
| @@ -26,8 +25,6 @@ public class AimServiceImpl implements AimService { | |||
| private String aimUrl; | |||
| @Value("${aim.proxyUrl}") | |||
| private String aimProxyUrl; | |||
| @Resource | |||
| private NewHttpUtils httpUtils; | |||
| @Override | |||
| public List<InsMetricInfoVo> getExpTrainInfos(Integer experimentId) throws Exception { | |||
| @@ -52,7 +49,7 @@ public class AimServiceImpl implements AimService { | |||
| } | |||
| String encodedUrlString = URLEncoder.encode("run.experiment==\"" + experimentName + "\"", "UTF-8"); | |||
| String url = aimProxyUrl + "/api/runs/search/run?query=" + encodedUrlString; | |||
| String s = httpUtils.sendGet(url, null); | |||
| String s = HttpUtils.sendGet(url, null); | |||
| List<Map<String, Object>> response = JacksonUtil.parseJSONStr2MapList(s); | |||
| System.out.println("response: " + JacksonUtil.toJSONString(response)); | |||
| if (response == null || response.size() == 0) { | |||
| @@ -142,7 +139,7 @@ public class AimServiceImpl implements AimService { | |||
| public List<InsMetricInfoVo> getExpTrainInfos1(boolean isTrain, Integer experimentId, String runId) throws Exception { | |||
| String encodedUrlString = URLEncoder.encode("run.id==\"" + runId + "\"", "UTF-8"); | |||
| String url = aimProxyUrl + "/api/runs/search/run?query=" + encodedUrlString; | |||
| String s = httpUtils.sendGet(url, null); | |||
| String s = HttpUtils.sendGet(url, null); | |||
| List<Map<String, Object>> response = JacksonUtil.parseJSONStr2MapList(s); | |||
| System.out.println("response: " + JacksonUtil.toJSONString(response)); | |||
| if (response == null || response.size() == 0) { | |||
| @@ -245,4 +242,44 @@ public class AimServiceImpl implements AimService { | |||
| } | |||
| return datasetList; | |||
| } | |||
| @Override | |||
| public HashMap<String, Object> queryMetricsParams(String runId) throws UnsupportedEncodingException { | |||
| String encodedUrlString = URLEncoder.encode("run.id==\"" + runId + "\"", "UTF-8"); | |||
| String url = aimProxyUrl + "/api/runs/search/run?query=" + encodedUrlString; | |||
| String s = HttpUtils.sendGet(url, null); | |||
| List<Map<String, Object>> response = JacksonUtil.parseJSONStr2MapList(s); | |||
| if (response == null || response.size() == 0) { | |||
| return new HashMap<>(); | |||
| } | |||
| HashMap<String, Object> resultMap = new HashMap<>(); | |||
| List<Map<String, Object>> paramList = new ArrayList<>(); | |||
| Map<String, Object> map = response.get(0); | |||
| LinkedHashMap<String, ArrayList> traces = (LinkedHashMap<String, ArrayList>) map.get("traces"); | |||
| if (traces != null) { | |||
| List<Map<String, Object>> metrics = traces.get("metric"); | |||
| for (Map<String, Object> metric : metrics) { | |||
| Map<String, Object> metricParam = new HashMap<>(); | |||
| metricParam.put("context", metric.get("context")); | |||
| metricParam.put("name", metric.get("name")); | |||
| paramList.add(metricParam); | |||
| } | |||
| resultMap.put("params", JSON.toJSONString(paramList)); | |||
| } | |||
| resultMap.put("run_hash", map.get("run_hash")); | |||
| return resultMap; | |||
| } | |||
| @Override | |||
| public List<Map<String, Object>> getBatchMetric(String runHash, String params) { | |||
| String url = aimUrl + "/api/runs/" + runHash + "/metric/get-batch"; | |||
| String response = HttpUtils.sendPost(url, params); | |||
| if (StringUtils.isNotEmpty(response)) { | |||
| return JacksonUtil.parseJSONStr2MapList(response); | |||
| } | |||
| return null; | |||
| } | |||
| } | |||
| @@ -36,13 +36,13 @@ public class ExperimentInsServiceImpl implements ExperimentInsService { | |||
| @Resource | |||
| private WorkflowService workflowService; | |||
| @Value("${argo.url}") | |||
| private String argoUrl; | |||
| private String argoUrl; | |||
| @Value("${argo.convert}") | |||
| private String argoConvert; | |||
| private String argoConvert; | |||
| @Value("${argo.workflowRun}") | |||
| private String argoWorkflowRun; | |||
| private String argoWorkflowRun; | |||
| @Value("${argo.workflowStatus}") | |||
| private String argoWorkflowStatus; | |||
| private String argoWorkflowStatus; | |||
| @Value("${argo.workflowTermination}") | |||
| private String argoWorkflowTermination; | |||
| @Value("${argo.workflowLog}") | |||
| @@ -60,6 +60,7 @@ public class ExperimentInsServiceImpl implements ExperimentInsService { | |||
| @Resource | |||
| private K8sClientUtil k8sClientUtil; | |||
| private final MinioUtil minioUtil; | |||
| public ExperimentInsServiceImpl(MinioUtil minioUtil) { | |||
| this.minioUtil = minioUtil; | |||
| } | |||
| @@ -97,8 +98,6 @@ public class ExperimentInsServiceImpl implements ExperimentInsService { | |||
| } | |||
| /** | |||
| * | |||
| * | |||
| * 根据实验ID查找所有具有相同ID的实例,并将它们添加到实验列表中 | |||
| * | |||
| * @param experimentId 实验ID | |||
| @@ -187,8 +186,6 @@ public class ExperimentInsServiceImpl implements ExperimentInsService { | |||
| } | |||
| /** | |||
| * 修改数据 | |||
| * | |||
| @@ -207,7 +204,7 @@ public class ExperimentInsServiceImpl implements ExperimentInsService { | |||
| @Override | |||
| public String removeById(Integer id) { | |||
| ExperimentIns experimentIns = experimentInsDao.queryById(id); | |||
| if (experimentIns == null){ | |||
| if (experimentIns == null) { | |||
| return "实验实例不存在"; | |||
| } | |||
| @@ -215,21 +212,31 @@ public class ExperimentInsServiceImpl implements ExperimentInsService { | |||
| LoginUser loginUser = SecurityUtils.getLoginUser(); | |||
| String username = loginUser.getUsername(); | |||
| String createdBy = experimentIns.getCreateBy(); | |||
| if (!(StringUtils.equals(username,"admin") || StringUtils.equals(username,createdBy))){ | |||
| if (!(StringUtils.equals(username, "admin") || StringUtils.equals(username, createdBy))) { | |||
| return "无权限删除该流水线"; | |||
| } | |||
| if (StringUtils.isEmpty(experimentIns.getStatus())){ | |||
| if (StringUtils.isEmpty(experimentIns.getStatus())) { | |||
| experimentIns = queryStatusFromArgo(experimentIns); | |||
| } | |||
| if (StringUtils.equals(experimentIns.getStatus(),"Running")){ | |||
| if (StringUtils.equals(experimentIns.getStatus(), "Running")) { | |||
| return "实验实例正在运行,不可删除"; | |||
| } | |||
| experimentIns.setState(0); | |||
| return this.experimentInsDao.update(experimentIns)>0?"删除成功":"删除失败"; | |||
| return this.experimentInsDao.update(experimentIns) > 0 ? "删除成功" : "删除失败"; | |||
| } | |||
| @Override | |||
| public String batchDelete(List<Integer> ids) { | |||
| for (Integer id : ids) { | |||
| String result = removeById(id); | |||
| if (!"删除成功".equals(result)) { | |||
| return result; | |||
| } | |||
| } | |||
| return "删除成功"; | |||
| } | |||
| /** | |||
| @@ -256,16 +263,16 @@ public class ExperimentInsServiceImpl implements ExperimentInsService { | |||
| Integer id = ins.getId(); | |||
| // 创建请求数据map | |||
| ExperimentIns experimentIns = this.experimentInsDao.queryById(id); | |||
| Map<String,Object> requestData = new HashMap<>(); | |||
| Map<String, Object> requestData = new HashMap<>(); | |||
| requestData.put("namespace", namespace); | |||
| requestData.put("name", name); | |||
| // 创建发送数据map,将请求数据作为"data"键的值 | |||
| Map<String,Object> res = new HashMap<>(); | |||
| Map<String, Object> res = new HashMap<>(); | |||
| res.put("data", requestData); | |||
| try { | |||
| // 发送POST请求到Argo工作流状态查询接口,并将请求数据转换为JSON | |||
| String req = httpUtils.sendPost(argoUrl + argoWorkflowStatus,null, JsonUtils.mapToJson(res)); | |||
| String req = httpUtils.sendPost(argoUrl + argoWorkflowStatus, null, JsonUtils.mapToJson(res)); | |||
| // 检查响应是否为空或无内容 | |||
| if (req == null || StringUtils.isEmpty(req)) { | |||
| throw new RuntimeException("工作流状态响应为空。"); | |||
| @@ -293,10 +300,10 @@ public class ExperimentInsServiceImpl implements ExperimentInsService { | |||
| // 解析nodes字段,提取节点状态并转换为JSON字符串 | |||
| Map<String, Object> nodes = (Map<String, Object>) status.get("nodes"); | |||
| Map<String, Object> modifiedNodes = new LinkedHashMap<>(); | |||
| if (nodes != null ) { | |||
| Map<String, Object> modifiedNodes = new LinkedHashMap<>(); | |||
| if (nodes != null) { | |||
| for (Map.Entry<String, Object> nodeEntry : nodes.entrySet()) { | |||
| Map<String,Object> nodeDetails = (Map<String, Object>) nodeEntry.getValue(); | |||
| Map<String, Object> nodeDetails = (Map<String, Object>) nodeEntry.getValue(); | |||
| String templateName = (String) nodeDetails.get("displayName"); | |||
| modifiedNodes.put(templateName, nodeDetails); | |||
| } | |||
| @@ -306,10 +313,10 @@ public class ExperimentInsServiceImpl implements ExperimentInsService { | |||
| experimentIns.setNodesStatus(nodeStatusJson); | |||
| //终止态为终止不改 | |||
| if (!StringUtils.equals(experimentIns.getStatus(),"Terminated")) { | |||
| experimentIns.setStatus(StringUtils.isNotEmpty((String) status.get("phase"))?(String) status.get("phase"):"Pending"); | |||
| if (!StringUtils.equals(experimentIns.getStatus(), "Terminated")) { | |||
| experimentIns.setStatus(StringUtils.isNotEmpty((String) status.get("phase")) ? (String) status.get("phase") : "Pending"); | |||
| } | |||
| if (StringUtils.equals(experimentIns.getStatus(),"Error")) { | |||
| if (StringUtils.equals(experimentIns.getStatus(), "Error")) { | |||
| experimentIns.setStatus("Failed"); | |||
| } | |||
| @@ -350,16 +357,16 @@ public class ExperimentInsServiceImpl implements ExperimentInsService { | |||
| } | |||
| // 创建请求数据map | |||
| Map<String,Object> requestData = new HashMap<>(); | |||
| Map<String, Object> requestData = new HashMap<>(); | |||
| requestData.put("namespace", namespace); | |||
| requestData.put("name", name); | |||
| // 创建发送数据map,将请求数据作为"data"键的值 | |||
| Map<String,Object> res = new HashMap<>(); | |||
| Map<String, Object> res = new HashMap<>(); | |||
| res.put("data", requestData); | |||
| try { | |||
| // 发送POST请求到Argo工作流状态查询接口,并将请求数据转换为JSON | |||
| String req = httpUtils.sendPost(argoUrl + argoWorkflowTermination,null, JsonUtils.mapToJson(res)); | |||
| String req = httpUtils.sendPost(argoUrl + argoWorkflowTermination, null, JsonUtils.mapToJson(res)); | |||
| // 检查响应是否为空或无内容 | |||
| if (StringUtils.isEmpty(req)) { | |||
| throw new RuntimeException("终止响应内容为空。"); | |||
| @@ -423,20 +430,20 @@ public class ExperimentInsServiceImpl implements ExperimentInsService { | |||
| String namespace = experimentIns.getArgoInsNs(); | |||
| // 创建请求数据的Json(map) | |||
| Map<String,Object> requestData = new HashMap<>(); | |||
| Map<String, Object> requestData = new HashMap<>(); | |||
| requestData.put("namespace", namespace); | |||
| requestData.put("name", name); | |||
| //先写死这两个数据项 | |||
| requestData.put("workflow-type","workflows"); | |||
| requestData.put("artifact-name","main-logs"); | |||
| requestData.put("component-id",componentId); | |||
| requestData.put("workflow-type", "workflows"); | |||
| requestData.put("artifact-name", "main-logs"); | |||
| requestData.put("component-id", componentId); | |||
| // 创建发送数据map,将请求数据作为"data"键的值 | |||
| Map<String,Object> res = new HashMap<>(); | |||
| Map<String, Object> res = new HashMap<>(); | |||
| res.put("data", requestData); | |||
| try { | |||
| // 发送POST请求到Argo工作流日志查询接口,并将请求数据转换为JSON | |||
| String req = httpUtils.sendPost(argoUrl + argoWorkflowLog, null,JsonUtils.mapToJson(res)); | |||
| String req = httpUtils.sendPost(argoUrl + argoWorkflowLog, null, JsonUtils.mapToJson(res)); | |||
| // 检查响应是否为空或无内容 | |||
| if (StringUtils.isEmpty(req)) { | |||
| throw new RuntimeException("响应内容为空"); | |||
| @@ -445,7 +452,7 @@ public class ExperimentInsServiceImpl implements ExperimentInsService { | |||
| Map<String, Object> runResMap = JsonUtils.jsonToMap(req); | |||
| // 从响应Map中获取"data"的值,也就是日志的值 | |||
| String experimentInsLog = (String) runResMap.get("data"); | |||
| if (experimentInsLog == null){ | |||
| if (experimentInsLog == null) { | |||
| throw new RuntimeException("日志为空。"); | |||
| } | |||
| //返回日志内容 | |||
| @@ -463,28 +470,28 @@ public class ExperimentInsServiceImpl implements ExperimentInsService { | |||
| //查询 实例 | |||
| ExperimentIns experimentIns = this.experimentInsDao.queryById(id); | |||
| if (experimentIns == null) { | |||
| throw new Exception("实验实例未查询到,id:" + id); | |||
| throw new Exception("实验实例未查询到,id:" + id); | |||
| } | |||
| //找到 节点 节点结果 | |||
| String nodesResultString = experimentIns.getNodesResult(); | |||
| if (StringUtils.isEmpty(nodesResultString)) { | |||
| throw new Exception("实验实例未查询到节点结果,id:" + id); | |||
| throw new Exception("实验实例未查询到节点结果,id:" + id); | |||
| } | |||
| Map<String, Object> nodesResult = JsonUtils.jsonToMap(nodesResultString); | |||
| Map<String, Object> paramOutput = (Map<String, Object>) nodesResult.get("param_output"); | |||
| List<Map<String,Object>> nodeList = (List<Map<String,Object>>) paramOutput.get(nodeId); | |||
| List<Map<String, Object>> nodeList = (List<Map<String, Object>>) paramOutput.get(nodeId); | |||
| //遍历 查询minio | |||
| for (int i = 0; i < nodeList.size(); i++) { | |||
| Map<String, Object> map = nodeList.get(i); | |||
| String path = (String) map.get("path"); | |||
| String bucketName = path.substring(0, path.indexOf("/")); | |||
| String prefix = path.substring(path.indexOf("/")+1,path.length())+"/"; | |||
| if (StringUtils.equals( (String)map.get("type"),"file")){ | |||
| String prefix = path.substring(path.indexOf("/") + 1, path.length()) + "/"; | |||
| if (StringUtils.equals((String) map.get("type"), "file")) { | |||
| List<Map> fileInfo = minioUtil.listFilesInDirectory(bucketName, prefix); | |||
| map.put("value",fileInfo); | |||
| }else if (StringUtils.equals( (String)map.get("type"),"string")){ | |||
| map.put("value", fileInfo); | |||
| } else if (StringUtils.equals((String) map.get("type"), "string")) { | |||
| String resultInfo = minioUtil.readObjectAsString(bucketName, prefix); | |||
| map.put("value",resultInfo); | |||
| map.put("value", resultInfo); | |||
| } | |||
| results.add(map); | |||
| //组装 | |||
| @@ -501,14 +508,14 @@ public class ExperimentInsServiceImpl implements ExperimentInsService { | |||
| String taskId = logRequest.getTaskId(); | |||
| String startTime = logRequest.getStartTime(); | |||
| // 创建请求数据的Json(map) | |||
| Map<String,Object> requestData = new HashMap<>(); | |||
| Map<String, Object> requestData = new HashMap<>(); | |||
| requestData.put("component_id", componentId); | |||
| requestData.put("namespace", nameSpace); | |||
| requestData.put("name", name); | |||
| requestData.put("task_id",taskId); | |||
| requestData.put("start_time",startTime); | |||
| requestData.put("task_id", taskId); | |||
| requestData.put("start_time", startTime); | |||
| // 创建发送数据map,将请求数据作为"data"键的值 | |||
| Map<String,Object> res = new HashMap<>(); | |||
| Map<String, Object> res = new HashMap<>(); | |||
| res.put("data", requestData); | |||
| try { | |||
| @@ -531,9 +538,9 @@ public class ExperimentInsServiceImpl implements ExperimentInsService { | |||
| @Override | |||
| public Map<String, Object> getRealtimePodLog(String podName, String startTime) { | |||
| Map<String,Object> requestData = new HashMap<>(); | |||
| requestData.put("pod_name",podName); | |||
| requestData.put("start_time",startTime); | |||
| Map<String, Object> requestData = new HashMap<>(); | |||
| requestData.put("pod_name", podName); | |||
| requestData.put("start_time", startTime); | |||
| try { | |||
| // 将Map转换为JSON字符串 | |||
| String req = HttpUtils.sendPost(argoUrl + argoWorkflowPodLog, JsonUtils.mapToJson(requestData)); | |||
| @@ -554,7 +561,7 @@ public class ExperimentInsServiceImpl implements ExperimentInsService { | |||
| @Override | |||
| public String getRealtimePodLogFromPod(PodLogVo podLogVo) { | |||
| return k8sClientUtil.getPodLogs(podLogVo.getPodName(), podLogVo.getNamespace(),podLogVo.getContainerName(), logsLines); | |||
| return k8sClientUtil.getPodLogs(podLogVo.getPodName(), podLogVo.getNamespace(), podLogVo.getContainerName(), logsLines); | |||
| } | |||
| @Override | |||
| @@ -573,7 +580,7 @@ public class ExperimentInsServiceImpl implements ExperimentInsService { | |||
| experimentIns.setStatus(status.toString()); // 设置实例的状态为枚举中的状态 | |||
| // 查询具有相同状态的实例数量 | |||
| Long count = experimentInsDao.count(experimentIns); | |||
| Long count = experimentInsDao.count(experimentIns); | |||
| // 将状态及其对应的实例数量放入map中 | |||
| statusCountMap.put(status.toString(), count); | |||
| @@ -588,7 +595,7 @@ public class ExperimentInsServiceImpl implements ExperimentInsService { | |||
| boolean flag = true; | |||
| List<String> terminatedStates = Arrays.asList("Succeeded", "Failed"); | |||
| flag = terminatedStates.contains(status); | |||
| if (StringUtils.equals(status, "Terminated")){ | |||
| if (StringUtils.equals(status, "Terminated")) { | |||
| //如果跟node_status里面不一样,就要去更新node_status的信息 | |||
| String nodesStatus = ins.getNodesStatus(); | |||
| Map<String, Object> nodeMap = JsonUtils.jsonToMap(nodesStatus); | |||
| @@ -597,7 +604,7 @@ public class ExperimentInsServiceImpl implements ExperimentInsService { | |||
| .findFirst() | |||
| .orElse(null); | |||
| Map workflowMap = (Map) nodeMap.get(keyStartsWithWorkflow); | |||
| if (workflowMap != null){ | |||
| if (workflowMap != null) { | |||
| flag = StringUtils.equals("Terminated", (String) workflowMap.get("phase")); | |||
| } | |||
| } | |||
| @@ -560,7 +560,7 @@ public class ExperimentServiceImpl implements ExperimentService { | |||
| //处理指标 | |||
| HashMap<String, Object> metricMap = JSON.parseObject(metricRecord, HashMap.class); | |||
| modelMetaVo.setMetrics(metricMap); | |||
| modelMetaVo.setMetricsParams(metricMap); | |||
| //训练数据集 | |||
| List<Map<String, Object>> trainDatasetList = (List<Map<String, Object>>) modelTrainMap.get("datasets"); | |||
| @@ -747,7 +747,7 @@ public class ModelsServiceImpl implements ModelsService { | |||
| if (buildingModel != null) { | |||
| modelMetaVo = JSON.parseObject(buildingModel.getMeta(), ModelMetaVo.class); | |||
| //获取指标 | |||
| getMetrics(modelMetaVo); | |||
| transMetrics(modelMetaVo); | |||
| } | |||
| //拼接生产的元数据后写入yaml文件 | |||
| @@ -806,7 +806,7 @@ public class ModelsServiceImpl implements ModelsService { | |||
| modelDependency1Dao.insert(modelDependency); | |||
| } else { | |||
| //更新模型依赖 | |||
| modelDependency1Dao.updateState(modelsVo.getId(), modelsVo.getIdentifier(), modelsVo.getVersion(), Constant.State_valid); | |||
| modelDependency1Dao.updateState(modelsVo.getId(), modelsVo.getIdentifier(), modelsVo.getVersion(), meta, Constant.State_valid); | |||
| } | |||
| } else { | |||
| //保存模型依赖 | |||
| @@ -973,11 +973,47 @@ public class ModelsServiceImpl implements ModelsService { | |||
| return new PageImpl<>(result, pageRequest, collect.size()); | |||
| } | |||
| @Override | |||
| public Page<Map<String, Object>> queryVersions(PageRequest pageRequest, String identifier, String owner) throws Exception { | |||
| String token = gitService.checkoutToken(); | |||
| List<Map<String, Object>> collect = gitService.getBrancheList(token, owner, identifier); | |||
| List<Map<String, Object>> result = collect.stream() | |||
| .filter(branch -> !"master".equals(branch.get("name"))) | |||
| .skip((pageRequest.getPageNumber()) * pageRequest.getPageSize()).limit(pageRequest.getPageSize()) | |||
| .collect(Collectors.toList()); | |||
| for (Map<String, Object> branch : result) { | |||
| String meta = modelDependency1Dao.getMeta(identifier, owner, (String) branch.get("name")); | |||
| ModelMetaVo modelMetaVo = JSON.parseObject(meta, ModelMetaVo.class); | |||
| if (modelMetaVo.getParams() != null) { | |||
| branch.putAll(modelMetaVo.getParams()); | |||
| } | |||
| if (modelMetaVo.getMetrics() != null) { | |||
| branch.putAll(modelMetaVo.getMetrics()); | |||
| } | |||
| if (modelMetaVo.getMetricsParams() != null) { | |||
| branch.putAll(modelMetaVo.getMetricsParams()); | |||
| } | |||
| } | |||
| return new PageImpl<>(result, pageRequest, collect.size()); | |||
| } | |||
| @Override | |||
| public List<List<Map<String, Object>>> queryVersionsMetrics(List<String> runIds) throws Exception { | |||
| List<List<Map<String, Object>>> batchMetrics = new ArrayList<>(); | |||
| for (String runId : runIds) { | |||
| HashMap<String, Object> map = aimsService.queryMetricsParams(runId); | |||
| List<Map<String, Object>> batchMetric = aimsService.getBatchMetric((String) map.get("run_hash"), (String) map.get("params")); | |||
| batchMetrics.add(batchMetric); | |||
| } | |||
| return batchMetrics; | |||
| } | |||
| @Override | |||
| public List<Map<String, Object>> getVersionList(String identifier, String owner) throws Exception { | |||
| String token = gitService.checkoutToken(); | |||
| List<Map<String, Object>> brancheList = gitService.getBrancheList(token, owner, identifier); | |||
| return brancheList.stream() | |||
| List<Map<String, Object>> branchList = gitService.getBrancheList(token, owner, identifier); | |||
| return branchList.stream() | |||
| .filter(branch -> !"master".equals(branch.get("name"))) | |||
| .collect(Collectors.toList()); | |||
| } | |||
| @@ -1179,12 +1215,12 @@ public class ModelsServiceImpl implements ModelsService { | |||
| return userInfo; | |||
| } | |||
| void getMetrics(ModelMetaVo modelMetaVo) throws Exception { | |||
| void transMetrics(ModelMetaVo modelMetaVo) throws Exception { | |||
| HashMap<String, Object> result = new HashMap<>(); | |||
| HashMap<String, Object> train = new HashMap<>(); | |||
| HashMap<String, Object> evaluate = new HashMap<>(); | |||
| HashMap<String, Object> metrics = modelMetaVo.getMetrics(); | |||
| HashMap<String, Object> metrics = modelMetaVo.getMetricsParams(); | |||
| JSONArray trainMetrics = (JSONArray) metrics.get("train"); | |||
| if (trainMetrics != null) { | |||
| for (int i = 0; i < trainMetrics.size(); i++) { | |||
| @@ -1,5 +1,6 @@ | |||
| package com.ruoyi.platform.utils; | |||
| import com.ruoyi.common.core.utils.StringUtils; | |||
| import lombok.extern.slf4j.Slf4j; | |||
| import org.apache.http.HttpEntity; | |||
| import org.apache.http.HttpResponse; | |||
| @@ -101,12 +102,13 @@ public class HttpUtils { | |||
| // public static String sendGetWithToken(String url, String param,String token) { | |||
| // return sendGet(url, param, "UTF-8",token); | |||
| // } | |||
| /** | |||
| * 向指定 URL 发送带 token 的 GET 方法的请求,使用 Apache HttpClient | |||
| * | |||
| * @param url 发送请求的 URL | |||
| * @param param 请求参数,请求参数应该是 name1=value1&name2=value2 的形式。 | |||
| * @param token 认证 token | |||
| * @param url 发送请求的 URL | |||
| * @param param 请求参数,请求参数应该是 name1=value1&name2=value2 的形式。 | |||
| * @param token 认证 token | |||
| * @return 所代表远程资源的响应结果 | |||
| */ | |||
| public static String sendGetWithToken(String url, String param, String token) { | |||
| @@ -148,6 +150,7 @@ public class HttpUtils { | |||
| } | |||
| return result; | |||
| } | |||
| /** | |||
| * 向指定 URL 发送带token的GET方法的请求 | |||
| * | |||
| @@ -156,7 +159,7 @@ public class HttpUtils { | |||
| * @param contentType 编码类型 | |||
| * @return 所代表远程资源的响应结果 | |||
| */ | |||
| public static String sendGet(String url, String param, String contentType,String token) { | |||
| public static String sendGet(String url, String param, String contentType, String token) { | |||
| StringBuilder result = new StringBuilder(); | |||
| BufferedReader in = null; | |||
| try { | |||
| @@ -196,7 +199,6 @@ public class HttpUtils { | |||
| } | |||
| /** | |||
| * 向指定 URL 发送GET方法的请求 | |||
| * | |||
| @@ -209,7 +211,10 @@ public class HttpUtils { | |||
| StringBuilder result = new StringBuilder(); | |||
| BufferedReader in = null; | |||
| try { | |||
| String urlNameString = url + "?" + param; | |||
| String urlNameString = url; | |||
| if (StringUtils.isNotEmpty(param)) { | |||
| urlNameString = url + "?" + param; | |||
| } | |||
| log.info("sendGet - {}", urlNameString); | |||
| URL realUrl = new URL(urlNameString); | |||
| URLConnection connection = realUrl.openConnection(); | |||
| @@ -245,11 +250,11 @@ public class HttpUtils { | |||
| /** | |||
| * 向指定 URL 发送带token的POST方法的请求 | |||
| * | |||
| * @param url 发送请求的 URL | |||
| * @param url 发送请求的 URL | |||
| * @param | |||
| * @return 所代表远程资源的响应结果 | |||
| */ | |||
| public static String sendPostWithToken(String url, Object params, String token) throws Exception { | |||
| public static String sendPostWithToken(String url, Object params, String token) throws Exception { | |||
| String resultStr = null; | |||
| HttpPost httpPost = new HttpPost(url); | |||
| if (params != null) { | |||
| @@ -518,11 +523,10 @@ public class HttpUtils { | |||
| } | |||
| /** | |||
| * 发送 HTTP 请求并返回二进制数据流(OutputStream)。 | |||
| * | |||
| * @param url 请求的 URL 地址。 | |||
| * @param url 请求的 URL 地址。 | |||
| * @param headers 头节点。 | |||
| * @return 服务器响应的二进制数据流(OutputStream)。 | |||
| * @throws IOException 如果请求失败或发生其他 I/O 错误。 | |||
| @@ -626,7 +630,6 @@ public class HttpUtils { | |||
| } | |||
| // 其他方法保持不变 | |||
| /** | |||
| @@ -59,6 +59,9 @@ public class ModelMetaVo implements Serializable { | |||
| @ApiModelProperty(value = "指标") | |||
| private HashMap<String, Object> metrics; | |||
| @ApiModelProperty(value = "指标查询参数") | |||
| private HashMap<String, Object> metricsParams; | |||
| @ApiModelProperty(value = "训练任务") | |||
| private TrainTaskDepency trainTask; | |||
| @@ -60,6 +60,15 @@ | |||
| order by create_time desc limit 1 | |||
| </select> | |||
| <select id="getMeta" resultType="java.lang.String"> | |||
| select meta from model_dependency1 | |||
| where identifier = #{identifier} | |||
| and owner = #{owner} | |||
| and version = #{version} | |||
| and state = 1 | |||
| order by create_time desc limit 1 | |||
| </select> | |||
| <update id="deleteModel"> | |||
| update model_dependency1 | |||
| set state = 0 | |||
| @@ -77,7 +86,8 @@ | |||
| <update id="updateState"> | |||
| update model_dependency1 | |||
| set state = 1 | |||
| set state = 1, | |||
| meta = #{meta} | |||
| where repo_id = #{repoId} | |||
| and identifier = #{identifier} | |||
| and version = #{version} | |||