Browse Source

Merge remote-tracking branch 'origin/dev' into test

dev-lhz
chenzhihang 1 year ago
parent
commit
e15486728b
15 changed files with 216 additions and 83 deletions
  1. +1
    -1
      k8s/deploy.sh
  2. +2
    -0
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/RuoYiManagementPlatformApplication.java
  3. +6
    -0
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/experiment/ExperimentInsController.java
  4. +17
    -3
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/model/NewModelFromGitController.java
  5. +3
    -1
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/ModelDependency1Dao.java
  6. +6
    -0
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AimService.java
  7. +3
    -0
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ExperimentInsService.java
  8. +4
    -0
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ModelsService.java
  9. +45
    -8
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AimServiceImpl.java
  10. +58
    -51
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentInsServiceImpl.java
  11. +1
    -1
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentServiceImpl.java
  12. +42
    -6
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java
  13. +14
    -11
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/HttpUtils.java
  14. +3
    -0
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelMetaVo.java
  15. +11
    -1
      ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ModelDependency1DaoMapper.xml

+ 1
- 1
k8s/deploy.sh View File

@@ -143,7 +143,7 @@ fi


if [ "$service" == "all" ]; then if [ "$service" == "all" ]; then
#部署前端 #部署前端
# build_and_deploy "nginx-dockerfile" "172.20.32.187/ci4s/ci4s-front:${tag}" "k8s-12front.yaml"
build_and_deploy "nginx-dockerfile" "172.20.32.187/ci4s/ci4s-front:${tag}" "k8s-12front.yaml"
#部署管理平台 #部署管理平台
build_and_deploy "managent-dockerfile" "172.20.32.187/ci4s/ci4s-managent:${tag}" "k8s-7management.yaml" build_and_deploy "managent-dockerfile" "172.20.32.187/ci4s/ci4s-managent:${tag}" "k8s-7management.yaml"
#部署认证中心 #部署认证中心


+ 2
- 0
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/RuoYiManagementPlatformApplication.java View File

@@ -5,6 +5,7 @@ import com.ruoyi.common.security.annotation.EnableRyFeignClients;
import com.ruoyi.common.swagger.annotation.EnableCustomSwagger2; import com.ruoyi.common.swagger.annotation.EnableCustomSwagger2;
import org.springframework.boot.SpringApplication; import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.scheduling.annotation.EnableAsync;
import org.springframework.scheduling.annotation.EnableScheduling; import org.springframework.scheduling.annotation.EnableScheduling;


/** /**
@@ -17,6 +18,7 @@ import org.springframework.scheduling.annotation.EnableScheduling;
@EnableRyFeignClients @EnableRyFeignClients
@SpringBootApplication @SpringBootApplication
@EnableScheduling @EnableScheduling
@EnableAsync
public class RuoYiManagementPlatformApplication { public class RuoYiManagementPlatformApplication {
public static void main(String[] args) { public static void main(String[] args) {
SpringApplication.run(RuoYiManagementPlatformApplication.class, args); SpringApplication.run(RuoYiManagementPlatformApplication.class, args);


+ 6
- 0
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/experiment/ExperimentInsController.java View File

@@ -106,6 +106,12 @@ public class ExperimentInsController extends BaseController {
return genericsSuccess(this.experimentInsService.removeById(id)); return genericsSuccess(this.experimentInsService.removeById(id));
} }


@DeleteMapping("batchDelete")
@ApiOperation("批量删除实验实例")
public GenericsAjaxResult<String> batchDelete(@RequestBody List<Integer> ids) throws Exception{
return genericsSuccess(this.experimentInsService.batchDelete(ids));
}

/** /**
* 终止实验实例 * 终止实验实例
* *


+ 17
- 3
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/model/NewModelFromGitController.java View File

@@ -2,11 +2,9 @@ package com.ruoyi.platform.controller.model;


import com.ruoyi.common.core.web.domain.AjaxResult; import com.ruoyi.common.core.web.domain.AjaxResult;
import com.ruoyi.platform.service.ModelsService; import com.ruoyi.platform.service.ModelsService;
import com.ruoyi.platform.utils.DVCUtils;
import com.ruoyi.platform.vo.ModelsVo; import com.ruoyi.platform.vo.ModelsVo;
import io.swagger.annotations.Api; import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiOperation;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.springframework.core.io.InputStreamResource; import org.springframework.core.io.InputStreamResource;
import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.PageRequest;
import org.springframework.http.ResponseEntity; import org.springframework.http.ResponseEntity;
@@ -14,7 +12,6 @@ import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile; import org.springframework.web.multipart.MultipartFile;


import javax.annotation.Resource; import javax.annotation.Resource;
import java.io.IOException;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;


@@ -77,6 +74,23 @@ public class NewModelFromGitController {
} }
} }


@GetMapping("/queryVersions")
@ApiOperation("分页查询模型版本")
public AjaxResult queryVersions(@RequestParam(value = "page") int page,
@RequestParam(value = "size") int size,
@RequestParam("identifier") String identifier,
@RequestParam("owner") String owner) throws Exception {
PageRequest pageRequest = PageRequest.of(page, size);
return AjaxResult.success(this.modelsService.queryVersions(pageRequest, identifier, owner));
}

@GetMapping("/queryVersionsMetrics")
@ApiOperation("查询版本指标")
public AjaxResult queryVersionsMetrics(@RequestParam("runIds") List<String> runIds) throws Exception {
return AjaxResult.success(this.modelsService.queryVersionsMetrics(runIds));
}


@GetMapping("/getVersionList") @GetMapping("/getVersionList")
@ApiOperation(value = "获取模型分支列表") @ApiOperation(value = "获取模型分支列表")
public AjaxResult getVersionList(@RequestParam("identifier") String identifier, @RequestParam("owner") String owner) throws Exception { public AjaxResult getVersionList(@RequestParam("identifier") String identifier, @RequestParam("owner") String owner) throws Exception {


+ 3
- 1
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/ModelDependency1Dao.java View File

@@ -9,7 +9,7 @@ public interface ModelDependency1Dao {


int insert(ModelDependency1 modelDependency1); int insert(ModelDependency1 modelDependency1);


int updateState(@Param("repoId") Integer repoId, @Param("identifier") String identifier, @Param("version") String version, @Param("state") Integer state);
int updateState(@Param("repoId") Integer repoId, @Param("identifier") String identifier, @Param("version") String version, @Param("meta") String meta, @Param("state") Integer state);


List<ModelDependency1> queryModelDependency(@Param("modelName") String modelName, @Param("repoId") Integer repoId, @Param("owner") String owner); List<ModelDependency1> queryModelDependency(@Param("modelName") String modelName, @Param("repoId") Integer repoId, @Param("owner") String owner);


@@ -24,4 +24,6 @@ public interface ModelDependency1Dao {
int deleteModel(@Param("repoId") Integer repoId, @Param("identifier") String identifier, @Param("owner") String owner, @Param("version") String version); int deleteModel(@Param("repoId") Integer repoId, @Param("identifier") String identifier, @Param("owner") String owner, @Param("version") String version);


int deleteModelDependency(@Param("parentModel") String parentModel); int deleteModelDependency(@Param("parentModel") String parentModel);

String getMeta(@Param("identifier") String identifier, @Param("owner") String owner, @Param("version") String version);
} }

+ 6
- 0
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AimService.java View File

@@ -2,7 +2,9 @@ package com.ruoyi.platform.service;


import com.ruoyi.platform.vo.InsMetricInfoVo; import com.ruoyi.platform.vo.InsMetricInfoVo;


import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map;


public interface AimService { public interface AimService {


@@ -13,4 +15,8 @@ public interface AimService {
List<InsMetricInfoVo> getExpEvaluateInfos(Integer experimentId) throws Exception; List<InsMetricInfoVo> getExpEvaluateInfos(Integer experimentId) throws Exception;


String getExpMetrics(List<String> runIds) throws Exception; String getExpMetrics(List<String> runIds) throws Exception;

HashMap<String, Object> queryMetricsParams(String runId) throws Exception;

List<Map<String, Object>> getBatchMetric(String runHash, String params);
} }

+ 3
- 0
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ExperimentInsService.java View File

@@ -73,6 +73,9 @@ public interface ExperimentInsService {
* @param id 主键 * @param id 主键
* @return 是否成功 * @return 是否成功
*/ */

String batchDelete(List<Integer> ids);

boolean deleteById(Integer id); boolean deleteById(Integer id);






+ 4
- 0
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ModelsService.java View File

@@ -99,6 +99,10 @@ public interface ModelsService {


Page<ModelsVo> newPersonalQueryByPage(ModelsVo modelsVo, PageRequest pageRequest) throws Exception; Page<ModelsVo> newPersonalQueryByPage(ModelsVo modelsVo, PageRequest pageRequest) throws Exception;


Page<Map<String, Object>> queryVersions(PageRequest pageRequest, String identifier, String owner) throws Exception;

List<List<Map<String, Object>>> queryVersionsMetrics(List<String> runIds) throws Exception;

List<Map<String, Object>> getVersionList(String identifier, String owner) throws Exception; List<Map<String, Object>> getVersionList(String identifier, String owner) throws Exception;


ModelsVo getModelDetail(Integer id, String identifier, String owner, String version) throws Exception; ModelsVo getModelDetail(Integer id, String identifier, String owner, String version) throws Exception;


+ 45
- 8
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AimServiceImpl.java View File

@@ -1,18 +1,17 @@
package com.ruoyi.platform.service.impl; package com.ruoyi.platform.service.impl;


import com.alibaba.fastjson2.JSON;
import com.ruoyi.platform.domain.ExperimentIns; import com.ruoyi.platform.domain.ExperimentIns;
import com.ruoyi.platform.service.AimService; import com.ruoyi.platform.service.AimService;
import com.ruoyi.platform.service.ExperimentInsService; import com.ruoyi.platform.service.ExperimentInsService;
import com.ruoyi.platform.utils.AIM64EncoderUtil;
import com.ruoyi.platform.utils.JacksonUtil;
import com.ruoyi.platform.utils.JsonUtils;
import com.ruoyi.platform.utils.NewHttpUtils;
import com.ruoyi.platform.utils.*;
import com.ruoyi.platform.vo.InsMetricInfoVo; import com.ruoyi.platform.vo.InsMetricInfoVo;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;


import javax.annotation.Resource; import javax.annotation.Resource;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder; import java.net.URLEncoder;
import java.util.*; import java.util.*;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@@ -26,8 +25,6 @@ public class AimServiceImpl implements AimService {
private String aimUrl; private String aimUrl;
@Value("${aim.proxyUrl}") @Value("${aim.proxyUrl}")
private String aimProxyUrl; private String aimProxyUrl;
@Resource
private NewHttpUtils httpUtils;


@Override @Override
public List<InsMetricInfoVo> getExpTrainInfos(Integer experimentId) throws Exception { public List<InsMetricInfoVo> getExpTrainInfos(Integer experimentId) throws Exception {
@@ -52,7 +49,7 @@ public class AimServiceImpl implements AimService {
} }
String encodedUrlString = URLEncoder.encode("run.experiment==\"" + experimentName + "\"", "UTF-8"); String encodedUrlString = URLEncoder.encode("run.experiment==\"" + experimentName + "\"", "UTF-8");
String url = aimProxyUrl + "/api/runs/search/run?query=" + encodedUrlString; String url = aimProxyUrl + "/api/runs/search/run?query=" + encodedUrlString;
String s = httpUtils.sendGet(url, null);
String s = HttpUtils.sendGet(url, null);
List<Map<String, Object>> response = JacksonUtil.parseJSONStr2MapList(s); List<Map<String, Object>> response = JacksonUtil.parseJSONStr2MapList(s);
System.out.println("response: " + JacksonUtil.toJSONString(response)); System.out.println("response: " + JacksonUtil.toJSONString(response));
if (response == null || response.size() == 0) { if (response == null || response.size() == 0) {
@@ -142,7 +139,7 @@ public class AimServiceImpl implements AimService {
public List<InsMetricInfoVo> getExpTrainInfos1(boolean isTrain, Integer experimentId, String runId) throws Exception { public List<InsMetricInfoVo> getExpTrainInfos1(boolean isTrain, Integer experimentId, String runId) throws Exception {
String encodedUrlString = URLEncoder.encode("run.id==\"" + runId + "\"", "UTF-8"); String encodedUrlString = URLEncoder.encode("run.id==\"" + runId + "\"", "UTF-8");
String url = aimProxyUrl + "/api/runs/search/run?query=" + encodedUrlString; String url = aimProxyUrl + "/api/runs/search/run?query=" + encodedUrlString;
String s = httpUtils.sendGet(url, null);
String s = HttpUtils.sendGet(url, null);
List<Map<String, Object>> response = JacksonUtil.parseJSONStr2MapList(s); List<Map<String, Object>> response = JacksonUtil.parseJSONStr2MapList(s);
System.out.println("response: " + JacksonUtil.toJSONString(response)); System.out.println("response: " + JacksonUtil.toJSONString(response));
if (response == null || response.size() == 0) { if (response == null || response.size() == 0) {
@@ -245,4 +242,44 @@ public class AimServiceImpl implements AimService {
} }
return datasetList; return datasetList;
} }

@Override
public HashMap<String, Object> queryMetricsParams(String runId) throws UnsupportedEncodingException {
String encodedUrlString = URLEncoder.encode("run.id==\"" + runId + "\"", "UTF-8");
String url = aimProxyUrl + "/api/runs/search/run?query=" + encodedUrlString;
String s = HttpUtils.sendGet(url, null);
List<Map<String, Object>> response = JacksonUtil.parseJSONStr2MapList(s);
if (response == null || response.size() == 0) {
return new HashMap<>();
}

HashMap<String, Object> resultMap = new HashMap<>();
List<Map<String, Object>> paramList = new ArrayList<>();

Map<String, Object> map = response.get(0);
LinkedHashMap<String, ArrayList> traces = (LinkedHashMap<String, ArrayList>) map.get("traces");
if (traces != null) {
List<Map<String, Object>> metrics = traces.get("metric");
for (Map<String, Object> metric : metrics) {
Map<String, Object> metricParam = new HashMap<>();
metricParam.put("context", metric.get("context"));
metricParam.put("name", metric.get("name"));
paramList.add(metricParam);
}
resultMap.put("params", JSON.toJSONString(paramList));
}
resultMap.put("run_hash", map.get("run_hash"));
return resultMap;
}

@Override
public List<Map<String, Object>> getBatchMetric(String runHash, String params) {
String url = aimUrl + "/api/runs/" + runHash + "/metric/get-batch";
String response = HttpUtils.sendPost(url, params);
if (StringUtils.isNotEmpty(response)) {
return JacksonUtil.parseJSONStr2MapList(response);
}
return null;
}

} }

+ 58
- 51
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentInsServiceImpl.java View File

@@ -36,13 +36,13 @@ public class ExperimentInsServiceImpl implements ExperimentInsService {
@Resource @Resource
private WorkflowService workflowService; private WorkflowService workflowService;
@Value("${argo.url}") @Value("${argo.url}")
private String argoUrl;
private String argoUrl;
@Value("${argo.convert}") @Value("${argo.convert}")
private String argoConvert;
private String argoConvert;
@Value("${argo.workflowRun}") @Value("${argo.workflowRun}")
private String argoWorkflowRun;
private String argoWorkflowRun;
@Value("${argo.workflowStatus}") @Value("${argo.workflowStatus}")
private String argoWorkflowStatus;
private String argoWorkflowStatus;
@Value("${argo.workflowTermination}") @Value("${argo.workflowTermination}")
private String argoWorkflowTermination; private String argoWorkflowTermination;
@Value("${argo.workflowLog}") @Value("${argo.workflowLog}")
@@ -60,6 +60,7 @@ public class ExperimentInsServiceImpl implements ExperimentInsService {
@Resource @Resource
private K8sClientUtil k8sClientUtil; private K8sClientUtil k8sClientUtil;
private final MinioUtil minioUtil; private final MinioUtil minioUtil;

public ExperimentInsServiceImpl(MinioUtil minioUtil) { public ExperimentInsServiceImpl(MinioUtil minioUtil) {
this.minioUtil = minioUtil; this.minioUtil = minioUtil;
} }
@@ -97,8 +98,6 @@ public class ExperimentInsServiceImpl implements ExperimentInsService {
} }


/** /**
*
*
* 根据实验ID查找所有具有相同ID的实例,并将它们添加到实验列表中 * 根据实验ID查找所有具有相同ID的实例,并将它们添加到实验列表中
* *
* @param experimentId 实验ID * @param experimentId 实验ID
@@ -187,8 +186,6 @@ public class ExperimentInsServiceImpl implements ExperimentInsService {
} }






/** /**
* 修改数据 * 修改数据
* *
@@ -207,7 +204,7 @@ public class ExperimentInsServiceImpl implements ExperimentInsService {
@Override @Override
public String removeById(Integer id) { public String removeById(Integer id) {
ExperimentIns experimentIns = experimentInsDao.queryById(id); ExperimentIns experimentIns = experimentInsDao.queryById(id);
if (experimentIns == null){
if (experimentIns == null) {
return "实验实例不存在"; return "实验实例不存在";
} }


@@ -215,21 +212,31 @@ public class ExperimentInsServiceImpl implements ExperimentInsService {
LoginUser loginUser = SecurityUtils.getLoginUser(); LoginUser loginUser = SecurityUtils.getLoginUser();
String username = loginUser.getUsername(); String username = loginUser.getUsername();
String createdBy = experimentIns.getCreateBy(); String createdBy = experimentIns.getCreateBy();
if (!(StringUtils.equals(username,"admin") || StringUtils.equals(username,createdBy))){
if (!(StringUtils.equals(username, "admin") || StringUtils.equals(username, createdBy))) {
return "无权限删除该流水线"; return "无权限删除该流水线";
} }


if (StringUtils.isEmpty(experimentIns.getStatus())){
if (StringUtils.isEmpty(experimentIns.getStatus())) {
experimentIns = queryStatusFromArgo(experimentIns); experimentIns = queryStatusFromArgo(experimentIns);
} }
if (StringUtils.equals(experimentIns.getStatus(),"Running")){
if (StringUtils.equals(experimentIns.getStatus(), "Running")) {
return "实验实例正在运行,不可删除"; return "实验实例正在运行,不可删除";
} }
experimentIns.setState(0); experimentIns.setState(0);
return this.experimentInsDao.update(experimentIns)>0?"删除成功":"删除失败";
return this.experimentInsDao.update(experimentIns) > 0 ? "删除成功" : "删除失败";


} }


@Override
public String batchDelete(List<Integer> ids) {
for (Integer id : ids) {
String result = removeById(id);
if (!"删除成功".equals(result)) {
return result;
}
}
return "删除成功";
}




/** /**
@@ -256,16 +263,16 @@ public class ExperimentInsServiceImpl implements ExperimentInsService {
Integer id = ins.getId(); Integer id = ins.getId();
// 创建请求数据map // 创建请求数据map
ExperimentIns experimentIns = this.experimentInsDao.queryById(id); ExperimentIns experimentIns = this.experimentInsDao.queryById(id);
Map<String,Object> requestData = new HashMap<>();
Map<String, Object> requestData = new HashMap<>();
requestData.put("namespace", namespace); requestData.put("namespace", namespace);
requestData.put("name", name); requestData.put("name", name);
// 创建发送数据map,将请求数据作为"data"键的值 // 创建发送数据map,将请求数据作为"data"键的值
Map<String,Object> res = new HashMap<>();
Map<String, Object> res = new HashMap<>();
res.put("data", requestData); res.put("data", requestData);


try { try {
// 发送POST请求到Argo工作流状态查询接口,并将请求数据转换为JSON // 发送POST请求到Argo工作流状态查询接口,并将请求数据转换为JSON
String req = httpUtils.sendPost(argoUrl + argoWorkflowStatus,null, JsonUtils.mapToJson(res));
String req = httpUtils.sendPost(argoUrl + argoWorkflowStatus, null, JsonUtils.mapToJson(res));
// 检查响应是否为空或无内容 // 检查响应是否为空或无内容
if (req == null || StringUtils.isEmpty(req)) { if (req == null || StringUtils.isEmpty(req)) {
throw new RuntimeException("工作流状态响应为空。"); throw new RuntimeException("工作流状态响应为空。");
@@ -293,10 +300,10 @@ public class ExperimentInsServiceImpl implements ExperimentInsService {


// 解析nodes字段,提取节点状态并转换为JSON字符串 // 解析nodes字段,提取节点状态并转换为JSON字符串
Map<String, Object> nodes = (Map<String, Object>) status.get("nodes"); Map<String, Object> nodes = (Map<String, Object>) status.get("nodes");
Map<String, Object> modifiedNodes = new LinkedHashMap<>();
if (nodes != null ) {
Map<String, Object> modifiedNodes = new LinkedHashMap<>();
if (nodes != null) {
for (Map.Entry<String, Object> nodeEntry : nodes.entrySet()) { for (Map.Entry<String, Object> nodeEntry : nodes.entrySet()) {
Map<String,Object> nodeDetails = (Map<String, Object>) nodeEntry.getValue();
Map<String, Object> nodeDetails = (Map<String, Object>) nodeEntry.getValue();
String templateName = (String) nodeDetails.get("displayName"); String templateName = (String) nodeDetails.get("displayName");
modifiedNodes.put(templateName, nodeDetails); modifiedNodes.put(templateName, nodeDetails);
} }
@@ -306,10 +313,10 @@ public class ExperimentInsServiceImpl implements ExperimentInsService {
experimentIns.setNodesStatus(nodeStatusJson); experimentIns.setNodesStatus(nodeStatusJson);


//终止态为终止不改 //终止态为终止不改
if (!StringUtils.equals(experimentIns.getStatus(),"Terminated")) {
experimentIns.setStatus(StringUtils.isNotEmpty((String) status.get("phase"))?(String) status.get("phase"):"Pending");
if (!StringUtils.equals(experimentIns.getStatus(), "Terminated")) {
experimentIns.setStatus(StringUtils.isNotEmpty((String) status.get("phase")) ? (String) status.get("phase") : "Pending");
} }
if (StringUtils.equals(experimentIns.getStatus(),"Error")) {
if (StringUtils.equals(experimentIns.getStatus(), "Error")) {
experimentIns.setStatus("Failed"); experimentIns.setStatus("Failed");
} }


@@ -350,16 +357,16 @@ public class ExperimentInsServiceImpl implements ExperimentInsService {
} }


// 创建请求数据map // 创建请求数据map
Map<String,Object> requestData = new HashMap<>();
Map<String, Object> requestData = new HashMap<>();
requestData.put("namespace", namespace); requestData.put("namespace", namespace);
requestData.put("name", name); requestData.put("name", name);
// 创建发送数据map,将请求数据作为"data"键的值 // 创建发送数据map,将请求数据作为"data"键的值
Map<String,Object> res = new HashMap<>();
Map<String, Object> res = new HashMap<>();
res.put("data", requestData); res.put("data", requestData);


try { try {
// 发送POST请求到Argo工作流状态查询接口,并将请求数据转换为JSON // 发送POST请求到Argo工作流状态查询接口,并将请求数据转换为JSON
String req = httpUtils.sendPost(argoUrl + argoWorkflowTermination,null, JsonUtils.mapToJson(res));
String req = httpUtils.sendPost(argoUrl + argoWorkflowTermination, null, JsonUtils.mapToJson(res));
// 检查响应是否为空或无内容 // 检查响应是否为空或无内容
if (StringUtils.isEmpty(req)) { if (StringUtils.isEmpty(req)) {
throw new RuntimeException("终止响应内容为空。"); throw new RuntimeException("终止响应内容为空。");
@@ -423,20 +430,20 @@ public class ExperimentInsServiceImpl implements ExperimentInsService {
String namespace = experimentIns.getArgoInsNs(); String namespace = experimentIns.getArgoInsNs();


// 创建请求数据的Json(map) // 创建请求数据的Json(map)
Map<String,Object> requestData = new HashMap<>();
Map<String, Object> requestData = new HashMap<>();
requestData.put("namespace", namespace); requestData.put("namespace", namespace);
requestData.put("name", name); requestData.put("name", name);
//先写死这两个数据项 //先写死这两个数据项
requestData.put("workflow-type","workflows");
requestData.put("artifact-name","main-logs");
requestData.put("component-id",componentId);
requestData.put("workflow-type", "workflows");
requestData.put("artifact-name", "main-logs");
requestData.put("component-id", componentId);
// 创建发送数据map,将请求数据作为"data"键的值 // 创建发送数据map,将请求数据作为"data"键的值
Map<String,Object> res = new HashMap<>();
Map<String, Object> res = new HashMap<>();
res.put("data", requestData); res.put("data", requestData);


try { try {
// 发送POST请求到Argo工作流日志查询接口,并将请求数据转换为JSON // 发送POST请求到Argo工作流日志查询接口,并将请求数据转换为JSON
String req = httpUtils.sendPost(argoUrl + argoWorkflowLog, null,JsonUtils.mapToJson(res));
String req = httpUtils.sendPost(argoUrl + argoWorkflowLog, null, JsonUtils.mapToJson(res));
// 检查响应是否为空或无内容 // 检查响应是否为空或无内容
if (StringUtils.isEmpty(req)) { if (StringUtils.isEmpty(req)) {
throw new RuntimeException("响应内容为空"); throw new RuntimeException("响应内容为空");
@@ -445,7 +452,7 @@ public class ExperimentInsServiceImpl implements ExperimentInsService {
Map<String, Object> runResMap = JsonUtils.jsonToMap(req); Map<String, Object> runResMap = JsonUtils.jsonToMap(req);
// 从响应Map中获取"data"的值,也就是日志的值 // 从响应Map中获取"data"的值,也就是日志的值
String experimentInsLog = (String) runResMap.get("data"); String experimentInsLog = (String) runResMap.get("data");
if (experimentInsLog == null){
if (experimentInsLog == null) {
throw new RuntimeException("日志为空。"); throw new RuntimeException("日志为空。");
} }
//返回日志内容 //返回日志内容
@@ -463,28 +470,28 @@ public class ExperimentInsServiceImpl implements ExperimentInsService {
//查询 实例 //查询 实例
ExperimentIns experimentIns = this.experimentInsDao.queryById(id); ExperimentIns experimentIns = this.experimentInsDao.queryById(id);
if (experimentIns == null) { if (experimentIns == null) {
throw new Exception("实验实例未查询到,id:" + id);
throw new Exception("实验实例未查询到,id:" + id);
} }
//找到 节点 节点结果 //找到 节点 节点结果
String nodesResultString = experimentIns.getNodesResult(); String nodesResultString = experimentIns.getNodesResult();
if (StringUtils.isEmpty(nodesResultString)) { if (StringUtils.isEmpty(nodesResultString)) {
throw new Exception("实验实例未查询到节点结果,id:" + id);
throw new Exception("实验实例未查询到节点结果,id:" + id);
} }
Map<String, Object> nodesResult = JsonUtils.jsonToMap(nodesResultString); Map<String, Object> nodesResult = JsonUtils.jsonToMap(nodesResultString);
Map<String, Object> paramOutput = (Map<String, Object>) nodesResult.get("param_output"); Map<String, Object> paramOutput = (Map<String, Object>) nodesResult.get("param_output");
List<Map<String,Object>> nodeList = (List<Map<String,Object>>) paramOutput.get(nodeId);
List<Map<String, Object>> nodeList = (List<Map<String, Object>>) paramOutput.get(nodeId);
//遍历 查询minio //遍历 查询minio
for (int i = 0; i < nodeList.size(); i++) { for (int i = 0; i < nodeList.size(); i++) {
Map<String, Object> map = nodeList.get(i); Map<String, Object> map = nodeList.get(i);
String path = (String) map.get("path"); String path = (String) map.get("path");
String bucketName = path.substring(0, path.indexOf("/")); String bucketName = path.substring(0, path.indexOf("/"));
String prefix = path.substring(path.indexOf("/")+1,path.length())+"/";
if (StringUtils.equals( (String)map.get("type"),"file")){
String prefix = path.substring(path.indexOf("/") + 1, path.length()) + "/";
if (StringUtils.equals((String) map.get("type"), "file")) {
List<Map> fileInfo = minioUtil.listFilesInDirectory(bucketName, prefix); List<Map> fileInfo = minioUtil.listFilesInDirectory(bucketName, prefix);
map.put("value",fileInfo);
}else if (StringUtils.equals( (String)map.get("type"),"string")){
map.put("value", fileInfo);
} else if (StringUtils.equals((String) map.get("type"), "string")) {
String resultInfo = minioUtil.readObjectAsString(bucketName, prefix); String resultInfo = minioUtil.readObjectAsString(bucketName, prefix);
map.put("value",resultInfo);
map.put("value", resultInfo);
} }
results.add(map); results.add(map);
//组装 //组装
@@ -501,14 +508,14 @@ public class ExperimentInsServiceImpl implements ExperimentInsService {
String taskId = logRequest.getTaskId(); String taskId = logRequest.getTaskId();
String startTime = logRequest.getStartTime(); String startTime = logRequest.getStartTime();
// 创建请求数据的Json(map) // 创建请求数据的Json(map)
Map<String,Object> requestData = new HashMap<>();
Map<String, Object> requestData = new HashMap<>();
requestData.put("component_id", componentId); requestData.put("component_id", componentId);
requestData.put("namespace", nameSpace); requestData.put("namespace", nameSpace);
requestData.put("name", name); requestData.put("name", name);
requestData.put("task_id",taskId);
requestData.put("start_time",startTime);
requestData.put("task_id", taskId);
requestData.put("start_time", startTime);
// 创建发送数据map,将请求数据作为"data"键的值 // 创建发送数据map,将请求数据作为"data"键的值
Map<String,Object> res = new HashMap<>();
Map<String, Object> res = new HashMap<>();
res.put("data", requestData); res.put("data", requestData);


try { try {
@@ -531,9 +538,9 @@ public class ExperimentInsServiceImpl implements ExperimentInsService {


@Override @Override
public Map<String, Object> getRealtimePodLog(String podName, String startTime) { public Map<String, Object> getRealtimePodLog(String podName, String startTime) {
Map<String,Object> requestData = new HashMap<>();
requestData.put("pod_name",podName);
requestData.put("start_time",startTime);
Map<String, Object> requestData = new HashMap<>();
requestData.put("pod_name", podName);
requestData.put("start_time", startTime);
try { try {
// 将Map转换为JSON字符串 // 将Map转换为JSON字符串
String req = HttpUtils.sendPost(argoUrl + argoWorkflowPodLog, JsonUtils.mapToJson(requestData)); String req = HttpUtils.sendPost(argoUrl + argoWorkflowPodLog, JsonUtils.mapToJson(requestData));
@@ -554,7 +561,7 @@ public class ExperimentInsServiceImpl implements ExperimentInsService {


@Override @Override
public String getRealtimePodLogFromPod(PodLogVo podLogVo) { public String getRealtimePodLogFromPod(PodLogVo podLogVo) {
return k8sClientUtil.getPodLogs(podLogVo.getPodName(), podLogVo.getNamespace(),podLogVo.getContainerName(), logsLines);
return k8sClientUtil.getPodLogs(podLogVo.getPodName(), podLogVo.getNamespace(), podLogVo.getContainerName(), logsLines);
} }


@Override @Override
@@ -573,7 +580,7 @@ public class ExperimentInsServiceImpl implements ExperimentInsService {
experimentIns.setStatus(status.toString()); // 设置实例的状态为枚举中的状态 experimentIns.setStatus(status.toString()); // 设置实例的状态为枚举中的状态


// 查询具有相同状态的实例数量 // 查询具有相同状态的实例数量
Long count = experimentInsDao.count(experimentIns);
Long count = experimentInsDao.count(experimentIns);


// 将状态及其对应的实例数量放入map中 // 将状态及其对应的实例数量放入map中
statusCountMap.put(status.toString(), count); statusCountMap.put(status.toString(), count);
@@ -588,7 +595,7 @@ public class ExperimentInsServiceImpl implements ExperimentInsService {
boolean flag = true; boolean flag = true;
List<String> terminatedStates = Arrays.asList("Succeeded", "Failed"); List<String> terminatedStates = Arrays.asList("Succeeded", "Failed");
flag = terminatedStates.contains(status); flag = terminatedStates.contains(status);
if (StringUtils.equals(status, "Terminated")){
if (StringUtils.equals(status, "Terminated")) {
//如果跟node_status里面不一样,就要去更新node_status的信息 //如果跟node_status里面不一样,就要去更新node_status的信息
String nodesStatus = ins.getNodesStatus(); String nodesStatus = ins.getNodesStatus();
Map<String, Object> nodeMap = JsonUtils.jsonToMap(nodesStatus); Map<String, Object> nodeMap = JsonUtils.jsonToMap(nodesStatus);
@@ -597,7 +604,7 @@ public class ExperimentInsServiceImpl implements ExperimentInsService {
.findFirst() .findFirst()
.orElse(null); .orElse(null);
Map workflowMap = (Map) nodeMap.get(keyStartsWithWorkflow); Map workflowMap = (Map) nodeMap.get(keyStartsWithWorkflow);
if (workflowMap != null){
if (workflowMap != null) {
flag = StringUtils.equals("Terminated", (String) workflowMap.get("phase")); flag = StringUtils.equals("Terminated", (String) workflowMap.get("phase"));
} }
} }


+ 1
- 1
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentServiceImpl.java View File

@@ -560,7 +560,7 @@ public class ExperimentServiceImpl implements ExperimentService {


//处理指标 //处理指标
HashMap<String, Object> metricMap = JSON.parseObject(metricRecord, HashMap.class); HashMap<String, Object> metricMap = JSON.parseObject(metricRecord, HashMap.class);
modelMetaVo.setMetrics(metricMap);
modelMetaVo.setMetricsParams(metricMap);


//训练数据集 //训练数据集
List<Map<String, Object>> trainDatasetList = (List<Map<String, Object>>) modelTrainMap.get("datasets"); List<Map<String, Object>> trainDatasetList = (List<Map<String, Object>>) modelTrainMap.get("datasets");


+ 42
- 6
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java View File

@@ -747,7 +747,7 @@ public class ModelsServiceImpl implements ModelsService {
if (buildingModel != null) { if (buildingModel != null) {
modelMetaVo = JSON.parseObject(buildingModel.getMeta(), ModelMetaVo.class); modelMetaVo = JSON.parseObject(buildingModel.getMeta(), ModelMetaVo.class);
//获取指标 //获取指标
getMetrics(modelMetaVo);
transMetrics(modelMetaVo);
} }


//拼接生产的元数据后写入yaml文件 //拼接生产的元数据后写入yaml文件
@@ -806,7 +806,7 @@ public class ModelsServiceImpl implements ModelsService {
modelDependency1Dao.insert(modelDependency); modelDependency1Dao.insert(modelDependency);
} else { } else {
//更新模型依赖 //更新模型依赖
modelDependency1Dao.updateState(modelsVo.getId(), modelsVo.getIdentifier(), modelsVo.getVersion(), Constant.State_valid);
modelDependency1Dao.updateState(modelsVo.getId(), modelsVo.getIdentifier(), modelsVo.getVersion(), meta, Constant.State_valid);
} }
} else { } else {
//保存模型依赖 //保存模型依赖
@@ -973,11 +973,47 @@ public class ModelsServiceImpl implements ModelsService {
return new PageImpl<>(result, pageRequest, collect.size()); return new PageImpl<>(result, pageRequest, collect.size());
} }


@Override
public Page<Map<String, Object>> queryVersions(PageRequest pageRequest, String identifier, String owner) throws Exception {
String token = gitService.checkoutToken();
List<Map<String, Object>> collect = gitService.getBrancheList(token, owner, identifier);
List<Map<String, Object>> result = collect.stream()
.filter(branch -> !"master".equals(branch.get("name")))
.skip((pageRequest.getPageNumber()) * pageRequest.getPageSize()).limit(pageRequest.getPageSize())
.collect(Collectors.toList());

for (Map<String, Object> branch : result) {
String meta = modelDependency1Dao.getMeta(identifier, owner, (String) branch.get("name"));
ModelMetaVo modelMetaVo = JSON.parseObject(meta, ModelMetaVo.class);
if (modelMetaVo.getParams() != null) {
branch.putAll(modelMetaVo.getParams());
}
if (modelMetaVo.getMetrics() != null) {
branch.putAll(modelMetaVo.getMetrics());
}
if (modelMetaVo.getMetricsParams() != null) {
branch.putAll(modelMetaVo.getMetricsParams());
}
}
return new PageImpl<>(result, pageRequest, collect.size());
}

@Override
public List<List<Map<String, Object>>> queryVersionsMetrics(List<String> runIds) throws Exception {
List<List<Map<String, Object>>> batchMetrics = new ArrayList<>();
for (String runId : runIds) {
HashMap<String, Object> map = aimsService.queryMetricsParams(runId);
List<Map<String, Object>> batchMetric = aimsService.getBatchMetric((String) map.get("run_hash"), (String) map.get("params"));
batchMetrics.add(batchMetric);
}
return batchMetrics;
}

@Override @Override
public List<Map<String, Object>> getVersionList(String identifier, String owner) throws Exception { public List<Map<String, Object>> getVersionList(String identifier, String owner) throws Exception {
String token = gitService.checkoutToken(); String token = gitService.checkoutToken();
List<Map<String, Object>> brancheList = gitService.getBrancheList(token, owner, identifier);
return brancheList.stream()
List<Map<String, Object>> branchList = gitService.getBrancheList(token, owner, identifier);
return branchList.stream()
.filter(branch -> !"master".equals(branch.get("name"))) .filter(branch -> !"master".equals(branch.get("name")))
.collect(Collectors.toList()); .collect(Collectors.toList());
} }
@@ -1179,12 +1215,12 @@ public class ModelsServiceImpl implements ModelsService {
return userInfo; return userInfo;
} }


void getMetrics(ModelMetaVo modelMetaVo) throws Exception {
void transMetrics(ModelMetaVo modelMetaVo) throws Exception {
HashMap<String, Object> result = new HashMap<>(); HashMap<String, Object> result = new HashMap<>();
HashMap<String, Object> train = new HashMap<>(); HashMap<String, Object> train = new HashMap<>();
HashMap<String, Object> evaluate = new HashMap<>(); HashMap<String, Object> evaluate = new HashMap<>();


HashMap<String, Object> metrics = modelMetaVo.getMetrics();
HashMap<String, Object> metrics = modelMetaVo.getMetricsParams();
JSONArray trainMetrics = (JSONArray) metrics.get("train"); JSONArray trainMetrics = (JSONArray) metrics.get("train");
if (trainMetrics != null) { if (trainMetrics != null) {
for (int i = 0; i < trainMetrics.size(); i++) { for (int i = 0; i < trainMetrics.size(); i++) {


+ 14
- 11
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/HttpUtils.java View File

@@ -1,5 +1,6 @@
package com.ruoyi.platform.utils; package com.ruoyi.platform.utils;


import com.ruoyi.common.core.utils.StringUtils;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.http.HttpEntity; import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse; import org.apache.http.HttpResponse;
@@ -101,12 +102,13 @@ public class HttpUtils {
// public static String sendGetWithToken(String url, String param,String token) { // public static String sendGetWithToken(String url, String param,String token) {
// return sendGet(url, param, "UTF-8",token); // return sendGet(url, param, "UTF-8",token);
// } // }

/** /**
* 向指定 URL 发送带 token 的 GET 方法的请求,使用 Apache HttpClient * 向指定 URL 发送带 token 的 GET 方法的请求,使用 Apache HttpClient
* *
* @param url 发送请求的 URL
* @param param 请求参数,请求参数应该是 name1=value1&name2=value2 的形式。
* @param token 认证 token
* @param url 发送请求的 URL
* @param param 请求参数,请求参数应该是 name1=value1&name2=value2 的形式。
* @param token 认证 token
* @return 所代表远程资源的响应结果 * @return 所代表远程资源的响应结果
*/ */
public static String sendGetWithToken(String url, String param, String token) { public static String sendGetWithToken(String url, String param, String token) {
@@ -148,6 +150,7 @@ public class HttpUtils {
} }
return result; return result;
} }

/** /**
* 向指定 URL 发送带token的GET方法的请求 * 向指定 URL 发送带token的GET方法的请求
* *
@@ -156,7 +159,7 @@ public class HttpUtils {
* @param contentType 编码类型 * @param contentType 编码类型
* @return 所代表远程资源的响应结果 * @return 所代表远程资源的响应结果
*/ */
public static String sendGet(String url, String param, String contentType,String token) {
public static String sendGet(String url, String param, String contentType, String token) {
StringBuilder result = new StringBuilder(); StringBuilder result = new StringBuilder();
BufferedReader in = null; BufferedReader in = null;
try { try {
@@ -196,7 +199,6 @@ public class HttpUtils {
} }





/** /**
* 向指定 URL 发送GET方法的请求 * 向指定 URL 发送GET方法的请求
* *
@@ -209,7 +211,10 @@ public class HttpUtils {
StringBuilder result = new StringBuilder(); StringBuilder result = new StringBuilder();
BufferedReader in = null; BufferedReader in = null;
try { try {
String urlNameString = url + "?" + param;
String urlNameString = url;
if (StringUtils.isNotEmpty(param)) {
urlNameString = url + "?" + param;
}
log.info("sendGet - {}", urlNameString); log.info("sendGet - {}", urlNameString);
URL realUrl = new URL(urlNameString); URL realUrl = new URL(urlNameString);
URLConnection connection = realUrl.openConnection(); URLConnection connection = realUrl.openConnection();
@@ -245,11 +250,11 @@ public class HttpUtils {
/** /**
* 向指定 URL 发送带token的POST方法的请求 * 向指定 URL 发送带token的POST方法的请求
* *
* @param url 发送请求的 URL
* @param url 发送请求的 URL
* @param * @param
* @return 所代表远程资源的响应结果 * @return 所代表远程资源的响应结果
*/ */
public static String sendPostWithToken(String url, Object params, String token) throws Exception {
public static String sendPostWithToken(String url, Object params, String token) throws Exception {
String resultStr = null; String resultStr = null;
HttpPost httpPost = new HttpPost(url); HttpPost httpPost = new HttpPost(url);
if (params != null) { if (params != null) {
@@ -518,11 +523,10 @@ public class HttpUtils {
} }





/** /**
* 发送 HTTP 请求并返回二进制数据流(OutputStream)。 * 发送 HTTP 请求并返回二进制数据流(OutputStream)。
* *
* @param url 请求的 URL 地址。
* @param url 请求的 URL 地址。
* @param headers 头节点。 * @param headers 头节点。
* @return 服务器响应的二进制数据流(OutputStream)。 * @return 服务器响应的二进制数据流(OutputStream)。
* @throws IOException 如果请求失败或发生其他 I/O 错误。 * @throws IOException 如果请求失败或发生其他 I/O 错误。
@@ -626,7 +630,6 @@ public class HttpUtils {
} }





// 其他方法保持不变 // 其他方法保持不变


/** /**


+ 3
- 0
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelMetaVo.java View File

@@ -59,6 +59,9 @@ public class ModelMetaVo implements Serializable {
@ApiModelProperty(value = "指标") @ApiModelProperty(value = "指标")
private HashMap<String, Object> metrics; private HashMap<String, Object> metrics;


@ApiModelProperty(value = "指标查询参数")
private HashMap<String, Object> metricsParams;

@ApiModelProperty(value = "训练任务") @ApiModelProperty(value = "训练任务")
private TrainTaskDepency trainTask; private TrainTaskDepency trainTask;




+ 11
- 1
ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ModelDependency1DaoMapper.xml View File

@@ -60,6 +60,15 @@
order by create_time desc limit 1 order by create_time desc limit 1
</select> </select>


<select id="getMeta" resultType="java.lang.String">
select meta from model_dependency1
where identifier = #{identifier}
and owner = #{owner}
and version = #{version}
and state = 1
order by create_time desc limit 1
</select>

<update id="deleteModel"> <update id="deleteModel">
update model_dependency1 update model_dependency1
set state = 0 set state = 0
@@ -77,7 +86,8 @@


<update id="updateState"> <update id="updateState">
update model_dependency1 update model_dependency1
set state = 1
set state = 1,
meta = #{meta}
where repo_id = #{repoId} where repo_id = #{repoId}
and identifier = #{identifier} and identifier = #{identifier}
and version = #{version} and version = #{version}


Loading…
Cancel
Save