Browse Source

Merge remote-tracking branch 'origin/dev' into test

dev-lhz
chenzhihang 1 year ago
parent
commit
e15486728b
15 changed files with 216 additions and 83 deletions
  1. +1
    -1
      k8s/deploy.sh
  2. +2
    -0
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/RuoYiManagementPlatformApplication.java
  3. +6
    -0
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/experiment/ExperimentInsController.java
  4. +17
    -3
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/model/NewModelFromGitController.java
  5. +3
    -1
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/ModelDependency1Dao.java
  6. +6
    -0
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AimService.java
  7. +3
    -0
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ExperimentInsService.java
  8. +4
    -0
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ModelsService.java
  9. +45
    -8
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AimServiceImpl.java
  10. +58
    -51
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentInsServiceImpl.java
  11. +1
    -1
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentServiceImpl.java
  12. +42
    -6
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java
  13. +14
    -11
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/HttpUtils.java
  14. +3
    -0
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelMetaVo.java
  15. +11
    -1
      ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ModelDependency1DaoMapper.xml

+ 1
- 1
k8s/deploy.sh View File

@@ -143,7 +143,7 @@ fi

if [ "$service" == "all" ]; then
#部署前端
# build_and_deploy "nginx-dockerfile" "172.20.32.187/ci4s/ci4s-front:${tag}" "k8s-12front.yaml"
build_and_deploy "nginx-dockerfile" "172.20.32.187/ci4s/ci4s-front:${tag}" "k8s-12front.yaml"
#部署管理平台
build_and_deploy "managent-dockerfile" "172.20.32.187/ci4s/ci4s-managent:${tag}" "k8s-7management.yaml"
#部署认证中心


+ 2
- 0
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/RuoYiManagementPlatformApplication.java View File

@@ -5,6 +5,7 @@ import com.ruoyi.common.security.annotation.EnableRyFeignClients;
import com.ruoyi.common.swagger.annotation.EnableCustomSwagger2;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.scheduling.annotation.EnableAsync;
import org.springframework.scheduling.annotation.EnableScheduling;

/**
@@ -17,6 +18,7 @@ import org.springframework.scheduling.annotation.EnableScheduling;
@EnableRyFeignClients
@SpringBootApplication
@EnableScheduling
@EnableAsync
public class RuoYiManagementPlatformApplication {
public static void main(String[] args) {
SpringApplication.run(RuoYiManagementPlatformApplication.class, args);


+ 6
- 0
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/experiment/ExperimentInsController.java View File

@@ -106,6 +106,12 @@ public class ExperimentInsController extends BaseController {
return genericsSuccess(this.experimentInsService.removeById(id));
}

@DeleteMapping("batchDelete")
@ApiOperation("批量删除实验实例")
public GenericsAjaxResult<String> batchDelete(@RequestBody List<Integer> ids) throws Exception{
return genericsSuccess(this.experimentInsService.batchDelete(ids));
}

/**
* 终止实验实例
*


+ 17
- 3
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/model/NewModelFromGitController.java View File

@@ -2,11 +2,9 @@ package com.ruoyi.platform.controller.model;

import com.ruoyi.common.core.web.domain.AjaxResult;
import com.ruoyi.platform.service.ModelsService;
import com.ruoyi.platform.utils.DVCUtils;
import com.ruoyi.platform.vo.ModelsVo;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.springframework.core.io.InputStreamResource;
import org.springframework.data.domain.PageRequest;
import org.springframework.http.ResponseEntity;
@@ -14,7 +12,6 @@ import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;

import javax.annotation.Resource;
import java.io.IOException;
import java.util.List;
import java.util.Map;

@@ -77,6 +74,23 @@ public class NewModelFromGitController {
}
}

@GetMapping("/queryVersions")
@ApiOperation("分页查询模型版本")
public AjaxResult queryVersions(@RequestParam(value = "page") int page,
@RequestParam(value = "size") int size,
@RequestParam("identifier") String identifier,
@RequestParam("owner") String owner) throws Exception {
PageRequest pageRequest = PageRequest.of(page, size);
return AjaxResult.success(this.modelsService.queryVersions(pageRequest, identifier, owner));
}

@GetMapping("/queryVersionsMetrics")
@ApiOperation("查询版本指标")
public AjaxResult queryVersionsMetrics(@RequestParam("runIds") List<String> runIds) throws Exception {
return AjaxResult.success(this.modelsService.queryVersionsMetrics(runIds));
}


@GetMapping("/getVersionList")
@ApiOperation(value = "获取模型分支列表")
public AjaxResult getVersionList(@RequestParam("identifier") String identifier, @RequestParam("owner") String owner) throws Exception {


+ 3
- 1
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/ModelDependency1Dao.java View File

@@ -9,7 +9,7 @@ public interface ModelDependency1Dao {

int insert(ModelDependency1 modelDependency1);

int updateState(@Param("repoId") Integer repoId, @Param("identifier") String identifier, @Param("version") String version, @Param("state") Integer state);
int updateState(@Param("repoId") Integer repoId, @Param("identifier") String identifier, @Param("version") String version, @Param("meta") String meta, @Param("state") Integer state);

List<ModelDependency1> queryModelDependency(@Param("modelName") String modelName, @Param("repoId") Integer repoId, @Param("owner") String owner);

@@ -24,4 +24,6 @@ public interface ModelDependency1Dao {
int deleteModel(@Param("repoId") Integer repoId, @Param("identifier") String identifier, @Param("owner") String owner, @Param("version") String version);

int deleteModelDependency(@Param("parentModel") String parentModel);

String getMeta(@Param("identifier") String identifier, @Param("owner") String owner, @Param("version") String version);
}

+ 6
- 0
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/AimService.java View File

@@ -2,7 +2,9 @@ package com.ruoyi.platform.service;

import com.ruoyi.platform.vo.InsMetricInfoVo;

import java.util.HashMap;
import java.util.List;
import java.util.Map;

public interface AimService {

@@ -13,4 +15,8 @@ public interface AimService {
List<InsMetricInfoVo> getExpEvaluateInfos(Integer experimentId) throws Exception;

String getExpMetrics(List<String> runIds) throws Exception;

HashMap<String, Object> queryMetricsParams(String runId) throws Exception;

List<Map<String, Object>> getBatchMetric(String runHash, String params);
}

+ 3
- 0
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ExperimentInsService.java View File

@@ -73,6 +73,9 @@ public interface ExperimentInsService {
* @param id 主键
* @return 是否成功
*/

String batchDelete(List<Integer> ids);

boolean deleteById(Integer id);




+ 4
- 0
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ModelsService.java View File

@@ -99,6 +99,10 @@ public interface ModelsService {

Page<ModelsVo> newPersonalQueryByPage(ModelsVo modelsVo, PageRequest pageRequest) throws Exception;

Page<Map<String, Object>> queryVersions(PageRequest pageRequest, String identifier, String owner) throws Exception;

List<List<Map<String, Object>>> queryVersionsMetrics(List<String> runIds) throws Exception;

List<Map<String, Object>> getVersionList(String identifier, String owner) throws Exception;

ModelsVo getModelDetail(Integer id, String identifier, String owner, String version) throws Exception;


+ 45
- 8
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/AimServiceImpl.java View File

@@ -1,18 +1,17 @@
package com.ruoyi.platform.service.impl;

import com.alibaba.fastjson2.JSON;
import com.ruoyi.platform.domain.ExperimentIns;
import com.ruoyi.platform.service.AimService;
import com.ruoyi.platform.service.ExperimentInsService;
import com.ruoyi.platform.utils.AIM64EncoderUtil;
import com.ruoyi.platform.utils.JacksonUtil;
import com.ruoyi.platform.utils.JsonUtils;
import com.ruoyi.platform.utils.NewHttpUtils;
import com.ruoyi.platform.utils.*;
import com.ruoyi.platform.vo.InsMetricInfoVo;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;

import javax.annotation.Resource;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.*;
import java.util.stream.Collectors;
@@ -26,8 +25,6 @@ public class AimServiceImpl implements AimService {
private String aimUrl;
@Value("${aim.proxyUrl}")
private String aimProxyUrl;
@Resource
private NewHttpUtils httpUtils;

@Override
public List<InsMetricInfoVo> getExpTrainInfos(Integer experimentId) throws Exception {
@@ -52,7 +49,7 @@ public class AimServiceImpl implements AimService {
}
String encodedUrlString = URLEncoder.encode("run.experiment==\"" + experimentName + "\"", "UTF-8");
String url = aimProxyUrl + "/api/runs/search/run?query=" + encodedUrlString;
String s = httpUtils.sendGet(url, null);
String s = HttpUtils.sendGet(url, null);
List<Map<String, Object>> response = JacksonUtil.parseJSONStr2MapList(s);
System.out.println("response: " + JacksonUtil.toJSONString(response));
if (response == null || response.size() == 0) {
@@ -142,7 +139,7 @@ public class AimServiceImpl implements AimService {
public List<InsMetricInfoVo> getExpTrainInfos1(boolean isTrain, Integer experimentId, String runId) throws Exception {
String encodedUrlString = URLEncoder.encode("run.id==\"" + runId + "\"", "UTF-8");
String url = aimProxyUrl + "/api/runs/search/run?query=" + encodedUrlString;
String s = httpUtils.sendGet(url, null);
String s = HttpUtils.sendGet(url, null);
List<Map<String, Object>> response = JacksonUtil.parseJSONStr2MapList(s);
System.out.println("response: " + JacksonUtil.toJSONString(response));
if (response == null || response.size() == 0) {
@@ -245,4 +242,44 @@ public class AimServiceImpl implements AimService {
}
return datasetList;
}

@Override
public HashMap<String, Object> queryMetricsParams(String runId) throws UnsupportedEncodingException {
String encodedUrlString = URLEncoder.encode("run.id==\"" + runId + "\"", "UTF-8");
String url = aimProxyUrl + "/api/runs/search/run?query=" + encodedUrlString;
String s = HttpUtils.sendGet(url, null);
List<Map<String, Object>> response = JacksonUtil.parseJSONStr2MapList(s);
if (response == null || response.size() == 0) {
return new HashMap<>();
}

HashMap<String, Object> resultMap = new HashMap<>();
List<Map<String, Object>> paramList = new ArrayList<>();

Map<String, Object> map = response.get(0);
LinkedHashMap<String, ArrayList> traces = (LinkedHashMap<String, ArrayList>) map.get("traces");
if (traces != null) {
List<Map<String, Object>> metrics = traces.get("metric");
for (Map<String, Object> metric : metrics) {
Map<String, Object> metricParam = new HashMap<>();
metricParam.put("context", metric.get("context"));
metricParam.put("name", metric.get("name"));
paramList.add(metricParam);
}
resultMap.put("params", JSON.toJSONString(paramList));
}
resultMap.put("run_hash", map.get("run_hash"));
return resultMap;
}

@Override
public List<Map<String, Object>> getBatchMetric(String runHash, String params) {
String url = aimUrl + "/api/runs/" + runHash + "/metric/get-batch";
String response = HttpUtils.sendPost(url, params);
if (StringUtils.isNotEmpty(response)) {
return JacksonUtil.parseJSONStr2MapList(response);
}
return null;
}

}

+ 58
- 51
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentInsServiceImpl.java View File

@@ -36,13 +36,13 @@ public class ExperimentInsServiceImpl implements ExperimentInsService {
@Resource
private WorkflowService workflowService;
@Value("${argo.url}")
private String argoUrl;
private String argoUrl;
@Value("${argo.convert}")
private String argoConvert;
private String argoConvert;
@Value("${argo.workflowRun}")
private String argoWorkflowRun;
private String argoWorkflowRun;
@Value("${argo.workflowStatus}")
private String argoWorkflowStatus;
private String argoWorkflowStatus;
@Value("${argo.workflowTermination}")
private String argoWorkflowTermination;
@Value("${argo.workflowLog}")
@@ -60,6 +60,7 @@ public class ExperimentInsServiceImpl implements ExperimentInsService {
@Resource
private K8sClientUtil k8sClientUtil;
private final MinioUtil minioUtil;

public ExperimentInsServiceImpl(MinioUtil minioUtil) {
this.minioUtil = minioUtil;
}
@@ -97,8 +98,6 @@ public class ExperimentInsServiceImpl implements ExperimentInsService {
}

/**
*
*
* 根据实验ID查找所有具有相同ID的实例,并将它们添加到实验列表中
*
* @param experimentId 实验ID
@@ -187,8 +186,6 @@ public class ExperimentInsServiceImpl implements ExperimentInsService {
}




/**
* 修改数据
*
@@ -207,7 +204,7 @@ public class ExperimentInsServiceImpl implements ExperimentInsService {
@Override
public String removeById(Integer id) {
ExperimentIns experimentIns = experimentInsDao.queryById(id);
if (experimentIns == null){
if (experimentIns == null) {
return "实验实例不存在";
}

@@ -215,21 +212,31 @@ public class ExperimentInsServiceImpl implements ExperimentInsService {
LoginUser loginUser = SecurityUtils.getLoginUser();
String username = loginUser.getUsername();
String createdBy = experimentIns.getCreateBy();
if (!(StringUtils.equals(username,"admin") || StringUtils.equals(username,createdBy))){
if (!(StringUtils.equals(username, "admin") || StringUtils.equals(username, createdBy))) {
return "无权限删除该流水线";
}

if (StringUtils.isEmpty(experimentIns.getStatus())){
if (StringUtils.isEmpty(experimentIns.getStatus())) {
experimentIns = queryStatusFromArgo(experimentIns);
}
if (StringUtils.equals(experimentIns.getStatus(),"Running")){
if (StringUtils.equals(experimentIns.getStatus(), "Running")) {
return "实验实例正在运行,不可删除";
}
experimentIns.setState(0);
return this.experimentInsDao.update(experimentIns)>0?"删除成功":"删除失败";
return this.experimentInsDao.update(experimentIns) > 0 ? "删除成功" : "删除失败";

}

@Override
public String batchDelete(List<Integer> ids) {
for (Integer id : ids) {
String result = removeById(id);
if (!"删除成功".equals(result)) {
return result;
}
}
return "删除成功";
}


/**
@@ -256,16 +263,16 @@ public class ExperimentInsServiceImpl implements ExperimentInsService {
Integer id = ins.getId();
// 创建请求数据map
ExperimentIns experimentIns = this.experimentInsDao.queryById(id);
Map<String,Object> requestData = new HashMap<>();
Map<String, Object> requestData = new HashMap<>();
requestData.put("namespace", namespace);
requestData.put("name", name);
// 创建发送数据map,将请求数据作为"data"键的值
Map<String,Object> res = new HashMap<>();
Map<String, Object> res = new HashMap<>();
res.put("data", requestData);

try {
// 发送POST请求到Argo工作流状态查询接口,并将请求数据转换为JSON
String req = httpUtils.sendPost(argoUrl + argoWorkflowStatus,null, JsonUtils.mapToJson(res));
String req = httpUtils.sendPost(argoUrl + argoWorkflowStatus, null, JsonUtils.mapToJson(res));
// 检查响应是否为空或无内容
if (req == null || StringUtils.isEmpty(req)) {
throw new RuntimeException("工作流状态响应为空。");
@@ -293,10 +300,10 @@ public class ExperimentInsServiceImpl implements ExperimentInsService {

// 解析nodes字段,提取节点状态并转换为JSON字符串
Map<String, Object> nodes = (Map<String, Object>) status.get("nodes");
Map<String, Object> modifiedNodes = new LinkedHashMap<>();
if (nodes != null ) {
Map<String, Object> modifiedNodes = new LinkedHashMap<>();
if (nodes != null) {
for (Map.Entry<String, Object> nodeEntry : nodes.entrySet()) {
Map<String,Object> nodeDetails = (Map<String, Object>) nodeEntry.getValue();
Map<String, Object> nodeDetails = (Map<String, Object>) nodeEntry.getValue();
String templateName = (String) nodeDetails.get("displayName");
modifiedNodes.put(templateName, nodeDetails);
}
@@ -306,10 +313,10 @@ public class ExperimentInsServiceImpl implements ExperimentInsService {
experimentIns.setNodesStatus(nodeStatusJson);

//终止态为终止不改
if (!StringUtils.equals(experimentIns.getStatus(),"Terminated")) {
experimentIns.setStatus(StringUtils.isNotEmpty((String) status.get("phase"))?(String) status.get("phase"):"Pending");
if (!StringUtils.equals(experimentIns.getStatus(), "Terminated")) {
experimentIns.setStatus(StringUtils.isNotEmpty((String) status.get("phase")) ? (String) status.get("phase") : "Pending");
}
if (StringUtils.equals(experimentIns.getStatus(),"Error")) {
if (StringUtils.equals(experimentIns.getStatus(), "Error")) {
experimentIns.setStatus("Failed");
}

@@ -350,16 +357,16 @@ public class ExperimentInsServiceImpl implements ExperimentInsService {
}

// 创建请求数据map
Map<String,Object> requestData = new HashMap<>();
Map<String, Object> requestData = new HashMap<>();
requestData.put("namespace", namespace);
requestData.put("name", name);
// 创建发送数据map,将请求数据作为"data"键的值
Map<String,Object> res = new HashMap<>();
Map<String, Object> res = new HashMap<>();
res.put("data", requestData);

try {
// 发送POST请求到Argo工作流状态查询接口,并将请求数据转换为JSON
String req = httpUtils.sendPost(argoUrl + argoWorkflowTermination,null, JsonUtils.mapToJson(res));
String req = httpUtils.sendPost(argoUrl + argoWorkflowTermination, null, JsonUtils.mapToJson(res));
// 检查响应是否为空或无内容
if (StringUtils.isEmpty(req)) {
throw new RuntimeException("终止响应内容为空。");
@@ -423,20 +430,20 @@ public class ExperimentInsServiceImpl implements ExperimentInsService {
String namespace = experimentIns.getArgoInsNs();

// 创建请求数据的Json(map)
Map<String,Object> requestData = new HashMap<>();
Map<String, Object> requestData = new HashMap<>();
requestData.put("namespace", namespace);
requestData.put("name", name);
//先写死这两个数据项
requestData.put("workflow-type","workflows");
requestData.put("artifact-name","main-logs");
requestData.put("component-id",componentId);
requestData.put("workflow-type", "workflows");
requestData.put("artifact-name", "main-logs");
requestData.put("component-id", componentId);
// 创建发送数据map,将请求数据作为"data"键的值
Map<String,Object> res = new HashMap<>();
Map<String, Object> res = new HashMap<>();
res.put("data", requestData);

try {
// 发送POST请求到Argo工作流日志查询接口,并将请求数据转换为JSON
String req = httpUtils.sendPost(argoUrl + argoWorkflowLog, null,JsonUtils.mapToJson(res));
String req = httpUtils.sendPost(argoUrl + argoWorkflowLog, null, JsonUtils.mapToJson(res));
// 检查响应是否为空或无内容
if (StringUtils.isEmpty(req)) {
throw new RuntimeException("响应内容为空");
@@ -445,7 +452,7 @@ public class ExperimentInsServiceImpl implements ExperimentInsService {
Map<String, Object> runResMap = JsonUtils.jsonToMap(req);
// 从响应Map中获取"data"的值,也就是日志的值
String experimentInsLog = (String) runResMap.get("data");
if (experimentInsLog == null){
if (experimentInsLog == null) {
throw new RuntimeException("日志为空。");
}
//返回日志内容
@@ -463,28 +470,28 @@ public class ExperimentInsServiceImpl implements ExperimentInsService {
//查询 实例
ExperimentIns experimentIns = this.experimentInsDao.queryById(id);
if (experimentIns == null) {
throw new Exception("实验实例未查询到,id:" + id);
throw new Exception("实验实例未查询到,id:" + id);
}
//找到 节点 节点结果
String nodesResultString = experimentIns.getNodesResult();
if (StringUtils.isEmpty(nodesResultString)) {
throw new Exception("实验实例未查询到节点结果,id:" + id);
throw new Exception("实验实例未查询到节点结果,id:" + id);
}
Map<String, Object> nodesResult = JsonUtils.jsonToMap(nodesResultString);
Map<String, Object> paramOutput = (Map<String, Object>) nodesResult.get("param_output");
List<Map<String,Object>> nodeList = (List<Map<String,Object>>) paramOutput.get(nodeId);
List<Map<String, Object>> nodeList = (List<Map<String, Object>>) paramOutput.get(nodeId);
//遍历 查询minio
for (int i = 0; i < nodeList.size(); i++) {
Map<String, Object> map = nodeList.get(i);
String path = (String) map.get("path");
String bucketName = path.substring(0, path.indexOf("/"));
String prefix = path.substring(path.indexOf("/")+1,path.length())+"/";
if (StringUtils.equals( (String)map.get("type"),"file")){
String prefix = path.substring(path.indexOf("/") + 1, path.length()) + "/";
if (StringUtils.equals((String) map.get("type"), "file")) {
List<Map> fileInfo = minioUtil.listFilesInDirectory(bucketName, prefix);
map.put("value",fileInfo);
}else if (StringUtils.equals( (String)map.get("type"),"string")){
map.put("value", fileInfo);
} else if (StringUtils.equals((String) map.get("type"), "string")) {
String resultInfo = minioUtil.readObjectAsString(bucketName, prefix);
map.put("value",resultInfo);
map.put("value", resultInfo);
}
results.add(map);
//组装
@@ -501,14 +508,14 @@ public class ExperimentInsServiceImpl implements ExperimentInsService {
String taskId = logRequest.getTaskId();
String startTime = logRequest.getStartTime();
// 创建请求数据的Json(map)
Map<String,Object> requestData = new HashMap<>();
Map<String, Object> requestData = new HashMap<>();
requestData.put("component_id", componentId);
requestData.put("namespace", nameSpace);
requestData.put("name", name);
requestData.put("task_id",taskId);
requestData.put("start_time",startTime);
requestData.put("task_id", taskId);
requestData.put("start_time", startTime);
// 创建发送数据map,将请求数据作为"data"键的值
Map<String,Object> res = new HashMap<>();
Map<String, Object> res = new HashMap<>();
res.put("data", requestData);

try {
@@ -531,9 +538,9 @@ public class ExperimentInsServiceImpl implements ExperimentInsService {

@Override
public Map<String, Object> getRealtimePodLog(String podName, String startTime) {
Map<String,Object> requestData = new HashMap<>();
requestData.put("pod_name",podName);
requestData.put("start_time",startTime);
Map<String, Object> requestData = new HashMap<>();
requestData.put("pod_name", podName);
requestData.put("start_time", startTime);
try {
// 将Map转换为JSON字符串
String req = HttpUtils.sendPost(argoUrl + argoWorkflowPodLog, JsonUtils.mapToJson(requestData));
@@ -554,7 +561,7 @@ public class ExperimentInsServiceImpl implements ExperimentInsService {

@Override
public String getRealtimePodLogFromPod(PodLogVo podLogVo) {
return k8sClientUtil.getPodLogs(podLogVo.getPodName(), podLogVo.getNamespace(),podLogVo.getContainerName(), logsLines);
return k8sClientUtil.getPodLogs(podLogVo.getPodName(), podLogVo.getNamespace(), podLogVo.getContainerName(), logsLines);
}

@Override
@@ -573,7 +580,7 @@ public class ExperimentInsServiceImpl implements ExperimentInsService {
experimentIns.setStatus(status.toString()); // 设置实例的状态为枚举中的状态

// 查询具有相同状态的实例数量
Long count = experimentInsDao.count(experimentIns);
Long count = experimentInsDao.count(experimentIns);

// 将状态及其对应的实例数量放入map中
statusCountMap.put(status.toString(), count);
@@ -588,7 +595,7 @@ public class ExperimentInsServiceImpl implements ExperimentInsService {
boolean flag = true;
List<String> terminatedStates = Arrays.asList("Succeeded", "Failed");
flag = terminatedStates.contains(status);
if (StringUtils.equals(status, "Terminated")){
if (StringUtils.equals(status, "Terminated")) {
//如果跟node_status里面不一样,就要去更新node_status的信息
String nodesStatus = ins.getNodesStatus();
Map<String, Object> nodeMap = JsonUtils.jsonToMap(nodesStatus);
@@ -597,7 +604,7 @@ public class ExperimentInsServiceImpl implements ExperimentInsService {
.findFirst()
.orElse(null);
Map workflowMap = (Map) nodeMap.get(keyStartsWithWorkflow);
if (workflowMap != null){
if (workflowMap != null) {
flag = StringUtils.equals("Terminated", (String) workflowMap.get("phase"));
}
}


+ 1
- 1
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentServiceImpl.java View File

@@ -560,7 +560,7 @@ public class ExperimentServiceImpl implements ExperimentService {

//处理指标
HashMap<String, Object> metricMap = JSON.parseObject(metricRecord, HashMap.class);
modelMetaVo.setMetrics(metricMap);
modelMetaVo.setMetricsParams(metricMap);

//训练数据集
List<Map<String, Object>> trainDatasetList = (List<Map<String, Object>>) modelTrainMap.get("datasets");


+ 42
- 6
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java View File

@@ -747,7 +747,7 @@ public class ModelsServiceImpl implements ModelsService {
if (buildingModel != null) {
modelMetaVo = JSON.parseObject(buildingModel.getMeta(), ModelMetaVo.class);
//获取指标
getMetrics(modelMetaVo);
transMetrics(modelMetaVo);
}

//拼接生产的元数据后写入yaml文件
@@ -806,7 +806,7 @@ public class ModelsServiceImpl implements ModelsService {
modelDependency1Dao.insert(modelDependency);
} else {
//更新模型依赖
modelDependency1Dao.updateState(modelsVo.getId(), modelsVo.getIdentifier(), modelsVo.getVersion(), Constant.State_valid);
modelDependency1Dao.updateState(modelsVo.getId(), modelsVo.getIdentifier(), modelsVo.getVersion(), meta, Constant.State_valid);
}
} else {
//保存模型依赖
@@ -973,11 +973,47 @@ public class ModelsServiceImpl implements ModelsService {
return new PageImpl<>(result, pageRequest, collect.size());
}

@Override
public Page<Map<String, Object>> queryVersions(PageRequest pageRequest, String identifier, String owner) throws Exception {
String token = gitService.checkoutToken();
List<Map<String, Object>> collect = gitService.getBrancheList(token, owner, identifier);
List<Map<String, Object>> result = collect.stream()
.filter(branch -> !"master".equals(branch.get("name")))
.skip((pageRequest.getPageNumber()) * pageRequest.getPageSize()).limit(pageRequest.getPageSize())
.collect(Collectors.toList());

for (Map<String, Object> branch : result) {
String meta = modelDependency1Dao.getMeta(identifier, owner, (String) branch.get("name"));
ModelMetaVo modelMetaVo = JSON.parseObject(meta, ModelMetaVo.class);
if (modelMetaVo.getParams() != null) {
branch.putAll(modelMetaVo.getParams());
}
if (modelMetaVo.getMetrics() != null) {
branch.putAll(modelMetaVo.getMetrics());
}
if (modelMetaVo.getMetricsParams() != null) {
branch.putAll(modelMetaVo.getMetricsParams());
}
}
return new PageImpl<>(result, pageRequest, collect.size());
}

@Override
public List<List<Map<String, Object>>> queryVersionsMetrics(List<String> runIds) throws Exception {
List<List<Map<String, Object>>> batchMetrics = new ArrayList<>();
for (String runId : runIds) {
HashMap<String, Object> map = aimsService.queryMetricsParams(runId);
List<Map<String, Object>> batchMetric = aimsService.getBatchMetric((String) map.get("run_hash"), (String) map.get("params"));
batchMetrics.add(batchMetric);
}
return batchMetrics;
}

@Override
public List<Map<String, Object>> getVersionList(String identifier, String owner) throws Exception {
String token = gitService.checkoutToken();
List<Map<String, Object>> brancheList = gitService.getBrancheList(token, owner, identifier);
return brancheList.stream()
List<Map<String, Object>> branchList = gitService.getBrancheList(token, owner, identifier);
return branchList.stream()
.filter(branch -> !"master".equals(branch.get("name")))
.collect(Collectors.toList());
}
@@ -1179,12 +1215,12 @@ public class ModelsServiceImpl implements ModelsService {
return userInfo;
}

void getMetrics(ModelMetaVo modelMetaVo) throws Exception {
void transMetrics(ModelMetaVo modelMetaVo) throws Exception {
HashMap<String, Object> result = new HashMap<>();
HashMap<String, Object> train = new HashMap<>();
HashMap<String, Object> evaluate = new HashMap<>();

HashMap<String, Object> metrics = modelMetaVo.getMetrics();
HashMap<String, Object> metrics = modelMetaVo.getMetricsParams();
JSONArray trainMetrics = (JSONArray) metrics.get("train");
if (trainMetrics != null) {
for (int i = 0; i < trainMetrics.size(); i++) {


+ 14
- 11
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/HttpUtils.java View File

@@ -1,5 +1,6 @@
package com.ruoyi.platform.utils;

import com.ruoyi.common.core.utils.StringUtils;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
@@ -101,12 +102,13 @@ public class HttpUtils {
// public static String sendGetWithToken(String url, String param,String token) {
// return sendGet(url, param, "UTF-8",token);
// }

/**
* 向指定 URL 发送带 token 的 GET 方法的请求,使用 Apache HttpClient
*
* @param url 发送请求的 URL
* @param param 请求参数,请求参数应该是 name1=value1&name2=value2 的形式。
* @param token 认证 token
* @param url 发送请求的 URL
* @param param 请求参数,请求参数应该是 name1=value1&name2=value2 的形式。
* @param token 认证 token
* @return 所代表远程资源的响应结果
*/
public static String sendGetWithToken(String url, String param, String token) {
@@ -148,6 +150,7 @@ public class HttpUtils {
}
return result;
}

/**
* 向指定 URL 发送带token的GET方法的请求
*
@@ -156,7 +159,7 @@ public class HttpUtils {
* @param contentType 编码类型
* @return 所代表远程资源的响应结果
*/
public static String sendGet(String url, String param, String contentType,String token) {
public static String sendGet(String url, String param, String contentType, String token) {
StringBuilder result = new StringBuilder();
BufferedReader in = null;
try {
@@ -196,7 +199,6 @@ public class HttpUtils {
}



/**
* 向指定 URL 发送GET方法的请求
*
@@ -209,7 +211,10 @@ public class HttpUtils {
StringBuilder result = new StringBuilder();
BufferedReader in = null;
try {
String urlNameString = url + "?" + param;
String urlNameString = url;
if (StringUtils.isNotEmpty(param)) {
urlNameString = url + "?" + param;
}
log.info("sendGet - {}", urlNameString);
URL realUrl = new URL(urlNameString);
URLConnection connection = realUrl.openConnection();
@@ -245,11 +250,11 @@ public class HttpUtils {
/**
* 向指定 URL 发送带token的POST方法的请求
*
* @param url 发送请求的 URL
* @param url 发送请求的 URL
* @param
* @return 所代表远程资源的响应结果
*/
public static String sendPostWithToken(String url, Object params, String token) throws Exception {
public static String sendPostWithToken(String url, Object params, String token) throws Exception {
String resultStr = null;
HttpPost httpPost = new HttpPost(url);
if (params != null) {
@@ -518,11 +523,10 @@ public class HttpUtils {
}



/**
* 发送 HTTP 请求并返回二进制数据流(OutputStream)。
*
* @param url 请求的 URL 地址。
* @param url 请求的 URL 地址。
* @param headers 头节点。
* @return 服务器响应的二进制数据流(OutputStream)。
* @throws IOException 如果请求失败或发生其他 I/O 错误。
@@ -626,7 +630,6 @@ public class HttpUtils {
}



// 其他方法保持不变

/**


+ 3
- 0
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelMetaVo.java View File

@@ -59,6 +59,9 @@ public class ModelMetaVo implements Serializable {
@ApiModelProperty(value = "指标")
private HashMap<String, Object> metrics;

@ApiModelProperty(value = "指标查询参数")
private HashMap<String, Object> metricsParams;

@ApiModelProperty(value = "训练任务")
private TrainTaskDepency trainTask;



+ 11
- 1
ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ModelDependency1DaoMapper.xml View File

@@ -60,6 +60,15 @@
order by create_time desc limit 1
</select>

<select id="getMeta" resultType="java.lang.String">
select meta from model_dependency1
where identifier = #{identifier}
and owner = #{owner}
and version = #{version}
and state = 1
order by create_time desc limit 1
</select>

<update id="deleteModel">
update model_dependency1
set state = 0
@@ -77,7 +86,8 @@

<update id="updateState">
update model_dependency1
set state = 1
set state = 1,
meta = #{meta}
where repo_id = #{repoId}
and identifier = #{identifier}
and version = #{version}


Loading…
Cancel
Save