ci4s
/
ci4sManagement-cloud
Not watched
1
0
Fork 0
Code
Releases 1 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
Browse Source

优化模型接口

dev-restore_mount
chenzhihang 1 year ago
parent
a888b88074
commit
006b625571
4 changed files with 352 additions and 172 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +51
    -21
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/model/NewModelFromGitController.java
  2. +13
    -3
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ModelsService.java
  3. +279
    -122
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java
  4. +9
    -26
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelsVo.java

+ 51
- 21
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/model/NewModelFromGitController.java View File

@@ -9,6 +9,7 @@ import org.springframework.core.io.InputStreamResource;
import org.springframework.data.domain.PageRequest;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;

import javax.annotation.Resource;

@@ -20,16 +21,17 @@ public class NewModelFromGitController {
@Resource
private ModelsService modelsService;

@PostMapping("/addModelAndVersion")
@ApiOperation("添加模型和版本")
@PostMapping("/addModel")
@ApiOperation("添加模型")
public AjaxResult addModelAndVersion(@RequestBody ModelsVo modelsVo) throws Exception {
return AjaxResult.success(this.modelsService.newCreateModel(modelsVo));
}

@PostMapping("/addVersion")
@ApiOperation("添加版本")
public AjaxResult addVersion(@RequestBody ModelsVo modelsVo) throws Exception {
return AjaxResult.success(this.modelsService.newCreateVersion(modelsVo));
@CrossOrigin(origins = "*", allowedHeaders = "*")
@PostMapping("/upload")
@ApiOperation(value = "上传模型", notes = "根据模型id上传模型文件，并将信息存入数据库。")
public AjaxResult uploadModel(@RequestParam("file") MultipartFile[] files, @RequestParam("uuid") String uuid) throws Exception {
return AjaxResult.success(this.modelsService.uploadModelLocal(files, uuid));
}

/**
@@ -40,11 +42,9 @@ public class NewModelFromGitController {
*/
@GetMapping("/downloadAllFiles")
@ApiOperation(value = "下载同一版本下所有模型，并打包")
public ResponseEntity<InputStreamResource> downloadAllDatasetFiles(@RequestParam(value = "repository_name") String repositoryName,
@RequestParam(value = "version") String version,
@RequestParam(value = "git_link_username") String gitLinkUsername,
@RequestParam(value = "git_link_password") String gitLinkPassword) throws Exception {
return modelsService.downloadAllModelFilesNew(repositoryName, version, gitLinkUsername, gitLinkPassword);
public ResponseEntity<InputStreamResource> downloadAllDatasetFiles(@RequestParam("identifier") String identifier,
@RequestParam(value = "version") String version) throws Exception {
return modelsService.downloadAllModelFilesNew(identifier, version);
}


@@ -61,21 +61,19 @@ public class NewModelFromGitController {
return modelsService.downloadModels(model_version_id);
}

@GetMapping("/queryDatasets")
@GetMapping("/queryModels")
@ApiOperation("模型广场公开模型分页查询,根据model_type,model_tag筛选，true公开false私有")
public AjaxResult queryDatasets(@RequestParam(value = "page") int page,
@RequestParam(value = "size") int size,
@RequestParam(value = "is_public") Boolean isPublic,
@RequestParam(value = "model_type", required = false) String modelType,
@RequestParam(value = "model_tag", required = false) String modelTag,
@RequestParam(value = "git_link_username") String gitLinkUsername,
@RequestParam(value = "git_link_password") String gitLinkPassword) throws Exception {
public AjaxResult queryModels(@RequestParam(value = "page") int page,
@RequestParam(value = "size") int size,
@RequestParam(value = "is_public", required = false) Boolean isPublic,
@RequestParam(value = "model_type", required = false) String modelType,
@RequestParam(value = "model_tag", required = false) String modelTag,
@RequestParam(value = "name", required = false) String name) throws Exception {
PageRequest pageRequest = PageRequest.of(page, size);
ModelsVo modelsVo = new ModelsVo();
modelsVo.setModelType(modelType);
modelsVo.setModelTag(modelTag);
modelsVo.setGitLinkUsername(gitLinkUsername);
modelsVo.setGitLinkPassword(gitLinkPassword);
modelsVo.setName(name);
if (isPublic) {
return AjaxResult.success(this.modelsService.newPubilcQueryByPage(modelsVo, pageRequest));
} else {
@@ -83,4 +81,36 @@ public class NewModelFromGitController {
}
}

@GetMapping("/getVersionList")
@ApiOperation(value = "获取模型分支列表")
public AjaxResult getVersionList(@RequestParam("identifier") String identifier, @RequestParam("owner")String owner) throws Exception {
return AjaxResult.success(this.modelsService.getVersionList(identifier, owner));
}


@GetMapping("/getModelDetail")
@ApiOperation(value = "获取模型详细信息")
public AjaxResult getModelDetail(@RequestParam("model_name") String modelName, @RequestParam("identifier") String identifier,@RequestParam("owner") String owner, @RequestParam("version") String version) throws Exception {
return AjaxResult.success(this.modelsService.getModelDetail(modelName, identifier, owner, version));
}

@GetMapping("/getModelDependencyTree")
@ApiOperation(value = "获取模型依赖关系树")
public AjaxResult getModelDependencyTree(@RequestParam("repoId") Integer repoId, @RequestParam("model_name") String modelName, @RequestParam("version") String version) throws Exception {
return AjaxResult.success(this.modelsService.getModelDependencyTree(repoId, modelName, version));
}

@DeleteMapping("/delete")
@ApiOperation(value = "删除模型")
public AjaxResult deleteModel(@RequestParam("repoId")Integer repoId, @RequestParam("identifier") String identifier, @RequestParam("owner") String owner) throws Exception {
this.modelsService.deleteModel(repoId,identifier, owner);
return AjaxResult.success();
}

@DeleteMapping("/deleteVersion")
@ApiOperation(value = "删除模型版本")
public AjaxResult deleteVersion(@RequestParam("repoId")Integer repoId, @RequestParam("identifier") String identifier, @RequestParam("owner") String owner, @RequestParam("version") String version) throws Exception {
this.modelsService.deleteVersion(repoId, identifier, owner, version);
return AjaxResult.success();
}
}

+ 13
- 3
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ModelsService.java View File

@@ -3,6 +3,7 @@ package com.ruoyi.platform.service;

import com.ruoyi.platform.domain.Models;
import com.ruoyi.platform.domain.ModelsVersion;
import com.ruoyi.platform.vo.ModelDependency1TreeVo;
import com.ruoyi.platform.vo.ModelsVo;
import org.springframework.core.io.InputStreamResource;
import org.springframework.data.domain.Page;
@@ -86,14 +87,23 @@ public interface ModelsService {

List<Map<String, String>> exportModels(String path, String uuid) throws Exception;


CompletableFuture<String> newCreateModel(ModelsVo modelsVo) throws Exception;

CompletableFuture<String> newCreateVersion(ModelsVo modelsVo);
List<Map<String, String>> uploadModelLocal(MultipartFile[] files, String uuid) throws Exception;

ResponseEntity<InputStreamResource> downloadAllModelFilesNew(String repositoryName, String version, String gitLinkUsername, String gitLinkPassword) throws IOException, Exception;
ResponseEntity<InputStreamResource> downloadAllModelFilesNew(String identifier, String version) throws IOException, Exception;

Page<ModelsVo> newPubilcQueryByPage(ModelsVo modelsVo, PageRequest pageRequest) throws Exception;

Page<ModelsVo> newPersonalQueryByPage(ModelsVo modelsVo, PageRequest pageRequest) throws Exception;

List<Map<String, Object>> getVersionList(String identifier, String owner) throws Exception;

ModelsVo getModelDetail(String modelName, String identifier, String owner, String version) throws Exception;

ModelDependency1TreeVo getModelDependencyTree(Integer repoId, String modelName, String version) throws Exception;

void deleteModel(Integer repoId, String identifier, String owner) throws Exception;

void deleteVersion(Integer repoId, String identifier, String owner, String version) throws Exception;
}

+ 279
- 122
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java View File

@@ -4,20 +4,22 @@ import com.ruoyi.common.core.utils.DateUtils;
import com.ruoyi.common.security.utils.SecurityUtils;
import com.ruoyi.platform.annotations.CheckDuplicate;
import com.ruoyi.platform.domain.AssetIcon;
import com.ruoyi.platform.domain.ModelDependency1;
import com.ruoyi.platform.domain.Models;
import com.ruoyi.platform.domain.ModelsVersion;
import com.ruoyi.platform.mapper.ModelDependency1Dao;
import com.ruoyi.platform.mapper.ModelsDao;
import com.ruoyi.platform.mapper.ModelsVersionDao;
import com.ruoyi.platform.service.*;
import com.ruoyi.platform.utils.*;
import com.ruoyi.platform.vo.GitProjectVo;
import com.ruoyi.platform.vo.ModelsVo;
import com.ruoyi.platform.vo.VersionVo;
import com.ruoyi.platform.vo.*;
import com.ruoyi.system.api.model.LoginUser;
import io.minio.messages.Item;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.io.InputStreamResource;
import org.springframework.data.domain.Page;
@@ -63,6 +65,9 @@ public class ModelsServiceImpl implements ModelsService {
@Resource
private ModelsVersionService modelsVersionService;

@Resource
private ModelDependency1Dao modelDependency1Dao;

@Resource
private MinioService minioService;

@@ -376,7 +381,7 @@ public class ModelsServiceImpl implements ModelsService {
@Override
@Transactional
public String insertModelAndVersion(ModelsVo modelsVo) throws Exception {
List<VersionVo> modelsVersionVos = modelsVo.getModelsVersionVos();
List<VersionVo> modelsVersionVos = modelsVo.getModelVersionVos();

if (modelsVersionVos == null || modelsVersionVos.isEmpty()) {
throw new Exception("模型版本信息错误");
@@ -511,65 +516,59 @@ public class ModelsServiceImpl implements ModelsService {
}

@Override
public CompletableFuture<String> newCreateModel(ModelsVo modelsVo) throws Exception {
public CompletableFuture<String> newCreateModel(ModelsVo modelsVo) {
return CompletableFuture.supplyAsync(() -> {
try {
gitService.checkoutToken();
LoginUser loginUser = SecurityUtils.getLoginUser();
String ci4sUsername = loginUser.getUsername();
Jedis jedis = new Jedis(redisHost);
String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername();
String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword();

Map<String, Object> userInfo = getUserInfo(ci4sUsername);
Integer userId = (Integer) userInfo.get("user_id");
String token = (String) userInfo.get("token");


String repositoryName = modelsVo.getRepositoryName() == null ? ci4sUsername + "_model_" + DateUtils.dateTimeNow() : modelsVo.getRepositoryName();
ModelDependency1 modelDependency = new ModelDependency1();
List<ModelDependency1> oldModelDependencys = modelDependency1Dao.queryByModelName(modelsVo.getName());

if (oldModelDependencys != null && !oldModelDependencys.isEmpty()) {
if (oldModelDependencys.stream().map(ModelDependency1::getVersion).anyMatch(version -> version.equals(modelsVo.getVersion()))) {
throw new RuntimeException("模型版本已存在，请勿重复创建");
}

// 拼接project
String repositoryName = ci4sUsername + "_model_" + DateUtils.dateTimeNow();
GitProjectVo gitProjectVo = new GitProjectVo();
gitProjectVo.setRepositoryName(repositoryName);
gitProjectVo.setName(modelsVo.getName());
gitProjectVo.setDescription(modelsVo.getDescription());
gitProjectVo.setPrivate(modelsVo.getAvailableRange() == 0);
gitProjectVo.setUserId(userId);

// 创建项目
Map project = gitService.createProject(gitProjectVo);

// 创建分支
String branchName = modelsVo.getVersion();
gitService.createBranch((String) userInfo.get("login"), repositoryName, branchName, "master");
// 定义标签 标签1：ci4s_model 标签2：ModelTag 标签3：ModelType
gitService.createTopic((Integer) project.get("id"), "ci4s_model");
gitService.createTopic((Integer) project.get("id"), "ModelTag_" + modelsVo.getModelTag());
gitService.createTopic((Integer) project.get("id"), "ModelType_" + modelsVo.getModelType());
// 得到项目地址
String projectUrl = gitendpoint + "/" + (String) userInfo.get("login") + "/" + repositoryName + ".git";
// 得到用户操作的路径
String url = modelsVo.getModelsVersionVos().get(0).getUrl();
String localPath1 = localPath + modelsVo.getName();
String sourcePath = url.substring(0, url.lastIndexOf("/"));
// 命令行操作 git clone 项目地址
DVCUtils.gitClone(localPath1, projectUrl, branchName, modelsVo.getGitLinkUsername(), modelsVo.getGitLinkPassword());
String s3Path = "management-platform-files/" + ci4sUsername + "/models/" + repositoryName + "/" + branchName;

// 拼接生产的元数据后写入yaml文件
YamlUtils.generateYamlFile(JsonUtils.objectToMap(modelsVo), sourcePath, "model");

DVCUtils.moveFiles(sourcePath, localPath1 + "/model");

// dvc init 初始化
DVCUtils.dvcInit(localPath1);
// 配置远程S3地址
DVCUtils.dvcRemoteAdd(localPath1, s3Path);
DVCUtils.dvcConfigS3Credentials(localPath1, endpoint);
DVCUtils.dvcConfigS3Credentials2(localPath1, accessKeyId);
DVCUtils.dvcConfigS3Credentials3(localPath1, secretAccessKey);
// dvc 跟踪
DVCUtils.dvcAdd(localPath1, "model");
// git commit
DVCUtils.gitAdd(localPath1, ".");
DVCUtils.gitCommit(localPath1, "commit from ci4s with " + ci4sUsername);
DVCUtils.gitPush(localPath1, modelsVo.getGitLinkUsername(), modelsVo.getGitLinkPassword());
// dvc push 到远程S3
DVCUtils.dvcPush(localPath1);
//新建版本
commonDvc(token, userInfo, ci4sUsername, modelsVo.getRepositoryName(), modelsVo, gitLinkUsername, gitLinkPassword, "CreateModelFromPipeline");
modelDependency.setRepoId(modelsVo.getRepoId());
} else {
//新建模型
// 拼接project
GitProjectVo gitProjectVo = new GitProjectVo();
gitProjectVo.setRepositoryName(repositoryName);
gitProjectVo.setName(modelsVo.getName());
gitProjectVo.setDescription(modelsVo.getDescription());
gitProjectVo.setPrivate(modelsVo.getAvailableRange() == 0);
gitProjectVo.setUserId(userId);

// 创建项目
Map project = gitService.createProject(gitProjectVo);
modelDependency.setRepoId((Integer) project.get("id"));
// 定义标签 标签1：ci4s_model 标签2：ModelTag 标签3：ModelType
gitService.createTopic((Integer) project.get("id"), "ci4s_model");
gitService.createTopic((Integer) project.get("id"), "modeltag_" + modelsVo.getModelTag());
gitService.createTopic((Integer) project.get("id"), "modeltype_" + modelsVo.getModelType());
commonDvc(token, userInfo, ci4sUsername, repositoryName, modelsVo, gitLinkUsername, gitLinkPassword, "createModel");
}

//保存模型依赖
modelDependency.setIdentifier(repositoryName);
modelDependency.setModelName(modelsVo.getName());
modelDependency.setVersion(modelsVo.getVersion());
modelDependency.setParentModel(modelsVo.getParentModel());
modelDependency.setOwner(ci4sUsername);
modelDependency1Dao.insert(modelDependency);
return "新增模型成功";
} catch (Exception e) {
logger.error(e.getMessage());
@@ -578,67 +577,45 @@ public class ModelsServiceImpl implements ModelsService {
});
}


@Override
public CompletableFuture<String> newCreateVersion(ModelsVo modelsVo) {
return CompletableFuture.supplyAsync(() -> {
try {
String token = gitService.login(modelsVo.getGitLinkUsername(), modelsVo.getGitLinkPassword());
LoginUser loginUser = SecurityUtils.getLoginUser();
String ci4sUsername = loginUser.getUsername();
Jedis jedis = new Jedis(redisHost);
String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
// 创建分支
String branchName = modelsVo.getVersion();
String repositoryName = modelsVo.getRepositoryName();
gitService.createBranch((String) userInfo.get("login"), repositoryName, branchName, "master");
// 得到项目地址
String projectUrl = gitendpoint + "/" + (String) userInfo.get("login") + "/" + repositoryName + ".git";
// 得到用户操作的路径
String url = modelsVo.getModelVersionVos().get(0).getUrl();
String localPath1 = localPath + ci4sUsername + "/model/" + modelsVo.getName();
String sourcePath = url.substring(0, url.lastIndexOf("/"));
// 命令行操作 git clone 项目地址
DVCUtils.gitClone(localPath1, projectUrl, branchName, modelsVo.getGitLinkUsername(), modelsVo.getGitLinkPassword());
String s3Path = "management-platform-files/" + ci4sUsername + "/model/" + repositoryName + "/" + branchName;
//拼接生产的元数据后写入yaml文件
YamlUtils.generateYamlFile(JsonUtils.objectToMap(modelsVo), sourcePath, "dataset");

DVCUtils.moveFiles(sourcePath, localPath1 + "/model");
// dvc init 初始化
DVCUtils.dvcInit(localPath1);
// 配置远程S3地址
DVCUtils.dvcRemoteAdd(localPath1, s3Path);
DVCUtils.dvcConfigS3Credentials(localPath1, endpoint);
DVCUtils.dvcConfigS3Credentials2(localPath1, accessKeyId);
DVCUtils.dvcConfigS3Credentials3(localPath1, secretAccessKey);
// dvc 跟踪
DVCUtils.dvcAdd(localPath1, "model");
// git commit
DVCUtils.gitAdd(localPath1, ".");
DVCUtils.gitCommit(localPath1, "commit from ci4s with " + ci4sUsername);
DVCUtils.gitPush(localPath1, modelsVo.getGitLinkUsername(), modelsVo.getGitLinkPassword());
// dvc push 到远程S3
DVCUtils.dvcPush(localPath1);
return "新增模型成功";
} catch (Exception e) {
throw new RuntimeException(e);
}
});
public List<Map<String, String>> uploadModelLocal(MultipartFile[] files, String uuid) throws Exception {
List<Map<String, String>> results = new ArrayList<>();

for (MultipartFile file : files) {
// 构建objectName
String username = SecurityUtils.getLoginUser().getUsername();
String fileName = file.getOriginalFilename();
String path = "/temp/" + username + "/models/" + uuid + "/model/";
long sizeInBytes = file.getSize();
String formattedSize = FileUtil.formatFileSize(sizeInBytes);
File targetFile = new File(path, file.getOriginalFilename());
// 确保目录存在
targetFile.getParentFile().mkdirs();
// 保存文件到目标路径
FileUtils.copyInputStreamToFile(file.getInputStream(), targetFile);
// 返回上传文件的路径
String absolutePath = targetFile.getAbsolutePath();
Map<String, String> result = new HashMap<>();
result.put("fileName", fileName);
result.put("url", absolutePath); // objectName根据实际情况定义
result.put("fileSize", formattedSize);
results.add(result);
}
return results;
}

@Override
public ResponseEntity<InputStreamResource> downloadAllModelFilesNew(String repositoryName, String version, String gitLinkUsername, String gitLinkPassword) throws IOException, Exception {
public ResponseEntity<InputStreamResource> downloadAllModelFilesNew(String identifier, String version) throws Exception {
// 命令行操作 git clone 项目地址
LoginUser loginUser = SecurityUtils.getLoginUser();
String ci4sUsername = loginUser.getUsername();
String token = gitService.login(gitLinkUsername, gitLinkPassword);
Jedis jedis = new Jedis(redisHost);
String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
Integer userId = (Integer) userInfo.get("user_id");
String projectUrl = gitendpoint + "/" + (String) userInfo.get("login") + "/" + repositoryName + ".git";
String localPath1 = localPath + ci4sUsername + "/model/" + repositoryName;
String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername();
String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword();
Map<String, Object> userInfo = getUserInfo(ci4sUsername);

String projectUrl = gitendpoint + "/" + userInfo.get("login") + "/" + identifier + ".git";
String localPath1 = localPath + ci4sUsername + "/model/" + identifier;
File folder = new File(localPath1);
if (folder.exists() && folder.isDirectory()) {
//切换分支
@@ -685,17 +662,18 @@ public class ModelsServiceImpl implements ModelsService {
public Page<ModelsVo> newPubilcQueryByPage(ModelsVo modelsVo, PageRequest pageRequest) throws Exception {
LoginUser loginUser = SecurityUtils.getLoginUser();
String ci4sUsername = loginUser.getUsername();
String token = gitService.login(modelsVo.getGitLinkUsername(), modelsVo.getGitLinkPassword());
Jedis jedis = new Jedis(redisHost);
String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
Map<String, Object> userInfo = getUserInfo(ci4sUsername);
String token = (String) userInfo.get("token");

//拼接查询url
String modelTagName = modelsVo.getModelTag();
String modelTypeName = modelsVo.getModelType();
String search = modelsVo.getName();
String topic_name = "ci4s_model";
topic_name = StringUtils.isEmpty(modelTagName) ? topic_name : topic_name + ",modeltag_" + modelTypeName;
topic_name = StringUtils.isEmpty(modelTagName) ? topic_name : topic_name + ",modeltype_" + modelTypeName;
String url = gitendpoint + "/api/users/" + (String) userInfo.get("login") + "/projects.json?page=" + pageRequest.getPageNumber() + "&limit=" + pageRequest.getPageSize() + "&category=manage&topic_name=" + topic_name;

String url = gitendpoint + "/api/users/" + userInfo.get("login") + "/projects.json?page=" + pageRequest.getPageNumber() + "&limit=" + pageRequest.getPageSize() + "&category=manage&topic_name=" + topic_name + "&search=" + search;
String req = HttpUtils.sendGetWithToken(url, null, token);
Map<String, Object> stringObjectMap = JacksonUtil.parseJSONStr2Map(req);
Integer total = (Integer) stringObjectMap.get("count");
@@ -707,19 +685,19 @@ public class ModelsServiceImpl implements ModelsService {
public Page<ModelsVo> newPersonalQueryByPage(ModelsVo modelsVo, PageRequest pageRequest) throws Exception {
LoginUser loginUser = SecurityUtils.getLoginUser();
String ci4sUsername = loginUser.getUsername();
String token = gitService.login(modelsVo.getGitLinkUsername(), modelsVo.getGitLinkPassword());
Jedis jedis = new Jedis(redisHost);
String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
Map<String, Object> userInfo = getUserInfo(ci4sUsername);
Integer userId = (Integer) userInfo.get("user_id");
String token = (String) userInfo.get("token");

//拼接查询url
String modelTagName = modelsVo.getModelTag();
String modelTypeName = modelsVo.getModelType();
String search = modelsVo.getName();
String topic_name = "ci4s_model";
topic_name = StringUtils.isEmpty(modelTagName) ? topic_name : topic_name + ",modeltag_" + modelTagName;
topic_name = StringUtils.isEmpty(modelTypeName) ? topic_name : topic_name + ",modeltype_" + modelTypeName;

String url = gitendpoint + "/api/projects.json?user_id=" + userId + "&page=" + pageRequest.getPageNumber() + "&limit=" + pageRequest.getPageSize() + "&sort_by=praises_count&topic_name=" + topic_name;
String url = gitendpoint + "/api/projects.json?user_id=" + userId + "&page=" + pageRequest.getPageNumber() + "&limit=" + pageRequest.getPageSize() + "&sort_by=praises_count&topic_name=" + topic_name + "&search=" + search;
String req = HttpUtils.sendGetWithToken(url, null, token);
Map<String, Object> stringObjectMap = JacksonUtil.parseJSONStr2Map(req);
Integer total = (Integer) stringObjectMap.get("total_count");
@@ -727,6 +705,147 @@ public class ModelsServiceImpl implements ModelsService {
return new PageImpl<>(convert(projects), pageRequest, total);
}

@Override
public List<Map<String, Object>> getVersionList(String identifier, String owner) throws Exception {
List<Map<String, Object>> brancheList = gitService.getBrancheList(owner, identifier);
return brancheList.stream()
.filter(branch -> "master".equals(branch.get("name")))
.collect(Collectors.toList());
}

@Override
public ModelsVo getModelDetail(String modelName, String identifier, String owner, String version) throws Exception {
LoginUser loginUser = SecurityUtils.getLoginUser();
String ci4sUsername = loginUser.getUsername();
String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername();
String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword();

if (StringUtils.isEmpty(version)) {
List<Map<String, Object>> versionList = this.getVersionList(owner, identifier);
if (versionList.size() == 0) {
throw new Exception("数据集文件不存在");
}
version = (String) versionList.get(0).get("name");
}

// git pull操作，然后读取里面的文件列表，列出每个文件的大小和名称，封装成MAP
List<Map<String, Object>> fileDetailsAfterGitPull = DVCUtils.getFileDetailsAfterGitPull(localPath + ci4sUsername + "/model/", modelName, version, "model", gitLinkUsername, gitLinkPassword);

Map<String, Object> stringObjectMap = YamlUtils.loadYamlFile(localPath + ci4sUsername + "/model/" + modelName + "/" + version + "/metadata/metadata.yaml");
assert stringObjectMap != null;

ModelMetaVo modelMetaVo = ConvertUtil.convertMapToObject(stringObjectMap, ModelMetaVo.class);
ModelsVo modelsVo = new ModelsVo();
BeanUtils.copyProperties(modelMetaVo, modelsVo);

List<VersionVo> versionVos = new ArrayList<>();
if (!fileDetailsAfterGitPull.isEmpty()) {
for (Map<String, Object> fileDetail : fileDetailsAfterGitPull) {
VersionVo versionVo = new VersionVo();
versionVo.setUrl((String) fileDetail.get("filePath"));
versionVo.setFileName((String) fileDetail.get("fileName"));
long size = (long) fileDetail.get("size");
versionVo.setFileSize(FileUtil.formatFileSize(size));
versionVos.add(versionVo);
}
}
modelsVo.setModelVersionVos(versionVos);

return modelsVo;
}

@Override
public ModelDependency1TreeVo getModelDependencyTree(Integer repoId, String modelName, String version) throws Exception {
ModelDependency1 modelDependency1 = modelDependency1Dao.queryByModelNameAndVersion(repoId, modelName, version);
ModelDependency1TreeVo modelDependency1TreeVo = new ModelDependency1TreeVo();
BeanUtils.copyProperties(modelDependency1, modelDependency1TreeVo);

//递归查询父模型
List<ModelDependency1> parentModelList = new ArrayList<>();
getParentModel(parentModelList, modelDependency1);
modelDependency1TreeVo.setPatrentModelList(parentModelList);

//递归查询子模型
getChildModel(modelDependency1TreeVo);

return modelDependency1TreeVo;
}

@Override
public void deleteModel(Integer repoId, String identifier, String owner) throws Exception {
LoginUser loginUser = SecurityUtils.getLoginUser();
String ci4sUsername = loginUser.getUsername();
Map<String, Object> userInfo = getUserInfo(ci4sUsername);
String token = (String) userInfo.get("token");


gitService.deleteProject(owner, identifier);
//删除模型依赖
modelDependency1Dao.deleteModel(repoId, identifier, owner, null);
modelDependency1Dao.deleteModelDependency(repoId, identifier, owner, null);
}

@Override
public void deleteVersion(Integer repoId, String identifier, String owner, String version) throws Exception {
LoginUser loginUser = SecurityUtils.getLoginUser();
String ci4sUsername = loginUser.getUsername();

gitService.deleteBranch(owner, identifier, version);
//删除模型依赖
modelDependency1Dao.deleteModel(repoId, identifier, owner, version);
modelDependency1Dao.deleteModelDependency(repoId, identifier, owner, version);
}

void commonDvc(String token, Map<String, Object> userInfo, String username, String repositoryName, ModelsVo modelsVo, String gitLinkUsername, String gitLinkPassword, String type) throws Exception {
String branchName = modelsVo.getVersion();

// 创建分支
gitService.createBranch((String) userInfo.get("login"), repositoryName, branchName, "master");

String projectUrl = gitendpoint + "/" + (String) userInfo.get("login") + "/" + repositoryName + ".git";
String url = modelsVo.getModelVersionVos().get(0).getUrl();
String sourcePath = url.substring(0, url.lastIndexOf("/"));
String rootPath = localPath + username + "/model/" + modelsVo.getName() + "/" + branchName;
String modelPath = rootPath + "/model";
String metaPath = rootPath + "/metadata";


if (type.equals("CreateModelFromPipeline") && FileUtil.checkDirectoryExists(rootPath)) {
DVCUtils.gitFetch(rootPath, gitLinkUsername, gitLinkPassword);
DVCUtils.gitCheckoutBranch(rootPath, branchName);
} else {
DVCUtils.gitClone(rootPath, projectUrl, branchName, modelsVo.getGitLinkUsername(), modelsVo.getGitLinkPassword());
}

DVCUtils.moveFiles(sourcePath, modelPath);

//拼接生产的元数据后写入yaml文件
ModelMetaVo modelMetaVo = new ModelMetaVo();
BeanUtils.copyProperties(modelsVo, modelMetaVo);
YamlUtils.generateYamlFile(JsonUtils.objectToMap(modelMetaVo), metaPath, "metadata");

// dvc init 初始化
DVCUtils.dvcInit(rootPath);

// 配置远程S3地址
String s3Path = "management-platform-files/" + username + "/model/" + repositoryName + "/" + branchName;
DVCUtils.dvcRemoteAdd(rootPath, s3Path);
DVCUtils.dvcConfigS3Credentials(rootPath, endpoint);
DVCUtils.dvcConfigS3Credentials2(rootPath, accessKeyId);
DVCUtils.dvcConfigS3Credentials3(rootPath, secretAccessKey);

// dvc 跟踪
DVCUtils.dvcAdd(rootPath, "model");

// git commit
DVCUtils.gitAdd(rootPath, ".");
DVCUtils.gitCommit(rootPath, "commit from ci4s with " + username);
DVCUtils.gitPush(rootPath, modelsVo.getGitLinkUsername(), modelsVo.getGitLinkPassword());

// dvc push 到远程S3
DVCUtils.dvcPush(rootPath);
}

public List<ModelsVo> convert(List<Map<String, Object>> lst) {
if (lst != null && lst.size() > 0) {
List<ModelsVo> newModelVos = ConvertUtil.convertListMapToObjectList(lst, ModelsVo.class);
@@ -753,10 +872,48 @@ public class ModelsServiceImpl implements ModelsService {
}
}
}

return newModelVos;
}
return new ArrayList<>();
}

void getParentModel(List<ModelDependency1> modelList, ModelDependency1 modelDependency1) {
if (modelDependency1.getParentModel() != null) {
String[] split = modelDependency1.getParentModel().split(":");
Integer parentRepoId = Integer.valueOf(split[0]);
String parentModelName = split[1];
String parentModelVersion = split[2];
ModelDependency1 parentModel = modelDependency1Dao.queryByModelNameAndVersion(parentRepoId, parentModelName, parentModelVersion);
modelList.add(parentModel);
if (parentModel != null && parentModel.getParentModel() != null) {
getParentModel(modelList, parentModel);
}
}
}

void getChildModel(ModelDependency1TreeVo modelDependency1TreeVo) {
List<ModelDependency1TreeVo> childModelList = new ArrayList<>();
String model = modelDependency1TreeVo.getRepoId() + ":" + modelDependency1TreeVo.getModelName() + ":" + modelDependency1TreeVo.getVersion();

List<ModelDependency1> children = modelDependency1Dao.queryByParentModel(model);
if (children != null && !children.isEmpty()) {
for (ModelDependency1 modelDependency1 : children) {
ModelDependency1TreeVo modelDependency1TreeVo1 = new ModelDependency1TreeVo();
BeanUtils.copyProperties(modelDependency1, modelDependency1TreeVo1);
getChildModel(modelDependency1TreeVo1);
childModelList.add(modelDependency1TreeVo1);
}
}
modelDependency1TreeVo.setChildModelList(childModelList);
}

Map<String, Object> getUserInfo(String ci4sUsername) throws IOException {
Jedis jedis = new Jedis(redisHost);
String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);

String token = gitService.checkoutToken();
userInfo.put("token", token);
return userInfo;
}
}

+ 9
- 26
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelsVo.java View File

@@ -10,15 +10,9 @@ import java.util.List;

@JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy.class)
@Data
public class ModelsVo implements Serializable {
public class ModelsVo extends ModelMetaVo implements Serializable {


@ApiModelProperty(name = "name")
private String name;

// private String version;
@ApiModelProperty(name = "description")
private String description;
private Integer id;

/**
* 模型可见范围
@@ -26,22 +20,6 @@ public class ModelsVo implements Serializable {
@ApiModelProperty(name = "available_range")
private int availableRange;

// private String url;
@ApiModelProperty(name = "model_type")
private String modelType;


@ApiModelProperty(name = "model_tag")
private String modelTag;
/**
* 版本
*/
@ApiModelProperty(name = "version")
private String version;

@ApiModelProperty(name = "model_version_vos")
private List<VersionVo> modelVersionVos;

/**
* 状态
*/
@@ -53,14 +31,19 @@ public class ModelsVo implements Serializable {
private String gitLinkPassword;

@ApiModelProperty(name = "models_version_vos")
private List<VersionVo> modelsVersionVos;
private List<VersionVo> modelVersionVos;

/**
* 数据集仓库名称
* 模型仓库名称
*/
@ApiModelProperty(name = "repository_name")
private String repositoryName;

@ApiModelProperty(name = "repo_id")
private Integer repoId;

/**
* 父模型
*/
private String parentModel;
}

Write Preview
Loading…
Cancel
Save