diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/model/NewModelFromGitController.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/model/NewModelFromGitController.java index 2966d2bb..145f84d8 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/model/NewModelFromGitController.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/model/NewModelFromGitController.java @@ -9,6 +9,7 @@ import org.springframework.core.io.InputStreamResource; import org.springframework.data.domain.PageRequest; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.*; +import org.springframework.web.multipart.MultipartFile; import javax.annotation.Resource; @@ -20,16 +21,17 @@ public class NewModelFromGitController { @Resource private ModelsService modelsService; - @PostMapping("/addModelAndVersion") - @ApiOperation("添加模型和版本") + @PostMapping("/addModel") + @ApiOperation("添加模型") public AjaxResult addModelAndVersion(@RequestBody ModelsVo modelsVo) throws Exception { return AjaxResult.success(this.modelsService.newCreateModel(modelsVo)); } - @PostMapping("/addVersion") - @ApiOperation("添加版本") - public AjaxResult addVersion(@RequestBody ModelsVo modelsVo) throws Exception { - return AjaxResult.success(this.modelsService.newCreateVersion(modelsVo)); + @CrossOrigin(origins = "*", allowedHeaders = "*") + @PostMapping("/upload") + @ApiOperation(value = "上传模型", notes = "根据模型id上传模型文件，并将信息存入数据库。") + public AjaxResult uploadModel(@RequestParam("file") MultipartFile[] files, @RequestParam("uuid") String uuid) throws Exception { + return AjaxResult.success(this.modelsService.uploadModelLocal(files, uuid)); } /** @@ -40,11 +42,9 @@ public class NewModelFromGitController { */ @GetMapping("/downloadAllFiles") @ApiOperation(value = "下载同一版本下所有模型，并打包") - public ResponseEntity downloadAllDatasetFiles(@RequestParam(value = "repository_name") String repositoryName, - @RequestParam(value = "version") String version, - @RequestParam(value = "git_link_username") String gitLinkUsername, - @RequestParam(value = "git_link_password") String gitLinkPassword) throws Exception { - return modelsService.downloadAllModelFilesNew(repositoryName, version, gitLinkUsername, gitLinkPassword); + public ResponseEntity downloadAllDatasetFiles(@RequestParam("identifier") String identifier, + @RequestParam(value = "version") String version) throws Exception { + return modelsService.downloadAllModelFilesNew(identifier, version); } @@ -61,21 +61,19 @@ public class NewModelFromGitController { return modelsService.downloadModels(model_version_id); } - @GetMapping("/queryDatasets") + @GetMapping("/queryModels") @ApiOperation("模型广场公开模型分页查询,根据model_type,model_tag筛选，true公开false私有") - public AjaxResult queryDatasets(@RequestParam(value = "page") int page, - @RequestParam(value = "size") int size, - @RequestParam(value = "is_public") Boolean isPublic, - @RequestParam(value = "model_type", required = false) String modelType, - @RequestParam(value = "model_tag", required = false) String modelTag, - @RequestParam(value = "git_link_username") String gitLinkUsername, - @RequestParam(value = "git_link_password") String gitLinkPassword) throws Exception { + public AjaxResult queryModels(@RequestParam(value = "page") int page, + @RequestParam(value = "size") int size, + @RequestParam(value = "is_public", required = false) Boolean isPublic, + @RequestParam(value = "model_type", required = false) String modelType, + @RequestParam(value = "model_tag", required = false) String modelTag, + @RequestParam(value = "name", required = false) String name) throws Exception { PageRequest pageRequest = PageRequest.of(page, size); ModelsVo modelsVo = new ModelsVo(); modelsVo.setModelType(modelType); modelsVo.setModelTag(modelTag); - modelsVo.setGitLinkUsername(gitLinkUsername); - modelsVo.setGitLinkPassword(gitLinkPassword); + modelsVo.setName(name); if (isPublic) { return AjaxResult.success(this.modelsService.newPubilcQueryByPage(modelsVo, pageRequest)); } else { @@ -83,4 +81,36 @@ public class NewModelFromGitController { } } + @GetMapping("/getVersionList") + @ApiOperation(value = "获取模型分支列表") + public AjaxResult getVersionList(@RequestParam("identifier") String identifier, @RequestParam("owner")String owner) throws Exception { + return AjaxResult.success(this.modelsService.getVersionList(identifier, owner)); + } + + + @GetMapping("/getModelDetail") + @ApiOperation(value = "获取模型详细信息") + public AjaxResult getModelDetail(@RequestParam("model_name") String modelName, @RequestParam("identifier") String identifier,@RequestParam("owner") String owner, @RequestParam("version") String version) throws Exception { + return AjaxResult.success(this.modelsService.getModelDetail(modelName, identifier, owner, version)); + } + + @GetMapping("/getModelDependencyTree") + @ApiOperation(value = "获取模型依赖关系树") + public AjaxResult getModelDependencyTree(@RequestParam("repoId") Integer repoId, @RequestParam("model_name") String modelName, @RequestParam("version") String version) throws Exception { + return AjaxResult.success(this.modelsService.getModelDependencyTree(repoId, modelName, version)); + } + + @DeleteMapping("/delete") + @ApiOperation(value = "删除模型") + public AjaxResult deleteModel(@RequestParam("repoId")Integer repoId, @RequestParam("identifier") String identifier, @RequestParam("owner") String owner) throws Exception { + this.modelsService.deleteModel(repoId,identifier, owner); + return AjaxResult.success(); + } + + @DeleteMapping("/deleteVersion") + @ApiOperation(value = "删除模型版本") + public AjaxResult deleteVersion(@RequestParam("repoId")Integer repoId, @RequestParam("identifier") String identifier, @RequestParam("owner") String owner, @RequestParam("version") String version) throws Exception { + this.modelsService.deleteVersion(repoId, identifier, owner, version); + return AjaxResult.success(); + } } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ModelsService.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ModelsService.java index 8502fd95..a2aac1b1 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ModelsService.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ModelsService.java @@ -3,6 +3,7 @@ package com.ruoyi.platform.service; import com.ruoyi.platform.domain.Models; import com.ruoyi.platform.domain.ModelsVersion; +import com.ruoyi.platform.vo.ModelDependency1TreeVo; import com.ruoyi.platform.vo.ModelsVo; import org.springframework.core.io.InputStreamResource; import org.springframework.data.domain.Page; @@ -86,14 +87,23 @@ public interface ModelsService { List> exportModels(String path, String uuid) throws Exception; - CompletableFuture newCreateModel(ModelsVo modelsVo) throws Exception; - CompletableFuture newCreateVersion(ModelsVo modelsVo); + List> uploadModelLocal(MultipartFile[] files, String uuid) throws Exception; - ResponseEntity downloadAllModelFilesNew(String repositoryName, String version, String gitLinkUsername, String gitLinkPassword) throws IOException, Exception; + ResponseEntity downloadAllModelFilesNew(String identifier, String version) throws IOException, Exception; Page newPubilcQueryByPage(ModelsVo modelsVo, PageRequest pageRequest) throws Exception; Page newPersonalQueryByPage(ModelsVo modelsVo, PageRequest pageRequest) throws Exception; + + List> getVersionList(String identifier, String owner) throws Exception; + + ModelsVo getModelDetail(String modelName, String identifier, String owner, String version) throws Exception; + + ModelDependency1TreeVo getModelDependencyTree(Integer repoId, String modelName, String version) throws Exception; + + void deleteModel(Integer repoId, String identifier, String owner) throws Exception; + + void deleteVersion(Integer repoId, String identifier, String owner, String version) throws Exception; } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/MinioServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/MinioServiceImpl.java index 6a1656b2..380f96c6 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/MinioServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/MinioServiceImpl.java @@ -63,7 +63,9 @@ public class MinioServiceImpl implements MinioService { try (InputStream inputStream = file.getInputStream()){ minioUtil.uploadObject(bucketName, objectName, inputStream); result.put("fileName", file.getOriginalFilename()); - result.put("url", objectName); // objectName根据实际情况定义 + int lastIndex = objectName.lastIndexOf('/'); + String url = objectName.substring(0, lastIndex); + result.put("url", url); // objectName根据实际情况定义 result.put("fileSize", formattedSize); } catch (Exception e) { throw new Exception("上传文件失败: " + e.getMessage(), e); diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java index 16472b8a..7d0bf9ae 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java @@ -4,20 +4,22 @@ import com.ruoyi.common.core.utils.DateUtils; import com.ruoyi.common.security.utils.SecurityUtils; import com.ruoyi.platform.annotations.CheckDuplicate; import com.ruoyi.platform.domain.AssetIcon; +import com.ruoyi.platform.domain.ModelDependency1; import com.ruoyi.platform.domain.Models; import com.ruoyi.platform.domain.ModelsVersion; +import com.ruoyi.platform.mapper.ModelDependency1Dao; import com.ruoyi.platform.mapper.ModelsDao; import com.ruoyi.platform.mapper.ModelsVersionDao; import com.ruoyi.platform.service.*; import com.ruoyi.platform.utils.*; -import com.ruoyi.platform.vo.GitProjectVo; -import com.ruoyi.platform.vo.ModelsVo; -import com.ruoyi.platform.vo.VersionVo; +import com.ruoyi.platform.vo.*; import com.ruoyi.system.api.model.LoginUser; import io.minio.messages.Item; +import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.core.io.InputStreamResource; import org.springframework.data.domain.Page; @@ -63,6 +65,9 @@ public class ModelsServiceImpl implements ModelsService { @Resource private ModelsVersionService modelsVersionService; + @Resource + private ModelDependency1Dao modelDependency1Dao; + @Resource private MinioService minioService; @@ -376,7 +381,7 @@ public class ModelsServiceImpl implements ModelsService { @Override @Transactional public String insertModelAndVersion(ModelsVo modelsVo) throws Exception { - List modelsVersionVos = modelsVo.getModelsVersionVos(); + List modelsVersionVos = modelsVo.getModelVersionVos(); if (modelsVersionVos == null || modelsVersionVos.isEmpty()) { throw new Exception("模型版本信息错误"); @@ -511,65 +516,59 @@ public class ModelsServiceImpl implements ModelsService { } @Override - public CompletableFuture newCreateModel(ModelsVo modelsVo) throws Exception { + public CompletableFuture newCreateModel(ModelsVo modelsVo) { return CompletableFuture.supplyAsync(() -> { try { + gitService.checkoutToken(); LoginUser loginUser = SecurityUtils.getLoginUser(); String ci4sUsername = loginUser.getUsername(); - Jedis jedis = new Jedis(redisHost); - String userReq = jedis.get(ci4sUsername + "_gitUserInfo"); - Map userInfo = JsonUtils.jsonToMap(userReq); + String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername(); + String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword(); + + Map userInfo = getUserInfo(ci4sUsername); Integer userId = (Integer) userInfo.get("user_id"); + String token = (String) userInfo.get("token"); + + + String repositoryName = modelsVo.getRepositoryName() == null ? ci4sUsername + "_model_" + DateUtils.dateTimeNow() : modelsVo.getRepositoryName(); + ModelDependency1 modelDependency = new ModelDependency1(); + List oldModelDependencys = modelDependency1Dao.queryByModelName(modelsVo.getName()); + + if (oldModelDependencys != null && !oldModelDependencys.isEmpty()) { + if (oldModelDependencys.stream().map(ModelDependency1::getVersion).anyMatch(version -> version.equals(modelsVo.getVersion()))) { + throw new RuntimeException("模型版本已存在，请勿重复创建"); + } - // 拼接project - String repositoryName = ci4sUsername + "_model_" + DateUtils.dateTimeNow(); - GitProjectVo gitProjectVo = new GitProjectVo(); - gitProjectVo.setRepositoryName(repositoryName); - gitProjectVo.setName(modelsVo.getName()); - gitProjectVo.setDescription(modelsVo.getDescription()); - gitProjectVo.setPrivate(modelsVo.getAvailableRange() == 0); - gitProjectVo.setUserId(userId); - - // 创建项目 - Map project = gitService.createProject(gitProjectVo); - - // 创建分支 - String branchName = modelsVo.getVersion(); - gitService.createBranch((String) userInfo.get("login"), repositoryName, branchName, "master"); - // 定义标签 标签1：ci4s_model 标签2：ModelTag 标签3：ModelType - gitService.createTopic((Integer) project.get("id"), "ci4s_model"); - gitService.createTopic((Integer) project.get("id"), "ModelTag_" + modelsVo.getModelTag()); - gitService.createTopic((Integer) project.get("id"), "ModelType_" + modelsVo.getModelType()); - // 得到项目地址 - String projectUrl = gitendpoint + "/" + (String) userInfo.get("login") + "/" + repositoryName + ".git"; - // 得到用户操作的路径 - String url = modelsVo.getModelsVersionVos().get(0).getUrl(); - String localPath1 = localPath + modelsVo.getName(); - String sourcePath = url.substring(0, url.lastIndexOf("/")); - // 命令行操作 git clone 项目地址 - DVCUtils.gitClone(localPath1, projectUrl, branchName, modelsVo.getGitLinkUsername(), modelsVo.getGitLinkPassword()); - String s3Path = "management-platform-files/" + ci4sUsername + "/models/" + repositoryName + "/" + branchName; - - // 拼接生产的元数据后写入yaml文件 - YamlUtils.generateYamlFile(JsonUtils.objectToMap(modelsVo), sourcePath, "model"); - - DVCUtils.moveFiles(sourcePath, localPath1 + "/model"); - - // dvc init 初始化 - DVCUtils.dvcInit(localPath1); - // 配置远程S3地址 - DVCUtils.dvcRemoteAdd(localPath1, s3Path); - DVCUtils.dvcConfigS3Credentials(localPath1, endpoint); - DVCUtils.dvcConfigS3Credentials2(localPath1, accessKeyId); - DVCUtils.dvcConfigS3Credentials3(localPath1, secretAccessKey); - // dvc 跟踪 - DVCUtils.dvcAdd(localPath1, "model"); - // git commit - DVCUtils.gitAdd(localPath1, "."); - DVCUtils.gitCommit(localPath1, "commit from ci4s with " + ci4sUsername); - DVCUtils.gitPush(localPath1, modelsVo.getGitLinkUsername(), modelsVo.getGitLinkPassword()); - // dvc push 到远程S3 - DVCUtils.dvcPush(localPath1); + //新建版本 + commonDvc(token, userInfo, ci4sUsername, modelsVo.getRepositoryName(), modelsVo, gitLinkUsername, gitLinkPassword, "CreateModelFromPipeline"); + modelDependency.setRepoId(modelsVo.getRepoId()); + } else { + //新建模型 + // 拼接project + GitProjectVo gitProjectVo = new GitProjectVo(); + gitProjectVo.setRepositoryName(repositoryName); + gitProjectVo.setName(modelsVo.getName()); + gitProjectVo.setDescription(modelsVo.getDescription()); + gitProjectVo.setPrivate(modelsVo.getAvailableRange() == 0); + gitProjectVo.setUserId(userId); + + // 创建项目 + Map project = gitService.createProject(gitProjectVo); + modelDependency.setRepoId((Integer) project.get("id")); + // 定义标签 标签1：ci4s_model 标签2：ModelTag 标签3：ModelType + gitService.createTopic((Integer) project.get("id"), "ci4s_model"); + gitService.createTopic((Integer) project.get("id"), "modeltag_" + modelsVo.getModelTag()); + gitService.createTopic((Integer) project.get("id"), "modeltype_" + modelsVo.getModelType()); + commonDvc(token, userInfo, ci4sUsername, repositoryName, modelsVo, gitLinkUsername, gitLinkPassword, "createModel"); + } + + //保存模型依赖 + modelDependency.setIdentifier(repositoryName); + modelDependency.setModelName(modelsVo.getName()); + modelDependency.setVersion(modelsVo.getVersion()); + modelDependency.setParentModel(modelsVo.getParentModel()); + modelDependency.setOwner(ci4sUsername); + modelDependency1Dao.insert(modelDependency); return "新增模型成功"; } catch (Exception e) { logger.error(e.getMessage()); @@ -578,67 +577,45 @@ public class ModelsServiceImpl implements ModelsService { }); } + @Override - public CompletableFuture newCreateVersion(ModelsVo modelsVo) { - return CompletableFuture.supplyAsync(() -> { - try { - String token = gitService.login(modelsVo.getGitLinkUsername(), modelsVo.getGitLinkPassword()); - LoginUser loginUser = SecurityUtils.getLoginUser(); - String ci4sUsername = loginUser.getUsername(); - Jedis jedis = new Jedis(redisHost); - String userReq = jedis.get(ci4sUsername + "_gitUserInfo"); - Map userInfo = JsonUtils.jsonToMap(userReq); - // 创建分支 - String branchName = modelsVo.getVersion(); - String repositoryName = modelsVo.getRepositoryName(); - gitService.createBranch((String) userInfo.get("login"), repositoryName, branchName, "master"); - // 得到项目地址 - String projectUrl = gitendpoint + "/" + (String) userInfo.get("login") + "/" + repositoryName + ".git"; - // 得到用户操作的路径 - String url = modelsVo.getModelVersionVos().get(0).getUrl(); - String localPath1 = localPath + ci4sUsername + "/model/" + modelsVo.getName(); - String sourcePath = url.substring(0, url.lastIndexOf("/")); - // 命令行操作 git clone 项目地址 - DVCUtils.gitClone(localPath1, projectUrl, branchName, modelsVo.getGitLinkUsername(), modelsVo.getGitLinkPassword()); - String s3Path = "management-platform-files/" + ci4sUsername + "/model/" + repositoryName + "/" + branchName; - //拼接生产的元数据后写入yaml文件 - YamlUtils.generateYamlFile(JsonUtils.objectToMap(modelsVo), sourcePath, "dataset"); - - DVCUtils.moveFiles(sourcePath, localPath1 + "/model"); - // dvc init 初始化 - DVCUtils.dvcInit(localPath1); - // 配置远程S3地址 - DVCUtils.dvcRemoteAdd(localPath1, s3Path); - DVCUtils.dvcConfigS3Credentials(localPath1, endpoint); - DVCUtils.dvcConfigS3Credentials2(localPath1, accessKeyId); - DVCUtils.dvcConfigS3Credentials3(localPath1, secretAccessKey); - // dvc 跟踪 - DVCUtils.dvcAdd(localPath1, "model"); - // git commit - DVCUtils.gitAdd(localPath1, "."); - DVCUtils.gitCommit(localPath1, "commit from ci4s with " + ci4sUsername); - DVCUtils.gitPush(localPath1, modelsVo.getGitLinkUsername(), modelsVo.getGitLinkPassword()); - // dvc push 到远程S3 - DVCUtils.dvcPush(localPath1); - return "新增模型成功"; - } catch (Exception e) { - throw new RuntimeException(e); - } - }); + public List> uploadModelLocal(MultipartFile[] files, String uuid) throws Exception { + List> results = new ArrayList<>(); + + for (MultipartFile file : files) { + // 构建objectName + String username = SecurityUtils.getLoginUser().getUsername(); + String fileName = file.getOriginalFilename(); + String path = "/temp/" + username + "/models/" + uuid + "/model/"; + long sizeInBytes = file.getSize(); + String formattedSize = FileUtil.formatFileSize(sizeInBytes); + File targetFile = new File(path, file.getOriginalFilename()); + // 确保目录存在 + targetFile.getParentFile().mkdirs(); + // 保存文件到目标路径 + FileUtils.copyInputStreamToFile(file.getInputStream(), targetFile); + // 返回上传文件的路径 + String absolutePath = targetFile.getAbsolutePath(); + Map result = new HashMap<>(); + result.put("fileName", fileName); + result.put("url", absolutePath); // objectName根据实际情况定义 + result.put("fileSize", formattedSize); + results.add(result); + } + return results; } @Override - public ResponseEntity downloadAllModelFilesNew(String repositoryName, String version, String gitLinkUsername, String gitLinkPassword) throws IOException, Exception { + public ResponseEntity downloadAllModelFilesNew(String identifier, String version) throws Exception { // 命令行操作 git clone 项目地址 LoginUser loginUser = SecurityUtils.getLoginUser(); String ci4sUsername = loginUser.getUsername(); - String token = gitService.login(gitLinkUsername, gitLinkPassword); - Jedis jedis = new Jedis(redisHost); - String userReq = jedis.get(ci4sUsername + "_gitUserInfo"); - Map userInfo = JsonUtils.jsonToMap(userReq); - Integer userId = (Integer) userInfo.get("user_id"); - String projectUrl = gitendpoint + "/" + (String) userInfo.get("login") + "/" + repositoryName + ".git"; - String localPath1 = localPath + ci4sUsername + "/model/" + repositoryName; + String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername(); + String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword(); + Map userInfo = getUserInfo(ci4sUsername); + + String projectUrl = gitendpoint + "/" + userInfo.get("login") + "/" + identifier + ".git"; + String localPath1 = localPath + ci4sUsername + "/model/" + identifier; File folder = new File(localPath1); if (folder.exists() && folder.isDirectory()) { //切换分支 @@ -685,17 +662,18 @@ public class ModelsServiceImpl implements ModelsService { public Page newPubilcQueryByPage(ModelsVo modelsVo, PageRequest pageRequest) throws Exception { LoginUser loginUser = SecurityUtils.getLoginUser(); String ci4sUsername = loginUser.getUsername(); - String token = gitService.login(modelsVo.getGitLinkUsername(), modelsVo.getGitLinkPassword()); - Jedis jedis = new Jedis(redisHost); - String userReq = jedis.get(ci4sUsername + "_gitUserInfo"); - Map userInfo = JsonUtils.jsonToMap(userReq); + Map userInfo = getUserInfo(ci4sUsername); + String token = (String) userInfo.get("token"); + //拼接查询url String modelTagName = modelsVo.getModelTag(); String modelTypeName = modelsVo.getModelType(); + String search = modelsVo.getName(); String topic_name = "ci4s_model"; topic_name = StringUtils.isEmpty(modelTagName) ? topic_name : topic_name + ",modeltag_" + modelTypeName; topic_name = StringUtils.isEmpty(modelTagName) ? topic_name : topic_name + ",modeltype_" + modelTypeName; - String url = gitendpoint + "/api/users/" + (String) userInfo.get("login") + "/projects.json?page=" + pageRequest.getPageNumber() + "&limit=" + pageRequest.getPageSize() + "&category=manage&topic_name=" + topic_name; + + String url = gitendpoint + "/api/users/" + userInfo.get("login") + "/projects.json?page=" + pageRequest.getPageNumber() + "&limit=" + pageRequest.getPageSize() + "&category=manage&topic_name=" + topic_name + "&search=" + search; String req = HttpUtils.sendGetWithToken(url, null, token); Map stringObjectMap = JacksonUtil.parseJSONStr2Map(req); Integer total = (Integer) stringObjectMap.get("count"); @@ -707,19 +685,19 @@ public class ModelsServiceImpl implements ModelsService { public Page newPersonalQueryByPage(ModelsVo modelsVo, PageRequest pageRequest) throws Exception { LoginUser loginUser = SecurityUtils.getLoginUser(); String ci4sUsername = loginUser.getUsername(); - String token = gitService.login(modelsVo.getGitLinkUsername(), modelsVo.getGitLinkPassword()); - Jedis jedis = new Jedis(redisHost); - String userReq = jedis.get(ci4sUsername + "_gitUserInfo"); - Map userInfo = JsonUtils.jsonToMap(userReq); + Map userInfo = getUserInfo(ci4sUsername); Integer userId = (Integer) userInfo.get("user_id"); + String token = (String) userInfo.get("token"); + //拼接查询url String modelTagName = modelsVo.getModelTag(); String modelTypeName = modelsVo.getModelType(); + String search = modelsVo.getName(); String topic_name = "ci4s_model"; topic_name = StringUtils.isEmpty(modelTagName) ? topic_name : topic_name + ",modeltag_" + modelTagName; topic_name = StringUtils.isEmpty(modelTypeName) ? topic_name : topic_name + ",modeltype_" + modelTypeName; - String url = gitendpoint + "/api/projects.json?user_id=" + userId + "&page=" + pageRequest.getPageNumber() + "&limit=" + pageRequest.getPageSize() + "&sort_by=praises_count&topic_name=" + topic_name; + String url = gitendpoint + "/api/projects.json?user_id=" + userId + "&page=" + pageRequest.getPageNumber() + "&limit=" + pageRequest.getPageSize() + "&sort_by=praises_count&topic_name=" + topic_name + "&search=" + search; String req = HttpUtils.sendGetWithToken(url, null, token); Map stringObjectMap = JacksonUtil.parseJSONStr2Map(req); Integer total = (Integer) stringObjectMap.get("total_count"); @@ -727,6 +705,147 @@ public class ModelsServiceImpl implements ModelsService { return new PageImpl<>(convert(projects), pageRequest, total); } + @Override + public List> getVersionList(String identifier, String owner) throws Exception { + List> brancheList = gitService.getBrancheList(owner, identifier); + return brancheList.stream() + .filter(branch -> "master".equals(branch.get("name"))) + .collect(Collectors.toList()); + } + + @Override + public ModelsVo getModelDetail(String modelName, String identifier, String owner, String version) throws Exception { + LoginUser loginUser = SecurityUtils.getLoginUser(); + String ci4sUsername = loginUser.getUsername(); + String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername(); + String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword(); + + if (StringUtils.isEmpty(version)) { + List> versionList = this.getVersionList(owner, identifier); + if (versionList.size() == 0) { + throw new Exception("数据集文件不存在"); + } + version = (String) versionList.get(0).get("name"); + } + + // git pull操作，然后读取里面的文件列表，列出每个文件的大小和名称，封装成MAP + List> fileDetailsAfterGitPull = DVCUtils.getFileDetailsAfterGitPull(localPath + ci4sUsername + "/model/", modelName, version, "model", gitLinkUsername, gitLinkPassword); + + Map stringObjectMap = YamlUtils.loadYamlFile(localPath + ci4sUsername + "/model/" + modelName + "/" + version + "/metadata/metadata.yaml"); + assert stringObjectMap != null; + + ModelMetaVo modelMetaVo = ConvertUtil.convertMapToObject(stringObjectMap, ModelMetaVo.class); + ModelsVo modelsVo = new ModelsVo(); + BeanUtils.copyProperties(modelMetaVo, modelsVo); + + List versionVos = new ArrayList<>(); + if (!fileDetailsAfterGitPull.isEmpty()) { + for (Map fileDetail : fileDetailsAfterGitPull) { + VersionVo versionVo = new VersionVo(); + versionVo.setUrl((String) fileDetail.get("filePath")); + versionVo.setFileName((String) fileDetail.get("fileName")); + long size = (long) fileDetail.get("size"); + versionVo.setFileSize(FileUtil.formatFileSize(size)); + versionVos.add(versionVo); + } + } + modelsVo.setModelVersionVos(versionVos); + + return modelsVo; + } + + @Override + public ModelDependency1TreeVo getModelDependencyTree(Integer repoId, String modelName, String version) throws Exception { + ModelDependency1 modelDependency1 = modelDependency1Dao.queryByModelNameAndVersion(repoId, modelName, version); + ModelDependency1TreeVo modelDependency1TreeVo = new ModelDependency1TreeVo(); + BeanUtils.copyProperties(modelDependency1, modelDependency1TreeVo); + + //递归查询父模型 + List parentModelList = new ArrayList<>(); + getParentModel(parentModelList, modelDependency1); + modelDependency1TreeVo.setPatrentModelList(parentModelList); + + //递归查询子模型 + getChildModel(modelDependency1TreeVo); + + return modelDependency1TreeVo; + } + + @Override + public void deleteModel(Integer repoId, String identifier, String owner) throws Exception { + LoginUser loginUser = SecurityUtils.getLoginUser(); + String ci4sUsername = loginUser.getUsername(); + Map userInfo = getUserInfo(ci4sUsername); + String token = (String) userInfo.get("token"); + + + gitService.deleteProject(owner, identifier); + //删除模型依赖 + modelDependency1Dao.deleteModel(repoId, identifier, owner, null); + modelDependency1Dao.deleteModelDependency(repoId, identifier, owner, null); + } + + @Override + public void deleteVersion(Integer repoId, String identifier, String owner, String version) throws Exception { + LoginUser loginUser = SecurityUtils.getLoginUser(); + String ci4sUsername = loginUser.getUsername(); + + gitService.deleteBranch(owner, identifier, version); + //删除模型依赖 + modelDependency1Dao.deleteModel(repoId, identifier, owner, version); + modelDependency1Dao.deleteModelDependency(repoId, identifier, owner, version); + } + + void commonDvc(String token, Map userInfo, String username, String repositoryName, ModelsVo modelsVo, String gitLinkUsername, String gitLinkPassword, String type) throws Exception { + String branchName = modelsVo.getVersion(); + + // 创建分支 + gitService.createBranch((String) userInfo.get("login"), repositoryName, branchName, "master"); + + String projectUrl = gitendpoint + "/" + (String) userInfo.get("login") + "/" + repositoryName + ".git"; + String url = modelsVo.getModelVersionVos().get(0).getUrl(); + String sourcePath = url.substring(0, url.lastIndexOf("/")); + String rootPath = localPath + username + "/model/" + modelsVo.getName() + "/" + branchName; + String modelPath = rootPath + "/model"; + String metaPath = rootPath + "/metadata"; + + + if (type.equals("CreateModelFromPipeline") && FileUtil.checkDirectoryExists(rootPath)) { + DVCUtils.gitFetch(rootPath, gitLinkUsername, gitLinkPassword); + DVCUtils.gitCheckoutBranch(rootPath, branchName); + } else { + DVCUtils.gitClone(rootPath, projectUrl, branchName, modelsVo.getGitLinkUsername(), modelsVo.getGitLinkPassword()); + } + + DVCUtils.moveFiles(sourcePath, modelPath); + + //拼接生产的元数据后写入yaml文件 + ModelMetaVo modelMetaVo = new ModelMetaVo(); + BeanUtils.copyProperties(modelsVo, modelMetaVo); + YamlUtils.generateYamlFile(JsonUtils.objectToMap(modelMetaVo), metaPath, "metadata"); + + // dvc init 初始化 + DVCUtils.dvcInit(rootPath); + + // 配置远程S3地址 + String s3Path = "management-platform-files/" + username + "/model/" + repositoryName + "/" + branchName; + DVCUtils.dvcRemoteAdd(rootPath, s3Path); + DVCUtils.dvcConfigS3Credentials(rootPath, endpoint); + DVCUtils.dvcConfigS3Credentials2(rootPath, accessKeyId); + DVCUtils.dvcConfigS3Credentials3(rootPath, secretAccessKey); + + // dvc 跟踪 + DVCUtils.dvcAdd(rootPath, "model"); + + // git commit + DVCUtils.gitAdd(rootPath, "."); + DVCUtils.gitCommit(rootPath, "commit from ci4s with " + username); + DVCUtils.gitPush(rootPath, modelsVo.getGitLinkUsername(), modelsVo.getGitLinkPassword()); + + // dvc push 到远程S3 + DVCUtils.dvcPush(rootPath); + } + public List convert(List> lst) { if (lst != null && lst.size() > 0) { List newModelVos = ConvertUtil.convertListMapToObjectList(lst, ModelsVo.class); @@ -753,10 +872,48 @@ public class ModelsServiceImpl implements ModelsService { } } } - return newModelVos; } return new ArrayList<>(); } + void getParentModel(List modelList, ModelDependency1 modelDependency1) { + if (modelDependency1.getParentModel() != null) { + String[] split = modelDependency1.getParentModel().split(":"); + Integer parentRepoId = Integer.valueOf(split[0]); + String parentModelName = split[1]; + String parentModelVersion = split[2]; + ModelDependency1 parentModel = modelDependency1Dao.queryByModelNameAndVersion(parentRepoId, parentModelName, parentModelVersion); + modelList.add(parentModel); + if (parentModel != null && parentModel.getParentModel() != null) { + getParentModel(modelList, parentModel); + } + } + } + + void getChildModel(ModelDependency1TreeVo modelDependency1TreeVo) { + List childModelList = new ArrayList<>(); + String model = modelDependency1TreeVo.getRepoId() + ":" + modelDependency1TreeVo.getModelName() + ":" + modelDependency1TreeVo.getVersion(); + + List children = modelDependency1Dao.queryByParentModel(model); + if (children != null && !children.isEmpty()) { + for (ModelDependency1 modelDependency1 : children) { + ModelDependency1TreeVo modelDependency1TreeVo1 = new ModelDependency1TreeVo(); + BeanUtils.copyProperties(modelDependency1, modelDependency1TreeVo1); + getChildModel(modelDependency1TreeVo1); + childModelList.add(modelDependency1TreeVo1); + } + } + modelDependency1TreeVo.setChildModelList(childModelList); + } + + Map getUserInfo(String ci4sUsername) throws IOException { + Jedis jedis = new Jedis(redisHost); + String userReq = jedis.get(ci4sUsername + "_gitUserInfo"); + Map userInfo = JsonUtils.jsonToMap(userReq); + + String token = gitService.checkoutToken(); + userInfo.put("token", token); + return userInfo; + } } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/K8sClientUtil.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/K8sClientUtil.java index 03648e6e..cb382fd8 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/K8sClientUtil.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/K8sClientUtil.java @@ -366,7 +366,7 @@ public class K8sClientUtil { .endContainer() .addNewVolume() .withName("workspace") - .withHostPath(new V1HostPathVolumeSource().path(hostPath + "/" + podName + "/tensorboard").type("DirectoryOrCreate")) + .withHostPath(new V1HostPathVolumeSource().path(pvcName).type("DirectoryOrCreate")) // .withPersistentVolumeClaim(new V1PersistentVolumeClaimVolumeSource().claimName(pvcName)) .endVolume() .withTerminationGracePeriodSeconds(14400L) @@ -452,13 +452,12 @@ public class K8sClientUtil { // 配置卷和卷挂载 List volumeMounts = new ArrayList<>(); volumeMounts.add(new V1VolumeMount().name("workspace").mountPath("/opt/notebooks")); - volumeMounts.add(new V1VolumeMount().name("dataset").mountPath("/opt/dataset").readOnly(true)); - volumeMounts.add(new V1VolumeMount().name("model").mountPath("/opt/model").readOnly(true)); + volumeMounts.add(new V1VolumeMount().name("data").mountPath("/opt/dataset").subPath(datasetPath).readOnly(true)); + volumeMounts.add(new V1VolumeMount().name("data").mountPath("/opt/model").subPath(modelPath).readOnly(true)); List volumes = new ArrayList<>(); volumes.add(new V1Volume().name("workspace").hostPath(new V1HostPathVolumeSource().path(hostPath + "/" + podName + "/notebooks").type("DirectoryOrCreate"))); - volumes.add(new V1Volume().name("dataset").hostPath(new V1HostPathVolumeSource().path(hostPath + "/" + podName + "/dataset").type("DirectoryOrCreate"))); - volumes.add(new V1Volume().name("model").hostPath(new V1HostPathVolumeSource().path(hostPath + "/" + podName + "/model").type("DirectoryOrCreate"))); + volumes.add(new V1Volume().name("data").hostPath(new V1HostPathVolumeSource().path(hostPath).type("DirectoryOrCreate"))); // 配置卷和卷挂载 // List volumeMounts = new ArrayList<>(); diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelsVo.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelsVo.java index 1189da76..a57184ca 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelsVo.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelsVo.java @@ -10,15 +10,9 @@ import java.util.List; @JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy.class) @Data -public class ModelsVo implements Serializable { +public class ModelsVo extends ModelMetaVo implements Serializable { - - @ApiModelProperty(name = "name") - private String name; - - // private String version; - @ApiModelProperty(name = "description") - private String description; + private Integer id; /** * 模型可见范围 @@ -26,22 +20,6 @@ public class ModelsVo implements Serializable { @ApiModelProperty(name = "available_range") private int availableRange; - // private String url; - @ApiModelProperty(name = "model_type") - private String modelType; - - - @ApiModelProperty(name = "model_tag") - private String modelTag; - /** - * 版本 - */ - @ApiModelProperty(name = "version") - private String version; - - @ApiModelProperty(name = "model_version_vos") - private List modelVersionVos; - /** * 状态 */ @@ -53,14 +31,19 @@ public class ModelsVo implements Serializable { private String gitLinkPassword; @ApiModelProperty(name = "models_version_vos") - private List modelsVersionVos; + private List modelVersionVos; /** - * 数据集仓库名称 + * 模型仓库名称 */ @ApiModelProperty(name = "repository_name") private String repositoryName; @ApiModelProperty(name = "repo_id") private Integer repoId; + + /** + * 父模型 + */ + private String parentModel; }