diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/dataset/NewDatasetFromGitController.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/dataset/NewDatasetFromGitController.java index 521cac30..cb6e4327 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/dataset/NewDatasetFromGitController.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/dataset/NewDatasetFromGitController.java @@ -26,7 +26,6 @@ public class NewDatasetFromGitController { private NewDatasetService newDatasetService; - /** * 新增数据集与版本新 * @@ -57,50 +56,51 @@ public class NewDatasetFromGitController { @GetMapping("/queryDatasets") @ApiOperation("数据集广场公开数据集分页查询,根据data_type,data_tag筛选,true公开false私有") public AjaxResult queryDatasets(Dataset dataset, @RequestParam("page") int page, - @RequestParam("size") int size, + @RequestParam("size") int size, @RequestParam(value = "is_public") Boolean isPublic, - @RequestParam(value = "data_type", required = false) String dataType, - @RequestParam(value = "data_tag", required = false) String dataTag) throws Exception { + @RequestParam(value = "data_type", required = false) String dataType, + @RequestParam(value = "data_tag", required = false) String dataTag) throws Exception { PageRequest pageRequest = PageRequest.of(page, size); - if(isPublic){ + if (isPublic) { return AjaxResult.success(this.newDatasetService.newPubilcQueryByPage(dataset, pageRequest)); - }else { + } else { return AjaxResult.success(this.newDatasetService.newPersonalQueryByPage(dataset, pageRequest)); } } @GetMapping("/getVersionList") @ApiOperation(value = "获取分支列表") - public AjaxResult getVersionList(@RequestParam("identifier") String repo,@RequestParam("owner")String owner) throws Exception { - return AjaxResult.success(this.newDatasetService.getVersionList(repo,owner)); + public AjaxResult getVersionList(@RequestParam("identifier") String repo, @RequestParam("owner") String owner) throws Exception { + return AjaxResult.success(this.newDatasetService.getVersionList(repo, owner)); } @GetMapping("/getDatasetDetail") @ApiOperation(value = "获取数据集详情") public AjaxResult getDatasetVersions(@RequestParam("name") String name, @RequestParam("id") Integer id, - @RequestParam(value ="owner",required = false)String owner, - @RequestParam(value = "identifier",required = false) String identifier, - @RequestParam(value = "version",required = false)String version) throws Exception { - return AjaxResult.success(this.newDatasetService.getNewDatasetDesc(id,name,identifier,owner,version)); + @RequestParam(value = "owner", required = false) String owner, + @RequestParam(value = "identifier", required = false) String identifier, + @RequestParam(value = "version", required = false) String version) throws Exception { + return AjaxResult.success(this.newDatasetService.getNewDatasetDesc(id, name, identifier, owner, version)); } @DeleteMapping("/deleteDataset") @ApiOperation(value = "删除数据集") - public AjaxResult deleteDataset(@RequestParam("identifier") String repo,@RequestParam("owner")String owner) throws Exception { - this.newDatasetService.deleteDatasetNew(repo,owner); + public AjaxResult deleteDataset(@RequestParam("identifier") String repo, @RequestParam("owner") String owner) throws Exception { + this.newDatasetService.deleteDatasetNew(repo, owner); return AjaxResult.success(); } @DeleteMapping("/deleteDatasetVersion") @ApiOperation(value = "删除数据集版本") - public AjaxResult deleteDatasetVersion(@RequestParam("identifier") String repo,@RequestParam("owner")String owner,@RequestParam("version")String version) throws Exception { + public AjaxResult deleteDatasetVersion(@RequestParam("identifier") String repo, @RequestParam("owner") String owner, @RequestParam("version") String version, + @RequestParam("relative_paths") String relativePaths) throws Exception { // 查询版本,如果是最后一个版本,则不能删除 List> versionList = this.newDatasetService.getVersionList(repo, owner); - if(versionList.size() == 1){ + if (versionList.size() == 1) { return AjaxResult.error("当前数据集只有一个版本,不能删除该版本"); } - this.newDatasetService.deleteDatasetVersionNew(repo,owner,version); + this.newDatasetService.deleteDatasetVersionNew(repo, owner, version, relativePaths); return AjaxResult.success(); } @@ -109,36 +109,36 @@ public class NewDatasetFromGitController { * 上传数据集 * * @param files 上传的数据集文件 - * @param uuid 上传唯一标识,构建url + * @param uuid 上传唯一标识,构建url * @return 上传结果 */ @CrossOrigin(origins = "*", allowedHeaders = "*") @PostMapping("/upload") @ApiOperation(value = "上传数据集") public AjaxResult uploadDataset(@RequestParam("file") MultipartFile[] files, @RequestParam("uuid") String uuid) throws Exception { - return AjaxResult.success(this.newDatasetService.uploadDatasetlocal(files,uuid)); + return AjaxResult.success(this.newDatasetService.uploadDatasetlocal(files, uuid)); } /** * 数据集打包下载 * - * @param version 数据集版本 + * @param version 数据集版本 * @return 单条数据 */ @GetMapping("/downloadAllFiles") @ApiOperation(value = "下载同一版本下所有数据集,并打包") - public ResponseEntity downloadAllDatasetFiles(@RequestParam("name") String name, @RequestParam("id") Integer id, @RequestParam("version") String version) throws Exception { - return newDatasetService.downloadAllDatasetFilesNew(name,id, version); + public ResponseEntity downloadAllDatasetFiles(@RequestParam("name") String name, @RequestParam("id") Integer id, @RequestParam("version") String version) throws Exception { + return newDatasetService.downloadAllDatasetFilesNew(name, id, version); } /** * 下载数据集 * - * @param url ps:路径 + * @param url ps:路径 * @return 单条数据 */ - @GetMapping("/downloadSinggerFile") + @GetMapping("/downloadSingleFile") @ApiOperation(value = "下载单个数据集文件", notes = "根据数据集版本表id下载单个数据集文件") public ResponseEntity downloadDataset(@RequestParam("url") String url) throws Exception { return newDatasetService.downloadDatasetlocal(url); diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/model/NewModelFromGitController.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/model/NewModelFromGitController.java index 145f84d8..732e2d97 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/model/NewModelFromGitController.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/model/NewModelFromGitController.java @@ -12,6 +12,8 @@ import org.springframework.web.bind.annotation.*; import org.springframework.web.multipart.MultipartFile; import javax.annotation.Resource; +import java.util.List; +import java.util.Map; @RestController @RequestMapping("newmodel") @@ -27,6 +29,12 @@ public class NewModelFromGitController { return AjaxResult.success(this.modelsService.newCreateModel(modelsVo)); } + @PostMapping("/addVersion") + @ApiOperation("添加版本") + public AjaxResult addVersion(@RequestBody ModelsVo modelsVo) throws Exception { + return AjaxResult.success(this.modelsService.newCreateVersion(modelsVo)); + } + @CrossOrigin(origins = "*", allowedHeaders = "*") @PostMapping("/upload") @ApiOperation(value = "上传模型", notes = "根据模型id上传模型文件,并将信息存入数据库。") @@ -34,31 +42,16 @@ public class NewModelFromGitController { return AjaxResult.success(this.modelsService.uploadModelLocal(files, uuid)); } - /** - * 模型打包下载 - * - * @param version 模型版本 - * @return 模型 - */ @GetMapping("/downloadAllFiles") @ApiOperation(value = "下载同一版本下所有模型,并打包") - public ResponseEntity downloadAllDatasetFiles(@RequestParam("identifier") String identifier, - @RequestParam(value = "version") String version) throws Exception { - return modelsService.downloadAllModelFilesNew(identifier, version); + public ResponseEntity downloadAllDatasetFiles(@RequestParam("name") String name, @RequestParam("id") Integer id, @RequestParam("version") String version) throws Exception { + return modelsService.downloadAllModelFilesNew(name, id, version); } - - /** - * 下载模型 - * - * @param model_version_id ps:这里的id是model_version表的主键 - * @return 模型 - */ - - @GetMapping("/download/{model_version_id}") + @GetMapping("/downloadSingleFile") @ApiOperation(value = "下载单个模型文件", notes = "根据模型版本表id下载单个模型文件") - public ResponseEntity downloadModel(@PathVariable("model_version_id") Integer model_version_id) throws Exception { - return modelsService.downloadModels(model_version_id); + public ResponseEntity downloadDatasetlocal(@RequestParam("url") String url) throws Exception { + return modelsService.downloadDatasetlocal(url); } @GetMapping("/queryModels") @@ -83,34 +76,40 @@ public class NewModelFromGitController { @GetMapping("/getVersionList") @ApiOperation(value = "获取模型分支列表") - public AjaxResult getVersionList(@RequestParam("identifier") String identifier, @RequestParam("owner")String owner) throws Exception { + public AjaxResult getVersionList(@RequestParam("identifier") String identifier, @RequestParam("owner") String owner) throws Exception { return AjaxResult.success(this.modelsService.getVersionList(identifier, owner)); } @GetMapping("/getModelDetail") @ApiOperation(value = "获取模型详细信息") - public AjaxResult getModelDetail(@RequestParam("model_name") String modelName, @RequestParam("identifier") String identifier,@RequestParam("owner") String owner, @RequestParam("version") String version) throws Exception { - return AjaxResult.success(this.modelsService.getModelDetail(modelName, identifier, owner, version)); + public AjaxResult getModelDetail(@RequestParam("id") Integer id, @RequestParam("name") String name, @RequestParam("identifier") String identifier, @RequestParam("owner") String owner, @RequestParam("version") String version) throws Exception { + return AjaxResult.success(this.modelsService.getModelDetail(id, name, identifier, owner, version)); } @GetMapping("/getModelDependencyTree") @ApiOperation(value = "获取模型依赖关系树") - public AjaxResult getModelDependencyTree(@RequestParam("repoId") Integer repoId, @RequestParam("model_name") String modelName, @RequestParam("version") String version) throws Exception { - return AjaxResult.success(this.modelsService.getModelDependencyTree(repoId, modelName, version)); + public AjaxResult getModelDependencyTree(@RequestParam("id") Integer id, @RequestParam("model_name") String modelName, @RequestParam("version") String version) throws Exception { + return AjaxResult.success(this.modelsService.getModelDependencyTree(id, modelName, version)); } @DeleteMapping("/delete") @ApiOperation(value = "删除模型") - public AjaxResult deleteModel(@RequestParam("repoId")Integer repoId, @RequestParam("identifier") String identifier, @RequestParam("owner") String owner) throws Exception { - this.modelsService.deleteModel(repoId,identifier, owner); + public AjaxResult deleteModel(@RequestParam("id") Integer id, @RequestParam("identifier") String identifier, + @RequestParam("owner") String owner) throws Exception { + this.modelsService.deleteModel(id, identifier, owner); return AjaxResult.success(); } @DeleteMapping("/deleteVersion") @ApiOperation(value = "删除模型版本") - public AjaxResult deleteVersion(@RequestParam("repoId")Integer repoId, @RequestParam("identifier") String identifier, @RequestParam("owner") String owner, @RequestParam("version") String version) throws Exception { - this.modelsService.deleteVersion(repoId, identifier, owner, version); + public AjaxResult deleteVersion(@RequestParam("id") Integer id, @RequestParam("identifier") String identifier, @RequestParam("owner") String owner, @RequestParam("version") String version + , @RequestParam("relative_paths") String relativePaths) throws Exception { + List> versionList = this.modelsService.getVersionList(identifier, owner); + if (versionList.size() == 1) { + return AjaxResult.error("当前模型只有一个版本,不能删除该版本"); + } + this.modelsService.deleteVersion(id, identifier, owner, version, relativePaths); return AjaxResult.success(); } } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/ModelDependency1.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/ModelDependency1.java index 60bedb72..387d6c92 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/ModelDependency1.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/ModelDependency1.java @@ -24,4 +24,6 @@ public class ModelDependency1 implements Serializable { private String owner; private Integer state; + + private String meta; } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/ModelDependency1Dao.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/ModelDependency1Dao.java index b3910b5f..2937743b 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/ModelDependency1Dao.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/ModelDependency1Dao.java @@ -9,13 +9,13 @@ public interface ModelDependency1Dao { int insert(ModelDependency1 modelDependency1); - List queryByModelName(@Param("modelName") String modelName); + List queryModelDependency(@Param("modelName") String modelName, @Param("repoId") Integer repoId, @Param("owner") String owner); ModelDependency1 queryByModelNameAndVersion(@Param("repoId") Integer repoId, @Param("modelName") String modelName, @Param("version") String version); List queryByParentModel(@Param("parentModel") String parentModel); - int deleteModel(@Param("repoId") Integer repoId, @Param("identifier")String identifier, @Param("owner") String owner, @Param("version") String version); + int deleteModel(@Param("repoId") Integer repoId, @Param("identifier") String identifier, @Param("owner") String owner, @Param("version") String version); - int deleteModelDependency(@Param("repoId") Integer repoId, @Param("identifier")String identifier, @Param("owner") String owner, @Param("version") String version); + int deleteModelDependency(@Param("repoId") Integer repoId, @Param("identifier") String identifier, @Param("owner") String owner, @Param("version") String version); } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/GitService.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/GitService.java index 735c894f..a8e2ab79 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/GitService.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/GitService.java @@ -2,7 +2,6 @@ package com.ruoyi.platform.service; import com.ruoyi.platform.vo.GitProjectVo; -import java.io.IOException; import java.util.List; import java.util.Map; @@ -23,7 +22,7 @@ public interface GitService { void deleteProject(String owner, String projectName) throws Exception; - void deleteBranch(String owner,String projectName ,String branchName) throws Exception; + void deleteBranch(String owner, String projectName, String branchName, String localPath) throws Exception; Map getUserInfo(String token) throws Exception; } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ModelsService.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ModelsService.java index a2aac1b1..ec330f1d 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ModelsService.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ModelsService.java @@ -14,7 +14,6 @@ import org.springframework.web.multipart.MultipartFile; import java.io.IOException; import java.util.List; import java.util.Map; -import java.util.concurrent.CompletableFuture; /** * (Models)表服务接口 @@ -70,6 +69,7 @@ public interface ModelsService { ResponseEntity downloadModels(Integer id) throws Exception; + ResponseEntity downloadDatasetlocal(String filePath) throws Exception; List> uploadModels(MultipartFile[] files, String uuid) throws Exception; @@ -87,11 +87,13 @@ public interface ModelsService { List> exportModels(String path, String uuid) throws Exception; - CompletableFuture newCreateModel(ModelsVo modelsVo) throws Exception; + String newCreateModel(ModelsVo modelsVo) throws Exception; + + String newCreateVersion(ModelsVo modelsVo) throws Exception; List> uploadModelLocal(MultipartFile[] files, String uuid) throws Exception; - ResponseEntity downloadAllModelFilesNew(String identifier, String version) throws IOException, Exception; + ResponseEntity downloadAllModelFilesNew(String identifier, Integer id, String version) throws IOException, Exception; Page newPubilcQueryByPage(ModelsVo modelsVo, PageRequest pageRequest) throws Exception; @@ -99,11 +101,11 @@ public interface ModelsService { List> getVersionList(String identifier, String owner) throws Exception; - ModelsVo getModelDetail(String modelName, String identifier, String owner, String version) throws Exception; + ModelsVo getModelDetail(Integer id, String name, String identifier, String owner, String version) throws Exception; ModelDependency1TreeVo getModelDependencyTree(Integer repoId, String modelName, String version) throws Exception; void deleteModel(Integer repoId, String identifier, String owner) throws Exception; - void deleteVersion(Integer repoId, String identifier, String owner, String version) throws Exception; + void deleteVersion(Integer repoId, String identifier, String owner, String version, String relativePath) throws Exception; } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/NewDatasetService.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/NewDatasetService.java index 71e96ccd..1da4dfee 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/NewDatasetService.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/NewDatasetService.java @@ -29,5 +29,5 @@ public interface NewDatasetService { void deleteDatasetNew(String repo, String owner) throws Exception; - void deleteDatasetVersionNew(String repo, String owner, String version) throws Exception; + void deleteDatasetVersionNew(String repo, String owner, String version, String relativePath) throws Exception; } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/GitServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/GitServiceImpl.java index 1ecccaa8..b7573c4e 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/GitServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/GitServiceImpl.java @@ -8,12 +8,15 @@ import com.ruoyi.platform.utils.JsonUtils; import com.ruoyi.platform.vo.GitProjectVo; import com.ruoyi.system.api.model.LoginUser; import org.apache.commons.lang3.StringUtils; +import org.eclipse.jgit.api.Git; +import org.eclipse.jgit.api.errors.GitAPIException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import redis.clients.jedis.Jedis; +import java.io.File; import java.io.IOException; import java.util.Collections; import java.util.HashMap; @@ -147,7 +150,13 @@ public class GitServiceImpl implements GitService { } @Override - public void deleteBranch(String owner, String projectName, String branchName) throws Exception { + public void deleteBranch(String owner, String projectName, String branchName, String localPath) throws Exception { + try (Git git = Git.open(new File(localPath))) { + git.checkout().setName("master").call(); + git.branchDelete().setBranchNames(branchName).setForce(true).call(); + } catch (IOException | GitAPIException e) { + log.error("Exception occurred while creating local branch based on master",e); + } String token = this.checkoutToken(); HttpUtils.sendDeleteRequest("https://www.gitlink.org.cn/api/v1/" + owner + "/" + projectName + "/branches/" + branchName + ".json", token); } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java index d78b1f1e..f4df9af8 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java @@ -41,7 +41,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.*; -import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; @@ -249,6 +248,29 @@ public class ModelsServiceImpl implements ModelsService { } } + @Override + public ResponseEntity downloadDatasetlocal(String filePath) throws Exception { + File file = new File(filePath); + + if (!file.exists()) { + throw new FileNotFoundException("File not found: " + filePath); + } + + InputStreamResource resource = new InputStreamResource(new FileInputStream(file)); + + HttpHeaders headers = new HttpHeaders(); + headers.add(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=" + file.getName()); + headers.add(HttpHeaders.CONTENT_LENGTH, String.valueOf(file.length())); + headers.add(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_OCTET_STREAM_VALUE); + + return ResponseEntity.ok() + .headers(headers) + .contentLength(file.length()) + .contentType(MediaType.APPLICATION_OCTET_STREAM) + .body(resource); + } + + /** * 上传模型文件 * @@ -389,7 +411,6 @@ public class ModelsServiceImpl implements ModelsService { Models models = new Models(); models.setName(modelsVo.getName()); models.setDescription(modelsVo.getDescription()); - models.setAvailableRange(modelsVo.getAvailableRange()); models.setModelType(modelsVo.getModelType()); models.setModelTag(modelsVo.getModelTag()); Models modelsInsert = this.insert(models); @@ -516,64 +537,206 @@ public class ModelsServiceImpl implements ModelsService { } @Override - public CompletableFuture newCreateModel(ModelsVo modelsVo) { - return CompletableFuture.supplyAsync(() -> { - try { - LoginUser loginUser = SecurityUtils.getLoginUser(); - String ci4sUsername = loginUser.getUsername(); - String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername(); - String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword(); + public String newCreateModel(ModelsVo modelsVo) { + try { + LoginUser loginUser = SecurityUtils.getLoginUser(); + String ci4sUsername = loginUser.getUsername(); + String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername(); + String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword(); + +// String ci4sUsername = "chenzhihang"; +// String gitLinkUsername = "chenzhihang"; +// String gitLinkPassword = "czh19970311"; + + Map userInfo = getUserInfo(ci4sUsername, gitLinkUsername, gitLinkPassword); + Integer userId = (Integer) userInfo.get("user_id"); + + String repositoryName = modelsVo.getIdentifier() == null ? ci4sUsername + "_model_" + DateUtils.dateTimeNow() : modelsVo.getIdentifier(); + ModelDependency1 modelDependency = new ModelDependency1(); + List oldModelDependencys = modelDependency1Dao.queryModelDependency(modelsVo.getName(), null, gitLinkUsername); + if (oldModelDependencys != null && !oldModelDependencys.isEmpty()) { + if (oldModelDependencys.stream().anyMatch(oldModelDependency -> oldModelDependency.getVersion().equals(modelsVo.getVersion()))) { + throw new RuntimeException("模型版本已存在,请勿重复创建"); + } + } - Map userInfo = getUserInfo(ci4sUsername); - Integer userId = (Integer) userInfo.get("user_id"); + //新建模型 + // 拼接project + GitProjectVo gitProjectVo = new GitProjectVo(); + gitProjectVo.setRepositoryName(repositoryName); + gitProjectVo.setName(modelsVo.getName()); + gitProjectVo.setDescription(modelsVo.getDescription()); + gitProjectVo.setPrivate(!modelsVo.getIsPublic()); + gitProjectVo.setUserId(userId); + + // 创建项目 + Map project = gitService.createProject(gitProjectVo); + Integer gitlinIid = (Integer) project.get("id"); + if (gitlinIid == null) { + throw new Exception("创建模型失败:" + project.get("message")); + } + // 创建分支 + gitService.createBranch((String) userInfo.get("login"), repositoryName, modelsVo.getVersion(), "master"); + // 定义标签 标签1:ci4s_model 标签2:ModelTag 标签3:ModelType + gitService.createTopic(gitlinIid, "ci4s_model"); + gitService.createTopic(gitlinIid, "modeltag_" + modelsVo.getModelTag()); + gitService.createTopic(gitlinIid, "modeltype_" + modelsVo.getModelType()); + + String branchName = modelsVo.getVersion(); + String owner = (String) userInfo.get("login"); + + String projectUrl = gitendpoint + "/" + owner + "/" + repositoryName + ".git"; + String sourcePath = modelsVo.getModelVersionVos().get(0).getUrl(); + String relatePath = ci4sUsername + "/model/" + gitlinIid + "/" + modelsVo.getName(); + String rootPath = localPath + relatePath; + String modelPath = rootPath + "/model"; + String metaPath = rootPath + "/metadata"; - String repositoryName = modelsVo.getRepositoryName() == null ? ci4sUsername + "_model_" + DateUtils.dateTimeNow() : modelsVo.getRepositoryName(); - ModelDependency1 modelDependency = new ModelDependency1(); - List oldModelDependencys = modelDependency1Dao.queryByModelName(modelsVo.getName()); + DVCUtils.gitClone(rootPath, projectUrl, branchName, gitLinkUsername, gitLinkPassword); + DVCUtils.moveFiles(sourcePath, modelPath); + + //拼接生产的元数据后写入yaml文件 + ModelMetaVo modelMetaVo = new ModelMetaVo(); + BeanUtils.copyProperties(modelsVo, modelMetaVo); + modelMetaVo.setId(gitlinIid); + modelMetaVo.setCreateBy(String.valueOf(StringUtils.isNotEmpty((String) userInfo.get("nickname")) ? userInfo.get("nickname") : userInfo.get("login"))); + modelMetaVo.setUpdateTime(DateUtils.getTime()); + modelMetaVo.setUsage("
" +
+                    "# 克隆模型配置文件与存储参数到本地\n" +
+                    "git clone -b " + branchName + " " + projectUrl + "\n" +
+                    "# 远程拉取配置文件\n" +
+                    "dvc pull\n" +
+                    "
"); + modelMetaVo.setIdentifier(repositoryName); + modelMetaVo.setOwner(owner); + modelMetaVo.setVersionDesc(modelMetaVo.getDescription()); + modelMetaVo.setRelativePaths(relatePath); + + Map metaMap = JsonUtils.objectToMap(modelMetaVo); + YamlUtils.generateYamlFile(metaMap, metaPath, "metadata"); + String meta = metaMap.toString(); + + DVCUtils.dvcInit(rootPath); + // 配置远程S3地址 + String s3Path = "management-platform-files/" + ci4sUsername + "/model/" + gitlinIid + "/" + repositoryName + "/" + branchName; + DVCUtils.dvcRemoteAdd(rootPath, s3Path); + DVCUtils.dvcConfigS3Credentials(rootPath, endpoint); + DVCUtils.dvcConfigS3Credentials2(rootPath, accessKeyId); + DVCUtils.dvcConfigS3Credentials3(rootPath, secretAccessKey); + + // dvc 跟踪 + DVCUtils.dvcAdd(rootPath, "model"); + // git commit + DVCUtils.gitAdd(rootPath, "."); + DVCUtils.gitCommit(rootPath, "commit from ci4s with " + ci4sUsername); + DVCUtils.gitPush(rootPath, gitLinkUsername, gitLinkPassword); + DVCUtils.dvcPush(rootPath); + + //保存模型依赖 + modelDependency.setRepoId(gitlinIid); + modelDependency.setIdentifier(repositoryName); + modelDependency.setModelName(modelsVo.getName()); + modelDependency.setVersion(modelsVo.getVersion()); + modelDependency.setParentModel(modelsVo.getParentModel()); + modelDependency.setOwner(gitLinkUsername); + modelDependency.setMeta(meta); + modelDependency1Dao.insert(modelDependency); + return "新增模型成功"; + } catch (Exception e) { + logger.error(e.getMessage()); + throw new RuntimeException(e); + } + } - if (oldModelDependencys != null && !oldModelDependencys.isEmpty()) { + @Override + public String newCreateVersion(ModelsVo modelsVo) throws Exception { + LoginUser loginUser = SecurityUtils.getLoginUser(); + String ci4sUsername = loginUser.getUsername(); + String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername(); + String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword(); - if (oldModelDependencys.stream().anyMatch(oldModelDependency -> oldModelDependency.getRepoId().equals(modelsVo.getRepoId()) && oldModelDependency.getVersion().equals(modelsVo.getVersion()))) { - throw new RuntimeException("模型版本已存在,请勿重复创建"); - } +// String ci4sUsername = "chenzhihang"; +// String gitLinkUsername = "chenzhihang"; +// String gitLinkPassword = "czh19970311"; - //新建版本 - commonDvc(userInfo, ci4sUsername, modelsVo.getRepoId(), modelsVo.getRepositoryName(), modelsVo, gitLinkUsername, gitLinkPassword, "CreateModelFromPipeline"); - modelDependency.setRepoId(modelsVo.getRepoId()); - } else { - //新建模型 - // 拼接project - GitProjectVo gitProjectVo = new GitProjectVo(); - gitProjectVo.setRepositoryName(repositoryName); - gitProjectVo.setName(modelsVo.getName()); - gitProjectVo.setDescription(modelsVo.getDescription()); - gitProjectVo.setPrivate(modelsVo.getAvailableRange() == 0); - gitProjectVo.setUserId(userId); - - // 创建项目 - Map project = gitService.createProject(gitProjectVo); - Integer gitlinIid = (Integer) project.get("id"); - modelDependency.setRepoId(gitlinIid); - // 定义标签 标签1:ci4s_model 标签2:ModelTag 标签3:ModelType - gitService.createTopic(gitlinIid, "ci4s_model"); - gitService.createTopic(gitlinIid, "modeltag_" + modelsVo.getModelTag()); - gitService.createTopic(gitlinIid, "modeltype_" + modelsVo.getModelType()); - commonDvc(userInfo, ci4sUsername, gitlinIid, repositoryName, modelsVo, gitLinkUsername, gitLinkPassword, "createModel"); - } + Map userInfo = getUserInfo(ci4sUsername, gitLinkUsername, gitLinkPassword); - //保存模型依赖 - modelDependency.setIdentifier(repositoryName); - modelDependency.setModelName(modelsVo.getName()); - modelDependency.setVersion(modelsVo.getVersion()); - modelDependency.setParentModel(modelsVo.getParentModel()); - modelDependency.setOwner(ci4sUsername); - modelDependency1Dao.insert(modelDependency); - return "新增模型成功"; - } catch (Exception e) { - logger.error(e.getMessage()); - throw new RuntimeException(e); + String repositoryName = modelsVo.getIdentifier() == null ? ci4sUsername + "_model_" + DateUtils.dateTimeNow() : modelsVo.getIdentifier(); + ModelDependency1 modelDependency = new ModelDependency1(); + List oldModelDependencys = modelDependency1Dao.queryModelDependency(modelsVo.getName(), modelsVo.getId(), gitLinkUsername); + if (oldModelDependencys != null && !oldModelDependencys.isEmpty()) { + if (oldModelDependencys.stream().anyMatch(oldModelDependency -> oldModelDependency.getVersion().equals(modelsVo.getVersion()))) { + throw new RuntimeException("模型版本已存在,请勿重复创建"); } - }); + + //新建版本 + String branchName = modelsVo.getVersion(); + String owner = (String) userInfo.get("login"); + + String projectUrl = gitendpoint + "/" + owner + "/" + repositoryName + ".git"; + String sourcePath = modelsVo.getModelVersionVos().get(0).getUrl(); + String relatePath = ci4sUsername + "/model/" + modelsVo.getId() + "/" + modelsVo.getName(); + String rootPath = localPath + relatePath; + String modelPath = rootPath + "/model"; + String metaPath = rootPath + "/metadata"; + + //部分信息在前面的版本里面,从那边取过来 + Map stringObjectMap = YamlUtils.loadYamlFile(metaPath + "/" + "metadata.yaml"); + ModelMetaVo oldModelVo = ConvertUtil.convertMapToObject(stringObjectMap, ModelMetaVo.class); + + // 创建本地分支 + DVCUtils.createLocalBranchBasedOnMaster(rootPath, branchName); + //dvc checkout + DVCUtils.dvcCheckout(rootPath); + DVCUtils.moveFiles(sourcePath, modelPath); + + //拼接生产的元数据后写入yaml文件 + ModelMetaVo modelMetaVo = new ModelMetaVo(); + BeanUtils.copyProperties(modelsVo, modelMetaVo); + modelMetaVo.setCreateBy(String.valueOf(StringUtils.isNotEmpty((String) userInfo.get("nickname")) ? userInfo.get("nickname") : userInfo.get("login"))); + modelMetaVo.setUpdateTime(DateUtils.getTime()); + modelMetaVo.setUsage("
" +
+                    "# 克隆模型配置文件与存储参数到本地\n" +
+                    "git clone -b " + branchName + " " + projectUrl + "\n" +
+                    "# 远程拉取配置文件\n" +
+                    "dvc pull\n" +
+                    "
"); + modelMetaVo.setIdentifier(repositoryName); + modelMetaVo.setOwner(owner); + modelMetaVo.setDescription(oldModelVo.getDescription()); + modelMetaVo.setModelTag(oldModelVo.getModelTag()); + modelMetaVo.setModelType(oldModelVo.getModelType()); + modelMetaVo.setRelativePaths(relatePath); + + Map metaMap = JsonUtils.objectToMap(modelMetaVo); + YamlUtils.generateYamlFile(metaMap, metaPath, "metadata"); + String meta = metaMap.toString(); + + // 配置远程S3地址 + String s3Path = "management-platform-files/" + ci4sUsername + "/model/" + modelsVo.getId() + "/" + repositoryName + "/" + branchName; + DVCUtils.dvcRemoteAdd(rootPath, s3Path); + DVCUtils.dvcConfigS3Credentials(rootPath, endpoint); + DVCUtils.dvcConfigS3Credentials2(rootPath, accessKeyId); + DVCUtils.dvcConfigS3Credentials3(rootPath, secretAccessKey); + // dvc 跟踪 + DVCUtils.dvcAdd(rootPath, "model"); + DVCUtils.pushNewBranchToRemote(rootPath, gitLinkUsername, gitLinkPassword, branchName); + //dvc push 到远程S3 + DVCUtils.dvcPush(rootPath); + + //保存模型依赖 + modelDependency.setRepoId(modelsVo.getId()); + modelDependency.setIdentifier(repositoryName); + modelDependency.setModelName(modelsVo.getName()); + modelDependency.setVersion(modelsVo.getVersion()); + modelDependency.setParentModel(modelsVo.getParentModel()); + modelDependency.setOwner(gitLinkUsername); + modelDependency.setMeta(meta); + modelDependency1Dao.insert(modelDependency); + + return "新增模型版本成功"; + } + return null; } @@ -584,8 +747,9 @@ public class ModelsServiceImpl implements ModelsService { for (MultipartFile file : files) { // 构建objectName String username = SecurityUtils.getLoginUser().getUsername(); +// String username = "chenzhihang"; String fileName = file.getOriginalFilename(); - String path = "/temp/" + username + "/models/" + uuid + "/model/"; + String path = localPath + "/temp/" + username + "/models/" + uuid; long sizeInBytes = file.getSize(); String formattedSize = FileUtil.formatFileSize(sizeInBytes); File targetFile = new File(path, file.getOriginalFilename()); @@ -594,10 +758,9 @@ public class ModelsServiceImpl implements ModelsService { // 保存文件到目标路径 FileUtils.copyInputStreamToFile(file.getInputStream(), targetFile); // 返回上传文件的路径 - String absolutePath = targetFile.getAbsolutePath(); Map result = new HashMap<>(); result.put("fileName", fileName); - result.put("url", absolutePath); // objectName根据实际情况定义 + result.put("url", path); // objectName根据实际情况定义 result.put("fileSize", formattedSize); results.add(result); } @@ -605,27 +768,12 @@ public class ModelsServiceImpl implements ModelsService { } @Override - public ResponseEntity downloadAllModelFilesNew(String identifier, String version) throws Exception { + public ResponseEntity downloadAllModelFilesNew(String identifier, Integer id, String version) throws Exception { // 命令行操作 git clone 项目地址 LoginUser loginUser = SecurityUtils.getLoginUser(); String ci4sUsername = loginUser.getUsername(); - String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername(); - String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword(); - Map userInfo = getUserInfo(ci4sUsername); - - String projectUrl = gitendpoint + "/" + userInfo.get("login") + "/" + identifier + ".git"; - String localPath1 = localPath + ci4sUsername + "/model/" + identifier; - File folder = new File(localPath1); - if (folder.exists() && folder.isDirectory()) { - //切换分支 - DVCUtils.gitCheckoutBranch(localPath1, version); - //pull - DVCUtils.gitPull(localPath1, gitLinkUsername, gitLinkPassword); - //dvc pull - DVCUtils.dvcPull(localPath1); - } else { - DVCUtils.gitClone(localPath1, projectUrl, version, gitLinkUsername, gitLinkPassword); - } + + String localPath1 = localPath + ci4sUsername + "/model/" + id + "/" + identifier; // 打包 data 文件夹 String dataFolderPath = localPath1 + "/model"; String zipFilePath = localPath1 + "/model.zip"; @@ -651,28 +799,40 @@ public class ModelsServiceImpl implements ModelsService { InputStreamResource resource = new InputStreamResource(new FileInputStream(zipFile)); return ResponseEntity.ok() - .header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=data.zip") + .header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=" + identifier + "_" + version + "_model.zip") .contentType(MediaType.APPLICATION_OCTET_STREAM) .contentLength(zipFile.length()) .body(resource); } @Override - public Page newPubilcQueryByPage(ModelsVo modelsVo, PageRequest pageRequest) throws Exception { + public Page newPersonalQueryByPage(ModelsVo modelsVo, PageRequest pageRequest) throws Exception { LoginUser loginUser = SecurityUtils.getLoginUser(); String ci4sUsername = loginUser.getUsername(); - Map userInfo = getUserInfo(ci4sUsername); + String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername(); + String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword(); + +// String ci4sUsername = "chenzhihang"; +// String gitLinkUsername = "chenzhihang"; +// String gitLinkPassword = "czh19970311"; + + Map userInfo = getUserInfo(ci4sUsername, gitLinkUsername, gitLinkPassword); String token = (String) userInfo.get("token"); //拼接查询url String modelTagName = modelsVo.getModelTag(); String modelTypeName = modelsVo.getModelType(); - String search = modelsVo.getName(); String topic_name = "ci4s_model"; topic_name = StringUtils.isEmpty(modelTagName) ? topic_name : topic_name + ",modeltag_" + modelTypeName; topic_name = StringUtils.isEmpty(modelTagName) ? topic_name : topic_name + ",modeltype_" + modelTypeName; - String url = gitendpoint + "/api/users/" + userInfo.get("login") + "/projects.json?page=" + pageRequest.getPageNumber() + "&limit=" + pageRequest.getPageSize() + "&category=manage&topic_name=" + topic_name + "&search=" + search; + String url = gitendpoint + "/api/users/" + userInfo.get("login") + "/projects.json?page=" + pageRequest.getPageNumber() + "&limit=" + pageRequest.getPageSize() + "&category=manage&topic_name=" + topic_name; + + String name = modelsVo.getName(); + if (StringUtils.isNotEmpty(name)) { + url = url + "&search=" + name; + } + String req = HttpUtils.sendGetWithToken(url, null, token); Map stringObjectMap = JacksonUtil.parseJSONStr2Map(req); Integer total = (Integer) stringObjectMap.get("count"); @@ -681,22 +841,29 @@ public class ModelsServiceImpl implements ModelsService { } @Override - public Page newPersonalQueryByPage(ModelsVo modelsVo, PageRequest pageRequest) throws Exception { + public Page newPubilcQueryByPage(ModelsVo modelsVo, PageRequest pageRequest) throws Exception { LoginUser loginUser = SecurityUtils.getLoginUser(); String ci4sUsername = loginUser.getUsername(); - Map userInfo = getUserInfo(ci4sUsername); + String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername(); + String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword(); + + Map userInfo = getUserInfo(ci4sUsername, gitLinkUsername, gitLinkPassword); Integer userId = (Integer) userInfo.get("user_id"); String token = (String) userInfo.get("token"); //拼接查询url String modelTagName = modelsVo.getModelTag(); String modelTypeName = modelsVo.getModelType(); - String search = modelsVo.getName(); String topic_name = "ci4s_model"; topic_name = StringUtils.isEmpty(modelTagName) ? topic_name : topic_name + ",modeltag_" + modelTagName; topic_name = StringUtils.isEmpty(modelTypeName) ? topic_name : topic_name + ",modeltype_" + modelTypeName; - String url = gitendpoint + "/api/projects.json?user_id=" + userId + "&page=" + pageRequest.getPageNumber() + "&limit=" + pageRequest.getPageSize() + "&sort_by=praises_count&topic_name=" + topic_name + "&search=" + search; + String url = gitendpoint + "/api/projects.json?user_id=" + userId + "&page=" + pageRequest.getPageNumber() + "&limit=" + pageRequest.getPageSize() + "&sort_by=praises_count&topic_name=" + topic_name; + String name = modelsVo.getName(); + if (StringUtils.isNotEmpty(name)) { + url = url + "&search=" + name; + } + String req = HttpUtils.sendGetWithToken(url, null, token); Map stringObjectMap = JacksonUtil.parseJSONStr2Map(req); Integer total = (Integer) stringObjectMap.get("total_count"); @@ -708,34 +875,34 @@ public class ModelsServiceImpl implements ModelsService { public List> getVersionList(String identifier, String owner) throws Exception { List> brancheList = gitService.getBrancheList(owner, identifier); return brancheList.stream() - .filter(branch -> "master".equals(branch.get("name"))) + .filter(branch -> !"master".equals(branch.get("name"))) .collect(Collectors.toList()); } @Override - public ModelsVo getModelDetail(String modelName, String identifier, String owner, String version) throws Exception { + public ModelsVo getModelDetail(Integer id, String name, String identifier, String owner, String version) throws Exception { LoginUser loginUser = SecurityUtils.getLoginUser(); String ci4sUsername = loginUser.getUsername(); String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername(); String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword(); +// String ci4sUsername = "chenzhihang"; +// String gitLinkUsername = "chenzhihang"; +// String gitLinkPassword = "czh19970311"; if (StringUtils.isEmpty(version)) { - List> versionList = this.getVersionList(owner, identifier); + List> versionList = this.getVersionList(identifier, owner); if (versionList.size() == 0) { - throw new Exception("数据集文件不存在"); + throw new Exception("模型文件不存在"); } version = (String) versionList.get(0).get("name"); } // git pull操作,然后读取里面的文件列表,列出每个文件的大小和名称,封装成MAP - List> fileDetailsAfterGitPull = DVCUtils.getFileDetailsAfterGitPull(localPath + ci4sUsername + "/model/", modelName, version, "model", gitLinkUsername, gitLinkPassword); + List> fileDetailsAfterGitPull = DVCUtils.getFileDetailsAfterGitPull(localPath + ci4sUsername + "/model/" + id, name, version, "model", gitLinkUsername, gitLinkPassword); - Map stringObjectMap = YamlUtils.loadYamlFile(localPath + ci4sUsername + "/model/" + modelName + "/" + version + "/metadata/metadata.yaml"); - assert stringObjectMap != null; + Map stringObjectMap = YamlUtils.loadYamlFile(localPath + ci4sUsername + "/model/" + id + "/" + name + "/metadata/metadata.yaml"); - ModelMetaVo modelMetaVo = ConvertUtil.convertMapToObject(stringObjectMap, ModelMetaVo.class); - ModelsVo modelsVo = new ModelsVo(); - BeanUtils.copyProperties(modelMetaVo, modelsVo); + ModelsVo modelsVo = ConvertUtil.convertMapToObject(stringObjectMap, ModelsVo.class); List versionVos = new ArrayList<>(); if (!fileDetailsAfterGitPull.isEmpty()) { @@ -772,12 +939,6 @@ public class ModelsServiceImpl implements ModelsService { @Override public void deleteModel(Integer repoId, String identifier, String owner) throws Exception { - LoginUser loginUser = SecurityUtils.getLoginUser(); - String ci4sUsername = loginUser.getUsername(); - Map userInfo = getUserInfo(ci4sUsername); - String token = (String) userInfo.get("token"); - - gitService.deleteProject(owner, identifier); //删除模型依赖 modelDependency1Dao.deleteModel(repoId, identifier, owner, null); @@ -785,73 +946,20 @@ public class ModelsServiceImpl implements ModelsService { } @Override - public void deleteVersion(Integer repoId, String identifier, String owner, String version) throws Exception { - LoginUser loginUser = SecurityUtils.getLoginUser(); - String ci4sUsername = loginUser.getUsername(); - - gitService.deleteBranch(owner, identifier, version); + public void deleteVersion(Integer repoId, String identifier, String owner, String version, String relativePath) throws Exception { + gitService.deleteBranch(owner, identifier, version, localPath + relativePath); //删除模型依赖 modelDependency1Dao.deleteModel(repoId, identifier, owner, version); modelDependency1Dao.deleteModelDependency(repoId, identifier, owner, version); } - void commonDvc(Map userInfo, String username, Integer gitlinIid, String repositoryName, ModelsVo modelsVo, String gitLinkUsername, String gitLinkPassword, String type) throws Exception { - String branchName = modelsVo.getVersion(); - - // 创建分支 - gitService.createBranch((String) userInfo.get("login"), repositoryName, branchName, "master"); - - String projectUrl = gitendpoint + "/" + (String) userInfo.get("login") + "/" + repositoryName + ".git"; - String url = modelsVo.getModelVersionVos().get(0).getUrl(); - String sourcePath = url.substring(0, url.lastIndexOf("/")); - String rootPath = localPath + username + "/model/" + modelsVo.getName() + "/" + branchName; - String modelPath = rootPath + "/model"; - String metaPath = rootPath + "/metadata"; - - - if (type.equals("CreateModelFromPipeline") && FileUtil.checkDirectoryExists(rootPath)) { - DVCUtils.gitFetch(rootPath, gitLinkUsername, gitLinkPassword); - DVCUtils.gitCheckoutBranch(rootPath, branchName); - } else { - DVCUtils.gitClone(rootPath, projectUrl, branchName, gitLinkUsername, gitLinkPassword); - } - - DVCUtils.moveFiles(sourcePath, modelPath); - - //拼接生产的元数据后写入yaml文件 - ModelMetaVo modelMetaVo = new ModelMetaVo(); - BeanUtils.copyProperties(modelsVo, modelMetaVo); - YamlUtils.generateYamlFile(JsonUtils.objectToMap(modelMetaVo), metaPath, "metadata"); - - // dvc init 初始化 - DVCUtils.dvcInit(rootPath); - - // 配置远程S3地址 - String s3Path = "management-platform-files/" + username + "/model/" + gitlinIid + repositoryName + "/" + branchName; - DVCUtils.dvcRemoteAdd(rootPath, s3Path); - DVCUtils.dvcConfigS3Credentials(rootPath, endpoint); - DVCUtils.dvcConfigS3Credentials2(rootPath, accessKeyId); - DVCUtils.dvcConfigS3Credentials3(rootPath, secretAccessKey); - - // dvc 跟踪 - DVCUtils.dvcAdd(rootPath, "model"); - - // git commit - DVCUtils.gitAdd(rootPath, "."); - DVCUtils.gitCommit(rootPath, "commit from ci4s with " + username); - DVCUtils.gitPush(rootPath, gitLinkUsername, gitLinkPassword); - - // dvc push 到远程S3 - DVCUtils.dvcPush(rootPath); - } - public List convert(List> lst) { if (lst != null && lst.size() > 0) { List newModelVos = ConvertUtil.convertListMapToObjectList(lst, ModelsVo.class); for (ModelsVo newModelVo : newModelVos) { Map map = lst.stream() - .filter(m -> m.get("repo_id").equals(newModelVo.getRepoId())) + .filter(m -> m.get("id").equals(newModelVo.getId())) .findFirst() .orElse(null); @@ -869,6 +977,9 @@ public class ModelsServiceImpl implements ModelsService { } }); } + Map author = (Map) map.get("author"); + newModelVo.setCreateBy((String) author.get("name")); + newModelVo.setOwner((String) author.get("login")); } } return newModelVos; @@ -906,9 +1017,13 @@ public class ModelsServiceImpl implements ModelsService { modelDependency1TreeVo.setChildModelList(childModelList); } - Map getUserInfo(String ci4sUsername) throws IOException { + Map getUserInfo(String ci4sUsername, String gitLinkUsername, String gitLinkPassword) throws IOException { Jedis jedis = new Jedis(redisHost); String userReq = jedis.get(ci4sUsername + "_gitUserInfo"); + if (userReq == null) { + gitService.login(gitLinkUsername, gitLinkPassword); + userReq = jedis.get(ci4sUsername + "_gitUserInfo"); + } Map userInfo = JsonUtils.jsonToMap(userReq); String token = gitService.checkoutToken(); diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java index a71e4310..6930efbc 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java @@ -64,7 +64,6 @@ public class NewDatasetServiceImpl implements NewDatasetService { private DatasetTempStorageService datasetTempStorageService; @Override public String newCreateDataset(NewDatasetVo datasetVo) throws Exception { - Jedis jedis = new Jedis(redisHost); LoginUser loginUser = SecurityUtils.getLoginUser(); String ci4sUsername = loginUser.getUsername(); @@ -85,8 +84,6 @@ public class NewDatasetServiceImpl implements NewDatasetService { // 创建项目 Map project = gitService.createProject(gitProjectVo); Integer gitlinIid = (Integer) project.get("id"); - //相对路径!!! - String relativePath = ci4sUsername + "/" + "datasets" + "/" + gitlinIid + "/" + datasetVo.getName() + "/"; if (gitlinIid == null){ throw new Exception("创建数据集失败:"+project.get("message")); } @@ -101,14 +98,16 @@ public class NewDatasetServiceImpl implements NewDatasetService { String projectUrl = gitendpoint + "/" + (String) userInfo.get("login") + "/"+ repositoryName + ".git"; // 得到用户操作的路径 - String url = datasetVo.getDatasetVersionVos().get(0).getUrl(); - String localPath = localPathlocal+ ci4sUsername+"/datasets/" + gitlinIid + "/" + datasetVo.getName(); - String sourcePath = url.substring(0, url.lastIndexOf("/")); + String sourcePath = datasetVo.getDatasetVersionVos().get(0).getUrl(); + String relatePath = ci4sUsername+"/datasets/" + gitlinIid + "/" + datasetVo.getName(); + String localPath = localPathlocal+ relatePath; + String datasetPath = localPath + "/dataset"; + // 命令行操作 git clone 项目地址 DVCUtils.gitClone(localPath, projectUrl, branchName, gitLinkUsername, gitLinkPassword); - String s3Path = "management-platform-files" + "/" + relativePath + branchName; + String s3Path = "management-platform-files" + "/" + relatePath + "/" + branchName; - DVCUtils.moveFiles(sourcePath, localPath); + DVCUtils.moveFiles(sourcePath, datasetPath); // 拼接生产的元数据后写入yaml文件 datasetVo.setCreateBy(String.valueOf(StringUtils.isNotEmpty((String) userInfo.get("nickname")) ? userInfo.get("nickname") : userInfo.get("login"))); datasetVo.setUpdateTime(DateUtils.getTime()); @@ -122,7 +121,7 @@ public class NewDatasetServiceImpl implements NewDatasetService { datasetVo.setIdentifier(repositoryName); datasetVo.setId(gitlinIid); datasetVo.setOwner((String) userInfo.get("login")); - datasetVo.setRelativePaths(relativePath); + datasetVo.setRelativePaths(relatePath); addDatasetSourceToDataVo(datasetVo); @@ -135,7 +134,7 @@ public class NewDatasetServiceImpl implements NewDatasetService { DVCUtils.dvcConfigS3Credentials2(localPath, accessKeyId); DVCUtils.dvcConfigS3Credentials3(localPath, secretAccessKey); // dvc 跟踪 - DVCUtils.dvcAdd(localPath, "data"); + DVCUtils.dvcAdd(localPath, "dataset"); // git commit DVCUtils.gitAdd(localPath, "."); DVCUtils.gitCommit(localPath, "commit from ci4s with " + loginUser.getUsername()); @@ -157,12 +156,11 @@ public class NewDatasetServiceImpl implements NewDatasetService { // 创建分支 String branchName = StringUtils.isEmpty(datasetVo.getVersion()) ? "master" : datasetVo.getVersion(); String repositoryName = datasetVo.getIdentifier(); - String url = datasetVo.getDatasetVersionVos().get(0).getUrl(); - //相对路径!!! - String relativePath = ci4sUsername + "/" + "datasets" + "/" + datasetVo.getId() + "/" + datasetVo.getName() + "/"; - String localPath = localPathlocal + relativePath; - String sourcePath = url.substring(0, url.lastIndexOf("/")); - String projectUrl = gitendpoint + "/" + (String) userInfo.get("login") + "/" + repositoryName + ".git"; + String sourcePath = datasetVo.getDatasetVersionVos().get(0).getUrl(); + String relatePath = ci4sUsername +"/datasets/"+ datasetVo.getId() + "/" + datasetVo.getName(); + String localPath = localPathlocal + relatePath; + String datasetPath = localPath + "/dataset"; + String projectUrl = gitendpoint + "/" + userInfo.get("login") + "/" + repositoryName + ".git"; //部分信息在前面的版本里面,从那边取过来 Map stringObjectMap = YamlUtils.loadYamlFile(localPath + "/" + "dataset.yaml"); @@ -175,9 +173,9 @@ public class NewDatasetServiceImpl implements NewDatasetService { DVCUtils.dvcCheckout(localPath); // 准备数据 - String s3Path = "management-platform-files" + "/" + relativePath + branchName; + String s3Path = "management-platform-files" + "/" + relatePath + "/" + branchName; - DVCUtils.moveFiles(sourcePath, localPath); + DVCUtils.moveFiles(sourcePath, datasetPath); // 拼接生产的元数据后写入yaml文件 datasetVo.setCreateBy(String.valueOf(StringUtils.isNotEmpty((String) userInfo.get("nickname")) ? userInfo.get("nickname") : userInfo.get("login"))); datasetVo.setUpdateTime(DateUtils.getTime()); @@ -188,7 +186,6 @@ public class NewDatasetServiceImpl implements NewDatasetService { "dvc pull\n" + ""); datasetVo.setIdentifier(repositoryName); - datasetVo.setId(datasetVo.getId()); datasetVo.setOwner((String) userInfo.get("login")); if (StringUtils.isEmpty(datasetVo.getDatasetSource())){ datasetVo.setDatasetSource("用户上传"); @@ -196,7 +193,7 @@ public class NewDatasetServiceImpl implements NewDatasetService { datasetVo.setDescription(newDatasetVo.getDescription()); datasetVo.setDataTag(newDatasetVo.getDataTag()); datasetVo.setDataType(newDatasetVo.getDataType()); - datasetVo.setRelativePaths(relativePath); + datasetVo.setRelativePaths(relatePath); addDatasetSourceToDataVo(datasetVo); @@ -208,7 +205,7 @@ public class NewDatasetServiceImpl implements NewDatasetService { DVCUtils.dvcConfigS3Credentials2(localPath, accessKeyId); DVCUtils.dvcConfigS3Credentials3(localPath, secretAccessKey); // dvc 跟踪 - DVCUtils.dvcAdd(localPath, "data"); + DVCUtils.dvcAdd(localPath, "dataset"); DVCUtils.pushNewBranchToRemote(localPath,gitLinkUsername,gitLinkPassword,branchName); //dvc push 到远程S3 DVCUtils.dvcPush(localPath); @@ -308,10 +305,6 @@ public class NewDatasetServiceImpl implements NewDatasetService { @Override public List> getVersionList(String repo, String owner) throws Exception { - Jedis jedis = new Jedis(redisHost); - LoginUser loginUser = SecurityUtils.getLoginUser(); - String ci4sUsername = loginUser.getUsername(); - String token = jedis.get(ci4sUsername+"_gitToken"); List> brancheList = gitService.getBrancheList(owner, repo); return brancheList.stream() .filter(branch -> !"master".equals(branch.get("name"))) @@ -328,12 +321,12 @@ public class NewDatasetServiceImpl implements NewDatasetService { } @Override - public void deleteDatasetVersionNew(String repo, String owner, String version) throws Exception { + public void deleteDatasetVersionNew(String repo, String owner, String version, String relativePath) throws Exception { Jedis jedis = new Jedis(redisHost); LoginUser loginUser = SecurityUtils.getLoginUser(); String ci4sUsername = loginUser.getUsername(); String token = jedis.get(ci4sUsername+"_gitToken"); - gitService.deleteBranch(owner, repo, version); + gitService.deleteBranch(owner, repo, version, localPathlocal + relativePath); } @Override @@ -344,16 +337,15 @@ public class NewDatasetServiceImpl implements NewDatasetService { // 构建objectName String username = SecurityUtils.getLoginUser().getUsername(); String fileName = file.getOriginalFilename(); - String path = localPathlocal + "temp" + "/" + username + "/" + "datasets" + "/" + uuid + "/" + "data" + "/" + fileName; + String path = localPathlocal + "temp" + "/" + username + "/" + "datasets" + "/" + uuid; long sizeInBytes = file.getSize(); String formattedSize = FileUtil.formatFileSize(sizeInBytes); - File targetFile = new File(path); + File targetFile = new File(path, file.getOriginalFilename()); // 确保目录存在 targetFile.getParentFile().mkdirs(); // 保存文件到目标路径 FileUtils.copyInputStreamToFile(file.getInputStream(), targetFile); // 返回上传文件的路径 - String absolutePath = targetFile.getAbsolutePath(); Map result = new HashMap<>(); result.put("fileName", fileName); result.put("url", path); // objectName根据实际情况定义 diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/TensorBoardServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/TensorBoardServiceImpl.java index c998af87..747ee5e2 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/TensorBoardServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/TensorBoardServiceImpl.java @@ -59,9 +59,6 @@ public class TensorBoardServiceImpl implements TensorBoardService { @Override public String runTensorBoard(FrameLogPathVo frameLogPathVo) throws Exception { -// if (StringUtils.isEmpty(frameLogPathVo.getPath())||StringUtils.isEmpty(frameLogPathVo.getPvcName())){ -// throw new Exception("存储路径或存储为空"); -// } if (StringUtils.isEmpty(frameLogPathVo.getPath())) { throw new Exception("存储路径为空"); } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/ConvertUtil.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/ConvertUtil.java index 24cede7e..e1b9f62f 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/ConvertUtil.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/ConvertUtil.java @@ -100,14 +100,13 @@ public class ConvertUtil { private static boolean hasProperty(Class clazz, String propertyName) { try { + Field field = clazz.getDeclaredField(propertyName); + return field != null; + } catch (NoSuchFieldException e) { Class superClazz = clazz.getSuperclass(); - if(superClazz != Object.class){ + if (superClazz != Object.class) { return hasProperty(superClazz, propertyName); - } else { - Field field = clazz.getDeclaredField(propertyName); - return field != null; } - } catch (NoSuchFieldException e) { return false; } } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/DVCUtils.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/DVCUtils.java index 73e40d28..c877721c 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/DVCUtils.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/DVCUtils.java @@ -45,7 +45,7 @@ public class DVCUtils { public static void moveFiles(String sourcePath, String targetPath) throws Exception { Path sourceDir = Paths.get(sourcePath); - Path targetDir = Paths.get(targetPath).resolve(sourceDir.getFileName()); + Path targetDir = Paths.get(targetPath); if (!Files.exists(targetDir)) { Files.createDirectories(targetDir); diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/YamlUtils.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/YamlUtils.java index 03ec45b8..f54238de 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/YamlUtils.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/YamlUtils.java @@ -16,6 +16,15 @@ public class YamlUtils { */ public static void generateYamlFile(Map data, String path, String fileName) { Yaml yaml = new Yaml(); + + File directory = new File(path); + if (!directory.exists()) { + boolean isCreated = directory.mkdirs(); + if (!isCreated) { + throw new RuntimeException("创建路径失败: " + path); + } + } + String fullPath = path + "/" + fileName + ".yaml"; try (FileWriter writer = new FileWriter(fullPath)) { diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelMetaVo.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelMetaVo.java index 2716bf50..19d1e554 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelMetaVo.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelMetaVo.java @@ -13,6 +13,8 @@ import java.util.List; @Data @JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy.class) public class ModelMetaVo implements Serializable { + private Integer id; + @ApiModelProperty(name = "模型名称") private String name; @@ -20,13 +22,13 @@ public class ModelMetaVo implements Serializable { private String version; @ApiModelProperty(name = "版本描述") - private String versionDescription; + private String versionDesc; @ApiModelProperty(value = "创建人") private String createBy; @ApiModelProperty(value = "更新时间") - private Date updateTime; + private String updateTime; @ApiModelProperty(value = "训练镜像") private String image; @@ -62,6 +64,21 @@ public class ModelMetaVo implements Serializable { private String description; @ApiModelProperty(value = "示例用法") - private String examples; + private String usage; + + @ApiModelProperty(name = "time_ago", value = "更新时间") + private String timeAgo; + + @ApiModelProperty(name = "owner", value = "项目所有者") + private String owner; + @ApiModelProperty(name = "identifier", value = "模型仓库名称") + private String identifier; + + @ApiModelProperty(name = "is_public", value = "是否公开1公开,0私有") + private Boolean isPublic; + + @ApiModelProperty(name = "relative_paths",value = "文件相对路径") + private String relativePaths; } + diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelsVo.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelsVo.java index 8a21b24e..9859bf77 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelsVo.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelsVo.java @@ -12,34 +12,13 @@ import java.util.List; @Data public class ModelsVo extends ModelMetaVo implements Serializable { - private Integer id; - - /** - * 模型可见范围 - */ - @ApiModelProperty(name = "available_range") - private int availableRange; - - /** - * 状态 - */ - @ApiModelProperty(name = "status") + @ApiModelProperty(name = "status", value = "状态") private Integer status; - @ApiModelProperty(name = "models_version_vos") + @ApiModelProperty(name = "model_version_vos") private List modelVersionVos; - /** - * 模型仓库名称 - */ - @ApiModelProperty(name = "repository_name") - private String repositoryName; - - @ApiModelProperty(name = "repo_id") - private Integer repoId; - - /** - * 父模型 - */ + @ApiModelProperty(name = "parent_model", value = "父模型") private String parentModel; + } diff --git a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ModelDependency1DaoMapper.xml b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ModelDependency1DaoMapper.xml index 9a57aa3a..5bdd32d6 100644 --- a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ModelDependency1DaoMapper.xml +++ b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ModelDependency1DaoMapper.xml @@ -4,15 +4,24 @@ - insert into model_dependency1(parent_model, model_name, version, repo_id, identifier, owner) - values (#{parentModel}, #{modelName}, #{version}, #{repoId}, #{identifier}, #{owner}) + insert into model_dependency1(parent_model, model_name, version, repo_id, identifier, owner, meta) + values (#{parentModel}, #{modelName}, #{version}, #{repoId}, #{identifier}, #{owner}, #{meta}) - select * from model_dependency1 - where model_name = #{modelName} - and state = 1 + where + state = 1 + + and model_name = #{modelName} + + + and repo_id = #{repoId} + + + and owner = #{owner} +