diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/dataset/NewDatasetFromGitController.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/dataset/NewDatasetFromGitController.java index 5144cac7..62f5122a 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/dataset/NewDatasetFromGitController.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/dataset/NewDatasetFromGitController.java @@ -91,8 +91,8 @@ public class NewDatasetFromGitController { @DeleteMapping("/deleteDataset") @ApiOperation(value = "删除数据集") - public AjaxResult deleteDataset(@RequestParam("identifier") String repo, @RequestParam("owner") String owner) throws Exception { - this.newDatasetService.deleteDatasetNew(repo, owner); + public AjaxResult deleteDataset(@RequestParam("id") Integer id, @RequestParam("identifier") String identifier, @RequestParam("owner") String owner) throws Exception { + this.newDatasetService.deleteDatasetNew(id, identifier, owner); return AjaxResult.success(); } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/GitService.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/GitService.java index 4d9e0952..85eea493 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/GitService.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/GitService.java @@ -2,6 +2,7 @@ package com.ruoyi.platform.service; import com.ruoyi.platform.vo.GitProjectVo; +import java.io.IOException; import java.util.List; import java.util.Map; @@ -25,4 +26,6 @@ public interface GitService { void deleteBranch(String token,String owner, String projectName, String branchName, String localPath) throws Exception; Map getUserInfo(String token) throws Exception; + + Map getProjectDetail(String owner, String identifier, String token) throws IOException; } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/NewDatasetService.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/NewDatasetService.java index ff2003de..f43891ee 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/NewDatasetService.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/NewDatasetService.java @@ -27,7 +27,7 @@ public interface NewDatasetService { List> getVersionList(String repo, String owner) throws Exception; - void deleteDatasetNew(String repo, String owner) throws Exception; + void deleteDatasetNew(Integer repoId, String repo, String owner) throws Exception; void deleteDatasetVersionNew(String repo, String owner, String version, String relativePath) throws Exception; } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/GitServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/GitServiceImpl.java index ce582396..3fd29369 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/GitServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/GitServiceImpl.java @@ -47,7 +47,7 @@ public class GitServiceImpl implements GitService { params.put("client_secret", "L3wBKTNnRo-wPen7bxR3F1myCvtVDgpWa6MnpfyWeJE"); try { // 发送POST请求 - String req = httpUtils.sendPost(gitendpoint + "/oauth/token", null,JsonUtils.mapToJson(params)); + String req = httpUtils.sendPost(gitendpoint + "/oauth/token", null, JsonUtils.mapToJson(params)); // 解析响应JSON if (StringUtils.isEmpty(req)) { throw new RuntimeException("终止响应内容为空。"); @@ -66,7 +66,7 @@ public class GitServiceImpl implements GitService { LoginUser loginUser = SecurityUtils.getLoginUser(); String ci4sUsername = loginUser.getUsername(); // 将access_token存入Redis - Jedis jedis = new Jedis(redisHost,redisPort); + Jedis jedis = new Jedis(redisHost, redisPort); jedis.set(ci4sUsername + "_gitToken", accessToken); jedis.set(ci4sUsername + "_gitUserInfo", userReq); return accessToken; @@ -77,7 +77,7 @@ public class GitServiceImpl implements GitService { } public String checkoutToken() { - Jedis jedis = new Jedis(redisHost,redisPort); + Jedis jedis = new Jedis(redisHost, redisPort); LoginUser loginUser = SecurityUtils.getLoginUser(); String ci4sUsername = loginUser.getUsername(); String token = jedis.get(ci4sUsername + "_gitToken"); @@ -103,13 +103,13 @@ public class GitServiceImpl implements GitService { } @Override - public Map createProject(String token,GitProjectVo gitProjectVo) throws Exception { - String userReq = httpUtils.sendPostWithToken(gitendpoint + "/api/projects.json",null, token,JsonUtils.objectToJson(gitProjectVo)); + public Map createProject(String token, GitProjectVo gitProjectVo) throws Exception { + String userReq = httpUtils.sendPostWithToken(gitendpoint + "/api/projects.json", null, token, JsonUtils.objectToJson(gitProjectVo)); return JsonUtils.jsonToMap(userReq); } @Override - public void createBranch(String token,String owner, String projectName, String branchName, String oldBranchName) throws Exception { + public void createBranch(String token, String owner, String projectName, String branchName, String oldBranchName) throws Exception { //https://www.gitlink.org.cn/api/v1/fanshuai/testdssa8755/branches.json // { // "new_branch_name": "SsS", @@ -119,21 +119,21 @@ public class GitServiceImpl implements GitService { Map resMap = new HashMap<>(); resMap.put("new_branch_name", branchName); resMap.put("old_branch_name", oldBranchName); - String req = httpUtils.sendPostWithToken(createBranchUrl, null, token,JsonUtils.objectToJson(resMap)); + String req = httpUtils.sendPostWithToken(createBranchUrl, null, token, JsonUtils.objectToJson(resMap)); System.out.println(req); } @Override - public void createTopic(String token,Integer id, String topicName) throws Exception { + public void createTopic(String token, Integer id, String topicName) throws Exception { // https://www.gitlink.org.cn/api/v1/project_topics.json Map resMap = new HashMap<>(); resMap.put("project_id", id); resMap.put("name", topicName); - String req = httpUtils.sendPostWithToken(gitendpoint + "/api/v1/project_topics.json",null, token ,JsonUtils.objectToJson(resMap)); + String req = httpUtils.sendPostWithToken(gitendpoint + "/api/v1/project_topics.json", null, token, JsonUtils.objectToJson(resMap)); } @Override - public List> getBrancheList(String token,String owner, String projectName) throws Exception { + public List> getBrancheList(String token, String owner, String projectName) throws Exception { String req = httpUtils.sendGetWithToken(gitendpoint + "/api/v1/" + owner + "/" + projectName + "/branches/all.json", null, token); // 解析响应JSON if (StringUtils.isEmpty(req)) { @@ -145,25 +145,35 @@ public class GitServiceImpl implements GitService { } @Override - public void deleteProject(String token,String owner, String projectName) throws Exception { + public void deleteProject(String token, String owner, String projectName) throws Exception { httpUtils.sendDeleteWithToken(gitendpoint + "/api/" + owner + "/" + projectName + ".json", null, token); } @Override - public void deleteBranch(String token,String owner, String projectName, String branchName, String localPath) throws Exception { + public void deleteBranch(String token, String owner, String projectName, String branchName, String localPath) throws Exception { try (Git git = Git.open(new File(localPath))) { git.checkout().setName("master").call(); git.branchDelete().setBranchNames(branchName).setForce(true).call(); } catch (IOException | GitAPIException e) { - log.error("Exception occurred while creating local branch based on master",e); + log.error("Exception occurred while creating local branch based on master", e); } httpUtils.sendDeleteWithToken(gitendpoint + "/api/v1/" + owner + "/" + projectName + "/branches/" + branchName + ".json", null, token); } @Override public Map getUserInfo(String token) throws Exception { - String userReq = httpUtils.sendGetWithToken(gitendpoint + "/api/users/get_user_info.json",null, token); - if (StringUtils.isEmpty(userReq)){ + String userReq = httpUtils.sendGetWithToken(gitendpoint + "/api/users/get_user_info.json", null, token); + if (StringUtils.isEmpty(userReq)) { + return null; + } + Map runResMap = JsonUtils.jsonToMap(userReq); + return runResMap; + } + + @Override + public Map getProjectDetail(String owner, String identifier, String token) throws IOException { + String userReq = httpUtils.sendGetWithToken(gitendpoint + "/api/" + owner + "/" + identifier + "/detail.json", null, token); + if (StringUtils.isEmpty(userReq)) { return null; } Map runResMap = JsonUtils.jsonToMap(userReq); diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java index 514a0d36..dbd557e0 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java @@ -602,12 +602,15 @@ public class ModelsServiceImpl implements ModelsService { String projectUrl = gitendpoint + "/" + owner + "/" + repositoryName + ".git"; String sourcePath = modelsVo.getModelVersionVos().get(0).getUrl(); - String rootPath = localPath + ci4sUsername + "/model/" + gitlinIid + "/" + repositoryName; + String rootPath = localPath + ci4sUsername + "/model/" + gitlinIid + "/" + repositoryName + "/" + branchName; String modelPath = rootPath + "/model"; String metaPath = rootPath + "/metadata"; - String relatePath = ci4sUsername + "/model/" + gitlinIid + "/" + repositoryName + "/model"; + String relatePath = ci4sUsername + "/model/" + gitlinIid + "/" + repositoryName + "/" + branchName + "/model"; + + dvcUtils.gitClone(rootPath, projectUrl, "master", gitLinkUsername, gitLinkPassword); + dvcUtils.createLocalBranchBasedOnMaster(rootPath, branchName); + dvcUtils.gitCheckoutBranch(rootPath, branchName); - dvcUtils.gitClone(rootPath, projectUrl, branchName, gitLinkUsername, gitLinkPassword); //干掉目标文件夹 dvcUtils.deleteDirectory(modelPath); dvcUtils.moveFiles(sourcePath, modelPath); @@ -661,11 +664,11 @@ public class ModelsServiceImpl implements ModelsService { // dvc 跟踪 dvcUtils.dvcAdd(rootPath, "model"); // git commit - dvcUtils.gitAdd(rootPath, "."); dvcUtils.gitCommit(rootPath, "commit from ci4s with " + ci4sUsername); dvcUtils.gitPush(rootPath, gitLinkUsername, gitLinkPassword); dvcUtils.dvcPush(rootPath); } catch (Exception e) { + logger.error(e.getMessage(), e); throw new RuntimeException(e); } return null; @@ -699,14 +702,18 @@ public class ModelsServiceImpl implements ModelsService { String owner = (String) userInfo.get("login"); String projectUrl = gitendpoint + "/" + owner + "/" + repositoryName + ".git"; - String rootPath = localPath + ci4sUsername + "/model/" + modelsVo.getId() + "/" + repositoryName; + String rootPath = localPath + ci4sUsername + "/model/" + modelsVo.getId() + "/" + repositoryName + "/" + branchName; String modelPath = rootPath + "/model"; String metaPath = rootPath + "/metadata"; - String relatePath = ci4sUsername + "/model/" + modelsVo.getId() + "/" + repositoryName + "/model"; + String relatePath = ci4sUsername + "/model/" + modelsVo.getId() + "/" + repositoryName + "/" + branchName + "/model"; - //部分信息在前面的版本里面,从那边取过来 - Map stringObjectMap = YamlUtils.loadYamlFile(metaPath + "/" + "metadata.yaml"); - ModelMetaVo oldModelVo = ConvertUtil.convertMapToObject(stringObjectMap, ModelMetaVo.class); + //克隆 + dvcUtils.gitClone(rootPath, projectUrl, oldModelDependencys.get(0).getVersion(), gitLinkUsername, gitLinkPassword); + dvcUtils.refreshRemoteBranches(rootPath, gitLinkUsername, gitLinkPassword, oldModelDependencys.get(0).getVersion()); + + //查询项目信息 + String token = gitService.checkoutToken(); + Map projectDetail = gitService.getProjectDetail(owner, repositoryName, token); //检查是否存在本地重名分支,有的话干掉 dvcUtils.deleteLocalBranch(rootPath, branchName); @@ -757,9 +764,20 @@ public class ModelsServiceImpl implements ModelsService { ""); modelMetaVo.setIdentifier(repositoryName); modelMetaVo.setOwner(owner); - modelMetaVo.setDescription(oldModelVo.getDescription()); - modelMetaVo.setModelTag(oldModelVo.getModelTag()); - modelMetaVo.setModelType(oldModelVo.getModelType()); + modelMetaVo.setDescription((String) projectDetail.get("description")); + List topics = (List) projectDetail.get("topics"); + if (topics != null && topics.size() > 0) { + for (Map topic : topics) { + String topicName = (String) topic.get("name"); + if (topicName.startsWith("modeltag-")) { + modelMetaVo.setModelTag(topicName.substring("modeltag-".length())); + } + if (topicName.startsWith("modeltype-")) { + modelMetaVo.setModelType(topicName.substring("modeltype-".length())); + } + } + } + modelMetaVo.setRelativePaths(relatePath); File folder = new File(modelPath); long folderSize = FileUtil.getFolderSize(folder); @@ -814,15 +832,18 @@ public class ModelsServiceImpl implements ModelsService { dvcUtils.dvcAdd(rootPath, "model"); dvcUtils.pushNewBranchToRemote(rootPath, gitLinkUsername, gitLinkPassword, branchName); - CompletableFuture.supplyAsync(() -> { - try { - //dvc push 到远程S3 - dvcUtils.dvcPush(rootPath); - } catch (Exception e) { - throw new RuntimeException(e); - } - return null; - }); + //dvc push 到远程S3 + dvcUtils.dvcPush(rootPath); + +// CompletableFuture.supplyAsync(() -> { +// try { +// //dvc push 到远程S3 +// dvcUtils.dvcPush(rootPath); +// } catch (Exception e) { +// throw new RuntimeException(e); +// } +// return null; +// }); } return "新增模型版本成功"; } @@ -860,7 +881,7 @@ public class ModelsServiceImpl implements ModelsService { LoginUser loginUser = SecurityUtils.getLoginUser(); String ci4sUsername = loginUser.getUsername(); - String localPath1 = localPath + ci4sUsername + "/model/" + id + "/" + identifier; + String localPath1 = localPath + ci4sUsername + "/model/" + id + "/" + identifier + "/" + version; // 打包 data 文件夹 String dataFolderPath = localPath1 + "/model"; String zipFilePath = localPath1 + "/model.zip"; @@ -980,7 +1001,7 @@ public class ModelsServiceImpl implements ModelsService { // git pull操作,然后读取里面的文件列表,列出每个文件的大小和名称,封装成MAP List> fileDetailsAfterGitPull = dvcUtils.getFileDetailsAfterGitPull(localPath + ci4sUsername + "/model/" + id, identifier, version, "model", gitLinkUsername, gitLinkPassword); - Map stringObjectMap = YamlUtils.loadYamlFile(localPath + ci4sUsername + "/model/" + id + "/" + identifier + "/metadata/metadata.yaml"); + Map stringObjectMap = YamlUtils.loadYamlFile(localPath + ci4sUsername + "/model/" + id + "/" + identifier + "/" + version + "/metadata/metadata.yaml"); String jsonString = JSON.toJSONString(stringObjectMap); ModelsVo modelsVo = JSON.parseObject(jsonString, ModelsVo.class); @@ -1030,12 +1051,17 @@ public class ModelsServiceImpl implements ModelsService { map.put("identifier", identifier); String parentModel = JSON.toJSONString(map); modelDependency1Dao.deleteModelDependency(parentModel); + + LoginUser loginUser = SecurityUtils.getLoginUser(); + String ci4sUsername = loginUser.getUsername(); + dvcUtils.deleteDirectory(localPath + "/" + ci4sUsername + "/model/" + repoId + "/" + identifier); } @Override public void deleteVersion(Integer repoId, String identifier, String owner, String version, String relativePath) throws Exception { String token = gitService.checkoutToken(); - gitService.deleteBranch(token, owner, identifier, version, localPath + relativePath); + String rootPath = Paths.get(localPath + "/" + relativePath).getParent().toString(); + gitService.deleteBranch(token, owner, identifier, version, rootPath); //删除模型依赖 modelDependency1Dao.deleteModel(repoId, identifier, owner, version); HashMap map = new HashMap<>(); @@ -1044,6 +1070,7 @@ public class ModelsServiceImpl implements ModelsService { map.put("version", version); String parentModel = JSON.toJSONString(map); modelDependency1Dao.deleteModelDependency(parentModel); + dvcUtils.deleteDirectory(rootPath); } public List convert(List> lst, String modelTopic, String modelTagName, String modelTypeName) { diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java index 030b5790..69dda5b8 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java @@ -1,23 +1,22 @@ package com.ruoyi.platform.service.impl; -import com.alibaba.fastjson2.JSON; import com.ruoyi.common.core.utils.DateUtils; import com.ruoyi.common.security.utils.SecurityUtils; import com.ruoyi.platform.constant.Constant; import com.ruoyi.platform.domain.Dataset; import com.ruoyi.platform.domain.DatasetTempStorage; import com.ruoyi.platform.service.DatasetTempStorageService; -import com.ruoyi.platform.service.DvcService; import com.ruoyi.platform.service.GitService; import com.ruoyi.platform.service.NewDatasetService; import com.ruoyi.platform.utils.*; import com.ruoyi.platform.vo.GitProjectVo; -import com.ruoyi.platform.vo.ModelsVo; import com.ruoyi.platform.vo.NewDatasetVo; import com.ruoyi.platform.vo.VersionVo; import com.ruoyi.system.api.model.LoginUser; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; import org.springframework.core.io.InputStreamResource; import org.springframework.data.domain.Page; @@ -47,11 +46,11 @@ import java.util.zip.ZipOutputStream; @Service public class NewDatasetServiceImpl implements NewDatasetService { + private static final Logger logger = LoggerFactory.getLogger(NewDatasetServiceImpl.class); + @Resource private GitService gitService; - @Resource - private DvcService dvcService; @Value("${spring.redis.host}") private String redisHost; @Value("${spring.redis.port}") @@ -105,17 +104,20 @@ public class NewDatasetServiceImpl implements NewDatasetService { } // 创建分支 String branchName = datasetVo.getVersion(); + String owner = (String) userInfo.get("login"); + gitService.createBranch(token, (String) userInfo.get("login"), repositoryName, branchName, "master"); // 定义标签 标签1:ci4s-dataset 标签2:DataTag 标签3:DataType gitService.createTopic(token, gitlinIid, "ci4s-dataset"); gitService.createTopic(token, gitlinIid, "datatag-" + datasetVo.getDataTag()); gitService.createTopic(token, gitlinIid, "datatype-" + datasetVo.getDataType()); + // 得到项目地址 - String projectUrl = gitendpoint + "/" + (String) userInfo.get("login") + "/" + repositoryName + ".git"; + String projectUrl = gitendpoint + "/" + owner + "/" + repositoryName + ".git"; // 得到用户操作的路径 String sourcePath = datasetVo.getDatasetVersionVos().get(0).getUrl(); - String relatePath = ci4sUsername + "/datasets/" + gitlinIid + "/" + repositoryName; + String relatePath = ci4sUsername + "/datasets/" + gitlinIid + "/" + repositoryName + "/" + branchName; String localPath = localPathlocal + relatePath; String datasetPath = localPath + "/dataset"; @@ -137,7 +139,7 @@ public class NewDatasetServiceImpl implements NewDatasetService { ""); datasetVo.setIdentifier(repositoryName); datasetVo.setId(gitlinIid); - datasetVo.setOwner((String) userInfo.get("login")); + datasetVo.setOwner(owner); datasetVo.setRelativePaths(relatePath + "/dataset"); addDatasetSourceToDataVo(datasetVo); @@ -161,6 +163,7 @@ public class NewDatasetServiceImpl implements NewDatasetService { // dvc push 到远程S3 dvcUtils.dvcPush(localPath); } catch (Exception e) { + logger.error(e.getMessage(), e); throw new RuntimeException(e); } return null; @@ -179,17 +182,26 @@ public class NewDatasetServiceImpl implements NewDatasetService { Map userInfo = JsonUtils.jsonToMap(userReq); // 创建分支 String branchName = StringUtils.isEmpty(datasetVo.getVersion()) ? "master" : datasetVo.getVersion(); + String owner = (String) userInfo.get("login"); + String repositoryName = datasetVo.getIdentifier(); String sourcePath = datasetVo.getDatasetVersionVos().get(0).getUrl(); - String relatePath = ci4sUsername + "/datasets/" + datasetVo.getId() + "/" + repositoryName; + String relatePath = ci4sUsername + "/datasets/" + datasetVo.getId() + "/" + repositoryName + "/" + branchName; String localPath = localPathlocal + relatePath; String datasetPath = localPath + "/dataset"; - String projectUrl = gitendpoint + "/" + userInfo.get("login") + "/" + repositoryName + ".git"; + String projectUrl = gitendpoint + "/" + owner + "/" + repositoryName + ".git"; + + //查询项目信息 + Map projectDetail = gitService.getProjectDetail(owner, repositoryName, token); + + //克隆 + List> brancheList = gitService.getBrancheList(token, owner, repositoryName); + String oldBranch = (String) brancheList.stream() + .filter(branch -> !"master".equals(branch.get("name"))).collect(Collectors.toList()).get(0).get("name"); + + dvcUtils.gitClone(localPath, projectUrl, oldBranch, gitLinkUsername, gitLinkPassword); + dvcUtils.refreshRemoteBranches(localPath, gitLinkUsername, gitLinkPassword, oldBranch); - //部分信息在前面的版本里面,从那边取过来 - Map stringObjectMap = YamlUtils.loadYamlFile(localPath + "/" + "dataset.yaml"); - String jsonString = JacksonUtil.toJSONString(stringObjectMap); - NewDatasetVo newDatasetVo = JsonUtils.jsonToObject(jsonString, NewDatasetVo.class); //检查是否存在本地重名分支,有的话干掉 dvcUtils.deleteLocalBranch(localPath, branchName); // 创建本地分支 @@ -225,9 +237,19 @@ public class NewDatasetServiceImpl implements NewDatasetService { datasetVo.setIdentifier(repositoryName); datasetVo.setOwner((String) userInfo.get("login")); - datasetVo.setDescription(newDatasetVo.getDescription()); - datasetVo.setDataTag(newDatasetVo.getDataTag()); - datasetVo.setDataType(newDatasetVo.getDataType()); + datasetVo.setDescription((String) projectDetail.get("description")); + List topics = (List) projectDetail.get("topics"); + if (topics != null && topics.size() > 0) { + for (Map topic : topics) { + String topicName = (String) topic.get("name"); + if (topicName.startsWith("datatag-")) { + datasetVo.setDataTag(topicName.substring("datatag-".length())); + } + if (topicName.startsWith("datatype-")) { + datasetVo.setDataType(topicName.substring("datatype-".length())); + } + } + } datasetVo.setRelativePaths(relatePath + "/dataset"); datasetVo = addDatasetSourceToDataVo(datasetVo); @@ -244,15 +266,17 @@ public class NewDatasetServiceImpl implements NewDatasetService { dvcUtils.dvcAdd(localPath, "dataset"); dvcUtils.pushNewBranchToRemote(localPath, gitLinkUsername, gitLinkPassword, branchName); - CompletableFuture.supplyAsync(() -> { - try { - //dvc push 到远程S3 - dvcUtils.dvcPush(localPath); - } catch (Exception e) { - throw new RuntimeException(e); - } - return null; - }); + dvcUtils.dvcPush(localPath); + +// CompletableFuture.supplyAsync(() -> { +// try { +// //dvc push 到远程S3 +// dvcUtils.dvcPush(localPath); +// } catch (Exception e) { +// throw new RuntimeException(e); +// } +// return null; +// }); return "新增数据集成功"; } @@ -326,7 +350,7 @@ public class NewDatasetServiceImpl implements NewDatasetService { } List> fileDetailsAfterGitPull = dvcUtils.getFileDetailsAfterGitPull(localPathlocal + loginUser.getUsername() + "/datasets/" + id, repo, version, "dataset", gitLinkUsername, gitLinkPassword); // 在localPathlocal+id+"/"+repositoryName目录下的dataset.yaml中取到元数据 - Map stringObjectMap = YamlUtils.loadYamlFile(localPathlocal + loginUser.getUsername() + "/datasets/" + id + "/" + repo + "/" + "dataset.yaml"); + Map stringObjectMap = YamlUtils.loadYamlFile(localPathlocal + loginUser.getUsername() + "/datasets/" + id + "/" + repo + "/" + version + "/dataset.yaml"); String jsonString = JacksonUtil.toJSONString(stringObjectMap); NewDatasetVo newDatasetVo = JsonUtils.jsonToObject(jsonString, NewDatasetVo.class); List versionVos = new ArrayList(); @@ -354,15 +378,22 @@ public class NewDatasetServiceImpl implements NewDatasetService { } @Override - public void deleteDatasetNew(String repo, String owner) throws Exception { + public void deleteDatasetNew(Integer repoId, String repo, String owner) throws Exception { String token = gitService.checkoutToken(); gitService.deleteProject(token, owner, repo); + + LoginUser loginUser = SecurityUtils.getLoginUser(); + String ci4sUsername = loginUser.getUsername(); + dvcUtils.deleteDirectory(localPathlocal + "/" + ci4sUsername + "/datasets/" + repoId + "/" + repo); + } @Override public void deleteDatasetVersionNew(String repo, String owner, String version, String relativePath) throws Exception { String token = gitService.checkoutToken(); - gitService.deleteBranch(token, owner, repo, version, localPathlocal + relativePath); + String rootPath = Paths.get(localPathlocal + "/" + relativePath).getParent().toString(); + gitService.deleteBranch(token, owner, repo, version, rootPath); + dvcUtils.deleteDirectory(rootPath); } @Override @@ -418,7 +449,7 @@ public class NewDatasetServiceImpl implements NewDatasetService { public ResponseEntity downloadAllDatasetFilesNew(String name, String identifier, Integer id, String version) throws Exception { // 命令行操作 git clone 项目地址 LoginUser loginUser = SecurityUtils.getLoginUser(); - String localPath = localPathlocal + loginUser.getUsername() + "/datasets/" + id + "/" + identifier; + String localPath = localPathlocal + loginUser.getUsername() + "/datasets/" + id + "/" + identifier + "/" + version; // 打包 data 文件夹 String dataFolderPath = localPath + "/dataset"; @@ -512,7 +543,7 @@ public class NewDatasetServiceImpl implements NewDatasetService { if (datasetTempStorage != null) { String source = datasetTempStorage.getSource(); Map sourceMap = JacksonUtil.parseJSONStr2Map(source); - Map preprocessCode = (Map) sourceMap.get("preprocess_code"); + Map preprocessCode = (Map) sourceMap.get("preprocess_code"); sourceMap.remove("preprocess_code"); datasetVo.setTrainTask((HashMap) sourceMap); datasetVo.setProcessingCode((HashMap) preprocessCode); diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/DVCUtils.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/DVCUtils.java index 730b5f5a..fd307914 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/DVCUtils.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/DVCUtils.java @@ -71,9 +71,9 @@ public class DVCUtils { Path sourceDir = Paths.get(sourcePath); Path targetDir = Paths.get(targetPath); -// if (!Files.exists(targetDir)) { -// Files.createDirectories(targetDir); -// } + if (!Files.exists(targetDir)) { + Files.createDirectories(targetDir); + } Files.move(sourceDir, targetDir, StandardCopyOption.REPLACE_EXISTING); } @@ -88,8 +88,6 @@ public class DVCUtils { .filter(path -> !path.equals(directory)) .sorted(Comparator.reverseOrder()) // 先删除子文件夹,再删除父文件夹 .forEach(this::deletePath); - } else { - Files.createDirectories(directory); } } @@ -629,9 +627,9 @@ public class DVCUtils { public List> getFileDetailsAfterGitPull(String localPath, String repoFolder, String branch, String filePath , String username, String password) { try { //刷新 - refreshRemoteBranches(localPath+"/"+repoFolder, username, password,branch); + refreshRemoteBranches(localPath+"/" + repoFolder + "/"+ branch, username, password,branch); // 读取data文件夹中的文件列表 - String path = localPath + "/" + repoFolder + "/" + filePath; + String path = localPath + "/" + repoFolder + "/"+ branch + "/" + filePath; return FileUtil.getFiles(path); diff --git a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ModelDependency1DaoMapper.xml b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ModelDependency1DaoMapper.xml index 596c931a..37f89d41 100644 --- a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ModelDependency1DaoMapper.xml +++ b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ModelDependency1DaoMapper.xml @@ -22,6 +22,7 @@ and owner = #{owner} + order by create_time