diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/dataset/NewDatasetFromGitController.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/dataset/NewDatasetFromGitController.java index 42cb47da..b182f405 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/dataset/NewDatasetFromGitController.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/dataset/NewDatasetFromGitController.java @@ -119,8 +119,8 @@ public class NewDatasetFromGitController { */ @GetMapping("/downloadAllFiles") @ApiOperation(value = "下载同一版本下所有数据集,并打包") - public ResponseEntity downloadAllDatasetFiles(@RequestParam("repository_name") String repositoryName, @RequestParam("version") String version) throws Exception { - return newDatasetService.downloadAllDatasetFilesNew(repositoryName, version); + public ResponseEntity downloadAllDatasetFiles(@RequestParam("name") String name, @RequestParam("id") Integer id, @RequestParam("version") String version) throws Exception { + return newDatasetService.downloadAllDatasetFilesNew(name,id, version); } /** diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/NewDatasetService.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/NewDatasetService.java index 0234f133..8fa97d00 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/NewDatasetService.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/NewDatasetService.java @@ -19,7 +19,7 @@ public interface NewDatasetService { String newCreateVersion(NewDatasetVo datasetVo) throws Exception; List> uploadDatasetlocal(MultipartFile[] files, String uuid) throws Exception; ResponseEntity downloadDatasetlocal(String filePath) throws Exception; - ResponseEntity downloadAllDatasetFilesNew(String repositoryName, String version) throws IOException, Exception; + ResponseEntity downloadAllDatasetFilesNew(String repositoryName,Integer id, String version) throws IOException, Exception; Page newPersonalQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception; Page newPubilcQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception; diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java index eb0cca04..40c31894 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java @@ -80,6 +80,9 @@ public class NewDatasetServiceImpl implements NewDatasetService { // 创建项目 Map project = gitService.createProject(gitProjectVo); Integer gitlinIid = (Integer) project.get("id"); + if (gitlinIid == null){ + throw new Exception("创建数据集失败:"+project.get("message")); + } // 创建分支 String branchName = datasetVo.getVersion(); gitService.createBranch((String) userInfo.get("login"), repositoryName, branchName, "master"); @@ -92,7 +95,7 @@ public class NewDatasetServiceImpl implements NewDatasetService { // 得到用户操作的路径 String url = datasetVo.getDatasetVersionVos().get(0).getUrl(); - String localPath = localPathlocal + gitlinIid + "/" + datasetVo.getName(); + String localPath = localPathlocal+ ci4sUsername+"/datasets/" + gitlinIid + "/" + datasetVo.getName(); String sourcePath = url.substring(0, url.lastIndexOf("/")); // 命令行操作 git clone 项目地址 DVCUtils.gitClone(localPath, projectUrl, branchName, gitLinkUsername, gitLinkPassword); @@ -112,6 +115,8 @@ public class NewDatasetServiceImpl implements NewDatasetService { datasetVo.setIdentifier(repositoryName); datasetVo.setId(gitlinIid); datasetVo.setOwner((String) userInfo.get("login")); + datasetVo.setRelativePaths(ci4sUsername+"/" + gitlinIid + "/" + datasetVo.getName()); + if (StringUtils.isEmpty(datasetVo.getDatasetSource())){ datasetVo.setDatasetSource("用户上传"); } @@ -147,10 +152,14 @@ public class NewDatasetServiceImpl implements NewDatasetService { String branchName = StringUtils.isEmpty(datasetVo.getVersion()) ? "master" : datasetVo.getVersion(); String repositoryName = datasetVo.getIdentifier(); String url = datasetVo.getDatasetVersionVos().get(0).getUrl(); - String localPath = localPathlocal + datasetVo.getId() + "/" + datasetVo.getName(); + String localPath = localPathlocal + ci4sUsername +"/datasets/"+ datasetVo.getId() + "/" + datasetVo.getName(); String sourcePath = url.substring(0, url.lastIndexOf("/")); String projectUrl = gitendpoint + "/" + (String) userInfo.get("login") + "/" + repositoryName + ".git"; + //部分信息在前面的版本里面,从那边取过来 + Map stringObjectMap = YamlUtils.loadYamlFile(localPath + "/" + "dataset.yaml"); + NewDatasetVo newDatasetVo = ConvertUtil.convertMapToObject(stringObjectMap, NewDatasetVo.class); + // 创建本地分支 DVCUtils.createLocalBranchBasedOnMaster(localPath, branchName); @@ -164,7 +173,6 @@ public class NewDatasetServiceImpl implements NewDatasetService { // 拼接生产的元数据后写入yaml文件 datasetVo.setCreateBy(String.valueOf(StringUtils.isNotEmpty((String) userInfo.get("nickname")) ? userInfo.get("nickname") : userInfo.get("login"))); datasetVo.setUpdateTime(DateUtils.getTime()); - datasetVo.setVersionDesc(datasetVo.getDescription()); datasetVo.setUsage("
" +
                 "# 克隆数据集配置文件与存储参数到本地\n" +
                 "git clone -b " + branchName + " "+ projectUrl + "\n" +
@@ -177,6 +185,10 @@ public class NewDatasetServiceImpl implements NewDatasetService {
         if (StringUtils.isEmpty(datasetVo.getDatasetSource())){
             datasetVo.setDatasetSource("用户上传");
         }
+        datasetVo.setDescription(newDatasetVo.getDescription());
+        datasetVo.setDataTag(newDatasetVo.getDataTag());
+        datasetVo.setDataType(newDatasetVo.getDataType());
+        datasetVo.setRelativePaths(ci4sUsername + "/" + datasetVo.getId() + "/" + datasetVo.getName());
         YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo), localPath, "dataset");
         //dvc数据跟踪
         // 配置远程S3地址
@@ -208,6 +220,11 @@ public class NewDatasetServiceImpl implements NewDatasetService {
         topic_name = StringUtils.isEmpty(datasetTagName) ? topic_name : topic_name + ",datatag_" + datasetTagName;
         topic_name = StringUtils.isEmpty(datasetTagName) ? topic_name : topic_name + ",datatype_" + datasetTypeName;
         String url = gitendpoint + "/api/users/" + (String) userInfo.get("login") + "/projects.json?page=" + pageRequest.getPageNumber() + "&limit=" + pageRequest.getPageSize() + "&category=manage&topic_name=" + topic_name;
+        String name = dataset.getName();
+
+        if (StringUtils.isNotEmpty(name)){
+            url =url + "&search=" + name;
+        }
         String req = HttpUtils.sendGetWithToken(url, null, token);
         Map stringObjectMap = JacksonUtil.parseJSONStr2Map(req);
         Integer total = (Integer) stringObjectMap.get("count");
@@ -230,8 +247,12 @@ public class NewDatasetServiceImpl implements NewDatasetService {
         String topic_name = "ci4s_dataset";
         topic_name = StringUtils.isEmpty(datasetTagName) ? topic_name : topic_name + ",datatag_" + datasetTagName;
         topic_name = StringUtils.isEmpty(datasetTagName) ? topic_name : topic_name + ",datatype_" + datasetTypeName;
-
         String url = gitendpoint + "/api/projects.json?user_id=" + userId + "&page=" + pageRequest.getPageNumber() + "&limit=" + pageRequest.getPageSize() + "&sort_by=praises_count&topic_name=" + topic_name;
+        String name = dataset.getName();
+        if (StringUtils.isNotEmpty(name)){
+            url =url + "&search=" + name;
+        }
+
         String req = HttpUtils.sendGetWithToken(url, null, token);
         Map stringObjectMap = JacksonUtil.parseJSONStr2Map(req);
         Integer total = (Integer) stringObjectMap.get("total_count");
@@ -255,9 +276,9 @@ public class NewDatasetServiceImpl implements NewDatasetService {
             }
             version = (String) versionList.get(0).get("name");
         }
-        List> fileDetailsAfterGitPull = DVCUtils.getFileDetailsAfterGitPull(localPathlocal +id, repositoryName, version, "data",gitLinkUsername, gitLinkPassword);
+        List> fileDetailsAfterGitPull = DVCUtils.getFileDetailsAfterGitPull(localPathlocal +loginUser.getUsername()+"/datasets/" + id, repositoryName, version, "data",gitLinkUsername, gitLinkPassword);
         // 在localPathlocal+id+"/"+repositoryName目录下的dataset.yaml中取到元数据
-        Map stringObjectMap = YamlUtils.loadYamlFile(localPathlocal + id + "\\" + repositoryName + "\\" + "dataset.yaml");
+        Map stringObjectMap = YamlUtils.loadYamlFile(localPathlocal +loginUser.getUsername()+"/datasets/" + id + "/" + repositoryName + "/" + "dataset.yaml");
         NewDatasetVo newDatasetVo = ConvertUtil.convertMapToObject(stringObjectMap, NewDatasetVo.class);
         List versionVos = new ArrayList();
         if (fileDetailsAfterGitPull!=null&&fileDetailsAfterGitPull.size()>0){
@@ -354,7 +375,7 @@ public class NewDatasetServiceImpl implements NewDatasetService {
     }
 
     @Override
-    public ResponseEntity downloadAllDatasetFilesNew(String repositoryName, String version) throws Exception {
+    public ResponseEntity downloadAllDatasetFilesNew(String repositoryName,Integer id, String version) throws Exception {
         // 命令行操作 git clone 项目地址
         LoginUser loginUser = SecurityUtils.getLoginUser();
         String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername();
@@ -363,20 +384,18 @@ public class NewDatasetServiceImpl implements NewDatasetService {
         Jedis jedis = new Jedis(redisHost);
         String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
         Map userInfo = JsonUtils.jsonToMap(userReq);
-        Integer userId = (Integer) userInfo.get("user_id");
-        String projectUrl = gitendpoint + "/" +(String) userInfo.get("login") + "/" + repositoryName + ".git";
-        String localPath = localPathlocal+ loginUser.getUsername()+"/datasets/" +repositoryName;
-        File folder = new File(localPath);
-        if(folder.exists() && folder.isDirectory()){
-            //切换分支
-            DVCUtils.gitCheckoutBranch(localPath, version);
-            //pull
-            DVCUtils.gitPull(localPath,gitLinkUsername, gitLinkPassword);
-            //dvc pull
-            DVCUtils.dvcPull(localPath);
-        }else {
-            DVCUtils.gitClone(localPath, projectUrl, version, gitLinkUsername, gitLinkPassword);
-        }
+        String localPath = localPathlocal+ loginUser.getUsername()+"/datasets/"+id+"/"+repositoryName;
+        //不需要,没意义,我查看详情已经是最新的了
+//        if(folder.exists() && folder.isDirectory()){
+//            //切换分支
+//            DVCUtils.gitCheckoutBranch(localPath, version);
+//            //pull
+//            DVCUtils.gitPull(localPath,gitLinkUsername, gitLinkPassword);
+//            //dvc pull
+//            DVCUtils.dvcPull(localPath);
+//        }else {
+//            DVCUtils.gitClone(localPath, projectUrl, version, gitLinkUsername, gitLinkPassword);
+//        }
 
         // 打包 data 文件夹
         String dataFolderPath = localPath + "/data";
@@ -403,7 +422,7 @@ public class NewDatasetServiceImpl implements NewDatasetService {
         InputStreamResource resource = new InputStreamResource(new FileInputStream(zipFile));
 
         return ResponseEntity.ok()
-                .header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=data.zip")
+                .header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename="+repositoryName+"_"+version+"_data.zip")
                 .contentType(MediaType.APPLICATION_OCTET_STREAM)
                 .contentLength(zipFile.length())
                 .body(resource);
diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/NewDatasetVo.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/NewDatasetVo.java
index 6f182f0a..0382a8bc 100644
--- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/NewDatasetVo.java
+++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/NewDatasetVo.java
@@ -60,5 +60,7 @@ public class NewDatasetVo implements Serializable {
     private String owner;
     @ApiModelProperty(name = "dataset_source",value = "数据集来源")
     private String datasetSource;
+    @ApiModelProperty(name = "relative_paths",value = "文件相对路径")
+    private String relativePaths;
 
 }