新增存储路径

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/dataset/NewDatasetFromGitController.java

@@ -119,8 +119,8 @@ public class NewDatasetFromGitController {
      */
     @GetMapping("/downloadAllFiles")
     @ApiOperation(value = "下载同一版本下所有数据集，并打包")
-    public ResponseEntity<InputStreamResource> downloadAllDatasetFiles(@RequestParam("repository_name") String repositoryName, @RequestParam("version") String version) throws Exception {
-        return newDatasetService.downloadAllDatasetFilesNew(repositoryName, version);
+    public ResponseEntity<InputStreamResource> downloadAllDatasetFiles(@RequestParam("name") String name,  @RequestParam("id") Integer id, @RequestParam("version") String version) throws Exception {
+        return newDatasetService.downloadAllDatasetFilesNew(name,id, version);
     }
 
     /**

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/NewDatasetService.java

@@ -19,7 +19,7 @@ public interface NewDatasetService {
     String newCreateVersion(NewDatasetVo datasetVo) throws Exception;
     List<Map<String, String>> uploadDatasetlocal(MultipartFile[] files, String uuid) throws Exception;
     ResponseEntity<InputStreamResource> downloadDatasetlocal(String filePath) throws Exception;
-    ResponseEntity<InputStreamResource> downloadAllDatasetFilesNew(String repositoryName, String version) throws IOException, Exception;
+    ResponseEntity<InputStreamResource> downloadAllDatasetFilesNew(String repositoryName,Integer id, String version) throws IOException, Exception;
     Page<NewDatasetVo> newPersonalQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception;
     Page<NewDatasetVo>  newPubilcQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception;
 

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java

@@ -80,6 +80,9 @@ public class NewDatasetServiceImpl implements NewDatasetService {
         // 创建项目
         Map project = gitService.createProject(gitProjectVo);
         Integer gitlinIid = (Integer) project.get("id");
+        if (gitlinIid == null){
+            throw new Exception("创建数据集失败:"+project.get("message"));
+        }
         // 创建分支
         String branchName = datasetVo.getVersion();
         gitService.createBranch((String) userInfo.get("login"), repositoryName, branchName, "master");
@@ -92,7 +95,7 @@ public class NewDatasetServiceImpl implements NewDatasetService {
 
         // 得到用户操作的路径
         String url = datasetVo.getDatasetVersionVos().get(0).getUrl();
-        String localPath = localPathlocal + gitlinIid + "/" + datasetVo.getName();
+        String localPath = localPathlocal+ ci4sUsername+"/datasets/" + gitlinIid + "/" + datasetVo.getName();
         String sourcePath = url.substring(0, url.lastIndexOf("/"));
         // 命令行操作 git clone 项目地址
         DVCUtils.gitClone(localPath, projectUrl, branchName, gitLinkUsername, gitLinkPassword);
@@ -112,6 +115,8 @@ public class NewDatasetServiceImpl implements NewDatasetService {
         datasetVo.setIdentifier(repositoryName);
         datasetVo.setId(gitlinIid);
         datasetVo.setOwner((String) userInfo.get("login"));
+        datasetVo.setRelativePaths(ci4sUsername+"/" + gitlinIid + "/" + datasetVo.getName());
+
         if (StringUtils.isEmpty(datasetVo.getDatasetSource())){
             datasetVo.setDatasetSource("用户上传");
         }
@@ -147,10 +152,14 @@ public class NewDatasetServiceImpl implements NewDatasetService {
         String branchName = StringUtils.isEmpty(datasetVo.getVersion()) ? "master" : datasetVo.getVersion();
         String repositoryName = datasetVo.getIdentifier();
         String url = datasetVo.getDatasetVersionVos().get(0).getUrl();
-        String localPath = localPathlocal + datasetVo.getId() + "/" + datasetVo.getName();
+        String localPath = localPathlocal +  ci4sUsername +"/datasets/"+ datasetVo.getId() + "/" + datasetVo.getName();
         String sourcePath = url.substring(0, url.lastIndexOf("/"));
         String projectUrl = gitendpoint + "/" + (String) userInfo.get("login") + "/" + repositoryName + ".git";
 
+        //部分信息在前面的版本里面，从那边取过来
+        Map<String, Object> stringObjectMap = YamlUtils.loadYamlFile(localPath + "/" + "dataset.yaml");
+        NewDatasetVo newDatasetVo = ConvertUtil.convertMapToObject(stringObjectMap, NewDatasetVo.class);
+
         // 创建本地分支
         DVCUtils.createLocalBranchBasedOnMaster(localPath, branchName);
 
@@ -164,7 +173,6 @@ public class NewDatasetServiceImpl implements NewDatasetService {
         // 拼接生产的元数据后写入yaml文件
         datasetVo.setCreateBy(String.valueOf(StringUtils.isNotEmpty((String) userInfo.get("nickname")) ? userInfo.get("nickname") : userInfo.get("login")));
         datasetVo.setUpdateTime(DateUtils.getTime());
-        datasetVo.setVersionDesc(datasetVo.getDescription());
         datasetVo.setUsage("<pre><code>" +
                 "# 克隆数据集配置文件与存储参数到本地\n" +
                 "git clone -b " + branchName + " "+ projectUrl + "\n" +
@@ -177,6 +185,10 @@ public class NewDatasetServiceImpl implements NewDatasetService {
         if (StringUtils.isEmpty(datasetVo.getDatasetSource())){
             datasetVo.setDatasetSource("用户上传");
         }
+        datasetVo.setDescription(newDatasetVo.getDescription());
+        datasetVo.setDataTag(newDatasetVo.getDataTag());
+        datasetVo.setDataType(newDatasetVo.getDataType());
+        datasetVo.setRelativePaths(ci4sUsername + "/" + datasetVo.getId() + "/" + datasetVo.getName());
         YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo), localPath, "dataset");
         //dvc数据跟踪
         // 配置远程S3地址
@@ -208,6 +220,11 @@ public class NewDatasetServiceImpl implements NewDatasetService {
         topic_name = StringUtils.isEmpty(datasetTagName) ? topic_name : topic_name + ",datatag_" + datasetTagName;
         topic_name = StringUtils.isEmpty(datasetTagName) ? topic_name : topic_name + ",datatype_" + datasetTypeName;
         String url = gitendpoint + "/api/users/" + (String) userInfo.get("login") + "/projects.json?page=" + pageRequest.getPageNumber() + "&limit=" + pageRequest.getPageSize() + "&category=manage&topic_name=" + topic_name;
+        String name = dataset.getName();
+
+        if (StringUtils.isNotEmpty(name)){
+            url =url + "&search=" + name;
+        }
         String req = HttpUtils.sendGetWithToken(url, null, token);
         Map<String, Object> stringObjectMap = JacksonUtil.parseJSONStr2Map(req);
         Integer total = (Integer) stringObjectMap.get("count");
@@ -230,8 +247,12 @@ public class NewDatasetServiceImpl implements NewDatasetService {
         String topic_name = "ci4s_dataset";
         topic_name = StringUtils.isEmpty(datasetTagName) ? topic_name : topic_name + ",datatag_" + datasetTagName;
         topic_name = StringUtils.isEmpty(datasetTagName) ? topic_name : topic_name + ",datatype_" + datasetTypeName;
-
         String url = gitendpoint + "/api/projects.json?user_id=" + userId + "&page=" + pageRequest.getPageNumber() + "&limit=" + pageRequest.getPageSize() + "&sort_by=praises_count&topic_name=" + topic_name;
+        String name = dataset.getName();
+        if (StringUtils.isNotEmpty(name)){
+            url =url + "&search=" + name;
+        }
+
         String req = HttpUtils.sendGetWithToken(url, null, token);
         Map<String, Object> stringObjectMap = JacksonUtil.parseJSONStr2Map(req);
         Integer total = (Integer) stringObjectMap.get("total_count");
@@ -255,9 +276,9 @@ public class NewDatasetServiceImpl implements NewDatasetService {
             }
             version = (String) versionList.get(0).get("name");
         }
-        List<Map<String, Object>> fileDetailsAfterGitPull = DVCUtils.getFileDetailsAfterGitPull(localPathlocal +id, repositoryName, version, "data",gitLinkUsername, gitLinkPassword);
+        List<Map<String, Object>> fileDetailsAfterGitPull = DVCUtils.getFileDetailsAfterGitPull(localPathlocal +loginUser.getUsername()+"/datasets/" + id, repositoryName, version, "data",gitLinkUsername, gitLinkPassword);
         // 在localPathlocal+id+"/"+repositoryName目录下的dataset.yaml中取到元数据
-        Map<String, Object> stringObjectMap = YamlUtils.loadYamlFile(localPathlocal + id + "\\" + repositoryName + "\\" + "dataset.yaml");
+        Map<String, Object> stringObjectMap = YamlUtils.loadYamlFile(localPathlocal +loginUser.getUsername()+"/datasets/" + id + "/" + repositoryName + "/" + "dataset.yaml");
         NewDatasetVo newDatasetVo = ConvertUtil.convertMapToObject(stringObjectMap, NewDatasetVo.class);
         List<VersionVo> versionVos = new ArrayList<VersionVo>();
         if (fileDetailsAfterGitPull!=null&&fileDetailsAfterGitPull.size()>0){
@@ -354,7 +375,7 @@ public class NewDatasetServiceImpl implements NewDatasetService {
     }
 
     @Override
-    public ResponseEntity<InputStreamResource> downloadAllDatasetFilesNew(String repositoryName, String version) throws Exception {
+    public ResponseEntity<InputStreamResource> downloadAllDatasetFilesNew(String repositoryName,Integer id, String version) throws Exception {
         // 命令行操作 git clone 项目地址
         LoginUser loginUser = SecurityUtils.getLoginUser();
         String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername();
@@ -363,20 +384,18 @@ public class NewDatasetServiceImpl implements NewDatasetService {
         Jedis jedis = new Jedis(redisHost);
         String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
         Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
-        Integer userId = (Integer) userInfo.get("user_id");
-        String projectUrl = gitendpoint + "/" +(String) userInfo.get("login") + "/" + repositoryName + ".git";
-        String localPath = localPathlocal+ loginUser.getUsername()+"/datasets/" +repositoryName;
-        File folder = new File(localPath);
-        if(folder.exists() && folder.isDirectory()){
-            //切换分支
-            DVCUtils.gitCheckoutBranch(localPath, version);
-            //pull
-            DVCUtils.gitPull(localPath,gitLinkUsername, gitLinkPassword);
-            //dvc pull
-            DVCUtils.dvcPull(localPath);
-        }else {
-            DVCUtils.gitClone(localPath, projectUrl, version, gitLinkUsername, gitLinkPassword);
-        }
+        String localPath = localPathlocal+ loginUser.getUsername()+"/datasets/"+id+"/"+repositoryName;
+        //不需要，没意义，我查看详情已经是最新的了
+//        if(folder.exists() && folder.isDirectory()){
+//            //切换分支
+//            DVCUtils.gitCheckoutBranch(localPath, version);
+//            //pull
+//            DVCUtils.gitPull(localPath,gitLinkUsername, gitLinkPassword);
+//            //dvc pull
+//            DVCUtils.dvcPull(localPath);
+//        }else {
+//            DVCUtils.gitClone(localPath, projectUrl, version, gitLinkUsername, gitLinkPassword);
+//        }
 
         // 打包 data 文件夹
         String dataFolderPath = localPath + "/data";
@@ -403,7 +422,7 @@ public class NewDatasetServiceImpl implements NewDatasetService {
         InputStreamResource resource = new InputStreamResource(new FileInputStream(zipFile));
 
         return ResponseEntity.ok()
-                .header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=data.zip")
+                .header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename="+repositoryName+"_"+version+"_data.zip")
                 .contentType(MediaType.APPLICATION_OCTET_STREAM)
                 .contentLength(zipFile.length())
                 .body(resource);

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/NewDatasetVo.java

@@ -60,5 +60,7 @@ public class NewDatasetVo implements Serializable {
     private String owner;
     @ApiModelProperty(name = "dataset_source",value = "数据集来源")
     private String datasetSource;
+    @ApiModelProperty(name = "relative_paths",value = "文件相对路径")
+    private String relativePaths;
 
 }