update dataset version

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/dataset/NewDatasetFromGitController.java

@@ -2,10 +2,8 @@ package com.ruoyi.platform.controller.dataset;
 
 import com.ruoyi.common.core.web.domain.AjaxResult;
 import com.ruoyi.platform.domain.Dataset;
-import com.ruoyi.platform.service.DatasetService;
-import com.ruoyi.platform.vo.DatasetVo;
+import com.ruoyi.platform.service.NewDatasetService;
 import com.ruoyi.platform.vo.NewDatasetVo;
-import io.swagger.annotations.Api;
 import io.swagger.annotations.ApiOperation;
 import org.springframework.core.io.InputStreamResource;
 import org.springframework.data.domain.PageRequest;
@@ -22,7 +20,7 @@ public class NewDatasetFromGitController {
      * 服务对象
      */
     @Resource
-    private DatasetService datasetService;
+    private NewDatasetService newDatasetService;
 
 
 
@@ -35,7 +33,7 @@ public class NewDatasetFromGitController {
     @PostMapping("/addDatasetAndVersion")
     @ApiOperation("添加数据集和版本")
     public AjaxResult addDatasetAndVersion(@RequestBody NewDatasetVo datasetVo) throws Exception {
-        return AjaxResult.success(this.datasetService.newCreateDataset(datasetVo));
+        return AjaxResult.success(this.newDatasetService.newCreateDataset(datasetVo));
 
     }
 
@@ -49,7 +47,7 @@ public class NewDatasetFromGitController {
     @PostMapping("/addVersion")
     @ApiOperation("添加版本")
     public AjaxResult addVersion(@RequestBody NewDatasetVo datasetVo) throws Exception {
-        return AjaxResult.success(this.datasetService.newCreateVersion(datasetVo));
+        return AjaxResult.success(this.newDatasetService.newCreateVersion(datasetVo));
 
     }
 
@@ -62,39 +60,39 @@ public class NewDatasetFromGitController {
                                      @RequestParam(value = "data_tag", required = false) String dataTag) throws Exception {
         PageRequest pageRequest = PageRequest.of(page, size);
         if(isPublic){
-            return AjaxResult.success(this.datasetService.newPubilcQueryByPage(dataset, pageRequest));
+            return AjaxResult.success(this.newDatasetService.newPubilcQueryByPage(dataset, pageRequest));
         }else {
-            return AjaxResult.success(this.datasetService.newPersonalQueryByPage(dataset, pageRequest));
+            return AjaxResult.success(this.newDatasetService.newPersonalQueryByPage(dataset, pageRequest));
         }
     }
 
     @GetMapping("/getVersionList")
     @ApiOperation(value = "获取分支列表")
     public AjaxResult getVersionList(@RequestParam("identifier") String repo,@RequestParam("owner")String owner) throws Exception {
-        return AjaxResult.success(this.datasetService.getVersionList(repo,owner));
+        return AjaxResult.success(this.newDatasetService.getVersionList(repo,owner));
     }
 
     @GetMapping("/getDatasetDetail")
     @ApiOperation(value = "获取数据集详情")
     public AjaxResult getDatasetVersions(@RequestParam("name") String name,
-                                         @RequestParam("repo_id") Integer repoId,
+                                         @RequestParam("id") Integer id,
                                          @RequestParam(value ="owner",required = false)String owner,
-                                         @RequestParam(value = "identifier",required = false) String repo,
+                                         @RequestParam(value = "identifier",required = false) String identifier,
                                          @RequestParam(value = "version",required = false)String version) throws Exception {
-        return AjaxResult.success(this.datasetService.getNewDatasetDesc(repoId,name,repo,owner,version));
+        return AjaxResult.success(this.newDatasetService.getNewDatasetDesc(id,name,identifier,owner,version));
     }
 
     @DeleteMapping("/deleteDataset")
     @ApiOperation(value = "删除数据集")
     public AjaxResult deleteDataset(@RequestParam("identifier") String repo,@RequestParam("owner")String owner) throws Exception {
-        this.datasetService.deleteDatasetNew(repo,owner);
+        this.newDatasetService.deleteDatasetNew(repo,owner);
         return AjaxResult.success();
     }
 
     @DeleteMapping("/deleteDatasetVersion")
     @ApiOperation(value = "删除数据集版本")
     public AjaxResult deleteDatasetVersion(@RequestParam("identifier") String repo,@RequestParam("owner")String owner,@RequestParam("version")String version) throws Exception {
-        this.datasetService.deleteDatasetVersionNew(repo,owner,version);
+        this.newDatasetService.deleteDatasetVersionNew(repo,owner,version);
         return AjaxResult.success();
     }
 
@@ -110,7 +108,7 @@ public class NewDatasetFromGitController {
     @PostMapping("/upload")
     @ApiOperation(value = "上传数据集")
     public AjaxResult uploadDataset(@RequestParam("file") MultipartFile[] files, @RequestParam("uuid") String uuid) throws Exception {
-        return AjaxResult.success(this.datasetService.uploadDatasetlocal(files,uuid));
+        return AjaxResult.success(this.newDatasetService.uploadDatasetlocal(files,uuid));
     }
 
     /**
@@ -122,7 +120,7 @@ public class NewDatasetFromGitController {
     @GetMapping("/downloadAllFiles")
     @ApiOperation(value = "下载同一版本下所有数据集，并打包")
     public ResponseEntity<InputStreamResource> downloadAllDatasetFiles(@RequestParam("repository_name") String repositoryName, @RequestParam("version") String version) throws Exception {
-        return datasetService.downloadAllDatasetFilesNew(repositoryName, version);
+        return newDatasetService.downloadAllDatasetFilesNew(repositoryName, version);
     }
 
     /**
@@ -135,6 +133,6 @@ public class NewDatasetFromGitController {
     @GetMapping("/downloadSinggerFile")
     @ApiOperation(value = "下载单个数据集文件", notes = "根据数据集版本表id下载单个数据集文件")
     public ResponseEntity<InputStreamResource> downloadDataset(@RequestParam("url") String url) throws Exception {
-        return datasetService.downloadDatasetlocal(url);
+        return newDatasetService.downloadDatasetlocal(url);
     }
 }

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/jupyter/JupyterController.java

@@ -4,9 +4,11 @@ import com.ruoyi.common.core.web.controller.BaseController;
 import com.ruoyi.common.core.web.domain.AjaxResult;
 import com.ruoyi.common.core.web.domain.GenericsAjaxResult;
 import com.ruoyi.platform.domain.DevEnvironment;
-import com.ruoyi.platform.service.DatasetService;
 import com.ruoyi.platform.service.JupyterService;
-import com.ruoyi.platform.vo.*;
+import com.ruoyi.platform.service.NewDatasetService;
+import com.ruoyi.platform.vo.NewDatasetVo;
+import com.ruoyi.platform.vo.PodStatusVo;
+import com.ruoyi.platform.vo.VersionVo;
 import io.swagger.annotations.Api;
 import io.swagger.annotations.ApiOperation;
 import io.swagger.v3.oas.annotations.responses.ApiResponse;
@@ -27,7 +29,7 @@ public class JupyterController extends BaseController {
     @Resource
     private JupyterService jupyterService;
     @Resource
-    private DatasetService datasetService;
+    private NewDatasetService newDatasetService;
     @GetMapping(value = "/getURL")
     @ApiOperation("得到访问地址")
     public GenericsAjaxResult<String> getURL() throws IOException {
@@ -101,6 +103,6 @@ public class JupyterController extends BaseController {
         datasetVo.setVersionDesc("this is a test");
         datasetVersionVos.add(versionVo);
         datasetVo.setDatasetVersionVos(datasetVersionVos);
-        return AjaxResult.success(datasetService.newCreateDataset(datasetVo));
+        return AjaxResult.success(newDatasetService.newCreateDataset(datasetVo));
     }
 }

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/DatasetService.java

@@ -21,11 +21,7 @@ import java.util.concurrent.CompletableFuture;
  * @author Xidaray
  * @since 2023-11-28 11:51:22
  */
-public interface
-DatasetService {
-
-
-
+public interface DatasetService {
 
 
     /**
@@ -39,12 +35,12 @@ DatasetService {
     /**
      * 分页查询
      *
-     * @param dataset 筛选条件
-     * @param pageRequest      分页对象
+     * @param dataset     筛选条件
+     * @param pageRequest 分页对象
      * @return 查询结果
      */
     Page<Dataset> queryByPage(Dataset dataset, PageRequest pageRequest);
-    
+
     /**
      * 新增数据
      *
@@ -54,7 +50,6 @@ DatasetService {
     Dataset insert(Dataset dataset) throws Exception;
 
 
-
     /**
      * 修改数据
      *
@@ -82,30 +77,11 @@ DatasetService {
     List<String> getDatasetVersions(Integer datasetId) throws Exception;
 
     String insertDatasetAndVersion(DatasetVo datasetVo) throws Exception;
+
     void checkDeclaredName(Dataset insert) throws Exception;
 
     ResponseEntity<InputStreamResource> downloadAllDatasetFiles(Integer datasetId, String version) throws Exception;
 
     List<Map<String, String>> exportDataset(String path, String uuid) throws Exception;
 
-
-
-
-
-    CompletableFuture<String> newCreateDataset(NewDatasetVo datasetVo) throws Exception;
-    CompletableFuture<String> newCreateVersion(NewDatasetVo datasetVo);
-
-    List<Map<String, String>> uploadDatasetlocal(MultipartFile[] files, String uuid) throws Exception;
-    ResponseEntity<InputStreamResource> downloadDatasetlocal(String filePath) throws Exception;
-    ResponseEntity<InputStreamResource> downloadAllDatasetFilesNew(String repositoryName, String version) throws IOException, Exception;
-    Page<NewDatasetVo> newPersonalQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception;
-    Page<NewDatasetVo>  newPubilcQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception;
-
-    NewDatasetVo getNewDatasetDesc(Integer repoId,String RepositoryName,String repo, String owner, String version)throws Exception;
-
-    List<Map<String, Object>> getVersionList(String repo, String owner)  throws Exception;
-
-     void deleteDatasetNew(String repo, String owner) throws Exception;
-
-    void deleteDatasetVersionNew(String repo, String owner, String version) throws Exception;
-}
+}

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/NewDatasetService.java

@@ -0,0 +1,35 @@
+package com.ruoyi.platform.service;
+
+import com.ruoyi.platform.domain.Dataset;
+import com.ruoyi.platform.vo.NewDatasetVo;
+import org.springframework.core.io.InputStreamResource;
+import org.springframework.data.domain.Page;
+import org.springframework.data.domain.PageRequest;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.multipart.MultipartFile;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+
+public interface NewDatasetService {
+
+
+    String newCreateDataset(NewDatasetVo datasetVo) throws Exception;
+    CompletableFuture<String> newCreateVersion(NewDatasetVo datasetVo)throws Exception;
+
+    List<Map<String, String>> uploadDatasetlocal(MultipartFile[] files, String uuid) throws Exception;
+    ResponseEntity<InputStreamResource> downloadDatasetlocal(String filePath) throws Exception;
+    ResponseEntity<InputStreamResource> downloadAllDatasetFilesNew(String repositoryName, String version) throws IOException, Exception;
+    Page<NewDatasetVo> newPersonalQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception;
+    Page<NewDatasetVo>  newPubilcQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception;
+
+    NewDatasetVo getNewDatasetDesc(Integer repoId,String RepositoryName,String repo, String owner, String version)throws Exception;
+
+    List<Map<String, Object>> getVersionList(String repo, String owner)  throws Exception;
+
+    void deleteDatasetNew(String repo, String owner) throws Exception;
+
+    void deleteDatasetVersionNew(String repo, String owner, String version) throws Exception;
+}

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/DatasetServiceImpl.java

@@ -1,6 +1,5 @@
 package com.ruoyi.platform.service.impl;
 
-import com.ruoyi.common.core.utils.DateUtils;
 import com.ruoyi.common.security.utils.SecurityUtils;
 import com.ruoyi.platform.annotations.CheckDuplicate;
 import com.ruoyi.platform.domain.AssetIcon;
@@ -9,14 +8,13 @@ import com.ruoyi.platform.domain.DatasetVersion;
 import com.ruoyi.platform.mapper.DatasetDao;
 import com.ruoyi.platform.mapper.DatasetVersionDao;
 import com.ruoyi.platform.service.*;
-import com.ruoyi.platform.utils.*;
+import com.ruoyi.platform.utils.BeansUtils;
+import com.ruoyi.platform.utils.FileUtil;
+import com.ruoyi.platform.utils.MinioUtil;
 import com.ruoyi.platform.vo.DatasetVo;
-import com.ruoyi.platform.vo.GitProjectVo;
-import com.ruoyi.platform.vo.NewDatasetVo;
 import com.ruoyi.platform.vo.VersionVo;
 import com.ruoyi.system.api.model.LoginUser;
 import io.minio.messages.Item;
-import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.core.io.InputStreamResource;
@@ -30,17 +28,13 @@ import org.springframework.http.ResponseEntity;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
 import org.springframework.web.multipart.MultipartFile;
-import redis.clients.jedis.Jedis;
 
 import javax.annotation.Resource;
-import java.io.*;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
 import java.lang.reflect.Field;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
 import java.text.SimpleDateFormat;
 import java.util.*;
-import java.util.concurrent.CompletableFuture;
 import java.util.stream.Collectors;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipOutputStream;
@@ -68,12 +62,6 @@ public class DatasetServiceImpl implements DatasetService {
     @Resource
     private MinioService minioService;
 
-    @Resource
-    private GitService gitService;
-
-    @Resource
-    private DvcService dvcService;
-
     // 固定存储桶名
     @Value("${minio.dataReleaseBucketName}")
     private String bucketName;
@@ -81,19 +69,6 @@ public class DatasetServiceImpl implements DatasetService {
     @Resource
     private  MinioUtil minioUtil;
 
-    @Value("${spring.redis.host}")
-    private String redisHost;
-    @Value("${minio.accessKey}")
-    String accessKeyId;
-    @Value("${minio.secretKey}")
-    String secretAccessKey;
-    @Value("${minio.endpoint}")
-    String endpoint;
-    @Value("${git.endpoint}")
-    String gitendpoint;
-    @Value("${git.localPath}")
-    String localPathlocal;
-
     /**
      * 通过ID查询单条数据
      *
@@ -457,447 +432,4 @@ public class DatasetServiceImpl implements DatasetService {
         }
         return results;
     }
-
-
-
-    @Override
-    public CompletableFuture<String> newCreateDataset(NewDatasetVo datasetVo) {
-        return CompletableFuture.supplyAsync(() -> {
-            try {
-                checkoutToken();
-                Jedis jedis = new Jedis(redisHost);
-                LoginUser loginUser = SecurityUtils.getLoginUser();
-                String ci4sUsername = loginUser.getUsername();
-                String token = jedis.get(ci4sUsername+"_gitToken");
-                String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername();
-                String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword();
-                String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
-                Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
-                Integer userId = (Integer) userInfo.get("user_id");
-                // 拼接project
-                String repositoryName = ci4sUsername + "_dataset_" + DateUtils.dateTimeNow();
-                GitProjectVo gitProjectVo = new GitProjectVo();
-                gitProjectVo.setRepositoryName(repositoryName);
-                gitProjectVo.setName(datasetVo.getName());
-                gitProjectVo.setDescription(datasetVo.getDescription());
-                gitProjectVo.setPrivate(!datasetVo.getIsPublic());
-                gitProjectVo.setUserId(userId);
-                // 创建项目
-                Map project = gitService.createProject(token, gitProjectVo);
-                Integer repoId = (Integer) project.get("id");
-                // 创建分支
-                String branchName = datasetVo.getVersion();
-                gitService.createBranch(token, (String) userInfo.get("login"), repositoryName, branchName, "master");
-                // 定义标签 标签1：ci4s_dataset 标签2：DataTag 标签3：DataType
-                gitService.createTopic(token, repoId, "ci4s_dataset");
-                gitService.createTopic(token, repoId, "DataTag_" + datasetVo.getDataTag());
-                gitService.createTopic(token, repoId, "DataType_" + datasetVo.getDataType());
-                // 得到项目地址
-                String projectUrl = gitendpoint + "/" +(String) userInfo.get("login") + "/" + repositoryName + ".git";
-
-                // 得到用户操作的路径
-                String url = datasetVo.getDatasetVersionVos().get(0).getUrl();
-                String localPath = localPathlocal + "/" + repoId+"/"+datasetVo.getName();
-                String sourcePath = url.substring(0, url.lastIndexOf("/"));
-                // 命令行操作 git clone 项目地址
-                DVCUtils.gitClone(localPath, projectUrl, branchName, gitLinkUsername, gitLinkPassword);
-                String s3Path = "management-platform-files/" + ci4sUsername + "/datasets/"+repoId+"/"+  repositoryName + "/" + branchName;
-
-                DVCUtils.moveFiles(sourcePath, localPath);
-                //拼接生产的元数据后写入yaml文件
-                datasetVo.setCreateBy(String.valueOf(StringUtils.isNotEmpty((String)userInfo.get("nickname"))? userInfo.get("nickname") : userInfo.get("login")));
-                datasetVo.setUpdateTime(DateUtils.getNowDate());
-                datasetVo.setVersionDesc(datasetVo.getDescription());
-                datasetVo.setUsage("```bash\n" +
-                        "# 克隆数据集配置文件与存储参数到本地\n" +
-                        "git clone -b " + branchName + " " + projectUrl + "\n" +
-                        "# 远程拉取配置文件\n" +
-                        "dvc pull\n" +
-                        "```");
-                YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo),localPath, "dataset");
-                // dvc init 初始化
-                DVCUtils.dvcInit(localPath);
-                // 配置远程S3地址
-                DVCUtils.dvcRemoteAdd(localPath, s3Path);
-                DVCUtils.dvcConfigS3Credentials(localPath, endpoint);
-                DVCUtils.dvcConfigS3Credentials2(localPath, accessKeyId);
-                DVCUtils.dvcConfigS3Credentials3(localPath, secretAccessKey);
-                // dvc 跟踪
-                DVCUtils.dvcAdd(localPath, "data");
-                // git commit
-                DVCUtils.gitAdd(localPath, ".");
-                DVCUtils.gitCommit(localPath, "commit from ci4s with " + loginUser.getUsername());
-                DVCUtils.gitPush(localPath, gitLinkUsername, gitLinkPassword);
-                // dvc push 到远程S3
-                DVCUtils.dvcPush(localPath);
-                return "新增数据集成功";
-            } catch (Exception e) {
-                throw new RuntimeException(e);
-            }
-        });
-    }
-
-    @Override
-    public CompletableFuture<String> newCreateVersion(NewDatasetVo datasetVo) {
-        return CompletableFuture.supplyAsync(() -> {
-            try {
-                checkoutToken();
-                Jedis jedis = new Jedis(redisHost);
-                LoginUser loginUser = SecurityUtils.getLoginUser();
-                String ci4sUsername = loginUser.getUsername();
-                String token = jedis.get(ci4sUsername+"_gitToken");
-                String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername();
-                String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword();
-                String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
-                Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
-                // 创建分支
-                String branchName = StringUtils.isEmpty(datasetVo.getVersion())? "master" : datasetVo.getVersion();
-                String repositoryName = datasetVo.getIdentifier();
-                if (StringUtils.equals(branchName, "master")) {
-                    gitService.createBranch(token, (String) userInfo.get("login"), repositoryName, branchName, "master");
-                }
-                // 得到项目地址
-                String projectUrl = gitendpoint + "/" +(String) userInfo.get("login") + "/" + repositoryName + ".git";
-
-                // 得到用户操作的路径
-                String url = datasetVo.getDatasetVersionVos().get(0).getUrl();
-                String localPath = localPathlocal+ loginUser.getUsername()+"/datasets/"+ datasetVo.getName();
-                String sourcePath = url.substring(0, url.lastIndexOf("/"));
-                // 命令行操作 git clone 项目地址
-                if(FileUtil.checkDirectoryExists(localPath)){
-                    DVCUtils.gitFetch(localPath,gitLinkUsername, gitLinkPassword);
-                    DVCUtils.gitCheckoutBranch(localPath,branchName);
-                }else {
-                    DVCUtils.gitClone(localPath, projectUrl, branchName, gitLinkUsername, gitLinkPassword);
-                }
-
-                String s3Path = "management-platform-files/" + ci4sUsername + "/datasets/"+ datasetVo.getRepoId()+"/"+  repositoryName + "/" + branchName;
-
-                DVCUtils.moveFiles(sourcePath, localPath);
-                //拼接生产的元数据后写入yaml文件
-                datasetVo.setCreateBy(String.valueOf(StringUtils.isNotEmpty((String)userInfo.get("nickname"))? userInfo.get("nickname") : userInfo.get("login")));
-                datasetVo.setUpdateTime(DateUtils.getNowDate());
-                datasetVo.setVersionDesc(datasetVo.getDescription());
-                datasetVo.setUsage("```bash\n" +
-                        "# 克隆数据集配置文件与存储参数到本地\n" +
-                        "git clone -b " + branchName + " " + projectUrl + "\n" +
-                        "# 远程拉取配置文件\n" +
-                        "dvc pull\n" +
-                        "```");
-                YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo),localPath, "dataset");
-
-                // dvc init 初始化
-                DVCUtils.dvcInit(localPath);
-                // 配置远程S3地址
-                DVCUtils.dvcRemoteAdd(localPath, s3Path);
-                DVCUtils.dvcConfigS3Credentials(localPath, endpoint);
-                DVCUtils.dvcConfigS3Credentials2(localPath, accessKeyId);
-                DVCUtils.dvcConfigS3Credentials3(localPath, secretAccessKey);
-                // dvc 跟踪
-                DVCUtils.dvcAdd(localPath, "data");
-                // git commit
-                DVCUtils.gitAdd(localPath, ".");
-                DVCUtils.gitCommit(localPath, "commit from ci4s with " + loginUser.getUsername());
-                DVCUtils.gitPush(localPath, gitLinkUsername, gitLinkPassword);
-                // dvc push 到远程S3
-                DVCUtils.dvcPush(localPath);
-                return "新增数据集成功";
-            } catch (Exception e) {
-                throw new RuntimeException(e);
-            }
-        });
-    }
-
-
-    @Override
-    public Page<NewDatasetVo> newPersonalQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception {
-        checkoutToken();
-        Jedis jedis = new Jedis(redisHost);
-        LoginUser loginUser = SecurityUtils.getLoginUser();
-        String ci4sUsername = loginUser.getUsername();
-        String token = jedis.get(ci4sUsername+"_gitToken");
-        String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
-        Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
-        //拼接查询url
-        String datasetTagName = dataset.getDatasetTagName();
-        String datasetTypeName = dataset.getDatasetTypeName();
-        String topic_name = "ci4s_dataset";
-        topic_name =StringUtils.isEmpty(datasetTagName)?topic_name : topic_name+",datatag_" + datasetTagName;
-        topic_name =StringUtils.isEmpty(datasetTagName)?topic_name : topic_name+",datatype_" + datasetTypeName;
-        String url = gitendpoint + "/api/users/"+(String) userInfo.get("login")+"/projects.json?page="+pageRequest.getPageNumber()+"&limit="+pageRequest.getPageSize()+"&category=manage&topic_name="+topic_name;
-        String req = HttpUtils.sendGetWithToken(url,null,token);
-        Map<String, Object> stringObjectMap = JacksonUtil.parseJSONStr2Map(req);
-        Integer total = (Integer) stringObjectMap.get("count");
-        List<Map<String, Object>> projects = (List<Map<String, Object>>) stringObjectMap.get("projects");
-        return new PageImpl<>(convert(projects), pageRequest, total);
-    }
-
-    @Override
-    public Page<NewDatasetVo> newPubilcQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception {
-        checkoutToken();
-        Jedis jedis = new Jedis(redisHost);
-        LoginUser loginUser = SecurityUtils.getLoginUser();
-        String ci4sUsername = loginUser.getUsername();
-        String token = jedis.get(ci4sUsername+"_gitToken");
-        String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
-        Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
-        Integer userId = (Integer) userInfo.get("user_id");
-        //拼接查询url
-        String datasetTagName = dataset.getDatasetTagName();
-        String datasetTypeName = dataset.getDatasetTypeName();
-        String topic_name = "ci4s_dataset";
-        topic_name =StringUtils.isEmpty(datasetTagName)?topic_name : topic_name+",datatag_" + datasetTagName;
-        topic_name =StringUtils.isEmpty(datasetTagName)?topic_name : topic_name+",datatype_" + datasetTypeName;
-
-        String url = gitendpoint + "/api/projects.json?user_id="+userId+"&page="+pageRequest.getPageNumber()+"&limit="+pageRequest.getPageSize()+"&sort_by=praises_count&topic_name="+topic_name;
-        String req = HttpUtils.sendGetWithToken(url,null,token);
-        Map<String, Object> stringObjectMap = JacksonUtil.parseJSONStr2Map(req);
-        Integer total = (Integer) stringObjectMap.get("total_count");
-        List<Map<String, Object>> projects = (List<Map<String, Object>>) stringObjectMap.get("projects");
-        if (projects == null){
-            return new PageImpl<>(new ArrayList<>(), pageRequest, 0);
-        }
-        return new PageImpl<>(convert(projects), pageRequest, total);
-    }
-
-    @Override
-    public NewDatasetVo getNewDatasetDesc(Integer repoId,String repositoryName,String repo, String owner,String version) throws Exception{
-        LoginUser loginUser = SecurityUtils.getLoginUser();
-        String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername();
-        String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword();
-        // cd到 localPathlocal/repoId/下面还有一个文件夹，然后做git pull操作，然后读取里面的文件列表，列出每个文件的大小和名称，封装成MAP
-        if (StringUtils.isEmpty(version)){
-            List<Map<String, Object>> versionList = this.getVersionList(repo, owner);
-            if (versionList.size() == 0){
-                throw new Exception("数据集文件不存在");
-            }
-            version = (String) versionList.get(0).get("name");
-        }
-
-        List<Map<String, Object>> fileDetailsAfterGitPull = DVCUtils.getFileDetailsAfterGitPull(localPathlocal+repoId, repositoryName, version,gitLinkUsername, gitLinkPassword);
-        //在localPathlocal+repoId+"/"+repositoryName目录下的dataset.yaml中取到元数据
-        Map<String, Object> stringObjectMap = YamlUtils.loadYamlFile(localPathlocal + repoId + "/" + repositoryName + "/dataset.yaml");
-        NewDatasetVo newDatasetVo = ConvertUtil.convertMapToObject(stringObjectMap, NewDatasetVo.class);
-        List<VersionVo> versionVos = new ArrayList<VersionVo>();
-        if (fileDetailsAfterGitPull!=null&&fileDetailsAfterGitPull.size()>0){
-            for(Map<String, Object> fileDetail : fileDetailsAfterGitPull){
-                VersionVo versionVo = new VersionVo();
-                versionVo.setUrl((String) fileDetail.get("filePath"));
-                versionVo.setFileName((String) fileDetail.get("fileName"));
-                long size = (long) fileDetail.get("size");
-                versionVo.setFileSize(FileUtil.formatFileSize(size));
-                versionVos.add(versionVo);
-            }
-        }
-        newDatasetVo.setDatasetVersionVos(versionVos);
-        return newDatasetVo;
-    }
-
-    @Override
-    public List<Map<String, Object>> getVersionList(String repo, String owner) throws Exception {
-        checkoutToken();
-        Jedis jedis = new Jedis(redisHost);
-        LoginUser loginUser = SecurityUtils.getLoginUser();
-        String ci4sUsername = loginUser.getUsername();
-        String token = jedis.get(ci4sUsername+"_gitToken");
-        List<Map<String, Object>> brancheList = gitService.getBrancheList(token, owner, repo);
-        return brancheList.stream()
-                .filter(branch -> "master".equals(branch.get("name")))
-                .collect(Collectors.toList());
-    }
-
-    @Override
-    public void deleteDatasetNew(String repo, String owner) throws Exception {
-        checkoutToken();
-        Jedis jedis = new Jedis(redisHost);
-        LoginUser loginUser = SecurityUtils.getLoginUser();
-        String ci4sUsername = loginUser.getUsername();
-        String token = jedis.get(ci4sUsername+"_gitToken");
-        gitService.deleteProject(token, owner, repo);
-    }
-
-    @Override
-    public void deleteDatasetVersionNew(String repo, String owner, String version) throws Exception {
-        checkoutToken();
-        Jedis jedis = new Jedis(redisHost);
-        LoginUser loginUser = SecurityUtils.getLoginUser();
-        String ci4sUsername = loginUser.getUsername();
-        String token = jedis.get(ci4sUsername+"_gitToken");
-        gitService.deleteBranch(token, owner, repo, version);
-    }
-
-    @Override
-    public List<Map<String, String>> uploadDatasetlocal(MultipartFile[] files, String uuid) throws Exception {
-        List<Map<String, String>> results = new ArrayList<>();
-
-        for (MultipartFile file:files){
-            // 构建objectName
-            String username = SecurityUtils.getLoginUser().getUsername();
-            String fileName = file.getOriginalFilename();
-            String path = "/temp/"+ username +"/datasets/"+ uuid + "/"+"/data/" + fileName;
-            long sizeInBytes = file.getSize();
-            String formattedSize = FileUtil.formatFileSize(sizeInBytes);
-            File targetFile = new File(path, file.getOriginalFilename());
-            // 确保目录存在
-            targetFile.getParentFile().mkdirs();
-            // 保存文件到目标路径
-            FileUtils.copyInputStreamToFile(file.getInputStream(), targetFile);
-            // 返回上传文件的路径
-            String absolutePath = targetFile.getAbsolutePath();
-            Map<String, String> result = new HashMap<>();
-            result.put("fileName", fileName);
-            result.put("url", absolutePath); // objectName根据实际情况定义
-            result.put("fileSize", formattedSize);
-            results.add(result);
-        }
-        return results;
-    }
-
-    @Override
-    public ResponseEntity<InputStreamResource> downloadDatasetlocal(String filePath) throws Exception {
-        File file = new File(filePath);
-
-        if (!file.exists()) {
-            throw new FileNotFoundException("File not found: " + filePath);
-        }
-
-        InputStreamResource resource = new InputStreamResource(new FileInputStream(file));
-
-        HttpHeaders headers = new HttpHeaders();
-        headers.add(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=" + file.getName());
-        headers.add(HttpHeaders.CONTENT_LENGTH, String.valueOf(file.length()));
-        headers.add(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_OCTET_STREAM_VALUE);
-
-        return ResponseEntity.ok()
-                .headers(headers)
-                .contentLength(file.length())
-                .contentType(MediaType.APPLICATION_OCTET_STREAM)
-                .body(resource);
-    }
-
-    @Override
-    public ResponseEntity<InputStreamResource> downloadAllDatasetFilesNew(String repositoryName, String version) throws Exception {
-        // 命令行操作 git clone 项目地址
-        checkoutToken();
-        LoginUser loginUser = SecurityUtils.getLoginUser();
-        String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername();
-        String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword();
-        String ci4sUsername = loginUser.getUsername();
-        Jedis jedis = new Jedis(redisHost);
-        String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
-        Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
-        Integer userId = (Integer) userInfo.get("user_id");
-        String projectUrl = gitendpoint + "/" +(String) userInfo.get("login") + "/" + repositoryName + ".git";
-        String localPath = localPathlocal+ loginUser.getUsername()+"/datasets/" +repositoryName;
-        File folder = new File(localPath);
-        if(folder.exists() && folder.isDirectory()){
-            //切换分支
-            DVCUtils.gitCheckoutBranch(localPath, version);
-            //pull
-            DVCUtils.gitPull(localPath,gitLinkUsername, gitLinkPassword);
-            //dvc pull
-            DVCUtils.dvcPull(localPath);
-        }else {
-            DVCUtils.gitClone(localPath, projectUrl, version, gitLinkUsername, gitLinkPassword);
-        }
-
-        // 打包 data 文件夹
-        String dataFolderPath = localPath + "/data";
-        String zipFilePath = localPath + "/data.zip";
-        try (FileOutputStream fos = new FileOutputStream(zipFilePath);
-             ZipOutputStream zos = new ZipOutputStream(fos)) {
-            Path sourcePath = Paths.get(dataFolderPath);
-            Files.walk(sourcePath).forEach(path -> {
-                if (!Files.isDirectory(path)) {
-                    ZipEntry zipEntry = new ZipEntry(sourcePath.relativize(path).toString());
-                    try {
-                        zos.putNextEntry(zipEntry);
-                        Files.copy(path, zos);
-                        zos.closeEntry();
-                    } catch (IOException e) {
-                        throw new RuntimeException("Error while zipping: " + path, e);
-                    }
-                }
-            });
-        }
-
-        // 返回压缩文件的输入流
-        File zipFile = new File(zipFilePath);
-        InputStreamResource resource = new InputStreamResource(new FileInputStream(zipFile));
-
-        return ResponseEntity.ok()
-                .header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=data.zip")
-                .contentType(MediaType.APPLICATION_OCTET_STREAM)
-                .contentLength(zipFile.length())
-                .body(resource);
-
-    }
-
-
-
-    public List<NewDatasetVo> convert(List<Map<String, Object>> lst) {
-        if (lst != null && lst.size() > 0) {
-            List<NewDatasetVo> newDatasetVos = ConvertUtil.convertListMapToObjectList(lst, NewDatasetVo.class);
-
-            for (NewDatasetVo newDatasetVo : newDatasetVos) {
-                Map<String, Object> map = lst.stream()
-                        .filter(m -> m.get("repo_id").equals(newDatasetVo.getRepoId()))
-                        .findFirst()
-                        .orElse(null);
-
-                if (map != null) {
-                    List<Map<String, Object>> topics = (List<Map<String, Object>>) map.get("topics");
-                    if (topics != null) {
-                        topics.forEach(topic -> {
-                            String name = (String) topic.get("name");
-                            if (name != null) {
-                                if (name.startsWith("datatag_")) {
-                                    newDatasetVo.setDataTag(name.substring("datatag_".length()));
-                                } else if (name.startsWith("datatype_")) {
-                                    newDatasetVo.setDataType(name.substring("datatype_".length()));
-                                }
-                            }
-                        });
-                    }
-
-                    Map<String, Object> author = (Map<String, Object>) map.get("author");
-                    newDatasetVo.setCreateBy((String) author.get("name"));
-                    newDatasetVo.setOwner((String) author.get("login"));
-                }
-            }
-
-            return newDatasetVos;
-        }
-        return new ArrayList<>();
-    }
-
-    private String checkoutToken(){
-        Jedis jedis = new Jedis(redisHost);
-        LoginUser loginUser = SecurityUtils.getLoginUser();
-        String ci4sUsername = loginUser.getUsername();
-        String token = jedis.get(ci4sUsername+"_gitToken");
-        String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername();
-        String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword();
-
-        if(StringUtils.isEmpty(token)){
-            gitService.login(gitLinkUsername, gitLinkPassword);
-            token = jedis.get(ci4sUsername+"_gitToken");
-        }else {
-            try {
-                Map userInfo = gitService.getUserInfo(token);
-                if (userInfo == null ||(userInfo.get("status")!=null&&401==(Integer) userInfo.get("status"))){
-                    gitService.login(gitLinkUsername, gitLinkPassword);
-                    token = jedis.get(ci4sUsername+"_gitToken");
-                }
-            } catch (Exception e) {
-                throw new RuntimeException(e);
-            }
-
-        }
-        return token;
-
-    }
-
-
-
 }

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/GitServiceImpl.java

@@ -122,6 +122,9 @@ public class GitServiceImpl implements GitService {
     @Override
     public Map getUserInfo(String token) throws Exception {
         String userReq = HttpUtils.sendGetWithToken("https://www.gitlink.org.cn/api/users/get_user_info.json",null, token);
+        if (StringUtils.isEmpty(userReq)){
+            return null;
+        }
         Map<String, Object> runResMap = JsonUtils.jsonToMap(userReq);
         return  runResMap;
     }

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java

@@ -0,0 +1,501 @@
+package com.ruoyi.platform.service.impl;
+
+import com.ruoyi.common.core.utils.DateUtils;
+import com.ruoyi.common.security.utils.SecurityUtils;
+import com.ruoyi.platform.domain.Dataset;
+import com.ruoyi.platform.service.DvcService;
+import com.ruoyi.platform.service.GitService;
+import com.ruoyi.platform.service.NewDatasetService;
+import com.ruoyi.platform.utils.*;
+import com.ruoyi.platform.vo.GitProjectVo;
+import com.ruoyi.platform.vo.NewDatasetVo;
+import com.ruoyi.platform.vo.VersionVo;
+import com.ruoyi.system.api.model.LoginUser;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.core.io.InputStreamResource;
+import org.springframework.data.domain.Page;
+import org.springframework.data.domain.PageImpl;
+import org.springframework.data.domain.PageRequest;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.MediaType;
+import org.springframework.http.ResponseEntity;
+import org.springframework.stereotype.Service;
+import org.springframework.web.multipart.MultipartFile;
+import redis.clients.jedis.Jedis;
+
+import javax.annotation.Resource;
+import java.io.*;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.stream.Collectors;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
+
+@Service
+public class NewDatasetServiceImpl implements NewDatasetService {
+
+    @Resource
+    private GitService gitService;
+
+    @Resource
+    private DvcService dvcService;
+    @Value("${spring.redis.host}")
+    private String redisHost;
+    @Value("${minio.accessKey}")
+    String accessKeyId;
+    @Value("${minio.secretKey}")
+    String secretAccessKey;
+    @Value("${minio.endpoint}")
+    String endpoint;
+    @Value("${git.endpoint}")
+    String gitendpoint;
+    @Value("${git.localPath}")
+    String localPathlocal;
+
+    @Override
+    public String newCreateDataset(NewDatasetVo datasetVo) throws Exception {
+        checkoutToken();
+        Jedis jedis = new Jedis(redisHost);
+        LoginUser loginUser = SecurityUtils.getLoginUser();
+        String ci4sUsername = loginUser.getUsername();
+        String token = jedis.get(ci4sUsername + "_gitToken");
+        String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername();
+        String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword();
+        String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
+        Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
+        Integer userId = (Integer) userInfo.get("user_id");
+        // 拼接project
+        String repositoryName = ci4sUsername + "_dataset_" + DateUtils.dateTimeNow();
+        GitProjectVo gitProjectVo = new GitProjectVo();
+        gitProjectVo.setRepositoryName(repositoryName);
+        gitProjectVo.setName(datasetVo.getName());
+        gitProjectVo.setDescription(datasetVo.getDescription());
+        gitProjectVo.setPrivate(!datasetVo.getIsPublic());
+        gitProjectVo.setUserId(userId);
+        // 创建项目
+        Map project = gitService.createProject(token, gitProjectVo);
+        Integer gitlinIid = (Integer) project.get("id");
+        // 创建分支
+        String branchName = datasetVo.getVersion();
+        gitService.createBranch(token, (String) userInfo.get("login"), repositoryName, branchName, "master");
+        // 定义标签 标签1：ci4s_dataset 标签2：DataTag 标签3：DataType
+        gitService.createTopic(token, gitlinIid, "ci4s_dataset");
+        gitService.createTopic(token, gitlinIid, "DataTag_" + datasetVo.getDataTag());
+        gitService.createTopic(token, gitlinIid, "DataType_" + datasetVo.getDataType());
+        // 得到项目地址
+        String projectUrl = gitendpoint + "/" + (String) userInfo.get("login") + "/"+ repositoryName + ".git";
+
+        // 得到用户操作的路径
+        String url = datasetVo.getDatasetVersionVos().get(0).getUrl();
+        String localPath = localPathlocal + gitlinIid + "/" + datasetVo.getName();
+        String sourcePath = url.substring(0, url.lastIndexOf("/"));
+        // 命令行操作 git clone 项目地址
+        DVCUtils.gitClone(localPath, projectUrl, branchName, gitLinkUsername, gitLinkPassword);
+        String s3Path = "management-platform-files" + "/" + ci4sUsername + "/" + "datasets" + "/" + gitlinIid + "/" + repositoryName + "/" + branchName;
+
+        DVCUtils.moveFiles(sourcePath, localPath);
+        // 拼接生产的元数据后写入yaml文件
+        datasetVo.setCreateBy(String.valueOf(StringUtils.isNotEmpty((String) userInfo.get("nickname")) ? userInfo.get("nickname") : userInfo.get("login")));
+        datasetVo.setUpdateTime(DateUtils.getTime());
+        datasetVo.setVersionDesc(datasetVo.getDescription());
+        datasetVo.setUsage("<pre><code>\n" +
+                "# 克隆数据集配置文件与存储参数到本地\n" +
+                "git clone -b " + branchName + " "+ projectUrl + "\n" +
+                "# 远程拉取配置文件\n" +
+                "dvc pull\n" +
+                "</code></pre>\n");
+        datasetVo.setIdentifier(repositoryName);
+        datasetVo.setId(gitlinIid);
+        datasetVo.setOwner((String) userInfo.get("login"));
+
+        YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo), localPath, "dataset");
+        // dvc init 初始化
+        DVCUtils.dvcInit(localPath);
+        // 配置远程S3地址
+        DVCUtils.dvcRemoteAdd(localPath, s3Path);
+        DVCUtils.dvcConfigS3Credentials(localPath, endpoint);
+        DVCUtils.dvcConfigS3Credentials2(localPath, accessKeyId);
+        DVCUtils.dvcConfigS3Credentials3(localPath, secretAccessKey);
+        // dvc 跟踪
+        DVCUtils.dvcAdd(localPath, "data");
+        // git commit
+        DVCUtils.gitAdd(localPath, ".");
+        DVCUtils.gitCommit(localPath, "commit from ci4s with " + loginUser.getUsername());
+        DVCUtils.gitPush(localPath, gitLinkUsername, gitLinkPassword);
+        // dvc push 到远程S3
+        DVCUtils.dvcPush(localPath);
+        return "新增数据集成功";
+    }
+
+    @Override
+    public CompletableFuture<String> newCreateVersion(NewDatasetVo datasetVo) {
+        return CompletableFuture.supplyAsync(() -> {
+            try {
+                checkoutToken();
+                Jedis jedis = new Jedis(redisHost);
+                LoginUser loginUser = SecurityUtils.getLoginUser();
+                String ci4sUsername = loginUser.getUsername();
+                String token = jedis.get(ci4sUsername + "_gitToken");
+                String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername();
+                String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword();
+                String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
+                Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
+                // 创建分支
+                String branchName = StringUtils.isEmpty(datasetVo.getVersion()) ? "master" : datasetVo.getVersion();
+                String repositoryName = datasetVo.getIdentifier();
+                if (StringUtils.equals(branchName, "master")) {
+                    gitService.createBranch(token, (String) userInfo.get("login"), repositoryName, branchName, "master");
+                }
+                // 得到项目地址
+                String projectUrl = gitendpoint + "/" + (String) userInfo.get("login") + "/" + repositoryName + ".git";
+
+                // 得到用户操作的路径
+                String url = datasetVo.getDatasetVersionVos().get(0).getUrl();
+                String localPath = localPathlocal + loginUser.getUsername() + "/" + "datasets" + "/" + datasetVo.getName();
+                String sourcePath = url.substring(0, url.lastIndexOf("/"));
+                // 命令行操作 git clone 项目地址
+                if (FileUtil.checkDirectoryExists(localPath)) {
+                    DVCUtils.gitFetch(localPath, gitLinkUsername, gitLinkPassword);
+                    DVCUtils.gitCheckoutBranch(localPath, branchName);
+                } else {
+                    DVCUtils.gitClone(localPath, projectUrl, branchName, gitLinkUsername, gitLinkPassword);
+                }
+
+                String s3Path = "management-platform-files" + "/" + ci4sUsername + "/" + "datasets" + "/" + datasetVo.getId() + "/" + repositoryName + "/" + branchName;
+
+                DVCUtils.moveFiles(sourcePath, localPath);
+                // 拼接生产的元数据后写入yaml文件
+                datasetVo.setCreateBy(String.valueOf(StringUtils.isNotEmpty((String) userInfo.get("nickname")) ? userInfo.get("nickname") : userInfo.get("login")));
+                datasetVo.setUpdateTime(DateUtils.getTime());
+                datasetVo.setVersionDesc(datasetVo.getDescription());
+                datasetVo.setUsage("<pre><code>\n" +
+                        "# 克隆数据集配置文件与存储参数到本地\n" +
+                        "git clone -b " + branchName + " "+ projectUrl + "\n" +
+                        "# 远程拉取配置文件\n" +
+                        "dvc pull\n" +
+                        "</code></pre>\n");
+                datasetVo.setIdentifier(repositoryName);
+                datasetVo.setId(datasetVo.getId());
+                datasetVo.setOwner((String) userInfo.get("login"));
+                YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo), localPath, "dataset");
+
+                // dvc init 初始化
+                DVCUtils.dvcInit(localPath);
+                // 配置远程S3地址
+                DVCUtils.dvcRemoteAdd(localPath, s3Path);
+                DVCUtils.dvcConfigS3Credentials(localPath, endpoint);
+                DVCUtils.dvcConfigS3Credentials2(localPath, accessKeyId);
+                DVCUtils.dvcConfigS3Credentials3(localPath, secretAccessKey);
+                // dvc 跟踪
+                DVCUtils.dvcAdd(localPath, "data");
+                // git commit
+                DVCUtils.gitAdd(localPath, ".");
+                DVCUtils.gitCommit(localPath, "commit from ci4s with " + loginUser.getUsername());
+                DVCUtils.gitPush(localPath, gitLinkUsername, gitLinkPassword);
+                // dvc push 到远程S3
+                DVCUtils.dvcPush(localPath);
+                return "新增数据集成功";
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+        });
+    }
+
+    @Override
+    public Page<NewDatasetVo> newPersonalQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception {
+        checkoutToken();
+        Jedis jedis = new Jedis(redisHost);
+        LoginUser loginUser = SecurityUtils.getLoginUser();
+        String ci4sUsername = loginUser.getUsername();
+        String token = jedis.get(ci4sUsername + "_gitToken");
+        String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
+        Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
+        // 拼接查询url
+        String datasetTagName = dataset.getDatasetTagName();
+        String datasetTypeName = dataset.getDatasetTypeName();
+        String topic_name = "ci4s_dataset";
+        topic_name = StringUtils.isEmpty(datasetTagName) ? topic_name : topic_name + ",datatag_" + datasetTagName;
+        topic_name = StringUtils.isEmpty(datasetTagName) ? topic_name : topic_name + ",datatype_" + datasetTypeName;
+        String url = gitendpoint + "/api/users/" + (String) userInfo.get("login") + "/projects.json?page=" + pageRequest.getPageNumber() + "&limit=" + pageRequest.getPageSize() + "&category=manage&topic_name=" + topic_name;
+        String req = HttpUtils.sendGetWithToken(url, null, token);
+        Map<String, Object> stringObjectMap = JacksonUtil.parseJSONStr2Map(req);
+        Integer total = (Integer) stringObjectMap.get("count");
+        List<Map<String, Object>> projects = (List<Map<String, Object>>) stringObjectMap.get("projects");
+        return new PageImpl<>(convert(projects), pageRequest, total);
+    }
+
+    @Override
+    public Page<NewDatasetVo> newPubilcQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception {
+        checkoutToken();
+        Jedis jedis = new Jedis(redisHost);
+        LoginUser loginUser = SecurityUtils.getLoginUser();
+        String ci4sUsername = loginUser.getUsername();
+        String token = jedis.get(ci4sUsername + "_gitToken");
+        String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
+        Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
+        Integer userId = (Integer) userInfo.get("user_id");
+        // 拼接查询url
+        String datasetTagName = dataset.getDatasetTagName();
+        String datasetTypeName = dataset.getDatasetTypeName();
+        String topic_name = "ci4s_dataset";
+        topic_name = StringUtils.isEmpty(datasetTagName) ? topic_name : topic_name + ",datatag_" + datasetTagName;
+        topic_name = StringUtils.isEmpty(datasetTagName) ? topic_name : topic_name + ",datatype_" + datasetTypeName;
+
+        String url = gitendpoint + "/api/projects.json?user_id=" + userId + "&page=" + pageRequest.getPageNumber() + "&limit=" + pageRequest.getPageSize() + "&sort_by=praises_count&topic_name=" + topic_name;
+        String req = HttpUtils.sendGetWithToken(url, null, token);
+        Map<String, Object> stringObjectMap = JacksonUtil.parseJSONStr2Map(req);
+        Integer total = (Integer) stringObjectMap.get("total_count");
+        List<Map<String, Object>> projects = (List<Map<String, Object>>) stringObjectMap.get("projects");
+        if (projects == null) {
+            return new PageImpl<>(new ArrayList<>(), pageRequest, 0);
+        }
+        return new PageImpl<>(convert(projects), pageRequest, total);
+    }
+
+    @Override
+    public NewDatasetVo getNewDatasetDesc(Integer id, String repositoryName, String repo, String owner, String version) throws Exception {
+        LoginUser loginUser = SecurityUtils.getLoginUser();
+        String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername();
+        String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword();
+        // cd到 localPathlocal/id/下面还有一个文件夹，然后做git pull操作，然后读取里面的文件列表，列出每个文件的大小和名称，封装成MAP
+        if (StringUtils.isEmpty(version)) {
+            List<Map<String, Object>> versionList = this.getVersionList(repo, owner);
+            if (versionList.size() == 0) {
+                throw new Exception("数据集文件不存在");
+            }
+            version = (String) versionList.get(0).get("name");
+        }
+
+        List<Map<String, Object>> fileDetailsAfterGitPull = DVCUtils.getFileDetailsAfterGitPull("E:\\test\\" +id, repositoryName, version, gitLinkUsername, gitLinkPassword);
+        // 在localPathlocal+id+"/"+repositoryName目录下的dataset.yaml中取到元数据
+        Map<String, Object> stringObjectMap = YamlUtils.loadYamlFile("E:\\test\\" + id + "\\" + repositoryName + "\\" + "dataset.yaml");
+        NewDatasetVo newDatasetVo = ConvertUtil.convertMapToObject(stringObjectMap, NewDatasetVo.class);
+        List<VersionVo> versionVos = new ArrayList<VersionVo>();
+        if (fileDetailsAfterGitPull!=null&&fileDetailsAfterGitPull.size()>0){
+            for(Map<String, Object> fileDetail : fileDetailsAfterGitPull){
+                VersionVo versionVo = new VersionVo();
+                versionVo.setUrl((String) fileDetail.get("filePath"));
+                versionVo.setFileName((String) fileDetail.get("fileName"));
+                long size = (long) fileDetail.get("size");
+                versionVo.setFileSize(FileUtil.formatFileSize(size));
+                versionVos.add(versionVo);
+            }
+        }
+        newDatasetVo.setDatasetVersionVos(versionVos);
+        return newDatasetVo;
+    }
+
+    @Override
+    public List<Map<String, Object>> getVersionList(String repo, String owner) throws Exception {
+        checkoutToken();
+        Jedis jedis = new Jedis(redisHost);
+        LoginUser loginUser = SecurityUtils.getLoginUser();
+        String ci4sUsername = loginUser.getUsername();
+        String token = jedis.get(ci4sUsername+"_gitToken");
+        List<Map<String, Object>> brancheList = gitService.getBrancheList(token, owner, repo);
+        return brancheList.stream()
+                .filter(branch -> !"master".equals(branch.get("name")))
+                .collect(Collectors.toList());
+    }
+
+    @Override
+    public void deleteDatasetNew(String repo, String owner) throws Exception {
+        checkoutToken();
+        Jedis jedis = new Jedis(redisHost);
+        LoginUser loginUser = SecurityUtils.getLoginUser();
+        String ci4sUsername = loginUser.getUsername();
+        String token = jedis.get(ci4sUsername+"_gitToken");
+        gitService.deleteProject(token, owner, repo);
+    }
+
+    @Override
+    public void deleteDatasetVersionNew(String repo, String owner, String version) throws Exception {
+        checkoutToken();
+        Jedis jedis = new Jedis(redisHost);
+        LoginUser loginUser = SecurityUtils.getLoginUser();
+        String ci4sUsername = loginUser.getUsername();
+        String token = jedis.get(ci4sUsername+"_gitToken");
+        gitService.deleteBranch(token, owner, repo, version);
+    }
+
+    @Override
+    public List<Map<String, String>> uploadDatasetlocal(MultipartFile[] files, String uuid) throws Exception {
+        List<Map<String, String>> results = new ArrayList<>();
+
+        for (MultipartFile file : files) {
+            // 构建objectName
+            String username = SecurityUtils.getLoginUser().getUsername();
+            String fileName = file.getOriginalFilename();
+            String path = localPathlocal + "temp" + "/" + username + "/" + "datasets" + "/" + uuid + "/" + "data" + "/" + fileName;
+            long sizeInBytes = file.getSize();
+            String formattedSize = FileUtil.formatFileSize(sizeInBytes);
+            File targetFile = new File(path);
+            // 确保目录存在
+            targetFile.getParentFile().mkdirs();
+            // 保存文件到目标路径
+            FileUtils.copyInputStreamToFile(file.getInputStream(), targetFile);
+            // 返回上传文件的路径
+            String absolutePath = targetFile.getAbsolutePath();
+            Map<String, String> result = new HashMap<>();
+            result.put("fileName", fileName);
+            result.put("url", path); // objectName根据实际情况定义
+            result.put("fileSize", formattedSize);
+            results.add(result);
+        }
+        return results;
+    }
+
+    @Override
+    public ResponseEntity<InputStreamResource> downloadDatasetlocal(String filePath) throws Exception {
+        File file = new File(filePath);
+
+        if (!file.exists()) {
+            throw new FileNotFoundException("File not found: " + filePath);
+        }
+
+        InputStreamResource resource = new InputStreamResource(new FileInputStream(file));
+
+        HttpHeaders headers = new HttpHeaders();
+        headers.add(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=" + file.getName());
+        headers.add(HttpHeaders.CONTENT_LENGTH, String.valueOf(file.length()));
+        headers.add(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_OCTET_STREAM_VALUE);
+
+        return ResponseEntity.ok()
+                .headers(headers)
+                .contentLength(file.length())
+                .contentType(MediaType.APPLICATION_OCTET_STREAM)
+                .body(resource);
+    }
+
+    @Override
+    public ResponseEntity<InputStreamResource> downloadAllDatasetFilesNew(String repositoryName, String version) throws Exception {
+        // 命令行操作 git clone 项目地址
+        checkoutToken();
+        LoginUser loginUser = SecurityUtils.getLoginUser();
+        String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername();
+        String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword();
+        String ci4sUsername = loginUser.getUsername();
+        Jedis jedis = new Jedis(redisHost);
+        String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
+        Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
+        Integer userId = (Integer) userInfo.get("user_id");
+        String projectUrl = gitendpoint + "/" +(String) userInfo.get("login") + "/" + repositoryName + ".git";
+        String localPath = localPathlocal+ loginUser.getUsername()+"/datasets/" +repositoryName;
+        File folder = new File(localPath);
+        if(folder.exists() && folder.isDirectory()){
+            //切换分支
+            DVCUtils.gitCheckoutBranch(localPath, version);
+            //pull
+            DVCUtils.gitPull(localPath,gitLinkUsername, gitLinkPassword);
+            //dvc pull
+            DVCUtils.dvcPull(localPath);
+        }else {
+            DVCUtils.gitClone(localPath, projectUrl, version, gitLinkUsername, gitLinkPassword);
+        }
+
+        // 打包 data 文件夹
+        String dataFolderPath = localPath + "/data";
+        String zipFilePath = localPath + "/data.zip";
+        try (FileOutputStream fos = new FileOutputStream(zipFilePath);
+             ZipOutputStream zos = new ZipOutputStream(fos)) {
+            Path sourcePath = Paths.get(dataFolderPath);
+            Files.walk(sourcePath).forEach(path -> {
+                if (!Files.isDirectory(path)) {
+                    ZipEntry zipEntry = new ZipEntry(sourcePath.relativize(path).toString());
+                    try {
+                        zos.putNextEntry(zipEntry);
+                        Files.copy(path, zos);
+                        zos.closeEntry();
+                    } catch (IOException e) {
+                        throw new RuntimeException("Error while zipping: " + path, e);
+                    }
+                }
+            });
+        }
+
+        // 返回压缩文件的输入流
+        File zipFile = new File(zipFilePath);
+        InputStreamResource resource = new InputStreamResource(new FileInputStream(zipFile));
+
+        return ResponseEntity.ok()
+                .header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=data.zip")
+                .contentType(MediaType.APPLICATION_OCTET_STREAM)
+                .contentLength(zipFile.length())
+                .body(resource);
+
+    }
+
+
+
+    public List<NewDatasetVo> convert(List<Map<String, Object>> lst) {
+        if (lst != null && lst.size() > 0) {
+            List<NewDatasetVo> newDatasetVos = ConvertUtil.convertListMapToObjectList(lst, NewDatasetVo.class);
+
+            for (NewDatasetVo newDatasetVo : newDatasetVos) {
+                Map<String, Object> map = lst.stream()
+                        .filter(m -> m.get("id").equals(newDatasetVo.getId()))
+                        .findFirst()
+                        .orElse(null);
+
+                if (map != null) {
+                    List<Map<String, Object>> topics = (List<Map<String, Object>>) map.get("topics");
+                    if (topics != null) {
+                        topics.forEach(topic -> {
+                            String name = (String) topic.get("name");
+                            if (name != null) {
+                                if (name.startsWith("datatag_")) {
+                                    newDatasetVo.setDataTag(name.substring("datatag_".length()));
+                                } else if (name.startsWith("datatype_")) {
+                                    newDatasetVo.setDataType(name.substring("datatype_".length()));
+                                }
+                            }
+                        });
+                    }
+
+                    Map<String, Object> author = (Map<String, Object>) map.get("author");
+                    newDatasetVo.setCreateBy((String) author.get("name"));
+                    newDatasetVo.setOwner((String) author.get("login"));
+                }
+            }
+
+            return newDatasetVos;
+        }
+        return new ArrayList<>();
+    }
+
+    private String checkoutToken(){
+        Jedis jedis = new Jedis(redisHost);
+        LoginUser loginUser = SecurityUtils.getLoginUser();
+        String ci4sUsername = loginUser.getUsername();
+        String token = jedis.get(ci4sUsername+"_gitToken");
+        String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername();
+        String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword();
+
+        if(StringUtils.isEmpty(token)){
+            gitService.login(gitLinkUsername, gitLinkPassword);
+            token = jedis.get(ci4sUsername+"_gitToken");
+        }else {
+            try {
+                Map userInfo = gitService.getUserInfo(token);
+                if (userInfo == null ||(userInfo.get("status")!=null&&401==(Integer) userInfo.get("status"))){
+                    gitService.login(gitLinkUsername, gitLinkPassword);
+                    token = jedis.get(ci4sUsername+"_gitToken");
+                }
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+
+        }
+        return token;
+
+    }
+
+}

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/DVCUtils.java

@@ -201,11 +201,11 @@ public class DVCUtils {
             Path repoPath = Paths.get(localPath, repoFolder);
 
             //刷新
-            gitFetch(localPath, username, password);
-            // 切换到指定分支
-            gitCheckoutBranch(localPath, branch);
-            // 执行git pull
-            gitPull(localPath, username, password);
+//            gitFetch(localPath, username, password);
+//            // 切换到指定分支
+//            gitCheckoutBranch(localPath, branch);
+//            // 执行git pull
+//            gitPull(localPath, username, password);
             // 读取data文件夹中的文件列表
             Path dataPath = Paths.get(repoPath.toString(), "data");
             File[] files = dataPath.toFile().listFiles();

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/NewDatasetVo.java

@@ -42,8 +42,8 @@ public class NewDatasetVo implements Serializable {
 
     @ApiModelProperty(name = "time_ago",value = "更新时间")
     private String timeAgo;
-    @ApiModelProperty(name = "repo_id",value = "数据集仓库id")
-    private Integer repoId;
+    @ApiModelProperty(name = "id",value = "数据集仓库id")
+    private Integer id;
     @ApiModelProperty(name = "visits",value = "访问次数")
     private Integer visits;
     @ApiModelProperty(name = "create_by",value = "创建者")
@@ -53,7 +53,7 @@ public class NewDatasetVo implements Serializable {
     @ApiModelProperty(name = "usage",value = "使用示例")
     private String usage;
     @ApiModelProperty(name = "update_time",value = "更新时间")
-    private Date updateTime;
+    private String updateTime;
     @ApiModelProperty(name = "processing_code",value = "处理代码")
     private String processingCode;
     @ApiModelProperty(name = "owner",value = "项目所有者")