dvc

ruoyi-modules/management-platform/pom.xml

@@ -142,6 +142,11 @@
             <artifactId>docker-java</artifactId>
             <version>3.2.13</version>
         </dependency>
+        <dependency>
+            <groupId>commons-beanutils</groupId>
+            <artifactId>commons-beanutils</artifactId>
+            <version>1.9.4</version>
+        </dependency>
         <dependency>
             <groupId>com.github.docker-java</groupId>
             <artifactId>docker-java-transport-httpclient5</artifactId>
@@ -227,7 +232,16 @@
             <version>2.6</version>
             <scope>compile</scope>
         </dependency>
-
+        <dependency>
+            <groupId>org.eclipse.jgit</groupId>
+            <artifactId>org.eclipse.jgit</artifactId>
+            <version>5.13.0.202109080827-r</version>
+        </dependency>
+        <dependency>
+            <groupId>redis.clients</groupId>
+            <artifactId>jedis</artifactId>
+            <version>3.6.0</version>
+        </dependency>
 
     </dependencies>
 

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/dataset/DatasetController.java

@@ -224,5 +224,7 @@ public class DatasetController {
     public AjaxResult uploadDatasetPipeline(@RequestBody(required =false) DatasetVersion datasetVersion) throws Exception {
         return AjaxResult.success(this.datasetService.uploadDatasetPipeline(datasetVersion));
     }
+
+
 }
 

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/dataset/NewDatasetFromGitController.java

@@ -0,0 +1,110 @@
+package com.ruoyi.platform.controller.dataset;
+
+import com.ruoyi.common.core.web.domain.AjaxResult;
+import com.ruoyi.platform.domain.Dataset;
+import com.ruoyi.platform.service.DatasetService;
+import com.ruoyi.platform.vo.DatasetVo;
+import io.swagger.annotations.Api;
+import io.swagger.annotations.ApiOperation;
+import org.springframework.core.io.InputStreamResource;
+import org.springframework.data.domain.PageRequest;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.*;
+import org.springframework.web.multipart.MultipartFile;
+
+import javax.annotation.Resource;
+@RestController
+@RequestMapping("newdataset")
+@Api(value = "新数据集管理")
+public class NewDatasetFromGitController {
+    /**
+     * 服务对象
+     */
+    @Resource
+    private DatasetService datasetService;
+
+
+
+    /**
+     * 新增数据集与版本新
+     *
+     * @param datasetVo 实体
+     * @return 新增结果
+     */
+    @PostMapping("/addDatasetAndVersion")
+    @ApiOperation("添加数据集和版本")
+    public AjaxResult addDatasetAndVersion(@RequestBody DatasetVo datasetVo) throws Exception {
+        return AjaxResult.success(this.datasetService.newCreateDataset(datasetVo));
+
+    }
+
+
+    /**
+     * 新增数据集与版本新
+     *
+     * @param datasetVo 实体
+     * @return 新增结果
+     */
+    @PostMapping("/addVersion")
+    @ApiOperation("添加版本")
+    public AjaxResult addVersion(@RequestBody DatasetVo datasetVo) throws Exception {
+        return AjaxResult.success(this.datasetService.newCreateVersion(datasetVo));
+
+    }
+
+
+    /**
+     * 上传数据集
+     *
+     * @param files 上传的数据集文件
+     * @param uuid 上传唯一标识，构建url
+     * @return 上传结果
+     */
+    @CrossOrigin(origins = "*", allowedHeaders = "*")
+    @PostMapping("/upload")
+    @ApiOperation(value = "上传数据集")
+    public AjaxResult uploadDataset(@RequestParam("file") MultipartFile[] files, @RequestParam("uuid") String uuid) throws Exception {
+        return AjaxResult.success(this.datasetService.uploadDatasetlocal(files,uuid));
+    }
+
+    /**
+     * 数据集打包下载
+     *
+     * @param version  数据集版本
+     * @return 单条数据
+     */
+    @GetMapping("/downloadAllFiles")
+    @ApiOperation(value = "下载同一版本下所有数据集，并打包")
+    public ResponseEntity<InputStreamResource> downloadAllDatasetFiles(@RequestParam("repository_name") String repositoryName, @RequestParam("version") String version) throws Exception {
+        return datasetService.downloadAllDatasetFilesNew(repositoryName, version);
+    }
+
+    /**
+     * 下载数据集
+     *
+     * @param dataset_version_id  ps:这里的id是dataset_version表的主键
+     * @return 单条数据
+     */
+
+    @GetMapping("/download/{dataset_version_id}")
+    @ApiOperation(value = "下载单个数据集文件", notes = "根据数据集版本表id下载单个数据集文件")
+    public ResponseEntity<InputStreamResource> downloadDataset(@PathVariable("dataset_version_id") Integer dataset_version_id) throws Exception {
+        return datasetService.downloadDataset(dataset_version_id);
+    }
+
+    @GetMapping("/queryDatasets")
+    @ApiOperation("数据集广场公开数据集分页查询,根据data_type,data_tag筛选，true公开false私有")
+    public AjaxResult queryDatasets(Dataset dataset, @RequestParam("page") int page,
+                                     @RequestParam("size") int size,
+                                    @RequestParam(value = "is_public") Boolean isPublic,
+                                     @RequestParam(value = "data_type", required = false) String dataType,
+                                     @RequestParam(value = "data_tag", required = false) String dataTag) throws Exception {
+        PageRequest pageRequest = PageRequest.of(page, size);
+        if(isPublic){
+            return AjaxResult.success(this.datasetService.newPubilcQueryByPage(dataset, pageRequest));
+        }else {
+            return AjaxResult.success(this.datasetService.newPersonalQueryByPage(dataset, pageRequest));
+        }
+    }
+
+}

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/jupyter/JupyterController.java

@@ -90,7 +90,7 @@ public class JupyterController extends BaseController {
     @GetMapping(value = "/testdvc")
     public AjaxResult testdvc() throws Exception {
         DatasetVo datasetVo = new DatasetVo();
-        datasetVo.setName("testdvc82sas155");
+        datasetVo.setName("testdassad23");
         datasetVo.setDescription("sss");
         datasetVo.setAvailableRange(0);
         datasetVo.setDataTag("计算机视觉");
@@ -98,10 +98,9 @@ public class JupyterController extends BaseController {
         datasetVo.setVersion("dev");
         List<VersionVo> datasetVersionVos = new ArrayList<>();
         VersionVo versionVo = new VersionVo();
-        versionVo.setUrl("/home/xxx.txt");
+        versionVo.setUrl("E:/test/bb/data/xssa.doc");
         datasetVersionVos.add(versionVo);
         datasetVo.setDatasetVersionVos(datasetVersionVos);
-        datasetService.newCreateDataset(datasetVo);
-        return AjaxResult.success();
+        return AjaxResult.success(datasetService.newCreateDataset(datasetVo));
     }
 }

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/DatasetService.java

@@ -3,12 +3,14 @@ package com.ruoyi.platform.service;
 import com.ruoyi.platform.domain.Dataset;
 import com.ruoyi.platform.domain.DatasetVersion;
 import com.ruoyi.platform.vo.DatasetVo;
+import com.ruoyi.platform.vo.NewDatasetVo;
 import org.springframework.core.io.InputStreamResource;
 import org.springframework.data.domain.Page;
 import org.springframework.data.domain.PageRequest;
 import org.springframework.http.ResponseEntity;
 import org.springframework.web.multipart.MultipartFile;
 
+import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.CompletableFuture;
@@ -91,6 +93,13 @@ DatasetService {
 
 
     CompletableFuture<String> newCreateDataset(DatasetVo datasetVo) throws Exception;
+    CompletableFuture<String> newCreateVersion(DatasetVo datasetVo);
 
 
+
+    List<Map<String, String>> uploadDatasetlocal(MultipartFile[] files, String uuid) throws Exception;
+
+    ResponseEntity<InputStreamResource> downloadAllDatasetFilesNew(String repositoryName, String version) throws IOException, Exception;
+    Page<NewDatasetVo> newPersonalQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception;
+    Page<NewDatasetVo>  newPubilcQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception;
 }

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/DatasetServiceImpl.java

@@ -1,6 +1,5 @@
 package com.ruoyi.platform.service.impl;
 
-
 import com.ruoyi.common.core.utils.DateUtils;
 import com.ruoyi.common.security.utils.SecurityUtils;
 import com.ruoyi.platform.annotations.CheckDuplicate;
@@ -13,6 +12,7 @@ import com.ruoyi.platform.service.*;
 import com.ruoyi.platform.utils.*;
 import com.ruoyi.platform.vo.DatasetVo;
 import com.ruoyi.platform.vo.GitProjectVo;
+import com.ruoyi.platform.vo.NewDatasetVo;
 import com.ruoyi.platform.vo.VersionVo;
 import com.ruoyi.system.api.model.LoginUser;
 import io.minio.messages.Item;
@@ -33,10 +33,11 @@ import org.springframework.web.multipart.MultipartFile;
 import redis.clients.jedis.Jedis;
 
 import javax.annotation.Resource;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.File;
+import java.io.*;
 import java.lang.reflect.Field;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.text.SimpleDateFormat;
 import java.util.*;
 import java.util.concurrent.CompletableFuture;
@@ -461,9 +462,7 @@ public class DatasetServiceImpl implements DatasetService {
     public CompletableFuture<String> newCreateDataset(DatasetVo datasetVo) {
         return CompletableFuture.supplyAsync(() -> {
             try {
-                String gitusername = datasetVo.getGitusername();
-                String gitpassword = datasetVo.getGitpassword();
-                String token = gitService.login(gitusername, gitpassword);
+                String token = gitService.login("fanshuai", "h1n2x3j4y5@");
                 LoginUser loginUser = SecurityUtils.getLoginUser();
                 String ci4sUsername = loginUser.getUsername();
                 Jedis jedis = new Jedis(redisHost);
@@ -492,12 +491,18 @@ public class DatasetServiceImpl implements DatasetService {
 
                 // 得到用户操作的路径
                 String url = datasetVo.getDatasetVersionVos().get(0).getUrl();
-                String localPath = "/home/tempwork" + datasetVo.getName();
+                String localPath = "E:/test/" + datasetVo.getName();
                 String sourcePath = url.substring(0, url.lastIndexOf("/"));
                 // 命令行操作 git clone 项目地址
-                DVCUtils.gitClone(localPath, projectUrl, branchName, gitusername, gitpassword);
+                DVCUtils.gitClone(localPath, projectUrl, branchName, "fanshuai", "h1n2x3j4y5@");
                 String s3Path = "management-platform-files/" + ci4sUsername + "/datasets/" + repositoryName + "/" + branchName;
+                //拼接生产的元数据后写入yaml文件
+                YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo),sourcePath, "dataset");
+
                 DVCUtils.moveFiles(sourcePath, localPath);
+
+
+
                 // dvc init 初始化
                 DVCUtils.dvcInit(localPath);
                 // 配置远程S3地址
@@ -510,7 +515,7 @@ public class DatasetServiceImpl implements DatasetService {
                 // git commit
                 DVCUtils.gitAdd(localPath, ".");
                 DVCUtils.gitCommit(localPath, "commit from ci4s with " + loginUser.getUsername());
-                DVCUtils.gitPush(localPath, gitusername, gitpassword);
+                DVCUtils.gitPush(localPath, "fanshuai", "h1n2x3j4y5@");
                 // dvc push 到远程S3
                 DVCUtils.dvcPush(localPath);
                 return "新增数据集成功";
@@ -520,6 +525,106 @@ public class DatasetServiceImpl implements DatasetService {
         });
     }
 
+    @Override
+    public CompletableFuture<String> newCreateVersion(DatasetVo datasetVo) {
+        return CompletableFuture.supplyAsync(() -> {
+            try {
+                String token = gitService.login("fanshuai", "h1n2x3j4y5@");
+                LoginUser loginUser = SecurityUtils.getLoginUser();
+                String ci4sUsername = loginUser.getUsername();
+                Jedis jedis = new Jedis(redisHost);
+                String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
+                Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
+                // 创建分支
+                String branchName = datasetVo.getVersion();
+                String repositoryName = datasetVo.getRepositoryName();
+                gitService.createBranch(token, (String) userInfo.get("login"), repositoryName, branchName, "master");
+                // 得到项目地址
+                String projectUrl = gitendpoint + "/" +(String) userInfo.get("login") + "/" + repositoryName + ".git";
+
+                // 得到用户操作的路径
+                String url = datasetVo.getDatasetVersionVos().get(0).getUrl();
+                String localPath = "E:/test/"+ loginUser.getUsername()+"/datasets/"+ datasetVo.getName();
+                String sourcePath = url.substring(0, url.lastIndexOf("/"));
+                // 命令行操作 git clone 项目地址
+                DVCUtils.gitClone(localPath, projectUrl, branchName, "fanshuai", "h1n2x3j4y5@");
+                String s3Path = "management-platform-files/" + ci4sUsername + "/datasets/" + repositoryName + "/" + branchName;
+                //拼接生产的元数据后写入yaml文件
+                YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo),sourcePath, "dataset");
+
+                DVCUtils.moveFiles(sourcePath, localPath);
+                // dvc init 初始化
+                DVCUtils.dvcInit(localPath);
+                // 配置远程S3地址
+                DVCUtils.dvcRemoteAdd(localPath, s3Path);
+                DVCUtils.dvcConfigS3Credentials(localPath, endpoint);
+                DVCUtils.dvcConfigS3Credentials2(localPath, accessKeyId);
+                DVCUtils.dvcConfigS3Credentials3(localPath, secretAccessKey);
+                // dvc 跟踪
+                DVCUtils.dvcAdd(localPath, "data");
+                // git commit
+                DVCUtils.gitAdd(localPath, ".");
+                DVCUtils.gitCommit(localPath, "commit from ci4s with " + loginUser.getUsername());
+                DVCUtils.gitPush(localPath, "fanshuai", "h1n2x3j4y5@");
+                // dvc push 到远程S3
+                DVCUtils.dvcPush(localPath);
+                return "新增数据集成功";
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+        });
+    }
+
+
+    @Override
+    public Page<NewDatasetVo> newPersonalQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception {
+        LoginUser loginUser = SecurityUtils.getLoginUser();
+        String token = gitService.login("fanshuai", "h1n2x3j4y5@");
+        String ci4sUsername = loginUser.getUsername();
+        Jedis jedis = new Jedis(redisHost);
+        String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
+        Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
+        //拼接查询url
+        String datasetTagName = dataset.getDatasetTagName();
+        String datasetTypeName = dataset.getDatasetTypeName();
+        String topic_name = "ci4s_dataset";
+        topic_name =StringUtils.isEmpty(datasetTagName)?topic_name : topic_name+",datatag_" + datasetTagName;
+        topic_name =StringUtils.isEmpty(datasetTagName)?topic_name : topic_name+",datatype_" + datasetTypeName;
+        String url = gitendpoint + "/api/users/"+(String) userInfo.get("login")+"/projects.json?page="+pageRequest.getPageNumber()+"&limit="+pageRequest.getPageSize()+"&category=manage&topic_name="+topic_name;
+        String req = HttpUtils.sendGetWithToken(url,null,token);
+        Map<String, Object> stringObjectMap = JacksonUtil.parseJSONStr2Map(req);
+        Integer total = (Integer) stringObjectMap.get("count");
+        List<Map<String, Object>> projects = (List<Map<String, Object>>) stringObjectMap.get("projects");
+        return new PageImpl<>(convert(projects), pageRequest, total);
+    }
+
+    @Override
+    public Page<NewDatasetVo> newPubilcQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception {
+
+        LoginUser loginUser = SecurityUtils.getLoginUser();
+        String token = gitService.login("fanshuai", "h1n2x3j4y5@");
+        String ci4sUsername = loginUser.getUsername();
+        Jedis jedis = new Jedis(redisHost);
+        String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
+        Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
+        Integer userId = (Integer) userInfo.get("user_id");
+        //拼接查询url
+        String datasetTagName = dataset.getDatasetTagName();
+        String datasetTypeName = dataset.getDatasetTypeName();
+        String topic_name = "ci4s_dataset";
+        topic_name =StringUtils.isEmpty(datasetTagName)?topic_name : topic_name+",datatag_" + datasetTagName;
+        topic_name =StringUtils.isEmpty(datasetTagName)?topic_name : topic_name+",datatype_" + datasetTypeName;
+
+        String url = gitendpoint + "/api/projects.json?user_id="+userId+"&page="+pageRequest.getPageNumber()+"&limit="+pageRequest.getPageSize()+"&sort_by=praises_count&topic_name="+topic_name;
+        String req = HttpUtils.sendGetWithToken(url,null,token);
+        Map<String, Object> stringObjectMap = JacksonUtil.parseJSONStr2Map(req);
+        Integer total = (Integer) stringObjectMap.get("total_count");
+        List<Map<String, Object>> projects = (List<Map<String, Object>>) stringObjectMap.get("projects");
+        return new PageImpl<>(convert(projects), pageRequest, total);
+
+    }
+
+    @Override
     public List<Map<String, String>> uploadDatasetlocal(MultipartFile[] files, String uuid) throws Exception {
         List<Map<String, String>> results = new ArrayList<>();
 
@@ -527,7 +632,7 @@ public class DatasetServiceImpl implements DatasetService {
             // 构建objectName
             String username = SecurityUtils.getLoginUser().getUsername();
             String fileName = file.getOriginalFilename();
-            String path = "/datasets/" + username + "/" + uuid + "/"+"/data/" + fileName;
+            String path = "/temp/"+ username +"/datasets/"+ uuid + "/"+"/data/" + fileName;
             long sizeInBytes = file.getSize();
             String formattedSize = FileUtil.formatFileSize(sizeInBytes);
             File targetFile = new File(path, file.getOriginalFilename());
@@ -546,4 +651,94 @@ public class DatasetServiceImpl implements DatasetService {
         return results;
     }
 
+    @Override
+    public ResponseEntity<InputStreamResource> downloadAllDatasetFilesNew(String repositoryName, String version) throws Exception {
+        // 命令行操作 git clone 项目地址
+        LoginUser loginUser = SecurityUtils.getLoginUser();
+        String token = gitService.login("fanshuai", "h1n2x3j4y5@");
+        String ci4sUsername = loginUser.getUsername();
+        Jedis jedis = new Jedis(redisHost);
+        String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
+        Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
+        Integer userId = (Integer) userInfo.get("user_id");
+        String projectUrl = gitendpoint + "/" +(String) userInfo.get("login") + "/" + repositoryName + ".git";
+        String localPath = "E:/test/"+ loginUser.getUsername()+"/datasets/" +repositoryName;
+        File folder = new File(localPath);
+        if(folder.exists() && folder.isDirectory()){
+            //切换分支
+            DVCUtils.gitCheckoutBranch(localPath, version);
+            //pull
+            DVCUtils.gitPull(localPath,"fanshuai", "h1n2x3j4y5@");
+            //dvc pull
+            DVCUtils.dvcPull(localPath);
+        }else {
+            DVCUtils.gitClone(localPath, projectUrl, version, "fanshuai", "h1n2x3j4y5@");
+        }
+
+        // 打包 data 文件夹
+        String dataFolderPath = localPath + "/data";
+        String zipFilePath = localPath + "/data.zip";
+        try (FileOutputStream fos = new FileOutputStream(zipFilePath);
+             ZipOutputStream zos = new ZipOutputStream(fos)) {
+            Path sourcePath = Paths.get(dataFolderPath);
+            Files.walk(sourcePath).forEach(path -> {
+                if (!Files.isDirectory(path)) {
+                    ZipEntry zipEntry = new ZipEntry(sourcePath.relativize(path).toString());
+                    try {
+                        zos.putNextEntry(zipEntry);
+                        Files.copy(path, zos);
+                        zos.closeEntry();
+                    } catch (IOException e) {
+                        throw new RuntimeException("Error while zipping: " + path, e);
+                    }
+                }
+            });
+        }
+
+        // 返回压缩文件的输入流
+        File zipFile = new File(zipFilePath);
+        InputStreamResource resource = new InputStreamResource(new FileInputStream(zipFile));
+
+        return ResponseEntity.ok()
+                .header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=data.zip")
+                .contentType(MediaType.APPLICATION_OCTET_STREAM)
+                .contentLength(zipFile.length())
+                .body(resource);
+
+    }
+
+
+
+    public List<NewDatasetVo> convert(List<Map<String, Object>> lst) {
+        if (lst != null && lst.size() > 0) {
+            List<NewDatasetVo> newDatasetVos = ConvertUtil.convertListMapToObjectList(lst, NewDatasetVo.class);
+
+            for (NewDatasetVo newDatasetVo : newDatasetVos) {
+                Map<String, Object> map = lst.stream()
+                        .filter(m -> m.get("repo_id").equals(newDatasetVo.getRepoId()))
+                        .findFirst()
+                        .orElse(null);
+
+                if (map != null) {
+                    List<Map<String, Object>> topics = (List<Map<String, Object>>) map.get("topics");
+                    if (topics != null) {
+                        topics.forEach(topic -> {
+                            String name = (String) topic.get("name");
+                            if (name != null) {
+                                if (name.startsWith("datatag_")) {
+                                    newDatasetVo.setDataTag(name.substring("datatag_".length()));
+                                } else if (name.startsWith("datatype_")) {
+                                    newDatasetVo.setDataType(name.substring("datatype_".length()));
+                                }
+                            }
+                        });
+                    }
+                }
+            }
+
+            return newDatasetVos;
+        }
+        return new ArrayList<>();
+    }
+
 }

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/ConvertUtil.java

@@ -4,10 +4,13 @@ package com.ruoyi.platform.utils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.BeanUtils;
-
+import org.apache.commons.beanutils.PropertyUtils;
+import java.lang.reflect.Field;
+import java.lang.reflect.InvocationTargetException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
+import java.util.Map;
 
 
 /**
@@ -60,5 +63,64 @@ public class ConvertUtil {
         return targetList;
     }
 
+    public static <T> List<T> convertListMapToObjectList(List<Map<String, Object>> listMap, Class<T> targetClass) {
+        List<T> resultList = new ArrayList<>();
+
+        if (listMap != null) {
+            for (Map<String, Object> map : listMap) {
+                T targetObject = convertMapToObject(map, targetClass);
+                if (targetObject != null) {
+                    resultList.add(targetObject);
+                }
+            }
+        }
+
+        return resultList;
+    }
+
+    private static <T> T convertMapToObject(Map<String, Object> map, Class<T> targetClass) {
+        try {
+            T targetObject = targetClass.newInstance();
+            for (Map.Entry<String, Object> entry : map.entrySet()) {
+                String key = entry.getKey();
+                Object value = entry.getValue();
+                String camelCaseKey = toCamelCase(key);
+                if (hasProperty(targetClass, camelCaseKey)) {
+                    PropertyUtils.setProperty(targetObject, camelCaseKey, value);
+                }
+            }
+            return targetObject;
+        } catch (InstantiationException | IllegalAccessException | InvocationTargetException | NoSuchMethodException e) {
+            e.printStackTrace();
+        }
+        return null;
+    }
+
+    private static boolean hasProperty(Class<?> clazz, String propertyName) {
+        try {
+            Field field = clazz.getDeclaredField(propertyName);
+            return field != null;
+        } catch (NoSuchFieldException e) {
+            return false;
+        }
+    }
+
+    private static String toCamelCase(String key) {
+        StringBuilder sb = new StringBuilder();
+        boolean nextUpperCase = false;
+        for (char c : key.toCharArray()) {
+            if (c == '_') {
+                nextUpperCase = true;
+            } else {
+                if (nextUpperCase) {
+                    sb.append(Character.toUpperCase(c));
+                    nextUpperCase = false;
+                } else {
+                    sb.append(c);
+                }
+            }
+        }
+        return sb.toString();
+    }
 }
 

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/DVCUtils.java

@@ -95,6 +95,33 @@ public class DVCUtils {
         }
     }
 
+    public static void gitCheckoutBranch(String localPath, String branchName) throws IOException, GitAPIException {
+        FileRepositoryBuilder builder = new FileRepositoryBuilder();
+        Repository repository = builder.setGitDir(new File(localPath, ".git"))
+                .readEnvironment()
+                .findGitDir()
+                .build();
+
+        try (Git git = new Git(repository)) {
+            CheckoutCommand checkoutCommand = git.checkout();
+            checkoutCommand.setName(branchName).call();
+        }
+    }
+
+    public static void gitPull(String localPath, String username, String password) throws IOException, GitAPIException {
+        FileRepositoryBuilder builder = new FileRepositoryBuilder();
+        Repository repository = builder.setGitDir(new File(localPath, ".git"))
+                .readEnvironment()
+                .findGitDir()
+                .build();
+
+        try (Git git = new Git(repository)) {
+            CredentialsProvider credentialsProvider = new UsernamePasswordCredentialsProvider(username, password);
+            PullCommand pullCommand = git.pull();
+            pullCommand.setCredentialsProvider(credentialsProvider).call();
+        }
+    }
+
     public static void dvcInit(String localPath) throws Exception {
         String command = "dvc init";
         runCommand(command, localPath);
@@ -111,13 +138,15 @@ public class DVCUtils {
     }
 
     public static void dvcConfigS3Credentials(String localPath, String endpointurl) throws Exception {
-        String command = "dvc remote modify myremote endpointurl " + endpointurl  ;
+        String command = "dvc remote modify myremote endpointurl " + endpointurl;
         runCommand(command, localPath);
     }
+
     public static void dvcConfigS3Credentials2(String localPath, String accessKeyId) throws Exception {
         String command = "dvc remote modify myremote access_key_id " + accessKeyId;
         runCommand(command, localPath);
     }
+
     public static void dvcConfigS3Credentials3(String localPath, String secretAccessKey) throws Exception {
         String command = "dvc remote modify myremote secret_access_key " + secretAccessKey;
         runCommand(command, localPath);
@@ -128,6 +157,7 @@ public class DVCUtils {
         runCommand(command, localPath);
     }
 
+    // 更新的 dvcPull 方法
     public static void dvcPull(String localPath) throws Exception {
         String command = "dvc pull";
         runCommand(command, localPath);

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/HttpUtils.java

@@ -11,6 +11,7 @@ import org.apache.http.client.entity.UrlEncodedFormEntity;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.utils.URIBuilder;
 import org.apache.http.entity.StringEntity;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.DefaultHttpClient;
@@ -89,15 +90,61 @@ public class HttpUtils {
     public static String sendGet(String url, String param) {
         return sendGet(url, param, "UTF-8");
     }
+//    /**
+//     * 向指定 URL 发送带token的GET方法的请求
+//     *
+//     * @param url   发送请求的 URL
+//     * @param param 请求参数，请求参数应该是 name1=value1&name2=value2 的形式。
+//     * @return 所代表远程资源的响应结果
+//     */
+//    public static String sendGetWithToken(String url, String param,String token) {
+//        return sendGet(url, param, "UTF-8",token);
+//    }
     /**
-     * 向指定 URL 发送带token的GET方法的请求
+     * 向指定 URL 发送带 token 的 GET 方法的请求，使用 Apache HttpClient
      *
-     * @param url   发送请求的 URL
-     * @param param 请求参数，请求参数应该是 name1=value1&name2=value2 的形式。
+     * @param url    发送请求的 URL
+     * @param param  请求参数，请求参数应该是 name1=value1&name2=value2 的形式。
+     * @param token  认证 token
      * @return 所代表远程资源的响应结果
      */
-    public static String sendGetWithToken(String url, String param,String token) {
-        return sendGet(url, param, "UTF-8",token);
+    public static String sendGetWithToken(String url, String param, String token) {
+        String result = "";
+        try (CloseableHttpClient httpClient = HttpClients.createDefault()) {
+            URIBuilder uriBuilder = new URIBuilder(url);
+            if (param != null && !param.isEmpty()) {
+                String[] pairs = param.split("&");
+                for (String pair : pairs) {
+                    int idx = pair.indexOf("=");
+                    if (idx != -1) {
+                        uriBuilder.setParameter(pair.substring(0, idx), pair.substring(idx + 1));
+                    }
+                }
+            }
+
+            URI uri = uriBuilder.build();
+            HttpGet httpGet = new HttpGet(uri);
+
+            httpGet.setHeader("Content-Type", "application/json");
+            httpGet.setHeader("Authorization", "Bearer " + token);
+
+            log.info("Executing request: " + httpGet.getRequestLine());
+
+            try (CloseableHttpResponse response = httpClient.execute(httpGet)) {
+                int statusCode = response.getStatusLine().getStatusCode();
+                if (statusCode == 200) {
+                    result = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
+                } else {
+                    throw new IOException("HTTP request failed with response code: " + statusCode);
+                }
+                log.info("Response: " + result);
+            }
+        } catch (URISyntaxException e) {
+            log.error("URISyntaxException, url=" + url + ", param=" + param, e);
+        } catch (IOException e) {
+            log.error("IOException, url=" + url + ", param=" + param, e);
+        }
+        return result;
     }
     /**
      * 向指定 URL 发送带token的GET方法的请求

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/JsonUtils.java

@@ -51,4 +51,9 @@ public class JsonUtils {
 
         return flatMap;
     }
+
+    // 将Object转换为Map
+    public static Map<String, Object> objectToMap(Object object) throws IOException {
+        return objectMapper.convertValue(object, Map.class);
+    }
 }

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/YamlUtils.java

@@ -0,0 +1,28 @@
+package com.ruoyi.platform.utils;
+
+import org.yaml.snakeyaml.Yaml;
+
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.Map;
+
+public class YamlUtils {
+
+    /**
+     * 将Map对象转换为YAML格式并写入指定路径的文件中
+     *
+     * @param data      Map对象
+     * @param path      文件路径
+     * @param fileName  文件名
+     */
+    public static void generateYamlFile(Map<String, Object> data, String path, String fileName) {
+        Yaml yaml = new Yaml();
+        String fullPath = path + "/" + fileName + ".yaml";
+
+        try (FileWriter writer = new FileWriter(fullPath)) {
+            yaml.dump(data, writer);
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+    }
+}

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/DatasetVo.java

@@ -28,10 +28,6 @@ public class DatasetVo implements Serializable {
     @ApiModelProperty(name = "data_tag")
     private String dataTag;
 
-    @ApiModelProperty(name = "gitusername")
-    private String gitusername;
-    @ApiModelProperty(name = "gitpassword")
-    private String gitpassword;
     /**
      * 版本
      */
@@ -47,4 +43,10 @@ public class DatasetVo implements Serializable {
     @ApiModelProperty(name = "available_cluster")
     private String availableCluster;
 
+
+    /**
+     * 数据集仓库名称
+     */
+    @ApiModelProperty(name = "repository_name")
+    private String repositoryName;
 }

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/GitProjectVo.java

@@ -0,0 +1,67 @@
+package com.ruoyi.platform.vo;
+
+import com.fasterxml.jackson.databind.PropertyNamingStrategy;
+import com.fasterxml.jackson.databind.annotation.JsonNaming;
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+import lombok.Data;
+
+import java.io.Serializable;
+
+@ApiModel(description = "Git项目信息")
+@Data
+@JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy.class)
+public class GitProjectVo implements Serializable {
+
+    @ApiModelProperty(value = "用户或组织ID", required = true)
+    private int userId;
+
+    @ApiModelProperty(value = "项目名称", required = true)
+    private String name;
+
+    @ApiModelProperty(value = "项目描述", required = false)
+    private String description;
+
+    @ApiModelProperty(value = "项目标识", required = true)
+    private String repositoryName;
+
+//    @ApiModelProperty(value = "项目分类ID", required = false)
+//    private int projectCategoryId;
+//
+//    @ApiModelProperty(value = "项目语言ID", required = false)
+//    private int projectLanguageId;
+
+//    @ApiModelProperty(value = "项目忽略文件ID", required = false)
+//    private int ignoreId;
+
+//    @ApiModelProperty(value = "项目许可证ID", required = false)
+//    private int licenseId;
+
+    @ApiModelProperty(value = "项目是否为私有项目", required = false)
+    private boolean isPrivate;
+
+//    @ApiModelProperty(value = "是否为区块链激励项目", required = false)
+//    private boolean blockchain;
+//
+//    @ApiModelProperty(value = "区块链激励金额总数", required = false)
+//    private int blockchainTokenAll;
+
+//
+//    @Override
+//    public String toString() {
+//        return "GitProjectVo{" +
+//                "userId=" + userId +
+//                ", name='" + name + '\'' +
+//                ", description='" + description + '\'' +
+//                ", repositoryName='" + repositoryName + '\'' +
+//                ", projectCategoryId=" + projectCategoryId +
+//                ", projectLanguageId=" + projectLanguageId +
+//                ", ignoreId=" + ignoreId +
+//                ", licenseId=" + licenseId +
+//                ", isPrivate=" + isPrivate +
+//                ", blockchain=" + blockchain +
+//                ", blockchainTokenAll=" + blockchainTokenAll +
+//                '}';
+//    }
+}
+

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/NewDatasetVo.java

@@ -0,0 +1,41 @@
+package com.ruoyi.platform.vo;
+
+import com.fasterxml.jackson.databind.PropertyNamingStrategy;
+import com.fasterxml.jackson.databind.annotation.JsonNaming;
+import io.swagger.annotations.ApiModelProperty;
+import lombok.Data;
+import org.apache.ibatis.annotations.MapKey;
+
+import java.io.Serializable;
+import java.util.List;
+
+@JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy.class)
+@Data
+public class NewDatasetVo implements Serializable {
+
+
+    @ApiModelProperty(name = "name")
+    private String name;
+    /**
+     * 数据集仓库名称
+     */
+    @ApiModelProperty(name = "identifier")
+    private String identifier;
+    @ApiModelProperty(name = "description")
+    private String description;
+    /**
+     * 是否公开1公开，0私有
+     */
+    @ApiModelProperty(name = "is_public")
+    private Boolean isPublic;
+    @ApiModelProperty(name = "data_type")
+    private String dataType;
+    @ApiModelProperty(name = "data_tag")
+    private String dataTag;
+    @ApiModelProperty(name = "time_ago")
+    private String timeAgo;
+    @ApiModelProperty(name = "repo_id")
+    private Integer repoId;
+
+
+}