Merge pull request '829' (#121) from dev into master

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/jupyter/JupyterController.java

@@ -4,9 +4,12 @@ import com.ruoyi.common.core.web.controller.BaseController;
 import com.ruoyi.common.core.web.domain.AjaxResult;
 import com.ruoyi.common.core.web.domain.GenericsAjaxResult;
 import com.ruoyi.platform.domain.DevEnvironment;
+import com.ruoyi.platform.service.DatasetService;
 import com.ruoyi.platform.service.JupyterService;
+import com.ruoyi.platform.vo.DatasetVo;
 import com.ruoyi.platform.vo.FrameLogPathVo;
 import com.ruoyi.platform.vo.PodStatusVo;
+import com.ruoyi.platform.vo.VersionVo;
 import io.swagger.annotations.Api;
 import io.swagger.annotations.ApiOperation;
 import io.swagger.v3.oas.annotations.responses.ApiResponse;
@@ -17,6 +20,8 @@ import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
 
 @RestController
 @RequestMapping("/jupyter")
@@ -24,6 +29,8 @@ import java.io.InputStream;
 public class JupyterController extends BaseController {
     @Resource
     private JupyterService jupyterService;
+    @Resource
+    private DatasetService datasetService;
     @GetMapping(value = "/getURL")
     @ApiOperation("得到访问地址")
     public GenericsAjaxResult<String> getURL() throws IOException {
@@ -79,4 +86,22 @@ public class JupyterController extends BaseController {
         jupyterService.mlflow();
         return AjaxResult.success();
     }
+
+    @GetMapping(value = "/testdvc")
+    public AjaxResult testdvc() throws Exception {
+        DatasetVo datasetVo = new DatasetVo();
+        datasetVo.setName("testdvc82sas155");
+        datasetVo.setDescription("sss");
+        datasetVo.setAvailableRange(0);
+        datasetVo.setDataTag("计算机视觉");
+        datasetVo.setDataType("机器翻译");
+        datasetVo.setVersion("dev");
+        List<VersionVo> datasetVersionVos = new ArrayList<>();
+        VersionVo versionVo = new VersionVo();
+        versionVo.setUrl("/home/xxx.txt");
+        datasetVersionVos.add(versionVo);
+        datasetVo.setDatasetVersionVos(datasetVersionVos);
+        datasetService.newCreateDataset(datasetVo);
+        return AjaxResult.success();
+    }
 }

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/DatasetService.java

@@ -11,6 +11,7 @@ import org.springframework.web.multipart.MultipartFile;
 
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.CompletableFuture;
 
 /**
  * (Dataset)表服务接口
@@ -84,4 +85,12 @@ DatasetService {
     ResponseEntity<InputStreamResource> downloadAllDatasetFiles(Integer datasetId, String version) throws Exception;
 
     List<Map<String, String>> exportDataset(String path, String uuid) throws Exception;
+
+
+
+
+
+    CompletableFuture<String> newCreateDataset(DatasetVo datasetVo) throws Exception;
+
+
 }

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/DvcService.java

@@ -0,0 +1,8 @@
+package com.ruoyi.platform.service;
+
+public interface DvcService {
+
+    //使用dvc初始化，跟踪，push到远程仓库的接口
+    public void initaddpushDvc(String localPath) throws Exception ;
+
+}

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/GitService.java

@@ -0,0 +1,17 @@
+package com.ruoyi.platform.service;
+
+import com.ruoyi.platform.vo.GitProjectVo;
+
+import java.util.Map;
+
+public interface GitService {
+    //登录方法，返回token
+    String login(String username, String password);
+
+    //输入token，项目名，tag，创建新项目,返回项目地址
+    Map createProject(String token, GitProjectVo gitProjectVo) throws Exception;
+
+    void createBranch(String token,String owner, String projectName, String branchName, String oldBranchName) throws Exception;
+
+    void createTopic(String token, Integer id, String topicName) throws Exception;
+}

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/DatasetServiceImpl.java

@@ -1,6 +1,7 @@
 package com.ruoyi.platform.service.impl;
 
 
+import com.ruoyi.common.core.utils.DateUtils;
 import com.ruoyi.common.security.utils.SecurityUtils;
 import com.ruoyi.platform.annotations.CheckDuplicate;
 import com.ruoyi.platform.domain.AssetIcon;
@@ -8,17 +9,14 @@ import com.ruoyi.platform.domain.Dataset;
 import com.ruoyi.platform.domain.DatasetVersion;
 import com.ruoyi.platform.mapper.DatasetDao;
 import com.ruoyi.platform.mapper.DatasetVersionDao;
-import com.ruoyi.platform.service.AssetIconService;
-import com.ruoyi.platform.service.DatasetService;
-import com.ruoyi.platform.service.DatasetVersionService;
-import com.ruoyi.platform.service.MinioService;
-import com.ruoyi.platform.utils.BeansUtils;
-import com.ruoyi.platform.utils.FileUtil;
-import com.ruoyi.platform.utils.MinioUtil;
-import com.ruoyi.platform.vo.VersionVo;
+import com.ruoyi.platform.service.*;
+import com.ruoyi.platform.utils.*;
 import com.ruoyi.platform.vo.DatasetVo;
+import com.ruoyi.platform.vo.GitProjectVo;
+import com.ruoyi.platform.vo.VersionVo;
 import com.ruoyi.system.api.model.LoginUser;
 import io.minio.messages.Item;
+import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.core.io.InputStreamResource;
@@ -32,14 +30,16 @@ import org.springframework.http.ResponseEntity;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
 import org.springframework.web.multipart.MultipartFile;
+import redis.clients.jedis.Jedis;
 
 import javax.annotation.Resource;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
-import java.io.InputStream;
+import java.io.File;
 import java.lang.reflect.Field;
 import java.text.SimpleDateFormat;
 import java.util.*;
+import java.util.concurrent.CompletableFuture;
 import java.util.stream.Collectors;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipOutputStream;
@@ -67,6 +67,12 @@ public class DatasetServiceImpl implements DatasetService {
     @Resource
     private MinioService minioService;
 
+    @Resource
+    private GitService gitService;
+
+    @Resource
+    private DvcService dvcService;
+
     // 固定存储桶名
     @Value("${minio.dataReleaseBucketName}")
     private String bucketName;
@@ -74,7 +80,16 @@ public class DatasetServiceImpl implements DatasetService {
     @Resource
     private  MinioUtil minioUtil;
 
-
+    @Value("${spring.redis.host}")
+    private String redisHost;
+    @Value("${minio.accessKey}")
+    String accessKeyId;
+    @Value("${minio.secretKey}")
+    String secretAccessKey;
+    @Value("${minio.endpoint}")
+    String endpoint;
+    @Value("${git.endpoint}")
+    String gitendpoint;
 
     /**
      * 通过ID查询单条数据
@@ -370,7 +385,7 @@ public class DatasetServiceImpl implements DatasetService {
     }
 
     @Override
-    public ResponseEntity<InputStreamResource> downloadAllDatasetFiles(Integer datasetId, String version) throws Exception {
+    public ResponseEntity<InputStreamResource>  downloadAllDatasetFiles(Integer datasetId, String version) throws Exception {
         // 根据数据集id查数据名
         Dataset dataset = this.datasetDao.queryById(datasetId);
         String datasetName = dataset.getName();
@@ -440,4 +455,95 @@ public class DatasetServiceImpl implements DatasetService {
         return results;
     }
 
+
+
+    @Override
+    public CompletableFuture<String> newCreateDataset(DatasetVo datasetVo) {
+        return CompletableFuture.supplyAsync(() -> {
+            try {
+                String gitusername = datasetVo.getGitusername();
+                String gitpassword = datasetVo.getGitpassword();
+                String token = gitService.login(gitusername, gitpassword);
+                LoginUser loginUser = SecurityUtils.getLoginUser();
+                String ci4sUsername = loginUser.getUsername();
+                Jedis jedis = new Jedis(redisHost);
+                String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
+                Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
+                Integer userId = (Integer) userInfo.get("user_id");
+                // 拼接project
+                String repositoryName = ci4sUsername + "_dataset_" + DateUtils.dateTimeNow();
+                GitProjectVo gitProjectVo = new GitProjectVo();
+                gitProjectVo.setRepositoryName(repositoryName);
+                gitProjectVo.setName(datasetVo.getName());
+                gitProjectVo.setDescription(datasetVo.getDescription());
+                gitProjectVo.setPrivate(datasetVo.getAvailableRange() == 0);
+                gitProjectVo.setUserId(userId);
+                // 创建项目
+                Map project = gitService.createProject(token, gitProjectVo);
+                // 创建分支
+                String branchName = datasetVo.getVersion();
+                gitService.createBranch(token, (String) userInfo.get("login"), repositoryName, branchName, "master");
+                // 定义标签 标签1：ci4s_dataset 标签2：DataTag 标签3：DataType
+                gitService.createTopic(token, (Integer) project.get("id"), "ci4s_dataset");
+                gitService.createTopic(token, (Integer) project.get("id"), "DataTag_" + datasetVo.getDataTag());
+                gitService.createTopic(token, (Integer) project.get("id"), "DataType_" + datasetVo.getDataType());
+                // 得到项目地址
+                String projectUrl = gitendpoint + "/" +(String) userInfo.get("login") + "/" + repositoryName + ".git";
+
+                // 得到用户操作的路径
+                String url = datasetVo.getDatasetVersionVos().get(0).getUrl();
+                String localPath = "/home/tempwork" + datasetVo.getName();
+                String sourcePath = url.substring(0, url.lastIndexOf("/"));
+                // 命令行操作 git clone 项目地址
+                DVCUtils.gitClone(localPath, projectUrl, branchName, gitusername, gitpassword);
+                String s3Path = "management-platform-files/" + ci4sUsername + "/datasets/" + repositoryName + "/" + branchName;
+                DVCUtils.moveFiles(sourcePath, localPath);
+                // dvc init 初始化
+                DVCUtils.dvcInit(localPath);
+                // 配置远程S3地址
+                DVCUtils.dvcRemoteAdd(localPath, s3Path);
+                DVCUtils.dvcConfigS3Credentials(localPath, endpoint);
+                DVCUtils.dvcConfigS3Credentials2(localPath, accessKeyId);
+                DVCUtils.dvcConfigS3Credentials3(localPath, secretAccessKey);
+                // dvc 跟踪
+                DVCUtils.dvcAdd(localPath, "data");
+                // git commit
+                DVCUtils.gitAdd(localPath, ".");
+                DVCUtils.gitCommit(localPath, "commit from ci4s with " + loginUser.getUsername());
+                DVCUtils.gitPush(localPath, gitusername, gitpassword);
+                // dvc push 到远程S3
+                DVCUtils.dvcPush(localPath);
+                return "新增数据集成功";
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+        });
+    }
+
+    public List<Map<String, String>> uploadDatasetlocal(MultipartFile[] files, String uuid) throws Exception {
+        List<Map<String, String>> results = new ArrayList<>();
+
+        for (MultipartFile file:files){
+            // 构建objectName
+            String username = SecurityUtils.getLoginUser().getUsername();
+            String fileName = file.getOriginalFilename();
+            String path = "/datasets/" + username + "/" + uuid + "/"+"/data/" + fileName;
+            long sizeInBytes = file.getSize();
+            String formattedSize = FileUtil.formatFileSize(sizeInBytes);
+            File targetFile = new File(path, file.getOriginalFilename());
+            // 确保目录存在
+            targetFile.getParentFile().mkdirs();
+            // 保存文件到目标路径
+            FileUtils.copyInputStreamToFile(file.getInputStream(), targetFile);
+            // 返回上传文件的路径
+            String absolutePath = targetFile.getAbsolutePath();
+            Map<String, String> result = new HashMap<>();
+            result.put("fileName", fileName);
+            result.put("url", absolutePath); // objectName根据实际情况定义
+            result.put("fileSize", formattedSize);
+            results.add(result);
+        }
+        return results;
+    }
+
 }

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/DvcServiceImpl.java

@@ -0,0 +1,34 @@
+package com.ruoyi.platform.service.impl;
+
+import com.ruoyi.common.security.utils.SecurityUtils;
+import com.ruoyi.platform.service.DvcService;
+import com.ruoyi.platform.utils.DVCUtils;
+import com.ruoyi.system.api.model.LoginUser;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Service;
+
+@Service
+public class DvcServiceImpl implements DvcService {
+    @Value("${minio.accessKey}")
+    String accessKeyId;
+    @Value("${minio.secretKey}")
+    String secretAccessKey;
+    @Value("${minio.endpoint}")
+    String endpoint;
+    @Override
+    public void initaddpushDvc(String localPath) throws Exception {
+        LoginUser loginUser = SecurityUtils.getLoginUser();
+        // dvc init 初始化
+        DVCUtils.dvcInit(localPath);
+        // 配置远程S3地址
+        DVCUtils.dvcRemoteAdd(localPath,"");
+//        DVCUtils.dvcConfigS3Credentials(localPath,endpoint, accessKeyId, secretAccessKey);
+        // dvc 跟踪
+        DVCUtils.dvcAdd(localPath , "data");
+        // git commit
+        DVCUtils.gitCommit(localPath, "commit from ci4s with "+loginUser.getUsername());
+        // dvc push 到远程S3
+        DVCUtils.dvcPush(localPath);
+    }
+
+}

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/GitServiceImpl.java

@@ -0,0 +1,98 @@
+package com.ruoyi.platform.service.impl;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.ruoyi.common.security.utils.SecurityUtils;
+import com.ruoyi.platform.service.GitService;
+import com.ruoyi.platform.utils.HttpUtils;
+import com.ruoyi.platform.utils.JsonUtils;
+import com.ruoyi.platform.vo.GitProjectVo;
+import com.ruoyi.system.api.model.LoginUser;
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Service;
+import redis.clients.jedis.Jedis;
+
+import java.util.HashMap;
+import java.util.Map;
+
+@Service
+public class GitServiceImpl implements GitService {
+
+    @Value("${spring.redis.host}")
+    private String redisHost;
+
+    private static final Logger log = LoggerFactory.getLogger(GitServiceImpl.class);
+
+    @Override
+    public String login(String username, String password) {
+        // 构建请求参数
+        Map<String, Object> params = new HashMap<>();
+        params.put("grant_type", "password");
+        params.put("username", username);
+        params.put("password", password);
+        params.put("client_id", "jEdGwrIJixCUIJM9vj2MwA6zmoTVhUdXxiSAaaCiXwA");
+        params.put("client_secret", "L3wBKTNnRo-wPen7bxR3F1myCvtVDgpWa6MnpfyWeJE");
+        try {
+            // 发送POST请求
+            String req = HttpUtils.sendPostRequest("https://www.gitlink.org.cn/oauth/token",null, JsonUtils.mapToJson(params));
+            // 解析响应JSON
+            if (StringUtils.isEmpty(req)) {
+                throw new RuntimeException("终止响应内容为空。");
+            }
+            // 将响应的JSON字符串转换为Map对象
+            Map<String, Object> runResMap = JsonUtils.jsonToMap(req);
+            // 提取access_token
+            // 提取access_token
+            String accessToken = (String) runResMap.get("access_token");
+
+            //通过access_token获取用户信息
+            String userReq = HttpUtils.sendGetWithToken("https://www.gitlink.org.cn/api/users/get_user_info.json",null, accessToken);
+            if (StringUtils.isEmpty(userReq)) {
+                throw new RuntimeException("终止响应内容为空。");
+            }
+            LoginUser loginUser = SecurityUtils.getLoginUser();
+            String ci4sUsername = loginUser.getUsername();
+            // 将access_token存入Redis
+            Jedis jedis = new Jedis(redisHost);
+            jedis.set(ci4sUsername+"_gitToken", accessToken);
+            jedis.set(ci4sUsername+"_gitUserInfo", userReq);
+            return accessToken;
+        } catch (Exception e) {
+            log.error("登录GitLink失败。", e);
+            return null;
+        }
+    }
+
+    @Override
+    public Map createProject(String token, GitProjectVo gitProjectVo) throws Exception {
+        String userReq = HttpUtils.sendPostWithToken("https://www.gitlink.org.cn/api/projects.json",JsonUtils.objectToJson(gitProjectVo),token);
+        return JsonUtils.jsonToMap(userReq);
+    }
+
+    @Override
+    public void createBranch(String token,String owner, String projectName, String branchName, String oldBranchName) throws Exception {
+        //https://www.gitlink.org.cn/api/v1/fanshuai/testdssa8755/branches.json
+//        {
+//            "new_branch_name": "SsS",
+//                "old_branch_name": "master"
+//        }
+        String createBranchUrl = "https://www.gitlink.org.cn/api/v1/"+ owner + "/" + projectName + "/branches.json";
+        Map<String, Object> resMap = new HashMap<>();
+        resMap.put("new_branch_name", branchName);
+        resMap.put("old_branch_name", oldBranchName);
+        String req = HttpUtils.sendPostWithToken(createBranchUrl,JsonUtils.objectToJson(resMap),token);
+        System.out.println(req);
+    }
+
+    @Override
+    public void createTopic(String token, Integer id, String topicName) throws Exception {
+        // https://www.gitlink.org.cn/api/v1/project_topics.json
+        Map<String, Object> resMap = new HashMap<>();
+        resMap.put("project_id", id);
+        resMap.put("name", topicName);
+        String req = HttpUtils.sendPostWithToken("https://www.gitlink.org.cn/api/v1/project_topics.json",JsonUtils.objectToJson(resMap),token);
+        System.out.println(req);
+    }
+}

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/JupyterServiceImpl.java

@@ -100,7 +100,7 @@ public class JupyterServiceImpl implements JupyterService {
         //TODO 设置镜像可配置，这里先用默认镜像启动pod
 
         // 调用修改后的 createPod 方法，传入额外的参数
-        Integer podPort = k8sClientUtil.createConfiguredPod(podName, namespace, port, mountPath, pvc, image, minioPvcName, datasetPath, modelPath);
+        Integer podPort = k8sClientUtil.createConfiguredPod(podName, namespace, port, mountPath, pvc, devEnvironment.getImage(), minioPvcName, datasetPath, modelPath);
         String url = masterIp + ":" + podPort;
         redisService.setCacheObject(podName,masterIp + ":" + podPort);
         devEnvironment.setStatus("Pending");

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/DVCUtils.java

@@ -0,0 +1,135 @@
+package com.ruoyi.platform.utils;
+
+import org.eclipse.jgit.api.*;
+import org.eclipse.jgit.api.errors.GitAPIException;
+import org.eclipse.jgit.lib.Repository;
+import org.eclipse.jgit.storage.file.FileRepositoryBuilder;
+import org.eclipse.jgit.transport.CredentialsProvider;
+import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider;
+
+import java.io.*;
+import java.nio.file.*;
+
+public class DVCUtils {
+
+    private static void runCommand(String command, String workingDir) throws Exception {
+        ProcessBuilder processBuilder = new ProcessBuilder(command.split(" "));
+        processBuilder.directory(new File(workingDir));
+        Process process = processBuilder.start();
+        BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()));
+        String line;
+        while ((line = reader.readLine()) != null) {
+            System.out.println(line);
+        }
+        int exitCode = process.waitFor();
+        if (exitCode != 0) {
+            throw new Exception("Command failed with exit code " + exitCode);
+        }
+    }
+
+    public static void moveFiles(String sourcePath, String targetPath) throws Exception {
+        Path sourceDir = Paths.get(sourcePath);
+        Path targetDir = Paths.get(targetPath).resolve(sourceDir.getFileName());
+
+        if (!Files.exists(targetDir)) {
+            Files.createDirectories(targetDir);
+        }
+
+        Files.move(sourceDir, targetDir, StandardCopyOption.REPLACE_EXISTING);
+    }
+
+    public static void gitClone(String localPath, String repoUrl, String branch, String username, String password) throws GitAPIException {
+        CloneCommand cloneCommand = Git.cloneRepository()
+                .setURI(repoUrl)
+                .setBranch(branch)
+                .setDirectory(new java.io.File(localPath))
+                .setCredentialsProvider(new UsernamePasswordCredentialsProvider(username, password));
+
+        cloneCommand.call();
+    }
+
+    public static void gitAdd(String localPath, String filePath) throws IOException, GitAPIException {
+        FileRepositoryBuilder builder = new FileRepositoryBuilder();
+        Repository repository = builder.setGitDir(new File(localPath, ".git"))
+                .readEnvironment()
+                .findGitDir()
+                .build();
+
+        try (Git git = new Git(repository)) {
+            AddCommand addCommand = git.add();
+            addCommand.addFilepattern(filePath).call();
+        }
+    }
+
+    public static void gitCommit(String localPath, String commitMessage) throws IOException, GitAPIException {
+        FileRepositoryBuilder builder = new FileRepositoryBuilder();
+        Repository repository = builder.setGitDir(new File(localPath, ".git"))
+                .readEnvironment()
+                .findGitDir()
+                .build();
+
+        try (Git git = new Git(repository)) {
+            // 添加所有文件
+            AddCommand addCommand = git.add();
+            addCommand.addFilepattern(".").call();
+
+            // 提交更改
+            CommitCommand commitCommand = git.commit();
+            commitCommand.setMessage(commitMessage).call();
+        }
+    }
+
+    public static void gitPush(String localPath, String username, String password) throws IOException, GitAPIException {
+        FileRepositoryBuilder builder = new FileRepositoryBuilder();
+        Repository repository = builder.setGitDir(new File(localPath, ".git"))
+                .readEnvironment()
+                .findGitDir()
+                .build();
+
+        try (Git git = new Git(repository)) {
+            CredentialsProvider credentialsProvider = new UsernamePasswordCredentialsProvider(username, password);
+            PushCommand pushCommand = git.push();
+            pushCommand.setCredentialsProvider(credentialsProvider)
+                    .setForce(true)
+                    .call();
+        }
+    }
+
+    public static void dvcInit(String localPath) throws Exception {
+        String command = "dvc init";
+        runCommand(command, localPath);
+    }
+
+    public static void dvcAdd(String localPath, String filePath) throws Exception {
+        String command = "dvc add " + filePath;
+        runCommand(command, localPath);
+    }
+
+    public static void dvcRemoteAdd(String localPath, String s3RemoteUrl) throws Exception {
+        String command = "dvc remote add -d myremote s3://" + s3RemoteUrl;
+        runCommand(command, localPath);
+    }
+
+    public static void dvcConfigS3Credentials(String localPath, String endpointurl) throws Exception {
+        String command = "dvc remote modify myremote endpointurl " + endpointurl  ;
+        runCommand(command, localPath);
+    }
+    public static void dvcConfigS3Credentials2(String localPath, String accessKeyId) throws Exception {
+        String command = "dvc remote modify myremote access_key_id " + accessKeyId;
+        runCommand(command, localPath);
+    }
+    public static void dvcConfigS3Credentials3(String localPath, String secretAccessKey) throws Exception {
+        String command = "dvc remote modify myremote secret_access_key " + secretAccessKey;
+        runCommand(command, localPath);
+    }
+
+    public static void dvcPush(String localPath) throws Exception {
+        String command = "dvc push";
+        runCommand(command, localPath);
+    }
+
+    public static void dvcPull(String localPath) throws Exception {
+        String command = "dvc pull";
+        runCommand(command, localPath);
+    }
+}

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/HttpUtils.java

@@ -89,6 +89,64 @@ public class HttpUtils {
     public static String sendGet(String url, String param) {
         return sendGet(url, param, "UTF-8");
     }
+    /**
+     * 向指定 URL 发送带token的GET方法的请求
+     *
+     * @param url   发送请求的 URL
+     * @param param 请求参数，请求参数应该是 name1=value1&name2=value2 的形式。
+     * @return 所代表远程资源的响应结果
+     */
+    public static String sendGetWithToken(String url, String param,String token) {
+        return sendGet(url, param, "UTF-8",token);
+    }
+    /**
+     * 向指定 URL 发送带token的GET方法的请求
+     *
+     * @param url         发送请求的 URL
+     * @param param       请求参数，请求参数应该是 name1=value1&name2=value2 的形式。
+     * @param contentType 编码类型
+     * @return 所代表远程资源的响应结果
+     */
+    public static String sendGet(String url, String param, String contentType,String token) {
+        StringBuilder result = new StringBuilder();
+        BufferedReader in = null;
+        try {
+            String urlNameString = url + "?" + param;
+            log.info("sendGet - {}", urlNameString);
+            URL realUrl = new URL(urlNameString);
+            URLConnection connection = realUrl.openConnection();
+            connection.setRequestProperty("accept", "*/*");
+            connection.setRequestProperty("connection", "Keep-Alive");
+            connection.setRequestProperty("user-agent", "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1;SV1)");
+            connection.setRequestProperty("Authorization", "Bearer " + token);
+            connection.connect();
+            in = new BufferedReader(new InputStreamReader(connection.getInputStream(), contentType));
+            String line;
+            while ((line = in.readLine()) != null) {
+                result.append(line);
+            }
+            log.info("recv - {}", result);
+        } catch (ConnectException e) {
+            log.error("调用HttpUtils.sendGet ConnectException, url=" + url + ",param=" + param, e);
+        } catch (SocketTimeoutException e) {
+            log.error("调用HttpUtils.sendGet SocketTimeoutException, url=" + url + ",param=" + param, e);
+        } catch (IOException e) {
+            log.error("调用HttpUtils.sendGet IOException, url=" + url + ",param=" + param, e);
+        } catch (Exception e) {
+            log.error("调用HttpsUtil.sendGet Exception, url=" + url + ",param=" + param, e);
+        } finally {
+            try {
+                if (in != null) {
+                    in.close();
+                }
+            } catch (Exception ex) {
+                log.error("调用in.close Exception, url=" + url + ",param=" + param, ex);
+            }
+        }
+        return result.toString();
+    }
+
+
 
     /**
      * 向指定 URL 发送GET方法的请求
@@ -108,7 +166,6 @@ public class HttpUtils {
             URLConnection connection = realUrl.openConnection();
             connection.setRequestProperty("accept", "*/*");
             connection.setRequestProperty("connection", "Keep-Alive");
-            connection.setRequestProperty("user-agent", "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1;SV1)");
             connection.connect();
             in = new BufferedReader(new InputStreamReader(connection.getInputStream(), contentType));
             String line;
@@ -136,6 +193,42 @@ public class HttpUtils {
         return result.toString();
     }
 
+    /**
+     * 向指定 URL 发送带token的POST方法的请求
+     *
+     * @param url   发送请求的 URL
+     * @param
+     * @return 所代表远程资源的响应结果
+     */
+    public static String sendPostWithToken(String url,  Object params, String token) throws Exception  {
+        String resultStr = null;
+        HttpPost httpPost = new HttpPost(url);
+        if (params != null) {
+            StringEntity entity;
+            if (params instanceof Map) {
+                entity = new StringEntity(dealPostParams((HashMap<String, String>) params, "UTF-8"));
+            } else if (params instanceof String) {
+                entity = new StringEntity((String) params, "UTF-8");
+            } else if (params instanceof List) {
+                entity = new UrlEncodedFormEntity((List<? extends NameValuePair>) params, "UTF-8");
+            } else {
+                throw new Exception("参数有误!");
+            }
+            httpPost.setHeader("Content-Type", "application/json");
+            httpPost.setHeader("Authorization", "Bearer " + token);
+            httpPost.setEntity(entity);
+        }
+        try {
+            HttpResponse response = httpClient.execute(httpPost);
+            if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
+                resultStr = EntityUtils.toString(response.getEntity());
+            }
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+        return resultStr;
+    }
+
     /**
      * 向指定 URL 发送POST方法的请求
      *

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/DatasetVo.java

@@ -3,11 +3,13 @@ package com.ruoyi.platform.vo;
 import com.fasterxml.jackson.databind.PropertyNamingStrategy;
 import com.fasterxml.jackson.databind.annotation.JsonNaming;
 import io.swagger.annotations.ApiModelProperty;
+import lombok.Data;
 
 import java.io.Serializable;
 import java.util.List;
 
 @JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy.class)
+@Data
 public class DatasetVo implements Serializable {
 
 
@@ -25,6 +27,11 @@ public class DatasetVo implements Serializable {
     private String dataType;
     @ApiModelProperty(name = "data_tag")
     private String dataTag;
+
+    @ApiModelProperty(name = "gitusername")
+    private String gitusername;
+    @ApiModelProperty(name = "gitpassword")
+    private String gitpassword;
     /**
      * 版本
      */
@@ -33,73 +40,11 @@ public class DatasetVo implements Serializable {
 
     @ApiModelProperty(name = "dataset_version_vos")
     private List<VersionVo> datasetVersionVos;
+
     /**
      * 可用集群
      */
     @ApiModelProperty(name = "available_cluster")
     private String availableCluster;
 
-    public String getName() {
-        return name;
-    }
-
-    public void setName(String name) {
-        this.name = name;
-    }
-
-    public String getDescription() {
-        return description;
-    }
-
-    public void setDescription(String description) {
-        this.description = description;
-    }
-
-    public int getAvailableRange() {
-        return availableRange;
-    }
-
-    public void setAvailableRange(int availableRange) {
-        this.availableRange = availableRange;
-    }
-
-    public String getDataType() {
-        return dataType;
-    }
-
-    public void setDataType(String dataType) {
-        this.dataType = dataType;
-    }
-
-    public String getDataTag() {
-        return dataTag;
-    }
-
-    public void setDataTag(String dataTag) {
-        this.dataTag = dataTag;
-    }
-
-    public String getVersion() {
-        return version;
-    }
-
-    public void setVersion(String version) {
-        this.version = version;
-    }
-
-    public List<VersionVo> getDatasetVersionVos() {
-        return datasetVersionVos;
-    }
-
-    public void setDatasetVersionVos(List<VersionVo> datasetVersionVos) {
-        this.datasetVersionVos = datasetVersionVos;
-    }
-
-    public String getAvailableCluster() {
-        return availableCluster;
-    }
-
-    public void setAvailableCluster(String availableCluster) {
-        this.availableCluster = availableCluster;
-    }
 }