优化dvc测试

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java

@@ -104,9 +104,12 @@ public class NewDatasetServiceImpl implements NewDatasetService {
         String gitLinkUsername = loginUser.getSysUser().getUserName();
         String gitLinkPassword = decrypt(loginUser.getSysUser().getOriginPassword());
         String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
+
         // 得到用户操作的路径
         Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
         Integer userId = (Integer) userInfo.get("user_id");
+        String owner = (String) userInfo.get("login");
+
         // 拼接project
         ci4sUsername = Boolean.TRUE.equals(datasetVo.getIsPublic()) ? Constant.Item_Public : loginUser.getUsername();
         String repositoryName = ci4sUsername + "_dataset_" + DateUtils.dateTimeNow();
@@ -117,17 +120,13 @@ public class NewDatasetServiceImpl implements NewDatasetService {
         gitProjectVo.setPrivate(!datasetVo.getIsPublic());
         gitProjectVo.setUserId(userId);
         gitProjectVo.setProjectCategoryId(projectDatasetId);
+
         // 创建项目
         Map project = gitService.createProject(token, gitProjectVo);
         Integer gitlinIid = (Integer) project.get("id");
         if (gitlinIid == null) {
             throw new Exception("创建数据集失败:" + project.get("message"));
         }
-        // 创建分支
-        String branchName = datasetVo.getVersion();
-        String owner = (String) userInfo.get("login");
-
-        gitService.createBranch(token, (String) userInfo.get("login"), repositoryName, branchName, "master");
         // 定义标签 标签1：dataset 标签2：DataTag 标签3：DataType
         gitService.createTopic(token, gitlinIid, Constant.Topic_Dataset);
         if (StringUtils.isNotEmpty(datasetVo.getDataTag())) {
@@ -137,23 +136,59 @@ public class NewDatasetServiceImpl implements NewDatasetService {
             gitService.createTopic(token, gitlinIid, "datatype-" + datasetVo.getDataType());
         }
 
+        String relatePath = ci4sUsername + "/datasets/" + gitlinIid + "/" + repositoryName + "/master";
+        String localPath = localPathlocal + relatePath;
+        String datasetPath = localPath + "/dataset";
+        String s3Path = bucketName + "/mini-model-management-platform-files" + "/" + relatePath;
+
         // 得到项目地址
         String projectUrl = gitCloneEndpoint + "/" + owner + "/" + repositoryName + ".git";
 
+        // 创建master空分支
+        // 拼接生产的元数据后写入yaml文件
+        datasetVo.setCreateBy(String.valueOf(StringUtils.isNotEmpty((String) userInfo.get("nickname")) ? userInfo.get("nickname") : userInfo.get("login")));
+        datasetVo.setUpdateTime(DateUtils.getTime());
+        datasetVo.setIdentifier(repositoryName);
+        datasetVo.setId(gitlinIid);
+        datasetVo.setOwner(owner);
+        datasetVo.setRelativePaths(relatePath + "/dataset");
+        addDatasetSourceToDataVo(datasetVo);
+        YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo), localPath, "dataset");
+
+        // dvc init 初始化
+        dvcUtils.mkdir(datasetPath);
+        dvcUtils.dvcInit(localPath);
+        // 配置远程S3地址
+        dvcUtils.dvcRemoteAdd(localPath, s3Path);
+        dvcUtils.dvcConfigS3Credentials(localPath, endpoint);
+        dvcUtils.dvcConfigS3Credentials2(localPath, accessKeyId);
+        dvcUtils.dvcConfigS3Credentials3(localPath, secretAccessKey);
+        // dvc 跟踪
+        dvcUtils.dvcAdd(localPath, "dataset");
+        // git commit
+        dvcUtils.gitCommit(localPath, "commit from ci4s with " + loginUser.getUsername());
+        dvcUtils.gitPush(localPath, gitLinkUsername, gitLinkPassword);
+        // dvc push 到远程S3
+        dvcUtils.dvcPush(localPath);
+
+
+        // 创建分支
+        String branchName = datasetVo.getVersion();
+        gitService.createBranch(token, owner, repositoryName, branchName, "master");
+
         // 得到用户操作的路径
         String sourcePath = datasetVo.getDatasetVersionVos().get(0).getUrl();
-        String relatePath = ci4sUsername + "/datasets/" + gitlinIid + "/" + repositoryName + "/" + branchName;
-        String localPath = localPathlocal + relatePath;
-        String datasetPath = localPath + "/dataset";
+        String relatePath1 = ci4sUsername + "/datasets/" + gitlinIid + "/" + repositoryName + "/" + branchName;
+        String localPath1 = localPathlocal + relatePath1;
+        String datasetPath1 = localPath1 + "/dataset";
 
         // 命令行操作 git clone 项目地址
-        dvcUtils.gitClone(localPath, projectUrl, branchName, gitLinkUsername, gitLinkPassword);
-        String s3Path = bucketName + "/mini-model-management-platform-files" + "/" + relatePath + "/" + branchName;
-        //干掉目标文件夹
-        dvcUtils.deleteDirectory(datasetPath);
-        dvcUtils.moveFiles(sourcePath, datasetPath);
+        dvcUtils.gitClone(localPath1, projectUrl, branchName, gitLinkUsername, gitLinkPassword);
+        String s3Path1 = bucketName + "/mini-model-management-platform-files" + "/" + relatePath1;
+        //删掉已存在文件夹
+        dvcUtils.deleteDirectory(datasetPath1);
+        dvcUtils.moveFiles(sourcePath, datasetPath1);
         // 拼接生产的元数据后写入yaml文件
-        datasetVo.setCreateBy(String.valueOf(StringUtils.isNotEmpty((String) userInfo.get("nickname")) ? userInfo.get("nickname") : userInfo.get("login")));
         datasetVo.setUpdateTime(DateUtils.getTime());
         datasetVo.setUsage("<pre><code>" +
                 "# 克隆数据集配置文件与存储参数到本地\n" +
@@ -161,31 +196,24 @@ public class NewDatasetServiceImpl implements NewDatasetService {
                 "# 远程拉取配置文件\n" +
                 "dvc pull\n" +
                 "</code></pre>");
-        datasetVo.setIdentifier(repositoryName);
-        datasetVo.setId(gitlinIid);
-        datasetVo.setOwner(owner);
-        datasetVo.setRelativePaths(relatePath + "/dataset");
-
-        addDatasetSourceToDataVo(datasetVo);
-
-        YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo), localPath, "dataset");
+        datasetVo.setRelativePaths(relatePath1 + "/dataset");
+        YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo), localPath1, "dataset");
 
         CompletableFuture.supplyAsync(() -> {
             try {
-                // dvc init 初始化
-                dvcUtils.dvcInit(localPath);
+                dvcUtils.dvcInit(localPath1);
                 // 配置远程S3地址
-                dvcUtils.dvcRemoteAdd(localPath, s3Path);
-                dvcUtils.dvcConfigS3Credentials(localPath, endpoint);
-                dvcUtils.dvcConfigS3Credentials2(localPath, accessKeyId);
-                dvcUtils.dvcConfigS3Credentials3(localPath, secretAccessKey);
+                dvcUtils.dvcRemoteAdd(localPath1, s3Path1);
+                dvcUtils.dvcConfigS3Credentials(localPath1, endpoint);
+                dvcUtils.dvcConfigS3Credentials2(localPath1, accessKeyId);
+                dvcUtils.dvcConfigS3Credentials3(localPath1, secretAccessKey);
                 // dvc 跟踪
-                dvcUtils.dvcAdd(localPath, "dataset");
+                dvcUtils.dvcAdd(localPath1, "dataset");
                 // git commit
-                dvcUtils.gitCommit(localPath, "commit from ci4s with " + loginUser.getUsername());
-                dvcUtils.gitPush(localPath, gitLinkUsername, gitLinkPassword);
+                dvcUtils.gitCommit(localPath1, "commit from ci4s with " + loginUser.getUsername());
+                dvcUtils.gitPush(localPath1, gitLinkUsername, gitLinkPassword);
                 // dvc push 到远程S3
-                dvcUtils.dvcPush(localPath);
+                dvcUtils.dvcPush(localPath1);
             } catch (Exception e) {
                 logger.error(e.getMessage(), e);
                 throw new RuntimeException(e.getMessage());

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/DVCUtils.java

@@ -68,6 +68,13 @@ public class DVCUtils {
         }
     }
 
+    public void mkdir(String path) throws IOException {
+        Path targetDir = Paths.get(path);
+        if (!Files.exists(targetDir)) {
+            Files.createDirectories(targetDir);
+        }
+    }
+
     public void moveFiles(String sourcePath, String targetPath) throws Exception {
         Path sourceDir = Paths.get(sourcePath);
         Path targetDir = Paths.get(targetPath);