From 4670b2bb5edd072647b2fe2944391bdaf859560b Mon Sep 17 00:00:00 2001 From: chenzhihang <709011834@qq.com> Date: Thu, 5 Jun 2025 20:22:27 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96dvc=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/impl/NewDatasetServiceImpl.java | 92 ++++++++++++------- .../com/ruoyi/platform/utils/DVCUtils.java | 7 ++ 2 files changed, 67 insertions(+), 32 deletions(-) diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java index 7f46d291..481bf03b 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java @@ -104,9 +104,12 @@ public class NewDatasetServiceImpl implements NewDatasetService { String gitLinkUsername = loginUser.getSysUser().getUserName(); String gitLinkPassword = decrypt(loginUser.getSysUser().getOriginPassword()); String userReq = jedis.get(ci4sUsername + "_gitUserInfo"); + // 得到用户操作的路径 Map userInfo = JsonUtils.jsonToMap(userReq); Integer userId = (Integer) userInfo.get("user_id"); + String owner = (String) userInfo.get("login"); + // 拼接project ci4sUsername = Boolean.TRUE.equals(datasetVo.getIsPublic()) ? Constant.Item_Public : loginUser.getUsername(); String repositoryName = ci4sUsername + "_dataset_" + DateUtils.dateTimeNow(); @@ -117,17 +120,13 @@ public class NewDatasetServiceImpl implements NewDatasetService { gitProjectVo.setPrivate(!datasetVo.getIsPublic()); gitProjectVo.setUserId(userId); gitProjectVo.setProjectCategoryId(projectDatasetId); + // 创建项目 Map project = gitService.createProject(token, gitProjectVo); Integer gitlinIid = (Integer) project.get("id"); if (gitlinIid == null) { throw new Exception("创建数据集失败:" + project.get("message")); } - // 创建分支 - String branchName = datasetVo.getVersion(); - String owner = (String) userInfo.get("login"); - - gitService.createBranch(token, (String) userInfo.get("login"), repositoryName, branchName, "master"); // 定义标签 标签1：dataset 标签2：DataTag 标签3：DataType gitService.createTopic(token, gitlinIid, Constant.Topic_Dataset); if (StringUtils.isNotEmpty(datasetVo.getDataTag())) { @@ -137,23 +136,59 @@ public class NewDatasetServiceImpl implements NewDatasetService { gitService.createTopic(token, gitlinIid, "datatype-" + datasetVo.getDataType()); } + String relatePath = ci4sUsername + "/datasets/" + gitlinIid + "/" + repositoryName + "/master"; + String localPath = localPathlocal + relatePath; + String datasetPath = localPath + "/dataset"; + String s3Path = bucketName + "/mini-model-management-platform-files" + "/" + relatePath; + // 得到项目地址 String projectUrl = gitCloneEndpoint + "/" + owner + "/" + repositoryName + ".git"; + // 创建master空分支 + // 拼接生产的元数据后写入yaml文件 + datasetVo.setCreateBy(String.valueOf(StringUtils.isNotEmpty((String) userInfo.get("nickname")) ? userInfo.get("nickname") : userInfo.get("login"))); + datasetVo.setUpdateTime(DateUtils.getTime()); + datasetVo.setIdentifier(repositoryName); + datasetVo.setId(gitlinIid); + datasetVo.setOwner(owner); + datasetVo.setRelativePaths(relatePath + "/dataset"); + addDatasetSourceToDataVo(datasetVo); + YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo), localPath, "dataset"); + + // dvc init 初始化 + dvcUtils.mkdir(datasetPath); + dvcUtils.dvcInit(localPath); + // 配置远程S3地址 + dvcUtils.dvcRemoteAdd(localPath, s3Path); + dvcUtils.dvcConfigS3Credentials(localPath, endpoint); + dvcUtils.dvcConfigS3Credentials2(localPath, accessKeyId); + dvcUtils.dvcConfigS3Credentials3(localPath, secretAccessKey); + // dvc 跟踪 + dvcUtils.dvcAdd(localPath, "dataset"); + // git commit + dvcUtils.gitCommit(localPath, "commit from ci4s with " + loginUser.getUsername()); + dvcUtils.gitPush(localPath, gitLinkUsername, gitLinkPassword); + // dvc push 到远程S3 + dvcUtils.dvcPush(localPath); + + + // 创建分支 + String branchName = datasetVo.getVersion(); + gitService.createBranch(token, owner, repositoryName, branchName, "master"); + // 得到用户操作的路径 String sourcePath = datasetVo.getDatasetVersionVos().get(0).getUrl(); - String relatePath = ci4sUsername + "/datasets/" + gitlinIid + "/" + repositoryName + "/" + branchName; - String localPath = localPathlocal + relatePath; - String datasetPath = localPath + "/dataset"; + String relatePath1 = ci4sUsername + "/datasets/" + gitlinIid + "/" + repositoryName + "/" + branchName; + String localPath1 = localPathlocal + relatePath1; + String datasetPath1 = localPath1 + "/dataset"; // 命令行操作 git clone 项目地址 - dvcUtils.gitClone(localPath, projectUrl, branchName, gitLinkUsername, gitLinkPassword); - String s3Path = bucketName + "/mini-model-management-platform-files" + "/" + relatePath + "/" + branchName; - //干掉目标文件夹 - dvcUtils.deleteDirectory(datasetPath); - dvcUtils.moveFiles(sourcePath, datasetPath); + dvcUtils.gitClone(localPath1, projectUrl, branchName, gitLinkUsername, gitLinkPassword); + String s3Path1 = bucketName + "/mini-model-management-platform-files" + "/" + relatePath1; + //删掉已存在文件夹 + dvcUtils.deleteDirectory(datasetPath1); + dvcUtils.moveFiles(sourcePath, datasetPath1); // 拼接生产的元数据后写入yaml文件 - datasetVo.setCreateBy(String.valueOf(StringUtils.isNotEmpty((String) userInfo.get("nickname")) ? userInfo.get("nickname") : userInfo.get("login"))); datasetVo.setUpdateTime(DateUtils.getTime()); datasetVo.setUsage("
" +
                 "# 克隆数据集配置文件与存储参数到本地\n" +
@@ -161,31 +196,24 @@ public class NewDatasetServiceImpl implements NewDatasetService {
                 "# 远程拉取配置文件\n" +
                 "dvc pull\n" +
                 "
"); - datasetVo.setIdentifier(repositoryName); - datasetVo.setId(gitlinIid); - datasetVo.setOwner(owner); - datasetVo.setRelativePaths(relatePath + "/dataset"); - - addDatasetSourceToDataVo(datasetVo); - - YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo), localPath, "dataset"); + datasetVo.setRelativePaths(relatePath1 + "/dataset"); + YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo), localPath1, "dataset"); CompletableFuture.supplyAsync(() -> { try { - // dvc init 初始化 - dvcUtils.dvcInit(localPath); + dvcUtils.dvcInit(localPath1); // 配置远程S3地址 - dvcUtils.dvcRemoteAdd(localPath, s3Path); - dvcUtils.dvcConfigS3Credentials(localPath, endpoint); - dvcUtils.dvcConfigS3Credentials2(localPath, accessKeyId); - dvcUtils.dvcConfigS3Credentials3(localPath, secretAccessKey); + dvcUtils.dvcRemoteAdd(localPath1, s3Path1); + dvcUtils.dvcConfigS3Credentials(localPath1, endpoint); + dvcUtils.dvcConfigS3Credentials2(localPath1, accessKeyId); + dvcUtils.dvcConfigS3Credentials3(localPath1, secretAccessKey); // dvc 跟踪 - dvcUtils.dvcAdd(localPath, "dataset"); + dvcUtils.dvcAdd(localPath1, "dataset"); // git commit - dvcUtils.gitCommit(localPath, "commit from ci4s with " + loginUser.getUsername()); - dvcUtils.gitPush(localPath, gitLinkUsername, gitLinkPassword); + dvcUtils.gitCommit(localPath1, "commit from ci4s with " + loginUser.getUsername()); + dvcUtils.gitPush(localPath1, gitLinkUsername, gitLinkPassword); // dvc push 到远程S3 - dvcUtils.dvcPush(localPath); + dvcUtils.dvcPush(localPath1); } catch (Exception e) { logger.error(e.getMessage(), e); throw new RuntimeException(e.getMessage()); diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/DVCUtils.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/DVCUtils.java index 45002332..819a5e53 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/DVCUtils.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/DVCUtils.java @@ -68,6 +68,13 @@ public class DVCUtils { } } + public void mkdir(String path) throws IOException { + Path targetDir = Paths.get(path); + if (!Files.exists(targetDir)) { + Files.createDirectories(targetDir); + } + } + public void moveFiles(String sourcePath, String targetPath) throws Exception { Path sourceDir = Paths.get(sourcePath); Path targetDir = Paths.get(targetPath);