From 35f7f4aa5e495bac77638bca92469d9af99d5659 Mon Sep 17 00:00:00 2001 From: chenzhihang <709011834@qq.com> Date: Fri, 6 Jun 2025 15:56:50 +0800 Subject: [PATCH] =?UTF-8?q?=E5=9B=9E=E9=80=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/impl/ModelsServiceImpl.java | 96 +++++++++--------- .../service/impl/NewDatasetServiceImpl.java | 98 ++++++++++++------- .../com/ruoyi/platform/utils/DVCUtils.java | 7 -- 3 files changed, 113 insertions(+), 88 deletions(-) diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java index 6df28844..ad9a1ac7 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java @@ -586,6 +586,7 @@ public class ModelsServiceImpl implements ModelsService { } //新建模型 + // 拼接project GitProjectVo gitProjectVo = new GitProjectVo(); gitProjectVo.setRepositoryName(repositoryName); gitProjectVo.setName(Constant.Topic_model + "-" + modelsVo.getName()); @@ -601,6 +602,18 @@ public class ModelsServiceImpl implements ModelsService { throw new RuntimeException("创建模型失败:" + project.get("message")); } + String branchName = modelsVo.getVersion(); + String owner = (String) userInfo.get("login"); + + String projectUrl = gitCloneEndpoint + "/" + owner + "/" + repositoryName + ".git"; + String sourcePath = modelsVo.getModelVersionVos().get(0).getUrl(); + String rootPath = localPath + ci4sUsername + "/model/" + gitlinIid + "/" + repositoryName + "/" + branchName; + String modelPath = rootPath + "/model"; + String metaPath = rootPath + "/metadata"; + String relatePath = ci4sUsername + "/model/" + gitlinIid + "/" + repositoryName + "/" + branchName + "/model"; + + // 创建分支 + gitService.createBranch(token, (String) userInfo.get("login"), repositoryName, branchName, "master"); // 定义标签 标签1：model 标签2：ModelTag 标签3：ModelType gitService.createTopic(token, gitlinIid, Constant.Topic_model); if (StringUtils.isNotEmpty(modelsVo.getModelTag())) { @@ -610,17 +623,11 @@ public class ModelsServiceImpl implements ModelsService { gitService.createTopic(token, gitlinIid, "modeltype-" + modelsVo.getModelType()); } - String owner = (String) userInfo.get("login"); - String projectUrl = gitCloneEndpoint + "/" + owner + "/" + repositoryName + ".git"; - String relationPath = ci4sUsername + "/model/" + gitlinIid + "/" + repositoryName + "/origin"; - String rootPath = localPath + relationPath; - String modelPath = rootPath + "/model"; - String metaPath = rootPath + "/metadata"; - String relatePath = relationPath + "/model"; + dvcUtils.gitClone(rootPath, projectUrl, branchName, gitLinkUsername, decryptGitLinkPassword); - // 创建origin分支 - gitService.createBranch(token, owner, repositoryName, "origin", "master"); - dvcUtils.gitClone(rootPath, projectUrl, "origin", gitLinkUsername, decryptGitLinkPassword); + //干掉目标文件夹 + dvcUtils.deleteDirectory(modelPath); + dvcUtils.moveFiles(sourcePath, modelPath); //拼接生产的元数据后写入yaml文件 ModelMetaVo modelMetaVo = new ModelMetaVo(); @@ -629,13 +636,19 @@ public class ModelsServiceImpl implements ModelsService { modelMetaVo.setCreateBy(String.valueOf(StringUtils.isNotEmpty((String) userInfo.get("nickname")) ? userInfo.get("nickname") : userInfo.get("login"))); modelMetaVo.setCreateTime(DateUtils.getTime()); modelMetaVo.setUpdateTime(DateUtils.getTime()); + String projectUrlIp = gitCloneEndpointIp + "/" + owner + "/" + repositoryName + ".git"; + modelMetaVo.setUsage("
" +
+                    "# 克隆模型配置文件与存储参数到本地\n" +
+                    "git clone -b " + branchName + " " + projectUrlIp + "\n" +
+                    "# 远程拉取配置文件\n" +
+                    "dvc pull\n" +
+                    "
"); modelMetaVo.setIdentifier(repositoryName); modelMetaVo.setOwner(owner); modelMetaVo.setRelativePaths(relatePath); File folder = new File(modelPath); long folderSize = FileUtil.getFolderSize(folder); modelMetaVo.setModelSize(FileUtil.formatFileSize(folderSize)); - BeanUtils.copyProperties(modelMetaVo, modelsVo); Map metaMap = JSON.parseObject(JSON.toJSONString(modelMetaVo), Map.class); YamlUtils.generateYamlFile(metaMap, metaPath, "metadata"); @@ -645,39 +658,35 @@ public class ModelsServiceImpl implements ModelsService { modelDependency.setRepoId(gitlinIid); modelDependency.setIdentifier(repositoryName); modelDependency.setModelName(modelsVo.getName()); - modelDependency.setVersion("origin"); + modelDependency.setVersion(modelsVo.getVersion()); modelDependency.setParentModel(modelsVo.getParentModel()); modelDependency.setOwner(gitLinkUsername); modelDependency.setMeta(meta); modelDependency.setState(Constant.State_valid); modelDependency1Dao.insert(modelDependency); - String s3Path = bucketName + "/mini-model-management-platform-files/" + relationPath; -// CompletableFuture.supplyAsync(() -> { -// try { - dvcUtils.mkdir(modelPath); - dvcUtils.dvcInit(rootPath); - // 配置远程S3地址 - dvcUtils.dvcRemoteAdd(rootPath, s3Path); - dvcUtils.dvcConfigS3Credentials(rootPath, endpoint); - dvcUtils.dvcConfigS3Credentials2(rootPath, accessKeyId); - dvcUtils.dvcConfigS3Credentials3(rootPath, secretAccessKey); - - // dvc 跟踪 - dvcUtils.dvcAdd(rootPath, "model"); - // git commit - dvcUtils.gitCommit(rootPath, "commit from ci4s with " + loginUser.getUsername()); - dvcUtils.gitPush(rootPath, gitLinkUsername, decryptGitLinkPassword); - dvcUtils.dvcPush(rootPath); - - // 创建分支 - newCreateVersion(modelsVo); -// } catch (Exception e) { -// logger.error(e.getMessage(), e); -// throw new RuntimeException(e.getMessage()); -// } -// return null; -// }); + String s3Path = bucketName + "/mini-model-management-platform-files/" + ci4sUsername + "/model/" + gitlinIid + "/" + repositoryName + "/" + branchName; + CompletableFuture.supplyAsync(() -> { + try { + dvcUtils.dvcInit(rootPath); + // 配置远程S3地址 + dvcUtils.dvcRemoteAdd(rootPath, s3Path); + dvcUtils.dvcConfigS3Credentials(rootPath, endpoint); + dvcUtils.dvcConfigS3Credentials2(rootPath, accessKeyId); + dvcUtils.dvcConfigS3Credentials3(rootPath, secretAccessKey); + + // dvc 跟踪 + dvcUtils.dvcAdd(rootPath, "model"); + // git commit + dvcUtils.gitCommit(rootPath, "commit from ci4s with " + loginUser.getUsername()); + dvcUtils.gitPush(rootPath, gitLinkUsername, decryptGitLinkPassword); + dvcUtils.dvcPush(rootPath); + } catch (Exception e) { + logger.error(e.getMessage(), e); + throw new RuntimeException(e.getMessage()); + } + return null; + }); return "新增模型成功"; } catch (Exception e) { @@ -718,8 +727,8 @@ public class ModelsServiceImpl implements ModelsService { FileUtil.renameFile(rootPath, rootPath + "_deleted_" + System.currentTimeMillis()); //克隆 - dvcUtils.gitClone(rootPath, projectUrl, "origin", gitLinkUsername, gitLinkPassword); - dvcUtils.refreshRemoteBranches(rootPath, gitLinkUsername, gitLinkPassword, "origin"); + dvcUtils.gitClone(rootPath, projectUrl, oldModelDependencys.get(0).getVersion(), gitLinkUsername, gitLinkPassword); + dvcUtils.refreshRemoteBranches(rootPath, gitLinkUsername, gitLinkPassword, oldModelDependencys.get(0).getVersion()); //查询项目信息 String token = gitService.checkoutToken(); @@ -766,10 +775,9 @@ public class ModelsServiceImpl implements ModelsService { modelMetaVo.setCreateBy(String.valueOf(StringUtils.isNotEmpty((String) userInfo.get("nickname")) ? userInfo.get("nickname") : userInfo.get("login"))); modelMetaVo.setCreateTime(DateUtils.getTime()); modelMetaVo.setUpdateTime(DateUtils.getTime()); - String projectUrlIp = gitCloneEndpointIp + "/" + owner + "/" + repositoryName + ".git"; modelMetaVo.setUsage("
" +
                     "# 克隆模型配置文件与存储参数到本地\n" +
-                    "git clone -b " + branchName + " " + projectUrlIp + "\n" +
+                    "git clone -b " + branchName + " " + projectUrl + "\n" +
                     "# 远程拉取配置文件\n" +
                     "dvc pull\n" +
                     "
"); @@ -976,7 +984,7 @@ public class ModelsServiceImpl implements ModelsService { String token = gitService.checkoutToken(); List> collect = gitService.getBrancheList(token, owner, identifier); List> result = collect.stream() - .filter(branch -> !"master".equals(branch.get("name")) && !"origin".equals(branch.get("name"))) + .filter(branch -> !"master".equals(branch.get("name"))) .skip((pageRequest.getPageNumber()) * pageRequest.getPageSize()).limit(pageRequest.getPageSize()) .collect(Collectors.toList()); @@ -1079,7 +1087,7 @@ public class ModelsServiceImpl implements ModelsService { String token = gitService.checkoutToken(); List> branchList = gitService.getBrancheList(token, owner, identifier); return branchList.stream() - .filter(branch -> !"master".equals(branch.get("name")) && !"origin".equals(branch.get("name"))) + .filter(branch -> !"master".equals(branch.get("name"))) .collect(Collectors.toList()); } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java index 24bc4fca..d238b7b4 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java @@ -104,12 +104,9 @@ public class NewDatasetServiceImpl implements NewDatasetService { String gitLinkUsername = loginUser.getSysUser().getUserName(); String gitLinkPassword = decrypt(loginUser.getSysUser().getOriginPassword()); String userReq = jedis.get(ci4sUsername + "_gitUserInfo"); - // 得到用户操作的路径 Map userInfo = JsonUtils.jsonToMap(userReq); Integer userId = (Integer) userInfo.get("user_id"); - String owner = (String) userInfo.get("login"); - // 拼接project ci4sUsername = Boolean.TRUE.equals(datasetVo.getIsPublic()) ? Constant.Item_Public : loginUser.getUsername(); String repositoryName = ci4sUsername + "_dataset_" + DateUtils.dateTimeNow(); @@ -120,13 +117,17 @@ public class NewDatasetServiceImpl implements NewDatasetService { gitProjectVo.setPrivate(!datasetVo.getIsPublic()); gitProjectVo.setUserId(userId); gitProjectVo.setProjectCategoryId(projectDatasetId); - // 创建项目 Map project = gitService.createProject(token, gitProjectVo); Integer gitlinIid = (Integer) project.get("id"); if (gitlinIid == null) { throw new Exception("创建数据集失败:" + project.get("message")); } + // 创建分支 + String branchName = datasetVo.getVersion(); + String owner = (String) userInfo.get("login"); + + gitService.createBranch(token, (String) userInfo.get("login"), repositoryName, branchName, "master"); // 定义标签 标签1：dataset 标签2：DataTag 标签3：DataType gitService.createTopic(token, gitlinIid, Constant.Topic_Dataset); if (StringUtils.isNotEmpty(datasetVo.getDataTag())) { @@ -136,44 +137,61 @@ public class NewDatasetServiceImpl implements NewDatasetService { gitService.createTopic(token, gitlinIid, "datatype-" + datasetVo.getDataType()); } - String relatePath = ci4sUsername + "/datasets/" + gitlinIid + "/" + repositoryName + "/origin"; - String localPath = localPathlocal + relatePath; - String datasetPath = localPath + "/dataset"; - String s3Path = bucketName + "/mini-model-management-platform-files" + "/" + relatePath; - // 得到项目地址 String projectUrl = gitCloneEndpoint + "/" + owner + "/" + repositoryName + ".git"; - // 创建origin分支 - gitService.createBranch(token, owner, repositoryName, "origin", "master"); - dvcUtils.gitClone(localPath, projectUrl, "origin", gitLinkUsername, gitLinkPassword); + // 得到用户操作的路径 + String sourcePath = datasetVo.getDatasetVersionVos().get(0).getUrl(); + String relatePath = ci4sUsername + "/datasets/" + gitlinIid + "/" + repositoryName + "/" + branchName; + String localPath = localPathlocal + relatePath; + String datasetPath = localPath + "/dataset"; + + // 命令行操作 git clone 项目地址 + dvcUtils.gitClone(localPath, projectUrl, branchName, gitLinkUsername, gitLinkPassword); + String s3Path = bucketName + "/mini-model-management-platform-files" + "/" + relatePath + "/" + branchName; + //干掉目标文件夹 + dvcUtils.deleteDirectory(datasetPath); + dvcUtils.moveFiles(sourcePath, datasetPath); // 拼接生产的元数据后写入yaml文件 datasetVo.setCreateBy(String.valueOf(StringUtils.isNotEmpty((String) userInfo.get("nickname")) ? userInfo.get("nickname") : userInfo.get("login"))); datasetVo.setUpdateTime(DateUtils.getTime()); + datasetVo.setUsage("
" +
+                "# 克隆数据集配置文件与存储参数到本地\n" +
+                "git clone -b " + branchName + " " + projectUrl + "\n" +
+                "# 远程拉取配置文件\n" +
+                "dvc pull\n" +
+                "
"); datasetVo.setIdentifier(repositoryName); datasetVo.setId(gitlinIid); datasetVo.setOwner(owner); datasetVo.setRelativePaths(relatePath + "/dataset"); - YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo), localPath, "dataset"); - // dvc init 初始化 - dvcUtils.mkdir(datasetPath); - dvcUtils.dvcInit(localPath); - // 配置远程S3地址 - dvcUtils.dvcRemoteAdd(localPath, s3Path); - dvcUtils.dvcConfigS3Credentials(localPath, endpoint); - dvcUtils.dvcConfigS3Credentials2(localPath, accessKeyId); - dvcUtils.dvcConfigS3Credentials3(localPath, secretAccessKey); - // dvc 跟踪 - dvcUtils.dvcAdd(localPath, "dataset"); - // git commit - dvcUtils.gitCommit(localPath, "commit from ci4s with " + loginUser.getUsername()); - dvcUtils.gitPush(localPath, gitLinkUsername, gitLinkPassword); - // dvc push 到远程S3 - dvcUtils.dvcPush(localPath); + addDatasetSourceToDataVo(datasetVo); - // 创建分支 - newCreateVersion(datasetVo); + YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo), localPath, "dataset"); + + CompletableFuture.supplyAsync(() -> { + try { + // dvc init 初始化 + dvcUtils.dvcInit(localPath); + // 配置远程S3地址 + dvcUtils.dvcRemoteAdd(localPath, s3Path); + dvcUtils.dvcConfigS3Credentials(localPath, endpoint); + dvcUtils.dvcConfigS3Credentials2(localPath, accessKeyId); + dvcUtils.dvcConfigS3Credentials3(localPath, secretAccessKey); + // dvc 跟踪 + dvcUtils.dvcAdd(localPath, "dataset"); + // git commit + dvcUtils.gitCommit(localPath, "commit from ci4s with " + loginUser.getUsername()); + dvcUtils.gitPush(localPath, gitLinkUsername, gitLinkPassword); + // dvc push 到远程S3 + dvcUtils.dvcPush(localPath); + } catch (Exception e) { + logger.error(e.getMessage(), e); + throw new RuntimeException(e.getMessage()); + } + return null; + }); return "新增数据集成功"; } @@ -196,7 +214,7 @@ public class NewDatasetServiceImpl implements NewDatasetService { ci4sUsername = Boolean.TRUE.equals(datasetVo.getIsPublic()) ? Constant.Item_Public : loginUser.getUsername(); Map userInfo = JsonUtils.jsonToMap(userReq); // 创建分支 - String branchName = datasetVo.getVersion(); + String branchName = StringUtils.isEmpty(datasetVo.getVersion()) ? "master" : datasetVo.getVersion(); String owner = (String) userInfo.get("login"); String repositoryName = datasetVo.getIdentifier(); @@ -214,10 +232,14 @@ public class NewDatasetServiceImpl implements NewDatasetService { Map projectDetail = gitService.getProjectDetail(owner, repositoryName, token); //克隆 - dvcUtils.gitClone(localPath, projectUrl, "origin", gitLinkUsername, gitLinkPassword); - dvcUtils.refreshRemoteBranches(localPath, gitLinkUsername, gitLinkPassword, "origin"); + List> brancheList = gitService.getBrancheList(token, owner, repositoryName); + String oldBranch = (String) brancheList.stream() + .filter(branch -> !"master".equals(branch.get("name"))).collect(Collectors.toList()).get(0).get("name"); + + dvcUtils.gitClone(localPath, projectUrl, oldBranch, gitLinkUsername, gitLinkPassword); + dvcUtils.refreshRemoteBranches(localPath, gitLinkUsername, gitLinkPassword, oldBranch); - //检查是否存在本地重名分支，有的话删掉 + //检查是否存在本地重名分支，有的话干掉 dvcUtils.deleteLocalBranch(localPath, branchName); // 创建本地分支 dvcUtils.createLocalBranchBasedOnMaster(localPath, branchName); @@ -227,8 +249,8 @@ public class NewDatasetServiceImpl implements NewDatasetService { dvcUtils.dvcCheckout(localPath); // 准备数据 - String s3Path = bucketName + "/mini-model-management-platform-files" + "/" + relatePath; - //删掉已存在文件夹 + String s3Path = bucketName + "/mini-model-management-platform-files" + "/" + relatePath + "/" + branchName; + //干掉目标文件夹 dvcUtils.deleteDirectory(datasetPath); if (Constant.Source_Hand_Export.equals(datasetVo.getDatasetSource())) { @@ -239,6 +261,7 @@ public class NewDatasetServiceImpl implements NewDatasetService { dvcUtils.moveFiles(sourcePath, datasetPath); } + // 拼接生产的元数据后写入yaml文件 datasetVo.setCreateBy(String.valueOf(StringUtils.isNotEmpty((String) userInfo.get("nickname")) ? userInfo.get("nickname") : userInfo.get("login"))); datasetVo.setUpdateTime(DateUtils.getTime()); @@ -280,6 +303,7 @@ public class NewDatasetServiceImpl implements NewDatasetService { // dvc 跟踪 dvcUtils.dvcAdd(localPath, "dataset"); dvcUtils.pushNewBranchToRemote(localPath, gitLinkUsername, gitLinkPassword, branchName); + dvcUtils.dvcPush(localPath); // CompletableFuture.supplyAsync(() -> { @@ -383,7 +407,7 @@ public class NewDatasetServiceImpl implements NewDatasetService { String token = gitService.checkoutToken(); List> brancheList = gitService.getBrancheList(token, owner, repo); return brancheList.stream() - .filter(branch -> !"master".equals(branch.get("name")) && !"origin".equals(branch.get("name"))) + .filter(branch -> !"master".equals(branch.get("name"))) .collect(Collectors.toList()); } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/DVCUtils.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/DVCUtils.java index d381764d..f3052f16 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/DVCUtils.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/DVCUtils.java @@ -68,13 +68,6 @@ public class DVCUtils { } } - public void mkdir(String path) throws IOException { - Path targetDir = Paths.get(path); - if (!Files.exists(targetDir)) { - Files.createDirectories(targetDir); - } - } - public void moveFiles(String sourcePath, String targetPath) throws Exception { Path sourceDir = Paths.get(sourcePath); Path targetDir = Paths.get(targetPath);