ci4s
/
ci4sManagement-cloud
Not watched
1
0
Fork 0
Code
Releases 1 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
Browse Source

回退

dev-check
chenzhihang 1 year ago
parent
18edf9d512
commit
35f7f4aa5e
3 changed files with 113 additions and 88 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +52
    -44
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java
  2. +61
    -37
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java
  3. +0
    -7
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/DVCUtils.java

+ 52
- 44
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java View File

@@ -586,6 +586,7 @@ public class ModelsServiceImpl implements ModelsService {
}

//新建模型
// 拼接project
GitProjectVo gitProjectVo = new GitProjectVo();
gitProjectVo.setRepositoryName(repositoryName);
gitProjectVo.setName(Constant.Topic_model + "-" + modelsVo.getName());
@@ -601,6 +602,18 @@ public class ModelsServiceImpl implements ModelsService {
throw new RuntimeException("创建模型失败:" + project.get("message"));
}

String branchName = modelsVo.getVersion();
String owner = (String) userInfo.get("login");

String projectUrl = gitCloneEndpoint + "/" + owner + "/" + repositoryName + ".git";
String sourcePath = modelsVo.getModelVersionVos().get(0).getUrl();
String rootPath = localPath + ci4sUsername + "/model/" + gitlinIid + "/" + repositoryName + "/" + branchName;
String modelPath = rootPath + "/model";
String metaPath = rootPath + "/metadata";
String relatePath = ci4sUsername + "/model/" + gitlinIid + "/" + repositoryName + "/" + branchName + "/model";

// 创建分支
gitService.createBranch(token, (String) userInfo.get("login"), repositoryName, branchName, "master");
// 定义标签 标签1：model 标签2：ModelTag 标签3：ModelType
gitService.createTopic(token, gitlinIid, Constant.Topic_model);
if (StringUtils.isNotEmpty(modelsVo.getModelTag())) {
@@ -610,17 +623,11 @@ public class ModelsServiceImpl implements ModelsService {
gitService.createTopic(token, gitlinIid, "modeltype-" + modelsVo.getModelType());
}

String owner = (String) userInfo.get("login");
String projectUrl = gitCloneEndpoint + "/" + owner + "/" + repositoryName + ".git";
String relationPath = ci4sUsername + "/model/" + gitlinIid + "/" + repositoryName + "/origin";
String rootPath = localPath + relationPath;
String modelPath = rootPath + "/model";
String metaPath = rootPath + "/metadata";
String relatePath = relationPath + "/model";
dvcUtils.gitClone(rootPath, projectUrl, branchName, gitLinkUsername, decryptGitLinkPassword);

// 创建origin分支
gitService.createBranch(token, owner, repositoryName, "origin", "master");
dvcUtils.gitClone(rootPath, projectUrl, "origin", gitLinkUsername, decryptGitLinkPassword);
//干掉目标文件夹
dvcUtils.deleteDirectory(modelPath);
dvcUtils.moveFiles(sourcePath, modelPath);

//拼接生产的元数据后写入yaml文件
ModelMetaVo modelMetaVo = new ModelMetaVo();
@@ -629,13 +636,19 @@ public class ModelsServiceImpl implements ModelsService {
modelMetaVo.setCreateBy(String.valueOf(StringUtils.isNotEmpty((String) userInfo.get("nickname")) ? userInfo.get("nickname") : userInfo.get("login")));
modelMetaVo.setCreateTime(DateUtils.getTime());
modelMetaVo.setUpdateTime(DateUtils.getTime());
String projectUrlIp = gitCloneEndpointIp + "/" + owner + "/" + repositoryName + ".git";
modelMetaVo.setUsage("<pre><code>" +
"# 克隆模型配置文件与存储参数到本地\n" +
"git clone -b " + branchName + " " + projectUrlIp + "\n" +
"# 远程拉取配置文件\n" +
"dvc pull\n" +
"</code></pre>");
modelMetaVo.setIdentifier(repositoryName);
modelMetaVo.setOwner(owner);
modelMetaVo.setRelativePaths(relatePath);
File folder = new File(modelPath);
long folderSize = FileUtil.getFolderSize(folder);
modelMetaVo.setModelSize(FileUtil.formatFileSize(folderSize));
BeanUtils.copyProperties(modelMetaVo, modelsVo);

Map<String, Object> metaMap = JSON.parseObject(JSON.toJSONString(modelMetaVo), Map.class);
YamlUtils.generateYamlFile(metaMap, metaPath, "metadata");
@@ -645,39 +658,35 @@ public class ModelsServiceImpl implements ModelsService {
modelDependency.setRepoId(gitlinIid);
modelDependency.setIdentifier(repositoryName);
modelDependency.setModelName(modelsVo.getName());
modelDependency.setVersion("origin");
modelDependency.setVersion(modelsVo.getVersion());
modelDependency.setParentModel(modelsVo.getParentModel());
modelDependency.setOwner(gitLinkUsername);
modelDependency.setMeta(meta);
modelDependency.setState(Constant.State_valid);
modelDependency1Dao.insert(modelDependency);

String s3Path = bucketName + "/mini-model-management-platform-files/" + relationPath;
// CompletableFuture.supplyAsync(() -> {
// try {
dvcUtils.mkdir(modelPath);
dvcUtils.dvcInit(rootPath);
// 配置远程S3地址
dvcUtils.dvcRemoteAdd(rootPath, s3Path);
dvcUtils.dvcConfigS3Credentials(rootPath, endpoint);
dvcUtils.dvcConfigS3Credentials2(rootPath, accessKeyId);
dvcUtils.dvcConfigS3Credentials3(rootPath, secretAccessKey);

// dvc 跟踪
dvcUtils.dvcAdd(rootPath, "model");
// git commit
dvcUtils.gitCommit(rootPath, "commit from ci4s with " + loginUser.getUsername());
dvcUtils.gitPush(rootPath, gitLinkUsername, decryptGitLinkPassword);
dvcUtils.dvcPush(rootPath);

// 创建分支
newCreateVersion(modelsVo);
// } catch (Exception e) {
// logger.error(e.getMessage(), e);
// throw new RuntimeException(e.getMessage());
// }
// return null;
// });
String s3Path = bucketName + "/mini-model-management-platform-files/" + ci4sUsername + "/model/" + gitlinIid + "/" + repositoryName + "/" + branchName;
CompletableFuture.supplyAsync(() -> {
try {
dvcUtils.dvcInit(rootPath);
// 配置远程S3地址
dvcUtils.dvcRemoteAdd(rootPath, s3Path);
dvcUtils.dvcConfigS3Credentials(rootPath, endpoint);
dvcUtils.dvcConfigS3Credentials2(rootPath, accessKeyId);
dvcUtils.dvcConfigS3Credentials3(rootPath, secretAccessKey);

// dvc 跟踪
dvcUtils.dvcAdd(rootPath, "model");
// git commit
dvcUtils.gitCommit(rootPath, "commit from ci4s with " + loginUser.getUsername());
dvcUtils.gitPush(rootPath, gitLinkUsername, decryptGitLinkPassword);
dvcUtils.dvcPush(rootPath);
} catch (Exception e) {
logger.error(e.getMessage(), e);
throw new RuntimeException(e.getMessage());
}
return null;
});

return "新增模型成功";
} catch (Exception e) {
@@ -718,8 +727,8 @@ public class ModelsServiceImpl implements ModelsService {
FileUtil.renameFile(rootPath, rootPath + "_deleted_" + System.currentTimeMillis());

//克隆
dvcUtils.gitClone(rootPath, projectUrl, "origin", gitLinkUsername, gitLinkPassword);
dvcUtils.refreshRemoteBranches(rootPath, gitLinkUsername, gitLinkPassword, "origin");
dvcUtils.gitClone(rootPath, projectUrl, oldModelDependencys.get(0).getVersion(), gitLinkUsername, gitLinkPassword);
dvcUtils.refreshRemoteBranches(rootPath, gitLinkUsername, gitLinkPassword, oldModelDependencys.get(0).getVersion());

//查询项目信息
String token = gitService.checkoutToken();
@@ -766,10 +775,9 @@ public class ModelsServiceImpl implements ModelsService {
modelMetaVo.setCreateBy(String.valueOf(StringUtils.isNotEmpty((String) userInfo.get("nickname")) ? userInfo.get("nickname") : userInfo.get("login")));
modelMetaVo.setCreateTime(DateUtils.getTime());
modelMetaVo.setUpdateTime(DateUtils.getTime());
String projectUrlIp = gitCloneEndpointIp + "/" + owner + "/" + repositoryName + ".git";
modelMetaVo.setUsage("<pre><code>" +
"# 克隆模型配置文件与存储参数到本地\n" +
"git clone -b " + branchName + " " + projectUrlIp + "\n" +
"git clone -b " + branchName + " " + projectUrl + "\n" +
"# 远程拉取配置文件\n" +
"dvc pull\n" +
"</code></pre>");
@@ -976,7 +984,7 @@ public class ModelsServiceImpl implements ModelsService {
String token = gitService.checkoutToken();
List<Map<String, Object>> collect = gitService.getBrancheList(token, owner, identifier);
List<Map<String, Object>> result = collect.stream()
.filter(branch -> !"master".equals(branch.get("name")) && !"origin".equals(branch.get("name")))
.filter(branch -> !"master".equals(branch.get("name")))
.skip((pageRequest.getPageNumber()) * pageRequest.getPageSize()).limit(pageRequest.getPageSize())
.collect(Collectors.toList());

@@ -1079,7 +1087,7 @@ public class ModelsServiceImpl implements ModelsService {
String token = gitService.checkoutToken();
List<Map<String, Object>> branchList = gitService.getBrancheList(token, owner, identifier);
return branchList.stream()
.filter(branch -> !"master".equals(branch.get("name")) && !"origin".equals(branch.get("name")))
.filter(branch -> !"master".equals(branch.get("name")))
.collect(Collectors.toList());
}



+ 61
- 37
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java View File

@@ -104,12 +104,9 @@ public class NewDatasetServiceImpl implements NewDatasetService {
String gitLinkUsername = loginUser.getSysUser().getUserName();
String gitLinkPassword = decrypt(loginUser.getSysUser().getOriginPassword());
String userReq = jedis.get(ci4sUsername + "_gitUserInfo");

// 得到用户操作的路径
Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
Integer userId = (Integer) userInfo.get("user_id");
String owner = (String) userInfo.get("login");

// 拼接project
ci4sUsername = Boolean.TRUE.equals(datasetVo.getIsPublic()) ? Constant.Item_Public : loginUser.getUsername();
String repositoryName = ci4sUsername + "_dataset_" + DateUtils.dateTimeNow();
@@ -120,13 +117,17 @@ public class NewDatasetServiceImpl implements NewDatasetService {
gitProjectVo.setPrivate(!datasetVo.getIsPublic());
gitProjectVo.setUserId(userId);
gitProjectVo.setProjectCategoryId(projectDatasetId);

// 创建项目
Map project = gitService.createProject(token, gitProjectVo);
Integer gitlinIid = (Integer) project.get("id");
if (gitlinIid == null) {
throw new Exception("创建数据集失败:" + project.get("message"));
}
// 创建分支
String branchName = datasetVo.getVersion();
String owner = (String) userInfo.get("login");

gitService.createBranch(token, (String) userInfo.get("login"), repositoryName, branchName, "master");
// 定义标签 标签1：dataset 标签2：DataTag 标签3：DataType
gitService.createTopic(token, gitlinIid, Constant.Topic_Dataset);
if (StringUtils.isNotEmpty(datasetVo.getDataTag())) {
@@ -136,44 +137,61 @@ public class NewDatasetServiceImpl implements NewDatasetService {
gitService.createTopic(token, gitlinIid, "datatype-" + datasetVo.getDataType());
}

String relatePath = ci4sUsername + "/datasets/" + gitlinIid + "/" + repositoryName + "/origin";
String localPath = localPathlocal + relatePath;
String datasetPath = localPath + "/dataset";
String s3Path = bucketName + "/mini-model-management-platform-files" + "/" + relatePath;

// 得到项目地址
String projectUrl = gitCloneEndpoint + "/" + owner + "/" + repositoryName + ".git";

// 创建origin分支
gitService.createBranch(token, owner, repositoryName, "origin", "master");
dvcUtils.gitClone(localPath, projectUrl, "origin", gitLinkUsername, gitLinkPassword);
// 得到用户操作的路径
String sourcePath = datasetVo.getDatasetVersionVos().get(0).getUrl();
String relatePath = ci4sUsername + "/datasets/" + gitlinIid + "/" + repositoryName + "/" + branchName;
String localPath = localPathlocal + relatePath;
String datasetPath = localPath + "/dataset";

// 命令行操作 git clone 项目地址
dvcUtils.gitClone(localPath, projectUrl, branchName, gitLinkUsername, gitLinkPassword);
String s3Path = bucketName + "/mini-model-management-platform-files" + "/" + relatePath + "/" + branchName;
//干掉目标文件夹
dvcUtils.deleteDirectory(datasetPath);
dvcUtils.moveFiles(sourcePath, datasetPath);
// 拼接生产的元数据后写入yaml文件
datasetVo.setCreateBy(String.valueOf(StringUtils.isNotEmpty((String) userInfo.get("nickname")) ? userInfo.get("nickname") : userInfo.get("login")));
datasetVo.setUpdateTime(DateUtils.getTime());
datasetVo.setUsage("<pre><code>" +
"# 克隆数据集配置文件与存储参数到本地\n" +
"git clone -b " + branchName + " " + projectUrl + "\n" +
"# 远程拉取配置文件\n" +
"dvc pull\n" +
"</code></pre>");
datasetVo.setIdentifier(repositoryName);
datasetVo.setId(gitlinIid);
datasetVo.setOwner(owner);
datasetVo.setRelativePaths(relatePath + "/dataset");
YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo), localPath, "dataset");

// dvc init 初始化
dvcUtils.mkdir(datasetPath);
dvcUtils.dvcInit(localPath);
// 配置远程S3地址
dvcUtils.dvcRemoteAdd(localPath, s3Path);
dvcUtils.dvcConfigS3Credentials(localPath, endpoint);
dvcUtils.dvcConfigS3Credentials2(localPath, accessKeyId);
dvcUtils.dvcConfigS3Credentials3(localPath, secretAccessKey);
// dvc 跟踪
dvcUtils.dvcAdd(localPath, "dataset");
// git commit
dvcUtils.gitCommit(localPath, "commit from ci4s with " + loginUser.getUsername());
dvcUtils.gitPush(localPath, gitLinkUsername, gitLinkPassword);
// dvc push 到远程S3
dvcUtils.dvcPush(localPath);
addDatasetSourceToDataVo(datasetVo);

// 创建分支
newCreateVersion(datasetVo);
YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo), localPath, "dataset");

CompletableFuture.supplyAsync(() -> {
try {
// dvc init 初始化
dvcUtils.dvcInit(localPath);
// 配置远程S3地址
dvcUtils.dvcRemoteAdd(localPath, s3Path);
dvcUtils.dvcConfigS3Credentials(localPath, endpoint);
dvcUtils.dvcConfigS3Credentials2(localPath, accessKeyId);
dvcUtils.dvcConfigS3Credentials3(localPath, secretAccessKey);
// dvc 跟踪
dvcUtils.dvcAdd(localPath, "dataset");
// git commit
dvcUtils.gitCommit(localPath, "commit from ci4s with " + loginUser.getUsername());
dvcUtils.gitPush(localPath, gitLinkUsername, gitLinkPassword);
// dvc push 到远程S3
dvcUtils.dvcPush(localPath);
} catch (Exception e) {
logger.error(e.getMessage(), e);
throw new RuntimeException(e.getMessage());
}
return null;
});
return "新增数据集成功";
}

@@ -196,7 +214,7 @@ public class NewDatasetServiceImpl implements NewDatasetService {
ci4sUsername = Boolean.TRUE.equals(datasetVo.getIsPublic()) ? Constant.Item_Public : loginUser.getUsername();
Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
// 创建分支
String branchName = datasetVo.getVersion();
String branchName = StringUtils.isEmpty(datasetVo.getVersion()) ? "master" : datasetVo.getVersion();
String owner = (String) userInfo.get("login");

String repositoryName = datasetVo.getIdentifier();
@@ -214,10 +232,14 @@ public class NewDatasetServiceImpl implements NewDatasetService {
Map projectDetail = gitService.getProjectDetail(owner, repositoryName, token);

//克隆
dvcUtils.gitClone(localPath, projectUrl, "origin", gitLinkUsername, gitLinkPassword);
dvcUtils.refreshRemoteBranches(localPath, gitLinkUsername, gitLinkPassword, "origin");
List<Map<String, Object>> brancheList = gitService.getBrancheList(token, owner, repositoryName);
String oldBranch = (String) brancheList.stream()
.filter(branch -> !"master".equals(branch.get("name"))).collect(Collectors.toList()).get(0).get("name");

dvcUtils.gitClone(localPath, projectUrl, oldBranch, gitLinkUsername, gitLinkPassword);
dvcUtils.refreshRemoteBranches(localPath, gitLinkUsername, gitLinkPassword, oldBranch);

//检查是否存在本地重名分支，有的话删掉
//检查是否存在本地重名分支，有的话干掉
dvcUtils.deleteLocalBranch(localPath, branchName);
// 创建本地分支
dvcUtils.createLocalBranchBasedOnMaster(localPath, branchName);
@@ -227,8 +249,8 @@ public class NewDatasetServiceImpl implements NewDatasetService {
dvcUtils.dvcCheckout(localPath);

// 准备数据
String s3Path = bucketName + "/mini-model-management-platform-files" + "/" + relatePath;
//删掉已存在文件夹
String s3Path = bucketName + "/mini-model-management-platform-files" + "/" + relatePath + "/" + branchName;
//干掉目标文件夹
dvcUtils.deleteDirectory(datasetPath);

if (Constant.Source_Hand_Export.equals(datasetVo.getDatasetSource())) {
@@ -239,6 +261,7 @@ public class NewDatasetServiceImpl implements NewDatasetService {
dvcUtils.moveFiles(sourcePath, datasetPath);
}


// 拼接生产的元数据后写入yaml文件
datasetVo.setCreateBy(String.valueOf(StringUtils.isNotEmpty((String) userInfo.get("nickname")) ? userInfo.get("nickname") : userInfo.get("login")));
datasetVo.setUpdateTime(DateUtils.getTime());
@@ -280,6 +303,7 @@ public class NewDatasetServiceImpl implements NewDatasetService {
// dvc 跟踪
dvcUtils.dvcAdd(localPath, "dataset");
dvcUtils.pushNewBranchToRemote(localPath, gitLinkUsername, gitLinkPassword, branchName);

dvcUtils.dvcPush(localPath);

// CompletableFuture.supplyAsync(() -> {
@@ -383,7 +407,7 @@ public class NewDatasetServiceImpl implements NewDatasetService {
String token = gitService.checkoutToken();
List<Map<String, Object>> brancheList = gitService.getBrancheList(token, owner, repo);
return brancheList.stream()
.filter(branch -> !"master".equals(branch.get("name")) && !"origin".equals(branch.get("name")))
.filter(branch -> !"master".equals(branch.get("name")))
.collect(Collectors.toList());
}



+ 0
- 7
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/DVCUtils.java View File

@@ -68,13 +68,6 @@ public class DVCUtils {
}
}

public void mkdir(String path) throws IOException {
Path targetDir = Paths.get(path);
if (!Files.exists(targetDir)) {
Files.createDirectories(targetDir);
}
}

public void moveFiles(String sourcePath, String targetPath) throws Exception {
Path sourceDir = Paths.get(sourcePath);
Path targetDir = Paths.get(targetPath);


Write Preview
Loading…
Cancel
Save