ci4s
/
ci4sManagement-cloud
Not watched
1
0
Fork 0
Code
Releases 1 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
Browse Source

异步执行创建

dev-DXTZYK
chenzhihang 1 year ago
parent
522ed32ff5
commit
24a51f4ead
2 changed files with 121 additions and 90 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +43
    -28
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java
  2. +78
    -62
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java

+ 43
- 28
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java View File

@@ -42,6 +42,7 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
@@ -623,22 +624,6 @@ public class ModelsServiceImpl implements ModelsService {
YamlUtils.generateYamlFile(metaMap, metaPath, "metadata");
String meta = JSON.toJSONString(modelMetaVo);

dvcUtils.dvcInit(rootPath);
// 配置远程S3地址
String s3Path = "management-platform-files/" + ci4sUsername + "/model/" + gitlinIid + "/" + repositoryName + "/" + branchName;
dvcUtils.dvcRemoteAdd(rootPath, s3Path);
dvcUtils.dvcConfigS3Credentials(rootPath, endpoint);
dvcUtils.dvcConfigS3Credentials2(rootPath, accessKeyId);
dvcUtils.dvcConfigS3Credentials3(rootPath, secretAccessKey);

// dvc 跟踪
dvcUtils.dvcAdd(rootPath, "model");
// git commit
dvcUtils.gitAdd(rootPath, ".");
dvcUtils.gitCommit(rootPath, "commit from ci4s with " + ci4sUsername);
dvcUtils.gitPush(rootPath, gitLinkUsername, gitLinkPassword);
dvcUtils.dvcPush(rootPath);

//保存模型依赖
modelDependency.setRepoId(gitlinIid);
modelDependency.setIdentifier(repositoryName);
@@ -648,6 +633,30 @@ public class ModelsServiceImpl implements ModelsService {
modelDependency.setOwner(gitLinkUsername);
modelDependency.setMeta(meta);
modelDependency1Dao.insert(modelDependency);

CompletableFuture.supplyAsync(() -> {
try {
dvcUtils.dvcInit(rootPath);
// 配置远程S3地址
String s3Path = "management-platform-files/" + ci4sUsername + "/model/" + gitlinIid + "/" + repositoryName + "/" + branchName;
dvcUtils.dvcRemoteAdd(rootPath, s3Path);
dvcUtils.dvcConfigS3Credentials(rootPath, endpoint);
dvcUtils.dvcConfigS3Credentials2(rootPath, accessKeyId);
dvcUtils.dvcConfigS3Credentials3(rootPath, secretAccessKey);

// dvc 跟踪
dvcUtils.dvcAdd(rootPath, "model");
// git commit
dvcUtils.gitAdd(rootPath, ".");
dvcUtils.gitCommit(rootPath, "commit from ci4s with " + ci4sUsername);
dvcUtils.gitPush(rootPath, gitLinkUsername, gitLinkPassword);
dvcUtils.dvcPush(rootPath);
} catch (Exception e) {
throw new RuntimeException(e);
}
return null;
});

return "新增模型成功";
} catch (Exception e) {
logger.error(e.getMessage());
@@ -719,18 +728,6 @@ public class ModelsServiceImpl implements ModelsService {
YamlUtils.generateYamlFile(metaMap, metaPath, "metadata");
String meta = JSON.toJSONString(modelMetaVo);

// 配置远程S3地址
String s3Path = "management-platform-files/" + ci4sUsername + "/model/" + modelsVo.getId() + "/" + repositoryName + "/" + branchName;
dvcUtils.dvcRemoteAdd(rootPath, s3Path);
dvcUtils.dvcConfigS3Credentials(rootPath, endpoint);
dvcUtils.dvcConfigS3Credentials2(rootPath, accessKeyId);
dvcUtils.dvcConfigS3Credentials3(rootPath, secretAccessKey);
// dvc 跟踪
dvcUtils.dvcAdd(rootPath, "model");
dvcUtils.pushNewBranchToRemote(rootPath, gitLinkUsername, gitLinkPassword, branchName);
//dvc push 到远程S3
dvcUtils.dvcPush(rootPath);

//保存模型依赖
modelDependency.setRepoId(modelsVo.getId());
modelDependency.setIdentifier(repositoryName);
@@ -741,6 +738,24 @@ public class ModelsServiceImpl implements ModelsService {
modelDependency.setMeta(meta);
modelDependency1Dao.insert(modelDependency);

CompletableFuture.supplyAsync(() -> {
try {
// 配置远程S3地址
String s3Path = "management-platform-files/" + ci4sUsername + "/model/" + modelsVo.getId() + "/" + repositoryName + "/" + branchName;
dvcUtils.dvcRemoteAdd(rootPath, s3Path);
dvcUtils.dvcConfigS3Credentials(rootPath, endpoint);
dvcUtils.dvcConfigS3Credentials2(rootPath, accessKeyId);
dvcUtils.dvcConfigS3Credentials3(rootPath, secretAccessKey);
// dvc 跟踪
dvcUtils.dvcAdd(rootPath, "model");
dvcUtils.pushNewBranchToRemote(rootPath, gitLinkUsername, gitLinkPassword, branchName);
//dvc push 到远程S3
dvcUtils.dvcPush(rootPath);
} catch (Exception e) {
throw new RuntimeException(e);
}
return null;
});
return "新增模型版本成功";
}
return null;


+ 78
- 62
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java View File

@@ -10,7 +10,6 @@ import com.ruoyi.platform.service.GitService;
import com.ruoyi.platform.service.NewDatasetService;
import com.ruoyi.platform.utils.*;
import com.ruoyi.platform.vo.GitProjectVo;
import com.ruoyi.platform.vo.ModelsVo;
import com.ruoyi.platform.vo.NewDatasetVo;
import com.ruoyi.platform.vo.VersionVo;
import com.ruoyi.system.api.model.LoginUser;
@@ -37,6 +36,7 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
@@ -69,10 +69,11 @@ public class NewDatasetServiceImpl implements NewDatasetService {
private DatasetTempStorageService datasetTempStorageService;
@Resource
private DVCUtils dvcUtils;

@Override
public String newCreateDataset(NewDatasetVo datasetVo) throws Exception {
String token = gitService.checkoutToken();
Jedis jedis = new Jedis(redisHost,redisPort);
Jedis jedis = new Jedis(redisHost, redisPort);
LoginUser loginUser = SecurityUtils.getLoginUser();
String ci4sUsername = loginUser.getUsername();
String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername();
@@ -90,25 +91,25 @@ public class NewDatasetServiceImpl implements NewDatasetService {
gitProjectVo.setPrivate(!datasetVo.getIsPublic());
gitProjectVo.setUserId(userId);
// 创建项目
Map project = gitService.createProject(token,gitProjectVo);
Map project = gitService.createProject(token, gitProjectVo);
Integer gitlinIid = (Integer) project.get("id");
if (gitlinIid == null){
throw new Exception("创建数据集失败:"+project.get("message"));
if (gitlinIid == null) {
throw new Exception("创建数据集失败:" + project.get("message"));
}
// 创建分支
String branchName = datasetVo.getVersion();
gitService.createBranch(token,(String) userInfo.get("login"), repositoryName, branchName, "master");
gitService.createBranch(token, (String) userInfo.get("login"), repositoryName, branchName, "master");
// 定义标签 标签1：ci4s-dataset 标签2：DataTag 标签3：DataType
gitService.createTopic(token,gitlinIid, "ci4s-dataset");
gitService.createTopic(token,gitlinIid, "datatag-" + datasetVo.getDataTag());
gitService.createTopic(token, gitlinIid, "ci4s-dataset");
gitService.createTopic(token, gitlinIid, "datatag-" + datasetVo.getDataTag());
gitService.createTopic(token, gitlinIid, "datatype-" + datasetVo.getDataType());
// 得到项目地址
String projectUrl = gitendpoint + "/" + (String) userInfo.get("login") + "/"+ repositoryName + ".git";
String projectUrl = gitendpoint + "/" + (String) userInfo.get("login") + "/" + repositoryName + ".git";

// 得到用户操作的路径
String sourcePath = datasetVo.getDatasetVersionVos().get(0).getUrl();
String relatePath = ci4sUsername+"/datasets/" + gitlinIid + "/" + repositoryName;
String localPath = localPathlocal+ relatePath;
String relatePath = ci4sUsername + "/datasets/" + gitlinIid + "/" + repositoryName;
String localPath = localPathlocal + relatePath;
String datasetPath = localPath + "/dataset";

// 命令行操作 git clone 项目地址
@@ -122,7 +123,7 @@ public class NewDatasetServiceImpl implements NewDatasetService {
datasetVo.setVersionDesc(datasetVo.getDescription());
datasetVo.setUsage("<pre><code>" +
"# 克隆数据集配置文件与存储参数到本地\n" +
"git clone -b " + branchName + " "+ projectUrl + "\n" +
"git clone -b " + branchName + " " + projectUrl + "\n" +
"# 远程拉取配置文件\n" +
"dvc pull\n" +
"</code></pre>");
@@ -134,26 +135,34 @@ public class NewDatasetServiceImpl implements NewDatasetService {
addDatasetSourceToDataVo(datasetVo);

YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo), localPath, "dataset");
// dvc init 初始化
dvcUtils.dvcInit(localPath);
// 配置远程S3地址
dvcUtils.dvcRemoteAdd(localPath, s3Path);
dvcUtils.dvcConfigS3Credentials(localPath, endpoint);
dvcUtils.dvcConfigS3Credentials2(localPath, accessKeyId);
dvcUtils.dvcConfigS3Credentials3(localPath, secretAccessKey);
// dvc 跟踪
dvcUtils.dvcAdd(localPath, "dataset");
// git commit
dvcUtils.gitCommit(localPath, "commit from ci4s with " + loginUser.getUsername());
dvcUtils.gitPush(localPath, gitLinkUsername, gitLinkPassword);
// dvc push 到远程S3
dvcUtils.dvcPush(localPath);

CompletableFuture.supplyAsync(() -> {
try {
// dvc init 初始化
dvcUtils.dvcInit(localPath);
// 配置远程S3地址
dvcUtils.dvcRemoteAdd(localPath, s3Path);
dvcUtils.dvcConfigS3Credentials(localPath, endpoint);
dvcUtils.dvcConfigS3Credentials2(localPath, accessKeyId);
dvcUtils.dvcConfigS3Credentials3(localPath, secretAccessKey);
// dvc 跟踪
dvcUtils.dvcAdd(localPath, "dataset");
// git commit
dvcUtils.gitCommit(localPath, "commit from ci4s with " + loginUser.getUsername());
dvcUtils.gitPush(localPath, gitLinkUsername, gitLinkPassword);
// dvc push 到远程S3
dvcUtils.dvcPush(localPath);
} catch (Exception e) {
throw new RuntimeException(e);
}
return null;
});
return "新增数据集成功";
}

public String newCreateVersion(NewDatasetVo datasetVo) throws Exception {
String token = gitService.checkoutToken();
Jedis jedis = new Jedis(redisHost,redisPort);
Jedis jedis = new Jedis(redisHost, redisPort);
LoginUser loginUser = SecurityUtils.getLoginUser();
String ci4sUsername = loginUser.getUsername();
String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername();
@@ -164,7 +173,7 @@ public class NewDatasetServiceImpl implements NewDatasetService {
String branchName = StringUtils.isEmpty(datasetVo.getVersion()) ? "master" : datasetVo.getVersion();
String repositoryName = datasetVo.getIdentifier();
String sourcePath = datasetVo.getDatasetVersionVos().get(0).getUrl();
String relatePath = ci4sUsername +"/datasets/"+ datasetVo.getId() + "/" + repositoryName;
String relatePath = ci4sUsername + "/datasets/" + datasetVo.getId() + "/" + repositoryName;
String localPath = localPathlocal + relatePath;
String datasetPath = localPath + "/dataset";
String projectUrl = gitendpoint + "/" + userInfo.get("login") + "/" + repositoryName + ".git";
@@ -188,13 +197,13 @@ public class NewDatasetServiceImpl implements NewDatasetService {
datasetVo.setUpdateTime(DateUtils.getTime());
datasetVo.setUsage("<pre><code>" +
"# 克隆数据集配置文件与存储参数到本地\n" +
"git clone -b " + branchName + " "+ projectUrl + "\n" +
"git clone -b " + branchName + " " + projectUrl + "\n" +
"# 远程拉取配置文件\n" +
"dvc pull\n" +
"</code></pre>");
datasetVo.setIdentifier(repositoryName);
datasetVo.setOwner((String) userInfo.get("login"));
if (StringUtils.isEmpty(datasetVo.getDatasetSource())){
if (StringUtils.isEmpty(datasetVo.getDatasetSource())) {
datasetVo.setDatasetSource("用户上传");
}
datasetVo.setDescription(newDatasetVo.getDescription());
@@ -205,17 +214,25 @@ public class NewDatasetServiceImpl implements NewDatasetService {
addDatasetSourceToDataVo(datasetVo);

YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo), localPath, "dataset");
//dvc数据跟踪
// 配置远程S3地址
dvcUtils.dvcRemoteAdd(localPath, s3Path);
dvcUtils.dvcConfigS3Credentials(localPath, endpoint);
dvcUtils.dvcConfigS3Credentials2(localPath, accessKeyId);
dvcUtils.dvcConfigS3Credentials3(localPath, secretAccessKey);
// dvc 跟踪
dvcUtils.dvcAdd(localPath, "dataset");
dvcUtils.pushNewBranchToRemote(localPath,gitLinkUsername,gitLinkPassword,branchName);
//dvc push 到远程S3
dvcUtils.dvcPush(localPath);

CompletableFuture.supplyAsync(() -> {
try {
//dvc数据跟踪
// 配置远程S3地址
dvcUtils.dvcRemoteAdd(localPath, s3Path);
dvcUtils.dvcConfigS3Credentials(localPath, endpoint);
dvcUtils.dvcConfigS3Credentials2(localPath, accessKeyId);
dvcUtils.dvcConfigS3Credentials3(localPath, secretAccessKey);
// dvc 跟踪
dvcUtils.dvcAdd(localPath, "dataset");
dvcUtils.pushNewBranchToRemote(localPath, gitLinkUsername, gitLinkPassword, branchName);
//dvc push 到远程S3
dvcUtils.dvcPush(localPath);
} catch (Exception e) {
throw new RuntimeException(e);
}
return null;
});
return "新增数据集成功";
}

@@ -223,7 +240,7 @@ public class NewDatasetServiceImpl implements NewDatasetService {
@Override
public Page<NewDatasetVo> newPersonalQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception {
String token = gitService.checkoutToken();
Jedis jedis = new Jedis(redisHost,redisPort);
Jedis jedis = new Jedis(redisHost, redisPort);
LoginUser loginUser = SecurityUtils.getLoginUser();
String ci4sUsername = loginUser.getUsername();
String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
@@ -232,8 +249,8 @@ public class NewDatasetServiceImpl implements NewDatasetService {
String url = gitendpoint + "/api/users/" + userInfo.get("login") + "/projects.json?category=manage";

String name = dataset.getName();
if (StringUtils.isNotEmpty(name)){
url =url + "&search=" + name;
if (StringUtils.isNotEmpty(name)) {
url = url + "&search=" + name;
}
String req = httpUtils.sendGetWithToken(url, null, token);
Map<String, Object> stringObjectMap = JacksonUtil.parseJSONStr2Map(req);
@@ -248,7 +265,7 @@ public class NewDatasetServiceImpl implements NewDatasetService {
@Override
public Page<NewDatasetVo> newPubilcQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception {
String token = gitService.checkoutToken();
Jedis jedis = new Jedis(redisHost,redisPort);
Jedis jedis = new Jedis(redisHost, redisPort);
LoginUser loginUser = SecurityUtils.getLoginUser();
String ci4sUsername = loginUser.getUsername();
String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
@@ -258,8 +275,8 @@ public class NewDatasetServiceImpl implements NewDatasetService {
String url = gitendpoint + "/api/projects.json?user_id=" + userId + "&sort_by=praises_count";

String name = dataset.getName();
if (StringUtils.isNotEmpty(name)){
url =url + "&search=" + name;
if (StringUtils.isNotEmpty(name)) {
url = url + "&search=" + name;
}

String req = httpUtils.sendGetWithToken(url, null, token);
@@ -287,14 +304,14 @@ public class NewDatasetServiceImpl implements NewDatasetService {
}
version = (String) versionList.get(0).get("name");
}
List<Map<String, Object>> fileDetailsAfterGitPull = dvcUtils.getFileDetailsAfterGitPull(localPathlocal +loginUser.getUsername()+"/datasets/" + id, repo, version, "dataset",gitLinkUsername, gitLinkPassword);
List<Map<String, Object>> fileDetailsAfterGitPull = dvcUtils.getFileDetailsAfterGitPull(localPathlocal + loginUser.getUsername() + "/datasets/" + id, repo, version, "dataset", gitLinkUsername, gitLinkPassword);
// 在localPathlocal+id+"/"+repositoryName目录下的dataset.yaml中取到元数据
System.out.println("!!!!!!!!!!!!!!!!!!!!!!!!!!!fileDetailsAfterGitPull:{}"+fileDetailsAfterGitPull);
Map<String, Object> stringObjectMap = YamlUtils.loadYamlFile(localPathlocal +loginUser.getUsername()+"/datasets/" + id + "/" + repo + "/" + "dataset.yaml");
System.out.println("!!!!!!!!!!!!!!!!!!!!!!!!!!!fileDetailsAfterGitPull:{}" + fileDetailsAfterGitPull);
Map<String, Object> stringObjectMap = YamlUtils.loadYamlFile(localPathlocal + loginUser.getUsername() + "/datasets/" + id + "/" + repo + "/" + "dataset.yaml");
NewDatasetVo newDatasetVo = ConvertUtil.convertMapToObject(stringObjectMap, NewDatasetVo.class);
List<VersionVo> versionVos = new ArrayList<VersionVo>();
if (fileDetailsAfterGitPull!=null&&fileDetailsAfterGitPull.size()>0){
for(Map<String, Object> fileDetail : fileDetailsAfterGitPull){
if (fileDetailsAfterGitPull != null && fileDetailsAfterGitPull.size() > 0) {
for (Map<String, Object> fileDetail : fileDetailsAfterGitPull) {
VersionVo versionVo = new VersionVo();
versionVo.setUrl((String) fileDetail.get("filePath"));
versionVo.setFileName((String) fileDetail.get("fileName"));
@@ -310,7 +327,7 @@ public class NewDatasetServiceImpl implements NewDatasetService {
@Override
public List<Map<String, Object>> getVersionList(String repo, String owner) throws Exception {
String token = gitService.checkoutToken();
List<Map<String, Object>> brancheList = gitService.getBrancheList(token,owner, repo);
List<Map<String, Object>> brancheList = gitService.getBrancheList(token, owner, repo);
return brancheList.stream()
.filter(branch -> !"master".equals(branch.get("name")))
.collect(Collectors.toList());
@@ -319,13 +336,13 @@ public class NewDatasetServiceImpl implements NewDatasetService {
@Override
public void deleteDatasetNew(String repo, String owner) throws Exception {
String token = gitService.checkoutToken();
gitService.deleteProject(token,owner, repo);
gitService.deleteProject(token, owner, repo);
}

@Override
public void deleteDatasetVersionNew(String repo, String owner, String version, String relativePath) throws Exception {
String token = gitService.checkoutToken();
gitService.deleteBranch(token,owner, repo, version, localPathlocal + relativePath);
gitService.deleteBranch(token, owner, repo, version, localPathlocal + relativePath);
}

@Override
@@ -378,14 +395,14 @@ public class NewDatasetServiceImpl implements NewDatasetService {
}

@Override
public ResponseEntity<InputStreamResource> downloadAllDatasetFilesNew(String name,Integer id, String version) throws Exception {
public ResponseEntity<InputStreamResource> downloadAllDatasetFilesNew(String name, Integer id, String version) throws Exception {
// 命令行操作 git clone 项目地址
LoginUser loginUser = SecurityUtils.getLoginUser();
String ci4sUsername = loginUser.getUsername();
Jedis jedis = new Jedis(redisHost,redisPort);
Jedis jedis = new Jedis(redisHost, redisPort);
String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
String localPath = localPathlocal+ loginUser.getUsername()+"/datasets/"+id+"/"+name;
String localPath = localPathlocal + loginUser.getUsername() + "/datasets/" + id + "/" + name;

// 打包 data 文件夹
String dataFolderPath = localPath + "/data";
@@ -412,7 +429,7 @@ public class NewDatasetServiceImpl implements NewDatasetService {
InputStreamResource resource = new InputStreamResource(new FileInputStream(zipFile));

return ResponseEntity.ok()
.header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename="+name+"_"+version+"_data.zip")
.header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=" + name + "_" + version + "_data.zip")
.contentType(MediaType.APPLICATION_OCTET_STREAM)
.contentLength(zipFile.length())
.body(resource);
@@ -420,7 +437,6 @@ public class NewDatasetServiceImpl implements NewDatasetService {
}



public List<NewDatasetVo> convert(List<Map<String, Object>> lst, String datasetTopic, String datasetTagName, String datasetTypeName) {
if (lst != null && lst.size() > 0) {
List<NewDatasetVo> result = new ArrayList<>();
@@ -460,12 +476,12 @@ public class NewDatasetServiceImpl implements NewDatasetService {
return new ArrayList<>();
}

private void addDatasetSourceToDataVo(NewDatasetVo datasetVo){
private void addDatasetSourceToDataVo(NewDatasetVo datasetVo) {
DatasetTempStorage queryDatasetTempStorage = new DatasetTempStorage();
queryDatasetTempStorage.setName(datasetVo.getName());
queryDatasetTempStorage.setVersion(datasetVo.getVersion());
DatasetTempStorage datasetTempStorage = datasetTempStorageService.queryByDatasetTempStorage(queryDatasetTempStorage);
if (datasetTempStorage != null){
if (datasetTempStorage != null) {
String datasetSourceString = datasetTempStorage.getSource();
Map<String, Object> datasetSourceMap = JacksonUtil.parseJSONStr2Map(datasetSourceString);
String preprocessCode = (String) datasetSourceMap.get("preprocess_code");


Write Preview
Loading…
Cancel
Save