diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java
index 405969bc..9caf1ef4 100644
--- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java
+++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java
@@ -42,6 +42,7 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;
+import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
@@ -623,22 +624,6 @@ public class ModelsServiceImpl implements ModelsService {
YamlUtils.generateYamlFile(metaMap, metaPath, "metadata");
String meta = JSON.toJSONString(modelMetaVo);
- dvcUtils.dvcInit(rootPath);
- // 配置远程S3地址
- String s3Path = "management-platform-files/" + ci4sUsername + "/model/" + gitlinIid + "/" + repositoryName + "/" + branchName;
- dvcUtils.dvcRemoteAdd(rootPath, s3Path);
- dvcUtils.dvcConfigS3Credentials(rootPath, endpoint);
- dvcUtils.dvcConfigS3Credentials2(rootPath, accessKeyId);
- dvcUtils.dvcConfigS3Credentials3(rootPath, secretAccessKey);
-
- // dvc 跟踪
- dvcUtils.dvcAdd(rootPath, "model");
- // git commit
- dvcUtils.gitAdd(rootPath, ".");
- dvcUtils.gitCommit(rootPath, "commit from ci4s with " + ci4sUsername);
- dvcUtils.gitPush(rootPath, gitLinkUsername, gitLinkPassword);
- dvcUtils.dvcPush(rootPath);
-
//保存模型依赖
modelDependency.setRepoId(gitlinIid);
modelDependency.setIdentifier(repositoryName);
@@ -648,6 +633,30 @@ public class ModelsServiceImpl implements ModelsService {
modelDependency.setOwner(gitLinkUsername);
modelDependency.setMeta(meta);
modelDependency1Dao.insert(modelDependency);
+
+ CompletableFuture.supplyAsync(() -> {
+ try {
+ dvcUtils.dvcInit(rootPath);
+ // 配置远程S3地址
+ String s3Path = "management-platform-files/" + ci4sUsername + "/model/" + gitlinIid + "/" + repositoryName + "/" + branchName;
+ dvcUtils.dvcRemoteAdd(rootPath, s3Path);
+ dvcUtils.dvcConfigS3Credentials(rootPath, endpoint);
+ dvcUtils.dvcConfigS3Credentials2(rootPath, accessKeyId);
+ dvcUtils.dvcConfigS3Credentials3(rootPath, secretAccessKey);
+
+ // dvc 跟踪
+ dvcUtils.dvcAdd(rootPath, "model");
+ // git commit
+ dvcUtils.gitAdd(rootPath, ".");
+ dvcUtils.gitCommit(rootPath, "commit from ci4s with " + ci4sUsername);
+ dvcUtils.gitPush(rootPath, gitLinkUsername, gitLinkPassword);
+ dvcUtils.dvcPush(rootPath);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ return null;
+ });
+
return "新增模型成功";
} catch (Exception e) {
logger.error(e.getMessage());
@@ -719,18 +728,6 @@ public class ModelsServiceImpl implements ModelsService {
YamlUtils.generateYamlFile(metaMap, metaPath, "metadata");
String meta = JSON.toJSONString(modelMetaVo);
- // 配置远程S3地址
- String s3Path = "management-platform-files/" + ci4sUsername + "/model/" + modelsVo.getId() + "/" + repositoryName + "/" + branchName;
- dvcUtils.dvcRemoteAdd(rootPath, s3Path);
- dvcUtils.dvcConfigS3Credentials(rootPath, endpoint);
- dvcUtils.dvcConfigS3Credentials2(rootPath, accessKeyId);
- dvcUtils.dvcConfigS3Credentials3(rootPath, secretAccessKey);
- // dvc 跟踪
- dvcUtils.dvcAdd(rootPath, "model");
- dvcUtils.pushNewBranchToRemote(rootPath, gitLinkUsername, gitLinkPassword, branchName);
- //dvc push 到远程S3
- dvcUtils.dvcPush(rootPath);
-
//保存模型依赖
modelDependency.setRepoId(modelsVo.getId());
modelDependency.setIdentifier(repositoryName);
@@ -741,6 +738,24 @@ public class ModelsServiceImpl implements ModelsService {
modelDependency.setMeta(meta);
modelDependency1Dao.insert(modelDependency);
+ CompletableFuture.supplyAsync(() -> {
+ try {
+ // 配置远程S3地址
+ String s3Path = "management-platform-files/" + ci4sUsername + "/model/" + modelsVo.getId() + "/" + repositoryName + "/" + branchName;
+ dvcUtils.dvcRemoteAdd(rootPath, s3Path);
+ dvcUtils.dvcConfigS3Credentials(rootPath, endpoint);
+ dvcUtils.dvcConfigS3Credentials2(rootPath, accessKeyId);
+ dvcUtils.dvcConfigS3Credentials3(rootPath, secretAccessKey);
+ // dvc 跟踪
+ dvcUtils.dvcAdd(rootPath, "model");
+ dvcUtils.pushNewBranchToRemote(rootPath, gitLinkUsername, gitLinkPassword, branchName);
+ //dvc push 到远程S3
+ dvcUtils.dvcPush(rootPath);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ return null;
+ });
return "新增模型版本成功";
}
return null;
diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java
index 06a245e2..02e43950 100644
--- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java
+++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java
@@ -10,7 +10,6 @@ import com.ruoyi.platform.service.GitService;
import com.ruoyi.platform.service.NewDatasetService;
import com.ruoyi.platform.utils.*;
import com.ruoyi.platform.vo.GitProjectVo;
-import com.ruoyi.platform.vo.ModelsVo;
import com.ruoyi.platform.vo.NewDatasetVo;
import com.ruoyi.platform.vo.VersionVo;
import com.ruoyi.system.api.model.LoginUser;
@@ -37,6 +36,7 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
@@ -69,10 +69,11 @@ public class NewDatasetServiceImpl implements NewDatasetService {
private DatasetTempStorageService datasetTempStorageService;
@Resource
private DVCUtils dvcUtils;
+
@Override
public String newCreateDataset(NewDatasetVo datasetVo) throws Exception {
String token = gitService.checkoutToken();
- Jedis jedis = new Jedis(redisHost,redisPort);
+ Jedis jedis = new Jedis(redisHost, redisPort);
LoginUser loginUser = SecurityUtils.getLoginUser();
String ci4sUsername = loginUser.getUsername();
String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername();
@@ -90,25 +91,25 @@ public class NewDatasetServiceImpl implements NewDatasetService {
gitProjectVo.setPrivate(!datasetVo.getIsPublic());
gitProjectVo.setUserId(userId);
// 创建项目
- Map project = gitService.createProject(token,gitProjectVo);
+ Map project = gitService.createProject(token, gitProjectVo);
Integer gitlinIid = (Integer) project.get("id");
- if (gitlinIid == null){
- throw new Exception("创建数据集失败:"+project.get("message"));
+ if (gitlinIid == null) {
+ throw new Exception("创建数据集失败:" + project.get("message"));
}
// 创建分支
String branchName = datasetVo.getVersion();
- gitService.createBranch(token,(String) userInfo.get("login"), repositoryName, branchName, "master");
+ gitService.createBranch(token, (String) userInfo.get("login"), repositoryName, branchName, "master");
// 定义标签 标签1：ci4s-dataset 标签2：DataTag 标签3：DataType
- gitService.createTopic(token,gitlinIid, "ci4s-dataset");
- gitService.createTopic(token,gitlinIid, "datatag-" + datasetVo.getDataTag());
+ gitService.createTopic(token, gitlinIid, "ci4s-dataset");
+ gitService.createTopic(token, gitlinIid, "datatag-" + datasetVo.getDataTag());
gitService.createTopic(token, gitlinIid, "datatype-" + datasetVo.getDataType());
// 得到项目地址
- String projectUrl = gitendpoint + "/" + (String) userInfo.get("login") + "/"+ repositoryName + ".git";
+ String projectUrl = gitendpoint + "/" + (String) userInfo.get("login") + "/" + repositoryName + ".git";
// 得到用户操作的路径
String sourcePath = datasetVo.getDatasetVersionVos().get(0).getUrl();
- String relatePath = ci4sUsername+"/datasets/" + gitlinIid + "/" + repositoryName;
- String localPath = localPathlocal+ relatePath;
+ String relatePath = ci4sUsername + "/datasets/" + gitlinIid + "/" + repositoryName;
+ String localPath = localPathlocal + relatePath;
String datasetPath = localPath + "/dataset";
// 命令行操作 git clone 项目地址
@@ -122,7 +123,7 @@ public class NewDatasetServiceImpl implements NewDatasetService {
datasetVo.setVersionDesc(datasetVo.getDescription());
datasetVo.setUsage("" +
"# 克隆数据集配置文件与存储参数到本地\n" +
- "git clone -b " + branchName + " "+ projectUrl + "\n" +
+ "git clone -b " + branchName + " " + projectUrl + "\n" +
"# 远程拉取配置文件\n" +
"dvc pull\n" +
"
");
@@ -134,26 +135,34 @@ public class NewDatasetServiceImpl implements NewDatasetService {
addDatasetSourceToDataVo(datasetVo);
YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo), localPath, "dataset");
- // dvc init 初始化
- dvcUtils.dvcInit(localPath);
- // 配置远程S3地址
- dvcUtils.dvcRemoteAdd(localPath, s3Path);
- dvcUtils.dvcConfigS3Credentials(localPath, endpoint);
- dvcUtils.dvcConfigS3Credentials2(localPath, accessKeyId);
- dvcUtils.dvcConfigS3Credentials3(localPath, secretAccessKey);
- // dvc 跟踪
- dvcUtils.dvcAdd(localPath, "dataset");
- // git commit
- dvcUtils.gitCommit(localPath, "commit from ci4s with " + loginUser.getUsername());
- dvcUtils.gitPush(localPath, gitLinkUsername, gitLinkPassword);
- // dvc push 到远程S3
- dvcUtils.dvcPush(localPath);
+
+ CompletableFuture.supplyAsync(() -> {
+ try {
+ // dvc init 初始化
+ dvcUtils.dvcInit(localPath);
+ // 配置远程S3地址
+ dvcUtils.dvcRemoteAdd(localPath, s3Path);
+ dvcUtils.dvcConfigS3Credentials(localPath, endpoint);
+ dvcUtils.dvcConfigS3Credentials2(localPath, accessKeyId);
+ dvcUtils.dvcConfigS3Credentials3(localPath, secretAccessKey);
+ // dvc 跟踪
+ dvcUtils.dvcAdd(localPath, "dataset");
+ // git commit
+ dvcUtils.gitCommit(localPath, "commit from ci4s with " + loginUser.getUsername());
+ dvcUtils.gitPush(localPath, gitLinkUsername, gitLinkPassword);
+ // dvc push 到远程S3
+ dvcUtils.dvcPush(localPath);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ return null;
+ });
return "新增数据集成功";
}
public String newCreateVersion(NewDatasetVo datasetVo) throws Exception {
String token = gitService.checkoutToken();
- Jedis jedis = new Jedis(redisHost,redisPort);
+ Jedis jedis = new Jedis(redisHost, redisPort);
LoginUser loginUser = SecurityUtils.getLoginUser();
String ci4sUsername = loginUser.getUsername();
String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername();
@@ -164,7 +173,7 @@ public class NewDatasetServiceImpl implements NewDatasetService {
String branchName = StringUtils.isEmpty(datasetVo.getVersion()) ? "master" : datasetVo.getVersion();
String repositoryName = datasetVo.getIdentifier();
String sourcePath = datasetVo.getDatasetVersionVos().get(0).getUrl();
- String relatePath = ci4sUsername +"/datasets/"+ datasetVo.getId() + "/" + repositoryName;
+ String relatePath = ci4sUsername + "/datasets/" + datasetVo.getId() + "/" + repositoryName;
String localPath = localPathlocal + relatePath;
String datasetPath = localPath + "/dataset";
String projectUrl = gitendpoint + "/" + userInfo.get("login") + "/" + repositoryName + ".git";
@@ -188,13 +197,13 @@ public class NewDatasetServiceImpl implements NewDatasetService {
datasetVo.setUpdateTime(DateUtils.getTime());
datasetVo.setUsage("" +
"# 克隆数据集配置文件与存储参数到本地\n" +
- "git clone -b " + branchName + " "+ projectUrl + "\n" +
+ "git clone -b " + branchName + " " + projectUrl + "\n" +
"# 远程拉取配置文件\n" +
"dvc pull\n" +
"
");
datasetVo.setIdentifier(repositoryName);
datasetVo.setOwner((String) userInfo.get("login"));
- if (StringUtils.isEmpty(datasetVo.getDatasetSource())){
+ if (StringUtils.isEmpty(datasetVo.getDatasetSource())) {
datasetVo.setDatasetSource("用户上传");
}
datasetVo.setDescription(newDatasetVo.getDescription());
@@ -205,17 +214,25 @@ public class NewDatasetServiceImpl implements NewDatasetService {
addDatasetSourceToDataVo(datasetVo);
YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo), localPath, "dataset");
- //dvc数据跟踪
- // 配置远程S3地址
- dvcUtils.dvcRemoteAdd(localPath, s3Path);
- dvcUtils.dvcConfigS3Credentials(localPath, endpoint);
- dvcUtils.dvcConfigS3Credentials2(localPath, accessKeyId);
- dvcUtils.dvcConfigS3Credentials3(localPath, secretAccessKey);
- // dvc 跟踪
- dvcUtils.dvcAdd(localPath, "dataset");
- dvcUtils.pushNewBranchToRemote(localPath,gitLinkUsername,gitLinkPassword,branchName);
- //dvc push 到远程S3
- dvcUtils.dvcPush(localPath);
+
+ CompletableFuture.supplyAsync(() -> {
+ try {
+ //dvc数据跟踪
+ // 配置远程S3地址
+ dvcUtils.dvcRemoteAdd(localPath, s3Path);
+ dvcUtils.dvcConfigS3Credentials(localPath, endpoint);
+ dvcUtils.dvcConfigS3Credentials2(localPath, accessKeyId);
+ dvcUtils.dvcConfigS3Credentials3(localPath, secretAccessKey);
+ // dvc 跟踪
+ dvcUtils.dvcAdd(localPath, "dataset");
+ dvcUtils.pushNewBranchToRemote(localPath, gitLinkUsername, gitLinkPassword, branchName);
+ //dvc push 到远程S3
+ dvcUtils.dvcPush(localPath);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ return null;
+ });
return "新增数据集成功";
}
@@ -223,7 +240,7 @@ public class NewDatasetServiceImpl implements NewDatasetService {
@Override
public Page newPersonalQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception {
String token = gitService.checkoutToken();
- Jedis jedis = new Jedis(redisHost,redisPort);
+ Jedis jedis = new Jedis(redisHost, redisPort);
LoginUser loginUser = SecurityUtils.getLoginUser();
String ci4sUsername = loginUser.getUsername();
String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
@@ -232,8 +249,8 @@ public class NewDatasetServiceImpl implements NewDatasetService {
String url = gitendpoint + "/api/users/" + userInfo.get("login") + "/projects.json?category=manage";
String name = dataset.getName();
- if (StringUtils.isNotEmpty(name)){
- url =url + "&search=" + name;
+ if (StringUtils.isNotEmpty(name)) {
+ url = url + "&search=" + name;
}
String req = httpUtils.sendGetWithToken(url, null, token);
Map stringObjectMap = JacksonUtil.parseJSONStr2Map(req);
@@ -248,7 +265,7 @@ public class NewDatasetServiceImpl implements NewDatasetService {
@Override
public Page newPubilcQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception {
String token = gitService.checkoutToken();
- Jedis jedis = new Jedis(redisHost,redisPort);
+ Jedis jedis = new Jedis(redisHost, redisPort);
LoginUser loginUser = SecurityUtils.getLoginUser();
String ci4sUsername = loginUser.getUsername();
String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
@@ -258,8 +275,8 @@ public class NewDatasetServiceImpl implements NewDatasetService {
String url = gitendpoint + "/api/projects.json?user_id=" + userId + "&sort_by=praises_count";
String name = dataset.getName();
- if (StringUtils.isNotEmpty(name)){
- url =url + "&search=" + name;
+ if (StringUtils.isNotEmpty(name)) {
+ url = url + "&search=" + name;
}
String req = httpUtils.sendGetWithToken(url, null, token);
@@ -287,14 +304,14 @@ public class NewDatasetServiceImpl implements NewDatasetService {
}
version = (String) versionList.get(0).get("name");
}
- List> fileDetailsAfterGitPull = dvcUtils.getFileDetailsAfterGitPull(localPathlocal +loginUser.getUsername()+"/datasets/" + id, repo, version, "dataset",gitLinkUsername, gitLinkPassword);
+ List> fileDetailsAfterGitPull = dvcUtils.getFileDetailsAfterGitPull(localPathlocal + loginUser.getUsername() + "/datasets/" + id, repo, version, "dataset", gitLinkUsername, gitLinkPassword);
// 在localPathlocal+id+"/"+repositoryName目录下的dataset.yaml中取到元数据
- System.out.println("!!!!!!!!!!!!!!!!!!!!!!!!!!!fileDetailsAfterGitPull:{}"+fileDetailsAfterGitPull);
- Map stringObjectMap = YamlUtils.loadYamlFile(localPathlocal +loginUser.getUsername()+"/datasets/" + id + "/" + repo + "/" + "dataset.yaml");
+ System.out.println("!!!!!!!!!!!!!!!!!!!!!!!!!!!fileDetailsAfterGitPull:{}" + fileDetailsAfterGitPull);
+ Map stringObjectMap = YamlUtils.loadYamlFile(localPathlocal + loginUser.getUsername() + "/datasets/" + id + "/" + repo + "/" + "dataset.yaml");
NewDatasetVo newDatasetVo = ConvertUtil.convertMapToObject(stringObjectMap, NewDatasetVo.class);
List versionVos = new ArrayList();
- if (fileDetailsAfterGitPull!=null&&fileDetailsAfterGitPull.size()>0){
- for(Map fileDetail : fileDetailsAfterGitPull){
+ if (fileDetailsAfterGitPull != null && fileDetailsAfterGitPull.size() > 0) {
+ for (Map fileDetail : fileDetailsAfterGitPull) {
VersionVo versionVo = new VersionVo();
versionVo.setUrl((String) fileDetail.get("filePath"));
versionVo.setFileName((String) fileDetail.get("fileName"));
@@ -310,7 +327,7 @@ public class NewDatasetServiceImpl implements NewDatasetService {
@Override
public List> getVersionList(String repo, String owner) throws Exception {
String token = gitService.checkoutToken();
- List> brancheList = gitService.getBrancheList(token,owner, repo);
+ List> brancheList = gitService.getBrancheList(token, owner, repo);
return brancheList.stream()
.filter(branch -> !"master".equals(branch.get("name")))
.collect(Collectors.toList());
@@ -319,13 +336,13 @@ public class NewDatasetServiceImpl implements NewDatasetService {
@Override
public void deleteDatasetNew(String repo, String owner) throws Exception {
String token = gitService.checkoutToken();
- gitService.deleteProject(token,owner, repo);
+ gitService.deleteProject(token, owner, repo);
}
@Override
public void deleteDatasetVersionNew(String repo, String owner, String version, String relativePath) throws Exception {
String token = gitService.checkoutToken();
- gitService.deleteBranch(token,owner, repo, version, localPathlocal + relativePath);
+ gitService.deleteBranch(token, owner, repo, version, localPathlocal + relativePath);
}
@Override
@@ -378,14 +395,14 @@ public class NewDatasetServiceImpl implements NewDatasetService {
}
@Override
- public ResponseEntity downloadAllDatasetFilesNew(String name,Integer id, String version) throws Exception {
+ public ResponseEntity downloadAllDatasetFilesNew(String name, Integer id, String version) throws Exception {
// 命令行操作 git clone 项目地址
LoginUser loginUser = SecurityUtils.getLoginUser();
String ci4sUsername = loginUser.getUsername();
- Jedis jedis = new Jedis(redisHost,redisPort);
+ Jedis jedis = new Jedis(redisHost, redisPort);
String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
Map userInfo = JsonUtils.jsonToMap(userReq);
- String localPath = localPathlocal+ loginUser.getUsername()+"/datasets/"+id+"/"+name;
+ String localPath = localPathlocal + loginUser.getUsername() + "/datasets/" + id + "/" + name;
// 打包 data 文件夹
String dataFolderPath = localPath + "/data";
@@ -412,7 +429,7 @@ public class NewDatasetServiceImpl implements NewDatasetService {
InputStreamResource resource = new InputStreamResource(new FileInputStream(zipFile));
return ResponseEntity.ok()
- .header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename="+name+"_"+version+"_data.zip")
+ .header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=" + name + "_" + version + "_data.zip")
.contentType(MediaType.APPLICATION_OCTET_STREAM)
.contentLength(zipFile.length())
.body(resource);
@@ -420,7 +437,6 @@ public class NewDatasetServiceImpl implements NewDatasetService {
}
-
public List convert(List> lst, String datasetTopic, String datasetTagName, String datasetTypeName) {
if (lst != null && lst.size() > 0) {
List result = new ArrayList<>();
@@ -460,12 +476,12 @@ public class NewDatasetServiceImpl implements NewDatasetService {
return new ArrayList<>();
}
- private void addDatasetSourceToDataVo(NewDatasetVo datasetVo){
+ private void addDatasetSourceToDataVo(NewDatasetVo datasetVo) {
DatasetTempStorage queryDatasetTempStorage = new DatasetTempStorage();
queryDatasetTempStorage.setName(datasetVo.getName());
queryDatasetTempStorage.setVersion(datasetVo.getVersion());
DatasetTempStorage datasetTempStorage = datasetTempStorageService.queryByDatasetTempStorage(queryDatasetTempStorage);
- if (datasetTempStorage != null){
+ if (datasetTempStorage != null) {
String datasetSourceString = datasetTempStorage.getSource();
Map datasetSourceMap = JacksonUtil.parseJSONStr2Map(datasetSourceString);
String preprocessCode = (String) datasetSourceMap.get("preprocess_code");