|
|
|
@@ -41,7 +41,6 @@ import java.nio.file.Files; |
|
|
|
import java.nio.file.Path; |
|
|
|
import java.nio.file.Paths; |
|
|
|
import java.util.*; |
|
|
|
import java.util.concurrent.CompletableFuture; |
|
|
|
import java.util.stream.Collectors; |
|
|
|
import java.util.zip.ZipEntry; |
|
|
|
import java.util.zip.ZipOutputStream; |
|
|
|
@@ -249,6 +248,29 @@ public class ModelsServiceImpl implements ModelsService { |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
@Override |
|
|
|
public ResponseEntity<InputStreamResource> downloadDatasetlocal(String filePath) throws Exception { |
|
|
|
File file = new File(filePath); |
|
|
|
|
|
|
|
if (!file.exists()) { |
|
|
|
throw new FileNotFoundException("File not found: " + filePath); |
|
|
|
} |
|
|
|
|
|
|
|
InputStreamResource resource = new InputStreamResource(new FileInputStream(file)); |
|
|
|
|
|
|
|
HttpHeaders headers = new HttpHeaders(); |
|
|
|
headers.add(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=" + file.getName()); |
|
|
|
headers.add(HttpHeaders.CONTENT_LENGTH, String.valueOf(file.length())); |
|
|
|
headers.add(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_OCTET_STREAM_VALUE); |
|
|
|
|
|
|
|
return ResponseEntity.ok() |
|
|
|
.headers(headers) |
|
|
|
.contentLength(file.length()) |
|
|
|
.contentType(MediaType.APPLICATION_OCTET_STREAM) |
|
|
|
.body(resource); |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
/** |
|
|
|
* 上传模型文件 |
|
|
|
* |
|
|
|
@@ -389,7 +411,6 @@ public class ModelsServiceImpl implements ModelsService { |
|
|
|
Models models = new Models(); |
|
|
|
models.setName(modelsVo.getName()); |
|
|
|
models.setDescription(modelsVo.getDescription()); |
|
|
|
models.setAvailableRange(modelsVo.getAvailableRange()); |
|
|
|
models.setModelType(modelsVo.getModelType()); |
|
|
|
models.setModelTag(modelsVo.getModelTag()); |
|
|
|
Models modelsInsert = this.insert(models); |
|
|
|
@@ -516,64 +537,206 @@ public class ModelsServiceImpl implements ModelsService { |
|
|
|
} |
|
|
|
|
|
|
|
@Override |
|
|
|
public CompletableFuture<String> newCreateModel(ModelsVo modelsVo) { |
|
|
|
return CompletableFuture.supplyAsync(() -> { |
|
|
|
try { |
|
|
|
LoginUser loginUser = SecurityUtils.getLoginUser(); |
|
|
|
String ci4sUsername = loginUser.getUsername(); |
|
|
|
String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername(); |
|
|
|
String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword(); |
|
|
|
public String newCreateModel(ModelsVo modelsVo) { |
|
|
|
try { |
|
|
|
LoginUser loginUser = SecurityUtils.getLoginUser(); |
|
|
|
String ci4sUsername = loginUser.getUsername(); |
|
|
|
String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername(); |
|
|
|
String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword(); |
|
|
|
|
|
|
|
// String ci4sUsername = "chenzhihang"; |
|
|
|
// String gitLinkUsername = "chenzhihang"; |
|
|
|
// String gitLinkPassword = "czh19970311"; |
|
|
|
|
|
|
|
Map<String, Object> userInfo = getUserInfo(ci4sUsername, gitLinkUsername, gitLinkPassword); |
|
|
|
Integer userId = (Integer) userInfo.get("user_id"); |
|
|
|
|
|
|
|
String repositoryName = modelsVo.getIdentifier() == null ? ci4sUsername + "_model_" + DateUtils.dateTimeNow() : modelsVo.getIdentifier(); |
|
|
|
ModelDependency1 modelDependency = new ModelDependency1(); |
|
|
|
List<ModelDependency1> oldModelDependencys = modelDependency1Dao.queryModelDependency(modelsVo.getName(), null, gitLinkUsername); |
|
|
|
if (oldModelDependencys != null && !oldModelDependencys.isEmpty()) { |
|
|
|
if (oldModelDependencys.stream().anyMatch(oldModelDependency -> oldModelDependency.getVersion().equals(modelsVo.getVersion()))) { |
|
|
|
throw new RuntimeException("模型版本已存在,请勿重复创建"); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
Map<String, Object> userInfo = getUserInfo(ci4sUsername); |
|
|
|
Integer userId = (Integer) userInfo.get("user_id"); |
|
|
|
//新建模型 |
|
|
|
// 拼接project |
|
|
|
GitProjectVo gitProjectVo = new GitProjectVo(); |
|
|
|
gitProjectVo.setRepositoryName(repositoryName); |
|
|
|
gitProjectVo.setName(modelsVo.getName()); |
|
|
|
gitProjectVo.setDescription(modelsVo.getDescription()); |
|
|
|
gitProjectVo.setPrivate(!modelsVo.getIsPublic()); |
|
|
|
gitProjectVo.setUserId(userId); |
|
|
|
|
|
|
|
// 创建项目 |
|
|
|
Map project = gitService.createProject(gitProjectVo); |
|
|
|
Integer gitlinIid = (Integer) project.get("id"); |
|
|
|
if (gitlinIid == null) { |
|
|
|
throw new Exception("创建模型失败:" + project.get("message")); |
|
|
|
} |
|
|
|
// 创建分支 |
|
|
|
gitService.createBranch((String) userInfo.get("login"), repositoryName, modelsVo.getVersion(), "master"); |
|
|
|
// 定义标签 标签1:ci4s_model 标签2:ModelTag 标签3:ModelType |
|
|
|
gitService.createTopic(gitlinIid, "ci4s_model"); |
|
|
|
gitService.createTopic(gitlinIid, "modeltag_" + modelsVo.getModelTag()); |
|
|
|
gitService.createTopic(gitlinIid, "modeltype_" + modelsVo.getModelType()); |
|
|
|
|
|
|
|
String branchName = modelsVo.getVersion(); |
|
|
|
String owner = (String) userInfo.get("login"); |
|
|
|
|
|
|
|
String projectUrl = gitendpoint + "/" + owner + "/" + repositoryName + ".git"; |
|
|
|
String sourcePath = modelsVo.getModelVersionVos().get(0).getUrl(); |
|
|
|
String relatePath = ci4sUsername + "/model/" + gitlinIid + "/" + modelsVo.getName(); |
|
|
|
String rootPath = localPath + relatePath; |
|
|
|
String modelPath = rootPath + "/model"; |
|
|
|
String metaPath = rootPath + "/metadata"; |
|
|
|
|
|
|
|
String repositoryName = modelsVo.getRepositoryName() == null ? ci4sUsername + "_model_" + DateUtils.dateTimeNow() : modelsVo.getRepositoryName(); |
|
|
|
ModelDependency1 modelDependency = new ModelDependency1(); |
|
|
|
List<ModelDependency1> oldModelDependencys = modelDependency1Dao.queryByModelName(modelsVo.getName()); |
|
|
|
DVCUtils.gitClone(rootPath, projectUrl, branchName, gitLinkUsername, gitLinkPassword); |
|
|
|
DVCUtils.moveFiles(sourcePath, modelPath); |
|
|
|
|
|
|
|
//拼接生产的元数据后写入yaml文件 |
|
|
|
ModelMetaVo modelMetaVo = new ModelMetaVo(); |
|
|
|
BeanUtils.copyProperties(modelsVo, modelMetaVo); |
|
|
|
modelMetaVo.setId(gitlinIid); |
|
|
|
modelMetaVo.setCreateBy(String.valueOf(StringUtils.isNotEmpty((String) userInfo.get("nickname")) ? userInfo.get("nickname") : userInfo.get("login"))); |
|
|
|
modelMetaVo.setUpdateTime(DateUtils.getTime()); |
|
|
|
modelMetaVo.setUsage("<pre><code>" + |
|
|
|
"# 克隆模型配置文件与存储参数到本地\n" + |
|
|
|
"git clone -b " + branchName + " " + projectUrl + "\n" + |
|
|
|
"# 远程拉取配置文件\n" + |
|
|
|
"dvc pull\n" + |
|
|
|
"</code></pre>"); |
|
|
|
modelMetaVo.setIdentifier(repositoryName); |
|
|
|
modelMetaVo.setOwner(owner); |
|
|
|
modelMetaVo.setVersionDesc(modelMetaVo.getDescription()); |
|
|
|
modelMetaVo.setRelativePaths(relatePath); |
|
|
|
|
|
|
|
Map<String, Object> metaMap = JsonUtils.objectToMap(modelMetaVo); |
|
|
|
YamlUtils.generateYamlFile(metaMap, metaPath, "metadata"); |
|
|
|
String meta = metaMap.toString(); |
|
|
|
|
|
|
|
DVCUtils.dvcInit(rootPath); |
|
|
|
// 配置远程S3地址 |
|
|
|
String s3Path = "management-platform-files/" + ci4sUsername + "/model/" + gitlinIid + "/" + repositoryName + "/" + branchName; |
|
|
|
DVCUtils.dvcRemoteAdd(rootPath, s3Path); |
|
|
|
DVCUtils.dvcConfigS3Credentials(rootPath, endpoint); |
|
|
|
DVCUtils.dvcConfigS3Credentials2(rootPath, accessKeyId); |
|
|
|
DVCUtils.dvcConfigS3Credentials3(rootPath, secretAccessKey); |
|
|
|
|
|
|
|
// dvc 跟踪 |
|
|
|
DVCUtils.dvcAdd(rootPath, "model"); |
|
|
|
// git commit |
|
|
|
DVCUtils.gitAdd(rootPath, "."); |
|
|
|
DVCUtils.gitCommit(rootPath, "commit from ci4s with " + ci4sUsername); |
|
|
|
DVCUtils.gitPush(rootPath, gitLinkUsername, gitLinkPassword); |
|
|
|
DVCUtils.dvcPush(rootPath); |
|
|
|
|
|
|
|
//保存模型依赖 |
|
|
|
modelDependency.setRepoId(gitlinIid); |
|
|
|
modelDependency.setIdentifier(repositoryName); |
|
|
|
modelDependency.setModelName(modelsVo.getName()); |
|
|
|
modelDependency.setVersion(modelsVo.getVersion()); |
|
|
|
modelDependency.setParentModel(modelsVo.getParentModel()); |
|
|
|
modelDependency.setOwner(gitLinkUsername); |
|
|
|
modelDependency.setMeta(meta); |
|
|
|
modelDependency1Dao.insert(modelDependency); |
|
|
|
return "新增模型成功"; |
|
|
|
} catch (Exception e) { |
|
|
|
logger.error(e.getMessage()); |
|
|
|
throw new RuntimeException(e); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
if (oldModelDependencys != null && !oldModelDependencys.isEmpty()) { |
|
|
|
@Override |
|
|
|
public String newCreateVersion(ModelsVo modelsVo) throws Exception { |
|
|
|
LoginUser loginUser = SecurityUtils.getLoginUser(); |
|
|
|
String ci4sUsername = loginUser.getUsername(); |
|
|
|
String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername(); |
|
|
|
String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword(); |
|
|
|
|
|
|
|
if (oldModelDependencys.stream().anyMatch(oldModelDependency -> oldModelDependency.getRepoId().equals(modelsVo.getRepoId()) && oldModelDependency.getVersion().equals(modelsVo.getVersion()))) { |
|
|
|
throw new RuntimeException("模型版本已存在,请勿重复创建"); |
|
|
|
} |
|
|
|
// String ci4sUsername = "chenzhihang"; |
|
|
|
// String gitLinkUsername = "chenzhihang"; |
|
|
|
// String gitLinkPassword = "czh19970311"; |
|
|
|
|
|
|
|
//新建版本 |
|
|
|
commonDvc(userInfo, ci4sUsername, modelsVo.getRepoId(), modelsVo.getRepositoryName(), modelsVo, gitLinkUsername, gitLinkPassword, "CreateModelFromPipeline"); |
|
|
|
modelDependency.setRepoId(modelsVo.getRepoId()); |
|
|
|
} else { |
|
|
|
//新建模型 |
|
|
|
// 拼接project |
|
|
|
GitProjectVo gitProjectVo = new GitProjectVo(); |
|
|
|
gitProjectVo.setRepositoryName(repositoryName); |
|
|
|
gitProjectVo.setName(modelsVo.getName()); |
|
|
|
gitProjectVo.setDescription(modelsVo.getDescription()); |
|
|
|
gitProjectVo.setPrivate(modelsVo.getAvailableRange() == 0); |
|
|
|
gitProjectVo.setUserId(userId); |
|
|
|
|
|
|
|
// 创建项目 |
|
|
|
Map project = gitService.createProject(gitProjectVo); |
|
|
|
Integer gitlinIid = (Integer) project.get("id"); |
|
|
|
modelDependency.setRepoId(gitlinIid); |
|
|
|
// 定义标签 标签1:ci4s_model 标签2:ModelTag 标签3:ModelType |
|
|
|
gitService.createTopic(gitlinIid, "ci4s_model"); |
|
|
|
gitService.createTopic(gitlinIid, "modeltag_" + modelsVo.getModelTag()); |
|
|
|
gitService.createTopic(gitlinIid, "modeltype_" + modelsVo.getModelType()); |
|
|
|
commonDvc(userInfo, ci4sUsername, gitlinIid, repositoryName, modelsVo, gitLinkUsername, gitLinkPassword, "createModel"); |
|
|
|
} |
|
|
|
Map<String, Object> userInfo = getUserInfo(ci4sUsername, gitLinkUsername, gitLinkPassword); |
|
|
|
|
|
|
|
//保存模型依赖 |
|
|
|
modelDependency.setIdentifier(repositoryName); |
|
|
|
modelDependency.setModelName(modelsVo.getName()); |
|
|
|
modelDependency.setVersion(modelsVo.getVersion()); |
|
|
|
modelDependency.setParentModel(modelsVo.getParentModel()); |
|
|
|
modelDependency.setOwner(ci4sUsername); |
|
|
|
modelDependency1Dao.insert(modelDependency); |
|
|
|
return "新增模型成功"; |
|
|
|
} catch (Exception e) { |
|
|
|
logger.error(e.getMessage()); |
|
|
|
throw new RuntimeException(e); |
|
|
|
String repositoryName = modelsVo.getIdentifier() == null ? ci4sUsername + "_model_" + DateUtils.dateTimeNow() : modelsVo.getIdentifier(); |
|
|
|
ModelDependency1 modelDependency = new ModelDependency1(); |
|
|
|
List<ModelDependency1> oldModelDependencys = modelDependency1Dao.queryModelDependency(modelsVo.getName(), modelsVo.getId(), gitLinkUsername); |
|
|
|
if (oldModelDependencys != null && !oldModelDependencys.isEmpty()) { |
|
|
|
if (oldModelDependencys.stream().anyMatch(oldModelDependency -> oldModelDependency.getVersion().equals(modelsVo.getVersion()))) { |
|
|
|
throw new RuntimeException("模型版本已存在,请勿重复创建"); |
|
|
|
} |
|
|
|
}); |
|
|
|
|
|
|
|
//新建版本 |
|
|
|
String branchName = modelsVo.getVersion(); |
|
|
|
String owner = (String) userInfo.get("login"); |
|
|
|
|
|
|
|
String projectUrl = gitendpoint + "/" + owner + "/" + repositoryName + ".git"; |
|
|
|
String sourcePath = modelsVo.getModelVersionVos().get(0).getUrl(); |
|
|
|
String relatePath = ci4sUsername + "/model/" + modelsVo.getId() + "/" + modelsVo.getName(); |
|
|
|
String rootPath = localPath + relatePath; |
|
|
|
String modelPath = rootPath + "/model"; |
|
|
|
String metaPath = rootPath + "/metadata"; |
|
|
|
|
|
|
|
//部分信息在前面的版本里面,从那边取过来 |
|
|
|
Map<String, Object> stringObjectMap = YamlUtils.loadYamlFile(metaPath + "/" + "metadata.yaml"); |
|
|
|
ModelMetaVo oldModelVo = ConvertUtil.convertMapToObject(stringObjectMap, ModelMetaVo.class); |
|
|
|
|
|
|
|
// 创建本地分支 |
|
|
|
DVCUtils.createLocalBranchBasedOnMaster(rootPath, branchName); |
|
|
|
//dvc checkout |
|
|
|
DVCUtils.dvcCheckout(rootPath); |
|
|
|
DVCUtils.moveFiles(sourcePath, modelPath); |
|
|
|
|
|
|
|
//拼接生产的元数据后写入yaml文件 |
|
|
|
ModelMetaVo modelMetaVo = new ModelMetaVo(); |
|
|
|
BeanUtils.copyProperties(modelsVo, modelMetaVo); |
|
|
|
modelMetaVo.setCreateBy(String.valueOf(StringUtils.isNotEmpty((String) userInfo.get("nickname")) ? userInfo.get("nickname") : userInfo.get("login"))); |
|
|
|
modelMetaVo.setUpdateTime(DateUtils.getTime()); |
|
|
|
modelMetaVo.setUsage("<pre><code>" + |
|
|
|
"# 克隆模型配置文件与存储参数到本地\n" + |
|
|
|
"git clone -b " + branchName + " " + projectUrl + "\n" + |
|
|
|
"# 远程拉取配置文件\n" + |
|
|
|
"dvc pull\n" + |
|
|
|
"</code></pre>"); |
|
|
|
modelMetaVo.setIdentifier(repositoryName); |
|
|
|
modelMetaVo.setOwner(owner); |
|
|
|
modelMetaVo.setDescription(oldModelVo.getDescription()); |
|
|
|
modelMetaVo.setModelTag(oldModelVo.getModelTag()); |
|
|
|
modelMetaVo.setModelType(oldModelVo.getModelType()); |
|
|
|
modelMetaVo.setRelativePaths(relatePath); |
|
|
|
|
|
|
|
Map<String, Object> metaMap = JsonUtils.objectToMap(modelMetaVo); |
|
|
|
YamlUtils.generateYamlFile(metaMap, metaPath, "metadata"); |
|
|
|
String meta = metaMap.toString(); |
|
|
|
|
|
|
|
// 配置远程S3地址 |
|
|
|
String s3Path = "management-platform-files/" + ci4sUsername + "/model/" + modelsVo.getId() + "/" + repositoryName + "/" + branchName; |
|
|
|
DVCUtils.dvcRemoteAdd(rootPath, s3Path); |
|
|
|
DVCUtils.dvcConfigS3Credentials(rootPath, endpoint); |
|
|
|
DVCUtils.dvcConfigS3Credentials2(rootPath, accessKeyId); |
|
|
|
DVCUtils.dvcConfigS3Credentials3(rootPath, secretAccessKey); |
|
|
|
// dvc 跟踪 |
|
|
|
DVCUtils.dvcAdd(rootPath, "model"); |
|
|
|
DVCUtils.pushNewBranchToRemote(rootPath, gitLinkUsername, gitLinkPassword, branchName); |
|
|
|
//dvc push 到远程S3 |
|
|
|
DVCUtils.dvcPush(rootPath); |
|
|
|
|
|
|
|
//保存模型依赖 |
|
|
|
modelDependency.setRepoId(modelsVo.getId()); |
|
|
|
modelDependency.setIdentifier(repositoryName); |
|
|
|
modelDependency.setModelName(modelsVo.getName()); |
|
|
|
modelDependency.setVersion(modelsVo.getVersion()); |
|
|
|
modelDependency.setParentModel(modelsVo.getParentModel()); |
|
|
|
modelDependency.setOwner(gitLinkUsername); |
|
|
|
modelDependency.setMeta(meta); |
|
|
|
modelDependency1Dao.insert(modelDependency); |
|
|
|
|
|
|
|
return "新增模型版本成功"; |
|
|
|
} |
|
|
|
return null; |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
@@ -584,8 +747,9 @@ public class ModelsServiceImpl implements ModelsService { |
|
|
|
for (MultipartFile file : files) { |
|
|
|
// 构建objectName |
|
|
|
String username = SecurityUtils.getLoginUser().getUsername(); |
|
|
|
// String username = "chenzhihang"; |
|
|
|
String fileName = file.getOriginalFilename(); |
|
|
|
String path = "/temp/" + username + "/models/" + uuid + "/model/"; |
|
|
|
String path = localPath + "/temp/" + username + "/models/" + uuid; |
|
|
|
long sizeInBytes = file.getSize(); |
|
|
|
String formattedSize = FileUtil.formatFileSize(sizeInBytes); |
|
|
|
File targetFile = new File(path, file.getOriginalFilename()); |
|
|
|
@@ -594,10 +758,9 @@ public class ModelsServiceImpl implements ModelsService { |
|
|
|
// 保存文件到目标路径 |
|
|
|
FileUtils.copyInputStreamToFile(file.getInputStream(), targetFile); |
|
|
|
// 返回上传文件的路径 |
|
|
|
String absolutePath = targetFile.getAbsolutePath(); |
|
|
|
Map<String, String> result = new HashMap<>(); |
|
|
|
result.put("fileName", fileName); |
|
|
|
result.put("url", absolutePath); // objectName根据实际情况定义 |
|
|
|
result.put("url", path); // objectName根据实际情况定义 |
|
|
|
result.put("fileSize", formattedSize); |
|
|
|
results.add(result); |
|
|
|
} |
|
|
|
@@ -605,27 +768,12 @@ public class ModelsServiceImpl implements ModelsService { |
|
|
|
} |
|
|
|
|
|
|
|
@Override |
|
|
|
public ResponseEntity<InputStreamResource> downloadAllModelFilesNew(String identifier, String version) throws Exception { |
|
|
|
public ResponseEntity<InputStreamResource> downloadAllModelFilesNew(String identifier, Integer id, String version) throws Exception { |
|
|
|
// 命令行操作 git clone 项目地址 |
|
|
|
LoginUser loginUser = SecurityUtils.getLoginUser(); |
|
|
|
String ci4sUsername = loginUser.getUsername(); |
|
|
|
String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername(); |
|
|
|
String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword(); |
|
|
|
Map<String, Object> userInfo = getUserInfo(ci4sUsername); |
|
|
|
|
|
|
|
String projectUrl = gitendpoint + "/" + userInfo.get("login") + "/" + identifier + ".git"; |
|
|
|
String localPath1 = localPath + ci4sUsername + "/model/" + identifier; |
|
|
|
File folder = new File(localPath1); |
|
|
|
if (folder.exists() && folder.isDirectory()) { |
|
|
|
//切换分支 |
|
|
|
DVCUtils.gitCheckoutBranch(localPath1, version); |
|
|
|
//pull |
|
|
|
DVCUtils.gitPull(localPath1, gitLinkUsername, gitLinkPassword); |
|
|
|
//dvc pull |
|
|
|
DVCUtils.dvcPull(localPath1); |
|
|
|
} else { |
|
|
|
DVCUtils.gitClone(localPath1, projectUrl, version, gitLinkUsername, gitLinkPassword); |
|
|
|
} |
|
|
|
|
|
|
|
String localPath1 = localPath + ci4sUsername + "/model/" + id + "/" + identifier; |
|
|
|
// 打包 data 文件夹 |
|
|
|
String dataFolderPath = localPath1 + "/model"; |
|
|
|
String zipFilePath = localPath1 + "/model.zip"; |
|
|
|
@@ -658,21 +806,33 @@ public class ModelsServiceImpl implements ModelsService { |
|
|
|
} |
|
|
|
|
|
|
|
@Override |
|
|
|
public Page<ModelsVo> newPubilcQueryByPage(ModelsVo modelsVo, PageRequest pageRequest) throws Exception { |
|
|
|
public Page<ModelsVo> newPersonalQueryByPage(ModelsVo modelsVo, PageRequest pageRequest) throws Exception { |
|
|
|
LoginUser loginUser = SecurityUtils.getLoginUser(); |
|
|
|
String ci4sUsername = loginUser.getUsername(); |
|
|
|
Map<String, Object> userInfo = getUserInfo(ci4sUsername); |
|
|
|
String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername(); |
|
|
|
String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword(); |
|
|
|
|
|
|
|
// String ci4sUsername = "chenzhihang"; |
|
|
|
// String gitLinkUsername = "chenzhihang"; |
|
|
|
// String gitLinkPassword = "czh19970311"; |
|
|
|
|
|
|
|
Map<String, Object> userInfo = getUserInfo(ci4sUsername, gitLinkUsername, gitLinkPassword); |
|
|
|
String token = (String) userInfo.get("token"); |
|
|
|
|
|
|
|
//拼接查询url |
|
|
|
String modelTagName = modelsVo.getModelTag(); |
|
|
|
String modelTypeName = modelsVo.getModelType(); |
|
|
|
String search = modelsVo.getName(); |
|
|
|
String topic_name = "ci4s_model"; |
|
|
|
topic_name = StringUtils.isEmpty(modelTagName) ? topic_name : topic_name + ",modeltag_" + modelTypeName; |
|
|
|
topic_name = StringUtils.isEmpty(modelTagName) ? topic_name : topic_name + ",modeltype_" + modelTypeName; |
|
|
|
|
|
|
|
String url = gitendpoint + "/api/users/" + userInfo.get("login") + "/projects.json?page=" + pageRequest.getPageNumber() + "&limit=" + pageRequest.getPageSize() + "&category=manage&topic_name=" + topic_name + "&search=" + search; |
|
|
|
String url = gitendpoint + "/api/users/" + userInfo.get("login") + "/projects.json?page=" + pageRequest.getPageNumber() + "&limit=" + pageRequest.getPageSize() + "&category=manage&topic_name=" + topic_name; |
|
|
|
|
|
|
|
String name = modelsVo.getName(); |
|
|
|
if (StringUtils.isNotEmpty(name)) { |
|
|
|
url = url + "&search=" + name; |
|
|
|
} |
|
|
|
|
|
|
|
String req = HttpUtils.sendGetWithToken(url, null, token); |
|
|
|
Map<String, Object> stringObjectMap = JacksonUtil.parseJSONStr2Map(req); |
|
|
|
Integer total = (Integer) stringObjectMap.get("count"); |
|
|
|
@@ -681,22 +841,29 @@ public class ModelsServiceImpl implements ModelsService { |
|
|
|
} |
|
|
|
|
|
|
|
@Override |
|
|
|
public Page<ModelsVo> newPersonalQueryByPage(ModelsVo modelsVo, PageRequest pageRequest) throws Exception { |
|
|
|
public Page<ModelsVo> newPubilcQueryByPage(ModelsVo modelsVo, PageRequest pageRequest) throws Exception { |
|
|
|
LoginUser loginUser = SecurityUtils.getLoginUser(); |
|
|
|
String ci4sUsername = loginUser.getUsername(); |
|
|
|
Map<String, Object> userInfo = getUserInfo(ci4sUsername); |
|
|
|
String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername(); |
|
|
|
String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword(); |
|
|
|
|
|
|
|
Map<String, Object> userInfo = getUserInfo(ci4sUsername, gitLinkUsername, gitLinkPassword); |
|
|
|
Integer userId = (Integer) userInfo.get("user_id"); |
|
|
|
String token = (String) userInfo.get("token"); |
|
|
|
|
|
|
|
//拼接查询url |
|
|
|
String modelTagName = modelsVo.getModelTag(); |
|
|
|
String modelTypeName = modelsVo.getModelType(); |
|
|
|
String search = modelsVo.getName(); |
|
|
|
String topic_name = "ci4s_model"; |
|
|
|
topic_name = StringUtils.isEmpty(modelTagName) ? topic_name : topic_name + ",modeltag_" + modelTagName; |
|
|
|
topic_name = StringUtils.isEmpty(modelTypeName) ? topic_name : topic_name + ",modeltype_" + modelTypeName; |
|
|
|
|
|
|
|
String url = gitendpoint + "/api/projects.json?user_id=" + userId + "&page=" + pageRequest.getPageNumber() + "&limit=" + pageRequest.getPageSize() + "&sort_by=praises_count&topic_name=" + topic_name + "&search=" + search; |
|
|
|
String url = gitendpoint + "/api/projects.json?user_id=" + userId + "&page=" + pageRequest.getPageNumber() + "&limit=" + pageRequest.getPageSize() + "&sort_by=praises_count&topic_name=" + topic_name; |
|
|
|
String name = modelsVo.getName(); |
|
|
|
if (StringUtils.isNotEmpty(name)) { |
|
|
|
url = url + "&search=" + name; |
|
|
|
} |
|
|
|
|
|
|
|
String req = HttpUtils.sendGetWithToken(url, null, token); |
|
|
|
Map<String, Object> stringObjectMap = JacksonUtil.parseJSONStr2Map(req); |
|
|
|
Integer total = (Integer) stringObjectMap.get("total_count"); |
|
|
|
@@ -705,37 +872,37 @@ public class ModelsServiceImpl implements ModelsService { |
|
|
|
} |
|
|
|
|
|
|
|
@Override |
|
|
|
public List<Map<String, Object>> getVersionList(String identifier, String owner) throws Exception { |
|
|
|
public List<Map<String, Object>> getVersionList(String identifier, String owner) throws Exception { |
|
|
|
List<Map<String, Object>> brancheList = gitService.getBrancheList(owner, identifier); |
|
|
|
return brancheList.stream() |
|
|
|
.filter(branch -> "master".equals(branch.get("name"))) |
|
|
|
.filter(branch -> !"master".equals(branch.get("name"))) |
|
|
|
.collect(Collectors.toList()); |
|
|
|
} |
|
|
|
|
|
|
|
@Override |
|
|
|
public ModelsVo getModelDetail(String modelName, String identifier, String owner, String version) throws Exception { |
|
|
|
public ModelsVo getModelDetail(Integer id, String name, String identifier, String owner, String version) throws Exception { |
|
|
|
LoginUser loginUser = SecurityUtils.getLoginUser(); |
|
|
|
String ci4sUsername = loginUser.getUsername(); |
|
|
|
String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername(); |
|
|
|
String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword(); |
|
|
|
// String ci4sUsername = "chenzhihang"; |
|
|
|
// String gitLinkUsername = "chenzhihang"; |
|
|
|
// String gitLinkPassword = "czh19970311"; |
|
|
|
|
|
|
|
if (StringUtils.isEmpty(version)) { |
|
|
|
List<Map<String, Object>> versionList = this.getVersionList(owner, identifier); |
|
|
|
List<Map<String, Object>> versionList = this.getVersionList(identifier, owner); |
|
|
|
if (versionList.size() == 0) { |
|
|
|
throw new Exception("数据集文件不存在"); |
|
|
|
throw new Exception("模型文件不存在"); |
|
|
|
} |
|
|
|
version = (String) versionList.get(0).get("name"); |
|
|
|
} |
|
|
|
|
|
|
|
// git pull操作,然后读取里面的文件列表,列出每个文件的大小和名称,封装成MAP |
|
|
|
List<Map<String, Object>> fileDetailsAfterGitPull = DVCUtils.getFileDetailsAfterGitPull(localPath + ci4sUsername + "/model/", modelName, version, "model", gitLinkUsername, gitLinkPassword); |
|
|
|
List<Map<String, Object>> fileDetailsAfterGitPull = DVCUtils.getFileDetailsAfterGitPull(localPath + ci4sUsername + "/model/" + id, name, version, "model", gitLinkUsername, gitLinkPassword); |
|
|
|
|
|
|
|
Map<String, Object> stringObjectMap = YamlUtils.loadYamlFile(localPath + ci4sUsername + "/model/" + modelName + "/" + version + "/metadata/metadata.yaml"); |
|
|
|
assert stringObjectMap != null; |
|
|
|
Map<String, Object> stringObjectMap = YamlUtils.loadYamlFile(localPath + ci4sUsername + "/model/" + id + "/" + name + "/metadata/metadata.yaml"); |
|
|
|
|
|
|
|
ModelMetaVo modelMetaVo = ConvertUtil.convertMapToObject(stringObjectMap, ModelMetaVo.class); |
|
|
|
ModelsVo modelsVo = new ModelsVo(); |
|
|
|
BeanUtils.copyProperties(modelMetaVo, modelsVo); |
|
|
|
ModelsVo modelsVo = ConvertUtil.convertMapToObject(stringObjectMap, ModelsVo.class); |
|
|
|
|
|
|
|
List<VersionVo> versionVos = new ArrayList<>(); |
|
|
|
if (!fileDetailsAfterGitPull.isEmpty()) { |
|
|
|
@@ -772,12 +939,6 @@ public class ModelsServiceImpl implements ModelsService { |
|
|
|
|
|
|
|
@Override |
|
|
|
public void deleteModel(Integer repoId, String identifier, String owner) throws Exception { |
|
|
|
LoginUser loginUser = SecurityUtils.getLoginUser(); |
|
|
|
String ci4sUsername = loginUser.getUsername(); |
|
|
|
Map<String, Object> userInfo = getUserInfo(ci4sUsername); |
|
|
|
String token = (String) userInfo.get("token"); |
|
|
|
|
|
|
|
|
|
|
|
gitService.deleteProject(owner, identifier); |
|
|
|
//删除模型依赖 |
|
|
|
modelDependency1Dao.deleteModel(repoId, identifier, owner, null); |
|
|
|
@@ -786,72 +947,19 @@ public class ModelsServiceImpl implements ModelsService { |
|
|
|
|
|
|
|
@Override |
|
|
|
public void deleteVersion(Integer repoId, String identifier, String owner, String version) throws Exception { |
|
|
|
LoginUser loginUser = SecurityUtils.getLoginUser(); |
|
|
|
String ci4sUsername = loginUser.getUsername(); |
|
|
|
|
|
|
|
gitService.deleteBranch(owner, identifier, version); |
|
|
|
//删除模型依赖 |
|
|
|
modelDependency1Dao.deleteModel(repoId, identifier, owner, version); |
|
|
|
modelDependency1Dao.deleteModelDependency(repoId, identifier, owner, version); |
|
|
|
} |
|
|
|
|
|
|
|
void commonDvc(Map<String, Object> userInfo, String username, Integer gitlinIid, String repositoryName, ModelsVo modelsVo, String gitLinkUsername, String gitLinkPassword, String type) throws Exception { |
|
|
|
String branchName = modelsVo.getVersion(); |
|
|
|
|
|
|
|
// 创建分支 |
|
|
|
gitService.createBranch((String) userInfo.get("login"), repositoryName, branchName, "master"); |
|
|
|
|
|
|
|
String projectUrl = gitendpoint + "/" + (String) userInfo.get("login") + "/" + repositoryName + ".git"; |
|
|
|
String url = modelsVo.getModelVersionVos().get(0).getUrl(); |
|
|
|
String sourcePath = url.substring(0, url.lastIndexOf("/")); |
|
|
|
String rootPath = localPath + username + "/model/" + modelsVo.getName() + "/" + branchName; |
|
|
|
String modelPath = rootPath + "/model"; |
|
|
|
String metaPath = rootPath + "/metadata"; |
|
|
|
|
|
|
|
|
|
|
|
if (type.equals("CreateModelFromPipeline") && FileUtil.checkDirectoryExists(rootPath)) { |
|
|
|
DVCUtils.gitFetch(rootPath, gitLinkUsername, gitLinkPassword); |
|
|
|
DVCUtils.gitCheckoutBranch(rootPath, branchName); |
|
|
|
} else { |
|
|
|
DVCUtils.gitClone(rootPath, projectUrl, branchName, gitLinkUsername, gitLinkPassword); |
|
|
|
} |
|
|
|
|
|
|
|
DVCUtils.moveFiles(sourcePath, modelPath); |
|
|
|
|
|
|
|
//拼接生产的元数据后写入yaml文件 |
|
|
|
ModelMetaVo modelMetaVo = new ModelMetaVo(); |
|
|
|
BeanUtils.copyProperties(modelsVo, modelMetaVo); |
|
|
|
YamlUtils.generateYamlFile(JsonUtils.objectToMap(modelMetaVo), metaPath, "metadata"); |
|
|
|
|
|
|
|
// dvc init 初始化 |
|
|
|
DVCUtils.dvcInit(rootPath); |
|
|
|
|
|
|
|
// 配置远程S3地址 |
|
|
|
String s3Path = "management-platform-files/" + username + "/model/" + gitlinIid + repositoryName + "/" + branchName; |
|
|
|
DVCUtils.dvcRemoteAdd(rootPath, s3Path); |
|
|
|
DVCUtils.dvcConfigS3Credentials(rootPath, endpoint); |
|
|
|
DVCUtils.dvcConfigS3Credentials2(rootPath, accessKeyId); |
|
|
|
DVCUtils.dvcConfigS3Credentials3(rootPath, secretAccessKey); |
|
|
|
|
|
|
|
// dvc 跟踪 |
|
|
|
DVCUtils.dvcAdd(rootPath, "model"); |
|
|
|
|
|
|
|
// git commit |
|
|
|
DVCUtils.gitAdd(rootPath, "."); |
|
|
|
DVCUtils.gitCommit(rootPath, "commit from ci4s with " + username); |
|
|
|
DVCUtils.gitPush(rootPath, gitLinkUsername, gitLinkPassword); |
|
|
|
|
|
|
|
// dvc push 到远程S3 |
|
|
|
DVCUtils.dvcPush(rootPath); |
|
|
|
} |
|
|
|
|
|
|
|
public List<ModelsVo> convert(List<Map<String, Object>> lst) { |
|
|
|
if (lst != null && lst.size() > 0) { |
|
|
|
List<ModelsVo> newModelVos = ConvertUtil.convertListMapToObjectList(lst, ModelsVo.class); |
|
|
|
|
|
|
|
for (ModelsVo newModelVo : newModelVos) { |
|
|
|
Map<String, Object> map = lst.stream() |
|
|
|
.filter(m -> m.get("repo_id").equals(newModelVo.getRepoId())) |
|
|
|
.filter(m -> m.get("id").equals(newModelVo.getId())) |
|
|
|
.findFirst() |
|
|
|
.orElse(null); |
|
|
|
|
|
|
|
@@ -869,6 +977,9 @@ public class ModelsServiceImpl implements ModelsService { |
|
|
|
} |
|
|
|
}); |
|
|
|
} |
|
|
|
Map<String, Object> author = (Map<String, Object>) map.get("author"); |
|
|
|
newModelVo.setCreateBy((String) author.get("name")); |
|
|
|
newModelVo.setOwner((String) author.get("login")); |
|
|
|
} |
|
|
|
} |
|
|
|
return newModelVos; |
|
|
|
@@ -906,9 +1017,13 @@ public class ModelsServiceImpl implements ModelsService { |
|
|
|
modelDependency1TreeVo.setChildModelList(childModelList); |
|
|
|
} |
|
|
|
|
|
|
|
Map<String, Object> getUserInfo(String ci4sUsername) throws IOException { |
|
|
|
Map<String, Object> getUserInfo(String ci4sUsername, String gitLinkUsername, String gitLinkPassword) throws IOException { |
|
|
|
Jedis jedis = new Jedis(redisHost); |
|
|
|
String userReq = jedis.get(ci4sUsername + "_gitUserInfo"); |
|
|
|
if (userReq == null) { |
|
|
|
gitService.login(gitLinkUsername, gitLinkPassword); |
|
|
|
userReq = jedis.get(ci4sUsername + "_gitUserInfo"); |
|
|
|
} |
|
|
|
Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq); |
|
|
|
|
|
|
|
String token = gitService.checkoutToken(); |
|
|
|
|