Browse Source

修复创建公开数据集模型bug

dev-complex-computation
chenzhihang 1 year ago
parent
commit
3b018f0565
7 changed files with 49 additions and 42 deletions
  1. +2
    -0
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/constant/Constant.java
  2. +7
    -6
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/dataset/NewDatasetFromGitController.java
  3. +6
    -7
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/model/NewModelFromGitController.java
  4. +3
    -3
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ModelsService.java
  5. +11
    -5
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/NewDatasetService.java
  6. +11
    -14
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java
  7. +9
    -7
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java

+ 2
- 0
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/constant/Constant.java View File

@@ -22,6 +22,8 @@ public class Constant {

public final static int Git_Category_Id = 39;

public final static String Item_Public = "public";

public final static String Source_Auto_Export = "auto_export";
public final static String Source_Hand_Export = "hand_export";
public final static String Source_Add = "add";


+ 7
- 6
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/dataset/NewDatasetFromGitController.java View File

@@ -85,14 +85,15 @@ public class NewDatasetFromGitController {
@RequestParam("id") Integer id,
@RequestParam(value = "owner", required = false) String owner,
@RequestParam(value = "identifier", required = false) String identifier,
@RequestParam(value = "version", required = false) String version) throws Exception {
return AjaxResult.success(this.newDatasetService.getNewDatasetDesc(id, name, identifier, owner, version));
@RequestParam(value = "version", required = false) String version,
@RequestParam(value = "is_public") Boolean isPublic) throws Exception {
return AjaxResult.success(this.newDatasetService.getNewDatasetDesc(id, name, identifier, owner, version, isPublic));
}

@DeleteMapping("/deleteDataset")
@ApiOperation(value = "删除数据集")
public AjaxResult deleteDataset(@RequestParam("id") Integer id, @RequestParam("identifier") String identifier, @RequestParam("owner") String owner) throws Exception {
this.newDatasetService.deleteDatasetNew(id, identifier, owner);
public AjaxResult deleteDataset(@RequestParam("id") Integer id, @RequestParam("identifier") String identifier, @RequestParam("owner") String owner, @RequestParam("is_public") Boolean isPublic) throws Exception {
this.newDatasetService.deleteDatasetNew(id, identifier, owner, isPublic);
return AjaxResult.success();
}

@@ -132,8 +133,8 @@ public class NewDatasetFromGitController {
*/
@GetMapping("/downloadAllFiles")
@ApiOperation(value = "下载同一版本下所有数据集,并打包")
public ResponseEntity<InputStreamResource> downloadAllDatasetFiles(@RequestParam("name") String name, @RequestParam("identifier") String identifier, @RequestParam("id") Integer id, @RequestParam("version") String version) throws Exception {
return newDatasetService.downloadAllDatasetFilesNew(name, identifier, id, version);
public ResponseEntity<InputStreamResource> downloadAllDatasetFiles(@RequestParam("name") String name, @RequestParam("identifier") String identifier, @RequestParam("id") Integer id, @RequestParam("version") String version, @RequestParam("is_public") Boolean isPublic) throws Exception {
return newDatasetService.downloadAllDatasetFilesNew(name, identifier, id, version, isPublic);
}

/**


+ 6
- 7
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/model/NewModelFromGitController.java View File

@@ -1,7 +1,6 @@
package com.ruoyi.platform.controller.model;

import com.ruoyi.common.core.web.domain.AjaxResult;
import com.ruoyi.platform.domain.ModelDependency1;
import com.ruoyi.platform.service.ModelsService;
import com.ruoyi.platform.vo.ModelsVo;
import com.ruoyi.platform.vo.QueryModelMetricsVo;
@@ -46,8 +45,8 @@ public class NewModelFromGitController {

@GetMapping("/downloadAllFiles")
@ApiOperation(value = "下载同一版本下所有模型,并打包")
public ResponseEntity<InputStreamResource> downloadAllDatasetFiles(@RequestParam("name") String name, @RequestParam("identifier") String identifier, @RequestParam("id") Integer id, @RequestParam("version") String version) throws Exception {
return modelsService.downloadAllModelFilesNew(name, identifier, id, version);
public ResponseEntity<InputStreamResource> downloadAllDatasetFiles(@RequestParam("name") String name, @RequestParam("identifier") String identifier, @RequestParam("id") Integer id, @RequestParam("version") String version, @RequestParam("is_public") Boolean isPublic) throws Exception {
return modelsService.downloadAllModelFilesNew(name, identifier, id, version, isPublic);
}

@GetMapping("/downloadSingleFile")
@@ -103,8 +102,8 @@ public class NewModelFromGitController {

@GetMapping("/getModelDetail")
@ApiOperation(value = "获取模型详细信息")
public AjaxResult getModelDetail(@RequestParam("id") Integer id, @RequestParam("identifier") String identifier, @RequestParam("owner") String owner, @RequestParam("version") String version) throws Exception {
return AjaxResult.success(this.modelsService.getModelDetail(id, identifier, owner, version));
public AjaxResult getModelDetail(@RequestParam("id") Integer id, @RequestParam("identifier") String identifier, @RequestParam("owner") String owner, @RequestParam("version") String version, @RequestParam("is_public") Boolean isPublic) throws Exception {
return AjaxResult.success(this.modelsService.getModelDetail(id, identifier, owner, version, isPublic));
}

@GetMapping("/getModelDependencyTree")
@@ -116,8 +115,8 @@ public class NewModelFromGitController {
@DeleteMapping("/delete")
@ApiOperation(value = "删除模型")
public AjaxResult deleteModel(@RequestParam("id") Integer id, @RequestParam("identifier") String identifier,
@RequestParam("owner") String owner) throws Exception {
this.modelsService.deleteModel(id, identifier, owner);
@RequestParam("owner") String owner, @RequestParam("is_public") Boolean isPublic) throws Exception {
this.modelsService.deleteModel(id, identifier, owner, isPublic);
return AjaxResult.success();
}



+ 3
- 3
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ModelsService.java View File

@@ -95,7 +95,7 @@ public interface ModelsService {

List<Map<String, String>> uploadModelLocal(MultipartFile[] files, String uuid) throws Exception;

ResponseEntity<InputStreamResource> downloadAllModelFilesNew(String name, String identifier, Integer id, String version) throws IOException, Exception;
ResponseEntity<InputStreamResource> downloadAllModelFilesNew(String name, String identifier, Integer id, String version, Boolean isPublic) throws IOException, Exception;

Page<ModelsVo> newPubilcQueryByPage(ModelsVo modelsVo, PageRequest pageRequest) throws Exception;

@@ -107,11 +107,11 @@ public interface ModelsService {

List<Map<String, Object>> getVersionList(String identifier, String owner) throws Exception;

ModelsVo getModelDetail(Integer id, String identifier, String owner, String version) throws Exception;
ModelsVo getModelDetail(Integer id, String identifier, String owner, String version, Boolean isPublic) throws Exception;

ModelDependency1TreeVo getModelDependencyTree(Integer repoId, String identifier, String version) throws Exception;

void deleteModel(Integer repoId, String identifier, String owner) throws Exception;
void deleteModel(Integer repoId, String identifier, String owner, Boolean isPublic) throws Exception;

void deleteVersion(Integer repoId, String identifier, String owner, String version, String relativePath) throws Exception;
}

+ 11
- 5
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/NewDatasetService.java View File

@@ -16,18 +16,24 @@ public interface NewDatasetService {


String newCreateDataset(NewDatasetVo datasetVo) throws Exception;

String newCreateVersion(NewDatasetVo datasetVo) throws Exception;

List<Map<String, String>> uploadDatasetlocal(MultipartFile[] files, String uuid) throws Exception;

ResponseEntity<InputStreamResource> downloadDatasetlocal(String filePath) throws Exception;
ResponseEntity<InputStreamResource> downloadAllDatasetFilesNew(String name, String identifier, Integer id, String version) throws IOException, Exception;

ResponseEntity<InputStreamResource> downloadAllDatasetFilesNew(String name, String identifier, Integer id, String version, Boolean isPublic) throws IOException, Exception;

Page<NewDatasetVo> newPersonalQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception;
Page<NewDatasetVo> newPubilcQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception;

NewDatasetVo getNewDatasetDesc(Integer repoId,String name,String repo, String owner, String version)throws Exception;
Page<NewDatasetVo> newPubilcQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception;

NewDatasetVo getNewDatasetDesc(Integer repoId, String name, String repo, String owner, String version, Boolean isPublic) throws Exception;

List<Map<String, Object>> getVersionList(String repo, String owner) throws Exception;
List<Map<String, Object>> getVersionList(String repo, String owner) throws Exception;

void deleteDatasetNew(Integer repoId, String repo, String owner) throws Exception;
void deleteDatasetNew(Integer repoId, String repo, String owner, Boolean isPublic) throws Exception;

void deleteDatasetVersionNew(String repo, String owner, String version, String relativePath) throws Exception;
}

+ 11
- 14
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java View File

@@ -561,10 +561,10 @@ public class ModelsServiceImpl implements ModelsService {
String ci4sUsername = loginUser.getUsername();
String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername();
String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword();

Map<String, Object> userInfo = getUserInfo(ci4sUsername, gitLinkUsername, gitLinkPassword);
Integer userId = (Integer) userInfo.get("user_id");

ci4sUsername = modelsVo.getIsPublic() ? Constant.Item_Public : ci4sUsername;
String repositoryName = modelsVo.getIdentifier() == null ? ci4sUsername + "_model_" + DateUtils.dateTimeNow() : modelsVo.getIdentifier();
ModelDependency1 modelDependency = new ModelDependency1();
List<ModelDependency1> oldModelDependencys = modelDependency1Dao.queryModelDependency(modelsVo.getName(), null, gitLinkUsername);
@@ -650,11 +650,11 @@ public class ModelsServiceImpl implements ModelsService {
modelDependency.setState(Constant.State_valid);
modelDependency1Dao.insert(modelDependency);

String s3Path = bucketName + "/mini-model-management-platform-files/" + ci4sUsername + "/model/" + gitlinIid + "/" + repositoryName + "/" + branchName;
CompletableFuture.supplyAsync(() -> {
try {
dvcUtils.dvcInit(rootPath);
// 配置远程S3地址
String s3Path = bucketName + "/mini-model-management-platform-files/" + ci4sUsername + "/model/" + gitlinIid + "/" + repositoryName + "/" + branchName;
dvcUtils.dvcRemoteAdd(rootPath, s3Path);
dvcUtils.dvcConfigS3Credentials(rootPath, endpoint);
dvcUtils.dvcConfigS3Credentials2(rootPath, accessKeyId);
@@ -663,7 +663,7 @@ public class ModelsServiceImpl implements ModelsService {
// dvc 跟踪
dvcUtils.dvcAdd(rootPath, "model");
// git commit
dvcUtils.gitCommit(rootPath, "commit from ci4s with " + ci4sUsername);
dvcUtils.gitCommit(rootPath, "commit from ci4s with " + loginUser.getUsername());
dvcUtils.gitPush(rootPath, gitLinkUsername, gitLinkPassword);
dvcUtils.dvcPush(rootPath);
} catch (Exception e) {
@@ -686,9 +686,9 @@ public class ModelsServiceImpl implements ModelsService {
String ci4sUsername = loginUser.getUsername();
String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername();
String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword();

Map<String, Object> userInfo = getUserInfo(ci4sUsername, gitLinkUsername, gitLinkPassword);

ci4sUsername = modelsVo.getIsPublic() ? Constant.Item_Public : loginUser.getUsername();
String repositoryName = modelsVo.getIdentifier() == null ? ci4sUsername + "_model_" + DateUtils.dateTimeNow() : modelsVo.getIdentifier();
ModelDependency1 modelDependency = new ModelDependency1();
List<ModelDependency1> oldModelDependencys = modelDependency1Dao.queryModelDependency(modelsVo.getName(), modelsVo.getId(), gitLinkUsername);
@@ -879,10 +879,9 @@ public class ModelsServiceImpl implements ModelsService {
}

@Override
public ResponseEntity<InputStreamResource> downloadAllModelFilesNew(String name, String identifier, Integer id, String version) throws Exception {
public ResponseEntity<InputStreamResource> downloadAllModelFilesNew(String name, String identifier, Integer id, String version, Boolean isPublic) throws Exception {
// 命令行操作 git clone 项目地址
LoginUser loginUser = SecurityUtils.getLoginUser();
String ci4sUsername = loginUser.getUsername();
String ci4sUsername = isPublic ? Constant.Item_Public : SecurityUtils.getLoginUser().getUsername();

String localPath1 = localPath + ci4sUsername + "/model/" + id + "/" + identifier + "/" + version;
// 打包 data 文件夹
@@ -1090,9 +1089,8 @@ public class ModelsServiceImpl implements ModelsService {
}

@Override
public ModelsVo getModelDetail(Integer id, String identifier, String owner, String version) throws Exception {
LoginUser loginUser = SecurityUtils.getLoginUser();
String ci4sUsername = loginUser.getUsername();
public ModelsVo getModelDetail(Integer id, String identifier, String owner, String version, Boolean isPublic) throws Exception {
String ci4sUsername = isPublic ? Constant.Item_Public : SecurityUtils.getLoginUser().getUsername();

if (StringUtils.isEmpty(version)) {
List<Map<String, Object>> versionList = this.getVersionList(identifier, owner);
@@ -1126,7 +1124,7 @@ public class ModelsServiceImpl implements ModelsService {
}

@Override
public ModelDependency1TreeVo getModelDependencyTree(Integer repoId, String identifier, String version) throws Exception {
public ModelDependency1TreeVo getModelDependencyTree(Integer repoId, String identifier, String version) {
ModelDependency1 modelDependency1 = modelDependency1Dao.queryByRepoAndVersion(repoId, identifier, version);
ModelDependency1TreeVo modelDependency1TreeVo = new ModelDependency1TreeVo();
BeanUtils.copyProperties(modelDependency1, modelDependency1TreeVo);
@@ -1143,7 +1141,7 @@ public class ModelsServiceImpl implements ModelsService {
}

@Override
public void deleteModel(Integer repoId, String identifier, String owner) throws Exception {
public void deleteModel(Integer repoId, String identifier, String owner, Boolean isPublic) throws Exception {
String token = gitService.checkoutToken();
gitService.deleteProject(token, owner, identifier);
//删除模型依赖
@@ -1155,8 +1153,7 @@ public class ModelsServiceImpl implements ModelsService {
String parentModel = JSON.toJSONString(map);
modelDependency1Dao.deleteModelDependency(parentModel);

LoginUser loginUser = SecurityUtils.getLoginUser();
String ci4sUsername = loginUser.getUsername();
String ci4sUsername = isPublic ? Constant.Item_Public : SecurityUtils.getLoginUser().getUsername();
dvcUtils.deleteDirectory(localPath + "/" + ci4sUsername + "/model/" + repoId + "/" + identifier);
}



+ 9
- 7
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java View File

@@ -90,6 +90,7 @@ public class NewDatasetServiceImpl implements NewDatasetService {
Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
Integer userId = (Integer) userInfo.get("user_id");
// 拼接project
ci4sUsername = datasetVo.getIsPublic() ? Constant.Item_Public : loginUser.getUsername();
String repositoryName = ci4sUsername + "_dataset_" + DateUtils.dateTimeNow();
GitProjectVo gitProjectVo = new GitProjectVo();
gitProjectVo.setRepositoryName(repositoryName);
@@ -181,6 +182,7 @@ public class NewDatasetServiceImpl implements NewDatasetService {
String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername();
String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword();
String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
ci4sUsername = datasetVo.getIsPublic() ? Constant.Item_Public : loginUser.getUsername();
Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
// 创建分支
String branchName = StringUtils.isEmpty(datasetVo.getVersion()) ? "master" : datasetVo.getVersion();
@@ -342,9 +344,9 @@ public class NewDatasetServiceImpl implements NewDatasetService {
}

@Override
public NewDatasetVo getNewDatasetDesc(Integer id, String name, String repo, String owner, String version) throws Exception {
public NewDatasetVo getNewDatasetDesc(Integer id, String name, String repo, String owner, String version, Boolean isPublic) throws Exception {
LoginUser loginUser = SecurityUtils.getLoginUser();
String ci4sUsername = loginUser.getUsername();
String ci4sUsername = isPublic ? Constant.Item_Public : loginUser.getUsername();
// cd到 localPathlocal/id/下面还有一个文件夹,然后做git pull操作,然后读取里面的文件列表,列出每个文件的大小和名称,封装成MAP
if (StringUtils.isEmpty(version)) {
List<Map<String, Object>> versionList = this.getVersionList(repo, owner);
@@ -384,12 +386,12 @@ public class NewDatasetServiceImpl implements NewDatasetService {
}

@Override
public void deleteDatasetNew(Integer repoId, String repo, String owner) throws Exception {
public void deleteDatasetNew(Integer repoId, String repo, String owner, Boolean isPublic) throws Exception {
String token = gitService.checkoutToken();
gitService.deleteProject(token, owner, repo);

LoginUser loginUser = SecurityUtils.getLoginUser();
String ci4sUsername = loginUser.getUsername();
String ci4sUsername = isPublic ? Constant.Item_Public : loginUser.getUsername();
dvcUtils.deleteDirectory(localPathlocal + "/" + ci4sUsername + "/datasets/" + repoId + "/" + repo);
}

@@ -452,10 +454,10 @@ public class NewDatasetServiceImpl implements NewDatasetService {
}

@Override
public ResponseEntity<InputStreamResource> downloadAllDatasetFilesNew(String name, String identifier, Integer id, String version) throws Exception {
public ResponseEntity<InputStreamResource> downloadAllDatasetFilesNew(String name, String identifier, Integer id, String version, Boolean isPublic) throws Exception {
// 命令行操作 git clone 项目地址
LoginUser loginUser = SecurityUtils.getLoginUser();
String localPath = localPathlocal + loginUser.getUsername() + "/datasets/" + id + "/" + identifier + "/" + version;
String ci4sUsername = isPublic ? Constant.Item_Public : SecurityUtils.getLoginUser().getUsername();
String localPath = localPathlocal + ci4sUsername + "/datasets/" + id + "/" + identifier + "/" + version;

// 打包 data 文件夹
String dataFolderPath = localPath + "/dataset";


Loading…
Cancel
Save