| @@ -22,6 +22,8 @@ public class Constant { | |||
| public final static int Git_Category_Id = 39; | |||
| public final static String Item_Public = "public"; | |||
| public final static String Source_Auto_Export = "auto_export"; | |||
| public final static String Source_Hand_Export = "hand_export"; | |||
| public final static String Source_Add = "add"; | |||
| @@ -85,14 +85,15 @@ public class NewDatasetFromGitController { | |||
| @RequestParam("id") Integer id, | |||
| @RequestParam(value = "owner", required = false) String owner, | |||
| @RequestParam(value = "identifier", required = false) String identifier, | |||
| @RequestParam(value = "version", required = false) String version) throws Exception { | |||
| return AjaxResult.success(this.newDatasetService.getNewDatasetDesc(id, name, identifier, owner, version)); | |||
| @RequestParam(value = "version", required = false) String version, | |||
| @RequestParam(value = "is_public") Boolean isPublic) throws Exception { | |||
| return AjaxResult.success(this.newDatasetService.getNewDatasetDesc(id, name, identifier, owner, version, isPublic)); | |||
| } | |||
| @DeleteMapping("/deleteDataset") | |||
| @ApiOperation(value = "删除数据集") | |||
| public AjaxResult deleteDataset(@RequestParam("id") Integer id, @RequestParam("identifier") String identifier, @RequestParam("owner") String owner) throws Exception { | |||
| this.newDatasetService.deleteDatasetNew(id, identifier, owner); | |||
| public AjaxResult deleteDataset(@RequestParam("id") Integer id, @RequestParam("identifier") String identifier, @RequestParam("owner") String owner, @RequestParam("is_public") Boolean isPublic) throws Exception { | |||
| this.newDatasetService.deleteDatasetNew(id, identifier, owner, isPublic); | |||
| return AjaxResult.success(); | |||
| } | |||
| @@ -132,8 +133,8 @@ public class NewDatasetFromGitController { | |||
| */ | |||
| @GetMapping("/downloadAllFiles") | |||
| @ApiOperation(value = "下载同一版本下所有数据集,并打包") | |||
| public ResponseEntity<InputStreamResource> downloadAllDatasetFiles(@RequestParam("name") String name, @RequestParam("identifier") String identifier, @RequestParam("id") Integer id, @RequestParam("version") String version) throws Exception { | |||
| return newDatasetService.downloadAllDatasetFilesNew(name, identifier, id, version); | |||
| public ResponseEntity<InputStreamResource> downloadAllDatasetFiles(@RequestParam("name") String name, @RequestParam("identifier") String identifier, @RequestParam("id") Integer id, @RequestParam("version") String version, @RequestParam("is_public") Boolean isPublic) throws Exception { | |||
| return newDatasetService.downloadAllDatasetFilesNew(name, identifier, id, version, isPublic); | |||
| } | |||
| /** | |||
| @@ -1,7 +1,6 @@ | |||
| package com.ruoyi.platform.controller.model; | |||
| import com.ruoyi.common.core.web.domain.AjaxResult; | |||
| import com.ruoyi.platform.domain.ModelDependency1; | |||
| import com.ruoyi.platform.service.ModelsService; | |||
| import com.ruoyi.platform.vo.ModelsVo; | |||
| import com.ruoyi.platform.vo.QueryModelMetricsVo; | |||
| @@ -46,8 +45,8 @@ public class NewModelFromGitController { | |||
| @GetMapping("/downloadAllFiles") | |||
| @ApiOperation(value = "下载同一版本下所有模型,并打包") | |||
| public ResponseEntity<InputStreamResource> downloadAllDatasetFiles(@RequestParam("name") String name, @RequestParam("identifier") String identifier, @RequestParam("id") Integer id, @RequestParam("version") String version) throws Exception { | |||
| return modelsService.downloadAllModelFilesNew(name, identifier, id, version); | |||
| public ResponseEntity<InputStreamResource> downloadAllDatasetFiles(@RequestParam("name") String name, @RequestParam("identifier") String identifier, @RequestParam("id") Integer id, @RequestParam("version") String version, @RequestParam("is_public") Boolean isPublic) throws Exception { | |||
| return modelsService.downloadAllModelFilesNew(name, identifier, id, version, isPublic); | |||
| } | |||
| @GetMapping("/downloadSingleFile") | |||
| @@ -103,8 +102,8 @@ public class NewModelFromGitController { | |||
| @GetMapping("/getModelDetail") | |||
| @ApiOperation(value = "获取模型详细信息") | |||
| public AjaxResult getModelDetail(@RequestParam("id") Integer id, @RequestParam("identifier") String identifier, @RequestParam("owner") String owner, @RequestParam("version") String version) throws Exception { | |||
| return AjaxResult.success(this.modelsService.getModelDetail(id, identifier, owner, version)); | |||
| public AjaxResult getModelDetail(@RequestParam("id") Integer id, @RequestParam("identifier") String identifier, @RequestParam("owner") String owner, @RequestParam("version") String version, @RequestParam("is_public") Boolean isPublic) throws Exception { | |||
| return AjaxResult.success(this.modelsService.getModelDetail(id, identifier, owner, version, isPublic)); | |||
| } | |||
| @GetMapping("/getModelDependencyTree") | |||
| @@ -116,8 +115,8 @@ public class NewModelFromGitController { | |||
| @DeleteMapping("/delete") | |||
| @ApiOperation(value = "删除模型") | |||
| public AjaxResult deleteModel(@RequestParam("id") Integer id, @RequestParam("identifier") String identifier, | |||
| @RequestParam("owner") String owner) throws Exception { | |||
| this.modelsService.deleteModel(id, identifier, owner); | |||
| @RequestParam("owner") String owner, @RequestParam("is_public") Boolean isPublic) throws Exception { | |||
| this.modelsService.deleteModel(id, identifier, owner, isPublic); | |||
| return AjaxResult.success(); | |||
| } | |||
| @@ -95,7 +95,7 @@ public interface ModelsService { | |||
| List<Map<String, String>> uploadModelLocal(MultipartFile[] files, String uuid) throws Exception; | |||
| ResponseEntity<InputStreamResource> downloadAllModelFilesNew(String name, String identifier, Integer id, String version) throws IOException, Exception; | |||
| ResponseEntity<InputStreamResource> downloadAllModelFilesNew(String name, String identifier, Integer id, String version, Boolean isPublic) throws IOException, Exception; | |||
| Page<ModelsVo> newPubilcQueryByPage(ModelsVo modelsVo, PageRequest pageRequest) throws Exception; | |||
| @@ -107,11 +107,11 @@ public interface ModelsService { | |||
| List<Map<String, Object>> getVersionList(String identifier, String owner) throws Exception; | |||
| ModelsVo getModelDetail(Integer id, String identifier, String owner, String version) throws Exception; | |||
| ModelsVo getModelDetail(Integer id, String identifier, String owner, String version, Boolean isPublic) throws Exception; | |||
| ModelDependency1TreeVo getModelDependencyTree(Integer repoId, String identifier, String version) throws Exception; | |||
| void deleteModel(Integer repoId, String identifier, String owner) throws Exception; | |||
| void deleteModel(Integer repoId, String identifier, String owner, Boolean isPublic) throws Exception; | |||
| void deleteVersion(Integer repoId, String identifier, String owner, String version, String relativePath) throws Exception; | |||
| } | |||
| @@ -16,18 +16,24 @@ public interface NewDatasetService { | |||
| String newCreateDataset(NewDatasetVo datasetVo) throws Exception; | |||
| String newCreateVersion(NewDatasetVo datasetVo) throws Exception; | |||
| List<Map<String, String>> uploadDatasetlocal(MultipartFile[] files, String uuid) throws Exception; | |||
| ResponseEntity<InputStreamResource> downloadDatasetlocal(String filePath) throws Exception; | |||
| ResponseEntity<InputStreamResource> downloadAllDatasetFilesNew(String name, String identifier, Integer id, String version) throws IOException, Exception; | |||
| ResponseEntity<InputStreamResource> downloadAllDatasetFilesNew(String name, String identifier, Integer id, String version, Boolean isPublic) throws IOException, Exception; | |||
| Page<NewDatasetVo> newPersonalQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception; | |||
| Page<NewDatasetVo> newPubilcQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception; | |||
| NewDatasetVo getNewDatasetDesc(Integer repoId,String name,String repo, String owner, String version)throws Exception; | |||
| Page<NewDatasetVo> newPubilcQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception; | |||
| NewDatasetVo getNewDatasetDesc(Integer repoId, String name, String repo, String owner, String version, Boolean isPublic) throws Exception; | |||
| List<Map<String, Object>> getVersionList(String repo, String owner) throws Exception; | |||
| List<Map<String, Object>> getVersionList(String repo, String owner) throws Exception; | |||
| void deleteDatasetNew(Integer repoId, String repo, String owner) throws Exception; | |||
| void deleteDatasetNew(Integer repoId, String repo, String owner, Boolean isPublic) throws Exception; | |||
| void deleteDatasetVersionNew(String repo, String owner, String version, String relativePath) throws Exception; | |||
| } | |||
| @@ -561,10 +561,10 @@ public class ModelsServiceImpl implements ModelsService { | |||
| String ci4sUsername = loginUser.getUsername(); | |||
| String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername(); | |||
| String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword(); | |||
| Map<String, Object> userInfo = getUserInfo(ci4sUsername, gitLinkUsername, gitLinkPassword); | |||
| Integer userId = (Integer) userInfo.get("user_id"); | |||
| ci4sUsername = modelsVo.getIsPublic() ? Constant.Item_Public : ci4sUsername; | |||
| String repositoryName = modelsVo.getIdentifier() == null ? ci4sUsername + "_model_" + DateUtils.dateTimeNow() : modelsVo.getIdentifier(); | |||
| ModelDependency1 modelDependency = new ModelDependency1(); | |||
| List<ModelDependency1> oldModelDependencys = modelDependency1Dao.queryModelDependency(modelsVo.getName(), null, gitLinkUsername); | |||
| @@ -650,11 +650,11 @@ public class ModelsServiceImpl implements ModelsService { | |||
| modelDependency.setState(Constant.State_valid); | |||
| modelDependency1Dao.insert(modelDependency); | |||
| String s3Path = bucketName + "/mini-model-management-platform-files/" + ci4sUsername + "/model/" + gitlinIid + "/" + repositoryName + "/" + branchName; | |||
| CompletableFuture.supplyAsync(() -> { | |||
| try { | |||
| dvcUtils.dvcInit(rootPath); | |||
| // 配置远程S3地址 | |||
| String s3Path = bucketName + "/mini-model-management-platform-files/" + ci4sUsername + "/model/" + gitlinIid + "/" + repositoryName + "/" + branchName; | |||
| dvcUtils.dvcRemoteAdd(rootPath, s3Path); | |||
| dvcUtils.dvcConfigS3Credentials(rootPath, endpoint); | |||
| dvcUtils.dvcConfigS3Credentials2(rootPath, accessKeyId); | |||
| @@ -663,7 +663,7 @@ public class ModelsServiceImpl implements ModelsService { | |||
| // dvc 跟踪 | |||
| dvcUtils.dvcAdd(rootPath, "model"); | |||
| // git commit | |||
| dvcUtils.gitCommit(rootPath, "commit from ci4s with " + ci4sUsername); | |||
| dvcUtils.gitCommit(rootPath, "commit from ci4s with " + loginUser.getUsername()); | |||
| dvcUtils.gitPush(rootPath, gitLinkUsername, gitLinkPassword); | |||
| dvcUtils.dvcPush(rootPath); | |||
| } catch (Exception e) { | |||
| @@ -686,9 +686,9 @@ public class ModelsServiceImpl implements ModelsService { | |||
| String ci4sUsername = loginUser.getUsername(); | |||
| String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername(); | |||
| String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword(); | |||
| Map<String, Object> userInfo = getUserInfo(ci4sUsername, gitLinkUsername, gitLinkPassword); | |||
| ci4sUsername = modelsVo.getIsPublic() ? Constant.Item_Public : loginUser.getUsername(); | |||
| String repositoryName = modelsVo.getIdentifier() == null ? ci4sUsername + "_model_" + DateUtils.dateTimeNow() : modelsVo.getIdentifier(); | |||
| ModelDependency1 modelDependency = new ModelDependency1(); | |||
| List<ModelDependency1> oldModelDependencys = modelDependency1Dao.queryModelDependency(modelsVo.getName(), modelsVo.getId(), gitLinkUsername); | |||
| @@ -879,10 +879,9 @@ public class ModelsServiceImpl implements ModelsService { | |||
| } | |||
| @Override | |||
| public ResponseEntity<InputStreamResource> downloadAllModelFilesNew(String name, String identifier, Integer id, String version) throws Exception { | |||
| public ResponseEntity<InputStreamResource> downloadAllModelFilesNew(String name, String identifier, Integer id, String version, Boolean isPublic) throws Exception { | |||
| // 命令行操作 git clone 项目地址 | |||
| LoginUser loginUser = SecurityUtils.getLoginUser(); | |||
| String ci4sUsername = loginUser.getUsername(); | |||
| String ci4sUsername = isPublic ? Constant.Item_Public : SecurityUtils.getLoginUser().getUsername(); | |||
| String localPath1 = localPath + ci4sUsername + "/model/" + id + "/" + identifier + "/" + version; | |||
| // 打包 data 文件夹 | |||
| @@ -1090,9 +1089,8 @@ public class ModelsServiceImpl implements ModelsService { | |||
| } | |||
| @Override | |||
| public ModelsVo getModelDetail(Integer id, String identifier, String owner, String version) throws Exception { | |||
| LoginUser loginUser = SecurityUtils.getLoginUser(); | |||
| String ci4sUsername = loginUser.getUsername(); | |||
| public ModelsVo getModelDetail(Integer id, String identifier, String owner, String version, Boolean isPublic) throws Exception { | |||
| String ci4sUsername = isPublic ? Constant.Item_Public : SecurityUtils.getLoginUser().getUsername(); | |||
| if (StringUtils.isEmpty(version)) { | |||
| List<Map<String, Object>> versionList = this.getVersionList(identifier, owner); | |||
| @@ -1126,7 +1124,7 @@ public class ModelsServiceImpl implements ModelsService { | |||
| } | |||
| @Override | |||
| public ModelDependency1TreeVo getModelDependencyTree(Integer repoId, String identifier, String version) throws Exception { | |||
| public ModelDependency1TreeVo getModelDependencyTree(Integer repoId, String identifier, String version) { | |||
| ModelDependency1 modelDependency1 = modelDependency1Dao.queryByRepoAndVersion(repoId, identifier, version); | |||
| ModelDependency1TreeVo modelDependency1TreeVo = new ModelDependency1TreeVo(); | |||
| BeanUtils.copyProperties(modelDependency1, modelDependency1TreeVo); | |||
| @@ -1143,7 +1141,7 @@ public class ModelsServiceImpl implements ModelsService { | |||
| } | |||
| @Override | |||
| public void deleteModel(Integer repoId, String identifier, String owner) throws Exception { | |||
| public void deleteModel(Integer repoId, String identifier, String owner, Boolean isPublic) throws Exception { | |||
| String token = gitService.checkoutToken(); | |||
| gitService.deleteProject(token, owner, identifier); | |||
| //删除模型依赖 | |||
| @@ -1155,8 +1153,7 @@ public class ModelsServiceImpl implements ModelsService { | |||
| String parentModel = JSON.toJSONString(map); | |||
| modelDependency1Dao.deleteModelDependency(parentModel); | |||
| LoginUser loginUser = SecurityUtils.getLoginUser(); | |||
| String ci4sUsername = loginUser.getUsername(); | |||
| String ci4sUsername = isPublic ? Constant.Item_Public : SecurityUtils.getLoginUser().getUsername(); | |||
| dvcUtils.deleteDirectory(localPath + "/" + ci4sUsername + "/model/" + repoId + "/" + identifier); | |||
| } | |||
| @@ -90,6 +90,7 @@ public class NewDatasetServiceImpl implements NewDatasetService { | |||
| Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq); | |||
| Integer userId = (Integer) userInfo.get("user_id"); | |||
| // 拼接project | |||
| ci4sUsername = datasetVo.getIsPublic() ? Constant.Item_Public : loginUser.getUsername(); | |||
| String repositoryName = ci4sUsername + "_dataset_" + DateUtils.dateTimeNow(); | |||
| GitProjectVo gitProjectVo = new GitProjectVo(); | |||
| gitProjectVo.setRepositoryName(repositoryName); | |||
| @@ -181,6 +182,7 @@ public class NewDatasetServiceImpl implements NewDatasetService { | |||
| String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername(); | |||
| String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword(); | |||
| String userReq = jedis.get(ci4sUsername + "_gitUserInfo"); | |||
| ci4sUsername = datasetVo.getIsPublic() ? Constant.Item_Public : loginUser.getUsername(); | |||
| Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq); | |||
| // 创建分支 | |||
| String branchName = StringUtils.isEmpty(datasetVo.getVersion()) ? "master" : datasetVo.getVersion(); | |||
| @@ -342,9 +344,9 @@ public class NewDatasetServiceImpl implements NewDatasetService { | |||
| } | |||
| @Override | |||
| public NewDatasetVo getNewDatasetDesc(Integer id, String name, String repo, String owner, String version) throws Exception { | |||
| public NewDatasetVo getNewDatasetDesc(Integer id, String name, String repo, String owner, String version, Boolean isPublic) throws Exception { | |||
| LoginUser loginUser = SecurityUtils.getLoginUser(); | |||
| String ci4sUsername = loginUser.getUsername(); | |||
| String ci4sUsername = isPublic ? Constant.Item_Public : loginUser.getUsername(); | |||
| // cd到 localPathlocal/id/下面还有一个文件夹,然后做git pull操作,然后读取里面的文件列表,列出每个文件的大小和名称,封装成MAP | |||
| if (StringUtils.isEmpty(version)) { | |||
| List<Map<String, Object>> versionList = this.getVersionList(repo, owner); | |||
| @@ -384,12 +386,12 @@ public class NewDatasetServiceImpl implements NewDatasetService { | |||
| } | |||
| @Override | |||
| public void deleteDatasetNew(Integer repoId, String repo, String owner) throws Exception { | |||
| public void deleteDatasetNew(Integer repoId, String repo, String owner, Boolean isPublic) throws Exception { | |||
| String token = gitService.checkoutToken(); | |||
| gitService.deleteProject(token, owner, repo); | |||
| LoginUser loginUser = SecurityUtils.getLoginUser(); | |||
| String ci4sUsername = loginUser.getUsername(); | |||
| String ci4sUsername = isPublic ? Constant.Item_Public : loginUser.getUsername(); | |||
| dvcUtils.deleteDirectory(localPathlocal + "/" + ci4sUsername + "/datasets/" + repoId + "/" + repo); | |||
| } | |||
| @@ -452,10 +454,10 @@ public class NewDatasetServiceImpl implements NewDatasetService { | |||
| } | |||
| @Override | |||
| public ResponseEntity<InputStreamResource> downloadAllDatasetFilesNew(String name, String identifier, Integer id, String version) throws Exception { | |||
| public ResponseEntity<InputStreamResource> downloadAllDatasetFilesNew(String name, String identifier, Integer id, String version, Boolean isPublic) throws Exception { | |||
| // 命令行操作 git clone 项目地址 | |||
| LoginUser loginUser = SecurityUtils.getLoginUser(); | |||
| String localPath = localPathlocal + loginUser.getUsername() + "/datasets/" + id + "/" + identifier + "/" + version; | |||
| String ci4sUsername = isPublic ? Constant.Item_Public : SecurityUtils.getLoginUser().getUsername(); | |||
| String localPath = localPathlocal + ci4sUsername + "/datasets/" + id + "/" + identifier + "/" + version; | |||
| // 打包 data 文件夹 | |||
| String dataFolderPath = localPath + "/dataset"; | |||