From 156888782eabb36f4d2c78f225db16771f594a77 Mon Sep 17 00:00:00 2001 From: fanshuai <1141904845@qq.com> Date: Fri, 6 Sep 2024 16:20:01 +0800 Subject: [PATCH] update dataset version --- .../dataset/NewDatasetFromGitController.java | 32 +- .../controller/jupyter/JupyterController.java | 10 +- .../platform/service/DatasetService.java | 36 +- .../platform/service/NewDatasetService.java | 35 ++ .../service/impl/DatasetServiceImpl.java | 478 +---------------- .../platform/service/impl/GitServiceImpl.java | 3 + .../service/impl/NewDatasetServiceImpl.java | 501 ++++++++++++++++++ .../com/ruoyi/platform/utils/DVCUtils.java | 10 +- .../com/ruoyi/platform/vo/NewDatasetVo.java | 6 +- 9 files changed, 579 insertions(+), 532 deletions(-) create mode 100644 ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/NewDatasetService.java create mode 100644 ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/dataset/NewDatasetFromGitController.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/dataset/NewDatasetFromGitController.java index 6b738bb5..42cb47da 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/dataset/NewDatasetFromGitController.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/dataset/NewDatasetFromGitController.java @@ -2,10 +2,8 @@ package com.ruoyi.platform.controller.dataset; import com.ruoyi.common.core.web.domain.AjaxResult; import com.ruoyi.platform.domain.Dataset; -import com.ruoyi.platform.service.DatasetService; -import com.ruoyi.platform.vo.DatasetVo; +import com.ruoyi.platform.service.NewDatasetService; import com.ruoyi.platform.vo.NewDatasetVo; -import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import org.springframework.core.io.InputStreamResource; import org.springframework.data.domain.PageRequest; @@ -22,7 +20,7 @@ public class NewDatasetFromGitController { * 服务对象 */ @Resource - private DatasetService datasetService; + private NewDatasetService newDatasetService; @@ -35,7 +33,7 @@ public class NewDatasetFromGitController { @PostMapping("/addDatasetAndVersion") @ApiOperation("添加数据集和版本") public AjaxResult addDatasetAndVersion(@RequestBody NewDatasetVo datasetVo) throws Exception { - return AjaxResult.success(this.datasetService.newCreateDataset(datasetVo)); + return AjaxResult.success(this.newDatasetService.newCreateDataset(datasetVo)); } @@ -49,7 +47,7 @@ public class NewDatasetFromGitController { @PostMapping("/addVersion") @ApiOperation("添加版本") public AjaxResult addVersion(@RequestBody NewDatasetVo datasetVo) throws Exception { - return AjaxResult.success(this.datasetService.newCreateVersion(datasetVo)); + return AjaxResult.success(this.newDatasetService.newCreateVersion(datasetVo)); } @@ -62,39 +60,39 @@ public class NewDatasetFromGitController { @RequestParam(value = "data_tag", required = false) String dataTag) throws Exception { PageRequest pageRequest = PageRequest.of(page, size); if(isPublic){ - return AjaxResult.success(this.datasetService.newPubilcQueryByPage(dataset, pageRequest)); + return AjaxResult.success(this.newDatasetService.newPubilcQueryByPage(dataset, pageRequest)); }else { - return AjaxResult.success(this.datasetService.newPersonalQueryByPage(dataset, pageRequest)); + return AjaxResult.success(this.newDatasetService.newPersonalQueryByPage(dataset, pageRequest)); } } @GetMapping("/getVersionList") @ApiOperation(value = "获取分支列表") public AjaxResult getVersionList(@RequestParam("identifier") String repo,@RequestParam("owner")String owner) throws Exception { - return AjaxResult.success(this.datasetService.getVersionList(repo,owner)); + return AjaxResult.success(this.newDatasetService.getVersionList(repo,owner)); } @GetMapping("/getDatasetDetail") @ApiOperation(value = "获取数据集详情") public AjaxResult getDatasetVersions(@RequestParam("name") String name, - @RequestParam("repo_id") Integer repoId, + @RequestParam("id") Integer id, @RequestParam(value ="owner",required = false)String owner, - @RequestParam(value = "identifier",required = false) String repo, + @RequestParam(value = "identifier",required = false) String identifier, @RequestParam(value = "version",required = false)String version) throws Exception { - return AjaxResult.success(this.datasetService.getNewDatasetDesc(repoId,name,repo,owner,version)); + return AjaxResult.success(this.newDatasetService.getNewDatasetDesc(id,name,identifier,owner,version)); } @DeleteMapping("/deleteDataset") @ApiOperation(value = "删除数据集") public AjaxResult deleteDataset(@RequestParam("identifier") String repo,@RequestParam("owner")String owner) throws Exception { - this.datasetService.deleteDatasetNew(repo,owner); + this.newDatasetService.deleteDatasetNew(repo,owner); return AjaxResult.success(); } @DeleteMapping("/deleteDatasetVersion") @ApiOperation(value = "删除数据集版本") public AjaxResult deleteDatasetVersion(@RequestParam("identifier") String repo,@RequestParam("owner")String owner,@RequestParam("version")String version) throws Exception { - this.datasetService.deleteDatasetVersionNew(repo,owner,version); + this.newDatasetService.deleteDatasetVersionNew(repo,owner,version); return AjaxResult.success(); } @@ -110,7 +108,7 @@ public class NewDatasetFromGitController { @PostMapping("/upload") @ApiOperation(value = "上传数据集") public AjaxResult uploadDataset(@RequestParam("file") MultipartFile[] files, @RequestParam("uuid") String uuid) throws Exception { - return AjaxResult.success(this.datasetService.uploadDatasetlocal(files,uuid)); + return AjaxResult.success(this.newDatasetService.uploadDatasetlocal(files,uuid)); } /** @@ -122,7 +120,7 @@ public class NewDatasetFromGitController { @GetMapping("/downloadAllFiles") @ApiOperation(value = "下载同一版本下所有数据集，并打包") public ResponseEntity downloadAllDatasetFiles(@RequestParam("repository_name") String repositoryName, @RequestParam("version") String version) throws Exception { - return datasetService.downloadAllDatasetFilesNew(repositoryName, version); + return newDatasetService.downloadAllDatasetFilesNew(repositoryName, version); } /** @@ -135,6 +133,6 @@ public class NewDatasetFromGitController { @GetMapping("/downloadSinggerFile") @ApiOperation(value = "下载单个数据集文件", notes = "根据数据集版本表id下载单个数据集文件") public ResponseEntity downloadDataset(@RequestParam("url") String url) throws Exception { - return datasetService.downloadDatasetlocal(url); + return newDatasetService.downloadDatasetlocal(url); } } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/jupyter/JupyterController.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/jupyter/JupyterController.java index 2a52ff87..52cad80b 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/jupyter/JupyterController.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/jupyter/JupyterController.java @@ -4,9 +4,11 @@ import com.ruoyi.common.core.web.controller.BaseController; import com.ruoyi.common.core.web.domain.AjaxResult; import com.ruoyi.common.core.web.domain.GenericsAjaxResult; import com.ruoyi.platform.domain.DevEnvironment; -import com.ruoyi.platform.service.DatasetService; import com.ruoyi.platform.service.JupyterService; -import com.ruoyi.platform.vo.*; +import com.ruoyi.platform.service.NewDatasetService; +import com.ruoyi.platform.vo.NewDatasetVo; +import com.ruoyi.platform.vo.PodStatusVo; +import com.ruoyi.platform.vo.VersionVo; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import io.swagger.v3.oas.annotations.responses.ApiResponse; @@ -27,7 +29,7 @@ public class JupyterController extends BaseController { @Resource private JupyterService jupyterService; @Resource - private DatasetService datasetService; + private NewDatasetService newDatasetService; @GetMapping(value = "/getURL") @ApiOperation("得到访问地址") public GenericsAjaxResult getURL() throws IOException { @@ -101,6 +103,6 @@ public class JupyterController extends BaseController { datasetVo.setVersionDesc("this is a test"); datasetVersionVos.add(versionVo); datasetVo.setDatasetVersionVos(datasetVersionVos); - return AjaxResult.success(datasetService.newCreateDataset(datasetVo)); + return AjaxResult.success(newDatasetService.newCreateDataset(datasetVo)); } } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/DatasetService.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/DatasetService.java index 64a1262c..4c66fd6c 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/DatasetService.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/DatasetService.java @@ -21,11 +21,7 @@ import java.util.concurrent.CompletableFuture; * @author Xidaray * @since 2023-11-28 11:51:22 */ -public interface -DatasetService { - - - +public interface DatasetService { /** @@ -39,12 +35,12 @@ DatasetService { /** * 分页查询 * - * @param dataset 筛选条件 - * @param pageRequest 分页对象 + * @param dataset 筛选条件 + * @param pageRequest 分页对象 * @return 查询结果 */ Page queryByPage(Dataset dataset, PageRequest pageRequest); - + /** * 新增数据 * @@ -54,7 +50,6 @@ DatasetService { Dataset insert(Dataset dataset) throws Exception; - /** * 修改数据 * @@ -82,30 +77,11 @@ DatasetService { List getDatasetVersions(Integer datasetId) throws Exception; String insertDatasetAndVersion(DatasetVo datasetVo) throws Exception; + void checkDeclaredName(Dataset insert) throws Exception; ResponseEntity downloadAllDatasetFiles(Integer datasetId, String version) throws Exception; List> exportDataset(String path, String uuid) throws Exception; - - - - - CompletableFuture newCreateDataset(NewDatasetVo datasetVo) throws Exception; - CompletableFuture newCreateVersion(NewDatasetVo datasetVo); - - List> uploadDatasetlocal(MultipartFile[] files, String uuid) throws Exception; - ResponseEntity downloadDatasetlocal(String filePath) throws Exception; - ResponseEntity downloadAllDatasetFilesNew(String repositoryName, String version) throws IOException, Exception; - Page newPersonalQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception; - Page newPubilcQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception; - - NewDatasetVo getNewDatasetDesc(Integer repoId,String RepositoryName,String repo, String owner, String version)throws Exception; - - List> getVersionList(String repo, String owner) throws Exception; - - void deleteDatasetNew(String repo, String owner) throws Exception; - - void deleteDatasetVersionNew(String repo, String owner, String version) throws Exception; -} +} \ No newline at end of file diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/NewDatasetService.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/NewDatasetService.java new file mode 100644 index 00000000..e41928d7 --- /dev/null +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/NewDatasetService.java @@ -0,0 +1,35 @@ +package com.ruoyi.platform.service; + +import com.ruoyi.platform.domain.Dataset; +import com.ruoyi.platform.vo.NewDatasetVo; +import org.springframework.core.io.InputStreamResource; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageRequest; +import org.springframework.http.ResponseEntity; +import org.springframework.web.multipart.MultipartFile; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; + +public interface NewDatasetService { + + + String newCreateDataset(NewDatasetVo datasetVo) throws Exception; + CompletableFuture newCreateVersion(NewDatasetVo datasetVo)throws Exception; + + List> uploadDatasetlocal(MultipartFile[] files, String uuid) throws Exception; + ResponseEntity downloadDatasetlocal(String filePath) throws Exception; + ResponseEntity downloadAllDatasetFilesNew(String repositoryName, String version) throws IOException, Exception; + Page newPersonalQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception; + Page newPubilcQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception; + + NewDatasetVo getNewDatasetDesc(Integer repoId,String RepositoryName,String repo, String owner, String version)throws Exception; + + List> getVersionList(String repo, String owner) throws Exception; + + void deleteDatasetNew(String repo, String owner) throws Exception; + + void deleteDatasetVersionNew(String repo, String owner, String version) throws Exception; +} diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/DatasetServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/DatasetServiceImpl.java index bf578a81..8eb3b9b8 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/DatasetServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/DatasetServiceImpl.java @@ -1,6 +1,5 @@ package com.ruoyi.platform.service.impl; -import com.ruoyi.common.core.utils.DateUtils; import com.ruoyi.common.security.utils.SecurityUtils; import com.ruoyi.platform.annotations.CheckDuplicate; import com.ruoyi.platform.domain.AssetIcon; @@ -9,14 +8,13 @@ import com.ruoyi.platform.domain.DatasetVersion; import com.ruoyi.platform.mapper.DatasetDao; import com.ruoyi.platform.mapper.DatasetVersionDao; import com.ruoyi.platform.service.*; -import com.ruoyi.platform.utils.*; +import com.ruoyi.platform.utils.BeansUtils; +import com.ruoyi.platform.utils.FileUtil; +import com.ruoyi.platform.utils.MinioUtil; import com.ruoyi.platform.vo.DatasetVo; -import com.ruoyi.platform.vo.GitProjectVo; -import com.ruoyi.platform.vo.NewDatasetVo; import com.ruoyi.platform.vo.VersionVo; import com.ruoyi.system.api.model.LoginUser; import io.minio.messages.Item; -import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.core.io.InputStreamResource; @@ -30,17 +28,13 @@ import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import org.springframework.web.multipart.MultipartFile; -import redis.clients.jedis.Jedis; import javax.annotation.Resource; -import java.io.*; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.lang.reflect.Field; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; import java.text.SimpleDateFormat; import java.util.*; -import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; @@ -68,12 +62,6 @@ public class DatasetServiceImpl implements DatasetService { @Resource private MinioService minioService; - @Resource - private GitService gitService; - - @Resource - private DvcService dvcService; - // 固定存储桶名 @Value("${minio.dataReleaseBucketName}") private String bucketName; @@ -81,19 +69,6 @@ public class DatasetServiceImpl implements DatasetService { @Resource private MinioUtil minioUtil; - @Value("${spring.redis.host}") - private String redisHost; - @Value("${minio.accessKey}") - String accessKeyId; - @Value("${minio.secretKey}") - String secretAccessKey; - @Value("${minio.endpoint}") - String endpoint; - @Value("${git.endpoint}") - String gitendpoint; - @Value("${git.localPath}") - String localPathlocal; - /** * 通过ID查询单条数据 * @@ -457,447 +432,4 @@ public class DatasetServiceImpl implements DatasetService { } return results; } - - - - @Override - public CompletableFuture newCreateDataset(NewDatasetVo datasetVo) { - return CompletableFuture.supplyAsync(() -> { - try { - checkoutToken(); - Jedis jedis = new Jedis(redisHost); - LoginUser loginUser = SecurityUtils.getLoginUser(); - String ci4sUsername = loginUser.getUsername(); - String token = jedis.get(ci4sUsername+"_gitToken"); - String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername(); - String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword(); - String userReq = jedis.get(ci4sUsername + "_gitUserInfo"); - Map userInfo = JsonUtils.jsonToMap(userReq); - Integer userId = (Integer) userInfo.get("user_id"); - // 拼接project - String repositoryName = ci4sUsername + "_dataset_" + DateUtils.dateTimeNow(); - GitProjectVo gitProjectVo = new GitProjectVo(); - gitProjectVo.setRepositoryName(repositoryName); - gitProjectVo.setName(datasetVo.getName()); - gitProjectVo.setDescription(datasetVo.getDescription()); - gitProjectVo.setPrivate(!datasetVo.getIsPublic()); - gitProjectVo.setUserId(userId); - // 创建项目 - Map project = gitService.createProject(token, gitProjectVo); - Integer repoId = (Integer) project.get("id"); - // 创建分支 - String branchName = datasetVo.getVersion(); - gitService.createBranch(token, (String) userInfo.get("login"), repositoryName, branchName, "master"); - // 定义标签 标签1：ci4s_dataset 标签2：DataTag 标签3：DataType - gitService.createTopic(token, repoId, "ci4s_dataset"); - gitService.createTopic(token, repoId, "DataTag_" + datasetVo.getDataTag()); - gitService.createTopic(token, repoId, "DataType_" + datasetVo.getDataType()); - // 得到项目地址 - String projectUrl = gitendpoint + "/" +(String) userInfo.get("login") + "/" + repositoryName + ".git"; - - // 得到用户操作的路径 - String url = datasetVo.getDatasetVersionVos().get(0).getUrl(); - String localPath = localPathlocal + "/" + repoId+"/"+datasetVo.getName(); - String sourcePath = url.substring(0, url.lastIndexOf("/")); - // 命令行操作 git clone 项目地址 - DVCUtils.gitClone(localPath, projectUrl, branchName, gitLinkUsername, gitLinkPassword); - String s3Path = "management-platform-files/" + ci4sUsername + "/datasets/"+repoId+"/"+ repositoryName + "/" + branchName; - - DVCUtils.moveFiles(sourcePath, localPath); - //拼接生产的元数据后写入yaml文件 - datasetVo.setCreateBy(String.valueOf(StringUtils.isNotEmpty((String)userInfo.get("nickname"))? userInfo.get("nickname") : userInfo.get("login"))); - datasetVo.setUpdateTime(DateUtils.getNowDate()); - datasetVo.setVersionDesc(datasetVo.getDescription()); - datasetVo.setUsage("```bash\n" + - "# 克隆数据集配置文件与存储参数到本地\n" + - "git clone -b " + branchName + " " + projectUrl + "\n" + - "# 远程拉取配置文件\n" + - "dvc pull\n" + - "```"); - YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo),localPath, "dataset"); - // dvc init 初始化 - DVCUtils.dvcInit(localPath); - // 配置远程S3地址 - DVCUtils.dvcRemoteAdd(localPath, s3Path); - DVCUtils.dvcConfigS3Credentials(localPath, endpoint); - DVCUtils.dvcConfigS3Credentials2(localPath, accessKeyId); - DVCUtils.dvcConfigS3Credentials3(localPath, secretAccessKey); - // dvc 跟踪 - DVCUtils.dvcAdd(localPath, "data"); - // git commit - DVCUtils.gitAdd(localPath, "."); - DVCUtils.gitCommit(localPath, "commit from ci4s with " + loginUser.getUsername()); - DVCUtils.gitPush(localPath, gitLinkUsername, gitLinkPassword); - // dvc push 到远程S3 - DVCUtils.dvcPush(localPath); - return "新增数据集成功"; - } catch (Exception e) { - throw new RuntimeException(e); - } - }); - } - - @Override - public CompletableFuture newCreateVersion(NewDatasetVo datasetVo) { - return CompletableFuture.supplyAsync(() -> { - try { - checkoutToken(); - Jedis jedis = new Jedis(redisHost); - LoginUser loginUser = SecurityUtils.getLoginUser(); - String ci4sUsername = loginUser.getUsername(); - String token = jedis.get(ci4sUsername+"_gitToken"); - String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername(); - String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword(); - String userReq = jedis.get(ci4sUsername + "_gitUserInfo"); - Map userInfo = JsonUtils.jsonToMap(userReq); - // 创建分支 - String branchName = StringUtils.isEmpty(datasetVo.getVersion())? "master" : datasetVo.getVersion(); - String repositoryName = datasetVo.getIdentifier(); - if (StringUtils.equals(branchName, "master")) { - gitService.createBranch(token, (String) userInfo.get("login"), repositoryName, branchName, "master"); - } - // 得到项目地址 - String projectUrl = gitendpoint + "/" +(String) userInfo.get("login") + "/" + repositoryName + ".git"; - - // 得到用户操作的路径 - String url = datasetVo.getDatasetVersionVos().get(0).getUrl(); - String localPath = localPathlocal+ loginUser.getUsername()+"/datasets/"+ datasetVo.getName(); - String sourcePath = url.substring(0, url.lastIndexOf("/")); - // 命令行操作 git clone 项目地址 - if(FileUtil.checkDirectoryExists(localPath)){ - DVCUtils.gitFetch(localPath,gitLinkUsername, gitLinkPassword); - DVCUtils.gitCheckoutBranch(localPath,branchName); - }else { - DVCUtils.gitClone(localPath, projectUrl, branchName, gitLinkUsername, gitLinkPassword); - } - - String s3Path = "management-platform-files/" + ci4sUsername + "/datasets/"+ datasetVo.getRepoId()+"/"+ repositoryName + "/" + branchName; - - DVCUtils.moveFiles(sourcePath, localPath); - //拼接生产的元数据后写入yaml文件 - datasetVo.setCreateBy(String.valueOf(StringUtils.isNotEmpty((String)userInfo.get("nickname"))? userInfo.get("nickname") : userInfo.get("login"))); - datasetVo.setUpdateTime(DateUtils.getNowDate()); - datasetVo.setVersionDesc(datasetVo.getDescription()); - datasetVo.setUsage("```bash\n" + - "# 克隆数据集配置文件与存储参数到本地\n" + - "git clone -b " + branchName + " " + projectUrl + "\n" + - "# 远程拉取配置文件\n" + - "dvc pull\n" + - "```"); - YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo),localPath, "dataset"); - - // dvc init 初始化 - DVCUtils.dvcInit(localPath); - // 配置远程S3地址 - DVCUtils.dvcRemoteAdd(localPath, s3Path); - DVCUtils.dvcConfigS3Credentials(localPath, endpoint); - DVCUtils.dvcConfigS3Credentials2(localPath, accessKeyId); - DVCUtils.dvcConfigS3Credentials3(localPath, secretAccessKey); - // dvc 跟踪 - DVCUtils.dvcAdd(localPath, "data"); - // git commit - DVCUtils.gitAdd(localPath, "."); - DVCUtils.gitCommit(localPath, "commit from ci4s with " + loginUser.getUsername()); - DVCUtils.gitPush(localPath, gitLinkUsername, gitLinkPassword); - // dvc push 到远程S3 - DVCUtils.dvcPush(localPath); - return "新增数据集成功"; - } catch (Exception e) { - throw new RuntimeException(e); - } - }); - } - - - @Override - public Page newPersonalQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception { - checkoutToken(); - Jedis jedis = new Jedis(redisHost); - LoginUser loginUser = SecurityUtils.getLoginUser(); - String ci4sUsername = loginUser.getUsername(); - String token = jedis.get(ci4sUsername+"_gitToken"); - String userReq = jedis.get(ci4sUsername + "_gitUserInfo"); - Map userInfo = JsonUtils.jsonToMap(userReq); - //拼接查询url - String datasetTagName = dataset.getDatasetTagName(); - String datasetTypeName = dataset.getDatasetTypeName(); - String topic_name = "ci4s_dataset"; - topic_name =StringUtils.isEmpty(datasetTagName)?topic_name : topic_name+",datatag_" + datasetTagName; - topic_name =StringUtils.isEmpty(datasetTagName)?topic_name : topic_name+",datatype_" + datasetTypeName; - String url = gitendpoint + "/api/users/"+(String) userInfo.get("login")+"/projects.json?page="+pageRequest.getPageNumber()+"&limit="+pageRequest.getPageSize()+"&category=manage&topic_name="+topic_name; - String req = HttpUtils.sendGetWithToken(url,null,token); - Map stringObjectMap = JacksonUtil.parseJSONStr2Map(req); - Integer total = (Integer) stringObjectMap.get("count"); - List> projects = (List>) stringObjectMap.get("projects"); - return new PageImpl<>(convert(projects), pageRequest, total); - } - - @Override - public Page newPubilcQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception { - checkoutToken(); - Jedis jedis = new Jedis(redisHost); - LoginUser loginUser = SecurityUtils.getLoginUser(); - String ci4sUsername = loginUser.getUsername(); - String token = jedis.get(ci4sUsername+"_gitToken"); - String userReq = jedis.get(ci4sUsername + "_gitUserInfo"); - Map userInfo = JsonUtils.jsonToMap(userReq); - Integer userId = (Integer) userInfo.get("user_id"); - //拼接查询url - String datasetTagName = dataset.getDatasetTagName(); - String datasetTypeName = dataset.getDatasetTypeName(); - String topic_name = "ci4s_dataset"; - topic_name =StringUtils.isEmpty(datasetTagName)?topic_name : topic_name+",datatag_" + datasetTagName; - topic_name =StringUtils.isEmpty(datasetTagName)?topic_name : topic_name+",datatype_" + datasetTypeName; - - String url = gitendpoint + "/api/projects.json?user_id="+userId+"&page="+pageRequest.getPageNumber()+"&limit="+pageRequest.getPageSize()+"&sort_by=praises_count&topic_name="+topic_name; - String req = HttpUtils.sendGetWithToken(url,null,token); - Map stringObjectMap = JacksonUtil.parseJSONStr2Map(req); - Integer total = (Integer) stringObjectMap.get("total_count"); - List> projects = (List>) stringObjectMap.get("projects"); - if (projects == null){ - return new PageImpl<>(new ArrayList<>(), pageRequest, 0); - } - return new PageImpl<>(convert(projects), pageRequest, total); - } - - @Override - public NewDatasetVo getNewDatasetDesc(Integer repoId,String repositoryName,String repo, String owner,String version) throws Exception{ - LoginUser loginUser = SecurityUtils.getLoginUser(); - String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername(); - String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword(); - // cd到 localPathlocal/repoId/下面还有一个文件夹，然后做git pull操作，然后读取里面的文件列表，列出每个文件的大小和名称，封装成MAP - if (StringUtils.isEmpty(version)){ - List> versionList = this.getVersionList(repo, owner); - if (versionList.size() == 0){ - throw new Exception("数据集文件不存在"); - } - version = (String) versionList.get(0).get("name"); - } - - List> fileDetailsAfterGitPull = DVCUtils.getFileDetailsAfterGitPull(localPathlocal+repoId, repositoryName, version,gitLinkUsername, gitLinkPassword); - //在localPathlocal+repoId+"/"+repositoryName目录下的dataset.yaml中取到元数据 - Map stringObjectMap = YamlUtils.loadYamlFile(localPathlocal + repoId + "/" + repositoryName + "/dataset.yaml"); - NewDatasetVo newDatasetVo = ConvertUtil.convertMapToObject(stringObjectMap, NewDatasetVo.class); - List versionVos = new ArrayList(); - if (fileDetailsAfterGitPull!=null&&fileDetailsAfterGitPull.size()>0){ - for(Map fileDetail : fileDetailsAfterGitPull){ - VersionVo versionVo = new VersionVo(); - versionVo.setUrl((String) fileDetail.get("filePath")); - versionVo.setFileName((String) fileDetail.get("fileName")); - long size = (long) fileDetail.get("size"); - versionVo.setFileSize(FileUtil.formatFileSize(size)); - versionVos.add(versionVo); - } - } - newDatasetVo.setDatasetVersionVos(versionVos); - return newDatasetVo; - } - - @Override - public List> getVersionList(String repo, String owner) throws Exception { - checkoutToken(); - Jedis jedis = new Jedis(redisHost); - LoginUser loginUser = SecurityUtils.getLoginUser(); - String ci4sUsername = loginUser.getUsername(); - String token = jedis.get(ci4sUsername+"_gitToken"); - List> brancheList = gitService.getBrancheList(token, owner, repo); - return brancheList.stream() - .filter(branch -> "master".equals(branch.get("name"))) - .collect(Collectors.toList()); - } - - @Override - public void deleteDatasetNew(String repo, String owner) throws Exception { - checkoutToken(); - Jedis jedis = new Jedis(redisHost); - LoginUser loginUser = SecurityUtils.getLoginUser(); - String ci4sUsername = loginUser.getUsername(); - String token = jedis.get(ci4sUsername+"_gitToken"); - gitService.deleteProject(token, owner, repo); - } - - @Override - public void deleteDatasetVersionNew(String repo, String owner, String version) throws Exception { - checkoutToken(); - Jedis jedis = new Jedis(redisHost); - LoginUser loginUser = SecurityUtils.getLoginUser(); - String ci4sUsername = loginUser.getUsername(); - String token = jedis.get(ci4sUsername+"_gitToken"); - gitService.deleteBranch(token, owner, repo, version); - } - - @Override - public List> uploadDatasetlocal(MultipartFile[] files, String uuid) throws Exception { - List> results = new ArrayList<>(); - - for (MultipartFile file:files){ - // 构建objectName - String username = SecurityUtils.getLoginUser().getUsername(); - String fileName = file.getOriginalFilename(); - String path = "/temp/"+ username +"/datasets/"+ uuid + "/"+"/data/" + fileName; - long sizeInBytes = file.getSize(); - String formattedSize = FileUtil.formatFileSize(sizeInBytes); - File targetFile = new File(path, file.getOriginalFilename()); - // 确保目录存在 - targetFile.getParentFile().mkdirs(); - // 保存文件到目标路径 - FileUtils.copyInputStreamToFile(file.getInputStream(), targetFile); - // 返回上传文件的路径 - String absolutePath = targetFile.getAbsolutePath(); - Map result = new HashMap<>(); - result.put("fileName", fileName); - result.put("url", absolutePath); // objectName根据实际情况定义 - result.put("fileSize", formattedSize); - results.add(result); - } - return results; - } - - @Override - public ResponseEntity downloadDatasetlocal(String filePath) throws Exception { - File file = new File(filePath); - - if (!file.exists()) { - throw new FileNotFoundException("File not found: " + filePath); - } - - InputStreamResource resource = new InputStreamResource(new FileInputStream(file)); - - HttpHeaders headers = new HttpHeaders(); - headers.add(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=" + file.getName()); - headers.add(HttpHeaders.CONTENT_LENGTH, String.valueOf(file.length())); - headers.add(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_OCTET_STREAM_VALUE); - - return ResponseEntity.ok() - .headers(headers) - .contentLength(file.length()) - .contentType(MediaType.APPLICATION_OCTET_STREAM) - .body(resource); - } - - @Override - public ResponseEntity downloadAllDatasetFilesNew(String repositoryName, String version) throws Exception { - // 命令行操作 git clone 项目地址 - checkoutToken(); - LoginUser loginUser = SecurityUtils.getLoginUser(); - String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername(); - String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword(); - String ci4sUsername = loginUser.getUsername(); - Jedis jedis = new Jedis(redisHost); - String userReq = jedis.get(ci4sUsername + "_gitUserInfo"); - Map userInfo = JsonUtils.jsonToMap(userReq); - Integer userId = (Integer) userInfo.get("user_id"); - String projectUrl = gitendpoint + "/" +(String) userInfo.get("login") + "/" + repositoryName + ".git"; - String localPath = localPathlocal+ loginUser.getUsername()+"/datasets/" +repositoryName; - File folder = new File(localPath); - if(folder.exists() && folder.isDirectory()){ - //切换分支 - DVCUtils.gitCheckoutBranch(localPath, version); - //pull - DVCUtils.gitPull(localPath,gitLinkUsername, gitLinkPassword); - //dvc pull - DVCUtils.dvcPull(localPath); - }else { - DVCUtils.gitClone(localPath, projectUrl, version, gitLinkUsername, gitLinkPassword); - } - - // 打包 data 文件夹 - String dataFolderPath = localPath + "/data"; - String zipFilePath = localPath + "/data.zip"; - try (FileOutputStream fos = new FileOutputStream(zipFilePath); - ZipOutputStream zos = new ZipOutputStream(fos)) { - Path sourcePath = Paths.get(dataFolderPath); - Files.walk(sourcePath).forEach(path -> { - if (!Files.isDirectory(path)) { - ZipEntry zipEntry = new ZipEntry(sourcePath.relativize(path).toString()); - try { - zos.putNextEntry(zipEntry); - Files.copy(path, zos); - zos.closeEntry(); - } catch (IOException e) { - throw new RuntimeException("Error while zipping: " + path, e); - } - } - }); - } - - // 返回压缩文件的输入流 - File zipFile = new File(zipFilePath); - InputStreamResource resource = new InputStreamResource(new FileInputStream(zipFile)); - - return ResponseEntity.ok() - .header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=data.zip") - .contentType(MediaType.APPLICATION_OCTET_STREAM) - .contentLength(zipFile.length()) - .body(resource); - - } - - - - public List convert(List> lst) { - if (lst != null && lst.size() > 0) { - List newDatasetVos = ConvertUtil.convertListMapToObjectList(lst, NewDatasetVo.class); - - for (NewDatasetVo newDatasetVo : newDatasetVos) { - Map map = lst.stream() - .filter(m -> m.get("repo_id").equals(newDatasetVo.getRepoId())) - .findFirst() - .orElse(null); - - if (map != null) { - List> topics = (List>) map.get("topics"); - if (topics != null) { - topics.forEach(topic -> { - String name = (String) topic.get("name"); - if (name != null) { - if (name.startsWith("datatag_")) { - newDatasetVo.setDataTag(name.substring("datatag_".length())); - } else if (name.startsWith("datatype_")) { - newDatasetVo.setDataType(name.substring("datatype_".length())); - } - } - }); - } - - Map author = (Map) map.get("author"); - newDatasetVo.setCreateBy((String) author.get("name")); - newDatasetVo.setOwner((String) author.get("login")); - } - } - - return newDatasetVos; - } - return new ArrayList<>(); - } - - private String checkoutToken(){ - Jedis jedis = new Jedis(redisHost); - LoginUser loginUser = SecurityUtils.getLoginUser(); - String ci4sUsername = loginUser.getUsername(); - String token = jedis.get(ci4sUsername+"_gitToken"); - String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername(); - String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword(); - - if(StringUtils.isEmpty(token)){ - gitService.login(gitLinkUsername, gitLinkPassword); - token = jedis.get(ci4sUsername+"_gitToken"); - }else { - try { - Map userInfo = gitService.getUserInfo(token); - if (userInfo == null ||(userInfo.get("status")!=null&&401==(Integer) userInfo.get("status"))){ - gitService.login(gitLinkUsername, gitLinkPassword); - token = jedis.get(ci4sUsername+"_gitToken"); - } - } catch (Exception e) { - throw new RuntimeException(e); - } - - } - return token; - - } - - - } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/GitServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/GitServiceImpl.java index 842e47da..3a4c6651 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/GitServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/GitServiceImpl.java @@ -122,6 +122,9 @@ public class GitServiceImpl implements GitService { @Override public Map getUserInfo(String token) throws Exception { String userReq = HttpUtils.sendGetWithToken("https://www.gitlink.org.cn/api/users/get_user_info.json",null, token); + if (StringUtils.isEmpty(userReq)){ + return null; + } Map runResMap = JsonUtils.jsonToMap(userReq); return runResMap; } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java new file mode 100644 index 00000000..b1e5e76e --- /dev/null +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java @@ -0,0 +1,501 @@ +package com.ruoyi.platform.service.impl; + +import com.ruoyi.common.core.utils.DateUtils; +import com.ruoyi.common.security.utils.SecurityUtils; +import com.ruoyi.platform.domain.Dataset; +import com.ruoyi.platform.service.DvcService; +import com.ruoyi.platform.service.GitService; +import com.ruoyi.platform.service.NewDatasetService; +import com.ruoyi.platform.utils.*; +import com.ruoyi.platform.vo.GitProjectVo; +import com.ruoyi.platform.vo.NewDatasetVo; +import com.ruoyi.platform.vo.VersionVo; +import com.ruoyi.system.api.model.LoginUser; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.core.io.InputStreamResource; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageImpl; +import org.springframework.data.domain.PageRequest; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Service; +import org.springframework.web.multipart.MultipartFile; +import redis.clients.jedis.Jedis; + +import javax.annotation.Resource; +import java.io.*; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +@Service +public class NewDatasetServiceImpl implements NewDatasetService { + + @Resource + private GitService gitService; + + @Resource + private DvcService dvcService; + @Value("${spring.redis.host}") + private String redisHost; + @Value("${minio.accessKey}") + String accessKeyId; + @Value("${minio.secretKey}") + String secretAccessKey; + @Value("${minio.endpoint}") + String endpoint; + @Value("${git.endpoint}") + String gitendpoint; + @Value("${git.localPath}") + String localPathlocal; + + @Override + public String newCreateDataset(NewDatasetVo datasetVo) throws Exception { + checkoutToken(); + Jedis jedis = new Jedis(redisHost); + LoginUser loginUser = SecurityUtils.getLoginUser(); + String ci4sUsername = loginUser.getUsername(); + String token = jedis.get(ci4sUsername + "_gitToken"); + String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername(); + String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword(); + String userReq = jedis.get(ci4sUsername + "_gitUserInfo"); + Map userInfo = JsonUtils.jsonToMap(userReq); + Integer userId = (Integer) userInfo.get("user_id"); + // 拼接project + String repositoryName = ci4sUsername + "_dataset_" + DateUtils.dateTimeNow(); + GitProjectVo gitProjectVo = new GitProjectVo(); + gitProjectVo.setRepositoryName(repositoryName); + gitProjectVo.setName(datasetVo.getName()); + gitProjectVo.setDescription(datasetVo.getDescription()); + gitProjectVo.setPrivate(!datasetVo.getIsPublic()); + gitProjectVo.setUserId(userId); + // 创建项目 + Map project = gitService.createProject(token, gitProjectVo); + Integer gitlinIid = (Integer) project.get("id"); + // 创建分支 + String branchName = datasetVo.getVersion(); + gitService.createBranch(token, (String) userInfo.get("login"), repositoryName, branchName, "master"); + // 定义标签 标签1：ci4s_dataset 标签2：DataTag 标签3：DataType + gitService.createTopic(token, gitlinIid, "ci4s_dataset"); + gitService.createTopic(token, gitlinIid, "DataTag_" + datasetVo.getDataTag()); + gitService.createTopic(token, gitlinIid, "DataType_" + datasetVo.getDataType()); + // 得到项目地址 + String projectUrl = gitendpoint + "/" + (String) userInfo.get("login") + "/"+ repositoryName + ".git"; + + // 得到用户操作的路径 + String url = datasetVo.getDatasetVersionVos().get(0).getUrl(); + String localPath = localPathlocal + gitlinIid + "/" + datasetVo.getName(); + String sourcePath = url.substring(0, url.lastIndexOf("/")); + // 命令行操作 git clone 项目地址 + DVCUtils.gitClone(localPath, projectUrl, branchName, gitLinkUsername, gitLinkPassword); + String s3Path = "management-platform-files" + "/" + ci4sUsername + "/" + "datasets" + "/" + gitlinIid + "/" + repositoryName + "/" + branchName; + + DVCUtils.moveFiles(sourcePath, localPath); + // 拼接生产的元数据后写入yaml文件 + datasetVo.setCreateBy(String.valueOf(StringUtils.isNotEmpty((String) userInfo.get("nickname")) ? userInfo.get("nickname") : userInfo.get("login"))); + datasetVo.setUpdateTime(DateUtils.getTime()); + datasetVo.setVersionDesc(datasetVo.getDescription()); + datasetVo.setUsage("
\n" +
+                "# 克隆数据集配置文件与存储参数到本地\n" +
+                "git clone -b " + branchName + " "+ projectUrl + "\n" +
+                "# 远程拉取配置文件\n" +
+                "dvc pull\n" +
+                "
\n"); + datasetVo.setIdentifier(repositoryName); + datasetVo.setId(gitlinIid); + datasetVo.setOwner((String) userInfo.get("login")); + + YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo), localPath, "dataset"); + // dvc init 初始化 + DVCUtils.dvcInit(localPath); + // 配置远程S3地址 + DVCUtils.dvcRemoteAdd(localPath, s3Path); + DVCUtils.dvcConfigS3Credentials(localPath, endpoint); + DVCUtils.dvcConfigS3Credentials2(localPath, accessKeyId); + DVCUtils.dvcConfigS3Credentials3(localPath, secretAccessKey); + // dvc 跟踪 + DVCUtils.dvcAdd(localPath, "data"); + // git commit + DVCUtils.gitAdd(localPath, "."); + DVCUtils.gitCommit(localPath, "commit from ci4s with " + loginUser.getUsername()); + DVCUtils.gitPush(localPath, gitLinkUsername, gitLinkPassword); + // dvc push 到远程S3 + DVCUtils.dvcPush(localPath); + return "新增数据集成功"; + } + + @Override + public CompletableFuture newCreateVersion(NewDatasetVo datasetVo) { + return CompletableFuture.supplyAsync(() -> { + try { + checkoutToken(); + Jedis jedis = new Jedis(redisHost); + LoginUser loginUser = SecurityUtils.getLoginUser(); + String ci4sUsername = loginUser.getUsername(); + String token = jedis.get(ci4sUsername + "_gitToken"); + String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername(); + String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword(); + String userReq = jedis.get(ci4sUsername + "_gitUserInfo"); + Map userInfo = JsonUtils.jsonToMap(userReq); + // 创建分支 + String branchName = StringUtils.isEmpty(datasetVo.getVersion()) ? "master" : datasetVo.getVersion(); + String repositoryName = datasetVo.getIdentifier(); + if (StringUtils.equals(branchName, "master")) { + gitService.createBranch(token, (String) userInfo.get("login"), repositoryName, branchName, "master"); + } + // 得到项目地址 + String projectUrl = gitendpoint + "/" + (String) userInfo.get("login") + "/" + repositoryName + ".git"; + + // 得到用户操作的路径 + String url = datasetVo.getDatasetVersionVos().get(0).getUrl(); + String localPath = localPathlocal + loginUser.getUsername() + "/" + "datasets" + "/" + datasetVo.getName(); + String sourcePath = url.substring(0, url.lastIndexOf("/")); + // 命令行操作 git clone 项目地址 + if (FileUtil.checkDirectoryExists(localPath)) { + DVCUtils.gitFetch(localPath, gitLinkUsername, gitLinkPassword); + DVCUtils.gitCheckoutBranch(localPath, branchName); + } else { + DVCUtils.gitClone(localPath, projectUrl, branchName, gitLinkUsername, gitLinkPassword); + } + + String s3Path = "management-platform-files" + "/" + ci4sUsername + "/" + "datasets" + "/" + datasetVo.getId() + "/" + repositoryName + "/" + branchName; + + DVCUtils.moveFiles(sourcePath, localPath); + // 拼接生产的元数据后写入yaml文件 + datasetVo.setCreateBy(String.valueOf(StringUtils.isNotEmpty((String) userInfo.get("nickname")) ? userInfo.get("nickname") : userInfo.get("login"))); + datasetVo.setUpdateTime(DateUtils.getTime()); + datasetVo.setVersionDesc(datasetVo.getDescription()); + datasetVo.setUsage("
\n" +
+                        "# 克隆数据集配置文件与存储参数到本地\n" +
+                        "git clone -b " + branchName + " "+ projectUrl + "\n" +
+                        "# 远程拉取配置文件\n" +
+                        "dvc pull\n" +
+                        "
\n"); + datasetVo.setIdentifier(repositoryName); + datasetVo.setId(datasetVo.getId()); + datasetVo.setOwner((String) userInfo.get("login")); + YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo), localPath, "dataset"); + + // dvc init 初始化 + DVCUtils.dvcInit(localPath); + // 配置远程S3地址 + DVCUtils.dvcRemoteAdd(localPath, s3Path); + DVCUtils.dvcConfigS3Credentials(localPath, endpoint); + DVCUtils.dvcConfigS3Credentials2(localPath, accessKeyId); + DVCUtils.dvcConfigS3Credentials3(localPath, secretAccessKey); + // dvc 跟踪 + DVCUtils.dvcAdd(localPath, "data"); + // git commit + DVCUtils.gitAdd(localPath, "."); + DVCUtils.gitCommit(localPath, "commit from ci4s with " + loginUser.getUsername()); + DVCUtils.gitPush(localPath, gitLinkUsername, gitLinkPassword); + // dvc push 到远程S3 + DVCUtils.dvcPush(localPath); + return "新增数据集成功"; + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + } + + @Override + public Page newPersonalQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception { + checkoutToken(); + Jedis jedis = new Jedis(redisHost); + LoginUser loginUser = SecurityUtils.getLoginUser(); + String ci4sUsername = loginUser.getUsername(); + String token = jedis.get(ci4sUsername + "_gitToken"); + String userReq = jedis.get(ci4sUsername + "_gitUserInfo"); + Map userInfo = JsonUtils.jsonToMap(userReq); + // 拼接查询url + String datasetTagName = dataset.getDatasetTagName(); + String datasetTypeName = dataset.getDatasetTypeName(); + String topic_name = "ci4s_dataset"; + topic_name = StringUtils.isEmpty(datasetTagName) ? topic_name : topic_name + ",datatag_" + datasetTagName; + topic_name = StringUtils.isEmpty(datasetTagName) ? topic_name : topic_name + ",datatype_" + datasetTypeName; + String url = gitendpoint + "/api/users/" + (String) userInfo.get("login") + "/projects.json?page=" + pageRequest.getPageNumber() + "&limit=" + pageRequest.getPageSize() + "&category=manage&topic_name=" + topic_name; + String req = HttpUtils.sendGetWithToken(url, null, token); + Map stringObjectMap = JacksonUtil.parseJSONStr2Map(req); + Integer total = (Integer) stringObjectMap.get("count"); + List> projects = (List>) stringObjectMap.get("projects"); + return new PageImpl<>(convert(projects), pageRequest, total); + } + + @Override + public Page newPubilcQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception { + checkoutToken(); + Jedis jedis = new Jedis(redisHost); + LoginUser loginUser = SecurityUtils.getLoginUser(); + String ci4sUsername = loginUser.getUsername(); + String token = jedis.get(ci4sUsername + "_gitToken"); + String userReq = jedis.get(ci4sUsername + "_gitUserInfo"); + Map userInfo = JsonUtils.jsonToMap(userReq); + Integer userId = (Integer) userInfo.get("user_id"); + // 拼接查询url + String datasetTagName = dataset.getDatasetTagName(); + String datasetTypeName = dataset.getDatasetTypeName(); + String topic_name = "ci4s_dataset"; + topic_name = StringUtils.isEmpty(datasetTagName) ? topic_name : topic_name + ",datatag_" + datasetTagName; + topic_name = StringUtils.isEmpty(datasetTagName) ? topic_name : topic_name + ",datatype_" + datasetTypeName; + + String url = gitendpoint + "/api/projects.json?user_id=" + userId + "&page=" + pageRequest.getPageNumber() + "&limit=" + pageRequest.getPageSize() + "&sort_by=praises_count&topic_name=" + topic_name; + String req = HttpUtils.sendGetWithToken(url, null, token); + Map stringObjectMap = JacksonUtil.parseJSONStr2Map(req); + Integer total = (Integer) stringObjectMap.get("total_count"); + List> projects = (List>) stringObjectMap.get("projects"); + if (projects == null) { + return new PageImpl<>(new ArrayList<>(), pageRequest, 0); + } + return new PageImpl<>(convert(projects), pageRequest, total); + } + + @Override + public NewDatasetVo getNewDatasetDesc(Integer id, String repositoryName, String repo, String owner, String version) throws Exception { + LoginUser loginUser = SecurityUtils.getLoginUser(); + String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername(); + String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword(); + // cd到 localPathlocal/id/下面还有一个文件夹，然后做git pull操作，然后读取里面的文件列表，列出每个文件的大小和名称，封装成MAP + if (StringUtils.isEmpty(version)) { + List> versionList = this.getVersionList(repo, owner); + if (versionList.size() == 0) { + throw new Exception("数据集文件不存在"); + } + version = (String) versionList.get(0).get("name"); + } + + List> fileDetailsAfterGitPull = DVCUtils.getFileDetailsAfterGitPull("E:\\test\\" +id, repositoryName, version, gitLinkUsername, gitLinkPassword); + // 在localPathlocal+id+"/"+repositoryName目录下的dataset.yaml中取到元数据 + Map stringObjectMap = YamlUtils.loadYamlFile("E:\\test\\" + id + "\\" + repositoryName + "\\" + "dataset.yaml"); + NewDatasetVo newDatasetVo = ConvertUtil.convertMapToObject(stringObjectMap, NewDatasetVo.class); + List versionVos = new ArrayList(); + if (fileDetailsAfterGitPull!=null&&fileDetailsAfterGitPull.size()>0){ + for(Map fileDetail : fileDetailsAfterGitPull){ + VersionVo versionVo = new VersionVo(); + versionVo.setUrl((String) fileDetail.get("filePath")); + versionVo.setFileName((String) fileDetail.get("fileName")); + long size = (long) fileDetail.get("size"); + versionVo.setFileSize(FileUtil.formatFileSize(size)); + versionVos.add(versionVo); + } + } + newDatasetVo.setDatasetVersionVos(versionVos); + return newDatasetVo; + } + + @Override + public List> getVersionList(String repo, String owner) throws Exception { + checkoutToken(); + Jedis jedis = new Jedis(redisHost); + LoginUser loginUser = SecurityUtils.getLoginUser(); + String ci4sUsername = loginUser.getUsername(); + String token = jedis.get(ci4sUsername+"_gitToken"); + List> brancheList = gitService.getBrancheList(token, owner, repo); + return brancheList.stream() + .filter(branch -> !"master".equals(branch.get("name"))) + .collect(Collectors.toList()); + } + + @Override + public void deleteDatasetNew(String repo, String owner) throws Exception { + checkoutToken(); + Jedis jedis = new Jedis(redisHost); + LoginUser loginUser = SecurityUtils.getLoginUser(); + String ci4sUsername = loginUser.getUsername(); + String token = jedis.get(ci4sUsername+"_gitToken"); + gitService.deleteProject(token, owner, repo); + } + + @Override + public void deleteDatasetVersionNew(String repo, String owner, String version) throws Exception { + checkoutToken(); + Jedis jedis = new Jedis(redisHost); + LoginUser loginUser = SecurityUtils.getLoginUser(); + String ci4sUsername = loginUser.getUsername(); + String token = jedis.get(ci4sUsername+"_gitToken"); + gitService.deleteBranch(token, owner, repo, version); + } + + @Override + public List> uploadDatasetlocal(MultipartFile[] files, String uuid) throws Exception { + List> results = new ArrayList<>(); + + for (MultipartFile file : files) { + // 构建objectName + String username = SecurityUtils.getLoginUser().getUsername(); + String fileName = file.getOriginalFilename(); + String path = localPathlocal + "temp" + "/" + username + "/" + "datasets" + "/" + uuid + "/" + "data" + "/" + fileName; + long sizeInBytes = file.getSize(); + String formattedSize = FileUtil.formatFileSize(sizeInBytes); + File targetFile = new File(path); + // 确保目录存在 + targetFile.getParentFile().mkdirs(); + // 保存文件到目标路径 + FileUtils.copyInputStreamToFile(file.getInputStream(), targetFile); + // 返回上传文件的路径 + String absolutePath = targetFile.getAbsolutePath(); + Map result = new HashMap<>(); + result.put("fileName", fileName); + result.put("url", path); // objectName根据实际情况定义 + result.put("fileSize", formattedSize); + results.add(result); + } + return results; + } + + @Override + public ResponseEntity downloadDatasetlocal(String filePath) throws Exception { + File file = new File(filePath); + + if (!file.exists()) { + throw new FileNotFoundException("File not found: " + filePath); + } + + InputStreamResource resource = new InputStreamResource(new FileInputStream(file)); + + HttpHeaders headers = new HttpHeaders(); + headers.add(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=" + file.getName()); + headers.add(HttpHeaders.CONTENT_LENGTH, String.valueOf(file.length())); + headers.add(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_OCTET_STREAM_VALUE); + + return ResponseEntity.ok() + .headers(headers) + .contentLength(file.length()) + .contentType(MediaType.APPLICATION_OCTET_STREAM) + .body(resource); + } + + @Override + public ResponseEntity downloadAllDatasetFilesNew(String repositoryName, String version) throws Exception { + // 命令行操作 git clone 项目地址 + checkoutToken(); + LoginUser loginUser = SecurityUtils.getLoginUser(); + String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername(); + String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword(); + String ci4sUsername = loginUser.getUsername(); + Jedis jedis = new Jedis(redisHost); + String userReq = jedis.get(ci4sUsername + "_gitUserInfo"); + Map userInfo = JsonUtils.jsonToMap(userReq); + Integer userId = (Integer) userInfo.get("user_id"); + String projectUrl = gitendpoint + "/" +(String) userInfo.get("login") + "/" + repositoryName + ".git"; + String localPath = localPathlocal+ loginUser.getUsername()+"/datasets/" +repositoryName; + File folder = new File(localPath); + if(folder.exists() && folder.isDirectory()){ + //切换分支 + DVCUtils.gitCheckoutBranch(localPath, version); + //pull + DVCUtils.gitPull(localPath,gitLinkUsername, gitLinkPassword); + //dvc pull + DVCUtils.dvcPull(localPath); + }else { + DVCUtils.gitClone(localPath, projectUrl, version, gitLinkUsername, gitLinkPassword); + } + + // 打包 data 文件夹 + String dataFolderPath = localPath + "/data"; + String zipFilePath = localPath + "/data.zip"; + try (FileOutputStream fos = new FileOutputStream(zipFilePath); + ZipOutputStream zos = new ZipOutputStream(fos)) { + Path sourcePath = Paths.get(dataFolderPath); + Files.walk(sourcePath).forEach(path -> { + if (!Files.isDirectory(path)) { + ZipEntry zipEntry = new ZipEntry(sourcePath.relativize(path).toString()); + try { + zos.putNextEntry(zipEntry); + Files.copy(path, zos); + zos.closeEntry(); + } catch (IOException e) { + throw new RuntimeException("Error while zipping: " + path, e); + } + } + }); + } + + // 返回压缩文件的输入流 + File zipFile = new File(zipFilePath); + InputStreamResource resource = new InputStreamResource(new FileInputStream(zipFile)); + + return ResponseEntity.ok() + .header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=data.zip") + .contentType(MediaType.APPLICATION_OCTET_STREAM) + .contentLength(zipFile.length()) + .body(resource); + + } + + + + public List convert(List> lst) { + if (lst != null && lst.size() > 0) { + List newDatasetVos = ConvertUtil.convertListMapToObjectList(lst, NewDatasetVo.class); + + for (NewDatasetVo newDatasetVo : newDatasetVos) { + Map map = lst.stream() + .filter(m -> m.get("id").equals(newDatasetVo.getId())) + .findFirst() + .orElse(null); + + if (map != null) { + List> topics = (List>) map.get("topics"); + if (topics != null) { + topics.forEach(topic -> { + String name = (String) topic.get("name"); + if (name != null) { + if (name.startsWith("datatag_")) { + newDatasetVo.setDataTag(name.substring("datatag_".length())); + } else if (name.startsWith("datatype_")) { + newDatasetVo.setDataType(name.substring("datatype_".length())); + } + } + }); + } + + Map author = (Map) map.get("author"); + newDatasetVo.setCreateBy((String) author.get("name")); + newDatasetVo.setOwner((String) author.get("login")); + } + } + + return newDatasetVos; + } + return new ArrayList<>(); + } + + private String checkoutToken(){ + Jedis jedis = new Jedis(redisHost); + LoginUser loginUser = SecurityUtils.getLoginUser(); + String ci4sUsername = loginUser.getUsername(); + String token = jedis.get(ci4sUsername+"_gitToken"); + String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername(); + String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword(); + + if(StringUtils.isEmpty(token)){ + gitService.login(gitLinkUsername, gitLinkPassword); + token = jedis.get(ci4sUsername+"_gitToken"); + }else { + try { + Map userInfo = gitService.getUserInfo(token); + if (userInfo == null ||(userInfo.get("status")!=null&&401==(Integer) userInfo.get("status"))){ + gitService.login(gitLinkUsername, gitLinkPassword); + token = jedis.get(ci4sUsername+"_gitToken"); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + + } + return token; + + } + +} diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/DVCUtils.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/DVCUtils.java index 2fa7780f..9aa2b03f 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/DVCUtils.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/DVCUtils.java @@ -201,11 +201,11 @@ public class DVCUtils { Path repoPath = Paths.get(localPath, repoFolder); //刷新 - gitFetch(localPath, username, password); - // 切换到指定分支 - gitCheckoutBranch(localPath, branch); - // 执行git pull - gitPull(localPath, username, password); +// gitFetch(localPath, username, password); +// // 切换到指定分支 +// gitCheckoutBranch(localPath, branch); +// // 执行git pull +// gitPull(localPath, username, password); // 读取data文件夹中的文件列表 Path dataPath = Paths.get(repoPath.toString(), "data"); File[] files = dataPath.toFile().listFiles(); diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/NewDatasetVo.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/NewDatasetVo.java index 74cccf6e..dddd325f 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/NewDatasetVo.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/NewDatasetVo.java @@ -42,8 +42,8 @@ public class NewDatasetVo implements Serializable { @ApiModelProperty(name = "time_ago",value = "更新时间") private String timeAgo; - @ApiModelProperty(name = "repo_id",value = "数据集仓库id") - private Integer repoId; + @ApiModelProperty(name = "id",value = "数据集仓库id") + private Integer id; @ApiModelProperty(name = "visits",value = "访问次数") private Integer visits; @ApiModelProperty(name = "create_by",value = "创建者") @@ -53,7 +53,7 @@ public class NewDatasetVo implements Serializable { @ApiModelProperty(name = "usage",value = "使用示例") private String usage; @ApiModelProperty(name = "update_time",value = "更新时间") - private Date updateTime; + private String updateTime; @ApiModelProperty(name = "processing_code",value = "处理代码") private String processingCode; @ApiModelProperty(name = "owner",value = "项目所有者")