diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/dataset/NewDatasetFromGitController.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/dataset/NewDatasetFromGitController.java index 26bf86be..faea384c 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/dataset/NewDatasetFromGitController.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/dataset/NewDatasetFromGitController.java @@ -53,6 +53,47 @@ public class NewDatasetFromGitController { } + @GetMapping("/queryDatasets") + @ApiOperation("数据集广场公开数据集分页查询,根据data_type,data_tag筛选，true公开false私有") + public AjaxResult queryDatasets(Dataset dataset, @RequestParam("page") int page, + @RequestParam("size") int size, + @RequestParam(value = "is_public") Boolean isPublic, + @RequestParam(value = "data_type", required = false) String dataType, + @RequestParam(value = "data_tag", required = false) String dataTag) throws Exception { + PageRequest pageRequest = PageRequest.of(page, size); + if(isPublic){ + return AjaxResult.success(this.datasetService.newPubilcQueryByPage(dataset, pageRequest)); + }else { + return AjaxResult.success(this.datasetService.newPersonalQueryByPage(dataset, pageRequest)); + } + } + + @GetMapping("/getVersionList") + @ApiOperation(value = "获取分支列表") + public AjaxResult getVersionList(@RequestParam("String") String repo,@RequestParam("owner")String owner) throws Exception { + return AjaxResult.success(this.datasetService.getVersionList(repo,owner)); + } + + @GetMapping("/getdatasetDetail") + @ApiOperation(value = "获取数据集详情") + public AjaxResult getDatasetVersions(@RequestParam("name") String name,@RequestParam("repo_id") Integer repoId,@RequestParam("version")String version) throws Exception { + return AjaxResult.success(this.datasetService.getNewDatasetDesc(repoId,name,version)); + } + + @DeleteMapping("/deleteDataset") + @ApiOperation(value = "删除数据集") + public AjaxResult deleteDataset(@RequestParam("String") String repo,@RequestParam("owner")String owner) throws Exception { + this.datasetService.deleteDatasetNew(repo,owner); + return AjaxResult.success(); + } + + @DeleteMapping("/deleteDatasetVersion") + @ApiOperation(value = "删除数据集版本") + public AjaxResult deleteDatasetVersion(@RequestParam("String") String repo,@RequestParam("owner")String owner,@RequestParam("version")String version) throws Exception { + this.datasetService.deleteDatasetVersionNew(repo,owner,version); + return AjaxResult.success(); + } + /** * 上传数据集 @@ -83,40 +124,13 @@ public class NewDatasetFromGitController { /** * 下载数据集 * - * @param dataset_version_id ps:这里的id是dataset_version表的主键 + * @param url ps:路径 * @return 单条数据 */ - @GetMapping("/download/{dataset_version_id}") + @GetMapping("/downloadSinggerFile") @ApiOperation(value = "下载单个数据集文件", notes = "根据数据集版本表id下载单个数据集文件") - public ResponseEntity downloadDataset(@PathVariable("dataset_version_id") Integer dataset_version_id) throws Exception { - return datasetService.downloadDataset(dataset_version_id); - } - - @GetMapping("/queryDatasets") - @ApiOperation("数据集广场公开数据集分页查询,根据data_type,data_tag筛选，true公开false私有") - public AjaxResult queryDatasets(Dataset dataset, @RequestParam("page") int page, - @RequestParam("size") int size, - @RequestParam(value = "is_public") Boolean isPublic, - @RequestParam(value = "data_type", required = false) String dataType, - @RequestParam(value = "data_tag", required = false) String dataTag) throws Exception { - PageRequest pageRequest = PageRequest.of(page, size); - if(isPublic){ - return AjaxResult.success(this.datasetService.newPubilcQueryByPage(dataset, pageRequest)); - }else { - return AjaxResult.success(this.datasetService.newPersonalQueryByPage(dataset, pageRequest)); - } - } - - @GetMapping("/getVersionList") - @ApiOperation(value = "获取分支列表") - public AjaxResult getVersionList(@RequestParam("name") String name,@RequestParam("repo_id") Integer repoId,@RequestParam("version")String version) throws Exception { - return AjaxResult.success(this.datasetService.getNewDatasetDesc(repoId,name,version)); - } - - @GetMapping("/getdatasetDetail") - @ApiOperation(value = "获取数据集详情") - public AjaxResult getDatasetVersions(@RequestParam("name") String name,@RequestParam("repo_id") Integer repoId,@RequestParam("version")String version) throws Exception { - return AjaxResult.success(this.datasetService.getNewDatasetDesc(repoId,name,version)); + public ResponseEntity downloadDataset(@RequestParam("url") String url) throws Exception { + return datasetService.downloadDatasetlocal(url); } } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/DatasetService.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/DatasetService.java index cddbd552..423e8956 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/DatasetService.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/DatasetService.java @@ -95,13 +95,17 @@ DatasetService { CompletableFuture newCreateDataset(NewDatasetVo datasetVo) throws Exception; CompletableFuture newCreateVersion(NewDatasetVo datasetVo); - - List> uploadDatasetlocal(MultipartFile[] files, String uuid) throws Exception; - + ResponseEntity downloadDatasetlocal(String filePath) throws Exception; ResponseEntity downloadAllDatasetFilesNew(String repositoryName, String version) throws IOException, Exception; Page newPersonalQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception; Page newPubilcQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception; NewDatasetVo getNewDatasetDesc(Integer repoId,String RepositoryName, String version); + + List> getVersionList(String repo, String owner) throws Exception; + + void deleteDatasetNew(String repo, String owner) throws Exception; + + void deleteDatasetVersionNew(String repo, String owner, String version) throws Exception; } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/GitService.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/GitService.java index e130896b..96af9291 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/GitService.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/GitService.java @@ -2,6 +2,8 @@ package com.ruoyi.platform.service; import com.ruoyi.platform.vo.GitProjectVo; +import java.io.IOException; +import java.util.List; import java.util.Map; public interface GitService { @@ -14,4 +16,10 @@ public interface GitService { void createBranch(String token,String owner, String projectName, String branchName, String oldBranchName) throws Exception; void createTopic(String token, Integer id, String topicName) throws Exception; + + List> getBrancheList(String token, String owner, String projectName) throws Exception; + + void deleteProject(String token, String owner, String projectName) throws Exception; + + void deleteBranch(String token, String owner,String projectName ,String branchName) throws Exception; } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/DatasetServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/DatasetServiceImpl.java index 887b3ed7..4313a462 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/DatasetServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/DatasetServiceImpl.java @@ -504,8 +504,8 @@ public class DatasetServiceImpl implements DatasetService { DVCUtils.moveFiles(sourcePath, localPath + "/data"); //拼接生产的元数据后写入yaml文件 - datasetVo.setCreateBy(String.valueOf(StringUtils.isNotEmpty((String) userInfo.get("nickname")) ? userInfo.get("nickname") : userInfo.get("login"))); - datasetVo.setUpdateTime(DateUtils.getTime()); + datasetVo.setCreateBy(String.valueOf(StringUtils.isNotEmpty((String)userInfo.get("nickname"))? userInfo.get("nickname") : userInfo.get("login"))); + datasetVo.setUpdateTime(DateUtils.getNowDate()); datasetVo.setVersionDesc(datasetVo.getDescription()); datasetVo.setUsage("```bash\n" + "# 克隆数据集配置文件与存储参数到本地\n" + @@ -562,10 +562,10 @@ public class DatasetServiceImpl implements DatasetService { String localPath = localPathlocal + loginUser.getUsername() + "/datasets/" + datasetVo.getName() + "/" + branchName; String sourcePath = url.substring(0, url.lastIndexOf("/")); // 命令行操作 git clone 项目地址 - if (FileUtil.checkDirectoryExists(localPath)) { - DVCUtils.gitFetch(localPath); - DVCUtils.gitCheckoutBranch(localPath, branchName); - } else { + if(FileUtil.checkDirectoryExists(localPath)){ + DVCUtils.gitFetch(localPath,gitLinkUsername, gitLinkPassword); + DVCUtils.gitCheckoutBranch(localPath,branchName); + }else { DVCUtils.gitClone(localPath, projectUrl, branchName, gitLinkUsername, gitLinkPassword); } @@ -573,8 +573,8 @@ public class DatasetServiceImpl implements DatasetService { DVCUtils.moveFiles(sourcePath, localPath + "/data"); //拼接生产的元数据后写入yaml文件 - datasetVo.setCreateBy(String.valueOf(StringUtils.isNotEmpty((String) userInfo.get("nickname")) ? userInfo.get("nickname") : userInfo.get("login"))); - datasetVo.setUpdateTime(DateUtils.getTime()); + datasetVo.setCreateBy(String.valueOf(StringUtils.isNotEmpty((String)userInfo.get("nickname"))? userInfo.get("nickname") : userInfo.get("login"))); + datasetVo.setUpdateTime(DateUtils.getNowDate()); datasetVo.setVersionDesc(datasetVo.getDescription()); datasetVo.setUsage("```bash\n" + "# 克隆数据集配置文件与存储参数到本地\n" + @@ -662,8 +662,10 @@ public class DatasetServiceImpl implements DatasetService { public NewDatasetVo getNewDatasetDesc(Integer repoId, String repositoryName, String version) { LoginUser loginUser = SecurityUtils.getLoginUser(); String ci4sUsername = loginUser.getUsername(); + String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername(); + String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword(); // cd到 localPathlocal/repoId/下面还有一个文件夹，然后做git pull操作，然后读取里面的文件列表，列出每个文件的大小和名称，封装成MAP - List> fileDetailsAfterGitPull = DVCUtils.getFileDetailsAfterGitPull(localPathlocal + ci4sUsername + "/datasets/", repositoryName, version, "data"); + List> fileDetailsAfterGitPull = DVCUtils.getFileDetailsAfterGitPull(localPathlocal + ci4sUsername + "/datasets/", repositoryName, version, "data",gitLinkUsername, gitLinkPassword); //在localPathlocal+repoId+"/"+repositoryName目录下的dataset.yaml中取到元数据 Map stringObjectMap = YamlUtils.loadYamlFile(localPathlocal + ci4sUsername + "/datasets/" + repositoryName + "/" + version + "/metadata/metadata.yaml"); NewDatasetVo newDatasetVo = ConvertUtil.convertMapToObject(stringObjectMap, NewDatasetVo.class); @@ -682,6 +684,34 @@ public class DatasetServiceImpl implements DatasetService { return newDatasetVo; } + @Override + public List> getVersionList(String repo, String owner) throws Exception { + LoginUser loginUser = SecurityUtils.getLoginUser(); + String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername(); + String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword(); + String token = gitService.login(gitLinkUsername, gitLinkPassword); + List> brancheList = gitService.getBrancheList(token, owner, repo); + return brancheList; + } + + @Override + public void deleteDatasetNew(String repo, String owner) throws Exception { + LoginUser loginUser = SecurityUtils.getLoginUser(); + String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername(); + String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword(); + String token = gitService.login(gitLinkUsername, gitLinkPassword); + gitService.deleteProject(token, owner, repo); + } + + @Override + public void deleteDatasetVersionNew(String repo, String owner, String version) throws Exception { + LoginUser loginUser = SecurityUtils.getLoginUser(); + String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername(); + String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword(); + String token = gitService.login(gitLinkUsername, gitLinkPassword); + gitService.deleteBranch(token, owner, repo, version); + } + @Override public List> uploadDatasetlocal(MultipartFile[] files, String uuid) throws Exception { List> results = new ArrayList<>(); @@ -709,6 +739,28 @@ public class DatasetServiceImpl implements DatasetService { return results; } + @Override + public ResponseEntity downloadDatasetlocal(String filePath) throws Exception { + File file = new File(filePath); + + if (!file.exists()) { + throw new FileNotFoundException("File not found: " + filePath); + } + + InputStreamResource resource = new InputStreamResource(new FileInputStream(file)); + + HttpHeaders headers = new HttpHeaders(); + headers.add(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=" + file.getName()); + headers.add(HttpHeaders.CONTENT_LENGTH, String.valueOf(file.length())); + headers.add(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_OCTET_STREAM_VALUE); + + return ResponseEntity.ok() + .headers(headers) + .contentLength(file.length()) + .contentType(MediaType.APPLICATION_OCTET_STREAM) + .body(resource); + } + @Override public ResponseEntity downloadAllDatasetFilesNew(String repositoryName, String version) throws Exception { // 命令行操作 git clone 项目地址 diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/GitServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/GitServiceImpl.java index 40980dc9..73e42491 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/GitServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/GitServiceImpl.java @@ -1,9 +1,9 @@ package com.ruoyi.platform.service.impl; -import com.fasterxml.jackson.core.JsonProcessingException; import com.ruoyi.common.security.utils.SecurityUtils; import com.ruoyi.platform.service.GitService; import com.ruoyi.platform.utils.HttpUtils; +import com.ruoyi.platform.utils.JacksonUtil; import com.ruoyi.platform.utils.JsonUtils; import com.ruoyi.platform.vo.GitProjectVo; import com.ruoyi.system.api.model.LoginUser; @@ -14,7 +14,9 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import redis.clients.jedis.Jedis; +import java.io.IOException; import java.util.HashMap; +import java.util.List; import java.util.Map; @Service @@ -95,4 +97,26 @@ public class GitServiceImpl implements GitService { String req = HttpUtils.sendPostWithToken("https://www.gitlink.org.cn/api/v1/project_topics.json",JsonUtils.objectToJson(resMap),token); System.out.println(req); } + + @Override + public List> getBrancheList(String token, String owner, String projectName) throws Exception { + String req = HttpUtils.sendGetWithToken("https://www.gitlink.org.cn/api/v1/"+owner+"/"+projectName+"/branches/all.json",null, token); + // 解析响应JSON + if (StringUtils.isEmpty(req)) { + throw new RuntimeException("终止响应内容为空。"); + } + // 将响应的JSON字符串转换为List对象 + List> mapList = JacksonUtil.parseJSONStr2MapList(req); + return mapList; + } + + @Override + public void deleteProject(String token, String owner, String projectName) throws Exception { + HttpUtils.sendDeleteRequest("https://www.gitlink.org.cn/api/"+owner+"/"+projectName+".json", token); + } + + @Override + public void deleteBranch(String token, String owner, String projectName, String branchName) throws Exception { + HttpUtils.sendDeleteRequest("https://www.gitlink.org.cn/api/v1/"+owner+"/"+projectName+"/branches/"+branchName+".json", token); + } } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/DVCUtils.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/DVCUtils.java index 1a6dd3bb..2d7d10a7 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/DVCUtils.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/DVCUtils.java @@ -6,6 +6,8 @@ import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.storage.file.FileRepositoryBuilder; import org.eclipse.jgit.transport.CredentialsProvider; import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.*; import java.nio.file.*; @@ -16,6 +18,8 @@ import java.util.Map; public class DVCUtils { + private static final Logger log = LoggerFactory.getLogger(DVCUtils.class); + private static void runCommand(String command, String workingDir) throws Exception { ProcessBuilder processBuilder = new ProcessBuilder(command.split(" ")); processBuilder.directory(new File(workingDir)); @@ -99,14 +103,17 @@ public class DVCUtils { } } - public static void gitFetch(String localPath) { - try { - ProcessBuilder pb = new ProcessBuilder("git", "fetch"); - pb.directory(new File(localPath)); - Process process = pb.start(); - process.waitFor(); - } catch (IOException | InterruptedException e) { - e.printStackTrace(); + public static void gitFetch(String localPath, String username, String password) throws IOException, GitAPIException { + FileRepositoryBuilder builder = new FileRepositoryBuilder(); + Repository repository = builder.setGitDir(new File(localPath, ".git")) + .readEnvironment() + .findGitDir() + .build(); + + try (Git git = new Git(repository)) { + CredentialsProvider credentialsProvider = new UsernamePasswordCredentialsProvider(username, password); + FetchCommand fetchCommand = git.fetch(); + fetchCommand.setCredentialsProvider(credentialsProvider).call(); } } public static void gitCheckoutBranch(String localPath, String branchName) throws IOException, GitAPIException { @@ -186,25 +193,19 @@ public class DVCUtils { * @param branch 分支名称 * @return 包含文件路径、名称和大小的List> */ - public static List> getFileDetailsAfterGitPull(String localPath, String repoFolder, String branch, String filePath) { + public static List> getFileDetailsAfterGitPull(String localPath, String repoFolder, String branch,String username, String password) { List> fileInfoList = new ArrayList<>(); try { // 切换到指定目录 Path repoPath = Paths.get(localPath, repoFolder, branch); + //刷新 + gitFetch(localPath, username, password); // 切换到指定分支 - ProcessBuilder pb = new ProcessBuilder("git", "checkout", branch); - pb.directory(repoPath.toFile()); - Process process = pb.start(); - process.waitFor(); - + gitCheckoutBranch(localPath, branch); // 执行git pull - pb = new ProcessBuilder("git", "pull"); - pb.directory(repoPath.toFile()); - process = pb.start(); - process.waitFor(); - + gitPull(localPath, username, password); // 读取data文件夹中的文件列表 Path dataPath = Paths.get(repoPath.toString(), filePath); File[] files = dataPath.toFile().listFiles(); @@ -226,8 +227,8 @@ public class DVCUtils { } } - } catch (IOException | InterruptedException e) { - e.printStackTrace(); + } catch (Exception e) { + log.error("Error occurred while getting file details after git pull", e); } return fileInfoList; diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/HttpUtils.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/HttpUtils.java index 504c4c6a..7a3d5518 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/HttpUtils.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/HttpUtils.java @@ -9,6 +9,7 @@ import org.apache.http.client.HttpClient; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.entity.UrlEncodedFormEntity; import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpDelete; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; import org.apache.http.client.utils.URIBuilder; @@ -622,4 +623,46 @@ public class HttpUtils { } return resultStr; } + + + + // 其他方法保持不变 + + /** + * 向指定 URL 发送带 token 的 DELETE 方法的请求 + * + * @param url 发送请求的 URL + * @param token 认证 token，作为请求头的一部分。 + * @return 所代表远程资源的响应结果 + * @throws IOException 如果请求失败或发生其他 I/O 错误。 + */ + public static String sendDeleteRequest(String url, String token) throws IOException { + String result = ""; + try (CloseableHttpClient httpClient = HttpClients.createDefault()) { + HttpDelete httpDelete = new HttpDelete(url); + + // 设置请求头 + httpDelete.setHeader("Authorization", "Bearer " + token); + httpDelete.setHeader("Content-Type", "application/json"); + + // 创建请求配置 + RequestConfig requestConfig = RequestConfig.custom() + .setConnectTimeout(15000) // 连接服务区主机超时时间 + .setConnectionRequestTimeout(60000) // 连接请求超时时间 + .setSocketTimeout(60000) // 设置读取响应数据超时时间 + .build(); + httpDelete.setConfig(requestConfig); + + // 执行请求 + try (CloseableHttpResponse response = httpClient.execute(httpDelete)) { + int statusCode = response.getStatusLine().getStatusCode(); + if (statusCode == 200) { + result = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8); + } else { + throw new IOException("HTTP request failed with response code: " + statusCode); + } + } + } + return result; + } } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/NewDatasetVo.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/NewDatasetVo.java index f4566b5f..bd329b12 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/NewDatasetVo.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/NewDatasetVo.java @@ -53,7 +53,7 @@ public class NewDatasetVo implements Serializable { @ApiModelProperty(name = "usage",value = "使用示例") private String usage; @ApiModelProperty(name = "update_time",value = "更新时间") - private String updateTime; + private Date updateTime; @ApiModelProperty(name = "processing_code",value = "处理代码") private String processingCode;