From 3d67f58e5bf732717247d1a338facd1656b36318 Mon Sep 17 00:00:00 2001 From: chenzhihang <709011834@qq.com> Date: Tue, 3 Sep 2024 16:35:32 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E6=A8=A1=E5=9E=8Bdvc=E7=AE=A1?= =?UTF-8?q?=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dataset/NewDatasetFromGitController.java | 2 +- .../model/NewModelFromGitController.java | 86 +++++ .../ruoyi/platform/service/ModelsService.java | 21 +- .../service/impl/ModelsServiceImpl.java | 362 +++++++++++++++--- .../java/com/ruoyi/platform/vo/ModelsVo.java | 84 +--- .../com/ruoyi/platform/vo/NewDatasetVo.java | 2 - 6 files changed, 433 insertions(+), 124 deletions(-) create mode 100644 ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/model/NewModelFromGitController.java diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/dataset/NewDatasetFromGitController.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/dataset/NewDatasetFromGitController.java index 0821d3ac..26e76b9a 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/dataset/NewDatasetFromGitController.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/dataset/NewDatasetFromGitController.java @@ -15,7 +15,7 @@ import org.springframework.web.multipart.MultipartFile; import javax.annotation.Resource; @RestController @RequestMapping("newdataset") -@Api(value = "新数据集管理") +//@Api(value = "新数据集管理") public class NewDatasetFromGitController { /** * 服务对象 diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/model/NewModelFromGitController.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/model/NewModelFromGitController.java new file mode 100644 index 00000000..2966d2bb --- /dev/null +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/model/NewModelFromGitController.java @@ -0,0 +1,86 @@ +package com.ruoyi.platform.controller.model; + +import com.ruoyi.common.core.web.domain.AjaxResult; +import com.ruoyi.platform.service.ModelsService; +import com.ruoyi.platform.vo.ModelsVo; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import org.springframework.core.io.InputStreamResource; +import org.springframework.data.domain.PageRequest; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.*; + +import javax.annotation.Resource; + +@RestController +@RequestMapping("newmodel") +@Api(value = "新模型管理") +public class NewModelFromGitController { + + @Resource + private ModelsService modelsService; + + @PostMapping("/addModelAndVersion") + @ApiOperation("添加模型和版本") + public AjaxResult addModelAndVersion(@RequestBody ModelsVo modelsVo) throws Exception { + return AjaxResult.success(this.modelsService.newCreateModel(modelsVo)); + } + + @PostMapping("/addVersion") + @ApiOperation("添加版本") + public AjaxResult addVersion(@RequestBody ModelsVo modelsVo) throws Exception { + return AjaxResult.success(this.modelsService.newCreateVersion(modelsVo)); + } + + /** + * 模型打包下载 + * + * @param version 模型版本 + * @return 模型 + */ + @GetMapping("/downloadAllFiles") + @ApiOperation(value = "下载同一版本下所有模型，并打包") + public ResponseEntity downloadAllDatasetFiles(@RequestParam(value = "repository_name") String repositoryName, + @RequestParam(value = "version") String version, + @RequestParam(value = "git_link_username") String gitLinkUsername, + @RequestParam(value = "git_link_password") String gitLinkPassword) throws Exception { + return modelsService.downloadAllModelFilesNew(repositoryName, version, gitLinkUsername, gitLinkPassword); + } + + + /** + * 下载模型 + * + * @param model_version_id ps:这里的id是model_version表的主键 + * @return 模型 + */ + + @GetMapping("/download/{model_version_id}") + @ApiOperation(value = "下载单个模型文件", notes = "根据模型版本表id下载单个模型文件") + public ResponseEntity downloadModel(@PathVariable("model_version_id") Integer model_version_id) throws Exception { + return modelsService.downloadModels(model_version_id); + } + + @GetMapping("/queryDatasets") + @ApiOperation("模型广场公开模型分页查询,根据model_type,model_tag筛选，true公开false私有") + public AjaxResult queryDatasets(@RequestParam(value = "page") int page, + @RequestParam(value = "size") int size, + @RequestParam(value = "is_public") Boolean isPublic, + @RequestParam(value = "model_type", required = false) String modelType, + @RequestParam(value = "model_tag", required = false) String modelTag, + @RequestParam(value = "git_link_username") String gitLinkUsername, + @RequestParam(value = "git_link_password") String gitLinkPassword) throws Exception { + PageRequest pageRequest = PageRequest.of(page, size); + ModelsVo modelsVo = new ModelsVo(); + modelsVo.setModelType(modelType); + modelsVo.setModelTag(modelTag); + modelsVo.setGitLinkUsername(gitLinkUsername); + modelsVo.setGitLinkPassword(gitLinkPassword); + if (isPublic) { + return AjaxResult.success(this.modelsService.newPubilcQueryByPage(modelsVo, pageRequest)); + } else { + return AjaxResult.success(this.modelsService.newPersonalQueryByPage(modelsVo, pageRequest)); + } + } + +} diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ModelsService.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ModelsService.java index c4e5b920..8502fd95 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ModelsService.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ModelsService.java @@ -1,8 +1,6 @@ package com.ruoyi.platform.service; - -import com.ruoyi.platform.domain.Dataset; import com.ruoyi.platform.domain.Models; import com.ruoyi.platform.domain.ModelsVersion; import com.ruoyi.platform.vo.ModelsVo; @@ -12,8 +10,10 @@ import org.springframework.data.domain.PageRequest; import org.springframework.http.ResponseEntity; import org.springframework.web.multipart.MultipartFile; +import java.io.IOException; import java.util.List; import java.util.Map; +import java.util.concurrent.CompletableFuture; /** * (Models)表服务接口 @@ -34,8 +34,8 @@ public interface ModelsService { /** * 分页查询 * - * @param models 筛选条件 - * @param pageRequest 分页对象 + * @param models 筛选条件 + * @param pageRequest 分页对象 * @return 查询结果 */ Page queryByPage(Models models, PageRequest pageRequest); @@ -70,12 +70,12 @@ public interface ModelsService { ResponseEntity downloadModels(Integer id) throws Exception; - List> uploadModels(MultipartFile[] files, String uuid) throws Exception; Map uploadModelsPipeline(ModelsVersion modelsVersion) throws Exception; ResponseEntity downloadAllModelFiles(Integer modelsId, String version) throws Exception; + List getModelVersions(Integer modelId) throws Exception; String insertModelAndVersion(ModelsVo modelsVo) throws Exception; @@ -85,4 +85,15 @@ public interface ModelsService { public void checkDeclaredName(Models insert) throws Exception; List> exportModels(String path, String uuid) throws Exception; + + + CompletableFuture newCreateModel(ModelsVo modelsVo) throws Exception; + + CompletableFuture newCreateVersion(ModelsVo modelsVo); + + ResponseEntity downloadAllModelFilesNew(String repositoryName, String version, String gitLinkUsername, String gitLinkPassword) throws IOException, Exception; + + Page newPubilcQueryByPage(ModelsVo modelsVo, PageRequest pageRequest) throws Exception; + + Page newPersonalQueryByPage(ModelsVo modelsVo, PageRequest pageRequest) throws Exception; } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java index b3603484..caef5b1d 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java @@ -1,26 +1,23 @@ package com.ruoyi.platform.service.impl; +import com.ruoyi.common.core.utils.DateUtils; import com.ruoyi.common.security.utils.SecurityUtils; import com.ruoyi.platform.annotations.CheckDuplicate; import com.ruoyi.platform.domain.AssetIcon; -import com.ruoyi.platform.domain.Dataset; import com.ruoyi.platform.domain.Models; import com.ruoyi.platform.domain.ModelsVersion; import com.ruoyi.platform.mapper.ModelsDao; import com.ruoyi.platform.mapper.ModelsVersionDao; -import com.ruoyi.platform.service.AssetIconService; -import com.ruoyi.platform.service.MinioService; -import com.ruoyi.platform.service.ModelsService; -import com.ruoyi.platform.service.ModelsVersionService; -import com.ruoyi.platform.utils.BeansUtils; -import com.ruoyi.platform.utils.FileUtil; -import com.ruoyi.platform.utils.MinioUtil; +import com.ruoyi.platform.service.*; +import com.ruoyi.platform.utils.*; +import com.ruoyi.platform.vo.GitProjectVo; import com.ruoyi.platform.vo.ModelsVo; import com.ruoyi.platform.vo.VersionVo; import com.ruoyi.system.api.model.LoginUser; import io.minio.messages.Item; -import io.netty.util.Version; import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; import org.springframework.core.io.InputStreamResource; import org.springframework.data.domain.Page; @@ -32,14 +29,17 @@ import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import org.springframework.web.multipart.MultipartFile; +import redis.clients.jedis.Jedis; import javax.annotation.Resource; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.InputStream; +import java.io.*; import java.lang.reflect.Field; import java.net.URLEncoder; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.*; +import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; @@ -52,6 +52,9 @@ import java.util.zip.ZipOutputStream; */ @Service("modelsService") public class ModelsServiceImpl implements ModelsService { + + private static final Logger logger = LoggerFactory.getLogger(ModelsServiceImpl.class); + @Resource private ModelsDao modelsDao; @Resource @@ -63,6 +66,9 @@ public class ModelsServiceImpl implements ModelsService { @Resource private MinioService minioService; + @Resource + private GitService gitService; + @Resource private AssetIconService assetIconService; @@ -71,8 +77,20 @@ public class ModelsServiceImpl implements ModelsService { @Value("${minio.dataReleaseBucketName}") private String bucketName; @Resource - private MinioUtil minioUtil; - + private MinioUtil minioUtil; + + @Value("${spring.redis.host}") + private String redisHost; + @Value("${git.endpoint}") + String gitendpoint; + @Value("${git.localPath}") + String localPath; + @Value("${minio.accessKey}") + String accessKeyId; + @Value("${minio.secretKey}") + String secretAccessKey; + @Value("${minio.endpoint}") + String endpoint; /** * 通过ID查询单条数据 @@ -86,15 +104,15 @@ public class ModelsServiceImpl implements ModelsService { String modelType = models.getModelType(); String modelTag = models.getModelTag(); //去资产管理表中查询对应的图标名,注意判空逻辑，只有当dataType和dataTag不为空时，才进行查询 - if(modelType != null && !modelType.isEmpty()){ + if (modelType != null && !modelType.isEmpty()) { AssetIcon modelTypeAssetIcon = assetIconService.queryById(Integer.valueOf(modelType)); - if (modelTypeAssetIcon != null){ + if (modelTypeAssetIcon != null) { models.setModelTypeName(modelTypeAssetIcon.getName()); } } - if(modelTag != null && !modelTag.isEmpty()){ + if (modelTag != null && !modelTag.isEmpty()) { AssetIcon modelTagAssetIcon = assetIconService.queryById(Integer.valueOf(modelTag)); - if (modelTagAssetIcon != null){ + if (modelTagAssetIcon != null) { models.setModelTagName(modelTagAssetIcon.getName()); } } @@ -104,8 +122,8 @@ public class ModelsServiceImpl implements ModelsService { /** * 分页查询 * - * @param models 筛选条件 - * @param pageRequest 分页对象 + * @param models 筛选条件 + * @param pageRequest 分页对象 * @return 查询结果 */ @Override @@ -142,7 +160,7 @@ public class ModelsServiceImpl implements ModelsService { @Override public Models update(Models models) { int currentState = models.getState(); - if(currentState == 0){ + if (currentState == 0) { throw new RuntimeException("模型已被删除，无法更新。"); } LoginUser loginUser = SecurityUtils.getLoginUser(); @@ -166,7 +184,7 @@ public class ModelsServiceImpl implements ModelsService { @Override public String removeById(Integer id) throws Exception { Models models = this.modelsDao.queryById(id); - if (models == null){ + if (models == null) { throw new Exception("模型不存在"); } @@ -176,15 +194,15 @@ public class ModelsServiceImpl implements ModelsService { String createdBy = models.getCreateBy(); - if (!(StringUtils.equals(username,"admin") || StringUtils.equals(username,createdBy))){ + if (!(StringUtils.equals(username, "admin") || StringUtils.equals(username, createdBy))) { throw new Exception("无权限删除该模型"); } //判断是否有版本文件 - if (!modelsVersionService.queryByModelsId(id).isEmpty()){ + if (!modelsVersionService.queryByModelsId(id).isEmpty()) { throw new Exception("请先删除该模型下的版本文件"); } models.setState(0); - return this.modelsDao.update(models)>0?"删除成功":"删除失败"; + return this.modelsDao.update(models) > 0 ? "删除成功" : "删除失败"; } /** @@ -192,7 +210,6 @@ public class ModelsServiceImpl implements ModelsService { * * @param id models_version表的主键 * @return 文件内容 - * */ @Override @@ -204,19 +221,19 @@ public class ModelsServiceImpl implements ModelsService { } // 从数据库中获取存储路径（即MinIO中的对象名称） String objectName = modelsVersion.getUrl(); - if(objectName == null || objectName.isEmpty() ){ + if (objectName == null || objectName.isEmpty()) { throw new Exception("未找到该版本模型文件"); } try { // 使用ByteArrayOutputStream来捕获下载的数据 ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - minioUtil.downloadObject(bucketName,objectName,outputStream); + minioUtil.downloadObject(bucketName, objectName, outputStream); ByteArrayInputStream inputStream = new ByteArrayInputStream(outputStream.toByteArray()); InputStreamResource resource = new InputStreamResource(inputStream); return ResponseEntity.ok() - .header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + URLEncoder.encode(extractFileName(objectName),"UTF-8") + "\"") + .header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + URLEncoder.encode(extractFileName(objectName), "UTF-8") + "\"") .contentType(MediaType.APPLICATION_OCTET_STREAM) .body(resource); @@ -231,13 +248,13 @@ public class ModelsServiceImpl implements ModelsService { * 上传模型文件 * * @param files 文件 - * @param uuid 唯一标识 + * @param uuid 唯一标识 * @return 是否成功 */ @Override public List> uploadModels(MultipartFile[] files, String uuid) throws Exception { List> results = new ArrayList<>(); - for (MultipartFile file:files) { + for (MultipartFile file : files) { // 构建objectName String username = SecurityUtils.getLoginUser().getUsername(); String fileName = file.getOriginalFilename(); @@ -265,7 +282,7 @@ public class ModelsServiceImpl implements ModelsService { //插表,因为这里是一次直接插表所以这里定掉date，然后用DAO插入 Date date = new Date(); //String timestamp = new SimpleDateFormat("yyyyMMdd-HHmmss").format(date); - url = "models/" + username + "/" + models.getName() + "-" + "/" + modelsVersion.getVersion() + "/" + modelsVersion.getFileName(); + url = "models/" + username + "/" + models.getName() + "-" + "/" + modelsVersion.getVersion() + "/" + modelsVersion.getFileName(); modelsVersion.setUrl(url); modelsVersion.setCreateBy(username); modelsVersion.setUpdateBy(username); @@ -273,7 +290,7 @@ public class ModelsServiceImpl implements ModelsService { modelsVersion.setUpdateTime(date); modelsVersion.setState(1); modelsVersionDao.insert(modelsVersion); - }else { + } else { //改表 BeansUtils.copyPropertiesIgnoreNull(modelsVersion, version); Date createTime = version.getCreateTime(); @@ -283,21 +300,21 @@ public class ModelsServiceImpl implements ModelsService { modelsVersionService.update(version); } Map result = new HashMap(); - result.put("url",url); + result.put("url", url); return result; } /** * 下载所有模型文件，用压缩包的方式返回 * - * @param modelsId 模型ID - * @param version 模型版本号 + * @param modelsId 模型ID + * @param version 模型版本号 * @return 文件内容 */ @Override public ResponseEntity downloadAllModelFiles(Integer modelsId, String version) throws Exception { // 根据模型id查模型名 - Models model = this.modelsDao.queryById(modelsId); + Models model = this.modelsDao.queryById(modelsId); String modelName = model.getName(); // 查询特定模型和版本对应的所有文件 List modelsVersionList = this.modelsVersionDao.queryAllByModelsVersion(modelsId, version); @@ -328,7 +345,7 @@ public class ModelsServiceImpl implements ModelsService { // 设置响应 return ResponseEntity.ok() - .header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + modelName + "_" + version + ".zip\"") + .header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + modelName + "_" + version + ".zip\"") .contentType(MediaType.APPLICATION_OCTET_STREAM) .body(resource); } catch (Exception e) { @@ -361,7 +378,7 @@ public class ModelsServiceImpl implements ModelsService { public String insertModelAndVersion(ModelsVo modelsVo) throws Exception { List modelsVersionVos = modelsVo.getModelsVersionVos(); - if (modelsVersionVos==null || modelsVersionVos.isEmpty()){ + if (modelsVersionVos == null || modelsVersionVos.isEmpty()) { throw new Exception("模型版本信息错误"); } Models models = new Models(); @@ -371,11 +388,11 @@ public class ModelsServiceImpl implements ModelsService { models.setModelType(modelsVo.getModelType()); models.setModelTag(modelsVo.getModelTag()); Models modelsInsert = this.insert(models); - if (modelsInsert == null){ + if (modelsInsert == null) { throw new Exception("新增模型失败"); } //遍历版本信息列表，把文件信息插入数据库 - for(VersionVo modelsVersionVo : modelsVersionVos){ + for (VersionVo modelsVersionVo : modelsVersionVos) { ModelsVersion modelsVersion = new ModelsVersion(); modelsVersion.setModelsId(modelsInsert.getId()); modelsVersion.setVersion(modelsVo.getVersion()); @@ -394,16 +411,16 @@ public class ModelsServiceImpl implements ModelsService { /** - * 根据模型id和版本读取文件内容 + * 根据模型id和版本读取文件内容 * - * @param modelsId 模型ID - * @param version 模型版本号 + * @param modelsId 模型ID + * @param version 模型版本号 * @return 文件内容 */ @Override public String readFileContent(Integer modelsId, String version) throws Exception { // 根据模型id查模型名 - Models model = this.modelsDao.queryById(modelsId); + Models model = this.modelsDao.queryById(modelsId); if (model == null) { throw new Exception("模型不存在"); } @@ -415,9 +432,9 @@ public class ModelsServiceImpl implements ModelsService { throw new Exception("对应模型版本不存在"); } //遍历文件列表 - for(ModelsVersion modelsVersion : modelsVersionList){ + for (ModelsVersion modelsVersion : modelsVersionList) { String fileName = modelsVersion.getFileName(); - if("readme.md".equalsIgnoreCase(fileName)){ + if ("readme.md".equalsIgnoreCase(fileName)) { //如果存在readme文件，读取minio中的url objectName = modelsVersion.getUrl(); // 读取MinIO中的对象为字符串 @@ -473,7 +490,7 @@ public class ModelsServiceImpl implements ModelsService { minioUtil.copyDirectory(srcBucketName, srcDir, bucketName, targetDir); List movedItems = minioUtil.getAllObjectsByPrefix(bucketName, targetDir, true); for (Item movedItem : movedItems) { - if(!movedItem.isDir() && movedItem.size() > 0){ // 检查是否为非目录且文件大小大于0 + if (!movedItem.isDir() && movedItem.size() > 0) { // 检查是否为非目录且文件大小大于0 Map result = new HashMap<>(); String url = movedItem.objectName(); String fileName = extractFileName(url); @@ -493,5 +510,254 @@ public class ModelsServiceImpl implements ModelsService { return urlStr.substring(urlStr.lastIndexOf('/') + 1); } + @Override + public CompletableFuture newCreateModel(ModelsVo modelsVo) throws Exception { + return CompletableFuture.supplyAsync(() -> { + try { + String token = gitService.login(modelsVo.getGitLinkUsername(), modelsVo.getGitLinkPassword()); + LoginUser loginUser = SecurityUtils.getLoginUser(); + String ci4sUsername = loginUser.getUsername(); + Jedis jedis = new Jedis(redisHost); + String userReq = jedis.get(ci4sUsername + "_gitUserInfo"); + Map userInfo = JsonUtils.jsonToMap(userReq); + Integer userId = (Integer) userInfo.get("user_id"); + + // 拼接project + String repositoryName = ci4sUsername + "_model_" + DateUtils.dateTimeNow(); + GitProjectVo gitProjectVo = new GitProjectVo(); + gitProjectVo.setRepositoryName(repositoryName); + gitProjectVo.setName(modelsVo.getName()); + gitProjectVo.setDescription(modelsVo.getDescription()); + gitProjectVo.setPrivate(modelsVo.getAvailableRange() == 0); + gitProjectVo.setUserId(userId); + + // 创建项目 + Map project = gitService.createProject(token, gitProjectVo); + + // 创建分支 + String branchName = modelsVo.getVersion(); + gitService.createBranch(token, (String) userInfo.get("login"), repositoryName, branchName, "master"); + // 定义标签 标签1：ci4s_model 标签2：ModelTag 标签3：ModelType + gitService.createTopic(token, (Integer) project.get("id"), "ci4s_model"); + gitService.createTopic(token, (Integer) project.get("id"), "ModelTag_" + modelsVo.getModelTag()); + gitService.createTopic(token, (Integer) project.get("id"), "ModelType_" + modelsVo.getModelType()); + // 得到项目地址 + String projectUrl = gitendpoint + "/" + (String) userInfo.get("login") + "/" + repositoryName + ".git"; + // 得到用户操作的路径 + String url = modelsVo.getModelsVersionVos().get(0).getUrl(); + String localPath1 = localPath + modelsVo.getName(); + String sourcePath = url.substring(0, url.lastIndexOf("/")); + // 命令行操作 git clone 项目地址 + DVCUtils.gitClone(localPath1, projectUrl, branchName, modelsVo.getGitLinkUsername(), modelsVo.getGitLinkPassword()); + String s3Path = "management-platform-files/" + ci4sUsername + "/models/" + repositoryName + "/" + branchName; + + // 拼接生产的元数据后写入yaml文件 + YamlUtils.generateYamlFile(JsonUtils.objectToMap(modelsVo), sourcePath, "model"); + + DVCUtils.moveFiles(sourcePath, localPath1 + "/model"); + + // dvc init 初始化 + DVCUtils.dvcInit(localPath1); + // 配置远程S3地址 + DVCUtils.dvcRemoteAdd(localPath1, s3Path); + DVCUtils.dvcConfigS3Credentials(localPath1, endpoint); + DVCUtils.dvcConfigS3Credentials2(localPath1, accessKeyId); + DVCUtils.dvcConfigS3Credentials3(localPath1, secretAccessKey); + // dvc 跟踪 + DVCUtils.dvcAdd(localPath1, "model"); + // git commit + DVCUtils.gitAdd(localPath1, "."); + DVCUtils.gitCommit(localPath1, "commit from ci4s with " + ci4sUsername); + DVCUtils.gitPush(localPath1, modelsVo.getGitLinkUsername(), modelsVo.getGitLinkPassword()); + // dvc push 到远程S3 + DVCUtils.dvcPush(localPath1); + return "新增模型成功"; + } catch (Exception e) { + logger.error(e.getMessage()); + throw new RuntimeException(e); + } + }); + } + + @Override + public CompletableFuture newCreateVersion(ModelsVo modelsVo) { + return CompletableFuture.supplyAsync(() -> { + try { + String token = gitService.login(modelsVo.getGitLinkUsername(), modelsVo.getGitLinkPassword()); + LoginUser loginUser = SecurityUtils.getLoginUser(); + String ci4sUsername = loginUser.getUsername(); + Jedis jedis = new Jedis(redisHost); + String userReq = jedis.get(ci4sUsername + "_gitUserInfo"); + Map userInfo = JsonUtils.jsonToMap(userReq); + // 创建分支 + String branchName = modelsVo.getVersion(); + String repositoryName = modelsVo.getRepositoryName(); + gitService.createBranch(token, (String) userInfo.get("login"), repositoryName, branchName, "master"); + // 得到项目地址 + String projectUrl = gitendpoint + "/" + (String) userInfo.get("login") + "/" + repositoryName + ".git"; + // 得到用户操作的路径 + String url = modelsVo.getModelVersionVos().get(0).getUrl(); + String localPath1 = localPath + ci4sUsername + "/model/" + modelsVo.getName(); + String sourcePath = url.substring(0, url.lastIndexOf("/")); + // 命令行操作 git clone 项目地址 + DVCUtils.gitClone(localPath1, projectUrl, branchName, modelsVo.getGitLinkUsername(), modelsVo.getGitLinkPassword()); + String s3Path = "management-platform-files/" + ci4sUsername + "/model/" + repositoryName + "/" + branchName; + //拼接生产的元数据后写入yaml文件 + YamlUtils.generateYamlFile(JsonUtils.objectToMap(modelsVo), sourcePath, "dataset"); + + DVCUtils.moveFiles(sourcePath, localPath1 + "/model"); + // dvc init 初始化 + DVCUtils.dvcInit(localPath1); + // 配置远程S3地址 + DVCUtils.dvcRemoteAdd(localPath1, s3Path); + DVCUtils.dvcConfigS3Credentials(localPath1, endpoint); + DVCUtils.dvcConfigS3Credentials2(localPath1, accessKeyId); + DVCUtils.dvcConfigS3Credentials3(localPath1, secretAccessKey); + // dvc 跟踪 + DVCUtils.dvcAdd(localPath1, "model"); + // git commit + DVCUtils.gitAdd(localPath1, "."); + DVCUtils.gitCommit(localPath1, "commit from ci4s with " + ci4sUsername); + DVCUtils.gitPush(localPath1, modelsVo.getGitLinkUsername(), modelsVo.getGitLinkPassword()); + // dvc push 到远程S3 + DVCUtils.dvcPush(localPath1); + return "新增模型成功"; + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + } + + @Override + public ResponseEntity downloadAllModelFilesNew(String repositoryName, String version, String gitLinkUsername, String gitLinkPassword) throws IOException, Exception { + // 命令行操作 git clone 项目地址 + LoginUser loginUser = SecurityUtils.getLoginUser(); + String ci4sUsername = loginUser.getUsername(); + String token = gitService.login(gitLinkUsername, gitLinkPassword); + Jedis jedis = new Jedis(redisHost); + String userReq = jedis.get(ci4sUsername + "_gitUserInfo"); + Map userInfo = JsonUtils.jsonToMap(userReq); + Integer userId = (Integer) userInfo.get("user_id"); + String projectUrl = gitendpoint + "/" + (String) userInfo.get("login") + "/" + repositoryName + ".git"; + String localPath1 = localPath + ci4sUsername + "/model/" + repositoryName; + File folder = new File(localPath1); + if (folder.exists() && folder.isDirectory()) { + //切换分支 + DVCUtils.gitCheckoutBranch(localPath1, version); + //pull + DVCUtils.gitPull(localPath1, gitLinkUsername, gitLinkPassword); + //dvc pull + DVCUtils.dvcPull(localPath1); + } else { + DVCUtils.gitClone(localPath1, projectUrl, version, gitLinkUsername, gitLinkPassword); + } + // 打包 data 文件夹 + String dataFolderPath = localPath1 + "/model"; + String zipFilePath = localPath1 + "/model.zip"; + try (FileOutputStream fos = new FileOutputStream(zipFilePath); + ZipOutputStream zos = new ZipOutputStream(fos)) { + Path sourcePath = Paths.get(dataFolderPath); + Files.walk(sourcePath).forEach(path -> { + if (!Files.isDirectory(path)) { + ZipEntry zipEntry = new ZipEntry(sourcePath.relativize(path).toString()); + try { + zos.putNextEntry(zipEntry); + Files.copy(path, zos); + zos.closeEntry(); + } catch (IOException e) { + throw new RuntimeException("Error while zipping: " + path, e); + } + } + }); + } + + // 返回压缩文件的输入流 + File zipFile = new File(zipFilePath); + InputStreamResource resource = new InputStreamResource(new FileInputStream(zipFile)); + + return ResponseEntity.ok() + .header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=data.zip") + .contentType(MediaType.APPLICATION_OCTET_STREAM) + .contentLength(zipFile.length()) + .body(resource); + } + + @Override + public Page newPubilcQueryByPage(ModelsVo modelsVo, PageRequest pageRequest) throws Exception { + LoginUser loginUser = SecurityUtils.getLoginUser(); + String ci4sUsername = loginUser.getUsername(); + String token = gitService.login(modelsVo.getGitLinkUsername(), modelsVo.getGitLinkPassword()); + Jedis jedis = new Jedis(redisHost); + String userReq = jedis.get(ci4sUsername + "_gitUserInfo"); + Map userInfo = JsonUtils.jsonToMap(userReq); + //拼接查询url + String modelTagName = modelsVo.getModelTag(); + String modelTypeName = modelsVo.getModelType(); + String topic_name = "ci4s_model"; + topic_name = StringUtils.isEmpty(modelTagName) ? topic_name : topic_name + ",modeltag_" + modelTypeName; + topic_name = StringUtils.isEmpty(modelTagName) ? topic_name : topic_name + ",modeltype_" + modelTypeName; + String url = gitendpoint + "/api/users/" + (String) userInfo.get("login") + "/projects.json?page=" + pageRequest.getPageNumber() + "&limit=" + pageRequest.getPageSize() + "&category=manage&topic_name=" + topic_name; + String req = HttpUtils.sendGetWithToken(url, null, token); + Map stringObjectMap = JacksonUtil.parseJSONStr2Map(req); + Integer total = (Integer) stringObjectMap.get("count"); + List> projects = (List>) stringObjectMap.get("projects"); + return new PageImpl<>(convert(projects), pageRequest, total); + } + + @Override + public Page newPersonalQueryByPage(ModelsVo modelsVo, PageRequest pageRequest) throws Exception { + LoginUser loginUser = SecurityUtils.getLoginUser(); + String ci4sUsername = loginUser.getUsername(); + String token = gitService.login(modelsVo.getGitLinkUsername(), modelsVo.getGitLinkPassword()); + Jedis jedis = new Jedis(redisHost); + String userReq = jedis.get(ci4sUsername + "_gitUserInfo"); + Map userInfo = JsonUtils.jsonToMap(userReq); + Integer userId = (Integer) userInfo.get("user_id"); + //拼接查询url + String modelTagName = modelsVo.getModelTag(); + String modelTypeName = modelsVo.getModelType(); + String topic_name = "ci4s_model"; + topic_name = StringUtils.isEmpty(modelTagName) ? topic_name : topic_name + ",modeltag_" + modelTagName; + topic_name = StringUtils.isEmpty(modelTypeName) ? topic_name : topic_name + ",modeltype_" + modelTypeName; + + String url = gitendpoint + "/api/projects.json?user_id=" + userId + "&page=" + pageRequest.getPageNumber() + "&limit=" + pageRequest.getPageSize() + "&sort_by=praises_count&topic_name=" + topic_name; + String req = HttpUtils.sendGetWithToken(url, null, token); + Map stringObjectMap = JacksonUtil.parseJSONStr2Map(req); + Integer total = (Integer) stringObjectMap.get("total_count"); + List> projects = (List>) stringObjectMap.get("projects"); + return new PageImpl<>(convert(projects), pageRequest, total); + } + + public List convert(List> lst) { + if (lst != null && lst.size() > 0) { + List newModelVos = ConvertUtil.convertListMapToObjectList(lst, ModelsVo.class); + + for (ModelsVo newModelVo : newModelVos) { + Map map = lst.stream() + .filter(m -> m.get("repo_id").equals(newModelVo.getRepoId())) + .findFirst() + .orElse(null); + + if (map != null) { + List> topics = (List>) map.get("topics"); + if (topics != null) { + topics.forEach(topic -> { + String name = (String) topic.get("name"); + if (name != null) { + if (name.startsWith("modeltag_")) { + newModelVo.setModelTag(name.substring("modeltag_".length())); + } else if (name.startsWith("modeltype_")) { + newModelVo.setModelType(name.substring("modeltype_".length())); + } + } + }); + } + } + } + + return newModelVos; + } + return new ArrayList<>(); + } } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelsVo.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelsVo.java index 315d60d4..1189da76 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelsVo.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelsVo.java @@ -2,13 +2,14 @@ package com.ruoyi.platform.vo; import com.fasterxml.jackson.databind.PropertyNamingStrategy; import com.fasterxml.jackson.databind.annotation.JsonNaming; -import com.ruoyi.platform.domain.ModelsVersion; import io.swagger.annotations.ApiModelProperty; +import lombok.Data; import java.io.Serializable; import java.util.List; @JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy.class) +@Data public class ModelsVo implements Serializable { @@ -25,7 +26,6 @@ public class ModelsVo implements Serializable { @ApiModelProperty(name = "available_range") private int availableRange; - // private String url; @ApiModelProperty(name = "model_type") private String modelType; @@ -39,80 +39,28 @@ public class ModelsVo implements Serializable { @ApiModelProperty(name = "version") private String version; + @ApiModelProperty(name = "model_version_vos") + private List modelVersionVos; + /** * 状态 */ @ApiModelProperty(name = "status") private Integer status; - @ApiModelProperty(name = "models_version_vos") - private List modelsVersionVos; - - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getDescription() { - return description; - } - - public void setDescription(String description) { - this.description = description; - } - - public int getAvailableRange() { - return availableRange; - } - - public void setAvailableRange(int availableRange) { - this.availableRange = availableRange; - } - + private String gitLinkUsername; - public String getModelType() { - return modelType; - } + private String gitLinkPassword; - public void setModelType(String modelType) { - this.modelType = modelType; - } - - public String getModelTag() { - return modelTag; - } - - public void setModelTag(String modelTag) { - this.modelTag = modelTag; - } - - public String getVersion() { - return version; - } - - public void setVersion(String version) { - this.version = version; - } - - public Integer getStatus() { - return status; - } - - public void setStatus(Integer status) { - this.status = status; - } - - public List getModelsVersionVos() { - return modelsVersionVos; - } - - public void setModelsVersionVos(List modelsVersionVos) { - this.modelsVersionVos = modelsVersionVos; - } + @ApiModelProperty(name = "models_version_vos") + private List modelsVersionVos; + /** + * 数据集仓库名称 + */ + @ApiModelProperty(name = "repository_name") + private String repositoryName; + @ApiModelProperty(name = "repo_id") + private Integer repoId; } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/NewDatasetVo.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/NewDatasetVo.java index 7ea72767..d8b5a2c5 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/NewDatasetVo.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/NewDatasetVo.java @@ -4,10 +4,8 @@ import com.fasterxml.jackson.databind.PropertyNamingStrategy; import com.fasterxml.jackson.databind.annotation.JsonNaming; import io.swagger.annotations.ApiModelProperty; import lombok.Data; -import org.apache.ibatis.annotations.MapKey; import java.io.Serializable; -import java.util.List; @JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy.class) @Data