diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/dataset/DatasetVersionController.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/dataset/DatasetVersionController.java index 87715d5d..f6e7c6e5 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/dataset/DatasetVersionController.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/dataset/DatasetVersionController.java @@ -136,12 +136,12 @@ public class DatasetVersionController extends BaseController { return genericsSuccess(this.datasetVersionService.deleteDatasetVersion(datasetId, version)); } - @PostMapping("/addDatasetVersionsFromLabel") - @ApiOperation("从数据标注添加数据集版本") - public GenericsAjaxResult addDatasetVersionsFromLabel(@RequestBody LabelDatasetVersionVo labelDatasetVersionVo) throws Exception { - datasetVersionService.addDatasetVersionsFromLabel(labelDatasetVersionVo); - - return GenericsAjaxResult.success(); - } +// @PostMapping("/addDatasetVersionsFromLabel") +// @ApiOperation("从数据标注添加数据集版本") +// public GenericsAjaxResult addDatasetVersionsFromLabel(@RequestBody LabelDatasetVersionVo labelDatasetVersionVo) throws Exception { +// datasetVersionService.addDatasetVersionsFromLabel(labelDatasetVersionVo); +// +// return GenericsAjaxResult.success(); +// } } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/dataset/NewDatasetFromGitController.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/dataset/NewDatasetFromGitController.java index 3e28b850..f2ba22d1 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/dataset/NewDatasetFromGitController.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/dataset/NewDatasetFromGitController.java @@ -3,6 +3,7 @@ package com.ruoyi.platform.controller.dataset; import com.ruoyi.common.core.web.domain.AjaxResult; import com.ruoyi.platform.domain.Dataset; import com.ruoyi.platform.service.NewDatasetService; +import com.ruoyi.platform.vo.LabelDatasetVersionVo; import com.ruoyi.platform.vo.NewDatasetVo; import com.ruoyi.platform.vo.QueryModelMetricsVo; import io.swagger.annotations.ApiOperation; @@ -54,6 +55,19 @@ public class NewDatasetFromGitController { } + /** + * 新增数据集与版本新 + * + * @param datasetVo 实体 + * @return 新增结果 + */ + @PostMapping("/addVersionFromLabelStudio") + @ApiOperation("从labelsudio添加版本") + public AjaxResult addVersionFromLabelStudio(@RequestBody LabelDatasetVersionVo datasetVo) throws Exception { + return AjaxResult.success(this.newDatasetService.newCreateVersionFromLabelStudio(datasetVo)); + + } + @GetMapping("/queryDatasets") @ApiOperation("数据集广场公开数据集分页查询,根据data_type,data_tag筛选，true公开false私有") public AjaxResult queryDatasets(@RequestParam("page") int page, diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/DatasetVersionService.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/DatasetVersionService.java index 3ae08ca3..31ec8ba2 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/DatasetVersionService.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/DatasetVersionService.java @@ -72,6 +72,5 @@ public interface DatasetVersionService { String addDatasetVersions(List datasetVersions) throws Exception; - void - addDatasetVersionsFromLabel(LabelDatasetVersionVo labelDatasetVersionVo) throws Exception; +// void addDatasetVersionsFromLabel(LabelDatasetVersionVo labelDatasetVersionVo) throws Exception; } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/NewDatasetService.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/NewDatasetService.java index c4967dba..a1116105 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/NewDatasetService.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/NewDatasetService.java @@ -1,6 +1,7 @@ package com.ruoyi.platform.service; import com.ruoyi.platform.domain.Dataset; +import com.ruoyi.platform.vo.LabelDatasetVersionVo; import com.ruoyi.platform.vo.NewDatasetVo; import com.ruoyi.platform.vo.QueryModelMetricsVo; import org.springframework.core.io.InputStreamResource; @@ -39,4 +40,6 @@ public interface NewDatasetService { void deleteDatasetVersionNew(Integer repoId, String repo, String owner, String version, String relativePath) throws Exception; Map getVersionsCompare(QueryModelMetricsVo querydatasetVo) throws Exception; + + String newCreateVersionFromLabelStudio(LabelDatasetVersionVo datasetVo) throws Exception; } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/DatasetVersionServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/DatasetVersionServiceImpl.java index 4a00ddb4..61d61031 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/DatasetVersionServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/DatasetVersionServiceImpl.java @@ -217,33 +217,33 @@ public class DatasetVersionServiceImpl implements DatasetVersionService { } } - @Override - public void addDatasetVersionsFromLabel(LabelDatasetVersionVo labelDatasetVersionVo) throws Exception{ - Dataset dataset = datasetDao.queryById(labelDatasetVersionVo.getDataset_id()); - if (dataset == null){ - throw new Exception("数据集不存在"); - } - // 获取label-studio数据流 - Map headers = new HashMap(); - headers.put("Authorization","Token "+labelDatasetVersionVo.getToken()); - InputStream inputStream = HttpUtils.getIntputStream(labelloaclUrl+"/api/projects/"+labelDatasetVersionVo.getProject_id()+"/export?exportType="+labelDatasetVersionVo.getExportType(), headers); - // 构建objectName - String username = SecurityUtils.getLoginUser().getUsername(); - String url = username + "/" + DateUtils.dateTimeNow() + "/" + dataset.getName()+"_"+labelDatasetVersionVo.getVersion()+"."+labelDatasetVersionVo.getExportType(); - String objectName = "datasets/" + url; - String formattedSize = FileUtil.formatFileSize(inputStream.available()); - minioService.uploaInputStream(bucketName,objectName,inputStream); - //保存DatasetVersion - DatasetVersion datasetVersion = new DatasetVersion(); - datasetVersion.setVersion(labelDatasetVersionVo.getVersion()); - datasetVersion.setDatasetId(labelDatasetVersionVo.getDataset_id()); - datasetVersion.setFileName(dataset.getName()+"_"+labelDatasetVersionVo.getVersion()+"."+labelDatasetVersionVo.getExportType()); - - datasetVersion.setFileSize(formattedSize); - datasetVersion.setUrl(objectName); - datasetVersion.setDescription(labelDatasetVersionVo.getDesc()); - this.insert(datasetVersion); - } +// @Override +// public void addDatasetVersionsFromLabel(LabelDatasetVersionVo labelDatasetVersionVo) throws Exception{ +// Dataset dataset = datasetDao.queryById(labelDatasetVersionVo.getDataset_id()); +// if (dataset == null){ +// throw new Exception("数据集不存在"); +// } +// // 获取label-studio数据流 +// Map headers = new HashMap(); +// headers.put("Authorization","Token "+labelDatasetVersionVo.getToken()); +// InputStream inputStream = HttpUtils.getIntputStream(labelloaclUrl+"/api/projects/"+labelDatasetVersionVo.getProject_id()+"/export?exportType="+labelDatasetVersionVo.getExportType(), headers); +// // 构建objectName +// String username = SecurityUtils.getLoginUser().getUsername(); +// String url = username + "/" + DateUtils.dateTimeNow() + "/" + dataset.getName()+"_"+labelDatasetVersionVo.getVersion()+"."+labelDatasetVersionVo.getExportType(); +// String objectName = "datasets/" + url; +// String formattedSize = FileUtil.formatFileSize(inputStream.available()); +// minioService.uploaInputStream(bucketName,objectName,inputStream); +// //保存DatasetVersion +// DatasetVersion datasetVersion = new DatasetVersion(); +// datasetVersion.setVersion(labelDatasetVersionVo.getVersion()); +// datasetVersion.setDatasetId(labelDatasetVersionVo.getDataset_id()); +// datasetVersion.setFileName(dataset.getName()+"_"+labelDatasetVersionVo.getVersion()+"."+labelDatasetVersionVo.getExportType()); +// +// datasetVersion.setFileSize(formattedSize); +// datasetVersion.setUrl(objectName); +// datasetVersion.setDescription(labelDatasetVersionVo.getDesc()); +// this.insert(datasetVersion); +// } private void insertPrepare(DatasetVersion datasetVersion) throws Exception { checkDeclaredVersion(datasetVersion); diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java index 9e0fa753..705a9668 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java @@ -14,10 +14,7 @@ import com.ruoyi.platform.service.DatasetTempStorageService; import com.ruoyi.platform.service.GitService; import com.ruoyi.platform.service.NewDatasetService; import com.ruoyi.platform.utils.*; -import com.ruoyi.platform.vo.GitProjectVo; -import com.ruoyi.platform.vo.NewDatasetVo; -import com.ruoyi.platform.vo.QueryModelMetricsVo; -import com.ruoyi.platform.vo.VersionVo; +import com.ruoyi.platform.vo.*; import com.ruoyi.system.api.model.LoginUser; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; @@ -40,10 +37,7 @@ import java.io.*; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; import java.util.zip.ZipEntry; @@ -79,6 +73,9 @@ public class NewDatasetServiceImpl implements NewDatasetService { private String bucketName; @Value("${git.localPath}") String localPathlocal; + + @Value("${labelStudio.loaclUrl}") + private String labelloaclUrl; @Resource private NewHttpUtils httpUtils; @Resource @@ -472,6 +469,44 @@ public class NewDatasetServiceImpl implements NewDatasetService { return result; } + // 在你的方法中调用 + public String newCreateVersionFromLabelStudio(LabelDatasetVersionVo labelDatasetVersionVo) throws Exception { + // 1. 下载数据集文件 + // 获取label-studio数据流 + Map headers = new HashMap(); + headers.put("Authorization", "Token " + labelDatasetVersionVo.getToken()); + InputStream inputStream = HttpUtils.getIntputStream(labelloaclUrl + "/api/projects/" + labelDatasetVersionVo.getProjectId() + "/export?exportType=" + labelDatasetVersionVo.getExportType(), headers); + + // 2. 打包成zip包 + String zipFileName = labelDatasetVersionVo.getName()+"_"+labelDatasetVersionVo.getVersion() + ".zip"; + MultipartFile[] files = FileUtil.toMultipartFiles(inputStream, zipFileName); + + // 3. 上传到minio + List> maps = uploadDatasetlocal(files, UUID.randomUUID().toString()); + + // 4. 组装DatasetVersionVo + NewDatasetVo newDatasetVo = new NewDatasetVo(); + newDatasetVo.setId(labelDatasetVersionVo.getId()); + newDatasetVo.setIdentifier(labelDatasetVersionVo.getIdentifier()); + newDatasetVo.setIsPublic(labelDatasetVersionVo.getIsPublic()); + newDatasetVo.setOwner(labelDatasetVersionVo.getOwner()); + newDatasetVo.setName(labelDatasetVersionVo.getName()); + newDatasetVo.setVersion(labelDatasetVersionVo.getVersion()); + newDatasetVo.setDatasetSource(labelDatasetVersionVo.getDatasetSource()); + newDatasetVo.setVersionDesc(labelDatasetVersionVo.getVersionDesc()); + List datasetVersionVos = new ArrayList<>(); + for (Map map : maps) { + VersionVo versionVo = new VersionVo(); + versionVo.setUrl(map.get("url")); + versionVo.setFileName(map.get("fileName")); + versionVo.setFileSize(map.get("fileSize")); + datasetVersionVos.add(versionVo); + } + newDatasetVo.setDatasetVersionVos(datasetVersionVos); + // 调用新增版本方法 + return newCreateVersion(newDatasetVo); + } + @Override public List> uploadDatasetlocal(MultipartFile[] files, String uuid) throws Exception { List> results = new ArrayList<>(); diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/FileUtil.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/FileUtil.java index e31eed5a..25c3cfe5 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/FileUtil.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/FileUtil.java @@ -2,8 +2,10 @@ package com.ruoyi.platform.utils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.core.io.ByteArrayResource; +import org.springframework.web.multipart.MultipartFile; -import java.io.File; +import java.io.*; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -11,6 +13,8 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; public class FileUtil { @@ -111,4 +115,75 @@ public class FileUtil { // 重命名文件夹 boolean renamed = oldFolder.renameTo(newFolder); } + + public static MultipartFile[] toMultipartFiles(InputStream inputStream, String fileName) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + ZipOutputStream zos = new ZipOutputStream(baos); + ZipEntry zipEntry = new ZipEntry(fileName); + zos.putNextEntry(zipEntry); + + byte[] bytes = new byte[1024]; + int length; + while ((length = inputStream.read(bytes)) >= 0) { + zos.write(bytes, 0, length); + } + + zos.closeEntry(); + zos.close(); + inputStream.close(); + + ByteArrayResource resource = new ByteArrayResource(baos.toByteArray()); + return new MultipartFile[]{new CustomMultipartFile(resource, fileName)}; + } + + private static class CustomMultipartFile implements MultipartFile { + + private final ByteArrayResource resource; + private final String fileName; + + CustomMultipartFile(ByteArrayResource resource, String fileName) { + this.resource = resource; + this.fileName = fileName; + } + + @Override + public String getName() { + return fileName; + } + + @Override + public String getOriginalFilename() { + return fileName; + } + + @Override + public String getContentType() { + return "application/zip"; + } + + @Override + public boolean isEmpty() { + return resource.contentLength() == 0; + } + + @Override + public long getSize() { + return resource.contentLength(); + } + + @Override + public byte[] getBytes() throws IOException { + return resource.getByteArray(); + } + + @Override + public InputStream getInputStream() throws IOException { + return new ByteArrayInputStream(resource.getByteArray()); + } + + @Override + public void transferTo(File dest) throws IOException, IllegalStateException { + resource.getByteArray(); + } + } } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/LabelDatasetVersionVo.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/LabelDatasetVersionVo.java index 11a2e718..1511d043 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/LabelDatasetVersionVo.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/LabelDatasetVersionVo.java @@ -2,62 +2,23 @@ package com.ruoyi.platform.vo; import com.fasterxml.jackson.databind.PropertyNamingStrategy; import com.fasterxml.jackson.databind.annotation.JsonNaming; +import lombok.Data; import java.io.Serializable; @JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy.class) +@Data public class LabelDatasetVersionVo implements Serializable { private String token; - private String project_id; - private Integer dataset_id; - private String version; - private String desc; + private String projectId; private String exportType; - public String getToken() { - return token; - } - - public void setToken(String token) { - this.token = token; - } - - public String getProject_id() { - return project_id; - } - - public void setProject_id(String project_id) { - this.project_id = project_id; - } - - public Integer getDataset_id() { - return dataset_id; - } - - public void setDataset_id(Integer dataset_id) { - this.dataset_id = dataset_id; - } - public String getVersion() { - return version; - } - - public void setVersion(String version) { - this.version = version; - } - - public String getDesc() { - return desc; - } - - public void setDesc(String desc) { - this.desc = desc; - } - - public String getExportType() { - return exportType; - } - - public void setExportType(String exportType) { - this.exportType = exportType; - } + private Integer id; + private String identifier; + private Boolean isPublic; + private String owner; + private String name; + private String version; + private String versionDesc; + private String datasetSource; }