Merge branch 'dev' of https://gitlink.org.cn/ci4s/ci4sManagement-cloud into dev

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/dataset/DatasetVersionController.java

@@ -136,12 +136,12 @@ public class DatasetVersionController extends BaseController {
         return genericsSuccess(this.datasetVersionService.deleteDatasetVersion(datasetId, version));
     }
 
-    @PostMapping("/addDatasetVersionsFromLabel")
-    @ApiOperation("从数据标注添加数据集版本")
-    public GenericsAjaxResult<?> addDatasetVersionsFromLabel(@RequestBody LabelDatasetVersionVo labelDatasetVersionVo) throws Exception {
-        datasetVersionService.addDatasetVersionsFromLabel(labelDatasetVersionVo);
-
-        return GenericsAjaxResult.success();
-    }
+//    @PostMapping("/addDatasetVersionsFromLabel")
+//    @ApiOperation("从数据标注添加数据集版本")
+//    public GenericsAjaxResult<?> addDatasetVersionsFromLabel(@RequestBody LabelDatasetVersionVo labelDatasetVersionVo) throws Exception {
+//        datasetVersionService.addDatasetVersionsFromLabel(labelDatasetVersionVo);
+//
+//        return GenericsAjaxResult.success();
+//    }
 }
 

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/dataset/NewDatasetFromGitController.java

@@ -3,6 +3,7 @@ package com.ruoyi.platform.controller.dataset;
 import com.ruoyi.common.core.web.domain.AjaxResult;
 import com.ruoyi.platform.domain.Dataset;
 import com.ruoyi.platform.service.NewDatasetService;
+import com.ruoyi.platform.vo.LabelDatasetVersionVo;
 import com.ruoyi.platform.vo.NewDatasetVo;
 import com.ruoyi.platform.vo.QueryModelMetricsVo;
 import io.swagger.annotations.ApiOperation;
@@ -54,6 +55,19 @@ public class NewDatasetFromGitController {
 
     }
 
+    /**
+     * 新增数据集与版本新
+     *
+     * @param datasetVo 实体
+     * @return 新增结果
+     */
+    @PostMapping("/addVersionFromLabelStudio")
+    @ApiOperation("从labelsudio添加版本")
+    public AjaxResult addVersionFromLabelStudio(@RequestBody LabelDatasetVersionVo datasetVo) throws Exception {
+        return AjaxResult.success(this.newDatasetService.newCreateVersionFromLabelStudio(datasetVo));
+
+    }
+
     @GetMapping("/queryDatasets")
     @ApiOperation("数据集广场公开数据集分页查询,根据data_type,data_tag筛选，true公开false私有")
     public AjaxResult queryDatasets(@RequestParam("page") int page,

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/DatasetVersionService.java

@@ -72,6 +72,5 @@ public interface DatasetVersionService {
 
     String addDatasetVersions(List<DatasetVersion> datasetVersions) throws Exception;
 
-    void
-    addDatasetVersionsFromLabel(LabelDatasetVersionVo labelDatasetVersionVo) throws Exception;
+//    void addDatasetVersionsFromLabel(LabelDatasetVersionVo labelDatasetVersionVo) throws Exception;
 }

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/NewDatasetService.java

@@ -1,6 +1,7 @@
 package com.ruoyi.platform.service;
 
 import com.ruoyi.platform.domain.Dataset;
+import com.ruoyi.platform.vo.LabelDatasetVersionVo;
 import com.ruoyi.platform.vo.NewDatasetVo;
 import com.ruoyi.platform.vo.QueryModelMetricsVo;
 import org.springframework.core.io.InputStreamResource;
@@ -39,4 +40,6 @@ public interface NewDatasetService {
     void deleteDatasetVersionNew(Integer repoId, String repo, String owner, String version, String relativePath) throws Exception;
 
     Map<String, Object> getVersionsCompare(QueryModelMetricsVo querydatasetVo) throws Exception;
+
+    String newCreateVersionFromLabelStudio(LabelDatasetVersionVo datasetVo) throws Exception;
 }

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/DatasetVersionServiceImpl.java

@@ -217,33 +217,33 @@ public class DatasetVersionServiceImpl implements DatasetVersionService {
         }
     }
 
-    @Override
-    public void addDatasetVersionsFromLabel(LabelDatasetVersionVo labelDatasetVersionVo) throws Exception{
-        Dataset dataset = datasetDao.queryById(labelDatasetVersionVo.getDataset_id());
-        if (dataset == null){
-            throw new Exception("数据集不存在");
-        }
-        // 获取label-studio数据流
-        Map<String, String> headers = new HashMap<String, String>();
-        headers.put("Authorization","Token "+labelDatasetVersionVo.getToken());
-        InputStream inputStream = HttpUtils.getIntputStream(labelloaclUrl+"/api/projects/"+labelDatasetVersionVo.getProject_id()+"/export?exportType="+labelDatasetVersionVo.getExportType(), headers);
-        // 构建objectName
-        String username = SecurityUtils.getLoginUser().getUsername();
-        String url = username + "/" + DateUtils.dateTimeNow() + "/" + dataset.getName()+"_"+labelDatasetVersionVo.getVersion()+"."+labelDatasetVersionVo.getExportType();
-        String objectName = "datasets/" + url;
-        String formattedSize = FileUtil.formatFileSize(inputStream.available());
-        minioService.uploaInputStream(bucketName,objectName,inputStream);
-        //保存DatasetVersion
-        DatasetVersion datasetVersion = new DatasetVersion();
-        datasetVersion.setVersion(labelDatasetVersionVo.getVersion());
-        datasetVersion.setDatasetId(labelDatasetVersionVo.getDataset_id());
-        datasetVersion.setFileName(dataset.getName()+"_"+labelDatasetVersionVo.getVersion()+"."+labelDatasetVersionVo.getExportType());
-
-        datasetVersion.setFileSize(formattedSize);
-        datasetVersion.setUrl(objectName);
-        datasetVersion.setDescription(labelDatasetVersionVo.getDesc());
-        this.insert(datasetVersion);
-    }
+//    @Override
+//    public void addDatasetVersionsFromLabel(LabelDatasetVersionVo labelDatasetVersionVo) throws Exception{
+//        Dataset dataset = datasetDao.queryById(labelDatasetVersionVo.getDataset_id());
+//        if (dataset == null){
+//            throw new Exception("数据集不存在");
+//        }
+//        // 获取label-studio数据流
+//        Map<String, String> headers = new HashMap<String, String>();
+//        headers.put("Authorization","Token "+labelDatasetVersionVo.getToken());
+//        InputStream inputStream = HttpUtils.getIntputStream(labelloaclUrl+"/api/projects/"+labelDatasetVersionVo.getProject_id()+"/export?exportType="+labelDatasetVersionVo.getExportType(), headers);
+//        // 构建objectName
+//        String username = SecurityUtils.getLoginUser().getUsername();
+//        String url = username + "/" + DateUtils.dateTimeNow() + "/" + dataset.getName()+"_"+labelDatasetVersionVo.getVersion()+"."+labelDatasetVersionVo.getExportType();
+//        String objectName = "datasets/" + url;
+//        String formattedSize = FileUtil.formatFileSize(inputStream.available());
+//        minioService.uploaInputStream(bucketName,objectName,inputStream);
+//        //保存DatasetVersion
+//        DatasetVersion datasetVersion = new DatasetVersion();
+//        datasetVersion.setVersion(labelDatasetVersionVo.getVersion());
+//        datasetVersion.setDatasetId(labelDatasetVersionVo.getDataset_id());
+//        datasetVersion.setFileName(dataset.getName()+"_"+labelDatasetVersionVo.getVersion()+"."+labelDatasetVersionVo.getExportType());
+//
+//        datasetVersion.setFileSize(formattedSize);
+//        datasetVersion.setUrl(objectName);
+//        datasetVersion.setDescription(labelDatasetVersionVo.getDesc());
+//        this.insert(datasetVersion);
+//    }
 
     private void insertPrepare(DatasetVersion datasetVersion) throws Exception {
         checkDeclaredVersion(datasetVersion);

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java

@@ -14,10 +14,7 @@ import com.ruoyi.platform.service.DatasetTempStorageService;
 import com.ruoyi.platform.service.GitService;
 import com.ruoyi.platform.service.NewDatasetService;
 import com.ruoyi.platform.utils.*;
-import com.ruoyi.platform.vo.GitProjectVo;
-import com.ruoyi.platform.vo.NewDatasetVo;
-import com.ruoyi.platform.vo.QueryModelMetricsVo;
-import com.ruoyi.platform.vo.VersionVo;
+import com.ruoyi.platform.vo.*;
 import com.ruoyi.system.api.model.LoginUser;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang3.StringUtils;
@@ -40,10 +37,7 @@ import java.io.*;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 import java.util.concurrent.CompletableFuture;
 import java.util.stream.Collectors;
 import java.util.zip.ZipEntry;
@@ -79,6 +73,9 @@ public class NewDatasetServiceImpl implements NewDatasetService {
     private String bucketName;
     @Value("${git.localPath}")
     String localPathlocal;
+
+    @Value("${labelStudio.loaclUrl}")
+    private String labelloaclUrl;
     @Resource
     private NewHttpUtils httpUtils;
     @Resource
@@ -472,6 +469,44 @@ public class NewDatasetServiceImpl implements NewDatasetService {
         return result;
     }
 
+    // 在你的方法中调用
+    public String newCreateVersionFromLabelStudio(LabelDatasetVersionVo labelDatasetVersionVo) throws Exception {
+        // 1. 下载数据集文件
+        // 获取label-studio数据流
+        Map<String, String> headers = new HashMap<String, String>();
+        headers.put("Authorization", "Token " + labelDatasetVersionVo.getToken());
+        InputStream inputStream = HttpUtils.getIntputStream(labelloaclUrl + "/api/projects/" + labelDatasetVersionVo.getProjectId() + "/export?exportType=" + labelDatasetVersionVo.getExportType(), headers);
+
+        // 2. 打包成zip包
+        String zipFileName = labelDatasetVersionVo.getName()+"_"+labelDatasetVersionVo.getVersion() + ".zip";
+        MultipartFile[] files = FileUtil.toMultipartFiles(inputStream, zipFileName);
+
+        // 3. 上传到minio
+        List<Map<String, String>> maps = uploadDatasetlocal(files, UUID.randomUUID().toString());
+
+        // 4. 组装DatasetVersionVo
+        NewDatasetVo newDatasetVo = new NewDatasetVo();
+        newDatasetVo.setId(labelDatasetVersionVo.getId());
+        newDatasetVo.setIdentifier(labelDatasetVersionVo.getIdentifier());
+        newDatasetVo.setIsPublic(labelDatasetVersionVo.getIsPublic());
+        newDatasetVo.setOwner(labelDatasetVersionVo.getOwner());
+        newDatasetVo.setName(labelDatasetVersionVo.getName());
+        newDatasetVo.setVersion(labelDatasetVersionVo.getVersion());
+        newDatasetVo.setDatasetSource(labelDatasetVersionVo.getDatasetSource());
+        newDatasetVo.setVersionDesc(labelDatasetVersionVo.getVersionDesc());
+        List<VersionVo> datasetVersionVos = new ArrayList<>();
+        for (Map<String, String> map : maps) {
+            VersionVo versionVo = new VersionVo();
+            versionVo.setUrl(map.get("url"));
+            versionVo.setFileName(map.get("fileName"));
+            versionVo.setFileSize(map.get("fileSize"));
+            datasetVersionVos.add(versionVo);
+        }
+        newDatasetVo.setDatasetVersionVos(datasetVersionVos);
+        // 调用新增版本方法
+        return newCreateVersion(newDatasetVo);
+    }
+
     @Override
     public List<Map<String, String>> uploadDatasetlocal(MultipartFile[] files, String uuid) throws Exception {
         List<Map<String, String>> results = new ArrayList<>();

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/FileUtil.java

@@ -2,8 +2,10 @@ package com.ruoyi.platform.utils;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.core.io.ByteArrayResource;
+import org.springframework.web.multipart.MultipartFile;
 
-import java.io.File;
+import java.io.*;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -11,6 +13,8 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
 
 public class FileUtil {
 
@@ -111,4 +115,75 @@ public class FileUtil {
         // 重命名文件夹
         boolean renamed = oldFolder.renameTo(newFolder);
     }
+
+    public static MultipartFile[] toMultipartFiles(InputStream inputStream, String fileName) throws IOException {
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        ZipOutputStream zos = new ZipOutputStream(baos);
+        ZipEntry zipEntry = new ZipEntry(fileName);
+        zos.putNextEntry(zipEntry);
+
+        byte[] bytes = new byte[1024];
+        int length;
+        while ((length = inputStream.read(bytes)) >= 0) {
+            zos.write(bytes, 0, length);
+        }
+
+        zos.closeEntry();
+        zos.close();
+        inputStream.close();
+
+        ByteArrayResource resource = new ByteArrayResource(baos.toByteArray());
+        return new MultipartFile[]{new CustomMultipartFile(resource, fileName)};
+    }
+
+    private static class CustomMultipartFile implements MultipartFile {
+
+        private final ByteArrayResource resource;
+        private final String fileName;
+
+        CustomMultipartFile(ByteArrayResource resource, String fileName) {
+            this.resource = resource;
+            this.fileName = fileName;
+        }
+
+        @Override
+        public String getName() {
+            return fileName;
+        }
+
+        @Override
+        public String getOriginalFilename() {
+            return fileName;
+        }
+
+        @Override
+        public String getContentType() {
+            return "application/zip";
+        }
+
+        @Override
+        public boolean isEmpty() {
+            return resource.contentLength() == 0;
+        }
+
+        @Override
+        public long getSize() {
+            return resource.contentLength();
+        }
+
+        @Override
+        public byte[] getBytes() throws IOException {
+            return resource.getByteArray();
+        }
+
+        @Override
+        public InputStream getInputStream() throws IOException {
+            return new ByteArrayInputStream(resource.getByteArray());
+        }
+
+        @Override
+        public void transferTo(File dest) throws IOException, IllegalStateException {
+            resource.getByteArray();
+        }
+    }
 }

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/LabelDatasetVersionVo.java

@@ -2,62 +2,23 @@ package com.ruoyi.platform.vo;
 
 import com.fasterxml.jackson.databind.PropertyNamingStrategy;
 import com.fasterxml.jackson.databind.annotation.JsonNaming;
+import lombok.Data;
 
 import java.io.Serializable;
 
 @JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy.class)
+@Data
 public class LabelDatasetVersionVo implements Serializable {
     private String token;
-    private String project_id;
-    private Integer dataset_id;
-    private String version;
-    private String desc;
+    private String projectId;
     private String exportType;
-    public String getToken() {
-        return token;
-    }
-
-    public void setToken(String token) {
-        this.token = token;
-    }
-
-    public String getProject_id() {
-        return project_id;
-    }
-
-    public void setProject_id(String project_id) {
-        this.project_id = project_id;
-    }
-
-    public Integer getDataset_id() {
-        return dataset_id;
-    }
-
-    public void setDataset_id(Integer dataset_id) {
-        this.dataset_id = dataset_id;
-    }
 
-    public String getVersion() {
-        return version;
-    }
-
-    public void setVersion(String version) {
-        this.version = version;
-    }
-
-    public String getDesc() {
-        return desc;
-    }
-
-    public void setDesc(String desc) {
-        this.desc = desc;
-    }
-
-    public String getExportType() {
-        return exportType;
-    }
-
-    public void setExportType(String exportType) {
-        this.exportType = exportType;
-    }
+    private Integer id;
+    private String identifier;
+    private Boolean isPublic;
+    private String owner;
+    private String name;
+    private String version;
+    private String versionDesc;
+    private String datasetSource;
 }