Merge branch 'dev' of code.gitlink.org.cn:ci4s/ci4sManagement-cloud into dev

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/dataset/DatasetController.java

@@ -194,6 +194,16 @@ public class DatasetController {
     }
 
 
+
+    @GetMapping("/exportDataset")
+    @ApiOperation(value = "导出数据集", notes = "将流水线产物导出到数据集。")
+    public AjaxResult exportDataset(@RequestParam("path") String path, @RequestParam("uuid") String uuid) throws Exception {
+        return AjaxResult.success(datasetService.exportDataset(path, uuid));
+    }
+
+
+
+
     /**
      * 从流水线上传数据集，不会给二进制文件，这边只存路径
      * @return 上传结果

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/DatasetService.java

@@ -82,4 +82,6 @@ DatasetService {
     public void checkDeclaredName(Dataset insert) throws Exception;
 
     ResponseEntity<InputStreamResource> downloadAllDatasetFiles(Integer datasetId, String version) throws Exception;
+
+    List<Map<String, String>> exportDataset(String path, String uuid) throws Exception;
 }

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/MinioService.java

@@ -11,4 +11,5 @@ public interface MinioService {
     ResponseEntity<InputStreamResource>  downloadZipFile(String bucketName , String path);
 
     Map<String, String> uploadFile(String bucketName, String objectName, MultipartFile file ) throws Exception;
+
 }

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/DatasetServiceImpl.java

@@ -9,12 +9,14 @@ import com.ruoyi.platform.mapper.DatasetDao;
 import com.ruoyi.platform.mapper.DatasetVersionDao;
 import com.ruoyi.platform.service.DatasetService;
 import com.ruoyi.platform.service.DatasetVersionService;
+import com.ruoyi.platform.service.MinioService;
 import com.ruoyi.platform.utils.BeansUtils;
 import com.ruoyi.platform.utils.FileUtil;
 import com.ruoyi.platform.utils.MinioUtil;
 import com.ruoyi.platform.vo.VersionVo;
 import com.ruoyi.platform.vo.DatasetVo;
 import com.ruoyi.system.api.model.LoginUser;
+import io.minio.messages.Item;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.core.io.InputStreamResource;
 import org.springframework.data.domain.Page;
@@ -56,6 +58,9 @@ public class DatasetServiceImpl implements DatasetService {
     @Resource
     private DatasetVersionService datasetVersionService;
 
+    @Resource
+    private MinioService minioService;
+
     // 固定存储桶名
     private final String bucketName = "platform-data";
 
@@ -204,44 +209,20 @@ public class DatasetServiceImpl implements DatasetService {
      * 上传数据集
      *
      * @param files 文件
-     * @param uuid
+     * @param uuid 唯一标识
      * @return 是否成功
      */
     @Override
     public List<Map<String, String>> uploadDataset(MultipartFile[] files, String uuid) throws Exception {
         List<Map<String, String>> results = new ArrayList<>();
-//        //时间戳统一定在外面，一次上传就定好
-//        Date createTime = new Date();
-//        String timestamp = new SimpleDateFormat("yyyyMMdd-HHmmss").format(createTime);
-
         for (MultipartFile file:files){
-            if (file.isEmpty()) {
-                throw new Exception("文件为空，无法上传");
-            }
-            // 获取文件大小并转换为可读形式
-            long sizeInBytes = file.getSize();
-            String formattedSize = FileUtil.formatFileSize(sizeInBytes); // 格式化文件大小
-
-            // 其余操作基于 modelsVersionToUse
+            // 构建objectName
             String username = SecurityUtils.getLoginUser().getUsername();
             String fileName = file.getOriginalFilename();
             String objectName = "datasets/" + username + "/" + uuid + "/" + fileName;
-
-            // 上传文件到MinIO并将记录新增到数据库中
-            try (InputStream inputStream = file.getInputStream()) {
-                minioUtil.uploadObject(bucketName, objectName, inputStream);
-
-                Map<String, String> fileResult = new HashMap<>();
-                fileResult.put("fileName", file.getOriginalFilename());
-                fileResult.put("url", objectName); // objectName根据实际情况定义
-                fileResult.put("fileSize", formattedSize);
-                results.add(fileResult);
-            } catch (Exception e) {
-                throw new Exception("上传数据集失败: " + e.getMessage(), e);
-            }
+            results.add(minioService.uploadFile(bucketName, objectName, file));
         }
         return results;
-
     }
 
 
@@ -286,19 +267,15 @@ public class DatasetServiceImpl implements DatasetService {
     public List<String> getDatasetVersions(Integer datasetId) throws Exception {
         // 获取所有相同模型ID的记录
         List<DatasetVersion> versions = datasetVersionDao.queryByDatasetId(datasetId);
-
         if (versions.isEmpty()) {
             throw new Exception("未找到相关数据集版本记录");
         }
-
         // 使用Stream API提取version字段，并去重
         return versions.stream()
                 .map(DatasetVersion::getVersion) // 提取每个DatasetVersion对象的version属性
                 .filter(version -> version != null && !version.isEmpty()) //忽略null或空字符串的version
                 .distinct() // 去重
                 .collect(Collectors.toList()); // 收集到List中
-
-
     }
 
 
@@ -308,6 +285,7 @@ public class DatasetServiceImpl implements DatasetService {
         List<VersionVo> datasetVersionVos = datasetVo.getDatasetVersionVos();
         if (datasetVersionVos==null || datasetVersionVos.isEmpty()){
             throw new Exception("数据集版本信息错误");
+
         }
 
         Dataset dataset = new Dataset();
@@ -334,7 +312,6 @@ public class DatasetServiceImpl implements DatasetService {
                 throw new Exception("新增数据集版本失败");
             }
         }
-
         return "新增数据集成功";
     }
 
@@ -396,7 +373,6 @@ public class DatasetServiceImpl implements DatasetService {
                     zos.closeEntry();
                 }
             }
-
             // 转换为输入流
             ByteArrayInputStream inputStream = new ByteArrayInputStream(baos.toByteArray());
             InputStreamResource resource = new InputStreamResource(inputStream);
@@ -412,4 +388,30 @@ public class DatasetServiceImpl implements DatasetService {
         }
     }
 
+    @Override
+    public List<Map<String, String>> exportDataset(String path, String uuid) throws Exception {
+        List<Map<String, String>> results = new ArrayList<>();
+        //根据path得到源文件所在桶名
+        String srcBucketName = path.substring(0, path.indexOf("/"));
+        //构建目标目录路径
+        String username = SecurityUtils.getLoginUser().getUsername();
+        String targetDir = "datasets/" + username + "/" + uuid;
+        // 递归拷贝整个原目录到目标目录下
+        minioUtil.copyDirectory(srcBucketName, path, bucketName, targetDir);
+        List<Item> movedItems = minioUtil.getAllObjectsByPrefix(bucketName, targetDir, true);
+        for (Item movedItem : movedItems) {
+            Map<String, String> result = new HashMap<>();
+            String url = movedItem.objectName();
+            String fileName = extractFileName(url);
+            // 获取文件大小并转换为可读形式
+            long sizeInBytes = movedItem.size();
+            String formattedSize = FileUtil.formatFileSize(sizeInBytes); // 格式化文件大小
+            result.put("fileName", fileName);
+            result.put("url", url); // objectName根据实际情况定义
+            result.put("fileSize", formattedSize);
+            results.add(result);
+        }
+        return results;
+    }
+
 }

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/MinioServiceImpl.java

@@ -50,7 +50,7 @@ public class MinioServiceImpl implements MinioService {
     }
 
     @Override
-    public Map<String, String> uploadFile(String bucketName, String objectName,MultipartFile file ) throws Exception {
+    public Map<String, String> uploadFile(String bucketName, String objectName, MultipartFile file ) throws Exception {
             if (file.isEmpty()) {
                 throw new Exception("文件为空，无法上传");
             }
@@ -65,7 +65,7 @@ public class MinioServiceImpl implements MinioService {
                 result.put("url", objectName); // objectName根据实际情况定义
                 result.put("fileSize", formattedSize);
             } catch (Exception e) {
-                throw new Exception("上传数据集失败: " + e.getMessage(), e);
+                throw new Exception("上传文件失败: " + e.getMessage(), e);
             }
         return result;
     }

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java

@@ -5,6 +5,7 @@ import com.ruoyi.platform.domain.Models;
 import com.ruoyi.platform.domain.ModelsVersion;
 import com.ruoyi.platform.mapper.ModelsDao;
 import com.ruoyi.platform.mapper.ModelsVersionDao;
+import com.ruoyi.platform.service.MinioService;
 import com.ruoyi.platform.service.ModelsService;
 import com.ruoyi.platform.service.ModelsVersionService;
 import com.ruoyi.platform.utils.BeansUtils;
@@ -52,6 +53,9 @@ public class ModelsServiceImpl implements ModelsService {
     @Resource
     private ModelsVersionService modelsVersionService;
 
+    @Resource
+    private MinioService minioService;
+
 
     // 固定存储桶名
     private final String bucketName = "platform-data";
@@ -194,50 +198,23 @@ public class ModelsServiceImpl implements ModelsService {
     }
 
     /**
-     * 上传模型
+     * 上传模型文件
      *
      * @param files 文件
-     * @param uuid
+     * @param uuid 唯一标识
      * @return 是否成功
      */
     @Override
     public List<Map<String, String>> uploadModels(MultipartFile[] files, String uuid) throws Exception {
 
         List<Map<String, String>> results = new ArrayList<>();
-//        //时间戳统一定在外面，一次上传就定好
-//        Date createTime = new Date();
-//        String timestamp = new SimpleDateFormat("yyyyMMdd-HHmmss").format(createTime);
-
-        for (MultipartFile file:files){
-            if (file.isEmpty()) {
-                throw new Exception("文件为空，无法上传");
-            }
-            // 获取文件大小并转换为KB
-            long sizeInBytes = file.getSize();
-            String formattedSize = FileUtil.formatFileSize(sizeInBytes); // 格式化文件大小
-
-
-
-
-            // 其余操作基于 modelsVersionToUse
+        for (MultipartFile file:files) {
+            // 构建objectName
             String username = SecurityUtils.getLoginUser().getUsername();
             String fileName = file.getOriginalFilename();
             String objectName = "models/" + username + "/" + uuid + "/" + fileName;
-
-            // 上传文件到MinIO并将记录新增到数据库中
-            try (InputStream inputStream = file.getInputStream()) {
-                minioUtil.uploadObject(bucketName, objectName, inputStream);
-//
-                Map<String, String> fileResult = new HashMap<>();
-                fileResult.put("fileName", file.getOriginalFilename());
-                fileResult.put("url", objectName); // objectName根据实际情况定义
-                fileResult.put("fileSize", formattedSize);
-                results.add(fileResult);
-            } catch (Exception e) {
-                throw new Exception("上传模型失败: " + e.getMessage(), e);
-            }
+            results.add(minioService.uploadFile(bucketName, objectName, file));
         }
-
         return results;
     }
 

ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/MinioUtil.java

@@ -3,27 +3,26 @@ package com.ruoyi.platform.utils;
 
 import io.minio.*;
 import io.minio.errors.MinioException;
+import io.minio.messages.DeleteObject;
 import io.minio.messages.Item;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
+import org.springframework.web.multipart.MultipartFile;
 
 import java.io.*;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
 import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipOutputStream;
 @Slf4j
 @Component
 public class MinioUtil {
 
-    private MinioClient minioClient;
+    private static MinioClient minioClient;
 
     @Autowired
     public MinioUtil(@Value("${minio.endpoint}")String minioEndpoint,@Value("${minio.accessKey}")String minioAccessKey,@Value("${minio.secretKey}") String minioSecretKey) {
@@ -43,6 +42,84 @@ public class MinioUtil {
         minioClient.removeBucket(RemoveBucketArgs.builder().bucket(bucketName).build());
     }
 
+    /**
+     * 验证bucketName是否存在
+     *
+     * @return boolean true:存在
+     */
+    public boolean bucketExists(String bucketName)
+            throws Exception {
+        return minioClient.bucketExists(BucketExistsArgs.builder().bucket(bucketName).build());
+    }
+
+    /**
+     * 判断文件是否存在
+     *
+     * @param bucketName 存储桶
+     * @param objectName 对象
+     * @return true：存在
+     */
+    public boolean doesObjectExist(String bucketName, String objectName) {
+        boolean exist = true;
+        try {
+            minioClient.statObject(StatObjectArgs.builder().bucket(bucketName).object(objectName).build());
+        } catch (Exception e) {
+            exist = false;
+        }
+        return exist;
+    }
+
+    /**
+     * 判断文件夹是否存在
+     *
+     * @param bucketName 存储桶
+     * @param objectName 文件夹名称（去掉/）
+     * @return true：存在
+     */
+    public boolean doesFolderExist(String bucketName, String objectName) {
+        boolean exist = false;
+        try {
+            Iterable<Result<Item>> results = minioClient.listObjects(
+                    ListObjectsArgs.builder().bucket(bucketName).prefix(objectName).recursive(false).build());
+            for (Result<Item> result : results) {
+                Item item = result.get();
+                if (item.isDir() && objectName.equals(item.objectName())) {
+                    exist = true;
+                }
+            }
+        } catch (Exception e) {
+            exist = false;
+        }
+        return exist;
+    }
+
+    /**
+     * 根据文件前置查询文件
+     *
+     * @param bucketName bucket名称
+     * @param prefix     前缀
+     * @param recursive  是否递归查询
+     * @return MinioItem 列表
+     */
+    public List<Item> getAllObjectsByPrefix(String bucketName, String prefix, boolean recursive) throws Exception {
+        List<Item> list = new ArrayList<>();
+        Iterable<Result<Item>> objectsIterator = minioClient.listObjects(
+                ListObjectsArgs.
+                        builder().
+                        bucket(bucketName).
+                        prefix(prefix).
+                        recursive(recursive).
+                        build());
+        if (objectsIterator != null) {
+            for (Result<Item> o : objectsIterator) {
+                Item item = o.get();
+                list.add(item);
+            }
+        }
+        return list;
+    }
+
+
     public void uploadObject(String bucketName, String objectName, InputStream stream) throws Exception {
         long size = stream.available();
         minioClient.putObject(
@@ -54,6 +131,36 @@ public class MinioUtil {
         );
     }
 
+    /**
+     * 通过MultipartFile，上传文件
+     *
+     * @param bucketName  存储桶
+     * @param file        文件
+     * @param objectName  对象名
+     * @param contentType 文件类型
+     */
+    public static ObjectWriteResponse putObject(String bucketName, MultipartFile file, String objectName, String contentType) throws Exception {
+        InputStream inputStream = file.getInputStream();
+        return minioClient.putObject(
+                PutObjectArgs.builder().bucket(bucketName).object(objectName).contentType(contentType)
+                        .stream(inputStream, inputStream.available(), -1).build());
+    }
+
+    /**
+     * 通过MultipartFile，上传文件
+     *
+     * @param bucketName 存储桶
+     * @param file       文件
+     * @param objectName 对象名
+     */
+    public static ObjectWriteResponse putObject(String bucketName, MultipartFile file, String objectName) throws Exception {
+        InputStream inputStream = file.getInputStream();
+        return minioClient.putObject(
+                PutObjectArgs.builder().bucket(bucketName).object(objectName)
+                        .stream(inputStream, inputStream.available(), -1).build());
+    }
+
+
     public void downloadObject(String bucketName, String objectName, OutputStream stream) throws Exception {
         try (InputStream inStream = minioClient.getObject(GetObjectArgs.builder().bucket(bucketName).object(objectName).build())) {
             byte[] buffer = new byte[1024];
@@ -64,6 +171,102 @@ public class MinioUtil {
         }
     }
 
+    /**
+     * 创建文件夹或目录
+     *
+     * @param bucketName 存储桶
+     * @param objectName 目录路径
+     */
+    public ObjectWriteResponse putDirObject(String bucketName, String objectName) throws Exception {
+        return minioClient.putObject(
+                PutObjectArgs.builder().bucket(bucketName).object(objectName).stream(
+                        new ByteArrayInputStream(new byte[]{}), 0, -1).build());
+    }
+
+    /**
+     * 拷贝文件
+     *
+     * @param bucketName    bucket名称
+     * @param objectName    文件名称
+     * @param srcBucketName 目标bucket名称
+     * @param srcObjectName 目标文件名称
+     */
+    public ObjectWriteResponse copyObject(String bucketName, String objectName,
+                                                 String srcBucketName, String srcObjectName) throws Exception {
+        return minioClient.copyObject(
+                CopyObjectArgs.builder()
+                        .source(CopySource.builder().bucket(bucketName).object(objectName).build())
+                        .bucket(srcBucketName)
+                        .object(srcObjectName)
+                        .build());
+    }
+
+
+
+    /**
+     * 递归拷贝
+     *
+     * @param sourceBucketName    源bucket名称
+     * @param sourceKeyPrefix    源目录路径
+     * @param targetBucketName 目标bucket名称
+     * @param targetKeyPrefix 目标目录名称
+     */
+    public void copyDirectory(String sourceBucketName, String sourceKeyPrefix,
+                              String targetBucketName, String targetKeyPrefix) throws Exception {
+        // 列出所有源目录下的对象
+        Iterable<Result<Item>> results = minioClient.listObjects(ListObjectsArgs.builder()
+                .bucket(sourceBucketName)
+                .prefix(sourceKeyPrefix)
+                .recursive(true)
+                .build());
+
+        for (Result<Item> result : results) {
+            Item item = result.get();
+            String sourceKey = item.objectName(); //文件的原始完整路径
+            String targetKey = targetKeyPrefix + sourceKey.substring(sourceKeyPrefix.length());
+
+            // 拷贝每个对象到目标路径
+            minioClient.copyObject(CopyObjectArgs.builder()
+                    .bucket(targetBucketName)
+                    .object(targetKey)
+                    .source(CopySource.builder()
+                            .bucket(sourceBucketName)
+                            .object(sourceKey)
+                            .build())
+                    .build());
+        }
+    }
+
+
+
+
+
+    /**
+     * 获取文件流
+     *
+     * @param bucketName bucket名称
+     * @param objectName 文件名称
+     * @return 二进制流
+     */
+    public static InputStream getObject(String bucketName, String objectName) throws Exception {
+        return minioClient.getObject(GetObjectArgs.builder().bucket(bucketName).object(objectName).build());
+    }
+
+    /**
+     * 断点下载
+     *
+     * @param bucketName bucket名称
+     * @param objectName 文件名称
+     * @param offset     起始字节的位置
+     * @param length     要读取的长度
+     * @return 流
+     */
+    public InputStream getObject(String bucketName, String objectName, long offset, long length)
+            throws Exception {
+        return minioClient.getObject(GetObjectArgs.builder().bucket(bucketName).object(objectName).offset(offset).length(length).build());
+    }
+
+
 
     public List<Map> listFilesInDirectory(String bucketName, String prefix) throws Exception {
         List<Map> fileInfoList = new ArrayList<>();
@@ -89,6 +292,37 @@ public class MinioUtil {
         return fileInfoList;
     }
 
+    /**
+     * 删除文件
+     *
+     * @param bucketName bucket名称
+     * @param objectName 文件名称
+     */
+    public static void removeObject(String bucketName, String objectName) throws Exception {
+        minioClient.removeObject(RemoveObjectArgs.builder().bucket(bucketName).object(objectName).build());
+    }
+
+
+    /**
+     * 批量删除文件
+     *
+     * @param bucketName bucket
+     * @param keys       需要删除的文件列表
+     */
+    public static void removeObjects(String bucketName, List<String> keys) {
+        List<DeleteObject> objects = new LinkedList<>();
+        keys.forEach(s -> {
+            objects.add(new DeleteObject(s));
+            try {
+                removeObject(bucketName, s);
+            } catch (Exception e) {
+                System.err.println("批量删除失败!");
+            }
+        });
+    }
+
+
+
 
     public String readObjectAsString(String bucketName, String objectName) throws Exception {
         try (InputStream inputStream = minioClient.getObject(