diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/constant/Constant.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/constant/Constant.java index caaa5abe..8ac1173d 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/constant/Constant.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/constant/Constant.java @@ -20,4 +20,8 @@ public class Constant { public final static String Computing_Resource_GPU = "GPU"; // 计算资源_GPU public final static int Git_Category_Id = 39; + + public final static String Source_Export = "export"; + + public final static String Source_Add = "add"; } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java index d9763af2..913b02ec 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java @@ -712,9 +712,16 @@ public class ModelsServiceImpl implements ModelsService { dvcUtils.dvcCheckout(rootPath); //干掉目标文件夹 dvcUtils.deleteDirectory(modelPath); - if (modelsVo.getModelVersionVos().size() != 0) { + if (modelsVo.getModelVersionVos()!= null && modelsVo.getModelVersionVos().size() != 0) { String sourcePath = modelsVo.getModelVersionVos().get(0).getUrl(); - dvcUtils.moveFiles(sourcePath, modelPath); + + if (Constant.Source_Export.equals(modelsVo.getModelSource())) { + String bucketName = sourcePath.split("/")[0]; + String root_path = sourcePath.split("/")[1]; + minioUtil.downloadFiles(bucketName, root_path, modelPath); + } else { + dvcUtils.moveFiles(sourcePath, modelPath); + } } //拼接生产的元数据后写入yaml文件 diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java index cd2f20ef..534ff9c0 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java @@ -71,6 +71,9 @@ public class NewDatasetServiceImpl implements NewDatasetService { @Resource private DVCUtils dvcUtils; + @Resource + private MinioUtil minioUtil; + @Override public String newCreateDataset(NewDatasetVo datasetVo) throws Exception { String token = gitService.checkoutToken(); @@ -197,7 +200,15 @@ public class NewDatasetServiceImpl implements NewDatasetService { String s3Path = "management-platform-files" + "/" + relatePath + "/" + branchName; //干掉目标文件夹 dvcUtils.deleteDirectory(datasetPath); - dvcUtils.moveFiles(sourcePath, datasetPath); + + if (Constant.Source_Export.equals(datasetVo.getDatasetSource())) { + String bucketName = sourcePath.split("/")[0]; + String root_path = sourcePath.split("/")[1]; + minioUtil.downloadFiles(bucketName, root_path, datasetPath); + } else { + dvcUtils.moveFiles(sourcePath, datasetPath); + } + // 拼接生产的元数据后写入yaml文件 datasetVo.setCreateBy(String.valueOf(StringUtils.isNotEmpty((String) userInfo.get("nickname")) ? userInfo.get("nickname") : userInfo.get("login"))); datasetVo.setUpdateTime(DateUtils.getTime()); diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/ConvertUtil.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/ConvertUtil.java index e1b9f62f..14b859e5 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/ConvertUtil.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/ConvertUtil.java @@ -8,10 +8,7 @@ import org.springframework.beans.BeanUtils; import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; +import java.util.*; /** @@ -26,6 +23,7 @@ public class ConvertUtil { /** * 将实体对象转换为VO对象 + * * @param source 源实体对象 * @param target 目标VO对象的类 * @return 转换后的VO对象 @@ -87,7 +85,9 @@ public class ConvertUtil { Object value = entry.getValue(); String camelCaseKey = toCamelCase(key); if (hasProperty(targetClass, camelCaseKey)) { - PropertyUtils.setProperty(targetObject, camelCaseKey, value); + if (!(value instanceof LinkedHashMap)) { + PropertyUtils.setProperty(targetObject, camelCaseKey, value); + } } } return targetObject; diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/MinioUtil.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/MinioUtil.java index bc5b234b..834984df 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/MinioUtil.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/MinioUtil.java @@ -1,31 +1,38 @@ package com.ruoyi.platform.utils; +import com.ruoyi.common.security.utils.SecurityUtils; import io.minio.*; import io.minio.errors.MinioException; import io.minio.messages.DeleteObject; import io.minio.messages.Item; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import org.springframework.web.multipart.MultipartFile; import java.io.*; +import java.net.URLDecoder; import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.nio.file.Paths; import java.util.*; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; + @Slf4j @Component public class MinioUtil { private static MinioClient minioClient; + @Value("${git.localPath}") + String localPath; + @Autowired - public MinioUtil(@Value("${minio.endpoint}")String minioEndpoint,@Value("${minio.accessKey}")String minioAccessKey,@Value("${minio.secretKey}") String minioSecretKey) { + public MinioUtil(@Value("${minio.endpoint}") String minioEndpoint, @Value("${minio.accessKey}") String minioAccessKey, @Value("${minio.secretKey}") String minioSecretKey) { this.minioClient = MinioClient.builder() .endpoint(minioEndpoint) .credentials(minioAccessKey, minioSecretKey) @@ -192,7 +199,7 @@ public class MinioUtil { * @param srcObjectName 目标文件名称 */ public ObjectWriteResponse copyObject(String bucketName, String objectName, - String srcBucketName, String srcObjectName) throws Exception { + String srcBucketName, String srcObjectName) throws Exception { return minioClient.copyObject( CopyObjectArgs.builder() .source(CopySource.builder().bucket(bucketName).object(objectName).build()) @@ -202,14 +209,13 @@ public class MinioUtil { } - /** * 递归拷贝 * - * @param sourceBucketName 源bucket名称 - * @param sourceKeyPrefix 源目录路径 + * @param sourceBucketName 源bucket名称 + * @param sourceKeyPrefix 源目录路径 * @param targetBucketName 目标bucket名称 - * @param targetKeyPrefix 目标目录名称 + * @param targetKeyPrefix 目标目录名称 */ public void copyDirectory(String sourceBucketName, String sourceKeyPrefix, String targetBucketName, String targetKeyPrefix) throws Exception { @@ -238,9 +244,6 @@ public class MinioUtil { } - - - /** * 获取文件流 * @@ -267,6 +270,36 @@ public class MinioUtil { } + public void downloadFiles(String bucketName, String root_path, String targetPath) throws Exception { + ListObjectsArgs listObjectsArgs = ListObjectsArgs.builder().bucket(bucketName).prefix(root_path).build(); + findAndSaveFile(listObjectsArgs, bucketName, targetPath); + } + + private void findAndSaveFile(ListObjectsArgs listObjectsArgs, String bucketName, String targetPath) throws Exception { + // List all objects in the bucket + Iterable> results = minioClient.listObjects(listObjectsArgs); + for (Result result : results) { + Item item = result.get(); + if (item.isDir()) { + System.out.println("文件夹：" + URLDecoder.decode(item.objectName(), "UTF-8"));//输出日志 + ListObjectsArgs args = ListObjectsArgs.builder().bucket(bucketName).prefix(item.objectName()).build(); + findAndSaveFile(args, bucketName, targetPath); + } else { + GetObjectArgs getObjectArgs = GetObjectArgs.builder().bucket(bucketName).object(URLDecoder.decode(item.objectName())).build(); + String objectName = URLDecoder.decode(item.objectName(), "UTF-8"); + // Create a local file with the same name as the object + File file = new File(targetPath); + // Create parent directories if needed + file.getParentFile().mkdirs(); + // Get the object as an input stream + try (InputStream stream = minioClient.getObject(getObjectArgs)) { + // Copy the input stream to the file + FileUtils.copyInputStreamToFile(stream, file); + } + System.out.printf("文件：%s 下载成功！\n", URLDecoder.decode(item.objectName()));//输出日志 + } + } + } public List listFilesInDirectory(String bucketName, String prefix) throws Exception { List fileInfoList = new ArrayList<>(); @@ -322,8 +355,6 @@ public class MinioUtil { } - - public String readObjectAsString(String bucketName, String objectName) throws Exception { try (InputStream inputStream = minioClient.getObject( GetObjectArgs.builder() @@ -341,6 +372,7 @@ public class MinioUtil { return content.toString(); } } + /** * Downloads files and folders from the specified bucket and path, and creates a zip archive. * @@ -350,7 +382,7 @@ public class MinioUtil { * @throws MinioException If an error occurs while communicating with Minio. * @throws IOException If an I/O error occurs during zip creation. */ - public InputStream downloadAndZip(String bucketName, String path) throws Exception{ + public InputStream downloadAndZip(String bucketName, String path) throws Exception { try (ByteArrayOutputStream zipOutputStream = new ByteArrayOutputStream(); ZipOutputStream zip = new ZipOutputStream(zipOutputStream)) {