package com.ruoyi.platform.service.impl;

import com.alibaba.fastjson2.JSON;
import com.ruoyi.common.core.utils.DateUtils;
import com.ruoyi.common.security.utils.SecurityUtils;
import com.ruoyi.platform.domain.*;
import com.ruoyi.platform.mapper.*;
import com.ruoyi.platform.service.DatasetTempStorageService;
import com.ruoyi.platform.service.GitService;
import com.ruoyi.platform.service.NewDatasetService;
import com.ruoyi.platform.utils.*;
import com.ruoyi.platform.vo.*;
import com.ruoyi.system.api.constant.Constant;
import com.ruoyi.system.api.model.LoginUser;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.io.InputStreamResource;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageImpl;
import org.springframework.data.domain.PageRequest;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;
import redis.clients.jedis.Jedis;

import javax.annotation.Resource;
import java.io.*;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;

import static com.ruoyi.common.security.utils.SecurityUtils.decrypt;

@Service
public class NewDatasetServiceImpl implements NewDatasetService {

    private static final Logger logger = LoggerFactory.getLogger(NewDatasetServiceImpl.class);

    @Resource
    private GitService gitService;
    @Resource
    private AssetWorkflowDao assetWorkflowDao;
    @Resource
    private MachineLearnDao machineLearnDao;
    @Resource
    private RayDao rayDao;
    @Resource
    private ActiveLearnDao activeLearnDao;
    @Resource
    private DevEnvironmentDao devEnvironmentDao;

    @Value("${spring.redis.host}")
    private String redisHost;
    @Value("${spring.redis.port}")
    private Integer redisPort;
    @Value("${minio.accessKey}")
    String accessKeyId;
    @Value("${minio.secretKey}")
    String secretAccessKey;
    @Value("${minio.endpoint}")
    String endpoint;
    @Value("${git.endpoint}")
    String gitendpoint;
    @Value("${git.cloneEndpoint}")
    String gitCloneEndpoint;
    @Value("${git.cloneEndpointIp}")
    String gitCloneEndpointIp;
    @Value("${git.projectDatasetId}")
    Integer projectDatasetId;

    @Value("${minio.dataReleaseBucketName}")
    private String bucketName;
    @Value("${git.localPath}")
    String localPathlocal;

    @Value("${labelStudio.loaclUrl}")
    private String labelloaclUrl;
    @Resource
    private NewHttpUtils httpUtils;
    @Resource
    private DatasetTempStorageService datasetTempStorageService;
    @Resource
    private DVCUtils dvcUtils;

    @Resource
    private MinioUtil minioUtil;

    @Override
    public String newCreateDataset(NewDatasetVo datasetVo) throws Exception {
        String token = gitService.checkoutToken();
        Jedis jedis = new Jedis(redisHost, redisPort);
        LoginUser loginUser = SecurityUtils.getLoginUser();
        String ci4sUsername = loginUser.getUsername();
        String gitLinkUsername = loginUser.getSysUser().getUserName();
        String gitLinkPassword = decrypt(loginUser.getSysUser().getOriginPassword());
        String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
        // 得到用户操作的路径
        Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
        Integer userId = (Integer) userInfo.get("user_id");
        // 拼接project
        ci4sUsername = Boolean.TRUE.equals(datasetVo.getIsPublic()) ? Constant.Item_Public : loginUser.getUsername();
        String repositoryName = ci4sUsername + "_dataset_" + DateUtils.dateTimeNow();
        GitProjectVo gitProjectVo = new GitProjectVo();
        gitProjectVo.setRepositoryName(repositoryName);
        gitProjectVo.setName(datasetVo.getName());
        gitProjectVo.setDescription(datasetVo.getDescription());
        gitProjectVo.setPrivate(!datasetVo.getIsPublic());
        gitProjectVo.setUserId(userId);
        gitProjectVo.setProjectCategoryId(projectDatasetId);
        // 创建项目
        Map project = gitService.createProject(token, gitProjectVo);
        Integer gitlinIid = (Integer) project.get("id");
        if (gitlinIid == null) {
            throw new Exception("创建数据集失败:" + project.get("message"));
        }
        // 创建分支
        String branchName = datasetVo.getVersion();
        String owner = (String) userInfo.get("login");

        gitService.createBranch(token, (String) userInfo.get("login"), repositoryName, branchName, "master");
        // 定义标签 标签1：dataset 标签2：DataTag 标签3：DataType
        gitService.createTopic(token, gitlinIid, Constant.Topic_Dataset);
        if (StringUtils.isNotEmpty(datasetVo.getDataTag())) {
            gitService.createTopic(token, gitlinIid, "datatag-" + datasetVo.getDataTag());
        }
        if (StringUtils.isNotEmpty(datasetVo.getDataType())) {
            gitService.createTopic(token, gitlinIid, "datatype-" + datasetVo.getDataType());
        }

        // 得到项目地址
        String projectUrl = gitCloneEndpoint + "/" + owner + "/" + repositoryName + ".git";

        // 得到用户操作的路径
        String sourcePath = datasetVo.getDatasetVersionVos().get(0).getUrl();
        String relatePath = ci4sUsername + "/datasets/" + gitlinIid + "/" + repositoryName + "/" + branchName;
        String localPath = localPathlocal + relatePath;
        String datasetPath = localPath + "/dataset";

        // 命令行操作 git clone 项目地址
        dvcUtils.gitClone(localPath, projectUrl, branchName, gitLinkUsername, gitLinkPassword);
        String s3Path = bucketName + "/mini-model-management-platform-files" + "/" + relatePath + "/" + branchName;
        //干掉目标文件夹
        dvcUtils.deleteDirectory(datasetPath);
        dvcUtils.moveFiles(sourcePath, datasetPath);
        // 拼接生产的元数据后写入yaml文件
        datasetVo.setCreateBy(String.valueOf(StringUtils.isNotEmpty((String) userInfo.get("nickname")) ? userInfo.get("nickname") : userInfo.get("login")));
        datasetVo.setUpdateTime(DateUtils.getTime());
        datasetVo.setUsage("<pre><code>" +
                "# 克隆数据集配置文件与存储参数到本地\n" +
                "git clone -b " + branchName + " " + projectUrl + "\n" +
                "# 远程拉取配置文件\n" +
                "dvc pull\n" +
                "</code></pre>");
        datasetVo.setIdentifier(repositoryName);
        datasetVo.setId(gitlinIid);
        datasetVo.setOwner(owner);
        datasetVo.setRelativePaths(relatePath + "/dataset");

        addDatasetSourceToDataVo(datasetVo);

        YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo), localPath, "dataset");

        CompletableFuture.supplyAsync(() -> {
            try {
                // dvc init 初始化
                dvcUtils.dvcInit(localPath);
                // 配置远程S3地址
                dvcUtils.dvcRemoteAdd(localPath, s3Path);
                dvcUtils.dvcConfigS3Credentials(localPath, endpoint);
                dvcUtils.dvcConfigS3Credentials2(localPath, accessKeyId);
                dvcUtils.dvcConfigS3Credentials3(localPath, secretAccessKey);
                // dvc 跟踪
                dvcUtils.dvcAdd(localPath, "dataset");
                // git commit
                dvcUtils.gitCommit(localPath, "commit from ci4s with " + loginUser.getUsername());
                dvcUtils.gitPush(localPath, gitLinkUsername, gitLinkPassword);
                // dvc push 到远程S3
                dvcUtils.dvcPush(localPath);
            } catch (Exception e) {
                logger.error(e.getMessage(), e);
                throw new RuntimeException(e.getMessage());
            }
            return null;
        });
        return "新增数据集成功";
    }

    public String newCreateVersion(NewDatasetVo datasetVo) throws Exception {
        List<Map<String, Object>> versionList = getVersionList(datasetVo.getIdentifier(), datasetVo.getOwner());
        for (Map<String, Object> versionMap : versionList) {
            String version = (String) versionMap.get("name");
            if (version.equals(datasetVo.getVersion())) {
                throw new Exception("数据集版本已存在，请勿重复创建");
            }
        }

        String token = gitService.checkoutToken();
        Jedis jedis = new Jedis(redisHost, redisPort);
        LoginUser loginUser = SecurityUtils.getLoginUser();
        String ci4sUsername = loginUser.getUsername();
        String gitLinkUsername = loginUser.getSysUser().getUserName();
        String gitLinkPassword = decrypt(loginUser.getSysUser().getOriginPassword());
        String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
        ci4sUsername = Boolean.TRUE.equals(datasetVo.getIsPublic()) ? Constant.Item_Public : loginUser.getUsername();
        Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
        // 创建分支
        String branchName = StringUtils.isEmpty(datasetVo.getVersion()) ? "master" : datasetVo.getVersion();
        String owner = (String) userInfo.get("login");

        String repositoryName = datasetVo.getIdentifier();
        String sourcePath = datasetVo.getDatasetVersionVos().get(0).getUrl();
        String relatePath = ci4sUsername + "/datasets/" + datasetVo.getId() + "/" + repositoryName + "/" + branchName;
        String localPath = localPathlocal + relatePath;
        String datasetPath = localPath + "/dataset";
        String projectUrl = gitCloneEndpoint + "/" + owner + "/" + repositoryName + ".git";

        //干掉目标文件夹
        dvcUtils.deleteDirectory(localPath);
        FileUtil.renameFile(localPath, localPath + "_deleted_" + System.currentTimeMillis());

        //查询项目信息
        Map projectDetail = gitService.getProjectDetail(owner, repositoryName, token);

        //克隆
        List<Map<String, Object>> brancheList = gitService.getBrancheList(token, owner, repositoryName);
        String oldBranch = (String) brancheList.stream()
                .filter(branch -> !"master".equals(branch.get("name"))).collect(Collectors.toList()).get(0).get("name");

        dvcUtils.gitClone(localPath, projectUrl, oldBranch, gitLinkUsername, gitLinkPassword);
        dvcUtils.refreshRemoteBranches(localPath, gitLinkUsername, gitLinkPassword, oldBranch);

        //检查是否存在本地重名分支，有的话干掉
        dvcUtils.deleteLocalBranch(localPath, branchName);
        // 创建本地分支
        dvcUtils.createLocalBranchBasedOnMaster(localPath, branchName);

        //dvc checkout
        dvcUtils.dvcPull(localPath);
        dvcUtils.dvcCheckout(localPath);

        // 准备数据
        String s3Path = bucketName + "/mini-model-management-platform-files" + "/" + relatePath + "/" + branchName;
        //干掉目标文件夹
        dvcUtils.deleteDirectory(datasetPath);

        if (Constant.Source_Hand_Export.equals(datasetVo.getDatasetSource())) {
            String bucketName = sourcePath.split("/")[0];
            String root_path = sourcePath.split("/")[1];
            minioUtil.downloadFiles(bucketName, root_path, datasetPath);
        } else {
            dvcUtils.moveFiles(sourcePath, datasetPath);
        }


        // 拼接生产的元数据后写入yaml文件
        datasetVo.setCreateBy(String.valueOf(StringUtils.isNotEmpty((String) userInfo.get("nickname")) ? userInfo.get("nickname") : userInfo.get("login")));
        datasetVo.setUpdateTime(DateUtils.getTime());
        String projectUrlIp = gitCloneEndpointIp + "/" + owner + "/" + repositoryName + ".git";
        datasetVo.setUsage("<pre><code>" +
                "# 克隆数据集配置文件与存储参数到本地\n" +
                "git clone -b " + branchName + " " + projectUrlIp + "\n" +
                "# 远程拉取配置文件\n" +
                "dvc pull\n" +
                "</code></pre>");
        datasetVo.setIdentifier(repositoryName);
        datasetVo.setOwner((String) userInfo.get("login"));

        datasetVo.setDescription((String) projectDetail.get("description"));
        List<Map> topics = (List<Map>) projectDetail.get("topics");
        if (topics != null && topics.size() > 0) {
            for (Map topic : topics) {
                String topicName = (String) topic.get("name");
                if (topicName.startsWith("datatag-")) {
                    datasetVo.setDataTag(topicName.substring("datatag-".length()));
                }
                if (topicName.startsWith("datatype-")) {
                    datasetVo.setDataType(topicName.substring("datatype-".length()));
                }
            }
        }
        datasetVo.setRelativePaths(relatePath + "/dataset");

        datasetVo = addDatasetSourceToDataVo(datasetVo);
        System.out.println(datasetVo);

        YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo), localPath, "dataset");
        //dvc数据跟踪
        // 配置远程S3地址
        dvcUtils.dvcRemoteAdd(localPath, s3Path);
        dvcUtils.dvcConfigS3Credentials(localPath, endpoint);
        dvcUtils.dvcConfigS3Credentials2(localPath, accessKeyId);
        dvcUtils.dvcConfigS3Credentials3(localPath, secretAccessKey);
        // dvc 跟踪
        dvcUtils.dvcAdd(localPath, "dataset");
        dvcUtils.pushNewBranchToRemote(localPath, gitLinkUsername, gitLinkPassword, branchName);

        dvcUtils.dvcPush(localPath);

//        CompletableFuture.supplyAsync(() -> {
//            try {
//                //dvc push 到远程S3
//                dvcUtils.dvcPush(localPath);
//            } catch (Exception e) {
//                throw new RuntimeException(e);
//            }
//            return null;
//        });
        return "新增数据集成功";
    }


    @Override
    public Page<NewDatasetVo> newPersonalQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception {
        String token = gitService.checkoutToken();
        Jedis jedis = new Jedis(redisHost, redisPort);
        LoginUser loginUser = SecurityUtils.getLoginUser();
        String ci4sUsername = loginUser.getUsername();
        String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
        Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
        String login = (String) userInfo.get("login");
        List<NewDatasetVo> collect = new ArrayList<>();
        for (int i = 1; ; i++) {
            List<Map<String, Object>> projects = PersonalQueryGitByPage(token, login, dataset.getName(), i);
            if (projects.isEmpty()) {
                break;
            } else {
                List<NewDatasetVo> collecti = convert(projects, Constant.Topic_Dataset, dataset.getDataTag(), dataset.getDataType());
                collect.addAll(collecti);
            }
        }
        List<NewDatasetVo> result = collect.stream().sorted(Comparator.comparing(NewDatasetVo::getPraisesCount).reversed()).skip((pageRequest.getPageNumber()) * pageRequest.getPageSize()).limit(pageRequest.getPageSize()).
                collect(Collectors.toList());
        return new PageImpl<>(result, pageRequest, collect.size());
    }

    List<Map<String, Object>> PersonalQueryGitByPage(String token, String login, String name, Integer page) throws Exception {
        // 拼接查询url
        String url = gitendpoint + "/api/users/" + login + "/projects.json?category=manage&sort_direction=praises_count&sort_by=desc";
        if (StringUtils.isNotEmpty(name)) {
            url = url + "&search=" + name;
        }
        String req = httpUtils.sendGetWithToken(url, null, token);
        Map<String, Object> stringObjectMap = JacksonUtil.parseJSONStr2Map(req);
        return (List<Map<String, Object>>) stringObjectMap.get("projects");
    }


    @Override
    public Page<NewDatasetVo> newPubilcQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception {
        String token = gitService.checkoutToken();

        List<NewDatasetVo> collect = new ArrayList<>();
        for (int i = 1; ; i++) {
            List<Map<String, Object>> projects = PubilcQueryGitByPage(token, dataset.getName(), i);
            if (projects.isEmpty()) {
                break;
            } else {
                List<NewDatasetVo> collecti = convert(projects, Constant.Topic_Dataset, dataset.getDataTag(), dataset.getDataType());
                collect.addAll(collecti);
            }
        }
        List<NewDatasetVo> result = collect.stream().sorted(Comparator.comparing(NewDatasetVo::getPraisesCount).reversed()).skip((long) pageRequest.getPageNumber() * pageRequest.getPageSize()).limit(pageRequest.getPageSize()).
                collect(Collectors.toList());
        return new PageImpl<>(result, pageRequest, collect.size());
    }

    List<Map<String, Object>> PubilcQueryGitByPage(String token, String name, Integer page) throws Exception {
        // 拼接查询url
        String url = gitendpoint + "/api/projects.json?page=" + page + "&limit=50sort_direction=praises_count&sort_by=desc&category_id=" + projectDatasetId;
        if (StringUtils.isNotEmpty(name)) {
            url = url + "&search=" + name;
        }
        String req = httpUtils.sendGetWithToken(url, null, token);
        Map<String, Object> stringObjectMap = JacksonUtil.parseJSONStr2Map(req);
        return (List<Map<String, Object>>) stringObjectMap.get("projects");
    }


    @Override
    public NewDatasetVo getNewDatasetDesc(Integer id, String name, String repo, String owner, String version, Boolean isPublic) throws Exception {
        LoginUser loginUser = SecurityUtils.getLoginUser();
        String ci4sUsername = isPublic ? Constant.Item_Public : loginUser.getUsername();
        // cd到 localPathlocal/id/下面还有一个文件夹，然后做git pull操作，然后读取里面的文件列表，列出每个文件的大小和名称，封装成MAP
        if (StringUtils.isEmpty(version)) {
            List<Map<String, Object>> versionList = this.getVersionList(repo, owner);
            if (versionList.size() == 0) {
                throw new Exception("数据集文件不存在");
            }
            version = (String) versionList.get(0).get("name");
        }
        List<Map<String, Object>> fileDetailsAfterGitPull = FileUtil.getFiles(localPathlocal + ci4sUsername + "/datasets/" + id + "/" + repo + "/" + version + "/dataset");

        // 在localPathlocal+id+"/"+repositoryName目录下的dataset.yaml中取到元数据
        Map<String, Object> stringObjectMap = YamlUtils.loadYamlFile(localPathlocal + ci4sUsername + "/datasets/" + id + "/" + repo + "/" + version + "/dataset.yaml");
        String jsonString = JacksonUtil.toJSONString(stringObjectMap);
        NewDatasetVo newDatasetVo = JsonUtils.jsonToObject(jsonString, NewDatasetVo.class);
        List<VersionVo> versionVos = new ArrayList<>();
        if (fileDetailsAfterGitPull != null && fileDetailsAfterGitPull.size() > 0) {
            for (Map<String, Object> fileDetail : fileDetailsAfterGitPull) {
                VersionVo versionVo = new VersionVo();
                versionVo.setUrl((String) fileDetail.get("filePath"));
                versionVo.setFileName((String) fileDetail.get("fileName"));
                long size = (long) fileDetail.get("size");
                versionVo.setFileSize(FileUtil.formatFileSize(size));
                versionVos.add(versionVo);
            }
        }
        newDatasetVo.setDatasetVersionVos(versionVos);

        String token = gitService.checkoutToken();
        String url = gitendpoint + "/api/" + owner + "/" + repo + "/detail.json";
        String req = httpUtils.sendGetWithToken(url, null, token);
        Map<String, Object> reqMap = JacksonUtil.parseJSONStr2Map(req);
        newDatasetVo.setPraisesCount((Integer) reqMap.get("praises_count"));
        newDatasetVo.setPraised((Boolean) reqMap.get("praised"));
        return newDatasetVo;
    }

    @Override
    public List<Map<String, Object>> getVersionList(String repo, String owner) throws Exception {
        String token = gitService.checkoutToken();
        List<Map<String, Object>> brancheList = gitService.getBrancheList(token, owner, repo);
        return brancheList.stream()
                .filter(branch -> !"master".equals(branch.get("name")))
                .collect(Collectors.toList());
    }

    @Override
    public void deleteDatasetNew(Integer repoId, String repo, String owner, Boolean isPublic) throws Exception {
        List<AssetWorkflow> assetWorkflow = assetWorkflowDao.getAssetWorkflow(Long.valueOf(repoId), Constant.Asset_Type_Dataset, null);
        if (assetWorkflow != null && !assetWorkflow.isEmpty()) {
            String workflows = String.join(",", assetWorkflow.stream().map(AssetWorkflow::getWorkflowName).collect(Collectors.toSet()));
            throw new Exception("该数据集被流水线：" + workflows + "使用，不能删除，请先删除流水线");
        }

        HashMap<String, String> map = new HashMap<>();
        map.put("id", String.valueOf(repoId));

        HashMap<Object, Object> mLQueryMap = new HashMap<>();
        mLQueryMap.put("dataset", map);
        List<MachineLearn> machineLearnList = machineLearnDao.queryByDatasetId(JSON.toJSONString(mLQueryMap));

        if (machineLearnList != null && !machineLearnList.isEmpty()) {
            String autoMls = String.join(",", machineLearnList.stream().map(MachineLearn::getName).collect(Collectors.toSet()));
            throw new Exception("该数据集被自动机器学习：" + autoMls + "使用，不能删除，请先删除自动机器学习");
        }

        List<Ray> rayList = rayDao.queryByDatasetId(JSON.toJSONString(map));
        if (rayList != null && !rayList.isEmpty()) {
            String rays = String.join(",", rayList.stream().map(Ray::getName).collect(Collectors.toSet()));
            throw new Exception("该数据集被超参数自动寻优：" + rays + "使用，不能删除，请先删除超参数自动寻优");
        }

        List<ActiveLearn> activeLearnList = activeLearnDao.queryByDatasetId(JSON.toJSONString(map));
        if (activeLearnList != null && !activeLearnList.isEmpty()) {
            String activeLearns = String.join(",", activeLearnList.stream().map(ActiveLearn::getName).collect(Collectors.toSet()));
            throw new Exception("该数据集被主动学习：" + activeLearns + "使用，不能删除，请先删除主动学习");
        }

        List<DevEnvironment> devEnvironmentList = devEnvironmentDao.queryByDatasetId(JSON.toJSONString(map));
        if (devEnvironmentList != null && !devEnvironmentList.isEmpty()) {
            String devEnvironments = String.join(",", devEnvironmentList.stream().map(DevEnvironment::getName).collect(Collectors.toSet()));
            throw new Exception("该数据集被开发环境：" + devEnvironments + "使用，不能删除，请先删除开发环境");
        }

        String token = gitService.checkoutToken();
        gitService.deleteProject(token, owner, repo);

        LoginUser loginUser = SecurityUtils.getLoginUser();
        String ci4sUsername = isPublic ? Constant.Item_Public : loginUser.getUsername();
        dvcUtils.deleteDirectory(localPathlocal + "/" + ci4sUsername + "/datasets/" + repoId + "/" + repo);
    }

    @Override
    public void deleteDatasetVersionNew(Integer repoId, String repo, String owner, String version, String relativePath) throws Exception {
        List<AssetWorkflow> assetWorkflow = assetWorkflowDao.getAssetWorkflow(Long.valueOf(repoId), Constant.Asset_Type_Dataset, version);
        if (assetWorkflow != null && !assetWorkflow.isEmpty()) {
            String workflows = String.join(",", assetWorkflow.stream().map(AssetWorkflow::getWorkflowName).collect(Collectors.toSet()));
            throw new Exception("该数据集版本被流水线：" + workflows + "使用，不能删除，请先删除流水线");
        }

        HashMap<String, String> map = new HashMap<>();
        map.put("id", String.valueOf(repoId));
        map.put("version", version);

        HashMap<Object, Object> mLQueryMap = new HashMap<>();
        mLQueryMap.put("dataset", map);
        List<MachineLearn> machineLearnList = machineLearnDao.queryByDatasetId(JSON.toJSONString(mLQueryMap));

        if (machineLearnList != null && !machineLearnList.isEmpty()) {
            String autoMls = String.join(",", machineLearnList.stream().map(MachineLearn::getName).collect(Collectors.toSet()));
            throw new Exception("该数据集版本被自动机器学习：" + autoMls + "使用，不能删除，请先删除自动机器学习");
        }

        List<Ray> rayList = rayDao.queryByDatasetId(JSON.toJSONString(map));
        if (rayList != null && !rayList.isEmpty()) {
            String rays = String.join(",", rayList.stream().map(Ray::getName).collect(Collectors.toSet()));
            throw new Exception("该数据集版本被超参数自动寻优：" + rays + "使用，不能删除，请先删除超参数自动寻优");
        }

        List<ActiveLearn> activeLearnList = activeLearnDao.queryByDatasetId(JSON.toJSONString(map));
        if (activeLearnList != null && !activeLearnList.isEmpty()) {
            String activeLearns = String.join(",", activeLearnList.stream().map(ActiveLearn::getName).collect(Collectors.toSet()));
            throw new Exception("该数据集版本被主动学习：" + activeLearns + "使用，不能删除，请先删除主动学习");
        }

        List<DevEnvironment> devEnvironmentList = devEnvironmentDao.queryByDatasetId(JSON.toJSONString(map));
        if (devEnvironmentList != null && !devEnvironmentList.isEmpty()) {
            String devEnvironments = String.join(",", devEnvironmentList.stream().map(DevEnvironment::getName).collect(Collectors.toSet()));
            throw new Exception("该数据集版本被开发环境：" + devEnvironments + "使用，不能删除，请先删除开发环境");
        }

        String token = gitService.checkoutToken();
        String rootPath = Paths.get(localPathlocal + "/" + relativePath).getParent().toString();
        gitService.deleteBranch(token, owner, repo, version, rootPath);
        dvcUtils.deleteDirectory(rootPath);
        FileUtil.renameFile(rootPath, rootPath + "_deleted_" + System.currentTimeMillis());
    }

    @Override
    public Map<String, Object> getVersionsCompare(QueryModelMetricsVo querydatasetVo) throws Exception {
        Map<String, Object> result = new HashMap<>();
        List<String> versions = querydatasetVo.getVersions();

        NewDatasetVo newDatasetDesc1 = getNewDatasetDesc(querydatasetVo.getRepoId(), null, querydatasetVo.getIdentifier(), querydatasetVo.getOwner(), versions.get(0), querydatasetVo.getIsPublic());
        NewDatasetVo newDatasetDesc2 = getNewDatasetDesc(querydatasetVo.getRepoId(), null, querydatasetVo.getIdentifier(), querydatasetVo.getOwner(), versions.get(1), querydatasetVo.getIsPublic());

        Map<String, String> compareMap = ConvertUtil.compareObjects(newDatasetDesc1, newDatasetDesc2);

        result.put("version1", newDatasetDesc1);
        result.put("version2", newDatasetDesc2);
        result.put("differences", compareMap);
        return result;
    }

    // 在你的方法中调用
    public String newCreateVersionFromLabelStudio(LabelDatasetVersionVo labelDatasetVersionVo) throws Exception {
        // 1. 下载数据集文件
        // 获取label-studio数据流
        Map<String, String> headers = new HashMap<>();
        headers.put("Authorization", "Token " + labelDatasetVersionVo.getToken());
        InputStream inputStream = HttpUtils.getIntputStream(labelloaclUrl + "/api/projects/" + labelDatasetVersionVo.getProjectId() + "/export?exportType=" + labelDatasetVersionVo.getExportType(), headers);

        // 2. 打包成zip包
//        String zipFileName = labelDatasetVersionVo.getName() + "_" + labelDatasetVersionVo.getVersion() + ".zip";
        String zipFileName = labelDatasetVersionVo.getName() + "_" + labelDatasetVersionVo.getVersion();
        MultipartFile[] files = FileUtil.toMultipartFiles(inputStream, zipFileName);

        // 3. 上传到minio
        List<Map<String, String>> maps = uploadDatasetlocal(files, UUID.randomUUID().toString());

        // 4. 组装DatasetVersionVo
        NewDatasetVo newDatasetVo = new NewDatasetVo();
        newDatasetVo.setId(labelDatasetVersionVo.getId());
        newDatasetVo.setIdentifier(labelDatasetVersionVo.getIdentifier());
        newDatasetVo.setIsPublic(labelDatasetVersionVo.getIsPublic());
        newDatasetVo.setOwner(labelDatasetVersionVo.getOwner());
        newDatasetVo.setName(labelDatasetVersionVo.getName());
        newDatasetVo.setVersion(labelDatasetVersionVo.getVersion());
        newDatasetVo.setDatasetSource(labelDatasetVersionVo.getDatasetSource());
        newDatasetVo.setVersionDesc(labelDatasetVersionVo.getVersionDesc());
        List<VersionVo> datasetVersionVos = new ArrayList<>();
        for (Map<String, String> map : maps) {
            VersionVo versionVo = new VersionVo();
            versionVo.setUrl(map.get("url"));
            versionVo.setFileName(map.get("fileName"));
            versionVo.setFileSize(map.get("fileSize"));
            datasetVersionVos.add(versionVo);
        }
        newDatasetVo.setDatasetVersionVos(datasetVersionVos);
        // 调用新增版本方法
        return newCreateVersion(newDatasetVo);
    }

    @Override
    public List<Map<String, String>> uploadDatasetlocal(MultipartFile[] files, String uuid) throws Exception {
        List<Map<String, String>> results = new ArrayList<>();

        for (MultipartFile file : files) {
            // 构建objectName
            String username = SecurityUtils.getLoginUser().getUsername();
            String fileName = file.getOriginalFilename();
            String path = localPathlocal + "temp" + "/" + username + "/" + "datasets" + "/" + uuid;
            long sizeInBytes = file.getSize();
            String formattedSize = FileUtil.formatFileSize(sizeInBytes);
            File targetFile = new File(path, file.getOriginalFilename());

            // 确保目录存在
            targetFile.getParentFile().mkdirs();
            // 保存文件到目标路径
            FileUtils.copyInputStreamToFile(file.getInputStream(), targetFile);
            // 返回上传文件的路径
            Map<String, String> result = new HashMap<>();
            result.put("fileName", fileName);
            result.put("url", path); // objectName根据实际情况定义
            result.put("fileSize", formattedSize);
            results.add(result);
        }
        return results;
    }

    @Override
    public ResponseEntity<InputStreamResource> downloadDatasetlocal(String filePath) throws Exception {
        File file = new File(filePath);

        if (!file.exists()) {
            throw new FileNotFoundException("File not found: " + filePath);
        }

        InputStreamResource resource = new InputStreamResource(new FileInputStream(file));

        HttpHeaders headers = new HttpHeaders();
        headers.add(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=" + file.getName());
        headers.add(HttpHeaders.CONTENT_LENGTH, String.valueOf(file.length()));
        headers.add(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_OCTET_STREAM_VALUE);

        return ResponseEntity.ok()
                .headers(headers)
                .contentLength(file.length())
                .contentType(MediaType.APPLICATION_OCTET_STREAM)
                .body(resource);
    }

    @Override
    public ResponseEntity<InputStreamResource> downloadAllDatasetFilesNew(String name, String identifier, Integer id, String version, Boolean isPublic) throws Exception {
        // 命令行操作 git clone 项目地址
        String ci4sUsername = isPublic ? Constant.Item_Public : SecurityUtils.getLoginUser().getUsername();
        String localPath = localPathlocal + ci4sUsername + "/datasets/" + id + "/" + identifier + "/" + version;

        // 打包 data 文件夹
        String dataFolderPath = localPath + "/dataset";
        String zipFilePath = localPath + "/data.zip";
        try (FileOutputStream fos = new FileOutputStream(zipFilePath);
             ZipOutputStream zos = new ZipOutputStream(fos)) {
            Path sourcePath = Paths.get(dataFolderPath);
            Files.walk(sourcePath).forEach(path -> {
                if (!Files.isDirectory(path)) {
                    ZipEntry zipEntry = new ZipEntry(sourcePath.relativize(path).toString());
                    try {
                        zos.putNextEntry(zipEntry);
                        Files.copy(path, zos);
                        zos.closeEntry();
                    } catch (IOException e) {
                        throw new RuntimeException("Error while zipping: " + path, e);
                    }
                }
            });
        }

        // 返回压缩文件的输入流
        File zipFile = new File(zipFilePath);
        InputStreamResource resource = new InputStreamResource(new FileInputStream(zipFile));

        return ResponseEntity.ok()
                .header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=" + name + "_" + version + "_dataset.zip")
                .contentType(MediaType.APPLICATION_OCTET_STREAM)
                .contentLength(zipFile.length())
                .body(resource);

    }

    @Override
    public List<NewDatasetVo> convert(List<Map<String, Object>> lst, String datasetTopic, String datasetTagName, String datasetTypeName) {
        if (lst != null && !lst.isEmpty()) {
            List<NewDatasetVo> result = new ArrayList<>();
            List<NewDatasetVo> newDatasetVos = ConvertUtil.convertListMapToObjectList(lst, NewDatasetVo.class);

            for (NewDatasetVo newDatasetVo : newDatasetVos) {
                Map<String, Object> map = lst.stream()
                        .filter(m -> m.get("id").equals(newDatasetVo.getId()))
                        .findFirst()
                        .orElse(null);

                if (map != null) {
                    List<Map<String, Object>> topics = (List<Map<String, Object>>) map.get("topics");
                    String datasetTopicName = null;
                    String datasetTag = null;
                    String datasetType = null;
                    for (Map<String, Object> topic : topics) {
                        String topicName = (String) topic.get("name");
                        if (datasetTopic.equals(topicName)) {
                            datasetTopicName = (String) topic.get("name");
                        }
                        if (topicName.startsWith("datatag-")) {
                            datasetTag = topicName.substring("datatag-".length());
                        }
                        if (topicName.startsWith("datatype-")) {
                            datasetType = topicName.substring("datatype-".length());
                        }
                    }

                    if (datasetTopic.equals(datasetTopicName)) {
                        if (StringUtils.isNotEmpty(datasetTagName) && !datasetTagName.toLowerCase().equals(datasetTag)) {
                            continue;
                        }
                        if (StringUtils.isNotEmpty(datasetTypeName) && !datasetTypeName.toLowerCase().equals(datasetType)) {
                            continue;
                        }
                        newDatasetVo.setDataTag(datasetTagName);
                        newDatasetVo.setDataType(datasetTypeName);

                        Map<String, Object> author = (Map<String, Object>) map.get("author");
                        newDatasetVo.setCreateBy((String) author.get("name"));
                        newDatasetVo.setOwner((String) author.get("login"));
                        String name = (String) map.get("name");
                        newDatasetVo.setName(name.substring((datasetTopic + "-").length()));
                        result.add(newDatasetVo);
                    }
                }
            }
            return result;
        }
        return new ArrayList<>();
    }

    private NewDatasetVo addDatasetSourceToDataVo(NewDatasetVo datasetVo) {
        DatasetTempStorage queryDatasetTempStorage = new DatasetTempStorage();
        queryDatasetTempStorage.setName(datasetVo.getIdentifier());
        queryDatasetTempStorage.setVersion(datasetVo.getVersion());
        DatasetTempStorage datasetTempStorage = datasetTempStorageService.queryByDatasetTempStorage(queryDatasetTempStorage);
        if (datasetTempStorage != null) {
            String source = datasetTempStorage.getSource();
            Map<String, Object> sourceMap = JacksonUtil.parseJSONStr2Map(source);
            Map<String, Object> preprocessCode = (Map<String, Object>) sourceMap.get("preprocess_code");
            sourceMap.remove("preprocess_code");
            datasetVo.setTrainTask((HashMap<String, Object>) sourceMap);
            datasetVo.setProcessingCode((HashMap<String, Object>) preprocessCode);
            datasetTempStorage.setState(Constant.State_invalid);
            datasetTempStorageService.update(datasetTempStorage);
        }
        return datasetVo;
    }

}
