From 5752c50bf29626174677174588154a0de28327dd Mon Sep 17 00:00:00 2001 From: fanshuai <1141904845@qq.com> Date: Thu, 12 Sep 2024 14:41:05 +0800 Subject: [PATCH] =?UTF-8?q?=E9=80=82=E9=85=8D=E6=B5=81=E6=B0=B4=E7=BA=BF?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dataset/DatasetTempStorageController.java | 84 ++++++++ .../platform/domain/DatasetTempStorage.java | 109 ++++++++++ .../mapper/DatasetTempStorageDao.java | 84 ++++++++ .../service/DatasetTempStorageService.java | 57 +++++ .../impl/DatasetTempStorageServiceImpl.java | 87 ++++++++ .../service/impl/ExperimentServiceImpl.java | 49 ++++- .../service/impl/NewDatasetServiceImpl.java | 29 ++- .../DatasetTempStorageDao.xml | 197 ++++++++++++++++++ 8 files changed, 691 insertions(+), 5 deletions(-) create mode 100644 ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/dataset/DatasetTempStorageController.java create mode 100644 ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/DatasetTempStorage.java create mode 100644 ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/DatasetTempStorageDao.java create mode 100644 ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/DatasetTempStorageService.java create mode 100644 ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/DatasetTempStorageServiceImpl.java create mode 100644 ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/DatasetTempStorageDao.xml diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/dataset/DatasetTempStorageController.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/dataset/DatasetTempStorageController.java new file mode 100644 index 00000000..0b1418da --- /dev/null +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/dataset/DatasetTempStorageController.java @@ -0,0 +1,84 @@ +package com.ruoyi.platform.controller.dataset; + +import com.ruoyi.platform.domain.DatasetTempStorage; +import com.ruoyi.platform.service.DatasetTempStorageService; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageRequest; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.*; + +import javax.annotation.Resource; + +/** + * 数据集暂存的数据(DatasetTempStorage)表控制层 + * + * @author makejava + * @since 2024-09-12 09:42:09 + */ +@RestController +@RequestMapping("datasetTempStorage") +public class DatasetTempStorageController { + /** + * 服务对象 + */ + @Resource + private DatasetTempStorageService datasetTempStorageService; + + /** + * 分页查询 + * + * @param datasetTempStorage 筛选条件 + * @param pageRequest 分页对象 + * @return 查询结果 + */ + @GetMapping + public ResponseEntity> queryByPage(DatasetTempStorage datasetTempStorage, PageRequest pageRequest) { + return ResponseEntity.ok(this.datasetTempStorageService.queryByPage(datasetTempStorage, pageRequest)); + } + + /** + * 通过主键查询单条数据 + * + * @param id 主键 + * @return 单条数据 + */ + @GetMapping("{id}") + public ResponseEntity queryById(@PathVariable("id") Integer id) { + return ResponseEntity.ok(this.datasetTempStorageService.queryById(id)); + } + + /** + * 新增数据 + * + * @param datasetTempStorage 实体 + * @return 新增结果 + */ + @PostMapping + public ResponseEntity add(DatasetTempStorage datasetTempStorage) { + return ResponseEntity.ok(this.datasetTempStorageService.insert(datasetTempStorage)); + } + + /** + * 编辑数据 + * + * @param datasetTempStorage 实体 + * @return 编辑结果 + */ + @PutMapping + public ResponseEntity edit(DatasetTempStorage datasetTempStorage) { + return ResponseEntity.ok(this.datasetTempStorageService.update(datasetTempStorage)); + } + + /** + * 删除数据 + * + * @param id 主键 + * @return 删除是否成功 + */ + @DeleteMapping + public ResponseEntity deleteById(Integer id) { + return ResponseEntity.ok(this.datasetTempStorageService.deleteById(id)); + } + +} + diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/DatasetTempStorage.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/DatasetTempStorage.java new file mode 100644 index 00000000..6b7255da --- /dev/null +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/DatasetTempStorage.java @@ -0,0 +1,109 @@ +package com.ruoyi.platform.domain; + +import java.util.Date; +import java.io.Serializable; + +/** + * 数据集暂存的数据(DatasetTempStorage)实体类 + * + * @author makejava + * @since 2024-09-12 09:42:09 + */ +public class DatasetTempStorage implements Serializable { + private static final long serialVersionUID = -44025176874790480L; + + private Integer id; + + private String name; + + private String version; + + private String source; + /** + * 为0失效，1生效 + */ + private Integer state; + + private String createBy; + + private Date createTime; + + private String updateBy; + + private Date updateTime; + + + public Integer getId() { + return id; + } + + public void setId(Integer id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public String getSource() { + return source; + } + + public void setSource(String source) { + this.source = source; + } + + public Integer getState() { + return state; + } + + public void setState(Integer state) { + this.state = state; + } + + public String getCreateBy() { + return createBy; + } + + public void setCreateBy(String createBy) { + this.createBy = createBy; + } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + public String getUpdateBy() { + return updateBy; + } + + public void setUpdateBy(String updateBy) { + this.updateBy = updateBy; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } + +} + diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/DatasetTempStorageDao.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/DatasetTempStorageDao.java new file mode 100644 index 00000000..8b747405 --- /dev/null +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/DatasetTempStorageDao.java @@ -0,0 +1,84 @@ +package com.ruoyi.platform.mapper; +import com.ruoyi.platform.domain.DatasetTempStorage; +import org.apache.ibatis.annotations.Param; +import org.springframework.data.domain.Pageable; + +import java.util.List; + +/** + * 数据集暂存的数据(DatasetTempStorage)表数据库访问层 + * + * @author makejava + * @since 2024-09-12 09:42:09 + */ +public interface DatasetTempStorageDao { + + /** + * 通过ID查询单条数据 + * + * @param id 主键 + * @return 实例对象 + */ + DatasetTempStorage queryById(Integer id); + + /** + * 查询指定行数据 + * + * @param datasetTempStorage 查询条件 + * @param pageable 分页对象 + * @return 对象列表 + */ + List queryAllByLimit(DatasetTempStorage datasetTempStorage, @Param("pageable") Pageable pageable); + + /** + * 统计总行数 + * + * @param datasetTempStorage 查询条件 + * @return 总行数 + */ + long count(DatasetTempStorage datasetTempStorage); + + /** + * 新增数据 + * + * @param datasetTempStorage 实例对象 + * @return 影响行数 + */ + int insert(DatasetTempStorage datasetTempStorage); + + /** + * 批量新增数据（MyBatis原生foreach方法） + * + * @param entities List 实例对象列表 + * @return 影响行数 + */ + int insertBatch(@Param("entities") List entities); + + /** + * 批量新增或按主键更新数据（MyBatis原生foreach方法） + * + * @param entities List 实例对象列表 + * @return 影响行数 + * @throws org.springframework.jdbc.BadSqlGrammarException 入参是空List的时候会抛SQL语句错误的异常，请自行校验入参 + */ + int insertOrUpdateBatch(@Param("entities") List entities); + + /** + * 修改数据 + * + * @param datasetTempStorage 实例对象 + * @return 影响行数 + */ + int update(DatasetTempStorage datasetTempStorage); + + /** + * 通过主键删除数据 + * + * @param id 主键 + * @return 影响行数 + */ + int deleteById(Integer id); + + DatasetTempStorage queryByDatasetTempStorage(DatasetTempStorage datasetTempStorage); +} + diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/DatasetTempStorageService.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/DatasetTempStorageService.java new file mode 100644 index 00000000..c5606208 --- /dev/null +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/DatasetTempStorageService.java @@ -0,0 +1,57 @@ +package com.ruoyi.platform.service; + +import com.ruoyi.platform.domain.DatasetTempStorage; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageRequest; + +/** + * 数据集暂存的数据(DatasetTempStorage)表服务接口 + * + * @author makejava + * @since 2024-09-12 09:42:10 + */ +public interface DatasetTempStorageService { + + /** + * 通过ID查询单条数据 + * + * @param id 主键 + * @return 实例对象 + */ + DatasetTempStorage queryById(Integer id); + + /** + * 分页查询 + * + * @param datasetTempStorage 筛选条件 + * @param pageRequest 分页对象 + * @return 查询结果 + */ + Page queryByPage(DatasetTempStorage datasetTempStorage, PageRequest pageRequest); + + DatasetTempStorage queryByDatasetTempStorage(DatasetTempStorage datasetTempStorage); + /** + * 新增数据 + * + * @param datasetTempStorage 实例对象 + * @return 实例对象 + */ + DatasetTempStorage insert(DatasetTempStorage datasetTempStorage); + + /** + * 修改数据 + * + * @param datasetTempStorage 实例对象 + * @return 实例对象 + */ + DatasetTempStorage update(DatasetTempStorage datasetTempStorage); + + /** + * 通过主键删除数据 + * + * @param id 主键 + * @return 是否成功 + */ + boolean deleteById(Integer id); + +} diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/DatasetTempStorageServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/DatasetTempStorageServiceImpl.java new file mode 100644 index 00000000..95f1f59e --- /dev/null +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/DatasetTempStorageServiceImpl.java @@ -0,0 +1,87 @@ +package com.ruoyi.platform.service.impl; + +import com.ruoyi.platform.domain.DatasetTempStorage; +import com.ruoyi.platform.mapper.DatasetTempStorageDao; +import com.ruoyi.platform.service.DatasetTempStorageService; +import org.springframework.stereotype.Service; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageImpl; +import org.springframework.data.domain.PageRequest; + +import javax.annotation.Resource; + +/** + * 数据集暂存的数据(DatasetTempStorage)表服务实现类 + * + * @author makejava + * @since 2024-09-12 09:42:10 + */ +@Service("datasetTempStorageService") +public class DatasetTempStorageServiceImpl implements DatasetTempStorageService { + @Resource + private DatasetTempStorageDao datasetTempStorageDao; + + /** + * 通过ID查询单条数据 + * + * @param id 主键 + * @return 实例对象 + */ + @Override + public DatasetTempStorage queryById(Integer id) { + return this.datasetTempStorageDao.queryById(id); + } + + /** + * 分页查询 + * + * @param datasetTempStorage 筛选条件 + * @param pageRequest 分页对象 + * @return 查询结果 + */ + @Override + public Page queryByPage(DatasetTempStorage datasetTempStorage, PageRequest pageRequest) { + long total = this.datasetTempStorageDao.count(datasetTempStorage); + return new PageImpl<>(this.datasetTempStorageDao.queryAllByLimit(datasetTempStorage, pageRequest), pageRequest, total); + } + + @Override + public DatasetTempStorage queryByDatasetTempStorage(DatasetTempStorage datasetTempStorage) { + return this.datasetTempStorageDao.queryByDatasetTempStorage(datasetTempStorage); + } + + /** + * 新增数据 + * + * @param datasetTempStorage 实例对象 + * @return 实例对象 + */ + @Override + public DatasetTempStorage insert(DatasetTempStorage datasetTempStorage) { + this.datasetTempStorageDao.insert(datasetTempStorage); + return datasetTempStorage; + } + + /** + * 修改数据 + * + * @param datasetTempStorage 实例对象 + * @return 实例对象 + */ + @Override + public DatasetTempStorage update(DatasetTempStorage datasetTempStorage) { + this.datasetTempStorageDao.update(datasetTempStorage); + return this.queryById(datasetTempStorage.getId()); + } + + /** + * 通过主键删除数据 + * + * @param id 主键 + * @return 是否成功 + */ + @Override + public boolean deleteById(Integer id) { + return this.datasetTempStorageDao.deleteById(id) > 0; + } +} diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentServiceImpl.java index 293b7826..41f9d68a 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentServiceImpl.java @@ -46,7 +46,8 @@ public class ExperimentServiceImpl implements ExperimentService { private DatasetService datasetService; @Resource private ModelDependencyService modelDependencyService; - + @Resource + private DatasetTempStorageService datasetTempStorageService; @Resource @Lazy private WorkflowService workflowService; @@ -288,7 +289,13 @@ public class ExperimentServiceImpl implements ExperimentService { if (dependendcy != null && trainInfo != null){ insertModelDependency(dependendcy,trainInfo,insert.getId(),experiment.getName()); } + + Map datasetDependendcy = (Map)converMap2.get("dataset_dependency"); //暂存数据集元数据{} + if (datasetDependendcy != null && trainInfo != null){ + insertDatasetTempStorage(datasetDependendcy,trainInfo,experiment.getId(),insert.getId(),experiment.getName()); + } + }catch (Exception e){ throw new RuntimeException(e); @@ -297,6 +304,7 @@ public class ExperimentServiceImpl implements ExperimentService { experiment.setExperimentInsList(updatedExperimentInsList); return experiment; } + private void addDatesetToMetric(Map metricRecord, Map trainInfo) { processMetricPart(metricRecord, trainInfo, "train", "model_train"); processMetricPart(metricRecord, trainInfo, "evaluate", "model_evaluate"); @@ -437,6 +445,45 @@ public class ExperimentServiceImpl implements ExperimentService { } } + /** + * 存储数据集元数据到临时表 + * + * + */ + private void insertDatasetTempStorage(Map datasetDependendcy, Map trainInfo, Integer experimentId,Integer experimentInsId, String experimentName) { + DatasetTempStorage datasetTempStorage = new DatasetTempStorage(); + + Iterator> dependendcyIterator = datasetDependendcy.entrySet().iterator(); + Map datasetExport = (Map) trainInfo.get("dataset_export"); + Map datasetPreprocess = (Map) trainInfo.get("dataset_preprocess"); + while (dependendcyIterator.hasNext()) { + Map.Entry entry = dependendcyIterator.next(); + Map modelDel = (Map) entry.getValue(); + Map source = (Map) modelDel.get("source"); //被处理数据集 +// List> test = (List>) modelDel.get("test"); + List> target = (List>) modelDel.get("target"); //导出的数据集 + + String sourceTaskId = (String) source.get("task_id"); + Map datasetPreprocessMap = (Map)datasetPreprocess.get(sourceTaskId); + //处理project数据 + Map projectMap = (Map) datasetPreprocessMap.get("project"); + Map datasets = (Map) datasetPreprocessMap.get("datasets"); + datasetTempStorage.setName((String) datasets.get("dataset_identifier")); + datasetTempStorage.setVersion((String) datasets.get("dataset_version")); + // 拼接需要的参数 + Map sourceParams = new HashMap<>(); + sourceParams.put("experiment_name", experimentName); + sourceParams.put("experiment_ins_id", experimentInsId); + sourceParams.put("experiment_id", experimentId); + sourceParams.put("train_name",sourceTaskId); + sourceParams.put("preprocess_code",projectMap); + datasetTempStorage.setSource(JacksonUtil.toJSONString(sourceParams)); + datasetTempStorage.setState(1); + datasetTempStorageService.insert(datasetTempStorage); + } + + } + /** * 被废弃的旧JSON * @param experiment diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java index 235f1b99..ad06aa06 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java @@ -3,6 +3,8 @@ package com.ruoyi.platform.service.impl; import com.ruoyi.common.core.utils.DateUtils; import com.ruoyi.common.security.utils.SecurityUtils; import com.ruoyi.platform.domain.Dataset; +import com.ruoyi.platform.domain.DatasetTempStorage; +import com.ruoyi.platform.service.DatasetTempStorageService; import com.ruoyi.platform.service.DvcService; import com.ruoyi.platform.service.GitService; import com.ruoyi.platform.service.NewDatasetService; @@ -58,7 +60,8 @@ public class NewDatasetServiceImpl implements NewDatasetService { String gitendpoint; @Value("${git.localPath}") String localPathlocal; - + @Resource + private DatasetTempStorageService datasetTempStorageService; @Override public String newCreateDataset(NewDatasetVo datasetVo) throws Exception { @@ -121,9 +124,8 @@ public class NewDatasetServiceImpl implements NewDatasetService { datasetVo.setOwner((String) userInfo.get("login")); datasetVo.setRelativePaths(relativePath); - if (StringUtils.isEmpty(datasetVo.getDatasetSource())){ - datasetVo.setDatasetSource("用户上传"); - } + addDatasetSourceToDataVo(datasetVo); + YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo), localPath, "dataset"); // dvc init 初始化 DVCUtils.dvcInit(localPath); @@ -195,6 +197,9 @@ public class NewDatasetServiceImpl implements NewDatasetService { datasetVo.setDataTag(newDatasetVo.getDataTag()); datasetVo.setDataType(newDatasetVo.getDataType()); datasetVo.setRelativePaths(relativePath); + + addDatasetSourceToDataVo(datasetVo); + YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo), localPath, "dataset"); //dvc数据跟踪 // 配置远程S3地址 @@ -473,5 +478,21 @@ public class NewDatasetServiceImpl implements NewDatasetService { return new ArrayList<>(); } + private void addDatasetSourceToDataVo(NewDatasetVo datasetVo){ + DatasetTempStorage queryDatasetTempStorage = new DatasetTempStorage(); + queryDatasetTempStorage.setName(datasetVo.getName()); + queryDatasetTempStorage.setVersion(datasetVo.getVersion()); + DatasetTempStorage datasetTempStorage = datasetTempStorageService.queryByDatasetTempStorage(queryDatasetTempStorage); + if (datasetTempStorage == null){ + String datasetSourceString = datasetTempStorage.getSource(); + Map datasetSourceMap = JacksonUtil.parseJSONStr2Map(datasetSourceString); + String preprocessCode = (String) datasetSourceMap.get("preprocess_code"); + datasetSourceMap.remove("preprocess_code"); + datasetVo.setProcessingCode(preprocessCode); + datasetVo.setDatasetSource(JacksonUtil.toJSONString(datasetSourceMap)); + datasetTempStorage.setState(0); + datasetTempStorageService.update(datasetTempStorage); + } + } } diff --git a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/DatasetTempStorageDao.xml b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/DatasetTempStorageDao.xml new file mode 100644 index 00000000..4d0a2d59 --- /dev/null +++ b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/DatasetTempStorageDao.xml @@ -0,0 +1,197 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + insert into dataset_temp_storage(nameversionsourcestatecreate_bycreate_timeupdate_byupdate_time) + values (#{name}#{version}#{source}#{state}#{createBy}#{createTime}#{updateBy}#{updateTime}) + + + + insert into dataset_temp_storage(nameversionsourcestatecreate_bycreate_timeupdate_byupdate_time) + values + + (#{entity.name}#{entity.version}#{entity.source}#{entity.state}#{entity.createBy}#{entity.createTime}#{entity.updateBy}#{entity.updateTime}) + + + + + insert into dataset_temp_storage(nameversionsourcestatecreate_bycreate_timeupdate_byupdate_time) + values + + (#{entity.name}#{entity.version}#{entity.source}#{entity.state}#{entity.createBy}#{entity.createTime}#{entity.updateBy}#{entity.updateTime}) + + on duplicate key update + name = values(name)version = values(version)source = values(source)state = values(state)create_by = + values(create_by)create_time = values(create_time)update_by = values(update_by)update_time = values(update_time) + + + + + update dataset_temp_storage + + + name = #{name}, + + + version = #{version}, + + + source = #{source}, + + + state = #{state}, + + + create_by = #{createBy}, + + + create_time = #{createTime}, + + + update_by = #{updateBy}, + + + update_time = #{updateTime}, + + + where id = #{id} + + + + + delete + from dataset_temp_storage + where id = #{id} + + + +