| @@ -1,6 +1,6 @@ | |||
| # 基础镜像 | |||
| FROM 172.20.32.187/ci4s/openjdk:8u162 | |||
| #FROM 172.20.32.187/ci4s/openjdk-dvc:2024829 | |||
| #FROM 172.20.32.187/ci4s/openjdk:8u162 | |||
| FROM 172.20.32.187/ci4s/openjdk-dvc:2024829 | |||
| # author | |||
| MAINTAINER ruoyi | |||
| @@ -15,10 +15,14 @@ spec: | |||
| spec: | |||
| containers: | |||
| - name: ci4s-management-platform | |||
| image: 172.20.32.187/ci4s/managent:202406121003 | |||
| image: 172.20.32.187/ci4s/ci4s-managent:202409181159 | |||
| ports: | |||
| - containerPort: 9213 | |||
| env: | |||
| - name: http_proxy | |||
| value: "http://172.20.32.253:3128" | |||
| - name: https_proxy | |||
| value: "http://172.20.32.253:3128" | |||
| --- | |||
| apiVersion: v1 | |||
| kind: Service | |||
| @@ -18,11 +18,6 @@ spec: | |||
| image: ${k8s-7management-image} | |||
| ports: | |||
| - containerPort: 9213 | |||
| env: | |||
| - name: http_proxy | |||
| value: "http://172.20.32.253:3128" | |||
| - name: https_proxy | |||
| value: "http://172.20.32.253:3128" | |||
| --- | |||
| apiVersion: v1 | |||
| kind: Service | |||
| @@ -46,7 +46,7 @@ function KFEmpty({ | |||
| return ( | |||
| <div className={classNames('kf-empty', className)} style={style}> | |||
| <img className="kf-empty__image" src={image} /> | |||
| <img className="kf-empty__image" src={image} draggable={false} alt="" /> | |||
| <div className="kf-empty__title">{title}</div> | |||
| <div className="kf-empty__content">{content}</div> | |||
| {hasFooter && ( | |||
| @@ -18,7 +18,7 @@ type ModalTitleProps = { | |||
| function ModalTitle({ title, image, style, className }: ModalTitleProps) { | |||
| return ( | |||
| <div className={classNames('kf-modal-title', className)} style={style}> | |||
| {image && <img className={'kf-modal-title__image'} src={image} alt="" />} | |||
| {image && <img className={'kf-modal-title__image'} src={image} draggable={false} alt="" />} | |||
| {title} | |||
| </div> | |||
| ); | |||
| @@ -17,7 +17,7 @@ type SubAreaTitleProps = { | |||
| function SubAreaTitle({ title, image, style, className }: SubAreaTitleProps) { | |||
| return ( | |||
| <div className={classNames('kf-subarea-title', className)} style={style}> | |||
| <img src={image} width={14} /> | |||
| <img src={image} width={14} draggable={false} alt="" /> | |||
| <span style={{ marginLeft: '8px' }}>{title}</span> | |||
| </div> | |||
| ); | |||
| @@ -63,11 +63,16 @@ function CodeConfigItem({ item, onClick, onEdit, onRemove }: CodeConfigItemProps | |||
| </div> | |||
| <Flex justify="space-between"> | |||
| <div className={styles['code-config-item__time']}> | |||
| <img style={{ width: '17px', marginRight: '6px' }} src={creatByImg} alt="" /> | |||
| <img | |||
| style={{ width: '17px', marginRight: '6px' }} | |||
| src={creatByImg} | |||
| alt="" | |||
| draggable={false} | |||
| /> | |||
| <span>{item.create_by}</span> | |||
| </div> | |||
| <div className={styles['code-config-item__time']}> | |||
| <img style={{ width: '12px', marginRight: '5px' }} src={clock} alt="" /> | |||
| <img style={{ width: '12px', marginRight: '5px' }} src={clock} alt="" draggable={false} /> | |||
| <span>最近更新: {formatDate(item.update_time, 'YYYY-MM-DD')}</span> | |||
| </div> | |||
| </Flex> | |||
| @@ -23,12 +23,14 @@ function CategoryItem({ resourceType, item, isSelected, onClick }: CategoryItemP | |||
| style={{ width: '22px' }} | |||
| src={`/assets/images/${config.prefix}/${item.path}.png`} | |||
| alt="" | |||
| draggable={false} | |||
| /> | |||
| <img | |||
| className={styles['category-item__active-icon']} | |||
| style={{ width: '22px' }} | |||
| src={`/assets/images/${config.prefix}/${item.path}-hover.png`} | |||
| alt="" | |||
| draggable={false} | |||
| /> | |||
| <span className={styles['category-item__name']}>{item.name}</span> | |||
| </div> | |||
| @@ -39,11 +39,16 @@ function ResourceItem({ item, isPublic, onClick, onRemove }: ResourceItemProps) | |||
| <div className={styles['resource-item__description']}>{item.description}</div> | |||
| <Flex justify="space-between"> | |||
| <div className={styles['resource-item__time']}> | |||
| <img style={{ width: '17px', marginRight: '6px' }} src={creatByImg} alt="" /> | |||
| <img | |||
| style={{ width: '17px', marginRight: '6px' }} | |||
| src={creatByImg} | |||
| draggable={false} | |||
| alt="" | |||
| /> | |||
| <span>{item.create_by ?? ''}</span> | |||
| </div> | |||
| <div className={styles['resource-item__time']}> | |||
| <img style={{ width: '12px', marginRight: '5px' }} src={clock} alt="" /> | |||
| <img style={{ width: '12px', marginRight: '5px' }} src={clock} draggable={false} alt="" /> | |||
| <span> | |||
| {'最近更新: '} | |||
| {item.update_time ? formatDate(item.update_time, 'YYYY-MM-DD') : item.time_ago ?? ''} | |||
| @@ -18,8 +18,8 @@ function ResourcePage({ resourceType }: ResourcePageProps) { | |||
| const [activeTab, setActiveTab] = useState<string>(cacheState?.activeTab ?? CommonTabKeys.Public); | |||
| const [typeList, setTypeList] = useState<CategoryData[]>([]); | |||
| const [tagList, setTagList] = useState<CategoryData[]>([]); | |||
| const [activeType, setActiveType] = useState<number | undefined>(cacheState?.activeType); | |||
| const [activeTag, setActiveTag] = useState<number | undefined>(cacheState?.activeTag); | |||
| const [activeType, setActiveType] = useState<string | undefined>(cacheState?.activeType); | |||
| const [activeTag, setActiveTag] = useState<string | undefined>(cacheState?.activeTag); | |||
| const dataListRef = useRef<ResourceListRef>(null); | |||
| const config = resourceConfig[resourceType]; | |||
| @@ -34,12 +34,12 @@ function ResourcePage({ resourceType }: ResourcePageProps) { | |||
| // 选择类型 | |||
| const chooseType = (record: CategoryData) => { | |||
| setActiveType((prev) => (prev === record.id ? undefined : record.id)); | |||
| setActiveType((prev) => (prev === record.name ? undefined : record.name)); | |||
| }; | |||
| // 选择 Tag | |||
| const chooseTag = (record: CategoryData) => { | |||
| setActiveTag((prev) => (prev === record.id ? undefined : record.id)); | |||
| setActiveTag((prev) => (prev === record.name ? undefined : record.name)); | |||
| }; | |||
| // 获取分类 | |||
| @@ -116,6 +116,8 @@ function ExperimentInstanceComponent({ | |||
| <img | |||
| style={{ width: '17px', marginRight: '7px' }} | |||
| src={experimentStatusInfo[item.status as ExperimentStatus]?.icon} | |||
| draggable={false} | |||
| alt="" | |||
| /> | |||
| <span | |||
| style={{ color: experimentStatusInfo[item.status as ExperimentStatus]?.color }} | |||
| @@ -14,7 +14,12 @@ function ExperimentStatusCell(status?: ExperimentStatus | null) { | |||
| } | |||
| return ( | |||
| <div className={styles['experiment-status-cell']}> | |||
| <img style={{ width: '17px', marginRight: '7px' }} src={statusInfo[status]?.icon} /> | |||
| <img | |||
| style={{ width: '17px', marginRight: '7px' }} | |||
| src={statusInfo[status]?.icon} | |||
| draggable={false} | |||
| alt="" | |||
| /> | |||
| <span | |||
| style={{ color: statusInfo[status]?.color }} | |||
| className={styles['experiment-status-cell__label']} | |||
| @@ -70,6 +70,8 @@ function TensorBoardStatusCell({ | |||
| className={styles['tensorBoard-status__icon']} | |||
| src={statusConfig[status].icon} | |||
| onClick={onClick} | |||
| draggable={false} | |||
| alt="" | |||
| /> | |||
| )} | |||
| </> | |||
| @@ -385,6 +385,8 @@ function Experiment() { | |||
| style={{ width: '17px', marginRight: '6px' }} | |||
| key={index} | |||
| src={experimentStatusInfo[item].icon} | |||
| draggable={false} | |||
| alt="" | |||
| /> | |||
| ); | |||
| }) | |||
| @@ -48,6 +48,7 @@ const ModelMenu = ({ onComponentDragEnd }: ModelMenuProps) => { | |||
| <img | |||
| style={{ height: '16px', marginRight: '15px' }} | |||
| src={`/assets/images/${ele.icon_path}.png`} | |||
| draggable={false} | |||
| alt="" | |||
| /> | |||
| )} | |||
| @@ -116,7 +116,12 @@ const AvatarCropperForm: React.FC<AvatarCropperProps> = (props) => { | |||
| </Col> | |||
| <Col span={12} order={2}> | |||
| <div className={styles.avatarPreview}> | |||
| <img src={previewData} style={{ height: '100%', width: '100%' }} /> | |||
| <img | |||
| src={previewData} | |||
| style={{ height: '100%', width: '100%' }} | |||
| draggable={false} | |||
| alt="" | |||
| /> | |||
| </div> | |||
| </Col> | |||
| </Row> | |||
| @@ -141,7 +141,7 @@ const Center: React.FC = () => { | |||
| setCropperModalOpen(true); | |||
| }} | |||
| > | |||
| <img alt="" src={currentUser.avatar} /> | |||
| <img src={currentUser.avatar} draggable={false} alt="" /> | |||
| </div> | |||
| {renderUserInfo(currentUser)} | |||
| <Divider dashed /> | |||
| @@ -11,7 +11,7 @@ import styles from './login.less'; | |||
| const LoginInputPrefix = ({ icon }: { icon: string }) => { | |||
| return ( | |||
| <div className={styles['login-input-prefix']}> | |||
| <img className={styles['login-input-prefix__icon']} src={icon} alt="" /> | |||
| <img className={styles['login-input-prefix__icon']} src={icon} alt="" draggable={false} /> | |||
| <div className={styles['login-input-prefix__line']}></div> | |||
| </div> | |||
| ); | |||
| @@ -98,6 +98,7 @@ const Login = () => { | |||
| <img | |||
| src={require('@/assets/img/logo.png')} | |||
| style={{ width: '32px', marginRight: '12px' }} | |||
| draggable={false} | |||
| alt="" | |||
| /> | |||
| 智能材料科研平台 | |||
| @@ -107,6 +108,7 @@ const Login = () => { | |||
| <img | |||
| src={require('@/assets/img/login-ai-logo.png')} | |||
| className={styles['user-login__left__title__img']} | |||
| draggable={false} | |||
| alt="" | |||
| /> | |||
| </div> | |||
| @@ -116,6 +118,7 @@ const Login = () => { | |||
| <img | |||
| className={styles['user-login__left__bottom-img']} | |||
| src={require('@/assets/img/login-left-image.png')} | |||
| draggable={false} | |||
| alt="" | |||
| /> | |||
| </div> | |||
| @@ -31,6 +31,8 @@ function ExperimentTable({ tableData = [], style }: ExperimentTableProps) { | |||
| src={experimentStatusInfo[item.status as ExperimentStatus]?.icon} | |||
| width={17} | |||
| height={17} | |||
| draggable={false} | |||
| alt="" | |||
| /> | |||
| </div> | |||
| <div className={styles['experiment-table__duration']}> | |||
| @@ -44,6 +44,7 @@ function WorkArrow({ | |||
| <img | |||
| className={styles['work-arrow__img']} | |||
| src={require('@/assets/img/blue-triangle.png')} | |||
| draggable={false} | |||
| alt="" | |||
| width={10} | |||
| height={9} | |||
| @@ -10,7 +10,7 @@ type TotalStatisticsProps = { | |||
| function TotalStatistics({ icon = '', title = '', count = 0, style }: TotalStatisticsProps) { | |||
| return ( | |||
| <div className={styles['total-statistics']} style={style}> | |||
| <img className={styles['total-statistics__icon']} src={icon} /> | |||
| <img className={styles['total-statistics__icon']} src={icon} draggable={false} alt="" /> | |||
| <div> | |||
| <div className={styles['total-statistics__title']}> | |||
| <span>{title}</span> | |||
| @@ -20,7 +20,15 @@ function UserSpace({ users = [] }: UserSpaceProps) { | |||
| className={styles['user-space__avatar']} | |||
| src={currentUser?.avatar} | |||
| alt="" | |||
| icon={<img src={require('@/assets/img/avatar-default.png')} width={56} height={56} />} | |||
| icon={ | |||
| <img | |||
| src={require('@/assets/img/avatar-default.png')} | |||
| width={56} | |||
| height={56} | |||
| draggable={false} | |||
| alt="" | |||
| /> | |||
| } | |||
| ></Avatar> | |||
| <div className={styles['user-space__name']}>{currentUser?.nickName}</div> | |||
| <div className={styles['user-space__role']}>{currentUser?.roleNames?.[0]?.roleName}</div> | |||
| @@ -34,7 +42,7 @@ function UserSpace({ users = [] }: UserSpaceProps) { | |||
| <div className={styles['user-space__participant__count']}>8</div> | |||
| </Space> | |||
| <Flex align="center" gap={12} wrap="wrap"> | |||
| {users?.map((item, index) => { | |||
| {users?.map((_item, index) => { | |||
| return ( | |||
| <Avatar | |||
| key={index} | |||
| @@ -42,7 +50,13 @@ function UserSpace({ users = [] }: UserSpaceProps) { | |||
| src={require(`@/assets/img/user-avatar/${index + 1}.png`)} | |||
| alt="" | |||
| icon={ | |||
| <img src={require('@/assets/img/avatar-default.png')} width={36} height={36} /> | |||
| <img | |||
| src={require('@/assets/img/avatar-default.png')} | |||
| width={36} | |||
| height={36} | |||
| draggable={false} | |||
| alt="" | |||
| /> | |||
| } | |||
| ></Avatar> | |||
| ); | |||
| @@ -15,7 +15,13 @@ function WorkspaceIntro() { | |||
| type="primary" | |||
| style={{ marginRight: '20px' }} | |||
| icon={ | |||
| <img src={require('@/assets/img/functional-material.png')} width={19} height={19} /> | |||
| <img | |||
| src={require('@/assets/img/functional-material.png')} | |||
| width={19} | |||
| height={19} | |||
| draggable={false} | |||
| alt="" | |||
| /> | |||
| } | |||
| > | |||
| 功能材料自主实验系统 | |||
| @@ -23,7 +29,13 @@ function WorkspaceIntro() { | |||
| <Button | |||
| type="default" | |||
| icon={ | |||
| <img src={require('@/assets/img/molecular-material.png')} width={19} height={19} /> | |||
| <img | |||
| src={require('@/assets/img/molecular-material.png')} | |||
| width={19} | |||
| height={19} | |||
| draggable={false} | |||
| alt="" | |||
| /> | |||
| } | |||
| > | |||
| 分子材料自主实验系统 | |||
| @@ -34,6 +46,8 @@ function WorkspaceIntro() { | |||
| <img | |||
| className={styles['workspace-intro__icon']} | |||
| src={require('@/assets/img/workspace-intro-icon.png')} | |||
| draggable={false} | |||
| alt="" | |||
| /> | |||
| </div> | |||
| </div> | |||
| @@ -53,6 +53,7 @@ | |||
| width: 64px; | |||
| height: 64px; | |||
| background-color: white; | |||
| border-radius: 10px; | |||
| cursor: pointer; | |||
| } | |||
| } | |||
| @@ -76,7 +76,8 @@ function Workspace() { | |||
| className={styles['workspace__robot-img']} | |||
| src={require('@/assets/img/robot.png')} | |||
| onClick={handleClick} | |||
| onDragStart={(e) => e.preventDefault()} | |||
| draggable={false} | |||
| alt="" | |||
| ></img> | |||
| </Draggable> | |||
| @@ -21,6 +21,7 @@ export function modalConfirm({ title, content, onOk, ...rest }: ModalFuncProps) | |||
| <img | |||
| src={require('@/assets/img/delete-icon.png')} | |||
| style={{ width: '120px', marginBottom: '24px' }} | |||
| draggable={false} | |||
| alt="" | |||
| /> | |||
| <div style={{ color: themes.textColor, fontSize: '16px', fontWeight: 500 }}>{title}</div> | |||
| @@ -7,6 +7,8 @@ import com.ruoyi.system.api.model.LoginUser; | |||
| import org.springframework.beans.factory.annotation.Value; | |||
| import org.springframework.stereotype.Service; | |||
| import javax.annotation.Resource; | |||
| @Service | |||
| public class DvcServiceImpl implements DvcService { | |||
| @Value("${minio.accessKey}") | |||
| @@ -15,20 +17,22 @@ public class DvcServiceImpl implements DvcService { | |||
| String secretAccessKey; | |||
| @Value("${minio.endpoint}") | |||
| String endpoint; | |||
| @Resource | |||
| private DVCUtils dvcUtils; | |||
| @Override | |||
| public void initaddpushDvc(String localPath) throws Exception { | |||
| LoginUser loginUser = SecurityUtils.getLoginUser(); | |||
| // dvc init 初始化 | |||
| DVCUtils.dvcInit(localPath); | |||
| dvcUtils.dvcInit(localPath); | |||
| // 配置远程S3地址 | |||
| DVCUtils.dvcRemoteAdd(localPath,""); | |||
| // DVCUtils.dvcConfigS3Credentials(localPath,endpoint, accessKeyId, secretAccessKey); | |||
| dvcUtils.dvcRemoteAdd(localPath,""); | |||
| // dvcUtils.dvcConfigS3Credentials(localPath,endpoint, accessKeyId, secretAccessKey); | |||
| // dvc 跟踪 | |||
| DVCUtils.dvcAdd(localPath , "data"); | |||
| dvcUtils.dvcAdd(localPath , "data"); | |||
| // git commit | |||
| DVCUtils.gitCommit(localPath, "commit from ci4s with "+loginUser.getUsername()); | |||
| dvcUtils.gitCommit(localPath, "commit from ci4s with "+loginUser.getUsername()); | |||
| // dvc push 到远程S3 | |||
| DVCUtils.dvcPush(localPath); | |||
| dvcUtils.dvcPush(localPath); | |||
| } | |||
| } | |||
| @@ -78,6 +78,8 @@ public class ModelsServiceImpl implements ModelsService { | |||
| private AssetIconService assetIconService; | |||
| @Resource | |||
| private NewHttpUtils httpUtils; | |||
| @Resource | |||
| private DVCUtils dvcUtils; | |||
| // 固定存储桶名 | |||
| @Value("${minio.dataReleaseBucketName}") | |||
| @@ -593,8 +595,8 @@ public class ModelsServiceImpl implements ModelsService { | |||
| String modelPath = rootPath + "/model"; | |||
| String metaPath = rootPath + "/metadata"; | |||
| DVCUtils.gitClone(rootPath, projectUrl, branchName, gitLinkUsername, gitLinkPassword); | |||
| DVCUtils.moveFiles(sourcePath, modelPath); | |||
| dvcUtils.gitClone(rootPath, projectUrl, branchName, gitLinkUsername, gitLinkPassword); | |||
| dvcUtils.moveFiles(sourcePath, modelPath); | |||
| //拼接生产的元数据后写入yaml文件 | |||
| ModelMetaVo modelMetaVo = new ModelMetaVo(); | |||
| @@ -621,21 +623,21 @@ public class ModelsServiceImpl implements ModelsService { | |||
| YamlUtils.generateYamlFile(metaMap, metaPath, "metadata"); | |||
| String meta = JSON.toJSONString(modelMetaVo); | |||
| DVCUtils.dvcInit(rootPath); | |||
| dvcUtils.dvcInit(rootPath); | |||
| // 配置远程S3地址 | |||
| String s3Path = "management-platform-files/" + ci4sUsername + "/model/" + gitlinIid + "/" + repositoryName + "/" + branchName; | |||
| DVCUtils.dvcRemoteAdd(rootPath, s3Path); | |||
| DVCUtils.dvcConfigS3Credentials(rootPath, endpoint); | |||
| DVCUtils.dvcConfigS3Credentials2(rootPath, accessKeyId); | |||
| DVCUtils.dvcConfigS3Credentials3(rootPath, secretAccessKey); | |||
| dvcUtils.dvcRemoteAdd(rootPath, s3Path); | |||
| dvcUtils.dvcConfigS3Credentials(rootPath, endpoint); | |||
| dvcUtils.dvcConfigS3Credentials2(rootPath, accessKeyId); | |||
| dvcUtils.dvcConfigS3Credentials3(rootPath, secretAccessKey); | |||
| // dvc 跟踪 | |||
| DVCUtils.dvcAdd(rootPath, "model"); | |||
| dvcUtils.dvcAdd(rootPath, "model"); | |||
| // git commit | |||
| DVCUtils.gitAdd(rootPath, "."); | |||
| DVCUtils.gitCommit(rootPath, "commit from ci4s with " + ci4sUsername); | |||
| DVCUtils.gitPush(rootPath, gitLinkUsername, gitLinkPassword); | |||
| DVCUtils.dvcPush(rootPath); | |||
| dvcUtils.gitAdd(rootPath, "."); | |||
| dvcUtils.gitCommit(rootPath, "commit from ci4s with " + ci4sUsername); | |||
| dvcUtils.gitPush(rootPath, gitLinkUsername, gitLinkPassword); | |||
| dvcUtils.dvcPush(rootPath); | |||
| //保存模型依赖 | |||
| modelDependency.setRepoId(gitlinIid); | |||
| @@ -686,10 +688,10 @@ public class ModelsServiceImpl implements ModelsService { | |||
| ModelMetaVo oldModelVo = ConvertUtil.convertMapToObject(stringObjectMap, ModelMetaVo.class); | |||
| // 创建本地分支 | |||
| DVCUtils.createLocalBranchBasedOnMaster(rootPath, branchName); | |||
| dvcUtils.createLocalBranchBasedOnMaster(rootPath, branchName); | |||
| //dvc checkout | |||
| DVCUtils.dvcCheckout(rootPath); | |||
| DVCUtils.moveFiles(sourcePath, modelPath); | |||
| dvcUtils.dvcCheckout(rootPath); | |||
| dvcUtils.moveFiles(sourcePath, modelPath); | |||
| //拼接生产的元数据后写入yaml文件 | |||
| ModelMetaVo modelMetaVo = new ModelMetaVo(); | |||
| @@ -719,15 +721,15 @@ public class ModelsServiceImpl implements ModelsService { | |||
| // 配置远程S3地址 | |||
| String s3Path = "management-platform-files/" + ci4sUsername + "/model/" + modelsVo.getId() + "/" + repositoryName + "/" + branchName; | |||
| DVCUtils.dvcRemoteAdd(rootPath, s3Path); | |||
| DVCUtils.dvcConfigS3Credentials(rootPath, endpoint); | |||
| DVCUtils.dvcConfigS3Credentials2(rootPath, accessKeyId); | |||
| DVCUtils.dvcConfigS3Credentials3(rootPath, secretAccessKey); | |||
| dvcUtils.dvcRemoteAdd(rootPath, s3Path); | |||
| dvcUtils.dvcConfigS3Credentials(rootPath, endpoint); | |||
| dvcUtils.dvcConfigS3Credentials2(rootPath, accessKeyId); | |||
| dvcUtils.dvcConfigS3Credentials3(rootPath, secretAccessKey); | |||
| // dvc 跟踪 | |||
| DVCUtils.dvcAdd(rootPath, "model"); | |||
| DVCUtils.pushNewBranchToRemote(rootPath, gitLinkUsername, gitLinkPassword, branchName); | |||
| dvcUtils.dvcAdd(rootPath, "model"); | |||
| dvcUtils.pushNewBranchToRemote(rootPath, gitLinkUsername, gitLinkPassword, branchName); | |||
| //dvc push 到远程S3 | |||
| DVCUtils.dvcPush(rootPath); | |||
| dvcUtils.dvcPush(rootPath); | |||
| //保存模型依赖 | |||
| modelDependency.setRepoId(modelsVo.getId()); | |||
| @@ -895,7 +897,7 @@ public class ModelsServiceImpl implements ModelsService { | |||
| } | |||
| // git pull操作,然后读取里面的文件列表,列出每个文件的大小和名称,封装成MAP | |||
| List<Map<String, Object>> fileDetailsAfterGitPull = DVCUtils.getFileDetailsAfterGitPull(localPath + ci4sUsername + "/model/" + id, name, version, "model", gitLinkUsername, gitLinkPassword); | |||
| List<Map<String, Object>> fileDetailsAfterGitPull = dvcUtils.getFileDetailsAfterGitPull(localPath + ci4sUsername + "/model/" + id, name, version, "model", gitLinkUsername, gitLinkPassword); | |||
| Map<String, Object> stringObjectMap = YamlUtils.loadYamlFile(localPath + ci4sUsername + "/model/" + id + "/" + name + "/metadata/metadata.yaml"); | |||
| @@ -67,6 +67,8 @@ public class NewDatasetServiceImpl implements NewDatasetService { | |||
| private NewHttpUtils httpUtils; | |||
| @Resource | |||
| private DatasetTempStorageService datasetTempStorageService; | |||
| @Resource | |||
| private DVCUtils dvcUtils; | |||
| @Override | |||
| public String newCreateDataset(NewDatasetVo datasetVo) throws Exception { | |||
| String token = gitService.checkoutToken(); | |||
| @@ -110,10 +112,10 @@ public class NewDatasetServiceImpl implements NewDatasetService { | |||
| String datasetPath = localPath + "/dataset"; | |||
| // 命令行操作 git clone 项目地址 | |||
| DVCUtils.gitClone(localPath, projectUrl, branchName, gitLinkUsername, gitLinkPassword); | |||
| dvcUtils.gitClone(localPath, projectUrl, branchName, gitLinkUsername, gitLinkPassword); | |||
| String s3Path = "management-platform-files" + "/" + relatePath + "/" + branchName; | |||
| DVCUtils.moveFiles(sourcePath, datasetPath); | |||
| dvcUtils.moveFiles(sourcePath, datasetPath); | |||
| // 拼接生产的元数据后写入yaml文件 | |||
| datasetVo.setCreateBy(String.valueOf(StringUtils.isNotEmpty((String) userInfo.get("nickname")) ? userInfo.get("nickname") : userInfo.get("login"))); | |||
| datasetVo.setUpdateTime(DateUtils.getTime()); | |||
| @@ -133,20 +135,19 @@ public class NewDatasetServiceImpl implements NewDatasetService { | |||
| YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo), localPath, "dataset"); | |||
| // dvc init 初始化 | |||
| DVCUtils.dvcInit(localPath); | |||
| dvcUtils.dvcInit(localPath); | |||
| // 配置远程S3地址 | |||
| DVCUtils.dvcRemoteAdd(localPath, s3Path); | |||
| DVCUtils.dvcConfigS3Credentials(localPath, endpoint); | |||
| DVCUtils.dvcConfigS3Credentials2(localPath, accessKeyId); | |||
| DVCUtils.dvcConfigS3Credentials3(localPath, secretAccessKey); | |||
| dvcUtils.dvcRemoteAdd(localPath, s3Path); | |||
| dvcUtils.dvcConfigS3Credentials(localPath, endpoint); | |||
| dvcUtils.dvcConfigS3Credentials2(localPath, accessKeyId); | |||
| dvcUtils.dvcConfigS3Credentials3(localPath, secretAccessKey); | |||
| // dvc 跟踪 | |||
| DVCUtils.dvcAdd(localPath, "dataset"); | |||
| dvcUtils.dvcAdd(localPath, "dataset"); | |||
| // git commit | |||
| DVCUtils.gitAdd(localPath, "."); | |||
| DVCUtils.gitCommit(localPath, "commit from ci4s with " + loginUser.getUsername()); | |||
| DVCUtils.gitPush(localPath, gitLinkUsername, gitLinkPassword); | |||
| dvcUtils.gitCommit(localPath, "commit from ci4s with " + loginUser.getUsername()); | |||
| dvcUtils.gitPush(localPath, gitLinkUsername, gitLinkPassword); | |||
| // dvc push 到远程S3 | |||
| DVCUtils.dvcPush(localPath); | |||
| dvcUtils.dvcPush(localPath); | |||
| return "新增数据集成功"; | |||
| } | |||
| @@ -173,15 +174,15 @@ public class NewDatasetServiceImpl implements NewDatasetService { | |||
| NewDatasetVo newDatasetVo = ConvertUtil.convertMapToObject(stringObjectMap, NewDatasetVo.class); | |||
| // 创建本地分支 | |||
| DVCUtils.createLocalBranchBasedOnMaster(localPath, branchName); | |||
| dvcUtils.createLocalBranchBasedOnMaster(localPath, branchName); | |||
| //dvc checkout | |||
| DVCUtils.dvcCheckout(localPath); | |||
| dvcUtils.dvcCheckout(localPath); | |||
| // 准备数据 | |||
| String s3Path = "management-platform-files" + "/" + relatePath + "/" + branchName; | |||
| DVCUtils.moveFiles(sourcePath, datasetPath); | |||
| dvcUtils.moveFiles(sourcePath, datasetPath); | |||
| // 拼接生产的元数据后写入yaml文件 | |||
| datasetVo.setCreateBy(String.valueOf(StringUtils.isNotEmpty((String) userInfo.get("nickname")) ? userInfo.get("nickname") : userInfo.get("login"))); | |||
| datasetVo.setUpdateTime(DateUtils.getTime()); | |||
| @@ -206,15 +207,15 @@ public class NewDatasetServiceImpl implements NewDatasetService { | |||
| YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo), localPath, "dataset"); | |||
| //dvc数据跟踪 | |||
| // 配置远程S3地址 | |||
| DVCUtils.dvcRemoteAdd(localPath, s3Path); | |||
| DVCUtils.dvcConfigS3Credentials(localPath, endpoint); | |||
| DVCUtils.dvcConfigS3Credentials2(localPath, accessKeyId); | |||
| DVCUtils.dvcConfigS3Credentials3(localPath, secretAccessKey); | |||
| dvcUtils.dvcRemoteAdd(localPath, s3Path); | |||
| dvcUtils.dvcConfigS3Credentials(localPath, endpoint); | |||
| dvcUtils.dvcConfigS3Credentials2(localPath, accessKeyId); | |||
| dvcUtils.dvcConfigS3Credentials3(localPath, secretAccessKey); | |||
| // dvc 跟踪 | |||
| DVCUtils.dvcAdd(localPath, "dataset"); | |||
| DVCUtils.pushNewBranchToRemote(localPath,gitLinkUsername,gitLinkPassword,branchName); | |||
| dvcUtils.dvcAdd(localPath, "dataset"); | |||
| dvcUtils.pushNewBranchToRemote(localPath,gitLinkUsername,gitLinkPassword,branchName); | |||
| //dvc push 到远程S3 | |||
| DVCUtils.dvcPush(localPath); | |||
| dvcUtils.dvcPush(localPath); | |||
| return "新增数据集成功"; | |||
| } | |||
| @@ -292,7 +293,7 @@ public class NewDatasetServiceImpl implements NewDatasetService { | |||
| } | |||
| version = (String) versionList.get(0).get("name"); | |||
| } | |||
| List<Map<String, Object>> fileDetailsAfterGitPull = DVCUtils.getFileDetailsAfterGitPull(localPathlocal +loginUser.getUsername()+"/datasets/" + id, name, version, "data",gitLinkUsername, gitLinkPassword); | |||
| List<Map<String, Object>> fileDetailsAfterGitPull = dvcUtils.getFileDetailsAfterGitPull(localPathlocal +loginUser.getUsername()+"/datasets/" + id, name, version, "data",gitLinkUsername, gitLinkPassword); | |||
| // 在localPathlocal+id+"/"+repositoryName目录下的dataset.yaml中取到元数据 | |||
| Map<String, Object> stringObjectMap = YamlUtils.loadYamlFile(localPathlocal +loginUser.getUsername()+"/datasets/" + id + "/" + name + "/" + "dataset.yaml"); | |||
| NewDatasetVo newDatasetVo = ConvertUtil.convertMapToObject(stringObjectMap, NewDatasetVo.class); | |||
| @@ -9,6 +9,8 @@ import org.eclipse.jgit.storage.file.FileRepositoryBuilder; | |||
| import org.eclipse.jgit.transport.*; | |||
| import org.slf4j.Logger; | |||
| import org.slf4j.LoggerFactory; | |||
| import org.springframework.beans.factory.annotation.Value; | |||
| import org.springframework.stereotype.Component; | |||
| import java.io.BufferedReader; | |||
| import java.io.File; | |||
| @@ -24,10 +26,32 @@ import java.util.concurrent.ExecutorService; | |||
| import java.util.concurrent.Executors; | |||
| import java.util.concurrent.TimeUnit; | |||
| @Component | |||
| public class DVCUtils { | |||
| private static final ExecutorService executorService = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); | |||
| private static final Logger log = LoggerFactory.getLogger(DVCUtils.class); | |||
| private static final ExecutorCompletionService<Void> completionService = new ExecutorCompletionService<>(executorService); | |||
| @Value("${proxy.useProxy:false}") | |||
| private boolean useProxy; | |||
| @Value("${proxy.host}") | |||
| private String host; | |||
| @Value("${proxy.port}") | |||
| private Integer port; | |||
| private class ProxyConfigCallback implements TransportConfigCallback { | |||
| @Override | |||
| public void configure(Transport transport) { | |||
| if (useProxy) { | |||
| System.setProperty("http.proxyHost", host); | |||
| System.setProperty("http.proxyPort", String.valueOf(port)); | |||
| System.setProperty("https.proxyHost", host); | |||
| System.setProperty("https.proxyPort", String.valueOf(port)); | |||
| log.info("Proxy configured: {}:{}", host, port); | |||
| } | |||
| } | |||
| } | |||
| private static void runCommand(String command, String workingDir) throws Exception { | |||
| ProcessBuilder processBuilder = new ProcessBuilder(command.split(" ")); | |||
| processBuilder.directory(new File(workingDir)); | |||
| @@ -43,7 +67,7 @@ public class DVCUtils { | |||
| } | |||
| } | |||
| public static void moveFiles(String sourcePath, String targetPath) throws Exception { | |||
| public void moveFiles(String sourcePath, String targetPath) throws Exception { | |||
| Path sourceDir = Paths.get(sourcePath); | |||
| Path targetDir = Paths.get(targetPath); | |||
| @@ -54,17 +78,18 @@ public class DVCUtils { | |||
| Files.move(sourceDir, targetDir, StandardCopyOption.REPLACE_EXISTING); | |||
| } | |||
| public static void gitClone(String localPath, String repoUrl, String branch, String username, String password) throws GitAPIException { | |||
| public void gitClone(String localPath, String repoUrl, String branch, String username, String password) throws GitAPIException { | |||
| CloneCommand cloneCommand = Git.cloneRepository() | |||
| .setURI(repoUrl) | |||
| .setBranch(branch) | |||
| .setDirectory(new java.io.File(localPath)) | |||
| .setTransportConfigCallback(new ProxyConfigCallback()) | |||
| .setCredentialsProvider(new UsernamePasswordCredentialsProvider(username, password)); | |||
| cloneCommand.call(); | |||
| } | |||
| public static void gitClone(String localPath, String repoUrl, String username, String password) throws GitAPIException { | |||
| public void gitClone(String localPath, String repoUrl, String username, String password) throws GitAPIException { | |||
| CloneCommand cloneCommand = Git.cloneRepository() | |||
| .setURI(repoUrl) | |||
| .setDirectory(new java.io.File(localPath)) | |||
| @@ -74,7 +99,7 @@ public class DVCUtils { | |||
| } | |||
| public static void gitAdd(String localPath, String filePath) throws IOException, GitAPIException { | |||
| public void gitAdd(String localPath, String filePath) throws IOException, GitAPIException { | |||
| FileRepositoryBuilder builder = new FileRepositoryBuilder(); | |||
| Repository repository = builder.setGitDir(new File(localPath, ".git")) | |||
| .readEnvironment() | |||
| @@ -87,7 +112,7 @@ public class DVCUtils { | |||
| } | |||
| } | |||
| public static void gitCommit(String localPath, String commitMessage) throws IOException, GitAPIException { | |||
| public void gitCommit(String localPath, String commitMessage) throws IOException, GitAPIException { | |||
| FileRepositoryBuilder builder = new FileRepositoryBuilder(); | |||
| Repository repository = builder.setGitDir(new File(localPath, ".git")) | |||
| .readEnvironment() | |||
| @@ -105,7 +130,7 @@ public class DVCUtils { | |||
| } | |||
| } | |||
| public static void gitPush(String localPath, String username, String password) throws IOException, GitAPIException { | |||
| public void gitPush(String localPath, String username, String password) throws IOException, GitAPIException { | |||
| FileRepositoryBuilder builder = new FileRepositoryBuilder(); | |||
| Repository repository = builder.setGitDir(new File(localPath, ".git")) | |||
| .readEnvironment() | |||
| @@ -116,6 +141,7 @@ public class DVCUtils { | |||
| CredentialsProvider credentialsProvider = new UsernamePasswordCredentialsProvider(username, password); | |||
| PushCommand pushCommand = git.push(); | |||
| pushCommand.setCredentialsProvider(credentialsProvider) | |||
| .setTransportConfigCallback(new ProxyConfigCallback()) | |||
| .setForce(true) | |||
| .call(); | |||
| } | |||
| @@ -129,7 +155,7 @@ public class DVCUtils { | |||
| * @param username 远程仓库用户名 | |||
| * @param password 远程仓库密码 | |||
| */ | |||
| public static void updateAllBranches(String localPath, String username, String password) { | |||
| public void updateAllBranches(String localPath, String username, String password) { | |||
| try (Git git = Git.open(new File(localPath))) { | |||
| // 设置凭证,用于远程仓库的认证 | |||
| UsernamePasswordCredentialsProvider credentialsProvider = new UsernamePasswordCredentialsProvider(username, password); | |||
| @@ -171,7 +197,7 @@ public class DVCUtils { | |||
| * @param password 远程仓库密码 | |||
| * @param branchName 需要更新的分支名称 | |||
| */ | |||
| public static void updateBranch(String localPath, String username, String password, String branchName) { | |||
| public void updateBranch(String localPath, String username, String password, String branchName) { | |||
| try (Git git = Git.open(new File(localPath))) { | |||
| // 设置凭证,用于远程仓库的认证 | |||
| UsernamePasswordCredentialsProvider credentialsProvider = new UsernamePasswordCredentialsProvider(username, password); | |||
| @@ -202,7 +228,7 @@ public class DVCUtils { | |||
| * @param localPath 本地仓库路径 | |||
| * @param branchName 要创建的分支名称 | |||
| */ | |||
| public static void createLocalBranch(String localPath, String branchName) { | |||
| public void createLocalBranch(String localPath, String branchName) { | |||
| try (Git git = Git.open(new File(localPath))) { | |||
| // 创建本地分支 | |||
| git.branchCreate().setName(branchName).call(); | |||
| @@ -219,7 +245,7 @@ public class DVCUtils { | |||
| * @param localPath 本地仓库路径 | |||
| * @param branchName 要创建的分支名称 | |||
| */ | |||
| public static void createLocalBranchBasedOnMaster(String localPath, String branchName) { | |||
| public void createLocalBranchBasedOnMaster(String localPath, String branchName) { | |||
| try (Git git = Git.open(new File(localPath))) { | |||
| // 切换到 master 分支 | |||
| git.checkout() | |||
| @@ -241,7 +267,7 @@ public class DVCUtils { | |||
| * @param password 远程仓库密码 | |||
| * @param branchName 要推送的分支名称 | |||
| */ | |||
| public static void pushLocalBranchToRemote(String localPath, String username, String password, String branchName) { | |||
| public void pushLocalBranchToRemote(String localPath, String username, String password, String branchName) { | |||
| try (Git git = Git.open(new File(localPath))) { | |||
| // 设置凭证,用于远程仓库的认证 | |||
| UsernamePasswordCredentialsProvider credentialsProvider = new UsernamePasswordCredentialsProvider(username, password); | |||
| @@ -279,7 +305,7 @@ public class DVCUtils { | |||
| * @throws IOException 如果仓库路径无效 | |||
| * @throws GitAPIException 如果Git操作失败 | |||
| */ | |||
| public static void pushNewBranchToRemote(String localPath, String username, String password, String branchName) throws IOException, GitAPIException { | |||
| public void pushNewBranchToRemote(String localPath, String username, String password, String branchName) throws IOException, GitAPIException { | |||
| try (Git git = Git.open(new File(localPath))) { | |||
| // 设置凭证,用于远程仓库的认证 | |||
| UsernamePasswordCredentialsProvider credentialsProvider = new UsernamePasswordCredentialsProvider(username, password); | |||
| @@ -300,6 +326,7 @@ public class DVCUtils { | |||
| .setRemote("origin") | |||
| .setRefSpecs(refSpec) | |||
| .setCredentialsProvider(credentialsProvider) | |||
| .setTransportConfigCallback(new ProxyConfigCallback()) | |||
| .call(); | |||
| // 打印结果 | |||
| @@ -326,7 +353,7 @@ public class DVCUtils { | |||
| return false; | |||
| } | |||
| public static void refreshRemoteBranches(String localPath, String username, String password, String branch) throws Exception { | |||
| public void refreshRemoteBranches(String localPath, String username, String password, String branch) throws Exception { | |||
| try (Repository repository = new FileRepositoryBuilder() | |||
| .setGitDir(new File(localPath + "/.git")) | |||
| .readEnvironment() | |||
| @@ -349,6 +376,7 @@ public class DVCUtils { | |||
| FetchResult fetchResult = git.fetch() | |||
| .setRemote("origin") | |||
| .setCredentialsProvider(credentialsProvider) | |||
| .setTransportConfigCallback(new ProxyConfigCallback()) // 设置代理 | |||
| .call(); | |||
| // 打印获取的远程分支 | |||
| @@ -394,7 +422,7 @@ public class DVCUtils { | |||
| dvcCheckout(localPath); | |||
| } | |||
| private static void processBranch(Git git, Repository repository, UsernamePasswordCredentialsProvider credentialsProvider, String fullBranchName, String branchName) throws Exception { | |||
| private void processBranch(Git git, Repository repository, UsernamePasswordCredentialsProvider credentialsProvider, String fullBranchName, String branchName) throws Exception { | |||
| // 检查本地分支是否存在 | |||
| Ref localRef = repository.findRef("refs/heads/" + branchName); | |||
| if (localRef != null) { | |||
| @@ -422,6 +450,7 @@ public class DVCUtils { | |||
| } | |||
| // 执行 git pull | |||
| PullCommand pullCommand = git.pull() | |||
| .setTransportConfigCallback(new ProxyConfigCallback()) | |||
| .setCredentialsProvider(credentialsProvider); | |||
| pullCommand.call(); | |||
| } | |||
| @@ -447,7 +476,7 @@ public class DVCUtils { | |||
| public static void gitFetch(String localPath, String username, String password) throws IOException, GitAPIException { | |||
| public void gitFetch(String localPath, String username, String password) throws IOException, GitAPIException { | |||
| FileRepositoryBuilder builder = new FileRepositoryBuilder(); | |||
| Repository repository = builder.setGitDir(new File(localPath, ".git")) | |||
| .readEnvironment() | |||
| @@ -460,7 +489,7 @@ public class DVCUtils { | |||
| fetchCommand.setCredentialsProvider(credentialsProvider).call(); | |||
| } | |||
| } | |||
| public static void gitCheckoutBranch(String localPath, String branchName) throws IOException, GitAPIException { | |||
| public void gitCheckoutBranch(String localPath, String branchName) throws IOException, GitAPIException { | |||
| FileRepositoryBuilder builder = new FileRepositoryBuilder(); | |||
| Repository repository = builder.setGitDir(new File(localPath, ".git")) | |||
| .readEnvironment() | |||
| @@ -473,7 +502,7 @@ public class DVCUtils { | |||
| } | |||
| } | |||
| public static void gitPull(String localPath, String username, String password) throws IOException, GitAPIException { | |||
| public void gitPull(String localPath, String username, String password) throws IOException, GitAPIException { | |||
| FileRepositoryBuilder builder = new FileRepositoryBuilder(); | |||
| Repository repository = builder.setGitDir(new File(localPath, ".git")) | |||
| .readEnvironment() | |||
| @@ -487,50 +516,50 @@ public class DVCUtils { | |||
| } | |||
| } | |||
| public static void dvcInit(String localPath) throws Exception { | |||
| public void dvcInit(String localPath) throws Exception { | |||
| String command = "dvc init"; | |||
| runCommand(command, localPath); | |||
| } | |||
| public static void dvcAdd(String localPath, String filePath) throws Exception { | |||
| public void dvcAdd(String localPath, String filePath) throws Exception { | |||
| String command = "dvc add " + filePath; | |||
| runCommand(command, localPath); | |||
| } | |||
| public static void dvcRemoteAdd(String localPath, String s3RemoteUrl) throws Exception { | |||
| public void dvcRemoteAdd(String localPath, String s3RemoteUrl) throws Exception { | |||
| String command = "dvc remote add -d myremote s3://" + s3RemoteUrl; | |||
| runCommand(command, localPath); | |||
| } | |||
| public static void dvcConfigS3Credentials(String localPath, String endpointurl) throws Exception { | |||
| public void dvcConfigS3Credentials(String localPath, String endpointurl) throws Exception { | |||
| String command = "dvc remote modify myremote endpointurl " + endpointurl; | |||
| runCommand(command, localPath); | |||
| } | |||
| public static void dvcConfigS3Credentials2(String localPath, String accessKeyId) throws Exception { | |||
| public void dvcConfigS3Credentials2(String localPath, String accessKeyId) throws Exception { | |||
| String command = "dvc remote modify myremote access_key_id " + accessKeyId; | |||
| runCommand(command, localPath); | |||
| } | |||
| public static void dvcConfigS3Credentials3(String localPath, String secretAccessKey) throws Exception { | |||
| public void dvcConfigS3Credentials3(String localPath, String secretAccessKey) throws Exception { | |||
| String command = "dvc remote modify myremote secret_access_key " + secretAccessKey; | |||
| runCommand(command, localPath); | |||
| } | |||
| public static void dvcPush(String localPath) throws Exception { | |||
| public void dvcPush(String localPath) throws Exception { | |||
| String command = "dvc push -v"; | |||
| runCommand(command, localPath); | |||
| } | |||
| // 更新的 dvcPull 方法 | |||
| public static void dvcPull(String localPath) throws Exception { | |||
| public void dvcPull(String localPath) throws Exception { | |||
| String command = "dvc pull"; | |||
| runCommand(command, localPath); | |||
| } | |||
| // 方法 | |||
| public static void dvcCheckout(String localPath) throws Exception { | |||
| public void dvcCheckout(String localPath) throws Exception { | |||
| String command = "dvc checkout"; | |||
| runCommand(command, localPath); | |||
| } | |||
| @@ -544,7 +573,7 @@ public class DVCUtils { | |||
| * @param branch 分支名称 | |||
| * @return 包含文件路径、名称和大小的List<Map<String, Object>> | |||
| */ | |||
| public static List<Map<String, Object>> getFileDetailsAfterGitPull(String localPath, String repoFolder, String branch, String filePath , String username, String password) { | |||
| public List<Map<String, Object>> getFileDetailsAfterGitPull(String localPath, String repoFolder, String branch, String filePath , String username, String password) { | |||
| List<Map<String, Object>> fileInfoList = new ArrayList<>(); | |||
| try { | |||