| @@ -141,12 +141,18 @@ export default [ | |||
| { | |||
| name: '实验对比', | |||
| path: 'compare', | |||
| component: './Experiment/Comparison/index', | |||
| }, | |||
| { | |||
| name: '实验可视化对比', | |||
| path: 'compare-visual', | |||
| component: './Experiment/Aim/index', | |||
| routes: [ | |||
| { | |||
| name: '实验对比', | |||
| path: '', | |||
| component: './Experiment/Comparison/index', | |||
| }, | |||
| { | |||
| name: '可视化对比', | |||
| path: 'compare-visual', | |||
| component: './Experiment/Aim/index', | |||
| }, | |||
| ], | |||
| }, | |||
| { | |||
| name: '可视化', | |||
| @@ -223,7 +229,18 @@ export default [ | |||
| { | |||
| name: '实验实例详情', | |||
| path: 'instance/:experimentId/:id', | |||
| component: './HyperParameter/Instance/index', | |||
| routes: [ | |||
| { | |||
| name: '实验实例详情', | |||
| path: '', | |||
| component: './HyperParameter/Instance/index', | |||
| }, | |||
| { | |||
| name: '可视化对比', | |||
| path: 'compare-visual', | |||
| component: './HyperParameter/Aim/index', | |||
| }, | |||
| ], | |||
| }, | |||
| ], | |||
| }, | |||
| @@ -21,6 +21,7 @@ import { | |||
| } from './services/session'; | |||
| import './styles/menu.less'; | |||
| import { needAuth } from './utils'; | |||
| // import { closeAllModals } from './utils/modal'; | |||
| import { gotoLoginPage } from './utils/ui'; | |||
| export { requestConfig as request } from './requestConfig'; | |||
| @@ -96,6 +97,7 @@ export const layout: RuntimeConfig['layout'] = ({ initialState }) => { | |||
| }, | |||
| onPageChange: () => { | |||
| const { location } = history; | |||
| // closeAllModals(); | |||
| // 如果没有登录,重定向到 login | |||
| if (!initialState?.currentUser && needAuth(location.pathname)) { | |||
| gotoLoginPage(); | |||
| @@ -98,7 +98,7 @@ function ResourcePage({ resourceType }: ResourcePageProps) { | |||
| dataType={activeType} | |||
| dataTag={activeTag} | |||
| initialSearchText={cacheState?.searchText} | |||
| initialPagination={cacheState?.initialPagination} | |||
| initialPagination={cacheState?.pagination} | |||
| setCacheState={setCacheState} | |||
| ></ResourceList> | |||
| </Flex> | |||
| @@ -77,7 +77,7 @@ function ExperimentComparison() { | |||
| const url = res.data; | |||
| // window.open(url, '_blank'); | |||
| SessionStorage.setItem(SessionStorage.aimUrlKey, url); | |||
| navigate('../compare-visual'); | |||
| navigate('compare-visual'); | |||
| } | |||
| }; | |||
| @@ -3,7 +3,6 @@ import { ExperimentStatus } from '@/enums'; | |||
| import { useStateRef } from '@/hooks/useStateRef'; | |||
| import { useVisible } from '@/hooks/useVisible'; | |||
| import { getExperimentIns } from '@/services/experiment/index.js'; | |||
| import { getWorkflowById } from '@/services/pipeline/index.js'; | |||
| import themes from '@/styles/theme.less'; | |||
| import { fittingString, parseJsonText } from '@/utils'; | |||
| import { formatDate } from '@/utils/date'; | |||
| @@ -40,7 +39,7 @@ function ExperimentText() { | |||
| useEffect(() => { | |||
| initGraph(); | |||
| getWorkflow(); | |||
| getExperimentInstance(); | |||
| return () => { | |||
| if (evtSourceRef.current) { | |||
| @@ -66,38 +65,54 @@ function ExperimentText() { | |||
| }, []); | |||
| // 获取流水线模版 | |||
| const getWorkflow = async () => { | |||
| const [res] = await to(getWorkflowById(locationParams.workflowId)); | |||
| if (res && res.data && res.data.dag) { | |||
| try { | |||
| const dag = JSON.parse(res.data.dag); | |||
| dag.nodes.forEach((item) => { | |||
| item.in_parameters = JSON.parse(item.in_parameters); | |||
| item.out_parameters = JSON.parse(item.out_parameters); | |||
| item.control_strategy = JSON.parse(item.control_strategy); | |||
| item.imgName = item.img.slice(0, item.img.length - 4); | |||
| }); | |||
| workflowRef.current = dag; | |||
| getExperimentInstance(); | |||
| } catch (error) { | |||
| // JSON.parse 错误 | |||
| console.error('JSON.parse error: ', error); | |||
| } | |||
| } | |||
| }; | |||
| // const getWorkflow = async () => { | |||
| // const [res] = await to(getWorkflowById(locationParams.workflowId)); | |||
| // if (res && res.data && res.data.dag) { | |||
| // try { | |||
| // const dag = JSON.parse(res.data.dag); | |||
| // dag.nodes.forEach((item) => { | |||
| // item.in_parameters = JSON.parse(item.in_parameters); | |||
| // item.out_parameters = JSON.parse(item.out_parameters); | |||
| // item.control_strategy = JSON.parse(item.control_strategy); | |||
| // item.imgName = item.img.slice(0, item.img.length - 4); | |||
| // }); | |||
| // workflowRef.current = dag; | |||
| // getExperimentInstance(); | |||
| // } catch (error) { | |||
| // // JSON.parse 错误 | |||
| // console.error('JSON.parse error: ', error); | |||
| // } | |||
| // } | |||
| // }; | |||
| // 获取实验实例 | |||
| const getExperimentInstance = async () => { | |||
| const [res] = await to(getExperimentIns(locationParams.id)); | |||
| if (res && res.data && workflowRef.current) { | |||
| if (res && res.data) { | |||
| setExperimentIns(res.data); | |||
| const { status, nodes_status, argo_ins_ns, argo_ins_name, finish_time } = res.data; | |||
| const workflowData = workflowRef.current; | |||
| const { status, nodes_status, argo_ins_ns, argo_ins_name, finish_time, dag } = res.data; | |||
| if (!dag) { | |||
| return; | |||
| } | |||
| const workflow = parseJsonText(dag); | |||
| const experimentStatusObjs = parseJsonText(nodes_status); | |||
| if (!workflow || !workflow.nodes) { | |||
| return; | |||
| } | |||
| workflow.nodes.forEach((item) => { | |||
| item.in_parameters = parseJsonText(item.in_parameters); | |||
| item.out_parameters = parseJsonText(item.out_parameters); | |||
| item.control_strategy = parseJsonText(item.control_strategy); | |||
| item.imgName = item.img.slice(0, item.img.length - 4); | |||
| }); | |||
| workflowRef.current = workflow; | |||
| if (experimentStatusObjs) { | |||
| // 更新各个节点 | |||
| workflowData.nodes.forEach((item) => { | |||
| const experimentNode = experimentStatusObjs?.[item.id]; | |||
| workflow.nodes.forEach((item) => { | |||
| const experimentNode = experimentStatusObjs[item.id]; | |||
| updateWorkflowNode(item, experimentNode); | |||
| }); | |||
| @@ -113,20 +128,20 @@ function ExperimentText() { | |||
| } | |||
| // 绘制图 | |||
| getGraphData(workflowData, true); | |||
| getGraphData(workflow, true); | |||
| if (status === ExperimentStatus.Pending) { | |||
| // 如果状态是 Pending, 打开第一个节点 | |||
| const node = workflowData.nodes[0]; | |||
| const node = workflow.nodes[0]; | |||
| if (node) { | |||
| setExperimentNodeData(node); | |||
| openPropsDrawer(); | |||
| } | |||
| } else if (status === ExperimentStatus.Running) { | |||
| // 如果状态是 Running,打开第一个运行中的节点,如果没有运行中的节点,则打开第一个节点 | |||
| // 如果状态是 Running,打开第一个 Running 或者 pending 的节点,如果没有,则打开第一个节点 | |||
| const node = | |||
| workflowData.nodes.find((item) => item.experimentStatus === ExperimentStatus.Running) ?? | |||
| workflowData.nodes[0]; | |||
| workflow.nodes.find((item) => item.experimentStatus === ExperimentStatus.Running || item.experimentStatus === ExperimentStatus.Pending) ?? | |||
| workflow.nodes[0]; | |||
| if (node) { | |||
| setExperimentNodeData(node); | |||
| openPropsDrawer(); | |||
| @@ -153,13 +168,13 @@ function ExperimentText() { | |||
| return; | |||
| } | |||
| try { | |||
| const dataJson = JSON.parse(data); | |||
| const dataJson = parseJsonText(data); | |||
| const statusData = dataJson?.result?.object?.status; | |||
| if (!statusData) { | |||
| return; | |||
| } | |||
| const { finishedAt, phase, nodes = {} } = statusData; | |||
| // 更新实验实例状态和结束时间 | |||
| // setExperimentIns((prev) => ({ | |||
| // ...prev, | |||
| @@ -11,9 +11,15 @@ type ExperimentParameterProps = { | |||
| function ExperimentParameter({ nodeData }: ExperimentParameterProps) { | |||
| // 控制策略 | |||
| const controlStrategyList = Object.entries(nodeData.control_strategy ?? {}).map( | |||
| ([key, value]) => ({ key, value }), | |||
| ); | |||
| // const controlStrategyList = Object.entries(nodeData.control_strategy ?? {}).map( | |||
| // ([key, value]) => ({ key, value }), | |||
| // ); | |||
| const nodeId = nodeData.id; | |||
| const hasTaskInfo = | |||
| nodeId && | |||
| !nodeId.startsWith('git-clone') && | |||
| !nodeId.startsWith('dataset-export') && | |||
| !nodeId.startsWith('model-export'); | |||
| // 输入参数 | |||
| const inParametersList = Object.entries(nodeData.in_parameters ?? {}).map(([key, value]) => ({ | |||
| @@ -74,54 +80,58 @@ function ExperimentParameter({ nodeData }: ExperimentParameterProps) { | |||
| > | |||
| <FormInfo /> | |||
| </Form.Item> | |||
| <div className={styles['experiment-parameter__title']}> | |||
| <SubAreaTitle | |||
| image={require('@/assets/img/duty-message.png')} | |||
| title="任务信息" | |||
| ></SubAreaTitle> | |||
| </div> | |||
| <Form.Item | |||
| label="镜像" | |||
| name="image" | |||
| rules={[ | |||
| { | |||
| required: true, | |||
| message: '请输入镜像', | |||
| }, | |||
| ]} | |||
| > | |||
| <FormInfo /> | |||
| </Form.Item> | |||
| <Form.Item label="工作目录" name="working_directory"> | |||
| <FormInfo /> | |||
| </Form.Item> | |||
| {hasTaskInfo && ( | |||
| <> | |||
| <div className={styles['experiment-parameter__title']}> | |||
| <SubAreaTitle | |||
| image={require('@/assets/img/duty-message.png')} | |||
| title="任务信息" | |||
| ></SubAreaTitle> | |||
| </div> | |||
| <Form.Item | |||
| label="镜像" | |||
| name="image" | |||
| rules={[ | |||
| { | |||
| required: true, | |||
| message: '请输入镜像', | |||
| }, | |||
| ]} | |||
| > | |||
| <FormInfo /> | |||
| </Form.Item> | |||
| <Form.Item label="工作目录" name="working_directory"> | |||
| <FormInfo /> | |||
| </Form.Item> | |||
| <Form.Item label="启动命令" name="command"> | |||
| <FormInfo textArea /> | |||
| </Form.Item> | |||
| <Form.Item | |||
| label="资源规格" | |||
| name="resources_standard" | |||
| rules={[ | |||
| { | |||
| required: true, | |||
| message: '请输入资源规格', | |||
| }, | |||
| ]} | |||
| > | |||
| <ParameterSelect dataType="resource" placeholder="请选择资源规格" display /> | |||
| </Form.Item> | |||
| <Form.Item label="挂载路径" name="mount_path"> | |||
| <FormInfo /> | |||
| </Form.Item> | |||
| <Form.Item label="环境变量" name="env_variables"> | |||
| <FormInfo textArea /> | |||
| </Form.Item> | |||
| {controlStrategyList.map((item) => ( | |||
| <Form.Item key={item.key} name={['control_strategy', item.key]} label={item.value.label}> | |||
| <FormInfo valuePropName="showValue" /> | |||
| </Form.Item> | |||
| ))} | |||
| <Form.Item label="启动命令" name="command"> | |||
| <FormInfo textArea /> | |||
| </Form.Item> | |||
| <Form.Item | |||
| label="资源规格" | |||
| name="resources_standard" | |||
| rules={[ | |||
| { | |||
| required: true, | |||
| message: '请输入资源规格', | |||
| }, | |||
| ]} | |||
| > | |||
| <ParameterSelect dataType="resource" placeholder="请选择资源规格" display /> | |||
| </Form.Item> | |||
| {/* <Form.Item label="挂载路径" name="mount_path"> | |||
| <FormInfo /> | |||
| </Form.Item> */} | |||
| <Form.Item label="环境变量" name="env_variables"> | |||
| <FormInfo textArea /> | |||
| </Form.Item> | |||
| {/* {controlStrategyList.map((item) => ( | |||
| <Form.Item key={item.key} name={['control_strategy', item.key]} label={item.value.label}> | |||
| <FormInfo valuePropName="showValue" /> | |||
| </Form.Item> | |||
| ))} */} | |||
| </> | |||
| )} | |||
| <div className={styles['experiment-parameter__title']}> | |||
| <SubAreaTitle | |||
| image={require('@/assets/img/duty-message.png')} | |||
| @@ -121,6 +121,8 @@ function ExportModelModal({ | |||
| const params = { | |||
| ...formData, | |||
| identifier: resource?.identifier, | |||
| owner: resource?.owner, | |||
| is_public: resource?.is_public, | |||
| name: resource?.name, | |||
| [config.sourceParamKey]: DataSource.HandExport, | |||
| train_task: { | |||
| @@ -174,6 +176,8 @@ function ExportModelModal({ | |||
| onChange={handleResourceChange} | |||
| options={resources} | |||
| fieldNames={{ label: 'name', value: 'id' }} | |||
| optionFilterProp="name" | |||
| showSearch | |||
| allowClear | |||
| ></Select> | |||
| </Form.Item> | |||
| @@ -0,0 +1,12 @@ | |||
| /* | |||
| * @Author: 赵伟 | |||
| * @Date: 2025-03-31 16:38:59 | |||
| * @Description: 实验对比 Aim | |||
| */ | |||
| import IframePage, { IframePageType } from '@/components/IFramePage'; | |||
| function AimPage() { | |||
| return <IframePage type={IframePageType.Aim}></IframePage>; | |||
| } | |||
| export default AimPage; | |||
| @@ -3,7 +3,9 @@ import TrialStatusCell from '@/pages/HyperParameter/components/TrialStatusCell'; | |||
| import { HyperParameterTrial } from '@/pages/HyperParameter/types'; | |||
| import { getExpMetricsReq } from '@/services/hyperParameter'; | |||
| import { to } from '@/utils/promise'; | |||
| import SessionStorage from '@/utils/sessionStorage'; | |||
| import tableCellRender, { TableCellValueType } from '@/utils/table'; | |||
| import { useNavigate } from '@umijs/max'; | |||
| import { App, Button, Table, type TableProps } from 'antd'; | |||
| import classNames from 'classnames'; | |||
| import { useEffect, useState } from 'react'; | |||
| @@ -36,6 +38,7 @@ function ExperimentHistory({ trialList = [] }: ExperimentHistoryProps) { | |||
| const metricAnalysis: Record<string, any> = first?.metric_analysis ?? {}; | |||
| const paramsNames = Object.keys(config); | |||
| const metricNames = Object.keys(metricAnalysis); | |||
| const navigate = useNavigate(); | |||
| const trialColumns: TableProps<HyperParameterTrial>['columns'] = [ | |||
| { | |||
| @@ -160,7 +163,8 @@ function ExperimentHistory({ trialList = [] }: ExperimentHistoryProps) { | |||
| const [res] = await to(getExpMetricsReq(selectedRowKeys)); | |||
| if (res && res.data) { | |||
| const url = res.data; | |||
| window.open(url, '_blank'); | |||
| SessionStorage.setItem(SessionStorage.aimUrlKey, url); | |||
| navigate('compare-visual'); | |||
| } | |||
| }; | |||
| @@ -20,5 +20,9 @@ | |||
| &__more { | |||
| padding: 20px 0; | |||
| } | |||
| &::-webkit-scrollbar-thumb { | |||
| background: rgba(255, 255, 255, 0.5); | |||
| } | |||
| } | |||
| } | |||
| @@ -35,6 +35,12 @@ const PipelineNodeParameter = forwardRef(({ onFormChange }: PipelineNodeParamete | |||
| const [stagingItem, setStagingItem] = useState<PipelineNodeModelSerialize>( | |||
| {} as PipelineNodeModelSerialize, | |||
| ); | |||
| const nodeId = Form.useWatch('id', form) as string; | |||
| const hasTaskInfo = | |||
| nodeId && | |||
| !nodeId.startsWith('git-clone') && | |||
| !nodeId.startsWith('dataset-export') && | |||
| !nodeId.startsWith('model-export'); | |||
| const [open, setOpen] = useState(false); | |||
| const [menuItems, setMenuItems] = useState<MenuProps['items']>([]); | |||
| @@ -46,10 +52,11 @@ const PipelineNodeParameter = forwardRef(({ onFormChange }: PipelineNodeParamete | |||
| const fields = form.getFieldsValue(); | |||
| // 保存字段顺序 | |||
| const control_strategy = { | |||
| ...stagingItem.control_strategy, | |||
| ...fields.control_strategy, | |||
| }; | |||
| // const control_strategy = { | |||
| // ...stagingItem.control_strategy, | |||
| // ...fields.control_strategy, | |||
| // }; | |||
| const in_parameters = { | |||
| ...stagingItem.in_parameters, | |||
| ...fields.in_parameters, | |||
| @@ -64,7 +71,7 @@ const PipelineNodeParameter = forwardRef(({ onFormChange }: PipelineNodeParamete | |||
| const res = { | |||
| ...stagingItem, | |||
| ...fields, | |||
| control_strategy: JSON.stringify(control_strategy), | |||
| // control_strategy: JSON.stringify(control_strategy), | |||
| in_parameters: JSON.stringify(in_parameters), | |||
| out_parameters: JSON.stringify(out_parameters), | |||
| formError: !!error, | |||
| @@ -91,7 +98,7 @@ const PipelineNodeParameter = forwardRef(({ onFormChange }: PipelineNodeParamete | |||
| ...model, | |||
| in_parameters: JSON.parse(model.in_parameters), | |||
| out_parameters: JSON.parse(model.out_parameters), | |||
| control_strategy: JSON.parse(model.control_strategy), | |||
| // control_strategy: JSON.parse(model.control_strategy), | |||
| }; | |||
| // console.log('model', nodeData); | |||
| setStagingItem({ | |||
| @@ -348,9 +355,9 @@ const PipelineNodeParameter = forwardRef(({ onFormChange }: PipelineNodeParamete | |||
| }; | |||
| // 控制策略 | |||
| const controlStrategyList = Object.entries(stagingItem.control_strategy ?? {}).map( | |||
| ([key, value]) => ({ key, value }), | |||
| ); | |||
| // const controlStrategyList = Object.entries(stagingItem.control_strategy ?? {}).map( | |||
| // ([key, value]) => ({ key, value }), | |||
| // ); | |||
| // 输入参数 | |||
| const inParametersList = Object.entries(stagingItem.in_parameters ?? {}).map(([key, value]) => ({ | |||
| @@ -422,71 +429,73 @@ const PipelineNodeParameter = forwardRef(({ onFormChange }: PipelineNodeParamete | |||
| > | |||
| <Input disabled /> | |||
| </Form.Item> | |||
| <div className={styles['pipeline-drawer__title']}> | |||
| <SubAreaTitle | |||
| image={require('@/assets/img/duty-message.png')} | |||
| title="任务信息" | |||
| ></SubAreaTitle> | |||
| </div> | |||
| <Form.Item label="镜像" required> | |||
| <div className={styles['pipeline-drawer__ref-row']}> | |||
| <Form.Item name="image" noStyle rules={[{ required: true, message: '请输入镜像' }]}> | |||
| <Input placeholder="请输入或选择镜像" allowClear /> | |||
| {hasTaskInfo && ( | |||
| <> | |||
| <div className={styles['pipeline-drawer__title']}> | |||
| <SubAreaTitle | |||
| image={require('@/assets/img/duty-message.png')} | |||
| title="任务信息" | |||
| ></SubAreaTitle> | |||
| </div> | |||
| <Form.Item label="镜像" required> | |||
| <div className={styles['pipeline-drawer__ref-row']}> | |||
| <Form.Item name="image" noStyle rules={[{ required: true, message: '请输入镜像' }]}> | |||
| <Input placeholder="请输入或选择镜像" allowClear /> | |||
| </Form.Item> | |||
| <Form.Item noStyle> | |||
| <Button | |||
| type="link" | |||
| size="small" | |||
| icon={getSelectBtnIcon({ item_type: 'image' })} | |||
| onClick={() => selectResource('image', { item_type: 'image' })} | |||
| className={styles['pipeline-drawer__ref-row__select-button']} | |||
| > | |||
| 选择镜像 | |||
| </Button> | |||
| </Form.Item> | |||
| </div> | |||
| </Form.Item> | |||
| <Form.Item noStyle> | |||
| <Button | |||
| type="link" | |||
| size="small" | |||
| icon={getSelectBtnIcon({ item_type: 'image' })} | |||
| onClick={() => selectResource('image', { item_type: 'image' })} | |||
| className={styles['pipeline-drawer__ref-row__select-button']} | |||
| > | |||
| 选择镜像 | |||
| </Button> | |||
| <Form.Item | |||
| name="working_directory" | |||
| label={ | |||
| <PropsLabel | |||
| menuItems={menuItems} | |||
| title="工作目录" | |||
| onClick={(value) => { | |||
| handleParameterClick('working_directory', value); | |||
| }} | |||
| /> | |||
| } | |||
| > | |||
| <Input placeholder="请输入工作目录" allowClear /> | |||
| </Form.Item> | |||
| </div> | |||
| </Form.Item> | |||
| <Form.Item | |||
| name="working_directory" | |||
| label={ | |||
| <PropsLabel | |||
| menuItems={menuItems} | |||
| title="工作目录" | |||
| onClick={(value) => { | |||
| handleParameterClick('working_directory', value); | |||
| }} | |||
| /> | |||
| } | |||
| > | |||
| <Input placeholder="请输入工作目录" allowClear /> | |||
| </Form.Item> | |||
| <Form.Item | |||
| name="command" | |||
| label={ | |||
| <PropsLabel | |||
| menuItems={menuItems} | |||
| title="启动命令" | |||
| onClick={(value) => { | |||
| handleParameterClick('command', value); | |||
| }} | |||
| /> | |||
| } | |||
| > | |||
| <TextArea placeholder="请输入启动命令" allowClear /> | |||
| </Form.Item> | |||
| <Form.Item | |||
| label="资源规格" | |||
| name="resources_standard" | |||
| rules={[ | |||
| { | |||
| required: true, | |||
| message: '请选择资源规格', | |||
| }, | |||
| ]} | |||
| > | |||
| <ParameterSelect dataType="resource" placeholder="请选择资源规格" /> | |||
| </Form.Item> | |||
| <Form.Item | |||
| <Form.Item | |||
| name="command" | |||
| label={ | |||
| <PropsLabel | |||
| menuItems={menuItems} | |||
| title="启动命令" | |||
| onClick={(value) => { | |||
| handleParameterClick('command', value); | |||
| }} | |||
| /> | |||
| } | |||
| > | |||
| <TextArea placeholder="请输入启动命令" allowClear /> | |||
| </Form.Item> | |||
| <Form.Item | |||
| label="资源规格" | |||
| name="resources_standard" | |||
| rules={[ | |||
| { | |||
| required: true, | |||
| message: '请选择资源规格', | |||
| }, | |||
| ]} | |||
| > | |||
| <ParameterSelect dataType="resource" placeholder="请选择资源规格" /> | |||
| </Form.Item> | |||
| {/* <Form.Item | |||
| name="mount_path" | |||
| label={ | |||
| <PropsLabel | |||
| @@ -499,23 +508,23 @@ const PipelineNodeParameter = forwardRef(({ onFormChange }: PipelineNodeParamete | |||
| } | |||
| > | |||
| <Input placeholder="请输入挂载路径" allowClear /> | |||
| </Form.Item> | |||
| <Form.Item | |||
| name="env_variables" | |||
| label={ | |||
| <PropsLabel | |||
| menuItems={menuItems} | |||
| title="环境变量" | |||
| onClick={(value) => { | |||
| handleParameterClick('env_variables', value); | |||
| }} | |||
| /> | |||
| } | |||
| > | |||
| <TextArea placeholder="请输入环境变量" allowClear /> | |||
| </Form.Item> | |||
| {/* 控制参数 */} | |||
| {controlStrategyList.map((item) => ( | |||
| </Form.Item> */} | |||
| <Form.Item | |||
| name="env_variables" | |||
| label={ | |||
| <PropsLabel | |||
| menuItems={menuItems} | |||
| title="环境变量" | |||
| onClick={(value) => { | |||
| handleParameterClick('env_variables', value); | |||
| }} | |||
| /> | |||
| } | |||
| > | |||
| <TextArea placeholder="请输入环境变量" allowClear /> | |||
| </Form.Item> | |||
| {/* 控制参数 */} | |||
| {/* {controlStrategyList.map((item) => ( | |||
| <Form.Item | |||
| key={item.key} | |||
| name={['control_strategy', item.key]} | |||
| @@ -525,7 +534,10 @@ const PipelineNodeParameter = forwardRef(({ onFormChange }: PipelineNodeParamete | |||
| > | |||
| <ParameterInput placeholder={item.value.placeholder} allowClear></ParameterInput> | |||
| </Form.Item> | |||
| ))} | |||
| ))} */} | |||
| </> | |||
| )} | |||
| {/* 输入参数 */} | |||
| {inParametersList.length > 0 && ( | |||
| <> | |||
| @@ -37,7 +37,18 @@ | |||
| &__summary { | |||
| display: flex; | |||
| flex-direction: column; | |||
| width: 33.33%; | |||
| width: 40%; | |||
| text-align: left; | |||
| &:nth-child(3n+2) { | |||
| text-align: center; | |||
| width: 30%; | |||
| } | |||
| &:nth-child(3n) { | |||
| text-align: right; | |||
| width: 30%; | |||
| } | |||
| &__title { | |||
| margin-bottom: 12px; | |||
| @@ -16,7 +16,7 @@ function AssetsManagement() { | |||
| }; | |||
| const [res] = await to(getWorkspaceAssetCountReq(params)); | |||
| if (res && res.data) { | |||
| const { dataset, image, model, workflow } = res.data; | |||
| const { dataset, image, model, workflow, codeConfig, service } = res.data; | |||
| const items = [ | |||
| { | |||
| title: '数据集', | |||
| @@ -42,6 +42,14 @@ function AssetsManagement() { | |||
| title: '流水线模版', | |||
| value: workflow, | |||
| }, | |||
| { | |||
| title: '代码配置', | |||
| value: codeConfig, | |||
| }, | |||
| { | |||
| title: '服务', | |||
| value: service, | |||
| }, | |||
| ]; | |||
| setAssetCounts(items); | |||
| } | |||
| @@ -53,7 +61,7 @@ function AssetsManagement() { | |||
| return ( | |||
| <div className={styles['assets-management']}> | |||
| <Flex justify="space-between"> | |||
| <div className={styles['assets-management__title']}>AI资产</div> | |||
| <div className={styles['assets-management__title']}>多形态资源库</div> | |||
| <Select | |||
| size="small" | |||
| value={type} | |||
| @@ -67,11 +75,11 @@ function AssetsManagement() { | |||
| /> | |||
| </Flex> | |||
| {/* <div className={styles['assets-management__increase']}>今日新增数量:5</div> */} | |||
| <Flex gap="22px 0" wrap="wrap" style={{ marginTop: '40px' }}> | |||
| <Flex gap="22px 0" wrap="wrap" style={{ marginTop: '40px', padding: '0 8px' }}> | |||
| {assetCounts.map((item, index) => ( | |||
| <div className={styles['assets-management__summary']} key={index}> | |||
| <div className={styles['assets-management__summary__title']}>{item.title}</div> | |||
| <div className={styles['assets-management__summary__value']}>{item.value}</div> | |||
| <div className={styles['assets-management__summary__value']}>{item.value ?? '-'}</div> | |||
| </div> | |||
| ))} | |||
| </Flex> | |||
| @@ -1,6 +1,6 @@ | |||
| .experiment-table { | |||
| flex: 1; | |||
| min-width: 378px; | |||
| min-width: 460px; | |||
| height: 140px; | |||
| padding: 12px; | |||
| background: @workspace-background; | |||
| @@ -32,7 +32,7 @@ | |||
| } | |||
| &__status { | |||
| width: 15%; | |||
| width: 25%; | |||
| } | |||
| &__duration { | |||
| @@ -40,11 +40,11 @@ | |||
| } | |||
| &__date { | |||
| width: calc(60% - 60px); | |||
| width: 40%; | |||
| } | |||
| &__operation { | |||
| width: 60px; | |||
| width: 10%; | |||
| :global { | |||
| .ant-btn-link { | |||
| @@ -55,7 +55,7 @@ function QuickStart() { | |||
| <WorkFlow | |||
| content="为开发者提供数据智能标注与数据回流服务" | |||
| buttonText="数据标注" | |||
| buttonTop={40} | |||
| buttonTop={20} | |||
| x={left} | |||
| y={309} | |||
| onClick={() => navigate('/datasetPreparation/datasetAnnotation')} | |||
| @@ -80,7 +80,7 @@ function QuickStart() { | |||
| <WorkFlow | |||
| content="开发者可以在这里运行流水线模板,产生实验实例,对比实验训练过程与产生的实验训练数据" | |||
| buttonText="实验" | |||
| buttonTop={40} | |||
| buttonTop={20} | |||
| x={left + 2 * (192 + space)} | |||
| y={295} | |||
| onClick={() => navigate('/pipeline/experiment')} | |||
| @@ -3,16 +3,16 @@ | |||
| align-items: center; | |||
| justify-content: center; | |||
| height: 140px; | |||
| padding: 0 16px; | |||
| padding: 0 35px; | |||
| // 媒体查询 | |||
| @media screen and (max-width: 1600px) { | |||
| flex: auto; | |||
| } | |||
| // // 媒体查询 | |||
| // @media screen and (max-width: 1600px) { | |||
| // flex: auto; | |||
| // } | |||
| &__icon { | |||
| width: 63px; | |||
| margin-right: 16px; | |||
| width: 80px; | |||
| margin-right: 20px; | |||
| } | |||
| &__title { | |||
| @@ -7,6 +7,7 @@ | |||
| height: 228px; | |||
| padding: 0 20px; | |||
| .backgroundFullImage(url(@/assets/img/user-points-bg.png)); | |||
| margin-bottom: 16px; | |||
| &__label { | |||
| margin-top: 60px; | |||
| @@ -4,9 +4,9 @@ function WorkspaceIntro() { | |||
| return ( | |||
| <div className={styles['workspace-intro']}> | |||
| <div className={styles['workspace-intro__left']}> | |||
| <div className={styles['workspace-intro__title']}>自主实验平台</div> | |||
| <div className={styles['workspace-intro__title']}>智能材料科研平台</div> | |||
| <div className={styles['workspace-intro__content']}> | |||
| 材料领域的自主实验系统是一种用于材料研究和开发的技术平台,它旨在提供实验数据收集、分析和可视化等功能, | |||
| 智能材料科研平台是用于材料研究和开发的技术平台,它旨在提供实验数据收集、分析和可视化等功能, | |||
| 以支持材料工程师、科学家和研究人员在材料设计、性能评估和工艺优化方面的工作 | |||
| </div> | |||
| {/* <div className={styles['workspace-intro__buttons']}> | |||
| @@ -27,7 +27,7 @@ | |||
| &__statistics { | |||
| flex: none; | |||
| min-width: 431px; | |||
| // min-width: 500px; | |||
| background: linear-gradient( | |||
| 123.08deg, | |||
| rgba(138, 138, 138, 0.06) 1.32%, | |||
| @@ -35,10 +35,10 @@ | |||
| ); | |||
| border-radius: 4px; | |||
| // 媒体查询 | |||
| @media screen and (max-width: 1600px) { | |||
| flex: 1; | |||
| } | |||
| // // 媒体查询 | |||
| // @media screen and (max-width: 1600px) { | |||
| // flex: 1; | |||
| // } | |||
| } | |||
| } | |||
| } | |||
| @@ -12,7 +12,6 @@ import QuickStart from './components/QuickStart'; | |||
| // import RobotFrame from './components/RobotFrame'; | |||
| import TotalStatistics from './components/TotalStatistics'; | |||
| import UserPoints from './components/UserPoints'; | |||
| import UserSpace from './components/UserSpace'; | |||
| import WorkspaceIntro from './components/WorkspaceIntro'; | |||
| import styles from './index.less'; | |||
| @@ -67,7 +66,6 @@ function Workspace() { | |||
| count={overviewData?.runningExperimentInsCount} | |||
| /> | |||
| </Flex> | |||
| <ExperitableTable | |||
| tableData={overviewData?.latestExperimentInsList || []} | |||
| ></ExperitableTable> | |||
| @@ -76,12 +74,12 @@ function Workspace() { | |||
| )} | |||
| </div> | |||
| </div> | |||
| <UserPoints /> | |||
| </Flex> | |||
| <div className={styles['workspace__quick-start']}> | |||
| <QuickStart></QuickStart> | |||
| <div className={styles['workspace__user']}> | |||
| <UserSpace users={users}></UserSpace> | |||
| <UserPoints /> | |||
| {/* <UserSpace users={users}></UserSpace> */} | |||
| <AssetsManagement></AssetsManagement> | |||
| </div> | |||
| </div> | |||
| @@ -112,9 +112,9 @@ export type KeysToCamelCase<T> = { | |||
| // 序列化后的流水线节点 | |||
| export type PipelineNodeModelSerialize = Omit< | |||
| PipelineNodeModel, | |||
| 'control_strategy' | 'in_parameters' | 'out_parameters' | |||
| 'in_parameters' | 'out_parameters' | |||
| > & { | |||
| control_strategy: Record<string, PipelineNodeModelParameter>; | |||
| // control_strategy: Record<string, PipelineNodeModelParameter>; | |||
| in_parameters: Record<string, PipelineNodeModelParameter>; | |||
| out_parameters: Record<string, PipelineNodeModelParameter>; | |||
| }; | |||
| @@ -0,0 +1,110 @@ | |||
| package com.ruoyi.platform.config; | |||
| import java.util.HashMap; | |||
| import java.util.Map; | |||
| import javax.sql.DataSource; | |||
| import org.slf4j.Logger; | |||
| import org.slf4j.LoggerFactory; | |||
| import org.springframework.beans.factory.annotation.Qualifier; | |||
| import org.springframework.beans.factory.annotation.Value; | |||
| import org.springframework.boot.context.properties.ConfigurationProperties; | |||
| import org.springframework.context.annotation.Bean; | |||
| import org.springframework.context.annotation.Configuration; | |||
| import org.springframework.jdbc.core.JdbcTemplate; | |||
| import org.springframework.jdbc.datasource.DataSourceTransactionManager; | |||
| import com.alibaba.druid.pool.DruidDataSource; | |||
| @Configuration | |||
| public class MultiDSConfig { | |||
| private Logger logger = LoggerFactory.getLogger(this.getClass()); | |||
| @Value("${mysql.datasource.master.url}") | |||
| private String masterUrl; | |||
| @Value("${mysql.datasource.master.username}") | |||
| private String username; | |||
| @Value("${mysql.datasource.master.password}") | |||
| private String password; | |||
| @Value("${mysql.datasource.master.driverClassName}") | |||
| private String driverClassName; | |||
| @Value("${mysql.datasource.master.initialSize}") | |||
| private int initialSize; | |||
| @Value("${mysql.datasource.master.minIdle}") | |||
| private int minIdle; | |||
| @Value("${mysql.datasource.master.maxActive}") | |||
| private int maxActive; | |||
| @Value("${mysql.datasource.master.maxWait}") | |||
| private int maxWait; | |||
| // 故意注释 | |||
| @Value("${mysql.datasource.master.timeBetweenEvictionRunsMillis}") | |||
| private int timeBetweenEvictionRunsMillis; | |||
| @Value("${mysql.datasource.master.validationQuery}") | |||
| private String validationQuery; | |||
| @Value("${mysql.datasource.master.testWhileIdle}") | |||
| private boolean testWhileIdle; | |||
| @Value("${mysql.datasource.master.testOnBorrow}") | |||
| private boolean testOnBorrow; | |||
| @Value("${mysql.datasource.master.testOnReturn}") | |||
| private boolean testOnReturn; | |||
| @Value("${mysql.datasource.master.ConnectionErrorRetryAttempts}") | |||
| private int connectionErrorRetryAttempts; | |||
| @Value("${mysql.datasource.master.NotFullTimeoutRetryCount}") | |||
| private int notFullTimeoutRetryCount; | |||
| // @Value("${mysql.datasource.master.removeAbandonedTimeoutMillis}") | |||
| // private long removeAbandonedTimeoutMillis; | |||
| // @Value("${mysql.datasource.master.removeAbandoned}") | |||
| // private boolean removeAbandoned; | |||
| @Value("${mysql.datasource.master.keepalive}") | |||
| private boolean keepAlive = true; | |||
| @Value("${mysql.datasource.master.minEvictableIdleTimeMillis}") | |||
| private int minEvictableIdleTimeMillis; | |||
| @Value("${mysql.datasource.master.maxEvictableIdleTimeMillis}") | |||
| private int maxEvictableIdleTimeMillis; | |||
| @Value("${mysql.datasource.master.keepAliveBetweenTimeMillis}") | |||
| private int keepAliveBetweenTimeMillis; | |||
| @Bean | |||
| @ConfigurationProperties(prefix = "mysql.datasource.master") | |||
| public DataSource masterDataSource() { | |||
| DruidDataSource datasource = new DruidDataSource(); | |||
| datasource.setUrl(masterUrl); | |||
| datasource.setUsername(username); | |||
| datasource.setPassword(password); | |||
| datasource.setDriverClassName(driverClassName); | |||
| // configuration | |||
| datasource.setInitialSize(initialSize); | |||
| datasource.setMinIdle(minIdle); | |||
| datasource.setMaxActive(maxActive); | |||
| datasource.setMaxWait(maxWait); | |||
| // 故意注释下面这一句 | |||
| datasource.setTimeBetweenEvictionRunsMillis(timeBetweenEvictionRunsMillis); | |||
| logger.info(">>>>>>timeBetweenEvictionRunsMillis->" + timeBetweenEvictionRunsMillis); | |||
| // datasource.setMinEvictableIdleTimeMillis(minEvictableIdleTimeMillis); | |||
| datasource.setValidationQuery(validationQuery); | |||
| datasource.setTestWhileIdle(testWhileIdle); | |||
| datasource.setTestOnBorrow(testOnBorrow); | |||
| datasource.setTestOnReturn(testOnReturn); | |||
| // datasource.setPoolPreparedStatements(poolPreparedStatements); | |||
| // datasource.setMaxPoolPreparedStatementPerConnectionSize(maxPoolPreparedStatementPerConnectionSize); | |||
| datasource.setConnectionErrorRetryAttempts(connectionErrorRetryAttempts); | |||
| datasource.setNotFullTimeoutRetryCount(notFullTimeoutRetryCount); | |||
| //datasource.setRemoveAbandonedTimeoutMillis(removeAbandonedTimeoutMillis); | |||
| //datasource.setRemoveAbandoned(removeAbandoned); | |||
| datasource.setMinEvictableIdleTimeMillis(minEvictableIdleTimeMillis); | |||
| datasource.setMaxEvictableIdleTimeMillis(maxEvictableIdleTimeMillis); | |||
| datasource.setKeepAliveBetweenTimeMillis(keepAliveBetweenTimeMillis); | |||
| datasource.setKeepAlive(keepAlive); | |||
| return datasource; | |||
| } | |||
| @Bean | |||
| public JdbcTemplate dataSource(DataSource myRoutingDataSource) { | |||
| return new JdbcTemplate(myRoutingDataSource); | |||
| } | |||
| @Bean | |||
| public DataSourceTransactionManager txManager(DataSource myRoutingDataSource) { | |||
| return new DataSourceTransactionManager(myRoutingDataSource); | |||
| } | |||
| } | |||
| @@ -80,6 +80,8 @@ public class ExperimentIns implements Serializable { | |||
| @ApiModelProperty(value = "状态:0失效,1生效") | |||
| private Integer state; | |||
| @ApiModelProperty(value = "DAG图") | |||
| private String dag; | |||
| @ApiModelProperty(value = "实验实例对应的流水线ID") | |||
| private Long workflowId; | |||
| @@ -280,6 +280,7 @@ public class ExperimentServiceImpl implements ExperimentService { | |||
| //获取训练参数 | |||
| experimentIns.setExperimentId(experiment.getId()); | |||
| experimentIns.setDag(dag); | |||
| experimentIns.setArgoInsNs((String) metadata.get("namespace")); | |||
| experimentIns.setArgoInsName((String) metadata.get("name")); | |||
| experimentIns.setStatus(Constant.Pending); | |||
| @@ -104,9 +104,12 @@ public class NewDatasetServiceImpl implements NewDatasetService { | |||
| String gitLinkUsername = loginUser.getSysUser().getUserName(); | |||
| String gitLinkPassword = decrypt(loginUser.getSysUser().getOriginPassword()); | |||
| String userReq = jedis.get(ci4sUsername + "_gitUserInfo"); | |||
| // 得到用户操作的路径 | |||
| Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq); | |||
| Integer userId = (Integer) userInfo.get("user_id"); | |||
| String owner = (String) userInfo.get("login"); | |||
| // 拼接project | |||
| ci4sUsername = Boolean.TRUE.equals(datasetVo.getIsPublic()) ? Constant.Item_Public : loginUser.getUsername(); | |||
| String repositoryName = ci4sUsername + "_dataset_" + DateUtils.dateTimeNow(); | |||
| @@ -117,17 +120,13 @@ public class NewDatasetServiceImpl implements NewDatasetService { | |||
| gitProjectVo.setPrivate(!datasetVo.getIsPublic()); | |||
| gitProjectVo.setUserId(userId); | |||
| gitProjectVo.setProjectCategoryId(projectDatasetId); | |||
| // 创建项目 | |||
| Map project = gitService.createProject(token, gitProjectVo); | |||
| Integer gitlinIid = (Integer) project.get("id"); | |||
| if (gitlinIid == null) { | |||
| throw new Exception("创建数据集失败:" + project.get("message")); | |||
| } | |||
| // 创建分支 | |||
| String branchName = datasetVo.getVersion(); | |||
| String owner = (String) userInfo.get("login"); | |||
| gitService.createBranch(token, (String) userInfo.get("login"), repositoryName, branchName, "master"); | |||
| // 定义标签 标签1:dataset 标签2:DataTag 标签3:DataType | |||
| gitService.createTopic(token, gitlinIid, Constant.Topic_Dataset); | |||
| if (StringUtils.isNotEmpty(datasetVo.getDataTag())) { | |||
| @@ -137,23 +136,65 @@ public class NewDatasetServiceImpl implements NewDatasetService { | |||
| gitService.createTopic(token, gitlinIid, "datatype-" + datasetVo.getDataType()); | |||
| } | |||
| String relatePath = ci4sUsername + "/datasets/" + gitlinIid + "/" + repositoryName + "/master"; | |||
| String localPath = localPathlocal + relatePath; | |||
| String datasetPath = localPath + "/dataset"; | |||
| String s3Path = bucketName + "/mini-model-management-platform-files" + "/" + relatePath; | |||
| // 得到项目地址 | |||
| String projectUrl = gitCloneEndpoint + "/" + owner + "/" + repositoryName + ".git"; | |||
| // 创建master分支 | |||
| dvcUtils.gitClone(localPath, projectUrl, "master", gitLinkUsername, gitLinkPassword); | |||
| // 拼接生产的元数据后写入yaml文件 | |||
| datasetVo.setCreateBy(String.valueOf(StringUtils.isNotEmpty((String) userInfo.get("nickname")) ? userInfo.get("nickname") : userInfo.get("login"))); | |||
| datasetVo.setUpdateTime(DateUtils.getTime()); | |||
| datasetVo.setIdentifier(repositoryName); | |||
| datasetVo.setId(gitlinIid); | |||
| datasetVo.setOwner(owner); | |||
| datasetVo.setRelativePaths(relatePath + "/dataset"); | |||
| // addDatasetSourceToDataVo(datasetVo); | |||
| YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo), localPath, "dataset"); | |||
| // dvc init 初始化 | |||
| dvcUtils.mkdir(datasetPath); | |||
| dvcUtils.dvcInit(localPath); | |||
| // 配置远程S3地址 | |||
| dvcUtils.dvcRemoteAdd(localPath, s3Path); | |||
| dvcUtils.dvcConfigS3Credentials(localPath, endpoint); | |||
| dvcUtils.dvcConfigS3Credentials2(localPath, accessKeyId); | |||
| dvcUtils.dvcConfigS3Credentials3(localPath, secretAccessKey); | |||
| // dvc 跟踪 | |||
| dvcUtils.dvcAdd(localPath, "dataset"); | |||
| // git commit | |||
| dvcUtils.gitCommit(localPath, "commit from ci4s with " + loginUser.getUsername()); | |||
| dvcUtils.gitPush(localPath, gitLinkUsername, gitLinkPassword); | |||
| // dvc push 到远程S3 | |||
| dvcUtils.dvcPush(localPath); | |||
| // 创建分支 | |||
| String branchName = datasetVo.getVersion(); | |||
| // 得到用户操作的路径 | |||
| String sourcePath = datasetVo.getDatasetVersionVos().get(0).getUrl(); | |||
| String relatePath = ci4sUsername + "/datasets/" + gitlinIid + "/" + repositoryName + "/" + branchName; | |||
| String localPath = localPathlocal + relatePath; | |||
| String datasetPath = localPath + "/dataset"; | |||
| String relatePath1 = ci4sUsername + "/datasets/" + gitlinIid + "/" + repositoryName + "/" + branchName; | |||
| String localPath1 = localPathlocal + relatePath1; | |||
| String datasetPath1 = localPath1 + "/dataset"; | |||
| // 命令行操作 git clone 项目地址 | |||
| dvcUtils.gitClone(localPath, projectUrl, branchName, gitLinkUsername, gitLinkPassword); | |||
| String s3Path = bucketName + "/mini-model-management-platform-files" + "/" + relatePath + "/" + branchName; | |||
| //干掉目标文件夹 | |||
| dvcUtils.deleteDirectory(datasetPath); | |||
| dvcUtils.moveFiles(sourcePath, datasetPath); | |||
| dvcUtils.gitClone(localPath1, projectUrl, "master", gitLinkUsername, gitLinkPassword); | |||
| dvcUtils.refreshRemoteBranches(localPath1, gitLinkUsername, gitLinkPassword, "master"); | |||
| // 创建本地分支 | |||
| dvcUtils.createLocalBranchBasedOnMaster(localPath1, branchName); | |||
| //dvc checkout | |||
| dvcUtils.dvcPull(localPath1); | |||
| dvcUtils.dvcCheckout(localPath1); | |||
| String s3Path1 = bucketName + "/mini-model-management-platform-files" + "/" + relatePath1; | |||
| //删掉已存在文件夹 | |||
| dvcUtils.deleteDirectory(datasetPath1); | |||
| dvcUtils.moveFiles(sourcePath, datasetPath1); | |||
| // 拼接生产的元数据后写入yaml文件 | |||
| datasetVo.setCreateBy(String.valueOf(StringUtils.isNotEmpty((String) userInfo.get("nickname")) ? userInfo.get("nickname") : userInfo.get("login"))); | |||
| datasetVo.setUpdateTime(DateUtils.getTime()); | |||
| datasetVo.setUsage("<pre><code>" + | |||
| "# 克隆数据集配置文件与存储参数到本地\n" + | |||
| @@ -161,37 +202,28 @@ public class NewDatasetServiceImpl implements NewDatasetService { | |||
| "# 远程拉取配置文件\n" + | |||
| "dvc pull\n" + | |||
| "</code></pre>"); | |||
| datasetVo.setIdentifier(repositoryName); | |||
| datasetVo.setId(gitlinIid); | |||
| datasetVo.setOwner(owner); | |||
| datasetVo.setRelativePaths(relatePath + "/dataset"); | |||
| datasetVo.setRelativePaths(relatePath1 + "/dataset"); | |||
| YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo), localPath1, "dataset"); | |||
| addDatasetSourceToDataVo(datasetVo); | |||
| YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo), localPath, "dataset"); | |||
| CompletableFuture.supplyAsync(() -> { | |||
| try { | |||
| // dvc init 初始化 | |||
| dvcUtils.dvcInit(localPath); | |||
| // CompletableFuture.supplyAsync(() -> { | |||
| // try { | |||
| // 配置远程S3地址 | |||
| dvcUtils.dvcRemoteAdd(localPath, s3Path); | |||
| dvcUtils.dvcConfigS3Credentials(localPath, endpoint); | |||
| dvcUtils.dvcConfigS3Credentials2(localPath, accessKeyId); | |||
| dvcUtils.dvcConfigS3Credentials3(localPath, secretAccessKey); | |||
| dvcUtils.dvcRemoteAdd(localPath1, s3Path1); | |||
| dvcUtils.dvcConfigS3Credentials(localPath1, endpoint); | |||
| dvcUtils.dvcConfigS3Credentials2(localPath1, accessKeyId); | |||
| dvcUtils.dvcConfigS3Credentials3(localPath1, secretAccessKey); | |||
| // dvc 跟踪 | |||
| dvcUtils.dvcAdd(localPath, "dataset"); | |||
| dvcUtils.dvcAdd(localPath1, "dataset"); | |||
| // git commit | |||
| dvcUtils.gitCommit(localPath, "commit from ci4s with " + loginUser.getUsername()); | |||
| dvcUtils.gitPush(localPath, gitLinkUsername, gitLinkPassword); | |||
| dvcUtils.pushNewBranchToRemote(localPath1, gitLinkUsername, gitLinkPassword, branchName); | |||
| // dvc push 到远程S3 | |||
| dvcUtils.dvcPush(localPath); | |||
| } catch (Exception e) { | |||
| logger.error(e.getMessage(), e); | |||
| throw new RuntimeException(e.getMessage()); | |||
| } | |||
| return null; | |||
| }); | |||
| dvcUtils.dvcPush(localPath1); | |||
| // } catch (Exception e) { | |||
| // logger.error(e.getMessage(), e); | |||
| // throw new RuntimeException(e.getMessage()); | |||
| // } | |||
| // return null; | |||
| // }); | |||
| return "新增数据集成功"; | |||
| } | |||
| @@ -214,7 +246,7 @@ public class NewDatasetServiceImpl implements NewDatasetService { | |||
| ci4sUsername = Boolean.TRUE.equals(datasetVo.getIsPublic()) ? Constant.Item_Public : loginUser.getUsername(); | |||
| Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq); | |||
| // 创建分支 | |||
| String branchName = StringUtils.isEmpty(datasetVo.getVersion()) ? "master" : datasetVo.getVersion(); | |||
| String branchName = datasetVo.getVersion(); | |||
| String owner = (String) userInfo.get("login"); | |||
| String repositoryName = datasetVo.getIdentifier(); | |||
| @@ -232,14 +264,14 @@ public class NewDatasetServiceImpl implements NewDatasetService { | |||
| Map projectDetail = gitService.getProjectDetail(owner, repositoryName, token); | |||
| //克隆 | |||
| List<Map<String, Object>> brancheList = gitService.getBrancheList(token, owner, repositoryName); | |||
| String oldBranch = (String) brancheList.stream() | |||
| .filter(branch -> !"master".equals(branch.get("name"))).collect(Collectors.toList()).get(0).get("name"); | |||
| // List<Map<String, Object>> brancheList = gitService.getBrancheList(token, owner, repositoryName); | |||
| // String oldBranch = (String) brancheList.stream() | |||
| // .filter(branch -> !"master".equals(branch.get("name"))).collect(Collectors.toList()).get(0).get("name"); | |||
| dvcUtils.gitClone(localPath, projectUrl, oldBranch, gitLinkUsername, gitLinkPassword); | |||
| dvcUtils.refreshRemoteBranches(localPath, gitLinkUsername, gitLinkPassword, oldBranch); | |||
| dvcUtils.gitClone(localPath, projectUrl, "master", gitLinkUsername, gitLinkPassword); | |||
| dvcUtils.refreshRemoteBranches(localPath, gitLinkUsername, gitLinkPassword, "master"); | |||
| //检查是否存在本地重名分支,有的话干掉 | |||
| //检查是否存在本地重名分支,有的话删掉 | |||
| dvcUtils.deleteLocalBranch(localPath, branchName); | |||
| // 创建本地分支 | |||
| dvcUtils.createLocalBranchBasedOnMaster(localPath, branchName); | |||
| @@ -250,7 +282,7 @@ public class NewDatasetServiceImpl implements NewDatasetService { | |||
| // 准备数据 | |||
| String s3Path = bucketName + "/mini-model-management-platform-files" + "/" + relatePath + "/" + branchName; | |||
| //干掉目标文件夹 | |||
| //删掉已存在文件夹 | |||
| dvcUtils.deleteDirectory(datasetPath); | |||
| if (Constant.Source_Hand_Export.equals(datasetVo.getDatasetSource())) { | |||
| @@ -261,7 +293,6 @@ public class NewDatasetServiceImpl implements NewDatasetService { | |||
| dvcUtils.moveFiles(sourcePath, datasetPath); | |||
| } | |||
| // 拼接生产的元数据后写入yaml文件 | |||
| datasetVo.setCreateBy(String.valueOf(StringUtils.isNotEmpty((String) userInfo.get("nickname")) ? userInfo.get("nickname") : userInfo.get("login"))); | |||
| datasetVo.setUpdateTime(DateUtils.getTime()); | |||
| @@ -303,7 +334,6 @@ public class NewDatasetServiceImpl implements NewDatasetService { | |||
| // dvc 跟踪 | |||
| dvcUtils.dvcAdd(localPath, "dataset"); | |||
| dvcUtils.pushNewBranchToRemote(localPath, gitLinkUsername, gitLinkPassword, branchName); | |||
| dvcUtils.dvcPush(localPath); | |||
| // CompletableFuture.supplyAsync(() -> { | |||
| @@ -4,8 +4,6 @@ import com.ruoyi.common.security.utils.SecurityUtils; | |||
| import com.ruoyi.platform.domain.*; | |||
| import com.ruoyi.platform.mapper.*; | |||
| import com.ruoyi.platform.service.*; | |||
| import com.ruoyi.platform.utils.JacksonUtil; | |||
| import com.ruoyi.platform.utils.JsonUtils; | |||
| import com.ruoyi.platform.utils.NewHttpUtils; | |||
| import com.ruoyi.platform.vo.ModelsVo; | |||
| import com.ruoyi.platform.vo.NewDatasetVo; | |||
| @@ -15,7 +13,6 @@ import org.springframework.beans.factory.annotation.Value; | |||
| import org.springframework.data.domain.Page; | |||
| import org.springframework.data.domain.PageRequest; | |||
| import org.springframework.stereotype.Service; | |||
| import redis.clients.jedis.Jedis; | |||
| import javax.annotation.Resource; | |||
| import java.util.HashMap; | |||
| @@ -38,20 +35,13 @@ public class WorkspaceServiceImpl implements WorkspaceService { | |||
| @Resource | |||
| private ComponentDao componentDao; | |||
| @Resource | |||
| private GitService gitService; | |||
| private CodeConfigDao codeConfigDao; | |||
| @Resource | |||
| private NewDatasetService newDatasetService; | |||
| @Resource | |||
| private ModelsService modelsService; | |||
| @Resource | |||
| private NewHttpUtils httpUtils; | |||
| @Value("${git.endpoint}") | |||
| String gitendpoint; | |||
| @Value("${spring.redis.host}") | |||
| private String redisHost; | |||
| @Value("${spring.redis.port}") | |||
| private Integer redisPort; | |||
| private ServiceDao serviceDao; | |||
| @Override | |||
| public Map<String, Object> getOverview() { | |||
| @@ -126,7 +116,15 @@ public class WorkspaceServiceImpl implements WorkspaceService { | |||
| Workflow workflow = new Workflow(); | |||
| Integer workflowCount = (int) this.workflowDao.count(workflow); | |||
| assetCountMap.put("workflow", workflowCount); | |||
| // 统计代码配置数量 | |||
| CodeConfig codeConfig = new CodeConfig(); | |||
| codeConfig.setIsPublic(isPublic); | |||
| Integer codeConfigCount = (int) this.codeConfigDao.count(codeConfig); | |||
| assetCountMap.put("codeConfig", codeConfigCount); | |||
| // 统计服务数量 | |||
| com.ruoyi.platform.domain.Service service = new com.ruoyi.platform.domain.Service(); | |||
| Integer serviceCount = (int) serviceDao.countService(service); | |||
| assetCountMap.put("service", serviceCount); | |||
| return assetCountMap; | |||
| } | |||
| } | |||
| @@ -68,6 +68,13 @@ public class DVCUtils { | |||
| } | |||
| } | |||
| public void mkdir(String path) throws IOException { | |||
| Path targetDir = Paths.get(path); | |||
| if (!Files.exists(targetDir)) { | |||
| Files.createDirectories(targetDir); | |||
| } | |||
| } | |||
| public void moveFiles(String sourcePath, String targetPath) throws Exception { | |||
| Path sourceDir = Paths.get(sourcePath); | |||
| Path targetDir = Paths.get(targetPath); | |||
| @@ -112,6 +112,9 @@ | |||
| <if test="codeConfig.updateTime != null"> | |||
| and update_time = #{codeConfig.updateTime} | |||
| </if> | |||
| <if test="codeConfig.isPublic != null"> | |||
| and is_public = #{codeConfig.isPublic} | |||
| </if> | |||
| </where> | |||
| </sql> | |||
| </mapper> | |||
| @@ -8,6 +8,7 @@ | |||
| <result property="argoInsName" column="argo_ins_name" jdbcType="VARCHAR"/> | |||
| <result property="argoInsNs" column="argo_ins_ns" jdbcType="VARCHAR"/> | |||
| <result property="status" column="status" jdbcType="VARCHAR"/> | |||
| <result property="dag" column="dag" jdbcType="VARCHAR"/> | |||
| <result property="nodesStatus" column="nodes_status" jdbcType="VARCHAR"/> | |||
| <result property="nodesResult" column="nodes_result" jdbcType="VARCHAR"/> | |||
| <result property="nodesLogs" column="nodes_logs" jdbcType="VARCHAR"/> | |||
| @@ -36,6 +37,7 @@ | |||
| <select id="queryById" resultMap="ExperimentInsMap"> | |||
| select a.id, | |||
| experiment_id, | |||
| a.dag, | |||
| argo_ins_name, | |||
| argo_ins_ns, | |||
| status, | |||
| @@ -282,16 +284,16 @@ | |||
| <!--新增所有列--> | |||
| <insert id="insert" keyProperty="id" useGeneratedKeys="true"> | |||
| insert into experiment_ins(experiment_id,argo_ins_name,argo_ins_ns,status,nodes_status,nodes_result,nodes_logs,global_param,metric_record,metric_value,start_time,finish_time,create_by,create_time,update_by,update_time,state) | |||
| values (#{experimentIns.experimentId},#{experimentIns.argoInsName},#{experimentIns.argoInsNs},#{experimentIns.status},#{experimentIns.nodesStatus},#{experimentIns.nodesResult},#{experimentIns.nodesLogs},#{experimentIns.globalParam},#{experimentIns.metricRecord},#{experimentIns.metricValue},#{experimentIns.startTime},#{experimentIns.finishTime},#{experimentIns.createBy},#{experimentIns.createTime},#{experimentIns.updateBy},#{experimentIns.updateTime},#{experimentIns.state}) | |||
| insert into experiment_ins(experiment_id,argo_ins_name,argo_ins_ns,status,nodes_status,nodes_result,nodes_logs,global_param,metric_record,metric_value,start_time,finish_time,create_by,create_time,update_by,update_time,state,dag) | |||
| values (#{experimentIns.experimentId},#{experimentIns.argoInsName},#{experimentIns.argoInsNs},#{experimentIns.status},#{experimentIns.nodesStatus},#{experimentIns.nodesResult},#{experimentIns.nodesLogs},#{experimentIns.globalParam},#{experimentIns.metricRecord},#{experimentIns.metricValue},#{experimentIns.startTime},#{experimentIns.finishTime},#{experimentIns.createBy},#{experimentIns.createTime},#{experimentIns.updateBy},#{experimentIns.updateTime},#{experimentIns.state},#{experimentIns.dag}) | |||
| </insert> | |||
| <insert id="insertBatch" keyProperty="id" useGeneratedKeys="true"> | |||
| insert into | |||
| experiment_ins(experiment_id,argo_ins_name,argo_ins_ns,status,nodes_status,nodes_result,nodes_logs,global_param,metric_record,metric_value,start_time,finish_time,create_by,create_time,update_by,update_time,state) | |||
| experiment_ins(experiment_id,argo_ins_name,argo_ins_ns,status,nodes_status,nodes_result,nodes_logs,global_param,metric_record,metric_value,start_time,finish_time,create_by,create_time,update_by,update_time,state,dag) | |||
| values | |||
| <foreach collection="entities" item="entity" separator=","> | |||
| (#{entity.experimentId},#{entity.argoInsName},#{entity.argoInsNs},#{entity.status},#{entity.nodesStatus},#{entity.nodesResult},#{entity.nodesLogs},#{entity.globalParam},#{entity.metricRecord},#{entity.metricValue},#{entity.startTime},#{entity.finishTime},#{entity.createBy},#{entity.createTime},#{entity.updateBy},#{entity.updateTime},#{entity.state}) | |||
| (#{entity.experimentId},#{entity.argoInsName},#{entity.argoInsNs},#{entity.status},#{entity.nodesStatus},#{entity.nodesResult},#{entity.nodesLogs},#{entity.globalParam},#{entity.metricRecord},#{entity.metricValue},#{entity.startTime},#{entity.finishTime},#{entity.createBy},#{entity.createTime},#{entity.updateBy},#{entity.updateTime},#{entity.state},#{entity.dag}) | |||
| </foreach> | |||
| </insert> | |||