From ea5464f42cf1593333ff63d4e205483b8d7ab4e0 Mon Sep 17 00:00:00 2001 From: chenzhihang <709011834@qq.com> Date: Fri, 22 Nov 2024 09:46:34 +0800 Subject: [PATCH] =?UTF-8?q?=E6=8C=82=E8=BD=BD=E4=BF=AE=E6=94=B9=E4=B8=BAju?= =?UTF-8?q?icefs=E5=AD=98=E5=82=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- k8s/template-yaml/k8s-7management.yaml | 5 +- .../impl/DatasetVersionServiceImpl.java | 6 +- .../service/impl/ImageServiceImpl.java | 1 - .../service/impl/JupyterServiceImpl.java | 16 ++-- .../service/impl/TensorBoardServiceImpl.java | 14 +-- .../ruoyi/platform/utils/K8sClientUtil.java | 90 +++++++++++-------- .../com/ruoyi/platform/vo/FrameLogPathVo.java | 16 ++-- 7 files changed, 80 insertions(+), 68 deletions(-) diff --git a/k8s/template-yaml/k8s-7management.yaml b/k8s/template-yaml/k8s-7management.yaml index 4fcddb15..edc1c621 100644 --- a/k8s/template-yaml/k8s-7management.yaml +++ b/k8s/template-yaml/k8s-7management.yaml @@ -26,10 +26,11 @@ spec: volumeMounts: - name: resource-volume mountPath: /home/resource/ + subPath: mini-model-platform-data volumes: - name: resource-volume - persistentVolumeClaim: - claimName: platform-data-pvc-nfs + hostPath: + path: /platform-data --- apiVersion: v1 kind: Service diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/DatasetVersionServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/DatasetVersionServiceImpl.java index 441ea786..4a00ddb4 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/DatasetVersionServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/DatasetVersionServiceImpl.java @@ -165,9 +165,9 @@ public class DatasetVersionServiceImpl implements DatasetVersionService { .findFirst() .ifPresent(datasetVersion -> { String url = datasetVersion.getUrl(); -// response.put("path", url); - String path = bucketName + '/' + url.substring(0, url.lastIndexOf('/')); - response.put("path", path); + response.put("path", url); +// String path = bucketName + '/' + url.substring(0, url.lastIndexOf('/')); +// response.put("path", path); }); response.put("content", datasetVersionList); diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ImageServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ImageServiceImpl.java index fc02f1f6..8eb6ab68 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ImageServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ImageServiceImpl.java @@ -61,7 +61,6 @@ public class ImageServiceImpl implements ImageService { @Resource private MinioService minioService; -// @Value("${minio.dataReleaseBucketName}") @Value("${minio.dataReleaseBucketName}") private String bucketName; // @Value("${harbor.bucketName}") diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/JupyterServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/JupyterServiceImpl.java index fb3ffd89..87111365 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/JupyterServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/JupyterServiceImpl.java @@ -90,25 +90,25 @@ public class JupyterServiceImpl implements JupyterService { // 提取数据集,模型信息，得到数据集模型的path Map dataset = JacksonUtil.parseJSONStr2Map(devEnvironment.getDataset()); - String datasetPath = "argo-workflow" + "/" + dataset.get("path"); -// String datasetPath = (String) dataset.get("path"); +// String datasetPath = "argo-workflow" + "/" + dataset.get("path"); + String datasetPath = (String) dataset.get("path"); Map model = JacksonUtil.parseJSONStr2Map(devEnvironment.getModel()); - String modelPath = "argo-workflow" + "/" + model.get("path"); -// String modelPath = (String) model.get("path"); +// String modelPath = "argo-workflow" + "/" + model.get("path"); + String modelPath = (String) model.get("path"); LoginUser loginUser = SecurityUtils.getLoginUser(); //构造pod名称 String podName = loginUser.getUsername().toLowerCase() + "-editor-pod" + "-" + id; //新建编辑器的pvc - String pvcName = loginUser.getUsername().toLowerCase() + "-editor-pvc"; - V1PersistentVolumeClaim pvc = k8sClientUtil.createPvc(namespace, pvcName, storage, storageClassName); +// String pvcName = loginUser.getUsername().toLowerCase() + "-editor-pvc"; +// V1PersistentVolumeClaim pvc = k8sClientUtil.createPvc(namespace, pvcName, storage, storageClassName); //TODO 设置镜像可配置，这里先用默认镜像启动pod // 调用修改后的 createPod 方法，传入额外的参数 - Integer podPort = k8sClientUtil.createConfiguredPod(podName, namespace, port, mountPath, pvc, devEnvironment, minioPvcName, datasetPath, modelPath); -// Integer podPort = k8sClientUtil.createConfiguredPod(podName, namespace, port, mountPath, null, devEnvironment, minioPvcName, datasetPath, modelPath); +// Integer podPort = k8sClientUtil.createConfiguredPod(podName, namespace, port, mountPath, pvc, devEnvironment, minioPvcName, datasetPath, modelPath); + Integer podPort = k8sClientUtil.createConfiguredPod(podName, namespace, port, mountPath, null, devEnvironment, minioPvcName, datasetPath, modelPath); String url = masterIp + ":" + podPort; redisService.setCacheObject(podName, masterIp + ":" + podPort); devEnvironment.setStatus("Pending"); diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/TensorBoardServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/TensorBoardServiceImpl.java index dca3651d..83b87d82 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/TensorBoardServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/TensorBoardServiceImpl.java @@ -59,17 +59,17 @@ public class TensorBoardServiceImpl implements TensorBoardService { @Override public String runTensorBoard(FrameLogPathVo frameLogPathVo) throws Exception { - if (StringUtils.isEmpty(frameLogPathVo.getPath())||StringUtils.isEmpty(frameLogPathVo.getPvcName())){ - throw new Exception("存储路径或存储为空"); - } -// if (StringUtils.isEmpty(frameLogPathVo.getPath())) { -// throw new Exception("存储路径为空"); +// if (StringUtils.isEmpty(frameLogPathVo.getPath())||StringUtils.isEmpty(frameLogPathVo.getPvcName())){ +// throw new Exception("存储路径或存储为空"); // } + if (StringUtils.isEmpty(frameLogPathVo.getPath())) { + throw new Exception("存储路径为空"); + } LoginUser loginUser = SecurityUtils.getLoginUser(); String podName = loginUser.getUsername().toLowerCase()+"-"+frameLogPathVo.getPath().split("/")[2]+ "-tensorboard-pod"; - Integer podPort = k8sClientUtil.createPodWithSubPath(podName, StringUtils.isEmpty(frameLogPathVo.getNamespace()) ? "default" : frameLogPathVo.getNamespace(), port, mountPath, frameLogPathVo.getPath(), frameLogPathVo.getPvcName(), image); -// Integer podPort = k8sClientUtil.createPodWithSubPath(podName, StringUtils.isEmpty(frameLogPathVo.getNamespace()) ? "default" : frameLogPathVo.getNamespace(), port, mountPath, frameLogPathVo.getPath(), image); +// Integer podPort = k8sClientUtil.createPodWithSubPath(podName, StringUtils.isEmpty(frameLogPathVo.getNamespace()) ? "default" : frameLogPathVo.getNamespace(), port, mountPath, frameLogPathVo.getPath(), frameLogPathVo.getPvcName(), image); + Integer podPort = k8sClientUtil.createPodWithSubPath(podName, StringUtils.isEmpty(frameLogPathVo.getNamespace()) ? "default" : frameLogPathVo.getNamespace(), port, mountPath, frameLogPathVo.getPath(), image); redisService.setCacheObject(podName, masterIp + ":" + podPort); return masterIp + ":" + podPort; } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/K8sClientUtil.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/K8sClientUtil.java index 45c2a69e..012b1c71 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/K8sClientUtil.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/K8sClientUtil.java @@ -1,5 +1,6 @@ package com.ruoyi.platform.utils; +import com.alibaba.fastjson2.JSON; import com.ruoyi.platform.constant.Constant; import com.ruoyi.platform.domain.DevEnvironment; import com.ruoyi.platform.mapper.ComputingResourceDao; @@ -8,6 +9,7 @@ import io.kubernetes.client.custom.IntOrString; import io.kubernetes.client.custom.Quantity; import io.kubernetes.client.openapi.ApiClient; import io.kubernetes.client.openapi.ApiException; +import io.kubernetes.client.openapi.apis.AppsV1Api; import io.kubernetes.client.openapi.apis.CoreV1Api; import io.kubernetes.client.openapi.models.*; import io.kubernetes.client.util.Config; @@ -33,8 +35,8 @@ import java.util.*; @Component public class K8sClientUtil { -// @Value("${jupyter.hostPath}") -// private String hostPath; + @Value("${jupyter.hostPath}") + private String hostPath; private String http; private String token; @@ -314,8 +316,8 @@ public class K8sClientUtil { * @param image 镜像 * @return 创建成功的pod，的nodePort端口 */ - public Integer createPodWithSubPath(String podName, String namespace, Integer port, String mountPath, String subPath, String pvcName, String image) { -// public Integer createPodWithSubPath(String podName, String namespace, Integer port, String mountPath, String subPath, String image) { +// public Integer createPodWithSubPath(String podName, String namespace, Integer port, String mountPath, String subPath, String pvcName, String image) { + public Integer createPodWithSubPath(String podName, String namespace, Integer port, String mountPath, String subPath, String image) { Map selector = new LinkedHashMap<>(); selector.put("k8s-jupyter", podName); @@ -339,6 +341,9 @@ public class K8sClientUtil { } } + int lastIndex = hostPath.lastIndexOf('/'); + String newPath = hostPath.substring(0, lastIndex); + V1Pod pod = new V1PodBuilder() .withNewMetadata() .withName(podName) @@ -366,8 +371,8 @@ public class K8sClientUtil { .endContainer() .addNewVolume() .withName("workspace") -// .withHostPath(new V1HostPathVolumeSource().path(hostPath).type("DirectoryOrCreate")) - .withPersistentVolumeClaim(new V1PersistentVolumeClaimVolumeSource().claimName(pvcName)) + .withHostPath(new V1HostPathVolumeSource().path(newPath).type("DirectoryOrCreate")) +// .withPersistentVolumeClaim(new V1PersistentVolumeClaimVolumeSource().claimName(pvcName)) .endVolume() .withTerminationGracePeriodSeconds(14400L) .endSpec() @@ -449,46 +454,29 @@ public class K8sClientUtil { } } - // 配置卷和卷挂载 -// List volumeMounts = new ArrayList<>(); -// volumeMounts.add(new V1VolumeMount().name("workspace").mountPath("/opt/notebooks")); -// volumeMounts.add(new V1VolumeMount().name("data").mountPath("/opt/dataset").subPath(datasetPath).readOnly(true)); -// volumeMounts.add(new V1VolumeMount().name("data").mountPath("/opt/model").subPath(modelPath).readOnly(true)); -// -// List volumes = new ArrayList<>(); -// volumes.add(new V1Volume().name("workspace").hostPath(new V1HostPathVolumeSource().path(hostPath + "/" + podName + "/notebooks").type("DirectoryOrCreate"))); -// volumes.add(new V1Volume().name("data").hostPath(new V1HostPathVolumeSource().path(hostPath).type("DirectoryOrCreate"))); - // 配置卷和卷挂载 List volumeMounts = new ArrayList<>(); - volumeMounts.add(new V1VolumeMount().name("workspace").mountPath(mountPath)); - volumeMounts.add(new V1VolumeMount().name("minio-pvc").mountPath("/opt/data").subPath(datasetPath).readOnly(true)); - volumeMounts.add(new V1VolumeMount().name("minio-pvc").mountPath("/opt/model").subPath(modelPath).readOnly(true)); + volumeMounts.add(new V1VolumeMount().name("workspace").mountPath("/opt/notebooks")); + volumeMounts.add(new V1VolumeMount().name("data").mountPath("/opt/dataset").subPath(datasetPath).readOnly(true)); + volumeMounts.add(new V1VolumeMount().name("data").mountPath("/opt/model").subPath(modelPath).readOnly(true)); List volumes = new ArrayList<>(); - volumes.add(new V1Volume().name("workspace").persistentVolumeClaim(new V1PersistentVolumeClaimVolumeSource().claimName(pvc.getMetadata().getName()))); - volumes.add(new V1Volume().name("minio-pvc").persistentVolumeClaim(new V1PersistentVolumeClaimVolumeSource().claimName(dataPvcName))); - + volumes.add(new V1Volume().name("workspace").hostPath(new V1HostPathVolumeSource().path(hostPath + "/" + podName + "/notebooks").type("DirectoryOrCreate"))); + volumes.add(new V1Volume().name("data").hostPath(new V1HostPathVolumeSource().path(hostPath).type("DirectoryOrCreate"))); - //配置资源 - JSONObject standardJson = new JSONObject(devEnvironment.getStandard()); - JSONObject valueJson = (JSONObject) standardJson.get("value"); - int cpu = (int) valueJson.get("cpu"); - String memory = (String) valueJson.get("memory"); - memory = memory.substring(0, memory.length() - 1).concat("i"); - int gpu = (int) valueJson.get("gpu"); + // 配置卷和卷挂载 +// List volumeMounts = new ArrayList<>(); +// volumeMounts.add(new V1VolumeMount().name("workspace").mountPath(mountPath)); +// volumeMounts.add(new V1VolumeMount().name("minio-pvc").mountPath("/opt/data").subPath(datasetPath).readOnly(true)); +// volumeMounts.add(new V1VolumeMount().name("minio-pvc").mountPath("/opt/model").subPath(modelPath).readOnly(true)); +// +// List volumes = new ArrayList<>(); +// volumes.add(new V1Volume().name("workspace").persistentVolumeClaim(new V1PersistentVolumeClaimVolumeSource().claimName(pvc.getMetadata().getName()))); +// volumes.add(new V1Volume().name("minio-pvc").persistentVolumeClaim(new V1PersistentVolumeClaimVolumeSource().claimName(dataPvcName))); - HashMap limitMap = new HashMap<>(); - if (Constant.Computing_Resource_GPU.equals(devEnvironment.getComputingResource())) { - limitMap.put("nvidia.com/gpu", new Quantity(String.valueOf(gpu))); - } - limitMap.put("cpu", new Quantity(String.valueOf(cpu))); - limitMap.put("memory", new Quantity(memory)); - limitMap.put("ephemeral-storage", new Quantity("100Gi")); - V1ResourceRequirements v1ResourceRequirements = new V1ResourceRequirements(); - v1ResourceRequirements.setRequests(limitMap); - v1ResourceRequirements.setLimits(limitMap); + //配置资源 + V1ResourceRequirements v1ResourceRequirements = setPodResource(devEnvironment.getStandard()); V1Pod pod = new V1PodBuilder() .withNewMetadata() @@ -688,6 +676,30 @@ public class K8sClientUtil { return pod; } + V1ResourceRequirements setPodResource(String resource) { + //配置pod资源 + JSONObject standardJson = new JSONObject(resource); + JSONObject valueJson = (JSONObject) standardJson.get("value"); + int cpu = (int) valueJson.get("cpu"); + String memory = (String) valueJson.get("memory"); + memory = memory.substring(0, memory.length() - 1).concat("i"); + Integer gpu = (Integer) valueJson.get("gpu"); + + HashMap limitMap = new HashMap<>(); + if (gpu != null && gpu != 0) { + limitMap.put("nvidia.com/gpu", new Quantity(String.valueOf(gpu))); + } + limitMap.put("cpu", new Quantity(String.valueOf(cpu))); + limitMap.put("memory", new Quantity(memory)); + limitMap.put("ephemeral-storage", new Quantity("100Gi")); + + V1ResourceRequirements v1ResourceRequirements = new V1ResourceRequirements(); + v1ResourceRequirements.setRequests(limitMap); + v1ResourceRequirements.setLimits(limitMap); + + return v1ResourceRequirements; + } + /** * 删除 Pod * diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/FrameLogPathVo.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/FrameLogPathVo.java index e4c8bd26..da32b84d 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/FrameLogPathVo.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/FrameLogPathVo.java @@ -10,7 +10,7 @@ public class FrameLogPathVo implements Serializable { String path; String namespace; - String pvcName; +// String pvcName; public String getPath() { return path; } @@ -27,11 +27,11 @@ public class FrameLogPathVo implements Serializable { this.namespace = namespace; } - public String getPvcName() { - return pvcName; - } - - public void setPvcName(String pvcName) { - this.pvcName = pvcName; - } +// public String getPvcName() { +// return pvcName; +// } +// +// public void setPvcName(String pvcName) { +// this.pvcName = pvcName; +// } }