From d42b8e918d2eb4b0749c4b46acf571f37ba1e355 Mon Sep 17 00:00:00 2001 From: chenzhihang <709011834@qq.com> Date: Thu, 29 Aug 2024 14:18:13 +0800 Subject: [PATCH] =?UTF-8?q?1=E3=80=81=E6=B7=BB=E5=8A=A0GPU=E5=8D=A0?= =?UTF-8?q?=E7=94=A8=E6=83=85=E5=86=B5=E8=AE=BE=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/ruoyi/platform/constant/Constant.java | 5 ++ .../service/impl/JupyterServiceImpl.java | 33 ++++--- .../ruoyi/platform/utils/K8sClientUtil.java | 88 +++++++++---------- 3 files changed, 65 insertions(+), 61 deletions(-) diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/constant/Constant.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/constant/Constant.java index b74c30ea..9d14aa30 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/constant/Constant.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/constant/Constant.java @@ -10,4 +10,9 @@ public class Constant { public final static int Used_State_used = 1; // 已占用 public final static int Used_State_unused = 0; // 未占用 + + + public final static String Computing_Resource_CPU = "CPU"; // 计算资源_CPU + + public final static String Computing_Resource_GPU = "GPU"; // 计算资源_GPU } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/JupyterServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/JupyterServiceImpl.java index 6e2dc717..b99bdb29 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/JupyterServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/JupyterServiceImpl.java @@ -75,7 +75,7 @@ public class JupyterServiceImpl implements JupyterService { LoginUser loginUser = SecurityUtils.getLoginUser(); String podName = loginUser.getUsername().toLowerCase() + "-editor-pod"; String pvcName = loginUser.getUsername().toLowerCase() + "-editor-pvc"; - V1PersistentVolumeClaim pvc = k8sClientUtil.createPvc(namespace, pvcName, storage,storageClassName); + V1PersistentVolumeClaim pvc = k8sClientUtil.createPvc(namespace, pvcName, storage, storageClassName); Integer podPort = k8sClientUtil.createPod(podName, namespace, port, mountPath, pvc, image); return masterIp + ":" + podPort; @@ -84,7 +84,7 @@ public class JupyterServiceImpl implements JupyterService { @Override public String runJupyterService(Integer id) throws Exception { DevEnvironment devEnvironment = this.devEnvironmentDao.queryById(id); - if(devEnvironment == null){ + if (devEnvironment == null) { throw new Exception("开发环境配置不存在"); } @@ -97,7 +97,7 @@ public class JupyterServiceImpl implements JupyterService { LoginUser loginUser = SecurityUtils.getLoginUser(); //构造pod名称 - String podName = loginUser.getUsername().toLowerCase() +"-editor-pod" + "-" + id; + String podName = loginUser.getUsername().toLowerCase() + "-editor-pod" + "-" + id; String pvcName = loginUser.getUsername().toLowerCase() + "-editor-pvc"; //新建编辑器的pvc V1PersistentVolumeClaim pvc = k8sClientUtil.createPvc(namespace, pvcName, storage, storageClassName); @@ -105,13 +105,13 @@ public class JupyterServiceImpl implements JupyterService { //TODO 设置镜像可配置，这里先用默认镜像启动pod // 调用修改后的 createPod 方法，传入额外的参数 - Integer podPort = k8sClientUtil.createConfiguredPod(podName, namespace, port, mountPath, pvc, devEnvironment.getImage(), minioPvcName, datasetPath, modelPath); + Integer podPort = k8sClientUtil.createConfiguredPod(podName, namespace, port, mountPath, pvc, devEnvironment.getImage(), devEnvironment.getComputingResource(), minioPvcName, datasetPath, modelPath); String url = masterIp + ":" + podPort; - redisService.setCacheObject(podName,masterIp + ":" + podPort); + redisService.setCacheObject(podName, masterIp + ":" + podPort); devEnvironment.setStatus("Pending"); devEnvironment.setUrl(url); this.devEnvironmentService.update(devEnvironment); - return url ; + return url; } @@ -119,20 +119,22 @@ public class JupyterServiceImpl implements JupyterService { @Override public String stopJupyterService(Integer id) throws Exception { DevEnvironment devEnvironment = this.devEnvironmentDao.queryById(id); - if (devEnvironment==null){ + if (devEnvironment == null) { throw new Exception("开发环境配置不存在"); } LoginUser loginUser = SecurityUtils.getLoginUser(); //构造pod和svc名称 - String podName = loginUser.getUsername().toLowerCase() +"-editor-pod" + "-" + id; - String svcName = loginUser.getUsername().toLowerCase() + "-editor-pod" + "-" + id + "-svc"; + String podName = loginUser.getUsername().toLowerCase() + "-editor-pod" + "-" + id; + String svcName = loginUser.getUsername().toLowerCase() + "-editor-pod" + "-" + id + "-svc"; //得到pod V1Pod pod = k8sClientUtil.getNSPodList(namespace, podName); - if(pod == null){ + if (pod == null) { return "pod不存在！"; } - computingResourceDao.updateUsedStateByNode(pod.getSpec().getNodeName(), Constant.Used_State_unused); + if (Constant.Computing_Resource_GPU.equals(devEnvironment.getComputingResource())) { + computingResourceDao.updateUsedStateByNode(pod.getSpec().getNodeName(), Constant.Used_State_unused); + } // 使用 Kubernetes API 删除 Pod String deleteResult = k8sClientUtil.deletePod(podName, namespace); @@ -150,11 +152,11 @@ public class JupyterServiceImpl implements JupyterService { String status = PodStatus.Terminated.getName(); PodStatusVo JupyterStatusVo = new PodStatusVo(); JupyterStatusVo.setStatus(status); - if (devEnvironment==null){ + if (devEnvironment == null) { return JupyterStatusVo; } LoginUser loginUser = SecurityUtils.getLoginUser(); - String podName = loginUser.getUsername().toLowerCase() +"-editor-pod" + "-" + devEnvironment.getId(); + String podName = loginUser.getUsername().toLowerCase() + "-editor-pod" + "-" + devEnvironment.getId(); try { // 查询相应pod状态 @@ -180,7 +182,7 @@ public class JupyterServiceImpl implements JupyterService { @Override public void upload(InputStream inputStream) { try { - minioUtil.uploadObject("platform-data","/pytorch/testupload4008208820",inputStream); + minioUtil.uploadObject("platform-data", "/pytorch/testupload4008208820", inputStream); } catch (Exception e) { e.printStackTrace(); } @@ -195,7 +197,4 @@ public class JupyterServiceImpl implements JupyterService { } - - - } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/K8sClientUtil.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/K8sClientUtil.java index d022bf37..d5b5ab21 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/K8sClientUtil.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/K8sClientUtil.java @@ -1,6 +1,5 @@ package com.ruoyi.platform.utils; -import com.alibaba.nacos.shaded.com.google.gson.reflect.TypeToken; import com.ruoyi.platform.constant.Constant; import com.ruoyi.platform.mapper.ComputingResourceDao; import io.kubernetes.client.Exec; @@ -8,20 +7,16 @@ import io.kubernetes.client.custom.IntOrString; import io.kubernetes.client.custom.Quantity; import io.kubernetes.client.openapi.ApiClient; import io.kubernetes.client.openapi.ApiException; -import io.kubernetes.client.openapi.ApiResponse; import io.kubernetes.client.openapi.apis.CoreV1Api; import io.kubernetes.client.openapi.models.*; import io.kubernetes.client.util.ClientBuilder; -import io.kubernetes.client.util.Watch; import io.kubernetes.client.util.credentials.AccessTokenAuthentication; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang.StringUtils; -import org.apache.poi.ss.formula.functions.T; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; -import javax.annotation.PostConstruct; import javax.annotation.Resource; import java.io.BufferedReader; import java.io.InputStreamReader; @@ -45,6 +40,7 @@ public class K8sClientUtil { @Resource private ComputingResourceDao computingResourceDao; + /** * 构建集群POD内通过SA访问的客户端 * loading the in-cluster config, including: @@ -138,11 +134,11 @@ public class K8sClientUtil { } catch (ApiException e) { log.error("获取 SVC 异常:", e); } - if (v1ServiceList!=null) { + if (v1ServiceList != null) { for (V1Service svc : v1ServiceList.getItems()) { if (StringUtils.equals(svc.getMetadata().getName(), serviceName)) { // SVC 已存在 - return svc; + return svc; } } } @@ -182,22 +178,22 @@ public class K8sClientUtil { /** * 创建k8s PVC * - * @param namespace 命名空间 - * @param pvcName 服务名称 + * @param namespace 命名空间 + * @param pvcName 服务名称 * @return 创建成功的service对象 */ - public V1PersistentVolumeClaim createPvc(String namespace, String pvcName ,String storage, String storageClassName){ + public V1PersistentVolumeClaim createPvc(String namespace, String pvcName, String storage, String storageClassName) { CoreV1Api api = new CoreV1Api(apiClient); V1PersistentVolumeClaimList pvcList = null; try { - pvcList = api.listNamespacedPersistentVolumeClaim(namespace, null,null, null, null, null,null,null, null, null, null); + pvcList = api.listNamespacedPersistentVolumeClaim(namespace, null, null, null, null, null, null, null, null, null, null); } catch (ApiException e) { log.error("获取 PVC 异常:", e); } - if (pvcList!=null) { + if (pvcList != null) { for (V1PersistentVolumeClaim pvc1 : pvcList.getItems()) { - if (StringUtils.equals(pvc1.getMetadata().getName(),pvcName)) { + if (StringUtils.equals(pvc1.getMetadata().getName(), pvcName)) { // PVC 已存在 return pvc1; } @@ -232,16 +228,17 @@ public class K8sClientUtil { /** * 创建k8s 临时POD + * * @param podName pod name - * @param namespace 命名空间 - * @param port port + * @param namespace 命名空间 + * @param port port * @param mountPath 映射路径 - * @param pvc 存储 - * @param image 镜像 + * @param pvc 存储 + * @param image 镜像 * @return 创建成功的pod，的nodePort端口 */ - public Integer createPod(String podName, String namespace, Integer port ,String mountPath, V1PersistentVolumeClaim pvc, String image){ + public Integer createPod(String podName, String namespace, Integer port, String mountPath, V1PersistentVolumeClaim pvc, String image) { Map selector = new LinkedHashMap(); selector.put("k8s-jupyter", podName); @@ -253,7 +250,7 @@ public class K8sClientUtil { } catch (ApiException e) { log.error("获取 POD 异常:", e); } - if (v1PodList!=null) { + if (v1PodList != null) { for (V1Pod pod1 : v1PodList.getItems()) { if (StringUtils.equals(pod1.getMetadata().getName(), podName)) { // PVC 已存在 @@ -301,17 +298,18 @@ public class K8sClientUtil { /** * 创建k8s 临时POD + * * @param podName pod name - * @param namespace 命名空间 - * @param port port + * @param namespace 命名空间 + * @param port port * @param mountPath 映射路径 - * @param subPath pvc子路径 - * @param pvcName 存储名 - * @param image 镜像 + * @param subPath pvc子路径 + * @param pvcName 存储名 + * @param image 镜像 * @return 创建成功的pod，的nodePort端口 */ - public Integer createPodWithSubPath(String podName, String namespace, Integer port ,String mountPath,String subPath,String pvcName, String image){ + public Integer createPodWithSubPath(String podName, String namespace, Integer port, String mountPath, String subPath, String pvcName, String image) { Map selector = new LinkedHashMap(); selector.put("k8s-jupyter", podName); @@ -323,7 +321,7 @@ public class K8sClientUtil { } catch (ApiException e) { log.error("获取 POD 异常:", e); } - if (v1PodList!=null) { + if (v1PodList != null) { for (V1Pod pod1 : v1PodList.getItems()) { if (StringUtils.equals(pod1.getMetadata().getName(), podName)) { // PVC 已存在 @@ -381,7 +379,7 @@ public class K8sClientUtil { } // 创建配置好的Pod - public Integer createConfiguredPod(String podName, String namespace, Integer port, String mountPath, V1PersistentVolumeClaim pvc, String image, String dataPvcName, String datasetPath, String modelPath) { + public Integer createConfiguredPod(String podName, String namespace, Integer port, String mountPath, V1PersistentVolumeClaim pvc, String image, String computingResource, String dataPvcName, String datasetPath, String modelPath) { //设置选择节点，pod反亲和性 Map selector = new LinkedHashMap<>(); @@ -473,7 +471,9 @@ public class K8sClientUtil { try { pod = api.createNamespacedPod(namespace, pod, null, null, null); String nodeName = getNodeName(podName, namespace); - computingResourceDao.updateUsedStateByNode(nodeName, Constant.Used_State_used); + if (Constant.Computing_Resource_GPU.equals(computingResource)) { + computingResourceDao.updateUsedStateByNode(nodeName, Constant.Used_State_used); + } } catch (ApiException e) { log.error("创建pod异常:" + e.getResponseBody(), e); } catch (Exception e) { @@ -485,14 +485,12 @@ public class K8sClientUtil { } - - /** * 根据获取namespace,deploymentName的Pod Name * * @return podList */ - public V1Pod getNSPodList(String namespace,String deploymentName) throws Exception { + public V1Pod getNSPodList(String namespace, String deploymentName) throws Exception { // new a CoreV1Api CoreV1Api api = new CoreV1Api(apiClient); V1PodList v1PodList = null; @@ -513,12 +511,12 @@ public class K8sClientUtil { return null; } - public String executeCommand(V1Pod item, String command) { + public String executeCommand(V1Pod item, String command) { try { // 创建API实例 // 创建Exec实例 Exec exec = new Exec(apiClient); - String[] cmd = { "/bin/sh", "-c", command}; + String[] cmd = {"/bin/sh", "-c", command}; Process proc = exec.exec(item, cmd, false); // 读取输出 BufferedReader reader = new BufferedReader(new InputStreamReader(proc.getInputStream())); @@ -543,10 +541,11 @@ public class K8sClientUtil { /** * 根据Pod的名称和Namespace查询Pod的状态 - * @param podName Pod的名称 + * + * @param podName Pod的名称 * @param namespace Pod所在的Namespace */ - public String getPodStatus(String podName, String namespace) throws Exception { + public String getPodStatus(String podName, String namespace) throws Exception { CoreV1Api api = new CoreV1Api(apiClient); V1Pod pod = api.readNamespacedPod(podName, namespace, null, null, null); return pod.getStatus().getPhase(); @@ -554,14 +553,15 @@ public class K8sClientUtil { /** * 根据Pod的名称和Namespace查询Pod的容器信息 - * @param podName Pod的名称 + * + * @param podName Pod的名称 * @param namespace Pod所在的Namespace */ public String getPodContainerId(String podName, String namespace) throws Exception { CoreV1Api api = new CoreV1Api(apiClient); V1Pod pod = api.readNamespacedPod(podName, namespace, null, null, null); - if(pod.getStatus().getContainerStatuses().size() !=1){ + if (pod.getStatus().getContainerStatuses().size() != 1) { throw new RuntimeException("容器错误"); } String containerId = pod.getStatus().getContainerStatuses().get(0).getContainerID().split("//")[1]; @@ -580,10 +580,10 @@ public class K8sClientUtil { return pod.getSpec().getNodeName(); } - public String getPodLogs(String podName,String namespace,String container,int line) { + public String getPodLogs(String podName, String namespace, String container, int line) { CoreV1Api api = new CoreV1Api(apiClient); try { - String log = api.readNamespacedPodLog(podName, namespace, StringUtils.isEmpty(container)?null:container, null, null, null, null,null, null, line, null); + String log = api.readNamespacedPodLog(podName, namespace, StringUtils.isEmpty(container) ? null : container, null, null, null, null, null, null, line, null); return log; } catch (ApiException e) { throw new RuntimeException("获取Pod日志异常", e); @@ -592,7 +592,7 @@ public class K8sClientUtil { } - public V1Pod createPodWithEnv(String podName,String namespace,String proxyUrl ,String mountPath,String pvcName, String image){ + public V1Pod createPodWithEnv(String podName, String namespace, String proxyUrl, String mountPath, String pvcName, String image) { CoreV1Api api = new CoreV1Api(apiClient); V1PodList v1PodList = null; V1Pod pod = new V1PodBuilder() @@ -636,7 +636,7 @@ public class K8sClientUtil { /** * 删除 Pod * - * @param podName Pod 名称 + * @param podName Pod 名称 * @param namespace 命名空间 * @throws ApiException 异常 */ @@ -655,7 +655,7 @@ public class K8sClientUtil { /** * 删除 Service * - * @param svcName Service 名称 + * @param svcName Service 名称 * @param namespace 命名空间 * @throws ApiException 异常 */ @@ -674,7 +674,7 @@ public class K8sClientUtil { /** * 检查 Pod 是否存在 * - * @param podName Pod 名称 + * @param podName Pod 名称 * @param namespace 命名空间 * @return 是否存在 * @throws ApiException 异常 @@ -682,7 +682,7 @@ public class K8sClientUtil { public boolean checkPodExists(String podName, String namespace) throws ApiException { CoreV1Api api = new CoreV1Api(apiClient); try { - api.readNamespacedPod(podName, namespace, null,false,false); + api.readNamespacedPod(podName, namespace, null, false, false); return true; } catch (ApiException e) { if (e.getCode() == 404) {