Browse Source

1、添加GPU占用情况设置

pull/127/head
chenzhihang 1 year ago
parent
commit
d42b8e918d
3 changed files with 65 additions and 61 deletions
  1. +5
    -0
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/constant/Constant.java
  2. +16
    -17
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/JupyterServiceImpl.java
  3. +44
    -44
      ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/K8sClientUtil.java

+ 5
- 0
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/constant/Constant.java View File

@@ -10,4 +10,9 @@ public class Constant {


public final static int Used_State_used = 1; // 已占用 public final static int Used_State_used = 1; // 已占用
public final static int Used_State_unused = 0; // 未占用 public final static int Used_State_unused = 0; // 未占用


public final static String Computing_Resource_CPU = "CPU"; // 计算资源_CPU

public final static String Computing_Resource_GPU = "GPU"; // 计算资源_GPU
} }

+ 16
- 17
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/JupyterServiceImpl.java View File

@@ -75,7 +75,7 @@ public class JupyterServiceImpl implements JupyterService {
LoginUser loginUser = SecurityUtils.getLoginUser(); LoginUser loginUser = SecurityUtils.getLoginUser();
String podName = loginUser.getUsername().toLowerCase() + "-editor-pod"; String podName = loginUser.getUsername().toLowerCase() + "-editor-pod";
String pvcName = loginUser.getUsername().toLowerCase() + "-editor-pvc"; String pvcName = loginUser.getUsername().toLowerCase() + "-editor-pvc";
V1PersistentVolumeClaim pvc = k8sClientUtil.createPvc(namespace, pvcName, storage,storageClassName);
V1PersistentVolumeClaim pvc = k8sClientUtil.createPvc(namespace, pvcName, storage, storageClassName);
Integer podPort = k8sClientUtil.createPod(podName, namespace, port, mountPath, pvc, image); Integer podPort = k8sClientUtil.createPod(podName, namespace, port, mountPath, pvc, image);
return masterIp + ":" + podPort; return masterIp + ":" + podPort;


@@ -84,7 +84,7 @@ public class JupyterServiceImpl implements JupyterService {
@Override @Override
public String runJupyterService(Integer id) throws Exception { public String runJupyterService(Integer id) throws Exception {
DevEnvironment devEnvironment = this.devEnvironmentDao.queryById(id); DevEnvironment devEnvironment = this.devEnvironmentDao.queryById(id);
if(devEnvironment == null){
if (devEnvironment == null) {
throw new Exception("开发环境配置不存在"); throw new Exception("开发环境配置不存在");
} }


@@ -97,7 +97,7 @@ public class JupyterServiceImpl implements JupyterService {


LoginUser loginUser = SecurityUtils.getLoginUser(); LoginUser loginUser = SecurityUtils.getLoginUser();
//构造pod名称 //构造pod名称
String podName = loginUser.getUsername().toLowerCase() +"-editor-pod" + "-" + id;
String podName = loginUser.getUsername().toLowerCase() + "-editor-pod" + "-" + id;
String pvcName = loginUser.getUsername().toLowerCase() + "-editor-pvc"; String pvcName = loginUser.getUsername().toLowerCase() + "-editor-pvc";
//新建编辑器的pvc //新建编辑器的pvc
V1PersistentVolumeClaim pvc = k8sClientUtil.createPvc(namespace, pvcName, storage, storageClassName); V1PersistentVolumeClaim pvc = k8sClientUtil.createPvc(namespace, pvcName, storage, storageClassName);
@@ -105,13 +105,13 @@ public class JupyterServiceImpl implements JupyterService {
//TODO 设置镜像可配置,这里先用默认镜像启动pod //TODO 设置镜像可配置,这里先用默认镜像启动pod


// 调用修改后的 createPod 方法,传入额外的参数 // 调用修改后的 createPod 方法,传入额外的参数
Integer podPort = k8sClientUtil.createConfiguredPod(podName, namespace, port, mountPath, pvc, devEnvironment.getImage(), minioPvcName, datasetPath, modelPath);
Integer podPort = k8sClientUtil.createConfiguredPod(podName, namespace, port, mountPath, pvc, devEnvironment.getImage(), devEnvironment.getComputingResource(), minioPvcName, datasetPath, modelPath);
String url = masterIp + ":" + podPort; String url = masterIp + ":" + podPort;
redisService.setCacheObject(podName,masterIp + ":" + podPort);
redisService.setCacheObject(podName, masterIp + ":" + podPort);
devEnvironment.setStatus("Pending"); devEnvironment.setStatus("Pending");
devEnvironment.setUrl(url); devEnvironment.setUrl(url);
this.devEnvironmentService.update(devEnvironment); this.devEnvironmentService.update(devEnvironment);
return url ;
return url;


} }


@@ -119,20 +119,22 @@ public class JupyterServiceImpl implements JupyterService {
@Override @Override
public String stopJupyterService(Integer id) throws Exception { public String stopJupyterService(Integer id) throws Exception {
DevEnvironment devEnvironment = this.devEnvironmentDao.queryById(id); DevEnvironment devEnvironment = this.devEnvironmentDao.queryById(id);
if (devEnvironment==null){
if (devEnvironment == null) {
throw new Exception("开发环境配置不存在"); throw new Exception("开发环境配置不存在");
} }
LoginUser loginUser = SecurityUtils.getLoginUser(); LoginUser loginUser = SecurityUtils.getLoginUser();
//构造pod和svc名称 //构造pod和svc名称
String podName = loginUser.getUsername().toLowerCase() +"-editor-pod" + "-" + id;
String svcName = loginUser.getUsername().toLowerCase() + "-editor-pod" + "-" + id + "-svc";
String podName = loginUser.getUsername().toLowerCase() + "-editor-pod" + "-" + id;
String svcName = loginUser.getUsername().toLowerCase() + "-editor-pod" + "-" + id + "-svc";
//得到pod //得到pod
V1Pod pod = k8sClientUtil.getNSPodList(namespace, podName); V1Pod pod = k8sClientUtil.getNSPodList(namespace, podName);
if(pod == null){
if (pod == null) {
return "pod不存在!"; return "pod不存在!";
} }


computingResourceDao.updateUsedStateByNode(pod.getSpec().getNodeName(), Constant.Used_State_unused);
if (Constant.Computing_Resource_GPU.equals(devEnvironment.getComputingResource())) {
computingResourceDao.updateUsedStateByNode(pod.getSpec().getNodeName(), Constant.Used_State_unused);
}


// 使用 Kubernetes API 删除 Pod // 使用 Kubernetes API 删除 Pod
String deleteResult = k8sClientUtil.deletePod(podName, namespace); String deleteResult = k8sClientUtil.deletePod(podName, namespace);
@@ -150,11 +152,11 @@ public class JupyterServiceImpl implements JupyterService {
String status = PodStatus.Terminated.getName(); String status = PodStatus.Terminated.getName();
PodStatusVo JupyterStatusVo = new PodStatusVo(); PodStatusVo JupyterStatusVo = new PodStatusVo();
JupyterStatusVo.setStatus(status); JupyterStatusVo.setStatus(status);
if (devEnvironment==null){
if (devEnvironment == null) {
return JupyterStatusVo; return JupyterStatusVo;
} }
LoginUser loginUser = SecurityUtils.getLoginUser(); LoginUser loginUser = SecurityUtils.getLoginUser();
String podName = loginUser.getUsername().toLowerCase() +"-editor-pod" + "-" + devEnvironment.getId();
String podName = loginUser.getUsername().toLowerCase() + "-editor-pod" + "-" + devEnvironment.getId();


try { try {
// 查询相应pod状态 // 查询相应pod状态
@@ -180,7 +182,7 @@ public class JupyterServiceImpl implements JupyterService {
@Override @Override
public void upload(InputStream inputStream) { public void upload(InputStream inputStream) {
try { try {
minioUtil.uploadObject("platform-data","/pytorch/testupload4008208820",inputStream);
minioUtil.uploadObject("platform-data", "/pytorch/testupload4008208820", inputStream);
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
} }
@@ -195,7 +197,4 @@ public class JupyterServiceImpl implements JupyterService {
} }







} }

+ 44
- 44
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/K8sClientUtil.java View File

@@ -1,6 +1,5 @@
package com.ruoyi.platform.utils; package com.ruoyi.platform.utils;


import com.alibaba.nacos.shaded.com.google.gson.reflect.TypeToken;
import com.ruoyi.platform.constant.Constant; import com.ruoyi.platform.constant.Constant;
import com.ruoyi.platform.mapper.ComputingResourceDao; import com.ruoyi.platform.mapper.ComputingResourceDao;
import io.kubernetes.client.Exec; import io.kubernetes.client.Exec;
@@ -8,20 +7,16 @@ import io.kubernetes.client.custom.IntOrString;
import io.kubernetes.client.custom.Quantity; import io.kubernetes.client.custom.Quantity;
import io.kubernetes.client.openapi.ApiClient; import io.kubernetes.client.openapi.ApiClient;
import io.kubernetes.client.openapi.ApiException; import io.kubernetes.client.openapi.ApiException;
import io.kubernetes.client.openapi.ApiResponse;
import io.kubernetes.client.openapi.apis.CoreV1Api; import io.kubernetes.client.openapi.apis.CoreV1Api;
import io.kubernetes.client.openapi.models.*; import io.kubernetes.client.openapi.models.*;
import io.kubernetes.client.util.ClientBuilder; import io.kubernetes.client.util.ClientBuilder;
import io.kubernetes.client.util.Watch;
import io.kubernetes.client.util.credentials.AccessTokenAuthentication; import io.kubernetes.client.util.credentials.AccessTokenAuthentication;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
import org.apache.poi.ss.formula.functions.T;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;


import javax.annotation.PostConstruct;
import javax.annotation.Resource; import javax.annotation.Resource;
import java.io.BufferedReader; import java.io.BufferedReader;
import java.io.InputStreamReader; import java.io.InputStreamReader;
@@ -45,6 +40,7 @@ public class K8sClientUtil {


@Resource @Resource
private ComputingResourceDao computingResourceDao; private ComputingResourceDao computingResourceDao;

/** /**
* 构建集群POD内通过SA访问的客户端 * 构建集群POD内通过SA访问的客户端
* loading the in-cluster config, including: * loading the in-cluster config, including:
@@ -138,11 +134,11 @@ public class K8sClientUtil {
} catch (ApiException e) { } catch (ApiException e) {
log.error("获取 SVC 异常:", e); log.error("获取 SVC 异常:", e);
} }
if (v1ServiceList!=null) {
if (v1ServiceList != null) {
for (V1Service svc : v1ServiceList.getItems()) { for (V1Service svc : v1ServiceList.getItems()) {
if (StringUtils.equals(svc.getMetadata().getName(), serviceName)) { if (StringUtils.equals(svc.getMetadata().getName(), serviceName)) {
// SVC 已存在 // SVC 已存在
return svc;
return svc;
} }
} }
} }
@@ -182,22 +178,22 @@ public class K8sClientUtil {
/** /**
* 创建k8s PVC * 创建k8s PVC
* *
* @param namespace 命名空间
* @param pvcName 服务名称
* @param namespace 命名空间
* @param pvcName 服务名称
* @return 创建成功的service对象 * @return 创建成功的service对象
*/ */
public V1PersistentVolumeClaim createPvc(String namespace, String pvcName ,String storage, String storageClassName){
public V1PersistentVolumeClaim createPvc(String namespace, String pvcName, String storage, String storageClassName) {
CoreV1Api api = new CoreV1Api(apiClient); CoreV1Api api = new CoreV1Api(apiClient);


V1PersistentVolumeClaimList pvcList = null; V1PersistentVolumeClaimList pvcList = null;
try { try {
pvcList = api.listNamespacedPersistentVolumeClaim(namespace, null,null, null, null, null,null,null, null, null, null);
pvcList = api.listNamespacedPersistentVolumeClaim(namespace, null, null, null, null, null, null, null, null, null, null);
} catch (ApiException e) { } catch (ApiException e) {
log.error("获取 PVC 异常:", e); log.error("获取 PVC 异常:", e);
} }
if (pvcList!=null) {
if (pvcList != null) {
for (V1PersistentVolumeClaim pvc1 : pvcList.getItems()) { for (V1PersistentVolumeClaim pvc1 : pvcList.getItems()) {
if (StringUtils.equals(pvc1.getMetadata().getName(),pvcName)) {
if (StringUtils.equals(pvc1.getMetadata().getName(), pvcName)) {
// PVC 已存在 // PVC 已存在
return pvc1; return pvc1;
} }
@@ -232,16 +228,17 @@ public class K8sClientUtil {


/** /**
* 创建k8s 临时POD * 创建k8s 临时POD
*
* @param podName pod name * @param podName pod name
* @param namespace 命名空间
* @param port port
* @param namespace 命名空间
* @param port port
* @param mountPath 映射路径 * @param mountPath 映射路径
* @param pvc 存储
* @param image 镜像
* @param pvc 存储
* @param image 镜像
* @return 创建成功的pod,的nodePort端口 * @return 创建成功的pod,的nodePort端口
*/ */


public Integer createPod(String podName, String namespace, Integer port ,String mountPath, V1PersistentVolumeClaim pvc, String image){
public Integer createPod(String podName, String namespace, Integer port, String mountPath, V1PersistentVolumeClaim pvc, String image) {


Map<String, String> selector = new LinkedHashMap<String, String>(); Map<String, String> selector = new LinkedHashMap<String, String>();
selector.put("k8s-jupyter", podName); selector.put("k8s-jupyter", podName);
@@ -253,7 +250,7 @@ public class K8sClientUtil {
} catch (ApiException e) { } catch (ApiException e) {
log.error("获取 POD 异常:", e); log.error("获取 POD 异常:", e);
} }
if (v1PodList!=null) {
if (v1PodList != null) {
for (V1Pod pod1 : v1PodList.getItems()) { for (V1Pod pod1 : v1PodList.getItems()) {
if (StringUtils.equals(pod1.getMetadata().getName(), podName)) { if (StringUtils.equals(pod1.getMetadata().getName(), podName)) {
// PVC 已存在 // PVC 已存在
@@ -301,17 +298,18 @@ public class K8sClientUtil {


/** /**
* 创建k8s 临时POD * 创建k8s 临时POD
*
* @param podName pod name * @param podName pod name
* @param namespace 命名空间
* @param port port
* @param namespace 命名空间
* @param port port
* @param mountPath 映射路径 * @param mountPath 映射路径
* @param subPath pvc子路径
* @param pvcName 存储名
* @param image 镜像
* @param subPath pvc子路径
* @param pvcName 存储名
* @param image 镜像
* @return 创建成功的pod,的nodePort端口 * @return 创建成功的pod,的nodePort端口
*/ */


public Integer createPodWithSubPath(String podName, String namespace, Integer port ,String mountPath,String subPath,String pvcName, String image){
public Integer createPodWithSubPath(String podName, String namespace, Integer port, String mountPath, String subPath, String pvcName, String image) {


Map<String, String> selector = new LinkedHashMap<String, String>(); Map<String, String> selector = new LinkedHashMap<String, String>();
selector.put("k8s-jupyter", podName); selector.put("k8s-jupyter", podName);
@@ -323,7 +321,7 @@ public class K8sClientUtil {
} catch (ApiException e) { } catch (ApiException e) {
log.error("获取 POD 异常:", e); log.error("获取 POD 异常:", e);
} }
if (v1PodList!=null) {
if (v1PodList != null) {
for (V1Pod pod1 : v1PodList.getItems()) { for (V1Pod pod1 : v1PodList.getItems()) {
if (StringUtils.equals(pod1.getMetadata().getName(), podName)) { if (StringUtils.equals(pod1.getMetadata().getName(), podName)) {
// PVC 已存在 // PVC 已存在
@@ -381,7 +379,7 @@ public class K8sClientUtil {
} }


// 创建配置好的Pod // 创建配置好的Pod
public Integer createConfiguredPod(String podName, String namespace, Integer port, String mountPath, V1PersistentVolumeClaim pvc, String image, String dataPvcName, String datasetPath, String modelPath) {
public Integer createConfiguredPod(String podName, String namespace, Integer port, String mountPath, V1PersistentVolumeClaim pvc, String image, String computingResource, String dataPvcName, String datasetPath, String modelPath) {


//设置选择节点,pod反亲和性 //设置选择节点,pod反亲和性
Map<String, String> selector = new LinkedHashMap<>(); Map<String, String> selector = new LinkedHashMap<>();
@@ -473,7 +471,9 @@ public class K8sClientUtil {
try { try {
pod = api.createNamespacedPod(namespace, pod, null, null, null); pod = api.createNamespacedPod(namespace, pod, null, null, null);
String nodeName = getNodeName(podName, namespace); String nodeName = getNodeName(podName, namespace);
computingResourceDao.updateUsedStateByNode(nodeName, Constant.Used_State_used);
if (Constant.Computing_Resource_GPU.equals(computingResource)) {
computingResourceDao.updateUsedStateByNode(nodeName, Constant.Used_State_used);
}
} catch (ApiException e) { } catch (ApiException e) {
log.error("创建pod异常:" + e.getResponseBody(), e); log.error("创建pod异常:" + e.getResponseBody(), e);
} catch (Exception e) { } catch (Exception e) {
@@ -485,14 +485,12 @@ public class K8sClientUtil {
} }






/** /**
* 根据获取namespace,deploymentName的Pod Name * 根据获取namespace,deploymentName的Pod Name
* *
* @return podList * @return podList
*/ */
public V1Pod getNSPodList(String namespace,String deploymentName) throws Exception {
public V1Pod getNSPodList(String namespace, String deploymentName) throws Exception {
// new a CoreV1Api // new a CoreV1Api
CoreV1Api api = new CoreV1Api(apiClient); CoreV1Api api = new CoreV1Api(apiClient);
V1PodList v1PodList = null; V1PodList v1PodList = null;
@@ -513,12 +511,12 @@ public class K8sClientUtil {
return null; return null;
} }


public String executeCommand(V1Pod item, String command) {
public String executeCommand(V1Pod item, String command) {
try { try {
// 创建API实例 // 创建API实例
// 创建Exec实例 // 创建Exec实例
Exec exec = new Exec(apiClient); Exec exec = new Exec(apiClient);
String[] cmd = { "/bin/sh", "-c", command};
String[] cmd = {"/bin/sh", "-c", command};
Process proc = exec.exec(item, cmd, false); Process proc = exec.exec(item, cmd, false);
// 读取输出 // 读取输出
BufferedReader reader = new BufferedReader(new InputStreamReader(proc.getInputStream())); BufferedReader reader = new BufferedReader(new InputStreamReader(proc.getInputStream()));
@@ -543,10 +541,11 @@ public class K8sClientUtil {


/** /**
* 根据Pod的名称和Namespace查询Pod的状态 * 根据Pod的名称和Namespace查询Pod的状态
* @param podName Pod的名称
*
* @param podName Pod的名称
* @param namespace Pod所在的Namespace * @param namespace Pod所在的Namespace
*/ */
public String getPodStatus(String podName, String namespace) throws Exception {
public String getPodStatus(String podName, String namespace) throws Exception {
CoreV1Api api = new CoreV1Api(apiClient); CoreV1Api api = new CoreV1Api(apiClient);
V1Pod pod = api.readNamespacedPod(podName, namespace, null, null, null); V1Pod pod = api.readNamespacedPod(podName, namespace, null, null, null);
return pod.getStatus().getPhase(); return pod.getStatus().getPhase();
@@ -554,14 +553,15 @@ public class K8sClientUtil {


/** /**
* 根据Pod的名称和Namespace查询Pod的容器信息 * 根据Pod的名称和Namespace查询Pod的容器信息
* @param podName Pod的名称
*
* @param podName Pod的名称
* @param namespace Pod所在的Namespace * @param namespace Pod所在的Namespace
*/ */
public String getPodContainerId(String podName, String namespace) throws Exception { public String getPodContainerId(String podName, String namespace) throws Exception {
CoreV1Api api = new CoreV1Api(apiClient); CoreV1Api api = new CoreV1Api(apiClient);
V1Pod pod = api.readNamespacedPod(podName, namespace, null, null, null); V1Pod pod = api.readNamespacedPod(podName, namespace, null, null, null);


if(pod.getStatus().getContainerStatuses().size() !=1){
if (pod.getStatus().getContainerStatuses().size() != 1) {
throw new RuntimeException("容器错误"); throw new RuntimeException("容器错误");
} }
String containerId = pod.getStatus().getContainerStatuses().get(0).getContainerID().split("//")[1]; String containerId = pod.getStatus().getContainerStatuses().get(0).getContainerID().split("//")[1];
@@ -580,10 +580,10 @@ public class K8sClientUtil {
return pod.getSpec().getNodeName(); return pod.getSpec().getNodeName();
} }


public String getPodLogs(String podName,String namespace,String container,int line) {
public String getPodLogs(String podName, String namespace, String container, int line) {
CoreV1Api api = new CoreV1Api(apiClient); CoreV1Api api = new CoreV1Api(apiClient);
try { try {
String log = api.readNamespacedPodLog(podName, namespace, StringUtils.isEmpty(container)?null:container, null, null, null, null,null, null, line, null);
String log = api.readNamespacedPodLog(podName, namespace, StringUtils.isEmpty(container) ? null : container, null, null, null, null, null, null, line, null);
return log; return log;
} catch (ApiException e) { } catch (ApiException e) {
throw new RuntimeException("获取Pod日志异常", e); throw new RuntimeException("获取Pod日志异常", e);
@@ -592,7 +592,7 @@ public class K8sClientUtil {
} }




public V1Pod createPodWithEnv(String podName,String namespace,String proxyUrl ,String mountPath,String pvcName, String image){
public V1Pod createPodWithEnv(String podName, String namespace, String proxyUrl, String mountPath, String pvcName, String image) {
CoreV1Api api = new CoreV1Api(apiClient); CoreV1Api api = new CoreV1Api(apiClient);
V1PodList v1PodList = null; V1PodList v1PodList = null;
V1Pod pod = new V1PodBuilder() V1Pod pod = new V1PodBuilder()
@@ -636,7 +636,7 @@ public class K8sClientUtil {
/** /**
* 删除 Pod * 删除 Pod
* *
* @param podName Pod 名称
* @param podName Pod 名称
* @param namespace 命名空间 * @param namespace 命名空间
* @throws ApiException 异常 * @throws ApiException 异常
*/ */
@@ -655,7 +655,7 @@ public class K8sClientUtil {
/** /**
* 删除 Service * 删除 Service
* *
* @param svcName Service 名称
* @param svcName Service 名称
* @param namespace 命名空间 * @param namespace 命名空间
* @throws ApiException 异常 * @throws ApiException 异常
*/ */
@@ -674,7 +674,7 @@ public class K8sClientUtil {
/** /**
* 检查 Pod 是否存在 * 检查 Pod 是否存在
* *
* @param podName Pod 名称
* @param podName Pod 名称
* @param namespace 命名空间 * @param namespace 命名空间
* @return 是否存在 * @return 是否存在
* @throws ApiException 异常 * @throws ApiException 异常
@@ -682,7 +682,7 @@ public class K8sClientUtil {
public boolean checkPodExists(String podName, String namespace) throws ApiException { public boolean checkPodExists(String podName, String namespace) throws ApiException {
CoreV1Api api = new CoreV1Api(apiClient); CoreV1Api api = new CoreV1Api(apiClient);
try { try {
api.readNamespacedPod(podName, namespace, null,false,false);
api.readNamespacedPod(podName, namespace, null, false, false);
return true; return true;
} catch (ApiException e) { } catch (ApiException e) {
if (e.getCode() == 404) { if (e.getCode() == 404) {


Loading…
Cancel
Save