From c594dca0ec33f6a26043e4fc26610ecba13e4825 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=A5=BF=E5=A4=A7=E9=94=90?= <1070211640@qq.com> Date: Tue, 27 Aug 2024 15:55:13 +0800 Subject: [PATCH] =?UTF-8?q?1=E3=80=81=E6=B7=BB=E5=8A=A0=E9=80=89=E6=8B=A9?= =?UTF-8?q?=E9=95=9C=E5=83=8F=E5=90=AF=E5=8A=A8=E7=8E=AF=E5=A2=83=E5=8A=9F?= =?UTF-8?q?=E8=83=BD=E3=80=82=202=E3=80=81pod=E9=80=89=E6=8B=A9GPU?= =?UTF-8?q?=E8=8A=82=E7=82=B9=E9=83=A8=E7=BD=B2=E3=80=82=203=E3=80=81?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E4=BF=9D=E5=AD=98=E9=95=9C=E5=83=8F=E6=8E=A5?= =?UTF-8?q?=E5=8F=A3=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ruoyi-modules/management-platform/pom.xml | 13 ++- .../controller/image/ImageController.java | 9 +- .../ruoyi/platform/service/ImageService.java | 3 +- .../service/impl/ImageServiceImpl.java | 29 ++++++ .../service/impl/JupyterServiceImpl.java | 2 +- .../platform/utils/DockerClientUtil.java | 98 +++++++++++++++++++ .../ruoyi/platform/utils/K8sClientUtil.java | 28 +++++- .../java/com/ruoyi/platform/vo/ImageVo.java | 14 ++- ruoyi-modules/ruoyi-job/pom.xml | 10 +- 9 files changed, 196 insertions(+), 10 deletions(-) create mode 100644 ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/DockerClientUtil.java diff --git a/ruoyi-modules/management-platform/pom.xml b/ruoyi-modules/management-platform/pom.xml index 37234ad0..c004f9ae 100644 --- a/ruoyi-modules/management-platform/pom.xml +++ b/ruoyi-modules/management-platform/pom.xml @@ -140,7 +140,12 @@ com.github.docker-java docker-java - 3.1.1 + 3.2.13 + + + com.github.docker-java + docker-java-transport-httpclient5 + 3.2.13 com.baomidou @@ -216,6 +221,12 @@ 3.0.8 compile + + commons-lang + commons-lang + 2.6 + compile + diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/image/ImageController.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/image/ImageController.java index a0b117d8..d389bad8 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/image/ImageController.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/image/ImageController.java @@ -78,16 +78,15 @@ public class ImageController extends BaseController { * @param image 实体 * @return 新增结果 */ - @PostMapping @ApiOperation("新增镜像,不包含镜像版本") public GenericsAjaxResult add(@RequestBody Image image) { return genericsSuccess(this.imageService.insert(image)); } /** - * 新增镜像和版本 * * @param imageVo 实体 + * 新增镜像和版本 @PostMapping * @return 新增结果 */ @PostMapping("/addImageAndVersion") @@ -149,5 +148,11 @@ public class ImageController extends BaseController { return genericsSuccess(this.imageService.uploadImageFiles(file)); } + + @PostMapping("/saveImage") + @ApiOperation(value = "保存环境为镜像", notes = "docker commit方式保存，并推送到horbor") + public GenericsAjaxResult saveImage(ImageVo imageVo){ + return genericsSuccess(this.imageService.saveImage(imageVo)); + } } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ImageService.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ImageService.java index 59614ac9..fb8984ed 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ImageService.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ImageService.java @@ -93,6 +93,5 @@ public interface ImageService { Map createImageFromNet(String imageName, String imageTag, String NetPath) throws Exception; Map uploadImageFiles(MultipartFile file) throws Exception; - - + String saveImage(ImageVo imageVo); } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ImageServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ImageServiceImpl.java index d95609ee..17bfdd48 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ImageServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ImageServiceImpl.java @@ -9,12 +9,14 @@ import com.ruoyi.platform.mapper.ImageVersionDao; import com.ruoyi.platform.service.ImageService; import com.ruoyi.platform.service.ImageVersionService; import com.ruoyi.platform.service.MinioService; +import com.ruoyi.platform.utils.DockerClientUtil; import com.ruoyi.platform.utils.FileUtil; import com.ruoyi.platform.utils.K8sClientUtil; import com.ruoyi.platform.vo.ImageVo; import com.ruoyi.system.api.model.LoginUser; import io.kubernetes.client.openapi.models.V1PersistentVolumeClaim; import io.kubernetes.client.openapi.models.V1Pod; +import lombok.Synchronized; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.data.domain.Page; @@ -50,6 +52,9 @@ public class ImageServiceImpl implements ImageService { private ImageVersionDao imageVersionDao; @Resource private K8sClientUtil k8sClientUtil; + @Resource + private DockerClientUtil dockerClientUtil; + @Resource private MinioService minioService; @Value("${harbor.bucketName}") @@ -75,6 +80,8 @@ public class ImageServiceImpl implements ImageService { private String proxyUrl; @Value("${minio.pvcName}") private String pvcName; + @Value("${jupyter.namespace}") + private String namespace; /** * 通过ID查询单条数据 * @@ -350,4 +357,26 @@ public class ImageServiceImpl implements ImageService { String path = loginUser.getUsername()+"/"+file.getOriginalFilename(); return minioService.uploadFile(bucketName, path, file); } + + @Override + @Synchronized + public String saveImage(ImageVo imageVo) { + if(imageDao.getByName(imageVo.getName()) != null){ + throw new IllegalStateException("镜像名称已存在"); + } + LoginUser loginUser = SecurityUtils.getLoginUser(); + String username = loginUser.getUsername().toLowerCase(); + String podName = username +"-editor-pod" + "-" + imageVo.getDevEnvironmentId(); + + try { + String containerId = k8sClientUtil.getPodContainerId(podName, namespace); + String hostIp = k8sClientUtil.getHostIp(podName, namespace); + + dockerClientUtil.commitImage(imageVo,containerId,hostIp,username); + dockerClientUtil.pushImageToHorbor(imageVo,hostIp); + } catch (Exception e) { + throw new RuntimeException(e); + } + return null; + } } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/JupyterServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/JupyterServiceImpl.java index be55468a..aa59b6ae 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/JupyterServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/JupyterServiceImpl.java @@ -100,7 +100,7 @@ public class JupyterServiceImpl implements JupyterService { //TODO 设置镜像可配置，这里先用默认镜像启动pod // 调用修改后的 createPod 方法，传入额外的参数 - Integer podPort = k8sClientUtil.createConfiguredPod(podName, namespace, port, mountPath, pvc, image, minioPvcName, datasetPath, modelPath); + Integer podPort = k8sClientUtil.createConfiguredPod(podName, namespace, port, mountPath, pvc, devEnvironment.getImage(), minioPvcName, datasetPath, modelPath); String url = masterIp + ":" + podPort; redisService.setCacheObject(podName,masterIp + ":" + podPort); devEnvironment.setStatus("Pending"); diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/DockerClientUtil.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/DockerClientUtil.java new file mode 100644 index 00000000..15a1134c --- /dev/null +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/DockerClientUtil.java @@ -0,0 +1,98 @@ +package com.ruoyi.platform.utils; + +import com.github.dockerjava.api.DockerClient; +import com.github.dockerjava.api.command.CommitCmd; +import com.github.dockerjava.api.model.AuthConfig; +import com.github.dockerjava.core.DefaultDockerClientConfig; +import com.github.dockerjava.core.DockerClientConfig; +import com.github.dockerjava.core.DockerClientImpl; +import com.github.dockerjava.httpclient5.ApacheDockerHttpClient; +import com.github.dockerjava.transport.DockerHttpClient; +import com.ruoyi.platform.vo.ImageVo; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import java.time.Duration; + +@Slf4j +@Component +public class DockerClientUtil { + + @Value("${harbor.bucketName}") + private String bucketName; + @Value("${harbor.repository}") + private String repository; + @Value("${harbor.harborUrl}") + private String harborUrl; + @Value("${harbor.harborUser}") + private String harborUser; + @Value("${harbor.harborpassword}") + private String harborpassword; + + public DockerClient getDockerClient(String dockerServerUrl) { + + //创建DefaultDockerClientConfig（指定docker服务器的配置） + DockerClientConfig config = DefaultDockerClientConfig + .createDefaultConfigBuilder() + .withDockerHost("tcp://"+ dockerServerUrl +":2375") + .withDockerTlsVerify(false) + .withApiVersion("1.40") +// .withDockerCertPath(dcokerCertPath) +// .withRegistryUsername(registryUser) +// .withRegistryPassword(registryPass) +// .withRegistryEmail(registryMail) +// .withRegistryUrl(registryUrl) + .build(); + + //创建DockerHttpClient + DockerHttpClient httpClient = new ApacheDockerHttpClient.Builder() + .dockerHost(config.getDockerHost()) + .sslConfig(config.getSSLConfig()) + .maxConnections(1000) + .connectionTimeout(Duration.ofSeconds(300)) + .responseTimeout(Duration.ofSeconds(450)) + .build(); + + //创建DockerClient + return DockerClientImpl.getInstance(config, httpClient); + + } + + public String commitImage(ImageVo imageVo, String containerId, String hostIp, String userName) { + DockerClient dockerClient = getDockerClient(hostIp); + +// dockerClient.startContainerCmd(containerId).exec(); + // 提交容器为镜像，这里的"new_image"和"new_tag"是新镜像的名字和标签 + CommitCmd commitCmd = dockerClient.commitCmd(containerId) + .withRepository(imageVo.getName()) + .withTag(imageVo.getTagName()) + .withAuthor(userName) + .withMessage(imageVo.getDescription()); + String exec = commitCmd.exec(); + return exec; + } + + public void pushImageToHorbor(ImageVo imageVo, String hostIp) { + + DockerClient dockerClient = getDockerClient(hostIp); + //Harbor登录信息 + AuthConfig autoConfig = new AuthConfig().withRegistryAddress(harborUrl).withUsername(harborUser).withPassword(harborpassword); + + + String localImageName = imageVo.getName() + ":" + imageVo.getTagName(); + String imageName = harborUrl + "/" + bucketName + "/" + imageVo.getName(); + + //给镜像打上tag + dockerClient.tagImageCmd(localImageName, imageName, imageVo.getTagName()).exec(); + //推送镜像至镜像仓库 + try { + dockerClient.pushImageCmd(imageName).withAuthConfig(autoConfig).start().awaitCompletion(); + //push成功后，删除本地加载的镜像 + dockerClient.removeImageCmd(localImageName).exec(); + } catch (InterruptedException e) { + throw new RuntimeException("推送镜像失败:"+e); + } + + } +} \ No newline at end of file diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/K8sClientUtil.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/K8sClientUtil.java index 23903e2a..e2f1f6ac 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/K8sClientUtil.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/K8sClientUtil.java @@ -136,7 +136,7 @@ public class K8sClientUtil { if (v1ServiceList!=null) { for (V1Service svc : v1ServiceList.getItems()) { if (StringUtils.equals(svc.getMetadata().getName(), serviceName)) { - // PVC 已存在 + // SVC 已存在 return svc; } } @@ -380,6 +380,9 @@ public class K8sClientUtil { Map selector = new LinkedHashMap<>(); selector.put("k8s-jupyter", podName); + Map nodeSelector = new LinkedHashMap<>(); + nodeSelector.put("resource-type", "CPU-GPU"); + CoreV1Api api = new CoreV1Api(apiClient); V1PodList v1PodList = null; try { @@ -423,6 +426,7 @@ public class K8sClientUtil { .withVolumeMounts(volumeMounts) .endContainer() .withVolumes(volumes) + .withNodeSelector(nodeSelector) .endSpec() .build(); @@ -506,6 +510,28 @@ public class K8sClientUtil { return pod.getStatus().getPhase(); } + /** + * 根据Pod的名称和Namespace查询Pod的容器信息 + * @param podName Pod的名称 + * @param namespace Pod所在的Namespace + */ + public String getPodContainerId(String podName, String namespace) throws Exception { + CoreV1Api api = new CoreV1Api(apiClient); + V1Pod pod = api.readNamespacedPod(podName, namespace, null, null, null); + + if(pod.getStatus().getContainerStatuses().size() !=1){ + throw new RuntimeException("容器错误"); + } + String containerId = pod.getStatus().getContainerStatuses().get(0).getContainerID().split("//")[1]; + return containerId; + } + + public String getHostIp(String podName, String namespace) throws Exception { + CoreV1Api api = new CoreV1Api(apiClient); + V1Pod pod = api.readNamespacedPod(podName, namespace, null, null, null); + return pod.getStatus().getHostIP(); + } + public String getPodLogs(String podName,String namespace,String container,int line) { CoreV1Api api = new CoreV1Api(apiClient); try { diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ImageVo.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ImageVo.java index c4b313c2..6ee88868 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ImageVo.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ImageVo.java @@ -43,7 +43,7 @@ public class ImageVo implements Serializable { /** * 镜像tag名称 */ - @ApiModelProperty(name = "tag_name") + @ApiModelProperty(name = "tagName") private String tagName; /** @@ -64,6 +64,8 @@ public class ImageVo implements Serializable { private String path; + @ApiModelProperty(value = "环境id") + private String devEnvironmentId; // public Integer getId() { // return id; // } @@ -147,7 +149,17 @@ public class ImageVo implements Serializable { public String getPath() { return path; } + public void setPath(String path) { this.path = path; } + + public String getDevEnvironmentId() { + return devEnvironmentId; + } + + public void setDevEnvironmentId(String devEnvironmentId) { + this.devEnvironmentId = devEnvironmentId; + } + } diff --git a/ruoyi-modules/ruoyi-job/pom.xml b/ruoyi-modules/ruoyi-job/pom.xml index e295d995..7b22bc72 100644 --- a/ruoyi-modules/ruoyi-job/pom.xml +++ b/ruoyi-modules/ruoyi-job/pom.xml @@ -65,7 +65,7 @@ com.mysql mysql-connector-j - + com.ruoyi @@ -77,7 +77,13 @@ com.ruoyi ruoyi-common-swagger - + + + + com.ruoyi + ruoyi-common-datasource + +