From 0e571126150acb88c4d0de444b57feb9efa5955d Mon Sep 17 00:00:00 2001 From: liuzx Date: Wed, 15 Nov 2023 09:59:37 +0800 Subject: [PATCH] =?UTF-8?q?=E6=9B=B4=E6=96=B0readme?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- gcu_mnist_example/gpu_mnist_example/README.md | 50 ++++++ gcu_mnist_example/gpu_mnist_example/model.py | 35 ++++ .../gpu_mnist_example/train_gcu.py | 166 ++++++++++++++++++ gpu_mnist_example/README.md | 5 +- npu_mnist_example/README.md | 2 +- 5 files changed, 254 insertions(+), 4 deletions(-) create mode 100644 gcu_mnist_example/gpu_mnist_example/README.md create mode 100644 gcu_mnist_example/gpu_mnist_example/model.py create mode 100644 gcu_mnist_example/gpu_mnist_example/train_gcu.py diff --git a/gcu_mnist_example/gpu_mnist_example/README.md b/gcu_mnist_example/gpu_mnist_example/README.md new file mode 100644 index 0000000..3b9351e --- /dev/null +++ b/gcu_mnist_example/gpu_mnist_example/README.md @@ -0,0 +1,50 @@ +# 如何在启智平台上进行模型训练—GCU示例 + +## 1.启智集群和智算集群的GCU训练样例 + +###### 启智集群的示例代码： + +- 训练示例请参考示例中[train_gcu.py](./train_gcu.py)的代码注释 + +## 2. 在openi上获取数据集，预训练模型，输出路径 + +安装openi包 + +``` +pip install -U openi +``` + +使用openi包 + +``` +#导入包 +from openi.context import prepare, upload_openi + +#初始化导入数据集和预训练模型到容器内 +openi_context = prepare() + +#获取数据集路径，预训练模型路径，输出路径 +dataset_path = openi_context.dataset_path +pretrain_model_path = openi_context.pretrain_model_path +output_path = openi_context.output_path + +#回传结果到openi +upload_openi() +``` + +## 3.FAQ + +### 3.1 关于启智平台公共库[openi](https://openi.pcl.ac.cn/OpenIOSSG/openi-pypi)： + +主要使用的方法有以下几个： + +``` +prepare 准备数据集，模型，输出路径 +upload_openi 将训练镜像的输出结果拷贝回启智平台 +``` + +### 3.2 解决参数报错问题： + +请在代码中加入 `args, unknown = parser.parse_known_args()`，可忽略掉 `--ckpt_url`， `--multi_date_url`等参数报错问题 + +## 对于示例代码有任何问题，欢迎在本项目中提issue。 diff --git a/gcu_mnist_example/gpu_mnist_example/model.py b/gcu_mnist_example/gpu_mnist_example/model.py new file mode 100644 index 0000000..ae424a7 --- /dev/null +++ b/gcu_mnist_example/gpu_mnist_example/model.py @@ -0,0 +1,35 @@ +from torch.nn import Module +from torch import nn + + +class Model(Module): + def __init__(self): + super(Model, self).__init__() + self.conv1 = nn.Conv2d(1, 6, 5) + self.relu1 = nn.ReLU() + self.pool1 = nn.MaxPool2d(2) + self.conv2 = nn.Conv2d(6, 16, 5) + self.relu2 = nn.ReLU() + self.pool2 = nn.MaxPool2d(2) + self.fc1 = nn.Linear(256, 120) + self.relu3 = nn.ReLU() + self.fc2 = nn.Linear(120, 84) + self.relu4 = nn.ReLU() + self.fc3 = nn.Linear(84, 10) + self.relu5 = nn.ReLU() + + def forward(self, x): + y = self.conv1(x) + y = self.relu1(y) + y = self.pool1(y) + y = self.conv2(y) + y = self.relu2(y) + y = self.pool2(y) + y = y.view(y.shape[0], -1) + y = self.fc1(y) + y = self.relu3(y) + y = self.fc2(y) + y = self.relu4(y) + y = self.fc3(y) + y = self.relu5(y) + return y diff --git a/gcu_mnist_example/gpu_mnist_example/train_gcu.py b/gcu_mnist_example/gpu_mnist_example/train_gcu.py new file mode 100644 index 0000000..0050dbf --- /dev/null +++ b/gcu_mnist_example/gpu_mnist_example/train_gcu.py @@ -0,0 +1,166 @@ +#!/usr/bin/python +#coding=utf-8 +''' +If there are Chinese comments in the code，please add at the beginning： +#!/usr/bin/python +#coding=utf-8 + +示例选用的数据集是MnistDataset.zip +数据集结构是： + MnistDataset.zip + ├── test + │ ├── MNIST/processed/test.pt + │ └── MNIST/processed/training.pt + │ ├── MNIST/raw/train-images-idx3-ubyte + │ └── MNIST/raw/train-labels-idx1-ubyte + │ ├── MNIST/raw/t10k-images-idx3-ubyte + │ └── MNIST/raw/t10k-labels-idx1-ubyte + ├── train + │ ├── MNIST/processed/test.pt + │ └── MNIST/processed/training.pt + │ ├── MNIST/raw/train-images-idx3-ubyte + │ └── MNIST/raw/train-labels-idx1-ubyte + │ ├── MNIST/raw/t10k-images-idx3-ubyte + │ └── MNIST/raw/t10k-labels-idx1-ubyte + + + +示例选用的预训练模型文件夹为MNIST_PytorchExample_GPU_test34_model_7f9j，模型文件为：mnist_epoch1_0.70.pkl + +''' + +import os +print("begin:") +os.system("pip uninstall openi-test") +os.system("pip install {}".format(os.getenv("OPENI_SDK_PATH"))) +import torch +from model import Model +import numpy as np +from torchvision.datasets import mnist +from torch.nn import CrossEntropyLoss +from torch.optim import SGD +from torch.utils.data import DataLoader +from torchvision.transforms import ToTensor +import argparse +from openi.context import prepare, upload_openi + +import importlib.util + +def is_torch_dtu_available(): + if importlib.util.find_spec("torch_dtu") is None: + return False + if importlib.util.find_spec("torch_dtu.core") is None: + return False + return importlib.util.find_spec("torch_dtu.core.dtu_model") is not None + +# Training settings +parser = argparse.ArgumentParser(description='PyTorch MNIST Example') +parser.add_argument('--epoch_size', type=int, default=1, help='how much epoch to train') +parser.add_argument('--batch_size', type=int, default=256, help='how much batch_size in epoch') + + +if __name__ == '__main__': + #获取参数并忽略超参数报错 + args, unknown = parser.parse_known_args() + + #初始化导入数据集和预训练模型到容器内 + openi_context = prepare() + + #获取数据集路径，预训练模型路径，输出路径 + dataset_path = openi_context.dataset_path + pretrain_model_path = openi_context.pretrain_model_path + output_path = openi_context.output_path + + dataset_path_A = dataset_path + "/MnistDataset" + pretrain_model_path_A = pretrain_model_path + "/MNIST_PytorchExample_GPU_test34_model_7f9j" + + print("dataset_path:") + print(os.listdir(dataset_path)) + os.listdir(dataset_path) + + print("pretrain_model_path:") + print(os.listdir(pretrain_model_path)) + os.listdir(pretrain_model_path) + + print("output_path:") + print(os.listdir(output_path)) + os.listdir(output_path) + # load DPU envs-xx.sh + DTU_FLAG = True + if is_torch_dtu_available(): + import torch_dtu + import torch_dtu.distributed as dist + import torch_dtu.core.dtu_model as dm + from torch_dtu.nn.parallel import DistributedDataParallel as torchDDP + print('dtu is available: True') + device = dm.dtu_device() + DTU_FLAG = True + else: + print('dtu is available: False') + device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") + DTU_FLAG = False + + + # 参数声明 + model = Model().to(device) + optimizer = SGD(model.parameters(), lr=1e-1) + #log output + batch_size = args.batch_size + train_dataset = mnist.MNIST(root=dataset_path_A + "/train", train=True, transform=ToTensor(),download=False) + test_dataset = mnist.MNIST(root=dataset_path_A + "/test", train=False, transform=ToTensor(),download=False) + train_loader = DataLoader(train_dataset, batch_size=batch_size) + test_loader = DataLoader(test_dataset, batch_size=batch_size) + model = Model().to(device) + sgd = SGD(model.parameters(), lr=1e-1) + cost = CrossEntropyLoss() + epochs = args.epoch_size + print('epoch_size is:{}'.format(epochs)) + + # 如果有保存的模型，则加载模型，并在其基础上继续训练 + if os.path.exists(pretrain_model_path_A+"/mnist_epoch1_0.70.pkl"): + checkpoint = torch.load(pretrain_model_path_A+"/mnist_epoch1_0.70.pkl") + model.load_state_dict(checkpoint['model']) + optimizer.load_state_dict(checkpoint['optimizer']) + start_epoch = checkpoint['epoch'] + print('加载 epoch {} 权重成功！'.format(start_epoch)) + else: + start_epoch = 0 + print('无保存模型，将从头开始训练！') + + for _epoch in range(start_epoch, epochs): + print('the {} epoch_size begin'.format(_epoch + 1)) + model.train() + for idx, (train_x, train_label) in enumerate(train_loader): + train_x = train_x.to(device) + train_label = train_label.to(device) + label_np = np.zeros((train_label.shape[0], 10)) + sgd.zero_grad() + predict_y = model(train_x.float()) + loss = cost(predict_y, train_label.long()) + if idx % 10 == 0: + print('idx: {}, loss: {}'.format(idx, loss.sum().item())) + loss.backward() + if DTU_FLAG: + dm.optimizer_step(sgd, barrier=True) + else: + sgd.step() + + + correct = 0 + _sum = 0 + model.eval() + for idx, (test_x, test_label) in enumerate(test_loader): + test_x = test_x + test_label = test_label + predict_y = model(test_x.to(device).float()).detach() + predict_ys = np.argmax(predict_y.cpu(), axis=-1) + label_np = test_label.numpy() + _ = predict_ys == test_label + correct += np.sum(_.numpy(), axis=-1) + _sum += _.shape[0] + print('accuracy: {:.2f}'.format(correct / _sum)) + #The model output location is placed under output_path + state = {'model':model.state_dict(), 'optimizer':optimizer.state_dict(), 'epoch':_epoch+1} + torch.save(state, '{}/mnist_epoch{}_{:.2f}.pkl'.format(output_path,_epoch+1, correct / _sum)) + print('test:') + print(os.listdir(output_path)) \ No newline at end of file diff --git a/gpu_mnist_example/README.md b/gpu_mnist_example/README.md index 2594aba..a12cb78 100644 --- a/gpu_mnist_example/README.md +++ b/gpu_mnist_example/README.md @@ -4,7 +4,7 @@ ###### 启智集群的示例代码： -- 训练示例请参考示例中[train_npu.py](./train_npu.py)的代码注释 +- 训练示例请参考示例中[train_gpu.py](./train_gpu.py)的代码注释 ## 2. 在openi上获取数据集，预训练模型，输出路径 @@ -43,8 +43,7 @@ upload_openi() ``` prepare 准备数据集，模型，输出路径 upload_openi 将训练镜像的输出结果拷贝回启智平台 -obs_copy_file 通过mox拷贝文件 -obs_copy_folder 通过mox拷贝文件夹 + ``` ### 4.2 解决参数报错问题： diff --git a/npu_mnist_example/README.md b/npu_mnist_example/README.md index d0a7e5d..0dffdda 100644 --- a/npu_mnist_example/README.md +++ b/npu_mnist_example/README.md @@ -59,7 +59,7 @@ obs_copy_folder 通过mox拷贝文件夹 ``` if local_rank%8==0: - 这里省略下载数据的代码... + 这里省略下载数据的代码...(openi.context.prepare()) f = open("/cache/download_input.txt", 'w') f.close() try: