|
|
|
@@ -6,6 +6,7 @@ import ( |
|
|
|
"errors" |
|
|
|
"fmt" |
|
|
|
"net/http" |
|
|
|
"net/url" |
|
|
|
"path" |
|
|
|
"strings" |
|
|
|
|
|
|
|
@@ -27,19 +28,22 @@ const ( |
|
|
|
MODEL_NOT_LATEST = 0 |
|
|
|
) |
|
|
|
|
|
|
|
func saveModelByParameters(jobId string, versionName string, name string, version string, label string, description string, ctx *context.Context) error { |
|
|
|
func saveModelByParameters(jobId string, versionName string, name string, version string, label string, description string, engine int, ctx *context.Context) error { |
|
|
|
aiTask, err := models.GetCloudbrainByJobIDAndVersionName(jobId, versionName) |
|
|
|
if err != nil { |
|
|
|
log.Info("query task error." + err.Error()) |
|
|
|
return err |
|
|
|
aiTask, err = models.GetRepoCloudBrainByJobID(ctx.Repo.Repository.ID, jobId) |
|
|
|
if err != nil { |
|
|
|
log.Info("query task error." + err.Error()) |
|
|
|
return err |
|
|
|
} else { |
|
|
|
log.Info("query gpu train task.") |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
uuid := uuid.NewV4() |
|
|
|
id := uuid.String() |
|
|
|
modelPath := id |
|
|
|
var lastNewModelId string |
|
|
|
var modelSize int64 |
|
|
|
cloudType := models.TypeCloudBrainTwo |
|
|
|
|
|
|
|
log.Info("find task name:" + aiTask.JobName) |
|
|
|
aimodels := models.QueryModelByName(name, aiTask.RepoID) |
|
|
|
@@ -53,7 +57,7 @@ func saveModelByParameters(jobId string, versionName string, name string, versio |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
cloudType = aiTask.Type |
|
|
|
cloudType := aiTask.Type |
|
|
|
//download model zip //train type |
|
|
|
if cloudType == models.TypeCloudBrainTwo { |
|
|
|
modelPath, modelSize, err = downloadModelFromCloudBrainTwo(id, aiTask.JobName, "", aiTask.TrainUrl) |
|
|
|
@@ -61,6 +65,12 @@ func saveModelByParameters(jobId string, versionName string, name string, versio |
|
|
|
log.Info("download model from CloudBrainTwo faild." + err.Error()) |
|
|
|
return err |
|
|
|
} |
|
|
|
} else if cloudType == models.TypeCloudBrainOne { |
|
|
|
modelPath, modelSize, err = downloadModelFromCloudBrainOne(id, aiTask.JobName, "", aiTask.TrainUrl) |
|
|
|
if err != nil { |
|
|
|
log.Info("download model from CloudBrainOne faild." + err.Error()) |
|
|
|
return err |
|
|
|
} |
|
|
|
} |
|
|
|
accuracy := make(map[string]string) |
|
|
|
accuracy["F1"] = "" |
|
|
|
@@ -143,8 +153,9 @@ func SaveModel(ctx *context.Context) { |
|
|
|
version := ctx.Query("Version") |
|
|
|
label := ctx.Query("Label") |
|
|
|
description := ctx.Query("Description") |
|
|
|
engine := ctx.QueryInt("Engine") |
|
|
|
trainTaskCreate := ctx.QueryBool("trainTaskCreate") |
|
|
|
|
|
|
|
log.Info("engine=" + fmt.Sprint(engine)) |
|
|
|
if !trainTaskCreate { |
|
|
|
if !ctx.Repo.CanWrite(models.UnitTypeModelManage) { |
|
|
|
//ctx.NotFound(ctx.Req.URL.RequestURI(), nil) |
|
|
|
@@ -163,7 +174,7 @@ func SaveModel(ctx *context.Context) { |
|
|
|
return |
|
|
|
} |
|
|
|
|
|
|
|
err := saveModelByParameters(JobId, VersionName, name, version, label, description, ctx) |
|
|
|
err := saveModelByParameters(JobId, VersionName, name, version, label, description, engine, ctx) |
|
|
|
|
|
|
|
if err != nil { |
|
|
|
log.Info("save model error." + err.Error()) |
|
|
|
@@ -199,6 +210,22 @@ func downloadModelFromCloudBrainTwo(modelUUID string, jobName string, parentDir |
|
|
|
return dataActualPath, size, nil |
|
|
|
} |
|
|
|
|
|
|
|
func downloadModelFromCloudBrainOne(modelUUID string, jobName string, parentDir string, trainUrl string) (string, int64, error) { |
|
|
|
modelActualPath := storage.GetMinioPath(jobName, "/model/") |
|
|
|
log.Info("modelActualPath=" + modelActualPath) |
|
|
|
modelSrcPrefix := setting.CBCodePathPrefix + jobName + "/model/" |
|
|
|
destKeyNamePrefix := Model_prefix + models.AttachmentRelativePath(modelUUID) + "/" |
|
|
|
bucketName := setting.Attachment.Minio.Bucket |
|
|
|
log.Info("destKeyNamePrefix=" + destKeyNamePrefix + " modelSrcPrefix=" + modelSrcPrefix + " bucket=" + bucketName) |
|
|
|
size, err := storage.MinioPathCopy(bucketName, modelSrcPrefix, destKeyNamePrefix) |
|
|
|
if err == nil { |
|
|
|
dataActualPath := bucketName + "/" + destKeyNamePrefix |
|
|
|
return dataActualPath, size, nil |
|
|
|
} else { |
|
|
|
return "", 0, nil |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
func DeleteModel(ctx *context.Context) { |
|
|
|
log.Info("delete model start.") |
|
|
|
id := ctx.Query("ID") |
|
|
|
@@ -277,51 +304,117 @@ func DownloadMultiModelFile(ctx *context.Context) { |
|
|
|
} |
|
|
|
|
|
|
|
path := Model_prefix + models.AttachmentRelativePath(id) + "/" |
|
|
|
if task.Type == models.TypeCloudBrainTwo { |
|
|
|
downloadFromCloudBrainTwo(path, task, ctx, id) |
|
|
|
} else if task.Type == models.TypeCloudBrainOne { |
|
|
|
downloadFromCloudBrainOne(path, task, ctx, id) |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
allFile, err := storage.GetAllObjectByBucketAndPrefix(setting.Bucket, path) |
|
|
|
func MinioDownloadManyFile(path string, ctx *context.Context, returnFileName string, allFile []storage.FileInfo) { |
|
|
|
ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+url.QueryEscape(returnFileName)) |
|
|
|
ctx.Resp.Header().Set("Content-Type", "application/octet-stream") |
|
|
|
w := zip.NewWriter(ctx.Resp) |
|
|
|
defer w.Close() |
|
|
|
for _, oneFile := range allFile { |
|
|
|
if oneFile.IsDir { |
|
|
|
log.Info("zip dir name:" + oneFile.FileName) |
|
|
|
} else { |
|
|
|
log.Info("zip file name:" + oneFile.FileName) |
|
|
|
fDest, err := w.Create(oneFile.FileName) |
|
|
|
if err != nil { |
|
|
|
log.Info("create zip entry error, download file failed: %s\n", err.Error()) |
|
|
|
ctx.ServerError("download file failed:", err) |
|
|
|
return |
|
|
|
} |
|
|
|
log.Info("minio file path=" + (path + oneFile.FileName)) |
|
|
|
body, err := storage.Attachments.DownloadAFile(setting.Attachment.Minio.Bucket, path+oneFile.FileName) |
|
|
|
if err != nil { |
|
|
|
log.Info("download file failed: %s\n", err.Error()) |
|
|
|
ctx.ServerError("download file failed:", err) |
|
|
|
return |
|
|
|
} else { |
|
|
|
defer body.Close() |
|
|
|
p := make([]byte, 1024) |
|
|
|
var readErr error |
|
|
|
var readCount int |
|
|
|
// 读取对象内容 |
|
|
|
for { |
|
|
|
readCount, readErr = body.Read(p) |
|
|
|
if readCount > 0 { |
|
|
|
fDest.Write(p[:readCount]) |
|
|
|
} |
|
|
|
if readErr != nil { |
|
|
|
break |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
func downloadFromCloudBrainOne(path string, task *models.AiModelManage, ctx *context.Context, id string) { |
|
|
|
allFile, err := storage.GetAllObjectByBucketAndPrefixMinio(setting.Attachment.Minio.Bucket, path) |
|
|
|
if err == nil { |
|
|
|
//count++ |
|
|
|
models.ModifyModelDownloadCount(id) |
|
|
|
|
|
|
|
returnFileName := task.Name + "_" + task.Version + ".zip" |
|
|
|
ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+returnFileName) |
|
|
|
ctx.Resp.Header().Set("Content-Type", "application/octet-stream") |
|
|
|
w := zip.NewWriter(ctx.Resp) |
|
|
|
defer w.Close() |
|
|
|
for _, oneFile := range allFile { |
|
|
|
if oneFile.IsDir { |
|
|
|
log.Info("zip dir name:" + oneFile.FileName) |
|
|
|
MinioDownloadManyFile(path, ctx, returnFileName, allFile) |
|
|
|
} else { |
|
|
|
log.Info("error,msg=" + err.Error()) |
|
|
|
ctx.ServerError("no file to download.", err) |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
func ObsDownloadManyFile(path string, ctx *context.Context, returnFileName string, allFile []storage.FileInfo) { |
|
|
|
ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+url.QueryEscape(returnFileName)) |
|
|
|
ctx.Resp.Header().Set("Content-Type", "application/octet-stream") |
|
|
|
w := zip.NewWriter(ctx.Resp) |
|
|
|
defer w.Close() |
|
|
|
for _, oneFile := range allFile { |
|
|
|
if oneFile.IsDir { |
|
|
|
log.Info("zip dir name:" + oneFile.FileName) |
|
|
|
} else { |
|
|
|
log.Info("zip file name:" + oneFile.FileName) |
|
|
|
fDest, err := w.Create(oneFile.FileName) |
|
|
|
if err != nil { |
|
|
|
log.Info("create zip entry error, download file failed: %s\n", err.Error()) |
|
|
|
ctx.ServerError("download file failed:", err) |
|
|
|
return |
|
|
|
} |
|
|
|
body, err := storage.ObsDownloadAFile(setting.Bucket, path+oneFile.FileName) |
|
|
|
if err != nil { |
|
|
|
log.Info("download file failed: %s\n", err.Error()) |
|
|
|
ctx.ServerError("download file failed:", err) |
|
|
|
return |
|
|
|
} else { |
|
|
|
log.Info("zip file name:" + oneFile.FileName) |
|
|
|
fDest, err := w.Create(oneFile.FileName) |
|
|
|
if err != nil { |
|
|
|
log.Info("create zip entry error, download file failed: %s\n", err.Error()) |
|
|
|
ctx.ServerError("download file failed:", err) |
|
|
|
return |
|
|
|
} |
|
|
|
body, err := storage.ObsDownloadAFile(setting.Bucket, path+oneFile.FileName) |
|
|
|
if err != nil { |
|
|
|
log.Info("download file failed: %s\n", err.Error()) |
|
|
|
ctx.ServerError("download file failed:", err) |
|
|
|
return |
|
|
|
} else { |
|
|
|
defer body.Close() |
|
|
|
p := make([]byte, 1024) |
|
|
|
var readErr error |
|
|
|
var readCount int |
|
|
|
// 读取对象内容 |
|
|
|
for { |
|
|
|
readCount, readErr = body.Read(p) |
|
|
|
if readCount > 0 { |
|
|
|
fDest.Write(p[:readCount]) |
|
|
|
} |
|
|
|
if readErr != nil { |
|
|
|
break |
|
|
|
} |
|
|
|
defer body.Close() |
|
|
|
p := make([]byte, 1024) |
|
|
|
var readErr error |
|
|
|
var readCount int |
|
|
|
// 读取对象内容 |
|
|
|
for { |
|
|
|
readCount, readErr = body.Read(p) |
|
|
|
if readCount > 0 { |
|
|
|
fDest.Write(p[:readCount]) |
|
|
|
} |
|
|
|
if readErr != nil { |
|
|
|
break |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
func downloadFromCloudBrainTwo(path string, task *models.AiModelManage, ctx *context.Context, id string) { |
|
|
|
allFile, err := storage.GetAllObjectByBucketAndPrefix(setting.Bucket, path) |
|
|
|
if err == nil { |
|
|
|
//count++ |
|
|
|
models.ModifyModelDownloadCount(id) |
|
|
|
returnFileName := task.Name + "_" + task.Version + ".zip" |
|
|
|
ObsDownloadManyFile(path, ctx, returnFileName, allFile) |
|
|
|
} else { |
|
|
|
log.Info("error,msg=" + err.Error()) |
|
|
|
ctx.ServerError("no file to download.", err) |
|
|
|
@@ -374,42 +467,55 @@ func DownloadSingleModelFile(ctx *context.Context) { |
|
|
|
ctx.NotFound(ctx.Req.URL.RequestURI(), nil) |
|
|
|
return |
|
|
|
} |
|
|
|
if setting.PROXYURL != "" { |
|
|
|
body, err := storage.ObsDownloadAFile(setting.Bucket, path) |
|
|
|
if err != nil { |
|
|
|
log.Info("download error.") |
|
|
|
if task.Type == models.TypeCloudBrainTwo { |
|
|
|
if setting.PROXYURL != "" { |
|
|
|
body, err := storage.ObsDownloadAFile(setting.Bucket, path) |
|
|
|
if err != nil { |
|
|
|
log.Info("download error.") |
|
|
|
} else { |
|
|
|
//count++ |
|
|
|
models.ModifyModelDownloadCount(id) |
|
|
|
defer body.Close() |
|
|
|
ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+fileName) |
|
|
|
ctx.Resp.Header().Set("Content-Type", "application/octet-stream") |
|
|
|
p := make([]byte, 1024) |
|
|
|
var readErr error |
|
|
|
var readCount int |
|
|
|
// 读取对象内容 |
|
|
|
for { |
|
|
|
readCount, readErr = body.Read(p) |
|
|
|
if readCount > 0 { |
|
|
|
ctx.Resp.Write(p[:readCount]) |
|
|
|
//fmt.Printf("%s", p[:readCount]) |
|
|
|
} |
|
|
|
if readErr != nil { |
|
|
|
break |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
} else { |
|
|
|
url, err := storage.GetObsCreateSignedUrlByBucketAndKey(setting.Bucket, path) |
|
|
|
if err != nil { |
|
|
|
log.Error("GetObsCreateSignedUrl failed: %v", err.Error(), ctx.Data["msgID"]) |
|
|
|
ctx.ServerError("GetObsCreateSignedUrl", err) |
|
|
|
return |
|
|
|
} |
|
|
|
//count++ |
|
|
|
models.ModifyModelDownloadCount(id) |
|
|
|
defer body.Close() |
|
|
|
ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+fileName) |
|
|
|
ctx.Resp.Header().Set("Content-Type", "application/octet-stream") |
|
|
|
p := make([]byte, 1024) |
|
|
|
var readErr error |
|
|
|
var readCount int |
|
|
|
// 读取对象内容 |
|
|
|
for { |
|
|
|
readCount, readErr = body.Read(p) |
|
|
|
if readCount > 0 { |
|
|
|
ctx.Resp.Write(p[:readCount]) |
|
|
|
//fmt.Printf("%s", p[:readCount]) |
|
|
|
} |
|
|
|
if readErr != nil { |
|
|
|
break |
|
|
|
} |
|
|
|
} |
|
|
|
http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently) |
|
|
|
} |
|
|
|
} else { |
|
|
|
url, err := storage.GetObsCreateSignedUrlByBucketAndKey(setting.Bucket, path) |
|
|
|
} else if task.Type == models.TypeCloudBrainOne { |
|
|
|
log.Info("start to down load minio file.") |
|
|
|
url, err := storage.Attachments.PresignedGetURL(path, fileName) |
|
|
|
if err != nil { |
|
|
|
log.Error("GetObsCreateSignedUrl failed: %v", err.Error(), ctx.Data["msgID"]) |
|
|
|
ctx.ServerError("GetObsCreateSignedUrl", err) |
|
|
|
log.Error("Get minio get SignedUrl failed: %v", err.Error(), ctx.Data["msgID"]) |
|
|
|
ctx.ServerError("Get minio get SignedUrl failed", err) |
|
|
|
return |
|
|
|
} |
|
|
|
//count++ |
|
|
|
models.ModifyModelDownloadCount(id) |
|
|
|
http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently) |
|
|
|
} |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
func ShowModelInfo(ctx *context.Context) { |
|
|
|
@@ -684,14 +790,22 @@ func QueryModelListForPredict(ctx *context.Context) { |
|
|
|
func QueryModelFileForPredict(ctx *context.Context) { |
|
|
|
id := ctx.Query("ID") |
|
|
|
model, err := models.QueryModelById(id) |
|
|
|
if err != nil { |
|
|
|
if err == nil { |
|
|
|
if model.Type == models.TypeCloudBrainTwo { |
|
|
|
prefix := model.Path[len(setting.Bucket)+1:] |
|
|
|
fileinfos, _ := storage.GetAllObjectByBucketAndPrefix(setting.Bucket, prefix) |
|
|
|
ctx.JSON(http.StatusOK, fileinfos) |
|
|
|
} else if model.Type == models.TypeCloudBrainOne { |
|
|
|
prefix := model.Path[len(setting.Attachment.Minio.Bucket)+1:] |
|
|
|
fileinfos, _ := storage.GetAllObjectByBucketAndPrefixMinio(setting.Attachment.Minio.Bucket, prefix) |
|
|
|
ctx.JSON(http.StatusOK, fileinfos) |
|
|
|
} |
|
|
|
} else { |
|
|
|
log.Error("no such model!", err.Error()) |
|
|
|
ctx.ServerError("no such model:", err) |
|
|
|
return |
|
|
|
} |
|
|
|
prefix := model.Path[len(setting.Bucket)+1:] |
|
|
|
fileinfos, err := storage.GetAllObjectByBucketAndPrefix(setting.Bucket, prefix) |
|
|
|
ctx.JSON(http.StatusOK, fileinfos) |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
func QueryOneLevelModelFile(ctx *context.Context) { |
|
|
|
@@ -703,7 +817,16 @@ func QueryOneLevelModelFile(ctx *context.Context) { |
|
|
|
ctx.ServerError("no such model:", err) |
|
|
|
return |
|
|
|
} |
|
|
|
prefix := model.Path[len(setting.Bucket)+1:] |
|
|
|
fileinfos, err := storage.GetOneLevelAllObjectUnderDir(setting.Bucket, prefix, parentDir) |
|
|
|
ctx.JSON(http.StatusOK, fileinfos) |
|
|
|
if model.Type == models.TypeCloudBrainTwo { |
|
|
|
log.Info("TypeCloudBrainTwo list model file.") |
|
|
|
prefix := model.Path[len(setting.Bucket)+1:] |
|
|
|
fileinfos, _ := storage.GetOneLevelAllObjectUnderDir(setting.Bucket, prefix, parentDir) |
|
|
|
ctx.JSON(http.StatusOK, fileinfos) |
|
|
|
} else if model.Type == models.TypeCloudBrainOne { |
|
|
|
log.Info("TypeCloudBrainOne list model file.") |
|
|
|
prefix := model.Path[len(setting.Attachment.Minio.Bucket)+1:] |
|
|
|
fileinfos, _ := storage.GetOneLevelAllObjectUnderDirMinio(setting.Attachment.Minio.Bucket, prefix, parentDir) |
|
|
|
ctx.JSON(http.StatusOK, fileinfos) |
|
|
|
} |
|
|
|
|
|
|
|
} |