Browse Source

提交代码,解决问题。

Signed-off-by: zouap <zouap@pcl.ac.cn>
tags/v1.22.11.2^2
zouap 3 years ago
parent
commit
5453f901f1
4 changed files with 95 additions and 64 deletions
  1. +2
    -2
      models/attachment.go
  2. +51
    -45
      modules/storage/obs.go
  3. +37
    -15
      routers/repo/ai_model_manage.go
  4. +5
    -2
      routers/routes/routes.go

+ 2
- 2
models/attachment.go View File

@@ -158,7 +158,8 @@ func (a *Attachment) S3DownloadURL() string {
if a.Type == TypeCloudBrainOne {
url, _ = storage.Attachments.PresignedGetURL(setting.Attachment.Minio.BasePath+AttachmentRelativePath(a.UUID), a.Name)
} else if a.Type == TypeCloudBrainTwo {
url, _ = storage.ObsGetPreSignedUrl(a.UUID, a.Name)
objectName := strings.TrimPrefix(path.Join(setting.BasePath, path.Join(a.UUID[0:1], a.UUID[1:2], a.UUID, a.Name)), "/")
url, _ = storage.ObsGetPreSignedUrl(objectName, a.Name)
}

return url
@@ -587,7 +588,6 @@ func AttachmentsByDatasetOption(datasets []int64, opts *SearchDatasetOptions) ([
)
}


attachments := make([]*Attachment, 0)
if err := sess.Table(&Attachment{}).Where(cond).Desc("id").
Find(&attachments); err != nil {


+ 51
- 45
modules/storage/obs.go View File

@@ -159,10 +159,11 @@ func CompleteObsMultiPartUpload(objectName, uploadID string, totalChunks int) er
return nil
}

func ObsMultiPartUpload(uuid string, uploadId string, partNumber int, fileName string, putBody io.ReadCloser) error {
func ObsMultiPartUpload(objectName string, uploadId string, partNumber int, fileName string, putBody io.ReadCloser) error {
input := &obs.UploadPartInput{}
input.Bucket = setting.Bucket
input.Key = strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, fileName)), "/")
input.Key = objectName
//strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, fileName)), "/")
input.UploadId = uploadId
input.PartNumber = partNumber
input.Body = putBody
@@ -242,11 +243,6 @@ func ObsDownloadAFile(bucket string, key string) (io.ReadCloser, error) {
}
}

func ObsDownload(uuid string, fileName string) (io.ReadCloser, error) {

return ObsDownloadAFile(setting.Bucket, strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, fileName)), "/"))
}

func ObsModelDownload(JobName string, fileName string) (io.ReadCloser, error) {
input := &obs.GetObjectInput{}
input.Bucket = setting.Bucket
@@ -368,51 +364,60 @@ func GetOneLevelAllObjectUnderDir(bucket string, prefixRootPath string, relative
if !strings.HasSuffix(input.Prefix, "/") {
input.Prefix += "/"
}
output, err := ObsCli.ListObjects(input)
fileInfos := make([]FileInfo, 0)
prefixLen := len(input.Prefix)
fileMap := make(map[string]bool, 0)
if err == nil {
for _, val := range output.Contents {
log.Info("val key=" + val.Key)
var isDir bool
var fileName string
if val.Key == input.Prefix {
continue
}
fileName = val.Key[prefixLen:]
log.Info("fileName =" + fileName)
files := strings.Split(fileName, "/")
if fileMap[files[0]] {
continue
} else {
fileMap[files[0]] = true
index := 1
for {
output, err := ObsCli.ListObjects(input)
if err == nil {
log.Info("Page:%d\n", index)
index++
for _, val := range output.Contents {
log.Info("val key=" + val.Key)
var isDir bool
var fileName string
if val.Key == input.Prefix {
continue
}
fileName = val.Key[prefixLen:]
//log.Info("fileName =" + fileName)
files := strings.Split(fileName, "/")
if fileMap[files[0]] {
continue
} else {
fileMap[files[0]] = true
}
ParenDir := relativePath
fileName = files[0]
if len(files) > 1 {
isDir = true
ParenDir += fileName + "/"
} else {
isDir = false
}
fileInfo := FileInfo{
ModTime: val.LastModified.Local().Format("2006-01-02 15:04:05"),
FileName: fileName,
Size: val.Size,
IsDir: isDir,
ParenDir: ParenDir,
}
fileInfos = append(fileInfos, fileInfo)
}
ParenDir := relativePath
fileName = files[0]
if len(files) > 1 {
isDir = true
ParenDir += fileName + "/"
if output.IsTruncated {
input.Marker = output.NextMarker
} else {
isDir = false
break
}
fileInfo := FileInfo{
ModTime: val.LastModified.Local().Format("2006-01-02 15:04:05"),
FileName: fileName,
Size: val.Size,
IsDir: isDir,
ParenDir: ParenDir,
} else {
if obsError, ok := err.(obs.ObsError); ok {
log.Error("Code:%s, Message:%s", obsError.Code, obsError.Message)
}
fileInfos = append(fileInfos, fileInfo)
}
return fileInfos, err
} else {
if obsError, ok := err.(obs.ObsError); ok {
log.Error("Code:%s, Message:%s", obsError.Code, obsError.Message)
return nil, err
}
return nil, err
}
return fileInfos, nil
}

func GetAllObjectByBucketAndPrefix(bucket string, prefix string) ([]FileInfo, error) {
@@ -580,10 +585,11 @@ func GetObsCreateSignedUrl(jobName, parentDir, fileName string) (string, error)
return GetObsCreateSignedUrlByBucketAndKey(setting.Bucket, strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir, fileName), "/"))
}

func ObsGetPreSignedUrl(uuid, fileName string) (string, error) {
func ObsGetPreSignedUrl(objectName, fileName string) (string, error) {
input := &obs.CreateSignedUrlInput{}
input.Method = obs.HttpMethodGet
input.Key = strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, fileName)), "/")
input.Key = objectName
//strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, fileName)), "/")
input.Bucket = setting.Bucket
input.Expires = 60 * 60



+ 37
- 15
routers/repo/ai_model_manage.go View File

@@ -22,17 +22,21 @@ import (
)

const (
Attachment_model = "model"
Model_prefix = "aimodels/"
tplModelManageIndex = "repo/modelmanage/index"
tplModelManageDownload = "repo/modelmanage/download"
tplModelInfo = "repo/modelmanage/showinfo"
MODEL_LATEST = 1
MODEL_NOT_LATEST = 0
MODEL_MAX_SIZE = 1024 * 1024 * 1024
STATUS_COPY_MODEL = 1
STATUS_FINISHED = 0
STATUS_ERROR = 2
Attachment_model = "model"
Model_prefix = "aimodels/"
tplModelManageIndex = "repo/modelmanage/index"
tplModelManageDownload = "repo/modelmanage/download"
tplModelInfo = "repo/modelmanage/showinfo"
tplCreateLocalModelInfo = "repo/modelmanage/create_local_1"
tplCreateLocalForUploadModelInfo = "repo/modelmanage/create_local_2"
tplCreateOnlineModelInfo = "repo/modelmanage/create_online"

MODEL_LATEST = 1
MODEL_NOT_LATEST = 0
MODEL_MAX_SIZE = 1024 * 1024 * 1024
STATUS_COPY_MODEL = 1
STATUS_FINISHED = 0
STATUS_ERROR = 2
)

func saveModelByParameters(jobId string, versionName string, name string, version string, label string, description string, engine int, ctx *context.Context) (string, error) {
@@ -287,10 +291,6 @@ func SaveLocalModel(ctx *context.Context) {
ctx.JSON(200, re)
}

func UploadLocalModel(ctx *context.Context) {

}

func SaveModel(ctx *context.Context) {
if !ctx.Repo.CanWrite(models.UnitTypeModelManage) {
ctx.Error(403, ctx.Tr("repo.model_noright"))
@@ -1106,3 +1106,25 @@ func QueryOneLevelModelFile(ctx *context.Context) {
ctx.JSON(http.StatusOK, fileinfos)
}
}

func CreateLocalModel(ctx *context.Context) {
ctx.Data["isModelManage"] = true
ctx.Data["ModelManageAccess"] = ctx.Repo.CanWrite(models.UnitTypeModelManage)

ctx.HTML(200, tplCreateLocalModelInfo)
}

func CreateLocalModelForUpload(ctx *context.Context) {
ctx.Data["uuid"] = ctx.Query("uuid")
ctx.Data["isModelManage"] = true
ctx.Data["ModelManageAccess"] = ctx.Repo.CanWrite(models.UnitTypeModelManage)

ctx.HTML(200, tplCreateLocalForUploadModelInfo)
}

func CreateOnlineModel(ctx *context.Context) {
ctx.Data["isModelManage"] = true
ctx.Data["ModelManageAccess"] = ctx.Repo.CanWrite(models.UnitTypeModelManage)

ctx.HTML(200, tplCreateOnlineModelInfo)
}

+ 5
- 2
routers/routes/routes.go View File

@@ -1232,9 +1232,12 @@ func RegisterRoutes(m *macaron.Macaron) {
})
}, context.RepoRef())
m.Group("/modelmanage", func() {
m.Post("/create_model", repo.SaveModel)
m.Post("/upload_local_model", repo.UploadLocalModel)
m.Get("/create_local_model_tmpl1", repo.CreateLocalModel)
m.Get("/create_local_model_tmpl2", repo.CreateLocalModelForUpload)
m.Get("/create_online_model_tmpl", repo.CreateOnlineModel)
m.Post("/create_local_model", repo.SaveLocalModel)

m.Post("/create_model", repo.SaveModel)
m.Post("/create_model_convert", reqWechatBind, reqRepoModelManageWriter, repo.SaveModelConvert)
m.Post("/create_new_model", repo.SaveNewNameModel)
m.Delete("/delete_model", repo.DeleteModel)


Loading…
Cancel
Save