diff --git a/models/attachment.go b/models/attachment.go index 55c6dbad3..e9b04e61d 100755 --- a/models/attachment.go +++ b/models/attachment.go @@ -158,7 +158,8 @@ func (a *Attachment) S3DownloadURL() string { if a.Type == TypeCloudBrainOne { url, _ = storage.Attachments.PresignedGetURL(setting.Attachment.Minio.BasePath+AttachmentRelativePath(a.UUID), a.Name) } else if a.Type == TypeCloudBrainTwo { - url, _ = storage.ObsGetPreSignedUrl(a.UUID, a.Name) + objectName := strings.TrimPrefix(path.Join(setting.BasePath, path.Join(a.UUID[0:1], a.UUID[1:2], a.UUID, a.Name)), "/") + url, _ = storage.ObsGetPreSignedUrl(objectName, a.Name) } return url @@ -587,7 +588,6 @@ func AttachmentsByDatasetOption(datasets []int64, opts *SearchDatasetOptions) ([ ) } - attachments := make([]*Attachment, 0) if err := sess.Table(&Attachment{}).Where(cond).Desc("id"). Find(&attachments); err != nil { diff --git a/modules/storage/obs.go b/modules/storage/obs.go index be9f2e1d6..2ed254706 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -159,10 +159,11 @@ func CompleteObsMultiPartUpload(objectName, uploadID string, totalChunks int) er return nil } -func ObsMultiPartUpload(uuid string, uploadId string, partNumber int, fileName string, putBody io.ReadCloser) error { +func ObsMultiPartUpload(objectName string, uploadId string, partNumber int, fileName string, putBody io.ReadCloser) error { input := &obs.UploadPartInput{} input.Bucket = setting.Bucket - input.Key = strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, fileName)), "/") + input.Key = objectName + //strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, fileName)), "/") input.UploadId = uploadId input.PartNumber = partNumber input.Body = putBody @@ -242,11 +243,6 @@ func ObsDownloadAFile(bucket string, key string) (io.ReadCloser, error) { } } -func ObsDownload(uuid string, fileName string) (io.ReadCloser, error) { - - return ObsDownloadAFile(setting.Bucket, strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, fileName)), "/")) -} - func ObsModelDownload(JobName string, fileName string) (io.ReadCloser, error) { input := &obs.GetObjectInput{} input.Bucket = setting.Bucket @@ -368,51 +364,60 @@ func GetOneLevelAllObjectUnderDir(bucket string, prefixRootPath string, relative if !strings.HasSuffix(input.Prefix, "/") { input.Prefix += "/" } - output, err := ObsCli.ListObjects(input) fileInfos := make([]FileInfo, 0) prefixLen := len(input.Prefix) fileMap := make(map[string]bool, 0) - if err == nil { - for _, val := range output.Contents { - log.Info("val key=" + val.Key) - var isDir bool - var fileName string - if val.Key == input.Prefix { - continue - } - fileName = val.Key[prefixLen:] - log.Info("fileName =" + fileName) - files := strings.Split(fileName, "/") - if fileMap[files[0]] { - continue - } else { - fileMap[files[0]] = true + index := 1 + for { + output, err := ObsCli.ListObjects(input) + if err == nil { + log.Info("Page:%d\n", index) + index++ + for _, val := range output.Contents { + log.Info("val key=" + val.Key) + var isDir bool + var fileName string + if val.Key == input.Prefix { + continue + } + fileName = val.Key[prefixLen:] + //log.Info("fileName =" + fileName) + files := strings.Split(fileName, "/") + if fileMap[files[0]] { + continue + } else { + fileMap[files[0]] = true + } + ParenDir := relativePath + fileName = files[0] + if len(files) > 1 { + isDir = true + ParenDir += fileName + "/" + } else { + isDir = false + } + fileInfo := FileInfo{ + ModTime: val.LastModified.Local().Format("2006-01-02 15:04:05"), + FileName: fileName, + Size: val.Size, + IsDir: isDir, + ParenDir: ParenDir, + } + fileInfos = append(fileInfos, fileInfo) } - ParenDir := relativePath - fileName = files[0] - if len(files) > 1 { - isDir = true - ParenDir += fileName + "/" + if output.IsTruncated { + input.Marker = output.NextMarker } else { - isDir = false + break } - fileInfo := FileInfo{ - ModTime: val.LastModified.Local().Format("2006-01-02 15:04:05"), - FileName: fileName, - Size: val.Size, - IsDir: isDir, - ParenDir: ParenDir, + } else { + if obsError, ok := err.(obs.ObsError); ok { + log.Error("Code:%s, Message:%s", obsError.Code, obsError.Message) } - fileInfos = append(fileInfos, fileInfo) - } - return fileInfos, err - } else { - if obsError, ok := err.(obs.ObsError); ok { - log.Error("Code:%s, Message:%s", obsError.Code, obsError.Message) + return nil, err } - return nil, err } - + return fileInfos, nil } func GetAllObjectByBucketAndPrefix(bucket string, prefix string) ([]FileInfo, error) { @@ -580,10 +585,11 @@ func GetObsCreateSignedUrl(jobName, parentDir, fileName string) (string, error) return GetObsCreateSignedUrlByBucketAndKey(setting.Bucket, strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir, fileName), "/")) } -func ObsGetPreSignedUrl(uuid, fileName string) (string, error) { +func ObsGetPreSignedUrl(objectName, fileName string) (string, error) { input := &obs.CreateSignedUrlInput{} input.Method = obs.HttpMethodGet - input.Key = strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, fileName)), "/") + input.Key = objectName + //strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, fileName)), "/") input.Bucket = setting.Bucket input.Expires = 60 * 60 diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index b366d5436..247caa010 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -22,17 +22,21 @@ import ( ) const ( - Attachment_model = "model" - Model_prefix = "aimodels/" - tplModelManageIndex = "repo/modelmanage/index" - tplModelManageDownload = "repo/modelmanage/download" - tplModelInfo = "repo/modelmanage/showinfo" - MODEL_LATEST = 1 - MODEL_NOT_LATEST = 0 - MODEL_MAX_SIZE = 1024 * 1024 * 1024 - STATUS_COPY_MODEL = 1 - STATUS_FINISHED = 0 - STATUS_ERROR = 2 + Attachment_model = "model" + Model_prefix = "aimodels/" + tplModelManageIndex = "repo/modelmanage/index" + tplModelManageDownload = "repo/modelmanage/download" + tplModelInfo = "repo/modelmanage/showinfo" + tplCreateLocalModelInfo = "repo/modelmanage/create_local_1" + tplCreateLocalForUploadModelInfo = "repo/modelmanage/create_local_2" + tplCreateOnlineModelInfo = "repo/modelmanage/create_online" + + MODEL_LATEST = 1 + MODEL_NOT_LATEST = 0 + MODEL_MAX_SIZE = 1024 * 1024 * 1024 + STATUS_COPY_MODEL = 1 + STATUS_FINISHED = 0 + STATUS_ERROR = 2 ) func saveModelByParameters(jobId string, versionName string, name string, version string, label string, description string, engine int, ctx *context.Context) (string, error) { @@ -287,10 +291,6 @@ func SaveLocalModel(ctx *context.Context) { ctx.JSON(200, re) } -func UploadLocalModel(ctx *context.Context) { - -} - func SaveModel(ctx *context.Context) { if !ctx.Repo.CanWrite(models.UnitTypeModelManage) { ctx.Error(403, ctx.Tr("repo.model_noright")) @@ -1106,3 +1106,25 @@ func QueryOneLevelModelFile(ctx *context.Context) { ctx.JSON(http.StatusOK, fileinfos) } } + +func CreateLocalModel(ctx *context.Context) { + ctx.Data["isModelManage"] = true + ctx.Data["ModelManageAccess"] = ctx.Repo.CanWrite(models.UnitTypeModelManage) + + ctx.HTML(200, tplCreateLocalModelInfo) +} + +func CreateLocalModelForUpload(ctx *context.Context) { + ctx.Data["uuid"] = ctx.Query("uuid") + ctx.Data["isModelManage"] = true + ctx.Data["ModelManageAccess"] = ctx.Repo.CanWrite(models.UnitTypeModelManage) + + ctx.HTML(200, tplCreateLocalForUploadModelInfo) +} + +func CreateOnlineModel(ctx *context.Context) { + ctx.Data["isModelManage"] = true + ctx.Data["ModelManageAccess"] = ctx.Repo.CanWrite(models.UnitTypeModelManage) + + ctx.HTML(200, tplCreateOnlineModelInfo) +} diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 588a00c18..4270245cd 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -1232,9 +1232,12 @@ func RegisterRoutes(m *macaron.Macaron) { }) }, context.RepoRef()) m.Group("/modelmanage", func() { - m.Post("/create_model", repo.SaveModel) - m.Post("/upload_local_model", repo.UploadLocalModel) + m.Get("/create_local_model_tmpl1", repo.CreateLocalModel) + m.Get("/create_local_model_tmpl2", repo.CreateLocalModelForUpload) + m.Get("/create_online_model_tmpl", repo.CreateOnlineModel) m.Post("/create_local_model", repo.SaveLocalModel) + + m.Post("/create_model", repo.SaveModel) m.Post("/create_model_convert", reqWechatBind, reqRepoModelManageWriter, repo.SaveModelConvert) m.Post("/create_new_model", repo.SaveNewNameModel) m.Delete("/delete_model", repo.DeleteModel)