From 674b4e1a4b64f758dc63ef19454472d532a26609 Mon Sep 17 00:00:00 2001 From: yanchao Date: Wed, 16 Mar 2022 17:28:02 +0800 Subject: [PATCH] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models/attachment.go | 14 +- routers/repo/attachment.go | 6 +- routers/repo/attachment.go.bak | 1086 -------------------------------- routers/repo/dataset.go.bak | 464 -------------- 4 files changed, 17 insertions(+), 1553 deletions(-) delete mode 100644 routers/repo/attachment.go.bak delete mode 100644 routers/repo/dataset.go.bak diff --git a/models/attachment.go b/models/attachment.go index 8de8ffd5c..a981aa987 100755 --- a/models/attachment.go +++ b/models/attachment.go @@ -347,6 +347,18 @@ func DeleteAttachmentsByComment(commentID int64, remove bool) (int, error) { func UpdateAttachment(atta *Attachment) error { return updateAttachment(x, atta) } +func UpdateAttachmentDescription(atta *Attachment) error { + return updateAttachmentDescription(x, atta) +} + +func updateAttachmentDescription(e Engine, atta *Attachment) error { + var sess *xorm.Session + + sess = e.ID(atta.ID) + + _, err := sess.Cols("description").Update(atta) + return err +} func updateAttachment(e Engine, atta *Attachment) error { var sess *xorm.Session @@ -356,7 +368,7 @@ func updateAttachment(e Engine, atta *Attachment) error { // Use uuid only if id is not set and uuid is set sess = e.Where("uuid = ?", atta.UUID) } - _, err := sess.Cols("name", "description", "issue_id", "release_id", "comment_id", "download_count", "is_private", "decompress_state").Update(atta) + _, err := sess.Cols("name", "issue_id", "release_id", "comment_id", "download_count", "is_private", "decompress_state").Update(atta) return err } diff --git a/routers/repo/attachment.go b/routers/repo/attachment.go index f0c1a2819..a188e2787 100755 --- a/routers/repo/attachment.go +++ b/routers/repo/attachment.go @@ -88,7 +88,9 @@ func EditAttachmentUI(ctx *context.Context) { if attachment == nil { ctx.Error(404, "The attachment does not exits.") } - + dataset, _ := models.GetDatasetByID(attachment.DatasetID) + r, _ := models.GetRepositoryByID(dataset.RepoID) + ctx.Data["Repo"] = r ctx.Data["Attachment"] = attachment ctx.HTML(200, tplAttachmentEdit) @@ -96,7 +98,7 @@ func EditAttachmentUI(ctx *context.Context) { func EditAttachment(ctx *context.Context, form auth.EditAttachmentForm) { - err := models.UpdateAttachment(&models.Attachment{ + err := models.UpdateAttachmentDescription(&models.Attachment{ ID: form.ID, Description: form.Description, }) diff --git a/routers/repo/attachment.go.bak b/routers/repo/attachment.go.bak deleted file mode 100644 index a4ffa6d29..000000000 --- a/routers/repo/attachment.go.bak +++ /dev/null @@ -1,1086 +0,0 @@ -// Copyright 2017 The Gitea Authors. All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. - -package repo - -import ( - contexExt "context" - "encoding/json" - "errors" - "fmt" - "mime/multipart" - "net/http" - "path" - "strconv" - "strings" - - "code.gitea.io/gitea/modules/auth" - - "code.gitea.io/gitea/modules/base" - - "code.gitea.io/gitea/models" - "code.gitea.io/gitea/modules/context" - "code.gitea.io/gitea/modules/labelmsg" - "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/minio_ext" - "code.gitea.io/gitea/modules/notification" - "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/storage" - "code.gitea.io/gitea/modules/upload" - "code.gitea.io/gitea/modules/worker" - gouuid "github.com/satori/go.uuid" -) - -const ( - //result of decompress - DecompressSuccess = "0" - DecompressFailed = "1" - tplAttachmentUpload base.TplName = "repo/attachment/upload" - tplAttachmentEdit base.TplName = "repo/attachment/edit" -) - -type CloudBrainDataset struct { - UUID string `json:"id"` - Name string `json:"name"` - Path string `json:"place"` - UserName string `json:"provider"` - CreateTime string `json:"created_at"` -} - -type UploadForm struct { - UploadID string `form:"uploadId"` - UuID string `form:"uuid"` - PartSize int64 `form:"size"` - Offset int64 `form:"offset"` - PartNumber int `form:"chunkNumber"` - PartFile multipart.File `form:"file"` -} - -func RenderAttachmentSettings(ctx *context.Context) { - renderAttachmentSettings(ctx) -} - -func renderAttachmentSettings(ctx *context.Context) { - ctx.Data["IsAttachmentEnabled"] = setting.Attachment.Enabled - ctx.Data["AttachmentStoreType"] = setting.Attachment.StoreType - ctx.Data["AttachmentAllowedTypes"] = setting.Attachment.AllowedTypes - ctx.Data["AttachmentMaxSize"] = setting.Attachment.MaxSize - ctx.Data["AttachmentMaxFiles"] = setting.Attachment.MaxFiles -} - -func UploadAttachmentUI(ctx *context.Context) { - ctx.Data["datasetId"] = ctx.Query("datasetId") - dataset, _ := models.GetDatasetByID(ctx.QueryInt64("datasetId")) - if dataset == nil { - ctx.Error(404, "The dataset does not exits.") - } - r, _ := models.GetRepositoryByID(dataset.RepoID) - ctx.Data["Repo"] = r - ctx.HTML(200, tplAttachmentUpload) - -} - -func EditAttachmentUI(ctx *context.Context) { - id := ctx.QueryInt64(":id") - attachment, _ := models.GetAttachmentByID(id) - if attachment == nil { - ctx.Error(404, "The attachment does not exits.") - } - - ctx.Data["Attachment"] = attachment - ctx.HTML(200, tplAttachmentEdit) - -} - -func EditAttachment(ctx *context.Context, form auth.EditAttachmentForm) { - - err := models.UpdateAttachment(&models.Attachment{ - ID: form.ID, - Description: form.Description, - }) - if err != nil { - ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("dataset.edit_attachment_fail"))) - } - ctx.JSON(http.StatusOK, models.BaseOKMessage) - -} - -// UploadAttachment response for uploading issue's attachment -func UploadAttachment(ctx *context.Context) { - if !setting.Attachment.Enabled { - ctx.Error(404, "attachment is not enabled") - return - } - - file, header, err := ctx.Req.FormFile("file") - if err != nil { - ctx.Error(500, fmt.Sprintf("FormFile: %v", err)) - return - } - defer file.Close() - - buf := make([]byte, 1024) - n, _ := file.Read(buf) - if n > 0 { - buf = buf[:n] - } - - err = upload.VerifyAllowedContentType(buf, strings.Split(setting.Attachment.AllowedTypes, ",")) - if err != nil { - ctx.Error(400, err.Error()) - return - } - - datasetID, _ := strconv.ParseInt(ctx.Req.FormValue("dataset_id"), 10, 64) - - attach, err := models.NewAttachment(&models.Attachment{ - IsPrivate: true, - UploaderID: ctx.User.ID, - Name: header.Filename, - DatasetID: datasetID, - }, buf, file) - if err != nil { - ctx.Error(500, fmt.Sprintf("NewAttachment: %v", err)) - return - } - - log.Trace("New attachment uploaded: %s", attach.UUID) - ctx.JSON(200, map[string]string{ - "uuid": attach.UUID, - }) -} - -func UpdatePublicAttachment(ctx *context.Context) { - file := ctx.Query("file") - isPrivate, _ := strconv.ParseBool(ctx.Query("is_private")) - attach, err := models.GetAttachmentByUUID(file) - if err != nil { - ctx.Error(404, err.Error()) - return - } - attach.IsPrivate = isPrivate - models.UpdateAttachment(attach) -} - -// DeleteAttachment response for deleting issue's attachment -func DeleteAttachment(ctx *context.Context) { - file := ctx.Query("file") - attach, err := models.GetAttachmentByUUID(file) - if err != nil { - ctx.Error(400, err.Error()) - return - } - - //issue 214: mod del-dataset permission - if !models.CanDelAttachment(ctx.IsSigned, ctx.User, attach) { - ctx.Error(403) - return - } - - err = models.DeleteAttachment(attach, true) - if err != nil { - ctx.Error(500, fmt.Sprintf("DeleteAttachment: %v", err)) - return - } - - attachjson, _ := json.Marshal(attach) - labelmsg.SendDeleteAttachToLabelSys(string(attachjson)) - - DeleteAllUnzipFile(attach, "") - - _, err = models.DeleteFileChunkById(attach.UUID) - if err != nil { - ctx.Error(500, fmt.Sprintf("DeleteFileChunkById: %v", err)) - return - } - ctx.JSON(200, map[string]string{ - "uuid": attach.UUID, - }) -} - -func DownloadUserIsOrgOrCollaboration(ctx *context.Context, attach *models.Attachment) bool { - dataset, err := models.GetDatasetByID(attach.DatasetID) - if err != nil { - log.Info("query dataset error") - } else { - repo, err := models.GetRepositoryByID(dataset.RepoID) - if err != nil { - log.Info("query repo error.") - } else { - repo.GetOwner() - if ctx.User != nil { - - if repo.Owner.IsOrganization() { - if repo.Owner.IsUserPartOfOrg(ctx.User.ID) { - log.Info("org user may visit the attach.") - return true - } - } - isCollaborator, _ := repo.IsCollaborator(ctx.User.ID) - if isCollaborator { - log.Info("Collaborator user may visit the attach.") - return true - } - } - } - } - return false -} - -// GetAttachment serve attachements -func GetAttachment(ctx *context.Context) { - typeCloudBrain := ctx.QueryInt("type") - err := checkTypeCloudBrain(typeCloudBrain) - if err != nil { - ctx.ServerError("checkTypeCloudBrain failed", err) - return - } - - attach, err := models.GetAttachmentByUUID(ctx.Params(":uuid")) - if err != nil { - if models.IsErrAttachmentNotExist(err) { - ctx.Error(404) - } else { - ctx.ServerError("GetAttachmentByUUID", err) - } - return - } - - repository, unitType, err := attach.LinkedRepository() - if err != nil { - ctx.ServerError("LinkedRepository", err) - return - } - dataSet, err := attach.LinkedDataSet() - if err != nil { - ctx.ServerError("LinkedDataSet", err) - return - } - - if repository == nil && dataSet != nil { - repository, _ = models.GetRepositoryByID(dataSet.RepoID) - unitType = models.UnitTypeDatasets - } - - if repository == nil { //If not linked - //if !(ctx.IsSigned && attach.UploaderID == ctx.User.ID) && attach.IsPrivate { //We block if not the uploader - //log.Info("ctx.IsSigned =" + fmt.Sprintf("%v", ctx.IsSigned)) - if !(ctx.IsSigned && attach.UploaderID == ctx.User.ID) && attach.IsPrivate && !DownloadUserIsOrgOrCollaboration(ctx, attach) { //We block if not the uploader - ctx.Error(http.StatusNotFound) - return - } - - } else { //If we have the repository we check access - perm, errPermission := models.GetUserRepoPermission(repository, ctx.User) - if errPermission != nil { - ctx.Error(http.StatusInternalServerError, "GetUserRepoPermission", errPermission.Error()) - return - } - if !perm.CanRead(unitType) { - ctx.Error(http.StatusNotFound) - return - } - } - - if dataSet != nil { - isPermit, err := models.GetUserDataSetPermission(dataSet, ctx.User) - if err != nil { - ctx.Error(http.StatusInternalServerError, "GetUserDataSetPermission", err.Error()) - return - } - if !isPermit { - ctx.Error(http.StatusNotFound) - return - } - } - - //If we have matched and access to release or issue - if setting.Attachment.StoreType == storage.MinioStorageType { - url := "" - if typeCloudBrain == models.TypeCloudBrainOne { - url, err = storage.Attachments.PresignedGetURL(setting.Attachment.Minio.BasePath+attach.RelativePath(), attach.Name) - if err != nil { - ctx.ServerError("PresignedGetURL", err) - return - } - } else { - if setting.PROXYURL != "" { - url = setting.PROXYURL + "/obs_proxy_download?uuid=" + attach.UUID + "&file_name=" + attach.Name - log.Info("return url=" + url) - } else { - url, err = storage.ObsGetPreSignedUrl(attach.UUID, attach.Name) - if err != nil { - ctx.ServerError("ObsGetPreSignedUrl", err) - return - } - } - } - - if err = increaseDownloadCount(attach, dataSet); err != nil { - ctx.ServerError("Update", err) - return - } - if dataSet != nil { - http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently) - } else { - fr, err := storage.Attachments.Open(attach.RelativePath()) - if err != nil { - ctx.ServerError("Open", err) - return - } - defer fr.Close() - if err = ServeData(ctx, attach.Name, fr); err != nil { - ctx.ServerError("ServeData", err) - return - } - } - - } else { - fr, err := storage.Attachments.Open(attach.RelativePath()) - if err != nil { - ctx.ServerError("Open", err) - return - } - defer fr.Close() - if err = increaseDownloadCount(attach, dataSet); err != nil { - ctx.ServerError("Update", err) - return - } - if err = ServeData(ctx, attach.Name, fr); err != nil { - ctx.ServerError("ServeData", err) - return - } - } - -} - -func increaseDownloadCount(attach *models.Attachment, dataSet *models.Dataset) error { - if err := attach.IncreaseDownloadCount(); err != nil { - return err - } - - if dataSet != nil { - if err := models.IncreaseDownloadCount(dataSet.ID); err != nil { - return err - } - } - - return nil -} - -// Get a presigned url for put object -func GetPresignedPutObjectURL(ctx *context.Context) { - if !setting.Attachment.Enabled { - ctx.Error(404, "attachment is not enabled") - return - } - - err := upload.VerifyFileType(ctx.Params("file_type"), strings.Split(setting.Attachment.AllowedTypes, ",")) - if err != nil { - ctx.Error(400, err.Error()) - return - } - - if setting.Attachment.StoreType == storage.MinioStorageType { - uuid := gouuid.NewV4().String() - url, err := storage.Attachments.PresignedPutURL(models.AttachmentRelativePath(uuid)) - if err != nil { - ctx.ServerError("PresignedPutURL", err) - return - } - - ctx.JSON(200, map[string]string{ - "uuid": uuid, - "url": url, - }) - } else { - ctx.Error(404, "storage type is not enabled") - return - } -} - -// AddAttachment response for add attachment record -func AddAttachment(ctx *context.Context) { - typeCloudBrain := ctx.QueryInt("type") - fileName := ctx.Query("file_name") - err := checkTypeCloudBrain(typeCloudBrain) - if err != nil { - ctx.ServerError("checkTypeCloudBrain failed", err) - return - } - - uuid := ctx.Query("uuid") - has := false - if typeCloudBrain == models.TypeCloudBrainOne { - has, err = storage.Attachments.HasObject(models.AttachmentRelativePath(uuid)) - if err != nil { - ctx.ServerError("HasObject", err) - return - } - } else { - has, err = storage.ObsHasObject(setting.BasePath + models.AttachmentRelativePath(uuid) + "/" + fileName) - if err != nil { - ctx.ServerError("ObsHasObject", err) - return - } - } - - if !has { - ctx.Error(404, "attachment has not been uploaded") - return - } - datasetId := ctx.QueryInt64("dataset_id") - dataset, err := models.GetDatasetByID(datasetId) - if err != nil { - ctx.Error(404, "dataset does not exist.") - return - } - - attachment, err := models.InsertAttachment(&models.Attachment{ - UUID: uuid, - UploaderID: ctx.User.ID, - IsPrivate: dataset.IsPrivate(), - Name: fileName, - Size: ctx.QueryInt64("size"), - DatasetID: ctx.QueryInt64("dataset_id"), - Type: typeCloudBrain, - }) - - if err != nil { - ctx.Error(500, fmt.Sprintf("InsertAttachment: %v", err)) - return - } - - if attachment.DatasetID != 0 { - if isCanDecompress(attachment.Name) { - if typeCloudBrain == models.TypeCloudBrainOne { - err = worker.SendDecompressTask(contexExt.Background(), uuid, attachment.Name) - if err != nil { - log.Error("SendDecompressTask(%s) failed:%s", uuid, err.Error()) - } else { - attachment.DecompressState = models.DecompressStateIng - err = models.UpdateAttachment(attachment) - if err != nil { - log.Error("UpdateAttachment state(%s) failed:%s", uuid, err.Error()) - } - } - } - //todo:decompress type_two - } - } - - ctx.JSON(200, map[string]string{ - "result_code": "0", - }) -} - -func isCanDecompress(name string) bool { - if strings.HasSuffix(name, ".zip") || strings.HasSuffix(name, ".tar.gz") || strings.HasSuffix(name, ".tgz") { - return true - } - return false -} - -func UpdateAttachmentDecompressState(ctx *context.Context) { - uuid := ctx.Query("uuid") - result := ctx.Query("result") - attach, err := models.GetAttachmentByUUID(uuid) - if err != nil { - log.Error("GetAttachmentByUUID(%s) failed:%s", uuid, err.Error()) - return - } - - if result == DecompressSuccess { - attach.DecompressState = models.DecompressStateDone - } else if result == DecompressFailed { - attach.DecompressState = models.DecompressStateFailed - } else { - log.Error("result is error:", result) - return - } - - err = models.UpdateAttachment(attach) - if err != nil { - log.Error("UpdateAttachment(%s) failed:%s", uuid, err.Error()) - return - } - log.Info("start to send msg to labelsystem ") - - dataset, _ := models.GetDatasetByID(attach.DatasetID) - - var labelMap map[string]string - labelMap = make(map[string]string) - labelMap["UUID"] = uuid - labelMap["Type"] = fmt.Sprint(attach.Type) - labelMap["UploaderID"] = fmt.Sprint(attach.UploaderID) - labelMap["RepoID"] = fmt.Sprint(dataset.RepoID) - labelMap["AttachName"] = attach.Name - attachjson, _ := json.Marshal(labelMap) - labelmsg.SendAddAttachToLabelSys(string(attachjson)) - - log.Info("end to send msg to labelsystem ") - - ctx.JSON(200, map[string]string{ - "result_code": "0", - }) -} - -func GetSuccessChunks(ctx *context.Context) { - fileMD5 := ctx.Query("md5") - typeCloudBrain := ctx.QueryInt("type") - fileName := ctx.Query("file_name") - var chunks string - - err := checkTypeCloudBrain(typeCloudBrain) - if err != nil { - ctx.ServerError("checkTypeCloudBrain failed", err) - return - } - - fileChunk, err := models.GetFileChunkByMD5AndUser(fileMD5, ctx.User.ID, typeCloudBrain) - if err != nil { - if models.IsErrFileChunkNotExist(err) { - ctx.JSON(200, map[string]string{ - "uuid": "", - "uploaded": "0", - "uploadID": "", - "chunks": "", - }) - } else { - ctx.ServerError("GetFileChunkByMD5", err) - } - return - } - - isExist := false - if typeCloudBrain == models.TypeCloudBrainOne { - isExist, err = storage.Attachments.HasObject(models.AttachmentRelativePath(fileChunk.UUID)) - if err != nil { - ctx.ServerError("HasObject failed", err) - return - } - } else { - isExist, err = storage.ObsHasObject(setting.BasePath + models.AttachmentRelativePath(fileChunk.UUID) + "/" + fileName) - if err != nil { - ctx.ServerError("ObsHasObject failed", err) - return - } - } - - if isExist { - if fileChunk.IsUploaded == models.FileNotUploaded { - log.Info("the file has been uploaded but not recorded") - fileChunk.IsUploaded = models.FileUploaded - if err = models.UpdateFileChunk(fileChunk); err != nil { - log.Error("UpdateFileChunk failed:", err.Error()) - } - } - } else { - if fileChunk.IsUploaded == models.FileUploaded { - log.Info("the file has been recorded but not uploaded") - fileChunk.IsUploaded = models.FileNotUploaded - if err = models.UpdateFileChunk(fileChunk); err != nil { - log.Error("UpdateFileChunk failed:", err.Error()) - } - } - - if typeCloudBrain == models.TypeCloudBrainOne { - chunks, err = storage.GetPartInfos(fileChunk.UUID, fileChunk.UploadID) - if err != nil { - log.Error("GetPartInfos failed:%v", err.Error()) - } - } else { - chunks, err = storage.GetObsPartInfos(fileChunk.UUID, fileChunk.UploadID, fileName) - if err != nil { - log.Error("GetObsPartInfos failed:%v", err.Error()) - } - } - - if err != nil { - models.DeleteFileChunk(fileChunk) - ctx.JSON(200, map[string]string{ - "uuid": "", - "uploaded": "0", - "uploadID": "", - "chunks": "", - }) - return - } - } - - var attachID int64 - attach, err := models.GetAttachmentByUUID(fileChunk.UUID) - if err != nil { - if models.IsErrAttachmentNotExist(err) { - attachID = 0 - } else { - ctx.ServerError("GetAttachmentByUUID", err) - return - } - } else { - attachID = attach.ID - } - - if attach == nil { - ctx.JSON(200, map[string]string{ - "uuid": fileChunk.UUID, - "uploaded": strconv.Itoa(fileChunk.IsUploaded), - "uploadID": fileChunk.UploadID, - "chunks": string(chunks), - "attachID": "0", - "datasetID": "0", - "fileName": "", - "datasetName": "", - }) - return - } - - dataset, err := models.GetDatasetByID(attach.DatasetID) - if err != nil { - ctx.ServerError("GetDatasetByID", err) - return - } - - ctx.JSON(200, map[string]string{ - "uuid": fileChunk.UUID, - "uploaded": strconv.Itoa(fileChunk.IsUploaded), - "uploadID": fileChunk.UploadID, - "chunks": string(chunks), - "attachID": strconv.Itoa(int(attachID)), - "datasetID": strconv.Itoa(int(attach.DatasetID)), - "fileName": attach.Name, - "datasetName": dataset.Title, - }) - -} - -func NewMultipart(ctx *context.Context) { - if !setting.Attachment.Enabled { - ctx.Error(404, "attachment is not enabled") - return - } - - err := upload.VerifyFileType(ctx.Query("fileType"), strings.Split(setting.Attachment.AllowedTypes, ",")) - if err != nil { - ctx.Error(400, err.Error()) - return - } - - typeCloudBrain := ctx.QueryInt("type") - err = checkTypeCloudBrain(typeCloudBrain) - if err != nil { - ctx.ServerError("checkTypeCloudBrain failed", err) - return - } - - fileName := ctx.Query("file_name") - - if setting.Attachment.StoreType == storage.MinioStorageType { - totalChunkCounts := ctx.QueryInt("totalChunkCounts") - if totalChunkCounts > minio_ext.MaxPartsCount { - ctx.Error(400, fmt.Sprintf("chunk counts(%d) is too much", totalChunkCounts)) - return - } - - fileSize := ctx.QueryInt64("size") - if fileSize > minio_ext.MaxMultipartPutObjectSize { - ctx.Error(400, fmt.Sprintf("file size(%d) is too big", fileSize)) - return - } - - uuid := gouuid.NewV4().String() - var uploadID string - if typeCloudBrain == models.TypeCloudBrainOne { - uploadID, err = storage.NewMultiPartUpload(uuid) - if err != nil { - ctx.ServerError("NewMultipart", err) - return - } - } else { - uploadID, err = storage.NewObsMultiPartUpload(uuid, fileName) - if err != nil { - ctx.ServerError("NewObsMultiPartUpload", err) - return - } - } - - _, err = models.InsertFileChunk(&models.FileChunk{ - UUID: uuid, - UserID: ctx.User.ID, - UploadID: uploadID, - Md5: ctx.Query("md5"), - Size: fileSize, - TotalChunks: totalChunkCounts, - Type: typeCloudBrain, - }) - - if err != nil { - ctx.Error(500, fmt.Sprintf("InsertFileChunk: %v", err)) - return - } - - ctx.JSON(200, map[string]string{ - "uuid": uuid, - "uploadID": uploadID, - }) - } else { - ctx.Error(404, "storage type is not enabled") - return - } -} - -func PutOBSProxyUpload(ctx *context.Context) { - uuid := ctx.Query("uuid") - uploadID := ctx.Query("uploadId") - partNumber := ctx.QueryInt("partNumber") - fileName := ctx.Query("file_name") - - RequestBody := ctx.Req.Body() - - if RequestBody == nil { - ctx.Error(500, fmt.Sprintf("FormFile: %v", RequestBody)) - return - } - - err := storage.ObsMultiPartUpload(uuid, uploadID, partNumber, fileName, RequestBody.ReadCloser()) - if err != nil { - log.Info("upload error.") - } -} - -func GetOBSProxyDownload(ctx *context.Context) { - uuid := ctx.Query("uuid") - fileName := ctx.Query("file_name") - - body, err := storage.ObsDownload(uuid, fileName) - if err != nil { - log.Info("upload error.") - } else { - defer body.Close() - ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+fileName) - ctx.Resp.Header().Set("Content-Type", "application/octet-stream") - p := make([]byte, 1024) - var readErr error - var readCount int - // 读取对象内容 - for { - readCount, readErr = body.Read(p) - if readCount > 0 { - ctx.Resp.Write(p[:readCount]) - //fmt.Printf("%s", p[:readCount]) - } - if readErr != nil { - break - } - } - } -} - -func GetMultipartUploadUrl(ctx *context.Context) { - uuid := ctx.Query("uuid") - uploadID := ctx.Query("uploadID") - partNumber := ctx.QueryInt("chunkNumber") - size := ctx.QueryInt64("size") - fileName := ctx.Query("file_name") - - typeCloudBrain := ctx.QueryInt("type") - err := checkTypeCloudBrain(typeCloudBrain) - if err != nil { - ctx.ServerError("checkTypeCloudBrain failed", err) - return - } - - url := "" - if typeCloudBrain == models.TypeCloudBrainOne { - if size > minio_ext.MinPartSize { - ctx.Error(400, fmt.Sprintf("chunk size(%d) is too big", size)) - return - } - - url, err = storage.GenMultiPartSignedUrl(uuid, uploadID, partNumber, size) - if err != nil { - ctx.Error(500, fmt.Sprintf("GenMultiPartSignedUrl failed: %v", err)) - return - } - } else { - if setting.PROXYURL != "" { - url = setting.PROXYURL + "/obs_proxy_multipart?uuid=" + uuid + "&uploadId=" + uploadID + "&partNumber=" + fmt.Sprint(partNumber) + "&file_name=" + fileName - log.Info("return url=" + url) - } else { - url, err = storage.ObsGenMultiPartSignedUrl(uuid, uploadID, partNumber, fileName) - if err != nil { - ctx.Error(500, fmt.Sprintf("ObsGenMultiPartSignedUrl failed: %v", err)) - return - } - log.Info("url=" + url) - } - } - - ctx.JSON(200, map[string]string{ - "url": url, - }) -} - -func GetObsKey(ctx *context.Context) { - uuid := gouuid.NewV4().String() - key := strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, uuid)), "/") - - ctx.JSON(200, map[string]string{ - "uuid": uuid, - "key": key, - "access_key_id": setting.AccessKeyID, - "secret_access_key": setting.SecretAccessKey, - "server": setting.Endpoint, - "bucket": setting.Bucket, - }) -} - -func CompleteMultipart(ctx *context.Context) { - uuid := ctx.Query("uuid") - uploadID := ctx.Query("uploadID") - typeCloudBrain := ctx.QueryInt("type") - fileName := ctx.Query("file_name") - - log.Warn("uuid:" + uuid) - log.Warn("typeCloudBrain:" + strconv.Itoa(typeCloudBrain)) - - err := checkTypeCloudBrain(typeCloudBrain) - if err != nil { - ctx.ServerError("checkTypeCloudBrain failed", err) - return - } - - fileChunk, err := models.GetFileChunkByUUID(uuid) - if err != nil { - if models.IsErrFileChunkNotExist(err) { - ctx.Error(404) - } else { - ctx.ServerError("GetFileChunkByUUID", err) - } - return - } - - if typeCloudBrain == models.TypeCloudBrainOne { - _, err = storage.CompleteMultiPartUpload(uuid, uploadID) - if err != nil { - ctx.Error(500, fmt.Sprintf("CompleteMultiPartUpload failed: %v", err)) - return - } - } else { - err = storage.CompleteObsMultiPartUpload(uuid, uploadID, fileName) - if err != nil { - ctx.Error(500, fmt.Sprintf("CompleteObsMultiPartUpload failed: %v", err)) - return - } - } - - fileChunk.IsUploaded = models.FileUploaded - - err = models.UpdateFileChunk(fileChunk) - if err != nil { - ctx.Error(500, fmt.Sprintf("UpdateFileChunk: %v", err)) - return - } - dataset, _ := models.GetDatasetByID(ctx.QueryInt64("dataset_id")) - log.Warn("insert attachment to datasetId:" + strconv.FormatInt(dataset.ID, 10)) - attachment, err := models.InsertAttachment(&models.Attachment{ - UUID: uuid, - UploaderID: ctx.User.ID, - IsPrivate: dataset.IsPrivate(), - Name: fileName, - Size: ctx.QueryInt64("size"), - DatasetID: ctx.QueryInt64("dataset_id"), - Description: ctx.Query("description"), - Type: typeCloudBrain, - }) - - if err != nil { - ctx.Error(500, fmt.Sprintf("InsertAttachment: %v", err)) - return - } - - repository, _ := models.GetRepositoryByID(dataset.RepoID) - notification.NotifyOtherTask(ctx.User, repository, fmt.Sprint(attachment.Type), attachment.Name, models.ActionUploadAttachment) - - if attachment.DatasetID != 0 { - if isCanDecompress(attachment.Name) { - if typeCloudBrain == models.TypeCloudBrainOne { - err = worker.SendDecompressTask(contexExt.Background(), uuid, attachment.Name) - if err != nil { - log.Error("SendDecompressTask(%s) failed:%s", uuid, err.Error()) - } else { - attachment.DecompressState = models.DecompressStateIng - err = models.UpdateAttachment(attachment) - if err != nil { - log.Error("UpdateAttachment state(%s) failed:%s", uuid, err.Error()) - } - } - } - if typeCloudBrain == models.TypeCloudBrainTwo { - attachjson, _ := json.Marshal(attachment) - labelmsg.SendDecompressAttachToLabelOBS(string(attachjson)) - } - } else { - var labelMap map[string]string - labelMap = make(map[string]string) - labelMap["UUID"] = uuid - labelMap["Type"] = fmt.Sprint(attachment.Type) - labelMap["UploaderID"] = fmt.Sprint(attachment.UploaderID) - labelMap["RepoID"] = fmt.Sprint(dataset.RepoID) - labelMap["AttachName"] = attachment.Name - attachjson, _ := json.Marshal(labelMap) - labelmsg.SendAddAttachToLabelSys(string(attachjson)) - } - } - - ctx.JSON(200, map[string]string{ - "result_code": "0", - }) -} - -func UpdateMultipart(ctx *context.Context) { - uuid := ctx.Query("uuid") - partNumber := ctx.QueryInt("chunkNumber") - etag := ctx.Query("etag") - - fileChunk, err := models.GetFileChunkByUUID(uuid) - if err != nil { - if models.IsErrFileChunkNotExist(err) { - ctx.Error(404) - } else { - ctx.ServerError("GetFileChunkByUUID", err) - } - return - } - - fileChunk.CompletedParts = append(fileChunk.CompletedParts, strconv.Itoa(partNumber)+"-"+strings.Replace(etag, "\"", "", -1)) - - err = models.UpdateFileChunk(fileChunk) - if err != nil { - ctx.Error(500, fmt.Sprintf("UpdateFileChunk: %v", err)) - return - } - - ctx.JSON(200, map[string]string{ - "result_code": "0", - }) -} - -func HandleUnDecompressAttachment() { - attachs, err := models.GetUnDecompressAttachments() - if err != nil { - log.Error("GetUnDecompressAttachments failed:", err.Error()) - return - } - - for _, attach := range attachs { - if attach.Type == models.TypeCloudBrainOne { - err = worker.SendDecompressTask(contexExt.Background(), attach.UUID, attach.Name) - if err != nil { - log.Error("SendDecompressTask(%s) failed:%s", attach.UUID, err.Error()) - } else { - attach.DecompressState = models.DecompressStateIng - err = models.UpdateAttachment(attach) - if err != nil { - log.Error("UpdateAttachment state(%s) failed:%s", attach.UUID, err.Error()) - } - } - } else if attach.Type == models.TypeCloudBrainTwo { - attachjson, _ := json.Marshal(attach) - labelmsg.SendDecompressAttachToLabelOBS(string(attachjson)) - } - - } - - return -} - -func QueryAllPublicDataset(ctx *context.Context) { - attachs, err := models.GetAllPublicAttachments() - if err != nil { - ctx.JSON(200, map[string]string{ - "result_code": "-1", - "error_msg": err.Error(), - "data": "", - }) - return - } - - queryDatasets(ctx, attachs) -} - -func QueryPrivateDataset(ctx *context.Context) { - username := ctx.Params(":username") - attachs, err := models.GetPrivateAttachments(username) - if err != nil { - ctx.JSON(200, map[string]string{ - "result_code": "-1", - "error_msg": err.Error(), - "data": "", - }) - return - } - - for _, attach := range attachs { - attach.Name = username - } - - queryDatasets(ctx, attachs) -} - -func queryDatasets(ctx *context.Context, attachs []*models.AttachmentUsername) { - var datasets []CloudBrainDataset - if len(attachs) == 0 { - log.Info("dataset is null") - ctx.JSON(200, map[string]string{ - "result_code": "0", - "error_msg": "", - "data": "", - }) - return - } - - for _, attch := range attachs { - has, err := storage.Attachments.HasObject(models.AttachmentRelativePath(attch.UUID)) - if err != nil || !has { - continue - } - - datasets = append(datasets, CloudBrainDataset{strconv.FormatInt(attch.ID, 10), - attch.Attachment.Name, - setting.Attachment.Minio.RealPath + - setting.Attachment.Minio.Bucket + "/" + - setting.Attachment.Minio.BasePath + - models.AttachmentRelativePath(attch.UUID) + - attch.UUID, - attch.Name, - attch.CreatedUnix.Format("2006-01-02 03:04:05 PM")}) - } - - data, err := json.Marshal(datasets) - if err != nil { - log.Error("json.Marshal failed:", err.Error()) - ctx.JSON(200, map[string]string{ - "result_code": "-1", - "error_msg": err.Error(), - "data": "", - }) - return - } - - ctx.JSON(200, map[string]string{ - "result_code": "0", - "error_msg": "", - "data": string(data), - }) - return -} - -func checkTypeCloudBrain(typeCloudBrain int) error { - if typeCloudBrain != models.TypeCloudBrainOne && typeCloudBrain != models.TypeCloudBrainTwo { - log.Error("type error:", typeCloudBrain) - return errors.New("type error") - } - return nil -} diff --git a/routers/repo/dataset.go.bak b/routers/repo/dataset.go.bak deleted file mode 100644 index 5a59e899e..000000000 --- a/routers/repo/dataset.go.bak +++ /dev/null @@ -1,464 +0,0 @@ -package repo - -import ( - "encoding/json" - "net/http" - "regexp" - "sort" - "strconv" - "strings" - "unicode/utf8" - - "code.gitea.io/gitea/models" - "code.gitea.io/gitea/modules/auth" - "code.gitea.io/gitea/modules/base" - "code.gitea.io/gitea/modules/context" - "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/setting" -) - -const ( - tplIndex base.TplName = "repo/datasets/index" - tplDatasetCreate base.TplName = "repo/datasets/create" - tplDatasetEdit base.TplName = "repo/datasets/edit" - taskstplIndex base.TplName = "repo/datasets/tasks/index" -) - -var titlePattern = regexp.MustCompile(`^[A-Za-z0-9-_\\.]{1,100}$`) - -// MustEnableDataset check if repository enable internal dataset -func MustEnableDataset(ctx *context.Context) { - if !ctx.Repo.CanRead(models.UnitTypeDatasets) { - ctx.NotFound("MustEnableDataset", nil) - return - } -} - -func newFilterPrivateAttachments(ctx *context.Context, list []*models.Attachment, repo *models.Repository) []*models.Attachment { - - if ctx.Repo.CanWrite(models.UnitTypeDatasets) { - log.Info("can write.") - return list - } else { - if repo.Owner == nil { - repo.GetOwner() - } - permission := false - if repo.Owner.IsOrganization() && ctx.User != nil { - if repo.Owner.IsUserPartOfOrg(ctx.User.ID) { - log.Info("user is member of org.") - permission = true - } - } - if !permission && ctx.User != nil { - isCollaborator, _ := repo.IsCollaborator(ctx.User.ID) - if isCollaborator { - log.Info("Collaborator user may visit the attach.") - permission = true - } - } - - var publicList []*models.Attachment - for _, attach := range list { - if !attach.IsPrivate { - publicList = append(publicList, attach) - } else { - if permission { - publicList = append(publicList, attach) - } - } - } - return publicList - } -} - -func QueryDataSet(ctx *context.Context) []*models.Attachment { - repo := ctx.Repo.Repository - - dataset, err := models.GetDatasetByRepo(repo) - if err != nil { - log.Error("zou not found dataset 1") - ctx.NotFound("GetDatasetByRepo", err) - return nil - } - - if ctx.Query("type") == "" { - log.Error("zou not found type 2") - ctx.NotFound("type error", nil) - return nil - } - err = models.GetDatasetAttachments(ctx.QueryInt("type"), ctx.IsSigned, ctx.User, dataset) - if err != nil { - ctx.ServerError("GetDatasetAttachments", err) - return nil - } - attachments := newFilterPrivateAttachments(ctx, dataset.Attachments, repo) - - ctx.Data["SortType"] = ctx.Query("sort") - - sort.Slice(attachments, func(i, j int) bool { - return attachments[i].CreatedUnix > attachments[j].CreatedUnix - }) - - return attachments -} - -func DatasetIndex(ctx *context.Context) { - log.Info("dataset index 1") - MustEnableDataset(ctx) - - repo := ctx.Repo.Repository - - dataset, err := models.GetDatasetByRepo(repo) - if err != nil { - log.Warn("query dataset, not found.") - ctx.HTML(200, tplIndex) - return - } - cloudbrainType := -1 - if ctx.Query("type") != "" { - - cloudbrainType = ctx.QueryInt("type") - } - err = models.GetDatasetAttachments(cloudbrainType, ctx.IsSigned, ctx.User, dataset) - if err != nil { - ctx.ServerError("GetDatasetAttachments", err) - return - } - - attachments := newFilterPrivateAttachments(ctx, dataset.Attachments, repo) - - sort.Slice(attachments, func(i, j int) bool { - return attachments[i].CreatedUnix > attachments[j].CreatedUnix - }) - - page := ctx.QueryInt("page") - if page <= 0 { - page = 1 - } - pagesize := ctx.QueryInt("pagesize") - if pagesize <= 0 { - pagesize = 10 - } - pager := context.NewPagination(len(attachments), pagesize, page, 5) - - pageAttachments := getPageAttachments(attachments, page, pagesize) - - //load attachment creator - for _, attachment := range pageAttachments { - uploader, _ := models.GetUserByID(attachment.UploaderID) - attachment.Uploader = uploader - } - - ctx.Data["Page"] = pager - ctx.Data["PageIsDataset"] = true - ctx.Data["Title"] = ctx.Tr("dataset.show_dataset") - ctx.Data["Link"] = ctx.Repo.RepoLink + "/datasets" - ctx.Data["dataset"] = dataset - ctx.Data["Attachments"] = pageAttachments - ctx.Data["IsOwner"] = true - ctx.Data["StoreType"] = setting.Attachment.StoreType - ctx.Data["Type"] = cloudbrainType - - renderAttachmentSettings(ctx) - - ctx.HTML(200, tplIndex) -} - -func getPageAttachments(attachments []*models.Attachment, page int, pagesize int) []*models.Attachment { - begin := (page - 1) * pagesize - end := (page) * pagesize - - if begin > len(attachments)-1 { - return nil - } - if end > len(attachments)-1 { - return attachments[begin:] - } else { - return attachments[begin:end] - } - -} - -func CreateDataset(ctx *context.Context) { - - MustEnableDataset(ctx) - - ctx.HTML(200, tplDatasetCreate) -} - -func EditDataset(ctx *context.Context) { - - MustEnableDataset(ctx) - datasetId, _ := strconv.ParseInt(ctx.Params(":id"), 10, 64) - - dataset, _ := models.GetDatasetByID(datasetId) - if dataset == nil { - ctx.Error(http.StatusNotFound, "") - return - } - ctx.Data["Dataset"] = dataset - - ctx.HTML(200, tplDatasetEdit) -} - -func CreateDatasetPost(ctx *context.Context, form auth.CreateDatasetForm) { - - dataset := &models.Dataset{} - - if !titlePattern.MatchString(form.Title) { - ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("dataset.title_format_err"))) - return - } - if utf8.RuneCountInString(form.Description) > 1024 { - ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("dataset.description_format_err"))) - return - } - - dataset.RepoID = ctx.Repo.Repository.ID - dataset.UserID = ctx.User.ID - dataset.Category = form.Category - dataset.Task = form.Task - dataset.Title = form.Title - dataset.License = form.License - dataset.Description = form.Description - dataset.DownloadTimes = 0 - if ctx.Repo.Repository.IsPrivate { - dataset.Status = 0 - } else { - dataset.Status = 1 - } - err := models.CreateDataset(dataset) - if err != nil { - log.Error("fail to create dataset", err) - ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("dataset.create_dataset_fail"))) - } else { - ctx.JSON(http.StatusOK, models.BaseOKMessage) - } - -} - -func EditDatasetPost(ctx *context.Context, form auth.EditDatasetForm) { - ctx.Data["PageIsDataset"] = true - - ctx.Data["Title"] = ctx.Tr("dataset.edit_dataset") - - if !titlePattern.MatchString(form.Title) { - ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("dataset.title_format_err"))) - return - } - if utf8.RuneCountInString(form.Description) > 1024 { - ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("dataset.description_format_err"))) - return - } - - rel, err := models.GetDatasetByID(form.ID) - ctx.Data["dataset"] = rel - - if err != nil { - log.Error("failed to query dataset", err) - ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("dataset.query_dataset_fail"))) - return - } - - rel.Title = form.Title - rel.Description = form.Description - rel.Category = form.Category - rel.Task = form.Task - rel.License = form.License - if err = models.UpdateDataset(models.DefaultDBContext(), rel); err != nil { - ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("dataset.query_dataset_fail"))) - } - ctx.JSON(http.StatusOK, models.BaseOKMessage) -} - -func DatasetAction(ctx *context.Context) { - var err error - datasetId, _ := strconv.ParseInt(ctx.Params(":id"), 10, 64) - switch ctx.Params(":action") { - case "star": - err = models.StarDataset(ctx.User.ID, datasetId, true) - case "unstar": - err = models.StarDataset(ctx.User.ID, datasetId, false) - - } - if err != nil { - ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("repo.star_fail", ctx.Params(":action")))) - } else { - ctx.JSON(http.StatusOK, models.BaseOKMessage) - } - -} - -func CurrentRepoDataset(ctx *context.Context) { - page := ctx.QueryInt("page") - cloudbrainType := ctx.QueryInt("type") - keyword := strings.Trim(ctx.Query("q"), " ") - - repo := ctx.Repo.Repository - var datasetIDs []int64 - dataset, err := models.GetDatasetByRepo(repo) - if err != nil { - ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("GetDatasetByRepo failed", err))) - } - datasetIDs = append(datasetIDs, dataset.ID) - uploaderID := ctx.User.ID - datasets, count, err := models.Attachments(&models.AttachmentsOptions{ - ListOptions: models.ListOptions{ - Page: page, - PageSize: setting.UI.IssuePagingNum, - }, - Keyword: keyword, - DatasetIDs: datasetIDs, - UploaderID: uploaderID, - Type: cloudbrainType, - NeedIsPrivate: false, - NeedRepoInfo: true, - }) - if err != nil { - ctx.ServerError("datasets", err) - return - } - - data, err := json.Marshal(datasets) - if err != nil { - log.Error("json.Marshal failed:", err.Error()) - ctx.JSON(200, map[string]string{ - "result_code": "-1", - "error_msg": err.Error(), - "data": "", - }) - return - } - ctx.JSON(200, map[string]string{ - "result_code": "0", - "data": string(data), - "count": strconv.FormatInt(count, 10), - }) -} - -func MyDatasets(ctx *context.Context) { - page := ctx.QueryInt("page") - cloudbrainType := ctx.QueryInt("type") - keyword := strings.Trim(ctx.Query("q"), " ") - - uploaderID := ctx.User.ID - datasets, count, err := models.Attachments(&models.AttachmentsOptions{ - ListOptions: models.ListOptions{ - Page: page, - PageSize: setting.UI.IssuePagingNum, - }, - Keyword: keyword, - UploaderID: uploaderID, - Type: cloudbrainType, - NeedIsPrivate: false, - NeedRepoInfo: true, - }) - if err != nil { - ctx.ServerError("datasets", err) - return - } - - data, err := json.Marshal(datasets) - if err != nil { - log.Error("json.Marshal failed:", err.Error()) - ctx.JSON(200, map[string]string{ - "result_code": "-1", - "error_msg": err.Error(), - "data": "", - }) - return - } - ctx.JSON(200, map[string]string{ - "result_code": "0", - "data": string(data), - "count": strconv.FormatInt(count, 10), - }) -} - -func PublicDataset(ctx *context.Context) { - page := ctx.QueryInt("page") - cloudbrainType := ctx.QueryInt("type") - keyword := strings.Trim(ctx.Query("q"), " ") - - datasets, count, err := models.Attachments(&models.AttachmentsOptions{ - ListOptions: models.ListOptions{ - Page: page, - PageSize: setting.UI.IssuePagingNum, - }, - Keyword: keyword, - NeedIsPrivate: true, - IsPrivate: false, - Type: cloudbrainType, - NeedRepoInfo: true, - }) - if err != nil { - ctx.ServerError("datasets", err) - return - } - - data, err := json.Marshal(datasets) - if err != nil { - log.Error("json.Marshal failed:", err.Error()) - ctx.JSON(200, map[string]string{ - "result_code": "-1", - "error_msg": err.Error(), - "data": "", - }) - return - } - ctx.JSON(200, map[string]string{ - "result_code": "0", - "data": string(data), - "count": strconv.FormatInt(count, 10), - }) -} - -func MyFavoriteDataset(ctx *context.Context) { - page := ctx.QueryInt("page") - cloudbrainType := ctx.QueryInt("type") - keyword := strings.Trim(ctx.Query("q"), " ") - - var datasetIDs []int64 - - datasetStars, err := models.GetDatasetStarByUser(ctx.User) - if err != nil { - ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("GetDatasetByRepo failed", err))) - } - for i, _ := range datasetStars { - datasetIDs = append(datasetIDs, datasetStars[i].DatasetID) - } - - datasets, count, err := models.Attachments(&models.AttachmentsOptions{ - ListOptions: models.ListOptions{ - Page: page, - PageSize: setting.UI.IssuePagingNum, - }, - Keyword: keyword, - DatasetIDs: datasetIDs, - NeedIsPrivate: true, - IsPrivate: false, - Type: cloudbrainType, - NeedRepoInfo: true, - }) - if err != nil { - ctx.ServerError("datasets", err) - return - } - - data, err := json.Marshal(datasets) - if err != nil { - log.Error("json.Marshal failed:", err.Error()) - ctx.JSON(200, map[string]string{ - "result_code": "-1", - "error_msg": err.Error(), - "data": "", - }) - return - } - ctx.JSON(200, map[string]string{ - "result_code": "0", - "data": string(data), - "count": strconv.FormatInt(count, 10), - }) -}