From f014c8290d72918cd87b039ac306bad49f10f533 Mon Sep 17 00:00:00 2001 From: yanchao Date: Wed, 16 Mar 2022 10:36:20 +0800 Subject: [PATCH] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- routers/repo/attachment.go | 3 +- routers/repo/attachment.go.bak | 1086 ++++++++++++++++++++++++++++++++ routers/repo/dataset.go | 2 + routers/repo/dataset.go.bak | 464 ++++++++++++++ 4 files changed, 1554 insertions(+), 1 deletion(-) create mode 100644 routers/repo/attachment.go.bak create mode 100644 routers/repo/dataset.go.bak diff --git a/routers/repo/attachment.go b/routers/repo/attachment.go index a4ffa6d29..f0c1a2819 100755 --- a/routers/repo/attachment.go +++ b/routers/repo/attachment.go @@ -82,7 +82,8 @@ func UploadAttachmentUI(ctx *context.Context) { } func EditAttachmentUI(ctx *context.Context) { - id := ctx.QueryInt64(":id") + + id, _ := strconv.ParseInt(ctx.Params(":id"), 10, 64) attachment, _ := models.GetAttachmentByID(id) if attachment == nil { ctx.Error(404, "The attachment does not exits.") diff --git a/routers/repo/attachment.go.bak b/routers/repo/attachment.go.bak new file mode 100644 index 000000000..a4ffa6d29 --- /dev/null +++ b/routers/repo/attachment.go.bak @@ -0,0 +1,1086 @@ +// Copyright 2017 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package repo + +import ( + contexExt "context" + "encoding/json" + "errors" + "fmt" + "mime/multipart" + "net/http" + "path" + "strconv" + "strings" + + "code.gitea.io/gitea/modules/auth" + + "code.gitea.io/gitea/modules/base" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/labelmsg" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/minio_ext" + "code.gitea.io/gitea/modules/notification" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/storage" + "code.gitea.io/gitea/modules/upload" + "code.gitea.io/gitea/modules/worker" + gouuid "github.com/satori/go.uuid" +) + +const ( + //result of decompress + DecompressSuccess = "0" + DecompressFailed = "1" + tplAttachmentUpload base.TplName = "repo/attachment/upload" + tplAttachmentEdit base.TplName = "repo/attachment/edit" +) + +type CloudBrainDataset struct { + UUID string `json:"id"` + Name string `json:"name"` + Path string `json:"place"` + UserName string `json:"provider"` + CreateTime string `json:"created_at"` +} + +type UploadForm struct { + UploadID string `form:"uploadId"` + UuID string `form:"uuid"` + PartSize int64 `form:"size"` + Offset int64 `form:"offset"` + PartNumber int `form:"chunkNumber"` + PartFile multipart.File `form:"file"` +} + +func RenderAttachmentSettings(ctx *context.Context) { + renderAttachmentSettings(ctx) +} + +func renderAttachmentSettings(ctx *context.Context) { + ctx.Data["IsAttachmentEnabled"] = setting.Attachment.Enabled + ctx.Data["AttachmentStoreType"] = setting.Attachment.StoreType + ctx.Data["AttachmentAllowedTypes"] = setting.Attachment.AllowedTypes + ctx.Data["AttachmentMaxSize"] = setting.Attachment.MaxSize + ctx.Data["AttachmentMaxFiles"] = setting.Attachment.MaxFiles +} + +func UploadAttachmentUI(ctx *context.Context) { + ctx.Data["datasetId"] = ctx.Query("datasetId") + dataset, _ := models.GetDatasetByID(ctx.QueryInt64("datasetId")) + if dataset == nil { + ctx.Error(404, "The dataset does not exits.") + } + r, _ := models.GetRepositoryByID(dataset.RepoID) + ctx.Data["Repo"] = r + ctx.HTML(200, tplAttachmentUpload) + +} + +func EditAttachmentUI(ctx *context.Context) { + id := ctx.QueryInt64(":id") + attachment, _ := models.GetAttachmentByID(id) + if attachment == nil { + ctx.Error(404, "The attachment does not exits.") + } + + ctx.Data["Attachment"] = attachment + ctx.HTML(200, tplAttachmentEdit) + +} + +func EditAttachment(ctx *context.Context, form auth.EditAttachmentForm) { + + err := models.UpdateAttachment(&models.Attachment{ + ID: form.ID, + Description: form.Description, + }) + if err != nil { + ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("dataset.edit_attachment_fail"))) + } + ctx.JSON(http.StatusOK, models.BaseOKMessage) + +} + +// UploadAttachment response for uploading issue's attachment +func UploadAttachment(ctx *context.Context) { + if !setting.Attachment.Enabled { + ctx.Error(404, "attachment is not enabled") + return + } + + file, header, err := ctx.Req.FormFile("file") + if err != nil { + ctx.Error(500, fmt.Sprintf("FormFile: %v", err)) + return + } + defer file.Close() + + buf := make([]byte, 1024) + n, _ := file.Read(buf) + if n > 0 { + buf = buf[:n] + } + + err = upload.VerifyAllowedContentType(buf, strings.Split(setting.Attachment.AllowedTypes, ",")) + if err != nil { + ctx.Error(400, err.Error()) + return + } + + datasetID, _ := strconv.ParseInt(ctx.Req.FormValue("dataset_id"), 10, 64) + + attach, err := models.NewAttachment(&models.Attachment{ + IsPrivate: true, + UploaderID: ctx.User.ID, + Name: header.Filename, + DatasetID: datasetID, + }, buf, file) + if err != nil { + ctx.Error(500, fmt.Sprintf("NewAttachment: %v", err)) + return + } + + log.Trace("New attachment uploaded: %s", attach.UUID) + ctx.JSON(200, map[string]string{ + "uuid": attach.UUID, + }) +} + +func UpdatePublicAttachment(ctx *context.Context) { + file := ctx.Query("file") + isPrivate, _ := strconv.ParseBool(ctx.Query("is_private")) + attach, err := models.GetAttachmentByUUID(file) + if err != nil { + ctx.Error(404, err.Error()) + return + } + attach.IsPrivate = isPrivate + models.UpdateAttachment(attach) +} + +// DeleteAttachment response for deleting issue's attachment +func DeleteAttachment(ctx *context.Context) { + file := ctx.Query("file") + attach, err := models.GetAttachmentByUUID(file) + if err != nil { + ctx.Error(400, err.Error()) + return + } + + //issue 214: mod del-dataset permission + if !models.CanDelAttachment(ctx.IsSigned, ctx.User, attach) { + ctx.Error(403) + return + } + + err = models.DeleteAttachment(attach, true) + if err != nil { + ctx.Error(500, fmt.Sprintf("DeleteAttachment: %v", err)) + return + } + + attachjson, _ := json.Marshal(attach) + labelmsg.SendDeleteAttachToLabelSys(string(attachjson)) + + DeleteAllUnzipFile(attach, "") + + _, err = models.DeleteFileChunkById(attach.UUID) + if err != nil { + ctx.Error(500, fmt.Sprintf("DeleteFileChunkById: %v", err)) + return + } + ctx.JSON(200, map[string]string{ + "uuid": attach.UUID, + }) +} + +func DownloadUserIsOrgOrCollaboration(ctx *context.Context, attach *models.Attachment) bool { + dataset, err := models.GetDatasetByID(attach.DatasetID) + if err != nil { + log.Info("query dataset error") + } else { + repo, err := models.GetRepositoryByID(dataset.RepoID) + if err != nil { + log.Info("query repo error.") + } else { + repo.GetOwner() + if ctx.User != nil { + + if repo.Owner.IsOrganization() { + if repo.Owner.IsUserPartOfOrg(ctx.User.ID) { + log.Info("org user may visit the attach.") + return true + } + } + isCollaborator, _ := repo.IsCollaborator(ctx.User.ID) + if isCollaborator { + log.Info("Collaborator user may visit the attach.") + return true + } + } + } + } + return false +} + +// GetAttachment serve attachements +func GetAttachment(ctx *context.Context) { + typeCloudBrain := ctx.QueryInt("type") + err := checkTypeCloudBrain(typeCloudBrain) + if err != nil { + ctx.ServerError("checkTypeCloudBrain failed", err) + return + } + + attach, err := models.GetAttachmentByUUID(ctx.Params(":uuid")) + if err != nil { + if models.IsErrAttachmentNotExist(err) { + ctx.Error(404) + } else { + ctx.ServerError("GetAttachmentByUUID", err) + } + return + } + + repository, unitType, err := attach.LinkedRepository() + if err != nil { + ctx.ServerError("LinkedRepository", err) + return + } + dataSet, err := attach.LinkedDataSet() + if err != nil { + ctx.ServerError("LinkedDataSet", err) + return + } + + if repository == nil && dataSet != nil { + repository, _ = models.GetRepositoryByID(dataSet.RepoID) + unitType = models.UnitTypeDatasets + } + + if repository == nil { //If not linked + //if !(ctx.IsSigned && attach.UploaderID == ctx.User.ID) && attach.IsPrivate { //We block if not the uploader + //log.Info("ctx.IsSigned =" + fmt.Sprintf("%v", ctx.IsSigned)) + if !(ctx.IsSigned && attach.UploaderID == ctx.User.ID) && attach.IsPrivate && !DownloadUserIsOrgOrCollaboration(ctx, attach) { //We block if not the uploader + ctx.Error(http.StatusNotFound) + return + } + + } else { //If we have the repository we check access + perm, errPermission := models.GetUserRepoPermission(repository, ctx.User) + if errPermission != nil { + ctx.Error(http.StatusInternalServerError, "GetUserRepoPermission", errPermission.Error()) + return + } + if !perm.CanRead(unitType) { + ctx.Error(http.StatusNotFound) + return + } + } + + if dataSet != nil { + isPermit, err := models.GetUserDataSetPermission(dataSet, ctx.User) + if err != nil { + ctx.Error(http.StatusInternalServerError, "GetUserDataSetPermission", err.Error()) + return + } + if !isPermit { + ctx.Error(http.StatusNotFound) + return + } + } + + //If we have matched and access to release or issue + if setting.Attachment.StoreType == storage.MinioStorageType { + url := "" + if typeCloudBrain == models.TypeCloudBrainOne { + url, err = storage.Attachments.PresignedGetURL(setting.Attachment.Minio.BasePath+attach.RelativePath(), attach.Name) + if err != nil { + ctx.ServerError("PresignedGetURL", err) + return + } + } else { + if setting.PROXYURL != "" { + url = setting.PROXYURL + "/obs_proxy_download?uuid=" + attach.UUID + "&file_name=" + attach.Name + log.Info("return url=" + url) + } else { + url, err = storage.ObsGetPreSignedUrl(attach.UUID, attach.Name) + if err != nil { + ctx.ServerError("ObsGetPreSignedUrl", err) + return + } + } + } + + if err = increaseDownloadCount(attach, dataSet); err != nil { + ctx.ServerError("Update", err) + return + } + if dataSet != nil { + http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently) + } else { + fr, err := storage.Attachments.Open(attach.RelativePath()) + if err != nil { + ctx.ServerError("Open", err) + return + } + defer fr.Close() + if err = ServeData(ctx, attach.Name, fr); err != nil { + ctx.ServerError("ServeData", err) + return + } + } + + } else { + fr, err := storage.Attachments.Open(attach.RelativePath()) + if err != nil { + ctx.ServerError("Open", err) + return + } + defer fr.Close() + if err = increaseDownloadCount(attach, dataSet); err != nil { + ctx.ServerError("Update", err) + return + } + if err = ServeData(ctx, attach.Name, fr); err != nil { + ctx.ServerError("ServeData", err) + return + } + } + +} + +func increaseDownloadCount(attach *models.Attachment, dataSet *models.Dataset) error { + if err := attach.IncreaseDownloadCount(); err != nil { + return err + } + + if dataSet != nil { + if err := models.IncreaseDownloadCount(dataSet.ID); err != nil { + return err + } + } + + return nil +} + +// Get a presigned url for put object +func GetPresignedPutObjectURL(ctx *context.Context) { + if !setting.Attachment.Enabled { + ctx.Error(404, "attachment is not enabled") + return + } + + err := upload.VerifyFileType(ctx.Params("file_type"), strings.Split(setting.Attachment.AllowedTypes, ",")) + if err != nil { + ctx.Error(400, err.Error()) + return + } + + if setting.Attachment.StoreType == storage.MinioStorageType { + uuid := gouuid.NewV4().String() + url, err := storage.Attachments.PresignedPutURL(models.AttachmentRelativePath(uuid)) + if err != nil { + ctx.ServerError("PresignedPutURL", err) + return + } + + ctx.JSON(200, map[string]string{ + "uuid": uuid, + "url": url, + }) + } else { + ctx.Error(404, "storage type is not enabled") + return + } +} + +// AddAttachment response for add attachment record +func AddAttachment(ctx *context.Context) { + typeCloudBrain := ctx.QueryInt("type") + fileName := ctx.Query("file_name") + err := checkTypeCloudBrain(typeCloudBrain) + if err != nil { + ctx.ServerError("checkTypeCloudBrain failed", err) + return + } + + uuid := ctx.Query("uuid") + has := false + if typeCloudBrain == models.TypeCloudBrainOne { + has, err = storage.Attachments.HasObject(models.AttachmentRelativePath(uuid)) + if err != nil { + ctx.ServerError("HasObject", err) + return + } + } else { + has, err = storage.ObsHasObject(setting.BasePath + models.AttachmentRelativePath(uuid) + "/" + fileName) + if err != nil { + ctx.ServerError("ObsHasObject", err) + return + } + } + + if !has { + ctx.Error(404, "attachment has not been uploaded") + return + } + datasetId := ctx.QueryInt64("dataset_id") + dataset, err := models.GetDatasetByID(datasetId) + if err != nil { + ctx.Error(404, "dataset does not exist.") + return + } + + attachment, err := models.InsertAttachment(&models.Attachment{ + UUID: uuid, + UploaderID: ctx.User.ID, + IsPrivate: dataset.IsPrivate(), + Name: fileName, + Size: ctx.QueryInt64("size"), + DatasetID: ctx.QueryInt64("dataset_id"), + Type: typeCloudBrain, + }) + + if err != nil { + ctx.Error(500, fmt.Sprintf("InsertAttachment: %v", err)) + return + } + + if attachment.DatasetID != 0 { + if isCanDecompress(attachment.Name) { + if typeCloudBrain == models.TypeCloudBrainOne { + err = worker.SendDecompressTask(contexExt.Background(), uuid, attachment.Name) + if err != nil { + log.Error("SendDecompressTask(%s) failed:%s", uuid, err.Error()) + } else { + attachment.DecompressState = models.DecompressStateIng + err = models.UpdateAttachment(attachment) + if err != nil { + log.Error("UpdateAttachment state(%s) failed:%s", uuid, err.Error()) + } + } + } + //todo:decompress type_two + } + } + + ctx.JSON(200, map[string]string{ + "result_code": "0", + }) +} + +func isCanDecompress(name string) bool { + if strings.HasSuffix(name, ".zip") || strings.HasSuffix(name, ".tar.gz") || strings.HasSuffix(name, ".tgz") { + return true + } + return false +} + +func UpdateAttachmentDecompressState(ctx *context.Context) { + uuid := ctx.Query("uuid") + result := ctx.Query("result") + attach, err := models.GetAttachmentByUUID(uuid) + if err != nil { + log.Error("GetAttachmentByUUID(%s) failed:%s", uuid, err.Error()) + return + } + + if result == DecompressSuccess { + attach.DecompressState = models.DecompressStateDone + } else if result == DecompressFailed { + attach.DecompressState = models.DecompressStateFailed + } else { + log.Error("result is error:", result) + return + } + + err = models.UpdateAttachment(attach) + if err != nil { + log.Error("UpdateAttachment(%s) failed:%s", uuid, err.Error()) + return + } + log.Info("start to send msg to labelsystem ") + + dataset, _ := models.GetDatasetByID(attach.DatasetID) + + var labelMap map[string]string + labelMap = make(map[string]string) + labelMap["UUID"] = uuid + labelMap["Type"] = fmt.Sprint(attach.Type) + labelMap["UploaderID"] = fmt.Sprint(attach.UploaderID) + labelMap["RepoID"] = fmt.Sprint(dataset.RepoID) + labelMap["AttachName"] = attach.Name + attachjson, _ := json.Marshal(labelMap) + labelmsg.SendAddAttachToLabelSys(string(attachjson)) + + log.Info("end to send msg to labelsystem ") + + ctx.JSON(200, map[string]string{ + "result_code": "0", + }) +} + +func GetSuccessChunks(ctx *context.Context) { + fileMD5 := ctx.Query("md5") + typeCloudBrain := ctx.QueryInt("type") + fileName := ctx.Query("file_name") + var chunks string + + err := checkTypeCloudBrain(typeCloudBrain) + if err != nil { + ctx.ServerError("checkTypeCloudBrain failed", err) + return + } + + fileChunk, err := models.GetFileChunkByMD5AndUser(fileMD5, ctx.User.ID, typeCloudBrain) + if err != nil { + if models.IsErrFileChunkNotExist(err) { + ctx.JSON(200, map[string]string{ + "uuid": "", + "uploaded": "0", + "uploadID": "", + "chunks": "", + }) + } else { + ctx.ServerError("GetFileChunkByMD5", err) + } + return + } + + isExist := false + if typeCloudBrain == models.TypeCloudBrainOne { + isExist, err = storage.Attachments.HasObject(models.AttachmentRelativePath(fileChunk.UUID)) + if err != nil { + ctx.ServerError("HasObject failed", err) + return + } + } else { + isExist, err = storage.ObsHasObject(setting.BasePath + models.AttachmentRelativePath(fileChunk.UUID) + "/" + fileName) + if err != nil { + ctx.ServerError("ObsHasObject failed", err) + return + } + } + + if isExist { + if fileChunk.IsUploaded == models.FileNotUploaded { + log.Info("the file has been uploaded but not recorded") + fileChunk.IsUploaded = models.FileUploaded + if err = models.UpdateFileChunk(fileChunk); err != nil { + log.Error("UpdateFileChunk failed:", err.Error()) + } + } + } else { + if fileChunk.IsUploaded == models.FileUploaded { + log.Info("the file has been recorded but not uploaded") + fileChunk.IsUploaded = models.FileNotUploaded + if err = models.UpdateFileChunk(fileChunk); err != nil { + log.Error("UpdateFileChunk failed:", err.Error()) + } + } + + if typeCloudBrain == models.TypeCloudBrainOne { + chunks, err = storage.GetPartInfos(fileChunk.UUID, fileChunk.UploadID) + if err != nil { + log.Error("GetPartInfos failed:%v", err.Error()) + } + } else { + chunks, err = storage.GetObsPartInfos(fileChunk.UUID, fileChunk.UploadID, fileName) + if err != nil { + log.Error("GetObsPartInfos failed:%v", err.Error()) + } + } + + if err != nil { + models.DeleteFileChunk(fileChunk) + ctx.JSON(200, map[string]string{ + "uuid": "", + "uploaded": "0", + "uploadID": "", + "chunks": "", + }) + return + } + } + + var attachID int64 + attach, err := models.GetAttachmentByUUID(fileChunk.UUID) + if err != nil { + if models.IsErrAttachmentNotExist(err) { + attachID = 0 + } else { + ctx.ServerError("GetAttachmentByUUID", err) + return + } + } else { + attachID = attach.ID + } + + if attach == nil { + ctx.JSON(200, map[string]string{ + "uuid": fileChunk.UUID, + "uploaded": strconv.Itoa(fileChunk.IsUploaded), + "uploadID": fileChunk.UploadID, + "chunks": string(chunks), + "attachID": "0", + "datasetID": "0", + "fileName": "", + "datasetName": "", + }) + return + } + + dataset, err := models.GetDatasetByID(attach.DatasetID) + if err != nil { + ctx.ServerError("GetDatasetByID", err) + return + } + + ctx.JSON(200, map[string]string{ + "uuid": fileChunk.UUID, + "uploaded": strconv.Itoa(fileChunk.IsUploaded), + "uploadID": fileChunk.UploadID, + "chunks": string(chunks), + "attachID": strconv.Itoa(int(attachID)), + "datasetID": strconv.Itoa(int(attach.DatasetID)), + "fileName": attach.Name, + "datasetName": dataset.Title, + }) + +} + +func NewMultipart(ctx *context.Context) { + if !setting.Attachment.Enabled { + ctx.Error(404, "attachment is not enabled") + return + } + + err := upload.VerifyFileType(ctx.Query("fileType"), strings.Split(setting.Attachment.AllowedTypes, ",")) + if err != nil { + ctx.Error(400, err.Error()) + return + } + + typeCloudBrain := ctx.QueryInt("type") + err = checkTypeCloudBrain(typeCloudBrain) + if err != nil { + ctx.ServerError("checkTypeCloudBrain failed", err) + return + } + + fileName := ctx.Query("file_name") + + if setting.Attachment.StoreType == storage.MinioStorageType { + totalChunkCounts := ctx.QueryInt("totalChunkCounts") + if totalChunkCounts > minio_ext.MaxPartsCount { + ctx.Error(400, fmt.Sprintf("chunk counts(%d) is too much", totalChunkCounts)) + return + } + + fileSize := ctx.QueryInt64("size") + if fileSize > minio_ext.MaxMultipartPutObjectSize { + ctx.Error(400, fmt.Sprintf("file size(%d) is too big", fileSize)) + return + } + + uuid := gouuid.NewV4().String() + var uploadID string + if typeCloudBrain == models.TypeCloudBrainOne { + uploadID, err = storage.NewMultiPartUpload(uuid) + if err != nil { + ctx.ServerError("NewMultipart", err) + return + } + } else { + uploadID, err = storage.NewObsMultiPartUpload(uuid, fileName) + if err != nil { + ctx.ServerError("NewObsMultiPartUpload", err) + return + } + } + + _, err = models.InsertFileChunk(&models.FileChunk{ + UUID: uuid, + UserID: ctx.User.ID, + UploadID: uploadID, + Md5: ctx.Query("md5"), + Size: fileSize, + TotalChunks: totalChunkCounts, + Type: typeCloudBrain, + }) + + if err != nil { + ctx.Error(500, fmt.Sprintf("InsertFileChunk: %v", err)) + return + } + + ctx.JSON(200, map[string]string{ + "uuid": uuid, + "uploadID": uploadID, + }) + } else { + ctx.Error(404, "storage type is not enabled") + return + } +} + +func PutOBSProxyUpload(ctx *context.Context) { + uuid := ctx.Query("uuid") + uploadID := ctx.Query("uploadId") + partNumber := ctx.QueryInt("partNumber") + fileName := ctx.Query("file_name") + + RequestBody := ctx.Req.Body() + + if RequestBody == nil { + ctx.Error(500, fmt.Sprintf("FormFile: %v", RequestBody)) + return + } + + err := storage.ObsMultiPartUpload(uuid, uploadID, partNumber, fileName, RequestBody.ReadCloser()) + if err != nil { + log.Info("upload error.") + } +} + +func GetOBSProxyDownload(ctx *context.Context) { + uuid := ctx.Query("uuid") + fileName := ctx.Query("file_name") + + body, err := storage.ObsDownload(uuid, fileName) + if err != nil { + log.Info("upload error.") + } else { + defer body.Close() + ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+fileName) + ctx.Resp.Header().Set("Content-Type", "application/octet-stream") + p := make([]byte, 1024) + var readErr error + var readCount int + // 读取对象内容 + for { + readCount, readErr = body.Read(p) + if readCount > 0 { + ctx.Resp.Write(p[:readCount]) + //fmt.Printf("%s", p[:readCount]) + } + if readErr != nil { + break + } + } + } +} + +func GetMultipartUploadUrl(ctx *context.Context) { + uuid := ctx.Query("uuid") + uploadID := ctx.Query("uploadID") + partNumber := ctx.QueryInt("chunkNumber") + size := ctx.QueryInt64("size") + fileName := ctx.Query("file_name") + + typeCloudBrain := ctx.QueryInt("type") + err := checkTypeCloudBrain(typeCloudBrain) + if err != nil { + ctx.ServerError("checkTypeCloudBrain failed", err) + return + } + + url := "" + if typeCloudBrain == models.TypeCloudBrainOne { + if size > minio_ext.MinPartSize { + ctx.Error(400, fmt.Sprintf("chunk size(%d) is too big", size)) + return + } + + url, err = storage.GenMultiPartSignedUrl(uuid, uploadID, partNumber, size) + if err != nil { + ctx.Error(500, fmt.Sprintf("GenMultiPartSignedUrl failed: %v", err)) + return + } + } else { + if setting.PROXYURL != "" { + url = setting.PROXYURL + "/obs_proxy_multipart?uuid=" + uuid + "&uploadId=" + uploadID + "&partNumber=" + fmt.Sprint(partNumber) + "&file_name=" + fileName + log.Info("return url=" + url) + } else { + url, err = storage.ObsGenMultiPartSignedUrl(uuid, uploadID, partNumber, fileName) + if err != nil { + ctx.Error(500, fmt.Sprintf("ObsGenMultiPartSignedUrl failed: %v", err)) + return + } + log.Info("url=" + url) + } + } + + ctx.JSON(200, map[string]string{ + "url": url, + }) +} + +func GetObsKey(ctx *context.Context) { + uuid := gouuid.NewV4().String() + key := strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, uuid)), "/") + + ctx.JSON(200, map[string]string{ + "uuid": uuid, + "key": key, + "access_key_id": setting.AccessKeyID, + "secret_access_key": setting.SecretAccessKey, + "server": setting.Endpoint, + "bucket": setting.Bucket, + }) +} + +func CompleteMultipart(ctx *context.Context) { + uuid := ctx.Query("uuid") + uploadID := ctx.Query("uploadID") + typeCloudBrain := ctx.QueryInt("type") + fileName := ctx.Query("file_name") + + log.Warn("uuid:" + uuid) + log.Warn("typeCloudBrain:" + strconv.Itoa(typeCloudBrain)) + + err := checkTypeCloudBrain(typeCloudBrain) + if err != nil { + ctx.ServerError("checkTypeCloudBrain failed", err) + return + } + + fileChunk, err := models.GetFileChunkByUUID(uuid) + if err != nil { + if models.IsErrFileChunkNotExist(err) { + ctx.Error(404) + } else { + ctx.ServerError("GetFileChunkByUUID", err) + } + return + } + + if typeCloudBrain == models.TypeCloudBrainOne { + _, err = storage.CompleteMultiPartUpload(uuid, uploadID) + if err != nil { + ctx.Error(500, fmt.Sprintf("CompleteMultiPartUpload failed: %v", err)) + return + } + } else { + err = storage.CompleteObsMultiPartUpload(uuid, uploadID, fileName) + if err != nil { + ctx.Error(500, fmt.Sprintf("CompleteObsMultiPartUpload failed: %v", err)) + return + } + } + + fileChunk.IsUploaded = models.FileUploaded + + err = models.UpdateFileChunk(fileChunk) + if err != nil { + ctx.Error(500, fmt.Sprintf("UpdateFileChunk: %v", err)) + return + } + dataset, _ := models.GetDatasetByID(ctx.QueryInt64("dataset_id")) + log.Warn("insert attachment to datasetId:" + strconv.FormatInt(dataset.ID, 10)) + attachment, err := models.InsertAttachment(&models.Attachment{ + UUID: uuid, + UploaderID: ctx.User.ID, + IsPrivate: dataset.IsPrivate(), + Name: fileName, + Size: ctx.QueryInt64("size"), + DatasetID: ctx.QueryInt64("dataset_id"), + Description: ctx.Query("description"), + Type: typeCloudBrain, + }) + + if err != nil { + ctx.Error(500, fmt.Sprintf("InsertAttachment: %v", err)) + return + } + + repository, _ := models.GetRepositoryByID(dataset.RepoID) + notification.NotifyOtherTask(ctx.User, repository, fmt.Sprint(attachment.Type), attachment.Name, models.ActionUploadAttachment) + + if attachment.DatasetID != 0 { + if isCanDecompress(attachment.Name) { + if typeCloudBrain == models.TypeCloudBrainOne { + err = worker.SendDecompressTask(contexExt.Background(), uuid, attachment.Name) + if err != nil { + log.Error("SendDecompressTask(%s) failed:%s", uuid, err.Error()) + } else { + attachment.DecompressState = models.DecompressStateIng + err = models.UpdateAttachment(attachment) + if err != nil { + log.Error("UpdateAttachment state(%s) failed:%s", uuid, err.Error()) + } + } + } + if typeCloudBrain == models.TypeCloudBrainTwo { + attachjson, _ := json.Marshal(attachment) + labelmsg.SendDecompressAttachToLabelOBS(string(attachjson)) + } + } else { + var labelMap map[string]string + labelMap = make(map[string]string) + labelMap["UUID"] = uuid + labelMap["Type"] = fmt.Sprint(attachment.Type) + labelMap["UploaderID"] = fmt.Sprint(attachment.UploaderID) + labelMap["RepoID"] = fmt.Sprint(dataset.RepoID) + labelMap["AttachName"] = attachment.Name + attachjson, _ := json.Marshal(labelMap) + labelmsg.SendAddAttachToLabelSys(string(attachjson)) + } + } + + ctx.JSON(200, map[string]string{ + "result_code": "0", + }) +} + +func UpdateMultipart(ctx *context.Context) { + uuid := ctx.Query("uuid") + partNumber := ctx.QueryInt("chunkNumber") + etag := ctx.Query("etag") + + fileChunk, err := models.GetFileChunkByUUID(uuid) + if err != nil { + if models.IsErrFileChunkNotExist(err) { + ctx.Error(404) + } else { + ctx.ServerError("GetFileChunkByUUID", err) + } + return + } + + fileChunk.CompletedParts = append(fileChunk.CompletedParts, strconv.Itoa(partNumber)+"-"+strings.Replace(etag, "\"", "", -1)) + + err = models.UpdateFileChunk(fileChunk) + if err != nil { + ctx.Error(500, fmt.Sprintf("UpdateFileChunk: %v", err)) + return + } + + ctx.JSON(200, map[string]string{ + "result_code": "0", + }) +} + +func HandleUnDecompressAttachment() { + attachs, err := models.GetUnDecompressAttachments() + if err != nil { + log.Error("GetUnDecompressAttachments failed:", err.Error()) + return + } + + for _, attach := range attachs { + if attach.Type == models.TypeCloudBrainOne { + err = worker.SendDecompressTask(contexExt.Background(), attach.UUID, attach.Name) + if err != nil { + log.Error("SendDecompressTask(%s) failed:%s", attach.UUID, err.Error()) + } else { + attach.DecompressState = models.DecompressStateIng + err = models.UpdateAttachment(attach) + if err != nil { + log.Error("UpdateAttachment state(%s) failed:%s", attach.UUID, err.Error()) + } + } + } else if attach.Type == models.TypeCloudBrainTwo { + attachjson, _ := json.Marshal(attach) + labelmsg.SendDecompressAttachToLabelOBS(string(attachjson)) + } + + } + + return +} + +func QueryAllPublicDataset(ctx *context.Context) { + attachs, err := models.GetAllPublicAttachments() + if err != nil { + ctx.JSON(200, map[string]string{ + "result_code": "-1", + "error_msg": err.Error(), + "data": "", + }) + return + } + + queryDatasets(ctx, attachs) +} + +func QueryPrivateDataset(ctx *context.Context) { + username := ctx.Params(":username") + attachs, err := models.GetPrivateAttachments(username) + if err != nil { + ctx.JSON(200, map[string]string{ + "result_code": "-1", + "error_msg": err.Error(), + "data": "", + }) + return + } + + for _, attach := range attachs { + attach.Name = username + } + + queryDatasets(ctx, attachs) +} + +func queryDatasets(ctx *context.Context, attachs []*models.AttachmentUsername) { + var datasets []CloudBrainDataset + if len(attachs) == 0 { + log.Info("dataset is null") + ctx.JSON(200, map[string]string{ + "result_code": "0", + "error_msg": "", + "data": "", + }) + return + } + + for _, attch := range attachs { + has, err := storage.Attachments.HasObject(models.AttachmentRelativePath(attch.UUID)) + if err != nil || !has { + continue + } + + datasets = append(datasets, CloudBrainDataset{strconv.FormatInt(attch.ID, 10), + attch.Attachment.Name, + setting.Attachment.Minio.RealPath + + setting.Attachment.Minio.Bucket + "/" + + setting.Attachment.Minio.BasePath + + models.AttachmentRelativePath(attch.UUID) + + attch.UUID, + attch.Name, + attch.CreatedUnix.Format("2006-01-02 03:04:05 PM")}) + } + + data, err := json.Marshal(datasets) + if err != nil { + log.Error("json.Marshal failed:", err.Error()) + ctx.JSON(200, map[string]string{ + "result_code": "-1", + "error_msg": err.Error(), + "data": "", + }) + return + } + + ctx.JSON(200, map[string]string{ + "result_code": "0", + "error_msg": "", + "data": string(data), + }) + return +} + +func checkTypeCloudBrain(typeCloudBrain int) error { + if typeCloudBrain != models.TypeCloudBrainOne && typeCloudBrain != models.TypeCloudBrainTwo { + log.Error("type error:", typeCloudBrain) + return errors.New("type error") + } + return nil +} diff --git a/routers/repo/dataset.go b/routers/repo/dataset.go index 5a59e899e..65ae5abb2 100755 --- a/routers/repo/dataset.go +++ b/routers/repo/dataset.go @@ -110,6 +110,8 @@ func DatasetIndex(ctx *context.Context) { repo := ctx.Repo.Repository dataset, err := models.GetDatasetByRepo(repo) + ctx.Data["CanRead"]=ctx.Repo.CanRead(models.UnitTypeDatasets) + ctx.Data["CanWrite"]=ctx.Repo.CanWrite(models.UnitTypeDatasets) if err != nil { log.Warn("query dataset, not found.") ctx.HTML(200, tplIndex) diff --git a/routers/repo/dataset.go.bak b/routers/repo/dataset.go.bak new file mode 100644 index 000000000..5a59e899e --- /dev/null +++ b/routers/repo/dataset.go.bak @@ -0,0 +1,464 @@ +package repo + +import ( + "encoding/json" + "net/http" + "regexp" + "sort" + "strconv" + "strings" + "unicode/utf8" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/auth" + "code.gitea.io/gitea/modules/base" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" +) + +const ( + tplIndex base.TplName = "repo/datasets/index" + tplDatasetCreate base.TplName = "repo/datasets/create" + tplDatasetEdit base.TplName = "repo/datasets/edit" + taskstplIndex base.TplName = "repo/datasets/tasks/index" +) + +var titlePattern = regexp.MustCompile(`^[A-Za-z0-9-_\\.]{1,100}$`) + +// MustEnableDataset check if repository enable internal dataset +func MustEnableDataset(ctx *context.Context) { + if !ctx.Repo.CanRead(models.UnitTypeDatasets) { + ctx.NotFound("MustEnableDataset", nil) + return + } +} + +func newFilterPrivateAttachments(ctx *context.Context, list []*models.Attachment, repo *models.Repository) []*models.Attachment { + + if ctx.Repo.CanWrite(models.UnitTypeDatasets) { + log.Info("can write.") + return list + } else { + if repo.Owner == nil { + repo.GetOwner() + } + permission := false + if repo.Owner.IsOrganization() && ctx.User != nil { + if repo.Owner.IsUserPartOfOrg(ctx.User.ID) { + log.Info("user is member of org.") + permission = true + } + } + if !permission && ctx.User != nil { + isCollaborator, _ := repo.IsCollaborator(ctx.User.ID) + if isCollaborator { + log.Info("Collaborator user may visit the attach.") + permission = true + } + } + + var publicList []*models.Attachment + for _, attach := range list { + if !attach.IsPrivate { + publicList = append(publicList, attach) + } else { + if permission { + publicList = append(publicList, attach) + } + } + } + return publicList + } +} + +func QueryDataSet(ctx *context.Context) []*models.Attachment { + repo := ctx.Repo.Repository + + dataset, err := models.GetDatasetByRepo(repo) + if err != nil { + log.Error("zou not found dataset 1") + ctx.NotFound("GetDatasetByRepo", err) + return nil + } + + if ctx.Query("type") == "" { + log.Error("zou not found type 2") + ctx.NotFound("type error", nil) + return nil + } + err = models.GetDatasetAttachments(ctx.QueryInt("type"), ctx.IsSigned, ctx.User, dataset) + if err != nil { + ctx.ServerError("GetDatasetAttachments", err) + return nil + } + attachments := newFilterPrivateAttachments(ctx, dataset.Attachments, repo) + + ctx.Data["SortType"] = ctx.Query("sort") + + sort.Slice(attachments, func(i, j int) bool { + return attachments[i].CreatedUnix > attachments[j].CreatedUnix + }) + + return attachments +} + +func DatasetIndex(ctx *context.Context) { + log.Info("dataset index 1") + MustEnableDataset(ctx) + + repo := ctx.Repo.Repository + + dataset, err := models.GetDatasetByRepo(repo) + if err != nil { + log.Warn("query dataset, not found.") + ctx.HTML(200, tplIndex) + return + } + cloudbrainType := -1 + if ctx.Query("type") != "" { + + cloudbrainType = ctx.QueryInt("type") + } + err = models.GetDatasetAttachments(cloudbrainType, ctx.IsSigned, ctx.User, dataset) + if err != nil { + ctx.ServerError("GetDatasetAttachments", err) + return + } + + attachments := newFilterPrivateAttachments(ctx, dataset.Attachments, repo) + + sort.Slice(attachments, func(i, j int) bool { + return attachments[i].CreatedUnix > attachments[j].CreatedUnix + }) + + page := ctx.QueryInt("page") + if page <= 0 { + page = 1 + } + pagesize := ctx.QueryInt("pagesize") + if pagesize <= 0 { + pagesize = 10 + } + pager := context.NewPagination(len(attachments), pagesize, page, 5) + + pageAttachments := getPageAttachments(attachments, page, pagesize) + + //load attachment creator + for _, attachment := range pageAttachments { + uploader, _ := models.GetUserByID(attachment.UploaderID) + attachment.Uploader = uploader + } + + ctx.Data["Page"] = pager + ctx.Data["PageIsDataset"] = true + ctx.Data["Title"] = ctx.Tr("dataset.show_dataset") + ctx.Data["Link"] = ctx.Repo.RepoLink + "/datasets" + ctx.Data["dataset"] = dataset + ctx.Data["Attachments"] = pageAttachments + ctx.Data["IsOwner"] = true + ctx.Data["StoreType"] = setting.Attachment.StoreType + ctx.Data["Type"] = cloudbrainType + + renderAttachmentSettings(ctx) + + ctx.HTML(200, tplIndex) +} + +func getPageAttachments(attachments []*models.Attachment, page int, pagesize int) []*models.Attachment { + begin := (page - 1) * pagesize + end := (page) * pagesize + + if begin > len(attachments)-1 { + return nil + } + if end > len(attachments)-1 { + return attachments[begin:] + } else { + return attachments[begin:end] + } + +} + +func CreateDataset(ctx *context.Context) { + + MustEnableDataset(ctx) + + ctx.HTML(200, tplDatasetCreate) +} + +func EditDataset(ctx *context.Context) { + + MustEnableDataset(ctx) + datasetId, _ := strconv.ParseInt(ctx.Params(":id"), 10, 64) + + dataset, _ := models.GetDatasetByID(datasetId) + if dataset == nil { + ctx.Error(http.StatusNotFound, "") + return + } + ctx.Data["Dataset"] = dataset + + ctx.HTML(200, tplDatasetEdit) +} + +func CreateDatasetPost(ctx *context.Context, form auth.CreateDatasetForm) { + + dataset := &models.Dataset{} + + if !titlePattern.MatchString(form.Title) { + ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("dataset.title_format_err"))) + return + } + if utf8.RuneCountInString(form.Description) > 1024 { + ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("dataset.description_format_err"))) + return + } + + dataset.RepoID = ctx.Repo.Repository.ID + dataset.UserID = ctx.User.ID + dataset.Category = form.Category + dataset.Task = form.Task + dataset.Title = form.Title + dataset.License = form.License + dataset.Description = form.Description + dataset.DownloadTimes = 0 + if ctx.Repo.Repository.IsPrivate { + dataset.Status = 0 + } else { + dataset.Status = 1 + } + err := models.CreateDataset(dataset) + if err != nil { + log.Error("fail to create dataset", err) + ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("dataset.create_dataset_fail"))) + } else { + ctx.JSON(http.StatusOK, models.BaseOKMessage) + } + +} + +func EditDatasetPost(ctx *context.Context, form auth.EditDatasetForm) { + ctx.Data["PageIsDataset"] = true + + ctx.Data["Title"] = ctx.Tr("dataset.edit_dataset") + + if !titlePattern.MatchString(form.Title) { + ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("dataset.title_format_err"))) + return + } + if utf8.RuneCountInString(form.Description) > 1024 { + ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("dataset.description_format_err"))) + return + } + + rel, err := models.GetDatasetByID(form.ID) + ctx.Data["dataset"] = rel + + if err != nil { + log.Error("failed to query dataset", err) + ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("dataset.query_dataset_fail"))) + return + } + + rel.Title = form.Title + rel.Description = form.Description + rel.Category = form.Category + rel.Task = form.Task + rel.License = form.License + if err = models.UpdateDataset(models.DefaultDBContext(), rel); err != nil { + ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("dataset.query_dataset_fail"))) + } + ctx.JSON(http.StatusOK, models.BaseOKMessage) +} + +func DatasetAction(ctx *context.Context) { + var err error + datasetId, _ := strconv.ParseInt(ctx.Params(":id"), 10, 64) + switch ctx.Params(":action") { + case "star": + err = models.StarDataset(ctx.User.ID, datasetId, true) + case "unstar": + err = models.StarDataset(ctx.User.ID, datasetId, false) + + } + if err != nil { + ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("repo.star_fail", ctx.Params(":action")))) + } else { + ctx.JSON(http.StatusOK, models.BaseOKMessage) + } + +} + +func CurrentRepoDataset(ctx *context.Context) { + page := ctx.QueryInt("page") + cloudbrainType := ctx.QueryInt("type") + keyword := strings.Trim(ctx.Query("q"), " ") + + repo := ctx.Repo.Repository + var datasetIDs []int64 + dataset, err := models.GetDatasetByRepo(repo) + if err != nil { + ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("GetDatasetByRepo failed", err))) + } + datasetIDs = append(datasetIDs, dataset.ID) + uploaderID := ctx.User.ID + datasets, count, err := models.Attachments(&models.AttachmentsOptions{ + ListOptions: models.ListOptions{ + Page: page, + PageSize: setting.UI.IssuePagingNum, + }, + Keyword: keyword, + DatasetIDs: datasetIDs, + UploaderID: uploaderID, + Type: cloudbrainType, + NeedIsPrivate: false, + NeedRepoInfo: true, + }) + if err != nil { + ctx.ServerError("datasets", err) + return + } + + data, err := json.Marshal(datasets) + if err != nil { + log.Error("json.Marshal failed:", err.Error()) + ctx.JSON(200, map[string]string{ + "result_code": "-1", + "error_msg": err.Error(), + "data": "", + }) + return + } + ctx.JSON(200, map[string]string{ + "result_code": "0", + "data": string(data), + "count": strconv.FormatInt(count, 10), + }) +} + +func MyDatasets(ctx *context.Context) { + page := ctx.QueryInt("page") + cloudbrainType := ctx.QueryInt("type") + keyword := strings.Trim(ctx.Query("q"), " ") + + uploaderID := ctx.User.ID + datasets, count, err := models.Attachments(&models.AttachmentsOptions{ + ListOptions: models.ListOptions{ + Page: page, + PageSize: setting.UI.IssuePagingNum, + }, + Keyword: keyword, + UploaderID: uploaderID, + Type: cloudbrainType, + NeedIsPrivate: false, + NeedRepoInfo: true, + }) + if err != nil { + ctx.ServerError("datasets", err) + return + } + + data, err := json.Marshal(datasets) + if err != nil { + log.Error("json.Marshal failed:", err.Error()) + ctx.JSON(200, map[string]string{ + "result_code": "-1", + "error_msg": err.Error(), + "data": "", + }) + return + } + ctx.JSON(200, map[string]string{ + "result_code": "0", + "data": string(data), + "count": strconv.FormatInt(count, 10), + }) +} + +func PublicDataset(ctx *context.Context) { + page := ctx.QueryInt("page") + cloudbrainType := ctx.QueryInt("type") + keyword := strings.Trim(ctx.Query("q"), " ") + + datasets, count, err := models.Attachments(&models.AttachmentsOptions{ + ListOptions: models.ListOptions{ + Page: page, + PageSize: setting.UI.IssuePagingNum, + }, + Keyword: keyword, + NeedIsPrivate: true, + IsPrivate: false, + Type: cloudbrainType, + NeedRepoInfo: true, + }) + if err != nil { + ctx.ServerError("datasets", err) + return + } + + data, err := json.Marshal(datasets) + if err != nil { + log.Error("json.Marshal failed:", err.Error()) + ctx.JSON(200, map[string]string{ + "result_code": "-1", + "error_msg": err.Error(), + "data": "", + }) + return + } + ctx.JSON(200, map[string]string{ + "result_code": "0", + "data": string(data), + "count": strconv.FormatInt(count, 10), + }) +} + +func MyFavoriteDataset(ctx *context.Context) { + page := ctx.QueryInt("page") + cloudbrainType := ctx.QueryInt("type") + keyword := strings.Trim(ctx.Query("q"), " ") + + var datasetIDs []int64 + + datasetStars, err := models.GetDatasetStarByUser(ctx.User) + if err != nil { + ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("GetDatasetByRepo failed", err))) + } + for i, _ := range datasetStars { + datasetIDs = append(datasetIDs, datasetStars[i].DatasetID) + } + + datasets, count, err := models.Attachments(&models.AttachmentsOptions{ + ListOptions: models.ListOptions{ + Page: page, + PageSize: setting.UI.IssuePagingNum, + }, + Keyword: keyword, + DatasetIDs: datasetIDs, + NeedIsPrivate: true, + IsPrivate: false, + Type: cloudbrainType, + NeedRepoInfo: true, + }) + if err != nil { + ctx.ServerError("datasets", err) + return + } + + data, err := json.Marshal(datasets) + if err != nil { + log.Error("json.Marshal failed:", err.Error()) + ctx.JSON(200, map[string]string{ + "result_code": "-1", + "error_msg": err.Error(), + "data": "", + }) + return + } + ctx.JSON(200, map[string]string{ + "result_code": "0", + "data": string(data), + "count": strconv.FormatInt(count, 10), + }) +}