You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.

attachment.go 22 kB

4 years ago
4 years ago
4 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
4 years ago
4 years ago
5 years ago
4 years ago
5 years ago
4 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
5 years ago
4 years ago
5 years ago
5 years ago
4 years ago
4 years ago
4 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
5 years ago
5 years ago
4 years ago
5 years ago
4 years ago
5 years ago
4 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
4 years ago
5 years ago
5 years ago
5 years ago
4 years ago
4 years ago
4 years ago
5 years ago
4 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
4 years ago
5 years ago
4 years ago
5 years ago
5 years ago
5 years ago
4 years ago
5 years ago
4 years ago
4 years ago
5 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
5 years ago
4 years ago
4 years ago
5 years ago
4 years ago
4 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
5 years ago
5 years ago
4 years ago
5 years ago
4 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
4 years ago
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855
  1. // Copyright 2017 The Gitea Authors. All rights reserved.
  2. // Use of this source code is governed by a MIT-style
  3. // license that can be found in the LICENSE file.
  4. package repo
  5. import (
  6. contexExt "context"
  7. "encoding/json"
  8. "errors"
  9. "fmt"
  10. "mime/multipart"
  11. "net/http"
  12. "path"
  13. "strconv"
  14. "strings"
  15. "code.gitea.io/gitea/models"
  16. "code.gitea.io/gitea/modules/context"
  17. "code.gitea.io/gitea/modules/log"
  18. "code.gitea.io/gitea/modules/minio_ext"
  19. "code.gitea.io/gitea/modules/setting"
  20. "code.gitea.io/gitea/modules/storage"
  21. "code.gitea.io/gitea/modules/upload"
  22. "code.gitea.io/gitea/modules/worker"
  23. gouuid "github.com/satori/go.uuid"
  24. )
  25. const (
  26. //result of decompress
  27. DecompressSuccess = "0"
  28. DecompressFailed = "1"
  29. )
  30. type CloudBrainDataset struct {
  31. UUID string `json:"id"`
  32. Name string `json:"name"`
  33. Path string `json:"place"`
  34. UserName string `json:"provider"`
  35. CreateTime string `json:"created_at"`
  36. }
  37. type UploadForm struct {
  38. UploadID string `form:"uploadId"`
  39. UuID string `form:"uuid"`
  40. PartSize int64 `form:"size"`
  41. Offset int64 `form:"offset"`
  42. PartNumber int `form:"chunkNumber"`
  43. PartFile multipart.File `form:"file"`
  44. }
  45. func RenderAttachmentSettings(ctx *context.Context) {
  46. renderAttachmentSettings(ctx)
  47. }
  48. func renderAttachmentSettings(ctx *context.Context) {
  49. ctx.Data["IsAttachmentEnabled"] = setting.Attachment.Enabled
  50. ctx.Data["AttachmentStoreType"] = setting.Attachment.StoreType
  51. ctx.Data["AttachmentAllowedTypes"] = setting.Attachment.AllowedTypes
  52. ctx.Data["AttachmentMaxSize"] = setting.Attachment.MaxSize
  53. ctx.Data["AttachmentMaxFiles"] = setting.Attachment.MaxFiles
  54. }
  55. // UploadAttachment response for uploading issue's attachment
  56. func UploadAttachment(ctx *context.Context) {
  57. if !setting.Attachment.Enabled {
  58. ctx.Error(404, "attachment is not enabled")
  59. return
  60. }
  61. file, header, err := ctx.Req.FormFile("file")
  62. if err != nil {
  63. ctx.Error(500, fmt.Sprintf("FormFile: %v", err))
  64. return
  65. }
  66. defer file.Close()
  67. buf := make([]byte, 1024)
  68. n, _ := file.Read(buf)
  69. if n > 0 {
  70. buf = buf[:n]
  71. }
  72. err = upload.VerifyAllowedContentType(buf, strings.Split(setting.Attachment.AllowedTypes, ","))
  73. if err != nil {
  74. ctx.Error(400, err.Error())
  75. return
  76. }
  77. datasetID, _ := strconv.ParseInt(ctx.Req.FormValue("dataset_id"), 10, 64)
  78. attach, err := models.NewAttachment(&models.Attachment{
  79. IsPrivate: true,
  80. UploaderID: ctx.User.ID,
  81. Name: header.Filename,
  82. DatasetID: datasetID,
  83. }, buf, file)
  84. if err != nil {
  85. ctx.Error(500, fmt.Sprintf("NewAttachment: %v", err))
  86. return
  87. }
  88. log.Trace("New attachment uploaded: %s", attach.UUID)
  89. ctx.JSON(200, map[string]string{
  90. "uuid": attach.UUID,
  91. })
  92. }
  93. func UpdatePublicAttachment(ctx *context.Context) {
  94. file := ctx.Query("file")
  95. isPrivate, _ := strconv.ParseBool(ctx.Query("is_private"))
  96. attach, err := models.GetAttachmentByUUID(file)
  97. if err != nil {
  98. ctx.Error(404, err.Error())
  99. return
  100. }
  101. attach.IsPrivate = isPrivate
  102. models.UpdateAttachment(attach)
  103. }
  104. // DeleteAttachment response for deleting issue's attachment
  105. func DeleteAttachment(ctx *context.Context) {
  106. file := ctx.Query("file")
  107. attach, err := models.GetAttachmentByUUID(file)
  108. if err != nil {
  109. ctx.Error(400, err.Error())
  110. return
  111. }
  112. if !ctx.IsSigned || (ctx.User.ID != attach.UploaderID) {
  113. ctx.Error(403)
  114. return
  115. }
  116. err = models.DeleteAttachment(attach, false)
  117. if err != nil {
  118. ctx.Error(500, fmt.Sprintf("DeleteAttachment: %v", err))
  119. return
  120. }
  121. ctx.JSON(200, map[string]string{
  122. "uuid": attach.UUID,
  123. })
  124. }
  125. // GetAttachment serve attachements
  126. func GetAttachment(ctx *context.Context) {
  127. typeCloudBrain := ctx.QueryInt("type")
  128. err := checkTypeCloudBrain(typeCloudBrain)
  129. if err != nil {
  130. ctx.ServerError("checkTypeCloudBrain failed", err)
  131. return
  132. }
  133. attach, err := models.GetAttachmentByUUID(ctx.Params(":uuid"))
  134. if err != nil {
  135. if models.IsErrAttachmentNotExist(err) {
  136. ctx.Error(404)
  137. } else {
  138. ctx.ServerError("GetAttachmentByUUID", err)
  139. }
  140. return
  141. }
  142. repository, unitType, err := attach.LinkedRepository()
  143. if err != nil {
  144. ctx.ServerError("LinkedRepository", err)
  145. return
  146. }
  147. if repository == nil { //If not linked
  148. if !(ctx.IsSigned && attach.UploaderID == ctx.User.ID) && attach.IsPrivate { //We block if not the uploader
  149. ctx.Error(http.StatusNotFound)
  150. return
  151. }
  152. } else { //If we have the repository we check access
  153. perm, err := models.GetUserRepoPermission(repository, ctx.User)
  154. if err != nil {
  155. ctx.Error(http.StatusInternalServerError, "GetUserRepoPermission", err.Error())
  156. return
  157. }
  158. if !perm.CanRead(unitType) {
  159. ctx.Error(http.StatusNotFound)
  160. return
  161. }
  162. }
  163. dataSet, err := attach.LinkedDataSet()
  164. if err != nil {
  165. ctx.ServerError("LinkedDataSet", err)
  166. return
  167. }
  168. if dataSet != nil {
  169. isPermit, err := models.GetUserDataSetPermission(dataSet, ctx.User)
  170. if err != nil {
  171. ctx.Error(http.StatusInternalServerError, "GetUserDataSetPermission", err.Error())
  172. return
  173. }
  174. if !isPermit {
  175. ctx.Error(http.StatusNotFound)
  176. return
  177. }
  178. }
  179. //If we have matched and access to release or issue
  180. if setting.Attachment.StoreType == storage.MinioStorageType {
  181. url := ""
  182. if typeCloudBrain == models.TypeCloudBrainOne {
  183. url, err = storage.Attachments.PresignedGetURL(attach.RelativePath(), attach.Name)
  184. if err != nil {
  185. ctx.ServerError("PresignedGetURL", err)
  186. return
  187. }
  188. } else {
  189. url, err = storage.ObsGetPreSignedUrl(attach.UUID, attach.Name)
  190. if err != nil {
  191. ctx.ServerError("ObsGetPreSignedUrl", err)
  192. return
  193. }
  194. }
  195. if err = increaseDownloadCount(attach, dataSet); err != nil {
  196. ctx.ServerError("Update", err)
  197. return
  198. }
  199. http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently)
  200. } else {
  201. fr, err := storage.Attachments.Open(attach.RelativePath())
  202. if err != nil {
  203. ctx.ServerError("Open", err)
  204. return
  205. }
  206. defer fr.Close()
  207. if err = increaseDownloadCount(attach, dataSet); err != nil {
  208. ctx.ServerError("Update", err)
  209. return
  210. }
  211. if err = ServeData(ctx, attach.Name, fr); err != nil {
  212. ctx.ServerError("ServeData", err)
  213. return
  214. }
  215. }
  216. }
  217. func increaseDownloadCount(attach *models.Attachment, dataSet *models.Dataset) error {
  218. if err := attach.IncreaseDownloadCount(); err != nil {
  219. return err
  220. }
  221. if dataSet != nil {
  222. if err := models.IncreaseDownloadCount(dataSet.ID); err != nil {
  223. return err
  224. }
  225. }
  226. return nil
  227. }
  228. // Get a presigned url for put object
  229. func GetPresignedPutObjectURL(ctx *context.Context) {
  230. if !setting.Attachment.Enabled {
  231. ctx.Error(404, "attachment is not enabled")
  232. return
  233. }
  234. err := upload.VerifyFileType(ctx.Params("file_type"), strings.Split(setting.Attachment.AllowedTypes, ","))
  235. if err != nil {
  236. ctx.Error(400, err.Error())
  237. return
  238. }
  239. if setting.Attachment.StoreType == storage.MinioStorageType {
  240. uuid := gouuid.NewV4().String()
  241. url, err := storage.Attachments.PresignedPutURL(models.AttachmentRelativePath(uuid))
  242. if err != nil {
  243. ctx.ServerError("PresignedPutURL", err)
  244. return
  245. }
  246. ctx.JSON(200, map[string]string{
  247. "uuid": uuid,
  248. "url": url,
  249. })
  250. } else {
  251. ctx.Error(404, "storage type is not enabled")
  252. return
  253. }
  254. }
  255. // AddAttachment response for add attachment record
  256. func AddAttachment(ctx *context.Context) {
  257. typeCloudBrain := ctx.QueryInt("type")
  258. err := checkTypeCloudBrain(typeCloudBrain)
  259. if err != nil {
  260. ctx.ServerError("checkTypeCloudBrain failed", err)
  261. return
  262. }
  263. uuid := ctx.Query("uuid")
  264. has := false
  265. if typeCloudBrain == models.TypeCloudBrainOne {
  266. has, err = storage.Attachments.HasObject(models.AttachmentRelativePath(uuid))
  267. if err != nil {
  268. ctx.ServerError("HasObject", err)
  269. return
  270. }
  271. } else {
  272. has, err = storage.ObsHasObject(setting.BasePath + models.AttachmentRelativePath(uuid) + "/" + uuid)
  273. if err != nil {
  274. ctx.ServerError("ObsHasObject", err)
  275. return
  276. }
  277. }
  278. if !has {
  279. ctx.Error(404, "attachment has not been uploaded")
  280. return
  281. }
  282. attachment, err := models.InsertAttachment(&models.Attachment{
  283. UUID: uuid,
  284. UploaderID: ctx.User.ID,
  285. IsPrivate: true,
  286. Name: ctx.Query("file_name"),
  287. Size: ctx.QueryInt64("size"),
  288. DatasetID: ctx.QueryInt64("dataset_id"),
  289. Type: typeCloudBrain,
  290. })
  291. if err != nil {
  292. ctx.Error(500, fmt.Sprintf("InsertAttachment: %v", err))
  293. return
  294. }
  295. if attachment.DatasetID != 0 {
  296. if strings.HasSuffix(attachment.Name, ".zip") {
  297. if typeCloudBrain == models.TypeCloudBrainOne {
  298. err = worker.SendDecompressTask(contexExt.Background(), uuid)
  299. if err != nil {
  300. log.Error("SendDecompressTask(%s) failed:%s", uuid, err.Error())
  301. } else {
  302. attachment.DecompressState = models.DecompressStateIng
  303. err = models.UpdateAttachment(attachment)
  304. if err != nil {
  305. log.Error("UpdateAttachment state(%s) failed:%s", uuid, err.Error())
  306. }
  307. }
  308. }
  309. //todo:decompress type_two
  310. }
  311. }
  312. ctx.JSON(200, map[string]string{
  313. "result_code": "0",
  314. })
  315. }
  316. func UpdateAttachmentDecompressState(ctx *context.Context) {
  317. uuid := ctx.Query("uuid")
  318. result := ctx.Query("result")
  319. attach, err := models.GetAttachmentByUUID(uuid)
  320. if err != nil {
  321. log.Error("GetAttachmentByUUID(%s) failed:%s", uuid, err.Error())
  322. return
  323. }
  324. if result == DecompressSuccess {
  325. attach.DecompressState = models.DecompressStateDone
  326. } else if result == DecompressFailed {
  327. attach.DecompressState = models.DecompressStateFailed
  328. } else {
  329. log.Error("result is error:", result)
  330. return
  331. }
  332. err = models.UpdateAttachment(attach)
  333. if err != nil {
  334. log.Error("UpdateAttachment(%s) failed:%s", uuid, err.Error())
  335. return
  336. }
  337. ctx.JSON(200, map[string]string{
  338. "result_code": "0",
  339. })
  340. }
  341. func GetSuccessChunks(ctx *context.Context) {
  342. fileMD5 := ctx.Query("md5")
  343. typeCloudBrain := ctx.QueryInt("type")
  344. var chunks string
  345. err := checkTypeCloudBrain(typeCloudBrain)
  346. if err != nil {
  347. ctx.ServerError("checkTypeCloudBrain failed", err)
  348. return
  349. }
  350. fileChunk, err := models.GetFileChunkByMD5AndUser(fileMD5, ctx.User.ID, typeCloudBrain)
  351. if err != nil {
  352. if models.IsErrFileChunkNotExist(err) {
  353. ctx.JSON(200, map[string]string{
  354. "uuid": "",
  355. "uploaded": "0",
  356. "uploadID": "",
  357. "chunks": "",
  358. })
  359. } else {
  360. ctx.ServerError("GetFileChunkByMD5", err)
  361. }
  362. return
  363. }
  364. isExist := false
  365. if typeCloudBrain == models.TypeCloudBrainOne {
  366. isExist, err = storage.Attachments.HasObject(models.AttachmentRelativePath(fileChunk.UUID))
  367. if err != nil {
  368. ctx.ServerError("HasObject failed", err)
  369. return
  370. }
  371. } else {
  372. isExist, err = storage.ObsHasObject(setting.BasePath + models.AttachmentRelativePath(fileChunk.UUID) + "/" + fileChunk.UUID)
  373. if err != nil {
  374. ctx.ServerError("ObsHasObject failed", err)
  375. return
  376. }
  377. }
  378. if isExist {
  379. if fileChunk.IsUploaded == models.FileNotUploaded {
  380. log.Info("the file has been uploaded but not recorded")
  381. fileChunk.IsUploaded = models.FileUploaded
  382. if err = models.UpdateFileChunk(fileChunk); err != nil {
  383. log.Error("UpdateFileChunk failed:", err.Error())
  384. }
  385. }
  386. } else {
  387. if fileChunk.IsUploaded == models.FileUploaded {
  388. log.Info("the file has been recorded but not uploaded")
  389. fileChunk.IsUploaded = models.FileNotUploaded
  390. if err = models.UpdateFileChunk(fileChunk); err != nil {
  391. log.Error("UpdateFileChunk failed:", err.Error())
  392. }
  393. }
  394. if typeCloudBrain == models.TypeCloudBrainOne {
  395. chunks, err = storage.GetPartInfos(fileChunk.UUID, fileChunk.UploadID)
  396. if err != nil {
  397. ctx.ServerError("GetPartInfos failed", err)
  398. return
  399. }
  400. } else {
  401. chunks, err = storage.GetObsPartInfos(fileChunk.UUID, fileChunk.UploadID)
  402. if err != nil {
  403. ctx.ServerError("GetObsPartInfos failed", err)
  404. return
  405. }
  406. }
  407. }
  408. var attachID int64
  409. attach, err := models.GetAttachmentByUUID(fileChunk.UUID)
  410. if err != nil {
  411. if models.IsErrAttachmentNotExist(err) {
  412. attachID = 0
  413. } else {
  414. ctx.ServerError("GetAttachmentByUUID", err)
  415. return
  416. }
  417. } else {
  418. attachID = attach.ID
  419. }
  420. if attach == nil {
  421. ctx.JSON(200, map[string]string{
  422. "uuid": fileChunk.UUID,
  423. "uploaded": strconv.Itoa(fileChunk.IsUploaded),
  424. "uploadID": fileChunk.UploadID,
  425. "chunks": string(chunks),
  426. "attachID": "0",
  427. "datasetID": "0",
  428. "fileName": "",
  429. "datasetName": "",
  430. })
  431. return
  432. }
  433. dataset, err := models.GetDatasetByID(attach.DatasetID)
  434. if err != nil {
  435. ctx.ServerError("GetDatasetByID", err)
  436. return
  437. }
  438. ctx.JSON(200, map[string]string{
  439. "uuid": fileChunk.UUID,
  440. "uploaded": strconv.Itoa(fileChunk.IsUploaded),
  441. "uploadID": fileChunk.UploadID,
  442. "chunks": string(chunks),
  443. "attachID": strconv.Itoa(int(attachID)),
  444. "datasetID": strconv.Itoa(int(attach.DatasetID)),
  445. "fileName": attach.Name,
  446. "datasetName": dataset.Title,
  447. })
  448. }
  449. func NewMultipart(ctx *context.Context) {
  450. if !setting.Attachment.Enabled {
  451. ctx.Error(404, "attachment is not enabled")
  452. return
  453. }
  454. err := upload.VerifyFileType(ctx.Query("fileType"), strings.Split(setting.Attachment.AllowedTypes, ","))
  455. if err != nil {
  456. ctx.Error(400, err.Error())
  457. return
  458. }
  459. typeCloudBrain := ctx.QueryInt("type")
  460. err = checkTypeCloudBrain(typeCloudBrain)
  461. if err != nil {
  462. ctx.ServerError("checkTypeCloudBrain failed", err)
  463. return
  464. }
  465. fileName := ctx.Query("file_name")
  466. if setting.Attachment.StoreType == storage.MinioStorageType {
  467. totalChunkCounts := ctx.QueryInt("totalChunkCounts")
  468. if totalChunkCounts > minio_ext.MaxPartsCount {
  469. ctx.Error(400, fmt.Sprintf("chunk counts(%d) is too much", totalChunkCounts))
  470. return
  471. }
  472. fileSize := ctx.QueryInt64("size")
  473. if fileSize > minio_ext.MaxMultipartPutObjectSize {
  474. ctx.Error(400, fmt.Sprintf("file size(%d) is too big", fileSize))
  475. return
  476. }
  477. uuid := gouuid.NewV4().String()
  478. var uploadID string
  479. if typeCloudBrain == models.TypeCloudBrainOne {
  480. uploadID, err = storage.NewMultiPartUpload(uuid)
  481. if err != nil {
  482. ctx.ServerError("NewMultipart", err)
  483. return
  484. }
  485. } else {
  486. uploadID, err = storage.NewObsMultiPartUpload(uuid, fileName)
  487. if err != nil {
  488. ctx.ServerError("NewObsMultiPartUpload", err)
  489. return
  490. }
  491. }
  492. _, err = models.InsertFileChunk(&models.FileChunk{
  493. UUID: uuid,
  494. UserID: ctx.User.ID,
  495. UploadID: uploadID,
  496. Md5: ctx.Query("md5"),
  497. Size: fileSize,
  498. TotalChunks: totalChunkCounts,
  499. Type: typeCloudBrain,
  500. })
  501. if err != nil {
  502. ctx.Error(500, fmt.Sprintf("InsertFileChunk: %v", err))
  503. return
  504. }
  505. ctx.JSON(200, map[string]string{
  506. "uuid": uuid,
  507. "uploadID": uploadID,
  508. })
  509. } else {
  510. ctx.Error(404, "storage type is not enabled")
  511. return
  512. }
  513. }
  514. func GetMultipartUploadUrl(ctx *context.Context) {
  515. uuid := ctx.Query("uuid")
  516. uploadID := ctx.Query("uploadID")
  517. partNumber := ctx.QueryInt("chunkNumber")
  518. size := ctx.QueryInt64("size")
  519. fileName := ctx.Query("file_name")
  520. typeCloudBrain := ctx.QueryInt("type")
  521. err := checkTypeCloudBrain(typeCloudBrain)
  522. if err != nil {
  523. ctx.ServerError("checkTypeCloudBrain failed", err)
  524. return
  525. }
  526. url := ""
  527. if typeCloudBrain == models.TypeCloudBrainOne {
  528. if size > minio_ext.MinPartSize {
  529. ctx.Error(400, fmt.Sprintf("chunk size(%d) is too big", size))
  530. return
  531. }
  532. url, err = storage.GenMultiPartSignedUrl(uuid, uploadID, partNumber, size)
  533. if err != nil {
  534. ctx.Error(500, fmt.Sprintf("GenMultiPartSignedUrl failed: %v", err))
  535. return
  536. }
  537. } else {
  538. url, err = storage.ObsGenMultiPartSignedUrl(uuid, uploadID, partNumber, fileName)
  539. if err != nil {
  540. ctx.Error(500, fmt.Sprintf("ObsGenMultiPartSignedUrl failed: %v", err))
  541. return
  542. }
  543. }
  544. ctx.JSON(200, map[string]string{
  545. "url": url,
  546. })
  547. }
  548. func GetObsKey(ctx *context.Context) {
  549. uuid := gouuid.NewV4().String()
  550. key := strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, uuid)), "/")
  551. ctx.JSON(200, map[string]string{
  552. "uuid": uuid,
  553. "key": key,
  554. "access_key_id": setting.AccessKeyID,
  555. "secret_access_key": setting.SecretAccessKey,
  556. "server": setting.Endpoint,
  557. "bucket": setting.Bucket,
  558. })
  559. }
  560. func CompleteMultipart(ctx *context.Context) {
  561. uuid := ctx.Query("uuid")
  562. uploadID := ctx.Query("uploadID")
  563. typeCloudBrain := ctx.QueryInt("type")
  564. fileName := ctx.Query("file_name")
  565. err := checkTypeCloudBrain(typeCloudBrain)
  566. if err != nil {
  567. ctx.ServerError("checkTypeCloudBrain failed", err)
  568. return
  569. }
  570. fileChunk, err := models.GetFileChunkByUUID(uuid)
  571. if err != nil {
  572. if models.IsErrFileChunkNotExist(err) {
  573. ctx.Error(404)
  574. } else {
  575. ctx.ServerError("GetFileChunkByUUID", err)
  576. }
  577. return
  578. }
  579. if typeCloudBrain == models.TypeCloudBrainOne {
  580. _, err = storage.CompleteMultiPartUpload(uuid, uploadID)
  581. if err != nil {
  582. ctx.Error(500, fmt.Sprintf("CompleteMultiPartUpload failed: %v", err))
  583. return
  584. }
  585. } else {
  586. err = storage.CompleteObsMultiPartUpload(uuid, uploadID, fileName)
  587. if err != nil {
  588. ctx.Error(500, fmt.Sprintf("CompleteObsMultiPartUpload failed: %v", err))
  589. return
  590. }
  591. }
  592. fileChunk.IsUploaded = models.FileUploaded
  593. err = models.UpdateFileChunk(fileChunk)
  594. if err != nil {
  595. ctx.Error(500, fmt.Sprintf("UpdateFileChunk: %v", err))
  596. return
  597. }
  598. attachment, err := models.InsertAttachment(&models.Attachment{
  599. UUID: uuid,
  600. UploaderID: ctx.User.ID,
  601. IsPrivate: true,
  602. Name: fileName,
  603. Size: ctx.QueryInt64("size"),
  604. DatasetID: ctx.QueryInt64("dataset_id"),
  605. Type: typeCloudBrain,
  606. })
  607. if err != nil {
  608. ctx.Error(500, fmt.Sprintf("InsertAttachment: %v", err))
  609. return
  610. }
  611. if attachment.DatasetID != 0 {
  612. if typeCloudBrain == models.TypeCloudBrainOne {
  613. if strings.HasSuffix(attachment.Name, ".zip") {
  614. err = worker.SendDecompressTask(contexExt.Background(), uuid)
  615. if err != nil {
  616. log.Error("SendDecompressTask(%s) failed:%s", uuid, err.Error())
  617. } else {
  618. attachment.DecompressState = models.DecompressStateIng
  619. err = models.UpdateAttachment(attachment)
  620. if err != nil {
  621. log.Error("UpdateAttachment state(%s) failed:%s", uuid, err.Error())
  622. }
  623. }
  624. }
  625. }
  626. }
  627. ctx.JSON(200, map[string]string{
  628. "result_code": "0",
  629. })
  630. }
  631. func UpdateMultipart(ctx *context.Context) {
  632. uuid := ctx.Query("uuid")
  633. partNumber := ctx.QueryInt("chunkNumber")
  634. etag := ctx.Query("etag")
  635. fileChunk, err := models.GetFileChunkByUUID(uuid)
  636. if err != nil {
  637. if models.IsErrFileChunkNotExist(err) {
  638. ctx.Error(404)
  639. } else {
  640. ctx.ServerError("GetFileChunkByUUID", err)
  641. }
  642. return
  643. }
  644. fileChunk.CompletedParts = append(fileChunk.CompletedParts, strconv.Itoa(partNumber)+"-"+strings.Replace(etag, "\"", "", -1))
  645. err = models.UpdateFileChunk(fileChunk)
  646. if err != nil {
  647. ctx.Error(500, fmt.Sprintf("UpdateFileChunk: %v", err))
  648. return
  649. }
  650. ctx.JSON(200, map[string]string{
  651. "result_code": "0",
  652. })
  653. }
  654. func HandleUnDecompressAttachment() {
  655. attachs, err := models.GetUnDecompressAttachments()
  656. if err != nil {
  657. log.Error("GetUnDecompressAttachments failed:", err.Error())
  658. return
  659. }
  660. for _, attach := range attachs {
  661. err = worker.SendDecompressTask(contexExt.Background(), attach.UUID)
  662. if err != nil {
  663. log.Error("SendDecompressTask(%s) failed:%s", attach.UUID, err.Error())
  664. } else {
  665. attach.DecompressState = models.DecompressStateIng
  666. err = models.UpdateAttachment(attach)
  667. if err != nil {
  668. log.Error("UpdateAttachment state(%s) failed:%s", attach.UUID, err.Error())
  669. }
  670. }
  671. }
  672. return
  673. }
  674. func QueryAllPublicDataset(ctx *context.Context) {
  675. attachs, err := models.GetAllPublicAttachments()
  676. if err != nil {
  677. ctx.JSON(200, map[string]string{
  678. "result_code": "-1",
  679. "error_msg": err.Error(),
  680. "data": "",
  681. })
  682. return
  683. }
  684. queryDatasets(ctx, attachs)
  685. }
  686. func QueryPrivateDataset(ctx *context.Context) {
  687. username := ctx.Params(":username")
  688. attachs, err := models.GetPrivateAttachments(username)
  689. if err != nil {
  690. ctx.JSON(200, map[string]string{
  691. "result_code": "-1",
  692. "error_msg": err.Error(),
  693. "data": "",
  694. })
  695. return
  696. }
  697. for _, attach := range attachs {
  698. attach.Name = username
  699. }
  700. queryDatasets(ctx, attachs)
  701. }
  702. func queryDatasets(ctx *context.Context, attachs []*models.AttachmentUsername) {
  703. var datasets []CloudBrainDataset
  704. if len(attachs) == 0 {
  705. log.Info("dataset is null")
  706. ctx.JSON(200, map[string]string{
  707. "result_code": "0",
  708. "error_msg": "",
  709. "data": "",
  710. })
  711. return
  712. }
  713. for _, attch := range attachs {
  714. has, err := storage.Attachments.HasObject(models.AttachmentRelativePath(attch.UUID))
  715. if err != nil || !has {
  716. continue
  717. }
  718. datasets = append(datasets, CloudBrainDataset{strconv.FormatInt(attch.ID, 10),
  719. attch.Attachment.Name,
  720. setting.Attachment.Minio.RealPath +
  721. setting.Attachment.Minio.Bucket + "/" +
  722. setting.Attachment.Minio.BasePath +
  723. models.AttachmentRelativePath(attch.UUID) +
  724. attch.UUID,
  725. attch.Name,
  726. attch.CreatedUnix.Format("2006-01-02 03:04:05 PM")})
  727. }
  728. data, err := json.Marshal(datasets)
  729. if err != nil {
  730. log.Error("json.Marshal failed:", err.Error())
  731. ctx.JSON(200, map[string]string{
  732. "result_code": "-1",
  733. "error_msg": err.Error(),
  734. "data": "",
  735. })
  736. return
  737. }
  738. ctx.JSON(200, map[string]string{
  739. "result_code": "0",
  740. "error_msg": "",
  741. "data": string(data),
  742. })
  743. return
  744. }
  745. func checkTypeCloudBrain(typeCloudBrain int) error {
  746. if typeCloudBrain != models.TypeCloudBrainOne && typeCloudBrain != models.TypeCloudBrainNotebook {
  747. log.Error("type error:", typeCloudBrain)
  748. return errors.New("type error")
  749. }
  750. return nil
  751. }