You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.

attachment.go 21 kB

4 years ago
4 years ago
4 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
5 years ago
4 years ago
5 years ago
5 years ago
4 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
5 years ago
5 years ago
4 years ago
5 years ago
4 years ago
5 years ago
4 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
4 years ago
5 years ago
5 years ago
5 years ago
4 years ago
5 years ago
4 years ago
4 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
5 years ago
4 years ago
5 years ago
5 years ago
5 years ago
4 years ago
5 years ago
4 years ago
5 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
5 years ago
4 years ago
5 years ago
4 years ago
5 years ago
5 years ago
5 years ago
4 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863
  1. // Copyright 2017 The Gitea Authors. All rights reserved.
  2. // Use of this source code is governed by a MIT-style
  3. // license that can be found in the LICENSE file.
  4. package repo
  5. import (
  6. contexExt "context"
  7. "encoding/json"
  8. "errors"
  9. "fmt"
  10. "mime/multipart"
  11. "net/http"
  12. "path"
  13. "strconv"
  14. "strings"
  15. "code.gitea.io/gitea/models"
  16. "code.gitea.io/gitea/modules/context"
  17. "code.gitea.io/gitea/modules/log"
  18. "code.gitea.io/gitea/modules/minio_ext"
  19. "code.gitea.io/gitea/modules/setting"
  20. "code.gitea.io/gitea/modules/storage"
  21. "code.gitea.io/gitea/modules/upload"
  22. "code.gitea.io/gitea/modules/worker"
  23. gouuid "github.com/satori/go.uuid"
  24. )
  25. const (
  26. //result of decompress
  27. DecompressSuccess = "0"
  28. DecompressFailed = "1"
  29. )
  30. type CloudBrainDataset struct {
  31. UUID string `json:"id"`
  32. Name string `json:"name"`
  33. Path string `json:"place"`
  34. UserName string `json:"provider"`
  35. CreateTime string `json:"created_at"`
  36. }
  37. type UploadForm struct {
  38. UploadID string `form:"uploadId"`
  39. UuID string `form:"uuid"`
  40. PartSize int64 `form:"size"`
  41. Offset int64 `form:"offset"`
  42. PartNumber int `form:"chunkNumber"`
  43. PartFile multipart.File `form:"file"`
  44. }
  45. func RenderAttachmentSettings(ctx *context.Context) {
  46. renderAttachmentSettings(ctx)
  47. }
  48. func renderAttachmentSettings(ctx *context.Context) {
  49. ctx.Data["IsAttachmentEnabled"] = setting.Attachment.Enabled
  50. ctx.Data["AttachmentStoreType"] = setting.Attachment.StoreType
  51. ctx.Data["AttachmentAllowedTypes"] = setting.Attachment.AllowedTypes
  52. ctx.Data["AttachmentMaxSize"] = setting.Attachment.MaxSize
  53. ctx.Data["AttachmentMaxFiles"] = setting.Attachment.MaxFiles
  54. }
  55. // UploadAttachment response for uploading issue's attachment
  56. func UploadAttachment(ctx *context.Context) {
  57. if !setting.Attachment.Enabled {
  58. ctx.Error(404, "attachment is not enabled")
  59. return
  60. }
  61. file, header, err := ctx.Req.FormFile("file")
  62. if err != nil {
  63. ctx.Error(500, fmt.Sprintf("FormFile: %v", err))
  64. return
  65. }
  66. defer file.Close()
  67. buf := make([]byte, 1024)
  68. n, _ := file.Read(buf)
  69. if n > 0 {
  70. buf = buf[:n]
  71. }
  72. err = upload.VerifyAllowedContentType(buf, strings.Split(setting.Attachment.AllowedTypes, ","))
  73. if err != nil {
  74. ctx.Error(400, err.Error())
  75. return
  76. }
  77. datasetID, _ := strconv.ParseInt(ctx.Req.FormValue("dataset_id"), 10, 64)
  78. attach, err := models.NewAttachment(&models.Attachment{
  79. IsPrivate: true,
  80. UploaderID: ctx.User.ID,
  81. Name: header.Filename,
  82. DatasetID: datasetID,
  83. }, buf, file)
  84. if err != nil {
  85. ctx.Error(500, fmt.Sprintf("NewAttachment: %v", err))
  86. return
  87. }
  88. log.Trace("New attachment uploaded: %s", attach.UUID)
  89. ctx.JSON(200, map[string]string{
  90. "uuid": attach.UUID,
  91. })
  92. }
  93. func UpdatePublicAttachment(ctx *context.Context) {
  94. file := ctx.Query("file")
  95. isPrivate, _ := strconv.ParseBool(ctx.Query("is_private"))
  96. attach, err := models.GetAttachmentByUUID(file)
  97. if err != nil {
  98. ctx.Error(404, err.Error())
  99. return
  100. }
  101. attach.IsPrivate = isPrivate
  102. models.UpdateAttachment(attach)
  103. }
  104. // DeleteAttachment response for deleting issue's attachment
  105. func DeleteAttachment(ctx *context.Context) {
  106. file := ctx.Query("file")
  107. attach, err := models.GetAttachmentByUUID(file)
  108. if err != nil {
  109. ctx.Error(400, err.Error())
  110. return
  111. }
  112. if !ctx.IsSigned || (ctx.User.ID != attach.UploaderID) {
  113. ctx.Error(403)
  114. return
  115. }
  116. err = models.DeleteAttachment(attach, false)
  117. if err != nil {
  118. ctx.Error(500, fmt.Sprintf("DeleteAttachment: %v", err))
  119. return
  120. }
  121. ctx.JSON(200, map[string]string{
  122. "uuid": attach.UUID,
  123. })
  124. }
  125. // GetAttachment serve attachements
  126. func GetAttachment(ctx *context.Context) {
  127. attach, err := models.GetAttachmentByUUID(ctx.Params(":uuid"))
  128. if err != nil {
  129. if models.IsErrAttachmentNotExist(err) {
  130. ctx.Error(404)
  131. } else {
  132. ctx.ServerError("GetAttachmentByUUID", err)
  133. }
  134. return
  135. }
  136. repository, unitType, err := attach.LinkedRepository()
  137. if err != nil {
  138. ctx.ServerError("LinkedRepository", err)
  139. return
  140. }
  141. if repository == nil { //If not linked
  142. if !(ctx.IsSigned && attach.UploaderID == ctx.User.ID) && attach.IsPrivate { //We block if not the uploader
  143. ctx.Error(http.StatusNotFound)
  144. return
  145. }
  146. } else { //If we have the repository we check access
  147. perm, err := models.GetUserRepoPermission(repository, ctx.User)
  148. if err != nil {
  149. ctx.Error(http.StatusInternalServerError, "GetUserRepoPermission", err.Error())
  150. return
  151. }
  152. if !perm.CanRead(unitType) {
  153. ctx.Error(http.StatusNotFound)
  154. return
  155. }
  156. }
  157. dataSet, err := attach.LinkedDataSet()
  158. if err != nil {
  159. ctx.ServerError("LinkedDataSet", err)
  160. return
  161. }
  162. if dataSet != nil {
  163. isPermit, err := models.GetUserDataSetPermission(dataSet, ctx.User)
  164. if err != nil {
  165. ctx.Error(http.StatusInternalServerError, "GetUserDataSetPermission", err.Error())
  166. return
  167. }
  168. if !isPermit {
  169. ctx.Error(http.StatusNotFound)
  170. return
  171. }
  172. }
  173. //If we have matched and access to release or issue
  174. if setting.Attachment.StoreType == storage.MinioStorageType {
  175. url, err := storage.Attachments.PresignedGetURL(attach.RelativePath(), attach.Name)
  176. if err != nil {
  177. ctx.ServerError("PresignedGetURL", err)
  178. return
  179. }
  180. if err = increaseDownloadCount(attach, dataSet); err != nil {
  181. ctx.ServerError("Update", err)
  182. return
  183. }
  184. http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently)
  185. } else {
  186. fr, err := storage.Attachments.Open(attach.RelativePath())
  187. if err != nil {
  188. ctx.ServerError("Open", err)
  189. return
  190. }
  191. defer fr.Close()
  192. if err = increaseDownloadCount(attach, dataSet); err != nil {
  193. ctx.ServerError("Update", err)
  194. return
  195. }
  196. if err = ServeData(ctx, attach.Name, fr); err != nil {
  197. ctx.ServerError("ServeData", err)
  198. return
  199. }
  200. }
  201. }
  202. func increaseDownloadCount(attach *models.Attachment, dataSet *models.Dataset) error {
  203. if err := attach.IncreaseDownloadCount(); err != nil {
  204. return err
  205. }
  206. if dataSet != nil {
  207. if err := models.IncreaseDownloadCount(dataSet.ID); err != nil {
  208. return err
  209. }
  210. }
  211. return nil
  212. }
  213. // Get a presigned url for put object
  214. func GetPresignedPutObjectURL(ctx *context.Context) {
  215. if !setting.Attachment.Enabled {
  216. ctx.Error(404, "attachment is not enabled")
  217. return
  218. }
  219. err := upload.VerifyFileType(ctx.Params("file_type"), strings.Split(setting.Attachment.AllowedTypes, ","))
  220. if err != nil {
  221. ctx.Error(400, err.Error())
  222. return
  223. }
  224. if setting.Attachment.StoreType == storage.MinioStorageType {
  225. uuid := gouuid.NewV4().String()
  226. url, err := storage.Attachments.PresignedPutURL(models.AttachmentRelativePath(uuid))
  227. if err != nil {
  228. ctx.ServerError("PresignedPutURL", err)
  229. return
  230. }
  231. ctx.JSON(200, map[string]string{
  232. "uuid": uuid,
  233. "url": url,
  234. })
  235. } else {
  236. ctx.Error(404, "storage type is not enabled")
  237. return
  238. }
  239. }
  240. // AddAttachment response for add attachment record
  241. func AddAttachment(ctx *context.Context) {
  242. typeCloudBrain := ctx.QueryInt("type")
  243. err := checkTypeCloudBrain(typeCloudBrain)
  244. if err != nil {
  245. ctx.ServerError("checkTypeCloudBrain failed", err)
  246. return
  247. }
  248. uuid := ctx.Query("uuid")
  249. has := false
  250. if typeCloudBrain == models.TypeCloudBrainOne {
  251. has, err = storage.Attachments.HasObject(models.AttachmentRelativePath(uuid))
  252. if err != nil {
  253. ctx.ServerError("HasObject", err)
  254. return
  255. }
  256. } else {
  257. has, err = storage.ObsHasObject(models.AttachmentRelativePath(uuid))
  258. if err != nil {
  259. ctx.ServerError("ObsHasObject", err)
  260. return
  261. }
  262. }
  263. if !has {
  264. ctx.Error(404, "attachment has not been uploaded")
  265. return
  266. }
  267. attachment, err := models.InsertAttachment(&models.Attachment{
  268. UUID: uuid,
  269. UploaderID: ctx.User.ID,
  270. IsPrivate: true,
  271. Name: ctx.Query("file_name"),
  272. Size: ctx.QueryInt64("size"),
  273. DatasetID: ctx.QueryInt64("dataset_id"),
  274. Type: typeCloudBrain,
  275. })
  276. if err != nil {
  277. ctx.Error(500, fmt.Sprintf("InsertAttachment: %v", err))
  278. return
  279. }
  280. if attachment.DatasetID != 0 {
  281. if strings.HasSuffix(attachment.Name, ".zip") {
  282. if typeCloudBrain == models.TypeCloudBrainOne {
  283. err = worker.SendDecompressTask(contexExt.Background(), uuid)
  284. if err != nil {
  285. log.Error("SendDecompressTask(%s) failed:%s", uuid, err.Error())
  286. } else {
  287. attachment.DecompressState = models.DecompressStateIng
  288. err = models.UpdateAttachment(attachment)
  289. if err != nil {
  290. log.Error("UpdateAttachment state(%s) failed:%s", uuid, err.Error())
  291. }
  292. }
  293. }
  294. //todo:decompress type_two
  295. }
  296. }
  297. ctx.JSON(200, map[string]string{
  298. "result_code": "0",
  299. })
  300. }
  301. func UpdateAttachmentDecompressState(ctx *context.Context) {
  302. uuid := ctx.Query("uuid")
  303. result := ctx.Query("result")
  304. attach, err := models.GetAttachmentByUUID(uuid)
  305. if err != nil {
  306. log.Error("GetAttachmentByUUID(%s) failed:%s", uuid, err.Error())
  307. return
  308. }
  309. if result == DecompressSuccess {
  310. attach.DecompressState = models.DecompressStateDone
  311. } else if result == DecompressFailed {
  312. attach.DecompressState = models.DecompressStateFailed
  313. } else {
  314. log.Error("result is error:", result)
  315. return
  316. }
  317. err = models.UpdateAttachment(attach)
  318. if err != nil {
  319. log.Error("UpdateAttachment(%s) failed:%s", uuid, err.Error())
  320. return
  321. }
  322. ctx.JSON(200, map[string]string{
  323. "result_code": "0",
  324. })
  325. }
  326. func GetSuccessChunks(ctx *context.Context) {
  327. fileMD5 := ctx.Query("md5")
  328. typeCloudBrain := ctx.QueryInt("type")
  329. var chunks string
  330. err := checkTypeCloudBrain(typeCloudBrain)
  331. if err != nil {
  332. ctx.ServerError("checkTypeCloudBrain failed", err)
  333. return
  334. }
  335. fileChunk, err := models.GetFileChunkByMD5AndUser(fileMD5, ctx.User.ID, typeCloudBrain)
  336. if err != nil {
  337. if models.IsErrFileChunkNotExist(err) {
  338. ctx.JSON(200, map[string]string{
  339. "uuid": "",
  340. "uploaded": "0",
  341. "uploadID": "",
  342. "chunks": "",
  343. })
  344. } else {
  345. ctx.ServerError("GetFileChunkByMD5", err)
  346. }
  347. return
  348. }
  349. isExist := false
  350. if typeCloudBrain == models.TypeCloudBrainOne {
  351. isExist, err = storage.Attachments.HasObject(models.AttachmentRelativePath(fileChunk.UUID))
  352. if err != nil {
  353. ctx.ServerError("HasObject failed", err)
  354. return
  355. }
  356. } else {
  357. isExist, err = storage.ObsHasObject(models.AttachmentRelativePath(fileChunk.UUID))
  358. if err != nil {
  359. ctx.ServerError("ObsHasObject failed", err)
  360. return
  361. }
  362. }
  363. if isExist {
  364. if fileChunk.IsUploaded == models.FileNotUploaded {
  365. log.Info("the file has been uploaded but not recorded")
  366. fileChunk.IsUploaded = models.FileUploaded
  367. if err = models.UpdateFileChunk(fileChunk); err != nil {
  368. log.Error("UpdateFileChunk failed:", err.Error())
  369. }
  370. }
  371. } else {
  372. if fileChunk.IsUploaded == models.FileUploaded {
  373. log.Info("the file has been recorded but not uploaded")
  374. fileChunk.IsUploaded = models.FileNotUploaded
  375. if err = models.UpdateFileChunk(fileChunk); err != nil {
  376. log.Error("UpdateFileChunk failed:", err.Error())
  377. }
  378. }
  379. if typeCloudBrain == models.TypeCloudBrainOne {
  380. chunks, err = storage.GetPartInfos(fileChunk.UUID, fileChunk.UploadID)
  381. if err != nil {
  382. ctx.ServerError("GetPartInfos failed", err)
  383. return
  384. }
  385. } else {
  386. chunks, err = storage.GetObsPartInfos(fileChunk.UUID, fileChunk.UploadID)
  387. if err != nil {
  388. ctx.ServerError("GetObsPartInfos failed", err)
  389. return
  390. }
  391. }
  392. }
  393. var attachID int64
  394. attach, err := models.GetAttachmentByUUID(fileChunk.UUID)
  395. if err != nil {
  396. if models.IsErrAttachmentNotExist(err) {
  397. attachID = 0
  398. } else {
  399. ctx.ServerError("GetAttachmentByUUID", err)
  400. return
  401. }
  402. } else {
  403. attachID = attach.ID
  404. }
  405. if attach == nil {
  406. ctx.JSON(200, map[string]string{
  407. "uuid": fileChunk.UUID,
  408. "uploaded": strconv.Itoa(fileChunk.IsUploaded),
  409. "uploadID": fileChunk.UploadID,
  410. "chunks": string(chunks),
  411. "attachID": "0",
  412. "datasetID": "0",
  413. "fileName": "",
  414. "datasetName": "",
  415. })
  416. return
  417. }
  418. dataset, err := models.GetDatasetByID(attach.DatasetID)
  419. if err != nil {
  420. ctx.ServerError("GetDatasetByID", err)
  421. return
  422. }
  423. ctx.JSON(200, map[string]string{
  424. "uuid": fileChunk.UUID,
  425. "uploaded": strconv.Itoa(fileChunk.IsUploaded),
  426. "uploadID": fileChunk.UploadID,
  427. "chunks": string(chunks),
  428. "attachID": strconv.Itoa(int(attachID)),
  429. "datasetID": strconv.Itoa(int(attach.DatasetID)),
  430. "fileName": attach.Name,
  431. "datasetName": dataset.Title,
  432. })
  433. }
  434. func NewMultipart(ctx *context.Context) {
  435. if !setting.Attachment.Enabled {
  436. ctx.Error(404, "attachment is not enabled")
  437. return
  438. }
  439. err := upload.VerifyFileType(ctx.Query("fileType"), strings.Split(setting.Attachment.AllowedTypes, ","))
  440. if err != nil {
  441. ctx.Error(400, err.Error())
  442. return
  443. }
  444. typeCloudBrain := ctx.QueryInt("type")
  445. err = checkTypeCloudBrain(typeCloudBrain)
  446. if err != nil {
  447. ctx.ServerError("checkTypeCloudBrain failed", err)
  448. return
  449. }
  450. if setting.Attachment.StoreType == storage.MinioStorageType {
  451. totalChunkCounts := ctx.QueryInt("totalChunkCounts")
  452. if totalChunkCounts > minio_ext.MaxPartsCount {
  453. ctx.Error(400, fmt.Sprintf("chunk counts(%d) is too much", totalChunkCounts))
  454. return
  455. }
  456. fileSize := ctx.QueryInt64("size")
  457. if fileSize > minio_ext.MaxMultipartPutObjectSize {
  458. ctx.Error(400, fmt.Sprintf("file size(%d) is too big", fileSize))
  459. return
  460. }
  461. uuid := gouuid.NewV4().String()
  462. var uploadID string
  463. if typeCloudBrain == models.TypeCloudBrainOne {
  464. uploadID, err = storage.NewMultiPartUpload(uuid)
  465. if err != nil {
  466. ctx.ServerError("NewMultipart", err)
  467. return
  468. }
  469. } else {
  470. uploadID, err = storage.NewObsMultiPartUpload(uuid)
  471. if err != nil {
  472. ctx.ServerError("NewObsMultiPartUpload", err)
  473. return
  474. }
  475. }
  476. _, err = models.InsertFileChunk(&models.FileChunk{
  477. UUID: uuid,
  478. UserID: ctx.User.ID,
  479. UploadID: uploadID,
  480. Md5: ctx.Query("md5"),
  481. Size: fileSize,
  482. TotalChunks: totalChunkCounts,
  483. Type: typeCloudBrain,
  484. })
  485. if err != nil {
  486. ctx.Error(500, fmt.Sprintf("InsertFileChunk: %v", err))
  487. return
  488. }
  489. ctx.JSON(200, map[string]string{
  490. "uuid": uuid,
  491. "uploadID": uploadID,
  492. })
  493. } else {
  494. ctx.Error(404, "storage type is not enabled")
  495. return
  496. }
  497. }
  498. func GetMultipartUploadUrl(ctx *context.Context) {
  499. uuid := ctx.Query("uuid")
  500. uploadID := ctx.Query("uploadID")
  501. partNumber := ctx.QueryInt("chunkNumber")
  502. size := ctx.QueryInt64("size")
  503. typeCloudBrain := ctx.QueryInt("type")
  504. err := checkTypeCloudBrain(typeCloudBrain)
  505. if err != nil {
  506. ctx.ServerError("checkTypeCloudBrain failed", err)
  507. return
  508. }
  509. url := ""
  510. if typeCloudBrain == models.TypeCloudBrainOne {
  511. if size > minio_ext.MinPartSize {
  512. ctx.Error(400, fmt.Sprintf("chunk size(%d) is too big", size))
  513. return
  514. }
  515. url, err = storage.GenMultiPartSignedUrl(uuid, uploadID, partNumber, size)
  516. if err != nil {
  517. ctx.Error(500, fmt.Sprintf("GenMultiPartSignedUrl failed: %v", err))
  518. return
  519. }
  520. } else {
  521. url, err = storage.ObsGenMultiPartSignedUrl(uuid, uploadID, partNumber, size)
  522. if err != nil {
  523. ctx.Error(500, fmt.Sprintf("ObsGenMultiPartSignedUrl failed: %v", err))
  524. return
  525. }
  526. }
  527. ctx.JSON(200, map[string]string{
  528. "url": url,
  529. })
  530. }
  531. func GetObsKey(ctx *context.Context) {
  532. uuid := gouuid.NewV4().String()
  533. key := strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid)), "/")
  534. ctx.JSON(200, map[string]string{
  535. "uuid": uuid,
  536. "key": key,
  537. })
  538. }
  539. func UploadPart(ctx *context.Context) {
  540. tmp, err := ctx.Req.Body().String()
  541. log.Info(tmp)
  542. err = ctx.Req.ParseMultipartForm(100*1024*1024)
  543. if err != nil {
  544. ctx.Error(http.StatusBadRequest, fmt.Sprintf("ParseMultipartForm failed: %v", err))
  545. return
  546. }
  547. file, fileHeader, err := ctx.Req.FormFile("file")
  548. log.Info(ctx.Req.Form.Get("file"))
  549. if err != nil {
  550. ctx.Error(http.StatusBadRequest, fmt.Sprintf("FormFile failed: %v", err))
  551. return
  552. }
  553. log.Info(fileHeader.Filename)
  554. etag, err := storage.ObsUploadPart("", "", 1, 1, file)
  555. if err != nil {
  556. ctx.Error(500, fmt.Sprintf("ObsUploadPart failed: %v", err))
  557. return
  558. }
  559. ctx.JSON(200, map[string]string{
  560. "etag": etag,
  561. })
  562. }
  563. func CompleteMultipart(ctx *context.Context) {
  564. uuid := ctx.Query("uuid")
  565. uploadID := ctx.Query("uploadID")
  566. typeCloudBrain := ctx.QueryInt("type")
  567. err := checkTypeCloudBrain(typeCloudBrain)
  568. if err != nil {
  569. ctx.ServerError("checkTypeCloudBrain failed", err)
  570. return
  571. }
  572. fileChunk, err := models.GetFileChunkByUUID(uuid)
  573. if err != nil {
  574. if models.IsErrFileChunkNotExist(err) {
  575. ctx.Error(404)
  576. } else {
  577. ctx.ServerError("GetFileChunkByUUID", err)
  578. }
  579. return
  580. }
  581. if typeCloudBrain == models.TypeCloudBrainOne {
  582. _, err = storage.CompleteMultiPartUpload(uuid, uploadID)
  583. if err != nil {
  584. ctx.Error(500, fmt.Sprintf("CompleteMultiPartUpload failed: %v", err))
  585. return
  586. }
  587. } else {
  588. err = storage.CompleteObsMultiPartUpload(uuid, uploadID)
  589. if err != nil {
  590. ctx.Error(500, fmt.Sprintf("CompleteObsMultiPartUpload failed: %v", err))
  591. return
  592. }
  593. }
  594. fileChunk.IsUploaded = models.FileUploaded
  595. err = models.UpdateFileChunk(fileChunk)
  596. if err != nil {
  597. ctx.Error(500, fmt.Sprintf("UpdateFileChunk: %v", err))
  598. return
  599. }
  600. attachment, err := models.InsertAttachment(&models.Attachment{
  601. UUID: uuid,
  602. UploaderID: ctx.User.ID,
  603. IsPrivate: true,
  604. Name: ctx.Query("file_name"),
  605. Size: ctx.QueryInt64("size"),
  606. DatasetID: ctx.QueryInt64("dataset_id"),
  607. Type: typeCloudBrain,
  608. })
  609. if err != nil {
  610. ctx.Error(500, fmt.Sprintf("InsertAttachment: %v", err))
  611. return
  612. }
  613. if attachment.DatasetID != 0 {
  614. if strings.HasSuffix(attachment.Name, ".zip") {
  615. err = worker.SendDecompressTask(contexExt.Background(), uuid)
  616. if err != nil {
  617. log.Error("SendDecompressTask(%s) failed:%s", uuid, err.Error())
  618. } else {
  619. attachment.DecompressState = models.DecompressStateIng
  620. err = models.UpdateAttachment(attachment)
  621. if err != nil {
  622. log.Error("UpdateAttachment state(%s) failed:%s", uuid, err.Error())
  623. }
  624. }
  625. }
  626. }
  627. ctx.JSON(200, map[string]string{
  628. "result_code": "0",
  629. })
  630. }
  631. func UpdateMultipart(ctx *context.Context) {
  632. uuid := ctx.Query("uuid")
  633. partNumber := ctx.QueryInt("chunkNumber")
  634. etag := ctx.Query("etag")
  635. fileChunk, err := models.GetFileChunkByUUID(uuid)
  636. if err != nil {
  637. if models.IsErrFileChunkNotExist(err) {
  638. ctx.Error(404)
  639. } else {
  640. ctx.ServerError("GetFileChunkByUUID", err)
  641. }
  642. return
  643. }
  644. fileChunk.CompletedParts = append(fileChunk.CompletedParts, strconv.Itoa(partNumber)+"-"+strings.Replace(etag, "\"", "", -1))
  645. err = models.UpdateFileChunk(fileChunk)
  646. if err != nil {
  647. ctx.Error(500, fmt.Sprintf("UpdateFileChunk: %v", err))
  648. return
  649. }
  650. ctx.JSON(200, map[string]string{
  651. "result_code": "0",
  652. })
  653. }
  654. func HandleUnDecompressAttachment() {
  655. attachs, err := models.GetUnDecompressAttachments()
  656. if err != nil {
  657. log.Error("GetUnDecompressAttachments failed:", err.Error())
  658. return
  659. }
  660. for _, attach := range attachs {
  661. err = worker.SendDecompressTask(contexExt.Background(), attach.UUID)
  662. if err != nil {
  663. log.Error("SendDecompressTask(%s) failed:%s", attach.UUID, err.Error())
  664. } else {
  665. attach.DecompressState = models.DecompressStateIng
  666. err = models.UpdateAttachment(attach)
  667. if err != nil {
  668. log.Error("UpdateAttachment state(%s) failed:%s", attach.UUID, err.Error())
  669. }
  670. }
  671. }
  672. return
  673. }
  674. func QueryAllPublicDataset(ctx *context.Context) {
  675. attachs, err := models.GetAllPublicAttachments()
  676. if err != nil {
  677. ctx.JSON(200, map[string]string{
  678. "result_code": "-1",
  679. "error_msg": err.Error(),
  680. "data": "",
  681. })
  682. return
  683. }
  684. queryDatasets(ctx, attachs)
  685. }
  686. func QueryPrivateDataset(ctx *context.Context) {
  687. username := ctx.Params(":username")
  688. attachs, err := models.GetPrivateAttachments(username)
  689. if err != nil {
  690. ctx.JSON(200, map[string]string{
  691. "result_code": "-1",
  692. "error_msg": err.Error(),
  693. "data": "",
  694. })
  695. return
  696. }
  697. for _, attach := range attachs {
  698. attach.Name = username
  699. }
  700. queryDatasets(ctx, attachs)
  701. }
  702. func queryDatasets(ctx *context.Context, attachs []*models.AttachmentUsername) {
  703. var datasets []CloudBrainDataset
  704. if len(attachs) == 0 {
  705. log.Info("dataset is null")
  706. ctx.JSON(200, map[string]string{
  707. "result_code": "0",
  708. "error_msg": "",
  709. "data": "",
  710. })
  711. return
  712. }
  713. for _, attch := range attachs {
  714. has, err := storage.Attachments.HasObject(models.AttachmentRelativePath(attch.UUID))
  715. if err != nil || !has {
  716. continue
  717. }
  718. datasets = append(datasets, CloudBrainDataset{strconv.FormatInt(attch.ID, 10),
  719. attch.Attachment.Name,
  720. setting.Attachment.Minio.RealPath +
  721. setting.Attachment.Minio.Bucket + "/" +
  722. setting.Attachment.Minio.BasePath +
  723. models.AttachmentRelativePath(attch.UUID) +
  724. attch.UUID,
  725. attch.Name,
  726. attch.CreatedUnix.Format("2006-01-02 03:04:05 PM")})
  727. }
  728. data, err := json.Marshal(datasets)
  729. if err != nil {
  730. log.Error("json.Marshal failed:", err.Error())
  731. ctx.JSON(200, map[string]string{
  732. "result_code": "-1",
  733. "error_msg": err.Error(),
  734. "data": "",
  735. })
  736. return
  737. }
  738. ctx.JSON(200, map[string]string{
  739. "result_code": "0",
  740. "error_msg": "",
  741. "data": string(data),
  742. })
  743. return
  744. }
  745. func checkTypeCloudBrain(typeCloudBrain int) error {
  746. if typeCloudBrain != models.TypeCloudBrainOne && typeCloudBrain != models.TypeCloudBrainTwo {
  747. log.Error("type error:", typeCloudBrain)
  748. return errors.New("type error")
  749. }
  750. return nil
  751. }