aiforge/routers/repo/dir.go

package repo

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"net/http"
	"path"
	"strings"

	"code.gitea.io/gitea/models"
	"code.gitea.io/gitea/modules/base"
	"code.gitea.io/gitea/modules/context"
	"code.gitea.io/gitea/modules/log"
	"code.gitea.io/gitea/modules/obs"
	"code.gitea.io/gitea/modules/setting"
	"code.gitea.io/gitea/modules/storage"
)

const (
	tplDirIndex base.TplName = "repo/datasets/dirs/index"
)

type FileInfo struct {
	FileName string `json:"FileName"`
	ModTime  string `json:"ModTime"`
	IsDir    bool   `json:"IsDir"`
	Size     int64  `json:"Size"`
	ParenDir string `json:"ParenDir"`
	UUID     string `json:"UUID"`
}

type RespGetDirs struct {
	ResultCode string `json:"resultCode"`
	FileInfos  string `json:"fileInfos"`
}

func DeleteAllUnzipFile(attachment *models.Attachment, parentDir string) {
	uuid := attachment.UUID
	dirArray := strings.Split(parentDir, "/")

	if !strings.HasSuffix(attachment.Name, ".zip") {
		log.Error("The file is not zip file, can not query the dir")
		return
	} else if attachment.DecompressState != models.DecompressStateDone {
		log.Error("The file has not been decompressed completely now")
		return
	}

	dirArray = append([]string{attachment.Name}, dirArray...)
	if parentDir == "" {
		dirArray = []string{attachment.Name}
	}
	if attachment.Type == models.TypeCloudBrainOne {
		dirs, err := GetDatasetDirs(uuid, parentDir)
		if err != nil {
			log.Error("getDatasetDirs failed:", err.Error())
			return
		}

		var fileInfos []FileInfo
		err = json.Unmarshal([]byte(dirs), &fileInfos)
		if err != nil {
			log.Error("json.Unmarshal failed:", err.Error())
			return
		}

		for _, fileInfo := range fileInfos {
			log.Info("fileName=" + fileInfo.FileName)
			log.Info("parentDir=" + fileInfo.ParenDir)
			if fileInfo.IsDir {
				DeleteAllUnzipFile(attachment, fileInfo.FileName)
			} else {
				absolutepath := path.Join(attachment.RelativePath()+attachment.UUID, fileInfo.ParenDir)
				log.Info("absolutepath=" + absolutepath)
				storage.Attachments.Delete(absolutepath)
			}
		}
	}
	if attachment.Type == models.TypeCloudBrainTwo {

		input := &obs.ListObjectsInput{}
		input.Bucket = setting.Bucket
		// 设置每页100个对象
		input.MaxKeys = 100
		input.Prefix = setting.BasePath + attachment.RelativePath() + attachment.UUID
		index := 1
		log.Info("prefix=" + input.Prefix)
		for {
			output, err := storage.ObsCli.ListObjects(input)
			if err == nil {
				log.Info("Page:%d\n", index)
				index++
				for _, val := range output.Contents {
					log.Info("delete obs file:" + val.Key)
					delObj := &obs.DeleteObjectInput{}
					delObj.Bucket = setting.Bucket
					delObj.Key = val.Key
					storage.ObsCli.DeleteObject(delObj)
				}
				if output.IsTruncated {
					input.Marker = output.NextMarker
				} else {
					break
				}
			} else {
				if obsError, ok := err.(obs.ObsError); ok {
					log.Info("Code:%s\n", obsError.Code)
					log.Info("Message:%s\n", obsError.Message)
				}
				break
			}
		}

	}

}

func DirIndex(ctx *context.Context) {
	uuid := ctx.Params("uuid")
	parentDir := ctx.Query("parentDir")
	dirArray := strings.Split(parentDir, "/")

	attachment, err := models.GetAttachmentByUUID(uuid)
	if err != nil {
		ctx.ServerError("GetDatasetAttachments", err)
		return
	}

	if !strings.HasSuffix(attachment.Name, ".zip") {
		log.Error("The file is not zip file, can not query the dir")
		ctx.ServerError("The file is not zip file, can not query the dir", errors.New("The file is not zip file, can not query the dir"))
		return
	} else if attachment.DecompressState != models.DecompressStateDone {
		log.Error("The file has not been decompressed completely now")
		ctx.ServerError("The file has not been decompressed completely now", errors.New("The file has not been decompressed completely now"))
		return
	}

	dirArray = append([]string{attachment.Name}, dirArray...)
	if parentDir == "" {
		dirArray = []string{attachment.Name}
	}

	/*
		dirs, err := GetDatasetDirs(uuid, parentDir)
		if err != nil {
			log.Error("getDatasetDirs failed:", err.Error())
			ctx.ServerError("getDatasetDirs failed:", err)
			return
		}
	*/

	//var fileInfos []FileInfo

	/*
		err = json.Unmarshal([]byte(dirs), &fileInfos)
		if err != nil {
			log.Error("json.Unmarshal failed:", err.Error())
			ctx.ServerError("json.Unmarshal failed:", err)
			return
		}
	*/

	ctx.Data["Path"] = dirArray
	ctx.Data["Dirs"] = true
	ctx.Data["Uuid"] = uuid
	ctx.Data["PageIsDataset"] = true

	ctx.HTML(200, tplDirIndex)
}

func GetDatasetDirs(uuid string, parentDir string) (string, error) {
	var req string
	dataActualPath := setting.Attachment.Minio.RealPath +
		setting.Attachment.Minio.Bucket + "/" +
		setting.Attachment.Minio.BasePath +
		models.AttachmentRelativePath(uuid) +
		uuid + "/"
	if parentDir == "" {
		req = "baseDir=" + dataActualPath
	} else {
		req = "baseDir=" + dataActualPath + "&parentDir=" + parentDir
	}

	return getDirs(req)
}

func getDirs(req string) (string, error) {
	var dirs string

	url := setting.DecompressAddress + "/dirs?" + req
	reqHttp, err := http.NewRequest(http.MethodGet, url, nil)
	if err != nil {
		log.Error("http.NewRequest failed:", err.Error())
		return dirs, err
	}

	reqHttp.SetBasicAuth(setting.AuthUser, setting.AuthPassword)
	res, err := http.DefaultClient.Do(reqHttp)
	if err != nil {
		log.Error("send http to decompress failed:", err.Error())
		return dirs, err
	}

	if res.StatusCode != http.StatusOK {
		log.Error("the response from decompress is failed")
		return dirs, errors.New("the response from decompress is failed")
	}

	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		log.Error("read resp body failed:", err.Error())
		return dirs, err
	}

	var resp RespGetDirs
	err = json.Unmarshal(body, &resp)
	if err != nil {
		log.Error("unmarshal resp failed:", err.Error())
		return dirs, err
	}

	if resp.ResultCode != "0" {
		log.Error("GetDirs failed:", resp.ResultCode)
		return dirs, errors.New("GetDirs failed")
	}

	dirs = resp.FileInfos
	return dirs, nil
}