Merge branch 'V20211101' of git.openi.org.cn:OpenI/aiforge into log-opt

models/user_business_analysis.go

@@ -71,6 +71,49 @@ type UserBusinessAnalysis struct {
 	Name string `xorm:"NOT NULL"`
 }
 
+func QueryUserStaticData(startTime int64, endTime int64) []*UserBusinessAnalysis {
+	log.Info("query startTime =" + fmt.Sprint(startTime) + " endTime=" + fmt.Sprint(endTime))
+	statictisSess := xStatistic.NewSession()
+	defer statictisSess.Close()
+
+	statictisSess.Select("*").Table("user_business_analysis").Where(" count_date>=" + fmt.Sprint(startTime) + " and count_date<=" + fmt.Sprint(endTime)).OrderBy("count_date desc")
+
+	userBusinessAnalysisList := make([]*UserBusinessAnalysis, 0)
+	statictisSess.Find(&userBusinessAnalysisList)
+
+	resultMap := make(map[int64]*UserBusinessAnalysis)
+	log.Info("query result size=" + fmt.Sprint(len(userBusinessAnalysisList)))
+	for _, userRecord := range userBusinessAnalysisList {
+		if _, ok := resultMap[userRecord.ID]; !ok {
+			resultMap[userRecord.ID] = userRecord
+		} else {
+			resultMap[userRecord.ID].CodeMergeCount += userRecord.CodeMergeCount
+			resultMap[userRecord.ID].CommitCount += userRecord.CommitCount
+			resultMap[userRecord.ID].IssueCount += userRecord.IssueCount
+			resultMap[userRecord.ID].CommentCount += userRecord.CommentCount
+			resultMap[userRecord.ID].FocusRepoCount += userRecord.FocusRepoCount
+			resultMap[userRecord.ID].StarRepoCount += userRecord.StarRepoCount
+			resultMap[userRecord.ID].WatchedCount += userRecord.WatchedCount
+			resultMap[userRecord.ID].CommitCodeSize += userRecord.CommitCodeSize
+			resultMap[userRecord.ID].CommitDatasetSize += userRecord.CommitDatasetSize
+			resultMap[userRecord.ID].CommitModelCount += userRecord.CommitModelCount
+			resultMap[userRecord.ID].SolveIssueCount += userRecord.SolveIssueCount
+			resultMap[userRecord.ID].EncyclopediasCount += userRecord.EncyclopediasCount
+			resultMap[userRecord.ID].CreateRepoCount += userRecord.CreateRepoCount
+			resultMap[userRecord.ID].LoginCount += userRecord.LoginCount
+		}
+	}
+
+	userBusinessAnalysisReturnList := make([]*UserBusinessAnalysis, len(resultMap))
+	index := 0
+	for _, v := range resultMap {
+		userBusinessAnalysisReturnList[index] = v
+		index += 1
+	}
+	log.Info("return size=" + fmt.Sprint(len(userBusinessAnalysisReturnList)))
+	return userBusinessAnalysisReturnList
+}
+
 func CountData(wikiCountMap map[string]int) {
 	log.Info("start to count other user info data")
 	sess := x.NewSession()
@@ -92,7 +135,7 @@ func CountData(wikiCountMap map[string]int) {
 
 	CountDate := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 1, 0, 0, currentTimeNow.Location())
 
-	CodeMergeCountMap := queryAction(start_unix, end_unix, 11)
+	CodeMergeCountMap := queryPullRequest(start_unix, end_unix)
 	CommitCountMap := queryAction(start_unix, end_unix, 5)
 	IssueCountMap := queryAction(start_unix, end_unix, 10)
 
@@ -223,6 +266,28 @@ func querySolveIssue(start_unix int64, end_unix int64) map[int64]int {
 
 }
 
+func queryPullRequest(start_unix int64, end_unix int64) map[int64]int {
+	sess := x.NewSession()
+	defer sess.Close()
+
+	sess.Select("issue.*").Table("issue").
+		Join("inner", "pull_request", "issue.id=pull_request.issue_id").
+		Where("pull_request.merged_unix>=" + fmt.Sprint(start_unix) + " and pull_request.merged_unix<=" + fmt.Sprint(end_unix))
+
+	issueList := make([]*Issue, 0)
+	sess.Find(&issueList)
+	resultMap := make(map[int64]int)
+	log.Info("query issue(PR) size=" + fmt.Sprint(len(issueList)))
+	for _, issueRecord := range issueList {
+		if _, ok := resultMap[issueRecord.PosterID]; !ok {
+			resultMap[issueRecord.PosterID] = 1
+		} else {
+			resultMap[issueRecord.PosterID] += 1
+		}
+	}
+	return resultMap
+}
+
 func queryAction(start_unix int64, end_unix int64, actionType int64) map[int64]int {
 	sess := x.NewSession()
 	defer sess.Close()
@@ -341,7 +406,7 @@ func queryDatasetSize(start_unix int64, end_unix int64) map[int64]int {
 func queryUserCreateRepo(start_unix int64, end_unix int64) map[int64]int {
 	sess := x.NewSession()
 	defer sess.Close()
-	sess.Select("id,owner_id,name").Table("repository").Where(" created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix))
+	sess.Select("id,owner_id,name").Table("repository").Where("is_fork=false and created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix))
 	repoList := make([]*Repository, 0)
 	sess.Find(&repoList)
 	resultMap := make(map[int64]int)
@@ -354,7 +419,6 @@ func queryUserCreateRepo(start_unix int64, end_unix int64) map[int64]int {
 		}
 	}
 	return resultMap
-
 }
 
 func subMonth(t1, t2 time.Time) (month int) {

modules/setting/setting.go

@@ -495,6 +495,9 @@ var (
 	Index         string
 	TimeField     string
 	ElkTimeFormat string
+
+	//nginx proxy
+	PROXYURL string
 )
 
 // DateLang transforms standard language locale name to corresponding value in datetime plugin.
@@ -1206,6 +1209,7 @@ func NewContext() {
 	Location = sec.Key("LOCATION").MustString("cn-south-222")
 	BasePath = sec.Key("BASE_PATH").MustString("attachment/")
 	UserBasePath = sec.Key("BASE_PATH_USER").MustString("users/")
+	PROXYURL = sec.Key("PROXY_URL").MustString("")
 
 	sec = Cfg.Section("modelarts")
 	ModelArtsHost = sec.Key("ENDPOINT").MustString("112.95.163.80")
@@ -1227,6 +1231,7 @@ func NewContext() {
 	Index = sec.Key("INDEX").MustString("filebeat-7.3.2*")
 	TimeField = sec.Key("TIMEFIELD").MustString(" @timestamptest")
 	ElkTimeFormat = sec.Key("ELKTIMEFORMAT").MustString("date_time")
+
 }
 
 func loadInternalToken(sec *ini.Section) string {

modules/storage/obs.go

@@ -5,11 +5,14 @@
 package storage
 
 import (
-	"github.com/unknwon/com"
+	"fmt"
+	"io"
 	"path"
 	"strconv"
 	"strings"
 
+	"github.com/unknwon/com"
+
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/obs"
 	"code.gitea.io/gitea/modules/setting"
@@ -102,6 +105,49 @@ func CompleteObsMultiPartUpload(uuid, uploadID, fileName string) error {
 	return nil
 }
 
+func ObsMultiPartUpload(uuid string, uploadId string, partNumber int, fileName string, putBody io.ReadCloser) error {
+	input := &obs.UploadPartInput{}
+	input.Bucket = setting.Bucket
+	input.Key = strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, fileName)), "/")
+	input.UploadId = uploadId
+	input.PartNumber = partNumber
+	input.Body = putBody
+	output, err := ObsCli.UploadPart(input)
+	if err == nil {
+		log.Info("RequestId:%s\n", output.RequestId)
+		log.Info("ETag:%s\n", output.ETag)
+		return nil
+	} else {
+		if obsError, ok := err.(obs.ObsError); ok {
+			log.Info(obsError.Code)
+			log.Info(obsError.Message)
+			return obsError
+		} else {
+			log.Error("error:", err.Error())
+			return err
+		}
+	}
+
+}
+
+func ObsDownload(uuid string, fileName string) (io.ReadCloser, error) {
+	input := &obs.GetObjectInput{}
+	input.Bucket = setting.Bucket
+	input.Key = strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, fileName)), "/")
+	output, err := ObsCli.GetObject(input)
+	if err == nil {
+		log.Info("StorageClass:%s, ETag:%s, ContentType:%s, ContentLength:%d, LastModified:%s\n",
+			output.StorageClass, output.ETag, output.ContentType, output.ContentLength, output.LastModified)
+		return output.Body, nil
+	} else if obsError, ok := err.(obs.ObsError); ok {
+		fmt.Printf("Code:%s\n", obsError.Code)
+		fmt.Printf("Message:%s\n", obsError.Message)
+		return nil, obsError
+	} else {
+		return nil, err
+	}
+}
+
 func ObsGenMultiPartSignedUrl(uuid string, uploadId string, partNumber int, fileName string) (string, error) {
 
 	input := &obs.CreateSignedUrlInput{}

public/self/labelTaskPage.js

@@ -309,11 +309,11 @@ function label_task_create(task_name, relate_task_id, taskType,assign_user_id,la
        success:function(res){
 		   console.log(res);
 		   if(res.code == 0){
-			  alert("自动标注任务创建成功!");
+			  alert("标注任务创建成功!");
 			  createsucced = true;
 		   }
 		   else{
-			  alert("创建自动标注任务失败，" + res.message);
+			  alert("创建标注任务失败，" + res.message);
 			  createsucced = false;
 		   }
       },

routers/repo/attachment.go

@@ -262,10 +262,15 @@ func GetAttachment(ctx *context.Context) {
 				return
 			}
 		} else {
-			url, err = storage.ObsGetPreSignedUrl(attach.UUID, attach.Name)
-			if err != nil {
-				ctx.ServerError("ObsGetPreSignedUrl", err)
-				return
+			if setting.PROXYURL != "" {
+				url = setting.PROXYURL + "/obs_proxy_download?uuid=" + attach.UUID + "&file_name=" + attach.Name
+				log.Info("return url=" + url)
+			} else {
+				url, err = storage.ObsGetPreSignedUrl(attach.UUID, attach.Name)
+				if err != nil {
+					ctx.ServerError("ObsGetPreSignedUrl", err)
+					return
+				}
 			}
 		}
 
@@ -273,7 +278,6 @@ func GetAttachment(ctx *context.Context) {
 			ctx.ServerError("Update", err)
 			return
 		}
-
 		http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently)
 	} else {
 		fr, err := storage.Attachments.Open(attach.RelativePath())
@@ -282,7 +286,6 @@ func GetAttachment(ctx *context.Context) {
 			return
 		}
 		defer fr.Close()
-
 		if err = increaseDownloadCount(attach, dataSet); err != nil {
 			ctx.ServerError("Update", err)
 			return
@@ -662,6 +665,53 @@ func NewMultipart(ctx *context.Context) {
 	}
 }
 
+func PutOBSProxyUpload(ctx *context.Context) {
+	uuid := ctx.Query("uuid")
+	uploadID := ctx.Query("uploadId")
+	partNumber := ctx.QueryInt("partNumber")
+	fileName := ctx.Query("file_name")
+
+	RequestBody := ctx.Req.Body()
+
+	if RequestBody == nil {
+		ctx.Error(500, fmt.Sprintf("FormFile: %v", RequestBody))
+		return
+	}
+
+	err := storage.ObsMultiPartUpload(uuid, uploadID, partNumber, fileName, RequestBody.ReadCloser())
+	if err != nil {
+		log.Info("upload error.")
+	}
+}
+
+func GetOBSProxyDownload(ctx *context.Context) {
+	uuid := ctx.Query("uuid")
+	fileName := ctx.Query("file_name")
+
+	body, err := storage.ObsDownload(uuid, fileName)
+	if err != nil {
+		log.Info("upload error.")
+	} else {
+		defer body.Close()
+		ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+fileName)
+		ctx.Resp.Header().Set("Content-Type", "application/octet-stream")
+		p := make([]byte, 1024)
+		var readErr error
+		var readCount int
+		// 读取对象内容
+		for {
+			readCount, readErr = body.Read(p)
+			if readCount > 0 {
+				ctx.Resp.Write(p[:readCount])
+				//fmt.Printf("%s", p[:readCount])
+			}
+			if readErr != nil {
+				break
+			}
+		}
+	}
+}
+
 func GetMultipartUploadUrl(ctx *context.Context) {
 	uuid := ctx.Query("uuid")
 	uploadID := ctx.Query("uploadID")
@@ -689,10 +739,16 @@ func GetMultipartUploadUrl(ctx *context.Context) {
 			return
 		}
 	} else {
-		url, err = storage.ObsGenMultiPartSignedUrl(uuid, uploadID, partNumber, fileName)
-		if err != nil {
-			ctx.Error(500, fmt.Sprintf("ObsGenMultiPartSignedUrl failed: %v", err))
-			return
+		if setting.PROXYURL != "" {
+			url = setting.PROXYURL + "/obs_proxy_multipart?uuid=" + uuid + "&uploadId=" + uploadID + "&partNumber=" + fmt.Sprint(partNumber) + "&file_name=" + fileName
+			log.Info("return url=" + url)
+		} else {
+			url, err = storage.ObsGenMultiPartSignedUrl(uuid, uploadID, partNumber, fileName)
+			if err != nil {
+				ctx.Error(500, fmt.Sprintf("ObsGenMultiPartSignedUrl failed: %v", err))
+				return
+			}
+			log.Info("url=" + url)
 		}
 	}
 

routers/routes/routes.go

@@ -6,13 +6,15 @@ package routes
 
 import (
 	"bytes"
-	"code.gitea.io/gitea/routers/operation"
 	"encoding/gob"
 	"net/http"
 	"path"
 	"text/template"
 	"time"
 
+	"code.gitea.io/gitea/routers/operation"
+	"code.gitea.io/gitea/routers/private"
+
 	"code.gitea.io/gitea/routers/secure"
 
 	"code.gitea.io/gitea/models"
@@ -33,7 +35,6 @@ import (
 	"code.gitea.io/gitea/routers/dev"
 	"code.gitea.io/gitea/routers/events"
 	"code.gitea.io/gitea/routers/org"
-	"code.gitea.io/gitea/routers/private"
 	"code.gitea.io/gitea/routers/repo"
 	"code.gitea.io/gitea/routers/user"
 	userSetting "code.gitea.io/gitea/routers/user/setting"
@@ -573,6 +574,8 @@ func RegisterRoutes(m *macaron.Macaron) {
 		m.Post("/private", repo.UpdatePublicAttachment)
 		m.Get("/get_chunks", repo.GetSuccessChunks)
 		m.Get("/new_multipart", repo.NewMultipart)
+		m.Put("/obs_proxy_multipart", repo.PutOBSProxyUpload)
+		m.Get("/obs_proxy_download", repo.GetOBSProxyDownload)
 		m.Get("/get_multipart_url", repo.GetMultipartUploadUrl)
 		m.Post("/complete_multipart", repo.CompleteMultipart)
 		m.Post("/update_chunk", repo.UpdateMultipart)

templates/repo/datasets/label/index.tmpl

@@ -123,7 +123,7 @@
                                                                     <select  name="pre_predict_task" id="dataset_list_auto"  onchange="dataset_auto_sele_Change(this)">
                                                                             {{if .Attachments}}
                                                                                 {{range .Attachments}}
-                                                                                        <option value="{{.UUID}}">{{.Name}}</option>
+                                                                                    <option value="{{.UUID}}">{{.Name}}</option>
                                                                                 {{end}}
                                                                             {{end}}
                                                                     </select>