diff --git a/custom/public/css/git.openi.css b/custom/public/css/git.openi.css index c6ada7b28..a4920eec0 100644 --- a/custom/public/css/git.openi.css +++ b/custom/public/css/git.openi.css @@ -44,12 +44,6 @@ -webkit-line-clamp: 2; -webkit-box-orient: vertical; } -.ui.label{ - font-weight: normal; -} -.active { - color: #0366D6 !important; -} .opacity5{ opacity:0.5;} .radius15{ border-radius:1.5rem !important; } @@ -287,70 +281,6 @@ position: relative; } -/**seach**/ -/**搜索导航条适配窄屏**/ -.seachnav{ - overflow-x: auto; - overflow-y: hidden; - scrollbar-width: none; /* firefox */ - -ms-overflow-style: none; /* IE 10+ */ -} -.seachnav::-webkit-scrollbar { - display: none; /* Chrome Safari */ -} -.ui.green.button, .ui.green.buttons .button{ - background-color: #5BB973; -} -.seach .repos--seach{ - padding-bottom: 0; - border-bottom: none; -} -.seach .ui.secondary.pointing.menu{ - border-bottom: none; -} -.seach .ui.secondary.pointing.menu .item > i{ - margin-right: 5px; -} -.seach .ui.secondary.pointing.menu .active.item{ - border-bottom-width: 2px; - margin: 0 0 -1px; -} -.seach .ui.menu .active.item>.label { - background: #1684FC; - color: #FFF; -} -.seach .ui.menu .item>.label:not(.active.item>.label) { - background: #e8e8e8; - color: rgba(0,0,0,.6); -} - -.highlight{ - color: red; -} -.ui.list .list>.item>img.image+.content, .ui.list>.item>img.image+.content { - width: calc(100% - 3.0em); - margin-left: 0; -} - -.seach .ui.list .list>.item .header, .seach .ui.list>.item .header{ - margin-bottom: 0.5em; - font-size: 1.4rem !important; - font-weight: normal; -} -.seach .time, .seach .time a{ - font-size: 12px; - color: grey; -} - -.seach .list .item.members .ui.avatar.image { - width: 3.2em; - height: 3.2em; -} -.ui.list .list>.item.members>img.image+.content, .ui.list>.item.members>img.image+.content { - width: calc(100% - 4.0em); - margin-left: 0; -} - @media only screen and (max-width: 767px) { .am-mt-30{ margin-top: 1.5rem !important;} .ui.secondary.hometop.segment{ diff --git a/models/action.go b/models/action.go index 2a9d88399..9b92b4192 100755 --- a/models/action.go +++ b/models/action.go @@ -57,6 +57,7 @@ const ( ActionCreateInferenceTask // 28 ActionCreateBenchMarkTask //29 ActionCreateNewModelTask //30 + ActionCreateGPUTrainTask //31 ) // Action represents user operation type and other information to diff --git a/models/attachment.go b/models/attachment.go index a3fc6fa01..7c95a73dd 100755 --- a/models/attachment.go +++ b/models/attachment.go @@ -51,6 +51,7 @@ type Attachment struct { FileChunk *FileChunk `xorm:"-"` CanDel bool `xorm:"-"` Uploader *User `xorm:"-"` + Md5 string `xorm:"-"` } type AttachmentUsername struct { diff --git a/models/cloudbrain.go b/models/cloudbrain.go index ea6d0338e..4a82a2031 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -20,9 +20,17 @@ type CloudbrainStatus string type JobType string type ModelArtsJobStatus string +const ( + TypeCloudBrainOne int = iota + TypeCloudBrainTwo + + TypeCloudBrainAll = -1 +) + const ( NPUResource = "NPU" GPUResource = "CPU/GPU" + AllResource = "all" //notebook storage category EVSCategory = "EVS" @@ -87,6 +95,8 @@ const ( ModelArtsTrainJobCheckRunning ModelArtsJobStatus = "CHECK_RUNNING" //审核作业正在运行中 ModelArtsTrainJobCheckRunningCompleted ModelArtsJobStatus = "CHECK_RUNNING_COMPLETED" //审核作业已经完成 ModelArtsTrainJobCheckFailed ModelArtsJobStatus = "CHECK_FAILED" //审核作业失败 + + DURATION_STR_ZERO = "00:00:00" ) type Cloudbrain struct { @@ -160,7 +170,9 @@ func (task *Cloudbrain) ComputeAndSetDuration() { if task.StartTime == 0 { d = 0 } else if task.EndTime == 0 { - d = time.Now().Unix() - task.StartTime.AsTime().Unix() + if !task.IsTerminal() { + d = time.Now().Unix() - task.StartTime.AsTime().Unix() + } } else { d = task.EndTime.AsTime().Unix() - task.StartTime.AsTime().Unix() } @@ -172,9 +184,14 @@ func (task *Cloudbrain) ComputeAndSetDuration() { task.TrainJobDuration = ConvertDurationToStr(d) } +func (task *Cloudbrain) IsTerminal() bool { + status := task.Status + return status == string(ModelArtsTrainJobCompleted) || status == string(ModelArtsTrainJobFailed) || status == string(ModelArtsTrainJobKilled) || status == string(ModelArtsStopped) || status == string(JobStopped) || status == string(JobFailed) || status == string(JobSucceeded) +} + func ConvertDurationToStr(duration int64) string { if duration == 0 { - return "00:00:00" + return DURATION_STR_ZERO } return util.AddZero(duration/3600) + ":" + util.AddZero(duration%3600/60) + ":" + util.AddZero(duration%60) } @@ -191,6 +208,19 @@ func IsCloudBrainOneDebugJobTerminal(status string) bool { return status == string(JobStopped) || status == string(JobFailed) || status == string(JobSucceeded) } +func ParseAndSetDurationFromCloudBrainOne(result JobResultPayload, task *Cloudbrain) { + isActivated := result.JobStatus.CreatedTime > 0 + if task.StartTime == 0 && isActivated { + task.StartTime = timeutil.TimeStamp(result.JobStatus.CreatedTime / 1000) + } + if task.EndTime == 0 && IsCloudBrainOneDebugJobTerminal(task.Status) && isActivated { + if result.JobStatus.CompletedTime > 0 { + task.EndTime = timeutil.TimeStamp(result.JobStatus.CompletedTime / 1000) + } + } + task.ComputeAndSetDuration() +} + type CloudbrainInfo struct { Cloudbrain `xorm:"extends"` User `xorm:"extends"` @@ -1323,6 +1353,7 @@ func CloudbrainsVersionList(opts *CloudbrainsOptions) ([]*CloudbrainInfo, int, e } func CreateCloudbrain(cloudbrain *Cloudbrain) (err error) { + cloudbrain.TrainJobDuration = DURATION_STR_ZERO if _, err = x.Insert(cloudbrain); err != nil { return err } @@ -1339,6 +1370,16 @@ func getRepoCloudBrain(cb *Cloudbrain) (*Cloudbrain, error) { return cb, nil } +func getRepoCloudBrainWithDeleted(cb *Cloudbrain) (*Cloudbrain, error) { + has, err := x.Unscoped().Get(cb) + if err != nil { + return nil, err + } else if !has { + return nil, ErrJobNotExist{} + } + return cb, nil +} + func GetRepoCloudBrainByJobID(repoID int64, jobID string) (*Cloudbrain, error) { cb := &Cloudbrain{JobID: jobID, RepoID: repoID} return getRepoCloudBrain(cb) @@ -1355,6 +1396,12 @@ func GetCloudbrainByID(id string) (*Cloudbrain, error) { return getRepoCloudBrain(cb) } +func GetCloudbrainByIDWithDeleted(id string) (*Cloudbrain, error) { + idInt64, _ := strconv.ParseInt(id, 10, 64) + cb := &Cloudbrain{ID: idInt64} + return getRepoCloudBrainWithDeleted(cb) +} + func GetCloudbrainByJobIDAndVersionName(jobID string, versionName string) (*Cloudbrain, error) { cb := &Cloudbrain{JobID: jobID, VersionName: versionName} return getRepoCloudBrain(cb) @@ -1467,6 +1514,15 @@ func GetCloudBrainUnStoppedJob() ([]*Cloudbrain, error) { Find(&cloudbrains) } +func GetStoppedJobWithNoDurationJob() ([]*Cloudbrain, error) { + cloudbrains := make([]*Cloudbrain, 0) + return cloudbrains, x. + In("status", ModelArtsTrainJobCompleted, ModelArtsTrainJobFailed, ModelArtsTrainJobKilled, ModelArtsStopped, JobStopped, JobFailed, JobSucceeded). + Where("train_job_duration is null or train_job_duration = '' "). + Limit(100). + Find(&cloudbrains) +} + func GetCloudbrainCountByUserID(userID int64, jobType string) (int, error) { count, err := x.In("status", JobWaiting, JobRunning).And("job_type = ? and user_id = ? and type = ?", jobType, userID, TypeCloudBrainOne).Count(new(Cloudbrain)) return int(count), err diff --git a/models/dataset.go b/models/dataset.go index af47c53fe..95800100c 100755 --- a/models/dataset.go +++ b/models/dataset.go @@ -1,6 +1,7 @@ package models import ( + "code.gitea.io/gitea/modules/log" "errors" "fmt" "sort" @@ -62,19 +63,20 @@ func (datasets DatasetList) loadAttributes(e Engine) error { } set := make(map[int64]struct{}) + userIdSet := make(map[int64]struct{}) datasetIDs := make([]int64, len(datasets)) for i := range datasets { - set[datasets[i].UserID] = struct{}{} + userIdSet[datasets[i].UserID] = struct{}{} set[datasets[i].RepoID] = struct{}{} datasetIDs[i] = datasets[i].ID } // Load owners. - users := make(map[int64]*User, len(set)) + users := make(map[int64]*User, len(userIdSet)) repos := make(map[int64]*Repository, len(set)) if err := e. Where("id > 0"). - In("id", keysInt64(set)). + In("id", keysInt64(userIdSet)). Find(&users); err != nil { return fmt.Errorf("find users: %v", err) } @@ -139,20 +141,7 @@ func SearchDatasetCondition(opts *SearchDatasetOptions) builder.Cond { var cond = builder.NewCond() cond = cond.And(builder.Neq{"dataset.status": DatasetStatusDeleted}) - if len(opts.Keyword) > 0 { - cond = cond.And(builder.Or(builder.Like{"dataset.title", opts.Keyword}, builder.Like{"dataset.description", opts.Keyword})) - } - - if len(opts.Category) > 0 { - cond = cond.And(builder.Eq{"dataset.category": opts.Category}) - } - - if len(opts.Task) > 0 { - cond = cond.And(builder.Eq{"dataset.task": opts.Task}) - } - if len(opts.License) > 0 { - cond = cond.And(builder.Eq{"dataset.license": opts.License}) - } + cond = generateFilterCond(opts, cond) if opts.RepoID > 0 { cond = cond.And(builder.Eq{"dataset.repo_id": opts.RepoID}) @@ -162,14 +151,12 @@ func SearchDatasetCondition(opts *SearchDatasetOptions) builder.Cond { cond = cond.And(builder.Eq{"dataset.status": DatasetStatusPublic}) cond = cond.And(builder.Eq{"attachment.is_private": false}) if opts.OwnerID > 0 { - if len(opts.Keyword) == 0 { - cond = cond.Or(builder.Eq{"repository.owner_id": opts.OwnerID}) - } else { - subCon := builder.NewCond() - subCon = subCon.And(builder.Eq{"repository.owner_id": opts.OwnerID}, builder.Or(builder.Like{"dataset.title", opts.Keyword}, builder.Like{"dataset.description", opts.Keyword})) - cond = cond.Or(subCon) - - } + + subCon := builder.NewCond() + subCon = subCon.And(builder.Eq{"repository.owner_id": opts.OwnerID}) + subCon = generateFilterCond(opts, subCon) + cond = cond.Or(subCon) + } } else if opts.OwnerID > 0 { cond = cond.And(builder.Eq{"repository.owner_id": opts.OwnerID}) @@ -182,6 +169,25 @@ func SearchDatasetCondition(opts *SearchDatasetOptions) builder.Cond { return cond } +func generateFilterCond(opts *SearchDatasetOptions, cond builder.Cond) builder.Cond { + if len(opts.Keyword) > 0 { + cond = cond.And(builder.Or(builder.Like{"dataset.title", opts.Keyword}, builder.Like{"dataset.description", opts.Keyword})) + } + + if len(opts.Category) > 0 { + cond = cond.And(builder.Eq{"dataset.category": opts.Category}) + } + + if len(opts.Task) > 0 { + cond = cond.And(builder.Eq{"dataset.task": opts.Task}) + } + if len(opts.License) > 0 { + cond = cond.And(builder.Eq{"dataset.license": opts.License}) + } + + return cond +} + func SearchDatasetByCondition(opts *SearchDatasetOptions, cond builder.Cond) (DatasetList, int64, error) { if opts.Page <= 0 { opts.Page = 1 @@ -292,7 +298,13 @@ func getDatasetAttachments(e Engine, typeCloudBrain int, isSigned bool, user *Us if err != nil { return err } - attachment.FileChunk = fileChunks[0] + if len(fileChunks) > 0 { + attachment.Md5 = fileChunks[0].Md5 + } else { + log.Error("has attachment record, but has no file_chunk record") + attachment.Md5 = "no_record" + } + attachment.CanDel = CanDelAttachment(isSigned, user, attachment) sortedRels.Rel[currentIndex].Attachments = append(sortedRels.Rel[currentIndex].Attachments, attachment) } @@ -348,7 +360,7 @@ func GetDatasetByRepo(repo *Repository) (*Dataset, error) { if has { return dataset, nil } else { - return nil, errors.New("Not Found") + return nil, ErrNotExist{repo.ID} } } diff --git a/models/dbsql/dataset_foreigntable_for_es.sql b/models/dbsql/dataset_foreigntable_for_es.sql new file mode 100644 index 000000000..02e5f0ddf --- /dev/null +++ b/models/dbsql/dataset_foreigntable_for_es.sql @@ -0,0 +1,199 @@ +DELETE FROM public.dataset_es; +DROP FOREIGN TABLE public.dataset_es; +DROP TRIGGER IF EXISTS es_insert_dataset on public.dataset; +DROP FUNCTION public.insert_dataset_data(); +DROP TRIGGER IF EXISTS es_udpate_dataset_file_name on public.attachment; +DROP FUNCTION public.udpate_dataset_file_name; + +DROP TRIGGER IF EXISTS es_update_dataset on public.dataset; +DROP FUNCTION public.update_dataset; + +DROP TRIGGER IF EXISTS es_delete_dataset on public.dataset; +DROP FUNCTION public.delete_dataset; + + +CREATE FOREIGN TABLE public.dataset_es +( + id bigint NOT NULL, + title character varying(255), + status integer, + category character varying(255), + description text, + download_times bigint, + license character varying(255), + task character varying(255), + release_id bigint, + user_id bigint, + repo_id bigint, + created_unix bigint, + updated_unix bigint, + file_name text, + file_desc text +)SERVER multicorn_es +OPTIONS + ( + host '192.168.207.94', + port '9200', + index 'dataset-es-index', + rowid_column 'id', + default_sort '_id' + ) +; +DELETE FROM public.dataset_es; + INSERT INTO public.dataset_es( + id, + title, + status, + category, + description, + download_times, + license, task, + release_id, + user_id, + repo_id, + created_unix, + updated_unix, + file_name, + file_desc + ) + SELECT + b.id, + b.title, + b.status, + b.category, + b.description, + b.download_times, + b.license, + b.task, + b.release_id, + b.user_id, + b.repo_id, + b.created_unix, + b.updated_unix, + (select array_to_string(array_agg(name order by created_unix desc),'-#,#-') from public.attachment a where a.dataset_id=b.id and a.is_private=false), + (select array_to_string(array_agg(description order by created_unix desc),'-#,#-') from public.attachment a where a.dataset_id=b.id and a.is_private=false) + FROM public.dataset b,public.repository c where b.repo_id=c.id and c.is_private=false; + + +DROP TRIGGER IF EXISTS es_insert_dataset on public.dataset; + +CREATE OR REPLACE FUNCTION public.insert_dataset_data() RETURNS trigger AS +$def$ + DECLARE + privateValue boolean=false; + BEGIN + select into privateValue is_private from public.repository where id=NEW.repo_id; + if not privateValue then + INSERT INTO public.dataset_es( + id, + title, + status, + category, + description, + download_times, + license, + task, + release_id, + user_id, + repo_id, + created_unix, + updated_unix) + VALUES ( + NEW.id, + NEW.title, + NEW.status, + NEW.category, + NEW.description, + NEW.download_times, + NEW.license, + NEW.task, + NEW.release_id, + NEW.user_id, + NEW.repo_id, + NEW.created_unix, + NEW.updated_unix + ); + end if; + RETURN NEW; + END; +$def$ +LANGUAGE plpgsql; + + + +CREATE TRIGGER es_insert_dataset + AFTER INSERT ON public.dataset + FOR EACH ROW EXECUTE PROCEDURE insert_dataset_data(); + +ALTER TABLE public.dataset ENABLE ALWAYS TRIGGER es_insert_dataset; + + +DROP TRIGGER IF EXISTS es_udpate_dataset_file_name on public.attachment; + +CREATE OR REPLACE FUNCTION public.udpate_dataset_file_name() RETURNS trigger AS +$def$ + BEGIN + if (TG_OP = 'UPDATE') then + update public.dataset_es SET file_desc=(select array_to_string(array_agg(description order by created_unix desc),'-#,#-') from public.attachment where dataset_id=NEW.dataset_id and is_private=false) where id=NEW.dataset_id; + elsif (TG_OP = 'INSERT') then + update public.dataset_es SET file_name=(select array_to_string(array_agg(name order by created_unix desc),'-#,#-') from public.attachment where dataset_id=NEW.dataset_id and is_private=false) where id=NEW.dataset_id; + elsif (TG_OP = 'DELETE') then + update public.dataset_es SET file_name=(select array_to_string(array_agg(name order by created_unix desc),'-#,#-') from public.attachment where dataset_id=OLD.dataset_id and is_private=false) where id=OLD.dataset_id; + update public.dataset_es SET file_desc=(select array_to_string(array_agg(description order by created_unix desc),'-#,#-') from public.attachment where dataset_id=OLD.dataset_id and is_private=false) where id=OLD.dataset_id; + end if; + return NEW; + END; +$def$ +LANGUAGE plpgsql; + + +CREATE TRIGGER es_udpate_dataset_file_name + AFTER INSERT OR UPDATE OR DELETE ON public.attachment + FOR EACH ROW EXECUTE PROCEDURE udpate_dataset_file_name(); + +ALTER TABLE public.attachment ENABLE ALWAYS TRIGGER es_udpate_dataset_file_name; + +DROP TRIGGER IF EXISTS es_update_dataset on public.dataset; + +CREATE OR REPLACE FUNCTION public.update_dataset() RETURNS trigger AS +$def$ + BEGIN + UPDATE public.dataset_es + SET description=NEW.description, + title=NEW.title, + category=NEW.category, + task=NEW.task, + download_times=NEW.download_times, + updated_unix=NEW.updated_unix, + file_name=(select array_to_string(array_agg(name order by created_unix desc),'-#,#-') from public.attachment where dataset_id=NEW.id and is_private=false), + file_desc=(select array_to_string(array_agg(description order by created_unix desc),'-#,#-') from public.attachment where dataset_id=NEW.id and is_private=false) + where id=NEW.id; + return new; + END +$def$ +LANGUAGE plpgsql; + +CREATE TRIGGER es_update_dataset + AFTER UPDATE ON public.dataset + FOR EACH ROW EXECUTE PROCEDURE update_dataset(); + +ALTER TABLE public.dataset ENABLE ALWAYS TRIGGER es_update_dataset; + +DROP TRIGGER IF EXISTS es_delete_dataset on public.dataset; + +CREATE OR REPLACE FUNCTION public.delete_dataset() RETURNS trigger AS +$def$ + declare + BEGIN + DELETE FROM public.dataset_es where id=OLD.id; + return new; + END +$def$ +LANGUAGE plpgsql; + + +CREATE TRIGGER es_delete_dataset + AFTER DELETE ON public.dataset + FOR EACH ROW EXECUTE PROCEDURE delete_dataset(); + +ALTER TABLE public.dataset ENABLE ALWAYS TRIGGER es_delete_dataset; diff --git a/models/dbsql/issue_foreigntable_for_es.sql b/models/dbsql/issue_foreigntable_for_es.sql new file mode 100644 index 000000000..d6a16cd27 --- /dev/null +++ b/models/dbsql/issue_foreigntable_for_es.sql @@ -0,0 +1,227 @@ +delete from public.issue_es; +DROP FOREIGN TABLE public.issue_es; +DROP TRIGGER IF EXISTS es_insert_issue on public.issue; +DROP FUNCTION public.insert_issue_data; +DROP TRIGGER IF EXISTS es_udpate_issue_comment on public.comment; +DROP FUNCTION udpate_issue_comment; +DROP TRIGGER IF EXISTS es_update_issue on public.issue; +DROP FUNCTION public.update_issue; +DROP TRIGGER IF EXISTS es_delete_issue on public.issue; +DROP FUNCTION public.delete_issue; + + +CREATE FOREIGN TABLE public.issue_es +( + id bigint NOT NULL, + repo_id bigint, + index bigint, + poster_id bigint, + original_author character varying(255), + original_author_id bigint, + name character varying(255) , + content text, + comment text, + milestone_id bigint, + priority integer, + is_closed boolean, + is_pull boolean, + pr_id bigint, + num_comments integer, + ref character varying(255), + deadline_unix bigint, + created_unix bigint, + updated_unix bigint, + closed_unix bigint, + is_locked boolean NOT NULL, + amount bigint, + is_transformed boolean NOT NULL +)SERVER multicorn_es +OPTIONS + ( + host '192.168.207.94', + port '9200', + index 'issue-es-index', + rowid_column 'id', + default_sort '_id' + ) +; + +delete from public.issue_es; +INSERT INTO public.issue_es( + id, + repo_id, + index, + poster_id, + original_author, + original_author_id, + name, + content, + milestone_id, + priority, + is_closed, + is_pull, + num_comments, + ref, + deadline_unix, + created_unix, + updated_unix, + closed_unix, + is_locked, + amount, + is_transformed,comment,pr_id) + SELECT + b.id, + b.repo_id, + b.index, + b.poster_id, + b.original_author, + b.original_author_id, + b.name, + b.content, + b.milestone_id, + b.priority, + b.is_closed, + b.is_pull, + b.num_comments, + b.ref, + b.deadline_unix, + b.created_unix, + b.updated_unix, + b.closed_unix, + b.is_locked, + b.amount, + b.is_transformed, + (select array_to_string(array_agg(content order by created_unix desc),',') from public.comment a where a.issue_id=b.id), + (select id from public.pull_request d where b.id=d.issue_id and b.is_pull=true) + FROM public.issue b,public.repository c where b.repo_id=c.id and c.is_private=false; + + +CREATE OR REPLACE FUNCTION public.insert_issue_data() RETURNS trigger AS +$def$ + DECLARE + privateValue boolean=false; + BEGIN + select into privateValue is_private from public.repository where id=NEW.repo_id; + if not privateValue then + INSERT INTO public.issue_es( + id, + repo_id, + index, + poster_id, + original_author, + original_author_id, + name, + content, + milestone_id, + priority, + is_closed, + is_pull, + num_comments, + ref, + deadline_unix, + created_unix, + updated_unix, + closed_unix, + is_locked, + amount, + is_transformed) + VALUES ( + NEW.id, + NEW.repo_id, + NEW.index, + NEW.poster_id, + NEW.original_author, + NEW.original_author_id, + NEW.name, + NEW.content, + NEW.milestone_id, + NEW.priority, + NEW.is_closed, + NEW.is_pull, + NEW.num_comments, + NEW.ref, + NEW.deadline_unix, + NEW.created_unix, + NEW.updated_unix, + NEW.closed_unix, + NEW.is_locked, + NEW.amount, + NEW.is_transformed + ); + end if; + RETURN NEW; + END; +$def$ +LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS es_insert_issue on public.issue; + +CREATE TRIGGER es_insert_issue + AFTER INSERT ON public.issue + FOR EACH ROW EXECUTE PROCEDURE insert_issue_data(); + +ALTER TABLE public.issue ENABLE ALWAYS TRIGGER es_insert_issue; + +CREATE OR REPLACE FUNCTION public.udpate_issue_comment() RETURNS trigger AS +$def$ + BEGIN + if (TG_OP = 'DELETE') then + update public.issue_es SET comment=(select array_to_string(array_agg(content order by created_unix desc),',') from public.comment where issue_id=OLD.issue_id) where id=OLD.issue_id; + elsif (TG_OP = 'UPDATE') then + update public.issue_es SET comment=(select array_to_string(array_agg(content order by created_unix desc),',') from public.comment where issue_id=NEW.issue_id) where id=NEW.issue_id; + end if; + + return null; + END; +$def$ +LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS es_udpate_issue_comment on public.comment; +CREATE TRIGGER es_udpate_issue_comment + AFTER DELETE OR UPDATE ON public.comment + FOR EACH ROW EXECUTE PROCEDURE udpate_issue_comment(); + +ALTER TABLE public.comment ENABLE ALWAYS TRIGGER es_udpate_issue_comment; + + +CREATE OR REPLACE FUNCTION public.update_issue() RETURNS trigger AS +$def$ + declare + BEGIN + UPDATE public.issue_es + SET content=NEW.content, + name=NEW.name, + is_closed=NEW.is_closed, + num_comments=NEW.num_comments, + updated_unix=NEW.updated_unix, + comment=(select array_to_string(array_agg(content order by created_unix desc),',') from public.comment where issue_id=NEW.id) + where id=NEW.id; + return new; + END +$def$ +LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS es_update_issue on public.issue; + +CREATE TRIGGER es_update_issue + AFTER UPDATE ON public.issue + FOR EACH ROW EXECUTE PROCEDURE update_issue(); + +ALTER TABLE public.issue ENABLE ALWAYS TRIGGER es_update_issue; + +CREATE OR REPLACE FUNCTION public.delete_issue() RETURNS trigger AS +$def$ + declare + BEGIN + DELETE FROM public.issue_es where id=OLD.id; + return new; + END +$def$ +LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS es_delete_issue on public.issue; +CREATE TRIGGER es_delete_issue + AFTER DELETE ON public.issue + FOR EACH ROW EXECUTE PROCEDURE delete_issue(); + +ALTER TABLE public.issue ENABLE ALWAYS TRIGGER es_delete_issue; \ No newline at end of file diff --git a/models/dbsql/repo_foreigntable_for_es.sql b/models/dbsql/repo_foreigntable_for_es.sql new file mode 100644 index 000000000..7e06fd99e --- /dev/null +++ b/models/dbsql/repo_foreigntable_for_es.sql @@ -0,0 +1,545 @@ +-- 要处理项目从私有变为公有，并且从公有变成私有的情况 +DELETE FROM public.repository_es; +DROP FOREIGN table if exists public.repository_es; +DROP TRIGGER IF EXISTS es_insert_repository on public.repository; +DROP FUNCTION public.insert_repository_data; +DROP TRIGGER IF EXISTS es_update_repository on public.repository; +DROP FUNCTION public.update_repository; + +DROP TRIGGER IF EXISTS es_delete_repository on public.repository; +DROP FUNCTION public.delete_repository; + +DROP TRIGGER IF EXISTS es_udpate_repository_lang on public.language_stat; +DROP FUNCTION public.udpate_repository_lang; + + +CREATE FOREIGN TABLE public.repository_es ( + id bigint NOT NULL, + owner_id bigint, + owner_name character varying(255), + lower_name character varying(255) NOT NULL, + name character varying(255) NOT NULL, + description text, + website character varying(2048), + original_service_type integer, + original_url character varying(2048), + default_branch character varying(255), + num_watches integer, + num_stars integer, + num_forks integer, + num_issues integer, + num_closed_issues integer, + num_pulls integer, + num_closed_pulls integer, + num_milestones integer DEFAULT 0 NOT NULL, + num_closed_milestones integer DEFAULT 0 NOT NULL, + is_private boolean, + is_empty boolean, + is_archived boolean, + is_mirror boolean, + status integer DEFAULT 0 NOT NULL, + is_fork boolean DEFAULT false NOT NULL, + fork_id bigint, + is_template boolean DEFAULT false NOT NULL, + template_id bigint, + size bigint DEFAULT 0 NOT NULL, + is_fsck_enabled boolean DEFAULT true NOT NULL, + close_issues_via_commit_in_any_branch boolean DEFAULT false NOT NULL, + topics text, + avatar character varying(64), + created_unix bigint, + updated_unix bigint, + contract_address character varying(255), + block_chain_status integer DEFAULT 0 NOT NULL, + balance character varying(255) DEFAULT '0'::character varying NOT NULL, + clone_cnt bigint DEFAULT 0 NOT NULL, + license character varying(100), + download_cnt bigint DEFAULT 0 NOT NULL, + num_commit bigint DEFAULT 0 NOT NULL, + git_clone_cnt bigint DEFAULT 0 NOT NULL, + creator_id bigint NOT NULL DEFAULT 0, + repo_type integer NOT NULL DEFAULT 0, + lang character varying(2048), + alias character varying(255), + lower_alias character varying(255) +) SERVER multicorn_es +OPTIONS + ( + host '192.168.207.94', + port '9200', + index 'repository-es-index', + rowid_column 'id', + default_sort '_id' + ) +; +delete from public.repository_es; + INSERT INTO public.repository_es (id, + owner_id, + owner_name, + lower_name, + name, + description, + website, + original_service_type, + original_url, + default_branch, + num_watches, + num_stars, + num_forks, + num_issues, + num_closed_issues, + num_pulls, + num_closed_pulls, + num_milestones, + num_closed_milestones, + is_private, + is_empty, + is_archived, + is_mirror, + status, + is_fork, + fork_id, + is_template, + template_id, + size, + is_fsck_enabled, + close_issues_via_commit_in_any_branch, + topics, + avatar, + created_unix, + updated_unix, + contract_address, + block_chain_status, + balance, + clone_cnt, + num_commit, + git_clone_cnt, + creator_id, + repo_type, + lang, + alias, + lower_alias + ) + SELECT + id, + owner_id, + owner_name, + lower_name, + name, + description, + website, + original_service_type, + original_url, + default_branch, + num_watches, + num_stars, + num_forks, + num_issues, + num_closed_issues, + num_pulls, + num_closed_pulls, + num_milestones, + num_closed_milestones, + is_private, + is_empty, + is_archived, + is_mirror, + status, + is_fork, + fork_id, + is_template, + template_id, + size, + is_fsck_enabled, + close_issues_via_commit_in_any_branch, + topics, + avatar, + created_unix, + updated_unix, + contract_address, + block_chain_status, + balance, + clone_cnt, + num_commit, + git_clone_cnt, + creator_id, + repo_type, + (select array_to_string(array_agg(language order by percentage desc),',') from public.language_stat a where a.repo_id=b.id), + alias, + lower_alias + FROM public.repository b where b.is_private=false; + +DROP TRIGGER IF EXISTS es_insert_repository on public.repository; + +CREATE OR REPLACE FUNCTION public.insert_repository_data() RETURNS trigger AS +$def$ + BEGIN + if not NEW.is_private then + INSERT INTO public.repository_es (id, + owner_id, + owner_name, + lower_name, + name, + description, + website, + original_service_type, + original_url, + default_branch, + num_watches, + num_stars, + num_forks, + num_issues, + num_closed_issues, + num_pulls, + num_closed_pulls, + num_milestones, + num_closed_milestones, + is_private, + is_empty, + is_archived, + is_mirror, + status, + is_fork, + fork_id, + is_template, + template_id, + size, + is_fsck_enabled, + close_issues_via_commit_in_any_branch, + topics, + avatar, + created_unix, + updated_unix, + contract_address, + block_chain_status, + balance, + clone_cnt, + num_commit, + git_clone_cnt, + creator_id, + repo_type, + alias, + lower_alias) VALUES + (NEW.id, + NEW.owner_id, + NEW.owner_name, + NEW.lower_name, + NEW.name, + NEW.description, + NEW.website, + NEW.original_service_type, + NEW.original_url, + NEW.default_branch, + NEW.num_watches, + NEW.num_stars, + NEW.num_forks, + NEW.num_issues, + NEW.num_closed_issues, + NEW.num_pulls, + NEW.num_closed_pulls, + NEW.num_milestones, + NEW.num_closed_milestones, + NEW.is_private, + NEW.is_empty, + NEW.is_archived, + NEW.is_mirror, + NEW.status, + NEW.is_fork, + NEW.fork_id, + NEW.is_template, + NEW.template_id, + NEW.size, + NEW.is_fsck_enabled, + NEW.close_issues_via_commit_in_any_branch, + NEW.topics, + NEW.avatar, + NEW.created_unix, + NEW.updated_unix, + NEW.contract_address, + NEW.block_chain_status, + NEW.balance, + NEW.clone_cnt, + NEW.num_commit, + NEW.git_clone_cnt, + NEW.creator_id, + NEW.repo_type, + NEW.alias, + NEW.lower_alias); + end if; + RETURN NEW; + END; +$def$ +LANGUAGE plpgsql; + + +CREATE TRIGGER es_insert_repository + AFTER INSERT ON public.repository + FOR EACH ROW EXECUTE PROCEDURE insert_repository_data(); + +ALTER TABLE public.repository ENABLE ALWAYS TRIGGER es_insert_repository; + +DROP TRIGGER IF EXISTS es_update_repository on public.repository; + +CREATE OR REPLACE FUNCTION public.update_repository() RETURNS trigger AS +$def$ + BEGIN + if OLD.is_private != NEW.is_private then + if OLD.is_private and not NEW.is_private then + --insert + INSERT INTO public.repository_es (id, + owner_id, + owner_name, + lower_name, + name, + description, + website, + original_service_type, + original_url, + default_branch, + num_watches, + num_stars, + num_forks, + num_issues, + num_closed_issues, + num_pulls, + num_closed_pulls, + num_milestones, + num_closed_milestones, + is_private, + is_empty, + is_archived, + is_mirror, + status, + is_fork, + fork_id, + is_template, + template_id, + size, + is_fsck_enabled, + close_issues_via_commit_in_any_branch, + topics, + avatar, + created_unix, + updated_unix, + contract_address, + block_chain_status, + balance, + clone_cnt, + num_commit, + git_clone_cnt, + creator_id, + repo_type, + lang, + alias, + lower_alias) + SELECT + id, + owner_id, + owner_name, + lower_name, + name, + description, + website, + original_service_type, + original_url, + default_branch, + num_watches, + num_stars, + num_forks, + num_issues, + num_closed_issues, + num_pulls, + num_closed_pulls, + num_milestones, + num_closed_milestones, + is_private, + is_empty, + is_archived, + is_mirror, + status, + is_fork, + fork_id, + is_template, + template_id, + size, + is_fsck_enabled, + close_issues_via_commit_in_any_branch, + topics, + avatar, + created_unix, + updated_unix, + contract_address, + block_chain_status, + balance, + clone_cnt, + num_commit, + git_clone_cnt, + creator_id, + repo_type, + (select array_to_string(array_agg(language order by percentage desc),',') from public.language_stat a where a.repo_id=b.id), + alias, + lower_alias + FROM public.repository b where b.id=NEW.id; + INSERT INTO public.dataset_es( + id, + title, + status, + category, + description, + download_times, + license, task, + release_id, + user_id, + repo_id, + created_unix, + updated_unix,file_name) + SELECT + b.id, + b.title, + b.status, + b.category, + b.description, + b.download_times, + b.license, + b.task, + b.release_id, + b.user_id, + b.repo_id, + b.created_unix, + b.updated_unix,(select array_to_string(array_agg(name order by created_unix desc),',') from public.attachment a where a.dataset_id=b.id and a.is_private=false) + FROM public.dataset b where b.repo_id=NEW.id; + + INSERT INTO public.issue_es( + id, + repo_id, + index, + poster_id, + original_author, + original_author_id, + name, + content, + milestone_id, + priority, + is_closed, + is_pull, + num_comments, + ref, + deadline_unix, + created_unix, + updated_unix, + closed_unix, + is_locked, + amount, + is_transformed,comment,pr_id) + SELECT + b.id, + b.repo_id, + b.index, + b.poster_id, + b.original_author, + b.original_author_id, + b.name, + b.content, + b.milestone_id, + b.priority, + b.is_closed, + b.is_pull, + b.num_comments, + b.ref, + b.deadline_unix, + b.created_unix, + b.updated_unix, + b.closed_unix, + b.is_locked, + b.amount, + b.is_transformed, + (select array_to_string(array_agg(content order by created_unix desc),',') from public.comment a where a.issue_id=b.id), + (select id from public.pull_request d where d.issue_id=b.id) + FROM public.issue b where b.repo_id=NEW.id; + + end if; + + if not OLD.is_private and NEW.is_private then + delete from public.issue_es where repo_id=NEW.id; + delete from public.dataset_es where repo_id=NEW.id; + delete from public.repository_es where id=NEW.id; + end if; + + end if; + + if not NEW.is_private then + raise notice 'update repo,the updated_unix is %',NEW.updated_unix; + update public.repository_es SET description=NEW.description, + name=NEW.name, + lower_name=NEW.lower_name, + owner_name=NEW.owner_name, + website=NEW.website, + updated_unix=NEW.updated_unix, + num_watches=NEW.num_watches, + num_stars=NEW.num_stars, + num_forks=NEW.num_forks, + topics=NEW.topics, + alias = NEW.alias, + lower_alias = NEW.lower_alias, + avatar=NEW.avatar + where id=NEW.id; + end if; + return new; + END +$def$ +LANGUAGE plpgsql; + +CREATE TRIGGER es_update_repository + AFTER UPDATE ON public.repository + FOR EACH ROW EXECUTE PROCEDURE update_repository(); + +ALTER TABLE public.repository ENABLE ALWAYS TRIGGER es_update_repository; + + +DROP TRIGGER IF EXISTS es_delete_repository on public.repository; + +CREATE OR REPLACE FUNCTION public.delete_repository() RETURNS trigger AS +$def$ + declare + BEGIN + delete from public.issue_es where repo_id=OLD.id; + delete from public.dataset_es where repo_id=OLD.id; + DELETE FROM public.repository_es where id=OLD.id; + return new; + END +$def$ +LANGUAGE plpgsql; + + +CREATE TRIGGER es_delete_repository + AFTER DELETE ON public.repository + FOR EACH ROW EXECUTE PROCEDURE delete_repository(); + +ALTER TABLE public.repository ENABLE ALWAYS TRIGGER es_delete_repository; + + + +DROP TRIGGER IF EXISTS es_udpate_repository_lang on public.language_stat; + +CREATE OR REPLACE FUNCTION public.udpate_repository_lang() RETURNS trigger AS +$def$ + BEGIN + if (TG_OP = 'UPDATE') then + update public.repository_es SET lang=(select array_to_string(array_agg(language order by percentage desc),',') from public.language_stat where repo_id=NEW.repo_id) where id=NEW.repo_id; + elsif (TG_OP = 'INSERT') then + update public.repository_es SET lang=(select array_to_string(array_agg(language order by percentage desc),',') from public.language_stat where repo_id=NEW.repo_id) where id=NEW.repo_id; + elsif (TG_OP = 'DELETE') then + if exists(select 1 from public.repository where id=OLD.repo_id) then + update public.repository_es SET lang=(select array_to_string(array_agg(language order by percentage desc),',') from public.language_stat where repo_id=OLD.repo_id) where id=OLD.repo_id; + end if; + end if; + return null; + END; +$def$ +LANGUAGE plpgsql; + +CREATE TRIGGER es_udpate_repository_lang + AFTER INSERT OR UPDATE OR DELETE ON public.language_stat + FOR EACH ROW EXECUTE PROCEDURE udpate_repository_lang(); + +ALTER TABLE public.language_stat ENABLE ALWAYS TRIGGER es_udpate_repository_lang; \ No newline at end of file diff --git a/models/dbsql/user_foreigntable_for_es.sql b/models/dbsql/user_foreigntable_for_es.sql new file mode 100644 index 000000000..5d77757f0 --- /dev/null +++ b/models/dbsql/user_foreigntable_for_es.sql @@ -0,0 +1,317 @@ +DELETE FROM public.user_es; +DROP FOREIGN table if exists public.user_es; +DROP TRIGGER IF EXISTS es_insert_user on public.user; +DROP FUNCTION public.insert_user_data; +DROP TRIGGER IF EXISTS es_update_user on public.user; +DROP FUNCTION public.update_user; + +DROP TRIGGER IF EXISTS es_delete_user on public.user; +DROP FUNCTION public.delete_user; + +CREATE FOREIGN TABLE public.user_es +( + id bigint NOT NULL , + lower_name character varying(255) NULL, + name character varying(255) NULL, + full_name character varying(255), + email character varying(255), + keep_email_private boolean, + email_notifications_preference character varying(20) , + passwd character varying(255) , + passwd_hash_algo character varying(255) , + must_change_password boolean NOT NULL DEFAULT false, + login_type integer, + login_source bigint NOT NULL DEFAULT 0, + login_name character varying(255) , + type integer, + location character varying(255), + website character varying(255), + rands character varying(10), + salt character varying(10), + language character varying(5), + description character varying(255), + created_unix bigint, + updated_unix bigint, + last_login_unix bigint, + last_repo_visibility boolean, + max_repo_creation integer, + is_active boolean, + is_admin boolean, + is_restricted boolean NOT NULL DEFAULT false, + allow_git_hook boolean, + allow_import_local boolean, + allow_create_organization boolean DEFAULT true, + prohibit_login boolean NOT NULL DEFAULT false, + avatar character varying(2048) , + avatar_email character varying(255), + use_custom_avatar boolean, + num_followers integer, + num_following integer NOT NULL DEFAULT 0, + num_stars integer, + num_repos integer, + num_teams integer, + num_members integer, + visibility integer NOT NULL DEFAULT 0, + repo_admin_change_team_access boolean NOT NULL DEFAULT false, + diff_view_style character varying(255), + theme character varying(255), + token character varying(1024) , + public_key character varying(255), + private_key character varying(255), + is_operator boolean NOT NULL DEFAULT false, + num_dataset_stars integer NOT NULL DEFAULT 0 +) SERVER multicorn_es +OPTIONS + ( + host '192.168.207.94', + port '9200', + index 'user-es-index', + rowid_column 'id', + default_sort '_id' + ) +; +delete from public.user_es; + INSERT INTO public.user_es( + id, + lower_name, + name, + full_name, + email, + keep_email_private, + email_notifications_preference, + must_change_password, + login_type, + login_source, + login_name, + type, + location, + website, + rands, + language, + description, + created_unix, + updated_unix, + last_login_unix, + last_repo_visibility, + max_repo_creation, + is_active, + is_restricted, + allow_git_hook, + allow_import_local, + allow_create_organization, + prohibit_login, + avatar, + avatar_email, + use_custom_avatar, + num_followers, + num_following, + num_stars, + num_repos, + num_teams, + num_members, + visibility, + repo_admin_change_team_access, + diff_view_style, + theme, + is_operator, + num_dataset_stars) + SELECT + id, + lower_name, + name, + full_name, + email, + keep_email_private, + email_notifications_preference, + must_change_password, + login_type, + login_source, + login_name, + type, + location, + website, + rands, + language, + description, + created_unix, + updated_unix, + last_login_unix, + last_repo_visibility, + max_repo_creation, + is_active, + is_restricted, + allow_git_hook, + allow_import_local, + allow_create_organization, + prohibit_login, + avatar, + avatar_email, + use_custom_avatar, + num_followers, + num_following, + num_stars, + num_repos, + num_teams, + num_members, + visibility, + repo_admin_change_team_access, + diff_view_style, + theme, + is_operator, + num_dataset_stars + FROM public.user; + +DROP TRIGGER IF EXISTS es_insert_user on public.user; + +CREATE OR REPLACE FUNCTION public.insert_user_data() RETURNS trigger AS +$def$ + BEGIN + INSERT INTO public."user_es"( + id, + lower_name, + name, + full_name, + email, + keep_email_private, + email_notifications_preference, + must_change_password, + login_type, + login_source, + login_name, + type, + location, + website, + rands, + language, + description, + created_unix, + updated_unix, + last_login_unix, + last_repo_visibility, + max_repo_creation, + is_active, + is_restricted, + allow_git_hook, + allow_import_local, + allow_create_organization, + prohibit_login, + avatar, + avatar_email, + use_custom_avatar, + num_followers, + num_following, + num_stars, + num_repos, + num_teams, + num_members, + visibility, + repo_admin_change_team_access, + diff_view_style, + theme, + is_operator, + num_dataset_stars) + VALUES ( + NEW.id, + NEW.lower_name, + NEW.name, + NEW.full_name, + NEW.email, + NEW.keep_email_private, + NEW.email_notifications_preference, + NEW.must_change_password, + NEW.login_type, + NEW.login_source, + NEW.login_name, + NEW.type, + NEW.location, + NEW.website, + NEW.rands, + NEW.language, + NEW.description, + NEW.created_unix, + NEW.updated_unix, + NEW.last_login_unix, + NEW.last_repo_visibility, + NEW.max_repo_creation, + NEW.is_active, + NEW.is_restricted, + NEW.allow_git_hook, + NEW.allow_import_local, + NEW.allow_create_organization, + NEW.prohibit_login, + NEW.avatar, + NEW.avatar_email, + NEW.use_custom_avatar, + NEW.num_followers, + NEW.num_following, + NEW.num_stars, + NEW.num_repos, + NEW.num_teams, + NEW.num_members, + NEW.visibility, + NEW.repo_admin_change_team_access, + NEW.diff_view_style, + NEW.theme, + NEW.is_operator, + NEW.num_dataset_stars + ); + + RETURN NEW; + END; +$def$ +LANGUAGE plpgsql; + + + +CREATE TRIGGER es_insert_user + AFTER INSERT ON public.user + FOR EACH ROW EXECUTE PROCEDURE insert_user_data(); + +ALTER TABLE public.user ENABLE ALWAYS TRIGGER es_insert_user; + +DROP TRIGGER IF EXISTS es_update_user on public.user; + +CREATE OR REPLACE FUNCTION public.update_user() RETURNS trigger AS +$def$ + BEGIN + UPDATE public.user_es + SET description=NEW.description, + name=NEW.name, + full_name=NEW.full_name, + location=NEW.location, + website=NEW.website, + email=NEW.email, + num_dataset_stars=NEW.num_dataset_stars, + updated_unix=NEW.updated_unix + where id=NEW.id; + return new; + END +$def$ +LANGUAGE plpgsql; + + + +CREATE TRIGGER es_update_user + AFTER UPDATE ON public.user + FOR EACH ROW EXECUTE PROCEDURE update_user(); + +ALTER TABLE public.user ENABLE ALWAYS TRIGGER es_update_user; + +DROP TRIGGER IF EXISTS es_delete_user on public.user; + +CREATE OR REPLACE FUNCTION public.delete_user() RETURNS trigger AS +$def$ + declare + BEGIN + DELETE FROM public.user_es where id=OLD.id; + return new; + END +$def$ +LANGUAGE plpgsql; + + +CREATE TRIGGER es_delete_user + AFTER DELETE ON public.user + FOR EACH ROW EXECUTE PROCEDURE delete_user(); + +ALTER TABLE public.user ENABLE ALWAYS TRIGGER es_delete_user; \ No newline at end of file diff --git a/models/file_chunk.go b/models/file_chunk.go index 76c926dc5..0fc3a8879 100755 --- a/models/file_chunk.go +++ b/models/file_chunk.go @@ -13,11 +13,6 @@ const ( FileUploaded ) -const ( - TypeCloudBrainOne int = iota - TypeCloudBrainTwo -) - type FileChunk struct { ID int64 `xorm:"pk autoincr"` UUID string `xorm:"uuid UNIQUE"` diff --git a/models/models.go b/models/models.go index 36527f78d..2ec61941d 100755 --- a/models/models.go +++ b/models/models.go @@ -138,6 +138,8 @@ func init() { new(OfficialTag), new(OfficialTagRepos), new(WechatBindLog), + new(OrgStatistic), + new(SearchRecord), ) tablesStatistic = append(tablesStatistic, @@ -152,6 +154,8 @@ func init() { new(UserBusinessAnalysisCurrentWeek), new(UserBusinessAnalysisYesterday), new(UserLoginLog), + new(UserMetrics), + new(UserAnalysisPara), ) gonicNames := []string{"SSL", "UID"} diff --git a/models/org.go b/models/org.go index 85fb157ae..8b3e60ef8 100755 --- a/models/org.go +++ b/models/org.go @@ -8,6 +8,7 @@ package models import ( "fmt" "os" + "strconv" "strings" "code.gitea.io/gitea/modules/log" @@ -19,6 +20,17 @@ import ( "xorm.io/xorm" ) +type OrgStatistic struct { + ID int64 `xorm:"pk autoincr"` + OrgID int64 `xorm:"UNIQUE"` + NumScore int `xorm:"INDEX NOT NULL DEFAULT 0"` +} + +type OrgScore struct { + *User + Score string +} + // IsOwnedBy returns true if given user is in the owner team. func (org *User) IsOwnedBy(uid int64) (bool, error) { return IsOrganizationOwner(org.ID, uid) @@ -135,6 +147,93 @@ func (org *User) RemoveOrgRepo(repoID int64) error { return org.removeOrgRepo(x, repoID) } +func UpdateOrgStatistics() { + ids, err := GetOrganizationsId() + if err != nil { + return + } + for _, id := range ids { + org := User{ID: id} + orgStat := &OrgStatistic{OrgID: id} + numScore, err := org.getOrgStatistics() + if err == nil { + has, _ := x.Get(orgStat) + + orgStat.NumScore = numScore + if has { + x.ID(orgStat.ID).Cols("num_score").Update(&orgStat) + } else { + x.Insert(orgStat) + } + + } + } + +} + +func (org *User) getOrgStatistics() (int, error) { + count, err := getRepositoryCount(x, org) + if err != nil { + return 0, err + } + + err = org.GetRepositories(ListOptions{int(count), 1}) + + if err != nil { + return 0, err + } + var numScore = 0 + for _, repo := range org.Repos { + + numScore += int(getOpenIByRepoId(repo.ID)) + } + + return numScore, nil + +} + +func FindTopNStarsOrgs(n int) ([]*OrgScore, error) { + sql := "select a.id,sum(b.num_stars) score from \"user\" a ,repository b where a.id=b.owner_id and a.type=1 group by a.id order by score desc limit " + strconv.Itoa(n) + + return findTopNOrgs(sql) +} +func FindTopNMembersOrgs(n int) ([]*OrgScore, error) { + sql := "select id, count(user_id) score from" + + " (select org_id as id, uid as user_id from org_user " + + "union select a.id,b.user_id from \"user\" a,collaboration b,repository c " + + "where a.type=1 and a.id=c.owner_id and b.repo_id=c.id) d " + + "group by id order by score desc limit " + strconv.Itoa(n) + + return findTopNOrgs(sql) +} + +func FindTopNOpenIOrgs(n int) ([]*OrgScore, error) { + sql := "select org_id id,num_score score from org_statistic order by num_score desc limit 10" + strconv.Itoa(n) + + return findTopNOrgs(sql) +} + +func findTopNOrgs(sql string) ([]*OrgScore, error) { + resutls, err := x.QueryString(sql) + + if err != nil { + return nil, err + } + var orgScore []*OrgScore + for _, record := range resutls { + id, _ := strconv.ParseInt(record["id"], 10, 64) + user, err := getUserByID(x, id) + if err != nil { + continue + } + orgScore = append(orgScore, &OrgScore{user, record["score"]}) + + } + + return orgScore, nil + +} + // CreateOrganization creates record of a new organization. func CreateOrganization(org, owner *User) (err error) { if !owner.CanCreateOrganization() { diff --git a/models/repo.go b/models/repo.go index 42e350fbe..25bfb4a74 100755 --- a/models/repo.go +++ b/models/repo.go @@ -6,13 +6,14 @@ package models import ( - "code.gitea.io/gitea/modules/git" "context" "crypto/md5" "errors" "fmt" "html/template" "math/rand" + + "code.gitea.io/gitea/modules/git" "xorm.io/xorm" "code.gitea.io/gitea/modules/blockchain" @@ -1606,14 +1607,16 @@ func updateRepository(e Engine, repo *Repository, visibilityChanged bool) (err e } dataset, err := GetDatasetByRepo(repo) - if err != nil { + if err != nil && !IsErrNotExist(err) { return err } - _, err = e.Where("dataset_id = ?", dataset.ID).Cols("is_private").Update(&Attachment{ - IsPrivate: true, - }) - if err != nil { - return err + if dataset != nil { + _, err = e.Where("dataset_id = ?", dataset.ID).Cols("is_private").Update(&Attachment{ + IsPrivate: true, + }) + if err != nil { + return err + } } } else { diff --git a/models/repo_list.go b/models/repo_list.go index 6fb9380de..5bf0ecf03 100755 --- a/models/repo_list.go +++ b/models/repo_list.go @@ -190,7 +190,8 @@ type SearchRepoOptions struct { // None -> include all repos // True -> include just courses // False -> include just no courses - Course util.OptionalBool + Course util.OptionalBool + OnlySearchPrivate bool } //SearchOrderBy is used to sort the result @@ -219,12 +220,15 @@ const ( SearchOrderByDownloadTimes SearchOrderBy = "download_times DESC" SearchOrderByHot SearchOrderBy = "(num_watches + num_stars + num_forks + clone_cnt) DESC" SearchOrderByActive SearchOrderBy = "(num_issues + num_pulls + num_commit) DESC" + SearchOrderByWatches SearchOrderBy = "num_watches DESC" ) // SearchRepositoryCondition creates a query condition according search repository options func SearchRepositoryCondition(opts *SearchRepoOptions) builder.Cond { var cond = builder.NewCond() - + if opts.OnlySearchPrivate { + cond = cond.And(builder.Eq{"is_private": true}) + } if opts.Private { if opts.Actor != nil && !opts.Actor.IsAdmin && opts.Actor.ID != opts.OwnerID { // OK we're in the context of a User @@ -337,7 +341,7 @@ func SearchRepositoryCondition(opts *SearchRepoOptions) builder.Cond { if !opts.TopicOnly { var likes = builder.NewCond() for _, v := range strings.Split(opts.Keyword, ",") { - likes = likes.Or(builder.Like{"lower_name", strings.ToLower(v)}) + likes = likes.Or(builder.Like{"lower_alias", strings.ToLower(v)}) likes = likes.Or(builder.Like{"alias", v}) if opts.IncludeDescription { likes = likes.Or(builder.Like{"LOWER(description)", strings.ToLower(v)}) diff --git a/models/repo_statistic.go b/models/repo_statistic.go index a9e9593af..4f8f13ed7 100755 --- a/models/repo_statistic.go +++ b/models/repo_statistic.go @@ -73,6 +73,16 @@ func (repo *RepoStatistic) DisplayName() string { return repo.Alias } +func getOpenIByRepoId(repoId int64) float64 { + repoStatistic := new(RepoStatistic) + has, err := xStatistic.Cols("radar_total").Where("repo_id=?", repoId).Desc("id").Limit(1).Get(repoStatistic) + if !has || err != nil { + return 0 + } + return repoStatistic.RadarTotal + +} + func DeleteRepoStatDaily(date string) error { sess := xStatistic.NewSession() defer sess.Close() diff --git a/models/search_record.go b/models/search_record.go new file mode 100644 index 000000000..d9d85a591 --- /dev/null +++ b/models/search_record.go @@ -0,0 +1,83 @@ +package models + +import ( + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/timeutil" + "xorm.io/xorm" +) + +type SearchRecord struct { + ID int64 `xorm:"pk autoincr"` + //user + Keyword string `xorm:"NOT NULL"` + // + CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` +} + +func SaveSearchKeywordToDb(keyword string) error { + record := &SearchRecord{ + Keyword: keyword, + } + sess := x.NewSession() + defer sess.Close() + _, err := sess.Insert(record) + if err != nil { + log.Info("insert error." + err.Error()) + return err + } + return nil +} + +func setIssueQueryCondition(sess *xorm.Session, Keyword string, isPull bool, userId int64) { + sess.And("issue.poster_id=?", userId) + sess.And("issue.is_pull=?", isPull) + sess.And("(issue.name like '%" + Keyword + "%' or issue.content like '%" + Keyword + "%')") + sess.Join("INNER", "repository", "issue.repo_id = repository.id").And("repository.is_private = ?", true) +} + +func SearchPrivateIssueOrPr(Page int, PageSize int, Keyword string, isPull bool, userId int64) ([]*Issue, int64, error) { + sess := x.NewSession() + defer sess.Close() + setIssueQueryCondition(sess, Keyword, isPull, userId) + count, err := sess.Count(new(Issue)) + if err != nil { + return nil, 0, err + } + + setIssueQueryCondition(sess, Keyword, isPull, userId) + sess.Desc("issue.created_unix") + sess.Limit(PageSize, (Page-1)*PageSize) + issues := make([]*Issue, 0) + if err := sess.Find(&issues); err != nil { + return nil, 0, err + } else { + return issues, count, nil + } +} + +func setDataSetQueryCondition(sess *xorm.Session, Keyword string, userId int64) { + sess.And("dataset.user_id=?", userId) + sess.And("(dataset.title like '%" + Keyword + "%' or dataset.description like '%" + Keyword + "%')") + sess.Join("INNER", "repository", "dataset.repo_id = repository.id").And("repository.is_private = ?", true) +} + +func SearchDatasetBySQL(Page int, PageSize int, Keyword string, userId int64) ([]*Dataset, int64, error) { + sess := x.NewSession() + defer sess.Close() + setDataSetQueryCondition(sess, Keyword, userId) + count, err := sess.Count(new(Dataset)) + if err != nil { + return nil, 0, err + } + + setDataSetQueryCondition(sess, Keyword, userId) + sess.Desc("dataset.created_unix") + sess.Limit(PageSize, (Page-1)*PageSize) + datasets := make([]*Dataset, 0) + if err := sess.Find(&datasets); err != nil { + return nil, 0, err + } else { + return datasets, count, nil + } + +} diff --git a/models/user.go b/models/user.go index f72462051..71885aeb1 100755 --- a/models/user.go +++ b/models/user.go @@ -2104,6 +2104,12 @@ func GetOrganizationsCount() (int64, error) { } +func GetOrganizationsId() ([]int64, error) { + var ids []int64 + err := x.Table("user").Where("type=1").Cols("id").Find(&ids) + return ids, err +} + func GetBlockChainUnSuccessUsers() ([]*User, error) { users := make([]*User, 0, 10) err := x.Where("public_key = ''"). diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index 288762161..65ce642d5 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -6,7 +6,6 @@ import ( "strconv" "time" - "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/timeutil" "xorm.io/builder" @@ -81,6 +80,19 @@ type UserBusinessAnalysisAll struct { Name string `xorm:"NOT NULL"` DataDate string `xorm:"NULL"` + + //cloudbraintask + CloudBrainTaskNum int `xorm:"NOT NULL DEFAULT 0"` + GpuDebugJob int `xorm:"NOT NULL DEFAULT 0"` + NpuDebugJob int `xorm:"NOT NULL DEFAULT 0"` + GpuTrainJob int `xorm:"NOT NULL DEFAULT 0"` + NpuTrainJob int `xorm:"NOT NULL DEFAULT 0"` + NpuInferenceJob int `xorm:"NOT NULL DEFAULT 0"` + GpuBenchMarkJob int `xorm:"NOT NULL DEFAULT 0"` + CloudBrainRunTime int `xorm:"NOT NULL DEFAULT 0"` + CommitDatasetNum int `xorm:"NOT NULL DEFAULT 0"` + UserIndex float64 `xorm:"NOT NULL DEFAULT 0"` + UserLocation string `xorm:"NULL"` } type UserBusinessAnalysis struct { @@ -146,6 +158,18 @@ type UserBusinessAnalysis struct { Name string `xorm:"NOT NULL"` DataDate string `xorm:"NULL"` + + CloudBrainTaskNum int `xorm:"NOT NULL DEFAULT 0"` + GpuDebugJob int `xorm:"NOT NULL DEFAULT 0"` + NpuDebugJob int `xorm:"NOT NULL DEFAULT 0"` + GpuTrainJob int `xorm:"NOT NULL DEFAULT 0"` + NpuTrainJob int `xorm:"NOT NULL DEFAULT 0"` + NpuInferenceJob int `xorm:"NOT NULL DEFAULT 0"` + GpuBenchMarkJob int `xorm:"NOT NULL DEFAULT 0"` + CloudBrainRunTime int `xorm:"NOT NULL DEFAULT 0"` + CommitDatasetNum int `xorm:"NOT NULL DEFAULT 0"` + UserIndex float64 `xorm:"NOT NULL DEFAULT 0"` + UserLocation string `xorm:"NULL"` } type UserBusinessAnalysisQueryOptions struct { @@ -183,6 +207,29 @@ func getLastCountDate() int64 { return pageStartTime.Unix() } +func QueryMetrics(start int64, end int64) ([]*UserMetrics, int64) { + statictisSess := xStatistic.NewSession() + defer statictisSess.Close() + userMetricsList := make([]*UserMetrics, 0) + if err := statictisSess.Table(new(UserMetrics)).Where("count_date >" + fmt.Sprint(start) + " and count_date<" + fmt.Sprint(end)).OrderBy("count_date desc"). + Find(&userMetricsList); err != nil { + return nil, 0 + } + return userMetricsList, int64(len(userMetricsList)) +} + +func QueryRankList(key string, tableName string, limit int) ([]*UserBusinessAnalysisAll, int64) { + statictisSess := xStatistic.NewSession() + defer statictisSess.Close() + + userBusinessAnalysisAllList := make([]*UserBusinessAnalysisAll, 0) + if err := statictisSess.Table(tableName).OrderBy(key+" desc,id desc").Limit(limit, 0). + Find(&userBusinessAnalysisAllList); err != nil { + return nil, 0 + } + return userBusinessAnalysisAllList, int64(len(userBusinessAnalysisAllList)) +} + func QueryUserStaticDataByTableName(start int, pageSize int, tableName string, queryObj interface{}, userName string) ([]*UserBusinessAnalysisAll, int64) { statictisSess := xStatistic.NewSession() defer statictisSess.Close() @@ -334,6 +381,7 @@ func QueryUserStaticDataPage(opts *UserBusinessAnalysisQueryOptions) ([]*UserBus resultMap[userRecord.ID].WatchedCount += userRecord.WatchedCount resultMap[userRecord.ID].CommitCodeSize += userRecord.CommitCodeSize resultMap[userRecord.ID].CommitDatasetSize += userRecord.CommitDatasetSize + resultMap[userRecord.ID].CommitDatasetNum += userRecord.CommitDatasetNum resultMap[userRecord.ID].CommitModelCount += userRecord.CommitModelCount resultMap[userRecord.ID].SolveIssueCount += userRecord.SolveIssueCount resultMap[userRecord.ID].EncyclopediasCount += userRecord.EncyclopediasCount @@ -353,7 +401,7 @@ func QueryUserStaticDataPage(opts *UserBusinessAnalysisQueryOptions) ([]*UserBus return userBusinessAnalysisReturnList, count } -func refreshUserStaticTable(wikiCountMap map[string]int, CommitCodeSizeMap map[string]*git.UserKPIStats, tableName string, pageStartTime time.Time, pageEndTime time.Time) { +func refreshUserStaticTable(wikiCountMap map[string]int, tableName string, pageStartTime time.Time, pageEndTime time.Time, userMetrics map[string]int) { sess := x.NewSession() defer sess.Close() @@ -379,14 +427,15 @@ func refreshUserStaticTable(wikiCountMap map[string]int, CommitCodeSizeMap map[s FocusRepoCountMap := queryWatch(start_unix, end_unix) StarRepoCountMap := queryStar(start_unix, end_unix) WatchedCountMap := queryFollow(start_unix, end_unix) - - CommitDatasetSizeMap := queryDatasetSize(start_unix, end_unix) + CommitCodeSizeMap := queryCommitCodeSize(start_unix, end_unix) + CommitDatasetSizeMap, CommitDatasetNumMap := queryDatasetSize(start_unix, end_unix) SolveIssueCountMap := querySolveIssue(start_unix, end_unix) CreateRepoCountMap := queryUserCreateRepo(start_unix, end_unix) LoginCountMap := queryLoginCount(start_unix, end_unix) OpenIIndexMap := queryUserRepoOpenIIndex(startTime.Unix(), end_unix) - + CloudBrainTaskMap, CloudBrainTaskItemMap := queryCloudBrainTask(start_unix, end_unix) + AiModelManageMap := queryUserModel(start_unix, end_unix) DataDate := currentTimeNow.Format("2006-01-02") + " 00:01" cond := "type != 1 and is_active=true" @@ -395,6 +444,7 @@ func refreshUserStaticTable(wikiCountMap map[string]int, CommitCodeSizeMap map[s log.Info("query user error. return.") return } + ParaWeight := getParaWeight() var indexTotal int64 indexTotal = 0 insertCount := 0 @@ -412,84 +462,22 @@ func refreshUserStaticTable(wikiCountMap map[string]int, CommitCodeSizeMap map[s dateRecordAll.Name = userRecord.Name dateRecordAll.GiteaAgeMonth = subMonth(currentTimeNow, userRecord.CreatedUnix.AsTime()) dateRecordAll.DataDate = DataDate - - if _, ok := CodeMergeCountMap[dateRecordAll.ID]; !ok { - dateRecordAll.CodeMergeCount = 0 - } else { - dateRecordAll.CodeMergeCount = CodeMergeCountMap[dateRecordAll.ID] - } - - if _, ok := CommitCountMap[dateRecordAll.ID]; !ok { - dateRecordAll.CommitCount = 0 - } else { - dateRecordAll.CommitCount = CommitCountMap[dateRecordAll.ID] - } - - if _, ok := IssueCountMap[dateRecordAll.ID]; !ok { - dateRecordAll.IssueCount = 0 - } else { - dateRecordAll.IssueCount = IssueCountMap[dateRecordAll.ID] - } - - if _, ok := CommentCountMap[dateRecordAll.ID]; !ok { - dateRecordAll.CommentCount = 0 - } else { - dateRecordAll.CommentCount = CommentCountMap[dateRecordAll.ID] - } - - if _, ok := FocusRepoCountMap[dateRecordAll.ID]; !ok { - dateRecordAll.FocusRepoCount = 0 - } else { - dateRecordAll.FocusRepoCount = FocusRepoCountMap[dateRecordAll.ID] - } - - if _, ok := StarRepoCountMap[dateRecordAll.ID]; !ok { - dateRecordAll.StarRepoCount = 0 - } else { - dateRecordAll.StarRepoCount = StarRepoCountMap[dateRecordAll.ID] - } - - if _, ok := WatchedCountMap[dateRecordAll.ID]; !ok { - dateRecordAll.WatchedCount = 0 - } else { - dateRecordAll.WatchedCount = WatchedCountMap[dateRecordAll.ID] - } - - if _, ok := CommitCodeSizeMap[dateRecordAll.Email]; !ok { - dateRecordAll.CommitCodeSize = 0 - } else { - dateRecordAll.CommitCodeSize = int(CommitCodeSizeMap[dateRecordAll.Email].CommitLines) - } - - if _, ok := CommitDatasetSizeMap[dateRecordAll.ID]; !ok { - dateRecordAll.CommitDatasetSize = 0 - } else { - dateRecordAll.CommitDatasetSize = CommitDatasetSizeMap[dateRecordAll.ID] - } - - if _, ok := SolveIssueCountMap[dateRecordAll.ID]; !ok { - dateRecordAll.SolveIssueCount = 0 - } else { - dateRecordAll.SolveIssueCount = SolveIssueCountMap[dateRecordAll.ID] - } - - if _, ok := wikiCountMap[dateRecordAll.Name]; !ok { - dateRecordAll.EncyclopediasCount = 0 - } else { - dateRecordAll.EncyclopediasCount = wikiCountMap[dateRecordAll.Name] - } - - if _, ok := CreateRepoCountMap[dateRecordAll.ID]; !ok { - dateRecordAll.CreateRepoCount = 0 - } else { - dateRecordAll.CreateRepoCount = CreateRepoCountMap[dateRecordAll.ID] - } - - if _, ok := LoginCountMap[dateRecordAll.ID]; !ok { - dateRecordAll.LoginCount = 0 - } else { - dateRecordAll.LoginCount = LoginCountMap[dateRecordAll.ID] - } + dateRecordAll.UserLocation = userRecord.Location + + dateRecordAll.CodeMergeCount = getMapValue(dateRecordAll.ID, CodeMergeCountMap) + dateRecordAll.CommitCount = getMapValue(dateRecordAll.ID, CommitCountMap) + dateRecordAll.IssueCount = getMapValue(dateRecordAll.ID, IssueCountMap) + dateRecordAll.CommentCount = getMapValue(dateRecordAll.ID, CommentCountMap) + dateRecordAll.FocusRepoCount = getMapValue(dateRecordAll.ID, FocusRepoCountMap) + dateRecordAll.StarRepoCount = getMapValue(dateRecordAll.ID, StarRepoCountMap) + dateRecordAll.WatchedCount = getMapValue(dateRecordAll.ID, WatchedCountMap) + dateRecordAll.CommitCodeSize = getMapValue(dateRecordAll.ID, CommitCodeSizeMap) + dateRecordAll.CommitDatasetSize = getMapValue(dateRecordAll.ID, CommitDatasetSizeMap) + dateRecordAll.CommitDatasetNum = getMapValue(dateRecordAll.ID, CommitDatasetNumMap) + dateRecordAll.SolveIssueCount = getMapValue(dateRecordAll.ID, SolveIssueCountMap) + dateRecordAll.EncyclopediasCount = getMapKeyStringValue(dateRecordAll.Name, wikiCountMap) + dateRecordAll.CreateRepoCount = getMapValue(dateRecordAll.ID, CreateRepoCountMap) + dateRecordAll.LoginCount = getMapValue(dateRecordAll.ID, LoginCountMap) if _, ok := OpenIIndexMap[dateRecordAll.ID]; !ok { dateRecordAll.OpenIIndex = 0 @@ -497,7 +485,15 @@ func refreshUserStaticTable(wikiCountMap map[string]int, CommitCodeSizeMap map[s dateRecordAll.OpenIIndex = OpenIIndexMap[dateRecordAll.ID] } - dateRecordAll.CommitModelCount = 0 + dateRecordAll.CloudBrainTaskNum = getMapValue(dateRecordAll.ID, CloudBrainTaskMap) + dateRecordAll.GpuDebugJob = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_GpuDebugJob", CloudBrainTaskItemMap) + dateRecordAll.NpuDebugJob = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_NpuDebugJob", CloudBrainTaskItemMap) + dateRecordAll.GpuTrainJob = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_GpuTrainJob", CloudBrainTaskItemMap) + dateRecordAll.NpuTrainJob = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_NpuTrainJob", CloudBrainTaskItemMap) + dateRecordAll.NpuInferenceJob = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_NpuInferenceJob", CloudBrainTaskItemMap) + dateRecordAll.GpuBenchMarkJob = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_GpuBenchMarkJob", CloudBrainTaskItemMap) + dateRecordAll.CommitModelCount = getMapValue(dateRecordAll.ID, AiModelManageMap) + dateRecordAll.UserIndex = getUserIndexFromAnalysisAll(dateRecordAll, ParaWeight) dateRecordBatch = append(dateRecordBatch, dateRecordAll) if len(dateRecordBatch) >= BATCH_INSERT_SIZE { @@ -508,6 +504,11 @@ func refreshUserStaticTable(wikiCountMap map[string]int, CommitCodeSizeMap map[s } dateRecordBatch = make([]UserBusinessAnalysisAll, 0) } + if tableName == "user_business_analysis_all" { + if dateRecordAll.UserIndex > 0 || dateRecordAll.LoginCount > 0 { + userMetrics["TotalHasActivityUser"] = getMapKeyStringValue("TotalHasActivityUser", userMetrics) + 1 + } + } } indexTotal += PAGE_SIZE if indexTotal >= count { @@ -529,7 +530,7 @@ func insertTable(dateRecords []UserBusinessAnalysisAll, tableName string, static insertBatchSql := "INSERT INTO public." + tableName + "(id, count_date, code_merge_count, commit_count, issue_count, comment_count, focus_repo_count, star_repo_count, watched_count, gitea_age_month, commit_code_size, commit_dataset_size, " + - "commit_model_count, solve_issue_count, encyclopedias_count, regist_date, create_repo_count, login_count, open_i_index, email, name, data_date) " + + "commit_model_count, solve_issue_count, encyclopedias_count, regist_date, create_repo_count, login_count, open_i_index, email, name, data_date,cloud_brain_task_num,gpu_debug_job,npu_debug_job,gpu_train_job,npu_train_job,npu_inference_job,gpu_bench_mark_job,cloud_brain_run_time,commit_dataset_num,user_index,user_location) " + "VALUES" for i, record := range dateRecords { @@ -537,7 +538,7 @@ func insertTable(dateRecords []UserBusinessAnalysisAll, tableName string, static ", " + fmt.Sprint(record.IssueCount) + ", " + fmt.Sprint(record.CommentCount) + ", " + fmt.Sprint(record.FocusRepoCount) + ", " + fmt.Sprint(record.StarRepoCount) + ", " + fmt.Sprint(record.WatchedCount) + ", " + fmt.Sprint(record.GiteaAgeMonth) + ", " + fmt.Sprint(record.CommitCodeSize) + ", " + fmt.Sprint(record.CommitDatasetSize) + ", " + fmt.Sprint(record.CommitModelCount) + ", " + fmt.Sprint(record.SolveIssueCount) + ", " + fmt.Sprint(record.EncyclopediasCount) + ", " + fmt.Sprint(record.RegistDate) + - ", " + fmt.Sprint(record.CreateRepoCount) + ", " + fmt.Sprint(record.LoginCount) + ", " + fmt.Sprint(record.OpenIIndex) + ", '" + record.Email + "', '" + record.Name + "', '" + record.DataDate + "')" + ", " + fmt.Sprint(record.CreateRepoCount) + ", " + fmt.Sprint(record.LoginCount) + ", " + fmt.Sprint(record.OpenIIndex) + ", '" + record.Email + "', '" + record.Name + "', '" + record.DataDate + "'," + fmt.Sprint(record.CloudBrainTaskNum) + "," + fmt.Sprint(record.GpuDebugJob) + "," + fmt.Sprint(record.NpuDebugJob) + "," + fmt.Sprint(record.GpuTrainJob) + "," + fmt.Sprint(record.NpuTrainJob) + "," + fmt.Sprint(record.NpuInferenceJob) + "," + fmt.Sprint(record.GpuBenchMarkJob) + "," + fmt.Sprint(record.CloudBrainRunTime) + "," + fmt.Sprint(record.CommitDatasetNum) + "," + fmt.Sprint(record.UserIndex) + ",'" + record.UserLocation + "')" if i < (len(dateRecords) - 1) { insertBatchSql += "," } @@ -546,36 +547,36 @@ func insertTable(dateRecords []UserBusinessAnalysisAll, tableName string, static statictisSess.Exec(insertBatchSql) } -func RefreshUserStaticAllTabel(wikiCountMap map[string]int, CommitCodeSizeMap map[string]*git.UserKPIStats) { +func RefreshUserStaticAllTabel(wikiCountMap map[string]int, userMetrics map[string]int) { currentTimeNow := time.Now() pageStartTime := time.Date(2021, 11, 5, 0, 0, 0, 0, currentTimeNow.Location()) pageEndTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 23, 59, 59, 0, currentTimeNow.Location()) - refreshUserStaticTable(wikiCountMap, CommitCodeSizeMap, "user_business_analysis_all", pageStartTime, pageEndTime) + refreshUserStaticTable(wikiCountMap, "user_business_analysis_all", pageStartTime, pageEndTime, userMetrics) log.Info("refresh all data finished.") pageStartTime = time.Date(currentTimeNow.Year(), 1, 1, 0, 0, 0, 0, currentTimeNow.Location()) - refreshUserStaticTable(wikiCountMap, CommitCodeSizeMap, "user_business_analysis_current_year", pageStartTime, pageEndTime) + refreshUserStaticTable(wikiCountMap, "user_business_analysis_current_year", pageStartTime, pageEndTime, userMetrics) thisMonth := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), 1, 0, 0, 0, 0, currentTimeNow.Location()) - refreshUserStaticTable(wikiCountMap, CommitCodeSizeMap, "user_business_analysis_current_month", thisMonth, pageEndTime) + refreshUserStaticTable(wikiCountMap, "user_business_analysis_current_month", thisMonth, pageEndTime, userMetrics) offset := int(time.Monday - currentTimeNow.Weekday()) if offset > 0 { offset = -6 } pageStartTime = time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 0, 0, 0, time.Local).AddDate(0, 0, offset) - refreshUserStaticTable(wikiCountMap, CommitCodeSizeMap, "user_business_analysis_current_week", pageStartTime, pageEndTime) + refreshUserStaticTable(wikiCountMap, "user_business_analysis_current_week", pageStartTime, pageEndTime, userMetrics) pageStartTime = time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 0, 0, 0, time.Local).AddDate(0, 0, -30) - refreshUserStaticTable(wikiCountMap, CommitCodeSizeMap, "user_business_analysis_last30_day", pageStartTime, pageEndTime) + refreshUserStaticTable(wikiCountMap, "user_business_analysis_last30_day", pageStartTime, pageEndTime, userMetrics) pageStartTime = time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 0, 0, 0, time.Local).AddDate(0, 0, -1) pageEndTime = time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 23, 59, 59, 0, currentTimeNow.Location()).AddDate(0, 0, -1) - refreshUserStaticTable(wikiCountMap, CommitCodeSizeMap, "user_business_analysis_yesterday", pageStartTime, pageEndTime) + refreshUserStaticTable(wikiCountMap, "user_business_analysis_yesterday", pageStartTime, pageEndTime, userMetrics) pageStartTime = thisMonth.AddDate(0, -1, 0) pageEndTime = time.Date(currentTimeNow.Year(), currentTimeNow.Month(), 1, 23, 59, 59, 0, currentTimeNow.Location()).AddDate(0, 0, -1) - refreshUserStaticTable(wikiCountMap, CommitCodeSizeMap, "user_business_analysis_last_month", pageStartTime, pageEndTime) + refreshUserStaticTable(wikiCountMap, "user_business_analysis_last_month", pageStartTime, pageEndTime, userMetrics) } @@ -613,12 +614,13 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, } else { log.Info("query commit code size, len=" + fmt.Sprint(len(CommitCodeSizeMap))) } - CommitDatasetSizeMap := queryDatasetSize(start_unix, end_unix) + CommitDatasetSizeMap, CommitDatasetNumMap := queryDatasetSize(start_unix, end_unix) SolveIssueCountMap := querySolveIssue(start_unix, end_unix) CreateRepoCountMap := queryUserCreateRepo(start_unix, end_unix) LoginCountMap := queryLoginCount(start_unix, end_unix) OpenIIndexMap := queryUserRepoOpenIIndex(start_unix, end_unix) - + CloudBrainTaskMap, CloudBrainTaskItemMap := queryCloudBrainTask(start_unix, end_unix) + AiModelManageMap := queryUserModel(start_unix, end_unix) statictisSess := xStatistic.NewSession() defer statictisSess.Close() @@ -628,6 +630,9 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, log.Info("query user error. return.") return err } + + ParaWeight := getParaWeight() + userMetrics := make(map[string]int) var indexTotal int64 indexTotal = 0 for { @@ -648,47 +653,14 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, dateRecord.Name = userRecord.Name dateRecord.GiteaAgeMonth = subMonth(currentTimeNow, userRecord.CreatedUnix.AsTime()) dateRecord.DataDate = DataDate - if _, ok := CodeMergeCountMap[dateRecord.ID]; !ok { - dateRecord.CodeMergeCount = 0 - } else { - dateRecord.CodeMergeCount = CodeMergeCountMap[dateRecord.ID] - } - - if _, ok := CommitCountMap[dateRecord.ID]; !ok { - dateRecord.CommitCount = 0 - } else { - dateRecord.CommitCount = CommitCountMap[dateRecord.ID] - } - - if _, ok := IssueCountMap[dateRecord.ID]; !ok { - dateRecord.IssueCount = 0 - } else { - dateRecord.IssueCount = IssueCountMap[dateRecord.ID] - } - - if _, ok := CommentCountMap[dateRecord.ID]; !ok { - dateRecord.CommentCount = 0 - } else { - dateRecord.CommentCount = CommentCountMap[dateRecord.ID] - } - - if _, ok := FocusRepoCountMap[dateRecord.ID]; !ok { - dateRecord.FocusRepoCount = 0 - } else { - dateRecord.FocusRepoCount = FocusRepoCountMap[dateRecord.ID] - } - if _, ok := StarRepoCountMap[dateRecord.ID]; !ok { - dateRecord.StarRepoCount = 0 - } else { - dateRecord.StarRepoCount = StarRepoCountMap[dateRecord.ID] - } - - if _, ok := WatchedCountMap[dateRecord.ID]; !ok { - dateRecord.WatchedCount = 0 - } else { - dateRecord.WatchedCount = WatchedCountMap[dateRecord.ID] - } + dateRecord.CodeMergeCount = getMapValue(dateRecord.ID, CodeMergeCountMap) + dateRecord.CommitCount = getMapValue(dateRecord.ID, CommitCountMap) + dateRecord.IssueCount = getMapValue(dateRecord.ID, IssueCountMap) + dateRecord.CommentCount = getMapValue(dateRecord.ID, CommentCountMap) + dateRecord.FocusRepoCount = getMapValue(dateRecord.ID, FocusRepoCountMap) + dateRecord.StarRepoCount = getMapValue(dateRecord.ID, StarRepoCountMap) + dateRecord.WatchedCount = getMapValue(dateRecord.ID, WatchedCountMap) if _, ok := CommitCodeSizeMap[dateRecord.Email]; !ok { dateRecord.CommitCodeSize = 0 @@ -696,35 +668,15 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, dateRecord.CommitCodeSize = int(CommitCodeSizeMap[dateRecord.Email].CommitLines) } - if _, ok := CommitDatasetSizeMap[dateRecord.ID]; !ok { - dateRecord.CommitDatasetSize = 0 - } else { - dateRecord.CommitDatasetSize = CommitDatasetSizeMap[dateRecord.ID] - } + dateRecord.CommitDatasetSize = getMapValue(dateRecord.ID, CommitDatasetSizeMap) + dateRecord.CommitDatasetNum = getMapValue(dateRecord.ID, CommitDatasetNumMap) + dateRecord.SolveIssueCount = getMapValue(dateRecord.ID, SolveIssueCountMap) - if _, ok := SolveIssueCountMap[dateRecord.ID]; !ok { - dateRecord.SolveIssueCount = 0 - } else { - dateRecord.SolveIssueCount = SolveIssueCountMap[dateRecord.ID] - } + dateRecord.EncyclopediasCount = getMapKeyStringValue(dateRecord.Name, wikiCountMap) - if _, ok := wikiCountMap[dateRecord.Name]; !ok { - dateRecord.EncyclopediasCount = 0 - } else { - dateRecord.EncyclopediasCount = wikiCountMap[dateRecord.Name] - } + dateRecord.CreateRepoCount = getMapValue(dateRecord.ID, CreateRepoCountMap) - if _, ok := CreateRepoCountMap[dateRecord.ID]; !ok { - dateRecord.CreateRepoCount = 0 - } else { - dateRecord.CreateRepoCount = CreateRepoCountMap[dateRecord.ID] - } - - if _, ok := LoginCountMap[dateRecord.ID]; !ok { - dateRecord.LoginCount = 0 - } else { - dateRecord.LoginCount = LoginCountMap[dateRecord.ID] - } + dateRecord.LoginCount = getMapValue(dateRecord.ID, LoginCountMap) if _, ok := OpenIIndexMap[dateRecord.ID]; !ok { dateRecord.OpenIIndex = 0 @@ -732,8 +684,17 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, dateRecord.OpenIIndex = OpenIIndexMap[dateRecord.ID] } - dateRecord.CommitModelCount = 0 - + dateRecord.CloudBrainTaskNum = getMapValue(dateRecord.ID, CloudBrainTaskMap) + dateRecord.GpuDebugJob = getMapKeyStringValue(fmt.Sprint(dateRecord.ID)+"_GpuDebugJob", CloudBrainTaskItemMap) + dateRecord.NpuDebugJob = getMapKeyStringValue(fmt.Sprint(dateRecord.ID)+"_NpuDebugJob", CloudBrainTaskItemMap) + dateRecord.GpuTrainJob = getMapKeyStringValue(fmt.Sprint(dateRecord.ID)+"_GpuTrainJob", CloudBrainTaskItemMap) + dateRecord.NpuTrainJob = getMapKeyStringValue(fmt.Sprint(dateRecord.ID)+"_NpuTrainJob", CloudBrainTaskItemMap) + dateRecord.NpuInferenceJob = getMapKeyStringValue(fmt.Sprint(dateRecord.ID)+"_NpuInferenceJob", CloudBrainTaskItemMap) + dateRecord.GpuBenchMarkJob = getMapKeyStringValue(fmt.Sprint(dateRecord.ID)+"_GpuBenchMarkJob", CloudBrainTaskItemMap) + dateRecord.CloudBrainRunTime = getMapKeyStringValue(fmt.Sprint(dateRecord.ID)+"_CloudBrainRunTime", CloudBrainTaskItemMap) + dateRecord.CommitModelCount = getMapValue(dateRecord.ID, AiModelManageMap) + dateRecord.UserIndex = getUserIndex(dateRecord, ParaWeight) + setUserMetrics(userMetrics, userRecord, start_unix, end_unix, dateRecord) _, err = statictisSess.Insert(&dateRecord) if err != nil { log.Info("insert daterecord failed." + err.Error()) @@ -747,11 +708,142 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, } } - RefreshUserStaticAllTabel(wikiCountMap, CommitCodeSizeMap) + RefreshUserStaticAllTabel(wikiCountMap, userMetrics) + + //insert userMetrics table + var useMetrics UserMetrics + useMetrics.CountDate = CountDate.Unix() + statictisSess.Delete(&useMetrics) + + useMetrics.ActivateRegistUser = getMapKeyStringValue("ActivateRegistUser", userMetrics) + useMetrics.HasActivityUser = getMapKeyStringValue("HasActivityUser", userMetrics) + useMetrics.NotActivateRegistUser = getMapKeyStringValue("NotActivateRegistUser", userMetrics) + useMetrics.TotalActivateRegistUser = getMapKeyStringValue("TotalActivateRegistUser", userMetrics) + useMetrics.TotalHasActivityUser = getMapKeyStringValue("TotalHasActivityUser", userMetrics) + statictisSess.Insert(&useMetrics) return nil } +func setUserMetrics(userMetrics map[string]int, user *User, start_time int64, end_time int64, dateRecord UserBusinessAnalysis) { + //ActivateRegistUser int `xorm:"NOT NULL DEFAULT 0"` + //NotActivateRegistUser int `xorm:"NOT NULL DEFAULT 0"` + //HasActivityUser int `xorm:"NOT NULL DEFAULT 0"` + //TotalActivateRegistUser int `xorm:"NOT NULL DEFAULT 0"` + //TotalHasActivityUser + regist_time := user.CreatedUnix.AsTime().Unix() + if regist_time >= start_time && regist_time <= end_time { + if user.IsActive { + userMetrics["ActivateRegistUser"] = getMapKeyStringValue("ActivateRegistUser", userMetrics) + 1 + } else { + userMetrics["NotActivateRegistUser"] = getMapKeyStringValue("NotActivateRegistUser", userMetrics) + 1 + } + } + if user.IsActive { + userMetrics["TotalActivateRegistUser"] = getMapKeyStringValue("TotalActivateRegistUser", userMetrics) + 1 + } + + if dateRecord.UserIndex > 0 || dateRecord.LoginCount > 0 { + userMetrics["HasActivityUser"] = getMapKeyStringValue("HasActivityUser", userMetrics) + 1 + } + +} + +func getParaWeight() map[string]float64 { + result := make(map[string]float64) + statictisSess := xStatistic.NewSession() + defer statictisSess.Close() + statictisSess.Select("*").Table(new(UserAnalysisPara)) + paraList := make([]*UserAnalysisPara, 0) + statictisSess.Find(¶List) + for _, paraRecord := range paraList { + result[paraRecord.Key] = paraRecord.Value + } + return result +} + +func getUserIndexFromAnalysisAll(dateRecord UserBusinessAnalysisAll, ParaWeight map[string]float64) float64 { + var result float64 + // PR数 0.20 + // commit数 0.20 + // 提出任务数 0.20 + // 评论数 0.20 + // 关注项目数 0.10 + // 点赞项目数 0.10 + // 登录次数 0.10 + result = float64(dateRecord.CodeMergeCount) * getParaWeightValue("CodeMergeCount", ParaWeight, 0.2) + result += float64(dateRecord.CommitCount) * getParaWeightValue("CommitCount", ParaWeight, 0.2) + log.Info("1 result=" + fmt.Sprint(result)) + result += float64(dateRecord.IssueCount) * getParaWeightValue("IssueCount", ParaWeight, 0.2) + result += float64(dateRecord.CommentCount) * getParaWeightValue("CommentCount", ParaWeight, 0.2) + result += float64(dateRecord.FocusRepoCount) * getParaWeightValue("FocusRepoCount", ParaWeight, 0.1) + result += float64(dateRecord.StarRepoCount) * getParaWeightValue("StarRepoCount", ParaWeight, 0.1) + result += float64(dateRecord.LoginCount) * getParaWeightValue("LoginCount", ParaWeight, 0.1) + result += float64(dateRecord.WatchedCount) * getParaWeightValue("WatchedCount", ParaWeight, 0.3) + result += float64(dateRecord.CommitCodeSize) * getParaWeightValue("CommitCodeSize", ParaWeight, 0.1) + result += float64(dateRecord.SolveIssueCount) * getParaWeightValue("SolveIssueCount", ParaWeight, 0.2) + result += float64(dateRecord.EncyclopediasCount) * getParaWeightValue("EncyclopediasCount", ParaWeight, 0.1) + result += float64(dateRecord.CreateRepoCount) * getParaWeightValue("CreateRepoCount", ParaWeight, 0.05) + result += float64(dateRecord.CloudBrainTaskNum) * getParaWeightValue("CloudBrainTaskNum", ParaWeight, 0.3) + result += float64(dateRecord.CommitModelCount) * getParaWeightValue("CommitModelCount", ParaWeight, 0.2) + result += dateRecord.OpenIIndex * getParaWeightValue("OpenIIndex", ParaWeight, 0.1) + + return result +} + +func getUserIndex(dateRecord UserBusinessAnalysis, ParaWeight map[string]float64) float64 { + var result float64 + // PR数 0.20 + // commit数 0.20 + // 提出任务数 0.20 + // 评论数 0.20 + // 关注项目数 0.10 + // 点赞项目数 0.10 + // 登录次数 0.10 + result = float64(dateRecord.CodeMergeCount) * getParaWeightValue("CodeMergeCount", ParaWeight, 0.2) + result += float64(dateRecord.CommitCount) * getParaWeightValue("CommitCount", ParaWeight, 0.2) + log.Info("2 result=" + fmt.Sprint(result)) + result += float64(dateRecord.IssueCount) * getParaWeightValue("IssueCount", ParaWeight, 0.2) + result += float64(dateRecord.CommentCount) * getParaWeightValue("CommentCount", ParaWeight, 0.2) + result += float64(dateRecord.FocusRepoCount) * getParaWeightValue("FocusRepoCount", ParaWeight, 0.1) + result += float64(dateRecord.StarRepoCount) * getParaWeightValue("StarRepoCount", ParaWeight, 0.1) + result += float64(dateRecord.LoginCount) * getParaWeightValue("LoginCount", ParaWeight, 0.1) + result += float64(dateRecord.WatchedCount) * getParaWeightValue("WatchedCount", ParaWeight, 0.3) + result += float64(dateRecord.CommitCodeSize) * getParaWeightValue("CommitCodeSize", ParaWeight, 0.1) + result += float64(dateRecord.SolveIssueCount) * getParaWeightValue("SolveIssueCount", ParaWeight, 0.2) + result += float64(dateRecord.EncyclopediasCount) * getParaWeightValue("EncyclopediasCount", ParaWeight, 0.1) + result += float64(dateRecord.CreateRepoCount) * getParaWeightValue("CreateRepoCount", ParaWeight, 0.05) + result += float64(dateRecord.CloudBrainTaskNum) * getParaWeightValue("CloudBrainTaskNum", ParaWeight, 0.3) + result += float64(dateRecord.CommitModelCount) * getParaWeightValue("CommitModelCount", ParaWeight, 0.2) + result += dateRecord.OpenIIndex * getParaWeightValue("OpenIIndex", ParaWeight, 0.1) + + return result +} + +func getParaWeightValue(key string, valueMap map[string]float64, defaultValue float64) float64 { + if _, ok := valueMap[key]; !ok { + return defaultValue + } else { + return valueMap[key] + } +} + +func getMapKeyStringValue(key string, valueMap map[string]int) int { + if _, ok := valueMap[key]; !ok { + return 0 + } else { + return valueMap[key] + } +} + +func getMapValue(userId int64, valueMap map[int64]int) int { + if _, ok := valueMap[userId]; !ok { + return 0 + } else { + return valueMap[userId] + } +} + func getInt(str string) int { re, err := strconv.ParseInt(str, 10, 32) if err != nil { @@ -1052,16 +1144,17 @@ func queryFollow(start_unix int64, end_unix int64) map[int64]int { return resultMap } -func queryDatasetSize(start_unix int64, end_unix int64) map[int64]int { +func queryDatasetSize(start_unix int64, end_unix int64) (map[int64]int, map[int64]int) { sess := x.NewSession() defer sess.Close() - resultMap := make(map[int64]int) + resultSizeMap := make(map[int64]int) + resultNumMap := make(map[int64]int) cond := " created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix) count, err := sess.Where(cond).Count(new(Attachment)) if err != nil { log.Info("query attachment error. return.") - return resultMap + return resultSizeMap, resultNumMap } var indexTotal int64 indexTotal = 0 @@ -1072,10 +1165,12 @@ func queryDatasetSize(start_unix int64, end_unix int64) map[int64]int { log.Info("query Attachment size=" + fmt.Sprint(len(attachmentList))) for _, attachRecord := range attachmentList { - if _, ok := resultMap[attachRecord.UploaderID]; !ok { - resultMap[attachRecord.UploaderID] = int(attachRecord.Size / (1024 * 1024)) //MB + if _, ok := resultSizeMap[attachRecord.UploaderID]; !ok { + resultSizeMap[attachRecord.UploaderID] = int(attachRecord.Size / (1024 * 1024)) //MB + resultNumMap[attachRecord.UploaderID] = 1 } else { - resultMap[attachRecord.UploaderID] += int(attachRecord.Size / (1024 * 1024)) //MB + resultSizeMap[attachRecord.UploaderID] += int(attachRecord.Size / (1024 * 1024)) //MB + resultNumMap[attachRecord.UploaderID] += 1 } } @@ -1085,7 +1180,7 @@ func queryDatasetSize(start_unix int64, end_unix int64) map[int64]int { } } - return resultMap + return resultSizeMap, resultNumMap } func queryUserCreateRepo(start_unix int64, end_unix int64) map[int64]int { @@ -1212,6 +1307,133 @@ func queryLoginCount(start_unix int64, end_unix int64) map[int64]int { return resultMap } +func queryCommitCodeSize(start_unix int64, end_unix int64) map[int64]int { + statictisSess := xStatistic.NewSession() + defer statictisSess.Close() + + resultMap := make(map[int64]int) + cond := "count_date>=" + fmt.Sprint(start_unix) + " and count_date<=" + fmt.Sprint(end_unix) + count, err := statictisSess.Where(cond).Count(new(UserBusinessAnalysis)) + if err != nil { + log.Info("query commit code size error. return.") + return resultMap + } + var indexTotal int64 + indexTotal = 0 + for { + statictisSess.Select("id,commit_code_size").Table("user_business_analysis").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal)) + userBusinessAnalysisList := make([]*UserBusinessAnalysis, 0) + statictisSess.Find(&userBusinessAnalysisList) + log.Info("query user login size=" + fmt.Sprint(len(userBusinessAnalysisList))) + for _, analysisRecord := range userBusinessAnalysisList { + if _, ok := resultMap[analysisRecord.ID]; !ok { + resultMap[analysisRecord.ID] = analysisRecord.CommitCodeSize + } else { + resultMap[analysisRecord.ID] += analysisRecord.CommitCodeSize + } + } + indexTotal += PAGE_SIZE + if indexTotal >= count { + break + } + } + log.Info("user commit code size=" + fmt.Sprint(len(resultMap))) + return resultMap +} + +func queryUserModel(start_unix int64, end_unix int64) map[int64]int { + sess := x.NewSession() + defer sess.Close() + resultMap := make(map[int64]int) + cond := " created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix) + count, err := sess.Where(cond).Count(new(AiModelManage)) + if err != nil { + log.Info("query AiModelManage error. return.") + return resultMap + } + var indexTotal int64 + indexTotal = 0 + for { + sess.Select("id,user_id").Table("ai_model_manage").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal)) + aiModelList := make([]*AiModelManage, 0) + sess.Find(&aiModelList) + log.Info("query AiModelManage size=" + fmt.Sprint(len(aiModelList))) + for _, aiModelRecord := range aiModelList { + if _, ok := resultMap[aiModelRecord.UserId]; !ok { + resultMap[aiModelRecord.UserId] = 1 + } else { + resultMap[aiModelRecord.UserId] += 1 + } + } + indexTotal += PAGE_SIZE + if indexTotal >= count { + break + } + } + return resultMap +} + +func queryCloudBrainTask(start_unix int64, end_unix int64) (map[int64]int, map[string]int) { + sess := x.NewSession() + defer sess.Close() + resultMap := make(map[int64]int) + resultItemMap := make(map[string]int) + + cond := " created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix) + count, err := sess.Where(cond).Count(new(Cloudbrain)) + if err != nil { + log.Info("query cloudbrain error. return.") + return resultMap, resultItemMap + } + var indexTotal int64 + indexTotal = 0 + for { + sess.Select("id,job_type,user_id,duration,train_job_duration,type").Table("cloudbrain").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal)) + cloudTaskList := make([]*Cloudbrain, 0) + sess.Find(&cloudTaskList) + log.Info("query cloudbrain size=" + fmt.Sprint(len(cloudTaskList))) + for _, cloudTaskRecord := range cloudTaskList { + if _, ok := resultMap[cloudTaskRecord.UserID]; !ok { + resultMap[cloudTaskRecord.UserID] = 1 + } else { + resultMap[cloudTaskRecord.UserID] += 1 + } + setMapKey("CloudBrainRunTime", cloudTaskRecord.UserID, int(cloudTaskRecord.Duration), resultItemMap) + if cloudTaskRecord.Type == 1 { //npu + if cloudTaskRecord.JobType == "TRAIN" { + setMapKey("NpuTrainJob", cloudTaskRecord.UserID, 1, resultItemMap) + } else if cloudTaskRecord.JobType == "INFERENCE" { + setMapKey("NpuInferenceJob", cloudTaskRecord.UserID, 1, resultItemMap) + } else { + setMapKey("NpuDebugJob", cloudTaskRecord.UserID, 1, resultItemMap) + } + } else { //type=0 gpu + if cloudTaskRecord.JobType == "TRAIN" { + setMapKey("GpuTrainJob", cloudTaskRecord.UserID, 1, resultItemMap) + } else if cloudTaskRecord.JobType == "BENCHMARK" { + setMapKey("GpuBenchMarkJob", cloudTaskRecord.UserID, 1, resultItemMap) + } else { + setMapKey("GpuDebugJob", cloudTaskRecord.UserID, 1, resultItemMap) + } + } + } + indexTotal += PAGE_SIZE + if indexTotal >= count { + break + } + } + + return resultMap, resultItemMap +} +func setMapKey(key string, userId int64, value int, resultItemMap map[string]int) { + newKey := fmt.Sprint(userId) + "_" + key + if _, ok := resultItemMap[newKey]; !ok { + resultItemMap[newKey] = value + } else { + resultItemMap[newKey] += value + } +} + func subMonth(t1, t2 time.Time) (month int) { y1 := t1.Year() y2 := t2.Year() diff --git a/models/user_business_struct.go b/models/user_business_struct.go index c435c0b07..17d9f046f 100644 --- a/models/user_business_struct.go +++ b/models/user_business_struct.go @@ -44,6 +44,18 @@ type UserBusinessAnalysisCurrentYear struct { //user Name string `xorm:"NOT NULL"` DataDate string `xorm:"NULL"` + + CloudBrainTaskNum int `xorm:"NOT NULL DEFAULT 0"` + GpuDebugJob int `xorm:"NOT NULL DEFAULT 0"` + NpuDebugJob int `xorm:"NOT NULL DEFAULT 0"` + GpuTrainJob int `xorm:"NOT NULL DEFAULT 0"` + NpuTrainJob int `xorm:"NOT NULL DEFAULT 0"` + NpuInferenceJob int `xorm:"NOT NULL DEFAULT 0"` + GpuBenchMarkJob int `xorm:"NOT NULL DEFAULT 0"` + CloudBrainRunTime int `xorm:"NOT NULL DEFAULT 0"` + CommitDatasetNum int `xorm:"NOT NULL DEFAULT 0"` + UserIndex float64 `xorm:"NOT NULL DEFAULT 0"` + UserLocation string `xorm:"NULL"` } type UserBusinessAnalysisLast30Day struct { @@ -88,6 +100,18 @@ type UserBusinessAnalysisLast30Day struct { //user Name string `xorm:"NOT NULL"` DataDate string `xorm:"NULL"` + + CloudBrainTaskNum int `xorm:"NOT NULL DEFAULT 0"` + GpuDebugJob int `xorm:"NOT NULL DEFAULT 0"` + NpuDebugJob int `xorm:"NOT NULL DEFAULT 0"` + GpuTrainJob int `xorm:"NOT NULL DEFAULT 0"` + NpuTrainJob int `xorm:"NOT NULL DEFAULT 0"` + NpuInferenceJob int `xorm:"NOT NULL DEFAULT 0"` + GpuBenchMarkJob int `xorm:"NOT NULL DEFAULT 0"` + CloudBrainRunTime int `xorm:"NOT NULL DEFAULT 0"` + CommitDatasetNum int `xorm:"NOT NULL DEFAULT 0"` + UserIndex float64 `xorm:"NOT NULL DEFAULT 0"` + UserLocation string `xorm:"NULL"` } type UserBusinessAnalysisLastMonth struct { @@ -132,6 +156,18 @@ type UserBusinessAnalysisLastMonth struct { //user Name string `xorm:"NOT NULL"` DataDate string `xorm:"NULL"` + + CloudBrainTaskNum int `xorm:"NOT NULL DEFAULT 0"` + GpuDebugJob int `xorm:"NOT NULL DEFAULT 0"` + NpuDebugJob int `xorm:"NOT NULL DEFAULT 0"` + GpuTrainJob int `xorm:"NOT NULL DEFAULT 0"` + NpuTrainJob int `xorm:"NOT NULL DEFAULT 0"` + NpuInferenceJob int `xorm:"NOT NULL DEFAULT 0"` + GpuBenchMarkJob int `xorm:"NOT NULL DEFAULT 0"` + CloudBrainRunTime int `xorm:"NOT NULL DEFAULT 0"` + CommitDatasetNum int `xorm:"NOT NULL DEFAULT 0"` + UserIndex float64 `xorm:"NOT NULL DEFAULT 0"` + UserLocation string `xorm:"NULL"` } type UserBusinessAnalysisCurrentMonth struct { @@ -176,6 +212,18 @@ type UserBusinessAnalysisCurrentMonth struct { //user Name string `xorm:"NOT NULL"` DataDate string `xorm:"NULL"` + + CloudBrainTaskNum int `xorm:"NOT NULL DEFAULT 0"` + GpuDebugJob int `xorm:"NOT NULL DEFAULT 0"` + NpuDebugJob int `xorm:"NOT NULL DEFAULT 0"` + GpuTrainJob int `xorm:"NOT NULL DEFAULT 0"` + NpuTrainJob int `xorm:"NOT NULL DEFAULT 0"` + NpuInferenceJob int `xorm:"NOT NULL DEFAULT 0"` + GpuBenchMarkJob int `xorm:"NOT NULL DEFAULT 0"` + CloudBrainRunTime int `xorm:"NOT NULL DEFAULT 0"` + CommitDatasetNum int `xorm:"NOT NULL DEFAULT 0"` + UserIndex float64 `xorm:"NOT NULL DEFAULT 0"` + UserLocation string `xorm:"NULL"` } type UserBusinessAnalysisCurrentWeek struct { @@ -220,6 +268,18 @@ type UserBusinessAnalysisCurrentWeek struct { //user Name string `xorm:"NOT NULL"` DataDate string `xorm:"NULL"` + + CloudBrainTaskNum int `xorm:"NOT NULL DEFAULT 0"` + GpuDebugJob int `xorm:"NOT NULL DEFAULT 0"` + NpuDebugJob int `xorm:"NOT NULL DEFAULT 0"` + GpuTrainJob int `xorm:"NOT NULL DEFAULT 0"` + NpuTrainJob int `xorm:"NOT NULL DEFAULT 0"` + NpuInferenceJob int `xorm:"NOT NULL DEFAULT 0"` + GpuBenchMarkJob int `xorm:"NOT NULL DEFAULT 0"` + CloudBrainRunTime int `xorm:"NOT NULL DEFAULT 0"` + CommitDatasetNum int `xorm:"NOT NULL DEFAULT 0"` + UserIndex float64 `xorm:"NOT NULL DEFAULT 0"` + UserLocation string `xorm:"NULL"` } type UserBusinessAnalysisYesterday struct { @@ -264,4 +324,30 @@ type UserBusinessAnalysisYesterday struct { //user Name string `xorm:"NOT NULL"` DataDate string `xorm:"NULL"` + + CloudBrainTaskNum int `xorm:"NOT NULL DEFAULT 0"` + GpuDebugJob int `xorm:"NOT NULL DEFAULT 0"` + NpuDebugJob int `xorm:"NOT NULL DEFAULT 0"` + GpuTrainJob int `xorm:"NOT NULL DEFAULT 0"` + NpuTrainJob int `xorm:"NOT NULL DEFAULT 0"` + NpuInferenceJob int `xorm:"NOT NULL DEFAULT 0"` + GpuBenchMarkJob int `xorm:"NOT NULL DEFAULT 0"` + CloudBrainRunTime int `xorm:"NOT NULL DEFAULT 0"` + CommitDatasetNum int `xorm:"NOT NULL DEFAULT 0"` + UserIndex float64 `xorm:"NOT NULL DEFAULT 0"` + UserLocation string `xorm:"NULL"` +} + +type UserAnalysisPara struct { + Key string `xorm:"NOT NULL"` + Value float64 `xorm:"NOT NULL DEFAULT 0"` +} + +type UserMetrics struct { + CountDate int64 `xorm:"pk"` + ActivateRegistUser int `xorm:"NOT NULL DEFAULT 0"` + NotActivateRegistUser int `xorm:"NOT NULL DEFAULT 0"` + HasActivityUser int `xorm:"NOT NULL DEFAULT 0"` + TotalActivateRegistUser int `xorm:"NOT NULL DEFAULT 0"` + TotalHasActivityUser int `xorm:"NOT NULL DEFAULT 0"` } diff --git a/modules/auth/cloudbrain.go b/modules/auth/cloudbrain.go index 9949feddc..9d3d6290f 100755 --- a/modules/auth/cloudbrain.go +++ b/modules/auth/cloudbrain.go @@ -20,6 +20,9 @@ type CreateCloudBrainForm struct { ResourceSpecId int `form:"resource_spec_id" binding:"Required"` BenchmarkTypeID int `form:"benchmark_types_id"` BenchmarkChildTypeID int `form:"benchmark_child_types_id"` + BootFile string `form:"boot_file"` + Params string `form:"run_para_list"` + BranchName string `form:"branch_name"` } type CommitImageCloudBrainForm struct { diff --git a/modules/cloudbrain/cloudbrain.go b/modules/cloudbrain/cloudbrain.go index 54ac0c7ac..4a89f9393 100755 --- a/modules/cloudbrain/cloudbrain.go +++ b/modules/cloudbrain/cloudbrain.go @@ -15,14 +15,13 @@ import ( ) const ( - Command = `pip3 install jupyterlab==2.2.5 -i https://pypi.tuna.tsinghua.edu.cn/simple; - service ssh stop; - jupyter lab --no-browser --ip=0.0.0.0 --allow-root --notebook-dir="/code" --port=80 --LabApp.token="" --LabApp.allow_origin="self https://cloudbrain.pcl.ac.cn"` + Command = `pip3 install jupyterlab==2.2.5 -i https://pypi.tuna.tsinghua.edu.cn/simple;service ssh stop;jupyter lab --no-browser --ip=0.0.0.0 --allow-root --notebook-dir="/code" --port=80 --LabApp.token="" --LabApp.allow_origin="self https://cloudbrain.pcl.ac.cn"` //CommandBenchmark = `echo "start benchmark";python /code/test.py;echo "end benchmark"` CommandBenchmark = `echo "start benchmark";cd /benchmark && bash run_bk.sh;echo "end benchmark"` CodeMountPath = "/code" DataSetMountPath = "/dataset" ModelMountPath = "/model" + LogFile = "log.txt" BenchMarkMountPath = "/benchmark" BenchMarkResourceID = 1 Snn4imagenetMountPath = "/snn4imagenet" @@ -32,10 +31,13 @@ const ( SubTaskName = "task1" Success = "S000" + + DefaultBranchName = "master" ) var ( - ResourceSpecs *models.ResourceSpecs + ResourceSpecs *models.ResourceSpecs + TrainResourceSpecs *models.ResourceSpecs ) func isAdminOrOwnerOrJobCreater(ctx *context.Context, job *models.Cloudbrain, err error) bool { @@ -147,7 +149,7 @@ func AdminOrJobCreaterRightForTrain(ctx *context.Context) { } -func GenerateTask(ctx *context.Context, displayJobName, jobName, image, command, uuid, codePath, modelPath, benchmarkPath, snn4imagenetPath, brainScorePath, jobType, gpuQueue, description string, benchmarkTypeID, benchmarkChildTypeID, resourceSpecId int) error { +func GenerateTask(ctx *context.Context, displayJobName, jobName, image, command, uuid, codePath, modelPath, benchmarkPath, snn4imagenetPath, brainScorePath, jobType, gpuQueue, description, branchName, bootFile, params string, benchmarkTypeID, benchmarkChildTypeID, resourceSpecId int) error { dataActualPath := setting.Attachment.Minio.RealPath + setting.Attachment.Minio.Bucket + "/" + setting.Attachment.Minio.BasePath + @@ -155,13 +157,27 @@ func GenerateTask(ctx *context.Context, displayJobName, jobName, image, command, uuid var resourceSpec *models.ResourceSpec - if ResourceSpecs == nil { - json.Unmarshal([]byte(setting.ResourceSpecs), &ResourceSpecs) - } - for _, spec := range ResourceSpecs.ResourceSpec { - if resourceSpecId == spec.Id { - resourceSpec = spec + var versionCount int + if jobType == string(models.JobTypeTrain) { + versionCount = 1 + if TrainResourceSpecs == nil { + json.Unmarshal([]byte(setting.TrainResourceSpecs), &TrainResourceSpecs) + } + for _, spec := range TrainResourceSpecs.ResourceSpec { + if resourceSpecId == spec.Id { + resourceSpec = spec + } + } + } else { + if ResourceSpecs == nil { + json.Unmarshal([]byte(setting.ResourceSpecs), &ResourceSpecs) + } + for _, spec := range ResourceSpecs.ResourceSpec { + if resourceSpecId == spec.Id { + resourceSpec = spec + } } + } if resourceSpec == nil { @@ -169,6 +185,15 @@ func GenerateTask(ctx *context.Context, displayJobName, jobName, image, command, return errors.New("no such resourceSpec") } + var datasetName string + attach, err := models.GetAttachmentByUUID(uuid) + if err != nil { + //for benchmark, do not return error + log.Error("GetAttachmentByUUID failed:%v", err) + } else { + datasetName = attach.Name + } + jobResult, err := CreateJob(jobName, models.CreateJobParams{ JobName: jobName, RetryCount: 1, @@ -263,13 +288,19 @@ func GenerateTask(ctx *context.Context, displayJobName, jobName, image, command, BenchmarkTypeID: benchmarkTypeID, BenchmarkChildTypeID: benchmarkChildTypeID, Description: description, + IsLatestVersion: "1", + VersionCount: versionCount, + BranchName: branchName, + BootFile: bootFile, + DatasetName: datasetName, + Parameters: params, }) if err != nil { return err } - task, err := models.GetCloudbrainByName(jobName) + task, err := models.GetCloudbrainByJobID(jobID) if err != nil { log.Error("GetCloudbrainByName failed: %v", err.Error()) return err @@ -278,6 +309,8 @@ func GenerateTask(ctx *context.Context, displayJobName, jobName, image, command, if string(models.JobTypeBenchmark) == jobType { notification.NotifyOtherTask(ctx.User, ctx.Repo.Repository, stringId, displayJobName, models.ActionCreateBenchMarkTask) + } else if string(models.JobTypeTrain) == jobType { + notification.NotifyOtherTask(ctx.User, ctx.Repo.Repository, jobID, displayJobName, models.ActionCreateGPUTrainTask) } else { notification.NotifyOtherTask(ctx.User, ctx.Repo.Repository, stringId, displayJobName, models.ActionCreateDebugGPUTask) } @@ -407,8 +440,10 @@ func RestartTask(ctx *context.Context, task *models.Cloudbrain, newID *string) e return err } - idString := strconv.FormatInt(newTask.ID, 10) - *newID = idString + stringId := strconv.FormatInt(newTask.ID, 10) + *newID = stringId + + notification.NotifyOtherTask(ctx.User, ctx.Repo.Repository, stringId, task.DisplayJobName, models.ActionCreateDebugGPUTask) return nil } diff --git a/modules/cron/tasks_basic.go b/modules/cron/tasks_basic.go index b9838e66f..b3a6c02a1 100755 --- a/modules/cron/tasks_basic.go +++ b/modules/cron/tasks_basic.go @@ -185,6 +185,17 @@ func registerHandleSummaryStatistic() { }) } +func registerHandleOrgStatistic() { + RegisterTaskFatal("handle_org_statistic", &BaseConfig{ + Enabled: true, + RunAtStart: false, + Schedule: "0 0 2 * * ?", + }, func(ctx context.Context, _ *models.User, _ Config) error { + models.UpdateOrgStatistics() + return nil + }) +} + func registerSyncCloudbrainStatus() { RegisterTaskFatal("sync_cloudbrain_status", &BaseConfig{ Enabled: true, @@ -215,4 +226,5 @@ func initBasicTasks() { registerHandleSummaryStatistic() registerSyncCloudbrainStatus() + registerHandleOrgStatistic() } diff --git a/modules/modelarts/modelarts.go b/modules/modelarts/modelarts.go index e30d0100c..b1e7b269e 100755 --- a/modules/modelarts/modelarts.go +++ b/modules/modelarts/modelarts.go @@ -56,7 +56,6 @@ const ( PerPage = 10 IsLatestVersion = "1" NotLatestVersion = "0" - DebugType = -1 VersionCount = 1 SortByCreateTime = "create_time" @@ -281,6 +280,7 @@ func GenerateNotebook2(ctx *context.Context, displayJobName, jobName, uuid, desc RepoID: ctx.Repo.Repository.ID, JobID: jobResult.ID, JobName: jobName, + FlavorCode: flavor, DisplayJobName: displayJobName, JobType: string(models.JobTypeDebug), Type: models.TypeCloudBrainTwo, diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 946fb73b4..26f068193 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -437,7 +437,7 @@ var ( //home page RecommentRepoAddr string - + ESSearchURL string //notice config UserNameOfNoticeRepo string RepoNameOfNoticeRepo string @@ -452,16 +452,18 @@ var ( DecompressOBSTaskName string //cloudbrain config - CBAuthUser string - CBAuthPassword string - RestServerHost string - JobPath string - CBCodePathPrefix string - JobType string - GpuTypes string - DebugServerHost string - ResourceSpecs string - MaxDuration int64 + CBAuthUser string + CBAuthPassword string + RestServerHost string + JobPath string + CBCodePathPrefix string + JobType string + GpuTypes string + DebugServerHost string + ResourceSpecs string + MaxDuration int64 + TrainGpuTypes string + TrainResourceSpecs string //benchmark config IsBenchmarkEnabled bool @@ -1265,6 +1267,7 @@ func NewContext() { sec = Cfg.Section("homepage") RecommentRepoAddr = sec.Key("Address").MustString("https://git.openi.org.cn/OpenIOSSG/promote/raw/branch/master/") + ESSearchURL = sec.Key("ESSearchURL").MustString("http://192.168.207.94:9200") sec = Cfg.Section("notice") UserNameOfNoticeRepo = sec.Key("USER_NAME").MustString("OpenIOSSG") @@ -1285,6 +1288,8 @@ func NewContext() { GpuTypes = sec.Key("GPU_TYPES").MustString("") ResourceSpecs = sec.Key("RESOURCE_SPECS").MustString("") MaxDuration = sec.Key("MAX_DURATION").MustInt64(14400) + TrainGpuTypes = sec.Key("TRAIN_GPU_TYPES").MustString("") + TrainResourceSpecs = sec.Key("TRAIN_RESOURCE_SPECS").MustString("") sec = Cfg.Section("benchmark") IsBenchmarkEnabled = sec.Key("ENABLED").MustBool(false) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 976387dd9..5d531d6e4 100755 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -254,6 +254,18 @@ page_dev_yunlao_desc3=Developers can freely choose the corresponding computing r page_dev_yunlao_desc4=If your model requires more computing resources, you can also apply for it separately. page_dev_yunlao_apply=Apply Separately +search=Search +search_repo=Repository +search_dataset=DataSet +search_issue=Issue +search_pr=Pull Request +search_user=User +search_org=Organization +search_finded=Find +search_related=related +search_maybe=maybe +search_ge= + [explore] repos = Repositories select_repos = Select the project @@ -481,6 +493,11 @@ static.encyclopediascount=Encyclopedias Count static.createrepocount=Create Repo Count static.openiindex=OpenI Index static.registdate=Regist Date +static.CloudBrainTaskNum=CloudBrain Task Count +static.CloudBrainRunTime=CloudBrain Run Time +static.CommitDatasetNum=Commit Dataset Count +static.CommitModelCount=Commit Model Count +static.UserIndex=User Index static.countdate=Count Date static.all=All static.public.user_business_analysis_current_month=Current_Month @@ -899,7 +916,9 @@ language_other = Other datasets = Datasets datasets.desc = Enable Dataset cloudbrain_helper=Use GPU/NPU resources to open notebooks, model training tasks, etc. - +cloudbrain.exitinfo=Exit Information +cloudbrain.platform=Platform +cloudbrain.endtime=End Time model_manager = Model model_noright=No right model_rename=Duplicate model name, please modify model name. @@ -1010,7 +1029,8 @@ modelarts.train_job.parameter_value=Parameter Value modelarts.train_job.resource_setting=resource_setting modelarts.train_job.resource_setting_info=resource_setting_info modelarts.train_job.resource_pool=resource_pool -modelarts.train_job.resource_type=resource_type +modelarts.train_job.resource_type=Resource Type +modelarts.train_job.train_dataset=Train Dataset modelarts.train_job.standard=Standard modelarts.train_job.NAS_address=NAS Address modelarts.train_job.NAS_mount_path=NAS Mount Path @@ -2182,6 +2202,16 @@ customize = Customize selected_project=Selected Projects fold = Fold unfold = Unfold +org_member = Member +org_members = Members +org_team = Team +org_teams = Teams +org_repository = Repository +org_repositories = Repositories + +star = Star Top10 +member = Members Top10 +active = Active Top10 form.name_reserved = The organization name '%s' is reserved. form.name_pattern_not_allowed = The pattern '%s' is not allowed in an organization name. @@ -2794,10 +2824,11 @@ reject_pull_request = `suggested changes for %s#%[2]s` upload_dataset=`upload dataset %s` task_gpudebugjob=`created CPU/GPU type debugging task%s` task_npudebugjob=`created NPU type debugging task %s` -task_trainjob=`created training task%s` +task_nputrainjob=`created NPU training task%s` task_inferencejob=`created reasoning task %s` task_benchmark=`created profiling task %s` task_createmodel=`created new model %s` +task_gputrainjob=`created CPU/GPU training task%s` [tool] ago = %s ago diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index 6e4b2dc59..3c0a98197 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -256,6 +256,18 @@ page_dev_yunlao_desc3=开发者可以根据使用需求，自由选择相应计 page_dev_yunlao_desc4=如果您的模型需要更多的计算资源，也可以单独申请 page_dev_yunlao_apply=单独申请 +search=搜索 +search_repo=项目 +search_dataset=数据集 +search_issue=任务 +search_pr=合并请求 +search_user=用户 +search_org=组织 +search_finded=找到 +search_related=相关 +search_maybe=约为 +search_ge=个 + [explore] repos=项目 select_repos=精选项目 @@ -484,6 +496,11 @@ static.encyclopediascount=百科页面贡献次数 static.createrepocount=创建项目数 static.openiindex=OpenI指数 static.registdate=用户注册时间 +static.CloudBrainTaskNum=云脑任务数 +static.CloudBrainRunTime=云脑运行时间（小时） +static.CommitDatasetNum=上传（提交）数据集文件数 +static.CommitModelCount=提交模型数 +static.UserIndex=用户指数 static.countdate=系统统计时间 static.all=所有 static.public.user_business_analysis_current_month=本月 @@ -946,7 +963,9 @@ cloudbrain_jobname_err=只能以小写字母或数字开头且只包含小写字 cloudbrain_query_fail=查询云脑任务失败。 cloudbrain.mirror_tag = 镜像标签 cloudbrain.mirror_description = 镜像描述 - +cloudbrain.exitinfo=退出信息 +cloudbrain.platform=平台 +cloudbrain.endtime=结束时间 record_begintime_get_err=无法获取统计开始时间。 parameter_is_wrong=输入参数错误，请检查输入参数。 total_count_get_error=查询总页数失败。 @@ -2187,6 +2206,16 @@ customize = 自定义 selected_project=精选项目 fold = 收起 unfold = 展开 +org_member = 成员 +org_members = 成员 +org_team = 团队 +org_teams = 团队 +org_repository = 项目 +org_repositories = 项目 + +star = 点赞榜 +member = 成员榜 +active = 活跃榜 form.name_reserved=组织名称 '%s' 是被保留的。 form.name_pattern_not_allowed=组织名称中不允许使用 "%s"。 @@ -2800,10 +2829,11 @@ reject_pull_request=`建议变更 %s#%[2]s` upload_dataset=`上传了数据集文件 %s` task_gpudebugjob=`创建了CPU/GPU类型调试任务 %s` task_npudebugjob=`创建了NPU类型调试任务 %s` -task_trainjob=`创建了训练任务 %s` +task_nputrainjob=`创建了NPU类型训练任务 %s` task_inferencejob=`创建了推理任务 %s` task_benchmark=`创建了评测任务 %s` task_createmodel=`导入了新模型 %s` +task_gputrainjob=`创建了CPU/GPU类型训练任务 %s` [tool] ago=%s前 diff --git a/public/home/home.js b/public/home/home.js old mode 100644 new mode 100755 index 7512a4423..478c70f21 --- a/public/home/home.js +++ b/public/home/home.js @@ -135,7 +135,7 @@ socket.onmessage = function (e) { html += recordPrefix + actionName; html += " " + getRepotext(record) + "" } - else if(record.OpType == "24" || record.OpType == "26" || record.OpType == "27" || record.OpType == "28" || record.OpType == "30"){ + else if(record.OpType == "24" || record.OpType == "26" || record.OpType == "27" || record.OpType == "28" || record.OpType == "30" || record.OpType == "31"){ html += recordPrefix + actionName; html += " " + record.RefName + "" } @@ -175,6 +175,8 @@ function getTaskLink(record){ re = re + "/cloudbrain/benchmark/" + record.Content; }else if(record.OpType == 30){ re = re + "/modelmanage/show_model_info?name=" + record.RefName; + }else if(record.OpType == 31){ + re = re + "/cloudbrain/train-job/" + record.Content; } re = encodeURI(re); return re; @@ -321,10 +323,11 @@ var actionNameZH={ "24":"上传了数据集文件", "25":"创建了CPU/GPU类型调试任务", "26":"创建了NPU类型调试任务", - "27":"创建了训练任务", + "27":"创建了NPU类型训练任务", "28":"创建了推理任务", "29":"创建了评测任务", - "30":"导入了新模型" + "30":"导入了新模型", + "31":"创建了CPU/GPU类型训练任务" }; var actionNameEN={ @@ -346,10 +349,11 @@ var actionNameEN={ "24":" upload dataset ", "25":" created CPU/GPU type debugging task ", "26":" created NPU type debugging task ", - "27":" created training task", + "27":" created NPU type training task", "28":" created reasoning task", "29":" created profiling task", - "30":" created new model" + "30":" created new model", + "31":" created CPU/GPU type training task", }; var repoAndOrgZH={ diff --git a/public/home/search.js b/public/home/search.js new file mode 100644 index 000000000..e23d27549 --- /dev/null +++ b/public/home/search.js @@ -0,0 +1,1305 @@ +var token; +if(isEmpty(token)){ + var meta = $("meta[name=_uid]"); + if(!isEmpty(meta)){ + token = meta.attr("content"); + console.log("token is uid:" + token); + } +} + +var html =document.documentElement; +var lang = html.attributes["lang"] +var isZh = true; +if(lang != null && lang.nodeValue =="en-US" ){ + console.log("the language is " + lang.nodeValue); + isZh=false; +}else{ + console.log("default lang=zh"); +} +function isEmpty(str){ + if(typeof str == "undefined" || str == null || str == ""){ + return true; + } + return false; +} + +var itemType={ + "1":"repository", + "2":"issue", + "3":"user", + "4":"org", + "5":"dataset", + "6":"pr" +}; + +var sortBy={ + "10":"default", + "11":"updated_unix.keyword", + "12":"num_watches", + "13":"num_stars", + "14":"num_forks", + "20":"default", + "21":"updated_unix.keyword", + "30":"default", + "31":"name.keyword", + "32":"name.keyword", + "33":"created_unix.keyword", + "34":"created_unix.keyword", + "40":"default", + "41":"name.keyword", + "42":"name.keyword", + "43":"created_unix.keyword", + "44":"created_unix.keyword", + "50":"default", + "51":"download_times", + "60":"default", + "61":"updated_unix.keyword" +}; + +var sortAscending={ + "10":"false", + "11":"false", + "12":"false", + "13":"false", + "14":"false", + "20":"false", + "21":"false", + "30":"false", + "31":"true", + "32":"false", + "33":"false", + "34":"true", + "40":"false", + "41":"true", + "42":"false", + "43":"false", + "44":"true", + "50":"false", + "51":"false", + "60":"false", + "61":"false" +}; + +var currentPage = 1; +var pageSize = 15; +var currentSearchTableName ="repository"; +var currentSearchKeyword=""; +var currentSearchSortBy=""; +var currentSearchAscending="false"; +var OnlySearchLabel=false; +var startIndex =1; +var endIndex = 5; +var totalPage = 1; +var totalNum = 0; +var privateTotal = 0; + +function initPageInfo(){ + currentPage = 1; + startIndex =1; + endIndex = 5; +} + +function searchItem(type,sortType){ + console.log("enter item 2."); + currentSearchKeyword = document.getElementById("keyword_input").value; + if(!isEmpty(currentSearchKeyword)){ + initPageInfo(); + currentSearchTableName = itemType[type]; + currentSearchSortBy = sortBy[sortType]; + currentSearchAscending = sortAscending[sortType]; + OnlySearchLabel =false; + page(currentPage); + }else{ + emptySearch(); + } +} + + + +function search(){ + console.log("enter here 1."); + currentSearchKeyword = document.getElementById("keyword_input").value; + if(!isEmpty(currentSearchKeyword)){ + currentSearchKeyword = currentSearchKeyword.trim(); + } + if(!isEmpty(currentSearchKeyword)){ + doSpcifySearch(currentSearchTableName,currentSearchKeyword,sortBy[10],"false"); + }else{ + emptySearch(); + } +} + +function emptySearch(){ + initDiv(false); + initPageInfo(); + $('#searchForm').addClass("hiddenSearch"); + document.getElementById("find_id").innerHTML=getLabel(isZh,"search_empty"); + $('#find_title').html(""); + document.getElementById("sort_type").innerHTML=""; + document.getElementById("child_search_item").innerHTML=""; + document.getElementById("page_menu").innerHTML=""; + $('#repo_total').text(""); + $('#pr_total').text(""); + $('#issue_total').text(""); + $('#dataset_total').text(""); + $('#user_total').text(""); + $('#org_total').text(""); + setActivate(null); +} + +function initDiv(isSearchLabel=false){ + if(isSearchLabel){ + document.getElementById("search_div").style.display="none"; + document.getElementById("search_label_div").style.display="block"; + document.getElementById("dataset_item").style.display="none"; + document.getElementById("issue_item").style.display="none"; + document.getElementById("pr_item").style.display="none"; + document.getElementById("user_item").style.display="none"; + document.getElementById("org_item").style.display="none"; + document.getElementById("find_id").innerHTML=""; + }else{ + document.getElementById("search_div").style.display="block"; + document.getElementById("search_label_div").style.display="none"; + document.getElementById("dataset_item").style.display="block"; + document.getElementById("issue_item").style.display="block"; + document.getElementById("pr_item").style.display="block"; + document.getElementById("user_item").style.display="block"; + document.getElementById("org_item").style.display="block"; + document.getElementById("find_id").innerHTML=getLabel(isZh,"search_finded"); + } +} + +function doSpcifySearch(tableName,keyword,sortBy="",ascending="false"){ + initDiv(false); + $('#searchForm').addClass("hiddenSearch"); + document.getElementById("find_id").innerHTML=getLabel(isZh,"search_finded"); + currentSearchKeyword = keyword; + initPageInfo(); + currentSearchTableName = tableName; + currentSearchSortBy = sortBy; + currentSearchAscending = ascending; + OnlySearchLabel =false; + + page(currentPage); + + if(currentSearchTableName != "repository"){ + doSearch("repository",currentSearchKeyword,1,pageSize,true,"",false); + } + if(currentSearchTableName != "issue"){ + doSearch("issue",currentSearchKeyword,1,pageSize,true,"",false); + } + if(currentSearchTableName != "user"){ + doSearch("user",currentSearchKeyword,1,pageSize,true,"",false); + } + if(currentSearchTableName != "org"){ + doSearch("org",currentSearchKeyword,1,pageSize,true,"",false); + } + if(currentSearchTableName != "dataset"){ + doSearch("dataset",currentSearchKeyword,1,pageSize,true,"",false); + } + if(currentSearchTableName != "pr"){ + doSearch("pr",currentSearchKeyword,1,pageSize,true,"",false); + } +} + +function doSearchLabel(tableName,keyword,sortBy="",ascending="false"){ + initDiv(true); + //document.getElementById("search_div").style.display="none"; + //document.getElementById("search_label_div").style.display="block"; + document.getElementById("search_label_div").innerHTML="

#" + keyword + "

"; + + currentSearchKeyword = keyword; + initPageInfo(); + currentSearchTableName = tableName; + currentSearchSortBy = sortBy; + currentSearchAscending = ascending; + OnlySearchLabel =true; + + page(currentPage); +} + +function searchLabel(tableName,keyword,sortBy="",ascending="false"){ + + sessionStorage.setItem("keyword",keyword); + sessionStorage.setItem("tableName",tableName); + sessionStorage.setItem("searchLabel",true); + sessionStorage.setItem("sortBy",sortBy); + sessionStorage.setItem("ascending",ascending); + console.log("enter label search."); + window.open("/all/search/"); +} + +function doSearch(tableName,keyword,page,pageSize=15,onlyReturnNum=true,sortBy="",OnlySearchLabel=false){ + var language = "zh-CN"; + if(!isZh){ + language="en-US"; + } + $.ajax({ + type:"GET", + url:"/all/dosearch/", + headers: { + authorization:token, + }, + dataType:"json", + data:{ + 'TableName': tableName, + 'Key': keyword, + 'Page': page, + 'PageSize': pageSize, + 'OnlyReturnNum':onlyReturnNum, + 'SortBy':sortBy, + 'OnlySearchLabel':OnlySearchLabel, + 'Ascending':currentSearchAscending, + 'WebTotal':totalNum, + 'PrivateTotal':privateTotal, + 'language':language + }, + async:true, + success:function(json){ + console.log("tableName=" + tableName); + console.log(json); + displayResult(tableName,page,json,onlyReturnNum,keyword); + }, + error:function(response) { + console.log(response); + } + }); +} + +function displayResult(tableName,page,jsonResult,onlyReturnNum,keyword){ + if(tableName == "repository") { + displayRepoResult(page,jsonResult,onlyReturnNum,keyword); + } else if (tableName == "issue") { + displayIssueResult(page,jsonResult,onlyReturnNum,keyword); + } else if (tableName == "user") { + displayUserResult(page,jsonResult,onlyReturnNum,keyword); + } else if (tableName == "org") { + displayOrgResult(page,jsonResult,onlyReturnNum,keyword); + } else if (tableName == "dataset") { + displayDataSetResult(page,jsonResult,onlyReturnNum,keyword); + } else if (tableName == "pr") { + displayPrResult(page,jsonResult,onlyReturnNum,keyword); + } + if(!onlyReturnNum){ + console.log("set total num." + tableName); + totalPage =Math.ceil(jsonResult.Total/pageSize); + totalNum = jsonResult.Total; + privateTotal = jsonResult.PrivateTotal; + setPage(page); + } + +} + +function displayPrResult(page,jsonResult,onlyReturnNum,keyword){ + var data = jsonResult.Result; + var total = jsonResult.Total; + $('#pr_total').text(total); + if(!onlyReturnNum){ + setActivate("pr_item"); + //$('#keyword_desc').text(keyword); + //$('#obj_desc').text(getLabel(isZh,"search_pr")); + //$('#child_total').text(total); + $('#find_title').html(getLabel(isZh,"find_title").replace('{keyword}',keyword).replace('{tablename}',getLabel(isZh,"search_pr")).replace('{total}',total)); + + setIssueOrPrInnerHtml(data,"pulls"); + } +} + +var categoryDesc={ + "computer_vision":"计算机视觉", + "natural_language_processing":"自然语言处理", + "speech_processing":"语音处理", + "computer_vision_natural_language_processing":"计算机视觉、自然语言处理" +}; + +var categoryENDesc={ + "computer_vision":"computer vision", + "natural_language_processing":"natural language processing", + "speech_processing":"speech processing", + "computer_vision_natural_language_processing":"computer vision and natural language processing" +}; + +var taskDesc={ + "machine_translation":"机器翻译", + "question_answering_system":"问答系统", + "information_retrieval":"信息检索", + "knowledge_graph":"知识图谱", + "text_annotation":"文本标注", + "text_categorization":"文本分类", + "emotion_analysis":"情感分析", + "language_modeling":"语言建模", + "speech_recognition":"语音识别", + "automatic_digest":"自动文摘", + "information_extraction":"信息抽取", + "description_generation":"说明生成", + "image_classification":"图像分类", + "face_recognition":"人脸识别", + "image_search":"图像搜索", + "target_detection":"目标检测", + "image_description_generation":"图像描述生成", + "vehicle_license_plate_recognition":"车辆车牌识别", + "medical_image_analysis":"医学图像分析", + "unmanned":"无人驾驶", + "unmanned_security":"无人安防", + "drone":"无人机", + "vr_ar":"VR/AR", + "2_d_vision":"2-D视觉", + "2_5_d_vision":"2.5-D视觉", + "3_d_reconstruction":"3D重构", + "image_processing":"图像处理", + "video_processing":"视频处理", + "visual_input_system":"视觉输入系统", + "speech_coding":"语音编码", + "speech_enhancement":"语音增强", + "speech_recognition":"语音识别", + "speech_synthesis":"语音合成" +}; + +var taskENDesc={ + "machine_translation":"machine translation", + "question_answering_system":"question answering system", + "information_retrieval":"information retrieval", + "knowledge_graph":"knowledge graph", + "text_annotation":"text annotation", + "text_categorization":"text categorization", + "emotion_analysis":"emotion analysis", + "language_modeling":"language modeling", + "speech_recognition":"speech recognition", + "automatic_digest":"automatic digest", + "information_extraction":"information extraction", + "description_generation":"description generation", + "image_classification":"image classification", + "face_recognition":"face recognition", + "image_search":"image search", + "target_detection":"target detection", + "image_description_generation":"image description generation", + "vehicle_license_plate_recognition":"vehicle license plate recognition", + "medical_image_analysis":"medical image analysis", + "unmanned":"unmanned", + "unmanned_security":"unmanned security", + "drone":"drone", + "vr_ar":"VR/AR", + "2_d_vision":"2.D vision", + "2.5_d_vision":"2.5D vision", + "3_d_reconstruction":"3Dreconstruction", + "image_processing":"image processing", + "video_processing":"video processing", + "visual_input_system":"visual input system", + "speech_coding":"speech coding", + "speech_enhancement":"speech enhancement", + "speech_recognition":"speech recognition", + "speech_synthesis":"speech synthesis" +}; + +function getCategoryDesc(isZh,key){ + var re = key; + if(isZh){ + re = categoryDesc[key]; + }else{ + re = categoryENDesc[key]; + } + if(isEmpty(re)){ + return key; + } + return re; +} + +function getTaskDesc(isZh,key){ + var re = key; + if(isZh){ + re = taskDesc[key]; + }else{ + re = taskENDesc[key]; + } + if(isEmpty(re)){ + return key; + } + return re; +} + +function getActiveItem(sort_type){ + console.log("currentSearchSortBy=" + currentSearchSortBy + " sort_type=" + sortBy[sort_type]); + if(currentSearchSortBy == sortBy[sort_type] && currentSearchAscending == sortAscending[sort_type]){ + return "active "; + }else{ + return ""; + } +} + +function displayDataSetResult(page,jsonResult,onlyReturnNum,keyword){ + var data = jsonResult.Result; + var total = jsonResult.Total; + $('#dataset_total').text(total); + if(!onlyReturnNum){ + setActivate("dataset_item"); + //$('#keyword_desc').text(keyword); + //$('#obj_desc').text(getLabel(isZh,"search_dataset")); + //$('#child_total').text(total); + $('#find_title').html(getLabel(isZh,"find_title").replace('{keyword}',keyword).replace('{tablename}',getLabel(isZh,"search_dataset")).replace('{total}',total)); + + var sortHtml = ""; + sortHtml +=""+ getLabel(isZh,"search_matched") + ""; + sortHtml +=""+ getLabel(isZh,"search_matched_download") + ""; + document.getElementById("sort_type").innerHTML=sortHtml; + + var html = ""; + var currentTime = new Date().getTime(); + for(var i = 0; i < data.length;i++){ + var recordMap = data[i]; + html += "
"; + html += "
"; + html += "
" ; + if(!isEmpty(recordMap["category"])){ + html += " " + getCategoryDesc(isZh,recordMap["category"]) + ""; + } + if(!isEmpty(recordMap["task"])){ + html += " " + getTaskDesc(isZh,recordMap["task"]) + ""; + } + html += " " +recordMap["download_times"] + " "; + html +="
"; + html += "
"; + html += " " + recordMap["title"] + ""; + html +=" "; + html +="
"; + html += "
"; + html += "

" + recordMap["description"] + "

"; + if(!isEmpty(recordMap["file_name"])){ + html += "

" + recordMap["file_name"] + "

"; + } + html +="

"; + html +=" "+ getLabel(isZh,"search_lasted_update") + " " + recordMap["updated_html"]; + html +="

"; + html +="
"; + html +="
"; + html +="
"; + } + document.getElementById("child_search_item").innerHTML=html; + } +} + +function displayOrgResult(page,jsonResult,onlyReturnNum,keyword){ + var data = jsonResult.Result; + var total = jsonResult.Total; + $('#org_total').text(total); + if(!onlyReturnNum){ + setActivate("org_item"); + //$('#keyword_desc').text(keyword); + //$('#obj_desc').text(getLabel(isZh,"search_org")); + //$('#child_total').text(total); + $('#find_title').html(getLabel(isZh,"find_title").replace('{keyword}',keyword).replace('{tablename}',getLabel(isZh,"search_org")).replace('{total}',total)); + + var sortHtml = ""; + sortHtml +=""+ getLabel(isZh,"search_matched") + ""; + sortHtml +=""+ getLabel(isZh,"search_letter_asc") + ""; + sortHtml +=""+ getLabel(isZh,"search_letter_desc") + ""; + sortHtml +=""+ getLabel(isZh,"search_lasted_create") + ""; + sortHtml +=""+ getLabel(isZh,"search_early_create") + ""; + document.getElementById("sort_type").innerHTML=sortHtml; + + var html = ""; + var currentTime = new Date().getTime(); + for(var i = 0; i < data.length;i++){ + var recordMap = data[i]; + html += "
"; + html += ""; + html += "
"; + html += "
"; + html += " " + recordMap["name"] + "  " + recordMap["full_name"] + ""; + html +="
"; + html += "
"; + html += "

" + recordMap["description"] + "

"; + html +="

"; + if(!isEmpty(recordMap["location"]) && recordMap["location"] != "null"){ + html +=" " + recordMap["location"]; + } + html +=" "; + if(!isEmpty(recordMap["website"]) && recordMap["website"] != "null"){ + html +=" " + "" + recordMap["website"] + ""; + } + html +=" "+ getLabel(isZh,"search_add_by") + " "; + html += recordMap["add_time"] + html +="

"; + html +="
"; + html +="
"; + html +="
"; + } + document.getElementById("child_search_item").innerHTML=html; + } +} +var monthDisplay=new Array("Jan","Feb","Mar","Apr","May","Jun","Jul","Aug","Spt","Oct","Nov","Dec"); +function displayUserResult(page,jsonResult,onlyReturnNum,keyword){ + var data = jsonResult.Result; + var total = jsonResult.Total; + $('#user_total').text(total); + if(!onlyReturnNum){ + setActivate("user_item"); + //$('#keyword_desc').text(keyword); + //$('#obj_desc').text(getLabel(isZh,"search_user")); + //$('#child_total').text(total); + + $('#find_title').html(getLabel(isZh,"find_title").replace('{keyword}',keyword).replace('{tablename}',getLabel(isZh,"search_user")).replace('{total}',total)); + + var sortHtml = "";//equal user sort by + sortHtml +=""+ getLabel(isZh,"search_matched") + ""; + sortHtml +=""+ getLabel(isZh,"search_letter_asc") + ""; + sortHtml +=""+ getLabel(isZh,"search_letter_desc") + ""; + sortHtml +=""+ getLabel(isZh,"search_lasted_create") + ""; + sortHtml +=""+ getLabel(isZh,"search_early_create") + ""; + + document.getElementById("sort_type").innerHTML=sortHtml; + + var html = ""; + var currentTime = new Date().getTime(); + for(var i = 0; i < data.length;i++){ + var recordMap = data[i]; + html += "
"; + html += ""; + html += "
"; + html += "
"; + html += " " + recordMap["name"] + "  " + recordMap["full_name"] + ""; + html +="
"; + html += "
"; + html += "

" + recordMap["description"] + "

"; + html +="

"; + if(!isEmpty(recordMap["email"]) && recordMap["email"] != "null"){ + html +="  " + recordMap["email"] + ""; + } + html +=" "+ getLabel(isZh,"search_add_by") + " "; + html += recordMap["add_time"] + html +="

"; + html +="
"; + html +="
"; + html +="
"; + } + document.getElementById("child_search_item").innerHTML=html; + } +} + +function setIssueOrPrInnerHtml(data,path){ + var sortHtml = ""; + if(path =="issues"){ + sortHtml +=""+ getLabel(isZh,"search_matched") + ""; + sortHtml +=""+ getLabel(isZh,"search_lasted") + ""; + }else{ + sortHtml +=""+ getLabel(isZh,"search_matched") + ""; + sortHtml +=""+ getLabel(isZh,"search_lasted") + ""; + } + + document.getElementById("sort_type").innerHTML=sortHtml; + + var html = ""; + var currentTime = new Date().getTime(); + for(var i = 0; i < data.length;i++){ + var recordMap = data[i]; + html += "
"; + html += "
"; + html += "
"; + html += " " + recordMap["name"] + ""; + html +="
"; + html += "
"; + html += "

" + recordMap["content"] + "

"; + html +="

"; + html +=" "; + html +=" " + addBlank(recordMap["repoUrl"]) +" #" + recordMap["index"] + "    "; + html +="  "; + if(recordMap["is_closed"] != null && (!(recordMap["is_closed"]) || recordMap["is_closed"]=="f")){ + html += getLabel(isZh,"search_open"); + }else{ + html += getLabel(isZh,"search_closed"); + } + html +="      " + recordMap["num_comments"]; + + html +="     "+ getLabel(isZh,"search_lasted_update") + " "+ recordMap["updated_html"]; + + html +="

"; + html +="
"; + html +="
"; + html +="
"; + } + document.getElementById("child_search_item").innerHTML=html; +} + +function addBlank(url){ + if(url == null){ + return url; + } + var tmps = url.split("/"); + if(tmps.length == 2){ + return tmps[0] + " / " + tmps[1]; + } + return url; +} + +function displayIssueResult(page,jsonResult,onlyReturnNum,keyword){ + var data = jsonResult.Result; + var total = jsonResult.Total; + $('#issue_total').text(total); + if(!onlyReturnNum){ + setActivate("issue_item"); + //$('#keyword_desc').text(keyword); + //$('#obj_desc').text(getLabel(isZh,"search_issue")); + //$('#child_total').text(total); + $('#find_title').html(getLabel(isZh,"find_title").replace('{keyword}',keyword).replace('{tablename}',getLabel(isZh,"search_issue")).replace('{total}',total)); + + setIssueOrPrInnerHtml(data,"issues"); + } +} + +function setActivate(name){ + $('#repo_item').removeClass("active"); + $('#user_item').removeClass("active"); + $('#issue_item').removeClass("active"); + $('#dataset_item').removeClass("active"); + $('#org_item').removeClass("active"); + $('#pr_item').removeClass("active"); + if(name==null){ + return; + } + var tmp = "#" + name; + $(tmp).addClass("active"); +} + +function displayRepoResult(page,jsonResult,onlyReturnNum,keyword){ + var data = jsonResult.Result; + var total = jsonResult.Total; + $('#repo_total').text(total); + + if(!onlyReturnNum){ + setActivate("repo_item"); + // $('#keyword_desc').text(keyword); + //$('#obj_desc').text(getLabel(isZh,"search_repo")); + //$('#child_total').text(total); + $('#find_title').html(getLabel(isZh,"find_title").replace('{keyword}',keyword).replace('{tablename}',getLabel(isZh,"search_repo")).replace('{total}',total)); + + var sortHtml = ""; + sortHtml +=""+ getLabel(isZh,"search_matched") + ""; + sortHtml +=""+ getLabel(isZh,"search_lasted") + ""; + sortHtml +=""+ getLabel(isZh,"search_watched") + ""; + sortHtml +=""+ getLabel(isZh,"search_star") + ""; + sortHtml +=""+ getLabel(isZh,"search_fork") + ""; + + document.getElementById("sort_type").innerHTML=sortHtml; + + var html = ""; + var currentTime = new Date().getTime(); + for(var i = 0; i < data.length;i++){ + var recordMap = data[i]; + html += "
"; + if(!isEmpty(recordMap['avatar'])){ + html += ""; + } + html += "
"; + html += "
"; + html += " " + recordMap["owner_name"] +" / " + recordMap["alias"] + ""; + if(recordMap["is_private"]){ + html +=" "; + } + html +="
"; + html += "
"; + html += "

" + recordMap["description"] + "

"; + html += "
"; + if(!isEmpty(recordMap["topics"]) && recordMap["topics"] !="null"){ + for(var j = 0; j < recordMap["topics"].length;j++){ + //function searchLabel(tableName,keyword,sortBy="",ascending=false) + html +="
"+ recordMap["hightTopics"][j] + "
"; + } + } + html +="
"; + html +="

"; + html +="  " + recordMap["num_watches"] + "   " + recordMap["num_stars"] + "   " + recordMap["num_forks"] +"  "; + html +="    "+ getLabel(isZh,"search_lasted_update") + " " + recordMap["updated_html"]; + if(!isEmpty(recordMap["lang"])){ + var lang = recordMap["lang"] + var tmpLang = recordMap["lang"].split(","); + if(tmpLang.length>0){ + lang = tmpLang[0] + } + var backColor = "#3572A5"; + if(LanguagesColor[lang] != null){ + backColor = LanguagesColor[lang]; + } + html +="  " + lang + ""; + } + html +="

"; + html +="
"; + html +="
"; + html +="
"; + } + + document.getElementById("child_search_item").innerHTML=html; + } +} + +function getTime(UpdatedUnix,currentTime){ + UpdatedUnix = UpdatedUnix; + currentTime = currentTime / 1000; + var timeEscSecond = currentTime - UpdatedUnix; + if( timeEscSecond < 0){ + timeEscSecond = 1; + } + console.log("currentTime=" + currentTime + " updateUnix=" + UpdatedUnix); + + var hours= Math.floor(timeEscSecond / 3600); + //计算相差分钟数 + var leave2 = Math.floor(timeEscSecond % (3600)); //计算小时数后剩余的秒数 + var minutes= Math.floor(leave2 / 60);//计算相差分钟数 + + var leave3=Math.floor(leave2 % 60); //计算分钟数后剩余的秒数 + var seconds= leave3; + + if(hours == 0 && minutes == 0){ + return seconds + getRepoOrOrg(6,isZh); + }else{ + if(hours > 0){ + if(hours >= 24){ + var days = Math.ceil(hours/24) + if (days >= 30 && days <365){ + return Math.ceil(days/30) + getRepoOrOrg(8,isZh); + }else if(days >= 365){ + return Math.ceil(days/365) + getRepoOrOrg(9,isZh); + } + return Math.ceil(hours/24) + getRepoOrOrg(7,isZh); + }else{ + return hours + getRepoOrOrg(4,isZh); + } + }else{ + return minutes + getRepoOrOrg(5,isZh); + } + } +} + +function getRepoOrOrg(key,isZhLang){ + if(isZhLang){ + return repoAndOrgZH[key]; + }else{ + return repoAndOrgEN[key]; + } +} + +var repoAndOrgZH={ + "1":"项目", + "2":"成员", + "3":"团队", + "4":"小时前", + "5":"分钟前", + "6":"秒前", + "7":"天前", + "8":"个月前", + "9":"年前" +}; + +var repoAndOrgEN={ + "1":"repository", + "2":"Members ", + "3":"Teams", + "4":" hours ago", + "5":" minutes ago", + "6":" seconds ago", + "7":" day ago", + "8":" month ago", + "9":" year ago" +}; + + + + +function page(current){ + + currentPage=current; + doSearch(currentSearchTableName,currentSearchKeyword,current,pageSize,false,currentSearchSortBy,OnlySearchLabel); + + } + + function nextPage(){ + currentPage = currentPage+1; + console.log("currentPage=" + currentPage); + if(currentPage >= endIndex){ + startIndex=startIndex+1; + endIndex = endIndex +1; + } + page(currentPage); + } + + function prePage(){ + console.log("currentPage=" + currentPage); + if(currentPage > 1){ + currentPage = currentPage-1; + if(currentPage <= startIndex && startIndex > 1){ + startIndex = startIndex -1; + endIndex = endIndex - 1; + } + console.log("currentPage=" + (currentPage)); + page(currentPage); + } + } + +function getXPosition(e){ + var x=e.offsetLeft; + while(e=e.offsetParent) + { + x+=e.offsetLeft; + } + return x+20;//-260防止屏幕超出 +} +//获取y坐标 +function getYPosition(e){ + var y=e.offsetTop; + while(e=e.offsetParent) + { + y+=e.offsetTop; + } + return y+20;//80为input高度 +} + + + function goPage(event){ + + var inputpage = document.getElementById("inputpage_div") + var left = getXPosition(event.target); + var top = getYPosition(event.target); + var goNum = $('#inputpage').val(); + if (goNum<=0){ + showTip(getLabel(isZh,"search_input_large_0"),"warning",left+5,top); + } + else if(goNum<=totalPage){ + page(goNum); + } + else{ + showTip(getLabel(isZh,"search_input_maxed"),"warning",left+5,top); + } + } + + function showTip(tip, type,left,top) { + var $tip = $('#tipmsg'); + var tipmsg = document.getElementById("tipmsg") + var style="z-index:10024;top:" + top + "px;left:" + left + "px;position:absolute;width:200px;height:60px;vertical-align:middle;"; + console.log(style); + tipmsg.style = style; + var html ="

" + tip + "

" + $tip.stop(true).prop('class', 'alert alert-' + type).html(html).fadeIn(500).delay(2000).fadeOut(500); + } + + function setPage(currentPage){ + console.log("totalPage=" + totalPage); + var html =""; + console.log("currentPage=" + currentPage); + console.log("privateTotal=" + privateTotal); + // if(totalPage==0){ + // return; + // } + html += "" + getLabel(isZh,"search_input_total") + " " + totalNum + " " + getLabel(isZh,"search_srtip") + "" + if(currentPage > 1){ + html += "" + getLabel(isZh,"search_home_page") + ""; + html += ""; + }else{ + html += "" + getLabel(isZh,"search_home_page") + ""; + html += ""; + } + + for(var i=startIndex; i <= endIndex; i++){ + var page_i = i; + if(page_i > totalPage){ + break; + } + if( page_i == currentPage){ + html += "" + page_i + ""; + }else{ + html += "" + page_i + ""; + } + } + + if(currentPage >=totalPage){ + html += ""; + html += "" + getLabel(isZh,"search_last_page") + ""; + }else{ + html += ""; + html += "" + getLabel(isZh,"search_last_page") + ""; + } + + html +="
" + getLabel(isZh,"search_go_to") + "
" + getLabel(isZh,"search_go_page") + "
"; + console.log("html=" + html) + document.getElementById("page_menu").innerHTML=html; + $('#inputpage').on('keypress',function(event){ + if(event.keyCode == 13){ + goPage(event); + } + }); + } + +$('#keyword_input').on('keypress',function(event){ + if(event.keyCode == 13){ + search(); + } +}); + + + + + +var LanguagesColor = { + "1C Enterprise": "#814CCC", + "ABAP": "#E8274B", + "AGS Script": "#B9D9FF", + "AMPL": "#E6EFBB", + "ANTLR": "#9DC3FF", + "API Blueprint": "#2ACCA8", + "APL": "#5A8164", + "ASP": "#6a40fd", + "ATS": "#1ac620", + "ActionScript": "#882B0F", + "Ada": "#02f88c", + "Agda": "#315665", + "Alloy": "#64C800", + "AngelScript": "#C7D7DC", + "AppleScript": "#101F1F", + "Arc": "#aa2afe", + "AspectJ": "#a957b0", + "Assembly": "#6E4C13", + "Asymptote": "#4a0c0c", + "AutoHotkey": "#6594b9", + "AutoIt": "#1C3552", + "Ballerina": "#FF5000", + "Batchfile": "#C1F12E", + "BlitzMax": "#cd6400", + "Boo": "#d4bec1", + "Brainfuck": "#2F2530", + "C": "#555555", + "C#": "#178600", + "C++": "#f34b7d", + "CSS": "#563d7c", + "Ceylon": "#dfa535", + "Chapel": "#8dc63f", + "Cirru": "#ccccff", + "Clarion": "#db901e", + "Clean": "#3F85AF", + "Click": "#E4E6F3", + "Clojure": "#db5855", + "CoffeeScript": "#244776", + "ColdFusion": "#ed2cd6", + "Common Lisp": "#3fb68b", + "Common Workflow Language": "#B5314C", + "Component Pascal": "#B0CE4E", + "Crystal": "#000100", + "Cuda": "#3A4E3A", + "D": "#ba595e", + "DM": "#447265", + "Dart": "#00B4AB", + "DataWeave": "#003a52", + "Dhall": "#dfafff", + "Dockerfile": "#384d54", + "Dogescript": "#cca760", + "Dylan": "#6c616e", + "E": "#ccce35", + "ECL": "#8a1267", + "EQ": "#a78649", + "Eiffel": "#946d57", + "Elixir": "#6e4a7e", + "Elm": "#60B5CC", + "Emacs Lisp": "#c065db", + "EmberScript": "#FFF4F3", + "Erlang": "#B83998", + "F#": "#b845fc", + "F*": "#572e30", + "FLUX": "#88ccff", + "Factor": "#636746", + "Fancy": "#7b9db4", + "Fantom": "#14253c", + "Faust": "#c37240", + "Forth": "#341708", + "Fortran": "#4d41b1", + "FreeMarker": "#0050b2", + "Frege": "#00cafe", + "G-code": "#D08CF2", + "GAML": "#FFC766", + "GDScript": "#355570", + "Game Maker Language": "#71b417", + "Genie": "#fb855d", + "Gherkin": "#5B2063", + "Glyph": "#c1ac7f", + "Gnuplot": "#f0a9f0", + "Go": "#00ADD8", + "Golo": "#88562A", + "Gosu": "#82937f", + "Grammatical Framework": "#79aa7a", + "Groovy": "#e69f56", + "HTML": "#e34c26", + "Hack": "#878787", + "Harbour": "#0e60e3", + "Haskell": "#5e5086", + "Haxe": "#df7900", + "HiveQL": "#dce200", + "HolyC": "#ffefaf", + "Hy": "#7790B2", + "IDL": "#a3522f", + "IGOR Pro": "#0000cc", + "Idris": "#b30000", + "Io": "#a9188d", + "Ioke": "#078193", + "Isabelle": "#FEFE00", + "J": "#9EEDFF", + "JSONiq": "#40d47e", + "Java": "#b07219", + "JavaScript": "#f1e05a", + "Jolie": "#843179", + "Jsonnet": "#0064bd", + "Julia": "#a270ba", + "Jupyter Notebook": "#DA5B0B", + "KRL": "#28430A", + "Kotlin": "#F18E33", + "LFE": "#4C3023", + "LLVM": "#185619", + "LOLCODE": "#cc9900", + "LSL": "#3d9970", + "Lasso": "#999999", + "Lex": "#DBCA00", + "LiveScript": "#499886", + "LookML": "#652B81", + "Lua": "#000080", + "MATLAB": "#e16737", + "MAXScript": "#00a6a6", + "MLIR": "#5EC8DB", + "MQL4": "#62A8D6", + "MQL5": "#4A76B8", + "MTML": "#b7e1f4", + "Makefile": "#427819", + "Mask": "#f97732", + "Max": "#c4a79c", + "Mercury": "#ff2b2b", + "Meson": "#007800", + "Metal": "#8f14e9", + "Mirah": "#c7a938", + "Modula-3": "#223388", + "NCL": "#28431f", + "Nearley": "#990000", + "Nemerle": "#3d3c6e", + "NetLinx": "#0aa0ff", + "NetLinx+ERB": "#747faa", + "NetLogo": "#ff6375", + "NewLisp": "#87AED7", + "Nextflow": "#3ac486", + "Nim": "#37775b", + "Nit": "#009917", + "Nix": "#7e7eff", + "Nu": "#c9df40", + "OCaml": "#3be133", + "ObjectScript": "#424893", + "Objective-C": "#438eff", + "Objective-C++": "#6866fb", + "Objective-J": "#ff0c5a", + "Odin": "#60AFFE", + "Omgrofl": "#cabbff", + "Opal": "#f7ede0", + "OpenQASM": "#AA70FF", + "Oxygene": "#cdd0e3", + "Oz": "#fab738", + "P4": "#7055b5", + "PHP": "#4F5D95", + "PLSQL": "#dad8d8", + "Pan": "#cc0000", + "Papyrus": "#6600cc", + "Parrot": "#f3ca0a", + "Pascal": "#E3F171", + "Pawn": "#dbb284", + "Pep8": "#C76F5B", + "Perl": "#0298c3", + "PigLatin": "#fcd7de", + "Pike": "#005390", + "PogoScript": "#d80074", + "PostScript": "#da291c", + "PowerBuilder": "#8f0f8d", + "PowerShell": "#012456", + "Processing": "#0096D8", + "Prolog": "#74283c", + "Propeller Spin": "#7fa2a7", + "Puppet": "#302B6D", + "PureBasic": "#5a6986", + "PureScript": "#1D222D", + "Python": "#3572A5", + "QML": "#44a51c", + "Quake": "#882233", + "R": "#198CE7", + "RAML": "#77d9fb", + "RUNOFF": "#665a4e", + "Racket": "#3c5caa", + "Ragel": "#9d5200", + "Raku": "#0000fb", + "Rascal": "#fffaa0", + "Reason": "#ff5847", + "Rebol": "#358a5b", + "Red": "#f50000", + "Ren'Py": "#ff7f7f", + "Ring": "#2D54CB", + "Riot": "#A71E49", + "Roff": "#ecdebe", + "Rouge": "#cc0088", + "Ruby": "#701516", + "Rust": "#dea584", + "SAS": "#B34936", + "SQF": "#3F3F3F", + "SRecode Template": "#348a34", + "SaltStack": "#646464", + "Scala": "#c22d40", + "Scheme": "#1e4aec", + "Self": "#0579aa", + "Shell": "#89e051", + "Shen": "#120F14", + "Slash": "#007eff", + "Slice": "#003fa2", + "SmPL": "#c94949", + "Smalltalk": "#596706", + "Solidity": "#AA6746", + "SourcePawn": "#5c7611", + "Squirrel": "#800000", + "Stan": "#b2011d", + "Standard ML": "#dc566d", + "Starlark": "#76d275", + "SuperCollider": "#46390b", + "Swift": "#ffac45", + "SystemVerilog": "#DAE1C2", + "TI Program": "#A0AA87", + "Tcl": "#e4cc98", + "TeX": "#3D6117", + "Terra": "#00004c", + "Turing": "#cf142b", + "TypeScript": "#2b7489", + "UnrealScript": "#a54c4d", + "V": "#5d87bd", + "VBA": "#867db1", + "VBScript": "#15dcdc", + "VCL": "#148AA8", + "VHDL": "#adb2cb", + "Vala": "#fbe5cd", + "Verilog": "#b2b7f8", + "Vim script": "#199f4b", + "Visual Basic .NET": "#945db7", + "Volt": "#1F1F1F", + "Vue": "#2c3e50", + "WebAssembly": "#04133b", + "Wollok": "#a23738", + "X10": "#4B6BEF", + "XC": "#99DA07", + "XQuery": "#5232e7", + "XSLT": "#EB8CEB", + "YARA": "#220000", + "YASnippet": "#32AB90", + "Yacc": "#4B6C4B", + "ZAP": "#0d665e", + "ZIL": "#dc75e5", + "ZenScript": "#00BCD1", + "Zephir": "#118f9e", + "Zig": "#ec915c", + "eC": "#913960", + "mIRC Script": "#926059", + "mcfunction": "#E22837", + "nesC": "#94B0C7", + "ooc": "#b0b77e", + "q": "#0040cd", + "sed": "#64b970", + "wdl": "#42f1f4", + "wisp": "#7582D1", + "xBase": "#403a40", +} + +function getLabel(isZh,key){ + if(isZh){ + return zhCN[key] + }else{ + return esUN[key] + } +} + +var zhCN={ + "search":"搜索", + "search_repo":"项目", + "search_dataset":"数据集", + "search_issue":"任务", + "search_pr":"合并请求", + "search_user":"用户", + "search_org":"组织", + "search_finded":"找到", + "search_matched":"最佳匹配", + "search_matched_download":"下载次数", + "search_lasted_update":"最后更新于", + "search_letter_asc":"字母顺序排序", + "search_letter_desc":"字母逆序排序", + "search_lasted_create":"最近创建", + "search_early_create":"最早创建", + "search_add_by":"加入于", + "search_lasted":"最近更新", + "search_open":"开启中", + "search_closed":"已关闭", + "search_watched":"关注数", + "search_star":"点赞数", + "search_fork":"Fork数", + "search_input_large_0":"请输入大于0的数值。", + "search_input_maxed":"不能超出总页数。", + "search_input_total":"共", + "search_srtip":"条", + "search_home_page":"首页", + "search_last_page":"末页", + "search_go_to":"前往", + "search_go_page":"页", + "find_title":"“{keyword}”相关{tablename}约为{total}个", + "search_empty":"请输入任意关键字开始搜索。" + } + + var esUN={ + "search":"Search", + "search_repo":"Repository", + "search_dataset":"DataSet", + "search_issue":"Issue", + "search_pr":"Pull Request", + "search_user":"User", + "search_org":"Organization", + "search_finded":"Find", + "search_matched":"Best Match", + "search_matched_download":"Most downloads", + "search_lasted_update":"Updated ", + "search_letter_asc":"Alphabetically", + "search_letter_desc":"Reverse alphabetically", + "search_lasted_create":"Recently created", + "search_early_create":"First created", + "search_add_by":"Joined on", + "search_lasted":"Recently updated", + "search_open":"Open", + "search_closed":"Closed", + "search_watched":"Watches", + "search_star":"Stars", + "search_fork":"Forks", + "search_input_large_0":"Please enter a value greater than 0.", + "search_input_maxed":"Cannot exceed total pages.", + "search_input_total":"Total", + "search_srtip":"", + "search_home_page":"First", + "search_last_page":"Last", + "search_go_to":"Go", + "search_go_page":"Page", + "find_title":" {total} \"{keyword}\" related {tablename}", + "search_empty":"Please enter any keyword to start the search." + } + initDiv(false); + document.onreadystatechange = function() { + if (document.readyState === "complete") { + var tmpSearchLabel = sessionStorage.getItem("searchLabel"); + console.log("tmpSearchLabel=" + tmpSearchLabel); + if(tmpSearchLabel){ + console.log("search label...."); + sessionStorage.removeItem("searchLabel"); + doSearchLabel(sessionStorage.getItem("tableName"),sessionStorage.getItem("keyword"),sessionStorage.getItem("sortBy"),sessionStorage.getItem("ascending")); + }else{ + var specifySearch = sessionStorage.getItem("specifySearch"); + if(specifySearch){ + sessionStorage.removeItem("specifySearch"); + console.log("search sepcial keyword=...." + sessionStorage.getItem("keyword")); + document.getElementById("keyword_input").value = sessionStorage.getItem("keyword"); + doSpcifySearch(sessionStorage.getItem("tableName"),sessionStorage.getItem("keyword"),sessionStorage.getItem("sortBy"),sessionStorage.getItem("ascending")); + }else{ + console.log("normal search...."); + search(); + } + + } + } + } + + diff --git a/public/self/dataset_preview.js b/public/self/dataset_preview.js index 3c9ded0aa..e6b79dd7d 100644 --- a/public/self/dataset_preview.js +++ b/public/self/dataset_preview.js @@ -620,10 +620,10 @@ function showfilelist(){ for (var i=0;i 70){ var tmpIndex = labeltastresult[i].pic_image_field.indexOf("/",70); - console.log(tmpIndex) + //console.log(tmpIndex) if(tmpIndex != -1){ fname = labeltastresult[i].pic_image_field.substring(tmpIndex + 1); fname = fname.substring(fname.indexOf('/')+1); @@ -679,7 +679,7 @@ function breadFiles(){ fname_full_path = tableData[fileindex].pic_image_field.substring(tmp_index + 1); } var fname_path = fname_full_path.split('/') - console.log(fname_path) + //console.log(fname_path) // var filename_text = tableData[fileindex].pic_image_field.substring(tableData[fileindex].pic_image_field.lastIndexOf('/')+1) var html_breadFile = '' // var source_name = filename_title+'.zip' diff --git a/routers/admin/cloudbrains.go b/routers/admin/cloudbrains.go old mode 100644 new mode 100755 index 6bbd534b9..884ed6b9b --- a/routers/admin/cloudbrains.go +++ b/routers/admin/cloudbrains.go @@ -41,7 +41,7 @@ func CloudBrains(ctx *context.Context) { if page <= 0 { page = 1 } - debugType := modelarts.DebugType + debugType := models.TypeCloudBrainAll if listType == models.GPUResource { debugType = models.TypeCloudBrainOne } else if listType == models.NPUResource { @@ -121,7 +121,7 @@ func DownloadCloudBrains(ctx *context.Context) { Page: page, PageSize: 1, }, - Type: modelarts.DebugType, + Type: models.TypeCloudBrainAll, NeedRepoInfo: false, IsLatestVersion: modelarts.IsLatestVersion, }) @@ -151,7 +151,7 @@ func DownloadCloudBrains(ctx *context.Context) { Page: page, PageSize: pageSize, }, - Type: modelarts.DebugType, + Type: models.TypeCloudBrainAll, NeedRepoInfo: true, IsLatestVersion: modelarts.IsLatestVersion, }) diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 306854af3..9de65662f 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -62,10 +62,10 @@ import ( "net/http" "strings" - "code.gitea.io/gitea/routers/authentication" - "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/auth" + + "code.gitea.io/gitea/modules/cloudbrain" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" @@ -77,6 +77,7 @@ import ( "code.gitea.io/gitea/routers/api/v1/repo" _ "code.gitea.io/gitea/routers/api/v1/swagger" // for swagger generation "code.gitea.io/gitea/routers/api/v1/user" + "code.gitea.io/gitea/routers/authentication" repo_ext "code.gitea.io/gitea/routers/repo" "gitea.com/macaron/binding" @@ -546,6 +547,8 @@ func RegisterRoutes(m *macaron.Macaron) { }) }, operationReq) + m.Get("/query_user_metrics", operationReq, repo_ext.QueryMetrics) + m.Get("/query_user_rank_list", operationReq, repo_ext.QueryRankingList) m.Get("/query_user_static_page", operationReq, repo_ext.QueryUserStaticDataPage) m.Get("/query_user_current_month", operationReq, repo_ext.QueryUserStaticCurrentMonth) m.Get("/query_user_current_week", operationReq, repo_ext.QueryUserStaticCurrentWeek) @@ -882,6 +885,13 @@ func RegisterRoutes(m *macaron.Macaron) { m.Group("/cloudbrain", func() { m.Get("/:id", repo.GetCloudbrainTask) m.Get("/:id/log", repo.CloudbrainGetLog) + m.Group("/train-job", func() { + m.Group("/:jobid", func() { + m.Get("", repo.GetModelArtsTrainJobVersion) + m.Get("/model_list", repo.CloudBrainModelList) + m.Post("/stop_version", cloudbrain.AdminOrOwnerOrJobCreaterRightForTrain, repo_ext.CloudBrainStop) + }) + }) }, reqRepoReader(models.UnitTypeCloudBrain)) m.Group("/modelarts", func() { m.Group("/notebook", func() { diff --git a/routers/api/v1/repo/cloudbrain.go b/routers/api/v1/repo/cloudbrain.go index b2f529dfb..a81263823 100755 --- a/routers/api/v1/repo/cloudbrain.go +++ b/routers/api/v1/repo/cloudbrain.go @@ -6,16 +6,18 @@ package repo import ( - "code.gitea.io/gitea/modules/timeutil" + "encoding/json" "net/http" "sort" + "strings" "time" - "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/cloudbrain" "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/storage" + routerRepo "code.gitea.io/gitea/routers/repo" ) // cloudbrain get job task by jobid @@ -63,7 +65,7 @@ func GetCloudbrainTask(ctx *context.APIContext) { log.Error("GetJob failed:", err) return } - result, err := models.ConvertToJobResultPayload(jobResult.Payload) + result, _ := models.ConvertToJobResultPayload(jobResult.Payload) if err != nil { ctx.NotFound(err) log.Error("ConvertToJobResultPayload failed:", err) @@ -71,24 +73,16 @@ func GetCloudbrainTask(ctx *context.APIContext) { } job.Status = result.JobStatus.State + taskRoles := result.TaskRoles + taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{})) if result.JobStatus.State != string(models.JobWaiting) && result.JobStatus.State != string(models.JobFailed) { - taskRoles := result.TaskRoles - taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{})) - job.ContainerIp = taskRes.TaskStatuses[0].ContainerIP job.ContainerID = taskRes.TaskStatuses[0].ContainerID job.Status = taskRes.TaskStatuses[0].State - - if job.StartTime == 0 && !taskRes.TaskStatuses[0].StartAt.IsZero() { - job.StartTime = timeutil.TimeStamp(taskRes.TaskStatuses[0].StartAt.Unix()) - } } if result.JobStatus.State != string(models.JobWaiting) { - if job.EndTime == 0 && models.IsCloudBrainOneDebugJobTerminal(job.Status) { - job.EndTime = timeutil.TimeStampNow() - } - job.ComputeAndSetDuration() + models.ParseAndSetDurationFromCloudBrainOne(result, job) err = models.UpdateJob(job) if err != nil { log.Error("UpdateJob failed:", err) @@ -161,3 +155,55 @@ func CloudbrainGetLog(ctx *context.Context) { return } + +func CloudBrainModelList(ctx *context.APIContext) { + var ( + err error + ) + + var jobID = ctx.Params(":jobid") + var versionName = ctx.Query("version_name") + parentDir := ctx.Query("parentDir") + dirArray := strings.Split(parentDir, "/") + + task, err := models.GetCloudbrainByJobIDAndVersionName(jobID, versionName) + if err != nil { + log.Error("GetCloudbrainByJobID(%s) failed:%v", task.JobName, err.Error()) + return + } + + //get dirs + dirs, err := routerRepo.GetModelDirs(task.JobName, parentDir) + if err != nil { + log.Error("GetModelDirs failed:%v", err.Error(), ctx.Data["msgID"]) + ctx.ServerError("GetModelDirs failed:", err) + return + } + + var fileInfos []storage.FileInfo + err = json.Unmarshal([]byte(dirs), &fileInfos) + if err != nil { + log.Error("json.Unmarshal failed:%v", err.Error(), ctx.Data["msgID"]) + ctx.ServerError("json.Unmarshal failed:", err) + return + } + + for i, fileInfo := range fileInfos { + temp, _ := time.Parse("2006-01-02 15:04:05", fileInfo.ModTime) + fileInfos[i].ModTime = temp.Local().Format("2006-01-02 15:04:05") + } + + sort.Slice(fileInfos, func(i, j int) bool { + return fileInfos[i].ModTime > fileInfos[j].ModTime + }) + + ctx.JSON(http.StatusOK, map[string]interface{}{ + "JobID": jobID, + "VersionName": versionName, + "StatusOK": 0, + "Path": dirArray, + "Dirs": fileInfos, + "task": task, + "PageIsCloudBrain": true, + }) +} diff --git a/routers/api/v1/repo/modelarts.go b/routers/api/v1/repo/modelarts.go index d7d011e07..e24ac95fb 100755 --- a/routers/api/v1/repo/modelarts.go +++ b/routers/api/v1/repo/modelarts.go @@ -6,16 +6,17 @@ package repo import ( - "code.gitea.io/gitea/modules/timeutil" "net/http" "strconv" "strings" "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/cloudbrain" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/modelarts" "code.gitea.io/gitea/modules/storage" + "code.gitea.io/gitea/modules/timeutil" routerRepo "code.gitea.io/gitea/routers/repo" ) @@ -66,8 +67,8 @@ func GetModelArtsNotebook2(ctx *context.APIContext) { ctx.NotFound(err) return } - if job.StartTime == 0 && result.Lease.CreateTime > 0 { - job.StartTime = timeutil.TimeStamp(result.Lease.CreateTime / 1000) + if job.StartTime == 0 && result.Lease.UpdateTime > 0 { + job.StartTime = timeutil.TimeStamp(result.Lease.UpdateTime / 1000) } job.Status = result.Status if job.EndTime == 0 && models.IsModelArtsDebugJobTerminal(job.Status) { @@ -133,27 +134,61 @@ func GetModelArtsTrainJobVersion(ctx *context.APIContext) { ctx.NotFound(err) return } - result, err := modelarts.GetTrainJob(jobID, strconv.FormatInt(job.VersionID, 10)) - if err != nil { - ctx.NotFound(err) - return - } - if job.StartTime == 0 && result.StartTime > 0 { - job.StartTime = timeutil.TimeStamp(result.StartTime / 1000) - } - job.Status = modelarts.TransTrainJobStatus(result.IntStatus) - job.Duration = result.Duration / 1000 - job.TrainJobDuration = result.TrainJobDuration - job.TrainJobDuration = models.ConvertDurationToStr(job.Duration) + if job.Type == models.TypeCloudBrainOne { + jobResult, err := cloudbrain.GetJob(job.JobID) + if err != nil { + ctx.NotFound(err) + log.Error("GetJob failed:", err) + return + } + result, err := models.ConvertToJobResultPayload(jobResult.Payload) + if err != nil { + ctx.NotFound(err) + log.Error("ConvertToJobResultPayload failed:", err) + return + } - if job.EndTime == 0 && models.IsTrainJobTerminal(job.Status) && job.StartTime > 0 { - job.EndTime = job.StartTime.Add(job.Duration) - } + job.Status = result.JobStatus.State + if result.JobStatus.State != string(models.JobWaiting) && result.JobStatus.State != string(models.JobFailed) { + taskRoles := result.TaskRoles + taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{})) - err = models.UpdateTrainJobVersion(job) - if err != nil { - log.Error("UpdateJob failed:", err) + job.ContainerIp = taskRes.TaskStatuses[0].ContainerIP + job.ContainerID = taskRes.TaskStatuses[0].ContainerID + job.Status = taskRes.TaskStatuses[0].State + } + + if result.JobStatus.State != string(models.JobWaiting) { + err = models.UpdateJob(job) + if err != nil { + log.Error("UpdateJob failed:", err) + } + } + } else { + result, err := modelarts.GetTrainJob(jobID, strconv.FormatInt(job.VersionID, 10)) + if err != nil { + ctx.NotFound(err) + return + } + + if job.StartTime == 0 && result.StartTime > 0 { + job.StartTime = timeutil.TimeStamp(result.StartTime / 1000) + } + job.Status = modelarts.TransTrainJobStatus(result.IntStatus) + job.Duration = result.Duration / 1000 + job.TrainJobDuration = result.TrainJobDuration + + job.TrainJobDuration = models.ConvertDurationToStr(job.Duration) + + if job.EndTime == 0 && models.IsTrainJobTerminal(job.Status) && job.StartTime > 0 { + job.EndTime = job.StartTime.Add(job.Duration) + } + + err = models.UpdateTrainJobVersion(job) + if err != nil { + log.Error("UpdateJob failed:", err) + } } ctx.JSON(http.StatusOK, map[string]interface{}{ @@ -377,9 +412,7 @@ func GetModelArtsInferenceJob(ctx *context.APIContext) { } job.Status = modelarts.TransTrainJobStatus(result.IntStatus) job.Duration = result.Duration / 1000 - job.TrainJobDuration = result.TrainJobDuration - - job.TrainJobDuration = models.ConvertDurationToStr(result.Duration) + job.TrainJobDuration = models.ConvertDurationToStr(job.Duration) if job.EndTime == 0 && models.IsTrainJobTerminal(job.Status) && job.StartTime > 0 { job.EndTime = job.StartTime.Add(job.Duration) diff --git a/routers/home.go b/routers/home.go index c33d7a049..324bb1032 100755 --- a/routers/home.go +++ b/routers/home.go @@ -49,7 +49,7 @@ func Home(ctx *context.Context) { ctx.HTML(200, tplHome) } -func setRecommendURL(ctx *context.Context) { +func setRecommendURLOnly(ctx *context.Context) { addr := setting.RecommentRepoAddr[10:] start := strings.Index(addr, "/") end := strings.Index(addr, "raw") @@ -58,7 +58,10 @@ func setRecommendURL(ctx *context.Context) { } else { ctx.Data["RecommendURL"] = setting.RecommentRepoAddr } +} +func setRecommendURL(ctx *context.Context) { + setRecommendURLOnly(ctx) ctx.Data["page_title"] = ctx.Tr("home.page_title") ctx.Data["page_small_title"] = ctx.Tr("home.page_small_title") ctx.Data["page_description"] = ctx.Tr("home.page_description") @@ -441,17 +444,39 @@ func ExploreOrganizations(ctx *context.Context) { ctx.Data["PageIsExploreOrganizations"] = true ctx.Data["IsRepoIndexerEnabled"] = setting.Indexer.RepoIndexerEnabled - visibleTypes := []structs.VisibleType{structs.VisibleTypePublic} - if ctx.User != nil { - visibleTypes = append(visibleTypes, structs.VisibleTypeLimited, structs.VisibleTypePrivate) + N := 10 + starInfo, err := models.FindTopNStarsOrgs(N) + if err != nil { + log.Error("GetStarOrgInfos failed:%v", err.Error(), ctx.Data["MsgID"]) + ctx.ServerError("GetStarOrgInfos", err) + return + } + memberInfo, err := models.FindTopNMembersOrgs(N) + if err != nil { + log.Error("GetMemberOrgInfos failed:%v", err.Error(), ctx.Data["MsgID"]) + ctx.ServerError("GetMemberOrgInfos", err) + return + } + openIInfo, err := models.FindTopNOpenIOrgs(N) + if err != nil { + log.Error("GetOpenIOrgInfos failed:%v", err.Error(), ctx.Data["MsgID"]) + ctx.ServerError("GetOpenIOrgInfos", err) + return } - RenderUserSearch(ctx, &models.SearchUserOptions{ - Actor: ctx.User, - Type: models.UserTypeOrganization, - ListOptions: models.ListOptions{PageSize: setting.UI.ExplorePagingNum}, - Visible: visibleTypes, - }, tplExploreOrganizations) + recommendOrgs, err := GetRecommendOrg() + if err != nil { + log.Error("GetRecommendOrgInfos failed:%v", err.Error(), ctx.Data["MsgID"]) + ctx.ServerError("GetRecommendOrgInfos", err) + return + } + setRecommendURLOnly(ctx) + ctx.Data["RecommendOrgs"] = recommendOrgs + ctx.Data["StarOrgs"] = starInfo + ctx.Data["MemberOrgs"] = memberInfo + ctx.Data["ActiveOrgs"] = openIInfo + + ctx.HTML(http.StatusOK, tplExploreOrganizations) } // ExploreCode render explore code page @@ -583,12 +608,12 @@ func NotFound(ctx *context.Context) { ctx.NotFound("home.NotFound", nil) } -func RecommendOrgFromPromote(ctx *context.Context) { +func GetRecommendOrg() ([]map[string]interface{}, error) { url := setting.RecommentRepoAddr + "organizations" result, err := repository.RecommendFromPromote(url) + if err != nil { - ctx.ServerError("500", err) - return + return nil, err } resultOrg := make([]map[string]interface{}, 0) for _, userName := range result { @@ -598,6 +623,7 @@ func RecommendOrgFromPromote(ctx *context.Context) { userMap["Name"] = user.Name userMap["Description"] = user.Description userMap["FullName"] = user.FullName + userMap["HomeLink"] = user.HomeLink() userMap["ID"] = user.ID userMap["Avatar"] = user.RelAvatarLink() userMap["NumRepos"] = user.NumRepos @@ -608,7 +634,15 @@ func RecommendOrgFromPromote(ctx *context.Context) { log.Info("query user error," + err.Error()) } } + return resultOrg, nil +} +func RecommendOrgFromPromote(ctx *context.Context) { + resultOrg, err := GetRecommendOrg() + if err != nil { + ctx.ServerError("500", err) + return + } ctx.JSON(200, resultOrg) } diff --git a/routers/init.go b/routers/init.go index 8b93b64d8..eab513c78 100755 --- a/routers/init.go +++ b/routers/init.go @@ -71,6 +71,8 @@ func NewServices() { log.Info("decompression.NewContext() succeed.") labelmsg.Init() log.Info("labelmsg.Init() succeed.") + InitESClient() + log.Info("ES Client succeed.") } // In case of problems connecting to DB, retry connection. Eg, PGSQL in Docker Container on Synology diff --git a/routers/private/internal.go b/routers/private/internal.go index 0dd725ca3..ace25c809 100755 --- a/routers/private/internal.go +++ b/routers/private/internal.go @@ -6,6 +6,7 @@ package private import ( + "code.gitea.io/gitea/routers/repo" "strings" "code.gitea.io/gitea/modules/log" @@ -44,7 +45,10 @@ func RegisterRoutes(m *macaron.Macaron) { m.Post("/manager/flush-queues", bind(private.FlushOptions{}), FlushQueues) m.Post("/tool/update_all_repo_commit_cnt", UpdateAllRepoCommitCnt) m.Post("/tool/repo_stat/:date", RepoStatisticManually) + + m.Get("/tool/org_stat", OrgStatisticManually) m.Post("/tool/update_repo_visit/:date", UpdateRepoVisit) + m.Post("/task/history_handle/duration", repo.HandleTaskWithNoDuration) }, CheckInternalToken) } diff --git a/routers/private/tool.go b/routers/private/tool.go index d01c5b2ab..122a41afe 100755 --- a/routers/private/tool.go +++ b/routers/private/tool.go @@ -45,6 +45,10 @@ func RepoStatisticManually(ctx *macaron.Context) { repo.TimingCountDataByDate(date) } +func OrgStatisticManually() { + models.UpdateOrgStatistics() +} + func UpdateRepoVisit(ctx *macaron.Context) { date := ctx.Params("date") log.Info("date(%s)", date) diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 0905efd54..0d007a27d 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -2,11 +2,9 @@ package repo import ( "bufio" - "code.gitea.io/gitea/modules/timeutil" "encoding/json" "errors" "fmt" - "github.com/unknwon/i18n" "io" "net/http" "os" @@ -16,6 +14,9 @@ import ( "strings" "time" + "code.gitea.io/gitea/modules/timeutil" + "github.com/unknwon/i18n" + "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/auth" "code.gitea.io/gitea/modules/base" @@ -37,6 +38,9 @@ const ( tplCloudBrainBenchmarkIndex base.TplName = "repo/cloudbrain/benchmark/index" tplCloudBrainBenchmarkNew base.TplName = "repo/cloudbrain/benchmark/new" tplCloudBrainBenchmarkShow base.TplName = "repo/cloudbrain/benchmark/show" + + tplCloudBrainTrainJobNew base.TplName = "repo/cloudbrain/trainjob/new" + tplCloudBrainTrainJobShow base.TplName = "repo/cloudbrain/trainjob/show" ) var ( @@ -45,6 +49,7 @@ var ( benchmarkTypes *models.BenchmarkTypes benchmarkGpuInfos *models.GpuInfos benchmarkResourceSpecs *models.ResourceSpecs + trainGpuInfos *models.GpuInfos ) const BENCHMARK_TYPE_CODE = "repo.cloudbrain.benchmark.types" @@ -143,6 +148,11 @@ func cloudBrainNewDataPrepare(ctx *context.Context) error { } ctx.Data["gpu_types"] = gpuInfos.GpuInfo + if trainGpuInfos == nil { + json.Unmarshal([]byte(setting.TrainGpuTypes), &trainGpuInfos) + } + ctx.Data["train_gpu_types"] = trainGpuInfos.GpuInfo + if benchmarkGpuInfos == nil { json.Unmarshal([]byte(setting.BenchmarkGpuTypes), &benchmarkGpuInfos) } @@ -157,6 +167,14 @@ func cloudBrainNewDataPrepare(ctx *context.Context) error { json.Unmarshal([]byte(setting.ResourceSpecs), &cloudbrain.ResourceSpecs) } ctx.Data["resource_specs"] = cloudbrain.ResourceSpecs.ResourceSpec + + if cloudbrain.TrainResourceSpecs == nil { + json.Unmarshal([]byte(setting.TrainResourceSpecs), &cloudbrain.TrainResourceSpecs) + } + ctx.Data["train_resource_specs"] = cloudbrain.TrainResourceSpecs.ResourceSpec + ctx.Data["params"] = "" + ctx.Data["branchName"] = ctx.Repo.BranchName + ctx.Data["snn4imagenet_path"] = cloudbrain.Snn4imagenetMountPath ctx.Data["is_snn4imagenet_enabled"] = setting.IsSnn4imagenetEnabled @@ -184,38 +202,52 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { image := form.Image uuid := form.Attachment jobType := form.JobType - command := cloudbrain.Command gpuQueue := form.GpuType codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath resourceSpecId := form.ResourceSpecId + branchName := form.BranchName repo := ctx.Repo.Repository - tasks, err := models.GetCloudbrainsByDisplayJobName(repo.ID, string(models.JobTypeDebug), displayJobName) + tpl := tplCloudBrainNew + command := cloudbrain.Command + if jobType == string(models.JobTypeTrain) { + tpl = tplCloudBrainTrainJobNew + commandTrain, err := getTrainJobCommand(form) + if err != nil { + log.Error("getTrainJobCommand failed: %v", err) + ctx.RenderWithErr(err.Error(), tpl, &form) + return + } + + command = commandTrain + } + + tasks, err := models.GetCloudbrainsByDisplayJobName(repo.ID, jobType, displayJobName) if err == nil { if len(tasks) != 0 { log.Error("the job name did already exist", ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) - ctx.RenderWithErr("the job name did already exist", tplCloudBrainNew, &form) + ctx.RenderWithErr("the job name did already exist", tpl, &form) return } } else { if !models.IsErrJobNotExist(err) { log.Error("system error, %v", err, ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) - ctx.RenderWithErr("system error", tplCloudBrainNew, &form) + ctx.RenderWithErr("system error", tpl, &form) return } } if !jobNamePattern.MatchString(displayJobName) { - ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tplCloudBrainNew, &form) + ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tpl, &form) return } - if jobType != string(models.JobTypeBenchmark) && jobType != string(models.JobTypeDebug) && jobType != string(models.JobTypeSnn4imagenet) && jobType != string(models.JobTypeBrainScore) { + if jobType != string(models.JobTypeBenchmark) && jobType != string(models.JobTypeDebug) && jobType != string(models.JobTypeSnn4imagenet) && jobType != string(models.JobTypeBrainScore) && jobType != string(models.JobTypeTrain) { log.Error("jobtype error:", jobType, ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) - ctx.RenderWithErr("jobtype error", tplCloudBrainNew, &form) + ctx.RenderWithErr("jobtype error", tpl, &form) return } @@ -223,18 +255,21 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { if err != nil { log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) - ctx.RenderWithErr("system error", tplCloudBrainNew, &form) + ctx.RenderWithErr("system error", tpl, &form) return } else { if count >= 1 { log.Error("the user already has running or waiting task", ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) - ctx.RenderWithErr("you have already a running or waiting task, can not create more", tplCloudBrainNew, &form) + ctx.RenderWithErr("you have already a running or waiting task, can not create more", tpl, &form) return } } - downloadCode(repo, codePath) + if branchName == "" { + branchName = cloudbrain.DefaultBranchName + } + downloadCode(repo, codePath, branchName) uploadCodeToMinio(codePath+"/", jobName, cloudbrain.CodeMountPath+"/") modelPath := setting.JobPath + jobName + cloudbrain.ModelMountPath + "/" @@ -268,15 +303,19 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { err = cloudbrain.GenerateTask(ctx, displayJobName, jobName, image, command, uuid, storage.GetMinioPath(jobName, cloudbrain.CodeMountPath+"/"), storage.GetMinioPath(jobName, cloudbrain.ModelMountPath+"/"), storage.GetMinioPath(jobName, cloudbrain.BenchMarkMountPath+"/"), storage.GetMinioPath(jobName, cloudbrain.Snn4imagenetMountPath+"/"), - storage.GetMinioPath(jobName, cloudbrain.BrainScoreMountPath+"/"), jobType, gpuQueue, form.Description, + storage.GetMinioPath(jobName, cloudbrain.BrainScoreMountPath+"/"), jobType, gpuQueue, form.Description, branchName, form.BootFile, form.Params, 0, 0, resourceSpecId) if err != nil { cloudBrainNewDataPrepare(ctx) - ctx.RenderWithErr(err.Error(), tplCloudBrainNew, &form) + ctx.RenderWithErr(err.Error(), tpl, &form) return } - ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/debugjob?debugListType=all") + if jobType == string(models.JobTypeTrain) { + ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/train-job?listType=all") + } else { + ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/debugjob?debugListType=all") + } } func CloudBrainRestart(ctx *context.Context) { @@ -307,6 +346,24 @@ func CloudBrainRestart(ctx *context.Context) { break } + var hasSameResource bool + if gpuInfos == nil { + json.Unmarshal([]byte(setting.GpuTypes), &gpuInfos) + } + for _, resourceType := range gpuInfos.GpuInfo { + if resourceType.Queue == task.GpuQueue { + hasSameResource = true + continue + } + } + + if !hasSameResource { + log.Error("has no same resource, can not restart", ctx.Data["MsgID"]) + resultCode = "-1" + errorMsg = "the job's version is too old and can not be restarted" + break + } + count, err := models.GetCloudbrainCountByUserID(ctx.User.ID, string(models.JobTypeDebug)) if err != nil { log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"]) @@ -342,32 +399,78 @@ func CloudBrainRestart(ctx *context.Context) { } func CloudBrainBenchMarkShow(ctx *context.Context) { - cloudBrainShow(ctx, tplCloudBrainBenchmarkShow) + cloudBrainShow(ctx, tplCloudBrainBenchmarkShow, models.JobTypeBenchmark) } func CloudBrainShow(ctx *context.Context) { - cloudBrainShow(ctx, tplCloudBrainShow) + cloudBrainShow(ctx, tplCloudBrainShow, models.JobTypeDebug) +} + +func CloudBrainTrainJobShow(ctx *context.Context) { + cloudBrainShow(ctx, tplCloudBrainTrainJobShow, models.JobTypeTrain) } -func cloudBrainShow(ctx *context.Context, tpName base.TplName) { +func cloudBrainShow(ctx *context.Context, tpName base.TplName, jobType models.JobType) { ctx.Data["PageIsCloudBrain"] = true - var ID = ctx.Params(":id") debugListType := ctx.Query("debugListType") - task, err := models.GetCloudbrainByID(ID) + + var task *models.Cloudbrain + var err error + if jobType == models.JobTypeTrain { + task, err = models.GetCloudbrainByJobID(ctx.Params(":jobid")) + } else { + task, err = models.GetCloudbrainByIDWithDeleted(ctx.Params(":id")) + } + if err != nil { log.Info("error:" + err.Error()) ctx.Data["error"] = err.Error() + return } + result, err := cloudbrain.GetJob(task.JobID) if err != nil { log.Info("error:" + err.Error()) ctx.Data["error"] = err.Error() + return } + + if cloudbrain.ResourceSpecs == nil { + json.Unmarshal([]byte(setting.ResourceSpecs), &cloudbrain.ResourceSpecs) + } + for _, tmp := range cloudbrain.ResourceSpecs.ResourceSpec { + if tmp.Id == task.ResourceSpecId { + ctx.Data["GpuNum"] = tmp.GpuNum + ctx.Data["CpuNum"] = tmp.CpuNum + ctx.Data["MemMiB"] = tmp.MemMiB + ctx.Data["ShareMemMiB"] = tmp.ShareMemMiB + } + } + if result != nil { jobRes, _ := models.ConvertToJobResultPayload(result.Payload) jobRes.Resource.Memory = strings.ReplaceAll(jobRes.Resource.Memory, "Mi", "MB") spec := "GPU数：" + strconv.Itoa(jobRes.Resource.NvidiaComGpu) + "，CPU数：" + strconv.Itoa(jobRes.Resource.CPU) + "，内存(MB):" + jobRes.Resource.Memory ctx.Data["resource_spec"] = spec + if task.JobType == string(models.JobTypeTrain) { + if trainGpuInfos == nil { + json.Unmarshal([]byte(setting.TrainGpuTypes), &trainGpuInfos) + } + for _, resourceType := range trainGpuInfos.GpuInfo { + if resourceType.Queue == jobRes.Config.GpuType { + ctx.Data["resource_type"] = resourceType.Value + } + } + } else { + if gpuInfos == nil { + json.Unmarshal([]byte(setting.GpuTypes), &gpuInfos) + } + for _, resourceType := range gpuInfos.GpuInfo { + if resourceType.Queue == jobRes.Config.GpuType { + ctx.Data["resource_type"] = resourceType.Value + } + } + } taskRoles := jobRes.TaskRoles if jobRes.JobStatus.State != string(models.JobFailed) { @@ -376,12 +479,16 @@ func cloudBrainShow(ctx *context.Context, tpName base.TplName) { task.Status = taskRes.TaskStatuses[0].State task.ContainerID = taskRes.TaskStatuses[0].ContainerID task.ContainerIp = taskRes.TaskStatuses[0].ContainerIP - if task.StartTime == 0 && !taskRes.TaskStatuses[0].StartAt.IsZero() { - task.StartTime = timeutil.TimeStamp(taskRes.TaskStatuses[0].StartAt.Unix()) - } - err = models.UpdateJob(task) - if err != nil { - ctx.Data["error"] = err.Error() + models.ParseAndSetDurationFromCloudBrainOne(jobRes, task) + + if task.DeletedAt.IsZero() { //normal record + err = models.UpdateJob(task) + if err != nil { + ctx.Data["error"] = err.Error() + return + } + } else { //deleted record + } } else { task.Status = jobRes.JobStatus.State @@ -398,7 +505,9 @@ func cloudBrainShow(ctx *context.Context, tpName base.TplName) { ctx.Data["result"] = jobRes } else { log.Info("error:" + err.Error()) + return } + user, err := models.GetUserByID(task.UserID) if err == nil { task.User = user @@ -419,15 +528,47 @@ func cloudBrainShow(ctx *context.Context, tpName base.TplName) { } } if task.TrainJobDuration == "" { - var duration int64 - if task.Status == string(models.JobRunning) { - duration = time.Now().Unix() - int64(task.CreatedUnix) - } else { - duration = int64(task.UpdatedUnix) - int64(task.CreatedUnix) + if task.Duration == 0 { + var duration int64 + if task.Status == string(models.JobRunning) { + duration = time.Now().Unix() - int64(task.CreatedUnix) + } else { + duration = int64(task.UpdatedUnix) - int64(task.CreatedUnix) + } + task.Duration = duration } - task.TrainJobDuration = models.ConvertDurationToStr(duration) + task.TrainJobDuration = models.ConvertDurationToStr(task.Duration) } ctx.Data["duration"] = task.TrainJobDuration + + if len(task.Parameters) > 0 { + var parameters models.Parameters + + err := json.Unmarshal([]byte(task.Parameters), ¶meters) + if err != nil { + log.Error("Failed to Unmarshal Parameters: %s (%v)", task.Parameters, err) + task.Parameters = "" + } else { + if len(parameters.Parameter) > 0 { + paramTemp := "" + for _, Parameter := range parameters.Parameter { + param := Parameter.Label + " = " + Parameter.Value + "; " + paramTemp = paramTemp + param + } + task.Parameters = paramTemp[:len(paramTemp)-2] + } else { + task.Parameters = "" + } + } + + } + attachment, err := models.GetAttachmentByUUID(task.Uuid) + if err == nil { + ctx.Data["datasetname"] = attachment.Name + } else { + ctx.Data["datasetname"] = "" + } + ctx.Data["task"] = task ctx.Data["jobName"] = task.JobName ctx.Data["displayJobName"] = task.DisplayJobName @@ -435,6 +576,10 @@ func cloudBrainShow(ctx *context.Context, tpName base.TplName) { version_list_task = append(version_list_task, task) ctx.Data["version_list_task"] = version_list_task ctx.Data["debugListType"] = debugListType + ctx.Data["code_path"] = cloudbrain.CodeMountPath + ctx.Data["dataset_path"] = cloudbrain.DataSetMountPath + ctx.Data["model_path"] = cloudbrain.ModelMountPath + ctx.Data["canDownload"] = cloudbrain.CanModifyJob(ctx, task) ctx.HTML(200, tpName) } @@ -506,11 +651,12 @@ func CloudBrainStop(ctx *context.Context) { break } - ctx.JSON(200, map[string]string{ + ctx.JSON(200, map[string]interface{}{ "result_code": resultCode, "error_msg": errorMsg, "status": status, "id": ID, + "StatusOK": 0, }) } @@ -762,8 +908,8 @@ func GetRate(ctx *context.Context) { } } -func downloadCode(repo *models.Repository, codePath string) error { - if err := git.Clone(repo.RepoPath(), codePath, git.CloneRepoOptions{}); err != nil { +func downloadCode(repo *models.Repository, codePath, branchName string) error { + if err := git.Clone(repo.RepoPath(), codePath, git.CloneRepoOptions{Branch: branchName}); err != nil { log.Error("Failed to clone repository: %s (%v)", repo.FullName(), err) return err } @@ -961,14 +1107,7 @@ func SyncCloudbrainStatus() { taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{})) task.Status = taskRes.TaskStatuses[0].State if task.Status != string(models.JobWaiting) { - task.Duration = time.Now().Unix() - taskRes.TaskStatuses[0].StartAt.Unix() - if task.StartTime == 0 && !taskRes.TaskStatuses[0].StartAt.IsZero() { - task.StartTime = timeutil.TimeStamp(taskRes.TaskStatuses[0].StartAt.Unix()) - } - if task.EndTime == 0 && models.IsCloudBrainOneDebugJobTerminal(task.Status) { - task.EndTime = timeutil.TimeStampNow() - } - task.ComputeAndSetDuration() + models.ParseAndSetDurationFromCloudBrainOne(jobRes, task) err = models.UpdateJob(task) if err != nil { log.Error("UpdateJob(%s) failed:%v", task.JobName, err) @@ -1011,8 +1150,8 @@ func SyncCloudbrainStatus() { if result != nil { task.Status = result.Status - if task.StartTime == 0 && result.Lease.CreateTime > 0 { - task.StartTime = timeutil.TimeStamp(result.Lease.CreateTime / 1000) + if task.StartTime == 0 && result.Lease.UpdateTime > 0 { + task.StartTime = timeutil.TimeStamp(result.Lease.UpdateTime / 1000) } if task.EndTime == 0 && models.IsModelArtsDebugJobTerminal(task.Status) { task.EndTime = timeutil.TimeStampNow() @@ -1024,7 +1163,7 @@ func SyncCloudbrainStatus() { continue } } - } else if task.JobType == string(models.JobTypeTrain) { + } else if task.JobType == string(models.JobTypeTrain) || task.JobType == string(models.JobTypeInference) { result, err := modelarts.GetTrainJob(task.JobID, strconv.FormatInt(task.VersionID, 10)) if err != nil { log.Error("GetTrainJob(%s) failed:%v", task.JobName, err) @@ -1062,6 +1201,158 @@ func SyncCloudbrainStatus() { return } +func HandleTaskWithNoDuration(ctx *context.Context) { + log.Info("HandleTaskWithNoDuration start") + count := 0 + start := time.Now().Unix() + for { + cloudBrains, err := models.GetStoppedJobWithNoDurationJob() + if err != nil { + log.Error("HandleTaskWithNoTrainJobDuration failed:", err.Error()) + break + } + if len(cloudBrains) == 0 { + log.Info("HandleTaskWithNoTrainJobDuration:no task need handle") + break + } + handleNoDurationTask(cloudBrains) + count += len(cloudBrains) + if len(cloudBrains) < 100 { + log.Info("HandleTaskWithNoTrainJobDuration:task less than 100") + break + } + if time.Now().Unix()-start > 600 { + log.Info("HandleTaskWithNoDuration : time out") + ctx.JSON(200, fmt.Sprintf("task stop for time out,count=%d", count)) + return + } + } + log.Info("HandleTaskWithNoTrainJobDuration:count=%d", count) + ctx.JSON(200, fmt.Sprintf("success,count=%d", count)) +} + +func handleNoDurationTask(cloudBrains []*models.Cloudbrain) { + for _, task := range cloudBrains { + time.Sleep(time.Millisecond * 100) + log.Info("Handle job ,%+v", task) + if task.Type == models.TypeCloudBrainOne { + result, err := cloudbrain.GetJob(task.JobID) + if err != nil { + log.Error("GetJob(%s) failed:%v", task.JobName, err) + updateDefaultDuration(task) + continue + } + + if result != nil { + if result.Msg != "success" { + updateDefaultDuration(task) + continue + } + jobRes, err := models.ConvertToJobResultPayload(result.Payload) + if err != nil || len(jobRes.TaskRoles) == 0 { + updateDefaultDuration(task) + continue + } + taskRoles := jobRes.TaskRoles + taskRes, err := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{})) + if err != nil || len(taskRes.TaskStatuses) == 0 { + updateDefaultDuration(task) + continue + } + task.Status = taskRes.TaskStatuses[0].State + log.Info("task startTime = %v endTime= %v ,jobId=%d", jobRes.JobStatus.StartTime, jobRes.JobStatus.EndTime, task.ID) + if jobRes.JobStatus.CreatedTime > 0 { + task.StartTime = timeutil.TimeStamp(jobRes.JobStatus.CreatedTime / 1000) + if jobRes.JobStatus.CompletedTime > 0 { + task.EndTime = timeutil.TimeStamp(jobRes.JobStatus.CompletedTime / 1000) + } else { + task.EndTime = task.UpdatedUnix + } + } else { + task.StartTime = 0 + task.EndTime = 0 + } + + if task.EndTime < task.StartTime { + log.Info("endTime[%v] is less than starTime[%v],jobId=%d", task.EndTime, task.StartTime, task.ID) + st := task.StartTime + task.StartTime = task.EndTime + task.EndTime = st + } + task.Duration = task.EndTime.AsTime().Unix() - task.StartTime.AsTime().Unix() + task.TrainJobDuration = models.ConvertDurationToStr(task.Duration) + err = models.UpdateJob(task) + if err != nil { + log.Error("UpdateJob(%s) failed:%v", task.JobName, err) + } + } + } else if task.Type == models.TypeCloudBrainTwo { + if task.JobType == string(models.JobTypeDebug) { + //result, err := modelarts.GetJob(task.JobID) + result, err := modelarts.GetNotebook2(task.JobID) + if err != nil { + log.Error("GetJob(%s) failed:%v", task.JobName, err) + updateDefaultDuration(task) + continue + } + + if result != nil { + task.Status = result.Status + startTime := result.Lease.CreateTime + duration := result.Lease.Duration / 1000 + if startTime > 0 { + task.StartTime = timeutil.TimeStamp(startTime / 1000) + task.EndTime = task.StartTime.Add(duration) + } + task.ComputeAndSetDuration() + err = models.UpdateJob(task) + if err != nil { + log.Error("UpdateJob(%s) failed:%v", task.JobName, err) + continue + } + } + } else if task.JobType == string(models.JobTypeTrain) || task.JobType == string(models.JobTypeInference) { + result, err := modelarts.GetTrainJob(task.JobID, strconv.FormatInt(task.VersionID, 10)) + if err != nil { + log.Error("GetTrainJob(%s) failed:%v", task.JobName, err) + updateDefaultDuration(task) + continue + } + + if result != nil { + startTime := result.StartTime / 1000 + if startTime > 0 { + task.StartTime = timeutil.TimeStamp(startTime) + task.EndTime = task.StartTime.Add(result.Duration / 1000) + } + task.ComputeAndSetDuration() + err = models.UpdateJob(task) + if err != nil { + log.Error("UpdateJob(%s) failed:%v", task.JobName, err) + continue + } + } + } else { + log.Error("task.JobType(%s) is error:%s", task.JobName, task.JobType) + } + + } else { + log.Error("task.Type(%s) is error:%d", task.JobName, task.Type) + } + } +} + +func updateDefaultDuration(task *models.Cloudbrain) { + log.Info("updateDefaultDuration: taskId=%d", task.ID) + task.StartTime = task.CreatedUnix + task.EndTime = task.UpdatedUnix + task.ComputeAndSetDuration() + err := models.UpdateJob(task) + if err != nil { + log.Error("UpdateJob(%s) failed:%v", task.JobName, err) + } +} + func CloudBrainBenchmarkIndex(ctx *context.Context) { MustEnableCloudbrain(ctx) repo := ctx.Repo.Repository @@ -1090,13 +1381,16 @@ func CloudBrainBenchmarkIndex(ctx *context.Context) { ciTasks[i].CanDel = cloudbrain.CanDeleteJob(ctx, &task.Cloudbrain) ciTasks[i].Cloudbrain.ComputeResource = task.ComputeResource if ciTasks[i].TrainJobDuration == "" { - var duration int64 - if task.Status == string(models.JobRunning) { - duration = time.Now().Unix() - int64(task.Cloudbrain.CreatedUnix) - } else { - duration = int64(task.Cloudbrain.UpdatedUnix) - int64(task.Cloudbrain.CreatedUnix) + if ciTasks[i].Duration == 0 { + var duration int64 + if task.Status == string(models.JobRunning) { + duration = time.Now().Unix() - int64(task.Cloudbrain.CreatedUnix) + } else { + duration = int64(task.Cloudbrain.UpdatedUnix) - int64(task.Cloudbrain.CreatedUnix) + } + ciTasks[i].Duration = duration } - ciTasks[i].TrainJobDuration = models.ConvertDurationToStr(duration) + ciTasks[i].TrainJobDuration = models.ConvertDurationToStr(ciTasks[i].Duration) } ciTasks[i].BenchmarkTypeName = "" @@ -1315,7 +1609,7 @@ func CloudBrainBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainF } os.RemoveAll(codePath) - if err := downloadCode(repo, codePath); err != nil { + if err := downloadCode(repo, codePath, cloudbrain.DefaultBranchName); err != nil { log.Error("downloadCode failed, %v", err, ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr("system error", tplCloudBrainBenchmarkNew, &form) @@ -1380,7 +1674,7 @@ func CloudBrainBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainF err = cloudbrain.GenerateTask(ctx, displayJobName, jobName, image, command, childInfo.Attachment, storage.GetMinioPath(jobName, cloudbrain.CodeMountPath+"/"), storage.GetMinioPath(jobName, cloudbrain.ModelMountPath+"/"), storage.GetMinioPath(jobName, cloudbrain.BenchMarkMountPath+"/"), storage.GetMinioPath(jobName, cloudbrain.Snn4imagenetMountPath+"/"), - storage.GetMinioPath(jobName, cloudbrain.BrainScoreMountPath+"/"), string(models.JobTypeBenchmark), gpuQueue, form.Description, + storage.GetMinioPath(jobName, cloudbrain.BrainScoreMountPath+"/"), string(models.JobTypeBenchmark), gpuQueue, form.Description, cloudbrain.DefaultBranchName, "", "", benchmarkTypeID, benchmarkChildTypeID, resourceSpecId) if err != nil { cloudBrainNewDataPrepare(ctx) @@ -1406,10 +1700,66 @@ func BenchmarkDel(ctx *context.Context) { } } +func CloudBrainTrainJobNew(ctx *context.Context) { + err := cloudBrainNewDataPrepare(ctx) + if err != nil { + ctx.ServerError("get new train-job info failed", err) + return + } + ctx.HTML(http.StatusOK, tplCloudBrainTrainJobNew) +} + +func getTrainJobCommand(form auth.CreateCloudBrainForm) (string, error) { + var command string + bootFile := form.BootFile + params := form.Params + + if !strings.HasSuffix(bootFile, ".py") { + log.Error("bootFile(%s) format error", bootFile) + return command, errors.New("bootFile format error") + } + + var parameters models.Parameters + var param string + if len(params) != 0 { + err := json.Unmarshal([]byte(params), ¶meters) + if err != nil { + log.Error("Failed to Unmarshal params: %s (%v)", params, err) + return command, err + } + + for _, parameter := range parameters.Parameter { + param += " --" + parameter.Label + "=" + parameter.Value + } + } + + command += "python /code/" + bootFile + param + " > " + cloudbrain.ModelMountPath + "/" + form.DisplayJobName + "-" + cloudbrain.LogFile + + return command, nil +} + +func CloudBrainTrainJobDel(ctx *context.Context) { + var listType = ctx.Query("listType") + if err := deleteCloudbrainJob(ctx); err != nil { + log.Error("deleteCloudbrainJob failed: %v", err, ctx.Data["msgID"]) + ctx.ServerError(err.Error(), err) + return + } + + var isAdminPage = ctx.Query("isadminpage") + if ctx.IsUserSiteAdmin() && isAdminPage == "true" { + ctx.Redirect(setting.AppSubURL + "/admin" + "/cloudbrains") + } else { + ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/train-job?listType=" + listType) + } +} + func GetBenchmarkTypes(ctx *context.Context) *models.BenchmarkTypes { var lang = ctx.Locale.Language() if benchmarkTypesMap[lang] == nil { var val = i18n.Tr(lang, BENCHMARK_TYPE_CODE) + //use config + val = setting.BenchmarkTypes var tempType *models.BenchmarkTypes if err := json.Unmarshal([]byte(val), &tempType); err != nil { log.Error("json.Unmarshal BenchmarkTypes(%s) failed:%v", val, err, ctx.Data["MsgID"]) diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 7d4c203bc..933571a0b 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -2,9 +2,9 @@ package repo import ( "archive/zip" - "code.gitea.io/gitea/modules/timeutil" "encoding/json" "errors" + "fmt" "io" "io/ioutil" "net/http" @@ -15,6 +15,9 @@ import ( "time" "unicode/utf8" + "code.gitea.io/gitea/modules/notification" + "code.gitea.io/gitea/modules/timeutil" + "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/auth" "code.gitea.io/gitea/modules/base" @@ -47,20 +50,26 @@ const ( ) func DebugJobIndex(ctx *context.Context) { - debugListType := ctx.Query("debugListType") - ctx.Data["ListType"] = debugListType + listType := ctx.Query("debugListType") + ctx.Data["ListType"] = listType MustEnableCloudbrain(ctx) repo := ctx.Repo.Repository page := ctx.QueryInt("page") if page <= 0 { page = 1 } - debugType := modelarts.DebugType + typeCloudBrain := models.TypeCloudBrainAll jobTypeNot := false - if debugListType == models.GPUResource { - debugType = models.TypeCloudBrainOne - } else if debugListType == models.NPUResource { - debugType = models.TypeCloudBrainTwo + if listType == models.GPUResource { + typeCloudBrain = models.TypeCloudBrainOne + } else if listType == models.NPUResource { + typeCloudBrain = models.TypeCloudBrainTwo + } else if listType == models.AllResource { + typeCloudBrain = models.TypeCloudBrainAll + } else { + log.Error("listType(%s) error", listType) + ctx.ServerError("listType error", errors.New("listType error")) + return } var jobTypes []string @@ -71,7 +80,7 @@ func DebugJobIndex(ctx *context.Context) { PageSize: setting.UI.IssuePagingNum, }, RepoID: repo.ID, - Type: debugType, + Type: typeCloudBrain, JobTypeNot: jobTypeNot, JobTypes: jobTypes, }) @@ -93,7 +102,7 @@ func DebugJobIndex(ctx *context.Context) { ctx.Data["Tasks"] = ciTasks ctx.Data["CanCreate"] = cloudbrain.CanCreateOrDebugJob(ctx) ctx.Data["RepoIsEmpty"] = repo.IsEmpty - ctx.Data["debugListType"] = debugListType + ctx.Data["debugListType"] = listType ctx.HTML(200, tplDebugJobIndex) } @@ -240,7 +249,7 @@ func NotebookShow(ctx *context.Context) { debugListType := ctx.Query("debugListType") var ID = ctx.Params(":id") - task, err := models.GetCloudbrainByID(ID) + task, err := models.GetCloudbrainByIDWithDeleted(ID) if err != nil { ctx.Data["error"] = err.Error() ctx.RenderWithErr(err.Error(), tplModelArtsNotebookShow, nil) @@ -255,16 +264,19 @@ func NotebookShow(ctx *context.Context) { } if result != nil { - task.Status = result.Status - err = models.UpdateJob(task) - if err != nil { - ctx.Data["error"] = err.Error() - ctx.RenderWithErr(err.Error(), tplModelArtsNotebookShow, nil) - return - } + if task.DeletedAt.IsZero() { //normal record + if task.Status != result.Status { + task.Status = result.Status + err = models.UpdateJob(task) + if err != nil { + ctx.Data["error"] = err.Error() + ctx.RenderWithErr(err.Error(), tplModelArtsNotebookShow, nil) + return + } + } + } else { //deleted record - result.CreateTime = time.Unix(int64(result.CreateAt/1000), 0).Format("2006-01-02 15:04:05") - result.LatestUpdateTime = time.Unix(int64(result.UpdateAt/1000), 0).Format("2006-01-02 15:04:05") + } } datasetDownloadLink := "-" @@ -272,16 +284,44 @@ func NotebookShow(ctx *context.Context) { if task.Uuid != "" && task.UserID == ctx.User.ID { attachment, err := models.GetAttachmentByUUID(task.Uuid) if err == nil { + task.DatasetName = attachment.Name datasetDownloadLink = attachment.S3DownloadURL() } } } - + user, err := models.GetUserByID(task.UserID) + if err == nil { + task.User = user + } + if modelarts.FlavorInfos == nil { + json.Unmarshal([]byte(setting.FlavorInfos), &modelarts.FlavorInfos) + } + if modelarts.FlavorInfos != nil { + ctx.Data["resource_spec"] = modelarts.FlavorInfos.FlavorInfo[0].Desc + for _, f := range modelarts.FlavorInfos.FlavorInfo { + if fmt.Sprint(f.Value) == task.FlavorCode { + ctx.Data["resource_spec"] = f.Desc + break + } + } + } + if task.TrainJobDuration == "" { + if task.Duration == 0 { + var duration int64 + if task.Status == string(models.JobRunning) { + duration = time.Now().Unix() - int64(task.CreatedUnix) + } else { + duration = int64(task.UpdatedUnix) - int64(task.CreatedUnix) + } + task.Duration = duration + } + task.TrainJobDuration = models.ConvertDurationToStr(task.Duration) + } + ctx.Data["duration"] = task.TrainJobDuration ctx.Data["datasetDownloadLink"] = datasetDownloadLink ctx.Data["task"] = task ctx.Data["ID"] = ID ctx.Data["jobName"] = task.JobName - ctx.Data["result"] = result ctx.Data["debugListType"] = debugListType ctx.HTML(200, tplModelArtsNotebookShow) } @@ -410,20 +450,46 @@ func NotebookManage(ctx *context.Context) { break } - task.Status = res.Status - if task.EndTime == 0 && models.IsModelArtsDebugJobTerminal(task.Status) { - task.EndTime = timeutil.TimeStampNow() - } - task.ComputeAndSetDuration() - err = models.UpdateJob(task) - if err != nil { - log.Error("UpdateJob(%s) failed:%v", task.JobName, err.Error(), ctx.Data["MsgID"]) - resultCode = "-1" - errorMsg = "system error" - break - } + status = res.Status + if action == models.ActionStart { + newTask := &models.Cloudbrain{ + Status: status, + UserID: task.UserID, + RepoID: task.RepoID, + JobID: task.JobID, + JobName: task.JobName, + DisplayJobName: task.DisplayJobName, + JobType: task.JobType, + Type: task.Type, + Uuid: task.Uuid, + Image: task.Image, + ComputeResource: task.ComputeResource, + Description: task.Description, + } - status = task.Status + err = models.RestartCloudbrain(task, newTask) + if err != nil { + log.Error("RestartCloudbrain(%s) failed:%v", task.JobName, err.Error(), ctx.Data["MsgID"]) + resultCode = "-1" + errorMsg = "system error" + break + } + ID = strconv.FormatInt(newTask.ID, 10) + notification.NotifyOtherTask(ctx.User, ctx.Repo.Repository, ID, task.DisplayJobName, models.ActionCreateDebugNPUTask) + } else { + task.Status = res.Status + if task.EndTime == 0 && models.IsModelArtsDebugJobTerminal(task.Status) { + task.EndTime = timeutil.TimeStampNow() + } + task.ComputeAndSetDuration() + err = models.UpdateJob(task) + if err != nil { + log.Error("UpdateJob(%s) failed:%v", task.JobName, err.Error(), ctx.Data["MsgID"]) + resultCode = "-1" + errorMsg = "system error" + break + } + } break } @@ -480,6 +546,26 @@ func TrainJobIndex(ctx *context.Context) { page = 1 } + listType := ctx.Query("listType") + if len(listType) == 0 { + listType = models.AllResource + } + ctx.Data["ListType"] = listType + + typeCloudBrain := models.TypeCloudBrainAll + if listType == models.GPUResource { + typeCloudBrain = models.TypeCloudBrainOne + } else if listType == models.NPUResource { + typeCloudBrain = models.TypeCloudBrainTwo + } else if listType == models.AllResource { + typeCloudBrain = models.TypeCloudBrainAll + } + //else { + // log.Error("listType(%s) error", listType) + // ctx.ServerError("listType error", errors.New("listType error")) + // return + //} + var jobTypes []string jobTypes = append(jobTypes, string(models.JobTypeTrain)) tasks, count, err := models.Cloudbrains(&models.CloudbrainsOptions{ @@ -488,7 +574,7 @@ func TrainJobIndex(ctx *context.Context) { PageSize: setting.UI.IssuePagingNum, }, RepoID: repo.ID, - Type: models.TypeCloudBrainTwo, + Type: typeCloudBrain, JobTypeNot: false, JobTypes: jobTypes, IsLatestVersion: modelarts.IsLatestVersion, @@ -501,11 +587,16 @@ func TrainJobIndex(ctx *context.Context) { for i, task := range tasks { tasks[i].CanDel = cloudbrain.CanDeleteJob(ctx, &task.Cloudbrain) tasks[i].CanModify = cloudbrain.CanModifyJob(ctx, &task.Cloudbrain) - tasks[i].ComputeResource = models.NPUResource + if task.Cloudbrain.Type == models.TypeCloudBrainOne { + tasks[i].ComputeResource = models.GPUResource + } else if task.Cloudbrain.Type == models.TypeCloudBrainTwo { + tasks[i].ComputeResource = models.NPUResource + } } pager := context.NewPagination(int(count), setting.UI.IssuePagingNum, page, 5) pager.SetDefaultParams(ctx) + pager.AddParam(ctx, "listType", "ListType") ctx.Data["Page"] = pager ctx.Data["PageIsCloudBrain"] = true @@ -1495,7 +1586,7 @@ func TrainJobShow(ctx *context.Context) { ctx.Data["displayJobName"] = VersionListTasks[0].DisplayJobName ctx.Data["version_list_task"] = VersionListTasks ctx.Data["version_list_count"] = VersionListCount - ctx.Data["canDownload"] = cloudbrain.CanDeleteJob(ctx, &VersionListTasks[0].Cloudbrain) + ctx.Data["canDownload"] = cloudbrain.CanModifyJob(ctx, &VersionListTasks[0].Cloudbrain) ctx.HTML(http.StatusOK, tplModelArtsTrainJobShow) } @@ -1555,6 +1646,7 @@ func trainJobGetLog(jobID string) (*models.GetTrainJobLogFileNamesResult, *model func TrainJobDel(ctx *context.Context) { var jobID = ctx.Params(":jobid") + var listType = ctx.Query("listType") repo := ctx.Repo.Repository var jobTypes []string @@ -1596,12 +1688,13 @@ func TrainJobDel(ctx *context.Context) { if ctx.IsUserSiteAdmin() && isAdminPage == "true" { ctx.Redirect(setting.AppSubURL + "/admin" + "/cloudbrains") } else { - ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/train-job") + ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/train-job?listType=" + listType) } } func TrainJobStop(ctx *context.Context) { var jobID = ctx.Params(":jobid") + var listType = ctx.Query("listType") task := ctx.Cloudbrain _, err := modelarts.StopTrainJob(jobID, strconv.FormatInt(task.VersionID, 10)) @@ -1611,7 +1704,7 @@ func TrainJobStop(ctx *context.Context) { return } - ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/train-job") + ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/train-job?listType=" + listType) } func canUserCreateTrainJob(uid int64) (bool, error) { @@ -2124,7 +2217,7 @@ func InferenceJobShow(ctx *context.Context) { ctx.Data["jobName"] = task.JobName ctx.Data["displayJobName"] = task.DisplayJobName ctx.Data["task"] = task - ctx.Data["canDownload"] = cloudbrain.CanDeleteJob(ctx, task) + ctx.Data["canDownload"] = cloudbrain.CanModifyJob(ctx, task) tempUids := []int64{} tempUids = append(tempUids, task.UserID) @@ -2276,7 +2369,7 @@ func SetJobCount(ctx *context.Context) { repoId := ctx.Repo.Repository.ID _, jobCount, err := models.Cloudbrains(&models.CloudbrainsOptions{ RepoID: repoId, - Type: modelarts.DebugType, + Type: models.TypeCloudBrainAll, }) if err != nil { ctx.ServerError("Get job faild:", err) diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index b4adfc347..9d906270f 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -54,7 +54,12 @@ func queryUserDataPage(ctx *context.Context, tableName string, queryObj interfac "N1": ctx.Tr("user.static.createrepocount"), "O1": ctx.Tr("user.static.openiindex"), "P1": ctx.Tr("user.static.registdate"), - "Q1": ctx.Tr("user.static.countdate"), + "Q1": ctx.Tr("user.static.CloudBrainTaskNum"), + "R1": ctx.Tr("user.static.CloudBrainRunTime"), + "S1": ctx.Tr("user.static.CommitDatasetNum"), + "T1": ctx.Tr("user.static.CommitModelCount"), + "U1": ctx.Tr("user.static.UserIndex"), + "V1": ctx.Tr("user.static.countdate"), } for k, v := range dataHeader { //设置单元格的值 @@ -89,8 +94,14 @@ func queryUserDataPage(ctx *context.Context, tableName string, queryObj interfac formatTime := userRecord.RegistDate.Format("2006-01-02 15:04:05") xlsx.SetCellValue(sheetName, "P"+rows, formatTime[0:len(formatTime)-3]) + xlsx.SetCellValue(sheetName, "Q"+rows, userRecord.CloudBrainTaskNum) + xlsx.SetCellValue(sheetName, "R"+rows, fmt.Sprintf("%.2f", float64(userRecord.CloudBrainRunTime)/3600)) + xlsx.SetCellValue(sheetName, "S"+rows, userRecord.CommitDatasetNum) + xlsx.SetCellValue(sheetName, "T"+rows, userRecord.CommitModelCount) + xlsx.SetCellValue(sheetName, "U"+rows, fmt.Sprintf("%.2f", userRecord.UserIndex)) + formatTime = userRecord.DataDate - xlsx.SetCellValue(sheetName, "Q"+rows, formatTime) + xlsx.SetCellValue(sheetName, "V"+rows, formatTime) } indexTotal += PAGE_SIZE @@ -115,6 +126,30 @@ func queryUserDataPage(ctx *context.Context, tableName string, queryObj interfac } } +func QueryMetrics(ctx *context.Context) { + startDate := ctx.Query("startDate") + endDate := ctx.Query("endDate") + startTime, _ := time.ParseInLocation("2006-01-02", startDate, time.Local) + endTime, _ := time.ParseInLocation("2006-01-02", endDate, time.Local) + result, count := models.QueryMetrics(startTime.Unix(), endTime.Unix()) + mapInterface := make(map[string]interface{}) + mapInterface["data"] = result + mapInterface["count"] = count + ctx.JSON(http.StatusOK, mapInterface) +} + +func QueryRankingList(ctx *context.Context) { + key := ctx.Query("key") + tableName := ctx.Query("tableName") + limit := ctx.QueryInt("limit") + + result, count := models.QueryRankList(key, tableName, limit) + mapInterface := make(map[string]interface{}) + mapInterface["data"] = result + mapInterface["count"] = count + ctx.JSON(http.StatusOK, mapInterface) +} + func QueryUserStaticCurrentMonth(ctx *context.Context) { queryUserDataPage(ctx, "public.user_business_analysis_current_month", new(models.UserBusinessAnalysisCurrentMonth)) } @@ -221,7 +256,12 @@ func QueryUserStaticDataPage(ctx *context.Context) { "N1": ctx.Tr("user.static.createrepocount"), "O1": ctx.Tr("user.static.openiindex"), "P1": ctx.Tr("user.static.registdate"), - "Q1": ctx.Tr("user.static.countdate"), + "Q1": ctx.Tr("user.static.CloudBrainTaskNum"), + "R1": ctx.Tr("user.static.CloudBrainRunTime"), + "S1": ctx.Tr("user.static.CommitDatasetNum"), + "T1": ctx.Tr("user.static.CommitModelCount"), + "U1": ctx.Tr("user.static.UserIndex"), + "V1": ctx.Tr("user.static.countdate"), } for k, v := range dataHeader { //设置单元格的值 @@ -249,9 +289,13 @@ func QueryUserStaticDataPage(ctx *context.Context) { formatTime := userRecord.RegistDate.Format("2006-01-02 15:04:05") xlsx.SetCellValue(sheetName, "P"+rows, formatTime[0:len(formatTime)-3]) - + xlsx.SetCellValue(sheetName, "Q"+rows, userRecord.CloudBrainTaskNum) + xlsx.SetCellValue(sheetName, "R"+rows, fmt.Sprintf("%.2f", float64(userRecord.CloudBrainRunTime)/3600)) + xlsx.SetCellValue(sheetName, "S"+rows, userRecord.CommitDatasetNum) + xlsx.SetCellValue(sheetName, "T"+rows, userRecord.CommitModelCount) + xlsx.SetCellValue(sheetName, "U"+rows, fmt.Sprintf("%.2f", userRecord.UserIndex)) formatTime = userRecord.DataDate - xlsx.SetCellValue(sheetName, "Q"+rows, formatTime+" 00:01") + xlsx.SetCellValue(sheetName, "V"+rows, formatTime) } //设置默认打开的表单 diff --git a/routers/repo/view.go b/routers/repo/view.go index 320102ba4..b28e21aa1 100755 --- a/routers/repo/view.go +++ b/routers/repo/view.go @@ -247,7 +247,11 @@ func renderDirectory(ctx *context.Context, treeLink string) { ctx.Data["ReadmeInList"] = true ctx.Data["ReadmeExist"] = true ctx.Data["FileIsSymlink"] = readmeFile.isSymlink - ctx.Data["ReadmeName"] = readmeFile.name + if ctx.Repo.TreePath == "" { + ctx.Data["ReadmeRelativePath"] = readmeFile.name + } else { + ctx.Data["ReadmeRelativePath"] = ctx.Repo.TreePath + "/" + readmeFile.name + } if ctx.Repo.CanEnableEditor() { ctx.Data["CanEditFile"] = true @@ -579,11 +583,11 @@ func safeURL(address string) string { } type ContributorInfo struct { - UserInfo *models.User // nil for contributor who is not a registered user - RelAvatarLink string `json:"rel_avatar_link"` - UserName string `json:"user_name"` - Email string `json:"email"` - CommitCnt int `json:"commit_cnt"` + UserInfo *models.User // nil for contributor who is not a registered user + RelAvatarLink string `json:"rel_avatar_link"` + UserName string `json:"user_name"` + Email string `json:"email"` + CommitCnt int `json:"commit_cnt"` } type GetContributorsInfo struct { @@ -642,7 +646,7 @@ func Home(ctx *context.Context) { existedContributorInfo.CommitCnt += c.CommitCnt } else { var newContributor = &ContributorInfo{ - user, "", "",c.Email, c.CommitCnt, + user, "", "", c.Email, c.CommitCnt, } count++ contributorInfos = append(contributorInfos, newContributor) @@ -839,7 +843,7 @@ func renderCode(ctx *context.Context) { compareInfo, err = baseGitRepo.GetCompareInfo(ctx.Repo.Repository.RepoPath(), ctx.Repo.BranchName, ctx.Repo.Repository.BaseRepo.DefaultBranch) ctx.Data["UpstreamSameBranchName"] = false } - if err==nil && compareInfo != nil { + if err == nil && compareInfo != nil { if compareInfo.Commits != nil { log.Info("compareInfoCommits数量:%d", compareInfo.Commits.Len()) ctx.Data["FetchUpstreamCnt"] = compareInfo.Commits.Len() @@ -950,7 +954,7 @@ func ContributorsAPI(ctx *context.Context) { } else { // new committer info var newContributor = &ContributorInfo{ - user, user.RelAvatarLink(),user.Name, user.Email,c.CommitCnt, + user, user.RelAvatarLink(), user.Name, user.Email, c.CommitCnt, } count++ contributorInfos = append(contributorInfos, newContributor) @@ -963,7 +967,7 @@ func ContributorsAPI(ctx *context.Context) { existedContributorInfo.CommitCnt += c.CommitCnt } else { var newContributor = &ContributorInfo{ - user, "", "",c.Email,c.CommitCnt, + user, "", "", c.Email, c.CommitCnt, } count++ contributorInfos = append(contributorInfos, newContributor) diff --git a/routers/routes/routes.go b/routers/routes/routes.go index c1ff24216..4cffcd10b 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -323,6 +323,9 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/action/notification", routers.ActionNotification) m.Get("/recommend/org", routers.RecommendOrgFromPromote) m.Get("/recommend/repo", routers.RecommendRepoFromPromote) + m.Post("/all/search/", routers.Search) + m.Get("/all/search/", routers.EmptySearch) + m.Get("/all/dosearch/", routers.SearchApi) m.Get("/home/term", routers.HomeTerm) m.Group("/explore", func() { m.Get("", func(ctx *context.Context) { @@ -1035,6 +1038,19 @@ func RegisterRoutes(m *macaron.Macaron) { m.Post("/create", reqWechatBind, reqRepoCloudBrainWriter, bindIgnErr(auth.CreateCloudBrainForm{}), repo.CloudBrainBenchmarkCreate) m.Get("/get_child_types", repo.GetChildTypes) }) + + m.Group("/train-job", func() { + m.Group("/:jobid", func() { + m.Get("", reqRepoCloudBrainReader, repo.CloudBrainTrainJobShow) + m.Post("/del", cloudbrain.AdminOrOwnerOrJobCreaterRightForTrain, repo.CloudBrainTrainJobDel) + //m.Get("/models", reqRepoCloudBrainReader, repo.CloudBrainShowModels) + m.Get("/download_model", cloudbrain.AdminOrOwnerOrJobCreaterRightForTrain, repo.CloudBrainDownloadModel) + //m.Get("/create_version", reqWechatBind, cloudbrain.AdminOrJobCreaterRightForTrain, repo.TrainJobNewVersion) + //m.Post("/create_version", reqWechatBind, cloudbrain.AdminOrJobCreaterRightForTrain, bindIgnErr(auth.CreateModelArtsTrainJobForm{}), repo.TrainJobCreateVersion) + }) + m.Get("/create", reqWechatBind, reqRepoCloudBrainWriter, repo.CloudBrainTrainJobNew) + m.Post("/create", reqWechatBind, reqRepoCloudBrainWriter, bindIgnErr(auth.CreateCloudBrainForm{}), repo.CloudBrainCreate) + }) }, context.RepoRef()) m.Group("/modelmanage", func() { m.Post("/create_model", reqRepoModelManageWriter, repo.SaveModel) diff --git a/routers/search.go b/routers/search.go new file mode 100644 index 000000000..c5655b9e1 --- /dev/null +++ b/routers/search.go @@ -0,0 +1,1194 @@ +package routers + +import ( + "encoding/json" + "fmt" + "sort" + "strconv" + "strings" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/timeutil" + "github.com/olivere/elastic/v7" +) + +type SearchRes struct { + Total int64 + Result []map[string]interface{} + PrivateTotal int64 +} + +var client *elastic.Client + +func InitESClient() { + ESSearchUrl := setting.ESSearchURL + var err error + client, err = elastic.NewClient(elastic.SetSniff(false), elastic.SetURL(ESSearchUrl)) + if err != nil { + log.Info("es init error.") + //panic(err) + } +} + +func EmptySearch(ctx *context.Context) { + log.Info("search template.") + ctx.Data["Keyword"] = "" + ctx.HTML(200, "explore/search_new") +} + +func Search(ctx *context.Context) { + log.Info("search template.") + keyword := strings.Trim(ctx.Query("q"), " ") + ctx.Data["Keyword"] = keyword + ctx.Data["SortType"] = "newest" + ctx.HTML(200, "explore/search_new") +} + +func SearchApi(ctx *context.Context) { + TableName := ctx.Query("TableName") + Key := ctx.Query("Key") + Page := ctx.QueryInt("Page") + PageSize := ctx.QueryInt("PageSize") + OnlyReturnNum := ctx.QueryBool("OnlyReturnNum") + OnlySearchLabel := ctx.QueryBool("OnlySearchLabel") + + if Page <= 0 { + Page = 1 + } + if PageSize <= 0 || PageSize > 200 { + PageSize = setting.UI.IssuePagingNum + } + if Key != "" && !OnlyReturnNum { + go models.SaveSearchKeywordToDb(Key) + } + if TableName == "repository" { + if OnlySearchLabel { + searchRepoByLabel(ctx, Key, Page, PageSize) + } else { + searchRepo(ctx, "repository-es-index", Key, Page, PageSize, OnlyReturnNum) + } + return + } else if TableName == "issue" { + searchIssueOrPr(ctx, "issue-es-index", Key, Page, PageSize, OnlyReturnNum, "f") + return + } else if TableName == "user" { + searchUserOrOrg(ctx, "user-es-index", Key, Page, PageSize, true, OnlyReturnNum) + return + } else if TableName == "org" { + searchUserOrOrg(ctx, "user-es-index", Key, Page, PageSize, false, OnlyReturnNum) + return + } else if TableName == "dataset" { + searchDataSet(ctx, "dataset-es-index", Key, Page, PageSize, OnlyReturnNum) + return + } else if TableName == "pr" { + searchIssueOrPr(ctx, "issue-es-index", Key, Page, PageSize, OnlyReturnNum, "t") + //searchPR(ctx, "issue-es-index", Key, Page, PageSize, OnlyReturnNum) + return + } +} + +func searchRepoByLabel(ctx *context.Context, Key string, Page int, PageSize int) { + /* + 项目， ES名称： repository-es-index + 搜索: + name character varying(255) , 项目名称 + description text, 项目描述 + topics json, 标签 + 排序: + updated_unix + num_watches, + num_stars, + num_forks, + */ + SortBy := ctx.Query("SortBy") + PrivateTotal := ctx.QueryInt("PrivateTotal") + WebTotal := ctx.QueryInt("WebTotal") + ascending := ctx.QueryBool("Ascending") + language := ctx.Query("language") + if language == "" { + language = "zh-CN" + } + from := (Page - 1) * PageSize + resultObj := &SearchRes{} + log.Info("WebTotal=" + fmt.Sprint(WebTotal)) + log.Info("PrivateTotal=" + fmt.Sprint(PrivateTotal)) + resultObj.Result = make([]map[string]interface{}, 0) + if from == 0 { + WebTotal = 0 + } + if ctx.User != nil && (from < PrivateTotal || from == 0) { + orderBy := models.SearchOrderByRecentUpdated + switch SortBy { + case "updated_unix.keyword": + orderBy = models.SearchOrderByRecentUpdated + case "num_stars": + orderBy = models.SearchOrderByStarsReverse + case "num_forks": + orderBy = models.SearchOrderByForksReverse + case "num_watches": + orderBy = models.SearchOrderByWatches + } + log.Info("actor is null?:" + fmt.Sprint(ctx.User == nil)) + repos, count, err := models.SearchRepository(&models.SearchRepoOptions{ + ListOptions: models.ListOptions{ + Page: Page, + PageSize: PageSize, + }, + Actor: ctx.User, + OrderBy: orderBy, + Private: true, + OnlyPrivate: true, + TopicOnly: true, + TopicName: Key, + IncludeDescription: setting.UI.SearchRepoDescription, + }) + if err != nil { + ctx.JSON(200, "") + return + } + resultObj.PrivateTotal = count + if repos.Len() > 0 { + log.Info("Query private repo number is:" + fmt.Sprint(repos.Len())) + makePrivateRepo(repos, resultObj, Key, language) + } else { + log.Info("not found private repo,keyword=" + Key) + } + if repos.Len() >= PageSize { + if WebTotal > 0 { + resultObj.Total = int64(WebTotal) + ctx.JSON(200, resultObj) + return + } + } + } else { + if ctx.User == nil { + resultObj.PrivateTotal = 0 + } else { + resultObj.PrivateTotal = int64(PrivateTotal) + } + } + + from = from - PrivateTotal + if from < 0 { + from = 0 + } + Size := PageSize - len(resultObj.Result) + + log.Info("query searchRepoByLabel start") + if Key != "" { + boolQ := elastic.NewBoolQuery() + topicsQuery := elastic.NewMatchQuery("topics", Key) + boolQ.Should(topicsQuery) + + res, err := client.Search("repository-es-index").Query(boolQ).SortBy(getSort(SortBy, ascending)).From(from).Size(Size).Highlight(queryHighlight("topics")).Do(ctx.Req.Context()) + if err == nil { + searchJson, _ := json.Marshal(res) + log.Info("searchJson=" + string(searchJson)) + esresult := makeRepoResult(res, "", false, language) + resultObj.Total = resultObj.PrivateTotal + esresult.Total + resultObj.Result = append(resultObj.Result, esresult.Result...) + ctx.JSON(200, resultObj) + } else { + log.Info("query es error," + err.Error()) + ctx.JSON(200, "") + } + } else { + ctx.JSON(200, "") + } +} + +func getSort(SortBy string, ascending bool) elastic.Sorter { + var sort elastic.Sorter + sort = elastic.NewScoreSort() + if SortBy != "" { + if SortBy == "default" { + return sort + } + return elastic.NewFieldSort(SortBy).Order(ascending) + } + return sort +} + +func searchRepo(ctx *context.Context, TableName string, Key string, Page int, PageSize int, OnlyReturnNum bool) { + /* + 项目， ES名称： repository-es-index + 搜索: + name character varying(255) , 项目名称 + description text, 项目描述 + topics json, 标签 + 排序: + updated_unix + num_watches, + num_stars, + num_forks, + */ + + SortBy := ctx.Query("SortBy") + PrivateTotal := ctx.QueryInt("PrivateTotal") + WebTotal := ctx.QueryInt("WebTotal") + ascending := ctx.QueryBool("Ascending") + from := (Page - 1) * PageSize + resultObj := &SearchRes{} + log.Info("WebTotal=" + fmt.Sprint(WebTotal)) + log.Info("PrivateTotal=" + fmt.Sprint(PrivateTotal)) + resultObj.Result = make([]map[string]interface{}, 0) + if from == 0 { + WebTotal = 0 + } + language := ctx.Query("language") + if language == "" { + language = "zh-CN" + } + if ctx.User != nil && (from < PrivateTotal || from == 0) { + orderBy := models.SearchOrderByRecentUpdated + switch SortBy { + case "updated_unix.keyword": + orderBy = models.SearchOrderByRecentUpdated + case "num_stars": + orderBy = models.SearchOrderByStarsReverse + case "num_forks": + orderBy = models.SearchOrderByForksReverse + case "num_watches": + orderBy = models.SearchOrderByWatches + } + log.Info("actor is null?:" + fmt.Sprint(ctx.User == nil)) + repos, count, err := models.SearchRepository(&models.SearchRepoOptions{ + ListOptions: models.ListOptions{ + Page: Page, + PageSize: PageSize, + }, + Actor: ctx.User, + OrderBy: orderBy, + Private: true, + OnlyPrivate: true, + Keyword: Key, + IncludeDescription: setting.UI.SearchRepoDescription, + OnlySearchPrivate: true, + }) + if err != nil { + ctx.JSON(200, "") + return + } + resultObj.PrivateTotal = count + if repos.Len() > 0 { + log.Info("Query private repo number is:" + fmt.Sprint(repos.Len())) + makePrivateRepo(repos, resultObj, Key, language) + } else { + log.Info("not found private repo,keyword=" + Key) + } + if repos.Len() >= PageSize { + if WebTotal > 0 { + resultObj.Total = int64(WebTotal) + ctx.JSON(200, resultObj) + return + } + } + } else { + if ctx.User == nil { + resultObj.PrivateTotal = 0 + } else { + resultObj.PrivateTotal = int64(PrivateTotal) + } + } + + from = from - PrivateTotal + if from < 0 { + from = 0 + } + Size := PageSize - len(resultObj.Result) + + log.Info("query searchRepo start") + if Key != "" { + boolQ := elastic.NewBoolQuery() + nameQuery := elastic.NewMatchQuery("alias", Key).Boost(1024).QueryName("f_first") + descriptionQuery := elastic.NewMatchQuery("description", Key).Boost(1.5).QueryName("f_second") + topicsQuery := elastic.NewMatchQuery("topics", Key).Boost(1).QueryName("f_third") + boolQ.Should(nameQuery, descriptionQuery, topicsQuery) + + res, err := client.Search(TableName).Query(boolQ).SortBy(getSort(SortBy, ascending)).From(from).Size(Size).Highlight(queryHighlight("alias", "description", "topics")).Do(ctx.Req.Context()) + if err == nil { + searchJson, _ := json.Marshal(res) + log.Info("searchJson=" + string(searchJson)) + esresult := makeRepoResult(res, Key, OnlyReturnNum, language) + resultObj.Total = resultObj.PrivateTotal + esresult.Total + isNeedSort := false + if len(resultObj.Result) > 0 { + isNeedSort = true + } + resultObj.Result = append(resultObj.Result, esresult.Result...) + if isNeedSort { + sortRepo(resultObj.Result, SortBy, ascending) + } + ctx.JSON(200, resultObj) + } else { + log.Info("query es error," + err.Error()) + ctx.JSON(200, "") + } + } else { + log.Info("query all content.") + //搜索的属性要指定{"timestamp":{"unmapped_type":"date"}} + res, err := client.Search(TableName).SortBy(getSort(SortBy, ascending)).From(from).Size(Size).Do(ctx.Req.Context()) + if err == nil { + searchJson, _ := json.Marshal(res) + log.Info("searchJson=" + string(searchJson)) + esresult := makeRepoResult(res, "", OnlyReturnNum, language) + resultObj.Total = resultObj.PrivateTotal + esresult.Total + resultObj.Result = append(resultObj.Result, esresult.Result...) + ctx.JSON(200, resultObj) + } else { + log.Info("query es error," + err.Error()) + ctx.JSON(200, "") + } + } +} + +func sortRepo(Result []map[string]interface{}, SortBy string, ascending bool) { + orderBy := "" + switch SortBy { + case "updated_unix.keyword": + orderBy = "updated_unix" + case "num_stars": + orderBy = "num_stars" + case "num_forks": + orderBy = "num_forks" + case "num_watches": + orderBy = "num_watches" + } + sort.Slice(Result, func(i, j int) bool { + return getInt(Result[i][orderBy], orderBy) > getInt(Result[j][orderBy], orderBy) + }) +} + +func getInt(tmp interface{}, orderBy string) int64 { + timeInt, err := strconv.ParseInt(fmt.Sprint(tmp), 10, 64) + if err == nil { + return timeInt + } else { + log.Info("convert " + orderBy + " error type=" + fmt.Sprint(tmp)) + } + return -1 +} + +func makePrivateRepo(repos models.RepositoryList, res *SearchRes, keyword string, language string) { + + for _, repo := range repos { + record := make(map[string]interface{}) + record["id"] = repo.ID + record["name"] = makeHighLight(keyword, repo.Name) + record["real_name"] = repo.Name + record["owner_name"] = repo.OwnerName + record["description"] = truncLongText(makeHighLight(keyword, repo.Description), true) + + hightTopics := make([]string, 0) + if len(repo.Topics) > 0 { + for _, t := range repo.Topics { + hightTopics = append(hightTopics, makeHighLight(keyword, t)) + } + } + record["hightTopics"] = hightTopics + + record["num_watches"] = repo.NumWatches + record["num_stars"] = repo.NumStars + record["num_forks"] = repo.NumForks + record["alias"] = truncLongText(makeHighLight(keyword, repo.Alias), true) + record["lower_alias"] = repo.LowerAlias + record["topics"] = repo.Topics + record["avatar"] = repo.RelAvatarLink() + if len(repo.RelAvatarLink()) == 0 { + record["avatar"] = setting.RepositoryAvatarFallbackImage + } + record["updated_unix"] = repo.UpdatedUnix + record["updated_html"] = timeutil.TimeSinceUnix(repo.UpdatedUnix, language) + lang, err := repo.GetTopLanguageStats(1) + if err == nil && len(lang) > 0 { + record["lang"] = lang[0].Language + } else { + record["lang"] = "" + } + record["is_private"] = true + res.Result = append(res.Result, record) + } +} + +func makeHighLight(keyword string, dest string) string { + + dest = replaceIngoreUpperOrLower(dest, strings.ToLower(dest), strings.ToLower(keyword)) + + return dest +} + +func replaceIngoreUpperOrLower(dest string, destLower string, keywordLower string) string { + re := "" + last := 0 + lenDestLower := len(destLower) + lenkeywordLower := len(keywordLower) + for i := 0; i < lenDestLower; i++ { + if destLower[i] == keywordLower[0] { + isFind := true + for j := 1; j < lenkeywordLower; j++ { + if (i+j) < lenDestLower && keywordLower[j] != destLower[i+j] { + isFind = false + break + } + } + if isFind && (i+lenkeywordLower) <= lenDestLower { + re += dest[last:i] + "\u003cfont color='red'\u003e" + dest[i:(i+lenkeywordLower)] + "\u003c/font\u003e" + i = i + lenkeywordLower + last = i + } + } + } + if last < lenDestLower { + re += dest[last:lenDestLower] + } + return re +} + +func makeRepoResult(sRes *elastic.SearchResult, Key string, OnlyReturnNum bool, language string) *SearchRes { + total := sRes.Hits.TotalHits.Value + result := make([]map[string]interface{}, 0) + if !OnlyReturnNum { + for i, hit := range sRes.Hits.Hits { + log.Info("this is repo query " + fmt.Sprint(i) + " result.") + recordSource := make(map[string]interface{}) + source, err := hit.Source.MarshalJSON() + + if err == nil { + err = json.Unmarshal(source, &recordSource) + if err == nil { + record := make(map[string]interface{}) + record["id"] = hit.Id + record["alias"] = getLabelValue("alias", recordSource, hit.Highlight) + record["real_name"] = recordSource["name"] + record["owner_name"] = recordSource["owner_name"] + if recordSource["description"] != nil { + desc := getLabelValue("description", recordSource, hit.Highlight) + record["description"] = dealLongText(desc, Key, hit.MatchedQueries) + } else { + record["description"] = "" + } + + record["hightTopics"] = jsonStrToArray(getLabelValue("topics", recordSource, hit.Highlight)) + record["num_watches"] = recordSource["num_watches"] + record["num_stars"] = recordSource["num_stars"] + record["num_forks"] = recordSource["num_forks"] + record["lower_alias"] = recordSource["lower_alias"] + if recordSource["topics"] != nil { + topicsStr := recordSource["topics"].(string) + log.Info("topicsStr=" + topicsStr) + if topicsStr != "null" { + record["topics"] = jsonStrToArray(topicsStr) + } + } + if recordSource["avatar"] != nil { + avatarstr := recordSource["avatar"].(string) + if len(avatarstr) == 0 { + record["avatar"] = setting.RepositoryAvatarFallbackImage + } else { + record["avatar"] = setting.AppSubURL + "/repo-avatars/" + avatarstr + } + } + record["updated_unix"] = recordSource["updated_unix"] + setUpdateHtml(record, recordSource["updated_unix"].(string), language) + + record["lang"] = recordSource["lang"] + record["is_private"] = false + result = append(result, record) + } else { + log.Info("deal repo source error," + err.Error()) + } + } else { + log.Info("deal repo source error," + err.Error()) + } + } + } + returnObj := &SearchRes{ + Total: total, + Result: result, + } + + return returnObj +} + +func setUpdateHtml(record map[string]interface{}, updated_unix string, language string) { + timeInt, err := strconv.ParseInt(updated_unix, 10, 64) + if err == nil { + record["updated_html"] = timeutil.TimeSinceUnix(timeutil.TimeStamp(timeInt), language) + } +} + +func jsonStrToArray(str string) []string { + b := []byte(str) + strs := make([]string, 0) + err := json.Unmarshal(b, &strs) + if err != nil { + log.Info("convert str arrar error, str=" + str) + } + return strs +} + +func dealLongText(text string, Key string, MatchedQueries []string) string { + var isNeedToDealText bool + isNeedToDealText = false + if len(MatchedQueries) > 0 && Key != "" { + if MatchedQueries[0] == "f_second" || MatchedQueries[0] == "f_third" { + isNeedToDealText = true + } + } + return truncLongText(text, isNeedToDealText) +} + +func truncLongText(text string, isNeedToDealText bool) string { + startStr := "color=" + textRune := []rune(text) + stringlen := len(textRune) + if isNeedToDealText && stringlen > 200 { + index := findFont(textRune, []rune(startStr)) + if index > 0 { + start := index - 50 + if start < 0 { + start = 0 + } + end := index + 150 + if end >= stringlen { + end = stringlen + } + return trimFontHtml(textRune[start:end]) + "..." + } else { + return trimFontHtml(textRune[0:200]) + "..." + } + } else { + if stringlen > 200 { + return trimFontHtml(textRune[0:200]) + "..." + } else { + return text + } + } +} + +func trimFontHtml(text []rune) string { + startRune := rune('<') + endRune := rune('>') + count := 0 + i := 0 + for ; i < len(text); i++ { + if text[i] == startRune { //start < + re := false + j := i + 1 + for ; j < len(text); j++ { + if text[j] == endRune { + re = true + break + } + } + if re { //found > + i = j + count++ + } else { + if count%2 == 1 { + return string(text[0:i]) + "" + } else { + return string(text[0:i]) + } + } + } + } + if count%2 == 1 { + return string(text[0:i]) + "" + } else { + return string(text[0:i]) + } +} + +func trimHrefHtml(result string) string { + result = strings.Replace(result, "", "", -1) + result = strings.Replace(result, "\n", "", -1) + var index int + for { + index = findSubstr(result, 0, "") + if sIndex != -1 { + result = result[0:index] + result[sIndex+1:] + } else { + result = result[0:index] + result[index+2:] + } + } else { + break + } + } + return result +} + +func findFont(text []rune, childText []rune) int { + for i := 0; i < len(text); i++ { + if text[i] == childText[0] { + re := true + for j, k := range childText { + if k != text[i+j] { + re = false + break + } + } + if re { + return i + } + } + } + return -1 +} + +func findSubstr(text string, startindex int, childText string) int { + for i := startindex; i < len(text); i++ { + if text[i] == childText[0] { + re := true + for k := range childText { + if childText[k] != text[i+k] { + re = false + break + } + } + if re { + return i + } + } + } + return -1 +} + +func searchUserOrOrg(ctx *context.Context, TableName string, Key string, Page int, PageSize int, IsQueryUser bool, OnlyReturnNum bool) { + /* + 用户或者组织 ES名称： user-es-index + 搜索: + name , 名称 + full_name 全名 + description 描述或者简介 + 排序: + created_unix + 名称字母序 + */ + SortBy := ctx.Query("SortBy") + ascending := ctx.QueryBool("Ascending") + boolQ := elastic.NewBoolQuery() + + typeValue := 1 + if IsQueryUser { + typeValue = 0 + } + UserOrOrgQuery := elastic.NewTermQuery("type", typeValue) + if Key != "" { + boolKeyQ := elastic.NewBoolQuery() + log.Info("user or org Key=" + Key) + nameQuery := elastic.NewMatchQuery("name", Key).Boost(2).QueryName("f_first") + full_nameQuery := elastic.NewMatchQuery("full_name", Key).Boost(1.5).QueryName("f_second") + descriptionQuery := elastic.NewMatchQuery("description", Key).Boost(1).QueryName("f_third") + boolKeyQ.Should(nameQuery, full_nameQuery, descriptionQuery) + boolQ.Must(UserOrOrgQuery, boolKeyQ) + } else { + boolQ.Must(UserOrOrgQuery) + } + + res, err := client.Search(TableName).Query(boolQ).SortBy(getSort(SortBy, ascending)).From((Page - 1) * PageSize).Size(PageSize).Highlight(queryHighlight("name", "full_name", "description")).Do(ctx.Req.Context()) + if err == nil { + searchJson, _ := json.Marshal(res) + log.Info("searchJson=" + string(searchJson)) + result := makeUserOrOrgResult(res, Key, ctx, OnlyReturnNum) + ctx.JSON(200, result) + } else { + log.Info("query es error," + err.Error()) + ctx.JSON(200, "") + } +} + +func getLabelValue(key string, recordSource map[string]interface{}, searchHighliht elastic.SearchHitHighlight) string { + if value, ok := searchHighliht[key]; !ok { + if recordSource[key] != nil { + return recordSource[key].(string) + } else { + return "" + } + } else { + return value[0] + } +} + +func makeUserOrOrgResult(sRes *elastic.SearchResult, Key string, ctx *context.Context, OnlyReturnNum bool) *SearchRes { + total := sRes.Hits.TotalHits.Value + result := make([]map[string]interface{}, 0) + if !OnlyReturnNum { + for i, hit := range sRes.Hits.Hits { + log.Info("this is user query " + fmt.Sprint(i) + " result.") + recordSource := make(map[string]interface{}) + source, err := hit.Source.MarshalJSON() + + if err == nil { + err = json.Unmarshal(source, &recordSource) + if err == nil { + record := make(map[string]interface{}) + record["id"] = hit.Id + record["name"] = getLabelValue("name", recordSource, hit.Highlight) + record["real_name"] = recordSource["name"] + record["full_name"] = getLabelValue("full_name", recordSource, hit.Highlight) + if recordSource["description"] != nil { + desc := getLabelValue("description", recordSource, hit.Highlight) + record["description"] = dealLongText(desc, Key, hit.MatchedQueries) + } else { + record["description"] = "" + } + if ctx.User != nil { + record["email"] = recordSource["email"] + } else { + record["email"] = "" + } + + record["location"] = recordSource["location"] + record["website"] = recordSource["website"] + record["num_repos"] = recordSource["num_repos"] + record["num_teams"] = recordSource["num_teams"] + record["num_members"] = recordSource["num_members"] + + record["avatar"] = strings.TrimRight(setting.AppSubURL, "/") + "/user/avatar/" + recordSource["name"].(string) + "/" + strconv.Itoa(-1) + record["updated_unix"] = recordSource["updated_unix"] + record["created_unix"] = recordSource["created_unix"] + record["add_time"] = getAddTime(recordSource["created_unix"].(string)) + result = append(result, record) + } else { + log.Info("deal user source error," + err.Error()) + } + } else { + log.Info("deal user source error," + err.Error()) + } + } + } + returnObj := &SearchRes{ + Total: total, + Result: result, + } + return returnObj +} + +func getAddTime(time string) string { + timeInt, err := strconv.ParseInt(time, 10, 64) + if err == nil { + t := timeutil.TimeStamp(timeInt) + return t.FormatShort() + } + return "" +} + +func searchDataSet(ctx *context.Context, TableName string, Key string, Page int, PageSize int, OnlyReturnNum bool) { + /* + 数据集，ES名称：dataset-es-index + 搜索: + title , 名称 + description 描述 + category 标签 + file_name 数据集文件名称 + 排序: + download_times + + */ + log.Info("query searchdataset start") + SortBy := ctx.Query("SortBy") + ascending := ctx.QueryBool("Ascending") + PrivateTotal := ctx.QueryInt("PrivateTotal") + WebTotal := ctx.QueryInt("WebTotal") + language := ctx.Query("language") + if language == "" { + language = "zh-CN" + } + from := (Page - 1) * PageSize + if from == 0 { + WebTotal = 0 + } + resultObj := &SearchRes{} + log.Info("WebTotal=" + fmt.Sprint(WebTotal)) + log.Info("PrivateTotal=" + fmt.Sprint(PrivateTotal)) + resultObj.Result = make([]map[string]interface{}, 0) + + if ctx.User != nil && (from < PrivateTotal || from == 0) { + + log.Info("actor is null?:" + fmt.Sprint(ctx.User == nil)) + datasets, count, err := models.SearchDatasetBySQL(Page, PageSize, Key, ctx.User.ID) + if err != nil { + ctx.JSON(200, "") + return + } + resultObj.PrivateTotal = count + datasetSize := len(datasets) + if datasetSize > 0 { + log.Info("Query private dataset number is:" + fmt.Sprint(datasetSize) + " count=" + fmt.Sprint(count)) + makePrivateDataSet(datasets, resultObj, Key, language) + } else { + log.Info("not found private dataset, keyword=" + Key) + } + if datasetSize >= PageSize { + if WebTotal > 0 { //next page， not first query. + resultObj.Total = int64(WebTotal) + ctx.JSON(200, resultObj) + return + } + } + } else { + resultObj.PrivateTotal = int64(PrivateTotal) + } + + from = from - PrivateTotal + if from < 0 { + from = 0 + } + Size := PageSize - len(resultObj.Result) + + boolQ := elastic.NewBoolQuery() + if Key != "" { + nameQuery := elastic.NewMatchQuery("title", Key).Boost(2).QueryName("f_first") + descQuery := elastic.NewMatchQuery("description", Key).Boost(1.5).QueryName("f_second") + fileNameQuery := elastic.NewMatchQuery("file_name", Key).Boost(1).QueryName("f_third") + categoryQuery := elastic.NewMatchQuery("category", Key).Boost(1).QueryName("f_fourth") + boolQ.Should(nameQuery, descQuery, categoryQuery, fileNameQuery) + res, err := client.Search(TableName).Query(boolQ).SortBy(getSort(SortBy, ascending)).From(from).Size(Size).Highlight(queryHighlight("title", "description", "file_name", "category")).Do(ctx.Req.Context()) + if err == nil { + searchJson, _ := json.Marshal(res) + log.Info("searchJson=" + string(searchJson)) + esresult := makeDatasetResult(res, Key, OnlyReturnNum, language) + resultObj.Total = resultObj.PrivateTotal + esresult.Total + log.Info("query dataset es count=" + fmt.Sprint(esresult.Total) + " total=" + fmt.Sprint(resultObj.Total)) + resultObj.Result = append(resultObj.Result, esresult.Result...) + ctx.JSON(200, resultObj) + } else { + log.Info("query es error," + err.Error()) + } + } else { + log.Info("query all datasets.") + //搜索的属性要指定{"timestamp":{"unmapped_type":"date"}} + res, err := client.Search(TableName).SortBy(getSort(SortBy, ascending)).From(from).Size(Size).Do(ctx.Req.Context()) + if err == nil { + searchJson, _ := json.Marshal(res) + log.Info("searchJson=" + string(searchJson)) + esresult := makeDatasetResult(res, "", OnlyReturnNum, language) + resultObj.Total = resultObj.PrivateTotal + esresult.Total + log.Info("query dataset es count=" + fmt.Sprint(esresult.Total) + " total=" + fmt.Sprint(resultObj.Total)) + resultObj.Result = append(resultObj.Result, esresult.Result...) + ctx.JSON(200, resultObj) + } else { + log.Info("query es error," + err.Error()) + ctx.JSON(200, "") + } + } + +} + +func makePrivateDataSet(datasets []*models.Dataset, res *SearchRes, Key string, language string) { + for _, dataset := range datasets { + record := make(map[string]interface{}) + + record["id"] = dataset.ID + userId := dataset.UserID + + user, errUser := models.GetUserByID(userId) + if errUser == nil { + record["owerName"] = user.GetDisplayName() + record["avatar"] = user.RelAvatarLink() + } + + repo, errRepo := models.GetRepositoryByID(dataset.RepoID) + if errRepo == nil { + log.Info("repo_url=" + repo.FullName()) + record["repoUrl"] = repo.FullName() + record["avatar"] = repo.RelAvatarLink() + } else { + log.Info("repo err=" + errRepo.Error()) + } + + record["title"] = makeHighLight(Key, dataset.Title) + record["description"] = truncLongText(makeHighLight(Key, dataset.Description), true) + + record["category"] = dataset.Category + record["task"] = dataset.Task + record["download_times"] = dataset.DownloadTimes + record["created_unix"] = dataset.CreatedUnix + record["updated_unix"] = repo.UpdatedUnix + record["updated_html"] = timeutil.TimeSinceUnix(repo.UpdatedUnix, language) + + res.Result = append(res.Result, record) + } +} + +func makeDatasetResult(sRes *elastic.SearchResult, Key string, OnlyReturnNum bool, language string) *SearchRes { + total := sRes.Hits.TotalHits.Value + result := make([]map[string]interface{}, 0) + if !OnlyReturnNum { + for i, hit := range sRes.Hits.Hits { + log.Info("this is dataset query " + fmt.Sprint(i) + " result.") + recordSource := make(map[string]interface{}) + source, err := hit.Source.MarshalJSON() + + if err == nil { + err = json.Unmarshal(source, &recordSource) + if err == nil { + record := make(map[string]interface{}) + record["id"] = hit.Id + userIdStr := recordSource["user_id"].(string) + userId, cerr := strconv.ParseInt(userIdStr, 10, 64) + if cerr == nil { + user, errUser := models.GetUserByID(userId) + if errUser == nil { + record["owerName"] = user.GetDisplayName() + record["avatar"] = user.RelAvatarLink() + } + } + setRepoInfo(recordSource, record) + record["title"] = getLabelValue("title", recordSource, hit.Highlight) + record["category"] = getLabelValue("category", recordSource, hit.Highlight) + if recordSource["description"] != nil { + desc := getLabelValue("description", recordSource, hit.Highlight) + record["description"] = dealLongText(desc, Key, hit.MatchedQueries) + } else { + record["description"] = "" + } + record["file_name"] = getDatasetFileName(getLabelValue("file_name", recordSource, hit.Highlight)) + record["task"] = recordSource["task"] + record["download_times"] = recordSource["download_times"] + record["created_unix"] = recordSource["created_unix"] + setUpdateHtml(record, recordSource["updated_unix"].(string), language) + result = append(result, record) + } else { + log.Info("deal dataset source error," + err.Error()) + } + } else { + log.Info("deal dataset source error," + err.Error()) + } + } + } + returnObj := &SearchRes{ + Total: total, + Result: result, + } + + return returnObj +} + +func getDatasetFileName(fileName string) string { + slices := strings.Split(fileName, "-#,#-") + fileName = strings.Join(slices, ", ") + return fileName +} + +func searchIssueOrPr(ctx *context.Context, TableName string, Key string, Page int, PageSize int, OnlyReturnNum bool, issueOrPr string) { + + /* + 任务，合并请求 ES名称：issue-es-index + 搜索: + name character varying(255) , 标题 + content text, 内容 + comment text, 评论 + 排序: + updated_unix + */ + SortBy := ctx.Query("SortBy") + ascending := ctx.QueryBool("Ascending") + PrivateTotal := ctx.QueryInt("PrivateTotal") + WebTotal := ctx.QueryInt("WebTotal") + language := ctx.Query("language") + if language == "" { + language = "zh-CN" + } + from := (Page - 1) * PageSize + if from == 0 { + WebTotal = 0 + } + resultObj := &SearchRes{} + log.Info("WebTotal=" + fmt.Sprint(WebTotal)) + log.Info("PrivateTotal=" + fmt.Sprint(PrivateTotal)) + resultObj.Result = make([]map[string]interface{}, 0) + isPull := false + if issueOrPr == "t" { + isPull = true + } + + if ctx.User != nil && (from < PrivateTotal || from == 0) { + + log.Info("actor is null?:" + fmt.Sprint(ctx.User == nil)) + issues, count, err := models.SearchPrivateIssueOrPr(Page, PageSize, Key, isPull, ctx.User.ID) + if err != nil { + ctx.JSON(200, "") + return + } + resultObj.PrivateTotal = count + issuesSize := len(issues) + if issuesSize > 0 { + log.Info("Query private repo issue number is:" + fmt.Sprint(issuesSize) + " count=" + fmt.Sprint(count)) + makePrivateIssueOrPr(issues, resultObj, Key, language) + } else { + log.Info("not found private repo issue,keyword=" + Key) + } + if issuesSize >= PageSize { + if WebTotal > 0 { //next page， not first query. + resultObj.Total = int64(WebTotal) + ctx.JSON(200, resultObj) + return + } + } + } else { + resultObj.PrivateTotal = int64(PrivateTotal) + } + + from = from - PrivateTotal + if from < 0 { + from = 0 + } + Size := PageSize - len(resultObj.Result) + + boolQ := elastic.NewBoolQuery() + isIssueQuery := elastic.NewTermQuery("is_pull", issueOrPr) + + if Key != "" { + boolKeyQ := elastic.NewBoolQuery() + log.Info("issue Key=" + Key) + nameQuery := elastic.NewMatchQuery("name", Key).Boost(2).QueryName("f_first") + contentQuery := elastic.NewMatchQuery("content", Key).Boost(1.5).QueryName("f_second") + commentQuery := elastic.NewMatchQuery("comment", Key).Boost(1).QueryName("f_third") + boolKeyQ.Should(nameQuery, contentQuery, commentQuery) + boolQ.Must(isIssueQuery, boolKeyQ) + } else { + boolQ.Must(isIssueQuery) + } + + res, err := client.Search(TableName).Query(boolQ).SortBy(getSort(SortBy, ascending)).From(from).Size(Size).Highlight(queryHighlight("name", "content", "comment")).Do(ctx.Req.Context()) + if err == nil { + searchJson, _ := json.Marshal(res) + log.Info("searchJson=" + string(searchJson)) + esresult := makeIssueResult(res, Key, OnlyReturnNum, language) + + resultObj.Total = resultObj.PrivateTotal + esresult.Total + log.Info("query issue es count=" + fmt.Sprint(esresult.Total) + " total=" + fmt.Sprint(resultObj.Total)) + resultObj.Result = append(resultObj.Result, esresult.Result...) + ctx.JSON(200, resultObj) + } else { + log.Info("query es error," + err.Error()) + } +} + +func queryHighlight(names ...string) *elastic.Highlight { + re := elastic.NewHighlight() + for i := 0; i < len(names); i++ { + field := &elastic.HighlighterField{ + Name: names[i], + } + re.Fields(field) + } + re.PreTags("") + re.PostTags("") + return re +} + +func setRepoInfo(recordSource map[string]interface{}, record map[string]interface{}) { + repoIdstr := recordSource["repo_id"].(string) + repoId, cerr := strconv.ParseInt(repoIdstr, 10, 64) + if cerr == nil { + repo, errRepo := models.GetRepositoryByID(repoId) + if errRepo == nil { + log.Info("repo_url=" + repo.FullName()) + record["repoUrl"] = repo.FullName() + record["avatar"] = repo.RelAvatarLink() + } else { + log.Info("repo err=" + errRepo.Error()) + } + } else { + log.Info("parse int err=" + cerr.Error()) + } +} + +func makePrivateIssueOrPr(issues []*models.Issue, res *SearchRes, Key string, language string) { + for _, issue := range issues { + record := make(map[string]interface{}) + record["id"] = issue.ID + record["repo_id"] = issue.RepoID + + repo, errRepo := models.GetRepositoryByID(issue.RepoID) + if errRepo == nil { + log.Info("repo_url=" + repo.FullName()) + record["repoUrl"] = repo.FullName() + record["avatar"] = repo.RelAvatarLink() + } else { + log.Info("repo err=" + errRepo.Error()) + } + record["name"] = makeHighLight(Key, issue.Title) + record["content"] = truncLongText(makeHighLight(Key, issue.Content), true) + + if issue.IsPull { + pr, err1 := issue.GetPullRequest() + if err1 == nil && pr != nil { + record["pr_id"] = pr.ID + } + } + record["index"] = issue.Index + record["num_comments"] = issue.NumComments + record["is_closed"] = issue.IsClosed + record["updated_unix"] = issue.UpdatedUnix + record["updated_html"] = timeutil.TimeSinceUnix(issue.UpdatedUnix, language) + res.Result = append(res.Result, record) + } +} + +func makeIssueResult(sRes *elastic.SearchResult, Key string, OnlyReturnNum bool, language string) *SearchRes { + total := sRes.Hits.TotalHits.Value + result := make([]map[string]interface{}, 0) + if !OnlyReturnNum { + for i, hit := range sRes.Hits.Hits { + log.Info("this is issue query " + fmt.Sprint(i) + " result.") + recordSource := make(map[string]interface{}) + source, err := hit.Source.MarshalJSON() + + if err == nil { + err = json.Unmarshal(source, &recordSource) + if err == nil { + record := make(map[string]interface{}) + record["id"] = hit.Id + record["repo_id"] = recordSource["repo_id"] + log.Info("recordSource[\"repo_id\"]=" + fmt.Sprint(recordSource["repo_id"])) + setRepoInfo(recordSource, record) + record["name"] = getLabelValue("name", recordSource, hit.Highlight) + if recordSource["content"] != nil { + desc := getLabelValue("content", recordSource, hit.Highlight) + record["content"] = dealLongText(desc, Key, hit.MatchedQueries) + if _, ok := hit.Highlight["content"]; !ok { + if _, ok_comment := hit.Highlight["comment"]; ok_comment { + desc := getLabelValue("comment", recordSource, hit.Highlight) + record["content"] = trimHrefHtml(dealLongText(desc, Key, hit.MatchedQueries)) + } + } + } else { + if recordSource["comment"] != nil { + desc := getLabelValue("comment", recordSource, hit.Highlight) + record["content"] = dealLongText(desc, Key, hit.MatchedQueries) + } + } + if recordSource["pr_id"] != nil { + record["pr_id"] = recordSource["pr_id"] + } + log.Info("index=" + recordSource["index"].(string)) + record["index"] = recordSource["index"] + record["num_comments"] = recordSource["num_comments"] + record["is_closed"] = recordSource["is_closed"] + record["updated_unix"] = recordSource["updated_unix"] + setUpdateHtml(record, recordSource["updated_unix"].(string), language) + result = append(result, record) + } else { + log.Info("deal issue source error," + err.Error()) + } + } else { + log.Info("deal issue source error," + err.Error()) + } + } + } + returnObj := &SearchRes{ + Total: total, + Result: result, + } + + return returnObj +} diff --git a/services/socketwrap/clientManager.go b/services/socketwrap/clientManager.go old mode 100644 new mode 100755 index 98b0e0aa9..61f356a66 --- a/services/socketwrap/clientManager.go +++ b/services/socketwrap/clientManager.go @@ -10,7 +10,7 @@ import ( "github.com/elliotchance/orderedmap" ) -var opTypes = []int{1, 2, 5, 6, 7, 9, 10, 11, 12, 13, 14, 15, 17, 22, 23, 25, 26, 27, 28, 29, 30} +var opTypes = []int{1, 2, 5, 6, 7, 9, 10, 11, 12, 13, 14, 15, 17, 22, 23, 25, 26, 27, 28, 29, 30, 31} type ClientsManager struct { Clients *orderedmap.OrderedMap diff --git a/templates/admin/cloudbrain/list.tmpl b/templates/admin/cloudbrain/list.tmpl old mode 100644 new mode 100755 index 39b2c21de..6fea2eef7 --- a/templates/admin/cloudbrain/list.tmpl +++ b/templates/admin/cloudbrain/list.tmpl @@ -81,7 +81,7 @@ {{.DisplayJobName}} {{else if eq .JobType "TRAIN"}} - + {{.DisplayJobName}} {{else if eq .JobType "BENCHMARK"}} @@ -155,13 +155,13 @@ {{else}} - + {{$.i18n.Tr "repo.stop"}} {{end}} -
+ {{$.CsrfTokenHtml}} {{$.i18n.Tr "repo.delete"}} diff --git a/templates/base/head_navbar.tmpl b/templates/base/head_navbar.tmpl index 0ce3f2fcf..18cf3ce0e 100755 --- a/templates/base/head_navbar.tmpl +++ b/templates/base/head_navbar.tmpl @@ -95,9 +95,9 @@ {{if .IsSigned}}
- +
- @@ -199,9 +199,9 @@
- +
- diff --git a/templates/base/head_navbar_fluid.tmpl b/templates/base/head_navbar_fluid.tmpl index 9475f8f5f..61326e386 100644 --- a/templates/base/head_navbar_fluid.tmpl +++ b/templates/base/head_navbar_fluid.tmpl @@ -93,9 +93,9 @@ {{if .IsSigned}}
- +
- @@ -196,9 +196,9 @@
- +
- diff --git a/templates/base/head_navbar_pro.tmpl b/templates/base/head_navbar_pro.tmpl index 0c1243fb0..fc9c401ca 100644 --- a/templates/base/head_navbar_pro.tmpl +++ b/templates/base/head_navbar_pro.tmpl @@ -96,9 +96,9 @@ {{if .IsSigned}}
- +
- @@ -199,9 +199,9 @@
- +
- diff --git a/templates/custom/select_dataset.tmpl b/templates/custom/select_dataset.tmpl index 17e4eee42..273477dd2 100644 --- a/templates/custom/select_dataset.tmpl +++ b/templates/custom/select_dataset.tmpl @@ -3,7 +3,11 @@
- + {{if eq .cloudbraintype 0}} + + {{else}} + + {{end}} {{.i18n.Tr "dataset.select_dataset"}} -
\ No newline at end of file +
diff --git a/templates/custom/select_dataset_train.tmpl b/templates/custom/select_dataset_train.tmpl new file mode 100644 index 000000000..2771200b6 --- /dev/null +++ b/templates/custom/select_dataset_train.tmpl @@ -0,0 +1,134 @@ + + +
+     + + + {{.i18n.Tr "dataset.select_dataset"}} + +
+ + +
+ + + +
+
+
${dataset.Repo.OwnerName}/${dataset.Repo.Alias} ${dataset.Name}
+
+ + + + ${dataset.Description} +
+
+
+ + + + 解压中 + + + + 解压失败 + +
+
+ + + + +
+ +
+ + + + 解压中 + + + + 解压失败 + +
+
+ + + +
+
+
${dataset.Repo.OwnerName}/${dataset.Repo.Alias}${dataset.Name}
+
+ + + + ${dataset.Description} +
+
+
+ + + + 解压中 + + + + 解压失败 + +
+
+ + + +
+
+
${dataset.Repo.OwnerName}/${dataset.Repo.Alias}${dataset.Name}
+
+ + + + ${dataset.Description} +
+
+
+ + + + 解压中 + + + + 解压失败 + +
+
+ + + +
+ + +
+ + + +
diff --git a/templates/explore/datasets.tmpl b/templates/explore/datasets.tmpl index ff18a3da3..a1692cf35 100644 --- a/templates/explore/datasets.tmpl +++ b/templates/explore/datasets.tmpl @@ -121,10 +121,10 @@
- {{.i18n.Tr "repo.issues.filter_sort.latest"}} - {{.i18n.Tr "repo.issues.filter_sort.oldest"}} - {{.i18n.Tr "repo.issues.filter_sort.recentupdate"}} - {{.i18n.Tr "repo.issues.filter_sort.leastupdate"}} + {{.i18n.Tr "repo.issues.filter_sort.latest"}} + {{.i18n.Tr "repo.issues.filter_sort.oldest"}} + {{.i18n.Tr "repo.issues.filter_sort.recentupdate"}} + {{.i18n.Tr "repo.issues.filter_sort.leastupdate"}}
@@ -184,9 +184,15 @@
+ {{if eq .UserID 0}} + {{else}} + + + + {{end}} 创建于：{{TimeSinceUnix1 .CreatedUnix}}
@@ -215,4 +221,4 @@
-{{template "base/footer" .}} +{{template "base/footer" .}} \ No newline at end of file diff --git a/templates/explore/organizations.tmpl b/templates/explore/organizations.tmpl index 1151c5a94..58615a8e5 100644 --- a/templates/explore/organizations.tmpl +++ b/templates/explore/organizations.tmpl @@ -1,68 +1,251 @@ + + + + + + {{template "base/head" .}}
- {{template "explore/search" .}} - -
-
- {{template "explore/navbar" .}} -
-

- {{.i18n.Tr "explore.organizations"}} -

-
- -
- - {{.i18n.Tr "repo.issues.filter_sort"}} - - -
- {{.i18n.Tr "repo.issues.filter_sort.latest"}} - {{.i18n.Tr "repo.issues.filter_sort.oldest"}} - {{.i18n.Tr "repo.issues.label.filter_sort.alphabetically"}} - {{.i18n.Tr "repo.issues.label.filter_sort.reverse_alphabetically"}} - {{.i18n.Tr "repo.issues.filter_sort.recentupdate"}} - {{.i18n.Tr "repo.issues.filter_sort.leastupdate"}} -
+ +
+
+
+
+
+ + +
+
+
+
-
- -
- {{range .Users}} -
- -
- - {{.Name}} {{.FullName}} - {{if .Visibility.IsPrivate}} - {{svg "octicon-lock" 16}} - {{end}} - -
- {{if .Location}} - {{svg "octicon-location" 16}} {{.Location}} - {{end}} - {{if and .Website}} - {{svg "octicon-link" 16}} - {{.Website}} - {{end}} - {{svg "octicon-clock" 16}} {{$.i18n.Tr "user.join_on"}} {{.CreatedUnix.FormatShort}} -
-
+
+
+

+ {{$.i18n.Tr "home.page_recommend_org"}} +

+

{{$.i18n.Tr "home.page_recommend_org_desc"}} {{$.i18n.Tr "home.page_recommend_org_commit"}}

+
+ +
+
+ - - {{template "base/paginate" .}} +
+
-
- {{template "explore/repo_right" .}} +
+
+
+
+
+
+
+

{{$.i18n.Tr "org.star"}}

+
+
  • + {{ range $i,$user :=.StarOrgs}} +
      + {{if eq $i 0}} +
      + +
      + {{else if eq $i 1}} +
      + +
      + {{else if eq $i 2}} +
      + +
      + {{else }} +
      + {{Add $i 1}} +
      + {{end}} +
    • + +
      • +
    • +

      {{$user.Name}}

      +
      • +
        +
      • +  

        {{$user.Score}}

        +
        • +
      + +
    + {{end}} +
    • +
    +
    +
    +

    {{$.i18n.Tr "org.member"}}

    +
    +
  • + {{ range $i,$user :=.StarOrgs}} +
      + {{if eq $i 0}} +
      + +
      + {{else if eq $i 1}} +
      + +
      + {{else if eq $i 2}} +
      + +
      + {{else }} +
      + {{Add $i 1}} +
      + {{end}} +
    • + +
      • +
    • +

      {{$user.Name}}

      +
      • +
        +
      • +  

        {{$user.Score}}

        +
        • +
      + +
    + {{end}} +
    • +
    +
    +
    +

    {{$.i18n.Tr "org.active"}}

    +
    +
  • + {{ range $i,$user :=.StarOrgs}} +
      + {{if eq $i 0}} +
      + +
      + {{else if eq $i 1}} +
      + +
      + {{else if eq $i 2}} +
      + +
      + {{else }} +
      + {{Add $i 1}} +
      + {{end}} +
    • + +
      • +
    • +

      {{$user.Name}}

      +
      • + +
    + {{end}} +
    • +
    + + +
    + {{template "base/footer" .}} + + diff --git a/templates/explore/repos.tmpl b/templates/explore/repos.tmpl index df305099a..23170b3d6 100644 --- a/templates/explore/repos.tmpl +++ b/templates/explore/repos.tmpl @@ -10,7 +10,19 @@
    {{template "explore/repo_list" .}} - {{template "base/paginate" .}} +
    +
    + + +
    +
    {{template "explore/repo_right" .}} diff --git a/templates/explore/search_new.tmpl b/templates/explore/search_new.tmpl new file mode 100644 index 000000000..7d099f3b8 --- /dev/null +++ b/templates/explore/search_new.tmpl @@ -0,0 +1,95 @@ +{{template "base/head" .}} + +
    +
    +
    +
    +
    +
    + + + + + +
    +
    + +
    + +
    + + + +
    +
    + + {{.i18n.Tr "home.search_repo"}} + + + + {{.i18n.Tr "home.search_dataset"}} + + + + {{.i18n.Tr "home.search_issue"}} + + + + {{.i18n.Tr "home.search_pr"}} + + + + {{.i18n.Tr "home.search_user"}} + + + + {{.i18n.Tr "home.search_org"}} + +
    +
    +
    + +
    + + {{.i18n.Tr "home.search_finded"}} + +
    + +
    + + {{.i18n.Tr "repo.issues.filter_sort"}} + + + +
    + +
    + +
    + +
    + + +
    + + +
    + +
    +
    +
    + +
    + + +
    + + + +{{template "base/footer" .}} \ No newline at end of file diff --git a/templates/repo/cloudbrain/benchmark/index.tmpl b/templates/repo/cloudbrain/benchmark/index.tmpl index 4e7d5b4e5..b840034e6 100755 --- a/templates/repo/cloudbrain/benchmark/index.tmpl +++ b/templates/repo/cloudbrain/benchmark/index.tmpl @@ -185,7 +185,19 @@
    --> - {{template "base/paginate" .}} +
    +
    + + +
    +
    diff --git a/templates/repo/cloudbrain/benchmark/show.tmpl b/templates/repo/cloudbrain/benchmark/show.tmpl index 3c8f91528..16e5a98f5 100755 --- a/templates/repo/cloudbrain/benchmark/show.tmpl +++ b/templates/repo/cloudbrain/benchmark/show.tmpl @@ -196,7 +196,13 @@ td, th {
    - {{TimeSinceUnix1 .CreatedUnix}} + + {{if not (eq .StartTime 0)}} + {{TimeSinceUnix1 .StartTime}} + {{else}} + {{TimeSinceUnix1 .CreatedUnix}} + {{end}} + {{$.i18n.Tr "repo.modelarts.status"}}： {{.Status}} @@ -252,7 +258,13 @@ td, th {
    - {{TimeSinceUnix1 .CreatedUnix}} + + {{if not (eq .StartTime 0)}} + {{TimeSinceUnix1 .StartTime}} + {{else}} + {{TimeSinceUnix1 .CreatedUnix}} + {{end}} +
    @@ -269,7 +281,7 @@ td, th { - 镜像 + {{.i18n.Tr "cloudbrain.mirror"}} @@ -280,7 +292,8 @@ td, th { - 类型 + {{.i18n.Tr "repo.cloudbrain.benchmark.evaluate_type"}} + @@ -301,7 +314,7 @@ td, th { - 训练程序 + {{.i18n.Tr "repo.cloudbrain.benchmark.evaluate_train"}} @@ -312,7 +325,7 @@ td, th { - 测试程序 + {{.i18n.Tr "repo.cloudbrain.benchmark.evaluate_test"}} @@ -348,7 +361,7 @@ td, th { - 创建者 + {{$.i18n.Tr "repo.cloudbrain_creator"}} @@ -359,7 +372,7 @@ td, th { - 子类型 + {{$.i18n.Tr "repo.cloudbrain.benchmark.evaluate_child_type"}} diff --git a/templates/repo/cloudbrain/show.tmpl b/templates/repo/cloudbrain/show.tmpl index d1e736c20..781c66d67 100755 --- a/templates/repo/cloudbrain/show.tmpl +++ b/templates/repo/cloudbrain/show.tmpl @@ -1,100 +1,501 @@ {{template "base/head" .}} + +
    +
    +
    +
    +
    +
    +
    +
    +
    {{template "repo/header" .}} -
    -
    - {{template "base/alert" .}} - +

    -
    - - {{.i18n.Tr "repo.cloudbrain"}} - -
    /
    - - {{$.i18n.Tr "repo.modelarts.notebook"}} - -
    /
    - {{with .task}} -
    {{.DisplayJobName}}
    - {{end}} -
    +
    + + {{.i18n.Tr "repo.cloudbrain"}} + +
    /
    + + {{$.i18n.Tr "repo.modelarts.notebook"}} + +
    /
    +
    {{.displayJobName}}
    +

    -
    -
    - {{with .task}} -

    任务名称: {{.DisplayJobName}}

    - {{end}} -
    -
    -

    任务结果:

    - {{with .taskRes}} - {{range .TaskStatuses}} - - - - - - - -
    状态 {{.State}}
    - {{end}} - {{end}} -
    -
    - {{with .result}} - - - - - - - - - - - - - - - - - - -
    硬件信息
    CPU {{.Resource.CPU}}
    Memory {{.Resource.Memory}}
    NvidiaComGpu {{.Resource.NvidiaComGpu}}
    - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    调试信息
    平台 {{.Platform}}
    开始时间 {{.JobStatus.StartTime}}
    结束时间 {{.JobStatus.EndTime}}
    ExitCode {{.JobStatus.AppExitCode}}
    退出信息 {{.JobStatus.AppExitDiagnostics | nl2br}}
    - {{end}} -
    + {{range $k ,$v := .version_list_task}} +
    + +
    +
    +
    + + + +
    + + {{if not (eq .StartTime 0)}} + {{TimeSinceUnix1 .StartTime}} + {{else}} + {{TimeSinceUnix1 .CreatedUnix}} + {{end}} + + + {{$.i18n.Tr "repo.modelarts.status"}}： + {{.Status}} + + {{$.i18n.Tr "repo.modelarts.train_job.dura_time"}}： + {{$.duration}} + +
    +     +     +
    +
    +
    +
    +
    +
    + {{$.i18n.Tr "repo.modelarts.train_job.config"}} +
    +
    +
    +
    +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + {{$.i18n.Tr "repo.cloudbrain_task"}} + +
    + {{.DisplayJobName}} +
    +
    + {{$.i18n.Tr "repo.modelarts.status"}} + +
    + {{.Status}} +
    +
    + {{$.i18n.Tr "repo.modelarts.train_job.start_time"}} + +
    + + {{if not (eq .StartTime 0)}} + {{TimeSinceUnix1 .StartTime}} + {{else}} + {{TimeSinceUnix1 .CreatedUnix}} + {{end}} + +
    +
    + {{$.i18n.Tr "repo.modelarts.train_job.dura_time"}} + +
    + {{$.duration}} +
    +
    + {{$.i18n.Tr "cloudbrain.mirror"}} + +
    + {{.Image}} +
    +
    + {{$.i18n.Tr "cloudbrain.dataset_storage_path"}} + +
    + {{$.dataset_path}} +
    +
    + {{$.i18n.Tr "cloudbrain.code_storage_path"}} + +
    + {{$.code_path}} +
    +
    +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + {{$.i18n.Tr "cloudbrain.gpu_type"}} + +
    + {{$.resource_type}} +
    +
    + ExitCode + +
    + {{$.result.JobStatus.AppExitCode}} +
    +
    + {{$.i18n.Tr "repo.cloudbrain.endtime"}} + +
    + {{if not (eq .EndTime 0)}} + {{TimeSinceUnix1 .EndTime}} + {{else}} + -- + {{end}} +
    +
    + {{$.i18n.Tr "repo.modelarts.train_job.dataset"}} + +
    + {{$.datasetname}} +
    +
    + {{$.i18n.Tr "repo.modelarts.train_job.standard"}} + +
    + {{$.i18n.Tr "cloudbrain.gpu_num"}}：{{$.GpuNum}}，{{$.i18n.Tr "cloudbrain.cpu_num"}}：{{$.CpuNum}}，{{$.i18n.Tr "cloudbrain.memory"}}(MB)：{{$.MemMiB}}，{{$.i18n.Tr "cloudbrain.shared_memory"}}(MB)：{{$.ShareMemMiB}} +
    +
    + {{$.i18n.Tr "repo.cloudbrain_creator"}} + +
    + {{.User.Name}} +
    +
    + {{$.i18n.Tr "cloudbrain.model_storage_path"}} + +
    + {{$.model_path}} +
    +
    +
    +
    +
    +
    + {{$.i18n.Tr "repo.cloudbrain.exitinfo"}} + +
    +
    +
    + + {{$.result.JobStatus.AppExitDiagnostics}} + +
    +
    + +
    +
    + + +
    +
    + {{end}} {{template "base/paginate" .}} +
    + +
    +
    +
    + {{.i18n.Tr "cloudbrain.delete_task"}} +
    +
    +

    {{.i18n.Tr "cloudbrain.task_delete_confirm"}}

    +
    +
    +
    + {{.i18n.Tr "cloudbrain.operate_cancel"}} +
    +
    + {{.i18n.Tr "cloudbrain.operate_confirm"}} +
    +
    +
    -
    + +
    {{template "base/footer" .}} + + diff --git a/templates/repo/cloudbrain/trainjob/new.tmpl b/templates/repo/cloudbrain/trainjob/new.tmpl new file mode 100755 index 000000000..e7254e410 --- /dev/null +++ b/templates/repo/cloudbrain/trainjob/new.tmpl @@ -0,0 +1,443 @@ +{{template "base/head" .}} + + +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    + {{template "repo/header" .}} +
    + {{template "base/alert" .}} +

    + {{.i18n.Tr "repo.modelarts.train_job.new"}} +

    +
    + + + {{.CsrfTokenHtml}} + + + +

    {{.i18n.Tr "repo.modelarts.train_job.basic_info"}}:

    +
    + +
    + + + + + + CPU/GPU + + + + + + + Ascend NPU +
    +
    +
    + + + {{.i18n.Tr "cloudbrain.job_name_rule"}} +
    + +
    + + +
    +
    + +

    {{.i18n.Tr "repo.modelarts.train_job.parameter_setting"}}:

    + + +
    + + +
    + + + +
    + + +
    + +
    + + + + + {{range .images}} + + {{end}} + {{range .public_images}} + + {{end}} + +
    + +
    + + {{if .bootFile}} + + {{else}} + + {{end}} + + + + 查看样例 +
    + + + {{template "custom/select_dataset_train" .}} + 训练脚本存储在/code中，数据集存储在/dataset中，训练输出请存储在/model中以供后续下载。 +
    + + {{.i18n.Tr "repo.modelarts.train_job.add_run_parameter"}} + +
    + {{if .params}} + {{if ne 0 (len .params)}} + {{range $k ,$v := .params}} +
    +
    + +
    +
    + +
    + + + + +
    + {{end}} + {{end}} + {{end}} +
    +
    + +
    + + +
    + +
    + + {{.i18n.Tr "repo.cloudbrain.cancel"}} +
    + + + + +
    +
    +
    +{{template "base/footer" .}} + + \ No newline at end of file diff --git a/templates/repo/cloudbrain/trainjob/show.tmpl b/templates/repo/cloudbrain/trainjob/show.tmpl new file mode 100755 index 000000000..324c19ac6 --- /dev/null +++ b/templates/repo/cloudbrain/trainjob/show.tmpl @@ -0,0 +1,653 @@ +{{template "base/head" .}} + +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +{{template "repo/header" .}} +
    +

    +
    + + {{.i18n.Tr "repo.cloudbrain"}} + +
    /
    + + {{$.i18n.Tr "repo.modelarts.train_job"}} + +
    /
    +
    {{.displayJobName}}
    +
    +

    + {{range $k ,$v := .version_list_task}} +
    + +
    +
    +
    + + + +
    + {{TimeSinceUnix1 .CreatedUnix}} + + {{$.i18n.Tr "repo.modelarts.status"}}： + {{.Status}} + + {{$.i18n.Tr "repo.modelarts.train_job.dura_time"}}： + {{$.duration}} + +
    +     +     +
    +
    +
    +
    +
    +
    + {{$.i18n.Tr "repo.modelarts.train_job.config"}} + {{$.i18n.Tr "repo.modelarts.log"}} + {{$.i18n.Tr "repo.model_download"}} +
    +
    +
    +
    +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + {{$.i18n.Tr "repo.cloudbrain_task"}} + +
    + {{.DisplayJobName}} +
    +
    + {{$.i18n.Tr "repo.modelarts.status"}} + +
    + {{.Status}} +
    +
    + {{$.i18n.Tr "repo.modelarts.train_job.start_time"}} + +
    + {{TimeSinceUnix1 .CreatedUnix}} +
    +
    + {{$.i18n.Tr "repo.modelarts.train_job.dura_time"}} + +
    + {{$.duration}} +
    +
    + {{$.i18n.Tr "repo.modelarts.train_job.resource_type"}} + +
    + {{$.resource_type}} +
    +
    + {{$.i18n.Tr "repo.modelarts.train_job.standard"}} + +
    + {{$.resource_spec}} +
    +
    +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + 镜像 + +
    + {{.Image}} +
    +
    + {{$.i18n.Tr "repo.modelarts.code_version"}} + +
    + {{.BranchName}} +
    +
    + {{$.i18n.Tr "repo.modelarts.train_job.start_file"}} + +
    + {{.BootFile}} +
    +
    + {{$.i18n.Tr "repo.modelarts.train_job.train_dataset"}} + +
    + {{.DatasetName}} +
    +
    + {{$.i18n.Tr "repo.modelarts.train_job.run_parameter"}} + +
    + {{.Parameters}} +
    +
    + {{$.i18n.Tr "repo.modelarts.train_job.description"}} + +
    + {{.Description}} +
    +
    +
    +
    +
    + +
    +
    +
    +
    + +
    + + + 
    
+                            
    
+                                        
+                        
    
+                        
+                    
    
+
+                    
    
+                        
+                        
+                        
+                        
    
+                            
+                        
    
+                    
    
+                    
+                
    
+            
    
+        
    
+        {{end}} {{template "base/paginate" .}}
+    
    
+    
+    
    
+        
    
+            
    
+                 删除任务
+            
    
+    
+            
    
+                
    你确认删除该任务么？此任务一旦删除不可恢复。
    
+            
    
+            
    
+                
    
+                     取消操作
+                
    
+                
    
+                     确定操作
+                
    
+            
    
+        
    
+    
    
+    
+    
+
    
+{{template "base/footer" .}}
+
+
\ No newline at end of file
diff --git a/templates/repo/datasets/index.tmpl b/templates/repo/datasets/index.tmpl
index a38e86525..e8d9b65e3 100755
--- a/templates/repo/datasets/index.tmpl
+++ b/templates/repo/datasets/index.tmpl
@@ -131,7 +131,7 @@
                 {{.i18n.Tr "repo.modelarts.modify"}}
               
           
    
-          {{if and (.dataset.Category) (.dataset.Task) (.dataset.License)}}
+          {{if or (.dataset.Category) (.dataset.Task) (.dataset.License)}}
           
    
               {{if .dataset.Category}}
               {{$category := .dataset.Category}}
@@ -199,7 +199,7 @@
                   
    
               
    
               {{range $k, $v :=.Attachments}}
-              
    
+              
    
                   
    
                       
 
@@ -266,8 +266,8 @@
                               {{$.i18n.Tr "preview"}}
                               {{end}}
                               {{if and (.CanDel) (not $.Repository.IsPrivate)}}
-                              {{$.i18n.Tr "dataset.set_public"}}
-                              {{$.i18n.Tr "dataset.set_private"}}
+                              {{$.i18n.Tr "dataset.set_public"}}
+                              {{$.i18n.Tr "dataset.set_private"}}
                               {{end}}
                               
-                
    
-                        
-                    
-                    
-                        
-                    
-                
    
-                
+                {{template "custom/select_dataset_train" .}}
+                {{.i18n.Tr "cloudbrain.dataset_path_rule"}}
                 
    
-                         
+                        
                         {{if .bootFile}}
                             
                         {{else}}
diff --git a/templates/repo/modelarts/inferencejob/show.tmpl b/templates/repo/modelarts/inferencejob/show.tmpl
index e11919b71..2e7a13eb1 100644
--- a/templates/repo/modelarts/inferencejob/show.tmpl
+++ b/templates/repo/modelarts/inferencejob/show.tmpl
@@ -232,7 +232,13 @@ td, th {
 
                                                 
                                                     
    
-                                                        {{TimeSinceUnix1 .CreatedUnix}}
+                                                        
+                                                        {{if not (eq .StartTime 0)}}
+															 {{TimeSinceUnix1 .StartTime}}
+															{{else}}
+															{{TimeSinceUnix1 .CreatedUnix}}
+															{{end}}
+                                                        
                                                     
    
                                                 
                                             
diff --git a/templates/repo/modelarts/notebook/new.tmpl b/templates/repo/modelarts/notebook/new.tmpl
index a66690f43..61ece0f8d 100755
--- a/templates/repo/modelarts/notebook/new.tmpl
+++ b/templates/repo/modelarts/notebook/new.tmpl
@@ -59,21 +59,7 @@
                             {{end}}
                         
                     
    
-<<<<<<< HEAD
                     {{template "custom/select_dataset" .}}
-=======
-
-                    
    
-                        
-                        
-                        
-                            {{range .attachments}}
-                                
-                            {{end}}
-                        
-                        
-                    
    
->>>>>>> V20220328
 
                     
+    
    
+        
    
+            
    
+                 {{.i18n.Tr "cloudbrain.delete_task"}}
+            
    
 
+            
    
+                
    {{.i18n.Tr "cloudbrain.task_delete_confirm"}}
    
+            
    
+            
    
+                
    
+                     {{.i18n.Tr "cloudbrain.operate_cancel"}}
+                
    
+                
    
+                     {{.i18n.Tr "cloudbrain.operate_confirm"}}
+                
    
+            
    
+        
    
     
    
-  
    
+
+
 
    
 {{template "base/footer" .}}
+
+
diff --git a/templates/repo/modelarts/trainjob/index.tmpl b/templates/repo/modelarts/trainjob/index.tmpl
index 19c6b3841..d27ee66fd 100755
--- a/templates/repo/modelarts/trainjob/index.tmpl
+++ b/templates/repo/modelarts/trainjob/index.tmpl
@@ -39,8 +39,18 @@
                     
    
                 
    
                 
    
+                    
    
+                        {{svg "octicon-server" 16}}
+                        
    
+                        
+                        
    
+                            
    {{$.i18n.Tr "repo.gpu_type_all"}}
    
+                            
    CPU/GPU
    
+                            
    NPU
    
+                        
    
+                    
    
                     {{if .Permission.CanWrite $.UnitTypeCloudBrain}}
-                    {{$.i18n.Tr "repo.modelarts.train_job.new_train"}}
+                    {{$.i18n.Tr "repo.modelarts.train_job.new_train"}}
                     {{else}}
                     {{$.i18n.Tr "repo.modelarts.train_job.new_train"}}
                     {{end}}
@@ -102,7 +112,7 @@
 
                                 
                                 
    
-                                    
+                                    
 
                                         {{.DisplayJobName}}
                                     
@@ -143,7 +153,7 @@
                                     
    
                                         {{$.CsrfTokenHtml}}
                                         {{if .CanDel}}
-                                        
+                                        
                                             {{$.i18n.Tr "repo.stop"}}
                                         
                                         {{else}}
@@ -154,7 +164,8 @@
 
                                     
    
                                     
-                                    
    
+                                    
+                                        
                                         {{$.CsrfTokenHtml}}
                                         {{if .CanDel}}
                                         
@@ -169,7 +180,20 @@
                                 
    
                             
    
                         
    
-                        {{end}} {{template "base/paginate" .}}
+                        {{end}}
+                        
    
+                            
    
+                                
+                                
+                            
    
+                        
    
                     
    
 
                 
    
@@ -206,3 +230,27 @@
 
    
 
    
 {{template "base/footer" .}}
+
+
diff --git a/templates/repo/modelarts/trainjob/new.tmpl b/templates/repo/modelarts/trainjob/new.tmpl
index afa09cf29..3629de442 100755
--- a/templates/repo/modelarts/trainjob/new.tmpl
+++ b/templates/repo/modelarts/trainjob/new.tmpl
@@ -78,6 +78,24 @@
                 
                 
                 
    {{.i18n.Tr "repo.modelarts.train_job.basic_info"}}:
    
+                
    
+                    
+                    
    
+                        
+                            
+                                
+                                
+                            
+                            CPU/GPU
+                        
+                        
+                            
+                                
+                                
+                            
+                            Ascend NPU
+                    
    
+                
    
                 
    
                     
                     
@@ -140,29 +158,18 @@
                 
    
                         
                         {{if .bootFile}}
-                            
+                            
                         {{else}}
-                            
+                            
                         {{end}}
                         
                             
                         
                         {{.i18n.Tr "cloudbrain.view_sample"}}
                 
    
-                
    
-                    
-                    
-                    {{.i18n.Tr "cloudbrain.dataset_path_rule"}}
-                
    
-
+                
+                {{template "custom/select_dataset_train" .}}
+                {{.i18n.Tr "cloudbrain.dataset_path_rule"}}
                 
    
                     
                     {{.i18n.Tr "repo.modelarts.train_job.add_run_parameter"}}
@@ -395,7 +402,7 @@
                     identifier  : 'display_job_name',
                     rules: [
                     {
-                        type: 'regExp[/^[a-zA-Z0-9-_]{1,64}[^-]$/]',
+                        type: 'regExp[/^[a-zA-Z0-9-_]{1,64}[a-zA-Z0-9_]$/]',
                     }
                     ]
                 },
diff --git a/templates/repo/modelarts/trainjob/show.tmpl b/templates/repo/modelarts/trainjob/show.tmpl
index a75ee5dad..0b420c484 100755
--- a/templates/repo/modelarts/trainjob/show.tmpl
+++ b/templates/repo/modelarts/trainjob/show.tmpl
@@ -223,7 +223,12 @@ td, th {
                                 
    
                                 
    
 
-                                    {{TimeSinceUnix1 .Cloudbrain.CreatedUnix}}
+                                    
+                                      {{if not (eq .Cloudbrain.StartTime 0)}}
+                                    	{{TimeSinceUnix1 .Cloudbrain.StartTime}}
+										{{else}}
+										{{TimeSinceUnix1 .Cloudbrain.CreatedUnix}}
+										{{end}}
                                      {{$.i18n.Tr "repo.modelarts.current_version"}}：{{.VersionName}}
                                       {{$.i18n.Tr "repo.modelarts.parent_version"}}：{{.PreVersionName}}
                                     {{$.i18n.Tr "repo.modelarts.status"}}：
@@ -293,7 +298,12 @@ td, th {
 
                                                     
                                                         
    
-                                                            {{TimeSinceUnix1 .Cloudbrain.CreatedUnix}}
+                                                            
+                                                              {{if not (eq .Cloudbrain.StartTime 0)}}
+                                                                {{TimeSinceUnix1 .Cloudbrain.StartTime}}
+																{{else}}
+																{{TimeSinceUnix1 .Cloudbrain.CreatedUnix}}
+																{{end}}
                                                         
    
                                                     
                                                 
diff --git a/templates/repo/view_file.tmpl b/templates/repo/view_file.tmpl
index a0cae31f3..a559dd611 100755
--- a/templates/repo/view_file.tmpl
+++ b/templates/repo/view_file.tmpl
@@ -40,7 +40,7 @@
 			
    
 				{{if .Repository.CanEnableEditor}}
 					{{if .CanEditFile}}
-						{{svg "octicon-pencil" 16}}
+						{{svg "octicon-pencil" 16}}
 					{{else}}
 						{{svg "octicon-pencil" 16}}
 					{{end}}
@@ -48,7 +48,7 @@
 			
    
 		
    
 		{{end}}
-		
+
 		{{if not .ReadmeInList}}
 		
    
 			
    
diff --git a/templates/user/dashboard/feeds.tmpl b/templates/user/dashboard/feeds.tmpl
old mode 100644
new mode 100755
index 9724de269..614c328c6
--- a/templates/user/dashboard/feeds.tmpl
+++ b/templates/user/dashboard/feeds.tmpl
@@ -77,13 +77,15 @@
 						{{else if eq .GetOpType 26}}
 							{{$.i18n.Tr "action.task_npudebugjob" .GetRepoLink .Content .RefName | Str2html}}
 						{{else if eq .GetOpType 27}}
-							{{$.i18n.Tr "action.task_trainjob" .GetRepoLink .Content .RefName | Str2html}}
+							{{$.i18n.Tr "action.task_nputrainjob" .GetRepoLink .Content .RefName | Str2html}}
 						{{else if eq .GetOpType 28}}
 							{{$.i18n.Tr "action.task_inferencejob" .GetRepoLink .Content .RefName | Str2html}}
 						{{else if eq .GetOpType 29}}
 							{{$.i18n.Tr "action.task_benchmark" .GetRepoLink .Content .RefName | Str2html}}
 						{{else if eq .GetOpType 30}}
 							{{$.i18n.Tr "action.task_createmodel" .GetRepoLink .RefName .RefName | Str2html}}
+						{{else if eq .GetOpType 31}}
+							{{$.i18n.Tr "action.task_gputrainjob" .GetRepoLink .Content .RefName | Str2html}}
 						{{end}}
 					
    
 					{{if or (eq .GetOpType 5) (eq .GetOpType 18)}}
@@ -129,6 +131,8 @@
 				     
 				{{else if eq .GetOpType 30}}
 				     
+				{{else if eq .GetOpType 31}}
+				     
 				{{else}}
                     {{svg (printf "octicon-%s" (ActionIcon .GetOpType)) 32}}
 				{{end}}
diff --git a/web_src/js/components/ProAnalysis.vue b/web_src/js/components/ProAnalysis.vue
index d92eb6df9..bdc874c27 100755
--- a/web_src/js/components/ProAnalysis.vue
+++ b/web_src/js/components/ProAnalysis.vue
@@ -150,21 +150,21 @@
                         align="center">
                     
                     
                         
-                      
+                            {{scope.row.isFork|changeType}}                      
+                        
+                     
                     
                         
-                      
+                       
                     
                         {{scope.row.RegistDate | transformTimestamp}}
                     
+                 
+                
                    
+                
+                    
+                 
+                
+                 
+                
+                 
+                
+                    
+                 
                 .item:first-child, .ui.pagination.menu .item:last-child {
+    background-color: #FFF !important;
+}
+.ui.ui.menu .item.disabled{
+    background-color: #F2F2F2;
+}
+.ui.pagination.menu .active.item {
+    background-color: #3291F8;
+    color: #FFF;
+}
+.ui.pagination.menu .item>.input {
+    margin: 0em .5em;
+    width: 3em;
+    height: 32px;
+}
 @media only screen and (max-width: 767px) {
 	.following.bar #navbar .brand{
 		padding-top: 6px;
@@ -783,4 +809,170 @@ display: block;
 	border: none !important;
 	color: #0366d6 !important;
 	box-shadow: -15px 0px 10px #fff;
+}
+
+.content_top10{
+	padding:10px;
+	padding-top:0px;
+}
+.re_con{
+	color: rgba(136, 136, 136, 100);
+	font-size: 14px;
+	text-align: center;
+	font-family: SourceHanSansSC-light;
+}
+.title_re{
+   margin-top: 50px !important;
+}
+.card_list {
+	width: calc(33.33333333333333% - 2em);
+	margin-left: 1em;
+	margin-right: 1em
+}
+.list_title{
+	height: 52px;
+	text-align: center
+}
+.star_title{
+	background-color: #3291F8;
+}
+.memb_title{
+	background-color: #706FE3;
+}
+.act_title{
+	background-color: #13C28D;
+}	
+.p_text{
+	line-height: 50px;
+	text-align:left;
+	padding-left:15px;
+	font-size:18px;
+	color:#FFFFFF;
+}
+.orgs {
+	display: flex;
+	flex-flow: row wrap;
+	padding: 0;
+	margin-top:20px
+}
+.orgs li {
+	display: flex;
+	border-bottom: 0!important;
+	padding: 3px!important;
+}
+.p_score{	
+	line-height: 28px;
+	width: 100%;
+	
+	/* padding-right: 20px; */
+	text-align: right;
+}
+.org_line_hight{
+	line-height: 28px;
+}
+.org_icon{
+	margin-top: 10px;
+	margin-right: 10px;
+	padding-left: 15px;
+}
+.org_icon_num{
+	margin-left: 2px;
+	margin-right: 13px;
+}
+.org_icon_color{
+	color: #FA8C16;
+}
+.li_name{
+	list-style:none;
+	width: 55%;
+}
+.li_avatar{
+	list-style: none;
+	width: 10%;
+}
+.li_score{
+	list-style:none;
+	margin-left: 2px;
+}
+/**seach**/
+/**搜索导航条适配窄屏**/
+.seachnav{
+	overflow-x: auto;   
+	overflow-y: hidden;  
+	scrollbar-width: none; /* firefox */
+	-ms-overflow-style: none; /* IE 10+ */
+  }
+  .seachnav::-webkit-scrollbar {
+	display: none; /* Chrome Safari */
+  }
+  .ui.green.button, .ui.green.buttons .button{
+	  background-color: #5BB973;
+  }
+  .seach .repos--seach{
+	padding-bottom: 0;
+	border-bottom: none;
+  }
+  .seach .ui.secondary.pointing.menu{
+	border-bottom: none;
+  }
+  .seach .ui.secondary.pointing.menu .item > i{
+	margin-right: 5px;
+  }
+  .seach .ui.secondary.pointing.menu .active.item{
+	border-bottom-width: 2px;
+	margin: 0 0 -1px;
+  }
+  .seach .ui.menu .active.item>.label {
+	background: #1684FC;
+	color: #FFF;
+  }
+  .seach .ui.menu .item>.label:not(.active.item>.label) {
+	background: #e8e8e8;
+	color: rgba(0,0,0,.6);
+  }
+  
+  .highlight{
+	color: red;
+  }
+  .ui.list .list>.item>img.image+.content, .ui.list>.item>img.image+.content {
+	width: calc(100% - 4.0em);
+	margin-left: 0;
+  }
+  
+  .seach .ui.list .list>.item .header, .seach .ui.list>.item .header{
+	margin-bottom: 0.5em;
+	font-size: 1.4rem !important;
+	font-weight: normal;
+  }
+  .seach .time, .seach .time a{
+	font-size: 12px;
+	color: grey;
+  }
+  
+  .seach .list .item.members .ui.avatar.image {
+	width: 3.2em;
+	height: 3.2em;
+  }
+  .ui.list .list>.item.members>img.image+.content, .ui.list>.item.members>img.image+.content {
+	width: calc(100% - 4.0em);
+	margin-left: 0;
+  }
+
+  .searchlabel{
+	color: rgba(16, 16, 16, 100);
+	font-size: 24px;
+	text-align: left;
+	font-family: SourceHanSansSC-medium;
+  }
+
+  .hiddenSearch{
+	margin: auto;
+    display: none;
+  }
+
+  #tipmsg {
+    display: none;
+    z-index: 9999;
+	width:150;
+	height: 80;
 }
\ No newline at end of file